1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/bpf.h> 11 #include <linux/btf.h> 12 #include <linux/bpf_verifier.h> 13 #include <linux/filter.h> 14 #include <net/netlink.h> 15 #include <linux/file.h> 16 #include <linux/vmalloc.h> 17 #include <linux/stringify.h> 18 #include <linux/bsearch.h> 19 #include <linux/sort.h> 20 #include <linux/perf_event.h> 21 #include <linux/ctype.h> 22 #include <linux/error-injection.h> 23 #include <linux/bpf_lsm.h> 24 #include <linux/btf_ids.h> 25 26 #include "disasm.h" 27 28 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 29 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 30 [_id] = & _name ## _verifier_ops, 31 #define BPF_MAP_TYPE(_id, _ops) 32 #define BPF_LINK_TYPE(_id, _name) 33 #include <linux/bpf_types.h> 34 #undef BPF_PROG_TYPE 35 #undef BPF_MAP_TYPE 36 #undef BPF_LINK_TYPE 37 }; 38 39 /* bpf_check() is a static code analyzer that walks eBPF program 40 * instruction by instruction and updates register/stack state. 41 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 42 * 43 * The first pass is depth-first-search to check that the program is a DAG. 44 * It rejects the following programs: 45 * - larger than BPF_MAXINSNS insns 46 * - if loop is present (detected via back-edge) 47 * - unreachable insns exist (shouldn't be a forest. program = one function) 48 * - out of bounds or malformed jumps 49 * The second pass is all possible path descent from the 1st insn. 50 * Since it's analyzing all pathes through the program, the length of the 51 * analysis is limited to 64k insn, which may be hit even if total number of 52 * insn is less then 4K, but there are too many branches that change stack/regs. 53 * Number of 'branches to be analyzed' is limited to 1k 54 * 55 * On entry to each instruction, each register has a type, and the instruction 56 * changes the types of the registers depending on instruction semantics. 57 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 58 * copied to R1. 59 * 60 * All registers are 64-bit. 61 * R0 - return register 62 * R1-R5 argument passing registers 63 * R6-R9 callee saved registers 64 * R10 - frame pointer read-only 65 * 66 * At the start of BPF program the register R1 contains a pointer to bpf_context 67 * and has type PTR_TO_CTX. 68 * 69 * Verifier tracks arithmetic operations on pointers in case: 70 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 71 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 72 * 1st insn copies R10 (which has FRAME_PTR) type into R1 73 * and 2nd arithmetic instruction is pattern matched to recognize 74 * that it wants to construct a pointer to some element within stack. 75 * So after 2nd insn, the register R1 has type PTR_TO_STACK 76 * (and -20 constant is saved for further stack bounds checking). 77 * Meaning that this reg is a pointer to stack plus known immediate constant. 78 * 79 * Most of the time the registers have SCALAR_VALUE type, which 80 * means the register has some value, but it's not a valid pointer. 81 * (like pointer plus pointer becomes SCALAR_VALUE type) 82 * 83 * When verifier sees load or store instructions the type of base register 84 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 85 * four pointer types recognized by check_mem_access() function. 86 * 87 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 88 * and the range of [ptr, ptr + map's value_size) is accessible. 89 * 90 * registers used to pass values to function calls are checked against 91 * function argument constraints. 92 * 93 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 94 * It means that the register type passed to this function must be 95 * PTR_TO_STACK and it will be used inside the function as 96 * 'pointer to map element key' 97 * 98 * For example the argument constraints for bpf_map_lookup_elem(): 99 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 100 * .arg1_type = ARG_CONST_MAP_PTR, 101 * .arg2_type = ARG_PTR_TO_MAP_KEY, 102 * 103 * ret_type says that this function returns 'pointer to map elem value or null' 104 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 105 * 2nd argument should be a pointer to stack, which will be used inside 106 * the helper function as a pointer to map element key. 107 * 108 * On the kernel side the helper function looks like: 109 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 110 * { 111 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 112 * void *key = (void *) (unsigned long) r2; 113 * void *value; 114 * 115 * here kernel can access 'key' and 'map' pointers safely, knowing that 116 * [key, key + map->key_size) bytes are valid and were initialized on 117 * the stack of eBPF program. 118 * } 119 * 120 * Corresponding eBPF program may look like: 121 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 122 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 123 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 124 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 125 * here verifier looks at prototype of map_lookup_elem() and sees: 126 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 127 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 128 * 129 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 130 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 131 * and were initialized prior to this call. 132 * If it's ok, then verifier allows this BPF_CALL insn and looks at 133 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 134 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 135 * returns ether pointer to map value or NULL. 136 * 137 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 138 * insn, the register holding that pointer in the true branch changes state to 139 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 140 * branch. See check_cond_jmp_op(). 141 * 142 * After the call R0 is set to return type of the function and registers R1-R5 143 * are set to NOT_INIT to indicate that they are no longer readable. 144 * 145 * The following reference types represent a potential reference to a kernel 146 * resource which, after first being allocated, must be checked and freed by 147 * the BPF program: 148 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 149 * 150 * When the verifier sees a helper call return a reference type, it allocates a 151 * pointer id for the reference and stores it in the current function state. 152 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 153 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 154 * passes through a NULL-check conditional. For the branch wherein the state is 155 * changed to CONST_IMM, the verifier releases the reference. 156 * 157 * For each helper function that allocates a reference, such as 158 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 159 * bpf_sk_release(). When a reference type passes into the release function, 160 * the verifier also releases the reference. If any unchecked or unreleased 161 * reference remains at the end of the program, the verifier rejects it. 162 */ 163 164 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 165 struct bpf_verifier_stack_elem { 166 /* verifer state is 'st' 167 * before processing instruction 'insn_idx' 168 * and after processing instruction 'prev_insn_idx' 169 */ 170 struct bpf_verifier_state st; 171 int insn_idx; 172 int prev_insn_idx; 173 struct bpf_verifier_stack_elem *next; 174 /* length of verifier log at the time this state was pushed on stack */ 175 u32 log_pos; 176 }; 177 178 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 179 #define BPF_COMPLEXITY_LIMIT_STATES 64 180 181 #define BPF_MAP_KEY_POISON (1ULL << 63) 182 #define BPF_MAP_KEY_SEEN (1ULL << 62) 183 184 #define BPF_MAP_PTR_UNPRIV 1UL 185 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 186 POISON_POINTER_DELTA)) 187 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 188 189 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 190 { 191 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 192 } 193 194 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 195 { 196 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 197 } 198 199 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 200 const struct bpf_map *map, bool unpriv) 201 { 202 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 203 unpriv |= bpf_map_ptr_unpriv(aux); 204 aux->map_ptr_state = (unsigned long)map | 205 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 206 } 207 208 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 209 { 210 return aux->map_key_state & BPF_MAP_KEY_POISON; 211 } 212 213 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 214 { 215 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 216 } 217 218 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 219 { 220 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 221 } 222 223 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 224 { 225 bool poisoned = bpf_map_key_poisoned(aux); 226 227 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 229 } 230 231 static bool bpf_pseudo_call(const struct bpf_insn *insn) 232 { 233 return insn->code == (BPF_JMP | BPF_CALL) && 234 insn->src_reg == BPF_PSEUDO_CALL; 235 } 236 237 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 238 { 239 return insn->code == (BPF_JMP | BPF_CALL) && 240 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 241 } 242 243 static bool bpf_pseudo_func(const struct bpf_insn *insn) 244 { 245 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 246 insn->src_reg == BPF_PSEUDO_FUNC; 247 } 248 249 struct bpf_call_arg_meta { 250 struct bpf_map *map_ptr; 251 bool raw_mode; 252 bool pkt_access; 253 int regno; 254 int access_size; 255 int mem_size; 256 u64 msize_max_value; 257 int ref_obj_id; 258 int func_id; 259 struct btf *btf; 260 u32 btf_id; 261 struct btf *ret_btf; 262 u32 ret_btf_id; 263 u32 subprogno; 264 }; 265 266 struct btf *btf_vmlinux; 267 268 static DEFINE_MUTEX(bpf_verifier_lock); 269 270 static const struct bpf_line_info * 271 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 272 { 273 const struct bpf_line_info *linfo; 274 const struct bpf_prog *prog; 275 u32 i, nr_linfo; 276 277 prog = env->prog; 278 nr_linfo = prog->aux->nr_linfo; 279 280 if (!nr_linfo || insn_off >= prog->len) 281 return NULL; 282 283 linfo = prog->aux->linfo; 284 for (i = 1; i < nr_linfo; i++) 285 if (insn_off < linfo[i].insn_off) 286 break; 287 288 return &linfo[i - 1]; 289 } 290 291 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 292 va_list args) 293 { 294 unsigned int n; 295 296 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 297 298 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 299 "verifier log line truncated - local buffer too short\n"); 300 301 n = min(log->len_total - log->len_used - 1, n); 302 log->kbuf[n] = '\0'; 303 304 if (log->level == BPF_LOG_KERNEL) { 305 pr_err("BPF:%s\n", log->kbuf); 306 return; 307 } 308 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 309 log->len_used += n; 310 else 311 log->ubuf = NULL; 312 } 313 314 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 315 { 316 char zero = 0; 317 318 if (!bpf_verifier_log_needed(log)) 319 return; 320 321 log->len_used = new_pos; 322 if (put_user(zero, log->ubuf + new_pos)) 323 log->ubuf = NULL; 324 } 325 326 /* log_level controls verbosity level of eBPF verifier. 327 * bpf_verifier_log_write() is used to dump the verification trace to the log, 328 * so the user can figure out what's wrong with the program 329 */ 330 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 331 const char *fmt, ...) 332 { 333 va_list args; 334 335 if (!bpf_verifier_log_needed(&env->log)) 336 return; 337 338 va_start(args, fmt); 339 bpf_verifier_vlog(&env->log, fmt, args); 340 va_end(args); 341 } 342 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 343 344 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 345 { 346 struct bpf_verifier_env *env = private_data; 347 va_list args; 348 349 if (!bpf_verifier_log_needed(&env->log)) 350 return; 351 352 va_start(args, fmt); 353 bpf_verifier_vlog(&env->log, fmt, args); 354 va_end(args); 355 } 356 357 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 358 const char *fmt, ...) 359 { 360 va_list args; 361 362 if (!bpf_verifier_log_needed(log)) 363 return; 364 365 va_start(args, fmt); 366 bpf_verifier_vlog(log, fmt, args); 367 va_end(args); 368 } 369 370 static const char *ltrim(const char *s) 371 { 372 while (isspace(*s)) 373 s++; 374 375 return s; 376 } 377 378 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 379 u32 insn_off, 380 const char *prefix_fmt, ...) 381 { 382 const struct bpf_line_info *linfo; 383 384 if (!bpf_verifier_log_needed(&env->log)) 385 return; 386 387 linfo = find_linfo(env, insn_off); 388 if (!linfo || linfo == env->prev_linfo) 389 return; 390 391 if (prefix_fmt) { 392 va_list args; 393 394 va_start(args, prefix_fmt); 395 bpf_verifier_vlog(&env->log, prefix_fmt, args); 396 va_end(args); 397 } 398 399 verbose(env, "%s\n", 400 ltrim(btf_name_by_offset(env->prog->aux->btf, 401 linfo->line_off))); 402 403 env->prev_linfo = linfo; 404 } 405 406 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 407 struct bpf_reg_state *reg, 408 struct tnum *range, const char *ctx, 409 const char *reg_name) 410 { 411 char tn_buf[48]; 412 413 verbose(env, "At %s the register %s ", ctx, reg_name); 414 if (!tnum_is_unknown(reg->var_off)) { 415 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 416 verbose(env, "has value %s", tn_buf); 417 } else { 418 verbose(env, "has unknown scalar value"); 419 } 420 tnum_strn(tn_buf, sizeof(tn_buf), *range); 421 verbose(env, " should have been in %s\n", tn_buf); 422 } 423 424 static bool type_is_pkt_pointer(enum bpf_reg_type type) 425 { 426 return type == PTR_TO_PACKET || 427 type == PTR_TO_PACKET_META; 428 } 429 430 static bool type_is_sk_pointer(enum bpf_reg_type type) 431 { 432 return type == PTR_TO_SOCKET || 433 type == PTR_TO_SOCK_COMMON || 434 type == PTR_TO_TCP_SOCK || 435 type == PTR_TO_XDP_SOCK; 436 } 437 438 static bool reg_type_not_null(enum bpf_reg_type type) 439 { 440 return type == PTR_TO_SOCKET || 441 type == PTR_TO_TCP_SOCK || 442 type == PTR_TO_MAP_VALUE || 443 type == PTR_TO_MAP_KEY || 444 type == PTR_TO_SOCK_COMMON; 445 } 446 447 static bool reg_type_may_be_null(enum bpf_reg_type type) 448 { 449 return type == PTR_TO_MAP_VALUE_OR_NULL || 450 type == PTR_TO_SOCKET_OR_NULL || 451 type == PTR_TO_SOCK_COMMON_OR_NULL || 452 type == PTR_TO_TCP_SOCK_OR_NULL || 453 type == PTR_TO_BTF_ID_OR_NULL || 454 type == PTR_TO_MEM_OR_NULL || 455 type == PTR_TO_RDONLY_BUF_OR_NULL || 456 type == PTR_TO_RDWR_BUF_OR_NULL; 457 } 458 459 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 460 { 461 return reg->type == PTR_TO_MAP_VALUE && 462 map_value_has_spin_lock(reg->map_ptr); 463 } 464 465 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) 466 { 467 return type == PTR_TO_SOCKET || 468 type == PTR_TO_SOCKET_OR_NULL || 469 type == PTR_TO_TCP_SOCK || 470 type == PTR_TO_TCP_SOCK_OR_NULL || 471 type == PTR_TO_MEM || 472 type == PTR_TO_MEM_OR_NULL; 473 } 474 475 static bool arg_type_may_be_refcounted(enum bpf_arg_type type) 476 { 477 return type == ARG_PTR_TO_SOCK_COMMON; 478 } 479 480 static bool arg_type_may_be_null(enum bpf_arg_type type) 481 { 482 return type == ARG_PTR_TO_MAP_VALUE_OR_NULL || 483 type == ARG_PTR_TO_MEM_OR_NULL || 484 type == ARG_PTR_TO_CTX_OR_NULL || 485 type == ARG_PTR_TO_SOCKET_OR_NULL || 486 type == ARG_PTR_TO_ALLOC_MEM_OR_NULL || 487 type == ARG_PTR_TO_STACK_OR_NULL; 488 } 489 490 /* Determine whether the function releases some resources allocated by another 491 * function call. The first reference type argument will be assumed to be 492 * released by release_reference(). 493 */ 494 static bool is_release_function(enum bpf_func_id func_id) 495 { 496 return func_id == BPF_FUNC_sk_release || 497 func_id == BPF_FUNC_ringbuf_submit || 498 func_id == BPF_FUNC_ringbuf_discard; 499 } 500 501 static bool may_be_acquire_function(enum bpf_func_id func_id) 502 { 503 return func_id == BPF_FUNC_sk_lookup_tcp || 504 func_id == BPF_FUNC_sk_lookup_udp || 505 func_id == BPF_FUNC_skc_lookup_tcp || 506 func_id == BPF_FUNC_map_lookup_elem || 507 func_id == BPF_FUNC_ringbuf_reserve; 508 } 509 510 static bool is_acquire_function(enum bpf_func_id func_id, 511 const struct bpf_map *map) 512 { 513 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 514 515 if (func_id == BPF_FUNC_sk_lookup_tcp || 516 func_id == BPF_FUNC_sk_lookup_udp || 517 func_id == BPF_FUNC_skc_lookup_tcp || 518 func_id == BPF_FUNC_ringbuf_reserve) 519 return true; 520 521 if (func_id == BPF_FUNC_map_lookup_elem && 522 (map_type == BPF_MAP_TYPE_SOCKMAP || 523 map_type == BPF_MAP_TYPE_SOCKHASH)) 524 return true; 525 526 return false; 527 } 528 529 static bool is_ptr_cast_function(enum bpf_func_id func_id) 530 { 531 return func_id == BPF_FUNC_tcp_sock || 532 func_id == BPF_FUNC_sk_fullsock || 533 func_id == BPF_FUNC_skc_to_tcp_sock || 534 func_id == BPF_FUNC_skc_to_tcp6_sock || 535 func_id == BPF_FUNC_skc_to_udp6_sock || 536 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 537 func_id == BPF_FUNC_skc_to_tcp_request_sock; 538 } 539 540 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 541 { 542 return BPF_CLASS(insn->code) == BPF_STX && 543 BPF_MODE(insn->code) == BPF_ATOMIC && 544 insn->imm == BPF_CMPXCHG; 545 } 546 547 /* string representation of 'enum bpf_reg_type' */ 548 static const char * const reg_type_str[] = { 549 [NOT_INIT] = "?", 550 [SCALAR_VALUE] = "inv", 551 [PTR_TO_CTX] = "ctx", 552 [CONST_PTR_TO_MAP] = "map_ptr", 553 [PTR_TO_MAP_VALUE] = "map_value", 554 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", 555 [PTR_TO_STACK] = "fp", 556 [PTR_TO_PACKET] = "pkt", 557 [PTR_TO_PACKET_META] = "pkt_meta", 558 [PTR_TO_PACKET_END] = "pkt_end", 559 [PTR_TO_FLOW_KEYS] = "flow_keys", 560 [PTR_TO_SOCKET] = "sock", 561 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", 562 [PTR_TO_SOCK_COMMON] = "sock_common", 563 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", 564 [PTR_TO_TCP_SOCK] = "tcp_sock", 565 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", 566 [PTR_TO_TP_BUFFER] = "tp_buffer", 567 [PTR_TO_XDP_SOCK] = "xdp_sock", 568 [PTR_TO_BTF_ID] = "ptr_", 569 [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", 570 [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", 571 [PTR_TO_MEM] = "mem", 572 [PTR_TO_MEM_OR_NULL] = "mem_or_null", 573 [PTR_TO_RDONLY_BUF] = "rdonly_buf", 574 [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", 575 [PTR_TO_RDWR_BUF] = "rdwr_buf", 576 [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", 577 [PTR_TO_FUNC] = "func", 578 [PTR_TO_MAP_KEY] = "map_key", 579 }; 580 581 static char slot_type_char[] = { 582 [STACK_INVALID] = '?', 583 [STACK_SPILL] = 'r', 584 [STACK_MISC] = 'm', 585 [STACK_ZERO] = '0', 586 }; 587 588 static void print_liveness(struct bpf_verifier_env *env, 589 enum bpf_reg_liveness live) 590 { 591 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 592 verbose(env, "_"); 593 if (live & REG_LIVE_READ) 594 verbose(env, "r"); 595 if (live & REG_LIVE_WRITTEN) 596 verbose(env, "w"); 597 if (live & REG_LIVE_DONE) 598 verbose(env, "D"); 599 } 600 601 static struct bpf_func_state *func(struct bpf_verifier_env *env, 602 const struct bpf_reg_state *reg) 603 { 604 struct bpf_verifier_state *cur = env->cur_state; 605 606 return cur->frame[reg->frameno]; 607 } 608 609 static const char *kernel_type_name(const struct btf* btf, u32 id) 610 { 611 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 612 } 613 614 static void print_verifier_state(struct bpf_verifier_env *env, 615 const struct bpf_func_state *state) 616 { 617 const struct bpf_reg_state *reg; 618 enum bpf_reg_type t; 619 int i; 620 621 if (state->frameno) 622 verbose(env, " frame%d:", state->frameno); 623 for (i = 0; i < MAX_BPF_REG; i++) { 624 reg = &state->regs[i]; 625 t = reg->type; 626 if (t == NOT_INIT) 627 continue; 628 verbose(env, " R%d", i); 629 print_liveness(env, reg->live); 630 verbose(env, "=%s", reg_type_str[t]); 631 if (t == SCALAR_VALUE && reg->precise) 632 verbose(env, "P"); 633 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 634 tnum_is_const(reg->var_off)) { 635 /* reg->off should be 0 for SCALAR_VALUE */ 636 verbose(env, "%lld", reg->var_off.value + reg->off); 637 } else { 638 if (t == PTR_TO_BTF_ID || 639 t == PTR_TO_BTF_ID_OR_NULL || 640 t == PTR_TO_PERCPU_BTF_ID) 641 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 642 verbose(env, "(id=%d", reg->id); 643 if (reg_type_may_be_refcounted_or_null(t)) 644 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); 645 if (t != SCALAR_VALUE) 646 verbose(env, ",off=%d", reg->off); 647 if (type_is_pkt_pointer(t)) 648 verbose(env, ",r=%d", reg->range); 649 else if (t == CONST_PTR_TO_MAP || 650 t == PTR_TO_MAP_KEY || 651 t == PTR_TO_MAP_VALUE || 652 t == PTR_TO_MAP_VALUE_OR_NULL) 653 verbose(env, ",ks=%d,vs=%d", 654 reg->map_ptr->key_size, 655 reg->map_ptr->value_size); 656 if (tnum_is_const(reg->var_off)) { 657 /* Typically an immediate SCALAR_VALUE, but 658 * could be a pointer whose offset is too big 659 * for reg->off 660 */ 661 verbose(env, ",imm=%llx", reg->var_off.value); 662 } else { 663 if (reg->smin_value != reg->umin_value && 664 reg->smin_value != S64_MIN) 665 verbose(env, ",smin_value=%lld", 666 (long long)reg->smin_value); 667 if (reg->smax_value != reg->umax_value && 668 reg->smax_value != S64_MAX) 669 verbose(env, ",smax_value=%lld", 670 (long long)reg->smax_value); 671 if (reg->umin_value != 0) 672 verbose(env, ",umin_value=%llu", 673 (unsigned long long)reg->umin_value); 674 if (reg->umax_value != U64_MAX) 675 verbose(env, ",umax_value=%llu", 676 (unsigned long long)reg->umax_value); 677 if (!tnum_is_unknown(reg->var_off)) { 678 char tn_buf[48]; 679 680 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 681 verbose(env, ",var_off=%s", tn_buf); 682 } 683 if (reg->s32_min_value != reg->smin_value && 684 reg->s32_min_value != S32_MIN) 685 verbose(env, ",s32_min_value=%d", 686 (int)(reg->s32_min_value)); 687 if (reg->s32_max_value != reg->smax_value && 688 reg->s32_max_value != S32_MAX) 689 verbose(env, ",s32_max_value=%d", 690 (int)(reg->s32_max_value)); 691 if (reg->u32_min_value != reg->umin_value && 692 reg->u32_min_value != U32_MIN) 693 verbose(env, ",u32_min_value=%d", 694 (int)(reg->u32_min_value)); 695 if (reg->u32_max_value != reg->umax_value && 696 reg->u32_max_value != U32_MAX) 697 verbose(env, ",u32_max_value=%d", 698 (int)(reg->u32_max_value)); 699 } 700 verbose(env, ")"); 701 } 702 } 703 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 704 char types_buf[BPF_REG_SIZE + 1]; 705 bool valid = false; 706 int j; 707 708 for (j = 0; j < BPF_REG_SIZE; j++) { 709 if (state->stack[i].slot_type[j] != STACK_INVALID) 710 valid = true; 711 types_buf[j] = slot_type_char[ 712 state->stack[i].slot_type[j]]; 713 } 714 types_buf[BPF_REG_SIZE] = 0; 715 if (!valid) 716 continue; 717 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 718 print_liveness(env, state->stack[i].spilled_ptr.live); 719 if (state->stack[i].slot_type[0] == STACK_SPILL) { 720 reg = &state->stack[i].spilled_ptr; 721 t = reg->type; 722 verbose(env, "=%s", reg_type_str[t]); 723 if (t == SCALAR_VALUE && reg->precise) 724 verbose(env, "P"); 725 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 726 verbose(env, "%lld", reg->var_off.value + reg->off); 727 } else { 728 verbose(env, "=%s", types_buf); 729 } 730 } 731 if (state->acquired_refs && state->refs[0].id) { 732 verbose(env, " refs=%d", state->refs[0].id); 733 for (i = 1; i < state->acquired_refs; i++) 734 if (state->refs[i].id) 735 verbose(env, ",%d", state->refs[i].id); 736 } 737 verbose(env, "\n"); 738 } 739 740 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 741 * small to hold src. This is different from krealloc since we don't want to preserve 742 * the contents of dst. 743 * 744 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 745 * not be allocated. 746 */ 747 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 748 { 749 size_t bytes; 750 751 if (ZERO_OR_NULL_PTR(src)) 752 goto out; 753 754 if (unlikely(check_mul_overflow(n, size, &bytes))) 755 return NULL; 756 757 if (ksize(dst) < bytes) { 758 kfree(dst); 759 dst = kmalloc_track_caller(bytes, flags); 760 if (!dst) 761 return NULL; 762 } 763 764 memcpy(dst, src, bytes); 765 out: 766 return dst ? dst : ZERO_SIZE_PTR; 767 } 768 769 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 770 * small to hold new_n items. new items are zeroed out if the array grows. 771 * 772 * Contrary to krealloc_array, does not free arr if new_n is zero. 773 */ 774 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 775 { 776 if (!new_n || old_n == new_n) 777 goto out; 778 779 arr = krealloc_array(arr, new_n, size, GFP_KERNEL); 780 if (!arr) 781 return NULL; 782 783 if (new_n > old_n) 784 memset(arr + old_n * size, 0, (new_n - old_n) * size); 785 786 out: 787 return arr ? arr : ZERO_SIZE_PTR; 788 } 789 790 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 791 { 792 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 793 sizeof(struct bpf_reference_state), GFP_KERNEL); 794 if (!dst->refs) 795 return -ENOMEM; 796 797 dst->acquired_refs = src->acquired_refs; 798 return 0; 799 } 800 801 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 802 { 803 size_t n = src->allocated_stack / BPF_REG_SIZE; 804 805 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 806 GFP_KERNEL); 807 if (!dst->stack) 808 return -ENOMEM; 809 810 dst->allocated_stack = src->allocated_stack; 811 return 0; 812 } 813 814 static int resize_reference_state(struct bpf_func_state *state, size_t n) 815 { 816 state->refs = realloc_array(state->refs, state->acquired_refs, n, 817 sizeof(struct bpf_reference_state)); 818 if (!state->refs) 819 return -ENOMEM; 820 821 state->acquired_refs = n; 822 return 0; 823 } 824 825 static int grow_stack_state(struct bpf_func_state *state, int size) 826 { 827 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 828 829 if (old_n >= n) 830 return 0; 831 832 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 833 if (!state->stack) 834 return -ENOMEM; 835 836 state->allocated_stack = size; 837 return 0; 838 } 839 840 /* Acquire a pointer id from the env and update the state->refs to include 841 * this new pointer reference. 842 * On success, returns a valid pointer id to associate with the register 843 * On failure, returns a negative errno. 844 */ 845 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 846 { 847 struct bpf_func_state *state = cur_func(env); 848 int new_ofs = state->acquired_refs; 849 int id, err; 850 851 err = resize_reference_state(state, state->acquired_refs + 1); 852 if (err) 853 return err; 854 id = ++env->id_gen; 855 state->refs[new_ofs].id = id; 856 state->refs[new_ofs].insn_idx = insn_idx; 857 858 return id; 859 } 860 861 /* release function corresponding to acquire_reference_state(). Idempotent. */ 862 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 863 { 864 int i, last_idx; 865 866 last_idx = state->acquired_refs - 1; 867 for (i = 0; i < state->acquired_refs; i++) { 868 if (state->refs[i].id == ptr_id) { 869 if (last_idx && i != last_idx) 870 memcpy(&state->refs[i], &state->refs[last_idx], 871 sizeof(*state->refs)); 872 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 873 state->acquired_refs--; 874 return 0; 875 } 876 } 877 return -EINVAL; 878 } 879 880 static void free_func_state(struct bpf_func_state *state) 881 { 882 if (!state) 883 return; 884 kfree(state->refs); 885 kfree(state->stack); 886 kfree(state); 887 } 888 889 static void clear_jmp_history(struct bpf_verifier_state *state) 890 { 891 kfree(state->jmp_history); 892 state->jmp_history = NULL; 893 state->jmp_history_cnt = 0; 894 } 895 896 static void free_verifier_state(struct bpf_verifier_state *state, 897 bool free_self) 898 { 899 int i; 900 901 for (i = 0; i <= state->curframe; i++) { 902 free_func_state(state->frame[i]); 903 state->frame[i] = NULL; 904 } 905 clear_jmp_history(state); 906 if (free_self) 907 kfree(state); 908 } 909 910 /* copy verifier state from src to dst growing dst stack space 911 * when necessary to accommodate larger src stack 912 */ 913 static int copy_func_state(struct bpf_func_state *dst, 914 const struct bpf_func_state *src) 915 { 916 int err; 917 918 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 919 err = copy_reference_state(dst, src); 920 if (err) 921 return err; 922 return copy_stack_state(dst, src); 923 } 924 925 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 926 const struct bpf_verifier_state *src) 927 { 928 struct bpf_func_state *dst; 929 int i, err; 930 931 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 932 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 933 GFP_USER); 934 if (!dst_state->jmp_history) 935 return -ENOMEM; 936 dst_state->jmp_history_cnt = src->jmp_history_cnt; 937 938 /* if dst has more stack frames then src frame, free them */ 939 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 940 free_func_state(dst_state->frame[i]); 941 dst_state->frame[i] = NULL; 942 } 943 dst_state->speculative = src->speculative; 944 dst_state->curframe = src->curframe; 945 dst_state->active_spin_lock = src->active_spin_lock; 946 dst_state->branches = src->branches; 947 dst_state->parent = src->parent; 948 dst_state->first_insn_idx = src->first_insn_idx; 949 dst_state->last_insn_idx = src->last_insn_idx; 950 for (i = 0; i <= src->curframe; i++) { 951 dst = dst_state->frame[i]; 952 if (!dst) { 953 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 954 if (!dst) 955 return -ENOMEM; 956 dst_state->frame[i] = dst; 957 } 958 err = copy_func_state(dst, src->frame[i]); 959 if (err) 960 return err; 961 } 962 return 0; 963 } 964 965 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 966 { 967 while (st) { 968 u32 br = --st->branches; 969 970 /* WARN_ON(br > 1) technically makes sense here, 971 * but see comment in push_stack(), hence: 972 */ 973 WARN_ONCE((int)br < 0, 974 "BUG update_branch_counts:branches_to_explore=%d\n", 975 br); 976 if (br) 977 break; 978 st = st->parent; 979 } 980 } 981 982 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 983 int *insn_idx, bool pop_log) 984 { 985 struct bpf_verifier_state *cur = env->cur_state; 986 struct bpf_verifier_stack_elem *elem, *head = env->head; 987 int err; 988 989 if (env->head == NULL) 990 return -ENOENT; 991 992 if (cur) { 993 err = copy_verifier_state(cur, &head->st); 994 if (err) 995 return err; 996 } 997 if (pop_log) 998 bpf_vlog_reset(&env->log, head->log_pos); 999 if (insn_idx) 1000 *insn_idx = head->insn_idx; 1001 if (prev_insn_idx) 1002 *prev_insn_idx = head->prev_insn_idx; 1003 elem = head->next; 1004 free_verifier_state(&head->st, false); 1005 kfree(head); 1006 env->head = elem; 1007 env->stack_size--; 1008 return 0; 1009 } 1010 1011 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1012 int insn_idx, int prev_insn_idx, 1013 bool speculative) 1014 { 1015 struct bpf_verifier_state *cur = env->cur_state; 1016 struct bpf_verifier_stack_elem *elem; 1017 int err; 1018 1019 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1020 if (!elem) 1021 goto err; 1022 1023 elem->insn_idx = insn_idx; 1024 elem->prev_insn_idx = prev_insn_idx; 1025 elem->next = env->head; 1026 elem->log_pos = env->log.len_used; 1027 env->head = elem; 1028 env->stack_size++; 1029 err = copy_verifier_state(&elem->st, cur); 1030 if (err) 1031 goto err; 1032 elem->st.speculative |= speculative; 1033 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1034 verbose(env, "The sequence of %d jumps is too complex.\n", 1035 env->stack_size); 1036 goto err; 1037 } 1038 if (elem->st.parent) { 1039 ++elem->st.parent->branches; 1040 /* WARN_ON(branches > 2) technically makes sense here, 1041 * but 1042 * 1. speculative states will bump 'branches' for non-branch 1043 * instructions 1044 * 2. is_state_visited() heuristics may decide not to create 1045 * a new state for a sequence of branches and all such current 1046 * and cloned states will be pointing to a single parent state 1047 * which might have large 'branches' count. 1048 */ 1049 } 1050 return &elem->st; 1051 err: 1052 free_verifier_state(env->cur_state, true); 1053 env->cur_state = NULL; 1054 /* pop all elements and return */ 1055 while (!pop_stack(env, NULL, NULL, false)); 1056 return NULL; 1057 } 1058 1059 #define CALLER_SAVED_REGS 6 1060 static const int caller_saved[CALLER_SAVED_REGS] = { 1061 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1062 }; 1063 1064 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1065 struct bpf_reg_state *reg); 1066 1067 /* This helper doesn't clear reg->id */ 1068 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1069 { 1070 reg->var_off = tnum_const(imm); 1071 reg->smin_value = (s64)imm; 1072 reg->smax_value = (s64)imm; 1073 reg->umin_value = imm; 1074 reg->umax_value = imm; 1075 1076 reg->s32_min_value = (s32)imm; 1077 reg->s32_max_value = (s32)imm; 1078 reg->u32_min_value = (u32)imm; 1079 reg->u32_max_value = (u32)imm; 1080 } 1081 1082 /* Mark the unknown part of a register (variable offset or scalar value) as 1083 * known to have the value @imm. 1084 */ 1085 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1086 { 1087 /* Clear id, off, and union(map_ptr, range) */ 1088 memset(((u8 *)reg) + sizeof(reg->type), 0, 1089 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1090 ___mark_reg_known(reg, imm); 1091 } 1092 1093 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1094 { 1095 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1096 reg->s32_min_value = (s32)imm; 1097 reg->s32_max_value = (s32)imm; 1098 reg->u32_min_value = (u32)imm; 1099 reg->u32_max_value = (u32)imm; 1100 } 1101 1102 /* Mark the 'variable offset' part of a register as zero. This should be 1103 * used only on registers holding a pointer type. 1104 */ 1105 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1106 { 1107 __mark_reg_known(reg, 0); 1108 } 1109 1110 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1111 { 1112 __mark_reg_known(reg, 0); 1113 reg->type = SCALAR_VALUE; 1114 } 1115 1116 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1117 struct bpf_reg_state *regs, u32 regno) 1118 { 1119 if (WARN_ON(regno >= MAX_BPF_REG)) { 1120 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1121 /* Something bad happened, let's kill all regs */ 1122 for (regno = 0; regno < MAX_BPF_REG; regno++) 1123 __mark_reg_not_init(env, regs + regno); 1124 return; 1125 } 1126 __mark_reg_known_zero(regs + regno); 1127 } 1128 1129 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1130 { 1131 switch (reg->type) { 1132 case PTR_TO_MAP_VALUE_OR_NULL: { 1133 const struct bpf_map *map = reg->map_ptr; 1134 1135 if (map->inner_map_meta) { 1136 reg->type = CONST_PTR_TO_MAP; 1137 reg->map_ptr = map->inner_map_meta; 1138 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1139 reg->type = PTR_TO_XDP_SOCK; 1140 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1141 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1142 reg->type = PTR_TO_SOCKET; 1143 } else { 1144 reg->type = PTR_TO_MAP_VALUE; 1145 } 1146 break; 1147 } 1148 case PTR_TO_SOCKET_OR_NULL: 1149 reg->type = PTR_TO_SOCKET; 1150 break; 1151 case PTR_TO_SOCK_COMMON_OR_NULL: 1152 reg->type = PTR_TO_SOCK_COMMON; 1153 break; 1154 case PTR_TO_TCP_SOCK_OR_NULL: 1155 reg->type = PTR_TO_TCP_SOCK; 1156 break; 1157 case PTR_TO_BTF_ID_OR_NULL: 1158 reg->type = PTR_TO_BTF_ID; 1159 break; 1160 case PTR_TO_MEM_OR_NULL: 1161 reg->type = PTR_TO_MEM; 1162 break; 1163 case PTR_TO_RDONLY_BUF_OR_NULL: 1164 reg->type = PTR_TO_RDONLY_BUF; 1165 break; 1166 case PTR_TO_RDWR_BUF_OR_NULL: 1167 reg->type = PTR_TO_RDWR_BUF; 1168 break; 1169 default: 1170 WARN_ONCE(1, "unknown nullable register type"); 1171 } 1172 } 1173 1174 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1175 { 1176 return type_is_pkt_pointer(reg->type); 1177 } 1178 1179 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1180 { 1181 return reg_is_pkt_pointer(reg) || 1182 reg->type == PTR_TO_PACKET_END; 1183 } 1184 1185 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1186 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1187 enum bpf_reg_type which) 1188 { 1189 /* The register can already have a range from prior markings. 1190 * This is fine as long as it hasn't been advanced from its 1191 * origin. 1192 */ 1193 return reg->type == which && 1194 reg->id == 0 && 1195 reg->off == 0 && 1196 tnum_equals_const(reg->var_off, 0); 1197 } 1198 1199 /* Reset the min/max bounds of a register */ 1200 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1201 { 1202 reg->smin_value = S64_MIN; 1203 reg->smax_value = S64_MAX; 1204 reg->umin_value = 0; 1205 reg->umax_value = U64_MAX; 1206 1207 reg->s32_min_value = S32_MIN; 1208 reg->s32_max_value = S32_MAX; 1209 reg->u32_min_value = 0; 1210 reg->u32_max_value = U32_MAX; 1211 } 1212 1213 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1214 { 1215 reg->smin_value = S64_MIN; 1216 reg->smax_value = S64_MAX; 1217 reg->umin_value = 0; 1218 reg->umax_value = U64_MAX; 1219 } 1220 1221 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1222 { 1223 reg->s32_min_value = S32_MIN; 1224 reg->s32_max_value = S32_MAX; 1225 reg->u32_min_value = 0; 1226 reg->u32_max_value = U32_MAX; 1227 } 1228 1229 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1230 { 1231 struct tnum var32_off = tnum_subreg(reg->var_off); 1232 1233 /* min signed is max(sign bit) | min(other bits) */ 1234 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1235 var32_off.value | (var32_off.mask & S32_MIN)); 1236 /* max signed is min(sign bit) | max(other bits) */ 1237 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1238 var32_off.value | (var32_off.mask & S32_MAX)); 1239 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1240 reg->u32_max_value = min(reg->u32_max_value, 1241 (u32)(var32_off.value | var32_off.mask)); 1242 } 1243 1244 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1245 { 1246 /* min signed is max(sign bit) | min(other bits) */ 1247 reg->smin_value = max_t(s64, reg->smin_value, 1248 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1249 /* max signed is min(sign bit) | max(other bits) */ 1250 reg->smax_value = min_t(s64, reg->smax_value, 1251 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1252 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1253 reg->umax_value = min(reg->umax_value, 1254 reg->var_off.value | reg->var_off.mask); 1255 } 1256 1257 static void __update_reg_bounds(struct bpf_reg_state *reg) 1258 { 1259 __update_reg32_bounds(reg); 1260 __update_reg64_bounds(reg); 1261 } 1262 1263 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1264 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1265 { 1266 /* Learn sign from signed bounds. 1267 * If we cannot cross the sign boundary, then signed and unsigned bounds 1268 * are the same, so combine. This works even in the negative case, e.g. 1269 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1270 */ 1271 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1272 reg->s32_min_value = reg->u32_min_value = 1273 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1274 reg->s32_max_value = reg->u32_max_value = 1275 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1276 return; 1277 } 1278 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1279 * boundary, so we must be careful. 1280 */ 1281 if ((s32)reg->u32_max_value >= 0) { 1282 /* Positive. We can't learn anything from the smin, but smax 1283 * is positive, hence safe. 1284 */ 1285 reg->s32_min_value = reg->u32_min_value; 1286 reg->s32_max_value = reg->u32_max_value = 1287 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1288 } else if ((s32)reg->u32_min_value < 0) { 1289 /* Negative. We can't learn anything from the smax, but smin 1290 * is negative, hence safe. 1291 */ 1292 reg->s32_min_value = reg->u32_min_value = 1293 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1294 reg->s32_max_value = reg->u32_max_value; 1295 } 1296 } 1297 1298 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1299 { 1300 /* Learn sign from signed bounds. 1301 * If we cannot cross the sign boundary, then signed and unsigned bounds 1302 * are the same, so combine. This works even in the negative case, e.g. 1303 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1304 */ 1305 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1306 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1307 reg->umin_value); 1308 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1309 reg->umax_value); 1310 return; 1311 } 1312 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1313 * boundary, so we must be careful. 1314 */ 1315 if ((s64)reg->umax_value >= 0) { 1316 /* Positive. We can't learn anything from the smin, but smax 1317 * is positive, hence safe. 1318 */ 1319 reg->smin_value = reg->umin_value; 1320 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1321 reg->umax_value); 1322 } else if ((s64)reg->umin_value < 0) { 1323 /* Negative. We can't learn anything from the smax, but smin 1324 * is negative, hence safe. 1325 */ 1326 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1327 reg->umin_value); 1328 reg->smax_value = reg->umax_value; 1329 } 1330 } 1331 1332 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1333 { 1334 __reg32_deduce_bounds(reg); 1335 __reg64_deduce_bounds(reg); 1336 } 1337 1338 /* Attempts to improve var_off based on unsigned min/max information */ 1339 static void __reg_bound_offset(struct bpf_reg_state *reg) 1340 { 1341 struct tnum var64_off = tnum_intersect(reg->var_off, 1342 tnum_range(reg->umin_value, 1343 reg->umax_value)); 1344 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1345 tnum_range(reg->u32_min_value, 1346 reg->u32_max_value)); 1347 1348 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1349 } 1350 1351 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1352 { 1353 reg->umin_value = reg->u32_min_value; 1354 reg->umax_value = reg->u32_max_value; 1355 /* Attempt to pull 32-bit signed bounds into 64-bit bounds 1356 * but must be positive otherwise set to worse case bounds 1357 * and refine later from tnum. 1358 */ 1359 if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0) 1360 reg->smax_value = reg->s32_max_value; 1361 else 1362 reg->smax_value = U32_MAX; 1363 if (reg->s32_min_value >= 0) 1364 reg->smin_value = reg->s32_min_value; 1365 else 1366 reg->smin_value = 0; 1367 } 1368 1369 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1370 { 1371 /* special case when 64-bit register has upper 32-bit register 1372 * zeroed. Typically happens after zext or <<32, >>32 sequence 1373 * allowing us to use 32-bit bounds directly, 1374 */ 1375 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1376 __reg_assign_32_into_64(reg); 1377 } else { 1378 /* Otherwise the best we can do is push lower 32bit known and 1379 * unknown bits into register (var_off set from jmp logic) 1380 * then learn as much as possible from the 64-bit tnum 1381 * known and unknown bits. The previous smin/smax bounds are 1382 * invalid here because of jmp32 compare so mark them unknown 1383 * so they do not impact tnum bounds calculation. 1384 */ 1385 __mark_reg64_unbounded(reg); 1386 __update_reg_bounds(reg); 1387 } 1388 1389 /* Intersecting with the old var_off might have improved our bounds 1390 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1391 * then new var_off is (0; 0x7f...fc) which improves our umax. 1392 */ 1393 __reg_deduce_bounds(reg); 1394 __reg_bound_offset(reg); 1395 __update_reg_bounds(reg); 1396 } 1397 1398 static bool __reg64_bound_s32(s64 a) 1399 { 1400 return a > S32_MIN && a < S32_MAX; 1401 } 1402 1403 static bool __reg64_bound_u32(u64 a) 1404 { 1405 return a > U32_MIN && a < U32_MAX; 1406 } 1407 1408 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1409 { 1410 __mark_reg32_unbounded(reg); 1411 1412 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1413 reg->s32_min_value = (s32)reg->smin_value; 1414 reg->s32_max_value = (s32)reg->smax_value; 1415 } 1416 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 1417 reg->u32_min_value = (u32)reg->umin_value; 1418 reg->u32_max_value = (u32)reg->umax_value; 1419 } 1420 1421 /* Intersecting with the old var_off might have improved our bounds 1422 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1423 * then new var_off is (0; 0x7f...fc) which improves our umax. 1424 */ 1425 __reg_deduce_bounds(reg); 1426 __reg_bound_offset(reg); 1427 __update_reg_bounds(reg); 1428 } 1429 1430 /* Mark a register as having a completely unknown (scalar) value. */ 1431 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1432 struct bpf_reg_state *reg) 1433 { 1434 /* 1435 * Clear type, id, off, and union(map_ptr, range) and 1436 * padding between 'type' and union 1437 */ 1438 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1439 reg->type = SCALAR_VALUE; 1440 reg->var_off = tnum_unknown; 1441 reg->frameno = 0; 1442 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; 1443 __mark_reg_unbounded(reg); 1444 } 1445 1446 static void mark_reg_unknown(struct bpf_verifier_env *env, 1447 struct bpf_reg_state *regs, u32 regno) 1448 { 1449 if (WARN_ON(regno >= MAX_BPF_REG)) { 1450 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1451 /* Something bad happened, let's kill all regs except FP */ 1452 for (regno = 0; regno < BPF_REG_FP; regno++) 1453 __mark_reg_not_init(env, regs + regno); 1454 return; 1455 } 1456 __mark_reg_unknown(env, regs + regno); 1457 } 1458 1459 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1460 struct bpf_reg_state *reg) 1461 { 1462 __mark_reg_unknown(env, reg); 1463 reg->type = NOT_INIT; 1464 } 1465 1466 static void mark_reg_not_init(struct bpf_verifier_env *env, 1467 struct bpf_reg_state *regs, u32 regno) 1468 { 1469 if (WARN_ON(regno >= MAX_BPF_REG)) { 1470 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1471 /* Something bad happened, let's kill all regs except FP */ 1472 for (regno = 0; regno < BPF_REG_FP; regno++) 1473 __mark_reg_not_init(env, regs + regno); 1474 return; 1475 } 1476 __mark_reg_not_init(env, regs + regno); 1477 } 1478 1479 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 1480 struct bpf_reg_state *regs, u32 regno, 1481 enum bpf_reg_type reg_type, 1482 struct btf *btf, u32 btf_id) 1483 { 1484 if (reg_type == SCALAR_VALUE) { 1485 mark_reg_unknown(env, regs, regno); 1486 return; 1487 } 1488 mark_reg_known_zero(env, regs, regno); 1489 regs[regno].type = PTR_TO_BTF_ID; 1490 regs[regno].btf = btf; 1491 regs[regno].btf_id = btf_id; 1492 } 1493 1494 #define DEF_NOT_SUBREG (0) 1495 static void init_reg_state(struct bpf_verifier_env *env, 1496 struct bpf_func_state *state) 1497 { 1498 struct bpf_reg_state *regs = state->regs; 1499 int i; 1500 1501 for (i = 0; i < MAX_BPF_REG; i++) { 1502 mark_reg_not_init(env, regs, i); 1503 regs[i].live = REG_LIVE_NONE; 1504 regs[i].parent = NULL; 1505 regs[i].subreg_def = DEF_NOT_SUBREG; 1506 } 1507 1508 /* frame pointer */ 1509 regs[BPF_REG_FP].type = PTR_TO_STACK; 1510 mark_reg_known_zero(env, regs, BPF_REG_FP); 1511 regs[BPF_REG_FP].frameno = state->frameno; 1512 } 1513 1514 #define BPF_MAIN_FUNC (-1) 1515 static void init_func_state(struct bpf_verifier_env *env, 1516 struct bpf_func_state *state, 1517 int callsite, int frameno, int subprogno) 1518 { 1519 state->callsite = callsite; 1520 state->frameno = frameno; 1521 state->subprogno = subprogno; 1522 init_reg_state(env, state); 1523 } 1524 1525 enum reg_arg_type { 1526 SRC_OP, /* register is used as source operand */ 1527 DST_OP, /* register is used as destination operand */ 1528 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1529 }; 1530 1531 static int cmp_subprogs(const void *a, const void *b) 1532 { 1533 return ((struct bpf_subprog_info *)a)->start - 1534 ((struct bpf_subprog_info *)b)->start; 1535 } 1536 1537 static int find_subprog(struct bpf_verifier_env *env, int off) 1538 { 1539 struct bpf_subprog_info *p; 1540 1541 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1542 sizeof(env->subprog_info[0]), cmp_subprogs); 1543 if (!p) 1544 return -ENOENT; 1545 return p - env->subprog_info; 1546 1547 } 1548 1549 static int add_subprog(struct bpf_verifier_env *env, int off) 1550 { 1551 int insn_cnt = env->prog->len; 1552 int ret; 1553 1554 if (off >= insn_cnt || off < 0) { 1555 verbose(env, "call to invalid destination\n"); 1556 return -EINVAL; 1557 } 1558 ret = find_subprog(env, off); 1559 if (ret >= 0) 1560 return ret; 1561 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1562 verbose(env, "too many subprograms\n"); 1563 return -E2BIG; 1564 } 1565 /* determine subprog starts. The end is one before the next starts */ 1566 env->subprog_info[env->subprog_cnt++].start = off; 1567 sort(env->subprog_info, env->subprog_cnt, 1568 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1569 return env->subprog_cnt - 1; 1570 } 1571 1572 struct bpf_kfunc_desc { 1573 struct btf_func_model func_model; 1574 u32 func_id; 1575 s32 imm; 1576 }; 1577 1578 #define MAX_KFUNC_DESCS 256 1579 struct bpf_kfunc_desc_tab { 1580 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 1581 u32 nr_descs; 1582 }; 1583 1584 static int kfunc_desc_cmp_by_id(const void *a, const void *b) 1585 { 1586 const struct bpf_kfunc_desc *d0 = a; 1587 const struct bpf_kfunc_desc *d1 = b; 1588 1589 /* func_id is not greater than BTF_MAX_TYPE */ 1590 return d0->func_id - d1->func_id; 1591 } 1592 1593 static const struct bpf_kfunc_desc * 1594 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id) 1595 { 1596 struct bpf_kfunc_desc desc = { 1597 .func_id = func_id, 1598 }; 1599 struct bpf_kfunc_desc_tab *tab; 1600 1601 tab = prog->aux->kfunc_tab; 1602 return bsearch(&desc, tab->descs, tab->nr_descs, 1603 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id); 1604 } 1605 1606 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id) 1607 { 1608 const struct btf_type *func, *func_proto; 1609 struct bpf_kfunc_desc_tab *tab; 1610 struct bpf_prog_aux *prog_aux; 1611 struct bpf_kfunc_desc *desc; 1612 const char *func_name; 1613 unsigned long addr; 1614 int err; 1615 1616 prog_aux = env->prog->aux; 1617 tab = prog_aux->kfunc_tab; 1618 if (!tab) { 1619 if (!btf_vmlinux) { 1620 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 1621 return -ENOTSUPP; 1622 } 1623 1624 if (!env->prog->jit_requested) { 1625 verbose(env, "JIT is required for calling kernel function\n"); 1626 return -ENOTSUPP; 1627 } 1628 1629 if (!bpf_jit_supports_kfunc_call()) { 1630 verbose(env, "JIT does not support calling kernel function\n"); 1631 return -ENOTSUPP; 1632 } 1633 1634 if (!env->prog->gpl_compatible) { 1635 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 1636 return -EINVAL; 1637 } 1638 1639 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 1640 if (!tab) 1641 return -ENOMEM; 1642 prog_aux->kfunc_tab = tab; 1643 } 1644 1645 if (find_kfunc_desc(env->prog, func_id)) 1646 return 0; 1647 1648 if (tab->nr_descs == MAX_KFUNC_DESCS) { 1649 verbose(env, "too many different kernel function calls\n"); 1650 return -E2BIG; 1651 } 1652 1653 func = btf_type_by_id(btf_vmlinux, func_id); 1654 if (!func || !btf_type_is_func(func)) { 1655 verbose(env, "kernel btf_id %u is not a function\n", 1656 func_id); 1657 return -EINVAL; 1658 } 1659 func_proto = btf_type_by_id(btf_vmlinux, func->type); 1660 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 1661 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 1662 func_id); 1663 return -EINVAL; 1664 } 1665 1666 func_name = btf_name_by_offset(btf_vmlinux, func->name_off); 1667 addr = kallsyms_lookup_name(func_name); 1668 if (!addr) { 1669 verbose(env, "cannot find address for kernel function %s\n", 1670 func_name); 1671 return -EINVAL; 1672 } 1673 1674 desc = &tab->descs[tab->nr_descs++]; 1675 desc->func_id = func_id; 1676 desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base; 1677 err = btf_distill_func_proto(&env->log, btf_vmlinux, 1678 func_proto, func_name, 1679 &desc->func_model); 1680 if (!err) 1681 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1682 kfunc_desc_cmp_by_id, NULL); 1683 return err; 1684 } 1685 1686 static int kfunc_desc_cmp_by_imm(const void *a, const void *b) 1687 { 1688 const struct bpf_kfunc_desc *d0 = a; 1689 const struct bpf_kfunc_desc *d1 = b; 1690 1691 if (d0->imm > d1->imm) 1692 return 1; 1693 else if (d0->imm < d1->imm) 1694 return -1; 1695 return 0; 1696 } 1697 1698 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) 1699 { 1700 struct bpf_kfunc_desc_tab *tab; 1701 1702 tab = prog->aux->kfunc_tab; 1703 if (!tab) 1704 return; 1705 1706 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1707 kfunc_desc_cmp_by_imm, NULL); 1708 } 1709 1710 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 1711 { 1712 return !!prog->aux->kfunc_tab; 1713 } 1714 1715 const struct btf_func_model * 1716 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 1717 const struct bpf_insn *insn) 1718 { 1719 const struct bpf_kfunc_desc desc = { 1720 .imm = insn->imm, 1721 }; 1722 const struct bpf_kfunc_desc *res; 1723 struct bpf_kfunc_desc_tab *tab; 1724 1725 tab = prog->aux->kfunc_tab; 1726 res = bsearch(&desc, tab->descs, tab->nr_descs, 1727 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); 1728 1729 return res ? &res->func_model : NULL; 1730 } 1731 1732 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 1733 { 1734 struct bpf_subprog_info *subprog = env->subprog_info; 1735 struct bpf_insn *insn = env->prog->insnsi; 1736 int i, ret, insn_cnt = env->prog->len; 1737 1738 /* Add entry function. */ 1739 ret = add_subprog(env, 0); 1740 if (ret) 1741 return ret; 1742 1743 for (i = 0; i < insn_cnt; i++, insn++) { 1744 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 1745 !bpf_pseudo_kfunc_call(insn)) 1746 continue; 1747 1748 if (!env->bpf_capable) { 1749 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 1750 return -EPERM; 1751 } 1752 1753 if (bpf_pseudo_func(insn)) { 1754 ret = add_subprog(env, i + insn->imm + 1); 1755 if (ret >= 0) 1756 /* remember subprog */ 1757 insn[1].imm = ret; 1758 } else if (bpf_pseudo_call(insn)) { 1759 ret = add_subprog(env, i + insn->imm + 1); 1760 } else { 1761 ret = add_kfunc_call(env, insn->imm); 1762 } 1763 1764 if (ret < 0) 1765 return ret; 1766 } 1767 1768 /* Add a fake 'exit' subprog which could simplify subprog iteration 1769 * logic. 'subprog_cnt' should not be increased. 1770 */ 1771 subprog[env->subprog_cnt].start = insn_cnt; 1772 1773 if (env->log.level & BPF_LOG_LEVEL2) 1774 for (i = 0; i < env->subprog_cnt; i++) 1775 verbose(env, "func#%d @%d\n", i, subprog[i].start); 1776 1777 return 0; 1778 } 1779 1780 static int check_subprogs(struct bpf_verifier_env *env) 1781 { 1782 int i, subprog_start, subprog_end, off, cur_subprog = 0; 1783 struct bpf_subprog_info *subprog = env->subprog_info; 1784 struct bpf_insn *insn = env->prog->insnsi; 1785 int insn_cnt = env->prog->len; 1786 1787 /* now check that all jumps are within the same subprog */ 1788 subprog_start = subprog[cur_subprog].start; 1789 subprog_end = subprog[cur_subprog + 1].start; 1790 for (i = 0; i < insn_cnt; i++) { 1791 u8 code = insn[i].code; 1792 1793 if (code == (BPF_JMP | BPF_CALL) && 1794 insn[i].imm == BPF_FUNC_tail_call && 1795 insn[i].src_reg != BPF_PSEUDO_CALL) 1796 subprog[cur_subprog].has_tail_call = true; 1797 if (BPF_CLASS(code) == BPF_LD && 1798 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 1799 subprog[cur_subprog].has_ld_abs = true; 1800 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 1801 goto next; 1802 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 1803 goto next; 1804 off = i + insn[i].off + 1; 1805 if (off < subprog_start || off >= subprog_end) { 1806 verbose(env, "jump out of range from insn %d to %d\n", i, off); 1807 return -EINVAL; 1808 } 1809 next: 1810 if (i == subprog_end - 1) { 1811 /* to avoid fall-through from one subprog into another 1812 * the last insn of the subprog should be either exit 1813 * or unconditional jump back 1814 */ 1815 if (code != (BPF_JMP | BPF_EXIT) && 1816 code != (BPF_JMP | BPF_JA)) { 1817 verbose(env, "last insn is not an exit or jmp\n"); 1818 return -EINVAL; 1819 } 1820 subprog_start = subprog_end; 1821 cur_subprog++; 1822 if (cur_subprog < env->subprog_cnt) 1823 subprog_end = subprog[cur_subprog + 1].start; 1824 } 1825 } 1826 return 0; 1827 } 1828 1829 /* Parentage chain of this register (or stack slot) should take care of all 1830 * issues like callee-saved registers, stack slot allocation time, etc. 1831 */ 1832 static int mark_reg_read(struct bpf_verifier_env *env, 1833 const struct bpf_reg_state *state, 1834 struct bpf_reg_state *parent, u8 flag) 1835 { 1836 bool writes = parent == state->parent; /* Observe write marks */ 1837 int cnt = 0; 1838 1839 while (parent) { 1840 /* if read wasn't screened by an earlier write ... */ 1841 if (writes && state->live & REG_LIVE_WRITTEN) 1842 break; 1843 if (parent->live & REG_LIVE_DONE) { 1844 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 1845 reg_type_str[parent->type], 1846 parent->var_off.value, parent->off); 1847 return -EFAULT; 1848 } 1849 /* The first condition is more likely to be true than the 1850 * second, checked it first. 1851 */ 1852 if ((parent->live & REG_LIVE_READ) == flag || 1853 parent->live & REG_LIVE_READ64) 1854 /* The parentage chain never changes and 1855 * this parent was already marked as LIVE_READ. 1856 * There is no need to keep walking the chain again and 1857 * keep re-marking all parents as LIVE_READ. 1858 * This case happens when the same register is read 1859 * multiple times without writes into it in-between. 1860 * Also, if parent has the stronger REG_LIVE_READ64 set, 1861 * then no need to set the weak REG_LIVE_READ32. 1862 */ 1863 break; 1864 /* ... then we depend on parent's value */ 1865 parent->live |= flag; 1866 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 1867 if (flag == REG_LIVE_READ64) 1868 parent->live &= ~REG_LIVE_READ32; 1869 state = parent; 1870 parent = state->parent; 1871 writes = true; 1872 cnt++; 1873 } 1874 1875 if (env->longest_mark_read_walk < cnt) 1876 env->longest_mark_read_walk = cnt; 1877 return 0; 1878 } 1879 1880 /* This function is supposed to be used by the following 32-bit optimization 1881 * code only. It returns TRUE if the source or destination register operates 1882 * on 64-bit, otherwise return FALSE. 1883 */ 1884 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 1885 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 1886 { 1887 u8 code, class, op; 1888 1889 code = insn->code; 1890 class = BPF_CLASS(code); 1891 op = BPF_OP(code); 1892 if (class == BPF_JMP) { 1893 /* BPF_EXIT for "main" will reach here. Return TRUE 1894 * conservatively. 1895 */ 1896 if (op == BPF_EXIT) 1897 return true; 1898 if (op == BPF_CALL) { 1899 /* BPF to BPF call will reach here because of marking 1900 * caller saved clobber with DST_OP_NO_MARK for which we 1901 * don't care the register def because they are anyway 1902 * marked as NOT_INIT already. 1903 */ 1904 if (insn->src_reg == BPF_PSEUDO_CALL) 1905 return false; 1906 /* Helper call will reach here because of arg type 1907 * check, conservatively return TRUE. 1908 */ 1909 if (t == SRC_OP) 1910 return true; 1911 1912 return false; 1913 } 1914 } 1915 1916 if (class == BPF_ALU64 || class == BPF_JMP || 1917 /* BPF_END always use BPF_ALU class. */ 1918 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 1919 return true; 1920 1921 if (class == BPF_ALU || class == BPF_JMP32) 1922 return false; 1923 1924 if (class == BPF_LDX) { 1925 if (t != SRC_OP) 1926 return BPF_SIZE(code) == BPF_DW; 1927 /* LDX source must be ptr. */ 1928 return true; 1929 } 1930 1931 if (class == BPF_STX) { 1932 /* BPF_STX (including atomic variants) has multiple source 1933 * operands, one of which is a ptr. Check whether the caller is 1934 * asking about it. 1935 */ 1936 if (t == SRC_OP && reg->type != SCALAR_VALUE) 1937 return true; 1938 return BPF_SIZE(code) == BPF_DW; 1939 } 1940 1941 if (class == BPF_LD) { 1942 u8 mode = BPF_MODE(code); 1943 1944 /* LD_IMM64 */ 1945 if (mode == BPF_IMM) 1946 return true; 1947 1948 /* Both LD_IND and LD_ABS return 32-bit data. */ 1949 if (t != SRC_OP) 1950 return false; 1951 1952 /* Implicit ctx ptr. */ 1953 if (regno == BPF_REG_6) 1954 return true; 1955 1956 /* Explicit source could be any width. */ 1957 return true; 1958 } 1959 1960 if (class == BPF_ST) 1961 /* The only source register for BPF_ST is a ptr. */ 1962 return true; 1963 1964 /* Conservatively return true at default. */ 1965 return true; 1966 } 1967 1968 /* Return the regno defined by the insn, or -1. */ 1969 static int insn_def_regno(const struct bpf_insn *insn) 1970 { 1971 switch (BPF_CLASS(insn->code)) { 1972 case BPF_JMP: 1973 case BPF_JMP32: 1974 case BPF_ST: 1975 return -1; 1976 case BPF_STX: 1977 if (BPF_MODE(insn->code) == BPF_ATOMIC && 1978 (insn->imm & BPF_FETCH)) { 1979 if (insn->imm == BPF_CMPXCHG) 1980 return BPF_REG_0; 1981 else 1982 return insn->src_reg; 1983 } else { 1984 return -1; 1985 } 1986 default: 1987 return insn->dst_reg; 1988 } 1989 } 1990 1991 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 1992 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 1993 { 1994 int dst_reg = insn_def_regno(insn); 1995 1996 if (dst_reg == -1) 1997 return false; 1998 1999 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 2000 } 2001 2002 static void mark_insn_zext(struct bpf_verifier_env *env, 2003 struct bpf_reg_state *reg) 2004 { 2005 s32 def_idx = reg->subreg_def; 2006 2007 if (def_idx == DEF_NOT_SUBREG) 2008 return; 2009 2010 env->insn_aux_data[def_idx - 1].zext_dst = true; 2011 /* The dst will be zero extended, so won't be sub-register anymore. */ 2012 reg->subreg_def = DEF_NOT_SUBREG; 2013 } 2014 2015 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 2016 enum reg_arg_type t) 2017 { 2018 struct bpf_verifier_state *vstate = env->cur_state; 2019 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2020 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 2021 struct bpf_reg_state *reg, *regs = state->regs; 2022 bool rw64; 2023 2024 if (regno >= MAX_BPF_REG) { 2025 verbose(env, "R%d is invalid\n", regno); 2026 return -EINVAL; 2027 } 2028 2029 reg = ®s[regno]; 2030 rw64 = is_reg64(env, insn, regno, reg, t); 2031 if (t == SRC_OP) { 2032 /* check whether register used as source operand can be read */ 2033 if (reg->type == NOT_INIT) { 2034 verbose(env, "R%d !read_ok\n", regno); 2035 return -EACCES; 2036 } 2037 /* We don't need to worry about FP liveness because it's read-only */ 2038 if (regno == BPF_REG_FP) 2039 return 0; 2040 2041 if (rw64) 2042 mark_insn_zext(env, reg); 2043 2044 return mark_reg_read(env, reg, reg->parent, 2045 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 2046 } else { 2047 /* check whether register used as dest operand can be written to */ 2048 if (regno == BPF_REG_FP) { 2049 verbose(env, "frame pointer is read only\n"); 2050 return -EACCES; 2051 } 2052 reg->live |= REG_LIVE_WRITTEN; 2053 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 2054 if (t == DST_OP) 2055 mark_reg_unknown(env, regs, regno); 2056 } 2057 return 0; 2058 } 2059 2060 /* for any branch, call, exit record the history of jmps in the given state */ 2061 static int push_jmp_history(struct bpf_verifier_env *env, 2062 struct bpf_verifier_state *cur) 2063 { 2064 u32 cnt = cur->jmp_history_cnt; 2065 struct bpf_idx_pair *p; 2066 2067 cnt++; 2068 p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); 2069 if (!p) 2070 return -ENOMEM; 2071 p[cnt - 1].idx = env->insn_idx; 2072 p[cnt - 1].prev_idx = env->prev_insn_idx; 2073 cur->jmp_history = p; 2074 cur->jmp_history_cnt = cnt; 2075 return 0; 2076 } 2077 2078 /* Backtrack one insn at a time. If idx is not at the top of recorded 2079 * history then previous instruction came from straight line execution. 2080 */ 2081 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 2082 u32 *history) 2083 { 2084 u32 cnt = *history; 2085 2086 if (cnt && st->jmp_history[cnt - 1].idx == i) { 2087 i = st->jmp_history[cnt - 1].prev_idx; 2088 (*history)--; 2089 } else { 2090 i--; 2091 } 2092 return i; 2093 } 2094 2095 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 2096 { 2097 const struct btf_type *func; 2098 2099 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 2100 return NULL; 2101 2102 func = btf_type_by_id(btf_vmlinux, insn->imm); 2103 return btf_name_by_offset(btf_vmlinux, func->name_off); 2104 } 2105 2106 /* For given verifier state backtrack_insn() is called from the last insn to 2107 * the first insn. Its purpose is to compute a bitmask of registers and 2108 * stack slots that needs precision in the parent verifier state. 2109 */ 2110 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 2111 u32 *reg_mask, u64 *stack_mask) 2112 { 2113 const struct bpf_insn_cbs cbs = { 2114 .cb_call = disasm_kfunc_name, 2115 .cb_print = verbose, 2116 .private_data = env, 2117 }; 2118 struct bpf_insn *insn = env->prog->insnsi + idx; 2119 u8 class = BPF_CLASS(insn->code); 2120 u8 opcode = BPF_OP(insn->code); 2121 u8 mode = BPF_MODE(insn->code); 2122 u32 dreg = 1u << insn->dst_reg; 2123 u32 sreg = 1u << insn->src_reg; 2124 u32 spi; 2125 2126 if (insn->code == 0) 2127 return 0; 2128 if (env->log.level & BPF_LOG_LEVEL) { 2129 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 2130 verbose(env, "%d: ", idx); 2131 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 2132 } 2133 2134 if (class == BPF_ALU || class == BPF_ALU64) { 2135 if (!(*reg_mask & dreg)) 2136 return 0; 2137 if (opcode == BPF_MOV) { 2138 if (BPF_SRC(insn->code) == BPF_X) { 2139 /* dreg = sreg 2140 * dreg needs precision after this insn 2141 * sreg needs precision before this insn 2142 */ 2143 *reg_mask &= ~dreg; 2144 *reg_mask |= sreg; 2145 } else { 2146 /* dreg = K 2147 * dreg needs precision after this insn. 2148 * Corresponding register is already marked 2149 * as precise=true in this verifier state. 2150 * No further markings in parent are necessary 2151 */ 2152 *reg_mask &= ~dreg; 2153 } 2154 } else { 2155 if (BPF_SRC(insn->code) == BPF_X) { 2156 /* dreg += sreg 2157 * both dreg and sreg need precision 2158 * before this insn 2159 */ 2160 *reg_mask |= sreg; 2161 } /* else dreg += K 2162 * dreg still needs precision before this insn 2163 */ 2164 } 2165 } else if (class == BPF_LDX) { 2166 if (!(*reg_mask & dreg)) 2167 return 0; 2168 *reg_mask &= ~dreg; 2169 2170 /* scalars can only be spilled into stack w/o losing precision. 2171 * Load from any other memory can be zero extended. 2172 * The desire to keep that precision is already indicated 2173 * by 'precise' mark in corresponding register of this state. 2174 * No further tracking necessary. 2175 */ 2176 if (insn->src_reg != BPF_REG_FP) 2177 return 0; 2178 if (BPF_SIZE(insn->code) != BPF_DW) 2179 return 0; 2180 2181 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 2182 * that [fp - off] slot contains scalar that needs to be 2183 * tracked with precision 2184 */ 2185 spi = (-insn->off - 1) / BPF_REG_SIZE; 2186 if (spi >= 64) { 2187 verbose(env, "BUG spi %d\n", spi); 2188 WARN_ONCE(1, "verifier backtracking bug"); 2189 return -EFAULT; 2190 } 2191 *stack_mask |= 1ull << spi; 2192 } else if (class == BPF_STX || class == BPF_ST) { 2193 if (*reg_mask & dreg) 2194 /* stx & st shouldn't be using _scalar_ dst_reg 2195 * to access memory. It means backtracking 2196 * encountered a case of pointer subtraction. 2197 */ 2198 return -ENOTSUPP; 2199 /* scalars can only be spilled into stack */ 2200 if (insn->dst_reg != BPF_REG_FP) 2201 return 0; 2202 if (BPF_SIZE(insn->code) != BPF_DW) 2203 return 0; 2204 spi = (-insn->off - 1) / BPF_REG_SIZE; 2205 if (spi >= 64) { 2206 verbose(env, "BUG spi %d\n", spi); 2207 WARN_ONCE(1, "verifier backtracking bug"); 2208 return -EFAULT; 2209 } 2210 if (!(*stack_mask & (1ull << spi))) 2211 return 0; 2212 *stack_mask &= ~(1ull << spi); 2213 if (class == BPF_STX) 2214 *reg_mask |= sreg; 2215 } else if (class == BPF_JMP || class == BPF_JMP32) { 2216 if (opcode == BPF_CALL) { 2217 if (insn->src_reg == BPF_PSEUDO_CALL) 2218 return -ENOTSUPP; 2219 /* regular helper call sets R0 */ 2220 *reg_mask &= ~1; 2221 if (*reg_mask & 0x3f) { 2222 /* if backtracing was looking for registers R1-R5 2223 * they should have been found already. 2224 */ 2225 verbose(env, "BUG regs %x\n", *reg_mask); 2226 WARN_ONCE(1, "verifier backtracking bug"); 2227 return -EFAULT; 2228 } 2229 } else if (opcode == BPF_EXIT) { 2230 return -ENOTSUPP; 2231 } 2232 } else if (class == BPF_LD) { 2233 if (!(*reg_mask & dreg)) 2234 return 0; 2235 *reg_mask &= ~dreg; 2236 /* It's ld_imm64 or ld_abs or ld_ind. 2237 * For ld_imm64 no further tracking of precision 2238 * into parent is necessary 2239 */ 2240 if (mode == BPF_IND || mode == BPF_ABS) 2241 /* to be analyzed */ 2242 return -ENOTSUPP; 2243 } 2244 return 0; 2245 } 2246 2247 /* the scalar precision tracking algorithm: 2248 * . at the start all registers have precise=false. 2249 * . scalar ranges are tracked as normal through alu and jmp insns. 2250 * . once precise value of the scalar register is used in: 2251 * . ptr + scalar alu 2252 * . if (scalar cond K|scalar) 2253 * . helper_call(.., scalar, ...) where ARG_CONST is expected 2254 * backtrack through the verifier states and mark all registers and 2255 * stack slots with spilled constants that these scalar regisers 2256 * should be precise. 2257 * . during state pruning two registers (or spilled stack slots) 2258 * are equivalent if both are not precise. 2259 * 2260 * Note the verifier cannot simply walk register parentage chain, 2261 * since many different registers and stack slots could have been 2262 * used to compute single precise scalar. 2263 * 2264 * The approach of starting with precise=true for all registers and then 2265 * backtrack to mark a register as not precise when the verifier detects 2266 * that program doesn't care about specific value (e.g., when helper 2267 * takes register as ARG_ANYTHING parameter) is not safe. 2268 * 2269 * It's ok to walk single parentage chain of the verifier states. 2270 * It's possible that this backtracking will go all the way till 1st insn. 2271 * All other branches will be explored for needing precision later. 2272 * 2273 * The backtracking needs to deal with cases like: 2274 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 2275 * r9 -= r8 2276 * r5 = r9 2277 * if r5 > 0x79f goto pc+7 2278 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 2279 * r5 += 1 2280 * ... 2281 * call bpf_perf_event_output#25 2282 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 2283 * 2284 * and this case: 2285 * r6 = 1 2286 * call foo // uses callee's r6 inside to compute r0 2287 * r0 += r6 2288 * if r0 == 0 goto 2289 * 2290 * to track above reg_mask/stack_mask needs to be independent for each frame. 2291 * 2292 * Also if parent's curframe > frame where backtracking started, 2293 * the verifier need to mark registers in both frames, otherwise callees 2294 * may incorrectly prune callers. This is similar to 2295 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 2296 * 2297 * For now backtracking falls back into conservative marking. 2298 */ 2299 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 2300 struct bpf_verifier_state *st) 2301 { 2302 struct bpf_func_state *func; 2303 struct bpf_reg_state *reg; 2304 int i, j; 2305 2306 /* big hammer: mark all scalars precise in this path. 2307 * pop_stack may still get !precise scalars. 2308 */ 2309 for (; st; st = st->parent) 2310 for (i = 0; i <= st->curframe; i++) { 2311 func = st->frame[i]; 2312 for (j = 0; j < BPF_REG_FP; j++) { 2313 reg = &func->regs[j]; 2314 if (reg->type != SCALAR_VALUE) 2315 continue; 2316 reg->precise = true; 2317 } 2318 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2319 if (func->stack[j].slot_type[0] != STACK_SPILL) 2320 continue; 2321 reg = &func->stack[j].spilled_ptr; 2322 if (reg->type != SCALAR_VALUE) 2323 continue; 2324 reg->precise = true; 2325 } 2326 } 2327 } 2328 2329 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, 2330 int spi) 2331 { 2332 struct bpf_verifier_state *st = env->cur_state; 2333 int first_idx = st->first_insn_idx; 2334 int last_idx = env->insn_idx; 2335 struct bpf_func_state *func; 2336 struct bpf_reg_state *reg; 2337 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 2338 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 2339 bool skip_first = true; 2340 bool new_marks = false; 2341 int i, err; 2342 2343 if (!env->bpf_capable) 2344 return 0; 2345 2346 func = st->frame[st->curframe]; 2347 if (regno >= 0) { 2348 reg = &func->regs[regno]; 2349 if (reg->type != SCALAR_VALUE) { 2350 WARN_ONCE(1, "backtracing misuse"); 2351 return -EFAULT; 2352 } 2353 if (!reg->precise) 2354 new_marks = true; 2355 else 2356 reg_mask = 0; 2357 reg->precise = true; 2358 } 2359 2360 while (spi >= 0) { 2361 if (func->stack[spi].slot_type[0] != STACK_SPILL) { 2362 stack_mask = 0; 2363 break; 2364 } 2365 reg = &func->stack[spi].spilled_ptr; 2366 if (reg->type != SCALAR_VALUE) { 2367 stack_mask = 0; 2368 break; 2369 } 2370 if (!reg->precise) 2371 new_marks = true; 2372 else 2373 stack_mask = 0; 2374 reg->precise = true; 2375 break; 2376 } 2377 2378 if (!new_marks) 2379 return 0; 2380 if (!reg_mask && !stack_mask) 2381 return 0; 2382 for (;;) { 2383 DECLARE_BITMAP(mask, 64); 2384 u32 history = st->jmp_history_cnt; 2385 2386 if (env->log.level & BPF_LOG_LEVEL) 2387 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 2388 for (i = last_idx;;) { 2389 if (skip_first) { 2390 err = 0; 2391 skip_first = false; 2392 } else { 2393 err = backtrack_insn(env, i, ®_mask, &stack_mask); 2394 } 2395 if (err == -ENOTSUPP) { 2396 mark_all_scalars_precise(env, st); 2397 return 0; 2398 } else if (err) { 2399 return err; 2400 } 2401 if (!reg_mask && !stack_mask) 2402 /* Found assignment(s) into tracked register in this state. 2403 * Since this state is already marked, just return. 2404 * Nothing to be tracked further in the parent state. 2405 */ 2406 return 0; 2407 if (i == first_idx) 2408 break; 2409 i = get_prev_insn_idx(st, i, &history); 2410 if (i >= env->prog->len) { 2411 /* This can happen if backtracking reached insn 0 2412 * and there are still reg_mask or stack_mask 2413 * to backtrack. 2414 * It means the backtracking missed the spot where 2415 * particular register was initialized with a constant. 2416 */ 2417 verbose(env, "BUG backtracking idx %d\n", i); 2418 WARN_ONCE(1, "verifier backtracking bug"); 2419 return -EFAULT; 2420 } 2421 } 2422 st = st->parent; 2423 if (!st) 2424 break; 2425 2426 new_marks = false; 2427 func = st->frame[st->curframe]; 2428 bitmap_from_u64(mask, reg_mask); 2429 for_each_set_bit(i, mask, 32) { 2430 reg = &func->regs[i]; 2431 if (reg->type != SCALAR_VALUE) { 2432 reg_mask &= ~(1u << i); 2433 continue; 2434 } 2435 if (!reg->precise) 2436 new_marks = true; 2437 reg->precise = true; 2438 } 2439 2440 bitmap_from_u64(mask, stack_mask); 2441 for_each_set_bit(i, mask, 64) { 2442 if (i >= func->allocated_stack / BPF_REG_SIZE) { 2443 /* the sequence of instructions: 2444 * 2: (bf) r3 = r10 2445 * 3: (7b) *(u64 *)(r3 -8) = r0 2446 * 4: (79) r4 = *(u64 *)(r10 -8) 2447 * doesn't contain jmps. It's backtracked 2448 * as a single block. 2449 * During backtracking insn 3 is not recognized as 2450 * stack access, so at the end of backtracking 2451 * stack slot fp-8 is still marked in stack_mask. 2452 * However the parent state may not have accessed 2453 * fp-8 and it's "unallocated" stack space. 2454 * In such case fallback to conservative. 2455 */ 2456 mark_all_scalars_precise(env, st); 2457 return 0; 2458 } 2459 2460 if (func->stack[i].slot_type[0] != STACK_SPILL) { 2461 stack_mask &= ~(1ull << i); 2462 continue; 2463 } 2464 reg = &func->stack[i].spilled_ptr; 2465 if (reg->type != SCALAR_VALUE) { 2466 stack_mask &= ~(1ull << i); 2467 continue; 2468 } 2469 if (!reg->precise) 2470 new_marks = true; 2471 reg->precise = true; 2472 } 2473 if (env->log.level & BPF_LOG_LEVEL) { 2474 print_verifier_state(env, func); 2475 verbose(env, "parent %s regs=%x stack=%llx marks\n", 2476 new_marks ? "didn't have" : "already had", 2477 reg_mask, stack_mask); 2478 } 2479 2480 if (!reg_mask && !stack_mask) 2481 break; 2482 if (!new_marks) 2483 break; 2484 2485 last_idx = st->last_insn_idx; 2486 first_idx = st->first_insn_idx; 2487 } 2488 return 0; 2489 } 2490 2491 static int mark_chain_precision(struct bpf_verifier_env *env, int regno) 2492 { 2493 return __mark_chain_precision(env, regno, -1); 2494 } 2495 2496 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) 2497 { 2498 return __mark_chain_precision(env, -1, spi); 2499 } 2500 2501 static bool is_spillable_regtype(enum bpf_reg_type type) 2502 { 2503 switch (type) { 2504 case PTR_TO_MAP_VALUE: 2505 case PTR_TO_MAP_VALUE_OR_NULL: 2506 case PTR_TO_STACK: 2507 case PTR_TO_CTX: 2508 case PTR_TO_PACKET: 2509 case PTR_TO_PACKET_META: 2510 case PTR_TO_PACKET_END: 2511 case PTR_TO_FLOW_KEYS: 2512 case CONST_PTR_TO_MAP: 2513 case PTR_TO_SOCKET: 2514 case PTR_TO_SOCKET_OR_NULL: 2515 case PTR_TO_SOCK_COMMON: 2516 case PTR_TO_SOCK_COMMON_OR_NULL: 2517 case PTR_TO_TCP_SOCK: 2518 case PTR_TO_TCP_SOCK_OR_NULL: 2519 case PTR_TO_XDP_SOCK: 2520 case PTR_TO_BTF_ID: 2521 case PTR_TO_BTF_ID_OR_NULL: 2522 case PTR_TO_RDONLY_BUF: 2523 case PTR_TO_RDONLY_BUF_OR_NULL: 2524 case PTR_TO_RDWR_BUF: 2525 case PTR_TO_RDWR_BUF_OR_NULL: 2526 case PTR_TO_PERCPU_BTF_ID: 2527 case PTR_TO_MEM: 2528 case PTR_TO_MEM_OR_NULL: 2529 case PTR_TO_FUNC: 2530 case PTR_TO_MAP_KEY: 2531 return true; 2532 default: 2533 return false; 2534 } 2535 } 2536 2537 /* Does this register contain a constant zero? */ 2538 static bool register_is_null(struct bpf_reg_state *reg) 2539 { 2540 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 2541 } 2542 2543 static bool register_is_const(struct bpf_reg_state *reg) 2544 { 2545 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 2546 } 2547 2548 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 2549 { 2550 return tnum_is_unknown(reg->var_off) && 2551 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 2552 reg->umin_value == 0 && reg->umax_value == U64_MAX && 2553 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 2554 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 2555 } 2556 2557 static bool register_is_bounded(struct bpf_reg_state *reg) 2558 { 2559 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 2560 } 2561 2562 static bool __is_pointer_value(bool allow_ptr_leaks, 2563 const struct bpf_reg_state *reg) 2564 { 2565 if (allow_ptr_leaks) 2566 return false; 2567 2568 return reg->type != SCALAR_VALUE; 2569 } 2570 2571 static void save_register_state(struct bpf_func_state *state, 2572 int spi, struct bpf_reg_state *reg) 2573 { 2574 int i; 2575 2576 state->stack[spi].spilled_ptr = *reg; 2577 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2578 2579 for (i = 0; i < BPF_REG_SIZE; i++) 2580 state->stack[spi].slot_type[i] = STACK_SPILL; 2581 } 2582 2583 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 2584 * stack boundary and alignment are checked in check_mem_access() 2585 */ 2586 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 2587 /* stack frame we're writing to */ 2588 struct bpf_func_state *state, 2589 int off, int size, int value_regno, 2590 int insn_idx) 2591 { 2592 struct bpf_func_state *cur; /* state of the current function */ 2593 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 2594 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; 2595 struct bpf_reg_state *reg = NULL; 2596 2597 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 2598 if (err) 2599 return err; 2600 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 2601 * so it's aligned access and [off, off + size) are within stack limits 2602 */ 2603 if (!env->allow_ptr_leaks && 2604 state->stack[spi].slot_type[0] == STACK_SPILL && 2605 size != BPF_REG_SIZE) { 2606 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 2607 return -EACCES; 2608 } 2609 2610 cur = env->cur_state->frame[env->cur_state->curframe]; 2611 if (value_regno >= 0) 2612 reg = &cur->regs[value_regno]; 2613 2614 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && 2615 !register_is_null(reg) && env->bpf_capable) { 2616 if (dst_reg != BPF_REG_FP) { 2617 /* The backtracking logic can only recognize explicit 2618 * stack slot address like [fp - 8]. Other spill of 2619 * scalar via different register has to be conervative. 2620 * Backtrack from here and mark all registers as precise 2621 * that contributed into 'reg' being a constant. 2622 */ 2623 err = mark_chain_precision(env, value_regno); 2624 if (err) 2625 return err; 2626 } 2627 save_register_state(state, spi, reg); 2628 } else if (reg && is_spillable_regtype(reg->type)) { 2629 /* register containing pointer is being spilled into stack */ 2630 if (size != BPF_REG_SIZE) { 2631 verbose_linfo(env, insn_idx, "; "); 2632 verbose(env, "invalid size of register spill\n"); 2633 return -EACCES; 2634 } 2635 2636 if (state != cur && reg->type == PTR_TO_STACK) { 2637 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 2638 return -EINVAL; 2639 } 2640 2641 if (!env->bypass_spec_v4) { 2642 bool sanitize = false; 2643 2644 if (state->stack[spi].slot_type[0] == STACK_SPILL && 2645 register_is_const(&state->stack[spi].spilled_ptr)) 2646 sanitize = true; 2647 for (i = 0; i < BPF_REG_SIZE; i++) 2648 if (state->stack[spi].slot_type[i] == STACK_MISC) { 2649 sanitize = true; 2650 break; 2651 } 2652 if (sanitize) { 2653 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; 2654 int soff = (-spi - 1) * BPF_REG_SIZE; 2655 2656 /* detected reuse of integer stack slot with a pointer 2657 * which means either llvm is reusing stack slot or 2658 * an attacker is trying to exploit CVE-2018-3639 2659 * (speculative store bypass) 2660 * Have to sanitize that slot with preemptive 2661 * store of zero. 2662 */ 2663 if (*poff && *poff != soff) { 2664 /* disallow programs where single insn stores 2665 * into two different stack slots, since verifier 2666 * cannot sanitize them 2667 */ 2668 verbose(env, 2669 "insn %d cannot access two stack slots fp%d and fp%d", 2670 insn_idx, *poff, soff); 2671 return -EINVAL; 2672 } 2673 *poff = soff; 2674 } 2675 } 2676 save_register_state(state, spi, reg); 2677 } else { 2678 u8 type = STACK_MISC; 2679 2680 /* regular write of data into stack destroys any spilled ptr */ 2681 state->stack[spi].spilled_ptr.type = NOT_INIT; 2682 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 2683 if (state->stack[spi].slot_type[0] == STACK_SPILL) 2684 for (i = 0; i < BPF_REG_SIZE; i++) 2685 state->stack[spi].slot_type[i] = STACK_MISC; 2686 2687 /* only mark the slot as written if all 8 bytes were written 2688 * otherwise read propagation may incorrectly stop too soon 2689 * when stack slots are partially written. 2690 * This heuristic means that read propagation will be 2691 * conservative, since it will add reg_live_read marks 2692 * to stack slots all the way to first state when programs 2693 * writes+reads less than 8 bytes 2694 */ 2695 if (size == BPF_REG_SIZE) 2696 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2697 2698 /* when we zero initialize stack slots mark them as such */ 2699 if (reg && register_is_null(reg)) { 2700 /* backtracking doesn't work for STACK_ZERO yet. */ 2701 err = mark_chain_precision(env, value_regno); 2702 if (err) 2703 return err; 2704 type = STACK_ZERO; 2705 } 2706 2707 /* Mark slots affected by this stack write. */ 2708 for (i = 0; i < size; i++) 2709 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 2710 type; 2711 } 2712 return 0; 2713 } 2714 2715 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 2716 * known to contain a variable offset. 2717 * This function checks whether the write is permitted and conservatively 2718 * tracks the effects of the write, considering that each stack slot in the 2719 * dynamic range is potentially written to. 2720 * 2721 * 'off' includes 'regno->off'. 2722 * 'value_regno' can be -1, meaning that an unknown value is being written to 2723 * the stack. 2724 * 2725 * Spilled pointers in range are not marked as written because we don't know 2726 * what's going to be actually written. This means that read propagation for 2727 * future reads cannot be terminated by this write. 2728 * 2729 * For privileged programs, uninitialized stack slots are considered 2730 * initialized by this write (even though we don't know exactly what offsets 2731 * are going to be written to). The idea is that we don't want the verifier to 2732 * reject future reads that access slots written to through variable offsets. 2733 */ 2734 static int check_stack_write_var_off(struct bpf_verifier_env *env, 2735 /* func where register points to */ 2736 struct bpf_func_state *state, 2737 int ptr_regno, int off, int size, 2738 int value_regno, int insn_idx) 2739 { 2740 struct bpf_func_state *cur; /* state of the current function */ 2741 int min_off, max_off; 2742 int i, err; 2743 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 2744 bool writing_zero = false; 2745 /* set if the fact that we're writing a zero is used to let any 2746 * stack slots remain STACK_ZERO 2747 */ 2748 bool zero_used = false; 2749 2750 cur = env->cur_state->frame[env->cur_state->curframe]; 2751 ptr_reg = &cur->regs[ptr_regno]; 2752 min_off = ptr_reg->smin_value + off; 2753 max_off = ptr_reg->smax_value + off + size; 2754 if (value_regno >= 0) 2755 value_reg = &cur->regs[value_regno]; 2756 if (value_reg && register_is_null(value_reg)) 2757 writing_zero = true; 2758 2759 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 2760 if (err) 2761 return err; 2762 2763 2764 /* Variable offset writes destroy any spilled pointers in range. */ 2765 for (i = min_off; i < max_off; i++) { 2766 u8 new_type, *stype; 2767 int slot, spi; 2768 2769 slot = -i - 1; 2770 spi = slot / BPF_REG_SIZE; 2771 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 2772 2773 if (!env->allow_ptr_leaks 2774 && *stype != NOT_INIT 2775 && *stype != SCALAR_VALUE) { 2776 /* Reject the write if there's are spilled pointers in 2777 * range. If we didn't reject here, the ptr status 2778 * would be erased below (even though not all slots are 2779 * actually overwritten), possibly opening the door to 2780 * leaks. 2781 */ 2782 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 2783 insn_idx, i); 2784 return -EINVAL; 2785 } 2786 2787 /* Erase all spilled pointers. */ 2788 state->stack[spi].spilled_ptr.type = NOT_INIT; 2789 2790 /* Update the slot type. */ 2791 new_type = STACK_MISC; 2792 if (writing_zero && *stype == STACK_ZERO) { 2793 new_type = STACK_ZERO; 2794 zero_used = true; 2795 } 2796 /* If the slot is STACK_INVALID, we check whether it's OK to 2797 * pretend that it will be initialized by this write. The slot 2798 * might not actually be written to, and so if we mark it as 2799 * initialized future reads might leak uninitialized memory. 2800 * For privileged programs, we will accept such reads to slots 2801 * that may or may not be written because, if we're reject 2802 * them, the error would be too confusing. 2803 */ 2804 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 2805 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 2806 insn_idx, i); 2807 return -EINVAL; 2808 } 2809 *stype = new_type; 2810 } 2811 if (zero_used) { 2812 /* backtracking doesn't work for STACK_ZERO yet. */ 2813 err = mark_chain_precision(env, value_regno); 2814 if (err) 2815 return err; 2816 } 2817 return 0; 2818 } 2819 2820 /* When register 'dst_regno' is assigned some values from stack[min_off, 2821 * max_off), we set the register's type according to the types of the 2822 * respective stack slots. If all the stack values are known to be zeros, then 2823 * so is the destination reg. Otherwise, the register is considered to be 2824 * SCALAR. This function does not deal with register filling; the caller must 2825 * ensure that all spilled registers in the stack range have been marked as 2826 * read. 2827 */ 2828 static void mark_reg_stack_read(struct bpf_verifier_env *env, 2829 /* func where src register points to */ 2830 struct bpf_func_state *ptr_state, 2831 int min_off, int max_off, int dst_regno) 2832 { 2833 struct bpf_verifier_state *vstate = env->cur_state; 2834 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2835 int i, slot, spi; 2836 u8 *stype; 2837 int zeros = 0; 2838 2839 for (i = min_off; i < max_off; i++) { 2840 slot = -i - 1; 2841 spi = slot / BPF_REG_SIZE; 2842 stype = ptr_state->stack[spi].slot_type; 2843 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 2844 break; 2845 zeros++; 2846 } 2847 if (zeros == max_off - min_off) { 2848 /* any access_size read into register is zero extended, 2849 * so the whole register == const_zero 2850 */ 2851 __mark_reg_const_zero(&state->regs[dst_regno]); 2852 /* backtracking doesn't support STACK_ZERO yet, 2853 * so mark it precise here, so that later 2854 * backtracking can stop here. 2855 * Backtracking may not need this if this register 2856 * doesn't participate in pointer adjustment. 2857 * Forward propagation of precise flag is not 2858 * necessary either. This mark is only to stop 2859 * backtracking. Any register that contributed 2860 * to const 0 was marked precise before spill. 2861 */ 2862 state->regs[dst_regno].precise = true; 2863 } else { 2864 /* have read misc data from the stack */ 2865 mark_reg_unknown(env, state->regs, dst_regno); 2866 } 2867 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 2868 } 2869 2870 /* Read the stack at 'off' and put the results into the register indicated by 2871 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 2872 * spilled reg. 2873 * 2874 * 'dst_regno' can be -1, meaning that the read value is not going to a 2875 * register. 2876 * 2877 * The access is assumed to be within the current stack bounds. 2878 */ 2879 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 2880 /* func where src register points to */ 2881 struct bpf_func_state *reg_state, 2882 int off, int size, int dst_regno) 2883 { 2884 struct bpf_verifier_state *vstate = env->cur_state; 2885 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2886 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 2887 struct bpf_reg_state *reg; 2888 u8 *stype; 2889 2890 stype = reg_state->stack[spi].slot_type; 2891 reg = ®_state->stack[spi].spilled_ptr; 2892 2893 if (stype[0] == STACK_SPILL) { 2894 if (size != BPF_REG_SIZE) { 2895 if (reg->type != SCALAR_VALUE) { 2896 verbose_linfo(env, env->insn_idx, "; "); 2897 verbose(env, "invalid size of register fill\n"); 2898 return -EACCES; 2899 } 2900 if (dst_regno >= 0) { 2901 mark_reg_unknown(env, state->regs, dst_regno); 2902 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 2903 } 2904 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2905 return 0; 2906 } 2907 for (i = 1; i < BPF_REG_SIZE; i++) { 2908 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { 2909 verbose(env, "corrupted spill memory\n"); 2910 return -EACCES; 2911 } 2912 } 2913 2914 if (dst_regno >= 0) { 2915 /* restore register state from stack */ 2916 state->regs[dst_regno] = *reg; 2917 /* mark reg as written since spilled pointer state likely 2918 * has its liveness marks cleared by is_state_visited() 2919 * which resets stack/reg liveness for state transitions 2920 */ 2921 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 2922 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 2923 /* If dst_regno==-1, the caller is asking us whether 2924 * it is acceptable to use this value as a SCALAR_VALUE 2925 * (e.g. for XADD). 2926 * We must not allow unprivileged callers to do that 2927 * with spilled pointers. 2928 */ 2929 verbose(env, "leaking pointer from stack off %d\n", 2930 off); 2931 return -EACCES; 2932 } 2933 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2934 } else { 2935 u8 type; 2936 2937 for (i = 0; i < size; i++) { 2938 type = stype[(slot - i) % BPF_REG_SIZE]; 2939 if (type == STACK_MISC) 2940 continue; 2941 if (type == STACK_ZERO) 2942 continue; 2943 verbose(env, "invalid read from stack off %d+%d size %d\n", 2944 off, i, size); 2945 return -EACCES; 2946 } 2947 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2948 if (dst_regno >= 0) 2949 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 2950 } 2951 return 0; 2952 } 2953 2954 enum stack_access_src { 2955 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 2956 ACCESS_HELPER = 2, /* the access is performed by a helper */ 2957 }; 2958 2959 static int check_stack_range_initialized(struct bpf_verifier_env *env, 2960 int regno, int off, int access_size, 2961 bool zero_size_allowed, 2962 enum stack_access_src type, 2963 struct bpf_call_arg_meta *meta); 2964 2965 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 2966 { 2967 return cur_regs(env) + regno; 2968 } 2969 2970 /* Read the stack at 'ptr_regno + off' and put the result into the register 2971 * 'dst_regno'. 2972 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 2973 * but not its variable offset. 2974 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 2975 * 2976 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 2977 * filling registers (i.e. reads of spilled register cannot be detected when 2978 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 2979 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 2980 * offset; for a fixed offset check_stack_read_fixed_off should be used 2981 * instead. 2982 */ 2983 static int check_stack_read_var_off(struct bpf_verifier_env *env, 2984 int ptr_regno, int off, int size, int dst_regno) 2985 { 2986 /* The state of the source register. */ 2987 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 2988 struct bpf_func_state *ptr_state = func(env, reg); 2989 int err; 2990 int min_off, max_off; 2991 2992 /* Note that we pass a NULL meta, so raw access will not be permitted. 2993 */ 2994 err = check_stack_range_initialized(env, ptr_regno, off, size, 2995 false, ACCESS_DIRECT, NULL); 2996 if (err) 2997 return err; 2998 2999 min_off = reg->smin_value + off; 3000 max_off = reg->smax_value + off; 3001 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 3002 return 0; 3003 } 3004 3005 /* check_stack_read dispatches to check_stack_read_fixed_off or 3006 * check_stack_read_var_off. 3007 * 3008 * The caller must ensure that the offset falls within the allocated stack 3009 * bounds. 3010 * 3011 * 'dst_regno' is a register which will receive the value from the stack. It 3012 * can be -1, meaning that the read value is not going to a register. 3013 */ 3014 static int check_stack_read(struct bpf_verifier_env *env, 3015 int ptr_regno, int off, int size, 3016 int dst_regno) 3017 { 3018 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3019 struct bpf_func_state *state = func(env, reg); 3020 int err; 3021 /* Some accesses are only permitted with a static offset. */ 3022 bool var_off = !tnum_is_const(reg->var_off); 3023 3024 /* The offset is required to be static when reads don't go to a 3025 * register, in order to not leak pointers (see 3026 * check_stack_read_fixed_off). 3027 */ 3028 if (dst_regno < 0 && var_off) { 3029 char tn_buf[48]; 3030 3031 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3032 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 3033 tn_buf, off, size); 3034 return -EACCES; 3035 } 3036 /* Variable offset is prohibited for unprivileged mode for simplicity 3037 * since it requires corresponding support in Spectre masking for stack 3038 * ALU. See also retrieve_ptr_limit(). 3039 */ 3040 if (!env->bypass_spec_v1 && var_off) { 3041 char tn_buf[48]; 3042 3043 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3044 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 3045 ptr_regno, tn_buf); 3046 return -EACCES; 3047 } 3048 3049 if (!var_off) { 3050 off += reg->var_off.value; 3051 err = check_stack_read_fixed_off(env, state, off, size, 3052 dst_regno); 3053 } else { 3054 /* Variable offset stack reads need more conservative handling 3055 * than fixed offset ones. Note that dst_regno >= 0 on this 3056 * branch. 3057 */ 3058 err = check_stack_read_var_off(env, ptr_regno, off, size, 3059 dst_regno); 3060 } 3061 return err; 3062 } 3063 3064 3065 /* check_stack_write dispatches to check_stack_write_fixed_off or 3066 * check_stack_write_var_off. 3067 * 3068 * 'ptr_regno' is the register used as a pointer into the stack. 3069 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 3070 * 'value_regno' is the register whose value we're writing to the stack. It can 3071 * be -1, meaning that we're not writing from a register. 3072 * 3073 * The caller must ensure that the offset falls within the maximum stack size. 3074 */ 3075 static int check_stack_write(struct bpf_verifier_env *env, 3076 int ptr_regno, int off, int size, 3077 int value_regno, int insn_idx) 3078 { 3079 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3080 struct bpf_func_state *state = func(env, reg); 3081 int err; 3082 3083 if (tnum_is_const(reg->var_off)) { 3084 off += reg->var_off.value; 3085 err = check_stack_write_fixed_off(env, state, off, size, 3086 value_regno, insn_idx); 3087 } else { 3088 /* Variable offset stack reads need more conservative handling 3089 * than fixed offset ones. 3090 */ 3091 err = check_stack_write_var_off(env, state, 3092 ptr_regno, off, size, 3093 value_regno, insn_idx); 3094 } 3095 return err; 3096 } 3097 3098 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 3099 int off, int size, enum bpf_access_type type) 3100 { 3101 struct bpf_reg_state *regs = cur_regs(env); 3102 struct bpf_map *map = regs[regno].map_ptr; 3103 u32 cap = bpf_map_flags_to_cap(map); 3104 3105 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 3106 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 3107 map->value_size, off, size); 3108 return -EACCES; 3109 } 3110 3111 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 3112 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 3113 map->value_size, off, size); 3114 return -EACCES; 3115 } 3116 3117 return 0; 3118 } 3119 3120 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 3121 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 3122 int off, int size, u32 mem_size, 3123 bool zero_size_allowed) 3124 { 3125 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 3126 struct bpf_reg_state *reg; 3127 3128 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 3129 return 0; 3130 3131 reg = &cur_regs(env)[regno]; 3132 switch (reg->type) { 3133 case PTR_TO_MAP_KEY: 3134 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 3135 mem_size, off, size); 3136 break; 3137 case PTR_TO_MAP_VALUE: 3138 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 3139 mem_size, off, size); 3140 break; 3141 case PTR_TO_PACKET: 3142 case PTR_TO_PACKET_META: 3143 case PTR_TO_PACKET_END: 3144 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 3145 off, size, regno, reg->id, off, mem_size); 3146 break; 3147 case PTR_TO_MEM: 3148 default: 3149 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 3150 mem_size, off, size); 3151 } 3152 3153 return -EACCES; 3154 } 3155 3156 /* check read/write into a memory region with possible variable offset */ 3157 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 3158 int off, int size, u32 mem_size, 3159 bool zero_size_allowed) 3160 { 3161 struct bpf_verifier_state *vstate = env->cur_state; 3162 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3163 struct bpf_reg_state *reg = &state->regs[regno]; 3164 int err; 3165 3166 /* We may have adjusted the register pointing to memory region, so we 3167 * need to try adding each of min_value and max_value to off 3168 * to make sure our theoretical access will be safe. 3169 */ 3170 if (env->log.level & BPF_LOG_LEVEL) 3171 print_verifier_state(env, state); 3172 3173 /* The minimum value is only important with signed 3174 * comparisons where we can't assume the floor of a 3175 * value is 0. If we are using signed variables for our 3176 * index'es we need to make sure that whatever we use 3177 * will have a set floor within our range. 3178 */ 3179 if (reg->smin_value < 0 && 3180 (reg->smin_value == S64_MIN || 3181 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 3182 reg->smin_value + off < 0)) { 3183 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3184 regno); 3185 return -EACCES; 3186 } 3187 err = __check_mem_access(env, regno, reg->smin_value + off, size, 3188 mem_size, zero_size_allowed); 3189 if (err) { 3190 verbose(env, "R%d min value is outside of the allowed memory range\n", 3191 regno); 3192 return err; 3193 } 3194 3195 /* If we haven't set a max value then we need to bail since we can't be 3196 * sure we won't do bad things. 3197 * If reg->umax_value + off could overflow, treat that as unbounded too. 3198 */ 3199 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 3200 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 3201 regno); 3202 return -EACCES; 3203 } 3204 err = __check_mem_access(env, regno, reg->umax_value + off, size, 3205 mem_size, zero_size_allowed); 3206 if (err) { 3207 verbose(env, "R%d max value is outside of the allowed memory range\n", 3208 regno); 3209 return err; 3210 } 3211 3212 return 0; 3213 } 3214 3215 /* check read/write into a map element with possible variable offset */ 3216 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 3217 int off, int size, bool zero_size_allowed) 3218 { 3219 struct bpf_verifier_state *vstate = env->cur_state; 3220 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3221 struct bpf_reg_state *reg = &state->regs[regno]; 3222 struct bpf_map *map = reg->map_ptr; 3223 int err; 3224 3225 err = check_mem_region_access(env, regno, off, size, map->value_size, 3226 zero_size_allowed); 3227 if (err) 3228 return err; 3229 3230 if (map_value_has_spin_lock(map)) { 3231 u32 lock = map->spin_lock_off; 3232 3233 /* if any part of struct bpf_spin_lock can be touched by 3234 * load/store reject this program. 3235 * To check that [x1, x2) overlaps with [y1, y2) 3236 * it is sufficient to check x1 < y2 && y1 < x2. 3237 */ 3238 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && 3239 lock < reg->umax_value + off + size) { 3240 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); 3241 return -EACCES; 3242 } 3243 } 3244 return err; 3245 } 3246 3247 #define MAX_PACKET_OFF 0xffff 3248 3249 static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) 3250 { 3251 return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; 3252 } 3253 3254 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 3255 const struct bpf_call_arg_meta *meta, 3256 enum bpf_access_type t) 3257 { 3258 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 3259 3260 switch (prog_type) { 3261 /* Program types only with direct read access go here! */ 3262 case BPF_PROG_TYPE_LWT_IN: 3263 case BPF_PROG_TYPE_LWT_OUT: 3264 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 3265 case BPF_PROG_TYPE_SK_REUSEPORT: 3266 case BPF_PROG_TYPE_FLOW_DISSECTOR: 3267 case BPF_PROG_TYPE_CGROUP_SKB: 3268 if (t == BPF_WRITE) 3269 return false; 3270 fallthrough; 3271 3272 /* Program types with direct read + write access go here! */ 3273 case BPF_PROG_TYPE_SCHED_CLS: 3274 case BPF_PROG_TYPE_SCHED_ACT: 3275 case BPF_PROG_TYPE_XDP: 3276 case BPF_PROG_TYPE_LWT_XMIT: 3277 case BPF_PROG_TYPE_SK_SKB: 3278 case BPF_PROG_TYPE_SK_MSG: 3279 if (meta) 3280 return meta->pkt_access; 3281 3282 env->seen_direct_write = true; 3283 return true; 3284 3285 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3286 if (t == BPF_WRITE) 3287 env->seen_direct_write = true; 3288 3289 return true; 3290 3291 default: 3292 return false; 3293 } 3294 } 3295 3296 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 3297 int size, bool zero_size_allowed) 3298 { 3299 struct bpf_reg_state *regs = cur_regs(env); 3300 struct bpf_reg_state *reg = ®s[regno]; 3301 int err; 3302 3303 /* We may have added a variable offset to the packet pointer; but any 3304 * reg->range we have comes after that. We are only checking the fixed 3305 * offset. 3306 */ 3307 3308 /* We don't allow negative numbers, because we aren't tracking enough 3309 * detail to prove they're safe. 3310 */ 3311 if (reg->smin_value < 0) { 3312 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3313 regno); 3314 return -EACCES; 3315 } 3316 3317 err = reg->range < 0 ? -EINVAL : 3318 __check_mem_access(env, regno, off, size, reg->range, 3319 zero_size_allowed); 3320 if (err) { 3321 verbose(env, "R%d offset is outside of the packet\n", regno); 3322 return err; 3323 } 3324 3325 /* __check_mem_access has made sure "off + size - 1" is within u16. 3326 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 3327 * otherwise find_good_pkt_pointers would have refused to set range info 3328 * that __check_mem_access would have rejected this pkt access. 3329 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 3330 */ 3331 env->prog->aux->max_pkt_offset = 3332 max_t(u32, env->prog->aux->max_pkt_offset, 3333 off + reg->umax_value + size - 1); 3334 3335 return err; 3336 } 3337 3338 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 3339 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 3340 enum bpf_access_type t, enum bpf_reg_type *reg_type, 3341 struct btf **btf, u32 *btf_id) 3342 { 3343 struct bpf_insn_access_aux info = { 3344 .reg_type = *reg_type, 3345 .log = &env->log, 3346 }; 3347 3348 if (env->ops->is_valid_access && 3349 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 3350 /* A non zero info.ctx_field_size indicates that this field is a 3351 * candidate for later verifier transformation to load the whole 3352 * field and then apply a mask when accessed with a narrower 3353 * access than actual ctx access size. A zero info.ctx_field_size 3354 * will only allow for whole field access and rejects any other 3355 * type of narrower access. 3356 */ 3357 *reg_type = info.reg_type; 3358 3359 if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) { 3360 *btf = info.btf; 3361 *btf_id = info.btf_id; 3362 } else { 3363 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 3364 } 3365 /* remember the offset of last byte accessed in ctx */ 3366 if (env->prog->aux->max_ctx_offset < off + size) 3367 env->prog->aux->max_ctx_offset = off + size; 3368 return 0; 3369 } 3370 3371 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 3372 return -EACCES; 3373 } 3374 3375 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 3376 int size) 3377 { 3378 if (size < 0 || off < 0 || 3379 (u64)off + size > sizeof(struct bpf_flow_keys)) { 3380 verbose(env, "invalid access to flow keys off=%d size=%d\n", 3381 off, size); 3382 return -EACCES; 3383 } 3384 return 0; 3385 } 3386 3387 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 3388 u32 regno, int off, int size, 3389 enum bpf_access_type t) 3390 { 3391 struct bpf_reg_state *regs = cur_regs(env); 3392 struct bpf_reg_state *reg = ®s[regno]; 3393 struct bpf_insn_access_aux info = {}; 3394 bool valid; 3395 3396 if (reg->smin_value < 0) { 3397 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3398 regno); 3399 return -EACCES; 3400 } 3401 3402 switch (reg->type) { 3403 case PTR_TO_SOCK_COMMON: 3404 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 3405 break; 3406 case PTR_TO_SOCKET: 3407 valid = bpf_sock_is_valid_access(off, size, t, &info); 3408 break; 3409 case PTR_TO_TCP_SOCK: 3410 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 3411 break; 3412 case PTR_TO_XDP_SOCK: 3413 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 3414 break; 3415 default: 3416 valid = false; 3417 } 3418 3419 3420 if (valid) { 3421 env->insn_aux_data[insn_idx].ctx_field_size = 3422 info.ctx_field_size; 3423 return 0; 3424 } 3425 3426 verbose(env, "R%d invalid %s access off=%d size=%d\n", 3427 regno, reg_type_str[reg->type], off, size); 3428 3429 return -EACCES; 3430 } 3431 3432 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 3433 { 3434 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 3435 } 3436 3437 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 3438 { 3439 const struct bpf_reg_state *reg = reg_state(env, regno); 3440 3441 return reg->type == PTR_TO_CTX; 3442 } 3443 3444 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 3445 { 3446 const struct bpf_reg_state *reg = reg_state(env, regno); 3447 3448 return type_is_sk_pointer(reg->type); 3449 } 3450 3451 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 3452 { 3453 const struct bpf_reg_state *reg = reg_state(env, regno); 3454 3455 return type_is_pkt_pointer(reg->type); 3456 } 3457 3458 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 3459 { 3460 const struct bpf_reg_state *reg = reg_state(env, regno); 3461 3462 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 3463 return reg->type == PTR_TO_FLOW_KEYS; 3464 } 3465 3466 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 3467 const struct bpf_reg_state *reg, 3468 int off, int size, bool strict) 3469 { 3470 struct tnum reg_off; 3471 int ip_align; 3472 3473 /* Byte size accesses are always allowed. */ 3474 if (!strict || size == 1) 3475 return 0; 3476 3477 /* For platforms that do not have a Kconfig enabling 3478 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 3479 * NET_IP_ALIGN is universally set to '2'. And on platforms 3480 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 3481 * to this code only in strict mode where we want to emulate 3482 * the NET_IP_ALIGN==2 checking. Therefore use an 3483 * unconditional IP align value of '2'. 3484 */ 3485 ip_align = 2; 3486 3487 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 3488 if (!tnum_is_aligned(reg_off, size)) { 3489 char tn_buf[48]; 3490 3491 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3492 verbose(env, 3493 "misaligned packet access off %d+%s+%d+%d size %d\n", 3494 ip_align, tn_buf, reg->off, off, size); 3495 return -EACCES; 3496 } 3497 3498 return 0; 3499 } 3500 3501 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 3502 const struct bpf_reg_state *reg, 3503 const char *pointer_desc, 3504 int off, int size, bool strict) 3505 { 3506 struct tnum reg_off; 3507 3508 /* Byte size accesses are always allowed. */ 3509 if (!strict || size == 1) 3510 return 0; 3511 3512 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 3513 if (!tnum_is_aligned(reg_off, size)) { 3514 char tn_buf[48]; 3515 3516 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3517 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 3518 pointer_desc, tn_buf, reg->off, off, size); 3519 return -EACCES; 3520 } 3521 3522 return 0; 3523 } 3524 3525 static int check_ptr_alignment(struct bpf_verifier_env *env, 3526 const struct bpf_reg_state *reg, int off, 3527 int size, bool strict_alignment_once) 3528 { 3529 bool strict = env->strict_alignment || strict_alignment_once; 3530 const char *pointer_desc = ""; 3531 3532 switch (reg->type) { 3533 case PTR_TO_PACKET: 3534 case PTR_TO_PACKET_META: 3535 /* Special case, because of NET_IP_ALIGN. Given metadata sits 3536 * right in front, treat it the very same way. 3537 */ 3538 return check_pkt_ptr_alignment(env, reg, off, size, strict); 3539 case PTR_TO_FLOW_KEYS: 3540 pointer_desc = "flow keys "; 3541 break; 3542 case PTR_TO_MAP_KEY: 3543 pointer_desc = "key "; 3544 break; 3545 case PTR_TO_MAP_VALUE: 3546 pointer_desc = "value "; 3547 break; 3548 case PTR_TO_CTX: 3549 pointer_desc = "context "; 3550 break; 3551 case PTR_TO_STACK: 3552 pointer_desc = "stack "; 3553 /* The stack spill tracking logic in check_stack_write_fixed_off() 3554 * and check_stack_read_fixed_off() relies on stack accesses being 3555 * aligned. 3556 */ 3557 strict = true; 3558 break; 3559 case PTR_TO_SOCKET: 3560 pointer_desc = "sock "; 3561 break; 3562 case PTR_TO_SOCK_COMMON: 3563 pointer_desc = "sock_common "; 3564 break; 3565 case PTR_TO_TCP_SOCK: 3566 pointer_desc = "tcp_sock "; 3567 break; 3568 case PTR_TO_XDP_SOCK: 3569 pointer_desc = "xdp_sock "; 3570 break; 3571 default: 3572 break; 3573 } 3574 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 3575 strict); 3576 } 3577 3578 static int update_stack_depth(struct bpf_verifier_env *env, 3579 const struct bpf_func_state *func, 3580 int off) 3581 { 3582 u16 stack = env->subprog_info[func->subprogno].stack_depth; 3583 3584 if (stack >= -off) 3585 return 0; 3586 3587 /* update known max for given subprogram */ 3588 env->subprog_info[func->subprogno].stack_depth = -off; 3589 return 0; 3590 } 3591 3592 /* starting from main bpf function walk all instructions of the function 3593 * and recursively walk all callees that given function can call. 3594 * Ignore jump and exit insns. 3595 * Since recursion is prevented by check_cfg() this algorithm 3596 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 3597 */ 3598 static int check_max_stack_depth(struct bpf_verifier_env *env) 3599 { 3600 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 3601 struct bpf_subprog_info *subprog = env->subprog_info; 3602 struct bpf_insn *insn = env->prog->insnsi; 3603 bool tail_call_reachable = false; 3604 int ret_insn[MAX_CALL_FRAMES]; 3605 int ret_prog[MAX_CALL_FRAMES]; 3606 int j; 3607 3608 process_func: 3609 /* protect against potential stack overflow that might happen when 3610 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 3611 * depth for such case down to 256 so that the worst case scenario 3612 * would result in 8k stack size (32 which is tailcall limit * 256 = 3613 * 8k). 3614 * 3615 * To get the idea what might happen, see an example: 3616 * func1 -> sub rsp, 128 3617 * subfunc1 -> sub rsp, 256 3618 * tailcall1 -> add rsp, 256 3619 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 3620 * subfunc2 -> sub rsp, 64 3621 * subfunc22 -> sub rsp, 128 3622 * tailcall2 -> add rsp, 128 3623 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 3624 * 3625 * tailcall will unwind the current stack frame but it will not get rid 3626 * of caller's stack as shown on the example above. 3627 */ 3628 if (idx && subprog[idx].has_tail_call && depth >= 256) { 3629 verbose(env, 3630 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 3631 depth); 3632 return -EACCES; 3633 } 3634 /* round up to 32-bytes, since this is granularity 3635 * of interpreter stack size 3636 */ 3637 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 3638 if (depth > MAX_BPF_STACK) { 3639 verbose(env, "combined stack size of %d calls is %d. Too large\n", 3640 frame + 1, depth); 3641 return -EACCES; 3642 } 3643 continue_func: 3644 subprog_end = subprog[idx + 1].start; 3645 for (; i < subprog_end; i++) { 3646 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 3647 continue; 3648 /* remember insn and function to return to */ 3649 ret_insn[frame] = i + 1; 3650 ret_prog[frame] = idx; 3651 3652 /* find the callee */ 3653 i = i + insn[i].imm + 1; 3654 idx = find_subprog(env, i); 3655 if (idx < 0) { 3656 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 3657 i); 3658 return -EFAULT; 3659 } 3660 3661 if (subprog[idx].has_tail_call) 3662 tail_call_reachable = true; 3663 3664 frame++; 3665 if (frame >= MAX_CALL_FRAMES) { 3666 verbose(env, "the call stack of %d frames is too deep !\n", 3667 frame); 3668 return -E2BIG; 3669 } 3670 goto process_func; 3671 } 3672 /* if tail call got detected across bpf2bpf calls then mark each of the 3673 * currently present subprog frames as tail call reachable subprogs; 3674 * this info will be utilized by JIT so that we will be preserving the 3675 * tail call counter throughout bpf2bpf calls combined with tailcalls 3676 */ 3677 if (tail_call_reachable) 3678 for (j = 0; j < frame; j++) 3679 subprog[ret_prog[j]].tail_call_reachable = true; 3680 3681 /* end of for() loop means the last insn of the 'subprog' 3682 * was reached. Doesn't matter whether it was JA or EXIT 3683 */ 3684 if (frame == 0) 3685 return 0; 3686 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 3687 frame--; 3688 i = ret_insn[frame]; 3689 idx = ret_prog[frame]; 3690 goto continue_func; 3691 } 3692 3693 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 3694 static int get_callee_stack_depth(struct bpf_verifier_env *env, 3695 const struct bpf_insn *insn, int idx) 3696 { 3697 int start = idx + insn->imm + 1, subprog; 3698 3699 subprog = find_subprog(env, start); 3700 if (subprog < 0) { 3701 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 3702 start); 3703 return -EFAULT; 3704 } 3705 return env->subprog_info[subprog].stack_depth; 3706 } 3707 #endif 3708 3709 int check_ctx_reg(struct bpf_verifier_env *env, 3710 const struct bpf_reg_state *reg, int regno) 3711 { 3712 /* Access to ctx or passing it to a helper is only allowed in 3713 * its original, unmodified form. 3714 */ 3715 3716 if (reg->off) { 3717 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", 3718 regno, reg->off); 3719 return -EACCES; 3720 } 3721 3722 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3723 char tn_buf[48]; 3724 3725 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3726 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); 3727 return -EACCES; 3728 } 3729 3730 return 0; 3731 } 3732 3733 static int __check_buffer_access(struct bpf_verifier_env *env, 3734 const char *buf_info, 3735 const struct bpf_reg_state *reg, 3736 int regno, int off, int size) 3737 { 3738 if (off < 0) { 3739 verbose(env, 3740 "R%d invalid %s buffer access: off=%d, size=%d\n", 3741 regno, buf_info, off, size); 3742 return -EACCES; 3743 } 3744 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3745 char tn_buf[48]; 3746 3747 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3748 verbose(env, 3749 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 3750 regno, off, tn_buf); 3751 return -EACCES; 3752 } 3753 3754 return 0; 3755 } 3756 3757 static int check_tp_buffer_access(struct bpf_verifier_env *env, 3758 const struct bpf_reg_state *reg, 3759 int regno, int off, int size) 3760 { 3761 int err; 3762 3763 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 3764 if (err) 3765 return err; 3766 3767 if (off + size > env->prog->aux->max_tp_access) 3768 env->prog->aux->max_tp_access = off + size; 3769 3770 return 0; 3771 } 3772 3773 static int check_buffer_access(struct bpf_verifier_env *env, 3774 const struct bpf_reg_state *reg, 3775 int regno, int off, int size, 3776 bool zero_size_allowed, 3777 const char *buf_info, 3778 u32 *max_access) 3779 { 3780 int err; 3781 3782 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 3783 if (err) 3784 return err; 3785 3786 if (off + size > *max_access) 3787 *max_access = off + size; 3788 3789 return 0; 3790 } 3791 3792 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 3793 static void zext_32_to_64(struct bpf_reg_state *reg) 3794 { 3795 reg->var_off = tnum_subreg(reg->var_off); 3796 __reg_assign_32_into_64(reg); 3797 } 3798 3799 /* truncate register to smaller size (in bytes) 3800 * must be called with size < BPF_REG_SIZE 3801 */ 3802 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 3803 { 3804 u64 mask; 3805 3806 /* clear high bits in bit representation */ 3807 reg->var_off = tnum_cast(reg->var_off, size); 3808 3809 /* fix arithmetic bounds */ 3810 mask = ((u64)1 << (size * 8)) - 1; 3811 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 3812 reg->umin_value &= mask; 3813 reg->umax_value &= mask; 3814 } else { 3815 reg->umin_value = 0; 3816 reg->umax_value = mask; 3817 } 3818 reg->smin_value = reg->umin_value; 3819 reg->smax_value = reg->umax_value; 3820 3821 /* If size is smaller than 32bit register the 32bit register 3822 * values are also truncated so we push 64-bit bounds into 3823 * 32-bit bounds. Above were truncated < 32-bits already. 3824 */ 3825 if (size >= 4) 3826 return; 3827 __reg_combine_64_into_32(reg); 3828 } 3829 3830 static bool bpf_map_is_rdonly(const struct bpf_map *map) 3831 { 3832 return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen; 3833 } 3834 3835 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 3836 { 3837 void *ptr; 3838 u64 addr; 3839 int err; 3840 3841 err = map->ops->map_direct_value_addr(map, &addr, off); 3842 if (err) 3843 return err; 3844 ptr = (void *)(long)addr + off; 3845 3846 switch (size) { 3847 case sizeof(u8): 3848 *val = (u64)*(u8 *)ptr; 3849 break; 3850 case sizeof(u16): 3851 *val = (u64)*(u16 *)ptr; 3852 break; 3853 case sizeof(u32): 3854 *val = (u64)*(u32 *)ptr; 3855 break; 3856 case sizeof(u64): 3857 *val = *(u64 *)ptr; 3858 break; 3859 default: 3860 return -EINVAL; 3861 } 3862 return 0; 3863 } 3864 3865 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 3866 struct bpf_reg_state *regs, 3867 int regno, int off, int size, 3868 enum bpf_access_type atype, 3869 int value_regno) 3870 { 3871 struct bpf_reg_state *reg = regs + regno; 3872 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 3873 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 3874 u32 btf_id; 3875 int ret; 3876 3877 if (off < 0) { 3878 verbose(env, 3879 "R%d is ptr_%s invalid negative access: off=%d\n", 3880 regno, tname, off); 3881 return -EACCES; 3882 } 3883 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3884 char tn_buf[48]; 3885 3886 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3887 verbose(env, 3888 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 3889 regno, tname, off, tn_buf); 3890 return -EACCES; 3891 } 3892 3893 if (env->ops->btf_struct_access) { 3894 ret = env->ops->btf_struct_access(&env->log, reg->btf, t, 3895 off, size, atype, &btf_id); 3896 } else { 3897 if (atype != BPF_READ) { 3898 verbose(env, "only read is supported\n"); 3899 return -EACCES; 3900 } 3901 3902 ret = btf_struct_access(&env->log, reg->btf, t, off, size, 3903 atype, &btf_id); 3904 } 3905 3906 if (ret < 0) 3907 return ret; 3908 3909 if (atype == BPF_READ && value_regno >= 0) 3910 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id); 3911 3912 return 0; 3913 } 3914 3915 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 3916 struct bpf_reg_state *regs, 3917 int regno, int off, int size, 3918 enum bpf_access_type atype, 3919 int value_regno) 3920 { 3921 struct bpf_reg_state *reg = regs + regno; 3922 struct bpf_map *map = reg->map_ptr; 3923 const struct btf_type *t; 3924 const char *tname; 3925 u32 btf_id; 3926 int ret; 3927 3928 if (!btf_vmlinux) { 3929 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 3930 return -ENOTSUPP; 3931 } 3932 3933 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 3934 verbose(env, "map_ptr access not supported for map type %d\n", 3935 map->map_type); 3936 return -ENOTSUPP; 3937 } 3938 3939 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 3940 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 3941 3942 if (!env->allow_ptr_to_map_access) { 3943 verbose(env, 3944 "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 3945 tname); 3946 return -EPERM; 3947 } 3948 3949 if (off < 0) { 3950 verbose(env, "R%d is %s invalid negative access: off=%d\n", 3951 regno, tname, off); 3952 return -EACCES; 3953 } 3954 3955 if (atype != BPF_READ) { 3956 verbose(env, "only read from %s is supported\n", tname); 3957 return -EACCES; 3958 } 3959 3960 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id); 3961 if (ret < 0) 3962 return ret; 3963 3964 if (value_regno >= 0) 3965 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id); 3966 3967 return 0; 3968 } 3969 3970 /* Check that the stack access at the given offset is within bounds. The 3971 * maximum valid offset is -1. 3972 * 3973 * The minimum valid offset is -MAX_BPF_STACK for writes, and 3974 * -state->allocated_stack for reads. 3975 */ 3976 static int check_stack_slot_within_bounds(int off, 3977 struct bpf_func_state *state, 3978 enum bpf_access_type t) 3979 { 3980 int min_valid_off; 3981 3982 if (t == BPF_WRITE) 3983 min_valid_off = -MAX_BPF_STACK; 3984 else 3985 min_valid_off = -state->allocated_stack; 3986 3987 if (off < min_valid_off || off > -1) 3988 return -EACCES; 3989 return 0; 3990 } 3991 3992 /* Check that the stack access at 'regno + off' falls within the maximum stack 3993 * bounds. 3994 * 3995 * 'off' includes `regno->offset`, but not its dynamic part (if any). 3996 */ 3997 static int check_stack_access_within_bounds( 3998 struct bpf_verifier_env *env, 3999 int regno, int off, int access_size, 4000 enum stack_access_src src, enum bpf_access_type type) 4001 { 4002 struct bpf_reg_state *regs = cur_regs(env); 4003 struct bpf_reg_state *reg = regs + regno; 4004 struct bpf_func_state *state = func(env, reg); 4005 int min_off, max_off; 4006 int err; 4007 char *err_extra; 4008 4009 if (src == ACCESS_HELPER) 4010 /* We don't know if helpers are reading or writing (or both). */ 4011 err_extra = " indirect access to"; 4012 else if (type == BPF_READ) 4013 err_extra = " read from"; 4014 else 4015 err_extra = " write to"; 4016 4017 if (tnum_is_const(reg->var_off)) { 4018 min_off = reg->var_off.value + off; 4019 if (access_size > 0) 4020 max_off = min_off + access_size - 1; 4021 else 4022 max_off = min_off; 4023 } else { 4024 if (reg->smax_value >= BPF_MAX_VAR_OFF || 4025 reg->smin_value <= -BPF_MAX_VAR_OFF) { 4026 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 4027 err_extra, regno); 4028 return -EACCES; 4029 } 4030 min_off = reg->smin_value + off; 4031 if (access_size > 0) 4032 max_off = reg->smax_value + off + access_size - 1; 4033 else 4034 max_off = min_off; 4035 } 4036 4037 err = check_stack_slot_within_bounds(min_off, state, type); 4038 if (!err) 4039 err = check_stack_slot_within_bounds(max_off, state, type); 4040 4041 if (err) { 4042 if (tnum_is_const(reg->var_off)) { 4043 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 4044 err_extra, regno, off, access_size); 4045 } else { 4046 char tn_buf[48]; 4047 4048 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4049 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 4050 err_extra, regno, tn_buf, access_size); 4051 } 4052 } 4053 return err; 4054 } 4055 4056 /* check whether memory at (regno + off) is accessible for t = (read | write) 4057 * if t==write, value_regno is a register which value is stored into memory 4058 * if t==read, value_regno is a register which will receive the value from memory 4059 * if t==write && value_regno==-1, some unknown value is stored into memory 4060 * if t==read && value_regno==-1, don't care what we read from memory 4061 */ 4062 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 4063 int off, int bpf_size, enum bpf_access_type t, 4064 int value_regno, bool strict_alignment_once) 4065 { 4066 struct bpf_reg_state *regs = cur_regs(env); 4067 struct bpf_reg_state *reg = regs + regno; 4068 struct bpf_func_state *state; 4069 int size, err = 0; 4070 4071 size = bpf_size_to_bytes(bpf_size); 4072 if (size < 0) 4073 return size; 4074 4075 /* alignment checks will add in reg->off themselves */ 4076 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 4077 if (err) 4078 return err; 4079 4080 /* for access checks, reg->off is just part of off */ 4081 off += reg->off; 4082 4083 if (reg->type == PTR_TO_MAP_KEY) { 4084 if (t == BPF_WRITE) { 4085 verbose(env, "write to change key R%d not allowed\n", regno); 4086 return -EACCES; 4087 } 4088 4089 err = check_mem_region_access(env, regno, off, size, 4090 reg->map_ptr->key_size, false); 4091 if (err) 4092 return err; 4093 if (value_regno >= 0) 4094 mark_reg_unknown(env, regs, value_regno); 4095 } else if (reg->type == PTR_TO_MAP_VALUE) { 4096 if (t == BPF_WRITE && value_regno >= 0 && 4097 is_pointer_value(env, value_regno)) { 4098 verbose(env, "R%d leaks addr into map\n", value_regno); 4099 return -EACCES; 4100 } 4101 err = check_map_access_type(env, regno, off, size, t); 4102 if (err) 4103 return err; 4104 err = check_map_access(env, regno, off, size, false); 4105 if (!err && t == BPF_READ && value_regno >= 0) { 4106 struct bpf_map *map = reg->map_ptr; 4107 4108 /* if map is read-only, track its contents as scalars */ 4109 if (tnum_is_const(reg->var_off) && 4110 bpf_map_is_rdonly(map) && 4111 map->ops->map_direct_value_addr) { 4112 int map_off = off + reg->var_off.value; 4113 u64 val = 0; 4114 4115 err = bpf_map_direct_read(map, map_off, size, 4116 &val); 4117 if (err) 4118 return err; 4119 4120 regs[value_regno].type = SCALAR_VALUE; 4121 __mark_reg_known(®s[value_regno], val); 4122 } else { 4123 mark_reg_unknown(env, regs, value_regno); 4124 } 4125 } 4126 } else if (reg->type == PTR_TO_MEM) { 4127 if (t == BPF_WRITE && value_regno >= 0 && 4128 is_pointer_value(env, value_regno)) { 4129 verbose(env, "R%d leaks addr into mem\n", value_regno); 4130 return -EACCES; 4131 } 4132 err = check_mem_region_access(env, regno, off, size, 4133 reg->mem_size, false); 4134 if (!err && t == BPF_READ && value_regno >= 0) 4135 mark_reg_unknown(env, regs, value_regno); 4136 } else if (reg->type == PTR_TO_CTX) { 4137 enum bpf_reg_type reg_type = SCALAR_VALUE; 4138 struct btf *btf = NULL; 4139 u32 btf_id = 0; 4140 4141 if (t == BPF_WRITE && value_regno >= 0 && 4142 is_pointer_value(env, value_regno)) { 4143 verbose(env, "R%d leaks addr into ctx\n", value_regno); 4144 return -EACCES; 4145 } 4146 4147 err = check_ctx_reg(env, reg, regno); 4148 if (err < 0) 4149 return err; 4150 4151 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, &btf_id); 4152 if (err) 4153 verbose_linfo(env, insn_idx, "; "); 4154 if (!err && t == BPF_READ && value_regno >= 0) { 4155 /* ctx access returns either a scalar, or a 4156 * PTR_TO_PACKET[_META,_END]. In the latter 4157 * case, we know the offset is zero. 4158 */ 4159 if (reg_type == SCALAR_VALUE) { 4160 mark_reg_unknown(env, regs, value_regno); 4161 } else { 4162 mark_reg_known_zero(env, regs, 4163 value_regno); 4164 if (reg_type_may_be_null(reg_type)) 4165 regs[value_regno].id = ++env->id_gen; 4166 /* A load of ctx field could have different 4167 * actual load size with the one encoded in the 4168 * insn. When the dst is PTR, it is for sure not 4169 * a sub-register. 4170 */ 4171 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 4172 if (reg_type == PTR_TO_BTF_ID || 4173 reg_type == PTR_TO_BTF_ID_OR_NULL) { 4174 regs[value_regno].btf = btf; 4175 regs[value_regno].btf_id = btf_id; 4176 } 4177 } 4178 regs[value_regno].type = reg_type; 4179 } 4180 4181 } else if (reg->type == PTR_TO_STACK) { 4182 /* Basic bounds checks. */ 4183 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 4184 if (err) 4185 return err; 4186 4187 state = func(env, reg); 4188 err = update_stack_depth(env, state, off); 4189 if (err) 4190 return err; 4191 4192 if (t == BPF_READ) 4193 err = check_stack_read(env, regno, off, size, 4194 value_regno); 4195 else 4196 err = check_stack_write(env, regno, off, size, 4197 value_regno, insn_idx); 4198 } else if (reg_is_pkt_pointer(reg)) { 4199 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 4200 verbose(env, "cannot write into packet\n"); 4201 return -EACCES; 4202 } 4203 if (t == BPF_WRITE && value_regno >= 0 && 4204 is_pointer_value(env, value_regno)) { 4205 verbose(env, "R%d leaks addr into packet\n", 4206 value_regno); 4207 return -EACCES; 4208 } 4209 err = check_packet_access(env, regno, off, size, false); 4210 if (!err && t == BPF_READ && value_regno >= 0) 4211 mark_reg_unknown(env, regs, value_regno); 4212 } else if (reg->type == PTR_TO_FLOW_KEYS) { 4213 if (t == BPF_WRITE && value_regno >= 0 && 4214 is_pointer_value(env, value_regno)) { 4215 verbose(env, "R%d leaks addr into flow keys\n", 4216 value_regno); 4217 return -EACCES; 4218 } 4219 4220 err = check_flow_keys_access(env, off, size); 4221 if (!err && t == BPF_READ && value_regno >= 0) 4222 mark_reg_unknown(env, regs, value_regno); 4223 } else if (type_is_sk_pointer(reg->type)) { 4224 if (t == BPF_WRITE) { 4225 verbose(env, "R%d cannot write into %s\n", 4226 regno, reg_type_str[reg->type]); 4227 return -EACCES; 4228 } 4229 err = check_sock_access(env, insn_idx, regno, off, size, t); 4230 if (!err && value_regno >= 0) 4231 mark_reg_unknown(env, regs, value_regno); 4232 } else if (reg->type == PTR_TO_TP_BUFFER) { 4233 err = check_tp_buffer_access(env, reg, regno, off, size); 4234 if (!err && t == BPF_READ && value_regno >= 0) 4235 mark_reg_unknown(env, regs, value_regno); 4236 } else if (reg->type == PTR_TO_BTF_ID) { 4237 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 4238 value_regno); 4239 } else if (reg->type == CONST_PTR_TO_MAP) { 4240 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 4241 value_regno); 4242 } else if (reg->type == PTR_TO_RDONLY_BUF) { 4243 if (t == BPF_WRITE) { 4244 verbose(env, "R%d cannot write into %s\n", 4245 regno, reg_type_str[reg->type]); 4246 return -EACCES; 4247 } 4248 err = check_buffer_access(env, reg, regno, off, size, false, 4249 "rdonly", 4250 &env->prog->aux->max_rdonly_access); 4251 if (!err && value_regno >= 0) 4252 mark_reg_unknown(env, regs, value_regno); 4253 } else if (reg->type == PTR_TO_RDWR_BUF) { 4254 err = check_buffer_access(env, reg, regno, off, size, false, 4255 "rdwr", 4256 &env->prog->aux->max_rdwr_access); 4257 if (!err && t == BPF_READ && value_regno >= 0) 4258 mark_reg_unknown(env, regs, value_regno); 4259 } else { 4260 verbose(env, "R%d invalid mem access '%s'\n", regno, 4261 reg_type_str[reg->type]); 4262 return -EACCES; 4263 } 4264 4265 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 4266 regs[value_regno].type == SCALAR_VALUE) { 4267 /* b/h/w load zero-extends, mark upper bits as known 0 */ 4268 coerce_reg_to_size(®s[value_regno], size); 4269 } 4270 return err; 4271 } 4272 4273 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 4274 { 4275 int load_reg; 4276 int err; 4277 4278 switch (insn->imm) { 4279 case BPF_ADD: 4280 case BPF_ADD | BPF_FETCH: 4281 case BPF_AND: 4282 case BPF_AND | BPF_FETCH: 4283 case BPF_OR: 4284 case BPF_OR | BPF_FETCH: 4285 case BPF_XOR: 4286 case BPF_XOR | BPF_FETCH: 4287 case BPF_XCHG: 4288 case BPF_CMPXCHG: 4289 break; 4290 default: 4291 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 4292 return -EINVAL; 4293 } 4294 4295 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 4296 verbose(env, "invalid atomic operand size\n"); 4297 return -EINVAL; 4298 } 4299 4300 /* check src1 operand */ 4301 err = check_reg_arg(env, insn->src_reg, SRC_OP); 4302 if (err) 4303 return err; 4304 4305 /* check src2 operand */ 4306 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 4307 if (err) 4308 return err; 4309 4310 if (insn->imm == BPF_CMPXCHG) { 4311 /* Check comparison of R0 with memory location */ 4312 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 4313 if (err) 4314 return err; 4315 } 4316 4317 if (is_pointer_value(env, insn->src_reg)) { 4318 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 4319 return -EACCES; 4320 } 4321 4322 if (is_ctx_reg(env, insn->dst_reg) || 4323 is_pkt_reg(env, insn->dst_reg) || 4324 is_flow_key_reg(env, insn->dst_reg) || 4325 is_sk_reg(env, insn->dst_reg)) { 4326 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 4327 insn->dst_reg, 4328 reg_type_str[reg_state(env, insn->dst_reg)->type]); 4329 return -EACCES; 4330 } 4331 4332 if (insn->imm & BPF_FETCH) { 4333 if (insn->imm == BPF_CMPXCHG) 4334 load_reg = BPF_REG_0; 4335 else 4336 load_reg = insn->src_reg; 4337 4338 /* check and record load of old value */ 4339 err = check_reg_arg(env, load_reg, DST_OP); 4340 if (err) 4341 return err; 4342 } else { 4343 /* This instruction accesses a memory location but doesn't 4344 * actually load it into a register. 4345 */ 4346 load_reg = -1; 4347 } 4348 4349 /* check whether we can read the memory */ 4350 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4351 BPF_SIZE(insn->code), BPF_READ, load_reg, true); 4352 if (err) 4353 return err; 4354 4355 /* check whether we can write into the same memory */ 4356 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4357 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 4358 if (err) 4359 return err; 4360 4361 return 0; 4362 } 4363 4364 /* When register 'regno' is used to read the stack (either directly or through 4365 * a helper function) make sure that it's within stack boundary and, depending 4366 * on the access type, that all elements of the stack are initialized. 4367 * 4368 * 'off' includes 'regno->off', but not its dynamic part (if any). 4369 * 4370 * All registers that have been spilled on the stack in the slots within the 4371 * read offsets are marked as read. 4372 */ 4373 static int check_stack_range_initialized( 4374 struct bpf_verifier_env *env, int regno, int off, 4375 int access_size, bool zero_size_allowed, 4376 enum stack_access_src type, struct bpf_call_arg_meta *meta) 4377 { 4378 struct bpf_reg_state *reg = reg_state(env, regno); 4379 struct bpf_func_state *state = func(env, reg); 4380 int err, min_off, max_off, i, j, slot, spi; 4381 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 4382 enum bpf_access_type bounds_check_type; 4383 /* Some accesses can write anything into the stack, others are 4384 * read-only. 4385 */ 4386 bool clobber = false; 4387 4388 if (access_size == 0 && !zero_size_allowed) { 4389 verbose(env, "invalid zero-sized read\n"); 4390 return -EACCES; 4391 } 4392 4393 if (type == ACCESS_HELPER) { 4394 /* The bounds checks for writes are more permissive than for 4395 * reads. However, if raw_mode is not set, we'll do extra 4396 * checks below. 4397 */ 4398 bounds_check_type = BPF_WRITE; 4399 clobber = true; 4400 } else { 4401 bounds_check_type = BPF_READ; 4402 } 4403 err = check_stack_access_within_bounds(env, regno, off, access_size, 4404 type, bounds_check_type); 4405 if (err) 4406 return err; 4407 4408 4409 if (tnum_is_const(reg->var_off)) { 4410 min_off = max_off = reg->var_off.value + off; 4411 } else { 4412 /* Variable offset is prohibited for unprivileged mode for 4413 * simplicity since it requires corresponding support in 4414 * Spectre masking for stack ALU. 4415 * See also retrieve_ptr_limit(). 4416 */ 4417 if (!env->bypass_spec_v1) { 4418 char tn_buf[48]; 4419 4420 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4421 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 4422 regno, err_extra, tn_buf); 4423 return -EACCES; 4424 } 4425 /* Only initialized buffer on stack is allowed to be accessed 4426 * with variable offset. With uninitialized buffer it's hard to 4427 * guarantee that whole memory is marked as initialized on 4428 * helper return since specific bounds are unknown what may 4429 * cause uninitialized stack leaking. 4430 */ 4431 if (meta && meta->raw_mode) 4432 meta = NULL; 4433 4434 min_off = reg->smin_value + off; 4435 max_off = reg->smax_value + off; 4436 } 4437 4438 if (meta && meta->raw_mode) { 4439 meta->access_size = access_size; 4440 meta->regno = regno; 4441 return 0; 4442 } 4443 4444 for (i = min_off; i < max_off + access_size; i++) { 4445 u8 *stype; 4446 4447 slot = -i - 1; 4448 spi = slot / BPF_REG_SIZE; 4449 if (state->allocated_stack <= slot) 4450 goto err; 4451 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 4452 if (*stype == STACK_MISC) 4453 goto mark; 4454 if (*stype == STACK_ZERO) { 4455 if (clobber) { 4456 /* helper can write anything into the stack */ 4457 *stype = STACK_MISC; 4458 } 4459 goto mark; 4460 } 4461 4462 if (state->stack[spi].slot_type[0] == STACK_SPILL && 4463 state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID) 4464 goto mark; 4465 4466 if (state->stack[spi].slot_type[0] == STACK_SPILL && 4467 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 4468 env->allow_ptr_leaks)) { 4469 if (clobber) { 4470 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 4471 for (j = 0; j < BPF_REG_SIZE; j++) 4472 state->stack[spi].slot_type[j] = STACK_MISC; 4473 } 4474 goto mark; 4475 } 4476 4477 err: 4478 if (tnum_is_const(reg->var_off)) { 4479 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 4480 err_extra, regno, min_off, i - min_off, access_size); 4481 } else { 4482 char tn_buf[48]; 4483 4484 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4485 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 4486 err_extra, regno, tn_buf, i - min_off, access_size); 4487 } 4488 return -EACCES; 4489 mark: 4490 /* reading any byte out of 8-byte 'spill_slot' will cause 4491 * the whole slot to be marked as 'read' 4492 */ 4493 mark_reg_read(env, &state->stack[spi].spilled_ptr, 4494 state->stack[spi].spilled_ptr.parent, 4495 REG_LIVE_READ64); 4496 } 4497 return update_stack_depth(env, state, min_off); 4498 } 4499 4500 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 4501 int access_size, bool zero_size_allowed, 4502 struct bpf_call_arg_meta *meta) 4503 { 4504 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4505 4506 switch (reg->type) { 4507 case PTR_TO_PACKET: 4508 case PTR_TO_PACKET_META: 4509 return check_packet_access(env, regno, reg->off, access_size, 4510 zero_size_allowed); 4511 case PTR_TO_MAP_KEY: 4512 return check_mem_region_access(env, regno, reg->off, access_size, 4513 reg->map_ptr->key_size, false); 4514 case PTR_TO_MAP_VALUE: 4515 if (check_map_access_type(env, regno, reg->off, access_size, 4516 meta && meta->raw_mode ? BPF_WRITE : 4517 BPF_READ)) 4518 return -EACCES; 4519 return check_map_access(env, regno, reg->off, access_size, 4520 zero_size_allowed); 4521 case PTR_TO_MEM: 4522 return check_mem_region_access(env, regno, reg->off, 4523 access_size, reg->mem_size, 4524 zero_size_allowed); 4525 case PTR_TO_RDONLY_BUF: 4526 if (meta && meta->raw_mode) 4527 return -EACCES; 4528 return check_buffer_access(env, reg, regno, reg->off, 4529 access_size, zero_size_allowed, 4530 "rdonly", 4531 &env->prog->aux->max_rdonly_access); 4532 case PTR_TO_RDWR_BUF: 4533 return check_buffer_access(env, reg, regno, reg->off, 4534 access_size, zero_size_allowed, 4535 "rdwr", 4536 &env->prog->aux->max_rdwr_access); 4537 case PTR_TO_STACK: 4538 return check_stack_range_initialized( 4539 env, 4540 regno, reg->off, access_size, 4541 zero_size_allowed, ACCESS_HELPER, meta); 4542 default: /* scalar_value or invalid ptr */ 4543 /* Allow zero-byte read from NULL, regardless of pointer type */ 4544 if (zero_size_allowed && access_size == 0 && 4545 register_is_null(reg)) 4546 return 0; 4547 4548 verbose(env, "R%d type=%s expected=%s\n", regno, 4549 reg_type_str[reg->type], 4550 reg_type_str[PTR_TO_STACK]); 4551 return -EACCES; 4552 } 4553 } 4554 4555 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 4556 u32 regno, u32 mem_size) 4557 { 4558 if (register_is_null(reg)) 4559 return 0; 4560 4561 if (reg_type_may_be_null(reg->type)) { 4562 /* Assuming that the register contains a value check if the memory 4563 * access is safe. Temporarily save and restore the register's state as 4564 * the conversion shouldn't be visible to a caller. 4565 */ 4566 const struct bpf_reg_state saved_reg = *reg; 4567 int rv; 4568 4569 mark_ptr_not_null_reg(reg); 4570 rv = check_helper_mem_access(env, regno, mem_size, true, NULL); 4571 *reg = saved_reg; 4572 return rv; 4573 } 4574 4575 return check_helper_mem_access(env, regno, mem_size, true, NULL); 4576 } 4577 4578 /* Implementation details: 4579 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL 4580 * Two bpf_map_lookups (even with the same key) will have different reg->id. 4581 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after 4582 * value_or_null->value transition, since the verifier only cares about 4583 * the range of access to valid map value pointer and doesn't care about actual 4584 * address of the map element. 4585 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 4586 * reg->id > 0 after value_or_null->value transition. By doing so 4587 * two bpf_map_lookups will be considered two different pointers that 4588 * point to different bpf_spin_locks. 4589 * The verifier allows taking only one bpf_spin_lock at a time to avoid 4590 * dead-locks. 4591 * Since only one bpf_spin_lock is allowed the checks are simpler than 4592 * reg_is_refcounted() logic. The verifier needs to remember only 4593 * one spin_lock instead of array of acquired_refs. 4594 * cur_state->active_spin_lock remembers which map value element got locked 4595 * and clears it after bpf_spin_unlock. 4596 */ 4597 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 4598 bool is_lock) 4599 { 4600 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4601 struct bpf_verifier_state *cur = env->cur_state; 4602 bool is_const = tnum_is_const(reg->var_off); 4603 struct bpf_map *map = reg->map_ptr; 4604 u64 val = reg->var_off.value; 4605 4606 if (!is_const) { 4607 verbose(env, 4608 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 4609 regno); 4610 return -EINVAL; 4611 } 4612 if (!map->btf) { 4613 verbose(env, 4614 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 4615 map->name); 4616 return -EINVAL; 4617 } 4618 if (!map_value_has_spin_lock(map)) { 4619 if (map->spin_lock_off == -E2BIG) 4620 verbose(env, 4621 "map '%s' has more than one 'struct bpf_spin_lock'\n", 4622 map->name); 4623 else if (map->spin_lock_off == -ENOENT) 4624 verbose(env, 4625 "map '%s' doesn't have 'struct bpf_spin_lock'\n", 4626 map->name); 4627 else 4628 verbose(env, 4629 "map '%s' is not a struct type or bpf_spin_lock is mangled\n", 4630 map->name); 4631 return -EINVAL; 4632 } 4633 if (map->spin_lock_off != val + reg->off) { 4634 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", 4635 val + reg->off); 4636 return -EINVAL; 4637 } 4638 if (is_lock) { 4639 if (cur->active_spin_lock) { 4640 verbose(env, 4641 "Locking two bpf_spin_locks are not allowed\n"); 4642 return -EINVAL; 4643 } 4644 cur->active_spin_lock = reg->id; 4645 } else { 4646 if (!cur->active_spin_lock) { 4647 verbose(env, "bpf_spin_unlock without taking a lock\n"); 4648 return -EINVAL; 4649 } 4650 if (cur->active_spin_lock != reg->id) { 4651 verbose(env, "bpf_spin_unlock of different lock\n"); 4652 return -EINVAL; 4653 } 4654 cur->active_spin_lock = 0; 4655 } 4656 return 0; 4657 } 4658 4659 static bool arg_type_is_mem_ptr(enum bpf_arg_type type) 4660 { 4661 return type == ARG_PTR_TO_MEM || 4662 type == ARG_PTR_TO_MEM_OR_NULL || 4663 type == ARG_PTR_TO_UNINIT_MEM; 4664 } 4665 4666 static bool arg_type_is_mem_size(enum bpf_arg_type type) 4667 { 4668 return type == ARG_CONST_SIZE || 4669 type == ARG_CONST_SIZE_OR_ZERO; 4670 } 4671 4672 static bool arg_type_is_alloc_size(enum bpf_arg_type type) 4673 { 4674 return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; 4675 } 4676 4677 static bool arg_type_is_int_ptr(enum bpf_arg_type type) 4678 { 4679 return type == ARG_PTR_TO_INT || 4680 type == ARG_PTR_TO_LONG; 4681 } 4682 4683 static int int_ptr_type_to_size(enum bpf_arg_type type) 4684 { 4685 if (type == ARG_PTR_TO_INT) 4686 return sizeof(u32); 4687 else if (type == ARG_PTR_TO_LONG) 4688 return sizeof(u64); 4689 4690 return -EINVAL; 4691 } 4692 4693 static int resolve_map_arg_type(struct bpf_verifier_env *env, 4694 const struct bpf_call_arg_meta *meta, 4695 enum bpf_arg_type *arg_type) 4696 { 4697 if (!meta->map_ptr) { 4698 /* kernel subsystem misconfigured verifier */ 4699 verbose(env, "invalid map_ptr to access map->type\n"); 4700 return -EACCES; 4701 } 4702 4703 switch (meta->map_ptr->map_type) { 4704 case BPF_MAP_TYPE_SOCKMAP: 4705 case BPF_MAP_TYPE_SOCKHASH: 4706 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 4707 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 4708 } else { 4709 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 4710 return -EINVAL; 4711 } 4712 break; 4713 4714 default: 4715 break; 4716 } 4717 return 0; 4718 } 4719 4720 struct bpf_reg_types { 4721 const enum bpf_reg_type types[10]; 4722 u32 *btf_id; 4723 }; 4724 4725 static const struct bpf_reg_types map_key_value_types = { 4726 .types = { 4727 PTR_TO_STACK, 4728 PTR_TO_PACKET, 4729 PTR_TO_PACKET_META, 4730 PTR_TO_MAP_KEY, 4731 PTR_TO_MAP_VALUE, 4732 }, 4733 }; 4734 4735 static const struct bpf_reg_types sock_types = { 4736 .types = { 4737 PTR_TO_SOCK_COMMON, 4738 PTR_TO_SOCKET, 4739 PTR_TO_TCP_SOCK, 4740 PTR_TO_XDP_SOCK, 4741 }, 4742 }; 4743 4744 #ifdef CONFIG_NET 4745 static const struct bpf_reg_types btf_id_sock_common_types = { 4746 .types = { 4747 PTR_TO_SOCK_COMMON, 4748 PTR_TO_SOCKET, 4749 PTR_TO_TCP_SOCK, 4750 PTR_TO_XDP_SOCK, 4751 PTR_TO_BTF_ID, 4752 }, 4753 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 4754 }; 4755 #endif 4756 4757 static const struct bpf_reg_types mem_types = { 4758 .types = { 4759 PTR_TO_STACK, 4760 PTR_TO_PACKET, 4761 PTR_TO_PACKET_META, 4762 PTR_TO_MAP_KEY, 4763 PTR_TO_MAP_VALUE, 4764 PTR_TO_MEM, 4765 PTR_TO_RDONLY_BUF, 4766 PTR_TO_RDWR_BUF, 4767 }, 4768 }; 4769 4770 static const struct bpf_reg_types int_ptr_types = { 4771 .types = { 4772 PTR_TO_STACK, 4773 PTR_TO_PACKET, 4774 PTR_TO_PACKET_META, 4775 PTR_TO_MAP_KEY, 4776 PTR_TO_MAP_VALUE, 4777 }, 4778 }; 4779 4780 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 4781 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 4782 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 4783 static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; 4784 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 4785 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; 4786 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; 4787 static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } }; 4788 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 4789 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 4790 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 4791 4792 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 4793 [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, 4794 [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, 4795 [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, 4796 [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types, 4797 [ARG_CONST_SIZE] = &scalar_types, 4798 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 4799 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 4800 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 4801 [ARG_PTR_TO_CTX] = &context_types, 4802 [ARG_PTR_TO_CTX_OR_NULL] = &context_types, 4803 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 4804 #ifdef CONFIG_NET 4805 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 4806 #endif 4807 [ARG_PTR_TO_SOCKET] = &fullsock_types, 4808 [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, 4809 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 4810 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 4811 [ARG_PTR_TO_MEM] = &mem_types, 4812 [ARG_PTR_TO_MEM_OR_NULL] = &mem_types, 4813 [ARG_PTR_TO_UNINIT_MEM] = &mem_types, 4814 [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, 4815 [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types, 4816 [ARG_PTR_TO_INT] = &int_ptr_types, 4817 [ARG_PTR_TO_LONG] = &int_ptr_types, 4818 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 4819 [ARG_PTR_TO_FUNC] = &func_ptr_types, 4820 [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types, 4821 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 4822 }; 4823 4824 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 4825 enum bpf_arg_type arg_type, 4826 const u32 *arg_btf_id) 4827 { 4828 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4829 enum bpf_reg_type expected, type = reg->type; 4830 const struct bpf_reg_types *compatible; 4831 int i, j; 4832 4833 compatible = compatible_reg_types[arg_type]; 4834 if (!compatible) { 4835 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 4836 return -EFAULT; 4837 } 4838 4839 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 4840 expected = compatible->types[i]; 4841 if (expected == NOT_INIT) 4842 break; 4843 4844 if (type == expected) 4845 goto found; 4846 } 4847 4848 verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); 4849 for (j = 0; j + 1 < i; j++) 4850 verbose(env, "%s, ", reg_type_str[compatible->types[j]]); 4851 verbose(env, "%s\n", reg_type_str[compatible->types[j]]); 4852 return -EACCES; 4853 4854 found: 4855 if (type == PTR_TO_BTF_ID) { 4856 if (!arg_btf_id) { 4857 if (!compatible->btf_id) { 4858 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 4859 return -EFAULT; 4860 } 4861 arg_btf_id = compatible->btf_id; 4862 } 4863 4864 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 4865 btf_vmlinux, *arg_btf_id)) { 4866 verbose(env, "R%d is of type %s but %s is expected\n", 4867 regno, kernel_type_name(reg->btf, reg->btf_id), 4868 kernel_type_name(btf_vmlinux, *arg_btf_id)); 4869 return -EACCES; 4870 } 4871 4872 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4873 verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", 4874 regno); 4875 return -EACCES; 4876 } 4877 } 4878 4879 return 0; 4880 } 4881 4882 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 4883 struct bpf_call_arg_meta *meta, 4884 const struct bpf_func_proto *fn) 4885 { 4886 u32 regno = BPF_REG_1 + arg; 4887 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4888 enum bpf_arg_type arg_type = fn->arg_type[arg]; 4889 enum bpf_reg_type type = reg->type; 4890 int err = 0; 4891 4892 if (arg_type == ARG_DONTCARE) 4893 return 0; 4894 4895 err = check_reg_arg(env, regno, SRC_OP); 4896 if (err) 4897 return err; 4898 4899 if (arg_type == ARG_ANYTHING) { 4900 if (is_pointer_value(env, regno)) { 4901 verbose(env, "R%d leaks addr into helper function\n", 4902 regno); 4903 return -EACCES; 4904 } 4905 return 0; 4906 } 4907 4908 if (type_is_pkt_pointer(type) && 4909 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 4910 verbose(env, "helper access to the packet is not allowed\n"); 4911 return -EACCES; 4912 } 4913 4914 if (arg_type == ARG_PTR_TO_MAP_VALUE || 4915 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || 4916 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { 4917 err = resolve_map_arg_type(env, meta, &arg_type); 4918 if (err) 4919 return err; 4920 } 4921 4922 if (register_is_null(reg) && arg_type_may_be_null(arg_type)) 4923 /* A NULL register has a SCALAR_VALUE type, so skip 4924 * type checking. 4925 */ 4926 goto skip_type_check; 4927 4928 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); 4929 if (err) 4930 return err; 4931 4932 if (type == PTR_TO_CTX) { 4933 err = check_ctx_reg(env, reg, regno); 4934 if (err < 0) 4935 return err; 4936 } 4937 4938 skip_type_check: 4939 if (reg->ref_obj_id) { 4940 if (meta->ref_obj_id) { 4941 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 4942 regno, reg->ref_obj_id, 4943 meta->ref_obj_id); 4944 return -EFAULT; 4945 } 4946 meta->ref_obj_id = reg->ref_obj_id; 4947 } 4948 4949 if (arg_type == ARG_CONST_MAP_PTR) { 4950 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 4951 meta->map_ptr = reg->map_ptr; 4952 } else if (arg_type == ARG_PTR_TO_MAP_KEY) { 4953 /* bpf_map_xxx(..., map_ptr, ..., key) call: 4954 * check that [key, key + map->key_size) are within 4955 * stack limits and initialized 4956 */ 4957 if (!meta->map_ptr) { 4958 /* in function declaration map_ptr must come before 4959 * map_key, so that it's verified and known before 4960 * we have to check map_key here. Otherwise it means 4961 * that kernel subsystem misconfigured verifier 4962 */ 4963 verbose(env, "invalid map_ptr to access map->key\n"); 4964 return -EACCES; 4965 } 4966 err = check_helper_mem_access(env, regno, 4967 meta->map_ptr->key_size, false, 4968 NULL); 4969 } else if (arg_type == ARG_PTR_TO_MAP_VALUE || 4970 (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && 4971 !register_is_null(reg)) || 4972 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { 4973 /* bpf_map_xxx(..., map_ptr, ..., value) call: 4974 * check [value, value + map->value_size) validity 4975 */ 4976 if (!meta->map_ptr) { 4977 /* kernel subsystem misconfigured verifier */ 4978 verbose(env, "invalid map_ptr to access map->value\n"); 4979 return -EACCES; 4980 } 4981 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); 4982 err = check_helper_mem_access(env, regno, 4983 meta->map_ptr->value_size, false, 4984 meta); 4985 } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { 4986 if (!reg->btf_id) { 4987 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 4988 return -EACCES; 4989 } 4990 meta->ret_btf = reg->btf; 4991 meta->ret_btf_id = reg->btf_id; 4992 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { 4993 if (meta->func_id == BPF_FUNC_spin_lock) { 4994 if (process_spin_lock(env, regno, true)) 4995 return -EACCES; 4996 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 4997 if (process_spin_lock(env, regno, false)) 4998 return -EACCES; 4999 } else { 5000 verbose(env, "verifier internal error\n"); 5001 return -EFAULT; 5002 } 5003 } else if (arg_type == ARG_PTR_TO_FUNC) { 5004 meta->subprogno = reg->subprogno; 5005 } else if (arg_type_is_mem_ptr(arg_type)) { 5006 /* The access to this pointer is only checked when we hit the 5007 * next is_mem_size argument below. 5008 */ 5009 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM); 5010 } else if (arg_type_is_mem_size(arg_type)) { 5011 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); 5012 5013 /* This is used to refine r0 return value bounds for helpers 5014 * that enforce this value as an upper bound on return values. 5015 * See do_refine_retval_range() for helpers that can refine 5016 * the return value. C type of helper is u32 so we pull register 5017 * bound from umax_value however, if negative verifier errors 5018 * out. Only upper bounds can be learned because retval is an 5019 * int type and negative retvals are allowed. 5020 */ 5021 meta->msize_max_value = reg->umax_value; 5022 5023 /* The register is SCALAR_VALUE; the access check 5024 * happens using its boundaries. 5025 */ 5026 if (!tnum_is_const(reg->var_off)) 5027 /* For unprivileged variable accesses, disable raw 5028 * mode so that the program is required to 5029 * initialize all the memory that the helper could 5030 * just partially fill up. 5031 */ 5032 meta = NULL; 5033 5034 if (reg->smin_value < 0) { 5035 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 5036 regno); 5037 return -EACCES; 5038 } 5039 5040 if (reg->umin_value == 0) { 5041 err = check_helper_mem_access(env, regno - 1, 0, 5042 zero_size_allowed, 5043 meta); 5044 if (err) 5045 return err; 5046 } 5047 5048 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 5049 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 5050 regno); 5051 return -EACCES; 5052 } 5053 err = check_helper_mem_access(env, regno - 1, 5054 reg->umax_value, 5055 zero_size_allowed, meta); 5056 if (!err) 5057 err = mark_chain_precision(env, regno); 5058 } else if (arg_type_is_alloc_size(arg_type)) { 5059 if (!tnum_is_const(reg->var_off)) { 5060 verbose(env, "R%d is not a known constant'\n", 5061 regno); 5062 return -EACCES; 5063 } 5064 meta->mem_size = reg->var_off.value; 5065 } else if (arg_type_is_int_ptr(arg_type)) { 5066 int size = int_ptr_type_to_size(arg_type); 5067 5068 err = check_helper_mem_access(env, regno, size, false, meta); 5069 if (err) 5070 return err; 5071 err = check_ptr_alignment(env, reg, 0, size, true); 5072 } else if (arg_type == ARG_PTR_TO_CONST_STR) { 5073 struct bpf_map *map = reg->map_ptr; 5074 int map_off; 5075 u64 map_addr; 5076 char *str_ptr; 5077 5078 if (!bpf_map_is_rdonly(map)) { 5079 verbose(env, "R%d does not point to a readonly map'\n", regno); 5080 return -EACCES; 5081 } 5082 5083 if (!tnum_is_const(reg->var_off)) { 5084 verbose(env, "R%d is not a constant address'\n", regno); 5085 return -EACCES; 5086 } 5087 5088 if (!map->ops->map_direct_value_addr) { 5089 verbose(env, "no direct value access support for this map type\n"); 5090 return -EACCES; 5091 } 5092 5093 err = check_map_access(env, regno, reg->off, 5094 map->value_size - reg->off, false); 5095 if (err) 5096 return err; 5097 5098 map_off = reg->off + reg->var_off.value; 5099 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 5100 if (err) { 5101 verbose(env, "direct value access on string failed\n"); 5102 return err; 5103 } 5104 5105 str_ptr = (char *)(long)(map_addr); 5106 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 5107 verbose(env, "string is not zero-terminated\n"); 5108 return -EINVAL; 5109 } 5110 } 5111 5112 return err; 5113 } 5114 5115 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 5116 { 5117 enum bpf_attach_type eatype = env->prog->expected_attach_type; 5118 enum bpf_prog_type type = resolve_prog_type(env->prog); 5119 5120 if (func_id != BPF_FUNC_map_update_elem) 5121 return false; 5122 5123 /* It's not possible to get access to a locked struct sock in these 5124 * contexts, so updating is safe. 5125 */ 5126 switch (type) { 5127 case BPF_PROG_TYPE_TRACING: 5128 if (eatype == BPF_TRACE_ITER) 5129 return true; 5130 break; 5131 case BPF_PROG_TYPE_SOCKET_FILTER: 5132 case BPF_PROG_TYPE_SCHED_CLS: 5133 case BPF_PROG_TYPE_SCHED_ACT: 5134 case BPF_PROG_TYPE_XDP: 5135 case BPF_PROG_TYPE_SK_REUSEPORT: 5136 case BPF_PROG_TYPE_FLOW_DISSECTOR: 5137 case BPF_PROG_TYPE_SK_LOOKUP: 5138 return true; 5139 default: 5140 break; 5141 } 5142 5143 verbose(env, "cannot update sockmap in this context\n"); 5144 return false; 5145 } 5146 5147 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 5148 { 5149 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); 5150 } 5151 5152 static int check_map_func_compatibility(struct bpf_verifier_env *env, 5153 struct bpf_map *map, int func_id) 5154 { 5155 if (!map) 5156 return 0; 5157 5158 /* We need a two way check, first is from map perspective ... */ 5159 switch (map->map_type) { 5160 case BPF_MAP_TYPE_PROG_ARRAY: 5161 if (func_id != BPF_FUNC_tail_call) 5162 goto error; 5163 break; 5164 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 5165 if (func_id != BPF_FUNC_perf_event_read && 5166 func_id != BPF_FUNC_perf_event_output && 5167 func_id != BPF_FUNC_skb_output && 5168 func_id != BPF_FUNC_perf_event_read_value && 5169 func_id != BPF_FUNC_xdp_output) 5170 goto error; 5171 break; 5172 case BPF_MAP_TYPE_RINGBUF: 5173 if (func_id != BPF_FUNC_ringbuf_output && 5174 func_id != BPF_FUNC_ringbuf_reserve && 5175 func_id != BPF_FUNC_ringbuf_submit && 5176 func_id != BPF_FUNC_ringbuf_discard && 5177 func_id != BPF_FUNC_ringbuf_query) 5178 goto error; 5179 break; 5180 case BPF_MAP_TYPE_STACK_TRACE: 5181 if (func_id != BPF_FUNC_get_stackid) 5182 goto error; 5183 break; 5184 case BPF_MAP_TYPE_CGROUP_ARRAY: 5185 if (func_id != BPF_FUNC_skb_under_cgroup && 5186 func_id != BPF_FUNC_current_task_under_cgroup) 5187 goto error; 5188 break; 5189 case BPF_MAP_TYPE_CGROUP_STORAGE: 5190 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 5191 if (func_id != BPF_FUNC_get_local_storage) 5192 goto error; 5193 break; 5194 case BPF_MAP_TYPE_DEVMAP: 5195 case BPF_MAP_TYPE_DEVMAP_HASH: 5196 if (func_id != BPF_FUNC_redirect_map && 5197 func_id != BPF_FUNC_map_lookup_elem) 5198 goto error; 5199 break; 5200 /* Restrict bpf side of cpumap and xskmap, open when use-cases 5201 * appear. 5202 */ 5203 case BPF_MAP_TYPE_CPUMAP: 5204 if (func_id != BPF_FUNC_redirect_map) 5205 goto error; 5206 break; 5207 case BPF_MAP_TYPE_XSKMAP: 5208 if (func_id != BPF_FUNC_redirect_map && 5209 func_id != BPF_FUNC_map_lookup_elem) 5210 goto error; 5211 break; 5212 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 5213 case BPF_MAP_TYPE_HASH_OF_MAPS: 5214 if (func_id != BPF_FUNC_map_lookup_elem) 5215 goto error; 5216 break; 5217 case BPF_MAP_TYPE_SOCKMAP: 5218 if (func_id != BPF_FUNC_sk_redirect_map && 5219 func_id != BPF_FUNC_sock_map_update && 5220 func_id != BPF_FUNC_map_delete_elem && 5221 func_id != BPF_FUNC_msg_redirect_map && 5222 func_id != BPF_FUNC_sk_select_reuseport && 5223 func_id != BPF_FUNC_map_lookup_elem && 5224 !may_update_sockmap(env, func_id)) 5225 goto error; 5226 break; 5227 case BPF_MAP_TYPE_SOCKHASH: 5228 if (func_id != BPF_FUNC_sk_redirect_hash && 5229 func_id != BPF_FUNC_sock_hash_update && 5230 func_id != BPF_FUNC_map_delete_elem && 5231 func_id != BPF_FUNC_msg_redirect_hash && 5232 func_id != BPF_FUNC_sk_select_reuseport && 5233 func_id != BPF_FUNC_map_lookup_elem && 5234 !may_update_sockmap(env, func_id)) 5235 goto error; 5236 break; 5237 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 5238 if (func_id != BPF_FUNC_sk_select_reuseport) 5239 goto error; 5240 break; 5241 case BPF_MAP_TYPE_QUEUE: 5242 case BPF_MAP_TYPE_STACK: 5243 if (func_id != BPF_FUNC_map_peek_elem && 5244 func_id != BPF_FUNC_map_pop_elem && 5245 func_id != BPF_FUNC_map_push_elem) 5246 goto error; 5247 break; 5248 case BPF_MAP_TYPE_SK_STORAGE: 5249 if (func_id != BPF_FUNC_sk_storage_get && 5250 func_id != BPF_FUNC_sk_storage_delete) 5251 goto error; 5252 break; 5253 case BPF_MAP_TYPE_INODE_STORAGE: 5254 if (func_id != BPF_FUNC_inode_storage_get && 5255 func_id != BPF_FUNC_inode_storage_delete) 5256 goto error; 5257 break; 5258 case BPF_MAP_TYPE_TASK_STORAGE: 5259 if (func_id != BPF_FUNC_task_storage_get && 5260 func_id != BPF_FUNC_task_storage_delete) 5261 goto error; 5262 break; 5263 default: 5264 break; 5265 } 5266 5267 /* ... and second from the function itself. */ 5268 switch (func_id) { 5269 case BPF_FUNC_tail_call: 5270 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 5271 goto error; 5272 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 5273 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 5274 return -EINVAL; 5275 } 5276 break; 5277 case BPF_FUNC_perf_event_read: 5278 case BPF_FUNC_perf_event_output: 5279 case BPF_FUNC_perf_event_read_value: 5280 case BPF_FUNC_skb_output: 5281 case BPF_FUNC_xdp_output: 5282 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 5283 goto error; 5284 break; 5285 case BPF_FUNC_get_stackid: 5286 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 5287 goto error; 5288 break; 5289 case BPF_FUNC_current_task_under_cgroup: 5290 case BPF_FUNC_skb_under_cgroup: 5291 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 5292 goto error; 5293 break; 5294 case BPF_FUNC_redirect_map: 5295 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 5296 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 5297 map->map_type != BPF_MAP_TYPE_CPUMAP && 5298 map->map_type != BPF_MAP_TYPE_XSKMAP) 5299 goto error; 5300 break; 5301 case BPF_FUNC_sk_redirect_map: 5302 case BPF_FUNC_msg_redirect_map: 5303 case BPF_FUNC_sock_map_update: 5304 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 5305 goto error; 5306 break; 5307 case BPF_FUNC_sk_redirect_hash: 5308 case BPF_FUNC_msg_redirect_hash: 5309 case BPF_FUNC_sock_hash_update: 5310 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 5311 goto error; 5312 break; 5313 case BPF_FUNC_get_local_storage: 5314 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 5315 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 5316 goto error; 5317 break; 5318 case BPF_FUNC_sk_select_reuseport: 5319 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 5320 map->map_type != BPF_MAP_TYPE_SOCKMAP && 5321 map->map_type != BPF_MAP_TYPE_SOCKHASH) 5322 goto error; 5323 break; 5324 case BPF_FUNC_map_peek_elem: 5325 case BPF_FUNC_map_pop_elem: 5326 case BPF_FUNC_map_push_elem: 5327 if (map->map_type != BPF_MAP_TYPE_QUEUE && 5328 map->map_type != BPF_MAP_TYPE_STACK) 5329 goto error; 5330 break; 5331 case BPF_FUNC_sk_storage_get: 5332 case BPF_FUNC_sk_storage_delete: 5333 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 5334 goto error; 5335 break; 5336 case BPF_FUNC_inode_storage_get: 5337 case BPF_FUNC_inode_storage_delete: 5338 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 5339 goto error; 5340 break; 5341 case BPF_FUNC_task_storage_get: 5342 case BPF_FUNC_task_storage_delete: 5343 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 5344 goto error; 5345 break; 5346 default: 5347 break; 5348 } 5349 5350 return 0; 5351 error: 5352 verbose(env, "cannot pass map_type %d into func %s#%d\n", 5353 map->map_type, func_id_name(func_id), func_id); 5354 return -EINVAL; 5355 } 5356 5357 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 5358 { 5359 int count = 0; 5360 5361 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 5362 count++; 5363 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 5364 count++; 5365 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 5366 count++; 5367 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 5368 count++; 5369 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 5370 count++; 5371 5372 /* We only support one arg being in raw mode at the moment, 5373 * which is sufficient for the helper functions we have 5374 * right now. 5375 */ 5376 return count <= 1; 5377 } 5378 5379 static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, 5380 enum bpf_arg_type arg_next) 5381 { 5382 return (arg_type_is_mem_ptr(arg_curr) && 5383 !arg_type_is_mem_size(arg_next)) || 5384 (!arg_type_is_mem_ptr(arg_curr) && 5385 arg_type_is_mem_size(arg_next)); 5386 } 5387 5388 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 5389 { 5390 /* bpf_xxx(..., buf, len) call will access 'len' 5391 * bytes from memory 'buf'. Both arg types need 5392 * to be paired, so make sure there's no buggy 5393 * helper function specification. 5394 */ 5395 if (arg_type_is_mem_size(fn->arg1_type) || 5396 arg_type_is_mem_ptr(fn->arg5_type) || 5397 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || 5398 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || 5399 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || 5400 check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) 5401 return false; 5402 5403 return true; 5404 } 5405 5406 static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) 5407 { 5408 int count = 0; 5409 5410 if (arg_type_may_be_refcounted(fn->arg1_type)) 5411 count++; 5412 if (arg_type_may_be_refcounted(fn->arg2_type)) 5413 count++; 5414 if (arg_type_may_be_refcounted(fn->arg3_type)) 5415 count++; 5416 if (arg_type_may_be_refcounted(fn->arg4_type)) 5417 count++; 5418 if (arg_type_may_be_refcounted(fn->arg5_type)) 5419 count++; 5420 5421 /* A reference acquiring function cannot acquire 5422 * another refcounted ptr. 5423 */ 5424 if (may_be_acquire_function(func_id) && count) 5425 return false; 5426 5427 /* We only support one arg being unreferenced at the moment, 5428 * which is sufficient for the helper functions we have right now. 5429 */ 5430 return count <= 1; 5431 } 5432 5433 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 5434 { 5435 int i; 5436 5437 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 5438 if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) 5439 return false; 5440 5441 if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) 5442 return false; 5443 } 5444 5445 return true; 5446 } 5447 5448 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 5449 { 5450 return check_raw_mode_ok(fn) && 5451 check_arg_pair_ok(fn) && 5452 check_btf_id_ok(fn) && 5453 check_refcount_ok(fn, func_id) ? 0 : -EINVAL; 5454 } 5455 5456 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 5457 * are now invalid, so turn them into unknown SCALAR_VALUE. 5458 */ 5459 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, 5460 struct bpf_func_state *state) 5461 { 5462 struct bpf_reg_state *regs = state->regs, *reg; 5463 int i; 5464 5465 for (i = 0; i < MAX_BPF_REG; i++) 5466 if (reg_is_pkt_pointer_any(®s[i])) 5467 mark_reg_unknown(env, regs, i); 5468 5469 bpf_for_each_spilled_reg(i, state, reg) { 5470 if (!reg) 5471 continue; 5472 if (reg_is_pkt_pointer_any(reg)) 5473 __mark_reg_unknown(env, reg); 5474 } 5475 } 5476 5477 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 5478 { 5479 struct bpf_verifier_state *vstate = env->cur_state; 5480 int i; 5481 5482 for (i = 0; i <= vstate->curframe; i++) 5483 __clear_all_pkt_pointers(env, vstate->frame[i]); 5484 } 5485 5486 enum { 5487 AT_PKT_END = -1, 5488 BEYOND_PKT_END = -2, 5489 }; 5490 5491 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 5492 { 5493 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 5494 struct bpf_reg_state *reg = &state->regs[regn]; 5495 5496 if (reg->type != PTR_TO_PACKET) 5497 /* PTR_TO_PACKET_META is not supported yet */ 5498 return; 5499 5500 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 5501 * How far beyond pkt_end it goes is unknown. 5502 * if (!range_open) it's the case of pkt >= pkt_end 5503 * if (range_open) it's the case of pkt > pkt_end 5504 * hence this pointer is at least 1 byte bigger than pkt_end 5505 */ 5506 if (range_open) 5507 reg->range = BEYOND_PKT_END; 5508 else 5509 reg->range = AT_PKT_END; 5510 } 5511 5512 static void release_reg_references(struct bpf_verifier_env *env, 5513 struct bpf_func_state *state, 5514 int ref_obj_id) 5515 { 5516 struct bpf_reg_state *regs = state->regs, *reg; 5517 int i; 5518 5519 for (i = 0; i < MAX_BPF_REG; i++) 5520 if (regs[i].ref_obj_id == ref_obj_id) 5521 mark_reg_unknown(env, regs, i); 5522 5523 bpf_for_each_spilled_reg(i, state, reg) { 5524 if (!reg) 5525 continue; 5526 if (reg->ref_obj_id == ref_obj_id) 5527 __mark_reg_unknown(env, reg); 5528 } 5529 } 5530 5531 /* The pointer with the specified id has released its reference to kernel 5532 * resources. Identify all copies of the same pointer and clear the reference. 5533 */ 5534 static int release_reference(struct bpf_verifier_env *env, 5535 int ref_obj_id) 5536 { 5537 struct bpf_verifier_state *vstate = env->cur_state; 5538 int err; 5539 int i; 5540 5541 err = release_reference_state(cur_func(env), ref_obj_id); 5542 if (err) 5543 return err; 5544 5545 for (i = 0; i <= vstate->curframe; i++) 5546 release_reg_references(env, vstate->frame[i], ref_obj_id); 5547 5548 return 0; 5549 } 5550 5551 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 5552 struct bpf_reg_state *regs) 5553 { 5554 int i; 5555 5556 /* after the call registers r0 - r5 were scratched */ 5557 for (i = 0; i < CALLER_SAVED_REGS; i++) { 5558 mark_reg_not_init(env, regs, caller_saved[i]); 5559 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 5560 } 5561 } 5562 5563 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 5564 struct bpf_func_state *caller, 5565 struct bpf_func_state *callee, 5566 int insn_idx); 5567 5568 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5569 int *insn_idx, int subprog, 5570 set_callee_state_fn set_callee_state_cb) 5571 { 5572 struct bpf_verifier_state *state = env->cur_state; 5573 struct bpf_func_info_aux *func_info_aux; 5574 struct bpf_func_state *caller, *callee; 5575 int err; 5576 bool is_global = false; 5577 5578 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 5579 verbose(env, "the call stack of %d frames is too deep\n", 5580 state->curframe + 2); 5581 return -E2BIG; 5582 } 5583 5584 caller = state->frame[state->curframe]; 5585 if (state->frame[state->curframe + 1]) { 5586 verbose(env, "verifier bug. Frame %d already allocated\n", 5587 state->curframe + 1); 5588 return -EFAULT; 5589 } 5590 5591 func_info_aux = env->prog->aux->func_info_aux; 5592 if (func_info_aux) 5593 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 5594 err = btf_check_subprog_arg_match(env, subprog, caller->regs); 5595 if (err == -EFAULT) 5596 return err; 5597 if (is_global) { 5598 if (err) { 5599 verbose(env, "Caller passes invalid args into func#%d\n", 5600 subprog); 5601 return err; 5602 } else { 5603 if (env->log.level & BPF_LOG_LEVEL) 5604 verbose(env, 5605 "Func#%d is global and valid. Skipping.\n", 5606 subprog); 5607 clear_caller_saved_regs(env, caller->regs); 5608 5609 /* All global functions return a 64-bit SCALAR_VALUE */ 5610 mark_reg_unknown(env, caller->regs, BPF_REG_0); 5611 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 5612 5613 /* continue with next insn after call */ 5614 return 0; 5615 } 5616 } 5617 5618 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 5619 if (!callee) 5620 return -ENOMEM; 5621 state->frame[state->curframe + 1] = callee; 5622 5623 /* callee cannot access r0, r6 - r9 for reading and has to write 5624 * into its own stack before reading from it. 5625 * callee can read/write into caller's stack 5626 */ 5627 init_func_state(env, callee, 5628 /* remember the callsite, it will be used by bpf_exit */ 5629 *insn_idx /* callsite */, 5630 state->curframe + 1 /* frameno within this callchain */, 5631 subprog /* subprog number within this prog */); 5632 5633 /* Transfer references to the callee */ 5634 err = copy_reference_state(callee, caller); 5635 if (err) 5636 return err; 5637 5638 err = set_callee_state_cb(env, caller, callee, *insn_idx); 5639 if (err) 5640 return err; 5641 5642 clear_caller_saved_regs(env, caller->regs); 5643 5644 /* only increment it after check_reg_arg() finished */ 5645 state->curframe++; 5646 5647 /* and go analyze first insn of the callee */ 5648 *insn_idx = env->subprog_info[subprog].start - 1; 5649 5650 if (env->log.level & BPF_LOG_LEVEL) { 5651 verbose(env, "caller:\n"); 5652 print_verifier_state(env, caller); 5653 verbose(env, "callee:\n"); 5654 print_verifier_state(env, callee); 5655 } 5656 return 0; 5657 } 5658 5659 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 5660 struct bpf_func_state *caller, 5661 struct bpf_func_state *callee) 5662 { 5663 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 5664 * void *callback_ctx, u64 flags); 5665 * callback_fn(struct bpf_map *map, void *key, void *value, 5666 * void *callback_ctx); 5667 */ 5668 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 5669 5670 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 5671 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 5672 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 5673 5674 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 5675 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 5676 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 5677 5678 /* pointer to stack or null */ 5679 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 5680 5681 /* unused */ 5682 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 5683 return 0; 5684 } 5685 5686 static int set_callee_state(struct bpf_verifier_env *env, 5687 struct bpf_func_state *caller, 5688 struct bpf_func_state *callee, int insn_idx) 5689 { 5690 int i; 5691 5692 /* copy r1 - r5 args that callee can access. The copy includes parent 5693 * pointers, which connects us up to the liveness chain 5694 */ 5695 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 5696 callee->regs[i] = caller->regs[i]; 5697 return 0; 5698 } 5699 5700 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5701 int *insn_idx) 5702 { 5703 int subprog, target_insn; 5704 5705 target_insn = *insn_idx + insn->imm + 1; 5706 subprog = find_subprog(env, target_insn); 5707 if (subprog < 0) { 5708 verbose(env, "verifier bug. No program starts at insn %d\n", 5709 target_insn); 5710 return -EFAULT; 5711 } 5712 5713 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 5714 } 5715 5716 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 5717 struct bpf_func_state *caller, 5718 struct bpf_func_state *callee, 5719 int insn_idx) 5720 { 5721 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 5722 struct bpf_map *map; 5723 int err; 5724 5725 if (bpf_map_ptr_poisoned(insn_aux)) { 5726 verbose(env, "tail_call abusing map_ptr\n"); 5727 return -EINVAL; 5728 } 5729 5730 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 5731 if (!map->ops->map_set_for_each_callback_args || 5732 !map->ops->map_for_each_callback) { 5733 verbose(env, "callback function not allowed for map\n"); 5734 return -ENOTSUPP; 5735 } 5736 5737 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 5738 if (err) 5739 return err; 5740 5741 callee->in_callback_fn = true; 5742 return 0; 5743 } 5744 5745 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 5746 { 5747 struct bpf_verifier_state *state = env->cur_state; 5748 struct bpf_func_state *caller, *callee; 5749 struct bpf_reg_state *r0; 5750 int err; 5751 5752 callee = state->frame[state->curframe]; 5753 r0 = &callee->regs[BPF_REG_0]; 5754 if (r0->type == PTR_TO_STACK) { 5755 /* technically it's ok to return caller's stack pointer 5756 * (or caller's caller's pointer) back to the caller, 5757 * since these pointers are valid. Only current stack 5758 * pointer will be invalid as soon as function exits, 5759 * but let's be conservative 5760 */ 5761 verbose(env, "cannot return stack pointer to the caller\n"); 5762 return -EINVAL; 5763 } 5764 5765 state->curframe--; 5766 caller = state->frame[state->curframe]; 5767 if (callee->in_callback_fn) { 5768 /* enforce R0 return value range [0, 1]. */ 5769 struct tnum range = tnum_range(0, 1); 5770 5771 if (r0->type != SCALAR_VALUE) { 5772 verbose(env, "R0 not a scalar value\n"); 5773 return -EACCES; 5774 } 5775 if (!tnum_in(range, r0->var_off)) { 5776 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 5777 return -EINVAL; 5778 } 5779 } else { 5780 /* return to the caller whatever r0 had in the callee */ 5781 caller->regs[BPF_REG_0] = *r0; 5782 } 5783 5784 /* Transfer references to the caller */ 5785 err = copy_reference_state(caller, callee); 5786 if (err) 5787 return err; 5788 5789 *insn_idx = callee->callsite + 1; 5790 if (env->log.level & BPF_LOG_LEVEL) { 5791 verbose(env, "returning from callee:\n"); 5792 print_verifier_state(env, callee); 5793 verbose(env, "to caller at %d:\n", *insn_idx); 5794 print_verifier_state(env, caller); 5795 } 5796 /* clear everything in the callee */ 5797 free_func_state(callee); 5798 state->frame[state->curframe + 1] = NULL; 5799 return 0; 5800 } 5801 5802 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 5803 int func_id, 5804 struct bpf_call_arg_meta *meta) 5805 { 5806 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 5807 5808 if (ret_type != RET_INTEGER || 5809 (func_id != BPF_FUNC_get_stack && 5810 func_id != BPF_FUNC_get_task_stack && 5811 func_id != BPF_FUNC_probe_read_str && 5812 func_id != BPF_FUNC_probe_read_kernel_str && 5813 func_id != BPF_FUNC_probe_read_user_str)) 5814 return; 5815 5816 ret_reg->smax_value = meta->msize_max_value; 5817 ret_reg->s32_max_value = meta->msize_max_value; 5818 ret_reg->smin_value = -MAX_ERRNO; 5819 ret_reg->s32_min_value = -MAX_ERRNO; 5820 __reg_deduce_bounds(ret_reg); 5821 __reg_bound_offset(ret_reg); 5822 __update_reg_bounds(ret_reg); 5823 } 5824 5825 static int 5826 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 5827 int func_id, int insn_idx) 5828 { 5829 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 5830 struct bpf_map *map = meta->map_ptr; 5831 5832 if (func_id != BPF_FUNC_tail_call && 5833 func_id != BPF_FUNC_map_lookup_elem && 5834 func_id != BPF_FUNC_map_update_elem && 5835 func_id != BPF_FUNC_map_delete_elem && 5836 func_id != BPF_FUNC_map_push_elem && 5837 func_id != BPF_FUNC_map_pop_elem && 5838 func_id != BPF_FUNC_map_peek_elem && 5839 func_id != BPF_FUNC_for_each_map_elem && 5840 func_id != BPF_FUNC_redirect_map) 5841 return 0; 5842 5843 if (map == NULL) { 5844 verbose(env, "kernel subsystem misconfigured verifier\n"); 5845 return -EINVAL; 5846 } 5847 5848 /* In case of read-only, some additional restrictions 5849 * need to be applied in order to prevent altering the 5850 * state of the map from program side. 5851 */ 5852 if ((map->map_flags & BPF_F_RDONLY_PROG) && 5853 (func_id == BPF_FUNC_map_delete_elem || 5854 func_id == BPF_FUNC_map_update_elem || 5855 func_id == BPF_FUNC_map_push_elem || 5856 func_id == BPF_FUNC_map_pop_elem)) { 5857 verbose(env, "write into map forbidden\n"); 5858 return -EACCES; 5859 } 5860 5861 if (!BPF_MAP_PTR(aux->map_ptr_state)) 5862 bpf_map_ptr_store(aux, meta->map_ptr, 5863 !meta->map_ptr->bypass_spec_v1); 5864 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 5865 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 5866 !meta->map_ptr->bypass_spec_v1); 5867 return 0; 5868 } 5869 5870 static int 5871 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 5872 int func_id, int insn_idx) 5873 { 5874 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 5875 struct bpf_reg_state *regs = cur_regs(env), *reg; 5876 struct bpf_map *map = meta->map_ptr; 5877 struct tnum range; 5878 u64 val; 5879 int err; 5880 5881 if (func_id != BPF_FUNC_tail_call) 5882 return 0; 5883 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 5884 verbose(env, "kernel subsystem misconfigured verifier\n"); 5885 return -EINVAL; 5886 } 5887 5888 range = tnum_range(0, map->max_entries - 1); 5889 reg = ®s[BPF_REG_3]; 5890 5891 if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { 5892 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 5893 return 0; 5894 } 5895 5896 err = mark_chain_precision(env, BPF_REG_3); 5897 if (err) 5898 return err; 5899 5900 val = reg->var_off.value; 5901 if (bpf_map_key_unseen(aux)) 5902 bpf_map_key_store(aux, val); 5903 else if (!bpf_map_key_poisoned(aux) && 5904 bpf_map_key_immediate(aux) != val) 5905 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 5906 return 0; 5907 } 5908 5909 static int check_reference_leak(struct bpf_verifier_env *env) 5910 { 5911 struct bpf_func_state *state = cur_func(env); 5912 int i; 5913 5914 for (i = 0; i < state->acquired_refs; i++) { 5915 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 5916 state->refs[i].id, state->refs[i].insn_idx); 5917 } 5918 return state->acquired_refs ? -EINVAL : 0; 5919 } 5920 5921 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 5922 struct bpf_reg_state *regs) 5923 { 5924 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 5925 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 5926 struct bpf_map *fmt_map = fmt_reg->map_ptr; 5927 int err, fmt_map_off, num_args; 5928 u64 fmt_addr; 5929 char *fmt; 5930 5931 /* data must be an array of u64 */ 5932 if (data_len_reg->var_off.value % 8) 5933 return -EINVAL; 5934 num_args = data_len_reg->var_off.value / 8; 5935 5936 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 5937 * and map_direct_value_addr is set. 5938 */ 5939 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 5940 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 5941 fmt_map_off); 5942 if (err) { 5943 verbose(env, "verifier bug\n"); 5944 return -EFAULT; 5945 } 5946 fmt = (char *)(long)fmt_addr + fmt_map_off; 5947 5948 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 5949 * can focus on validating the format specifiers. 5950 */ 5951 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args); 5952 if (err < 0) 5953 verbose(env, "Invalid format string\n"); 5954 5955 return err; 5956 } 5957 5958 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5959 int *insn_idx_p) 5960 { 5961 const struct bpf_func_proto *fn = NULL; 5962 struct bpf_reg_state *regs; 5963 struct bpf_call_arg_meta meta; 5964 int insn_idx = *insn_idx_p; 5965 bool changes_data; 5966 int i, err, func_id; 5967 5968 /* find function prototype */ 5969 func_id = insn->imm; 5970 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 5971 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 5972 func_id); 5973 return -EINVAL; 5974 } 5975 5976 if (env->ops->get_func_proto) 5977 fn = env->ops->get_func_proto(func_id, env->prog); 5978 if (!fn) { 5979 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 5980 func_id); 5981 return -EINVAL; 5982 } 5983 5984 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 5985 if (!env->prog->gpl_compatible && fn->gpl_only) { 5986 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 5987 return -EINVAL; 5988 } 5989 5990 if (fn->allowed && !fn->allowed(env->prog)) { 5991 verbose(env, "helper call is not allowed in probe\n"); 5992 return -EINVAL; 5993 } 5994 5995 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 5996 changes_data = bpf_helper_changes_pkt_data(fn->func); 5997 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 5998 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 5999 func_id_name(func_id), func_id); 6000 return -EINVAL; 6001 } 6002 6003 memset(&meta, 0, sizeof(meta)); 6004 meta.pkt_access = fn->pkt_access; 6005 6006 err = check_func_proto(fn, func_id); 6007 if (err) { 6008 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 6009 func_id_name(func_id), func_id); 6010 return err; 6011 } 6012 6013 meta.func_id = func_id; 6014 /* check args */ 6015 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 6016 err = check_func_arg(env, i, &meta, fn); 6017 if (err) 6018 return err; 6019 } 6020 6021 err = record_func_map(env, &meta, func_id, insn_idx); 6022 if (err) 6023 return err; 6024 6025 err = record_func_key(env, &meta, func_id, insn_idx); 6026 if (err) 6027 return err; 6028 6029 /* Mark slots with STACK_MISC in case of raw mode, stack offset 6030 * is inferred from register state. 6031 */ 6032 for (i = 0; i < meta.access_size; i++) { 6033 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 6034 BPF_WRITE, -1, false); 6035 if (err) 6036 return err; 6037 } 6038 6039 if (func_id == BPF_FUNC_tail_call) { 6040 err = check_reference_leak(env); 6041 if (err) { 6042 verbose(env, "tail_call would lead to reference leak\n"); 6043 return err; 6044 } 6045 } else if (is_release_function(func_id)) { 6046 err = release_reference(env, meta.ref_obj_id); 6047 if (err) { 6048 verbose(env, "func %s#%d reference has not been acquired before\n", 6049 func_id_name(func_id), func_id); 6050 return err; 6051 } 6052 } 6053 6054 regs = cur_regs(env); 6055 6056 /* check that flags argument in get_local_storage(map, flags) is 0, 6057 * this is required because get_local_storage() can't return an error. 6058 */ 6059 if (func_id == BPF_FUNC_get_local_storage && 6060 !register_is_null(®s[BPF_REG_2])) { 6061 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 6062 return -EINVAL; 6063 } 6064 6065 if (func_id == BPF_FUNC_for_each_map_elem) { 6066 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 6067 set_map_elem_callback_state); 6068 if (err < 0) 6069 return -EINVAL; 6070 } 6071 6072 if (func_id == BPF_FUNC_snprintf) { 6073 err = check_bpf_snprintf_call(env, regs); 6074 if (err < 0) 6075 return err; 6076 } 6077 6078 /* reset caller saved regs */ 6079 for (i = 0; i < CALLER_SAVED_REGS; i++) { 6080 mark_reg_not_init(env, regs, caller_saved[i]); 6081 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 6082 } 6083 6084 /* helper call returns 64-bit value. */ 6085 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 6086 6087 /* update return register (already marked as written above) */ 6088 if (fn->ret_type == RET_INTEGER) { 6089 /* sets type to SCALAR_VALUE */ 6090 mark_reg_unknown(env, regs, BPF_REG_0); 6091 } else if (fn->ret_type == RET_VOID) { 6092 regs[BPF_REG_0].type = NOT_INIT; 6093 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || 6094 fn->ret_type == RET_PTR_TO_MAP_VALUE) { 6095 /* There is no offset yet applied, variable or fixed */ 6096 mark_reg_known_zero(env, regs, BPF_REG_0); 6097 /* remember map_ptr, so that check_map_access() 6098 * can check 'value_size' boundary of memory access 6099 * to map element returned from bpf_map_lookup_elem() 6100 */ 6101 if (meta.map_ptr == NULL) { 6102 verbose(env, 6103 "kernel subsystem misconfigured verifier\n"); 6104 return -EINVAL; 6105 } 6106 regs[BPF_REG_0].map_ptr = meta.map_ptr; 6107 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { 6108 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; 6109 if (map_value_has_spin_lock(meta.map_ptr)) 6110 regs[BPF_REG_0].id = ++env->id_gen; 6111 } else { 6112 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; 6113 } 6114 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { 6115 mark_reg_known_zero(env, regs, BPF_REG_0); 6116 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; 6117 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { 6118 mark_reg_known_zero(env, regs, BPF_REG_0); 6119 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; 6120 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { 6121 mark_reg_known_zero(env, regs, BPF_REG_0); 6122 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; 6123 } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) { 6124 mark_reg_known_zero(env, regs, BPF_REG_0); 6125 regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; 6126 regs[BPF_REG_0].mem_size = meta.mem_size; 6127 } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL || 6128 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) { 6129 const struct btf_type *t; 6130 6131 mark_reg_known_zero(env, regs, BPF_REG_0); 6132 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 6133 if (!btf_type_is_struct(t)) { 6134 u32 tsize; 6135 const struct btf_type *ret; 6136 const char *tname; 6137 6138 /* resolve the type size of ksym. */ 6139 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 6140 if (IS_ERR(ret)) { 6141 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 6142 verbose(env, "unable to resolve the size of type '%s': %ld\n", 6143 tname, PTR_ERR(ret)); 6144 return -EINVAL; 6145 } 6146 regs[BPF_REG_0].type = 6147 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? 6148 PTR_TO_MEM : PTR_TO_MEM_OR_NULL; 6149 regs[BPF_REG_0].mem_size = tsize; 6150 } else { 6151 regs[BPF_REG_0].type = 6152 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? 6153 PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL; 6154 regs[BPF_REG_0].btf = meta.ret_btf; 6155 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 6156 } 6157 } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL || 6158 fn->ret_type == RET_PTR_TO_BTF_ID) { 6159 int ret_btf_id; 6160 6161 mark_reg_known_zero(env, regs, BPF_REG_0); 6162 regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ? 6163 PTR_TO_BTF_ID : 6164 PTR_TO_BTF_ID_OR_NULL; 6165 ret_btf_id = *fn->ret_btf_id; 6166 if (ret_btf_id == 0) { 6167 verbose(env, "invalid return type %d of func %s#%d\n", 6168 fn->ret_type, func_id_name(func_id), func_id); 6169 return -EINVAL; 6170 } 6171 /* current BPF helper definitions are only coming from 6172 * built-in code with type IDs from vmlinux BTF 6173 */ 6174 regs[BPF_REG_0].btf = btf_vmlinux; 6175 regs[BPF_REG_0].btf_id = ret_btf_id; 6176 } else { 6177 verbose(env, "unknown return type %d of func %s#%d\n", 6178 fn->ret_type, func_id_name(func_id), func_id); 6179 return -EINVAL; 6180 } 6181 6182 if (reg_type_may_be_null(regs[BPF_REG_0].type)) 6183 regs[BPF_REG_0].id = ++env->id_gen; 6184 6185 if (is_ptr_cast_function(func_id)) { 6186 /* For release_reference() */ 6187 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 6188 } else if (is_acquire_function(func_id, meta.map_ptr)) { 6189 int id = acquire_reference_state(env, insn_idx); 6190 6191 if (id < 0) 6192 return id; 6193 /* For mark_ptr_or_null_reg() */ 6194 regs[BPF_REG_0].id = id; 6195 /* For release_reference() */ 6196 regs[BPF_REG_0].ref_obj_id = id; 6197 } 6198 6199 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 6200 6201 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 6202 if (err) 6203 return err; 6204 6205 if ((func_id == BPF_FUNC_get_stack || 6206 func_id == BPF_FUNC_get_task_stack) && 6207 !env->prog->has_callchain_buf) { 6208 const char *err_str; 6209 6210 #ifdef CONFIG_PERF_EVENTS 6211 err = get_callchain_buffers(sysctl_perf_event_max_stack); 6212 err_str = "cannot get callchain buffer for func %s#%d\n"; 6213 #else 6214 err = -ENOTSUPP; 6215 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 6216 #endif 6217 if (err) { 6218 verbose(env, err_str, func_id_name(func_id), func_id); 6219 return err; 6220 } 6221 6222 env->prog->has_callchain_buf = true; 6223 } 6224 6225 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 6226 env->prog->call_get_stack = true; 6227 6228 if (changes_data) 6229 clear_all_pkt_pointers(env); 6230 return 0; 6231 } 6232 6233 /* mark_btf_func_reg_size() is used when the reg size is determined by 6234 * the BTF func_proto's return value size and argument. 6235 */ 6236 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 6237 size_t reg_size) 6238 { 6239 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 6240 6241 if (regno == BPF_REG_0) { 6242 /* Function return value */ 6243 reg->live |= REG_LIVE_WRITTEN; 6244 reg->subreg_def = reg_size == sizeof(u64) ? 6245 DEF_NOT_SUBREG : env->insn_idx + 1; 6246 } else { 6247 /* Function argument */ 6248 if (reg_size == sizeof(u64)) { 6249 mark_insn_zext(env, reg); 6250 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 6251 } else { 6252 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 6253 } 6254 } 6255 } 6256 6257 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn) 6258 { 6259 const struct btf_type *t, *func, *func_proto, *ptr_type; 6260 struct bpf_reg_state *regs = cur_regs(env); 6261 const char *func_name, *ptr_type_name; 6262 u32 i, nargs, func_id, ptr_type_id; 6263 const struct btf_param *args; 6264 int err; 6265 6266 func_id = insn->imm; 6267 func = btf_type_by_id(btf_vmlinux, func_id); 6268 func_name = btf_name_by_offset(btf_vmlinux, func->name_off); 6269 func_proto = btf_type_by_id(btf_vmlinux, func->type); 6270 6271 if (!env->ops->check_kfunc_call || 6272 !env->ops->check_kfunc_call(func_id)) { 6273 verbose(env, "calling kernel function %s is not allowed\n", 6274 func_name); 6275 return -EACCES; 6276 } 6277 6278 /* Check the arguments */ 6279 err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs); 6280 if (err) 6281 return err; 6282 6283 for (i = 0; i < CALLER_SAVED_REGS; i++) 6284 mark_reg_not_init(env, regs, caller_saved[i]); 6285 6286 /* Check return type */ 6287 t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL); 6288 if (btf_type_is_scalar(t)) { 6289 mark_reg_unknown(env, regs, BPF_REG_0); 6290 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 6291 } else if (btf_type_is_ptr(t)) { 6292 ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type, 6293 &ptr_type_id); 6294 if (!btf_type_is_struct(ptr_type)) { 6295 ptr_type_name = btf_name_by_offset(btf_vmlinux, 6296 ptr_type->name_off); 6297 verbose(env, "kernel function %s returns pointer type %s %s is not supported\n", 6298 func_name, btf_type_str(ptr_type), 6299 ptr_type_name); 6300 return -EINVAL; 6301 } 6302 mark_reg_known_zero(env, regs, BPF_REG_0); 6303 regs[BPF_REG_0].btf = btf_vmlinux; 6304 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 6305 regs[BPF_REG_0].btf_id = ptr_type_id; 6306 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 6307 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ 6308 6309 nargs = btf_type_vlen(func_proto); 6310 args = (const struct btf_param *)(func_proto + 1); 6311 for (i = 0; i < nargs; i++) { 6312 u32 regno = i + 1; 6313 6314 t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL); 6315 if (btf_type_is_ptr(t)) 6316 mark_btf_func_reg_size(env, regno, sizeof(void *)); 6317 else 6318 /* scalar. ensured by btf_check_kfunc_arg_match() */ 6319 mark_btf_func_reg_size(env, regno, t->size); 6320 } 6321 6322 return 0; 6323 } 6324 6325 static bool signed_add_overflows(s64 a, s64 b) 6326 { 6327 /* Do the add in u64, where overflow is well-defined */ 6328 s64 res = (s64)((u64)a + (u64)b); 6329 6330 if (b < 0) 6331 return res > a; 6332 return res < a; 6333 } 6334 6335 static bool signed_add32_overflows(s32 a, s32 b) 6336 { 6337 /* Do the add in u32, where overflow is well-defined */ 6338 s32 res = (s32)((u32)a + (u32)b); 6339 6340 if (b < 0) 6341 return res > a; 6342 return res < a; 6343 } 6344 6345 static bool signed_sub_overflows(s64 a, s64 b) 6346 { 6347 /* Do the sub in u64, where overflow is well-defined */ 6348 s64 res = (s64)((u64)a - (u64)b); 6349 6350 if (b < 0) 6351 return res < a; 6352 return res > a; 6353 } 6354 6355 static bool signed_sub32_overflows(s32 a, s32 b) 6356 { 6357 /* Do the sub in u32, where overflow is well-defined */ 6358 s32 res = (s32)((u32)a - (u32)b); 6359 6360 if (b < 0) 6361 return res < a; 6362 return res > a; 6363 } 6364 6365 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 6366 const struct bpf_reg_state *reg, 6367 enum bpf_reg_type type) 6368 { 6369 bool known = tnum_is_const(reg->var_off); 6370 s64 val = reg->var_off.value; 6371 s64 smin = reg->smin_value; 6372 6373 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 6374 verbose(env, "math between %s pointer and %lld is not allowed\n", 6375 reg_type_str[type], val); 6376 return false; 6377 } 6378 6379 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 6380 verbose(env, "%s pointer offset %d is not allowed\n", 6381 reg_type_str[type], reg->off); 6382 return false; 6383 } 6384 6385 if (smin == S64_MIN) { 6386 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 6387 reg_type_str[type]); 6388 return false; 6389 } 6390 6391 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 6392 verbose(env, "value %lld makes %s pointer be out of bounds\n", 6393 smin, reg_type_str[type]); 6394 return false; 6395 } 6396 6397 return true; 6398 } 6399 6400 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 6401 { 6402 return &env->insn_aux_data[env->insn_idx]; 6403 } 6404 6405 enum { 6406 REASON_BOUNDS = -1, 6407 REASON_TYPE = -2, 6408 REASON_PATHS = -3, 6409 REASON_LIMIT = -4, 6410 REASON_STACK = -5, 6411 }; 6412 6413 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 6414 const struct bpf_reg_state *off_reg, 6415 u32 *alu_limit, u8 opcode) 6416 { 6417 bool off_is_neg = off_reg->smin_value < 0; 6418 bool mask_to_left = (opcode == BPF_ADD && off_is_neg) || 6419 (opcode == BPF_SUB && !off_is_neg); 6420 u32 max = 0, ptr_limit = 0; 6421 6422 if (!tnum_is_const(off_reg->var_off) && 6423 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 6424 return REASON_BOUNDS; 6425 6426 switch (ptr_reg->type) { 6427 case PTR_TO_STACK: 6428 /* Offset 0 is out-of-bounds, but acceptable start for the 6429 * left direction, see BPF_REG_FP. Also, unknown scalar 6430 * offset where we would need to deal with min/max bounds is 6431 * currently prohibited for unprivileged. 6432 */ 6433 max = MAX_BPF_STACK + mask_to_left; 6434 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 6435 break; 6436 case PTR_TO_MAP_VALUE: 6437 max = ptr_reg->map_ptr->value_size; 6438 ptr_limit = (mask_to_left ? 6439 ptr_reg->smin_value : 6440 ptr_reg->umax_value) + ptr_reg->off; 6441 break; 6442 default: 6443 return REASON_TYPE; 6444 } 6445 6446 if (ptr_limit >= max) 6447 return REASON_LIMIT; 6448 *alu_limit = ptr_limit; 6449 return 0; 6450 } 6451 6452 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 6453 const struct bpf_insn *insn) 6454 { 6455 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 6456 } 6457 6458 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 6459 u32 alu_state, u32 alu_limit) 6460 { 6461 /* If we arrived here from different branches with different 6462 * state or limits to sanitize, then this won't work. 6463 */ 6464 if (aux->alu_state && 6465 (aux->alu_state != alu_state || 6466 aux->alu_limit != alu_limit)) 6467 return REASON_PATHS; 6468 6469 /* Corresponding fixup done in do_misc_fixups(). */ 6470 aux->alu_state = alu_state; 6471 aux->alu_limit = alu_limit; 6472 return 0; 6473 } 6474 6475 static int sanitize_val_alu(struct bpf_verifier_env *env, 6476 struct bpf_insn *insn) 6477 { 6478 struct bpf_insn_aux_data *aux = cur_aux(env); 6479 6480 if (can_skip_alu_sanitation(env, insn)) 6481 return 0; 6482 6483 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 6484 } 6485 6486 static bool sanitize_needed(u8 opcode) 6487 { 6488 return opcode == BPF_ADD || opcode == BPF_SUB; 6489 } 6490 6491 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 6492 struct bpf_insn *insn, 6493 const struct bpf_reg_state *ptr_reg, 6494 const struct bpf_reg_state *off_reg, 6495 struct bpf_reg_state *dst_reg, 6496 struct bpf_insn_aux_data *tmp_aux, 6497 const bool commit_window) 6498 { 6499 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : tmp_aux; 6500 struct bpf_verifier_state *vstate = env->cur_state; 6501 bool off_is_imm = tnum_is_const(off_reg->var_off); 6502 bool off_is_neg = off_reg->smin_value < 0; 6503 bool ptr_is_dst_reg = ptr_reg == dst_reg; 6504 u8 opcode = BPF_OP(insn->code); 6505 u32 alu_state, alu_limit; 6506 struct bpf_reg_state tmp; 6507 bool ret; 6508 int err; 6509 6510 if (can_skip_alu_sanitation(env, insn)) 6511 return 0; 6512 6513 /* We already marked aux for masking from non-speculative 6514 * paths, thus we got here in the first place. We only care 6515 * to explore bad access from here. 6516 */ 6517 if (vstate->speculative) 6518 goto do_sim; 6519 6520 err = retrieve_ptr_limit(ptr_reg, off_reg, &alu_limit, opcode); 6521 if (err < 0) 6522 return err; 6523 6524 if (commit_window) { 6525 /* In commit phase we narrow the masking window based on 6526 * the observed pointer move after the simulated operation. 6527 */ 6528 alu_state = tmp_aux->alu_state; 6529 alu_limit = abs(tmp_aux->alu_limit - alu_limit); 6530 } else { 6531 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 6532 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 6533 alu_state |= ptr_is_dst_reg ? 6534 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 6535 } 6536 6537 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 6538 if (err < 0) 6539 return err; 6540 do_sim: 6541 /* If we're in commit phase, we're done here given we already 6542 * pushed the truncated dst_reg into the speculative verification 6543 * stack. 6544 */ 6545 if (commit_window) 6546 return 0; 6547 6548 /* Simulate and find potential out-of-bounds access under 6549 * speculative execution from truncation as a result of 6550 * masking when off was not within expected range. If off 6551 * sits in dst, then we temporarily need to move ptr there 6552 * to simulate dst (== 0) +/-= ptr. Needed, for example, 6553 * for cases where we use K-based arithmetic in one direction 6554 * and truncated reg-based in the other in order to explore 6555 * bad access. 6556 */ 6557 if (!ptr_is_dst_reg) { 6558 tmp = *dst_reg; 6559 *dst_reg = *ptr_reg; 6560 } 6561 ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true); 6562 if (!ptr_is_dst_reg && ret) 6563 *dst_reg = tmp; 6564 return !ret ? REASON_STACK : 0; 6565 } 6566 6567 static int sanitize_err(struct bpf_verifier_env *env, 6568 const struct bpf_insn *insn, int reason, 6569 const struct bpf_reg_state *off_reg, 6570 const struct bpf_reg_state *dst_reg) 6571 { 6572 static const char *err = "pointer arithmetic with it prohibited for !root"; 6573 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 6574 u32 dst = insn->dst_reg, src = insn->src_reg; 6575 6576 switch (reason) { 6577 case REASON_BOUNDS: 6578 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 6579 off_reg == dst_reg ? dst : src, err); 6580 break; 6581 case REASON_TYPE: 6582 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 6583 off_reg == dst_reg ? src : dst, err); 6584 break; 6585 case REASON_PATHS: 6586 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 6587 dst, op, err); 6588 break; 6589 case REASON_LIMIT: 6590 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 6591 dst, op, err); 6592 break; 6593 case REASON_STACK: 6594 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 6595 dst, err); 6596 break; 6597 default: 6598 verbose(env, "verifier internal error: unknown reason (%d)\n", 6599 reason); 6600 break; 6601 } 6602 6603 return -EACCES; 6604 } 6605 6606 /* check that stack access falls within stack limits and that 'reg' doesn't 6607 * have a variable offset. 6608 * 6609 * Variable offset is prohibited for unprivileged mode for simplicity since it 6610 * requires corresponding support in Spectre masking for stack ALU. See also 6611 * retrieve_ptr_limit(). 6612 * 6613 * 6614 * 'off' includes 'reg->off'. 6615 */ 6616 static int check_stack_access_for_ptr_arithmetic( 6617 struct bpf_verifier_env *env, 6618 int regno, 6619 const struct bpf_reg_state *reg, 6620 int off) 6621 { 6622 if (!tnum_is_const(reg->var_off)) { 6623 char tn_buf[48]; 6624 6625 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6626 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 6627 regno, tn_buf, off); 6628 return -EACCES; 6629 } 6630 6631 if (off >= 0 || off < -MAX_BPF_STACK) { 6632 verbose(env, "R%d stack pointer arithmetic goes out of range, " 6633 "prohibited for !root; off=%d\n", regno, off); 6634 return -EACCES; 6635 } 6636 6637 return 0; 6638 } 6639 6640 static int sanitize_check_bounds(struct bpf_verifier_env *env, 6641 const struct bpf_insn *insn, 6642 const struct bpf_reg_state *dst_reg) 6643 { 6644 u32 dst = insn->dst_reg; 6645 6646 /* For unprivileged we require that resulting offset must be in bounds 6647 * in order to be able to sanitize access later on. 6648 */ 6649 if (env->bypass_spec_v1) 6650 return 0; 6651 6652 switch (dst_reg->type) { 6653 case PTR_TO_STACK: 6654 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 6655 dst_reg->off + dst_reg->var_off.value)) 6656 return -EACCES; 6657 break; 6658 case PTR_TO_MAP_VALUE: 6659 if (check_map_access(env, dst, dst_reg->off, 1, false)) { 6660 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 6661 "prohibited for !root\n", dst); 6662 return -EACCES; 6663 } 6664 break; 6665 default: 6666 break; 6667 } 6668 6669 return 0; 6670 } 6671 6672 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 6673 * Caller should also handle BPF_MOV case separately. 6674 * If we return -EACCES, caller may want to try again treating pointer as a 6675 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 6676 */ 6677 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 6678 struct bpf_insn *insn, 6679 const struct bpf_reg_state *ptr_reg, 6680 const struct bpf_reg_state *off_reg) 6681 { 6682 struct bpf_verifier_state *vstate = env->cur_state; 6683 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 6684 struct bpf_reg_state *regs = state->regs, *dst_reg; 6685 bool known = tnum_is_const(off_reg->var_off); 6686 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 6687 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 6688 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 6689 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 6690 struct bpf_insn_aux_data tmp_aux = {}; 6691 u8 opcode = BPF_OP(insn->code); 6692 u32 dst = insn->dst_reg; 6693 int ret; 6694 6695 dst_reg = ®s[dst]; 6696 6697 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 6698 smin_val > smax_val || umin_val > umax_val) { 6699 /* Taint dst register if offset had invalid bounds derived from 6700 * e.g. dead branches. 6701 */ 6702 __mark_reg_unknown(env, dst_reg); 6703 return 0; 6704 } 6705 6706 if (BPF_CLASS(insn->code) != BPF_ALU64) { 6707 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 6708 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 6709 __mark_reg_unknown(env, dst_reg); 6710 return 0; 6711 } 6712 6713 verbose(env, 6714 "R%d 32-bit pointer arithmetic prohibited\n", 6715 dst); 6716 return -EACCES; 6717 } 6718 6719 switch (ptr_reg->type) { 6720 case PTR_TO_MAP_VALUE_OR_NULL: 6721 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 6722 dst, reg_type_str[ptr_reg->type]); 6723 return -EACCES; 6724 case CONST_PTR_TO_MAP: 6725 /* smin_val represents the known value */ 6726 if (known && smin_val == 0 && opcode == BPF_ADD) 6727 break; 6728 fallthrough; 6729 case PTR_TO_PACKET_END: 6730 case PTR_TO_SOCKET: 6731 case PTR_TO_SOCKET_OR_NULL: 6732 case PTR_TO_SOCK_COMMON: 6733 case PTR_TO_SOCK_COMMON_OR_NULL: 6734 case PTR_TO_TCP_SOCK: 6735 case PTR_TO_TCP_SOCK_OR_NULL: 6736 case PTR_TO_XDP_SOCK: 6737 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 6738 dst, reg_type_str[ptr_reg->type]); 6739 return -EACCES; 6740 default: 6741 break; 6742 } 6743 6744 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 6745 * The id may be overwritten later if we create a new variable offset. 6746 */ 6747 dst_reg->type = ptr_reg->type; 6748 dst_reg->id = ptr_reg->id; 6749 6750 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 6751 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 6752 return -EINVAL; 6753 6754 /* pointer types do not carry 32-bit bounds at the moment. */ 6755 __mark_reg32_unbounded(dst_reg); 6756 6757 if (sanitize_needed(opcode)) { 6758 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 6759 &tmp_aux, false); 6760 if (ret < 0) 6761 return sanitize_err(env, insn, ret, off_reg, dst_reg); 6762 } 6763 6764 switch (opcode) { 6765 case BPF_ADD: 6766 /* We can take a fixed offset as long as it doesn't overflow 6767 * the s32 'off' field 6768 */ 6769 if (known && (ptr_reg->off + smin_val == 6770 (s64)(s32)(ptr_reg->off + smin_val))) { 6771 /* pointer += K. Accumulate it into fixed offset */ 6772 dst_reg->smin_value = smin_ptr; 6773 dst_reg->smax_value = smax_ptr; 6774 dst_reg->umin_value = umin_ptr; 6775 dst_reg->umax_value = umax_ptr; 6776 dst_reg->var_off = ptr_reg->var_off; 6777 dst_reg->off = ptr_reg->off + smin_val; 6778 dst_reg->raw = ptr_reg->raw; 6779 break; 6780 } 6781 /* A new variable offset is created. Note that off_reg->off 6782 * == 0, since it's a scalar. 6783 * dst_reg gets the pointer type and since some positive 6784 * integer value was added to the pointer, give it a new 'id' 6785 * if it's a PTR_TO_PACKET. 6786 * this creates a new 'base' pointer, off_reg (variable) gets 6787 * added into the variable offset, and we copy the fixed offset 6788 * from ptr_reg. 6789 */ 6790 if (signed_add_overflows(smin_ptr, smin_val) || 6791 signed_add_overflows(smax_ptr, smax_val)) { 6792 dst_reg->smin_value = S64_MIN; 6793 dst_reg->smax_value = S64_MAX; 6794 } else { 6795 dst_reg->smin_value = smin_ptr + smin_val; 6796 dst_reg->smax_value = smax_ptr + smax_val; 6797 } 6798 if (umin_ptr + umin_val < umin_ptr || 6799 umax_ptr + umax_val < umax_ptr) { 6800 dst_reg->umin_value = 0; 6801 dst_reg->umax_value = U64_MAX; 6802 } else { 6803 dst_reg->umin_value = umin_ptr + umin_val; 6804 dst_reg->umax_value = umax_ptr + umax_val; 6805 } 6806 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 6807 dst_reg->off = ptr_reg->off; 6808 dst_reg->raw = ptr_reg->raw; 6809 if (reg_is_pkt_pointer(ptr_reg)) { 6810 dst_reg->id = ++env->id_gen; 6811 /* something was added to pkt_ptr, set range to zero */ 6812 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 6813 } 6814 break; 6815 case BPF_SUB: 6816 if (dst_reg == off_reg) { 6817 /* scalar -= pointer. Creates an unknown scalar */ 6818 verbose(env, "R%d tried to subtract pointer from scalar\n", 6819 dst); 6820 return -EACCES; 6821 } 6822 /* We don't allow subtraction from FP, because (according to 6823 * test_verifier.c test "invalid fp arithmetic", JITs might not 6824 * be able to deal with it. 6825 */ 6826 if (ptr_reg->type == PTR_TO_STACK) { 6827 verbose(env, "R%d subtraction from stack pointer prohibited\n", 6828 dst); 6829 return -EACCES; 6830 } 6831 if (known && (ptr_reg->off - smin_val == 6832 (s64)(s32)(ptr_reg->off - smin_val))) { 6833 /* pointer -= K. Subtract it from fixed offset */ 6834 dst_reg->smin_value = smin_ptr; 6835 dst_reg->smax_value = smax_ptr; 6836 dst_reg->umin_value = umin_ptr; 6837 dst_reg->umax_value = umax_ptr; 6838 dst_reg->var_off = ptr_reg->var_off; 6839 dst_reg->id = ptr_reg->id; 6840 dst_reg->off = ptr_reg->off - smin_val; 6841 dst_reg->raw = ptr_reg->raw; 6842 break; 6843 } 6844 /* A new variable offset is created. If the subtrahend is known 6845 * nonnegative, then any reg->range we had before is still good. 6846 */ 6847 if (signed_sub_overflows(smin_ptr, smax_val) || 6848 signed_sub_overflows(smax_ptr, smin_val)) { 6849 /* Overflow possible, we know nothing */ 6850 dst_reg->smin_value = S64_MIN; 6851 dst_reg->smax_value = S64_MAX; 6852 } else { 6853 dst_reg->smin_value = smin_ptr - smax_val; 6854 dst_reg->smax_value = smax_ptr - smin_val; 6855 } 6856 if (umin_ptr < umax_val) { 6857 /* Overflow possible, we know nothing */ 6858 dst_reg->umin_value = 0; 6859 dst_reg->umax_value = U64_MAX; 6860 } else { 6861 /* Cannot overflow (as long as bounds are consistent) */ 6862 dst_reg->umin_value = umin_ptr - umax_val; 6863 dst_reg->umax_value = umax_ptr - umin_val; 6864 } 6865 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 6866 dst_reg->off = ptr_reg->off; 6867 dst_reg->raw = ptr_reg->raw; 6868 if (reg_is_pkt_pointer(ptr_reg)) { 6869 dst_reg->id = ++env->id_gen; 6870 /* something was added to pkt_ptr, set range to zero */ 6871 if (smin_val < 0) 6872 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 6873 } 6874 break; 6875 case BPF_AND: 6876 case BPF_OR: 6877 case BPF_XOR: 6878 /* bitwise ops on pointers are troublesome, prohibit. */ 6879 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 6880 dst, bpf_alu_string[opcode >> 4]); 6881 return -EACCES; 6882 default: 6883 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 6884 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 6885 dst, bpf_alu_string[opcode >> 4]); 6886 return -EACCES; 6887 } 6888 6889 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 6890 return -EINVAL; 6891 6892 __update_reg_bounds(dst_reg); 6893 __reg_deduce_bounds(dst_reg); 6894 __reg_bound_offset(dst_reg); 6895 6896 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 6897 return -EACCES; 6898 if (sanitize_needed(opcode)) { 6899 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 6900 &tmp_aux, true); 6901 if (ret < 0) 6902 return sanitize_err(env, insn, ret, off_reg, dst_reg); 6903 } 6904 6905 return 0; 6906 } 6907 6908 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 6909 struct bpf_reg_state *src_reg) 6910 { 6911 s32 smin_val = src_reg->s32_min_value; 6912 s32 smax_val = src_reg->s32_max_value; 6913 u32 umin_val = src_reg->u32_min_value; 6914 u32 umax_val = src_reg->u32_max_value; 6915 6916 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 6917 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 6918 dst_reg->s32_min_value = S32_MIN; 6919 dst_reg->s32_max_value = S32_MAX; 6920 } else { 6921 dst_reg->s32_min_value += smin_val; 6922 dst_reg->s32_max_value += smax_val; 6923 } 6924 if (dst_reg->u32_min_value + umin_val < umin_val || 6925 dst_reg->u32_max_value + umax_val < umax_val) { 6926 dst_reg->u32_min_value = 0; 6927 dst_reg->u32_max_value = U32_MAX; 6928 } else { 6929 dst_reg->u32_min_value += umin_val; 6930 dst_reg->u32_max_value += umax_val; 6931 } 6932 } 6933 6934 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 6935 struct bpf_reg_state *src_reg) 6936 { 6937 s64 smin_val = src_reg->smin_value; 6938 s64 smax_val = src_reg->smax_value; 6939 u64 umin_val = src_reg->umin_value; 6940 u64 umax_val = src_reg->umax_value; 6941 6942 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 6943 signed_add_overflows(dst_reg->smax_value, smax_val)) { 6944 dst_reg->smin_value = S64_MIN; 6945 dst_reg->smax_value = S64_MAX; 6946 } else { 6947 dst_reg->smin_value += smin_val; 6948 dst_reg->smax_value += smax_val; 6949 } 6950 if (dst_reg->umin_value + umin_val < umin_val || 6951 dst_reg->umax_value + umax_val < umax_val) { 6952 dst_reg->umin_value = 0; 6953 dst_reg->umax_value = U64_MAX; 6954 } else { 6955 dst_reg->umin_value += umin_val; 6956 dst_reg->umax_value += umax_val; 6957 } 6958 } 6959 6960 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 6961 struct bpf_reg_state *src_reg) 6962 { 6963 s32 smin_val = src_reg->s32_min_value; 6964 s32 smax_val = src_reg->s32_max_value; 6965 u32 umin_val = src_reg->u32_min_value; 6966 u32 umax_val = src_reg->u32_max_value; 6967 6968 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 6969 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 6970 /* Overflow possible, we know nothing */ 6971 dst_reg->s32_min_value = S32_MIN; 6972 dst_reg->s32_max_value = S32_MAX; 6973 } else { 6974 dst_reg->s32_min_value -= smax_val; 6975 dst_reg->s32_max_value -= smin_val; 6976 } 6977 if (dst_reg->u32_min_value < umax_val) { 6978 /* Overflow possible, we know nothing */ 6979 dst_reg->u32_min_value = 0; 6980 dst_reg->u32_max_value = U32_MAX; 6981 } else { 6982 /* Cannot overflow (as long as bounds are consistent) */ 6983 dst_reg->u32_min_value -= umax_val; 6984 dst_reg->u32_max_value -= umin_val; 6985 } 6986 } 6987 6988 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 6989 struct bpf_reg_state *src_reg) 6990 { 6991 s64 smin_val = src_reg->smin_value; 6992 s64 smax_val = src_reg->smax_value; 6993 u64 umin_val = src_reg->umin_value; 6994 u64 umax_val = src_reg->umax_value; 6995 6996 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 6997 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 6998 /* Overflow possible, we know nothing */ 6999 dst_reg->smin_value = S64_MIN; 7000 dst_reg->smax_value = S64_MAX; 7001 } else { 7002 dst_reg->smin_value -= smax_val; 7003 dst_reg->smax_value -= smin_val; 7004 } 7005 if (dst_reg->umin_value < umax_val) { 7006 /* Overflow possible, we know nothing */ 7007 dst_reg->umin_value = 0; 7008 dst_reg->umax_value = U64_MAX; 7009 } else { 7010 /* Cannot overflow (as long as bounds are consistent) */ 7011 dst_reg->umin_value -= umax_val; 7012 dst_reg->umax_value -= umin_val; 7013 } 7014 } 7015 7016 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 7017 struct bpf_reg_state *src_reg) 7018 { 7019 s32 smin_val = src_reg->s32_min_value; 7020 u32 umin_val = src_reg->u32_min_value; 7021 u32 umax_val = src_reg->u32_max_value; 7022 7023 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 7024 /* Ain't nobody got time to multiply that sign */ 7025 __mark_reg32_unbounded(dst_reg); 7026 return; 7027 } 7028 /* Both values are positive, so we can work with unsigned and 7029 * copy the result to signed (unless it exceeds S32_MAX). 7030 */ 7031 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 7032 /* Potential overflow, we know nothing */ 7033 __mark_reg32_unbounded(dst_reg); 7034 return; 7035 } 7036 dst_reg->u32_min_value *= umin_val; 7037 dst_reg->u32_max_value *= umax_val; 7038 if (dst_reg->u32_max_value > S32_MAX) { 7039 /* Overflow possible, we know nothing */ 7040 dst_reg->s32_min_value = S32_MIN; 7041 dst_reg->s32_max_value = S32_MAX; 7042 } else { 7043 dst_reg->s32_min_value = dst_reg->u32_min_value; 7044 dst_reg->s32_max_value = dst_reg->u32_max_value; 7045 } 7046 } 7047 7048 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 7049 struct bpf_reg_state *src_reg) 7050 { 7051 s64 smin_val = src_reg->smin_value; 7052 u64 umin_val = src_reg->umin_value; 7053 u64 umax_val = src_reg->umax_value; 7054 7055 if (smin_val < 0 || dst_reg->smin_value < 0) { 7056 /* Ain't nobody got time to multiply that sign */ 7057 __mark_reg64_unbounded(dst_reg); 7058 return; 7059 } 7060 /* Both values are positive, so we can work with unsigned and 7061 * copy the result to signed (unless it exceeds S64_MAX). 7062 */ 7063 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 7064 /* Potential overflow, we know nothing */ 7065 __mark_reg64_unbounded(dst_reg); 7066 return; 7067 } 7068 dst_reg->umin_value *= umin_val; 7069 dst_reg->umax_value *= umax_val; 7070 if (dst_reg->umax_value > S64_MAX) { 7071 /* Overflow possible, we know nothing */ 7072 dst_reg->smin_value = S64_MIN; 7073 dst_reg->smax_value = S64_MAX; 7074 } else { 7075 dst_reg->smin_value = dst_reg->umin_value; 7076 dst_reg->smax_value = dst_reg->umax_value; 7077 } 7078 } 7079 7080 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 7081 struct bpf_reg_state *src_reg) 7082 { 7083 bool src_known = tnum_subreg_is_const(src_reg->var_off); 7084 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 7085 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 7086 s32 smin_val = src_reg->s32_min_value; 7087 u32 umax_val = src_reg->u32_max_value; 7088 7089 /* Assuming scalar64_min_max_and will be called so its safe 7090 * to skip updating register for known 32-bit case. 7091 */ 7092 if (src_known && dst_known) 7093 return; 7094 7095 /* We get our minimum from the var_off, since that's inherently 7096 * bitwise. Our maximum is the minimum of the operands' maxima. 7097 */ 7098 dst_reg->u32_min_value = var32_off.value; 7099 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 7100 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 7101 /* Lose signed bounds when ANDing negative numbers, 7102 * ain't nobody got time for that. 7103 */ 7104 dst_reg->s32_min_value = S32_MIN; 7105 dst_reg->s32_max_value = S32_MAX; 7106 } else { 7107 /* ANDing two positives gives a positive, so safe to 7108 * cast result into s64. 7109 */ 7110 dst_reg->s32_min_value = dst_reg->u32_min_value; 7111 dst_reg->s32_max_value = dst_reg->u32_max_value; 7112 } 7113 7114 } 7115 7116 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 7117 struct bpf_reg_state *src_reg) 7118 { 7119 bool src_known = tnum_is_const(src_reg->var_off); 7120 bool dst_known = tnum_is_const(dst_reg->var_off); 7121 s64 smin_val = src_reg->smin_value; 7122 u64 umax_val = src_reg->umax_value; 7123 7124 if (src_known && dst_known) { 7125 __mark_reg_known(dst_reg, dst_reg->var_off.value); 7126 return; 7127 } 7128 7129 /* We get our minimum from the var_off, since that's inherently 7130 * bitwise. Our maximum is the minimum of the operands' maxima. 7131 */ 7132 dst_reg->umin_value = dst_reg->var_off.value; 7133 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 7134 if (dst_reg->smin_value < 0 || smin_val < 0) { 7135 /* Lose signed bounds when ANDing negative numbers, 7136 * ain't nobody got time for that. 7137 */ 7138 dst_reg->smin_value = S64_MIN; 7139 dst_reg->smax_value = S64_MAX; 7140 } else { 7141 /* ANDing two positives gives a positive, so safe to 7142 * cast result into s64. 7143 */ 7144 dst_reg->smin_value = dst_reg->umin_value; 7145 dst_reg->smax_value = dst_reg->umax_value; 7146 } 7147 /* We may learn something more from the var_off */ 7148 __update_reg_bounds(dst_reg); 7149 } 7150 7151 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 7152 struct bpf_reg_state *src_reg) 7153 { 7154 bool src_known = tnum_subreg_is_const(src_reg->var_off); 7155 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 7156 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 7157 s32 smin_val = src_reg->s32_min_value; 7158 u32 umin_val = src_reg->u32_min_value; 7159 7160 /* Assuming scalar64_min_max_or will be called so it is safe 7161 * to skip updating register for known case. 7162 */ 7163 if (src_known && dst_known) 7164 return; 7165 7166 /* We get our maximum from the var_off, and our minimum is the 7167 * maximum of the operands' minima 7168 */ 7169 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 7170 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 7171 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 7172 /* Lose signed bounds when ORing negative numbers, 7173 * ain't nobody got time for that. 7174 */ 7175 dst_reg->s32_min_value = S32_MIN; 7176 dst_reg->s32_max_value = S32_MAX; 7177 } else { 7178 /* ORing two positives gives a positive, so safe to 7179 * cast result into s64. 7180 */ 7181 dst_reg->s32_min_value = dst_reg->u32_min_value; 7182 dst_reg->s32_max_value = dst_reg->u32_max_value; 7183 } 7184 } 7185 7186 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 7187 struct bpf_reg_state *src_reg) 7188 { 7189 bool src_known = tnum_is_const(src_reg->var_off); 7190 bool dst_known = tnum_is_const(dst_reg->var_off); 7191 s64 smin_val = src_reg->smin_value; 7192 u64 umin_val = src_reg->umin_value; 7193 7194 if (src_known && dst_known) { 7195 __mark_reg_known(dst_reg, dst_reg->var_off.value); 7196 return; 7197 } 7198 7199 /* We get our maximum from the var_off, and our minimum is the 7200 * maximum of the operands' minima 7201 */ 7202 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 7203 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 7204 if (dst_reg->smin_value < 0 || smin_val < 0) { 7205 /* Lose signed bounds when ORing negative numbers, 7206 * ain't nobody got time for that. 7207 */ 7208 dst_reg->smin_value = S64_MIN; 7209 dst_reg->smax_value = S64_MAX; 7210 } else { 7211 /* ORing two positives gives a positive, so safe to 7212 * cast result into s64. 7213 */ 7214 dst_reg->smin_value = dst_reg->umin_value; 7215 dst_reg->smax_value = dst_reg->umax_value; 7216 } 7217 /* We may learn something more from the var_off */ 7218 __update_reg_bounds(dst_reg); 7219 } 7220 7221 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 7222 struct bpf_reg_state *src_reg) 7223 { 7224 bool src_known = tnum_subreg_is_const(src_reg->var_off); 7225 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 7226 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 7227 s32 smin_val = src_reg->s32_min_value; 7228 7229 /* Assuming scalar64_min_max_xor will be called so it is safe 7230 * to skip updating register for known case. 7231 */ 7232 if (src_known && dst_known) 7233 return; 7234 7235 /* We get both minimum and maximum from the var32_off. */ 7236 dst_reg->u32_min_value = var32_off.value; 7237 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 7238 7239 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 7240 /* XORing two positive sign numbers gives a positive, 7241 * so safe to cast u32 result into s32. 7242 */ 7243 dst_reg->s32_min_value = dst_reg->u32_min_value; 7244 dst_reg->s32_max_value = dst_reg->u32_max_value; 7245 } else { 7246 dst_reg->s32_min_value = S32_MIN; 7247 dst_reg->s32_max_value = S32_MAX; 7248 } 7249 } 7250 7251 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 7252 struct bpf_reg_state *src_reg) 7253 { 7254 bool src_known = tnum_is_const(src_reg->var_off); 7255 bool dst_known = tnum_is_const(dst_reg->var_off); 7256 s64 smin_val = src_reg->smin_value; 7257 7258 if (src_known && dst_known) { 7259 /* dst_reg->var_off.value has been updated earlier */ 7260 __mark_reg_known(dst_reg, dst_reg->var_off.value); 7261 return; 7262 } 7263 7264 /* We get both minimum and maximum from the var_off. */ 7265 dst_reg->umin_value = dst_reg->var_off.value; 7266 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 7267 7268 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 7269 /* XORing two positive sign numbers gives a positive, 7270 * so safe to cast u64 result into s64. 7271 */ 7272 dst_reg->smin_value = dst_reg->umin_value; 7273 dst_reg->smax_value = dst_reg->umax_value; 7274 } else { 7275 dst_reg->smin_value = S64_MIN; 7276 dst_reg->smax_value = S64_MAX; 7277 } 7278 7279 __update_reg_bounds(dst_reg); 7280 } 7281 7282 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 7283 u64 umin_val, u64 umax_val) 7284 { 7285 /* We lose all sign bit information (except what we can pick 7286 * up from var_off) 7287 */ 7288 dst_reg->s32_min_value = S32_MIN; 7289 dst_reg->s32_max_value = S32_MAX; 7290 /* If we might shift our top bit out, then we know nothing */ 7291 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 7292 dst_reg->u32_min_value = 0; 7293 dst_reg->u32_max_value = U32_MAX; 7294 } else { 7295 dst_reg->u32_min_value <<= umin_val; 7296 dst_reg->u32_max_value <<= umax_val; 7297 } 7298 } 7299 7300 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 7301 struct bpf_reg_state *src_reg) 7302 { 7303 u32 umax_val = src_reg->u32_max_value; 7304 u32 umin_val = src_reg->u32_min_value; 7305 /* u32 alu operation will zext upper bits */ 7306 struct tnum subreg = tnum_subreg(dst_reg->var_off); 7307 7308 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 7309 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 7310 /* Not required but being careful mark reg64 bounds as unknown so 7311 * that we are forced to pick them up from tnum and zext later and 7312 * if some path skips this step we are still safe. 7313 */ 7314 __mark_reg64_unbounded(dst_reg); 7315 __update_reg32_bounds(dst_reg); 7316 } 7317 7318 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 7319 u64 umin_val, u64 umax_val) 7320 { 7321 /* Special case <<32 because it is a common compiler pattern to sign 7322 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 7323 * positive we know this shift will also be positive so we can track 7324 * bounds correctly. Otherwise we lose all sign bit information except 7325 * what we can pick up from var_off. Perhaps we can generalize this 7326 * later to shifts of any length. 7327 */ 7328 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 7329 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 7330 else 7331 dst_reg->smax_value = S64_MAX; 7332 7333 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 7334 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 7335 else 7336 dst_reg->smin_value = S64_MIN; 7337 7338 /* If we might shift our top bit out, then we know nothing */ 7339 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 7340 dst_reg->umin_value = 0; 7341 dst_reg->umax_value = U64_MAX; 7342 } else { 7343 dst_reg->umin_value <<= umin_val; 7344 dst_reg->umax_value <<= umax_val; 7345 } 7346 } 7347 7348 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 7349 struct bpf_reg_state *src_reg) 7350 { 7351 u64 umax_val = src_reg->umax_value; 7352 u64 umin_val = src_reg->umin_value; 7353 7354 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 7355 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 7356 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 7357 7358 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 7359 /* We may learn something more from the var_off */ 7360 __update_reg_bounds(dst_reg); 7361 } 7362 7363 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 7364 struct bpf_reg_state *src_reg) 7365 { 7366 struct tnum subreg = tnum_subreg(dst_reg->var_off); 7367 u32 umax_val = src_reg->u32_max_value; 7368 u32 umin_val = src_reg->u32_min_value; 7369 7370 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 7371 * be negative, then either: 7372 * 1) src_reg might be zero, so the sign bit of the result is 7373 * unknown, so we lose our signed bounds 7374 * 2) it's known negative, thus the unsigned bounds capture the 7375 * signed bounds 7376 * 3) the signed bounds cross zero, so they tell us nothing 7377 * about the result 7378 * If the value in dst_reg is known nonnegative, then again the 7379 * unsigned bounds capture the signed bounds. 7380 * Thus, in all cases it suffices to blow away our signed bounds 7381 * and rely on inferring new ones from the unsigned bounds and 7382 * var_off of the result. 7383 */ 7384 dst_reg->s32_min_value = S32_MIN; 7385 dst_reg->s32_max_value = S32_MAX; 7386 7387 dst_reg->var_off = tnum_rshift(subreg, umin_val); 7388 dst_reg->u32_min_value >>= umax_val; 7389 dst_reg->u32_max_value >>= umin_val; 7390 7391 __mark_reg64_unbounded(dst_reg); 7392 __update_reg32_bounds(dst_reg); 7393 } 7394 7395 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 7396 struct bpf_reg_state *src_reg) 7397 { 7398 u64 umax_val = src_reg->umax_value; 7399 u64 umin_val = src_reg->umin_value; 7400 7401 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 7402 * be negative, then either: 7403 * 1) src_reg might be zero, so the sign bit of the result is 7404 * unknown, so we lose our signed bounds 7405 * 2) it's known negative, thus the unsigned bounds capture the 7406 * signed bounds 7407 * 3) the signed bounds cross zero, so they tell us nothing 7408 * about the result 7409 * If the value in dst_reg is known nonnegative, then again the 7410 * unsigned bounds capture the signed bounds. 7411 * Thus, in all cases it suffices to blow away our signed bounds 7412 * and rely on inferring new ones from the unsigned bounds and 7413 * var_off of the result. 7414 */ 7415 dst_reg->smin_value = S64_MIN; 7416 dst_reg->smax_value = S64_MAX; 7417 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 7418 dst_reg->umin_value >>= umax_val; 7419 dst_reg->umax_value >>= umin_val; 7420 7421 /* Its not easy to operate on alu32 bounds here because it depends 7422 * on bits being shifted in. Take easy way out and mark unbounded 7423 * so we can recalculate later from tnum. 7424 */ 7425 __mark_reg32_unbounded(dst_reg); 7426 __update_reg_bounds(dst_reg); 7427 } 7428 7429 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 7430 struct bpf_reg_state *src_reg) 7431 { 7432 u64 umin_val = src_reg->u32_min_value; 7433 7434 /* Upon reaching here, src_known is true and 7435 * umax_val is equal to umin_val. 7436 */ 7437 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 7438 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 7439 7440 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 7441 7442 /* blow away the dst_reg umin_value/umax_value and rely on 7443 * dst_reg var_off to refine the result. 7444 */ 7445 dst_reg->u32_min_value = 0; 7446 dst_reg->u32_max_value = U32_MAX; 7447 7448 __mark_reg64_unbounded(dst_reg); 7449 __update_reg32_bounds(dst_reg); 7450 } 7451 7452 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 7453 struct bpf_reg_state *src_reg) 7454 { 7455 u64 umin_val = src_reg->umin_value; 7456 7457 /* Upon reaching here, src_known is true and umax_val is equal 7458 * to umin_val. 7459 */ 7460 dst_reg->smin_value >>= umin_val; 7461 dst_reg->smax_value >>= umin_val; 7462 7463 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 7464 7465 /* blow away the dst_reg umin_value/umax_value and rely on 7466 * dst_reg var_off to refine the result. 7467 */ 7468 dst_reg->umin_value = 0; 7469 dst_reg->umax_value = U64_MAX; 7470 7471 /* Its not easy to operate on alu32 bounds here because it depends 7472 * on bits being shifted in from upper 32-bits. Take easy way out 7473 * and mark unbounded so we can recalculate later from tnum. 7474 */ 7475 __mark_reg32_unbounded(dst_reg); 7476 __update_reg_bounds(dst_reg); 7477 } 7478 7479 /* WARNING: This function does calculations on 64-bit values, but the actual 7480 * execution may occur on 32-bit values. Therefore, things like bitshifts 7481 * need extra checks in the 32-bit case. 7482 */ 7483 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 7484 struct bpf_insn *insn, 7485 struct bpf_reg_state *dst_reg, 7486 struct bpf_reg_state src_reg) 7487 { 7488 struct bpf_reg_state *regs = cur_regs(env); 7489 u8 opcode = BPF_OP(insn->code); 7490 bool src_known; 7491 s64 smin_val, smax_val; 7492 u64 umin_val, umax_val; 7493 s32 s32_min_val, s32_max_val; 7494 u32 u32_min_val, u32_max_val; 7495 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 7496 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 7497 int ret; 7498 7499 smin_val = src_reg.smin_value; 7500 smax_val = src_reg.smax_value; 7501 umin_val = src_reg.umin_value; 7502 umax_val = src_reg.umax_value; 7503 7504 s32_min_val = src_reg.s32_min_value; 7505 s32_max_val = src_reg.s32_max_value; 7506 u32_min_val = src_reg.u32_min_value; 7507 u32_max_val = src_reg.u32_max_value; 7508 7509 if (alu32) { 7510 src_known = tnum_subreg_is_const(src_reg.var_off); 7511 if ((src_known && 7512 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 7513 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 7514 /* Taint dst register if offset had invalid bounds 7515 * derived from e.g. dead branches. 7516 */ 7517 __mark_reg_unknown(env, dst_reg); 7518 return 0; 7519 } 7520 } else { 7521 src_known = tnum_is_const(src_reg.var_off); 7522 if ((src_known && 7523 (smin_val != smax_val || umin_val != umax_val)) || 7524 smin_val > smax_val || umin_val > umax_val) { 7525 /* Taint dst register if offset had invalid bounds 7526 * derived from e.g. dead branches. 7527 */ 7528 __mark_reg_unknown(env, dst_reg); 7529 return 0; 7530 } 7531 } 7532 7533 if (!src_known && 7534 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 7535 __mark_reg_unknown(env, dst_reg); 7536 return 0; 7537 } 7538 7539 if (sanitize_needed(opcode)) { 7540 ret = sanitize_val_alu(env, insn); 7541 if (ret < 0) 7542 return sanitize_err(env, insn, ret, NULL, NULL); 7543 } 7544 7545 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 7546 * There are two classes of instructions: The first class we track both 7547 * alu32 and alu64 sign/unsigned bounds independently this provides the 7548 * greatest amount of precision when alu operations are mixed with jmp32 7549 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 7550 * and BPF_OR. This is possible because these ops have fairly easy to 7551 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 7552 * See alu32 verifier tests for examples. The second class of 7553 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 7554 * with regards to tracking sign/unsigned bounds because the bits may 7555 * cross subreg boundaries in the alu64 case. When this happens we mark 7556 * the reg unbounded in the subreg bound space and use the resulting 7557 * tnum to calculate an approximation of the sign/unsigned bounds. 7558 */ 7559 switch (opcode) { 7560 case BPF_ADD: 7561 scalar32_min_max_add(dst_reg, &src_reg); 7562 scalar_min_max_add(dst_reg, &src_reg); 7563 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 7564 break; 7565 case BPF_SUB: 7566 scalar32_min_max_sub(dst_reg, &src_reg); 7567 scalar_min_max_sub(dst_reg, &src_reg); 7568 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 7569 break; 7570 case BPF_MUL: 7571 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 7572 scalar32_min_max_mul(dst_reg, &src_reg); 7573 scalar_min_max_mul(dst_reg, &src_reg); 7574 break; 7575 case BPF_AND: 7576 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 7577 scalar32_min_max_and(dst_reg, &src_reg); 7578 scalar_min_max_and(dst_reg, &src_reg); 7579 break; 7580 case BPF_OR: 7581 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 7582 scalar32_min_max_or(dst_reg, &src_reg); 7583 scalar_min_max_or(dst_reg, &src_reg); 7584 break; 7585 case BPF_XOR: 7586 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 7587 scalar32_min_max_xor(dst_reg, &src_reg); 7588 scalar_min_max_xor(dst_reg, &src_reg); 7589 break; 7590 case BPF_LSH: 7591 if (umax_val >= insn_bitness) { 7592 /* Shifts greater than 31 or 63 are undefined. 7593 * This includes shifts by a negative number. 7594 */ 7595 mark_reg_unknown(env, regs, insn->dst_reg); 7596 break; 7597 } 7598 if (alu32) 7599 scalar32_min_max_lsh(dst_reg, &src_reg); 7600 else 7601 scalar_min_max_lsh(dst_reg, &src_reg); 7602 break; 7603 case BPF_RSH: 7604 if (umax_val >= insn_bitness) { 7605 /* Shifts greater than 31 or 63 are undefined. 7606 * This includes shifts by a negative number. 7607 */ 7608 mark_reg_unknown(env, regs, insn->dst_reg); 7609 break; 7610 } 7611 if (alu32) 7612 scalar32_min_max_rsh(dst_reg, &src_reg); 7613 else 7614 scalar_min_max_rsh(dst_reg, &src_reg); 7615 break; 7616 case BPF_ARSH: 7617 if (umax_val >= insn_bitness) { 7618 /* Shifts greater than 31 or 63 are undefined. 7619 * This includes shifts by a negative number. 7620 */ 7621 mark_reg_unknown(env, regs, insn->dst_reg); 7622 break; 7623 } 7624 if (alu32) 7625 scalar32_min_max_arsh(dst_reg, &src_reg); 7626 else 7627 scalar_min_max_arsh(dst_reg, &src_reg); 7628 break; 7629 default: 7630 mark_reg_unknown(env, regs, insn->dst_reg); 7631 break; 7632 } 7633 7634 /* ALU32 ops are zero extended into 64bit register */ 7635 if (alu32) 7636 zext_32_to_64(dst_reg); 7637 7638 __update_reg_bounds(dst_reg); 7639 __reg_deduce_bounds(dst_reg); 7640 __reg_bound_offset(dst_reg); 7641 return 0; 7642 } 7643 7644 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 7645 * and var_off. 7646 */ 7647 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 7648 struct bpf_insn *insn) 7649 { 7650 struct bpf_verifier_state *vstate = env->cur_state; 7651 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7652 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 7653 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 7654 u8 opcode = BPF_OP(insn->code); 7655 int err; 7656 7657 dst_reg = ®s[insn->dst_reg]; 7658 src_reg = NULL; 7659 if (dst_reg->type != SCALAR_VALUE) 7660 ptr_reg = dst_reg; 7661 else 7662 /* Make sure ID is cleared otherwise dst_reg min/max could be 7663 * incorrectly propagated into other registers by find_equal_scalars() 7664 */ 7665 dst_reg->id = 0; 7666 if (BPF_SRC(insn->code) == BPF_X) { 7667 src_reg = ®s[insn->src_reg]; 7668 if (src_reg->type != SCALAR_VALUE) { 7669 if (dst_reg->type != SCALAR_VALUE) { 7670 /* Combining two pointers by any ALU op yields 7671 * an arbitrary scalar. Disallow all math except 7672 * pointer subtraction 7673 */ 7674 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 7675 mark_reg_unknown(env, regs, insn->dst_reg); 7676 return 0; 7677 } 7678 verbose(env, "R%d pointer %s pointer prohibited\n", 7679 insn->dst_reg, 7680 bpf_alu_string[opcode >> 4]); 7681 return -EACCES; 7682 } else { 7683 /* scalar += pointer 7684 * This is legal, but we have to reverse our 7685 * src/dest handling in computing the range 7686 */ 7687 err = mark_chain_precision(env, insn->dst_reg); 7688 if (err) 7689 return err; 7690 return adjust_ptr_min_max_vals(env, insn, 7691 src_reg, dst_reg); 7692 } 7693 } else if (ptr_reg) { 7694 /* pointer += scalar */ 7695 err = mark_chain_precision(env, insn->src_reg); 7696 if (err) 7697 return err; 7698 return adjust_ptr_min_max_vals(env, insn, 7699 dst_reg, src_reg); 7700 } 7701 } else { 7702 /* Pretend the src is a reg with a known value, since we only 7703 * need to be able to read from this state. 7704 */ 7705 off_reg.type = SCALAR_VALUE; 7706 __mark_reg_known(&off_reg, insn->imm); 7707 src_reg = &off_reg; 7708 if (ptr_reg) /* pointer += K */ 7709 return adjust_ptr_min_max_vals(env, insn, 7710 ptr_reg, src_reg); 7711 } 7712 7713 /* Got here implies adding two SCALAR_VALUEs */ 7714 if (WARN_ON_ONCE(ptr_reg)) { 7715 print_verifier_state(env, state); 7716 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 7717 return -EINVAL; 7718 } 7719 if (WARN_ON(!src_reg)) { 7720 print_verifier_state(env, state); 7721 verbose(env, "verifier internal error: no src_reg\n"); 7722 return -EINVAL; 7723 } 7724 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 7725 } 7726 7727 /* check validity of 32-bit and 64-bit arithmetic operations */ 7728 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 7729 { 7730 struct bpf_reg_state *regs = cur_regs(env); 7731 u8 opcode = BPF_OP(insn->code); 7732 int err; 7733 7734 if (opcode == BPF_END || opcode == BPF_NEG) { 7735 if (opcode == BPF_NEG) { 7736 if (BPF_SRC(insn->code) != 0 || 7737 insn->src_reg != BPF_REG_0 || 7738 insn->off != 0 || insn->imm != 0) { 7739 verbose(env, "BPF_NEG uses reserved fields\n"); 7740 return -EINVAL; 7741 } 7742 } else { 7743 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 7744 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 7745 BPF_CLASS(insn->code) == BPF_ALU64) { 7746 verbose(env, "BPF_END uses reserved fields\n"); 7747 return -EINVAL; 7748 } 7749 } 7750 7751 /* check src operand */ 7752 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 7753 if (err) 7754 return err; 7755 7756 if (is_pointer_value(env, insn->dst_reg)) { 7757 verbose(env, "R%d pointer arithmetic prohibited\n", 7758 insn->dst_reg); 7759 return -EACCES; 7760 } 7761 7762 /* check dest operand */ 7763 err = check_reg_arg(env, insn->dst_reg, DST_OP); 7764 if (err) 7765 return err; 7766 7767 } else if (opcode == BPF_MOV) { 7768 7769 if (BPF_SRC(insn->code) == BPF_X) { 7770 if (insn->imm != 0 || insn->off != 0) { 7771 verbose(env, "BPF_MOV uses reserved fields\n"); 7772 return -EINVAL; 7773 } 7774 7775 /* check src operand */ 7776 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7777 if (err) 7778 return err; 7779 } else { 7780 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 7781 verbose(env, "BPF_MOV uses reserved fields\n"); 7782 return -EINVAL; 7783 } 7784 } 7785 7786 /* check dest operand, mark as required later */ 7787 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 7788 if (err) 7789 return err; 7790 7791 if (BPF_SRC(insn->code) == BPF_X) { 7792 struct bpf_reg_state *src_reg = regs + insn->src_reg; 7793 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 7794 7795 if (BPF_CLASS(insn->code) == BPF_ALU64) { 7796 /* case: R1 = R2 7797 * copy register state to dest reg 7798 */ 7799 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 7800 /* Assign src and dst registers the same ID 7801 * that will be used by find_equal_scalars() 7802 * to propagate min/max range. 7803 */ 7804 src_reg->id = ++env->id_gen; 7805 *dst_reg = *src_reg; 7806 dst_reg->live |= REG_LIVE_WRITTEN; 7807 dst_reg->subreg_def = DEF_NOT_SUBREG; 7808 } else { 7809 /* R1 = (u32) R2 */ 7810 if (is_pointer_value(env, insn->src_reg)) { 7811 verbose(env, 7812 "R%d partial copy of pointer\n", 7813 insn->src_reg); 7814 return -EACCES; 7815 } else if (src_reg->type == SCALAR_VALUE) { 7816 *dst_reg = *src_reg; 7817 /* Make sure ID is cleared otherwise 7818 * dst_reg min/max could be incorrectly 7819 * propagated into src_reg by find_equal_scalars() 7820 */ 7821 dst_reg->id = 0; 7822 dst_reg->live |= REG_LIVE_WRITTEN; 7823 dst_reg->subreg_def = env->insn_idx + 1; 7824 } else { 7825 mark_reg_unknown(env, regs, 7826 insn->dst_reg); 7827 } 7828 zext_32_to_64(dst_reg); 7829 } 7830 } else { 7831 /* case: R = imm 7832 * remember the value we stored into this reg 7833 */ 7834 /* clear any state __mark_reg_known doesn't set */ 7835 mark_reg_unknown(env, regs, insn->dst_reg); 7836 regs[insn->dst_reg].type = SCALAR_VALUE; 7837 if (BPF_CLASS(insn->code) == BPF_ALU64) { 7838 __mark_reg_known(regs + insn->dst_reg, 7839 insn->imm); 7840 } else { 7841 __mark_reg_known(regs + insn->dst_reg, 7842 (u32)insn->imm); 7843 } 7844 } 7845 7846 } else if (opcode > BPF_END) { 7847 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 7848 return -EINVAL; 7849 7850 } else { /* all other ALU ops: and, sub, xor, add, ... */ 7851 7852 if (BPF_SRC(insn->code) == BPF_X) { 7853 if (insn->imm != 0 || insn->off != 0) { 7854 verbose(env, "BPF_ALU uses reserved fields\n"); 7855 return -EINVAL; 7856 } 7857 /* check src1 operand */ 7858 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7859 if (err) 7860 return err; 7861 } else { 7862 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 7863 verbose(env, "BPF_ALU uses reserved fields\n"); 7864 return -EINVAL; 7865 } 7866 } 7867 7868 /* check src2 operand */ 7869 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 7870 if (err) 7871 return err; 7872 7873 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 7874 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 7875 verbose(env, "div by zero\n"); 7876 return -EINVAL; 7877 } 7878 7879 if ((opcode == BPF_LSH || opcode == BPF_RSH || 7880 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 7881 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 7882 7883 if (insn->imm < 0 || insn->imm >= size) { 7884 verbose(env, "invalid shift %d\n", insn->imm); 7885 return -EINVAL; 7886 } 7887 } 7888 7889 /* check dest operand */ 7890 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 7891 if (err) 7892 return err; 7893 7894 return adjust_reg_min_max_vals(env, insn); 7895 } 7896 7897 return 0; 7898 } 7899 7900 static void __find_good_pkt_pointers(struct bpf_func_state *state, 7901 struct bpf_reg_state *dst_reg, 7902 enum bpf_reg_type type, int new_range) 7903 { 7904 struct bpf_reg_state *reg; 7905 int i; 7906 7907 for (i = 0; i < MAX_BPF_REG; i++) { 7908 reg = &state->regs[i]; 7909 if (reg->type == type && reg->id == dst_reg->id) 7910 /* keep the maximum range already checked */ 7911 reg->range = max(reg->range, new_range); 7912 } 7913 7914 bpf_for_each_spilled_reg(i, state, reg) { 7915 if (!reg) 7916 continue; 7917 if (reg->type == type && reg->id == dst_reg->id) 7918 reg->range = max(reg->range, new_range); 7919 } 7920 } 7921 7922 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 7923 struct bpf_reg_state *dst_reg, 7924 enum bpf_reg_type type, 7925 bool range_right_open) 7926 { 7927 int new_range, i; 7928 7929 if (dst_reg->off < 0 || 7930 (dst_reg->off == 0 && range_right_open)) 7931 /* This doesn't give us any range */ 7932 return; 7933 7934 if (dst_reg->umax_value > MAX_PACKET_OFF || 7935 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 7936 /* Risk of overflow. For instance, ptr + (1<<63) may be less 7937 * than pkt_end, but that's because it's also less than pkt. 7938 */ 7939 return; 7940 7941 new_range = dst_reg->off; 7942 if (range_right_open) 7943 new_range--; 7944 7945 /* Examples for register markings: 7946 * 7947 * pkt_data in dst register: 7948 * 7949 * r2 = r3; 7950 * r2 += 8; 7951 * if (r2 > pkt_end) goto <handle exception> 7952 * <access okay> 7953 * 7954 * r2 = r3; 7955 * r2 += 8; 7956 * if (r2 < pkt_end) goto <access okay> 7957 * <handle exception> 7958 * 7959 * Where: 7960 * r2 == dst_reg, pkt_end == src_reg 7961 * r2=pkt(id=n,off=8,r=0) 7962 * r3=pkt(id=n,off=0,r=0) 7963 * 7964 * pkt_data in src register: 7965 * 7966 * r2 = r3; 7967 * r2 += 8; 7968 * if (pkt_end >= r2) goto <access okay> 7969 * <handle exception> 7970 * 7971 * r2 = r3; 7972 * r2 += 8; 7973 * if (pkt_end <= r2) goto <handle exception> 7974 * <access okay> 7975 * 7976 * Where: 7977 * pkt_end == dst_reg, r2 == src_reg 7978 * r2=pkt(id=n,off=8,r=0) 7979 * r3=pkt(id=n,off=0,r=0) 7980 * 7981 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 7982 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 7983 * and [r3, r3 + 8-1) respectively is safe to access depending on 7984 * the check. 7985 */ 7986 7987 /* If our ids match, then we must have the same max_value. And we 7988 * don't care about the other reg's fixed offset, since if it's too big 7989 * the range won't allow anything. 7990 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 7991 */ 7992 for (i = 0; i <= vstate->curframe; i++) 7993 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, 7994 new_range); 7995 } 7996 7997 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 7998 { 7999 struct tnum subreg = tnum_subreg(reg->var_off); 8000 s32 sval = (s32)val; 8001 8002 switch (opcode) { 8003 case BPF_JEQ: 8004 if (tnum_is_const(subreg)) 8005 return !!tnum_equals_const(subreg, val); 8006 break; 8007 case BPF_JNE: 8008 if (tnum_is_const(subreg)) 8009 return !tnum_equals_const(subreg, val); 8010 break; 8011 case BPF_JSET: 8012 if ((~subreg.mask & subreg.value) & val) 8013 return 1; 8014 if (!((subreg.mask | subreg.value) & val)) 8015 return 0; 8016 break; 8017 case BPF_JGT: 8018 if (reg->u32_min_value > val) 8019 return 1; 8020 else if (reg->u32_max_value <= val) 8021 return 0; 8022 break; 8023 case BPF_JSGT: 8024 if (reg->s32_min_value > sval) 8025 return 1; 8026 else if (reg->s32_max_value <= sval) 8027 return 0; 8028 break; 8029 case BPF_JLT: 8030 if (reg->u32_max_value < val) 8031 return 1; 8032 else if (reg->u32_min_value >= val) 8033 return 0; 8034 break; 8035 case BPF_JSLT: 8036 if (reg->s32_max_value < sval) 8037 return 1; 8038 else if (reg->s32_min_value >= sval) 8039 return 0; 8040 break; 8041 case BPF_JGE: 8042 if (reg->u32_min_value >= val) 8043 return 1; 8044 else if (reg->u32_max_value < val) 8045 return 0; 8046 break; 8047 case BPF_JSGE: 8048 if (reg->s32_min_value >= sval) 8049 return 1; 8050 else if (reg->s32_max_value < sval) 8051 return 0; 8052 break; 8053 case BPF_JLE: 8054 if (reg->u32_max_value <= val) 8055 return 1; 8056 else if (reg->u32_min_value > val) 8057 return 0; 8058 break; 8059 case BPF_JSLE: 8060 if (reg->s32_max_value <= sval) 8061 return 1; 8062 else if (reg->s32_min_value > sval) 8063 return 0; 8064 break; 8065 } 8066 8067 return -1; 8068 } 8069 8070 8071 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 8072 { 8073 s64 sval = (s64)val; 8074 8075 switch (opcode) { 8076 case BPF_JEQ: 8077 if (tnum_is_const(reg->var_off)) 8078 return !!tnum_equals_const(reg->var_off, val); 8079 break; 8080 case BPF_JNE: 8081 if (tnum_is_const(reg->var_off)) 8082 return !tnum_equals_const(reg->var_off, val); 8083 break; 8084 case BPF_JSET: 8085 if ((~reg->var_off.mask & reg->var_off.value) & val) 8086 return 1; 8087 if (!((reg->var_off.mask | reg->var_off.value) & val)) 8088 return 0; 8089 break; 8090 case BPF_JGT: 8091 if (reg->umin_value > val) 8092 return 1; 8093 else if (reg->umax_value <= val) 8094 return 0; 8095 break; 8096 case BPF_JSGT: 8097 if (reg->smin_value > sval) 8098 return 1; 8099 else if (reg->smax_value <= sval) 8100 return 0; 8101 break; 8102 case BPF_JLT: 8103 if (reg->umax_value < val) 8104 return 1; 8105 else if (reg->umin_value >= val) 8106 return 0; 8107 break; 8108 case BPF_JSLT: 8109 if (reg->smax_value < sval) 8110 return 1; 8111 else if (reg->smin_value >= sval) 8112 return 0; 8113 break; 8114 case BPF_JGE: 8115 if (reg->umin_value >= val) 8116 return 1; 8117 else if (reg->umax_value < val) 8118 return 0; 8119 break; 8120 case BPF_JSGE: 8121 if (reg->smin_value >= sval) 8122 return 1; 8123 else if (reg->smax_value < sval) 8124 return 0; 8125 break; 8126 case BPF_JLE: 8127 if (reg->umax_value <= val) 8128 return 1; 8129 else if (reg->umin_value > val) 8130 return 0; 8131 break; 8132 case BPF_JSLE: 8133 if (reg->smax_value <= sval) 8134 return 1; 8135 else if (reg->smin_value > sval) 8136 return 0; 8137 break; 8138 } 8139 8140 return -1; 8141 } 8142 8143 /* compute branch direction of the expression "if (reg opcode val) goto target;" 8144 * and return: 8145 * 1 - branch will be taken and "goto target" will be executed 8146 * 0 - branch will not be taken and fall-through to next insn 8147 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 8148 * range [0,10] 8149 */ 8150 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 8151 bool is_jmp32) 8152 { 8153 if (__is_pointer_value(false, reg)) { 8154 if (!reg_type_not_null(reg->type)) 8155 return -1; 8156 8157 /* If pointer is valid tests against zero will fail so we can 8158 * use this to direct branch taken. 8159 */ 8160 if (val != 0) 8161 return -1; 8162 8163 switch (opcode) { 8164 case BPF_JEQ: 8165 return 0; 8166 case BPF_JNE: 8167 return 1; 8168 default: 8169 return -1; 8170 } 8171 } 8172 8173 if (is_jmp32) 8174 return is_branch32_taken(reg, val, opcode); 8175 return is_branch64_taken(reg, val, opcode); 8176 } 8177 8178 static int flip_opcode(u32 opcode) 8179 { 8180 /* How can we transform "a <op> b" into "b <op> a"? */ 8181 static const u8 opcode_flip[16] = { 8182 /* these stay the same */ 8183 [BPF_JEQ >> 4] = BPF_JEQ, 8184 [BPF_JNE >> 4] = BPF_JNE, 8185 [BPF_JSET >> 4] = BPF_JSET, 8186 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 8187 [BPF_JGE >> 4] = BPF_JLE, 8188 [BPF_JGT >> 4] = BPF_JLT, 8189 [BPF_JLE >> 4] = BPF_JGE, 8190 [BPF_JLT >> 4] = BPF_JGT, 8191 [BPF_JSGE >> 4] = BPF_JSLE, 8192 [BPF_JSGT >> 4] = BPF_JSLT, 8193 [BPF_JSLE >> 4] = BPF_JSGE, 8194 [BPF_JSLT >> 4] = BPF_JSGT 8195 }; 8196 return opcode_flip[opcode >> 4]; 8197 } 8198 8199 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 8200 struct bpf_reg_state *src_reg, 8201 u8 opcode) 8202 { 8203 struct bpf_reg_state *pkt; 8204 8205 if (src_reg->type == PTR_TO_PACKET_END) { 8206 pkt = dst_reg; 8207 } else if (dst_reg->type == PTR_TO_PACKET_END) { 8208 pkt = src_reg; 8209 opcode = flip_opcode(opcode); 8210 } else { 8211 return -1; 8212 } 8213 8214 if (pkt->range >= 0) 8215 return -1; 8216 8217 switch (opcode) { 8218 case BPF_JLE: 8219 /* pkt <= pkt_end */ 8220 fallthrough; 8221 case BPF_JGT: 8222 /* pkt > pkt_end */ 8223 if (pkt->range == BEYOND_PKT_END) 8224 /* pkt has at last one extra byte beyond pkt_end */ 8225 return opcode == BPF_JGT; 8226 break; 8227 case BPF_JLT: 8228 /* pkt < pkt_end */ 8229 fallthrough; 8230 case BPF_JGE: 8231 /* pkt >= pkt_end */ 8232 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 8233 return opcode == BPF_JGE; 8234 break; 8235 } 8236 return -1; 8237 } 8238 8239 /* Adjusts the register min/max values in the case that the dst_reg is the 8240 * variable register that we are working on, and src_reg is a constant or we're 8241 * simply doing a BPF_K check. 8242 * In JEQ/JNE cases we also adjust the var_off values. 8243 */ 8244 static void reg_set_min_max(struct bpf_reg_state *true_reg, 8245 struct bpf_reg_state *false_reg, 8246 u64 val, u32 val32, 8247 u8 opcode, bool is_jmp32) 8248 { 8249 struct tnum false_32off = tnum_subreg(false_reg->var_off); 8250 struct tnum false_64off = false_reg->var_off; 8251 struct tnum true_32off = tnum_subreg(true_reg->var_off); 8252 struct tnum true_64off = true_reg->var_off; 8253 s64 sval = (s64)val; 8254 s32 sval32 = (s32)val32; 8255 8256 /* If the dst_reg is a pointer, we can't learn anything about its 8257 * variable offset from the compare (unless src_reg were a pointer into 8258 * the same object, but we don't bother with that. 8259 * Since false_reg and true_reg have the same type by construction, we 8260 * only need to check one of them for pointerness. 8261 */ 8262 if (__is_pointer_value(false, false_reg)) 8263 return; 8264 8265 switch (opcode) { 8266 case BPF_JEQ: 8267 case BPF_JNE: 8268 { 8269 struct bpf_reg_state *reg = 8270 opcode == BPF_JEQ ? true_reg : false_reg; 8271 8272 /* JEQ/JNE comparison doesn't change the register equivalence. 8273 * r1 = r2; 8274 * if (r1 == 42) goto label; 8275 * ... 8276 * label: // here both r1 and r2 are known to be 42. 8277 * 8278 * Hence when marking register as known preserve it's ID. 8279 */ 8280 if (is_jmp32) 8281 __mark_reg32_known(reg, val32); 8282 else 8283 ___mark_reg_known(reg, val); 8284 break; 8285 } 8286 case BPF_JSET: 8287 if (is_jmp32) { 8288 false_32off = tnum_and(false_32off, tnum_const(~val32)); 8289 if (is_power_of_2(val32)) 8290 true_32off = tnum_or(true_32off, 8291 tnum_const(val32)); 8292 } else { 8293 false_64off = tnum_and(false_64off, tnum_const(~val)); 8294 if (is_power_of_2(val)) 8295 true_64off = tnum_or(true_64off, 8296 tnum_const(val)); 8297 } 8298 break; 8299 case BPF_JGE: 8300 case BPF_JGT: 8301 { 8302 if (is_jmp32) { 8303 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 8304 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 8305 8306 false_reg->u32_max_value = min(false_reg->u32_max_value, 8307 false_umax); 8308 true_reg->u32_min_value = max(true_reg->u32_min_value, 8309 true_umin); 8310 } else { 8311 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 8312 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 8313 8314 false_reg->umax_value = min(false_reg->umax_value, false_umax); 8315 true_reg->umin_value = max(true_reg->umin_value, true_umin); 8316 } 8317 break; 8318 } 8319 case BPF_JSGE: 8320 case BPF_JSGT: 8321 { 8322 if (is_jmp32) { 8323 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 8324 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 8325 8326 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 8327 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 8328 } else { 8329 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 8330 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 8331 8332 false_reg->smax_value = min(false_reg->smax_value, false_smax); 8333 true_reg->smin_value = max(true_reg->smin_value, true_smin); 8334 } 8335 break; 8336 } 8337 case BPF_JLE: 8338 case BPF_JLT: 8339 { 8340 if (is_jmp32) { 8341 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 8342 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 8343 8344 false_reg->u32_min_value = max(false_reg->u32_min_value, 8345 false_umin); 8346 true_reg->u32_max_value = min(true_reg->u32_max_value, 8347 true_umax); 8348 } else { 8349 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 8350 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 8351 8352 false_reg->umin_value = max(false_reg->umin_value, false_umin); 8353 true_reg->umax_value = min(true_reg->umax_value, true_umax); 8354 } 8355 break; 8356 } 8357 case BPF_JSLE: 8358 case BPF_JSLT: 8359 { 8360 if (is_jmp32) { 8361 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 8362 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 8363 8364 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 8365 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 8366 } else { 8367 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 8368 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 8369 8370 false_reg->smin_value = max(false_reg->smin_value, false_smin); 8371 true_reg->smax_value = min(true_reg->smax_value, true_smax); 8372 } 8373 break; 8374 } 8375 default: 8376 return; 8377 } 8378 8379 if (is_jmp32) { 8380 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 8381 tnum_subreg(false_32off)); 8382 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 8383 tnum_subreg(true_32off)); 8384 __reg_combine_32_into_64(false_reg); 8385 __reg_combine_32_into_64(true_reg); 8386 } else { 8387 false_reg->var_off = false_64off; 8388 true_reg->var_off = true_64off; 8389 __reg_combine_64_into_32(false_reg); 8390 __reg_combine_64_into_32(true_reg); 8391 } 8392 } 8393 8394 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 8395 * the variable reg. 8396 */ 8397 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 8398 struct bpf_reg_state *false_reg, 8399 u64 val, u32 val32, 8400 u8 opcode, bool is_jmp32) 8401 { 8402 opcode = flip_opcode(opcode); 8403 /* This uses zero as "not present in table"; luckily the zero opcode, 8404 * BPF_JA, can't get here. 8405 */ 8406 if (opcode) 8407 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 8408 } 8409 8410 /* Regs are known to be equal, so intersect their min/max/var_off */ 8411 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 8412 struct bpf_reg_state *dst_reg) 8413 { 8414 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 8415 dst_reg->umin_value); 8416 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 8417 dst_reg->umax_value); 8418 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 8419 dst_reg->smin_value); 8420 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 8421 dst_reg->smax_value); 8422 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 8423 dst_reg->var_off); 8424 /* We might have learned new bounds from the var_off. */ 8425 __update_reg_bounds(src_reg); 8426 __update_reg_bounds(dst_reg); 8427 /* We might have learned something about the sign bit. */ 8428 __reg_deduce_bounds(src_reg); 8429 __reg_deduce_bounds(dst_reg); 8430 /* We might have learned some bits from the bounds. */ 8431 __reg_bound_offset(src_reg); 8432 __reg_bound_offset(dst_reg); 8433 /* Intersecting with the old var_off might have improved our bounds 8434 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 8435 * then new var_off is (0; 0x7f...fc) which improves our umax. 8436 */ 8437 __update_reg_bounds(src_reg); 8438 __update_reg_bounds(dst_reg); 8439 } 8440 8441 static void reg_combine_min_max(struct bpf_reg_state *true_src, 8442 struct bpf_reg_state *true_dst, 8443 struct bpf_reg_state *false_src, 8444 struct bpf_reg_state *false_dst, 8445 u8 opcode) 8446 { 8447 switch (opcode) { 8448 case BPF_JEQ: 8449 __reg_combine_min_max(true_src, true_dst); 8450 break; 8451 case BPF_JNE: 8452 __reg_combine_min_max(false_src, false_dst); 8453 break; 8454 } 8455 } 8456 8457 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 8458 struct bpf_reg_state *reg, u32 id, 8459 bool is_null) 8460 { 8461 if (reg_type_may_be_null(reg->type) && reg->id == id && 8462 !WARN_ON_ONCE(!reg->id)) { 8463 /* Old offset (both fixed and variable parts) should 8464 * have been known-zero, because we don't allow pointer 8465 * arithmetic on pointers that might be NULL. 8466 */ 8467 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || 8468 !tnum_equals_const(reg->var_off, 0) || 8469 reg->off)) { 8470 __mark_reg_known_zero(reg); 8471 reg->off = 0; 8472 } 8473 if (is_null) { 8474 reg->type = SCALAR_VALUE; 8475 /* We don't need id and ref_obj_id from this point 8476 * onwards anymore, thus we should better reset it, 8477 * so that state pruning has chances to take effect. 8478 */ 8479 reg->id = 0; 8480 reg->ref_obj_id = 0; 8481 8482 return; 8483 } 8484 8485 mark_ptr_not_null_reg(reg); 8486 8487 if (!reg_may_point_to_spin_lock(reg)) { 8488 /* For not-NULL ptr, reg->ref_obj_id will be reset 8489 * in release_reg_references(). 8490 * 8491 * reg->id is still used by spin_lock ptr. Other 8492 * than spin_lock ptr type, reg->id can be reset. 8493 */ 8494 reg->id = 0; 8495 } 8496 } 8497 } 8498 8499 static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, 8500 bool is_null) 8501 { 8502 struct bpf_reg_state *reg; 8503 int i; 8504 8505 for (i = 0; i < MAX_BPF_REG; i++) 8506 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); 8507 8508 bpf_for_each_spilled_reg(i, state, reg) { 8509 if (!reg) 8510 continue; 8511 mark_ptr_or_null_reg(state, reg, id, is_null); 8512 } 8513 } 8514 8515 /* The logic is similar to find_good_pkt_pointers(), both could eventually 8516 * be folded together at some point. 8517 */ 8518 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 8519 bool is_null) 8520 { 8521 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 8522 struct bpf_reg_state *regs = state->regs; 8523 u32 ref_obj_id = regs[regno].ref_obj_id; 8524 u32 id = regs[regno].id; 8525 int i; 8526 8527 if (ref_obj_id && ref_obj_id == id && is_null) 8528 /* regs[regno] is in the " == NULL" branch. 8529 * No one could have freed the reference state before 8530 * doing the NULL check. 8531 */ 8532 WARN_ON_ONCE(release_reference_state(state, id)); 8533 8534 for (i = 0; i <= vstate->curframe; i++) 8535 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); 8536 } 8537 8538 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 8539 struct bpf_reg_state *dst_reg, 8540 struct bpf_reg_state *src_reg, 8541 struct bpf_verifier_state *this_branch, 8542 struct bpf_verifier_state *other_branch) 8543 { 8544 if (BPF_SRC(insn->code) != BPF_X) 8545 return false; 8546 8547 /* Pointers are always 64-bit. */ 8548 if (BPF_CLASS(insn->code) == BPF_JMP32) 8549 return false; 8550 8551 switch (BPF_OP(insn->code)) { 8552 case BPF_JGT: 8553 if ((dst_reg->type == PTR_TO_PACKET && 8554 src_reg->type == PTR_TO_PACKET_END) || 8555 (dst_reg->type == PTR_TO_PACKET_META && 8556 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8557 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 8558 find_good_pkt_pointers(this_branch, dst_reg, 8559 dst_reg->type, false); 8560 mark_pkt_end(other_branch, insn->dst_reg, true); 8561 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8562 src_reg->type == PTR_TO_PACKET) || 8563 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8564 src_reg->type == PTR_TO_PACKET_META)) { 8565 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 8566 find_good_pkt_pointers(other_branch, src_reg, 8567 src_reg->type, true); 8568 mark_pkt_end(this_branch, insn->src_reg, false); 8569 } else { 8570 return false; 8571 } 8572 break; 8573 case BPF_JLT: 8574 if ((dst_reg->type == PTR_TO_PACKET && 8575 src_reg->type == PTR_TO_PACKET_END) || 8576 (dst_reg->type == PTR_TO_PACKET_META && 8577 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8578 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 8579 find_good_pkt_pointers(other_branch, dst_reg, 8580 dst_reg->type, true); 8581 mark_pkt_end(this_branch, insn->dst_reg, false); 8582 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8583 src_reg->type == PTR_TO_PACKET) || 8584 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8585 src_reg->type == PTR_TO_PACKET_META)) { 8586 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 8587 find_good_pkt_pointers(this_branch, src_reg, 8588 src_reg->type, false); 8589 mark_pkt_end(other_branch, insn->src_reg, true); 8590 } else { 8591 return false; 8592 } 8593 break; 8594 case BPF_JGE: 8595 if ((dst_reg->type == PTR_TO_PACKET && 8596 src_reg->type == PTR_TO_PACKET_END) || 8597 (dst_reg->type == PTR_TO_PACKET_META && 8598 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8599 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 8600 find_good_pkt_pointers(this_branch, dst_reg, 8601 dst_reg->type, true); 8602 mark_pkt_end(other_branch, insn->dst_reg, false); 8603 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8604 src_reg->type == PTR_TO_PACKET) || 8605 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8606 src_reg->type == PTR_TO_PACKET_META)) { 8607 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 8608 find_good_pkt_pointers(other_branch, src_reg, 8609 src_reg->type, false); 8610 mark_pkt_end(this_branch, insn->src_reg, true); 8611 } else { 8612 return false; 8613 } 8614 break; 8615 case BPF_JLE: 8616 if ((dst_reg->type == PTR_TO_PACKET && 8617 src_reg->type == PTR_TO_PACKET_END) || 8618 (dst_reg->type == PTR_TO_PACKET_META && 8619 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8620 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 8621 find_good_pkt_pointers(other_branch, dst_reg, 8622 dst_reg->type, false); 8623 mark_pkt_end(this_branch, insn->dst_reg, true); 8624 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8625 src_reg->type == PTR_TO_PACKET) || 8626 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8627 src_reg->type == PTR_TO_PACKET_META)) { 8628 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 8629 find_good_pkt_pointers(this_branch, src_reg, 8630 src_reg->type, true); 8631 mark_pkt_end(other_branch, insn->src_reg, false); 8632 } else { 8633 return false; 8634 } 8635 break; 8636 default: 8637 return false; 8638 } 8639 8640 return true; 8641 } 8642 8643 static void find_equal_scalars(struct bpf_verifier_state *vstate, 8644 struct bpf_reg_state *known_reg) 8645 { 8646 struct bpf_func_state *state; 8647 struct bpf_reg_state *reg; 8648 int i, j; 8649 8650 for (i = 0; i <= vstate->curframe; i++) { 8651 state = vstate->frame[i]; 8652 for (j = 0; j < MAX_BPF_REG; j++) { 8653 reg = &state->regs[j]; 8654 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 8655 *reg = *known_reg; 8656 } 8657 8658 bpf_for_each_spilled_reg(j, state, reg) { 8659 if (!reg) 8660 continue; 8661 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 8662 *reg = *known_reg; 8663 } 8664 } 8665 } 8666 8667 static int check_cond_jmp_op(struct bpf_verifier_env *env, 8668 struct bpf_insn *insn, int *insn_idx) 8669 { 8670 struct bpf_verifier_state *this_branch = env->cur_state; 8671 struct bpf_verifier_state *other_branch; 8672 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 8673 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 8674 u8 opcode = BPF_OP(insn->code); 8675 bool is_jmp32; 8676 int pred = -1; 8677 int err; 8678 8679 /* Only conditional jumps are expected to reach here. */ 8680 if (opcode == BPF_JA || opcode > BPF_JSLE) { 8681 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 8682 return -EINVAL; 8683 } 8684 8685 if (BPF_SRC(insn->code) == BPF_X) { 8686 if (insn->imm != 0) { 8687 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 8688 return -EINVAL; 8689 } 8690 8691 /* check src1 operand */ 8692 err = check_reg_arg(env, insn->src_reg, SRC_OP); 8693 if (err) 8694 return err; 8695 8696 if (is_pointer_value(env, insn->src_reg)) { 8697 verbose(env, "R%d pointer comparison prohibited\n", 8698 insn->src_reg); 8699 return -EACCES; 8700 } 8701 src_reg = ®s[insn->src_reg]; 8702 } else { 8703 if (insn->src_reg != BPF_REG_0) { 8704 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 8705 return -EINVAL; 8706 } 8707 } 8708 8709 /* check src2 operand */ 8710 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 8711 if (err) 8712 return err; 8713 8714 dst_reg = ®s[insn->dst_reg]; 8715 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 8716 8717 if (BPF_SRC(insn->code) == BPF_K) { 8718 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 8719 } else if (src_reg->type == SCALAR_VALUE && 8720 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 8721 pred = is_branch_taken(dst_reg, 8722 tnum_subreg(src_reg->var_off).value, 8723 opcode, 8724 is_jmp32); 8725 } else if (src_reg->type == SCALAR_VALUE && 8726 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 8727 pred = is_branch_taken(dst_reg, 8728 src_reg->var_off.value, 8729 opcode, 8730 is_jmp32); 8731 } else if (reg_is_pkt_pointer_any(dst_reg) && 8732 reg_is_pkt_pointer_any(src_reg) && 8733 !is_jmp32) { 8734 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 8735 } 8736 8737 if (pred >= 0) { 8738 /* If we get here with a dst_reg pointer type it is because 8739 * above is_branch_taken() special cased the 0 comparison. 8740 */ 8741 if (!__is_pointer_value(false, dst_reg)) 8742 err = mark_chain_precision(env, insn->dst_reg); 8743 if (BPF_SRC(insn->code) == BPF_X && !err && 8744 !__is_pointer_value(false, src_reg)) 8745 err = mark_chain_precision(env, insn->src_reg); 8746 if (err) 8747 return err; 8748 } 8749 if (pred == 1) { 8750 /* only follow the goto, ignore fall-through */ 8751 *insn_idx += insn->off; 8752 return 0; 8753 } else if (pred == 0) { 8754 /* only follow fall-through branch, since 8755 * that's where the program will go 8756 */ 8757 return 0; 8758 } 8759 8760 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 8761 false); 8762 if (!other_branch) 8763 return -EFAULT; 8764 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 8765 8766 /* detect if we are comparing against a constant value so we can adjust 8767 * our min/max values for our dst register. 8768 * this is only legit if both are scalars (or pointers to the same 8769 * object, I suppose, but we don't support that right now), because 8770 * otherwise the different base pointers mean the offsets aren't 8771 * comparable. 8772 */ 8773 if (BPF_SRC(insn->code) == BPF_X) { 8774 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 8775 8776 if (dst_reg->type == SCALAR_VALUE && 8777 src_reg->type == SCALAR_VALUE) { 8778 if (tnum_is_const(src_reg->var_off) || 8779 (is_jmp32 && 8780 tnum_is_const(tnum_subreg(src_reg->var_off)))) 8781 reg_set_min_max(&other_branch_regs[insn->dst_reg], 8782 dst_reg, 8783 src_reg->var_off.value, 8784 tnum_subreg(src_reg->var_off).value, 8785 opcode, is_jmp32); 8786 else if (tnum_is_const(dst_reg->var_off) || 8787 (is_jmp32 && 8788 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 8789 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 8790 src_reg, 8791 dst_reg->var_off.value, 8792 tnum_subreg(dst_reg->var_off).value, 8793 opcode, is_jmp32); 8794 else if (!is_jmp32 && 8795 (opcode == BPF_JEQ || opcode == BPF_JNE)) 8796 /* Comparing for equality, we can combine knowledge */ 8797 reg_combine_min_max(&other_branch_regs[insn->src_reg], 8798 &other_branch_regs[insn->dst_reg], 8799 src_reg, dst_reg, opcode); 8800 if (src_reg->id && 8801 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 8802 find_equal_scalars(this_branch, src_reg); 8803 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 8804 } 8805 8806 } 8807 } else if (dst_reg->type == SCALAR_VALUE) { 8808 reg_set_min_max(&other_branch_regs[insn->dst_reg], 8809 dst_reg, insn->imm, (u32)insn->imm, 8810 opcode, is_jmp32); 8811 } 8812 8813 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 8814 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 8815 find_equal_scalars(this_branch, dst_reg); 8816 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 8817 } 8818 8819 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 8820 * NOTE: these optimizations below are related with pointer comparison 8821 * which will never be JMP32. 8822 */ 8823 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 8824 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 8825 reg_type_may_be_null(dst_reg->type)) { 8826 /* Mark all identical registers in each branch as either 8827 * safe or unknown depending R == 0 or R != 0 conditional. 8828 */ 8829 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 8830 opcode == BPF_JNE); 8831 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 8832 opcode == BPF_JEQ); 8833 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 8834 this_branch, other_branch) && 8835 is_pointer_value(env, insn->dst_reg)) { 8836 verbose(env, "R%d pointer comparison prohibited\n", 8837 insn->dst_reg); 8838 return -EACCES; 8839 } 8840 if (env->log.level & BPF_LOG_LEVEL) 8841 print_verifier_state(env, this_branch->frame[this_branch->curframe]); 8842 return 0; 8843 } 8844 8845 /* verify BPF_LD_IMM64 instruction */ 8846 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 8847 { 8848 struct bpf_insn_aux_data *aux = cur_aux(env); 8849 struct bpf_reg_state *regs = cur_regs(env); 8850 struct bpf_reg_state *dst_reg; 8851 struct bpf_map *map; 8852 int err; 8853 8854 if (BPF_SIZE(insn->code) != BPF_DW) { 8855 verbose(env, "invalid BPF_LD_IMM insn\n"); 8856 return -EINVAL; 8857 } 8858 if (insn->off != 0) { 8859 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 8860 return -EINVAL; 8861 } 8862 8863 err = check_reg_arg(env, insn->dst_reg, DST_OP); 8864 if (err) 8865 return err; 8866 8867 dst_reg = ®s[insn->dst_reg]; 8868 if (insn->src_reg == 0) { 8869 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 8870 8871 dst_reg->type = SCALAR_VALUE; 8872 __mark_reg_known(®s[insn->dst_reg], imm); 8873 return 0; 8874 } 8875 8876 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 8877 mark_reg_known_zero(env, regs, insn->dst_reg); 8878 8879 dst_reg->type = aux->btf_var.reg_type; 8880 switch (dst_reg->type) { 8881 case PTR_TO_MEM: 8882 dst_reg->mem_size = aux->btf_var.mem_size; 8883 break; 8884 case PTR_TO_BTF_ID: 8885 case PTR_TO_PERCPU_BTF_ID: 8886 dst_reg->btf = aux->btf_var.btf; 8887 dst_reg->btf_id = aux->btf_var.btf_id; 8888 break; 8889 default: 8890 verbose(env, "bpf verifier is misconfigured\n"); 8891 return -EFAULT; 8892 } 8893 return 0; 8894 } 8895 8896 if (insn->src_reg == BPF_PSEUDO_FUNC) { 8897 struct bpf_prog_aux *aux = env->prog->aux; 8898 u32 subprogno = insn[1].imm; 8899 8900 if (!aux->func_info) { 8901 verbose(env, "missing btf func_info\n"); 8902 return -EINVAL; 8903 } 8904 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 8905 verbose(env, "callback function not static\n"); 8906 return -EINVAL; 8907 } 8908 8909 dst_reg->type = PTR_TO_FUNC; 8910 dst_reg->subprogno = subprogno; 8911 return 0; 8912 } 8913 8914 map = env->used_maps[aux->map_index]; 8915 mark_reg_known_zero(env, regs, insn->dst_reg); 8916 dst_reg->map_ptr = map; 8917 8918 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { 8919 dst_reg->type = PTR_TO_MAP_VALUE; 8920 dst_reg->off = aux->map_off; 8921 if (map_value_has_spin_lock(map)) 8922 dst_reg->id = ++env->id_gen; 8923 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { 8924 dst_reg->type = CONST_PTR_TO_MAP; 8925 } else { 8926 verbose(env, "bpf verifier is misconfigured\n"); 8927 return -EINVAL; 8928 } 8929 8930 return 0; 8931 } 8932 8933 static bool may_access_skb(enum bpf_prog_type type) 8934 { 8935 switch (type) { 8936 case BPF_PROG_TYPE_SOCKET_FILTER: 8937 case BPF_PROG_TYPE_SCHED_CLS: 8938 case BPF_PROG_TYPE_SCHED_ACT: 8939 return true; 8940 default: 8941 return false; 8942 } 8943 } 8944 8945 /* verify safety of LD_ABS|LD_IND instructions: 8946 * - they can only appear in the programs where ctx == skb 8947 * - since they are wrappers of function calls, they scratch R1-R5 registers, 8948 * preserve R6-R9, and store return value into R0 8949 * 8950 * Implicit input: 8951 * ctx == skb == R6 == CTX 8952 * 8953 * Explicit input: 8954 * SRC == any register 8955 * IMM == 32-bit immediate 8956 * 8957 * Output: 8958 * R0 - 8/16/32-bit skb data converted to cpu endianness 8959 */ 8960 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 8961 { 8962 struct bpf_reg_state *regs = cur_regs(env); 8963 static const int ctx_reg = BPF_REG_6; 8964 u8 mode = BPF_MODE(insn->code); 8965 int i, err; 8966 8967 if (!may_access_skb(resolve_prog_type(env->prog))) { 8968 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 8969 return -EINVAL; 8970 } 8971 8972 if (!env->ops->gen_ld_abs) { 8973 verbose(env, "bpf verifier is misconfigured\n"); 8974 return -EINVAL; 8975 } 8976 8977 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 8978 BPF_SIZE(insn->code) == BPF_DW || 8979 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 8980 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 8981 return -EINVAL; 8982 } 8983 8984 /* check whether implicit source operand (register R6) is readable */ 8985 err = check_reg_arg(env, ctx_reg, SRC_OP); 8986 if (err) 8987 return err; 8988 8989 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 8990 * gen_ld_abs() may terminate the program at runtime, leading to 8991 * reference leak. 8992 */ 8993 err = check_reference_leak(env); 8994 if (err) { 8995 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 8996 return err; 8997 } 8998 8999 if (env->cur_state->active_spin_lock) { 9000 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 9001 return -EINVAL; 9002 } 9003 9004 if (regs[ctx_reg].type != PTR_TO_CTX) { 9005 verbose(env, 9006 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 9007 return -EINVAL; 9008 } 9009 9010 if (mode == BPF_IND) { 9011 /* check explicit source operand */ 9012 err = check_reg_arg(env, insn->src_reg, SRC_OP); 9013 if (err) 9014 return err; 9015 } 9016 9017 err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); 9018 if (err < 0) 9019 return err; 9020 9021 /* reset caller saved regs to unreadable */ 9022 for (i = 0; i < CALLER_SAVED_REGS; i++) { 9023 mark_reg_not_init(env, regs, caller_saved[i]); 9024 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 9025 } 9026 9027 /* mark destination R0 register as readable, since it contains 9028 * the value fetched from the packet. 9029 * Already marked as written above. 9030 */ 9031 mark_reg_unknown(env, regs, BPF_REG_0); 9032 /* ld_abs load up to 32-bit skb data. */ 9033 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 9034 return 0; 9035 } 9036 9037 static int check_return_code(struct bpf_verifier_env *env) 9038 { 9039 struct tnum enforce_attach_type_range = tnum_unknown; 9040 const struct bpf_prog *prog = env->prog; 9041 struct bpf_reg_state *reg; 9042 struct tnum range = tnum_range(0, 1); 9043 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 9044 int err; 9045 const bool is_subprog = env->cur_state->frame[0]->subprogno; 9046 9047 /* LSM and struct_ops func-ptr's return type could be "void" */ 9048 if (!is_subprog && 9049 (prog_type == BPF_PROG_TYPE_STRUCT_OPS || 9050 prog_type == BPF_PROG_TYPE_LSM) && 9051 !prog->aux->attach_func_proto->type) 9052 return 0; 9053 9054 /* eBPF calling convetion is such that R0 is used 9055 * to return the value from eBPF program. 9056 * Make sure that it's readable at this time 9057 * of bpf_exit, which means that program wrote 9058 * something into it earlier 9059 */ 9060 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 9061 if (err) 9062 return err; 9063 9064 if (is_pointer_value(env, BPF_REG_0)) { 9065 verbose(env, "R0 leaks addr as return value\n"); 9066 return -EACCES; 9067 } 9068 9069 reg = cur_regs(env) + BPF_REG_0; 9070 if (is_subprog) { 9071 if (reg->type != SCALAR_VALUE) { 9072 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 9073 reg_type_str[reg->type]); 9074 return -EINVAL; 9075 } 9076 return 0; 9077 } 9078 9079 switch (prog_type) { 9080 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 9081 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 9082 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 9083 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 9084 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 9085 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 9086 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 9087 range = tnum_range(1, 1); 9088 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 9089 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 9090 range = tnum_range(0, 3); 9091 break; 9092 case BPF_PROG_TYPE_CGROUP_SKB: 9093 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 9094 range = tnum_range(0, 3); 9095 enforce_attach_type_range = tnum_range(2, 3); 9096 } 9097 break; 9098 case BPF_PROG_TYPE_CGROUP_SOCK: 9099 case BPF_PROG_TYPE_SOCK_OPS: 9100 case BPF_PROG_TYPE_CGROUP_DEVICE: 9101 case BPF_PROG_TYPE_CGROUP_SYSCTL: 9102 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 9103 break; 9104 case BPF_PROG_TYPE_RAW_TRACEPOINT: 9105 if (!env->prog->aux->attach_btf_id) 9106 return 0; 9107 range = tnum_const(0); 9108 break; 9109 case BPF_PROG_TYPE_TRACING: 9110 switch (env->prog->expected_attach_type) { 9111 case BPF_TRACE_FENTRY: 9112 case BPF_TRACE_FEXIT: 9113 range = tnum_const(0); 9114 break; 9115 case BPF_TRACE_RAW_TP: 9116 case BPF_MODIFY_RETURN: 9117 return 0; 9118 case BPF_TRACE_ITER: 9119 break; 9120 default: 9121 return -ENOTSUPP; 9122 } 9123 break; 9124 case BPF_PROG_TYPE_SK_LOOKUP: 9125 range = tnum_range(SK_DROP, SK_PASS); 9126 break; 9127 case BPF_PROG_TYPE_EXT: 9128 /* freplace program can return anything as its return value 9129 * depends on the to-be-replaced kernel func or bpf program. 9130 */ 9131 default: 9132 return 0; 9133 } 9134 9135 if (reg->type != SCALAR_VALUE) { 9136 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 9137 reg_type_str[reg->type]); 9138 return -EINVAL; 9139 } 9140 9141 if (!tnum_in(range, reg->var_off)) { 9142 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 9143 return -EINVAL; 9144 } 9145 9146 if (!tnum_is_unknown(enforce_attach_type_range) && 9147 tnum_in(enforce_attach_type_range, reg->var_off)) 9148 env->prog->enforce_expected_attach_type = 1; 9149 return 0; 9150 } 9151 9152 /* non-recursive DFS pseudo code 9153 * 1 procedure DFS-iterative(G,v): 9154 * 2 label v as discovered 9155 * 3 let S be a stack 9156 * 4 S.push(v) 9157 * 5 while S is not empty 9158 * 6 t <- S.pop() 9159 * 7 if t is what we're looking for: 9160 * 8 return t 9161 * 9 for all edges e in G.adjacentEdges(t) do 9162 * 10 if edge e is already labelled 9163 * 11 continue with the next edge 9164 * 12 w <- G.adjacentVertex(t,e) 9165 * 13 if vertex w is not discovered and not explored 9166 * 14 label e as tree-edge 9167 * 15 label w as discovered 9168 * 16 S.push(w) 9169 * 17 continue at 5 9170 * 18 else if vertex w is discovered 9171 * 19 label e as back-edge 9172 * 20 else 9173 * 21 // vertex w is explored 9174 * 22 label e as forward- or cross-edge 9175 * 23 label t as explored 9176 * 24 S.pop() 9177 * 9178 * convention: 9179 * 0x10 - discovered 9180 * 0x11 - discovered and fall-through edge labelled 9181 * 0x12 - discovered and fall-through and branch edges labelled 9182 * 0x20 - explored 9183 */ 9184 9185 enum { 9186 DISCOVERED = 0x10, 9187 EXPLORED = 0x20, 9188 FALLTHROUGH = 1, 9189 BRANCH = 2, 9190 }; 9191 9192 static u32 state_htab_size(struct bpf_verifier_env *env) 9193 { 9194 return env->prog->len; 9195 } 9196 9197 static struct bpf_verifier_state_list **explored_state( 9198 struct bpf_verifier_env *env, 9199 int idx) 9200 { 9201 struct bpf_verifier_state *cur = env->cur_state; 9202 struct bpf_func_state *state = cur->frame[cur->curframe]; 9203 9204 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 9205 } 9206 9207 static void init_explored_state(struct bpf_verifier_env *env, int idx) 9208 { 9209 env->insn_aux_data[idx].prune_point = true; 9210 } 9211 9212 enum { 9213 DONE_EXPLORING = 0, 9214 KEEP_EXPLORING = 1, 9215 }; 9216 9217 /* t, w, e - match pseudo-code above: 9218 * t - index of current instruction 9219 * w - next instruction 9220 * e - edge 9221 */ 9222 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 9223 bool loop_ok) 9224 { 9225 int *insn_stack = env->cfg.insn_stack; 9226 int *insn_state = env->cfg.insn_state; 9227 9228 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 9229 return DONE_EXPLORING; 9230 9231 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 9232 return DONE_EXPLORING; 9233 9234 if (w < 0 || w >= env->prog->len) { 9235 verbose_linfo(env, t, "%d: ", t); 9236 verbose(env, "jump out of range from insn %d to %d\n", t, w); 9237 return -EINVAL; 9238 } 9239 9240 if (e == BRANCH) 9241 /* mark branch target for state pruning */ 9242 init_explored_state(env, w); 9243 9244 if (insn_state[w] == 0) { 9245 /* tree-edge */ 9246 insn_state[t] = DISCOVERED | e; 9247 insn_state[w] = DISCOVERED; 9248 if (env->cfg.cur_stack >= env->prog->len) 9249 return -E2BIG; 9250 insn_stack[env->cfg.cur_stack++] = w; 9251 return KEEP_EXPLORING; 9252 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 9253 if (loop_ok && env->bpf_capable) 9254 return DONE_EXPLORING; 9255 verbose_linfo(env, t, "%d: ", t); 9256 verbose_linfo(env, w, "%d: ", w); 9257 verbose(env, "back-edge from insn %d to %d\n", t, w); 9258 return -EINVAL; 9259 } else if (insn_state[w] == EXPLORED) { 9260 /* forward- or cross-edge */ 9261 insn_state[t] = DISCOVERED | e; 9262 } else { 9263 verbose(env, "insn state internal bug\n"); 9264 return -EFAULT; 9265 } 9266 return DONE_EXPLORING; 9267 } 9268 9269 static int visit_func_call_insn(int t, int insn_cnt, 9270 struct bpf_insn *insns, 9271 struct bpf_verifier_env *env, 9272 bool visit_callee) 9273 { 9274 int ret; 9275 9276 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 9277 if (ret) 9278 return ret; 9279 9280 if (t + 1 < insn_cnt) 9281 init_explored_state(env, t + 1); 9282 if (visit_callee) { 9283 init_explored_state(env, t); 9284 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, 9285 env, false); 9286 } 9287 return ret; 9288 } 9289 9290 /* Visits the instruction at index t and returns one of the following: 9291 * < 0 - an error occurred 9292 * DONE_EXPLORING - the instruction was fully explored 9293 * KEEP_EXPLORING - there is still work to be done before it is fully explored 9294 */ 9295 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) 9296 { 9297 struct bpf_insn *insns = env->prog->insnsi; 9298 int ret; 9299 9300 if (bpf_pseudo_func(insns + t)) 9301 return visit_func_call_insn(t, insn_cnt, insns, env, true); 9302 9303 /* All non-branch instructions have a single fall-through edge. */ 9304 if (BPF_CLASS(insns[t].code) != BPF_JMP && 9305 BPF_CLASS(insns[t].code) != BPF_JMP32) 9306 return push_insn(t, t + 1, FALLTHROUGH, env, false); 9307 9308 switch (BPF_OP(insns[t].code)) { 9309 case BPF_EXIT: 9310 return DONE_EXPLORING; 9311 9312 case BPF_CALL: 9313 return visit_func_call_insn(t, insn_cnt, insns, env, 9314 insns[t].src_reg == BPF_PSEUDO_CALL); 9315 9316 case BPF_JA: 9317 if (BPF_SRC(insns[t].code) != BPF_K) 9318 return -EINVAL; 9319 9320 /* unconditional jump with single edge */ 9321 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 9322 true); 9323 if (ret) 9324 return ret; 9325 9326 /* unconditional jmp is not a good pruning point, 9327 * but it's marked, since backtracking needs 9328 * to record jmp history in is_state_visited(). 9329 */ 9330 init_explored_state(env, t + insns[t].off + 1); 9331 /* tell verifier to check for equivalent states 9332 * after every call and jump 9333 */ 9334 if (t + 1 < insn_cnt) 9335 init_explored_state(env, t + 1); 9336 9337 return ret; 9338 9339 default: 9340 /* conditional jump with two edges */ 9341 init_explored_state(env, t); 9342 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 9343 if (ret) 9344 return ret; 9345 9346 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 9347 } 9348 } 9349 9350 /* non-recursive depth-first-search to detect loops in BPF program 9351 * loop == back-edge in directed graph 9352 */ 9353 static int check_cfg(struct bpf_verifier_env *env) 9354 { 9355 int insn_cnt = env->prog->len; 9356 int *insn_stack, *insn_state; 9357 int ret = 0; 9358 int i; 9359 9360 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 9361 if (!insn_state) 9362 return -ENOMEM; 9363 9364 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 9365 if (!insn_stack) { 9366 kvfree(insn_state); 9367 return -ENOMEM; 9368 } 9369 9370 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 9371 insn_stack[0] = 0; /* 0 is the first instruction */ 9372 env->cfg.cur_stack = 1; 9373 9374 while (env->cfg.cur_stack > 0) { 9375 int t = insn_stack[env->cfg.cur_stack - 1]; 9376 9377 ret = visit_insn(t, insn_cnt, env); 9378 switch (ret) { 9379 case DONE_EXPLORING: 9380 insn_state[t] = EXPLORED; 9381 env->cfg.cur_stack--; 9382 break; 9383 case KEEP_EXPLORING: 9384 break; 9385 default: 9386 if (ret > 0) { 9387 verbose(env, "visit_insn internal bug\n"); 9388 ret = -EFAULT; 9389 } 9390 goto err_free; 9391 } 9392 } 9393 9394 if (env->cfg.cur_stack < 0) { 9395 verbose(env, "pop stack internal bug\n"); 9396 ret = -EFAULT; 9397 goto err_free; 9398 } 9399 9400 for (i = 0; i < insn_cnt; i++) { 9401 if (insn_state[i] != EXPLORED) { 9402 verbose(env, "unreachable insn %d\n", i); 9403 ret = -EINVAL; 9404 goto err_free; 9405 } 9406 } 9407 ret = 0; /* cfg looks good */ 9408 9409 err_free: 9410 kvfree(insn_state); 9411 kvfree(insn_stack); 9412 env->cfg.insn_state = env->cfg.insn_stack = NULL; 9413 return ret; 9414 } 9415 9416 static int check_abnormal_return(struct bpf_verifier_env *env) 9417 { 9418 int i; 9419 9420 for (i = 1; i < env->subprog_cnt; i++) { 9421 if (env->subprog_info[i].has_ld_abs) { 9422 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 9423 return -EINVAL; 9424 } 9425 if (env->subprog_info[i].has_tail_call) { 9426 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 9427 return -EINVAL; 9428 } 9429 } 9430 return 0; 9431 } 9432 9433 /* The minimum supported BTF func info size */ 9434 #define MIN_BPF_FUNCINFO_SIZE 8 9435 #define MAX_FUNCINFO_REC_SIZE 252 9436 9437 static int check_btf_func(struct bpf_verifier_env *env, 9438 const union bpf_attr *attr, 9439 union bpf_attr __user *uattr) 9440 { 9441 const struct btf_type *type, *func_proto, *ret_type; 9442 u32 i, nfuncs, urec_size, min_size; 9443 u32 krec_size = sizeof(struct bpf_func_info); 9444 struct bpf_func_info *krecord; 9445 struct bpf_func_info_aux *info_aux = NULL; 9446 struct bpf_prog *prog; 9447 const struct btf *btf; 9448 void __user *urecord; 9449 u32 prev_offset = 0; 9450 bool scalar_return; 9451 int ret = -ENOMEM; 9452 9453 nfuncs = attr->func_info_cnt; 9454 if (!nfuncs) { 9455 if (check_abnormal_return(env)) 9456 return -EINVAL; 9457 return 0; 9458 } 9459 9460 if (nfuncs != env->subprog_cnt) { 9461 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 9462 return -EINVAL; 9463 } 9464 9465 urec_size = attr->func_info_rec_size; 9466 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 9467 urec_size > MAX_FUNCINFO_REC_SIZE || 9468 urec_size % sizeof(u32)) { 9469 verbose(env, "invalid func info rec size %u\n", urec_size); 9470 return -EINVAL; 9471 } 9472 9473 prog = env->prog; 9474 btf = prog->aux->btf; 9475 9476 urecord = u64_to_user_ptr(attr->func_info); 9477 min_size = min_t(u32, krec_size, urec_size); 9478 9479 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 9480 if (!krecord) 9481 return -ENOMEM; 9482 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 9483 if (!info_aux) 9484 goto err_free; 9485 9486 for (i = 0; i < nfuncs; i++) { 9487 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 9488 if (ret) { 9489 if (ret == -E2BIG) { 9490 verbose(env, "nonzero tailing record in func info"); 9491 /* set the size kernel expects so loader can zero 9492 * out the rest of the record. 9493 */ 9494 if (put_user(min_size, &uattr->func_info_rec_size)) 9495 ret = -EFAULT; 9496 } 9497 goto err_free; 9498 } 9499 9500 if (copy_from_user(&krecord[i], urecord, min_size)) { 9501 ret = -EFAULT; 9502 goto err_free; 9503 } 9504 9505 /* check insn_off */ 9506 ret = -EINVAL; 9507 if (i == 0) { 9508 if (krecord[i].insn_off) { 9509 verbose(env, 9510 "nonzero insn_off %u for the first func info record", 9511 krecord[i].insn_off); 9512 goto err_free; 9513 } 9514 } else if (krecord[i].insn_off <= prev_offset) { 9515 verbose(env, 9516 "same or smaller insn offset (%u) than previous func info record (%u)", 9517 krecord[i].insn_off, prev_offset); 9518 goto err_free; 9519 } 9520 9521 if (env->subprog_info[i].start != krecord[i].insn_off) { 9522 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 9523 goto err_free; 9524 } 9525 9526 /* check type_id */ 9527 type = btf_type_by_id(btf, krecord[i].type_id); 9528 if (!type || !btf_type_is_func(type)) { 9529 verbose(env, "invalid type id %d in func info", 9530 krecord[i].type_id); 9531 goto err_free; 9532 } 9533 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 9534 9535 func_proto = btf_type_by_id(btf, type->type); 9536 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 9537 /* btf_func_check() already verified it during BTF load */ 9538 goto err_free; 9539 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 9540 scalar_return = 9541 btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); 9542 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 9543 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 9544 goto err_free; 9545 } 9546 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 9547 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 9548 goto err_free; 9549 } 9550 9551 prev_offset = krecord[i].insn_off; 9552 urecord += urec_size; 9553 } 9554 9555 prog->aux->func_info = krecord; 9556 prog->aux->func_info_cnt = nfuncs; 9557 prog->aux->func_info_aux = info_aux; 9558 return 0; 9559 9560 err_free: 9561 kvfree(krecord); 9562 kfree(info_aux); 9563 return ret; 9564 } 9565 9566 static void adjust_btf_func(struct bpf_verifier_env *env) 9567 { 9568 struct bpf_prog_aux *aux = env->prog->aux; 9569 int i; 9570 9571 if (!aux->func_info) 9572 return; 9573 9574 for (i = 0; i < env->subprog_cnt; i++) 9575 aux->func_info[i].insn_off = env->subprog_info[i].start; 9576 } 9577 9578 #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ 9579 sizeof(((struct bpf_line_info *)(0))->line_col)) 9580 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 9581 9582 static int check_btf_line(struct bpf_verifier_env *env, 9583 const union bpf_attr *attr, 9584 union bpf_attr __user *uattr) 9585 { 9586 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 9587 struct bpf_subprog_info *sub; 9588 struct bpf_line_info *linfo; 9589 struct bpf_prog *prog; 9590 const struct btf *btf; 9591 void __user *ulinfo; 9592 int err; 9593 9594 nr_linfo = attr->line_info_cnt; 9595 if (!nr_linfo) 9596 return 0; 9597 9598 rec_size = attr->line_info_rec_size; 9599 if (rec_size < MIN_BPF_LINEINFO_SIZE || 9600 rec_size > MAX_LINEINFO_REC_SIZE || 9601 rec_size & (sizeof(u32) - 1)) 9602 return -EINVAL; 9603 9604 /* Need to zero it in case the userspace may 9605 * pass in a smaller bpf_line_info object. 9606 */ 9607 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 9608 GFP_KERNEL | __GFP_NOWARN); 9609 if (!linfo) 9610 return -ENOMEM; 9611 9612 prog = env->prog; 9613 btf = prog->aux->btf; 9614 9615 s = 0; 9616 sub = env->subprog_info; 9617 ulinfo = u64_to_user_ptr(attr->line_info); 9618 expected_size = sizeof(struct bpf_line_info); 9619 ncopy = min_t(u32, expected_size, rec_size); 9620 for (i = 0; i < nr_linfo; i++) { 9621 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 9622 if (err) { 9623 if (err == -E2BIG) { 9624 verbose(env, "nonzero tailing record in line_info"); 9625 if (put_user(expected_size, 9626 &uattr->line_info_rec_size)) 9627 err = -EFAULT; 9628 } 9629 goto err_free; 9630 } 9631 9632 if (copy_from_user(&linfo[i], ulinfo, ncopy)) { 9633 err = -EFAULT; 9634 goto err_free; 9635 } 9636 9637 /* 9638 * Check insn_off to ensure 9639 * 1) strictly increasing AND 9640 * 2) bounded by prog->len 9641 * 9642 * The linfo[0].insn_off == 0 check logically falls into 9643 * the later "missing bpf_line_info for func..." case 9644 * because the first linfo[0].insn_off must be the 9645 * first sub also and the first sub must have 9646 * subprog_info[0].start == 0. 9647 */ 9648 if ((i && linfo[i].insn_off <= prev_offset) || 9649 linfo[i].insn_off >= prog->len) { 9650 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 9651 i, linfo[i].insn_off, prev_offset, 9652 prog->len); 9653 err = -EINVAL; 9654 goto err_free; 9655 } 9656 9657 if (!prog->insnsi[linfo[i].insn_off].code) { 9658 verbose(env, 9659 "Invalid insn code at line_info[%u].insn_off\n", 9660 i); 9661 err = -EINVAL; 9662 goto err_free; 9663 } 9664 9665 if (!btf_name_by_offset(btf, linfo[i].line_off) || 9666 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 9667 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 9668 err = -EINVAL; 9669 goto err_free; 9670 } 9671 9672 if (s != env->subprog_cnt) { 9673 if (linfo[i].insn_off == sub[s].start) { 9674 sub[s].linfo_idx = i; 9675 s++; 9676 } else if (sub[s].start < linfo[i].insn_off) { 9677 verbose(env, "missing bpf_line_info for func#%u\n", s); 9678 err = -EINVAL; 9679 goto err_free; 9680 } 9681 } 9682 9683 prev_offset = linfo[i].insn_off; 9684 ulinfo += rec_size; 9685 } 9686 9687 if (s != env->subprog_cnt) { 9688 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 9689 env->subprog_cnt - s, s); 9690 err = -EINVAL; 9691 goto err_free; 9692 } 9693 9694 prog->aux->linfo = linfo; 9695 prog->aux->nr_linfo = nr_linfo; 9696 9697 return 0; 9698 9699 err_free: 9700 kvfree(linfo); 9701 return err; 9702 } 9703 9704 static int check_btf_info(struct bpf_verifier_env *env, 9705 const union bpf_attr *attr, 9706 union bpf_attr __user *uattr) 9707 { 9708 struct btf *btf; 9709 int err; 9710 9711 if (!attr->func_info_cnt && !attr->line_info_cnt) { 9712 if (check_abnormal_return(env)) 9713 return -EINVAL; 9714 return 0; 9715 } 9716 9717 btf = btf_get_by_fd(attr->prog_btf_fd); 9718 if (IS_ERR(btf)) 9719 return PTR_ERR(btf); 9720 if (btf_is_kernel(btf)) { 9721 btf_put(btf); 9722 return -EACCES; 9723 } 9724 env->prog->aux->btf = btf; 9725 9726 err = check_btf_func(env, attr, uattr); 9727 if (err) 9728 return err; 9729 9730 err = check_btf_line(env, attr, uattr); 9731 if (err) 9732 return err; 9733 9734 return 0; 9735 } 9736 9737 /* check %cur's range satisfies %old's */ 9738 static bool range_within(struct bpf_reg_state *old, 9739 struct bpf_reg_state *cur) 9740 { 9741 return old->umin_value <= cur->umin_value && 9742 old->umax_value >= cur->umax_value && 9743 old->smin_value <= cur->smin_value && 9744 old->smax_value >= cur->smax_value && 9745 old->u32_min_value <= cur->u32_min_value && 9746 old->u32_max_value >= cur->u32_max_value && 9747 old->s32_min_value <= cur->s32_min_value && 9748 old->s32_max_value >= cur->s32_max_value; 9749 } 9750 9751 /* If in the old state two registers had the same id, then they need to have 9752 * the same id in the new state as well. But that id could be different from 9753 * the old state, so we need to track the mapping from old to new ids. 9754 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 9755 * regs with old id 5 must also have new id 9 for the new state to be safe. But 9756 * regs with a different old id could still have new id 9, we don't care about 9757 * that. 9758 * So we look through our idmap to see if this old id has been seen before. If 9759 * so, we require the new id to match; otherwise, we add the id pair to the map. 9760 */ 9761 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 9762 { 9763 unsigned int i; 9764 9765 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 9766 if (!idmap[i].old) { 9767 /* Reached an empty slot; haven't seen this id before */ 9768 idmap[i].old = old_id; 9769 idmap[i].cur = cur_id; 9770 return true; 9771 } 9772 if (idmap[i].old == old_id) 9773 return idmap[i].cur == cur_id; 9774 } 9775 /* We ran out of idmap slots, which should be impossible */ 9776 WARN_ON_ONCE(1); 9777 return false; 9778 } 9779 9780 static void clean_func_state(struct bpf_verifier_env *env, 9781 struct bpf_func_state *st) 9782 { 9783 enum bpf_reg_liveness live; 9784 int i, j; 9785 9786 for (i = 0; i < BPF_REG_FP; i++) { 9787 live = st->regs[i].live; 9788 /* liveness must not touch this register anymore */ 9789 st->regs[i].live |= REG_LIVE_DONE; 9790 if (!(live & REG_LIVE_READ)) 9791 /* since the register is unused, clear its state 9792 * to make further comparison simpler 9793 */ 9794 __mark_reg_not_init(env, &st->regs[i]); 9795 } 9796 9797 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 9798 live = st->stack[i].spilled_ptr.live; 9799 /* liveness must not touch this stack slot anymore */ 9800 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 9801 if (!(live & REG_LIVE_READ)) { 9802 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 9803 for (j = 0; j < BPF_REG_SIZE; j++) 9804 st->stack[i].slot_type[j] = STACK_INVALID; 9805 } 9806 } 9807 } 9808 9809 static void clean_verifier_state(struct bpf_verifier_env *env, 9810 struct bpf_verifier_state *st) 9811 { 9812 int i; 9813 9814 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 9815 /* all regs in this state in all frames were already marked */ 9816 return; 9817 9818 for (i = 0; i <= st->curframe; i++) 9819 clean_func_state(env, st->frame[i]); 9820 } 9821 9822 /* the parentage chains form a tree. 9823 * the verifier states are added to state lists at given insn and 9824 * pushed into state stack for future exploration. 9825 * when the verifier reaches bpf_exit insn some of the verifer states 9826 * stored in the state lists have their final liveness state already, 9827 * but a lot of states will get revised from liveness point of view when 9828 * the verifier explores other branches. 9829 * Example: 9830 * 1: r0 = 1 9831 * 2: if r1 == 100 goto pc+1 9832 * 3: r0 = 2 9833 * 4: exit 9834 * when the verifier reaches exit insn the register r0 in the state list of 9835 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 9836 * of insn 2 and goes exploring further. At the insn 4 it will walk the 9837 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 9838 * 9839 * Since the verifier pushes the branch states as it sees them while exploring 9840 * the program the condition of walking the branch instruction for the second 9841 * time means that all states below this branch were already explored and 9842 * their final liveness markes are already propagated. 9843 * Hence when the verifier completes the search of state list in is_state_visited() 9844 * we can call this clean_live_states() function to mark all liveness states 9845 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 9846 * will not be used. 9847 * This function also clears the registers and stack for states that !READ 9848 * to simplify state merging. 9849 * 9850 * Important note here that walking the same branch instruction in the callee 9851 * doesn't meant that the states are DONE. The verifier has to compare 9852 * the callsites 9853 */ 9854 static void clean_live_states(struct bpf_verifier_env *env, int insn, 9855 struct bpf_verifier_state *cur) 9856 { 9857 struct bpf_verifier_state_list *sl; 9858 int i; 9859 9860 sl = *explored_state(env, insn); 9861 while (sl) { 9862 if (sl->state.branches) 9863 goto next; 9864 if (sl->state.insn_idx != insn || 9865 sl->state.curframe != cur->curframe) 9866 goto next; 9867 for (i = 0; i <= cur->curframe; i++) 9868 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 9869 goto next; 9870 clean_verifier_state(env, &sl->state); 9871 next: 9872 sl = sl->next; 9873 } 9874 } 9875 9876 /* Returns true if (rold safe implies rcur safe) */ 9877 static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, 9878 struct bpf_id_pair *idmap) 9879 { 9880 bool equal; 9881 9882 if (!(rold->live & REG_LIVE_READ)) 9883 /* explored state didn't use this */ 9884 return true; 9885 9886 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 9887 9888 if (rold->type == PTR_TO_STACK) 9889 /* two stack pointers are equal only if they're pointing to 9890 * the same stack frame, since fp-8 in foo != fp-8 in bar 9891 */ 9892 return equal && rold->frameno == rcur->frameno; 9893 9894 if (equal) 9895 return true; 9896 9897 if (rold->type == NOT_INIT) 9898 /* explored state can't have used this */ 9899 return true; 9900 if (rcur->type == NOT_INIT) 9901 return false; 9902 switch (rold->type) { 9903 case SCALAR_VALUE: 9904 if (rcur->type == SCALAR_VALUE) { 9905 if (!rold->precise && !rcur->precise) 9906 return true; 9907 /* new val must satisfy old val knowledge */ 9908 return range_within(rold, rcur) && 9909 tnum_in(rold->var_off, rcur->var_off); 9910 } else { 9911 /* We're trying to use a pointer in place of a scalar. 9912 * Even if the scalar was unbounded, this could lead to 9913 * pointer leaks because scalars are allowed to leak 9914 * while pointers are not. We could make this safe in 9915 * special cases if root is calling us, but it's 9916 * probably not worth the hassle. 9917 */ 9918 return false; 9919 } 9920 case PTR_TO_MAP_KEY: 9921 case PTR_TO_MAP_VALUE: 9922 /* If the new min/max/var_off satisfy the old ones and 9923 * everything else matches, we are OK. 9924 * 'id' is not compared, since it's only used for maps with 9925 * bpf_spin_lock inside map element and in such cases if 9926 * the rest of the prog is valid for one map element then 9927 * it's valid for all map elements regardless of the key 9928 * used in bpf_map_lookup() 9929 */ 9930 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 9931 range_within(rold, rcur) && 9932 tnum_in(rold->var_off, rcur->var_off); 9933 case PTR_TO_MAP_VALUE_OR_NULL: 9934 /* a PTR_TO_MAP_VALUE could be safe to use as a 9935 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 9936 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 9937 * checked, doing so could have affected others with the same 9938 * id, and we can't check for that because we lost the id when 9939 * we converted to a PTR_TO_MAP_VALUE. 9940 */ 9941 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) 9942 return false; 9943 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 9944 return false; 9945 /* Check our ids match any regs they're supposed to */ 9946 return check_ids(rold->id, rcur->id, idmap); 9947 case PTR_TO_PACKET_META: 9948 case PTR_TO_PACKET: 9949 if (rcur->type != rold->type) 9950 return false; 9951 /* We must have at least as much range as the old ptr 9952 * did, so that any accesses which were safe before are 9953 * still safe. This is true even if old range < old off, 9954 * since someone could have accessed through (ptr - k), or 9955 * even done ptr -= k in a register, to get a safe access. 9956 */ 9957 if (rold->range > rcur->range) 9958 return false; 9959 /* If the offsets don't match, we can't trust our alignment; 9960 * nor can we be sure that we won't fall out of range. 9961 */ 9962 if (rold->off != rcur->off) 9963 return false; 9964 /* id relations must be preserved */ 9965 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 9966 return false; 9967 /* new val must satisfy old val knowledge */ 9968 return range_within(rold, rcur) && 9969 tnum_in(rold->var_off, rcur->var_off); 9970 case PTR_TO_CTX: 9971 case CONST_PTR_TO_MAP: 9972 case PTR_TO_PACKET_END: 9973 case PTR_TO_FLOW_KEYS: 9974 case PTR_TO_SOCKET: 9975 case PTR_TO_SOCKET_OR_NULL: 9976 case PTR_TO_SOCK_COMMON: 9977 case PTR_TO_SOCK_COMMON_OR_NULL: 9978 case PTR_TO_TCP_SOCK: 9979 case PTR_TO_TCP_SOCK_OR_NULL: 9980 case PTR_TO_XDP_SOCK: 9981 /* Only valid matches are exact, which memcmp() above 9982 * would have accepted 9983 */ 9984 default: 9985 /* Don't know what's going on, just say it's not safe */ 9986 return false; 9987 } 9988 9989 /* Shouldn't get here; if we do, say it's not safe */ 9990 WARN_ON_ONCE(1); 9991 return false; 9992 } 9993 9994 static bool stacksafe(struct bpf_func_state *old, 9995 struct bpf_func_state *cur, 9996 struct bpf_id_pair *idmap) 9997 { 9998 int i, spi; 9999 10000 /* walk slots of the explored stack and ignore any additional 10001 * slots in the current stack, since explored(safe) state 10002 * didn't use them 10003 */ 10004 for (i = 0; i < old->allocated_stack; i++) { 10005 spi = i / BPF_REG_SIZE; 10006 10007 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 10008 i += BPF_REG_SIZE - 1; 10009 /* explored state didn't use this */ 10010 continue; 10011 } 10012 10013 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 10014 continue; 10015 10016 /* explored stack has more populated slots than current stack 10017 * and these slots were used 10018 */ 10019 if (i >= cur->allocated_stack) 10020 return false; 10021 10022 /* if old state was safe with misc data in the stack 10023 * it will be safe with zero-initialized stack. 10024 * The opposite is not true 10025 */ 10026 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 10027 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 10028 continue; 10029 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 10030 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 10031 /* Ex: old explored (safe) state has STACK_SPILL in 10032 * this stack slot, but current has STACK_MISC -> 10033 * this verifier states are not equivalent, 10034 * return false to continue verification of this path 10035 */ 10036 return false; 10037 if (i % BPF_REG_SIZE) 10038 continue; 10039 if (old->stack[spi].slot_type[0] != STACK_SPILL) 10040 continue; 10041 if (!regsafe(&old->stack[spi].spilled_ptr, 10042 &cur->stack[spi].spilled_ptr, 10043 idmap)) 10044 /* when explored and current stack slot are both storing 10045 * spilled registers, check that stored pointers types 10046 * are the same as well. 10047 * Ex: explored safe path could have stored 10048 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 10049 * but current path has stored: 10050 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 10051 * such verifier states are not equivalent. 10052 * return false to continue verification of this path 10053 */ 10054 return false; 10055 } 10056 return true; 10057 } 10058 10059 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 10060 { 10061 if (old->acquired_refs != cur->acquired_refs) 10062 return false; 10063 return !memcmp(old->refs, cur->refs, 10064 sizeof(*old->refs) * old->acquired_refs); 10065 } 10066 10067 /* compare two verifier states 10068 * 10069 * all states stored in state_list are known to be valid, since 10070 * verifier reached 'bpf_exit' instruction through them 10071 * 10072 * this function is called when verifier exploring different branches of 10073 * execution popped from the state stack. If it sees an old state that has 10074 * more strict register state and more strict stack state then this execution 10075 * branch doesn't need to be explored further, since verifier already 10076 * concluded that more strict state leads to valid finish. 10077 * 10078 * Therefore two states are equivalent if register state is more conservative 10079 * and explored stack state is more conservative than the current one. 10080 * Example: 10081 * explored current 10082 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 10083 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 10084 * 10085 * In other words if current stack state (one being explored) has more 10086 * valid slots than old one that already passed validation, it means 10087 * the verifier can stop exploring and conclude that current state is valid too 10088 * 10089 * Similarly with registers. If explored state has register type as invalid 10090 * whereas register type in current state is meaningful, it means that 10091 * the current state will reach 'bpf_exit' instruction safely 10092 */ 10093 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 10094 struct bpf_func_state *cur) 10095 { 10096 int i; 10097 10098 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 10099 for (i = 0; i < MAX_BPF_REG; i++) 10100 if (!regsafe(&old->regs[i], &cur->regs[i], env->idmap_scratch)) 10101 return false; 10102 10103 if (!stacksafe(old, cur, env->idmap_scratch)) 10104 return false; 10105 10106 if (!refsafe(old, cur)) 10107 return false; 10108 10109 return true; 10110 } 10111 10112 static bool states_equal(struct bpf_verifier_env *env, 10113 struct bpf_verifier_state *old, 10114 struct bpf_verifier_state *cur) 10115 { 10116 int i; 10117 10118 if (old->curframe != cur->curframe) 10119 return false; 10120 10121 /* Verification state from speculative execution simulation 10122 * must never prune a non-speculative execution one. 10123 */ 10124 if (old->speculative && !cur->speculative) 10125 return false; 10126 10127 if (old->active_spin_lock != cur->active_spin_lock) 10128 return false; 10129 10130 /* for states to be equal callsites have to be the same 10131 * and all frame states need to be equivalent 10132 */ 10133 for (i = 0; i <= old->curframe; i++) { 10134 if (old->frame[i]->callsite != cur->frame[i]->callsite) 10135 return false; 10136 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 10137 return false; 10138 } 10139 return true; 10140 } 10141 10142 /* Return 0 if no propagation happened. Return negative error code if error 10143 * happened. Otherwise, return the propagated bit. 10144 */ 10145 static int propagate_liveness_reg(struct bpf_verifier_env *env, 10146 struct bpf_reg_state *reg, 10147 struct bpf_reg_state *parent_reg) 10148 { 10149 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 10150 u8 flag = reg->live & REG_LIVE_READ; 10151 int err; 10152 10153 /* When comes here, read flags of PARENT_REG or REG could be any of 10154 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 10155 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 10156 */ 10157 if (parent_flag == REG_LIVE_READ64 || 10158 /* Or if there is no read flag from REG. */ 10159 !flag || 10160 /* Or if the read flag from REG is the same as PARENT_REG. */ 10161 parent_flag == flag) 10162 return 0; 10163 10164 err = mark_reg_read(env, reg, parent_reg, flag); 10165 if (err) 10166 return err; 10167 10168 return flag; 10169 } 10170 10171 /* A write screens off any subsequent reads; but write marks come from the 10172 * straight-line code between a state and its parent. When we arrive at an 10173 * equivalent state (jump target or such) we didn't arrive by the straight-line 10174 * code, so read marks in the state must propagate to the parent regardless 10175 * of the state's write marks. That's what 'parent == state->parent' comparison 10176 * in mark_reg_read() is for. 10177 */ 10178 static int propagate_liveness(struct bpf_verifier_env *env, 10179 const struct bpf_verifier_state *vstate, 10180 struct bpf_verifier_state *vparent) 10181 { 10182 struct bpf_reg_state *state_reg, *parent_reg; 10183 struct bpf_func_state *state, *parent; 10184 int i, frame, err = 0; 10185 10186 if (vparent->curframe != vstate->curframe) { 10187 WARN(1, "propagate_live: parent frame %d current frame %d\n", 10188 vparent->curframe, vstate->curframe); 10189 return -EFAULT; 10190 } 10191 /* Propagate read liveness of registers... */ 10192 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 10193 for (frame = 0; frame <= vstate->curframe; frame++) { 10194 parent = vparent->frame[frame]; 10195 state = vstate->frame[frame]; 10196 parent_reg = parent->regs; 10197 state_reg = state->regs; 10198 /* We don't need to worry about FP liveness, it's read-only */ 10199 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 10200 err = propagate_liveness_reg(env, &state_reg[i], 10201 &parent_reg[i]); 10202 if (err < 0) 10203 return err; 10204 if (err == REG_LIVE_READ64) 10205 mark_insn_zext(env, &parent_reg[i]); 10206 } 10207 10208 /* Propagate stack slots. */ 10209 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 10210 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 10211 parent_reg = &parent->stack[i].spilled_ptr; 10212 state_reg = &state->stack[i].spilled_ptr; 10213 err = propagate_liveness_reg(env, state_reg, 10214 parent_reg); 10215 if (err < 0) 10216 return err; 10217 } 10218 } 10219 return 0; 10220 } 10221 10222 /* find precise scalars in the previous equivalent state and 10223 * propagate them into the current state 10224 */ 10225 static int propagate_precision(struct bpf_verifier_env *env, 10226 const struct bpf_verifier_state *old) 10227 { 10228 struct bpf_reg_state *state_reg; 10229 struct bpf_func_state *state; 10230 int i, err = 0; 10231 10232 state = old->frame[old->curframe]; 10233 state_reg = state->regs; 10234 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 10235 if (state_reg->type != SCALAR_VALUE || 10236 !state_reg->precise) 10237 continue; 10238 if (env->log.level & BPF_LOG_LEVEL2) 10239 verbose(env, "propagating r%d\n", i); 10240 err = mark_chain_precision(env, i); 10241 if (err < 0) 10242 return err; 10243 } 10244 10245 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 10246 if (state->stack[i].slot_type[0] != STACK_SPILL) 10247 continue; 10248 state_reg = &state->stack[i].spilled_ptr; 10249 if (state_reg->type != SCALAR_VALUE || 10250 !state_reg->precise) 10251 continue; 10252 if (env->log.level & BPF_LOG_LEVEL2) 10253 verbose(env, "propagating fp%d\n", 10254 (-i - 1) * BPF_REG_SIZE); 10255 err = mark_chain_precision_stack(env, i); 10256 if (err < 0) 10257 return err; 10258 } 10259 return 0; 10260 } 10261 10262 static bool states_maybe_looping(struct bpf_verifier_state *old, 10263 struct bpf_verifier_state *cur) 10264 { 10265 struct bpf_func_state *fold, *fcur; 10266 int i, fr = cur->curframe; 10267 10268 if (old->curframe != fr) 10269 return false; 10270 10271 fold = old->frame[fr]; 10272 fcur = cur->frame[fr]; 10273 for (i = 0; i < MAX_BPF_REG; i++) 10274 if (memcmp(&fold->regs[i], &fcur->regs[i], 10275 offsetof(struct bpf_reg_state, parent))) 10276 return false; 10277 return true; 10278 } 10279 10280 10281 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 10282 { 10283 struct bpf_verifier_state_list *new_sl; 10284 struct bpf_verifier_state_list *sl, **pprev; 10285 struct bpf_verifier_state *cur = env->cur_state, *new; 10286 int i, j, err, states_cnt = 0; 10287 bool add_new_state = env->test_state_freq ? true : false; 10288 10289 cur->last_insn_idx = env->prev_insn_idx; 10290 if (!env->insn_aux_data[insn_idx].prune_point) 10291 /* this 'insn_idx' instruction wasn't marked, so we will not 10292 * be doing state search here 10293 */ 10294 return 0; 10295 10296 /* bpf progs typically have pruning point every 4 instructions 10297 * http://vger.kernel.org/bpfconf2019.html#session-1 10298 * Do not add new state for future pruning if the verifier hasn't seen 10299 * at least 2 jumps and at least 8 instructions. 10300 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 10301 * In tests that amounts to up to 50% reduction into total verifier 10302 * memory consumption and 20% verifier time speedup. 10303 */ 10304 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 10305 env->insn_processed - env->prev_insn_processed >= 8) 10306 add_new_state = true; 10307 10308 pprev = explored_state(env, insn_idx); 10309 sl = *pprev; 10310 10311 clean_live_states(env, insn_idx, cur); 10312 10313 while (sl) { 10314 states_cnt++; 10315 if (sl->state.insn_idx != insn_idx) 10316 goto next; 10317 if (sl->state.branches) { 10318 if (states_maybe_looping(&sl->state, cur) && 10319 states_equal(env, &sl->state, cur)) { 10320 verbose_linfo(env, insn_idx, "; "); 10321 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 10322 return -EINVAL; 10323 } 10324 /* if the verifier is processing a loop, avoid adding new state 10325 * too often, since different loop iterations have distinct 10326 * states and may not help future pruning. 10327 * This threshold shouldn't be too low to make sure that 10328 * a loop with large bound will be rejected quickly. 10329 * The most abusive loop will be: 10330 * r1 += 1 10331 * if r1 < 1000000 goto pc-2 10332 * 1M insn_procssed limit / 100 == 10k peak states. 10333 * This threshold shouldn't be too high either, since states 10334 * at the end of the loop are likely to be useful in pruning. 10335 */ 10336 if (env->jmps_processed - env->prev_jmps_processed < 20 && 10337 env->insn_processed - env->prev_insn_processed < 100) 10338 add_new_state = false; 10339 goto miss; 10340 } 10341 if (states_equal(env, &sl->state, cur)) { 10342 sl->hit_cnt++; 10343 /* reached equivalent register/stack state, 10344 * prune the search. 10345 * Registers read by the continuation are read by us. 10346 * If we have any write marks in env->cur_state, they 10347 * will prevent corresponding reads in the continuation 10348 * from reaching our parent (an explored_state). Our 10349 * own state will get the read marks recorded, but 10350 * they'll be immediately forgotten as we're pruning 10351 * this state and will pop a new one. 10352 */ 10353 err = propagate_liveness(env, &sl->state, cur); 10354 10355 /* if previous state reached the exit with precision and 10356 * current state is equivalent to it (except precsion marks) 10357 * the precision needs to be propagated back in 10358 * the current state. 10359 */ 10360 err = err ? : push_jmp_history(env, cur); 10361 err = err ? : propagate_precision(env, &sl->state); 10362 if (err) 10363 return err; 10364 return 1; 10365 } 10366 miss: 10367 /* when new state is not going to be added do not increase miss count. 10368 * Otherwise several loop iterations will remove the state 10369 * recorded earlier. The goal of these heuristics is to have 10370 * states from some iterations of the loop (some in the beginning 10371 * and some at the end) to help pruning. 10372 */ 10373 if (add_new_state) 10374 sl->miss_cnt++; 10375 /* heuristic to determine whether this state is beneficial 10376 * to keep checking from state equivalence point of view. 10377 * Higher numbers increase max_states_per_insn and verification time, 10378 * but do not meaningfully decrease insn_processed. 10379 */ 10380 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 10381 /* the state is unlikely to be useful. Remove it to 10382 * speed up verification 10383 */ 10384 *pprev = sl->next; 10385 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 10386 u32 br = sl->state.branches; 10387 10388 WARN_ONCE(br, 10389 "BUG live_done but branches_to_explore %d\n", 10390 br); 10391 free_verifier_state(&sl->state, false); 10392 kfree(sl); 10393 env->peak_states--; 10394 } else { 10395 /* cannot free this state, since parentage chain may 10396 * walk it later. Add it for free_list instead to 10397 * be freed at the end of verification 10398 */ 10399 sl->next = env->free_list; 10400 env->free_list = sl; 10401 } 10402 sl = *pprev; 10403 continue; 10404 } 10405 next: 10406 pprev = &sl->next; 10407 sl = *pprev; 10408 } 10409 10410 if (env->max_states_per_insn < states_cnt) 10411 env->max_states_per_insn = states_cnt; 10412 10413 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 10414 return push_jmp_history(env, cur); 10415 10416 if (!add_new_state) 10417 return push_jmp_history(env, cur); 10418 10419 /* There were no equivalent states, remember the current one. 10420 * Technically the current state is not proven to be safe yet, 10421 * but it will either reach outer most bpf_exit (which means it's safe) 10422 * or it will be rejected. When there are no loops the verifier won't be 10423 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 10424 * again on the way to bpf_exit. 10425 * When looping the sl->state.branches will be > 0 and this state 10426 * will not be considered for equivalence until branches == 0. 10427 */ 10428 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 10429 if (!new_sl) 10430 return -ENOMEM; 10431 env->total_states++; 10432 env->peak_states++; 10433 env->prev_jmps_processed = env->jmps_processed; 10434 env->prev_insn_processed = env->insn_processed; 10435 10436 /* add new state to the head of linked list */ 10437 new = &new_sl->state; 10438 err = copy_verifier_state(new, cur); 10439 if (err) { 10440 free_verifier_state(new, false); 10441 kfree(new_sl); 10442 return err; 10443 } 10444 new->insn_idx = insn_idx; 10445 WARN_ONCE(new->branches != 1, 10446 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 10447 10448 cur->parent = new; 10449 cur->first_insn_idx = insn_idx; 10450 clear_jmp_history(cur); 10451 new_sl->next = *explored_state(env, insn_idx); 10452 *explored_state(env, insn_idx) = new_sl; 10453 /* connect new state to parentage chain. Current frame needs all 10454 * registers connected. Only r6 - r9 of the callers are alive (pushed 10455 * to the stack implicitly by JITs) so in callers' frames connect just 10456 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 10457 * the state of the call instruction (with WRITTEN set), and r0 comes 10458 * from callee with its full parentage chain, anyway. 10459 */ 10460 /* clear write marks in current state: the writes we did are not writes 10461 * our child did, so they don't screen off its reads from us. 10462 * (There are no read marks in current state, because reads always mark 10463 * their parent and current state never has children yet. Only 10464 * explored_states can get read marks.) 10465 */ 10466 for (j = 0; j <= cur->curframe; j++) { 10467 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 10468 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 10469 for (i = 0; i < BPF_REG_FP; i++) 10470 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 10471 } 10472 10473 /* all stack frames are accessible from callee, clear them all */ 10474 for (j = 0; j <= cur->curframe; j++) { 10475 struct bpf_func_state *frame = cur->frame[j]; 10476 struct bpf_func_state *newframe = new->frame[j]; 10477 10478 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 10479 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 10480 frame->stack[i].spilled_ptr.parent = 10481 &newframe->stack[i].spilled_ptr; 10482 } 10483 } 10484 return 0; 10485 } 10486 10487 /* Return true if it's OK to have the same insn return a different type. */ 10488 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 10489 { 10490 switch (type) { 10491 case PTR_TO_CTX: 10492 case PTR_TO_SOCKET: 10493 case PTR_TO_SOCKET_OR_NULL: 10494 case PTR_TO_SOCK_COMMON: 10495 case PTR_TO_SOCK_COMMON_OR_NULL: 10496 case PTR_TO_TCP_SOCK: 10497 case PTR_TO_TCP_SOCK_OR_NULL: 10498 case PTR_TO_XDP_SOCK: 10499 case PTR_TO_BTF_ID: 10500 case PTR_TO_BTF_ID_OR_NULL: 10501 return false; 10502 default: 10503 return true; 10504 } 10505 } 10506 10507 /* If an instruction was previously used with particular pointer types, then we 10508 * need to be careful to avoid cases such as the below, where it may be ok 10509 * for one branch accessing the pointer, but not ok for the other branch: 10510 * 10511 * R1 = sock_ptr 10512 * goto X; 10513 * ... 10514 * R1 = some_other_valid_ptr; 10515 * goto X; 10516 * ... 10517 * R2 = *(u32 *)(R1 + 0); 10518 */ 10519 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 10520 { 10521 return src != prev && (!reg_type_mismatch_ok(src) || 10522 !reg_type_mismatch_ok(prev)); 10523 } 10524 10525 static int do_check(struct bpf_verifier_env *env) 10526 { 10527 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 10528 struct bpf_verifier_state *state = env->cur_state; 10529 struct bpf_insn *insns = env->prog->insnsi; 10530 struct bpf_reg_state *regs; 10531 int insn_cnt = env->prog->len; 10532 bool do_print_state = false; 10533 int prev_insn_idx = -1; 10534 10535 for (;;) { 10536 struct bpf_insn *insn; 10537 u8 class; 10538 int err; 10539 10540 env->prev_insn_idx = prev_insn_idx; 10541 if (env->insn_idx >= insn_cnt) { 10542 verbose(env, "invalid insn idx %d insn_cnt %d\n", 10543 env->insn_idx, insn_cnt); 10544 return -EFAULT; 10545 } 10546 10547 insn = &insns[env->insn_idx]; 10548 class = BPF_CLASS(insn->code); 10549 10550 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 10551 verbose(env, 10552 "BPF program is too large. Processed %d insn\n", 10553 env->insn_processed); 10554 return -E2BIG; 10555 } 10556 10557 err = is_state_visited(env, env->insn_idx); 10558 if (err < 0) 10559 return err; 10560 if (err == 1) { 10561 /* found equivalent state, can prune the search */ 10562 if (env->log.level & BPF_LOG_LEVEL) { 10563 if (do_print_state) 10564 verbose(env, "\nfrom %d to %d%s: safe\n", 10565 env->prev_insn_idx, env->insn_idx, 10566 env->cur_state->speculative ? 10567 " (speculative execution)" : ""); 10568 else 10569 verbose(env, "%d: safe\n", env->insn_idx); 10570 } 10571 goto process_bpf_exit; 10572 } 10573 10574 if (signal_pending(current)) 10575 return -EAGAIN; 10576 10577 if (need_resched()) 10578 cond_resched(); 10579 10580 if (env->log.level & BPF_LOG_LEVEL2 || 10581 (env->log.level & BPF_LOG_LEVEL && do_print_state)) { 10582 if (env->log.level & BPF_LOG_LEVEL2) 10583 verbose(env, "%d:", env->insn_idx); 10584 else 10585 verbose(env, "\nfrom %d to %d%s:", 10586 env->prev_insn_idx, env->insn_idx, 10587 env->cur_state->speculative ? 10588 " (speculative execution)" : ""); 10589 print_verifier_state(env, state->frame[state->curframe]); 10590 do_print_state = false; 10591 } 10592 10593 if (env->log.level & BPF_LOG_LEVEL) { 10594 const struct bpf_insn_cbs cbs = { 10595 .cb_call = disasm_kfunc_name, 10596 .cb_print = verbose, 10597 .private_data = env, 10598 }; 10599 10600 verbose_linfo(env, env->insn_idx, "; "); 10601 verbose(env, "%d: ", env->insn_idx); 10602 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 10603 } 10604 10605 if (bpf_prog_is_dev_bound(env->prog->aux)) { 10606 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 10607 env->prev_insn_idx); 10608 if (err) 10609 return err; 10610 } 10611 10612 regs = cur_regs(env); 10613 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 10614 prev_insn_idx = env->insn_idx; 10615 10616 if (class == BPF_ALU || class == BPF_ALU64) { 10617 err = check_alu_op(env, insn); 10618 if (err) 10619 return err; 10620 10621 } else if (class == BPF_LDX) { 10622 enum bpf_reg_type *prev_src_type, src_reg_type; 10623 10624 /* check for reserved fields is already done */ 10625 10626 /* check src operand */ 10627 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10628 if (err) 10629 return err; 10630 10631 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 10632 if (err) 10633 return err; 10634 10635 src_reg_type = regs[insn->src_reg].type; 10636 10637 /* check that memory (src_reg + off) is readable, 10638 * the state of dst_reg will be updated by this func 10639 */ 10640 err = check_mem_access(env, env->insn_idx, insn->src_reg, 10641 insn->off, BPF_SIZE(insn->code), 10642 BPF_READ, insn->dst_reg, false); 10643 if (err) 10644 return err; 10645 10646 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 10647 10648 if (*prev_src_type == NOT_INIT) { 10649 /* saw a valid insn 10650 * dst_reg = *(u32 *)(src_reg + off) 10651 * save type to validate intersecting paths 10652 */ 10653 *prev_src_type = src_reg_type; 10654 10655 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 10656 /* ABuser program is trying to use the same insn 10657 * dst_reg = *(u32*) (src_reg + off) 10658 * with different pointer types: 10659 * src_reg == ctx in one branch and 10660 * src_reg == stack|map in some other branch. 10661 * Reject it. 10662 */ 10663 verbose(env, "same insn cannot be used with different pointers\n"); 10664 return -EINVAL; 10665 } 10666 10667 } else if (class == BPF_STX) { 10668 enum bpf_reg_type *prev_dst_type, dst_reg_type; 10669 10670 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 10671 err = check_atomic(env, env->insn_idx, insn); 10672 if (err) 10673 return err; 10674 env->insn_idx++; 10675 continue; 10676 } 10677 10678 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 10679 verbose(env, "BPF_STX uses reserved fields\n"); 10680 return -EINVAL; 10681 } 10682 10683 /* check src1 operand */ 10684 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10685 if (err) 10686 return err; 10687 /* check src2 operand */ 10688 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 10689 if (err) 10690 return err; 10691 10692 dst_reg_type = regs[insn->dst_reg].type; 10693 10694 /* check that memory (dst_reg + off) is writeable */ 10695 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 10696 insn->off, BPF_SIZE(insn->code), 10697 BPF_WRITE, insn->src_reg, false); 10698 if (err) 10699 return err; 10700 10701 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 10702 10703 if (*prev_dst_type == NOT_INIT) { 10704 *prev_dst_type = dst_reg_type; 10705 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 10706 verbose(env, "same insn cannot be used with different pointers\n"); 10707 return -EINVAL; 10708 } 10709 10710 } else if (class == BPF_ST) { 10711 if (BPF_MODE(insn->code) != BPF_MEM || 10712 insn->src_reg != BPF_REG_0) { 10713 verbose(env, "BPF_ST uses reserved fields\n"); 10714 return -EINVAL; 10715 } 10716 /* check src operand */ 10717 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 10718 if (err) 10719 return err; 10720 10721 if (is_ctx_reg(env, insn->dst_reg)) { 10722 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 10723 insn->dst_reg, 10724 reg_type_str[reg_state(env, insn->dst_reg)->type]); 10725 return -EACCES; 10726 } 10727 10728 /* check that memory (dst_reg + off) is writeable */ 10729 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 10730 insn->off, BPF_SIZE(insn->code), 10731 BPF_WRITE, -1, false); 10732 if (err) 10733 return err; 10734 10735 } else if (class == BPF_JMP || class == BPF_JMP32) { 10736 u8 opcode = BPF_OP(insn->code); 10737 10738 env->jmps_processed++; 10739 if (opcode == BPF_CALL) { 10740 if (BPF_SRC(insn->code) != BPF_K || 10741 insn->off != 0 || 10742 (insn->src_reg != BPF_REG_0 && 10743 insn->src_reg != BPF_PSEUDO_CALL && 10744 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 10745 insn->dst_reg != BPF_REG_0 || 10746 class == BPF_JMP32) { 10747 verbose(env, "BPF_CALL uses reserved fields\n"); 10748 return -EINVAL; 10749 } 10750 10751 if (env->cur_state->active_spin_lock && 10752 (insn->src_reg == BPF_PSEUDO_CALL || 10753 insn->imm != BPF_FUNC_spin_unlock)) { 10754 verbose(env, "function calls are not allowed while holding a lock\n"); 10755 return -EINVAL; 10756 } 10757 if (insn->src_reg == BPF_PSEUDO_CALL) 10758 err = check_func_call(env, insn, &env->insn_idx); 10759 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 10760 err = check_kfunc_call(env, insn); 10761 else 10762 err = check_helper_call(env, insn, &env->insn_idx); 10763 if (err) 10764 return err; 10765 } else if (opcode == BPF_JA) { 10766 if (BPF_SRC(insn->code) != BPF_K || 10767 insn->imm != 0 || 10768 insn->src_reg != BPF_REG_0 || 10769 insn->dst_reg != BPF_REG_0 || 10770 class == BPF_JMP32) { 10771 verbose(env, "BPF_JA uses reserved fields\n"); 10772 return -EINVAL; 10773 } 10774 10775 env->insn_idx += insn->off + 1; 10776 continue; 10777 10778 } else if (opcode == BPF_EXIT) { 10779 if (BPF_SRC(insn->code) != BPF_K || 10780 insn->imm != 0 || 10781 insn->src_reg != BPF_REG_0 || 10782 insn->dst_reg != BPF_REG_0 || 10783 class == BPF_JMP32) { 10784 verbose(env, "BPF_EXIT uses reserved fields\n"); 10785 return -EINVAL; 10786 } 10787 10788 if (env->cur_state->active_spin_lock) { 10789 verbose(env, "bpf_spin_unlock is missing\n"); 10790 return -EINVAL; 10791 } 10792 10793 if (state->curframe) { 10794 /* exit from nested function */ 10795 err = prepare_func_exit(env, &env->insn_idx); 10796 if (err) 10797 return err; 10798 do_print_state = true; 10799 continue; 10800 } 10801 10802 err = check_reference_leak(env); 10803 if (err) 10804 return err; 10805 10806 err = check_return_code(env); 10807 if (err) 10808 return err; 10809 process_bpf_exit: 10810 update_branch_counts(env, env->cur_state); 10811 err = pop_stack(env, &prev_insn_idx, 10812 &env->insn_idx, pop_log); 10813 if (err < 0) { 10814 if (err != -ENOENT) 10815 return err; 10816 break; 10817 } else { 10818 do_print_state = true; 10819 continue; 10820 } 10821 } else { 10822 err = check_cond_jmp_op(env, insn, &env->insn_idx); 10823 if (err) 10824 return err; 10825 } 10826 } else if (class == BPF_LD) { 10827 u8 mode = BPF_MODE(insn->code); 10828 10829 if (mode == BPF_ABS || mode == BPF_IND) { 10830 err = check_ld_abs(env, insn); 10831 if (err) 10832 return err; 10833 10834 } else if (mode == BPF_IMM) { 10835 err = check_ld_imm(env, insn); 10836 if (err) 10837 return err; 10838 10839 env->insn_idx++; 10840 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 10841 } else { 10842 verbose(env, "invalid BPF_LD mode\n"); 10843 return -EINVAL; 10844 } 10845 } else { 10846 verbose(env, "unknown insn class %d\n", class); 10847 return -EINVAL; 10848 } 10849 10850 env->insn_idx++; 10851 } 10852 10853 return 0; 10854 } 10855 10856 static int find_btf_percpu_datasec(struct btf *btf) 10857 { 10858 const struct btf_type *t; 10859 const char *tname; 10860 int i, n; 10861 10862 /* 10863 * Both vmlinux and module each have their own ".data..percpu" 10864 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 10865 * types to look at only module's own BTF types. 10866 */ 10867 n = btf_nr_types(btf); 10868 if (btf_is_module(btf)) 10869 i = btf_nr_types(btf_vmlinux); 10870 else 10871 i = 1; 10872 10873 for(; i < n; i++) { 10874 t = btf_type_by_id(btf, i); 10875 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 10876 continue; 10877 10878 tname = btf_name_by_offset(btf, t->name_off); 10879 if (!strcmp(tname, ".data..percpu")) 10880 return i; 10881 } 10882 10883 return -ENOENT; 10884 } 10885 10886 /* replace pseudo btf_id with kernel symbol address */ 10887 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 10888 struct bpf_insn *insn, 10889 struct bpf_insn_aux_data *aux) 10890 { 10891 const struct btf_var_secinfo *vsi; 10892 const struct btf_type *datasec; 10893 struct btf_mod_pair *btf_mod; 10894 const struct btf_type *t; 10895 const char *sym_name; 10896 bool percpu = false; 10897 u32 type, id = insn->imm; 10898 struct btf *btf; 10899 s32 datasec_id; 10900 u64 addr; 10901 int i, btf_fd, err; 10902 10903 btf_fd = insn[1].imm; 10904 if (btf_fd) { 10905 btf = btf_get_by_fd(btf_fd); 10906 if (IS_ERR(btf)) { 10907 verbose(env, "invalid module BTF object FD specified.\n"); 10908 return -EINVAL; 10909 } 10910 } else { 10911 if (!btf_vmlinux) { 10912 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 10913 return -EINVAL; 10914 } 10915 btf = btf_vmlinux; 10916 btf_get(btf); 10917 } 10918 10919 t = btf_type_by_id(btf, id); 10920 if (!t) { 10921 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 10922 err = -ENOENT; 10923 goto err_put; 10924 } 10925 10926 if (!btf_type_is_var(t)) { 10927 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); 10928 err = -EINVAL; 10929 goto err_put; 10930 } 10931 10932 sym_name = btf_name_by_offset(btf, t->name_off); 10933 addr = kallsyms_lookup_name(sym_name); 10934 if (!addr) { 10935 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 10936 sym_name); 10937 err = -ENOENT; 10938 goto err_put; 10939 } 10940 10941 datasec_id = find_btf_percpu_datasec(btf); 10942 if (datasec_id > 0) { 10943 datasec = btf_type_by_id(btf, datasec_id); 10944 for_each_vsi(i, datasec, vsi) { 10945 if (vsi->type == id) { 10946 percpu = true; 10947 break; 10948 } 10949 } 10950 } 10951 10952 insn[0].imm = (u32)addr; 10953 insn[1].imm = addr >> 32; 10954 10955 type = t->type; 10956 t = btf_type_skip_modifiers(btf, type, NULL); 10957 if (percpu) { 10958 aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID; 10959 aux->btf_var.btf = btf; 10960 aux->btf_var.btf_id = type; 10961 } else if (!btf_type_is_struct(t)) { 10962 const struct btf_type *ret; 10963 const char *tname; 10964 u32 tsize; 10965 10966 /* resolve the type size of ksym. */ 10967 ret = btf_resolve_size(btf, t, &tsize); 10968 if (IS_ERR(ret)) { 10969 tname = btf_name_by_offset(btf, t->name_off); 10970 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 10971 tname, PTR_ERR(ret)); 10972 err = -EINVAL; 10973 goto err_put; 10974 } 10975 aux->btf_var.reg_type = PTR_TO_MEM; 10976 aux->btf_var.mem_size = tsize; 10977 } else { 10978 aux->btf_var.reg_type = PTR_TO_BTF_ID; 10979 aux->btf_var.btf = btf; 10980 aux->btf_var.btf_id = type; 10981 } 10982 10983 /* check whether we recorded this BTF (and maybe module) already */ 10984 for (i = 0; i < env->used_btf_cnt; i++) { 10985 if (env->used_btfs[i].btf == btf) { 10986 btf_put(btf); 10987 return 0; 10988 } 10989 } 10990 10991 if (env->used_btf_cnt >= MAX_USED_BTFS) { 10992 err = -E2BIG; 10993 goto err_put; 10994 } 10995 10996 btf_mod = &env->used_btfs[env->used_btf_cnt]; 10997 btf_mod->btf = btf; 10998 btf_mod->module = NULL; 10999 11000 /* if we reference variables from kernel module, bump its refcount */ 11001 if (btf_is_module(btf)) { 11002 btf_mod->module = btf_try_get_module(btf); 11003 if (!btf_mod->module) { 11004 err = -ENXIO; 11005 goto err_put; 11006 } 11007 } 11008 11009 env->used_btf_cnt++; 11010 11011 return 0; 11012 err_put: 11013 btf_put(btf); 11014 return err; 11015 } 11016 11017 static int check_map_prealloc(struct bpf_map *map) 11018 { 11019 return (map->map_type != BPF_MAP_TYPE_HASH && 11020 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 11021 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || 11022 !(map->map_flags & BPF_F_NO_PREALLOC); 11023 } 11024 11025 static bool is_tracing_prog_type(enum bpf_prog_type type) 11026 { 11027 switch (type) { 11028 case BPF_PROG_TYPE_KPROBE: 11029 case BPF_PROG_TYPE_TRACEPOINT: 11030 case BPF_PROG_TYPE_PERF_EVENT: 11031 case BPF_PROG_TYPE_RAW_TRACEPOINT: 11032 return true; 11033 default: 11034 return false; 11035 } 11036 } 11037 11038 static bool is_preallocated_map(struct bpf_map *map) 11039 { 11040 if (!check_map_prealloc(map)) 11041 return false; 11042 if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) 11043 return false; 11044 return true; 11045 } 11046 11047 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 11048 struct bpf_map *map, 11049 struct bpf_prog *prog) 11050 11051 { 11052 enum bpf_prog_type prog_type = resolve_prog_type(prog); 11053 /* 11054 * Validate that trace type programs use preallocated hash maps. 11055 * 11056 * For programs attached to PERF events this is mandatory as the 11057 * perf NMI can hit any arbitrary code sequence. 11058 * 11059 * All other trace types using preallocated hash maps are unsafe as 11060 * well because tracepoint or kprobes can be inside locked regions 11061 * of the memory allocator or at a place where a recursion into the 11062 * memory allocator would see inconsistent state. 11063 * 11064 * On RT enabled kernels run-time allocation of all trace type 11065 * programs is strictly prohibited due to lock type constraints. On 11066 * !RT kernels it is allowed for backwards compatibility reasons for 11067 * now, but warnings are emitted so developers are made aware of 11068 * the unsafety and can fix their programs before this is enforced. 11069 */ 11070 if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { 11071 if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { 11072 verbose(env, "perf_event programs can only use preallocated hash map\n"); 11073 return -EINVAL; 11074 } 11075 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 11076 verbose(env, "trace type programs can only use preallocated hash map\n"); 11077 return -EINVAL; 11078 } 11079 WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); 11080 verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); 11081 } 11082 11083 if (map_value_has_spin_lock(map)) { 11084 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 11085 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 11086 return -EINVAL; 11087 } 11088 11089 if (is_tracing_prog_type(prog_type)) { 11090 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 11091 return -EINVAL; 11092 } 11093 11094 if (prog->aux->sleepable) { 11095 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 11096 return -EINVAL; 11097 } 11098 } 11099 11100 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 11101 !bpf_offload_prog_map_match(prog, map)) { 11102 verbose(env, "offload device mismatch between prog and map\n"); 11103 return -EINVAL; 11104 } 11105 11106 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 11107 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 11108 return -EINVAL; 11109 } 11110 11111 if (prog->aux->sleepable) 11112 switch (map->map_type) { 11113 case BPF_MAP_TYPE_HASH: 11114 case BPF_MAP_TYPE_LRU_HASH: 11115 case BPF_MAP_TYPE_ARRAY: 11116 case BPF_MAP_TYPE_PERCPU_HASH: 11117 case BPF_MAP_TYPE_PERCPU_ARRAY: 11118 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 11119 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 11120 case BPF_MAP_TYPE_HASH_OF_MAPS: 11121 if (!is_preallocated_map(map)) { 11122 verbose(env, 11123 "Sleepable programs can only use preallocated maps\n"); 11124 return -EINVAL; 11125 } 11126 break; 11127 case BPF_MAP_TYPE_RINGBUF: 11128 break; 11129 default: 11130 verbose(env, 11131 "Sleepable programs can only use array, hash, and ringbuf maps\n"); 11132 return -EINVAL; 11133 } 11134 11135 return 0; 11136 } 11137 11138 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 11139 { 11140 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 11141 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 11142 } 11143 11144 /* find and rewrite pseudo imm in ld_imm64 instructions: 11145 * 11146 * 1. if it accesses map FD, replace it with actual map pointer. 11147 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 11148 * 11149 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 11150 */ 11151 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 11152 { 11153 struct bpf_insn *insn = env->prog->insnsi; 11154 int insn_cnt = env->prog->len; 11155 int i, j, err; 11156 11157 err = bpf_prog_calc_tag(env->prog); 11158 if (err) 11159 return err; 11160 11161 for (i = 0; i < insn_cnt; i++, insn++) { 11162 if (BPF_CLASS(insn->code) == BPF_LDX && 11163 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 11164 verbose(env, "BPF_LDX uses reserved fields\n"); 11165 return -EINVAL; 11166 } 11167 11168 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 11169 struct bpf_insn_aux_data *aux; 11170 struct bpf_map *map; 11171 struct fd f; 11172 u64 addr; 11173 11174 if (i == insn_cnt - 1 || insn[1].code != 0 || 11175 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 11176 insn[1].off != 0) { 11177 verbose(env, "invalid bpf_ld_imm64 insn\n"); 11178 return -EINVAL; 11179 } 11180 11181 if (insn[0].src_reg == 0) 11182 /* valid generic load 64-bit imm */ 11183 goto next_insn; 11184 11185 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 11186 aux = &env->insn_aux_data[i]; 11187 err = check_pseudo_btf_id(env, insn, aux); 11188 if (err) 11189 return err; 11190 goto next_insn; 11191 } 11192 11193 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 11194 aux = &env->insn_aux_data[i]; 11195 aux->ptr_type = PTR_TO_FUNC; 11196 goto next_insn; 11197 } 11198 11199 /* In final convert_pseudo_ld_imm64() step, this is 11200 * converted into regular 64-bit imm load insn. 11201 */ 11202 if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && 11203 insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || 11204 (insn[0].src_reg == BPF_PSEUDO_MAP_FD && 11205 insn[1].imm != 0)) { 11206 verbose(env, 11207 "unrecognized bpf_ld_imm64 insn\n"); 11208 return -EINVAL; 11209 } 11210 11211 f = fdget(insn[0].imm); 11212 map = __bpf_map_get(f); 11213 if (IS_ERR(map)) { 11214 verbose(env, "fd %d is not pointing to valid bpf_map\n", 11215 insn[0].imm); 11216 return PTR_ERR(map); 11217 } 11218 11219 err = check_map_prog_compatibility(env, map, env->prog); 11220 if (err) { 11221 fdput(f); 11222 return err; 11223 } 11224 11225 aux = &env->insn_aux_data[i]; 11226 if (insn->src_reg == BPF_PSEUDO_MAP_FD) { 11227 addr = (unsigned long)map; 11228 } else { 11229 u32 off = insn[1].imm; 11230 11231 if (off >= BPF_MAX_VAR_OFF) { 11232 verbose(env, "direct value offset of %u is not allowed\n", off); 11233 fdput(f); 11234 return -EINVAL; 11235 } 11236 11237 if (!map->ops->map_direct_value_addr) { 11238 verbose(env, "no direct value access support for this map type\n"); 11239 fdput(f); 11240 return -EINVAL; 11241 } 11242 11243 err = map->ops->map_direct_value_addr(map, &addr, off); 11244 if (err) { 11245 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 11246 map->value_size, off); 11247 fdput(f); 11248 return err; 11249 } 11250 11251 aux->map_off = off; 11252 addr += off; 11253 } 11254 11255 insn[0].imm = (u32)addr; 11256 insn[1].imm = addr >> 32; 11257 11258 /* check whether we recorded this map already */ 11259 for (j = 0; j < env->used_map_cnt; j++) { 11260 if (env->used_maps[j] == map) { 11261 aux->map_index = j; 11262 fdput(f); 11263 goto next_insn; 11264 } 11265 } 11266 11267 if (env->used_map_cnt >= MAX_USED_MAPS) { 11268 fdput(f); 11269 return -E2BIG; 11270 } 11271 11272 /* hold the map. If the program is rejected by verifier, 11273 * the map will be released by release_maps() or it 11274 * will be used by the valid program until it's unloaded 11275 * and all maps are released in free_used_maps() 11276 */ 11277 bpf_map_inc(map); 11278 11279 aux->map_index = env->used_map_cnt; 11280 env->used_maps[env->used_map_cnt++] = map; 11281 11282 if (bpf_map_is_cgroup_storage(map) && 11283 bpf_cgroup_storage_assign(env->prog->aux, map)) { 11284 verbose(env, "only one cgroup storage of each type is allowed\n"); 11285 fdput(f); 11286 return -EBUSY; 11287 } 11288 11289 fdput(f); 11290 next_insn: 11291 insn++; 11292 i++; 11293 continue; 11294 } 11295 11296 /* Basic sanity check before we invest more work here. */ 11297 if (!bpf_opcode_in_insntable(insn->code)) { 11298 verbose(env, "unknown opcode %02x\n", insn->code); 11299 return -EINVAL; 11300 } 11301 } 11302 11303 /* now all pseudo BPF_LD_IMM64 instructions load valid 11304 * 'struct bpf_map *' into a register instead of user map_fd. 11305 * These pointers will be used later by verifier to validate map access. 11306 */ 11307 return 0; 11308 } 11309 11310 /* drop refcnt of maps used by the rejected program */ 11311 static void release_maps(struct bpf_verifier_env *env) 11312 { 11313 __bpf_free_used_maps(env->prog->aux, env->used_maps, 11314 env->used_map_cnt); 11315 } 11316 11317 /* drop refcnt of maps used by the rejected program */ 11318 static void release_btfs(struct bpf_verifier_env *env) 11319 { 11320 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 11321 env->used_btf_cnt); 11322 } 11323 11324 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 11325 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 11326 { 11327 struct bpf_insn *insn = env->prog->insnsi; 11328 int insn_cnt = env->prog->len; 11329 int i; 11330 11331 for (i = 0; i < insn_cnt; i++, insn++) { 11332 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 11333 continue; 11334 if (insn->src_reg == BPF_PSEUDO_FUNC) 11335 continue; 11336 insn->src_reg = 0; 11337 } 11338 } 11339 11340 /* single env->prog->insni[off] instruction was replaced with the range 11341 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 11342 * [0, off) and [off, end) to new locations, so the patched range stays zero 11343 */ 11344 static int adjust_insn_aux_data(struct bpf_verifier_env *env, 11345 struct bpf_prog *new_prog, u32 off, u32 cnt) 11346 { 11347 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; 11348 struct bpf_insn *insn = new_prog->insnsi; 11349 u32 prog_len; 11350 int i; 11351 11352 /* aux info at OFF always needs adjustment, no matter fast path 11353 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 11354 * original insn at old prog. 11355 */ 11356 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 11357 11358 if (cnt == 1) 11359 return 0; 11360 prog_len = new_prog->len; 11361 new_data = vzalloc(array_size(prog_len, 11362 sizeof(struct bpf_insn_aux_data))); 11363 if (!new_data) 11364 return -ENOMEM; 11365 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 11366 memcpy(new_data + off + cnt - 1, old_data + off, 11367 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 11368 for (i = off; i < off + cnt - 1; i++) { 11369 new_data[i].seen = env->pass_cnt; 11370 new_data[i].zext_dst = insn_has_def32(env, insn + i); 11371 } 11372 env->insn_aux_data = new_data; 11373 vfree(old_data); 11374 return 0; 11375 } 11376 11377 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 11378 { 11379 int i; 11380 11381 if (len == 1) 11382 return; 11383 /* NOTE: fake 'exit' subprog should be updated as well. */ 11384 for (i = 0; i <= env->subprog_cnt; i++) { 11385 if (env->subprog_info[i].start <= off) 11386 continue; 11387 env->subprog_info[i].start += len - 1; 11388 } 11389 } 11390 11391 static void adjust_poke_descs(struct bpf_prog *prog, u32 len) 11392 { 11393 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 11394 int i, sz = prog->aux->size_poke_tab; 11395 struct bpf_jit_poke_descriptor *desc; 11396 11397 for (i = 0; i < sz; i++) { 11398 desc = &tab[i]; 11399 desc->insn_idx += len - 1; 11400 } 11401 } 11402 11403 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 11404 const struct bpf_insn *patch, u32 len) 11405 { 11406 struct bpf_prog *new_prog; 11407 11408 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 11409 if (IS_ERR(new_prog)) { 11410 if (PTR_ERR(new_prog) == -ERANGE) 11411 verbose(env, 11412 "insn %d cannot be patched due to 16-bit range\n", 11413 env->insn_aux_data[off].orig_idx); 11414 return NULL; 11415 } 11416 if (adjust_insn_aux_data(env, new_prog, off, len)) 11417 return NULL; 11418 adjust_subprog_starts(env, off, len); 11419 adjust_poke_descs(new_prog, len); 11420 return new_prog; 11421 } 11422 11423 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 11424 u32 off, u32 cnt) 11425 { 11426 int i, j; 11427 11428 /* find first prog starting at or after off (first to remove) */ 11429 for (i = 0; i < env->subprog_cnt; i++) 11430 if (env->subprog_info[i].start >= off) 11431 break; 11432 /* find first prog starting at or after off + cnt (first to stay) */ 11433 for (j = i; j < env->subprog_cnt; j++) 11434 if (env->subprog_info[j].start >= off + cnt) 11435 break; 11436 /* if j doesn't start exactly at off + cnt, we are just removing 11437 * the front of previous prog 11438 */ 11439 if (env->subprog_info[j].start != off + cnt) 11440 j--; 11441 11442 if (j > i) { 11443 struct bpf_prog_aux *aux = env->prog->aux; 11444 int move; 11445 11446 /* move fake 'exit' subprog as well */ 11447 move = env->subprog_cnt + 1 - j; 11448 11449 memmove(env->subprog_info + i, 11450 env->subprog_info + j, 11451 sizeof(*env->subprog_info) * move); 11452 env->subprog_cnt -= j - i; 11453 11454 /* remove func_info */ 11455 if (aux->func_info) { 11456 move = aux->func_info_cnt - j; 11457 11458 memmove(aux->func_info + i, 11459 aux->func_info + j, 11460 sizeof(*aux->func_info) * move); 11461 aux->func_info_cnt -= j - i; 11462 /* func_info->insn_off is set after all code rewrites, 11463 * in adjust_btf_func() - no need to adjust 11464 */ 11465 } 11466 } else { 11467 /* convert i from "first prog to remove" to "first to adjust" */ 11468 if (env->subprog_info[i].start == off) 11469 i++; 11470 } 11471 11472 /* update fake 'exit' subprog as well */ 11473 for (; i <= env->subprog_cnt; i++) 11474 env->subprog_info[i].start -= cnt; 11475 11476 return 0; 11477 } 11478 11479 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 11480 u32 cnt) 11481 { 11482 struct bpf_prog *prog = env->prog; 11483 u32 i, l_off, l_cnt, nr_linfo; 11484 struct bpf_line_info *linfo; 11485 11486 nr_linfo = prog->aux->nr_linfo; 11487 if (!nr_linfo) 11488 return 0; 11489 11490 linfo = prog->aux->linfo; 11491 11492 /* find first line info to remove, count lines to be removed */ 11493 for (i = 0; i < nr_linfo; i++) 11494 if (linfo[i].insn_off >= off) 11495 break; 11496 11497 l_off = i; 11498 l_cnt = 0; 11499 for (; i < nr_linfo; i++) 11500 if (linfo[i].insn_off < off + cnt) 11501 l_cnt++; 11502 else 11503 break; 11504 11505 /* First live insn doesn't match first live linfo, it needs to "inherit" 11506 * last removed linfo. prog is already modified, so prog->len == off 11507 * means no live instructions after (tail of the program was removed). 11508 */ 11509 if (prog->len != off && l_cnt && 11510 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 11511 l_cnt--; 11512 linfo[--i].insn_off = off + cnt; 11513 } 11514 11515 /* remove the line info which refer to the removed instructions */ 11516 if (l_cnt) { 11517 memmove(linfo + l_off, linfo + i, 11518 sizeof(*linfo) * (nr_linfo - i)); 11519 11520 prog->aux->nr_linfo -= l_cnt; 11521 nr_linfo = prog->aux->nr_linfo; 11522 } 11523 11524 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 11525 for (i = l_off; i < nr_linfo; i++) 11526 linfo[i].insn_off -= cnt; 11527 11528 /* fix up all subprogs (incl. 'exit') which start >= off */ 11529 for (i = 0; i <= env->subprog_cnt; i++) 11530 if (env->subprog_info[i].linfo_idx > l_off) { 11531 /* program may have started in the removed region but 11532 * may not be fully removed 11533 */ 11534 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 11535 env->subprog_info[i].linfo_idx -= l_cnt; 11536 else 11537 env->subprog_info[i].linfo_idx = l_off; 11538 } 11539 11540 return 0; 11541 } 11542 11543 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 11544 { 11545 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11546 unsigned int orig_prog_len = env->prog->len; 11547 int err; 11548 11549 if (bpf_prog_is_dev_bound(env->prog->aux)) 11550 bpf_prog_offload_remove_insns(env, off, cnt); 11551 11552 err = bpf_remove_insns(env->prog, off, cnt); 11553 if (err) 11554 return err; 11555 11556 err = adjust_subprog_starts_after_remove(env, off, cnt); 11557 if (err) 11558 return err; 11559 11560 err = bpf_adj_linfo_after_remove(env, off, cnt); 11561 if (err) 11562 return err; 11563 11564 memmove(aux_data + off, aux_data + off + cnt, 11565 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 11566 11567 return 0; 11568 } 11569 11570 /* The verifier does more data flow analysis than llvm and will not 11571 * explore branches that are dead at run time. Malicious programs can 11572 * have dead code too. Therefore replace all dead at-run-time code 11573 * with 'ja -1'. 11574 * 11575 * Just nops are not optimal, e.g. if they would sit at the end of the 11576 * program and through another bug we would manage to jump there, then 11577 * we'd execute beyond program memory otherwise. Returning exception 11578 * code also wouldn't work since we can have subprogs where the dead 11579 * code could be located. 11580 */ 11581 static void sanitize_dead_code(struct bpf_verifier_env *env) 11582 { 11583 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11584 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 11585 struct bpf_insn *insn = env->prog->insnsi; 11586 const int insn_cnt = env->prog->len; 11587 int i; 11588 11589 for (i = 0; i < insn_cnt; i++) { 11590 if (aux_data[i].seen) 11591 continue; 11592 memcpy(insn + i, &trap, sizeof(trap)); 11593 } 11594 } 11595 11596 static bool insn_is_cond_jump(u8 code) 11597 { 11598 u8 op; 11599 11600 if (BPF_CLASS(code) == BPF_JMP32) 11601 return true; 11602 11603 if (BPF_CLASS(code) != BPF_JMP) 11604 return false; 11605 11606 op = BPF_OP(code); 11607 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 11608 } 11609 11610 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 11611 { 11612 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11613 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 11614 struct bpf_insn *insn = env->prog->insnsi; 11615 const int insn_cnt = env->prog->len; 11616 int i; 11617 11618 for (i = 0; i < insn_cnt; i++, insn++) { 11619 if (!insn_is_cond_jump(insn->code)) 11620 continue; 11621 11622 if (!aux_data[i + 1].seen) 11623 ja.off = insn->off; 11624 else if (!aux_data[i + 1 + insn->off].seen) 11625 ja.off = 0; 11626 else 11627 continue; 11628 11629 if (bpf_prog_is_dev_bound(env->prog->aux)) 11630 bpf_prog_offload_replace_insn(env, i, &ja); 11631 11632 memcpy(insn, &ja, sizeof(ja)); 11633 } 11634 } 11635 11636 static int opt_remove_dead_code(struct bpf_verifier_env *env) 11637 { 11638 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11639 int insn_cnt = env->prog->len; 11640 int i, err; 11641 11642 for (i = 0; i < insn_cnt; i++) { 11643 int j; 11644 11645 j = 0; 11646 while (i + j < insn_cnt && !aux_data[i + j].seen) 11647 j++; 11648 if (!j) 11649 continue; 11650 11651 err = verifier_remove_insns(env, i, j); 11652 if (err) 11653 return err; 11654 insn_cnt = env->prog->len; 11655 } 11656 11657 return 0; 11658 } 11659 11660 static int opt_remove_nops(struct bpf_verifier_env *env) 11661 { 11662 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 11663 struct bpf_insn *insn = env->prog->insnsi; 11664 int insn_cnt = env->prog->len; 11665 int i, err; 11666 11667 for (i = 0; i < insn_cnt; i++) { 11668 if (memcmp(&insn[i], &ja, sizeof(ja))) 11669 continue; 11670 11671 err = verifier_remove_insns(env, i, 1); 11672 if (err) 11673 return err; 11674 insn_cnt--; 11675 i--; 11676 } 11677 11678 return 0; 11679 } 11680 11681 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 11682 const union bpf_attr *attr) 11683 { 11684 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 11685 struct bpf_insn_aux_data *aux = env->insn_aux_data; 11686 int i, patch_len, delta = 0, len = env->prog->len; 11687 struct bpf_insn *insns = env->prog->insnsi; 11688 struct bpf_prog *new_prog; 11689 bool rnd_hi32; 11690 11691 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 11692 zext_patch[1] = BPF_ZEXT_REG(0); 11693 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 11694 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 11695 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 11696 for (i = 0; i < len; i++) { 11697 int adj_idx = i + delta; 11698 struct bpf_insn insn; 11699 int load_reg; 11700 11701 insn = insns[adj_idx]; 11702 load_reg = insn_def_regno(&insn); 11703 if (!aux[adj_idx].zext_dst) { 11704 u8 code, class; 11705 u32 imm_rnd; 11706 11707 if (!rnd_hi32) 11708 continue; 11709 11710 code = insn.code; 11711 class = BPF_CLASS(code); 11712 if (load_reg == -1) 11713 continue; 11714 11715 /* NOTE: arg "reg" (the fourth one) is only used for 11716 * BPF_STX + SRC_OP, so it is safe to pass NULL 11717 * here. 11718 */ 11719 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 11720 if (class == BPF_LD && 11721 BPF_MODE(code) == BPF_IMM) 11722 i++; 11723 continue; 11724 } 11725 11726 /* ctx load could be transformed into wider load. */ 11727 if (class == BPF_LDX && 11728 aux[adj_idx].ptr_type == PTR_TO_CTX) 11729 continue; 11730 11731 imm_rnd = get_random_int(); 11732 rnd_hi32_patch[0] = insn; 11733 rnd_hi32_patch[1].imm = imm_rnd; 11734 rnd_hi32_patch[3].dst_reg = load_reg; 11735 patch = rnd_hi32_patch; 11736 patch_len = 4; 11737 goto apply_patch_buffer; 11738 } 11739 11740 /* Add in an zero-extend instruction if a) the JIT has requested 11741 * it or b) it's a CMPXCHG. 11742 * 11743 * The latter is because: BPF_CMPXCHG always loads a value into 11744 * R0, therefore always zero-extends. However some archs' 11745 * equivalent instruction only does this load when the 11746 * comparison is successful. This detail of CMPXCHG is 11747 * orthogonal to the general zero-extension behaviour of the 11748 * CPU, so it's treated independently of bpf_jit_needs_zext. 11749 */ 11750 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 11751 continue; 11752 11753 if (WARN_ON(load_reg == -1)) { 11754 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 11755 return -EFAULT; 11756 } 11757 11758 zext_patch[0] = insn; 11759 zext_patch[1].dst_reg = load_reg; 11760 zext_patch[1].src_reg = load_reg; 11761 patch = zext_patch; 11762 patch_len = 2; 11763 apply_patch_buffer: 11764 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 11765 if (!new_prog) 11766 return -ENOMEM; 11767 env->prog = new_prog; 11768 insns = new_prog->insnsi; 11769 aux = env->insn_aux_data; 11770 delta += patch_len - 1; 11771 } 11772 11773 return 0; 11774 } 11775 11776 /* convert load instructions that access fields of a context type into a 11777 * sequence of instructions that access fields of the underlying structure: 11778 * struct __sk_buff -> struct sk_buff 11779 * struct bpf_sock_ops -> struct sock 11780 */ 11781 static int convert_ctx_accesses(struct bpf_verifier_env *env) 11782 { 11783 const struct bpf_verifier_ops *ops = env->ops; 11784 int i, cnt, size, ctx_field_size, delta = 0; 11785 const int insn_cnt = env->prog->len; 11786 struct bpf_insn insn_buf[16], *insn; 11787 u32 target_size, size_default, off; 11788 struct bpf_prog *new_prog; 11789 enum bpf_access_type type; 11790 bool is_narrower_load; 11791 11792 if (ops->gen_prologue || env->seen_direct_write) { 11793 if (!ops->gen_prologue) { 11794 verbose(env, "bpf verifier is misconfigured\n"); 11795 return -EINVAL; 11796 } 11797 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 11798 env->prog); 11799 if (cnt >= ARRAY_SIZE(insn_buf)) { 11800 verbose(env, "bpf verifier is misconfigured\n"); 11801 return -EINVAL; 11802 } else if (cnt) { 11803 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 11804 if (!new_prog) 11805 return -ENOMEM; 11806 11807 env->prog = new_prog; 11808 delta += cnt - 1; 11809 } 11810 } 11811 11812 if (bpf_prog_is_dev_bound(env->prog->aux)) 11813 return 0; 11814 11815 insn = env->prog->insnsi + delta; 11816 11817 for (i = 0; i < insn_cnt; i++, insn++) { 11818 bpf_convert_ctx_access_t convert_ctx_access; 11819 11820 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 11821 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 11822 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 11823 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) 11824 type = BPF_READ; 11825 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 11826 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 11827 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 11828 insn->code == (BPF_STX | BPF_MEM | BPF_DW)) 11829 type = BPF_WRITE; 11830 else 11831 continue; 11832 11833 if (type == BPF_WRITE && 11834 env->insn_aux_data[i + delta].sanitize_stack_off) { 11835 struct bpf_insn patch[] = { 11836 /* Sanitize suspicious stack slot with zero. 11837 * There are no memory dependencies for this store, 11838 * since it's only using frame pointer and immediate 11839 * constant of zero 11840 */ 11841 BPF_ST_MEM(BPF_DW, BPF_REG_FP, 11842 env->insn_aux_data[i + delta].sanitize_stack_off, 11843 0), 11844 /* the original STX instruction will immediately 11845 * overwrite the same stack slot with appropriate value 11846 */ 11847 *insn, 11848 }; 11849 11850 cnt = ARRAY_SIZE(patch); 11851 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 11852 if (!new_prog) 11853 return -ENOMEM; 11854 11855 delta += cnt - 1; 11856 env->prog = new_prog; 11857 insn = new_prog->insnsi + i + delta; 11858 continue; 11859 } 11860 11861 switch (env->insn_aux_data[i + delta].ptr_type) { 11862 case PTR_TO_CTX: 11863 if (!ops->convert_ctx_access) 11864 continue; 11865 convert_ctx_access = ops->convert_ctx_access; 11866 break; 11867 case PTR_TO_SOCKET: 11868 case PTR_TO_SOCK_COMMON: 11869 convert_ctx_access = bpf_sock_convert_ctx_access; 11870 break; 11871 case PTR_TO_TCP_SOCK: 11872 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 11873 break; 11874 case PTR_TO_XDP_SOCK: 11875 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 11876 break; 11877 case PTR_TO_BTF_ID: 11878 if (type == BPF_READ) { 11879 insn->code = BPF_LDX | BPF_PROBE_MEM | 11880 BPF_SIZE((insn)->code); 11881 env->prog->aux->num_exentries++; 11882 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { 11883 verbose(env, "Writes through BTF pointers are not allowed\n"); 11884 return -EINVAL; 11885 } 11886 continue; 11887 default: 11888 continue; 11889 } 11890 11891 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 11892 size = BPF_LDST_BYTES(insn); 11893 11894 /* If the read access is a narrower load of the field, 11895 * convert to a 4/8-byte load, to minimum program type specific 11896 * convert_ctx_access changes. If conversion is successful, 11897 * we will apply proper mask to the result. 11898 */ 11899 is_narrower_load = size < ctx_field_size; 11900 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 11901 off = insn->off; 11902 if (is_narrower_load) { 11903 u8 size_code; 11904 11905 if (type == BPF_WRITE) { 11906 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 11907 return -EINVAL; 11908 } 11909 11910 size_code = BPF_H; 11911 if (ctx_field_size == 4) 11912 size_code = BPF_W; 11913 else if (ctx_field_size == 8) 11914 size_code = BPF_DW; 11915 11916 insn->off = off & ~(size_default - 1); 11917 insn->code = BPF_LDX | BPF_MEM | size_code; 11918 } 11919 11920 target_size = 0; 11921 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 11922 &target_size); 11923 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 11924 (ctx_field_size && !target_size)) { 11925 verbose(env, "bpf verifier is misconfigured\n"); 11926 return -EINVAL; 11927 } 11928 11929 if (is_narrower_load && size < target_size) { 11930 u8 shift = bpf_ctx_narrow_access_offset( 11931 off, size, size_default) * 8; 11932 if (ctx_field_size <= 4) { 11933 if (shift) 11934 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 11935 insn->dst_reg, 11936 shift); 11937 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 11938 (1 << size * 8) - 1); 11939 } else { 11940 if (shift) 11941 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 11942 insn->dst_reg, 11943 shift); 11944 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 11945 (1ULL << size * 8) - 1); 11946 } 11947 } 11948 11949 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 11950 if (!new_prog) 11951 return -ENOMEM; 11952 11953 delta += cnt - 1; 11954 11955 /* keep walking new program and skip insns we just inserted */ 11956 env->prog = new_prog; 11957 insn = new_prog->insnsi + i + delta; 11958 } 11959 11960 return 0; 11961 } 11962 11963 static int jit_subprogs(struct bpf_verifier_env *env) 11964 { 11965 struct bpf_prog *prog = env->prog, **func, *tmp; 11966 int i, j, subprog_start, subprog_end = 0, len, subprog; 11967 struct bpf_map *map_ptr; 11968 struct bpf_insn *insn; 11969 void *old_bpf_func; 11970 int err, num_exentries; 11971 11972 if (env->subprog_cnt <= 1) 11973 return 0; 11974 11975 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 11976 if (bpf_pseudo_func(insn)) { 11977 env->insn_aux_data[i].call_imm = insn->imm; 11978 /* subprog is encoded in insn[1].imm */ 11979 continue; 11980 } 11981 11982 if (!bpf_pseudo_call(insn)) 11983 continue; 11984 /* Upon error here we cannot fall back to interpreter but 11985 * need a hard reject of the program. Thus -EFAULT is 11986 * propagated in any case. 11987 */ 11988 subprog = find_subprog(env, i + insn->imm + 1); 11989 if (subprog < 0) { 11990 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 11991 i + insn->imm + 1); 11992 return -EFAULT; 11993 } 11994 /* temporarily remember subprog id inside insn instead of 11995 * aux_data, since next loop will split up all insns into funcs 11996 */ 11997 insn->off = subprog; 11998 /* remember original imm in case JIT fails and fallback 11999 * to interpreter will be needed 12000 */ 12001 env->insn_aux_data[i].call_imm = insn->imm; 12002 /* point imm to __bpf_call_base+1 from JITs point of view */ 12003 insn->imm = 1; 12004 } 12005 12006 err = bpf_prog_alloc_jited_linfo(prog); 12007 if (err) 12008 goto out_undo_insn; 12009 12010 err = -ENOMEM; 12011 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 12012 if (!func) 12013 goto out_undo_insn; 12014 12015 for (i = 0; i < env->subprog_cnt; i++) { 12016 subprog_start = subprog_end; 12017 subprog_end = env->subprog_info[i + 1].start; 12018 12019 len = subprog_end - subprog_start; 12020 /* BPF_PROG_RUN doesn't call subprogs directly, 12021 * hence main prog stats include the runtime of subprogs. 12022 * subprogs don't have IDs and not reachable via prog_get_next_id 12023 * func[i]->stats will never be accessed and stays NULL 12024 */ 12025 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 12026 if (!func[i]) 12027 goto out_free; 12028 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 12029 len * sizeof(struct bpf_insn)); 12030 func[i]->type = prog->type; 12031 func[i]->len = len; 12032 if (bpf_prog_calc_tag(func[i])) 12033 goto out_free; 12034 func[i]->is_func = 1; 12035 func[i]->aux->func_idx = i; 12036 /* the btf and func_info will be freed only at prog->aux */ 12037 func[i]->aux->btf = prog->aux->btf; 12038 func[i]->aux->func_info = prog->aux->func_info; 12039 12040 for (j = 0; j < prog->aux->size_poke_tab; j++) { 12041 u32 insn_idx = prog->aux->poke_tab[j].insn_idx; 12042 int ret; 12043 12044 if (!(insn_idx >= subprog_start && 12045 insn_idx <= subprog_end)) 12046 continue; 12047 12048 ret = bpf_jit_add_poke_descriptor(func[i], 12049 &prog->aux->poke_tab[j]); 12050 if (ret < 0) { 12051 verbose(env, "adding tail call poke descriptor failed\n"); 12052 goto out_free; 12053 } 12054 12055 func[i]->insnsi[insn_idx - subprog_start].imm = ret + 1; 12056 12057 map_ptr = func[i]->aux->poke_tab[ret].tail_call.map; 12058 ret = map_ptr->ops->map_poke_track(map_ptr, func[i]->aux); 12059 if (ret < 0) { 12060 verbose(env, "tracking tail call prog failed\n"); 12061 goto out_free; 12062 } 12063 } 12064 12065 /* Use bpf_prog_F_tag to indicate functions in stack traces. 12066 * Long term would need debug info to populate names 12067 */ 12068 func[i]->aux->name[0] = 'F'; 12069 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 12070 func[i]->jit_requested = 1; 12071 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 12072 func[i]->aux->linfo = prog->aux->linfo; 12073 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 12074 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 12075 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 12076 num_exentries = 0; 12077 insn = func[i]->insnsi; 12078 for (j = 0; j < func[i]->len; j++, insn++) { 12079 if (BPF_CLASS(insn->code) == BPF_LDX && 12080 BPF_MODE(insn->code) == BPF_PROBE_MEM) 12081 num_exentries++; 12082 } 12083 func[i]->aux->num_exentries = num_exentries; 12084 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 12085 func[i] = bpf_int_jit_compile(func[i]); 12086 if (!func[i]->jited) { 12087 err = -ENOTSUPP; 12088 goto out_free; 12089 } 12090 cond_resched(); 12091 } 12092 12093 /* Untrack main program's aux structs so that during map_poke_run() 12094 * we will not stumble upon the unfilled poke descriptors; each 12095 * of the main program's poke descs got distributed across subprogs 12096 * and got tracked onto map, so we are sure that none of them will 12097 * be missed after the operation below 12098 */ 12099 for (i = 0; i < prog->aux->size_poke_tab; i++) { 12100 map_ptr = prog->aux->poke_tab[i].tail_call.map; 12101 12102 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 12103 } 12104 12105 /* at this point all bpf functions were successfully JITed 12106 * now populate all bpf_calls with correct addresses and 12107 * run last pass of JIT 12108 */ 12109 for (i = 0; i < env->subprog_cnt; i++) { 12110 insn = func[i]->insnsi; 12111 for (j = 0; j < func[i]->len; j++, insn++) { 12112 if (bpf_pseudo_func(insn)) { 12113 subprog = insn[1].imm; 12114 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 12115 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 12116 continue; 12117 } 12118 if (!bpf_pseudo_call(insn)) 12119 continue; 12120 subprog = insn->off; 12121 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - 12122 __bpf_call_base; 12123 } 12124 12125 /* we use the aux data to keep a list of the start addresses 12126 * of the JITed images for each function in the program 12127 * 12128 * for some architectures, such as powerpc64, the imm field 12129 * might not be large enough to hold the offset of the start 12130 * address of the callee's JITed image from __bpf_call_base 12131 * 12132 * in such cases, we can lookup the start address of a callee 12133 * by using its subprog id, available from the off field of 12134 * the call instruction, as an index for this list 12135 */ 12136 func[i]->aux->func = func; 12137 func[i]->aux->func_cnt = env->subprog_cnt; 12138 } 12139 for (i = 0; i < env->subprog_cnt; i++) { 12140 old_bpf_func = func[i]->bpf_func; 12141 tmp = bpf_int_jit_compile(func[i]); 12142 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 12143 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 12144 err = -ENOTSUPP; 12145 goto out_free; 12146 } 12147 cond_resched(); 12148 } 12149 12150 /* finally lock prog and jit images for all functions and 12151 * populate kallsysm 12152 */ 12153 for (i = 0; i < env->subprog_cnt; i++) { 12154 bpf_prog_lock_ro(func[i]); 12155 bpf_prog_kallsyms_add(func[i]); 12156 } 12157 12158 /* Last step: make now unused interpreter insns from main 12159 * prog consistent for later dump requests, so they can 12160 * later look the same as if they were interpreted only. 12161 */ 12162 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 12163 if (bpf_pseudo_func(insn)) { 12164 insn[0].imm = env->insn_aux_data[i].call_imm; 12165 insn[1].imm = find_subprog(env, i + insn[0].imm + 1); 12166 continue; 12167 } 12168 if (!bpf_pseudo_call(insn)) 12169 continue; 12170 insn->off = env->insn_aux_data[i].call_imm; 12171 subprog = find_subprog(env, i + insn->off + 1); 12172 insn->imm = subprog; 12173 } 12174 12175 prog->jited = 1; 12176 prog->bpf_func = func[0]->bpf_func; 12177 prog->aux->func = func; 12178 prog->aux->func_cnt = env->subprog_cnt; 12179 bpf_prog_jit_attempt_done(prog); 12180 return 0; 12181 out_free: 12182 for (i = 0; i < env->subprog_cnt; i++) { 12183 if (!func[i]) 12184 continue; 12185 12186 for (j = 0; j < func[i]->aux->size_poke_tab; j++) { 12187 map_ptr = func[i]->aux->poke_tab[j].tail_call.map; 12188 map_ptr->ops->map_poke_untrack(map_ptr, func[i]->aux); 12189 } 12190 bpf_jit_free(func[i]); 12191 } 12192 kfree(func); 12193 out_undo_insn: 12194 /* cleanup main prog to be interpreted */ 12195 prog->jit_requested = 0; 12196 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 12197 if (!bpf_pseudo_call(insn)) 12198 continue; 12199 insn->off = 0; 12200 insn->imm = env->insn_aux_data[i].call_imm; 12201 } 12202 bpf_prog_jit_attempt_done(prog); 12203 return err; 12204 } 12205 12206 static int fixup_call_args(struct bpf_verifier_env *env) 12207 { 12208 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 12209 struct bpf_prog *prog = env->prog; 12210 struct bpf_insn *insn = prog->insnsi; 12211 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 12212 int i, depth; 12213 #endif 12214 int err = 0; 12215 12216 if (env->prog->jit_requested && 12217 !bpf_prog_is_dev_bound(env->prog->aux)) { 12218 err = jit_subprogs(env); 12219 if (err == 0) 12220 return 0; 12221 if (err == -EFAULT) 12222 return err; 12223 } 12224 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 12225 if (has_kfunc_call) { 12226 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 12227 return -EINVAL; 12228 } 12229 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 12230 /* When JIT fails the progs with bpf2bpf calls and tail_calls 12231 * have to be rejected, since interpreter doesn't support them yet. 12232 */ 12233 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 12234 return -EINVAL; 12235 } 12236 for (i = 0; i < prog->len; i++, insn++) { 12237 if (bpf_pseudo_func(insn)) { 12238 /* When JIT fails the progs with callback calls 12239 * have to be rejected, since interpreter doesn't support them yet. 12240 */ 12241 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 12242 return -EINVAL; 12243 } 12244 12245 if (!bpf_pseudo_call(insn)) 12246 continue; 12247 depth = get_callee_stack_depth(env, insn, i); 12248 if (depth < 0) 12249 return depth; 12250 bpf_patch_call_args(insn, depth); 12251 } 12252 err = 0; 12253 #endif 12254 return err; 12255 } 12256 12257 static int fixup_kfunc_call(struct bpf_verifier_env *env, 12258 struct bpf_insn *insn) 12259 { 12260 const struct bpf_kfunc_desc *desc; 12261 12262 /* insn->imm has the btf func_id. Replace it with 12263 * an address (relative to __bpf_base_call). 12264 */ 12265 desc = find_kfunc_desc(env->prog, insn->imm); 12266 if (!desc) { 12267 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 12268 insn->imm); 12269 return -EFAULT; 12270 } 12271 12272 insn->imm = desc->imm; 12273 12274 return 0; 12275 } 12276 12277 /* Do various post-verification rewrites in a single program pass. 12278 * These rewrites simplify JIT and interpreter implementations. 12279 */ 12280 static int do_misc_fixups(struct bpf_verifier_env *env) 12281 { 12282 struct bpf_prog *prog = env->prog; 12283 bool expect_blinding = bpf_jit_blinding_enabled(prog); 12284 struct bpf_insn *insn = prog->insnsi; 12285 const struct bpf_func_proto *fn; 12286 const int insn_cnt = prog->len; 12287 const struct bpf_map_ops *ops; 12288 struct bpf_insn_aux_data *aux; 12289 struct bpf_insn insn_buf[16]; 12290 struct bpf_prog *new_prog; 12291 struct bpf_map *map_ptr; 12292 int i, ret, cnt, delta = 0; 12293 12294 for (i = 0; i < insn_cnt; i++, insn++) { 12295 /* Make divide-by-zero exceptions impossible. */ 12296 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 12297 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 12298 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 12299 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 12300 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 12301 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 12302 struct bpf_insn *patchlet; 12303 struct bpf_insn chk_and_div[] = { 12304 /* [R,W]x div 0 -> 0 */ 12305 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 12306 BPF_JNE | BPF_K, insn->src_reg, 12307 0, 2, 0), 12308 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 12309 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 12310 *insn, 12311 }; 12312 struct bpf_insn chk_and_mod[] = { 12313 /* [R,W]x mod 0 -> [R,W]x */ 12314 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 12315 BPF_JEQ | BPF_K, insn->src_reg, 12316 0, 1 + (is64 ? 0 : 1), 0), 12317 *insn, 12318 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 12319 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 12320 }; 12321 12322 patchlet = isdiv ? chk_and_div : chk_and_mod; 12323 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 12324 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 12325 12326 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 12327 if (!new_prog) 12328 return -ENOMEM; 12329 12330 delta += cnt - 1; 12331 env->prog = prog = new_prog; 12332 insn = new_prog->insnsi + i + delta; 12333 continue; 12334 } 12335 12336 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 12337 if (BPF_CLASS(insn->code) == BPF_LD && 12338 (BPF_MODE(insn->code) == BPF_ABS || 12339 BPF_MODE(insn->code) == BPF_IND)) { 12340 cnt = env->ops->gen_ld_abs(insn, insn_buf); 12341 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 12342 verbose(env, "bpf verifier is misconfigured\n"); 12343 return -EINVAL; 12344 } 12345 12346 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12347 if (!new_prog) 12348 return -ENOMEM; 12349 12350 delta += cnt - 1; 12351 env->prog = prog = new_prog; 12352 insn = new_prog->insnsi + i + delta; 12353 continue; 12354 } 12355 12356 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 12357 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 12358 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 12359 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 12360 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 12361 struct bpf_insn *patch = &insn_buf[0]; 12362 bool issrc, isneg, isimm; 12363 u32 off_reg; 12364 12365 aux = &env->insn_aux_data[i + delta]; 12366 if (!aux->alu_state || 12367 aux->alu_state == BPF_ALU_NON_POINTER) 12368 continue; 12369 12370 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 12371 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 12372 BPF_ALU_SANITIZE_SRC; 12373 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 12374 12375 off_reg = issrc ? insn->src_reg : insn->dst_reg; 12376 if (isimm) { 12377 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 12378 } else { 12379 if (isneg) 12380 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 12381 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 12382 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 12383 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 12384 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 12385 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 12386 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 12387 } 12388 if (!issrc) 12389 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 12390 insn->src_reg = BPF_REG_AX; 12391 if (isneg) 12392 insn->code = insn->code == code_add ? 12393 code_sub : code_add; 12394 *patch++ = *insn; 12395 if (issrc && isneg && !isimm) 12396 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 12397 cnt = patch - insn_buf; 12398 12399 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12400 if (!new_prog) 12401 return -ENOMEM; 12402 12403 delta += cnt - 1; 12404 env->prog = prog = new_prog; 12405 insn = new_prog->insnsi + i + delta; 12406 continue; 12407 } 12408 12409 if (insn->code != (BPF_JMP | BPF_CALL)) 12410 continue; 12411 if (insn->src_reg == BPF_PSEUDO_CALL) 12412 continue; 12413 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 12414 ret = fixup_kfunc_call(env, insn); 12415 if (ret) 12416 return ret; 12417 continue; 12418 } 12419 12420 if (insn->imm == BPF_FUNC_get_route_realm) 12421 prog->dst_needed = 1; 12422 if (insn->imm == BPF_FUNC_get_prandom_u32) 12423 bpf_user_rnd_init_once(); 12424 if (insn->imm == BPF_FUNC_override_return) 12425 prog->kprobe_override = 1; 12426 if (insn->imm == BPF_FUNC_tail_call) { 12427 /* If we tail call into other programs, we 12428 * cannot make any assumptions since they can 12429 * be replaced dynamically during runtime in 12430 * the program array. 12431 */ 12432 prog->cb_access = 1; 12433 if (!allow_tail_call_in_subprogs(env)) 12434 prog->aux->stack_depth = MAX_BPF_STACK; 12435 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 12436 12437 /* mark bpf_tail_call as different opcode to avoid 12438 * conditional branch in the interpeter for every normal 12439 * call and to prevent accidental JITing by JIT compiler 12440 * that doesn't support bpf_tail_call yet 12441 */ 12442 insn->imm = 0; 12443 insn->code = BPF_JMP | BPF_TAIL_CALL; 12444 12445 aux = &env->insn_aux_data[i + delta]; 12446 if (env->bpf_capable && !expect_blinding && 12447 prog->jit_requested && 12448 !bpf_map_key_poisoned(aux) && 12449 !bpf_map_ptr_poisoned(aux) && 12450 !bpf_map_ptr_unpriv(aux)) { 12451 struct bpf_jit_poke_descriptor desc = { 12452 .reason = BPF_POKE_REASON_TAIL_CALL, 12453 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 12454 .tail_call.key = bpf_map_key_immediate(aux), 12455 .insn_idx = i + delta, 12456 }; 12457 12458 ret = bpf_jit_add_poke_descriptor(prog, &desc); 12459 if (ret < 0) { 12460 verbose(env, "adding tail call poke descriptor failed\n"); 12461 return ret; 12462 } 12463 12464 insn->imm = ret + 1; 12465 continue; 12466 } 12467 12468 if (!bpf_map_ptr_unpriv(aux)) 12469 continue; 12470 12471 /* instead of changing every JIT dealing with tail_call 12472 * emit two extra insns: 12473 * if (index >= max_entries) goto out; 12474 * index &= array->index_mask; 12475 * to avoid out-of-bounds cpu speculation 12476 */ 12477 if (bpf_map_ptr_poisoned(aux)) { 12478 verbose(env, "tail_call abusing map_ptr\n"); 12479 return -EINVAL; 12480 } 12481 12482 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 12483 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 12484 map_ptr->max_entries, 2); 12485 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 12486 container_of(map_ptr, 12487 struct bpf_array, 12488 map)->index_mask); 12489 insn_buf[2] = *insn; 12490 cnt = 3; 12491 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12492 if (!new_prog) 12493 return -ENOMEM; 12494 12495 delta += cnt - 1; 12496 env->prog = prog = new_prog; 12497 insn = new_prog->insnsi + i + delta; 12498 continue; 12499 } 12500 12501 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 12502 * and other inlining handlers are currently limited to 64 bit 12503 * only. 12504 */ 12505 if (prog->jit_requested && BITS_PER_LONG == 64 && 12506 (insn->imm == BPF_FUNC_map_lookup_elem || 12507 insn->imm == BPF_FUNC_map_update_elem || 12508 insn->imm == BPF_FUNC_map_delete_elem || 12509 insn->imm == BPF_FUNC_map_push_elem || 12510 insn->imm == BPF_FUNC_map_pop_elem || 12511 insn->imm == BPF_FUNC_map_peek_elem || 12512 insn->imm == BPF_FUNC_redirect_map)) { 12513 aux = &env->insn_aux_data[i + delta]; 12514 if (bpf_map_ptr_poisoned(aux)) 12515 goto patch_call_imm; 12516 12517 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 12518 ops = map_ptr->ops; 12519 if (insn->imm == BPF_FUNC_map_lookup_elem && 12520 ops->map_gen_lookup) { 12521 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 12522 if (cnt == -EOPNOTSUPP) 12523 goto patch_map_ops_generic; 12524 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 12525 verbose(env, "bpf verifier is misconfigured\n"); 12526 return -EINVAL; 12527 } 12528 12529 new_prog = bpf_patch_insn_data(env, i + delta, 12530 insn_buf, cnt); 12531 if (!new_prog) 12532 return -ENOMEM; 12533 12534 delta += cnt - 1; 12535 env->prog = prog = new_prog; 12536 insn = new_prog->insnsi + i + delta; 12537 continue; 12538 } 12539 12540 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 12541 (void *(*)(struct bpf_map *map, void *key))NULL)); 12542 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 12543 (int (*)(struct bpf_map *map, void *key))NULL)); 12544 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 12545 (int (*)(struct bpf_map *map, void *key, void *value, 12546 u64 flags))NULL)); 12547 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 12548 (int (*)(struct bpf_map *map, void *value, 12549 u64 flags))NULL)); 12550 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 12551 (int (*)(struct bpf_map *map, void *value))NULL)); 12552 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 12553 (int (*)(struct bpf_map *map, void *value))NULL)); 12554 BUILD_BUG_ON(!__same_type(ops->map_redirect, 12555 (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL)); 12556 12557 patch_map_ops_generic: 12558 switch (insn->imm) { 12559 case BPF_FUNC_map_lookup_elem: 12560 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - 12561 __bpf_call_base; 12562 continue; 12563 case BPF_FUNC_map_update_elem: 12564 insn->imm = BPF_CAST_CALL(ops->map_update_elem) - 12565 __bpf_call_base; 12566 continue; 12567 case BPF_FUNC_map_delete_elem: 12568 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - 12569 __bpf_call_base; 12570 continue; 12571 case BPF_FUNC_map_push_elem: 12572 insn->imm = BPF_CAST_CALL(ops->map_push_elem) - 12573 __bpf_call_base; 12574 continue; 12575 case BPF_FUNC_map_pop_elem: 12576 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - 12577 __bpf_call_base; 12578 continue; 12579 case BPF_FUNC_map_peek_elem: 12580 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - 12581 __bpf_call_base; 12582 continue; 12583 case BPF_FUNC_redirect_map: 12584 insn->imm = BPF_CAST_CALL(ops->map_redirect) - 12585 __bpf_call_base; 12586 continue; 12587 } 12588 12589 goto patch_call_imm; 12590 } 12591 12592 /* Implement bpf_jiffies64 inline. */ 12593 if (prog->jit_requested && BITS_PER_LONG == 64 && 12594 insn->imm == BPF_FUNC_jiffies64) { 12595 struct bpf_insn ld_jiffies_addr[2] = { 12596 BPF_LD_IMM64(BPF_REG_0, 12597 (unsigned long)&jiffies), 12598 }; 12599 12600 insn_buf[0] = ld_jiffies_addr[0]; 12601 insn_buf[1] = ld_jiffies_addr[1]; 12602 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 12603 BPF_REG_0, 0); 12604 cnt = 3; 12605 12606 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 12607 cnt); 12608 if (!new_prog) 12609 return -ENOMEM; 12610 12611 delta += cnt - 1; 12612 env->prog = prog = new_prog; 12613 insn = new_prog->insnsi + i + delta; 12614 continue; 12615 } 12616 12617 patch_call_imm: 12618 fn = env->ops->get_func_proto(insn->imm, env->prog); 12619 /* all functions that have prototype and verifier allowed 12620 * programs to call them, must be real in-kernel functions 12621 */ 12622 if (!fn->func) { 12623 verbose(env, 12624 "kernel subsystem misconfigured func %s#%d\n", 12625 func_id_name(insn->imm), insn->imm); 12626 return -EFAULT; 12627 } 12628 insn->imm = fn->func - __bpf_call_base; 12629 } 12630 12631 /* Since poke tab is now finalized, publish aux to tracker. */ 12632 for (i = 0; i < prog->aux->size_poke_tab; i++) { 12633 map_ptr = prog->aux->poke_tab[i].tail_call.map; 12634 if (!map_ptr->ops->map_poke_track || 12635 !map_ptr->ops->map_poke_untrack || 12636 !map_ptr->ops->map_poke_run) { 12637 verbose(env, "bpf verifier is misconfigured\n"); 12638 return -EINVAL; 12639 } 12640 12641 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 12642 if (ret < 0) { 12643 verbose(env, "tracking tail call prog failed\n"); 12644 return ret; 12645 } 12646 } 12647 12648 sort_kfunc_descs_by_imm(env->prog); 12649 12650 return 0; 12651 } 12652 12653 static void free_states(struct bpf_verifier_env *env) 12654 { 12655 struct bpf_verifier_state_list *sl, *sln; 12656 int i; 12657 12658 sl = env->free_list; 12659 while (sl) { 12660 sln = sl->next; 12661 free_verifier_state(&sl->state, false); 12662 kfree(sl); 12663 sl = sln; 12664 } 12665 env->free_list = NULL; 12666 12667 if (!env->explored_states) 12668 return; 12669 12670 for (i = 0; i < state_htab_size(env); i++) { 12671 sl = env->explored_states[i]; 12672 12673 while (sl) { 12674 sln = sl->next; 12675 free_verifier_state(&sl->state, false); 12676 kfree(sl); 12677 sl = sln; 12678 } 12679 env->explored_states[i] = NULL; 12680 } 12681 } 12682 12683 /* The verifier is using insn_aux_data[] to store temporary data during 12684 * verification and to store information for passes that run after the 12685 * verification like dead code sanitization. do_check_common() for subprogram N 12686 * may analyze many other subprograms. sanitize_insn_aux_data() clears all 12687 * temporary data after do_check_common() finds that subprogram N cannot be 12688 * verified independently. pass_cnt counts the number of times 12689 * do_check_common() was run and insn->aux->seen tells the pass number 12690 * insn_aux_data was touched. These variables are compared to clear temporary 12691 * data from failed pass. For testing and experiments do_check_common() can be 12692 * run multiple times even when prior attempt to verify is unsuccessful. 12693 */ 12694 static void sanitize_insn_aux_data(struct bpf_verifier_env *env) 12695 { 12696 struct bpf_insn *insn = env->prog->insnsi; 12697 struct bpf_insn_aux_data *aux; 12698 int i, class; 12699 12700 for (i = 0; i < env->prog->len; i++) { 12701 class = BPF_CLASS(insn[i].code); 12702 if (class != BPF_LDX && class != BPF_STX) 12703 continue; 12704 aux = &env->insn_aux_data[i]; 12705 if (aux->seen != env->pass_cnt) 12706 continue; 12707 memset(aux, 0, offsetof(typeof(*aux), orig_idx)); 12708 } 12709 } 12710 12711 static int do_check_common(struct bpf_verifier_env *env, int subprog) 12712 { 12713 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 12714 struct bpf_verifier_state *state; 12715 struct bpf_reg_state *regs; 12716 int ret, i; 12717 12718 env->prev_linfo = NULL; 12719 env->pass_cnt++; 12720 12721 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 12722 if (!state) 12723 return -ENOMEM; 12724 state->curframe = 0; 12725 state->speculative = false; 12726 state->branches = 1; 12727 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 12728 if (!state->frame[0]) { 12729 kfree(state); 12730 return -ENOMEM; 12731 } 12732 env->cur_state = state; 12733 init_func_state(env, state->frame[0], 12734 BPF_MAIN_FUNC /* callsite */, 12735 0 /* frameno */, 12736 subprog); 12737 12738 regs = state->frame[state->curframe]->regs; 12739 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 12740 ret = btf_prepare_func_args(env, subprog, regs); 12741 if (ret) 12742 goto out; 12743 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 12744 if (regs[i].type == PTR_TO_CTX) 12745 mark_reg_known_zero(env, regs, i); 12746 else if (regs[i].type == SCALAR_VALUE) 12747 mark_reg_unknown(env, regs, i); 12748 else if (regs[i].type == PTR_TO_MEM_OR_NULL) { 12749 const u32 mem_size = regs[i].mem_size; 12750 12751 mark_reg_known_zero(env, regs, i); 12752 regs[i].mem_size = mem_size; 12753 regs[i].id = ++env->id_gen; 12754 } 12755 } 12756 } else { 12757 /* 1st arg to a function */ 12758 regs[BPF_REG_1].type = PTR_TO_CTX; 12759 mark_reg_known_zero(env, regs, BPF_REG_1); 12760 ret = btf_check_subprog_arg_match(env, subprog, regs); 12761 if (ret == -EFAULT) 12762 /* unlikely verifier bug. abort. 12763 * ret == 0 and ret < 0 are sadly acceptable for 12764 * main() function due to backward compatibility. 12765 * Like socket filter program may be written as: 12766 * int bpf_prog(struct pt_regs *ctx) 12767 * and never dereference that ctx in the program. 12768 * 'struct pt_regs' is a type mismatch for socket 12769 * filter that should be using 'struct __sk_buff'. 12770 */ 12771 goto out; 12772 } 12773 12774 ret = do_check(env); 12775 out: 12776 /* check for NULL is necessary, since cur_state can be freed inside 12777 * do_check() under memory pressure. 12778 */ 12779 if (env->cur_state) { 12780 free_verifier_state(env->cur_state, true); 12781 env->cur_state = NULL; 12782 } 12783 while (!pop_stack(env, NULL, NULL, false)); 12784 if (!ret && pop_log) 12785 bpf_vlog_reset(&env->log, 0); 12786 free_states(env); 12787 if (ret) 12788 /* clean aux data in case subprog was rejected */ 12789 sanitize_insn_aux_data(env); 12790 return ret; 12791 } 12792 12793 /* Verify all global functions in a BPF program one by one based on their BTF. 12794 * All global functions must pass verification. Otherwise the whole program is rejected. 12795 * Consider: 12796 * int bar(int); 12797 * int foo(int f) 12798 * { 12799 * return bar(f); 12800 * } 12801 * int bar(int b) 12802 * { 12803 * ... 12804 * } 12805 * foo() will be verified first for R1=any_scalar_value. During verification it 12806 * will be assumed that bar() already verified successfully and call to bar() 12807 * from foo() will be checked for type match only. Later bar() will be verified 12808 * independently to check that it's safe for R1=any_scalar_value. 12809 */ 12810 static int do_check_subprogs(struct bpf_verifier_env *env) 12811 { 12812 struct bpf_prog_aux *aux = env->prog->aux; 12813 int i, ret; 12814 12815 if (!aux->func_info) 12816 return 0; 12817 12818 for (i = 1; i < env->subprog_cnt; i++) { 12819 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 12820 continue; 12821 env->insn_idx = env->subprog_info[i].start; 12822 WARN_ON_ONCE(env->insn_idx == 0); 12823 ret = do_check_common(env, i); 12824 if (ret) { 12825 return ret; 12826 } else if (env->log.level & BPF_LOG_LEVEL) { 12827 verbose(env, 12828 "Func#%d is safe for any args that match its prototype\n", 12829 i); 12830 } 12831 } 12832 return 0; 12833 } 12834 12835 static int do_check_main(struct bpf_verifier_env *env) 12836 { 12837 int ret; 12838 12839 env->insn_idx = 0; 12840 ret = do_check_common(env, 0); 12841 if (!ret) 12842 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 12843 return ret; 12844 } 12845 12846 12847 static void print_verification_stats(struct bpf_verifier_env *env) 12848 { 12849 int i; 12850 12851 if (env->log.level & BPF_LOG_STATS) { 12852 verbose(env, "verification time %lld usec\n", 12853 div_u64(env->verification_time, 1000)); 12854 verbose(env, "stack depth "); 12855 for (i = 0; i < env->subprog_cnt; i++) { 12856 u32 depth = env->subprog_info[i].stack_depth; 12857 12858 verbose(env, "%d", depth); 12859 if (i + 1 < env->subprog_cnt) 12860 verbose(env, "+"); 12861 } 12862 verbose(env, "\n"); 12863 } 12864 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 12865 "total_states %d peak_states %d mark_read %d\n", 12866 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 12867 env->max_states_per_insn, env->total_states, 12868 env->peak_states, env->longest_mark_read_walk); 12869 } 12870 12871 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 12872 { 12873 const struct btf_type *t, *func_proto; 12874 const struct bpf_struct_ops *st_ops; 12875 const struct btf_member *member; 12876 struct bpf_prog *prog = env->prog; 12877 u32 btf_id, member_idx; 12878 const char *mname; 12879 12880 if (!prog->gpl_compatible) { 12881 verbose(env, "struct ops programs must have a GPL compatible license\n"); 12882 return -EINVAL; 12883 } 12884 12885 btf_id = prog->aux->attach_btf_id; 12886 st_ops = bpf_struct_ops_find(btf_id); 12887 if (!st_ops) { 12888 verbose(env, "attach_btf_id %u is not a supported struct\n", 12889 btf_id); 12890 return -ENOTSUPP; 12891 } 12892 12893 t = st_ops->type; 12894 member_idx = prog->expected_attach_type; 12895 if (member_idx >= btf_type_vlen(t)) { 12896 verbose(env, "attach to invalid member idx %u of struct %s\n", 12897 member_idx, st_ops->name); 12898 return -EINVAL; 12899 } 12900 12901 member = &btf_type_member(t)[member_idx]; 12902 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 12903 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 12904 NULL); 12905 if (!func_proto) { 12906 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 12907 mname, member_idx, st_ops->name); 12908 return -EINVAL; 12909 } 12910 12911 if (st_ops->check_member) { 12912 int err = st_ops->check_member(t, member); 12913 12914 if (err) { 12915 verbose(env, "attach to unsupported member %s of struct %s\n", 12916 mname, st_ops->name); 12917 return err; 12918 } 12919 } 12920 12921 prog->aux->attach_func_proto = func_proto; 12922 prog->aux->attach_func_name = mname; 12923 env->ops = st_ops->verifier_ops; 12924 12925 return 0; 12926 } 12927 #define SECURITY_PREFIX "security_" 12928 12929 static int check_attach_modify_return(unsigned long addr, const char *func_name) 12930 { 12931 if (within_error_injection_list(addr) || 12932 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 12933 return 0; 12934 12935 return -EINVAL; 12936 } 12937 12938 /* list of non-sleepable functions that are otherwise on 12939 * ALLOW_ERROR_INJECTION list 12940 */ 12941 BTF_SET_START(btf_non_sleepable_error_inject) 12942 /* Three functions below can be called from sleepable and non-sleepable context. 12943 * Assume non-sleepable from bpf safety point of view. 12944 */ 12945 BTF_ID(func, __add_to_page_cache_locked) 12946 BTF_ID(func, should_fail_alloc_page) 12947 BTF_ID(func, should_failslab) 12948 BTF_SET_END(btf_non_sleepable_error_inject) 12949 12950 static int check_non_sleepable_error_inject(u32 btf_id) 12951 { 12952 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 12953 } 12954 12955 int bpf_check_attach_target(struct bpf_verifier_log *log, 12956 const struct bpf_prog *prog, 12957 const struct bpf_prog *tgt_prog, 12958 u32 btf_id, 12959 struct bpf_attach_target_info *tgt_info) 12960 { 12961 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 12962 const char prefix[] = "btf_trace_"; 12963 int ret = 0, subprog = -1, i; 12964 const struct btf_type *t; 12965 bool conservative = true; 12966 const char *tname; 12967 struct btf *btf; 12968 long addr = 0; 12969 12970 if (!btf_id) { 12971 bpf_log(log, "Tracing programs must provide btf_id\n"); 12972 return -EINVAL; 12973 } 12974 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 12975 if (!btf) { 12976 bpf_log(log, 12977 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 12978 return -EINVAL; 12979 } 12980 t = btf_type_by_id(btf, btf_id); 12981 if (!t) { 12982 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 12983 return -EINVAL; 12984 } 12985 tname = btf_name_by_offset(btf, t->name_off); 12986 if (!tname) { 12987 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 12988 return -EINVAL; 12989 } 12990 if (tgt_prog) { 12991 struct bpf_prog_aux *aux = tgt_prog->aux; 12992 12993 for (i = 0; i < aux->func_info_cnt; i++) 12994 if (aux->func_info[i].type_id == btf_id) { 12995 subprog = i; 12996 break; 12997 } 12998 if (subprog == -1) { 12999 bpf_log(log, "Subprog %s doesn't exist\n", tname); 13000 return -EINVAL; 13001 } 13002 conservative = aux->func_info_aux[subprog].unreliable; 13003 if (prog_extension) { 13004 if (conservative) { 13005 bpf_log(log, 13006 "Cannot replace static functions\n"); 13007 return -EINVAL; 13008 } 13009 if (!prog->jit_requested) { 13010 bpf_log(log, 13011 "Extension programs should be JITed\n"); 13012 return -EINVAL; 13013 } 13014 } 13015 if (!tgt_prog->jited) { 13016 bpf_log(log, "Can attach to only JITed progs\n"); 13017 return -EINVAL; 13018 } 13019 if (tgt_prog->type == prog->type) { 13020 /* Cannot fentry/fexit another fentry/fexit program. 13021 * Cannot attach program extension to another extension. 13022 * It's ok to attach fentry/fexit to extension program. 13023 */ 13024 bpf_log(log, "Cannot recursively attach\n"); 13025 return -EINVAL; 13026 } 13027 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 13028 prog_extension && 13029 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 13030 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 13031 /* Program extensions can extend all program types 13032 * except fentry/fexit. The reason is the following. 13033 * The fentry/fexit programs are used for performance 13034 * analysis, stats and can be attached to any program 13035 * type except themselves. When extension program is 13036 * replacing XDP function it is necessary to allow 13037 * performance analysis of all functions. Both original 13038 * XDP program and its program extension. Hence 13039 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 13040 * allowed. If extending of fentry/fexit was allowed it 13041 * would be possible to create long call chain 13042 * fentry->extension->fentry->extension beyond 13043 * reasonable stack size. Hence extending fentry is not 13044 * allowed. 13045 */ 13046 bpf_log(log, "Cannot extend fentry/fexit\n"); 13047 return -EINVAL; 13048 } 13049 } else { 13050 if (prog_extension) { 13051 bpf_log(log, "Cannot replace kernel functions\n"); 13052 return -EINVAL; 13053 } 13054 } 13055 13056 switch (prog->expected_attach_type) { 13057 case BPF_TRACE_RAW_TP: 13058 if (tgt_prog) { 13059 bpf_log(log, 13060 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 13061 return -EINVAL; 13062 } 13063 if (!btf_type_is_typedef(t)) { 13064 bpf_log(log, "attach_btf_id %u is not a typedef\n", 13065 btf_id); 13066 return -EINVAL; 13067 } 13068 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 13069 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 13070 btf_id, tname); 13071 return -EINVAL; 13072 } 13073 tname += sizeof(prefix) - 1; 13074 t = btf_type_by_id(btf, t->type); 13075 if (!btf_type_is_ptr(t)) 13076 /* should never happen in valid vmlinux build */ 13077 return -EINVAL; 13078 t = btf_type_by_id(btf, t->type); 13079 if (!btf_type_is_func_proto(t)) 13080 /* should never happen in valid vmlinux build */ 13081 return -EINVAL; 13082 13083 break; 13084 case BPF_TRACE_ITER: 13085 if (!btf_type_is_func(t)) { 13086 bpf_log(log, "attach_btf_id %u is not a function\n", 13087 btf_id); 13088 return -EINVAL; 13089 } 13090 t = btf_type_by_id(btf, t->type); 13091 if (!btf_type_is_func_proto(t)) 13092 return -EINVAL; 13093 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 13094 if (ret) 13095 return ret; 13096 break; 13097 default: 13098 if (!prog_extension) 13099 return -EINVAL; 13100 fallthrough; 13101 case BPF_MODIFY_RETURN: 13102 case BPF_LSM_MAC: 13103 case BPF_TRACE_FENTRY: 13104 case BPF_TRACE_FEXIT: 13105 if (!btf_type_is_func(t)) { 13106 bpf_log(log, "attach_btf_id %u is not a function\n", 13107 btf_id); 13108 return -EINVAL; 13109 } 13110 if (prog_extension && 13111 btf_check_type_match(log, prog, btf, t)) 13112 return -EINVAL; 13113 t = btf_type_by_id(btf, t->type); 13114 if (!btf_type_is_func_proto(t)) 13115 return -EINVAL; 13116 13117 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 13118 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 13119 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 13120 return -EINVAL; 13121 13122 if (tgt_prog && conservative) 13123 t = NULL; 13124 13125 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 13126 if (ret < 0) 13127 return ret; 13128 13129 if (tgt_prog) { 13130 if (subprog == 0) 13131 addr = (long) tgt_prog->bpf_func; 13132 else 13133 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 13134 } else { 13135 addr = kallsyms_lookup_name(tname); 13136 if (!addr) { 13137 bpf_log(log, 13138 "The address of function %s cannot be found\n", 13139 tname); 13140 return -ENOENT; 13141 } 13142 } 13143 13144 if (prog->aux->sleepable) { 13145 ret = -EINVAL; 13146 switch (prog->type) { 13147 case BPF_PROG_TYPE_TRACING: 13148 /* fentry/fexit/fmod_ret progs can be sleepable only if they are 13149 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 13150 */ 13151 if (!check_non_sleepable_error_inject(btf_id) && 13152 within_error_injection_list(addr)) 13153 ret = 0; 13154 break; 13155 case BPF_PROG_TYPE_LSM: 13156 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 13157 * Only some of them are sleepable. 13158 */ 13159 if (bpf_lsm_is_sleepable_hook(btf_id)) 13160 ret = 0; 13161 break; 13162 default: 13163 break; 13164 } 13165 if (ret) { 13166 bpf_log(log, "%s is not sleepable\n", tname); 13167 return ret; 13168 } 13169 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 13170 if (tgt_prog) { 13171 bpf_log(log, "can't modify return codes of BPF programs\n"); 13172 return -EINVAL; 13173 } 13174 ret = check_attach_modify_return(addr, tname); 13175 if (ret) { 13176 bpf_log(log, "%s() is not modifiable\n", tname); 13177 return ret; 13178 } 13179 } 13180 13181 break; 13182 } 13183 tgt_info->tgt_addr = addr; 13184 tgt_info->tgt_name = tname; 13185 tgt_info->tgt_type = t; 13186 return 0; 13187 } 13188 13189 static int check_attach_btf_id(struct bpf_verifier_env *env) 13190 { 13191 struct bpf_prog *prog = env->prog; 13192 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 13193 struct bpf_attach_target_info tgt_info = {}; 13194 u32 btf_id = prog->aux->attach_btf_id; 13195 struct bpf_trampoline *tr; 13196 int ret; 13197 u64 key; 13198 13199 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 13200 if (prog->aux->sleepable) 13201 /* attach_btf_id checked to be zero already */ 13202 return 0; 13203 verbose(env, "Syscall programs can only be sleepable\n"); 13204 return -EINVAL; 13205 } 13206 13207 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && 13208 prog->type != BPF_PROG_TYPE_LSM) { 13209 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); 13210 return -EINVAL; 13211 } 13212 13213 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 13214 return check_struct_ops_btf_id(env); 13215 13216 if (prog->type != BPF_PROG_TYPE_TRACING && 13217 prog->type != BPF_PROG_TYPE_LSM && 13218 prog->type != BPF_PROG_TYPE_EXT) 13219 return 0; 13220 13221 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 13222 if (ret) 13223 return ret; 13224 13225 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 13226 /* to make freplace equivalent to their targets, they need to 13227 * inherit env->ops and expected_attach_type for the rest of the 13228 * verification 13229 */ 13230 env->ops = bpf_verifier_ops[tgt_prog->type]; 13231 prog->expected_attach_type = tgt_prog->expected_attach_type; 13232 } 13233 13234 /* store info about the attachment target that will be used later */ 13235 prog->aux->attach_func_proto = tgt_info.tgt_type; 13236 prog->aux->attach_func_name = tgt_info.tgt_name; 13237 13238 if (tgt_prog) { 13239 prog->aux->saved_dst_prog_type = tgt_prog->type; 13240 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 13241 } 13242 13243 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 13244 prog->aux->attach_btf_trace = true; 13245 return 0; 13246 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 13247 if (!bpf_iter_prog_supported(prog)) 13248 return -EINVAL; 13249 return 0; 13250 } 13251 13252 if (prog->type == BPF_PROG_TYPE_LSM) { 13253 ret = bpf_lsm_verify_prog(&env->log, prog); 13254 if (ret < 0) 13255 return ret; 13256 } 13257 13258 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 13259 tr = bpf_trampoline_get(key, &tgt_info); 13260 if (!tr) 13261 return -ENOMEM; 13262 13263 prog->aux->dst_trampoline = tr; 13264 return 0; 13265 } 13266 13267 struct btf *bpf_get_btf_vmlinux(void) 13268 { 13269 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 13270 mutex_lock(&bpf_verifier_lock); 13271 if (!btf_vmlinux) 13272 btf_vmlinux = btf_parse_vmlinux(); 13273 mutex_unlock(&bpf_verifier_lock); 13274 } 13275 return btf_vmlinux; 13276 } 13277 13278 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, 13279 union bpf_attr __user *uattr) 13280 { 13281 u64 start_time = ktime_get_ns(); 13282 struct bpf_verifier_env *env; 13283 struct bpf_verifier_log *log; 13284 int i, len, ret = -EINVAL; 13285 bool is_priv; 13286 13287 /* no program is valid */ 13288 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 13289 return -EINVAL; 13290 13291 /* 'struct bpf_verifier_env' can be global, but since it's not small, 13292 * allocate/free it every time bpf_check() is called 13293 */ 13294 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 13295 if (!env) 13296 return -ENOMEM; 13297 log = &env->log; 13298 13299 len = (*prog)->len; 13300 env->insn_aux_data = 13301 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 13302 ret = -ENOMEM; 13303 if (!env->insn_aux_data) 13304 goto err_free_env; 13305 for (i = 0; i < len; i++) 13306 env->insn_aux_data[i].orig_idx = i; 13307 env->prog = *prog; 13308 env->ops = bpf_verifier_ops[env->prog->type]; 13309 is_priv = bpf_capable(); 13310 13311 bpf_get_btf_vmlinux(); 13312 13313 /* grab the mutex to protect few globals used by verifier */ 13314 if (!is_priv) 13315 mutex_lock(&bpf_verifier_lock); 13316 13317 if (attr->log_level || attr->log_buf || attr->log_size) { 13318 /* user requested verbose verifier output 13319 * and supplied buffer to store the verification trace 13320 */ 13321 log->level = attr->log_level; 13322 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 13323 log->len_total = attr->log_size; 13324 13325 ret = -EINVAL; 13326 /* log attributes have to be sane */ 13327 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || 13328 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) 13329 goto err_unlock; 13330 } 13331 13332 if (IS_ERR(btf_vmlinux)) { 13333 /* Either gcc or pahole or kernel are broken. */ 13334 verbose(env, "in-kernel BTF is malformed\n"); 13335 ret = PTR_ERR(btf_vmlinux); 13336 goto skip_full_check; 13337 } 13338 13339 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 13340 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 13341 env->strict_alignment = true; 13342 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 13343 env->strict_alignment = false; 13344 13345 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 13346 env->allow_uninit_stack = bpf_allow_uninit_stack(); 13347 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); 13348 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 13349 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 13350 env->bpf_capable = bpf_capable(); 13351 13352 if (is_priv) 13353 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 13354 13355 if (bpf_prog_is_dev_bound(env->prog->aux)) { 13356 ret = bpf_prog_offload_verifier_prep(env->prog); 13357 if (ret) 13358 goto skip_full_check; 13359 } 13360 13361 env->explored_states = kvcalloc(state_htab_size(env), 13362 sizeof(struct bpf_verifier_state_list *), 13363 GFP_USER); 13364 ret = -ENOMEM; 13365 if (!env->explored_states) 13366 goto skip_full_check; 13367 13368 ret = add_subprog_and_kfunc(env); 13369 if (ret < 0) 13370 goto skip_full_check; 13371 13372 ret = check_subprogs(env); 13373 if (ret < 0) 13374 goto skip_full_check; 13375 13376 ret = check_btf_info(env, attr, uattr); 13377 if (ret < 0) 13378 goto skip_full_check; 13379 13380 ret = check_attach_btf_id(env); 13381 if (ret) 13382 goto skip_full_check; 13383 13384 ret = resolve_pseudo_ldimm64(env); 13385 if (ret < 0) 13386 goto skip_full_check; 13387 13388 ret = check_cfg(env); 13389 if (ret < 0) 13390 goto skip_full_check; 13391 13392 ret = do_check_subprogs(env); 13393 ret = ret ?: do_check_main(env); 13394 13395 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 13396 ret = bpf_prog_offload_finalize(env); 13397 13398 skip_full_check: 13399 kvfree(env->explored_states); 13400 13401 if (ret == 0) 13402 ret = check_max_stack_depth(env); 13403 13404 /* instruction rewrites happen after this point */ 13405 if (is_priv) { 13406 if (ret == 0) 13407 opt_hard_wire_dead_code_branches(env); 13408 if (ret == 0) 13409 ret = opt_remove_dead_code(env); 13410 if (ret == 0) 13411 ret = opt_remove_nops(env); 13412 } else { 13413 if (ret == 0) 13414 sanitize_dead_code(env); 13415 } 13416 13417 if (ret == 0) 13418 /* program is valid, convert *(u32*)(ctx + off) accesses */ 13419 ret = convert_ctx_accesses(env); 13420 13421 if (ret == 0) 13422 ret = do_misc_fixups(env); 13423 13424 /* do 32-bit optimization after insn patching has done so those patched 13425 * insns could be handled correctly. 13426 */ 13427 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { 13428 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 13429 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 13430 : false; 13431 } 13432 13433 if (ret == 0) 13434 ret = fixup_call_args(env); 13435 13436 env->verification_time = ktime_get_ns() - start_time; 13437 print_verification_stats(env); 13438 13439 if (log->level && bpf_verifier_log_full(log)) 13440 ret = -ENOSPC; 13441 if (log->level && !log->ubuf) { 13442 ret = -EFAULT; 13443 goto err_release_maps; 13444 } 13445 13446 if (ret) 13447 goto err_release_maps; 13448 13449 if (env->used_map_cnt) { 13450 /* if program passed verifier, update used_maps in bpf_prog_info */ 13451 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 13452 sizeof(env->used_maps[0]), 13453 GFP_KERNEL); 13454 13455 if (!env->prog->aux->used_maps) { 13456 ret = -ENOMEM; 13457 goto err_release_maps; 13458 } 13459 13460 memcpy(env->prog->aux->used_maps, env->used_maps, 13461 sizeof(env->used_maps[0]) * env->used_map_cnt); 13462 env->prog->aux->used_map_cnt = env->used_map_cnt; 13463 } 13464 if (env->used_btf_cnt) { 13465 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 13466 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 13467 sizeof(env->used_btfs[0]), 13468 GFP_KERNEL); 13469 if (!env->prog->aux->used_btfs) { 13470 ret = -ENOMEM; 13471 goto err_release_maps; 13472 } 13473 13474 memcpy(env->prog->aux->used_btfs, env->used_btfs, 13475 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 13476 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 13477 } 13478 if (env->used_map_cnt || env->used_btf_cnt) { 13479 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 13480 * bpf_ld_imm64 instructions 13481 */ 13482 convert_pseudo_ld_imm64(env); 13483 } 13484 13485 adjust_btf_func(env); 13486 13487 err_release_maps: 13488 if (!env->prog->aux->used_maps) 13489 /* if we didn't copy map pointers into bpf_prog_info, release 13490 * them now. Otherwise free_used_maps() will release them. 13491 */ 13492 release_maps(env); 13493 if (!env->prog->aux->used_btfs) 13494 release_btfs(env); 13495 13496 /* extension progs temporarily inherit the attach_type of their targets 13497 for verification purposes, so set it back to zero before returning 13498 */ 13499 if (env->prog->type == BPF_PROG_TYPE_EXT) 13500 env->prog->expected_attach_type = 0; 13501 13502 *prog = env->prog; 13503 err_unlock: 13504 if (!is_priv) 13505 mutex_unlock(&bpf_verifier_lock); 13506 vfree(env->insn_aux_data); 13507 err_free_env: 13508 kfree(env); 13509 return ret; 13510 } 13511