1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/bpf.h> 11 #include <linux/btf.h> 12 #include <linux/bpf_verifier.h> 13 #include <linux/filter.h> 14 #include <net/netlink.h> 15 #include <linux/file.h> 16 #include <linux/vmalloc.h> 17 #include <linux/stringify.h> 18 #include <linux/bsearch.h> 19 #include <linux/sort.h> 20 #include <linux/perf_event.h> 21 #include <linux/ctype.h> 22 #include <linux/error-injection.h> 23 #include <linux/bpf_lsm.h> 24 #include <linux/btf_ids.h> 25 26 #include "disasm.h" 27 28 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 29 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 30 [_id] = & _name ## _verifier_ops, 31 #define BPF_MAP_TYPE(_id, _ops) 32 #define BPF_LINK_TYPE(_id, _name) 33 #include <linux/bpf_types.h> 34 #undef BPF_PROG_TYPE 35 #undef BPF_MAP_TYPE 36 #undef BPF_LINK_TYPE 37 }; 38 39 /* bpf_check() is a static code analyzer that walks eBPF program 40 * instruction by instruction and updates register/stack state. 41 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 42 * 43 * The first pass is depth-first-search to check that the program is a DAG. 44 * It rejects the following programs: 45 * - larger than BPF_MAXINSNS insns 46 * - if loop is present (detected via back-edge) 47 * - unreachable insns exist (shouldn't be a forest. program = one function) 48 * - out of bounds or malformed jumps 49 * The second pass is all possible path descent from the 1st insn. 50 * Since it's analyzing all paths through the program, the length of the 51 * analysis is limited to 64k insn, which may be hit even if total number of 52 * insn is less then 4K, but there are too many branches that change stack/regs. 53 * Number of 'branches to be analyzed' is limited to 1k 54 * 55 * On entry to each instruction, each register has a type, and the instruction 56 * changes the types of the registers depending on instruction semantics. 57 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 58 * copied to R1. 59 * 60 * All registers are 64-bit. 61 * R0 - return register 62 * R1-R5 argument passing registers 63 * R6-R9 callee saved registers 64 * R10 - frame pointer read-only 65 * 66 * At the start of BPF program the register R1 contains a pointer to bpf_context 67 * and has type PTR_TO_CTX. 68 * 69 * Verifier tracks arithmetic operations on pointers in case: 70 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 71 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 72 * 1st insn copies R10 (which has FRAME_PTR) type into R1 73 * and 2nd arithmetic instruction is pattern matched to recognize 74 * that it wants to construct a pointer to some element within stack. 75 * So after 2nd insn, the register R1 has type PTR_TO_STACK 76 * (and -20 constant is saved for further stack bounds checking). 77 * Meaning that this reg is a pointer to stack plus known immediate constant. 78 * 79 * Most of the time the registers have SCALAR_VALUE type, which 80 * means the register has some value, but it's not a valid pointer. 81 * (like pointer plus pointer becomes SCALAR_VALUE type) 82 * 83 * When verifier sees load or store instructions the type of base register 84 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 85 * four pointer types recognized by check_mem_access() function. 86 * 87 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 88 * and the range of [ptr, ptr + map's value_size) is accessible. 89 * 90 * registers used to pass values to function calls are checked against 91 * function argument constraints. 92 * 93 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 94 * It means that the register type passed to this function must be 95 * PTR_TO_STACK and it will be used inside the function as 96 * 'pointer to map element key' 97 * 98 * For example the argument constraints for bpf_map_lookup_elem(): 99 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 100 * .arg1_type = ARG_CONST_MAP_PTR, 101 * .arg2_type = ARG_PTR_TO_MAP_KEY, 102 * 103 * ret_type says that this function returns 'pointer to map elem value or null' 104 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 105 * 2nd argument should be a pointer to stack, which will be used inside 106 * the helper function as a pointer to map element key. 107 * 108 * On the kernel side the helper function looks like: 109 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 110 * { 111 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 112 * void *key = (void *) (unsigned long) r2; 113 * void *value; 114 * 115 * here kernel can access 'key' and 'map' pointers safely, knowing that 116 * [key, key + map->key_size) bytes are valid and were initialized on 117 * the stack of eBPF program. 118 * } 119 * 120 * Corresponding eBPF program may look like: 121 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 122 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 123 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 124 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 125 * here verifier looks at prototype of map_lookup_elem() and sees: 126 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 127 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 128 * 129 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 130 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 131 * and were initialized prior to this call. 132 * If it's ok, then verifier allows this BPF_CALL insn and looks at 133 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 134 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 135 * returns either pointer to map value or NULL. 136 * 137 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 138 * insn, the register holding that pointer in the true branch changes state to 139 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 140 * branch. See check_cond_jmp_op(). 141 * 142 * After the call R0 is set to return type of the function and registers R1-R5 143 * are set to NOT_INIT to indicate that they are no longer readable. 144 * 145 * The following reference types represent a potential reference to a kernel 146 * resource which, after first being allocated, must be checked and freed by 147 * the BPF program: 148 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 149 * 150 * When the verifier sees a helper call return a reference type, it allocates a 151 * pointer id for the reference and stores it in the current function state. 152 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 153 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 154 * passes through a NULL-check conditional. For the branch wherein the state is 155 * changed to CONST_IMM, the verifier releases the reference. 156 * 157 * For each helper function that allocates a reference, such as 158 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 159 * bpf_sk_release(). When a reference type passes into the release function, 160 * the verifier also releases the reference. If any unchecked or unreleased 161 * reference remains at the end of the program, the verifier rejects it. 162 */ 163 164 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 165 struct bpf_verifier_stack_elem { 166 /* verifer state is 'st' 167 * before processing instruction 'insn_idx' 168 * and after processing instruction 'prev_insn_idx' 169 */ 170 struct bpf_verifier_state st; 171 int insn_idx; 172 int prev_insn_idx; 173 struct bpf_verifier_stack_elem *next; 174 /* length of verifier log at the time this state was pushed on stack */ 175 u32 log_pos; 176 }; 177 178 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 179 #define BPF_COMPLEXITY_LIMIT_STATES 64 180 181 #define BPF_MAP_KEY_POISON (1ULL << 63) 182 #define BPF_MAP_KEY_SEEN (1ULL << 62) 183 184 #define BPF_MAP_PTR_UNPRIV 1UL 185 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 186 POISON_POINTER_DELTA)) 187 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 188 189 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 190 { 191 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 192 } 193 194 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 195 { 196 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 197 } 198 199 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 200 const struct bpf_map *map, bool unpriv) 201 { 202 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 203 unpriv |= bpf_map_ptr_unpriv(aux); 204 aux->map_ptr_state = (unsigned long)map | 205 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 206 } 207 208 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 209 { 210 return aux->map_key_state & BPF_MAP_KEY_POISON; 211 } 212 213 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 214 { 215 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 216 } 217 218 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 219 { 220 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 221 } 222 223 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 224 { 225 bool poisoned = bpf_map_key_poisoned(aux); 226 227 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 229 } 230 231 static bool bpf_pseudo_call(const struct bpf_insn *insn) 232 { 233 return insn->code == (BPF_JMP | BPF_CALL) && 234 insn->src_reg == BPF_PSEUDO_CALL; 235 } 236 237 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 238 { 239 return insn->code == (BPF_JMP | BPF_CALL) && 240 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 241 } 242 243 static bool bpf_pseudo_func(const struct bpf_insn *insn) 244 { 245 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 246 insn->src_reg == BPF_PSEUDO_FUNC; 247 } 248 249 struct bpf_call_arg_meta { 250 struct bpf_map *map_ptr; 251 bool raw_mode; 252 bool pkt_access; 253 int regno; 254 int access_size; 255 int mem_size; 256 u64 msize_max_value; 257 int ref_obj_id; 258 int map_uid; 259 int func_id; 260 struct btf *btf; 261 u32 btf_id; 262 struct btf *ret_btf; 263 u32 ret_btf_id; 264 u32 subprogno; 265 }; 266 267 struct btf *btf_vmlinux; 268 269 static DEFINE_MUTEX(bpf_verifier_lock); 270 271 static const struct bpf_line_info * 272 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 273 { 274 const struct bpf_line_info *linfo; 275 const struct bpf_prog *prog; 276 u32 i, nr_linfo; 277 278 prog = env->prog; 279 nr_linfo = prog->aux->nr_linfo; 280 281 if (!nr_linfo || insn_off >= prog->len) 282 return NULL; 283 284 linfo = prog->aux->linfo; 285 for (i = 1; i < nr_linfo; i++) 286 if (insn_off < linfo[i].insn_off) 287 break; 288 289 return &linfo[i - 1]; 290 } 291 292 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 293 va_list args) 294 { 295 unsigned int n; 296 297 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 298 299 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 300 "verifier log line truncated - local buffer too short\n"); 301 302 n = min(log->len_total - log->len_used - 1, n); 303 log->kbuf[n] = '\0'; 304 305 if (log->level == BPF_LOG_KERNEL) { 306 pr_err("BPF:%s\n", log->kbuf); 307 return; 308 } 309 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 310 log->len_used += n; 311 else 312 log->ubuf = NULL; 313 } 314 315 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 316 { 317 char zero = 0; 318 319 if (!bpf_verifier_log_needed(log)) 320 return; 321 322 log->len_used = new_pos; 323 if (put_user(zero, log->ubuf + new_pos)) 324 log->ubuf = NULL; 325 } 326 327 /* log_level controls verbosity level of eBPF verifier. 328 * bpf_verifier_log_write() is used to dump the verification trace to the log, 329 * so the user can figure out what's wrong with the program 330 */ 331 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 332 const char *fmt, ...) 333 { 334 va_list args; 335 336 if (!bpf_verifier_log_needed(&env->log)) 337 return; 338 339 va_start(args, fmt); 340 bpf_verifier_vlog(&env->log, fmt, args); 341 va_end(args); 342 } 343 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 344 345 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 346 { 347 struct bpf_verifier_env *env = private_data; 348 va_list args; 349 350 if (!bpf_verifier_log_needed(&env->log)) 351 return; 352 353 va_start(args, fmt); 354 bpf_verifier_vlog(&env->log, fmt, args); 355 va_end(args); 356 } 357 358 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 359 const char *fmt, ...) 360 { 361 va_list args; 362 363 if (!bpf_verifier_log_needed(log)) 364 return; 365 366 va_start(args, fmt); 367 bpf_verifier_vlog(log, fmt, args); 368 va_end(args); 369 } 370 371 static const char *ltrim(const char *s) 372 { 373 while (isspace(*s)) 374 s++; 375 376 return s; 377 } 378 379 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 380 u32 insn_off, 381 const char *prefix_fmt, ...) 382 { 383 const struct bpf_line_info *linfo; 384 385 if (!bpf_verifier_log_needed(&env->log)) 386 return; 387 388 linfo = find_linfo(env, insn_off); 389 if (!linfo || linfo == env->prev_linfo) 390 return; 391 392 if (prefix_fmt) { 393 va_list args; 394 395 va_start(args, prefix_fmt); 396 bpf_verifier_vlog(&env->log, prefix_fmt, args); 397 va_end(args); 398 } 399 400 verbose(env, "%s\n", 401 ltrim(btf_name_by_offset(env->prog->aux->btf, 402 linfo->line_off))); 403 404 env->prev_linfo = linfo; 405 } 406 407 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 408 struct bpf_reg_state *reg, 409 struct tnum *range, const char *ctx, 410 const char *reg_name) 411 { 412 char tn_buf[48]; 413 414 verbose(env, "At %s the register %s ", ctx, reg_name); 415 if (!tnum_is_unknown(reg->var_off)) { 416 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 417 verbose(env, "has value %s", tn_buf); 418 } else { 419 verbose(env, "has unknown scalar value"); 420 } 421 tnum_strn(tn_buf, sizeof(tn_buf), *range); 422 verbose(env, " should have been in %s\n", tn_buf); 423 } 424 425 static bool type_is_pkt_pointer(enum bpf_reg_type type) 426 { 427 return type == PTR_TO_PACKET || 428 type == PTR_TO_PACKET_META; 429 } 430 431 static bool type_is_sk_pointer(enum bpf_reg_type type) 432 { 433 return type == PTR_TO_SOCKET || 434 type == PTR_TO_SOCK_COMMON || 435 type == PTR_TO_TCP_SOCK || 436 type == PTR_TO_XDP_SOCK; 437 } 438 439 static bool reg_type_not_null(enum bpf_reg_type type) 440 { 441 return type == PTR_TO_SOCKET || 442 type == PTR_TO_TCP_SOCK || 443 type == PTR_TO_MAP_VALUE || 444 type == PTR_TO_MAP_KEY || 445 type == PTR_TO_SOCK_COMMON; 446 } 447 448 static bool reg_type_may_be_null(enum bpf_reg_type type) 449 { 450 return type == PTR_TO_MAP_VALUE_OR_NULL || 451 type == PTR_TO_SOCKET_OR_NULL || 452 type == PTR_TO_SOCK_COMMON_OR_NULL || 453 type == PTR_TO_TCP_SOCK_OR_NULL || 454 type == PTR_TO_BTF_ID_OR_NULL || 455 type == PTR_TO_MEM_OR_NULL || 456 type == PTR_TO_RDONLY_BUF_OR_NULL || 457 type == PTR_TO_RDWR_BUF_OR_NULL; 458 } 459 460 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 461 { 462 return reg->type == PTR_TO_MAP_VALUE && 463 map_value_has_spin_lock(reg->map_ptr); 464 } 465 466 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) 467 { 468 return type == PTR_TO_SOCKET || 469 type == PTR_TO_SOCKET_OR_NULL || 470 type == PTR_TO_TCP_SOCK || 471 type == PTR_TO_TCP_SOCK_OR_NULL || 472 type == PTR_TO_MEM || 473 type == PTR_TO_MEM_OR_NULL; 474 } 475 476 static bool arg_type_may_be_refcounted(enum bpf_arg_type type) 477 { 478 return type == ARG_PTR_TO_SOCK_COMMON; 479 } 480 481 static bool arg_type_may_be_null(enum bpf_arg_type type) 482 { 483 return type == ARG_PTR_TO_MAP_VALUE_OR_NULL || 484 type == ARG_PTR_TO_MEM_OR_NULL || 485 type == ARG_PTR_TO_CTX_OR_NULL || 486 type == ARG_PTR_TO_SOCKET_OR_NULL || 487 type == ARG_PTR_TO_ALLOC_MEM_OR_NULL || 488 type == ARG_PTR_TO_STACK_OR_NULL; 489 } 490 491 /* Determine whether the function releases some resources allocated by another 492 * function call. The first reference type argument will be assumed to be 493 * released by release_reference(). 494 */ 495 static bool is_release_function(enum bpf_func_id func_id) 496 { 497 return func_id == BPF_FUNC_sk_release || 498 func_id == BPF_FUNC_ringbuf_submit || 499 func_id == BPF_FUNC_ringbuf_discard; 500 } 501 502 static bool may_be_acquire_function(enum bpf_func_id func_id) 503 { 504 return func_id == BPF_FUNC_sk_lookup_tcp || 505 func_id == BPF_FUNC_sk_lookup_udp || 506 func_id == BPF_FUNC_skc_lookup_tcp || 507 func_id == BPF_FUNC_map_lookup_elem || 508 func_id == BPF_FUNC_ringbuf_reserve; 509 } 510 511 static bool is_acquire_function(enum bpf_func_id func_id, 512 const struct bpf_map *map) 513 { 514 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 515 516 if (func_id == BPF_FUNC_sk_lookup_tcp || 517 func_id == BPF_FUNC_sk_lookup_udp || 518 func_id == BPF_FUNC_skc_lookup_tcp || 519 func_id == BPF_FUNC_ringbuf_reserve) 520 return true; 521 522 if (func_id == BPF_FUNC_map_lookup_elem && 523 (map_type == BPF_MAP_TYPE_SOCKMAP || 524 map_type == BPF_MAP_TYPE_SOCKHASH)) 525 return true; 526 527 return false; 528 } 529 530 static bool is_ptr_cast_function(enum bpf_func_id func_id) 531 { 532 return func_id == BPF_FUNC_tcp_sock || 533 func_id == BPF_FUNC_sk_fullsock || 534 func_id == BPF_FUNC_skc_to_tcp_sock || 535 func_id == BPF_FUNC_skc_to_tcp6_sock || 536 func_id == BPF_FUNC_skc_to_udp6_sock || 537 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 538 func_id == BPF_FUNC_skc_to_tcp_request_sock; 539 } 540 541 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 542 { 543 return BPF_CLASS(insn->code) == BPF_STX && 544 BPF_MODE(insn->code) == BPF_ATOMIC && 545 insn->imm == BPF_CMPXCHG; 546 } 547 548 /* string representation of 'enum bpf_reg_type' */ 549 static const char * const reg_type_str[] = { 550 [NOT_INIT] = "?", 551 [SCALAR_VALUE] = "inv", 552 [PTR_TO_CTX] = "ctx", 553 [CONST_PTR_TO_MAP] = "map_ptr", 554 [PTR_TO_MAP_VALUE] = "map_value", 555 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", 556 [PTR_TO_STACK] = "fp", 557 [PTR_TO_PACKET] = "pkt", 558 [PTR_TO_PACKET_META] = "pkt_meta", 559 [PTR_TO_PACKET_END] = "pkt_end", 560 [PTR_TO_FLOW_KEYS] = "flow_keys", 561 [PTR_TO_SOCKET] = "sock", 562 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", 563 [PTR_TO_SOCK_COMMON] = "sock_common", 564 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", 565 [PTR_TO_TCP_SOCK] = "tcp_sock", 566 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", 567 [PTR_TO_TP_BUFFER] = "tp_buffer", 568 [PTR_TO_XDP_SOCK] = "xdp_sock", 569 [PTR_TO_BTF_ID] = "ptr_", 570 [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", 571 [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", 572 [PTR_TO_MEM] = "mem", 573 [PTR_TO_MEM_OR_NULL] = "mem_or_null", 574 [PTR_TO_RDONLY_BUF] = "rdonly_buf", 575 [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", 576 [PTR_TO_RDWR_BUF] = "rdwr_buf", 577 [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", 578 [PTR_TO_FUNC] = "func", 579 [PTR_TO_MAP_KEY] = "map_key", 580 }; 581 582 static char slot_type_char[] = { 583 [STACK_INVALID] = '?', 584 [STACK_SPILL] = 'r', 585 [STACK_MISC] = 'm', 586 [STACK_ZERO] = '0', 587 }; 588 589 static void print_liveness(struct bpf_verifier_env *env, 590 enum bpf_reg_liveness live) 591 { 592 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 593 verbose(env, "_"); 594 if (live & REG_LIVE_READ) 595 verbose(env, "r"); 596 if (live & REG_LIVE_WRITTEN) 597 verbose(env, "w"); 598 if (live & REG_LIVE_DONE) 599 verbose(env, "D"); 600 } 601 602 static struct bpf_func_state *func(struct bpf_verifier_env *env, 603 const struct bpf_reg_state *reg) 604 { 605 struct bpf_verifier_state *cur = env->cur_state; 606 607 return cur->frame[reg->frameno]; 608 } 609 610 static const char *kernel_type_name(const struct btf* btf, u32 id) 611 { 612 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 613 } 614 615 static void print_verifier_state(struct bpf_verifier_env *env, 616 const struct bpf_func_state *state) 617 { 618 const struct bpf_reg_state *reg; 619 enum bpf_reg_type t; 620 int i; 621 622 if (state->frameno) 623 verbose(env, " frame%d:", state->frameno); 624 for (i = 0; i < MAX_BPF_REG; i++) { 625 reg = &state->regs[i]; 626 t = reg->type; 627 if (t == NOT_INIT) 628 continue; 629 verbose(env, " R%d", i); 630 print_liveness(env, reg->live); 631 verbose(env, "=%s", reg_type_str[t]); 632 if (t == SCALAR_VALUE && reg->precise) 633 verbose(env, "P"); 634 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 635 tnum_is_const(reg->var_off)) { 636 /* reg->off should be 0 for SCALAR_VALUE */ 637 verbose(env, "%lld", reg->var_off.value + reg->off); 638 } else { 639 if (t == PTR_TO_BTF_ID || 640 t == PTR_TO_BTF_ID_OR_NULL || 641 t == PTR_TO_PERCPU_BTF_ID) 642 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 643 verbose(env, "(id=%d", reg->id); 644 if (reg_type_may_be_refcounted_or_null(t)) 645 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); 646 if (t != SCALAR_VALUE) 647 verbose(env, ",off=%d", reg->off); 648 if (type_is_pkt_pointer(t)) 649 verbose(env, ",r=%d", reg->range); 650 else if (t == CONST_PTR_TO_MAP || 651 t == PTR_TO_MAP_KEY || 652 t == PTR_TO_MAP_VALUE || 653 t == PTR_TO_MAP_VALUE_OR_NULL) 654 verbose(env, ",ks=%d,vs=%d", 655 reg->map_ptr->key_size, 656 reg->map_ptr->value_size); 657 if (tnum_is_const(reg->var_off)) { 658 /* Typically an immediate SCALAR_VALUE, but 659 * could be a pointer whose offset is too big 660 * for reg->off 661 */ 662 verbose(env, ",imm=%llx", reg->var_off.value); 663 } else { 664 if (reg->smin_value != reg->umin_value && 665 reg->smin_value != S64_MIN) 666 verbose(env, ",smin_value=%lld", 667 (long long)reg->smin_value); 668 if (reg->smax_value != reg->umax_value && 669 reg->smax_value != S64_MAX) 670 verbose(env, ",smax_value=%lld", 671 (long long)reg->smax_value); 672 if (reg->umin_value != 0) 673 verbose(env, ",umin_value=%llu", 674 (unsigned long long)reg->umin_value); 675 if (reg->umax_value != U64_MAX) 676 verbose(env, ",umax_value=%llu", 677 (unsigned long long)reg->umax_value); 678 if (!tnum_is_unknown(reg->var_off)) { 679 char tn_buf[48]; 680 681 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 682 verbose(env, ",var_off=%s", tn_buf); 683 } 684 if (reg->s32_min_value != reg->smin_value && 685 reg->s32_min_value != S32_MIN) 686 verbose(env, ",s32_min_value=%d", 687 (int)(reg->s32_min_value)); 688 if (reg->s32_max_value != reg->smax_value && 689 reg->s32_max_value != S32_MAX) 690 verbose(env, ",s32_max_value=%d", 691 (int)(reg->s32_max_value)); 692 if (reg->u32_min_value != reg->umin_value && 693 reg->u32_min_value != U32_MIN) 694 verbose(env, ",u32_min_value=%d", 695 (int)(reg->u32_min_value)); 696 if (reg->u32_max_value != reg->umax_value && 697 reg->u32_max_value != U32_MAX) 698 verbose(env, ",u32_max_value=%d", 699 (int)(reg->u32_max_value)); 700 } 701 verbose(env, ")"); 702 } 703 } 704 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 705 char types_buf[BPF_REG_SIZE + 1]; 706 bool valid = false; 707 int j; 708 709 for (j = 0; j < BPF_REG_SIZE; j++) { 710 if (state->stack[i].slot_type[j] != STACK_INVALID) 711 valid = true; 712 types_buf[j] = slot_type_char[ 713 state->stack[i].slot_type[j]]; 714 } 715 types_buf[BPF_REG_SIZE] = 0; 716 if (!valid) 717 continue; 718 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 719 print_liveness(env, state->stack[i].spilled_ptr.live); 720 if (state->stack[i].slot_type[0] == STACK_SPILL) { 721 reg = &state->stack[i].spilled_ptr; 722 t = reg->type; 723 verbose(env, "=%s", reg_type_str[t]); 724 if (t == SCALAR_VALUE && reg->precise) 725 verbose(env, "P"); 726 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 727 verbose(env, "%lld", reg->var_off.value + reg->off); 728 } else { 729 verbose(env, "=%s", types_buf); 730 } 731 } 732 if (state->acquired_refs && state->refs[0].id) { 733 verbose(env, " refs=%d", state->refs[0].id); 734 for (i = 1; i < state->acquired_refs; i++) 735 if (state->refs[i].id) 736 verbose(env, ",%d", state->refs[i].id); 737 } 738 if (state->in_callback_fn) 739 verbose(env, " cb"); 740 if (state->in_async_callback_fn) 741 verbose(env, " async_cb"); 742 verbose(env, "\n"); 743 } 744 745 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 746 * small to hold src. This is different from krealloc since we don't want to preserve 747 * the contents of dst. 748 * 749 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 750 * not be allocated. 751 */ 752 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 753 { 754 size_t bytes; 755 756 if (ZERO_OR_NULL_PTR(src)) 757 goto out; 758 759 if (unlikely(check_mul_overflow(n, size, &bytes))) 760 return NULL; 761 762 if (ksize(dst) < bytes) { 763 kfree(dst); 764 dst = kmalloc_track_caller(bytes, flags); 765 if (!dst) 766 return NULL; 767 } 768 769 memcpy(dst, src, bytes); 770 out: 771 return dst ? dst : ZERO_SIZE_PTR; 772 } 773 774 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 775 * small to hold new_n items. new items are zeroed out if the array grows. 776 * 777 * Contrary to krealloc_array, does not free arr if new_n is zero. 778 */ 779 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 780 { 781 if (!new_n || old_n == new_n) 782 goto out; 783 784 arr = krealloc_array(arr, new_n, size, GFP_KERNEL); 785 if (!arr) 786 return NULL; 787 788 if (new_n > old_n) 789 memset(arr + old_n * size, 0, (new_n - old_n) * size); 790 791 out: 792 return arr ? arr : ZERO_SIZE_PTR; 793 } 794 795 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 796 { 797 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 798 sizeof(struct bpf_reference_state), GFP_KERNEL); 799 if (!dst->refs) 800 return -ENOMEM; 801 802 dst->acquired_refs = src->acquired_refs; 803 return 0; 804 } 805 806 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 807 { 808 size_t n = src->allocated_stack / BPF_REG_SIZE; 809 810 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 811 GFP_KERNEL); 812 if (!dst->stack) 813 return -ENOMEM; 814 815 dst->allocated_stack = src->allocated_stack; 816 return 0; 817 } 818 819 static int resize_reference_state(struct bpf_func_state *state, size_t n) 820 { 821 state->refs = realloc_array(state->refs, state->acquired_refs, n, 822 sizeof(struct bpf_reference_state)); 823 if (!state->refs) 824 return -ENOMEM; 825 826 state->acquired_refs = n; 827 return 0; 828 } 829 830 static int grow_stack_state(struct bpf_func_state *state, int size) 831 { 832 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 833 834 if (old_n >= n) 835 return 0; 836 837 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 838 if (!state->stack) 839 return -ENOMEM; 840 841 state->allocated_stack = size; 842 return 0; 843 } 844 845 /* Acquire a pointer id from the env and update the state->refs to include 846 * this new pointer reference. 847 * On success, returns a valid pointer id to associate with the register 848 * On failure, returns a negative errno. 849 */ 850 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 851 { 852 struct bpf_func_state *state = cur_func(env); 853 int new_ofs = state->acquired_refs; 854 int id, err; 855 856 err = resize_reference_state(state, state->acquired_refs + 1); 857 if (err) 858 return err; 859 id = ++env->id_gen; 860 state->refs[new_ofs].id = id; 861 state->refs[new_ofs].insn_idx = insn_idx; 862 863 return id; 864 } 865 866 /* release function corresponding to acquire_reference_state(). Idempotent. */ 867 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 868 { 869 int i, last_idx; 870 871 last_idx = state->acquired_refs - 1; 872 for (i = 0; i < state->acquired_refs; i++) { 873 if (state->refs[i].id == ptr_id) { 874 if (last_idx && i != last_idx) 875 memcpy(&state->refs[i], &state->refs[last_idx], 876 sizeof(*state->refs)); 877 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 878 state->acquired_refs--; 879 return 0; 880 } 881 } 882 return -EINVAL; 883 } 884 885 static void free_func_state(struct bpf_func_state *state) 886 { 887 if (!state) 888 return; 889 kfree(state->refs); 890 kfree(state->stack); 891 kfree(state); 892 } 893 894 static void clear_jmp_history(struct bpf_verifier_state *state) 895 { 896 kfree(state->jmp_history); 897 state->jmp_history = NULL; 898 state->jmp_history_cnt = 0; 899 } 900 901 static void free_verifier_state(struct bpf_verifier_state *state, 902 bool free_self) 903 { 904 int i; 905 906 for (i = 0; i <= state->curframe; i++) { 907 free_func_state(state->frame[i]); 908 state->frame[i] = NULL; 909 } 910 clear_jmp_history(state); 911 if (free_self) 912 kfree(state); 913 } 914 915 /* copy verifier state from src to dst growing dst stack space 916 * when necessary to accommodate larger src stack 917 */ 918 static int copy_func_state(struct bpf_func_state *dst, 919 const struct bpf_func_state *src) 920 { 921 int err; 922 923 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 924 err = copy_reference_state(dst, src); 925 if (err) 926 return err; 927 return copy_stack_state(dst, src); 928 } 929 930 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 931 const struct bpf_verifier_state *src) 932 { 933 struct bpf_func_state *dst; 934 int i, err; 935 936 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 937 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 938 GFP_USER); 939 if (!dst_state->jmp_history) 940 return -ENOMEM; 941 dst_state->jmp_history_cnt = src->jmp_history_cnt; 942 943 /* if dst has more stack frames then src frame, free them */ 944 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 945 free_func_state(dst_state->frame[i]); 946 dst_state->frame[i] = NULL; 947 } 948 dst_state->speculative = src->speculative; 949 dst_state->curframe = src->curframe; 950 dst_state->active_spin_lock = src->active_spin_lock; 951 dst_state->branches = src->branches; 952 dst_state->parent = src->parent; 953 dst_state->first_insn_idx = src->first_insn_idx; 954 dst_state->last_insn_idx = src->last_insn_idx; 955 for (i = 0; i <= src->curframe; i++) { 956 dst = dst_state->frame[i]; 957 if (!dst) { 958 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 959 if (!dst) 960 return -ENOMEM; 961 dst_state->frame[i] = dst; 962 } 963 err = copy_func_state(dst, src->frame[i]); 964 if (err) 965 return err; 966 } 967 return 0; 968 } 969 970 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 971 { 972 while (st) { 973 u32 br = --st->branches; 974 975 /* WARN_ON(br > 1) technically makes sense here, 976 * but see comment in push_stack(), hence: 977 */ 978 WARN_ONCE((int)br < 0, 979 "BUG update_branch_counts:branches_to_explore=%d\n", 980 br); 981 if (br) 982 break; 983 st = st->parent; 984 } 985 } 986 987 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 988 int *insn_idx, bool pop_log) 989 { 990 struct bpf_verifier_state *cur = env->cur_state; 991 struct bpf_verifier_stack_elem *elem, *head = env->head; 992 int err; 993 994 if (env->head == NULL) 995 return -ENOENT; 996 997 if (cur) { 998 err = copy_verifier_state(cur, &head->st); 999 if (err) 1000 return err; 1001 } 1002 if (pop_log) 1003 bpf_vlog_reset(&env->log, head->log_pos); 1004 if (insn_idx) 1005 *insn_idx = head->insn_idx; 1006 if (prev_insn_idx) 1007 *prev_insn_idx = head->prev_insn_idx; 1008 elem = head->next; 1009 free_verifier_state(&head->st, false); 1010 kfree(head); 1011 env->head = elem; 1012 env->stack_size--; 1013 return 0; 1014 } 1015 1016 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1017 int insn_idx, int prev_insn_idx, 1018 bool speculative) 1019 { 1020 struct bpf_verifier_state *cur = env->cur_state; 1021 struct bpf_verifier_stack_elem *elem; 1022 int err; 1023 1024 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1025 if (!elem) 1026 goto err; 1027 1028 elem->insn_idx = insn_idx; 1029 elem->prev_insn_idx = prev_insn_idx; 1030 elem->next = env->head; 1031 elem->log_pos = env->log.len_used; 1032 env->head = elem; 1033 env->stack_size++; 1034 err = copy_verifier_state(&elem->st, cur); 1035 if (err) 1036 goto err; 1037 elem->st.speculative |= speculative; 1038 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1039 verbose(env, "The sequence of %d jumps is too complex.\n", 1040 env->stack_size); 1041 goto err; 1042 } 1043 if (elem->st.parent) { 1044 ++elem->st.parent->branches; 1045 /* WARN_ON(branches > 2) technically makes sense here, 1046 * but 1047 * 1. speculative states will bump 'branches' for non-branch 1048 * instructions 1049 * 2. is_state_visited() heuristics may decide not to create 1050 * a new state for a sequence of branches and all such current 1051 * and cloned states will be pointing to a single parent state 1052 * which might have large 'branches' count. 1053 */ 1054 } 1055 return &elem->st; 1056 err: 1057 free_verifier_state(env->cur_state, true); 1058 env->cur_state = NULL; 1059 /* pop all elements and return */ 1060 while (!pop_stack(env, NULL, NULL, false)); 1061 return NULL; 1062 } 1063 1064 #define CALLER_SAVED_REGS 6 1065 static const int caller_saved[CALLER_SAVED_REGS] = { 1066 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1067 }; 1068 1069 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1070 struct bpf_reg_state *reg); 1071 1072 /* This helper doesn't clear reg->id */ 1073 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1074 { 1075 reg->var_off = tnum_const(imm); 1076 reg->smin_value = (s64)imm; 1077 reg->smax_value = (s64)imm; 1078 reg->umin_value = imm; 1079 reg->umax_value = imm; 1080 1081 reg->s32_min_value = (s32)imm; 1082 reg->s32_max_value = (s32)imm; 1083 reg->u32_min_value = (u32)imm; 1084 reg->u32_max_value = (u32)imm; 1085 } 1086 1087 /* Mark the unknown part of a register (variable offset or scalar value) as 1088 * known to have the value @imm. 1089 */ 1090 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1091 { 1092 /* Clear id, off, and union(map_ptr, range) */ 1093 memset(((u8 *)reg) + sizeof(reg->type), 0, 1094 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1095 ___mark_reg_known(reg, imm); 1096 } 1097 1098 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1099 { 1100 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1101 reg->s32_min_value = (s32)imm; 1102 reg->s32_max_value = (s32)imm; 1103 reg->u32_min_value = (u32)imm; 1104 reg->u32_max_value = (u32)imm; 1105 } 1106 1107 /* Mark the 'variable offset' part of a register as zero. This should be 1108 * used only on registers holding a pointer type. 1109 */ 1110 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1111 { 1112 __mark_reg_known(reg, 0); 1113 } 1114 1115 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1116 { 1117 __mark_reg_known(reg, 0); 1118 reg->type = SCALAR_VALUE; 1119 } 1120 1121 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1122 struct bpf_reg_state *regs, u32 regno) 1123 { 1124 if (WARN_ON(regno >= MAX_BPF_REG)) { 1125 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1126 /* Something bad happened, let's kill all regs */ 1127 for (regno = 0; regno < MAX_BPF_REG; regno++) 1128 __mark_reg_not_init(env, regs + regno); 1129 return; 1130 } 1131 __mark_reg_known_zero(regs + regno); 1132 } 1133 1134 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1135 { 1136 switch (reg->type) { 1137 case PTR_TO_MAP_VALUE_OR_NULL: { 1138 const struct bpf_map *map = reg->map_ptr; 1139 1140 if (map->inner_map_meta) { 1141 reg->type = CONST_PTR_TO_MAP; 1142 reg->map_ptr = map->inner_map_meta; 1143 /* transfer reg's id which is unique for every map_lookup_elem 1144 * as UID of the inner map. 1145 */ 1146 reg->map_uid = reg->id; 1147 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1148 reg->type = PTR_TO_XDP_SOCK; 1149 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1150 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1151 reg->type = PTR_TO_SOCKET; 1152 } else { 1153 reg->type = PTR_TO_MAP_VALUE; 1154 } 1155 break; 1156 } 1157 case PTR_TO_SOCKET_OR_NULL: 1158 reg->type = PTR_TO_SOCKET; 1159 break; 1160 case PTR_TO_SOCK_COMMON_OR_NULL: 1161 reg->type = PTR_TO_SOCK_COMMON; 1162 break; 1163 case PTR_TO_TCP_SOCK_OR_NULL: 1164 reg->type = PTR_TO_TCP_SOCK; 1165 break; 1166 case PTR_TO_BTF_ID_OR_NULL: 1167 reg->type = PTR_TO_BTF_ID; 1168 break; 1169 case PTR_TO_MEM_OR_NULL: 1170 reg->type = PTR_TO_MEM; 1171 break; 1172 case PTR_TO_RDONLY_BUF_OR_NULL: 1173 reg->type = PTR_TO_RDONLY_BUF; 1174 break; 1175 case PTR_TO_RDWR_BUF_OR_NULL: 1176 reg->type = PTR_TO_RDWR_BUF; 1177 break; 1178 default: 1179 WARN_ONCE(1, "unknown nullable register type"); 1180 } 1181 } 1182 1183 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1184 { 1185 return type_is_pkt_pointer(reg->type); 1186 } 1187 1188 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1189 { 1190 return reg_is_pkt_pointer(reg) || 1191 reg->type == PTR_TO_PACKET_END; 1192 } 1193 1194 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1195 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1196 enum bpf_reg_type which) 1197 { 1198 /* The register can already have a range from prior markings. 1199 * This is fine as long as it hasn't been advanced from its 1200 * origin. 1201 */ 1202 return reg->type == which && 1203 reg->id == 0 && 1204 reg->off == 0 && 1205 tnum_equals_const(reg->var_off, 0); 1206 } 1207 1208 /* Reset the min/max bounds of a register */ 1209 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1210 { 1211 reg->smin_value = S64_MIN; 1212 reg->smax_value = S64_MAX; 1213 reg->umin_value = 0; 1214 reg->umax_value = U64_MAX; 1215 1216 reg->s32_min_value = S32_MIN; 1217 reg->s32_max_value = S32_MAX; 1218 reg->u32_min_value = 0; 1219 reg->u32_max_value = U32_MAX; 1220 } 1221 1222 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1223 { 1224 reg->smin_value = S64_MIN; 1225 reg->smax_value = S64_MAX; 1226 reg->umin_value = 0; 1227 reg->umax_value = U64_MAX; 1228 } 1229 1230 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1231 { 1232 reg->s32_min_value = S32_MIN; 1233 reg->s32_max_value = S32_MAX; 1234 reg->u32_min_value = 0; 1235 reg->u32_max_value = U32_MAX; 1236 } 1237 1238 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1239 { 1240 struct tnum var32_off = tnum_subreg(reg->var_off); 1241 1242 /* min signed is max(sign bit) | min(other bits) */ 1243 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1244 var32_off.value | (var32_off.mask & S32_MIN)); 1245 /* max signed is min(sign bit) | max(other bits) */ 1246 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1247 var32_off.value | (var32_off.mask & S32_MAX)); 1248 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1249 reg->u32_max_value = min(reg->u32_max_value, 1250 (u32)(var32_off.value | var32_off.mask)); 1251 } 1252 1253 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1254 { 1255 /* min signed is max(sign bit) | min(other bits) */ 1256 reg->smin_value = max_t(s64, reg->smin_value, 1257 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1258 /* max signed is min(sign bit) | max(other bits) */ 1259 reg->smax_value = min_t(s64, reg->smax_value, 1260 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1261 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1262 reg->umax_value = min(reg->umax_value, 1263 reg->var_off.value | reg->var_off.mask); 1264 } 1265 1266 static void __update_reg_bounds(struct bpf_reg_state *reg) 1267 { 1268 __update_reg32_bounds(reg); 1269 __update_reg64_bounds(reg); 1270 } 1271 1272 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1273 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1274 { 1275 /* Learn sign from signed bounds. 1276 * If we cannot cross the sign boundary, then signed and unsigned bounds 1277 * are the same, so combine. This works even in the negative case, e.g. 1278 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1279 */ 1280 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1281 reg->s32_min_value = reg->u32_min_value = 1282 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1283 reg->s32_max_value = reg->u32_max_value = 1284 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1285 return; 1286 } 1287 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1288 * boundary, so we must be careful. 1289 */ 1290 if ((s32)reg->u32_max_value >= 0) { 1291 /* Positive. We can't learn anything from the smin, but smax 1292 * is positive, hence safe. 1293 */ 1294 reg->s32_min_value = reg->u32_min_value; 1295 reg->s32_max_value = reg->u32_max_value = 1296 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1297 } else if ((s32)reg->u32_min_value < 0) { 1298 /* Negative. We can't learn anything from the smax, but smin 1299 * is negative, hence safe. 1300 */ 1301 reg->s32_min_value = reg->u32_min_value = 1302 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1303 reg->s32_max_value = reg->u32_max_value; 1304 } 1305 } 1306 1307 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1308 { 1309 /* Learn sign from signed bounds. 1310 * If we cannot cross the sign boundary, then signed and unsigned bounds 1311 * are the same, so combine. This works even in the negative case, e.g. 1312 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1313 */ 1314 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1315 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1316 reg->umin_value); 1317 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1318 reg->umax_value); 1319 return; 1320 } 1321 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1322 * boundary, so we must be careful. 1323 */ 1324 if ((s64)reg->umax_value >= 0) { 1325 /* Positive. We can't learn anything from the smin, but smax 1326 * is positive, hence safe. 1327 */ 1328 reg->smin_value = reg->umin_value; 1329 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1330 reg->umax_value); 1331 } else if ((s64)reg->umin_value < 0) { 1332 /* Negative. We can't learn anything from the smax, but smin 1333 * is negative, hence safe. 1334 */ 1335 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1336 reg->umin_value); 1337 reg->smax_value = reg->umax_value; 1338 } 1339 } 1340 1341 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1342 { 1343 __reg32_deduce_bounds(reg); 1344 __reg64_deduce_bounds(reg); 1345 } 1346 1347 /* Attempts to improve var_off based on unsigned min/max information */ 1348 static void __reg_bound_offset(struct bpf_reg_state *reg) 1349 { 1350 struct tnum var64_off = tnum_intersect(reg->var_off, 1351 tnum_range(reg->umin_value, 1352 reg->umax_value)); 1353 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1354 tnum_range(reg->u32_min_value, 1355 reg->u32_max_value)); 1356 1357 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1358 } 1359 1360 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1361 { 1362 reg->umin_value = reg->u32_min_value; 1363 reg->umax_value = reg->u32_max_value; 1364 /* Attempt to pull 32-bit signed bounds into 64-bit bounds 1365 * but must be positive otherwise set to worse case bounds 1366 * and refine later from tnum. 1367 */ 1368 if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0) 1369 reg->smax_value = reg->s32_max_value; 1370 else 1371 reg->smax_value = U32_MAX; 1372 if (reg->s32_min_value >= 0) 1373 reg->smin_value = reg->s32_min_value; 1374 else 1375 reg->smin_value = 0; 1376 } 1377 1378 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1379 { 1380 /* special case when 64-bit register has upper 32-bit register 1381 * zeroed. Typically happens after zext or <<32, >>32 sequence 1382 * allowing us to use 32-bit bounds directly, 1383 */ 1384 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1385 __reg_assign_32_into_64(reg); 1386 } else { 1387 /* Otherwise the best we can do is push lower 32bit known and 1388 * unknown bits into register (var_off set from jmp logic) 1389 * then learn as much as possible from the 64-bit tnum 1390 * known and unknown bits. The previous smin/smax bounds are 1391 * invalid here because of jmp32 compare so mark them unknown 1392 * so they do not impact tnum bounds calculation. 1393 */ 1394 __mark_reg64_unbounded(reg); 1395 __update_reg_bounds(reg); 1396 } 1397 1398 /* Intersecting with the old var_off might have improved our bounds 1399 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1400 * then new var_off is (0; 0x7f...fc) which improves our umax. 1401 */ 1402 __reg_deduce_bounds(reg); 1403 __reg_bound_offset(reg); 1404 __update_reg_bounds(reg); 1405 } 1406 1407 static bool __reg64_bound_s32(s64 a) 1408 { 1409 return a > S32_MIN && a < S32_MAX; 1410 } 1411 1412 static bool __reg64_bound_u32(u64 a) 1413 { 1414 return a > U32_MIN && a < U32_MAX; 1415 } 1416 1417 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1418 { 1419 __mark_reg32_unbounded(reg); 1420 1421 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1422 reg->s32_min_value = (s32)reg->smin_value; 1423 reg->s32_max_value = (s32)reg->smax_value; 1424 } 1425 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 1426 reg->u32_min_value = (u32)reg->umin_value; 1427 reg->u32_max_value = (u32)reg->umax_value; 1428 } 1429 1430 /* Intersecting with the old var_off might have improved our bounds 1431 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1432 * then new var_off is (0; 0x7f...fc) which improves our umax. 1433 */ 1434 __reg_deduce_bounds(reg); 1435 __reg_bound_offset(reg); 1436 __update_reg_bounds(reg); 1437 } 1438 1439 /* Mark a register as having a completely unknown (scalar) value. */ 1440 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1441 struct bpf_reg_state *reg) 1442 { 1443 /* 1444 * Clear type, id, off, and union(map_ptr, range) and 1445 * padding between 'type' and union 1446 */ 1447 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1448 reg->type = SCALAR_VALUE; 1449 reg->var_off = tnum_unknown; 1450 reg->frameno = 0; 1451 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; 1452 __mark_reg_unbounded(reg); 1453 } 1454 1455 static void mark_reg_unknown(struct bpf_verifier_env *env, 1456 struct bpf_reg_state *regs, u32 regno) 1457 { 1458 if (WARN_ON(regno >= MAX_BPF_REG)) { 1459 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1460 /* Something bad happened, let's kill all regs except FP */ 1461 for (regno = 0; regno < BPF_REG_FP; regno++) 1462 __mark_reg_not_init(env, regs + regno); 1463 return; 1464 } 1465 __mark_reg_unknown(env, regs + regno); 1466 } 1467 1468 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1469 struct bpf_reg_state *reg) 1470 { 1471 __mark_reg_unknown(env, reg); 1472 reg->type = NOT_INIT; 1473 } 1474 1475 static void mark_reg_not_init(struct bpf_verifier_env *env, 1476 struct bpf_reg_state *regs, u32 regno) 1477 { 1478 if (WARN_ON(regno >= MAX_BPF_REG)) { 1479 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1480 /* Something bad happened, let's kill all regs except FP */ 1481 for (regno = 0; regno < BPF_REG_FP; regno++) 1482 __mark_reg_not_init(env, regs + regno); 1483 return; 1484 } 1485 __mark_reg_not_init(env, regs + regno); 1486 } 1487 1488 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 1489 struct bpf_reg_state *regs, u32 regno, 1490 enum bpf_reg_type reg_type, 1491 struct btf *btf, u32 btf_id) 1492 { 1493 if (reg_type == SCALAR_VALUE) { 1494 mark_reg_unknown(env, regs, regno); 1495 return; 1496 } 1497 mark_reg_known_zero(env, regs, regno); 1498 regs[regno].type = PTR_TO_BTF_ID; 1499 regs[regno].btf = btf; 1500 regs[regno].btf_id = btf_id; 1501 } 1502 1503 #define DEF_NOT_SUBREG (0) 1504 static void init_reg_state(struct bpf_verifier_env *env, 1505 struct bpf_func_state *state) 1506 { 1507 struct bpf_reg_state *regs = state->regs; 1508 int i; 1509 1510 for (i = 0; i < MAX_BPF_REG; i++) { 1511 mark_reg_not_init(env, regs, i); 1512 regs[i].live = REG_LIVE_NONE; 1513 regs[i].parent = NULL; 1514 regs[i].subreg_def = DEF_NOT_SUBREG; 1515 } 1516 1517 /* frame pointer */ 1518 regs[BPF_REG_FP].type = PTR_TO_STACK; 1519 mark_reg_known_zero(env, regs, BPF_REG_FP); 1520 regs[BPF_REG_FP].frameno = state->frameno; 1521 } 1522 1523 #define BPF_MAIN_FUNC (-1) 1524 static void init_func_state(struct bpf_verifier_env *env, 1525 struct bpf_func_state *state, 1526 int callsite, int frameno, int subprogno) 1527 { 1528 state->callsite = callsite; 1529 state->frameno = frameno; 1530 state->subprogno = subprogno; 1531 init_reg_state(env, state); 1532 } 1533 1534 /* Similar to push_stack(), but for async callbacks */ 1535 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 1536 int insn_idx, int prev_insn_idx, 1537 int subprog) 1538 { 1539 struct bpf_verifier_stack_elem *elem; 1540 struct bpf_func_state *frame; 1541 1542 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1543 if (!elem) 1544 goto err; 1545 1546 elem->insn_idx = insn_idx; 1547 elem->prev_insn_idx = prev_insn_idx; 1548 elem->next = env->head; 1549 elem->log_pos = env->log.len_used; 1550 env->head = elem; 1551 env->stack_size++; 1552 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1553 verbose(env, 1554 "The sequence of %d jumps is too complex for async cb.\n", 1555 env->stack_size); 1556 goto err; 1557 } 1558 /* Unlike push_stack() do not copy_verifier_state(). 1559 * The caller state doesn't matter. 1560 * This is async callback. It starts in a fresh stack. 1561 * Initialize it similar to do_check_common(). 1562 */ 1563 elem->st.branches = 1; 1564 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 1565 if (!frame) 1566 goto err; 1567 init_func_state(env, frame, 1568 BPF_MAIN_FUNC /* callsite */, 1569 0 /* frameno within this callchain */, 1570 subprog /* subprog number within this prog */); 1571 elem->st.frame[0] = frame; 1572 return &elem->st; 1573 err: 1574 free_verifier_state(env->cur_state, true); 1575 env->cur_state = NULL; 1576 /* pop all elements and return */ 1577 while (!pop_stack(env, NULL, NULL, false)); 1578 return NULL; 1579 } 1580 1581 1582 enum reg_arg_type { 1583 SRC_OP, /* register is used as source operand */ 1584 DST_OP, /* register is used as destination operand */ 1585 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1586 }; 1587 1588 static int cmp_subprogs(const void *a, const void *b) 1589 { 1590 return ((struct bpf_subprog_info *)a)->start - 1591 ((struct bpf_subprog_info *)b)->start; 1592 } 1593 1594 static int find_subprog(struct bpf_verifier_env *env, int off) 1595 { 1596 struct bpf_subprog_info *p; 1597 1598 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1599 sizeof(env->subprog_info[0]), cmp_subprogs); 1600 if (!p) 1601 return -ENOENT; 1602 return p - env->subprog_info; 1603 1604 } 1605 1606 static int add_subprog(struct bpf_verifier_env *env, int off) 1607 { 1608 int insn_cnt = env->prog->len; 1609 int ret; 1610 1611 if (off >= insn_cnt || off < 0) { 1612 verbose(env, "call to invalid destination\n"); 1613 return -EINVAL; 1614 } 1615 ret = find_subprog(env, off); 1616 if (ret >= 0) 1617 return ret; 1618 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1619 verbose(env, "too many subprograms\n"); 1620 return -E2BIG; 1621 } 1622 /* determine subprog starts. The end is one before the next starts */ 1623 env->subprog_info[env->subprog_cnt++].start = off; 1624 sort(env->subprog_info, env->subprog_cnt, 1625 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1626 return env->subprog_cnt - 1; 1627 } 1628 1629 struct bpf_kfunc_desc { 1630 struct btf_func_model func_model; 1631 u32 func_id; 1632 s32 imm; 1633 }; 1634 1635 #define MAX_KFUNC_DESCS 256 1636 struct bpf_kfunc_desc_tab { 1637 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 1638 u32 nr_descs; 1639 }; 1640 1641 static int kfunc_desc_cmp_by_id(const void *a, const void *b) 1642 { 1643 const struct bpf_kfunc_desc *d0 = a; 1644 const struct bpf_kfunc_desc *d1 = b; 1645 1646 /* func_id is not greater than BTF_MAX_TYPE */ 1647 return d0->func_id - d1->func_id; 1648 } 1649 1650 static const struct bpf_kfunc_desc * 1651 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id) 1652 { 1653 struct bpf_kfunc_desc desc = { 1654 .func_id = func_id, 1655 }; 1656 struct bpf_kfunc_desc_tab *tab; 1657 1658 tab = prog->aux->kfunc_tab; 1659 return bsearch(&desc, tab->descs, tab->nr_descs, 1660 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id); 1661 } 1662 1663 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id) 1664 { 1665 const struct btf_type *func, *func_proto; 1666 struct bpf_kfunc_desc_tab *tab; 1667 struct bpf_prog_aux *prog_aux; 1668 struct bpf_kfunc_desc *desc; 1669 const char *func_name; 1670 unsigned long addr; 1671 int err; 1672 1673 prog_aux = env->prog->aux; 1674 tab = prog_aux->kfunc_tab; 1675 if (!tab) { 1676 if (!btf_vmlinux) { 1677 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 1678 return -ENOTSUPP; 1679 } 1680 1681 if (!env->prog->jit_requested) { 1682 verbose(env, "JIT is required for calling kernel function\n"); 1683 return -ENOTSUPP; 1684 } 1685 1686 if (!bpf_jit_supports_kfunc_call()) { 1687 verbose(env, "JIT does not support calling kernel function\n"); 1688 return -ENOTSUPP; 1689 } 1690 1691 if (!env->prog->gpl_compatible) { 1692 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 1693 return -EINVAL; 1694 } 1695 1696 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 1697 if (!tab) 1698 return -ENOMEM; 1699 prog_aux->kfunc_tab = tab; 1700 } 1701 1702 if (find_kfunc_desc(env->prog, func_id)) 1703 return 0; 1704 1705 if (tab->nr_descs == MAX_KFUNC_DESCS) { 1706 verbose(env, "too many different kernel function calls\n"); 1707 return -E2BIG; 1708 } 1709 1710 func = btf_type_by_id(btf_vmlinux, func_id); 1711 if (!func || !btf_type_is_func(func)) { 1712 verbose(env, "kernel btf_id %u is not a function\n", 1713 func_id); 1714 return -EINVAL; 1715 } 1716 func_proto = btf_type_by_id(btf_vmlinux, func->type); 1717 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 1718 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 1719 func_id); 1720 return -EINVAL; 1721 } 1722 1723 func_name = btf_name_by_offset(btf_vmlinux, func->name_off); 1724 addr = kallsyms_lookup_name(func_name); 1725 if (!addr) { 1726 verbose(env, "cannot find address for kernel function %s\n", 1727 func_name); 1728 return -EINVAL; 1729 } 1730 1731 desc = &tab->descs[tab->nr_descs++]; 1732 desc->func_id = func_id; 1733 desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base; 1734 err = btf_distill_func_proto(&env->log, btf_vmlinux, 1735 func_proto, func_name, 1736 &desc->func_model); 1737 if (!err) 1738 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1739 kfunc_desc_cmp_by_id, NULL); 1740 return err; 1741 } 1742 1743 static int kfunc_desc_cmp_by_imm(const void *a, const void *b) 1744 { 1745 const struct bpf_kfunc_desc *d0 = a; 1746 const struct bpf_kfunc_desc *d1 = b; 1747 1748 if (d0->imm > d1->imm) 1749 return 1; 1750 else if (d0->imm < d1->imm) 1751 return -1; 1752 return 0; 1753 } 1754 1755 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) 1756 { 1757 struct bpf_kfunc_desc_tab *tab; 1758 1759 tab = prog->aux->kfunc_tab; 1760 if (!tab) 1761 return; 1762 1763 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1764 kfunc_desc_cmp_by_imm, NULL); 1765 } 1766 1767 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 1768 { 1769 return !!prog->aux->kfunc_tab; 1770 } 1771 1772 const struct btf_func_model * 1773 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 1774 const struct bpf_insn *insn) 1775 { 1776 const struct bpf_kfunc_desc desc = { 1777 .imm = insn->imm, 1778 }; 1779 const struct bpf_kfunc_desc *res; 1780 struct bpf_kfunc_desc_tab *tab; 1781 1782 tab = prog->aux->kfunc_tab; 1783 res = bsearch(&desc, tab->descs, tab->nr_descs, 1784 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); 1785 1786 return res ? &res->func_model : NULL; 1787 } 1788 1789 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 1790 { 1791 struct bpf_subprog_info *subprog = env->subprog_info; 1792 struct bpf_insn *insn = env->prog->insnsi; 1793 int i, ret, insn_cnt = env->prog->len; 1794 1795 /* Add entry function. */ 1796 ret = add_subprog(env, 0); 1797 if (ret) 1798 return ret; 1799 1800 for (i = 0; i < insn_cnt; i++, insn++) { 1801 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 1802 !bpf_pseudo_kfunc_call(insn)) 1803 continue; 1804 1805 if (!env->bpf_capable) { 1806 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 1807 return -EPERM; 1808 } 1809 1810 if (bpf_pseudo_func(insn)) { 1811 ret = add_subprog(env, i + insn->imm + 1); 1812 if (ret >= 0) 1813 /* remember subprog */ 1814 insn[1].imm = ret; 1815 } else if (bpf_pseudo_call(insn)) { 1816 ret = add_subprog(env, i + insn->imm + 1); 1817 } else { 1818 ret = add_kfunc_call(env, insn->imm); 1819 } 1820 1821 if (ret < 0) 1822 return ret; 1823 } 1824 1825 /* Add a fake 'exit' subprog which could simplify subprog iteration 1826 * logic. 'subprog_cnt' should not be increased. 1827 */ 1828 subprog[env->subprog_cnt].start = insn_cnt; 1829 1830 if (env->log.level & BPF_LOG_LEVEL2) 1831 for (i = 0; i < env->subprog_cnt; i++) 1832 verbose(env, "func#%d @%d\n", i, subprog[i].start); 1833 1834 return 0; 1835 } 1836 1837 static int check_subprogs(struct bpf_verifier_env *env) 1838 { 1839 int i, subprog_start, subprog_end, off, cur_subprog = 0; 1840 struct bpf_subprog_info *subprog = env->subprog_info; 1841 struct bpf_insn *insn = env->prog->insnsi; 1842 int insn_cnt = env->prog->len; 1843 1844 /* now check that all jumps are within the same subprog */ 1845 subprog_start = subprog[cur_subprog].start; 1846 subprog_end = subprog[cur_subprog + 1].start; 1847 for (i = 0; i < insn_cnt; i++) { 1848 u8 code = insn[i].code; 1849 1850 if (code == (BPF_JMP | BPF_CALL) && 1851 insn[i].imm == BPF_FUNC_tail_call && 1852 insn[i].src_reg != BPF_PSEUDO_CALL) 1853 subprog[cur_subprog].has_tail_call = true; 1854 if (BPF_CLASS(code) == BPF_LD && 1855 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 1856 subprog[cur_subprog].has_ld_abs = true; 1857 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 1858 goto next; 1859 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 1860 goto next; 1861 off = i + insn[i].off + 1; 1862 if (off < subprog_start || off >= subprog_end) { 1863 verbose(env, "jump out of range from insn %d to %d\n", i, off); 1864 return -EINVAL; 1865 } 1866 next: 1867 if (i == subprog_end - 1) { 1868 /* to avoid fall-through from one subprog into another 1869 * the last insn of the subprog should be either exit 1870 * or unconditional jump back 1871 */ 1872 if (code != (BPF_JMP | BPF_EXIT) && 1873 code != (BPF_JMP | BPF_JA)) { 1874 verbose(env, "last insn is not an exit or jmp\n"); 1875 return -EINVAL; 1876 } 1877 subprog_start = subprog_end; 1878 cur_subprog++; 1879 if (cur_subprog < env->subprog_cnt) 1880 subprog_end = subprog[cur_subprog + 1].start; 1881 } 1882 } 1883 return 0; 1884 } 1885 1886 /* Parentage chain of this register (or stack slot) should take care of all 1887 * issues like callee-saved registers, stack slot allocation time, etc. 1888 */ 1889 static int mark_reg_read(struct bpf_verifier_env *env, 1890 const struct bpf_reg_state *state, 1891 struct bpf_reg_state *parent, u8 flag) 1892 { 1893 bool writes = parent == state->parent; /* Observe write marks */ 1894 int cnt = 0; 1895 1896 while (parent) { 1897 /* if read wasn't screened by an earlier write ... */ 1898 if (writes && state->live & REG_LIVE_WRITTEN) 1899 break; 1900 if (parent->live & REG_LIVE_DONE) { 1901 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 1902 reg_type_str[parent->type], 1903 parent->var_off.value, parent->off); 1904 return -EFAULT; 1905 } 1906 /* The first condition is more likely to be true than the 1907 * second, checked it first. 1908 */ 1909 if ((parent->live & REG_LIVE_READ) == flag || 1910 parent->live & REG_LIVE_READ64) 1911 /* The parentage chain never changes and 1912 * this parent was already marked as LIVE_READ. 1913 * There is no need to keep walking the chain again and 1914 * keep re-marking all parents as LIVE_READ. 1915 * This case happens when the same register is read 1916 * multiple times without writes into it in-between. 1917 * Also, if parent has the stronger REG_LIVE_READ64 set, 1918 * then no need to set the weak REG_LIVE_READ32. 1919 */ 1920 break; 1921 /* ... then we depend on parent's value */ 1922 parent->live |= flag; 1923 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 1924 if (flag == REG_LIVE_READ64) 1925 parent->live &= ~REG_LIVE_READ32; 1926 state = parent; 1927 parent = state->parent; 1928 writes = true; 1929 cnt++; 1930 } 1931 1932 if (env->longest_mark_read_walk < cnt) 1933 env->longest_mark_read_walk = cnt; 1934 return 0; 1935 } 1936 1937 /* This function is supposed to be used by the following 32-bit optimization 1938 * code only. It returns TRUE if the source or destination register operates 1939 * on 64-bit, otherwise return FALSE. 1940 */ 1941 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 1942 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 1943 { 1944 u8 code, class, op; 1945 1946 code = insn->code; 1947 class = BPF_CLASS(code); 1948 op = BPF_OP(code); 1949 if (class == BPF_JMP) { 1950 /* BPF_EXIT for "main" will reach here. Return TRUE 1951 * conservatively. 1952 */ 1953 if (op == BPF_EXIT) 1954 return true; 1955 if (op == BPF_CALL) { 1956 /* BPF to BPF call will reach here because of marking 1957 * caller saved clobber with DST_OP_NO_MARK for which we 1958 * don't care the register def because they are anyway 1959 * marked as NOT_INIT already. 1960 */ 1961 if (insn->src_reg == BPF_PSEUDO_CALL) 1962 return false; 1963 /* Helper call will reach here because of arg type 1964 * check, conservatively return TRUE. 1965 */ 1966 if (t == SRC_OP) 1967 return true; 1968 1969 return false; 1970 } 1971 } 1972 1973 if (class == BPF_ALU64 || class == BPF_JMP || 1974 /* BPF_END always use BPF_ALU class. */ 1975 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 1976 return true; 1977 1978 if (class == BPF_ALU || class == BPF_JMP32) 1979 return false; 1980 1981 if (class == BPF_LDX) { 1982 if (t != SRC_OP) 1983 return BPF_SIZE(code) == BPF_DW; 1984 /* LDX source must be ptr. */ 1985 return true; 1986 } 1987 1988 if (class == BPF_STX) { 1989 /* BPF_STX (including atomic variants) has multiple source 1990 * operands, one of which is a ptr. Check whether the caller is 1991 * asking about it. 1992 */ 1993 if (t == SRC_OP && reg->type != SCALAR_VALUE) 1994 return true; 1995 return BPF_SIZE(code) == BPF_DW; 1996 } 1997 1998 if (class == BPF_LD) { 1999 u8 mode = BPF_MODE(code); 2000 2001 /* LD_IMM64 */ 2002 if (mode == BPF_IMM) 2003 return true; 2004 2005 /* Both LD_IND and LD_ABS return 32-bit data. */ 2006 if (t != SRC_OP) 2007 return false; 2008 2009 /* Implicit ctx ptr. */ 2010 if (regno == BPF_REG_6) 2011 return true; 2012 2013 /* Explicit source could be any width. */ 2014 return true; 2015 } 2016 2017 if (class == BPF_ST) 2018 /* The only source register for BPF_ST is a ptr. */ 2019 return true; 2020 2021 /* Conservatively return true at default. */ 2022 return true; 2023 } 2024 2025 /* Return the regno defined by the insn, or -1. */ 2026 static int insn_def_regno(const struct bpf_insn *insn) 2027 { 2028 switch (BPF_CLASS(insn->code)) { 2029 case BPF_JMP: 2030 case BPF_JMP32: 2031 case BPF_ST: 2032 return -1; 2033 case BPF_STX: 2034 if (BPF_MODE(insn->code) == BPF_ATOMIC && 2035 (insn->imm & BPF_FETCH)) { 2036 if (insn->imm == BPF_CMPXCHG) 2037 return BPF_REG_0; 2038 else 2039 return insn->src_reg; 2040 } else { 2041 return -1; 2042 } 2043 default: 2044 return insn->dst_reg; 2045 } 2046 } 2047 2048 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 2049 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 2050 { 2051 int dst_reg = insn_def_regno(insn); 2052 2053 if (dst_reg == -1) 2054 return false; 2055 2056 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 2057 } 2058 2059 static void mark_insn_zext(struct bpf_verifier_env *env, 2060 struct bpf_reg_state *reg) 2061 { 2062 s32 def_idx = reg->subreg_def; 2063 2064 if (def_idx == DEF_NOT_SUBREG) 2065 return; 2066 2067 env->insn_aux_data[def_idx - 1].zext_dst = true; 2068 /* The dst will be zero extended, so won't be sub-register anymore. */ 2069 reg->subreg_def = DEF_NOT_SUBREG; 2070 } 2071 2072 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 2073 enum reg_arg_type t) 2074 { 2075 struct bpf_verifier_state *vstate = env->cur_state; 2076 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2077 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 2078 struct bpf_reg_state *reg, *regs = state->regs; 2079 bool rw64; 2080 2081 if (regno >= MAX_BPF_REG) { 2082 verbose(env, "R%d is invalid\n", regno); 2083 return -EINVAL; 2084 } 2085 2086 reg = ®s[regno]; 2087 rw64 = is_reg64(env, insn, regno, reg, t); 2088 if (t == SRC_OP) { 2089 /* check whether register used as source operand can be read */ 2090 if (reg->type == NOT_INIT) { 2091 verbose(env, "R%d !read_ok\n", regno); 2092 return -EACCES; 2093 } 2094 /* We don't need to worry about FP liveness because it's read-only */ 2095 if (regno == BPF_REG_FP) 2096 return 0; 2097 2098 if (rw64) 2099 mark_insn_zext(env, reg); 2100 2101 return mark_reg_read(env, reg, reg->parent, 2102 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 2103 } else { 2104 /* check whether register used as dest operand can be written to */ 2105 if (regno == BPF_REG_FP) { 2106 verbose(env, "frame pointer is read only\n"); 2107 return -EACCES; 2108 } 2109 reg->live |= REG_LIVE_WRITTEN; 2110 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 2111 if (t == DST_OP) 2112 mark_reg_unknown(env, regs, regno); 2113 } 2114 return 0; 2115 } 2116 2117 /* for any branch, call, exit record the history of jmps in the given state */ 2118 static int push_jmp_history(struct bpf_verifier_env *env, 2119 struct bpf_verifier_state *cur) 2120 { 2121 u32 cnt = cur->jmp_history_cnt; 2122 struct bpf_idx_pair *p; 2123 2124 cnt++; 2125 p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); 2126 if (!p) 2127 return -ENOMEM; 2128 p[cnt - 1].idx = env->insn_idx; 2129 p[cnt - 1].prev_idx = env->prev_insn_idx; 2130 cur->jmp_history = p; 2131 cur->jmp_history_cnt = cnt; 2132 return 0; 2133 } 2134 2135 /* Backtrack one insn at a time. If idx is not at the top of recorded 2136 * history then previous instruction came from straight line execution. 2137 */ 2138 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 2139 u32 *history) 2140 { 2141 u32 cnt = *history; 2142 2143 if (cnt && st->jmp_history[cnt - 1].idx == i) { 2144 i = st->jmp_history[cnt - 1].prev_idx; 2145 (*history)--; 2146 } else { 2147 i--; 2148 } 2149 return i; 2150 } 2151 2152 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 2153 { 2154 const struct btf_type *func; 2155 2156 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 2157 return NULL; 2158 2159 func = btf_type_by_id(btf_vmlinux, insn->imm); 2160 return btf_name_by_offset(btf_vmlinux, func->name_off); 2161 } 2162 2163 /* For given verifier state backtrack_insn() is called from the last insn to 2164 * the first insn. Its purpose is to compute a bitmask of registers and 2165 * stack slots that needs precision in the parent verifier state. 2166 */ 2167 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 2168 u32 *reg_mask, u64 *stack_mask) 2169 { 2170 const struct bpf_insn_cbs cbs = { 2171 .cb_call = disasm_kfunc_name, 2172 .cb_print = verbose, 2173 .private_data = env, 2174 }; 2175 struct bpf_insn *insn = env->prog->insnsi + idx; 2176 u8 class = BPF_CLASS(insn->code); 2177 u8 opcode = BPF_OP(insn->code); 2178 u8 mode = BPF_MODE(insn->code); 2179 u32 dreg = 1u << insn->dst_reg; 2180 u32 sreg = 1u << insn->src_reg; 2181 u32 spi; 2182 2183 if (insn->code == 0) 2184 return 0; 2185 if (env->log.level & BPF_LOG_LEVEL) { 2186 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 2187 verbose(env, "%d: ", idx); 2188 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 2189 } 2190 2191 if (class == BPF_ALU || class == BPF_ALU64) { 2192 if (!(*reg_mask & dreg)) 2193 return 0; 2194 if (opcode == BPF_MOV) { 2195 if (BPF_SRC(insn->code) == BPF_X) { 2196 /* dreg = sreg 2197 * dreg needs precision after this insn 2198 * sreg needs precision before this insn 2199 */ 2200 *reg_mask &= ~dreg; 2201 *reg_mask |= sreg; 2202 } else { 2203 /* dreg = K 2204 * dreg needs precision after this insn. 2205 * Corresponding register is already marked 2206 * as precise=true in this verifier state. 2207 * No further markings in parent are necessary 2208 */ 2209 *reg_mask &= ~dreg; 2210 } 2211 } else { 2212 if (BPF_SRC(insn->code) == BPF_X) { 2213 /* dreg += sreg 2214 * both dreg and sreg need precision 2215 * before this insn 2216 */ 2217 *reg_mask |= sreg; 2218 } /* else dreg += K 2219 * dreg still needs precision before this insn 2220 */ 2221 } 2222 } else if (class == BPF_LDX) { 2223 if (!(*reg_mask & dreg)) 2224 return 0; 2225 *reg_mask &= ~dreg; 2226 2227 /* scalars can only be spilled into stack w/o losing precision. 2228 * Load from any other memory can be zero extended. 2229 * The desire to keep that precision is already indicated 2230 * by 'precise' mark in corresponding register of this state. 2231 * No further tracking necessary. 2232 */ 2233 if (insn->src_reg != BPF_REG_FP) 2234 return 0; 2235 if (BPF_SIZE(insn->code) != BPF_DW) 2236 return 0; 2237 2238 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 2239 * that [fp - off] slot contains scalar that needs to be 2240 * tracked with precision 2241 */ 2242 spi = (-insn->off - 1) / BPF_REG_SIZE; 2243 if (spi >= 64) { 2244 verbose(env, "BUG spi %d\n", spi); 2245 WARN_ONCE(1, "verifier backtracking bug"); 2246 return -EFAULT; 2247 } 2248 *stack_mask |= 1ull << spi; 2249 } else if (class == BPF_STX || class == BPF_ST) { 2250 if (*reg_mask & dreg) 2251 /* stx & st shouldn't be using _scalar_ dst_reg 2252 * to access memory. It means backtracking 2253 * encountered a case of pointer subtraction. 2254 */ 2255 return -ENOTSUPP; 2256 /* scalars can only be spilled into stack */ 2257 if (insn->dst_reg != BPF_REG_FP) 2258 return 0; 2259 if (BPF_SIZE(insn->code) != BPF_DW) 2260 return 0; 2261 spi = (-insn->off - 1) / BPF_REG_SIZE; 2262 if (spi >= 64) { 2263 verbose(env, "BUG spi %d\n", spi); 2264 WARN_ONCE(1, "verifier backtracking bug"); 2265 return -EFAULT; 2266 } 2267 if (!(*stack_mask & (1ull << spi))) 2268 return 0; 2269 *stack_mask &= ~(1ull << spi); 2270 if (class == BPF_STX) 2271 *reg_mask |= sreg; 2272 } else if (class == BPF_JMP || class == BPF_JMP32) { 2273 if (opcode == BPF_CALL) { 2274 if (insn->src_reg == BPF_PSEUDO_CALL) 2275 return -ENOTSUPP; 2276 /* regular helper call sets R0 */ 2277 *reg_mask &= ~1; 2278 if (*reg_mask & 0x3f) { 2279 /* if backtracing was looking for registers R1-R5 2280 * they should have been found already. 2281 */ 2282 verbose(env, "BUG regs %x\n", *reg_mask); 2283 WARN_ONCE(1, "verifier backtracking bug"); 2284 return -EFAULT; 2285 } 2286 } else if (opcode == BPF_EXIT) { 2287 return -ENOTSUPP; 2288 } 2289 } else if (class == BPF_LD) { 2290 if (!(*reg_mask & dreg)) 2291 return 0; 2292 *reg_mask &= ~dreg; 2293 /* It's ld_imm64 or ld_abs or ld_ind. 2294 * For ld_imm64 no further tracking of precision 2295 * into parent is necessary 2296 */ 2297 if (mode == BPF_IND || mode == BPF_ABS) 2298 /* to be analyzed */ 2299 return -ENOTSUPP; 2300 } 2301 return 0; 2302 } 2303 2304 /* the scalar precision tracking algorithm: 2305 * . at the start all registers have precise=false. 2306 * . scalar ranges are tracked as normal through alu and jmp insns. 2307 * . once precise value of the scalar register is used in: 2308 * . ptr + scalar alu 2309 * . if (scalar cond K|scalar) 2310 * . helper_call(.., scalar, ...) where ARG_CONST is expected 2311 * backtrack through the verifier states and mark all registers and 2312 * stack slots with spilled constants that these scalar regisers 2313 * should be precise. 2314 * . during state pruning two registers (or spilled stack slots) 2315 * are equivalent if both are not precise. 2316 * 2317 * Note the verifier cannot simply walk register parentage chain, 2318 * since many different registers and stack slots could have been 2319 * used to compute single precise scalar. 2320 * 2321 * The approach of starting with precise=true for all registers and then 2322 * backtrack to mark a register as not precise when the verifier detects 2323 * that program doesn't care about specific value (e.g., when helper 2324 * takes register as ARG_ANYTHING parameter) is not safe. 2325 * 2326 * It's ok to walk single parentage chain of the verifier states. 2327 * It's possible that this backtracking will go all the way till 1st insn. 2328 * All other branches will be explored for needing precision later. 2329 * 2330 * The backtracking needs to deal with cases like: 2331 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 2332 * r9 -= r8 2333 * r5 = r9 2334 * if r5 > 0x79f goto pc+7 2335 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 2336 * r5 += 1 2337 * ... 2338 * call bpf_perf_event_output#25 2339 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 2340 * 2341 * and this case: 2342 * r6 = 1 2343 * call foo // uses callee's r6 inside to compute r0 2344 * r0 += r6 2345 * if r0 == 0 goto 2346 * 2347 * to track above reg_mask/stack_mask needs to be independent for each frame. 2348 * 2349 * Also if parent's curframe > frame where backtracking started, 2350 * the verifier need to mark registers in both frames, otherwise callees 2351 * may incorrectly prune callers. This is similar to 2352 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 2353 * 2354 * For now backtracking falls back into conservative marking. 2355 */ 2356 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 2357 struct bpf_verifier_state *st) 2358 { 2359 struct bpf_func_state *func; 2360 struct bpf_reg_state *reg; 2361 int i, j; 2362 2363 /* big hammer: mark all scalars precise in this path. 2364 * pop_stack may still get !precise scalars. 2365 */ 2366 for (; st; st = st->parent) 2367 for (i = 0; i <= st->curframe; i++) { 2368 func = st->frame[i]; 2369 for (j = 0; j < BPF_REG_FP; j++) { 2370 reg = &func->regs[j]; 2371 if (reg->type != SCALAR_VALUE) 2372 continue; 2373 reg->precise = true; 2374 } 2375 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2376 if (func->stack[j].slot_type[0] != STACK_SPILL) 2377 continue; 2378 reg = &func->stack[j].spilled_ptr; 2379 if (reg->type != SCALAR_VALUE) 2380 continue; 2381 reg->precise = true; 2382 } 2383 } 2384 } 2385 2386 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, 2387 int spi) 2388 { 2389 struct bpf_verifier_state *st = env->cur_state; 2390 int first_idx = st->first_insn_idx; 2391 int last_idx = env->insn_idx; 2392 struct bpf_func_state *func; 2393 struct bpf_reg_state *reg; 2394 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 2395 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 2396 bool skip_first = true; 2397 bool new_marks = false; 2398 int i, err; 2399 2400 if (!env->bpf_capable) 2401 return 0; 2402 2403 func = st->frame[st->curframe]; 2404 if (regno >= 0) { 2405 reg = &func->regs[regno]; 2406 if (reg->type != SCALAR_VALUE) { 2407 WARN_ONCE(1, "backtracing misuse"); 2408 return -EFAULT; 2409 } 2410 if (!reg->precise) 2411 new_marks = true; 2412 else 2413 reg_mask = 0; 2414 reg->precise = true; 2415 } 2416 2417 while (spi >= 0) { 2418 if (func->stack[spi].slot_type[0] != STACK_SPILL) { 2419 stack_mask = 0; 2420 break; 2421 } 2422 reg = &func->stack[spi].spilled_ptr; 2423 if (reg->type != SCALAR_VALUE) { 2424 stack_mask = 0; 2425 break; 2426 } 2427 if (!reg->precise) 2428 new_marks = true; 2429 else 2430 stack_mask = 0; 2431 reg->precise = true; 2432 break; 2433 } 2434 2435 if (!new_marks) 2436 return 0; 2437 if (!reg_mask && !stack_mask) 2438 return 0; 2439 for (;;) { 2440 DECLARE_BITMAP(mask, 64); 2441 u32 history = st->jmp_history_cnt; 2442 2443 if (env->log.level & BPF_LOG_LEVEL) 2444 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 2445 for (i = last_idx;;) { 2446 if (skip_first) { 2447 err = 0; 2448 skip_first = false; 2449 } else { 2450 err = backtrack_insn(env, i, ®_mask, &stack_mask); 2451 } 2452 if (err == -ENOTSUPP) { 2453 mark_all_scalars_precise(env, st); 2454 return 0; 2455 } else if (err) { 2456 return err; 2457 } 2458 if (!reg_mask && !stack_mask) 2459 /* Found assignment(s) into tracked register in this state. 2460 * Since this state is already marked, just return. 2461 * Nothing to be tracked further in the parent state. 2462 */ 2463 return 0; 2464 if (i == first_idx) 2465 break; 2466 i = get_prev_insn_idx(st, i, &history); 2467 if (i >= env->prog->len) { 2468 /* This can happen if backtracking reached insn 0 2469 * and there are still reg_mask or stack_mask 2470 * to backtrack. 2471 * It means the backtracking missed the spot where 2472 * particular register was initialized with a constant. 2473 */ 2474 verbose(env, "BUG backtracking idx %d\n", i); 2475 WARN_ONCE(1, "verifier backtracking bug"); 2476 return -EFAULT; 2477 } 2478 } 2479 st = st->parent; 2480 if (!st) 2481 break; 2482 2483 new_marks = false; 2484 func = st->frame[st->curframe]; 2485 bitmap_from_u64(mask, reg_mask); 2486 for_each_set_bit(i, mask, 32) { 2487 reg = &func->regs[i]; 2488 if (reg->type != SCALAR_VALUE) { 2489 reg_mask &= ~(1u << i); 2490 continue; 2491 } 2492 if (!reg->precise) 2493 new_marks = true; 2494 reg->precise = true; 2495 } 2496 2497 bitmap_from_u64(mask, stack_mask); 2498 for_each_set_bit(i, mask, 64) { 2499 if (i >= func->allocated_stack / BPF_REG_SIZE) { 2500 /* the sequence of instructions: 2501 * 2: (bf) r3 = r10 2502 * 3: (7b) *(u64 *)(r3 -8) = r0 2503 * 4: (79) r4 = *(u64 *)(r10 -8) 2504 * doesn't contain jmps. It's backtracked 2505 * as a single block. 2506 * During backtracking insn 3 is not recognized as 2507 * stack access, so at the end of backtracking 2508 * stack slot fp-8 is still marked in stack_mask. 2509 * However the parent state may not have accessed 2510 * fp-8 and it's "unallocated" stack space. 2511 * In such case fallback to conservative. 2512 */ 2513 mark_all_scalars_precise(env, st); 2514 return 0; 2515 } 2516 2517 if (func->stack[i].slot_type[0] != STACK_SPILL) { 2518 stack_mask &= ~(1ull << i); 2519 continue; 2520 } 2521 reg = &func->stack[i].spilled_ptr; 2522 if (reg->type != SCALAR_VALUE) { 2523 stack_mask &= ~(1ull << i); 2524 continue; 2525 } 2526 if (!reg->precise) 2527 new_marks = true; 2528 reg->precise = true; 2529 } 2530 if (env->log.level & BPF_LOG_LEVEL) { 2531 print_verifier_state(env, func); 2532 verbose(env, "parent %s regs=%x stack=%llx marks\n", 2533 new_marks ? "didn't have" : "already had", 2534 reg_mask, stack_mask); 2535 } 2536 2537 if (!reg_mask && !stack_mask) 2538 break; 2539 if (!new_marks) 2540 break; 2541 2542 last_idx = st->last_insn_idx; 2543 first_idx = st->first_insn_idx; 2544 } 2545 return 0; 2546 } 2547 2548 static int mark_chain_precision(struct bpf_verifier_env *env, int regno) 2549 { 2550 return __mark_chain_precision(env, regno, -1); 2551 } 2552 2553 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) 2554 { 2555 return __mark_chain_precision(env, -1, spi); 2556 } 2557 2558 static bool is_spillable_regtype(enum bpf_reg_type type) 2559 { 2560 switch (type) { 2561 case PTR_TO_MAP_VALUE: 2562 case PTR_TO_MAP_VALUE_OR_NULL: 2563 case PTR_TO_STACK: 2564 case PTR_TO_CTX: 2565 case PTR_TO_PACKET: 2566 case PTR_TO_PACKET_META: 2567 case PTR_TO_PACKET_END: 2568 case PTR_TO_FLOW_KEYS: 2569 case CONST_PTR_TO_MAP: 2570 case PTR_TO_SOCKET: 2571 case PTR_TO_SOCKET_OR_NULL: 2572 case PTR_TO_SOCK_COMMON: 2573 case PTR_TO_SOCK_COMMON_OR_NULL: 2574 case PTR_TO_TCP_SOCK: 2575 case PTR_TO_TCP_SOCK_OR_NULL: 2576 case PTR_TO_XDP_SOCK: 2577 case PTR_TO_BTF_ID: 2578 case PTR_TO_BTF_ID_OR_NULL: 2579 case PTR_TO_RDONLY_BUF: 2580 case PTR_TO_RDONLY_BUF_OR_NULL: 2581 case PTR_TO_RDWR_BUF: 2582 case PTR_TO_RDWR_BUF_OR_NULL: 2583 case PTR_TO_PERCPU_BTF_ID: 2584 case PTR_TO_MEM: 2585 case PTR_TO_MEM_OR_NULL: 2586 case PTR_TO_FUNC: 2587 case PTR_TO_MAP_KEY: 2588 return true; 2589 default: 2590 return false; 2591 } 2592 } 2593 2594 /* Does this register contain a constant zero? */ 2595 static bool register_is_null(struct bpf_reg_state *reg) 2596 { 2597 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 2598 } 2599 2600 static bool register_is_const(struct bpf_reg_state *reg) 2601 { 2602 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 2603 } 2604 2605 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 2606 { 2607 return tnum_is_unknown(reg->var_off) && 2608 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 2609 reg->umin_value == 0 && reg->umax_value == U64_MAX && 2610 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 2611 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 2612 } 2613 2614 static bool register_is_bounded(struct bpf_reg_state *reg) 2615 { 2616 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 2617 } 2618 2619 static bool __is_pointer_value(bool allow_ptr_leaks, 2620 const struct bpf_reg_state *reg) 2621 { 2622 if (allow_ptr_leaks) 2623 return false; 2624 2625 return reg->type != SCALAR_VALUE; 2626 } 2627 2628 static void save_register_state(struct bpf_func_state *state, 2629 int spi, struct bpf_reg_state *reg) 2630 { 2631 int i; 2632 2633 state->stack[spi].spilled_ptr = *reg; 2634 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2635 2636 for (i = 0; i < BPF_REG_SIZE; i++) 2637 state->stack[spi].slot_type[i] = STACK_SPILL; 2638 } 2639 2640 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 2641 * stack boundary and alignment are checked in check_mem_access() 2642 */ 2643 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 2644 /* stack frame we're writing to */ 2645 struct bpf_func_state *state, 2646 int off, int size, int value_regno, 2647 int insn_idx) 2648 { 2649 struct bpf_func_state *cur; /* state of the current function */ 2650 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 2651 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; 2652 struct bpf_reg_state *reg = NULL; 2653 2654 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 2655 if (err) 2656 return err; 2657 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 2658 * so it's aligned access and [off, off + size) are within stack limits 2659 */ 2660 if (!env->allow_ptr_leaks && 2661 state->stack[spi].slot_type[0] == STACK_SPILL && 2662 size != BPF_REG_SIZE) { 2663 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 2664 return -EACCES; 2665 } 2666 2667 cur = env->cur_state->frame[env->cur_state->curframe]; 2668 if (value_regno >= 0) 2669 reg = &cur->regs[value_regno]; 2670 if (!env->bypass_spec_v4) { 2671 bool sanitize = reg && is_spillable_regtype(reg->type); 2672 2673 for (i = 0; i < size; i++) { 2674 if (state->stack[spi].slot_type[i] == STACK_INVALID) { 2675 sanitize = true; 2676 break; 2677 } 2678 } 2679 2680 if (sanitize) 2681 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 2682 } 2683 2684 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && 2685 !register_is_null(reg) && env->bpf_capable) { 2686 if (dst_reg != BPF_REG_FP) { 2687 /* The backtracking logic can only recognize explicit 2688 * stack slot address like [fp - 8]. Other spill of 2689 * scalar via different register has to be conservative. 2690 * Backtrack from here and mark all registers as precise 2691 * that contributed into 'reg' being a constant. 2692 */ 2693 err = mark_chain_precision(env, value_regno); 2694 if (err) 2695 return err; 2696 } 2697 save_register_state(state, spi, reg); 2698 } else if (reg && is_spillable_regtype(reg->type)) { 2699 /* register containing pointer is being spilled into stack */ 2700 if (size != BPF_REG_SIZE) { 2701 verbose_linfo(env, insn_idx, "; "); 2702 verbose(env, "invalid size of register spill\n"); 2703 return -EACCES; 2704 } 2705 if (state != cur && reg->type == PTR_TO_STACK) { 2706 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 2707 return -EINVAL; 2708 } 2709 save_register_state(state, spi, reg); 2710 } else { 2711 u8 type = STACK_MISC; 2712 2713 /* regular write of data into stack destroys any spilled ptr */ 2714 state->stack[spi].spilled_ptr.type = NOT_INIT; 2715 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 2716 if (state->stack[spi].slot_type[0] == STACK_SPILL) 2717 for (i = 0; i < BPF_REG_SIZE; i++) 2718 state->stack[spi].slot_type[i] = STACK_MISC; 2719 2720 /* only mark the slot as written if all 8 bytes were written 2721 * otherwise read propagation may incorrectly stop too soon 2722 * when stack slots are partially written. 2723 * This heuristic means that read propagation will be 2724 * conservative, since it will add reg_live_read marks 2725 * to stack slots all the way to first state when programs 2726 * writes+reads less than 8 bytes 2727 */ 2728 if (size == BPF_REG_SIZE) 2729 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2730 2731 /* when we zero initialize stack slots mark them as such */ 2732 if (reg && register_is_null(reg)) { 2733 /* backtracking doesn't work for STACK_ZERO yet. */ 2734 err = mark_chain_precision(env, value_regno); 2735 if (err) 2736 return err; 2737 type = STACK_ZERO; 2738 } 2739 2740 /* Mark slots affected by this stack write. */ 2741 for (i = 0; i < size; i++) 2742 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 2743 type; 2744 } 2745 return 0; 2746 } 2747 2748 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 2749 * known to contain a variable offset. 2750 * This function checks whether the write is permitted and conservatively 2751 * tracks the effects of the write, considering that each stack slot in the 2752 * dynamic range is potentially written to. 2753 * 2754 * 'off' includes 'regno->off'. 2755 * 'value_regno' can be -1, meaning that an unknown value is being written to 2756 * the stack. 2757 * 2758 * Spilled pointers in range are not marked as written because we don't know 2759 * what's going to be actually written. This means that read propagation for 2760 * future reads cannot be terminated by this write. 2761 * 2762 * For privileged programs, uninitialized stack slots are considered 2763 * initialized by this write (even though we don't know exactly what offsets 2764 * are going to be written to). The idea is that we don't want the verifier to 2765 * reject future reads that access slots written to through variable offsets. 2766 */ 2767 static int check_stack_write_var_off(struct bpf_verifier_env *env, 2768 /* func where register points to */ 2769 struct bpf_func_state *state, 2770 int ptr_regno, int off, int size, 2771 int value_regno, int insn_idx) 2772 { 2773 struct bpf_func_state *cur; /* state of the current function */ 2774 int min_off, max_off; 2775 int i, err; 2776 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 2777 bool writing_zero = false; 2778 /* set if the fact that we're writing a zero is used to let any 2779 * stack slots remain STACK_ZERO 2780 */ 2781 bool zero_used = false; 2782 2783 cur = env->cur_state->frame[env->cur_state->curframe]; 2784 ptr_reg = &cur->regs[ptr_regno]; 2785 min_off = ptr_reg->smin_value + off; 2786 max_off = ptr_reg->smax_value + off + size; 2787 if (value_regno >= 0) 2788 value_reg = &cur->regs[value_regno]; 2789 if (value_reg && register_is_null(value_reg)) 2790 writing_zero = true; 2791 2792 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 2793 if (err) 2794 return err; 2795 2796 2797 /* Variable offset writes destroy any spilled pointers in range. */ 2798 for (i = min_off; i < max_off; i++) { 2799 u8 new_type, *stype; 2800 int slot, spi; 2801 2802 slot = -i - 1; 2803 spi = slot / BPF_REG_SIZE; 2804 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 2805 2806 if (!env->allow_ptr_leaks 2807 && *stype != NOT_INIT 2808 && *stype != SCALAR_VALUE) { 2809 /* Reject the write if there's are spilled pointers in 2810 * range. If we didn't reject here, the ptr status 2811 * would be erased below (even though not all slots are 2812 * actually overwritten), possibly opening the door to 2813 * leaks. 2814 */ 2815 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 2816 insn_idx, i); 2817 return -EINVAL; 2818 } 2819 2820 /* Erase all spilled pointers. */ 2821 state->stack[spi].spilled_ptr.type = NOT_INIT; 2822 2823 /* Update the slot type. */ 2824 new_type = STACK_MISC; 2825 if (writing_zero && *stype == STACK_ZERO) { 2826 new_type = STACK_ZERO; 2827 zero_used = true; 2828 } 2829 /* If the slot is STACK_INVALID, we check whether it's OK to 2830 * pretend that it will be initialized by this write. The slot 2831 * might not actually be written to, and so if we mark it as 2832 * initialized future reads might leak uninitialized memory. 2833 * For privileged programs, we will accept such reads to slots 2834 * that may or may not be written because, if we're reject 2835 * them, the error would be too confusing. 2836 */ 2837 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 2838 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 2839 insn_idx, i); 2840 return -EINVAL; 2841 } 2842 *stype = new_type; 2843 } 2844 if (zero_used) { 2845 /* backtracking doesn't work for STACK_ZERO yet. */ 2846 err = mark_chain_precision(env, value_regno); 2847 if (err) 2848 return err; 2849 } 2850 return 0; 2851 } 2852 2853 /* When register 'dst_regno' is assigned some values from stack[min_off, 2854 * max_off), we set the register's type according to the types of the 2855 * respective stack slots. If all the stack values are known to be zeros, then 2856 * so is the destination reg. Otherwise, the register is considered to be 2857 * SCALAR. This function does not deal with register filling; the caller must 2858 * ensure that all spilled registers in the stack range have been marked as 2859 * read. 2860 */ 2861 static void mark_reg_stack_read(struct bpf_verifier_env *env, 2862 /* func where src register points to */ 2863 struct bpf_func_state *ptr_state, 2864 int min_off, int max_off, int dst_regno) 2865 { 2866 struct bpf_verifier_state *vstate = env->cur_state; 2867 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2868 int i, slot, spi; 2869 u8 *stype; 2870 int zeros = 0; 2871 2872 for (i = min_off; i < max_off; i++) { 2873 slot = -i - 1; 2874 spi = slot / BPF_REG_SIZE; 2875 stype = ptr_state->stack[spi].slot_type; 2876 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 2877 break; 2878 zeros++; 2879 } 2880 if (zeros == max_off - min_off) { 2881 /* any access_size read into register is zero extended, 2882 * so the whole register == const_zero 2883 */ 2884 __mark_reg_const_zero(&state->regs[dst_regno]); 2885 /* backtracking doesn't support STACK_ZERO yet, 2886 * so mark it precise here, so that later 2887 * backtracking can stop here. 2888 * Backtracking may not need this if this register 2889 * doesn't participate in pointer adjustment. 2890 * Forward propagation of precise flag is not 2891 * necessary either. This mark is only to stop 2892 * backtracking. Any register that contributed 2893 * to const 0 was marked precise before spill. 2894 */ 2895 state->regs[dst_regno].precise = true; 2896 } else { 2897 /* have read misc data from the stack */ 2898 mark_reg_unknown(env, state->regs, dst_regno); 2899 } 2900 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 2901 } 2902 2903 /* Read the stack at 'off' and put the results into the register indicated by 2904 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 2905 * spilled reg. 2906 * 2907 * 'dst_regno' can be -1, meaning that the read value is not going to a 2908 * register. 2909 * 2910 * The access is assumed to be within the current stack bounds. 2911 */ 2912 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 2913 /* func where src register points to */ 2914 struct bpf_func_state *reg_state, 2915 int off, int size, int dst_regno) 2916 { 2917 struct bpf_verifier_state *vstate = env->cur_state; 2918 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2919 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 2920 struct bpf_reg_state *reg; 2921 u8 *stype; 2922 2923 stype = reg_state->stack[spi].slot_type; 2924 reg = ®_state->stack[spi].spilled_ptr; 2925 2926 if (stype[0] == STACK_SPILL) { 2927 if (size != BPF_REG_SIZE) { 2928 if (reg->type != SCALAR_VALUE) { 2929 verbose_linfo(env, env->insn_idx, "; "); 2930 verbose(env, "invalid size of register fill\n"); 2931 return -EACCES; 2932 } 2933 if (dst_regno >= 0) { 2934 mark_reg_unknown(env, state->regs, dst_regno); 2935 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 2936 } 2937 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2938 return 0; 2939 } 2940 for (i = 1; i < BPF_REG_SIZE; i++) { 2941 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { 2942 verbose(env, "corrupted spill memory\n"); 2943 return -EACCES; 2944 } 2945 } 2946 2947 if (dst_regno >= 0) { 2948 /* restore register state from stack */ 2949 state->regs[dst_regno] = *reg; 2950 /* mark reg as written since spilled pointer state likely 2951 * has its liveness marks cleared by is_state_visited() 2952 * which resets stack/reg liveness for state transitions 2953 */ 2954 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 2955 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 2956 /* If dst_regno==-1, the caller is asking us whether 2957 * it is acceptable to use this value as a SCALAR_VALUE 2958 * (e.g. for XADD). 2959 * We must not allow unprivileged callers to do that 2960 * with spilled pointers. 2961 */ 2962 verbose(env, "leaking pointer from stack off %d\n", 2963 off); 2964 return -EACCES; 2965 } 2966 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2967 } else { 2968 u8 type; 2969 2970 for (i = 0; i < size; i++) { 2971 type = stype[(slot - i) % BPF_REG_SIZE]; 2972 if (type == STACK_MISC) 2973 continue; 2974 if (type == STACK_ZERO) 2975 continue; 2976 verbose(env, "invalid read from stack off %d+%d size %d\n", 2977 off, i, size); 2978 return -EACCES; 2979 } 2980 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2981 if (dst_regno >= 0) 2982 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 2983 } 2984 return 0; 2985 } 2986 2987 enum stack_access_src { 2988 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 2989 ACCESS_HELPER = 2, /* the access is performed by a helper */ 2990 }; 2991 2992 static int check_stack_range_initialized(struct bpf_verifier_env *env, 2993 int regno, int off, int access_size, 2994 bool zero_size_allowed, 2995 enum stack_access_src type, 2996 struct bpf_call_arg_meta *meta); 2997 2998 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 2999 { 3000 return cur_regs(env) + regno; 3001 } 3002 3003 /* Read the stack at 'ptr_regno + off' and put the result into the register 3004 * 'dst_regno'. 3005 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 3006 * but not its variable offset. 3007 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 3008 * 3009 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 3010 * filling registers (i.e. reads of spilled register cannot be detected when 3011 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 3012 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 3013 * offset; for a fixed offset check_stack_read_fixed_off should be used 3014 * instead. 3015 */ 3016 static int check_stack_read_var_off(struct bpf_verifier_env *env, 3017 int ptr_regno, int off, int size, int dst_regno) 3018 { 3019 /* The state of the source register. */ 3020 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3021 struct bpf_func_state *ptr_state = func(env, reg); 3022 int err; 3023 int min_off, max_off; 3024 3025 /* Note that we pass a NULL meta, so raw access will not be permitted. 3026 */ 3027 err = check_stack_range_initialized(env, ptr_regno, off, size, 3028 false, ACCESS_DIRECT, NULL); 3029 if (err) 3030 return err; 3031 3032 min_off = reg->smin_value + off; 3033 max_off = reg->smax_value + off; 3034 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 3035 return 0; 3036 } 3037 3038 /* check_stack_read dispatches to check_stack_read_fixed_off or 3039 * check_stack_read_var_off. 3040 * 3041 * The caller must ensure that the offset falls within the allocated stack 3042 * bounds. 3043 * 3044 * 'dst_regno' is a register which will receive the value from the stack. It 3045 * can be -1, meaning that the read value is not going to a register. 3046 */ 3047 static int check_stack_read(struct bpf_verifier_env *env, 3048 int ptr_regno, int off, int size, 3049 int dst_regno) 3050 { 3051 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3052 struct bpf_func_state *state = func(env, reg); 3053 int err; 3054 /* Some accesses are only permitted with a static offset. */ 3055 bool var_off = !tnum_is_const(reg->var_off); 3056 3057 /* The offset is required to be static when reads don't go to a 3058 * register, in order to not leak pointers (see 3059 * check_stack_read_fixed_off). 3060 */ 3061 if (dst_regno < 0 && var_off) { 3062 char tn_buf[48]; 3063 3064 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3065 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 3066 tn_buf, off, size); 3067 return -EACCES; 3068 } 3069 /* Variable offset is prohibited for unprivileged mode for simplicity 3070 * since it requires corresponding support in Spectre masking for stack 3071 * ALU. See also retrieve_ptr_limit(). 3072 */ 3073 if (!env->bypass_spec_v1 && var_off) { 3074 char tn_buf[48]; 3075 3076 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3077 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 3078 ptr_regno, tn_buf); 3079 return -EACCES; 3080 } 3081 3082 if (!var_off) { 3083 off += reg->var_off.value; 3084 err = check_stack_read_fixed_off(env, state, off, size, 3085 dst_regno); 3086 } else { 3087 /* Variable offset stack reads need more conservative handling 3088 * than fixed offset ones. Note that dst_regno >= 0 on this 3089 * branch. 3090 */ 3091 err = check_stack_read_var_off(env, ptr_regno, off, size, 3092 dst_regno); 3093 } 3094 return err; 3095 } 3096 3097 3098 /* check_stack_write dispatches to check_stack_write_fixed_off or 3099 * check_stack_write_var_off. 3100 * 3101 * 'ptr_regno' is the register used as a pointer into the stack. 3102 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 3103 * 'value_regno' is the register whose value we're writing to the stack. It can 3104 * be -1, meaning that we're not writing from a register. 3105 * 3106 * The caller must ensure that the offset falls within the maximum stack size. 3107 */ 3108 static int check_stack_write(struct bpf_verifier_env *env, 3109 int ptr_regno, int off, int size, 3110 int value_regno, int insn_idx) 3111 { 3112 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3113 struct bpf_func_state *state = func(env, reg); 3114 int err; 3115 3116 if (tnum_is_const(reg->var_off)) { 3117 off += reg->var_off.value; 3118 err = check_stack_write_fixed_off(env, state, off, size, 3119 value_regno, insn_idx); 3120 } else { 3121 /* Variable offset stack reads need more conservative handling 3122 * than fixed offset ones. 3123 */ 3124 err = check_stack_write_var_off(env, state, 3125 ptr_regno, off, size, 3126 value_regno, insn_idx); 3127 } 3128 return err; 3129 } 3130 3131 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 3132 int off, int size, enum bpf_access_type type) 3133 { 3134 struct bpf_reg_state *regs = cur_regs(env); 3135 struct bpf_map *map = regs[regno].map_ptr; 3136 u32 cap = bpf_map_flags_to_cap(map); 3137 3138 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 3139 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 3140 map->value_size, off, size); 3141 return -EACCES; 3142 } 3143 3144 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 3145 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 3146 map->value_size, off, size); 3147 return -EACCES; 3148 } 3149 3150 return 0; 3151 } 3152 3153 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 3154 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 3155 int off, int size, u32 mem_size, 3156 bool zero_size_allowed) 3157 { 3158 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 3159 struct bpf_reg_state *reg; 3160 3161 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 3162 return 0; 3163 3164 reg = &cur_regs(env)[regno]; 3165 switch (reg->type) { 3166 case PTR_TO_MAP_KEY: 3167 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 3168 mem_size, off, size); 3169 break; 3170 case PTR_TO_MAP_VALUE: 3171 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 3172 mem_size, off, size); 3173 break; 3174 case PTR_TO_PACKET: 3175 case PTR_TO_PACKET_META: 3176 case PTR_TO_PACKET_END: 3177 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 3178 off, size, regno, reg->id, off, mem_size); 3179 break; 3180 case PTR_TO_MEM: 3181 default: 3182 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 3183 mem_size, off, size); 3184 } 3185 3186 return -EACCES; 3187 } 3188 3189 /* check read/write into a memory region with possible variable offset */ 3190 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 3191 int off, int size, u32 mem_size, 3192 bool zero_size_allowed) 3193 { 3194 struct bpf_verifier_state *vstate = env->cur_state; 3195 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3196 struct bpf_reg_state *reg = &state->regs[regno]; 3197 int err; 3198 3199 /* We may have adjusted the register pointing to memory region, so we 3200 * need to try adding each of min_value and max_value to off 3201 * to make sure our theoretical access will be safe. 3202 */ 3203 if (env->log.level & BPF_LOG_LEVEL) 3204 print_verifier_state(env, state); 3205 3206 /* The minimum value is only important with signed 3207 * comparisons where we can't assume the floor of a 3208 * value is 0. If we are using signed variables for our 3209 * index'es we need to make sure that whatever we use 3210 * will have a set floor within our range. 3211 */ 3212 if (reg->smin_value < 0 && 3213 (reg->smin_value == S64_MIN || 3214 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 3215 reg->smin_value + off < 0)) { 3216 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3217 regno); 3218 return -EACCES; 3219 } 3220 err = __check_mem_access(env, regno, reg->smin_value + off, size, 3221 mem_size, zero_size_allowed); 3222 if (err) { 3223 verbose(env, "R%d min value is outside of the allowed memory range\n", 3224 regno); 3225 return err; 3226 } 3227 3228 /* If we haven't set a max value then we need to bail since we can't be 3229 * sure we won't do bad things. 3230 * If reg->umax_value + off could overflow, treat that as unbounded too. 3231 */ 3232 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 3233 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 3234 regno); 3235 return -EACCES; 3236 } 3237 err = __check_mem_access(env, regno, reg->umax_value + off, size, 3238 mem_size, zero_size_allowed); 3239 if (err) { 3240 verbose(env, "R%d max value is outside of the allowed memory range\n", 3241 regno); 3242 return err; 3243 } 3244 3245 return 0; 3246 } 3247 3248 /* check read/write into a map element with possible variable offset */ 3249 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 3250 int off, int size, bool zero_size_allowed) 3251 { 3252 struct bpf_verifier_state *vstate = env->cur_state; 3253 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3254 struct bpf_reg_state *reg = &state->regs[regno]; 3255 struct bpf_map *map = reg->map_ptr; 3256 int err; 3257 3258 err = check_mem_region_access(env, regno, off, size, map->value_size, 3259 zero_size_allowed); 3260 if (err) 3261 return err; 3262 3263 if (map_value_has_spin_lock(map)) { 3264 u32 lock = map->spin_lock_off; 3265 3266 /* if any part of struct bpf_spin_lock can be touched by 3267 * load/store reject this program. 3268 * To check that [x1, x2) overlaps with [y1, y2) 3269 * it is sufficient to check x1 < y2 && y1 < x2. 3270 */ 3271 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && 3272 lock < reg->umax_value + off + size) { 3273 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); 3274 return -EACCES; 3275 } 3276 } 3277 if (map_value_has_timer(map)) { 3278 u32 t = map->timer_off; 3279 3280 if (reg->smin_value + off < t + sizeof(struct bpf_timer) && 3281 t < reg->umax_value + off + size) { 3282 verbose(env, "bpf_timer cannot be accessed directly by load/store\n"); 3283 return -EACCES; 3284 } 3285 } 3286 return err; 3287 } 3288 3289 #define MAX_PACKET_OFF 0xffff 3290 3291 static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) 3292 { 3293 return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; 3294 } 3295 3296 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 3297 const struct bpf_call_arg_meta *meta, 3298 enum bpf_access_type t) 3299 { 3300 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 3301 3302 switch (prog_type) { 3303 /* Program types only with direct read access go here! */ 3304 case BPF_PROG_TYPE_LWT_IN: 3305 case BPF_PROG_TYPE_LWT_OUT: 3306 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 3307 case BPF_PROG_TYPE_SK_REUSEPORT: 3308 case BPF_PROG_TYPE_FLOW_DISSECTOR: 3309 case BPF_PROG_TYPE_CGROUP_SKB: 3310 if (t == BPF_WRITE) 3311 return false; 3312 fallthrough; 3313 3314 /* Program types with direct read + write access go here! */ 3315 case BPF_PROG_TYPE_SCHED_CLS: 3316 case BPF_PROG_TYPE_SCHED_ACT: 3317 case BPF_PROG_TYPE_XDP: 3318 case BPF_PROG_TYPE_LWT_XMIT: 3319 case BPF_PROG_TYPE_SK_SKB: 3320 case BPF_PROG_TYPE_SK_MSG: 3321 if (meta) 3322 return meta->pkt_access; 3323 3324 env->seen_direct_write = true; 3325 return true; 3326 3327 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3328 if (t == BPF_WRITE) 3329 env->seen_direct_write = true; 3330 3331 return true; 3332 3333 default: 3334 return false; 3335 } 3336 } 3337 3338 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 3339 int size, bool zero_size_allowed) 3340 { 3341 struct bpf_reg_state *regs = cur_regs(env); 3342 struct bpf_reg_state *reg = ®s[regno]; 3343 int err; 3344 3345 /* We may have added a variable offset to the packet pointer; but any 3346 * reg->range we have comes after that. We are only checking the fixed 3347 * offset. 3348 */ 3349 3350 /* We don't allow negative numbers, because we aren't tracking enough 3351 * detail to prove they're safe. 3352 */ 3353 if (reg->smin_value < 0) { 3354 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3355 regno); 3356 return -EACCES; 3357 } 3358 3359 err = reg->range < 0 ? -EINVAL : 3360 __check_mem_access(env, regno, off, size, reg->range, 3361 zero_size_allowed); 3362 if (err) { 3363 verbose(env, "R%d offset is outside of the packet\n", regno); 3364 return err; 3365 } 3366 3367 /* __check_mem_access has made sure "off + size - 1" is within u16. 3368 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 3369 * otherwise find_good_pkt_pointers would have refused to set range info 3370 * that __check_mem_access would have rejected this pkt access. 3371 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 3372 */ 3373 env->prog->aux->max_pkt_offset = 3374 max_t(u32, env->prog->aux->max_pkt_offset, 3375 off + reg->umax_value + size - 1); 3376 3377 return err; 3378 } 3379 3380 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 3381 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 3382 enum bpf_access_type t, enum bpf_reg_type *reg_type, 3383 struct btf **btf, u32 *btf_id) 3384 { 3385 struct bpf_insn_access_aux info = { 3386 .reg_type = *reg_type, 3387 .log = &env->log, 3388 }; 3389 3390 if (env->ops->is_valid_access && 3391 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 3392 /* A non zero info.ctx_field_size indicates that this field is a 3393 * candidate for later verifier transformation to load the whole 3394 * field and then apply a mask when accessed with a narrower 3395 * access than actual ctx access size. A zero info.ctx_field_size 3396 * will only allow for whole field access and rejects any other 3397 * type of narrower access. 3398 */ 3399 *reg_type = info.reg_type; 3400 3401 if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) { 3402 *btf = info.btf; 3403 *btf_id = info.btf_id; 3404 } else { 3405 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 3406 } 3407 /* remember the offset of last byte accessed in ctx */ 3408 if (env->prog->aux->max_ctx_offset < off + size) 3409 env->prog->aux->max_ctx_offset = off + size; 3410 return 0; 3411 } 3412 3413 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 3414 return -EACCES; 3415 } 3416 3417 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 3418 int size) 3419 { 3420 if (size < 0 || off < 0 || 3421 (u64)off + size > sizeof(struct bpf_flow_keys)) { 3422 verbose(env, "invalid access to flow keys off=%d size=%d\n", 3423 off, size); 3424 return -EACCES; 3425 } 3426 return 0; 3427 } 3428 3429 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 3430 u32 regno, int off, int size, 3431 enum bpf_access_type t) 3432 { 3433 struct bpf_reg_state *regs = cur_regs(env); 3434 struct bpf_reg_state *reg = ®s[regno]; 3435 struct bpf_insn_access_aux info = {}; 3436 bool valid; 3437 3438 if (reg->smin_value < 0) { 3439 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3440 regno); 3441 return -EACCES; 3442 } 3443 3444 switch (reg->type) { 3445 case PTR_TO_SOCK_COMMON: 3446 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 3447 break; 3448 case PTR_TO_SOCKET: 3449 valid = bpf_sock_is_valid_access(off, size, t, &info); 3450 break; 3451 case PTR_TO_TCP_SOCK: 3452 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 3453 break; 3454 case PTR_TO_XDP_SOCK: 3455 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 3456 break; 3457 default: 3458 valid = false; 3459 } 3460 3461 3462 if (valid) { 3463 env->insn_aux_data[insn_idx].ctx_field_size = 3464 info.ctx_field_size; 3465 return 0; 3466 } 3467 3468 verbose(env, "R%d invalid %s access off=%d size=%d\n", 3469 regno, reg_type_str[reg->type], off, size); 3470 3471 return -EACCES; 3472 } 3473 3474 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 3475 { 3476 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 3477 } 3478 3479 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 3480 { 3481 const struct bpf_reg_state *reg = reg_state(env, regno); 3482 3483 return reg->type == PTR_TO_CTX; 3484 } 3485 3486 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 3487 { 3488 const struct bpf_reg_state *reg = reg_state(env, regno); 3489 3490 return type_is_sk_pointer(reg->type); 3491 } 3492 3493 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 3494 { 3495 const struct bpf_reg_state *reg = reg_state(env, regno); 3496 3497 return type_is_pkt_pointer(reg->type); 3498 } 3499 3500 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 3501 { 3502 const struct bpf_reg_state *reg = reg_state(env, regno); 3503 3504 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 3505 return reg->type == PTR_TO_FLOW_KEYS; 3506 } 3507 3508 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 3509 const struct bpf_reg_state *reg, 3510 int off, int size, bool strict) 3511 { 3512 struct tnum reg_off; 3513 int ip_align; 3514 3515 /* Byte size accesses are always allowed. */ 3516 if (!strict || size == 1) 3517 return 0; 3518 3519 /* For platforms that do not have a Kconfig enabling 3520 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 3521 * NET_IP_ALIGN is universally set to '2'. And on platforms 3522 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 3523 * to this code only in strict mode where we want to emulate 3524 * the NET_IP_ALIGN==2 checking. Therefore use an 3525 * unconditional IP align value of '2'. 3526 */ 3527 ip_align = 2; 3528 3529 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 3530 if (!tnum_is_aligned(reg_off, size)) { 3531 char tn_buf[48]; 3532 3533 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3534 verbose(env, 3535 "misaligned packet access off %d+%s+%d+%d size %d\n", 3536 ip_align, tn_buf, reg->off, off, size); 3537 return -EACCES; 3538 } 3539 3540 return 0; 3541 } 3542 3543 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 3544 const struct bpf_reg_state *reg, 3545 const char *pointer_desc, 3546 int off, int size, bool strict) 3547 { 3548 struct tnum reg_off; 3549 3550 /* Byte size accesses are always allowed. */ 3551 if (!strict || size == 1) 3552 return 0; 3553 3554 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 3555 if (!tnum_is_aligned(reg_off, size)) { 3556 char tn_buf[48]; 3557 3558 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3559 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 3560 pointer_desc, tn_buf, reg->off, off, size); 3561 return -EACCES; 3562 } 3563 3564 return 0; 3565 } 3566 3567 static int check_ptr_alignment(struct bpf_verifier_env *env, 3568 const struct bpf_reg_state *reg, int off, 3569 int size, bool strict_alignment_once) 3570 { 3571 bool strict = env->strict_alignment || strict_alignment_once; 3572 const char *pointer_desc = ""; 3573 3574 switch (reg->type) { 3575 case PTR_TO_PACKET: 3576 case PTR_TO_PACKET_META: 3577 /* Special case, because of NET_IP_ALIGN. Given metadata sits 3578 * right in front, treat it the very same way. 3579 */ 3580 return check_pkt_ptr_alignment(env, reg, off, size, strict); 3581 case PTR_TO_FLOW_KEYS: 3582 pointer_desc = "flow keys "; 3583 break; 3584 case PTR_TO_MAP_KEY: 3585 pointer_desc = "key "; 3586 break; 3587 case PTR_TO_MAP_VALUE: 3588 pointer_desc = "value "; 3589 break; 3590 case PTR_TO_CTX: 3591 pointer_desc = "context "; 3592 break; 3593 case PTR_TO_STACK: 3594 pointer_desc = "stack "; 3595 /* The stack spill tracking logic in check_stack_write_fixed_off() 3596 * and check_stack_read_fixed_off() relies on stack accesses being 3597 * aligned. 3598 */ 3599 strict = true; 3600 break; 3601 case PTR_TO_SOCKET: 3602 pointer_desc = "sock "; 3603 break; 3604 case PTR_TO_SOCK_COMMON: 3605 pointer_desc = "sock_common "; 3606 break; 3607 case PTR_TO_TCP_SOCK: 3608 pointer_desc = "tcp_sock "; 3609 break; 3610 case PTR_TO_XDP_SOCK: 3611 pointer_desc = "xdp_sock "; 3612 break; 3613 default: 3614 break; 3615 } 3616 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 3617 strict); 3618 } 3619 3620 static int update_stack_depth(struct bpf_verifier_env *env, 3621 const struct bpf_func_state *func, 3622 int off) 3623 { 3624 u16 stack = env->subprog_info[func->subprogno].stack_depth; 3625 3626 if (stack >= -off) 3627 return 0; 3628 3629 /* update known max for given subprogram */ 3630 env->subprog_info[func->subprogno].stack_depth = -off; 3631 return 0; 3632 } 3633 3634 /* starting from main bpf function walk all instructions of the function 3635 * and recursively walk all callees that given function can call. 3636 * Ignore jump and exit insns. 3637 * Since recursion is prevented by check_cfg() this algorithm 3638 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 3639 */ 3640 static int check_max_stack_depth(struct bpf_verifier_env *env) 3641 { 3642 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 3643 struct bpf_subprog_info *subprog = env->subprog_info; 3644 struct bpf_insn *insn = env->prog->insnsi; 3645 bool tail_call_reachable = false; 3646 int ret_insn[MAX_CALL_FRAMES]; 3647 int ret_prog[MAX_CALL_FRAMES]; 3648 int j; 3649 3650 process_func: 3651 /* protect against potential stack overflow that might happen when 3652 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 3653 * depth for such case down to 256 so that the worst case scenario 3654 * would result in 8k stack size (32 which is tailcall limit * 256 = 3655 * 8k). 3656 * 3657 * To get the idea what might happen, see an example: 3658 * func1 -> sub rsp, 128 3659 * subfunc1 -> sub rsp, 256 3660 * tailcall1 -> add rsp, 256 3661 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 3662 * subfunc2 -> sub rsp, 64 3663 * subfunc22 -> sub rsp, 128 3664 * tailcall2 -> add rsp, 128 3665 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 3666 * 3667 * tailcall will unwind the current stack frame but it will not get rid 3668 * of caller's stack as shown on the example above. 3669 */ 3670 if (idx && subprog[idx].has_tail_call && depth >= 256) { 3671 verbose(env, 3672 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 3673 depth); 3674 return -EACCES; 3675 } 3676 /* round up to 32-bytes, since this is granularity 3677 * of interpreter stack size 3678 */ 3679 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 3680 if (depth > MAX_BPF_STACK) { 3681 verbose(env, "combined stack size of %d calls is %d. Too large\n", 3682 frame + 1, depth); 3683 return -EACCES; 3684 } 3685 continue_func: 3686 subprog_end = subprog[idx + 1].start; 3687 for (; i < subprog_end; i++) { 3688 int next_insn; 3689 3690 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 3691 continue; 3692 /* remember insn and function to return to */ 3693 ret_insn[frame] = i + 1; 3694 ret_prog[frame] = idx; 3695 3696 /* find the callee */ 3697 next_insn = i + insn[i].imm + 1; 3698 idx = find_subprog(env, next_insn); 3699 if (idx < 0) { 3700 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 3701 next_insn); 3702 return -EFAULT; 3703 } 3704 if (subprog[idx].is_async_cb) { 3705 if (subprog[idx].has_tail_call) { 3706 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 3707 return -EFAULT; 3708 } 3709 /* async callbacks don't increase bpf prog stack size */ 3710 continue; 3711 } 3712 i = next_insn; 3713 3714 if (subprog[idx].has_tail_call) 3715 tail_call_reachable = true; 3716 3717 frame++; 3718 if (frame >= MAX_CALL_FRAMES) { 3719 verbose(env, "the call stack of %d frames is too deep !\n", 3720 frame); 3721 return -E2BIG; 3722 } 3723 goto process_func; 3724 } 3725 /* if tail call got detected across bpf2bpf calls then mark each of the 3726 * currently present subprog frames as tail call reachable subprogs; 3727 * this info will be utilized by JIT so that we will be preserving the 3728 * tail call counter throughout bpf2bpf calls combined with tailcalls 3729 */ 3730 if (tail_call_reachable) 3731 for (j = 0; j < frame; j++) 3732 subprog[ret_prog[j]].tail_call_reachable = true; 3733 if (subprog[0].tail_call_reachable) 3734 env->prog->aux->tail_call_reachable = true; 3735 3736 /* end of for() loop means the last insn of the 'subprog' 3737 * was reached. Doesn't matter whether it was JA or EXIT 3738 */ 3739 if (frame == 0) 3740 return 0; 3741 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 3742 frame--; 3743 i = ret_insn[frame]; 3744 idx = ret_prog[frame]; 3745 goto continue_func; 3746 } 3747 3748 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 3749 static int get_callee_stack_depth(struct bpf_verifier_env *env, 3750 const struct bpf_insn *insn, int idx) 3751 { 3752 int start = idx + insn->imm + 1, subprog; 3753 3754 subprog = find_subprog(env, start); 3755 if (subprog < 0) { 3756 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 3757 start); 3758 return -EFAULT; 3759 } 3760 return env->subprog_info[subprog].stack_depth; 3761 } 3762 #endif 3763 3764 int check_ctx_reg(struct bpf_verifier_env *env, 3765 const struct bpf_reg_state *reg, int regno) 3766 { 3767 /* Access to ctx or passing it to a helper is only allowed in 3768 * its original, unmodified form. 3769 */ 3770 3771 if (reg->off) { 3772 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", 3773 regno, reg->off); 3774 return -EACCES; 3775 } 3776 3777 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3778 char tn_buf[48]; 3779 3780 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3781 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); 3782 return -EACCES; 3783 } 3784 3785 return 0; 3786 } 3787 3788 static int __check_buffer_access(struct bpf_verifier_env *env, 3789 const char *buf_info, 3790 const struct bpf_reg_state *reg, 3791 int regno, int off, int size) 3792 { 3793 if (off < 0) { 3794 verbose(env, 3795 "R%d invalid %s buffer access: off=%d, size=%d\n", 3796 regno, buf_info, off, size); 3797 return -EACCES; 3798 } 3799 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3800 char tn_buf[48]; 3801 3802 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3803 verbose(env, 3804 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 3805 regno, off, tn_buf); 3806 return -EACCES; 3807 } 3808 3809 return 0; 3810 } 3811 3812 static int check_tp_buffer_access(struct bpf_verifier_env *env, 3813 const struct bpf_reg_state *reg, 3814 int regno, int off, int size) 3815 { 3816 int err; 3817 3818 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 3819 if (err) 3820 return err; 3821 3822 if (off + size > env->prog->aux->max_tp_access) 3823 env->prog->aux->max_tp_access = off + size; 3824 3825 return 0; 3826 } 3827 3828 static int check_buffer_access(struct bpf_verifier_env *env, 3829 const struct bpf_reg_state *reg, 3830 int regno, int off, int size, 3831 bool zero_size_allowed, 3832 const char *buf_info, 3833 u32 *max_access) 3834 { 3835 int err; 3836 3837 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 3838 if (err) 3839 return err; 3840 3841 if (off + size > *max_access) 3842 *max_access = off + size; 3843 3844 return 0; 3845 } 3846 3847 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 3848 static void zext_32_to_64(struct bpf_reg_state *reg) 3849 { 3850 reg->var_off = tnum_subreg(reg->var_off); 3851 __reg_assign_32_into_64(reg); 3852 } 3853 3854 /* truncate register to smaller size (in bytes) 3855 * must be called with size < BPF_REG_SIZE 3856 */ 3857 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 3858 { 3859 u64 mask; 3860 3861 /* clear high bits in bit representation */ 3862 reg->var_off = tnum_cast(reg->var_off, size); 3863 3864 /* fix arithmetic bounds */ 3865 mask = ((u64)1 << (size * 8)) - 1; 3866 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 3867 reg->umin_value &= mask; 3868 reg->umax_value &= mask; 3869 } else { 3870 reg->umin_value = 0; 3871 reg->umax_value = mask; 3872 } 3873 reg->smin_value = reg->umin_value; 3874 reg->smax_value = reg->umax_value; 3875 3876 /* If size is smaller than 32bit register the 32bit register 3877 * values are also truncated so we push 64-bit bounds into 3878 * 32-bit bounds. Above were truncated < 32-bits already. 3879 */ 3880 if (size >= 4) 3881 return; 3882 __reg_combine_64_into_32(reg); 3883 } 3884 3885 static bool bpf_map_is_rdonly(const struct bpf_map *map) 3886 { 3887 return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen; 3888 } 3889 3890 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 3891 { 3892 void *ptr; 3893 u64 addr; 3894 int err; 3895 3896 err = map->ops->map_direct_value_addr(map, &addr, off); 3897 if (err) 3898 return err; 3899 ptr = (void *)(long)addr + off; 3900 3901 switch (size) { 3902 case sizeof(u8): 3903 *val = (u64)*(u8 *)ptr; 3904 break; 3905 case sizeof(u16): 3906 *val = (u64)*(u16 *)ptr; 3907 break; 3908 case sizeof(u32): 3909 *val = (u64)*(u32 *)ptr; 3910 break; 3911 case sizeof(u64): 3912 *val = *(u64 *)ptr; 3913 break; 3914 default: 3915 return -EINVAL; 3916 } 3917 return 0; 3918 } 3919 3920 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 3921 struct bpf_reg_state *regs, 3922 int regno, int off, int size, 3923 enum bpf_access_type atype, 3924 int value_regno) 3925 { 3926 struct bpf_reg_state *reg = regs + regno; 3927 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 3928 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 3929 u32 btf_id; 3930 int ret; 3931 3932 if (off < 0) { 3933 verbose(env, 3934 "R%d is ptr_%s invalid negative access: off=%d\n", 3935 regno, tname, off); 3936 return -EACCES; 3937 } 3938 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3939 char tn_buf[48]; 3940 3941 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3942 verbose(env, 3943 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 3944 regno, tname, off, tn_buf); 3945 return -EACCES; 3946 } 3947 3948 if (env->ops->btf_struct_access) { 3949 ret = env->ops->btf_struct_access(&env->log, reg->btf, t, 3950 off, size, atype, &btf_id); 3951 } else { 3952 if (atype != BPF_READ) { 3953 verbose(env, "only read is supported\n"); 3954 return -EACCES; 3955 } 3956 3957 ret = btf_struct_access(&env->log, reg->btf, t, off, size, 3958 atype, &btf_id); 3959 } 3960 3961 if (ret < 0) 3962 return ret; 3963 3964 if (atype == BPF_READ && value_regno >= 0) 3965 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id); 3966 3967 return 0; 3968 } 3969 3970 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 3971 struct bpf_reg_state *regs, 3972 int regno, int off, int size, 3973 enum bpf_access_type atype, 3974 int value_regno) 3975 { 3976 struct bpf_reg_state *reg = regs + regno; 3977 struct bpf_map *map = reg->map_ptr; 3978 const struct btf_type *t; 3979 const char *tname; 3980 u32 btf_id; 3981 int ret; 3982 3983 if (!btf_vmlinux) { 3984 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 3985 return -ENOTSUPP; 3986 } 3987 3988 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 3989 verbose(env, "map_ptr access not supported for map type %d\n", 3990 map->map_type); 3991 return -ENOTSUPP; 3992 } 3993 3994 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 3995 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 3996 3997 if (!env->allow_ptr_to_map_access) { 3998 verbose(env, 3999 "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 4000 tname); 4001 return -EPERM; 4002 } 4003 4004 if (off < 0) { 4005 verbose(env, "R%d is %s invalid negative access: off=%d\n", 4006 regno, tname, off); 4007 return -EACCES; 4008 } 4009 4010 if (atype != BPF_READ) { 4011 verbose(env, "only read from %s is supported\n", tname); 4012 return -EACCES; 4013 } 4014 4015 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id); 4016 if (ret < 0) 4017 return ret; 4018 4019 if (value_regno >= 0) 4020 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id); 4021 4022 return 0; 4023 } 4024 4025 /* Check that the stack access at the given offset is within bounds. The 4026 * maximum valid offset is -1. 4027 * 4028 * The minimum valid offset is -MAX_BPF_STACK for writes, and 4029 * -state->allocated_stack for reads. 4030 */ 4031 static int check_stack_slot_within_bounds(int off, 4032 struct bpf_func_state *state, 4033 enum bpf_access_type t) 4034 { 4035 int min_valid_off; 4036 4037 if (t == BPF_WRITE) 4038 min_valid_off = -MAX_BPF_STACK; 4039 else 4040 min_valid_off = -state->allocated_stack; 4041 4042 if (off < min_valid_off || off > -1) 4043 return -EACCES; 4044 return 0; 4045 } 4046 4047 /* Check that the stack access at 'regno + off' falls within the maximum stack 4048 * bounds. 4049 * 4050 * 'off' includes `regno->offset`, but not its dynamic part (if any). 4051 */ 4052 static int check_stack_access_within_bounds( 4053 struct bpf_verifier_env *env, 4054 int regno, int off, int access_size, 4055 enum stack_access_src src, enum bpf_access_type type) 4056 { 4057 struct bpf_reg_state *regs = cur_regs(env); 4058 struct bpf_reg_state *reg = regs + regno; 4059 struct bpf_func_state *state = func(env, reg); 4060 int min_off, max_off; 4061 int err; 4062 char *err_extra; 4063 4064 if (src == ACCESS_HELPER) 4065 /* We don't know if helpers are reading or writing (or both). */ 4066 err_extra = " indirect access to"; 4067 else if (type == BPF_READ) 4068 err_extra = " read from"; 4069 else 4070 err_extra = " write to"; 4071 4072 if (tnum_is_const(reg->var_off)) { 4073 min_off = reg->var_off.value + off; 4074 if (access_size > 0) 4075 max_off = min_off + access_size - 1; 4076 else 4077 max_off = min_off; 4078 } else { 4079 if (reg->smax_value >= BPF_MAX_VAR_OFF || 4080 reg->smin_value <= -BPF_MAX_VAR_OFF) { 4081 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 4082 err_extra, regno); 4083 return -EACCES; 4084 } 4085 min_off = reg->smin_value + off; 4086 if (access_size > 0) 4087 max_off = reg->smax_value + off + access_size - 1; 4088 else 4089 max_off = min_off; 4090 } 4091 4092 err = check_stack_slot_within_bounds(min_off, state, type); 4093 if (!err) 4094 err = check_stack_slot_within_bounds(max_off, state, type); 4095 4096 if (err) { 4097 if (tnum_is_const(reg->var_off)) { 4098 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 4099 err_extra, regno, off, access_size); 4100 } else { 4101 char tn_buf[48]; 4102 4103 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4104 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 4105 err_extra, regno, tn_buf, access_size); 4106 } 4107 } 4108 return err; 4109 } 4110 4111 /* check whether memory at (regno + off) is accessible for t = (read | write) 4112 * if t==write, value_regno is a register which value is stored into memory 4113 * if t==read, value_regno is a register which will receive the value from memory 4114 * if t==write && value_regno==-1, some unknown value is stored into memory 4115 * if t==read && value_regno==-1, don't care what we read from memory 4116 */ 4117 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 4118 int off, int bpf_size, enum bpf_access_type t, 4119 int value_regno, bool strict_alignment_once) 4120 { 4121 struct bpf_reg_state *regs = cur_regs(env); 4122 struct bpf_reg_state *reg = regs + regno; 4123 struct bpf_func_state *state; 4124 int size, err = 0; 4125 4126 size = bpf_size_to_bytes(bpf_size); 4127 if (size < 0) 4128 return size; 4129 4130 /* alignment checks will add in reg->off themselves */ 4131 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 4132 if (err) 4133 return err; 4134 4135 /* for access checks, reg->off is just part of off */ 4136 off += reg->off; 4137 4138 if (reg->type == PTR_TO_MAP_KEY) { 4139 if (t == BPF_WRITE) { 4140 verbose(env, "write to change key R%d not allowed\n", regno); 4141 return -EACCES; 4142 } 4143 4144 err = check_mem_region_access(env, regno, off, size, 4145 reg->map_ptr->key_size, false); 4146 if (err) 4147 return err; 4148 if (value_regno >= 0) 4149 mark_reg_unknown(env, regs, value_regno); 4150 } else if (reg->type == PTR_TO_MAP_VALUE) { 4151 if (t == BPF_WRITE && value_regno >= 0 && 4152 is_pointer_value(env, value_regno)) { 4153 verbose(env, "R%d leaks addr into map\n", value_regno); 4154 return -EACCES; 4155 } 4156 err = check_map_access_type(env, regno, off, size, t); 4157 if (err) 4158 return err; 4159 err = check_map_access(env, regno, off, size, false); 4160 if (!err && t == BPF_READ && value_regno >= 0) { 4161 struct bpf_map *map = reg->map_ptr; 4162 4163 /* if map is read-only, track its contents as scalars */ 4164 if (tnum_is_const(reg->var_off) && 4165 bpf_map_is_rdonly(map) && 4166 map->ops->map_direct_value_addr) { 4167 int map_off = off + reg->var_off.value; 4168 u64 val = 0; 4169 4170 err = bpf_map_direct_read(map, map_off, size, 4171 &val); 4172 if (err) 4173 return err; 4174 4175 regs[value_regno].type = SCALAR_VALUE; 4176 __mark_reg_known(®s[value_regno], val); 4177 } else { 4178 mark_reg_unknown(env, regs, value_regno); 4179 } 4180 } 4181 } else if (reg->type == PTR_TO_MEM) { 4182 if (t == BPF_WRITE && value_regno >= 0 && 4183 is_pointer_value(env, value_regno)) { 4184 verbose(env, "R%d leaks addr into mem\n", value_regno); 4185 return -EACCES; 4186 } 4187 err = check_mem_region_access(env, regno, off, size, 4188 reg->mem_size, false); 4189 if (!err && t == BPF_READ && value_regno >= 0) 4190 mark_reg_unknown(env, regs, value_regno); 4191 } else if (reg->type == PTR_TO_CTX) { 4192 enum bpf_reg_type reg_type = SCALAR_VALUE; 4193 struct btf *btf = NULL; 4194 u32 btf_id = 0; 4195 4196 if (t == BPF_WRITE && value_regno >= 0 && 4197 is_pointer_value(env, value_regno)) { 4198 verbose(env, "R%d leaks addr into ctx\n", value_regno); 4199 return -EACCES; 4200 } 4201 4202 err = check_ctx_reg(env, reg, regno); 4203 if (err < 0) 4204 return err; 4205 4206 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, &btf_id); 4207 if (err) 4208 verbose_linfo(env, insn_idx, "; "); 4209 if (!err && t == BPF_READ && value_regno >= 0) { 4210 /* ctx access returns either a scalar, or a 4211 * PTR_TO_PACKET[_META,_END]. In the latter 4212 * case, we know the offset is zero. 4213 */ 4214 if (reg_type == SCALAR_VALUE) { 4215 mark_reg_unknown(env, regs, value_regno); 4216 } else { 4217 mark_reg_known_zero(env, regs, 4218 value_regno); 4219 if (reg_type_may_be_null(reg_type)) 4220 regs[value_regno].id = ++env->id_gen; 4221 /* A load of ctx field could have different 4222 * actual load size with the one encoded in the 4223 * insn. When the dst is PTR, it is for sure not 4224 * a sub-register. 4225 */ 4226 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 4227 if (reg_type == PTR_TO_BTF_ID || 4228 reg_type == PTR_TO_BTF_ID_OR_NULL) { 4229 regs[value_regno].btf = btf; 4230 regs[value_regno].btf_id = btf_id; 4231 } 4232 } 4233 regs[value_regno].type = reg_type; 4234 } 4235 4236 } else if (reg->type == PTR_TO_STACK) { 4237 /* Basic bounds checks. */ 4238 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 4239 if (err) 4240 return err; 4241 4242 state = func(env, reg); 4243 err = update_stack_depth(env, state, off); 4244 if (err) 4245 return err; 4246 4247 if (t == BPF_READ) 4248 err = check_stack_read(env, regno, off, size, 4249 value_regno); 4250 else 4251 err = check_stack_write(env, regno, off, size, 4252 value_regno, insn_idx); 4253 } else if (reg_is_pkt_pointer(reg)) { 4254 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 4255 verbose(env, "cannot write into packet\n"); 4256 return -EACCES; 4257 } 4258 if (t == BPF_WRITE && value_regno >= 0 && 4259 is_pointer_value(env, value_regno)) { 4260 verbose(env, "R%d leaks addr into packet\n", 4261 value_regno); 4262 return -EACCES; 4263 } 4264 err = check_packet_access(env, regno, off, size, false); 4265 if (!err && t == BPF_READ && value_regno >= 0) 4266 mark_reg_unknown(env, regs, value_regno); 4267 } else if (reg->type == PTR_TO_FLOW_KEYS) { 4268 if (t == BPF_WRITE && value_regno >= 0 && 4269 is_pointer_value(env, value_regno)) { 4270 verbose(env, "R%d leaks addr into flow keys\n", 4271 value_regno); 4272 return -EACCES; 4273 } 4274 4275 err = check_flow_keys_access(env, off, size); 4276 if (!err && t == BPF_READ && value_regno >= 0) 4277 mark_reg_unknown(env, regs, value_regno); 4278 } else if (type_is_sk_pointer(reg->type)) { 4279 if (t == BPF_WRITE) { 4280 verbose(env, "R%d cannot write into %s\n", 4281 regno, reg_type_str[reg->type]); 4282 return -EACCES; 4283 } 4284 err = check_sock_access(env, insn_idx, regno, off, size, t); 4285 if (!err && value_regno >= 0) 4286 mark_reg_unknown(env, regs, value_regno); 4287 } else if (reg->type == PTR_TO_TP_BUFFER) { 4288 err = check_tp_buffer_access(env, reg, regno, off, size); 4289 if (!err && t == BPF_READ && value_regno >= 0) 4290 mark_reg_unknown(env, regs, value_regno); 4291 } else if (reg->type == PTR_TO_BTF_ID) { 4292 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 4293 value_regno); 4294 } else if (reg->type == CONST_PTR_TO_MAP) { 4295 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 4296 value_regno); 4297 } else if (reg->type == PTR_TO_RDONLY_BUF) { 4298 if (t == BPF_WRITE) { 4299 verbose(env, "R%d cannot write into %s\n", 4300 regno, reg_type_str[reg->type]); 4301 return -EACCES; 4302 } 4303 err = check_buffer_access(env, reg, regno, off, size, false, 4304 "rdonly", 4305 &env->prog->aux->max_rdonly_access); 4306 if (!err && value_regno >= 0) 4307 mark_reg_unknown(env, regs, value_regno); 4308 } else if (reg->type == PTR_TO_RDWR_BUF) { 4309 err = check_buffer_access(env, reg, regno, off, size, false, 4310 "rdwr", 4311 &env->prog->aux->max_rdwr_access); 4312 if (!err && t == BPF_READ && value_regno >= 0) 4313 mark_reg_unknown(env, regs, value_regno); 4314 } else { 4315 verbose(env, "R%d invalid mem access '%s'\n", regno, 4316 reg_type_str[reg->type]); 4317 return -EACCES; 4318 } 4319 4320 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 4321 regs[value_regno].type == SCALAR_VALUE) { 4322 /* b/h/w load zero-extends, mark upper bits as known 0 */ 4323 coerce_reg_to_size(®s[value_regno], size); 4324 } 4325 return err; 4326 } 4327 4328 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 4329 { 4330 int load_reg; 4331 int err; 4332 4333 switch (insn->imm) { 4334 case BPF_ADD: 4335 case BPF_ADD | BPF_FETCH: 4336 case BPF_AND: 4337 case BPF_AND | BPF_FETCH: 4338 case BPF_OR: 4339 case BPF_OR | BPF_FETCH: 4340 case BPF_XOR: 4341 case BPF_XOR | BPF_FETCH: 4342 case BPF_XCHG: 4343 case BPF_CMPXCHG: 4344 break; 4345 default: 4346 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 4347 return -EINVAL; 4348 } 4349 4350 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 4351 verbose(env, "invalid atomic operand size\n"); 4352 return -EINVAL; 4353 } 4354 4355 /* check src1 operand */ 4356 err = check_reg_arg(env, insn->src_reg, SRC_OP); 4357 if (err) 4358 return err; 4359 4360 /* check src2 operand */ 4361 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 4362 if (err) 4363 return err; 4364 4365 if (insn->imm == BPF_CMPXCHG) { 4366 /* Check comparison of R0 with memory location */ 4367 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 4368 if (err) 4369 return err; 4370 } 4371 4372 if (is_pointer_value(env, insn->src_reg)) { 4373 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 4374 return -EACCES; 4375 } 4376 4377 if (is_ctx_reg(env, insn->dst_reg) || 4378 is_pkt_reg(env, insn->dst_reg) || 4379 is_flow_key_reg(env, insn->dst_reg) || 4380 is_sk_reg(env, insn->dst_reg)) { 4381 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 4382 insn->dst_reg, 4383 reg_type_str[reg_state(env, insn->dst_reg)->type]); 4384 return -EACCES; 4385 } 4386 4387 if (insn->imm & BPF_FETCH) { 4388 if (insn->imm == BPF_CMPXCHG) 4389 load_reg = BPF_REG_0; 4390 else 4391 load_reg = insn->src_reg; 4392 4393 /* check and record load of old value */ 4394 err = check_reg_arg(env, load_reg, DST_OP); 4395 if (err) 4396 return err; 4397 } else { 4398 /* This instruction accesses a memory location but doesn't 4399 * actually load it into a register. 4400 */ 4401 load_reg = -1; 4402 } 4403 4404 /* check whether we can read the memory */ 4405 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4406 BPF_SIZE(insn->code), BPF_READ, load_reg, true); 4407 if (err) 4408 return err; 4409 4410 /* check whether we can write into the same memory */ 4411 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4412 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 4413 if (err) 4414 return err; 4415 4416 return 0; 4417 } 4418 4419 /* When register 'regno' is used to read the stack (either directly or through 4420 * a helper function) make sure that it's within stack boundary and, depending 4421 * on the access type, that all elements of the stack are initialized. 4422 * 4423 * 'off' includes 'regno->off', but not its dynamic part (if any). 4424 * 4425 * All registers that have been spilled on the stack in the slots within the 4426 * read offsets are marked as read. 4427 */ 4428 static int check_stack_range_initialized( 4429 struct bpf_verifier_env *env, int regno, int off, 4430 int access_size, bool zero_size_allowed, 4431 enum stack_access_src type, struct bpf_call_arg_meta *meta) 4432 { 4433 struct bpf_reg_state *reg = reg_state(env, regno); 4434 struct bpf_func_state *state = func(env, reg); 4435 int err, min_off, max_off, i, j, slot, spi; 4436 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 4437 enum bpf_access_type bounds_check_type; 4438 /* Some accesses can write anything into the stack, others are 4439 * read-only. 4440 */ 4441 bool clobber = false; 4442 4443 if (access_size == 0 && !zero_size_allowed) { 4444 verbose(env, "invalid zero-sized read\n"); 4445 return -EACCES; 4446 } 4447 4448 if (type == ACCESS_HELPER) { 4449 /* The bounds checks for writes are more permissive than for 4450 * reads. However, if raw_mode is not set, we'll do extra 4451 * checks below. 4452 */ 4453 bounds_check_type = BPF_WRITE; 4454 clobber = true; 4455 } else { 4456 bounds_check_type = BPF_READ; 4457 } 4458 err = check_stack_access_within_bounds(env, regno, off, access_size, 4459 type, bounds_check_type); 4460 if (err) 4461 return err; 4462 4463 4464 if (tnum_is_const(reg->var_off)) { 4465 min_off = max_off = reg->var_off.value + off; 4466 } else { 4467 /* Variable offset is prohibited for unprivileged mode for 4468 * simplicity since it requires corresponding support in 4469 * Spectre masking for stack ALU. 4470 * See also retrieve_ptr_limit(). 4471 */ 4472 if (!env->bypass_spec_v1) { 4473 char tn_buf[48]; 4474 4475 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4476 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 4477 regno, err_extra, tn_buf); 4478 return -EACCES; 4479 } 4480 /* Only initialized buffer on stack is allowed to be accessed 4481 * with variable offset. With uninitialized buffer it's hard to 4482 * guarantee that whole memory is marked as initialized on 4483 * helper return since specific bounds are unknown what may 4484 * cause uninitialized stack leaking. 4485 */ 4486 if (meta && meta->raw_mode) 4487 meta = NULL; 4488 4489 min_off = reg->smin_value + off; 4490 max_off = reg->smax_value + off; 4491 } 4492 4493 if (meta && meta->raw_mode) { 4494 meta->access_size = access_size; 4495 meta->regno = regno; 4496 return 0; 4497 } 4498 4499 for (i = min_off; i < max_off + access_size; i++) { 4500 u8 *stype; 4501 4502 slot = -i - 1; 4503 spi = slot / BPF_REG_SIZE; 4504 if (state->allocated_stack <= slot) 4505 goto err; 4506 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 4507 if (*stype == STACK_MISC) 4508 goto mark; 4509 if (*stype == STACK_ZERO) { 4510 if (clobber) { 4511 /* helper can write anything into the stack */ 4512 *stype = STACK_MISC; 4513 } 4514 goto mark; 4515 } 4516 4517 if (state->stack[spi].slot_type[0] == STACK_SPILL && 4518 state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID) 4519 goto mark; 4520 4521 if (state->stack[spi].slot_type[0] == STACK_SPILL && 4522 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 4523 env->allow_ptr_leaks)) { 4524 if (clobber) { 4525 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 4526 for (j = 0; j < BPF_REG_SIZE; j++) 4527 state->stack[spi].slot_type[j] = STACK_MISC; 4528 } 4529 goto mark; 4530 } 4531 4532 err: 4533 if (tnum_is_const(reg->var_off)) { 4534 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 4535 err_extra, regno, min_off, i - min_off, access_size); 4536 } else { 4537 char tn_buf[48]; 4538 4539 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4540 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 4541 err_extra, regno, tn_buf, i - min_off, access_size); 4542 } 4543 return -EACCES; 4544 mark: 4545 /* reading any byte out of 8-byte 'spill_slot' will cause 4546 * the whole slot to be marked as 'read' 4547 */ 4548 mark_reg_read(env, &state->stack[spi].spilled_ptr, 4549 state->stack[spi].spilled_ptr.parent, 4550 REG_LIVE_READ64); 4551 } 4552 return update_stack_depth(env, state, min_off); 4553 } 4554 4555 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 4556 int access_size, bool zero_size_allowed, 4557 struct bpf_call_arg_meta *meta) 4558 { 4559 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4560 4561 switch (reg->type) { 4562 case PTR_TO_PACKET: 4563 case PTR_TO_PACKET_META: 4564 return check_packet_access(env, regno, reg->off, access_size, 4565 zero_size_allowed); 4566 case PTR_TO_MAP_KEY: 4567 return check_mem_region_access(env, regno, reg->off, access_size, 4568 reg->map_ptr->key_size, false); 4569 case PTR_TO_MAP_VALUE: 4570 if (check_map_access_type(env, regno, reg->off, access_size, 4571 meta && meta->raw_mode ? BPF_WRITE : 4572 BPF_READ)) 4573 return -EACCES; 4574 return check_map_access(env, regno, reg->off, access_size, 4575 zero_size_allowed); 4576 case PTR_TO_MEM: 4577 return check_mem_region_access(env, regno, reg->off, 4578 access_size, reg->mem_size, 4579 zero_size_allowed); 4580 case PTR_TO_RDONLY_BUF: 4581 if (meta && meta->raw_mode) 4582 return -EACCES; 4583 return check_buffer_access(env, reg, regno, reg->off, 4584 access_size, zero_size_allowed, 4585 "rdonly", 4586 &env->prog->aux->max_rdonly_access); 4587 case PTR_TO_RDWR_BUF: 4588 return check_buffer_access(env, reg, regno, reg->off, 4589 access_size, zero_size_allowed, 4590 "rdwr", 4591 &env->prog->aux->max_rdwr_access); 4592 case PTR_TO_STACK: 4593 return check_stack_range_initialized( 4594 env, 4595 regno, reg->off, access_size, 4596 zero_size_allowed, ACCESS_HELPER, meta); 4597 default: /* scalar_value or invalid ptr */ 4598 /* Allow zero-byte read from NULL, regardless of pointer type */ 4599 if (zero_size_allowed && access_size == 0 && 4600 register_is_null(reg)) 4601 return 0; 4602 4603 verbose(env, "R%d type=%s expected=%s\n", regno, 4604 reg_type_str[reg->type], 4605 reg_type_str[PTR_TO_STACK]); 4606 return -EACCES; 4607 } 4608 } 4609 4610 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 4611 u32 regno, u32 mem_size) 4612 { 4613 if (register_is_null(reg)) 4614 return 0; 4615 4616 if (reg_type_may_be_null(reg->type)) { 4617 /* Assuming that the register contains a value check if the memory 4618 * access is safe. Temporarily save and restore the register's state as 4619 * the conversion shouldn't be visible to a caller. 4620 */ 4621 const struct bpf_reg_state saved_reg = *reg; 4622 int rv; 4623 4624 mark_ptr_not_null_reg(reg); 4625 rv = check_helper_mem_access(env, regno, mem_size, true, NULL); 4626 *reg = saved_reg; 4627 return rv; 4628 } 4629 4630 return check_helper_mem_access(env, regno, mem_size, true, NULL); 4631 } 4632 4633 /* Implementation details: 4634 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL 4635 * Two bpf_map_lookups (even with the same key) will have different reg->id. 4636 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after 4637 * value_or_null->value transition, since the verifier only cares about 4638 * the range of access to valid map value pointer and doesn't care about actual 4639 * address of the map element. 4640 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 4641 * reg->id > 0 after value_or_null->value transition. By doing so 4642 * two bpf_map_lookups will be considered two different pointers that 4643 * point to different bpf_spin_locks. 4644 * The verifier allows taking only one bpf_spin_lock at a time to avoid 4645 * dead-locks. 4646 * Since only one bpf_spin_lock is allowed the checks are simpler than 4647 * reg_is_refcounted() logic. The verifier needs to remember only 4648 * one spin_lock instead of array of acquired_refs. 4649 * cur_state->active_spin_lock remembers which map value element got locked 4650 * and clears it after bpf_spin_unlock. 4651 */ 4652 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 4653 bool is_lock) 4654 { 4655 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4656 struct bpf_verifier_state *cur = env->cur_state; 4657 bool is_const = tnum_is_const(reg->var_off); 4658 struct bpf_map *map = reg->map_ptr; 4659 u64 val = reg->var_off.value; 4660 4661 if (!is_const) { 4662 verbose(env, 4663 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 4664 regno); 4665 return -EINVAL; 4666 } 4667 if (!map->btf) { 4668 verbose(env, 4669 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 4670 map->name); 4671 return -EINVAL; 4672 } 4673 if (!map_value_has_spin_lock(map)) { 4674 if (map->spin_lock_off == -E2BIG) 4675 verbose(env, 4676 "map '%s' has more than one 'struct bpf_spin_lock'\n", 4677 map->name); 4678 else if (map->spin_lock_off == -ENOENT) 4679 verbose(env, 4680 "map '%s' doesn't have 'struct bpf_spin_lock'\n", 4681 map->name); 4682 else 4683 verbose(env, 4684 "map '%s' is not a struct type or bpf_spin_lock is mangled\n", 4685 map->name); 4686 return -EINVAL; 4687 } 4688 if (map->spin_lock_off != val + reg->off) { 4689 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", 4690 val + reg->off); 4691 return -EINVAL; 4692 } 4693 if (is_lock) { 4694 if (cur->active_spin_lock) { 4695 verbose(env, 4696 "Locking two bpf_spin_locks are not allowed\n"); 4697 return -EINVAL; 4698 } 4699 cur->active_spin_lock = reg->id; 4700 } else { 4701 if (!cur->active_spin_lock) { 4702 verbose(env, "bpf_spin_unlock without taking a lock\n"); 4703 return -EINVAL; 4704 } 4705 if (cur->active_spin_lock != reg->id) { 4706 verbose(env, "bpf_spin_unlock of different lock\n"); 4707 return -EINVAL; 4708 } 4709 cur->active_spin_lock = 0; 4710 } 4711 return 0; 4712 } 4713 4714 static int process_timer_func(struct bpf_verifier_env *env, int regno, 4715 struct bpf_call_arg_meta *meta) 4716 { 4717 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4718 bool is_const = tnum_is_const(reg->var_off); 4719 struct bpf_map *map = reg->map_ptr; 4720 u64 val = reg->var_off.value; 4721 4722 if (!is_const) { 4723 verbose(env, 4724 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 4725 regno); 4726 return -EINVAL; 4727 } 4728 if (!map->btf) { 4729 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 4730 map->name); 4731 return -EINVAL; 4732 } 4733 if (!map_value_has_timer(map)) { 4734 if (map->timer_off == -E2BIG) 4735 verbose(env, 4736 "map '%s' has more than one 'struct bpf_timer'\n", 4737 map->name); 4738 else if (map->timer_off == -ENOENT) 4739 verbose(env, 4740 "map '%s' doesn't have 'struct bpf_timer'\n", 4741 map->name); 4742 else 4743 verbose(env, 4744 "map '%s' is not a struct type or bpf_timer is mangled\n", 4745 map->name); 4746 return -EINVAL; 4747 } 4748 if (map->timer_off != val + reg->off) { 4749 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 4750 val + reg->off, map->timer_off); 4751 return -EINVAL; 4752 } 4753 if (meta->map_ptr) { 4754 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 4755 return -EFAULT; 4756 } 4757 meta->map_uid = reg->map_uid; 4758 meta->map_ptr = map; 4759 return 0; 4760 } 4761 4762 static bool arg_type_is_mem_ptr(enum bpf_arg_type type) 4763 { 4764 return type == ARG_PTR_TO_MEM || 4765 type == ARG_PTR_TO_MEM_OR_NULL || 4766 type == ARG_PTR_TO_UNINIT_MEM; 4767 } 4768 4769 static bool arg_type_is_mem_size(enum bpf_arg_type type) 4770 { 4771 return type == ARG_CONST_SIZE || 4772 type == ARG_CONST_SIZE_OR_ZERO; 4773 } 4774 4775 static bool arg_type_is_alloc_size(enum bpf_arg_type type) 4776 { 4777 return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; 4778 } 4779 4780 static bool arg_type_is_int_ptr(enum bpf_arg_type type) 4781 { 4782 return type == ARG_PTR_TO_INT || 4783 type == ARG_PTR_TO_LONG; 4784 } 4785 4786 static int int_ptr_type_to_size(enum bpf_arg_type type) 4787 { 4788 if (type == ARG_PTR_TO_INT) 4789 return sizeof(u32); 4790 else if (type == ARG_PTR_TO_LONG) 4791 return sizeof(u64); 4792 4793 return -EINVAL; 4794 } 4795 4796 static int resolve_map_arg_type(struct bpf_verifier_env *env, 4797 const struct bpf_call_arg_meta *meta, 4798 enum bpf_arg_type *arg_type) 4799 { 4800 if (!meta->map_ptr) { 4801 /* kernel subsystem misconfigured verifier */ 4802 verbose(env, "invalid map_ptr to access map->type\n"); 4803 return -EACCES; 4804 } 4805 4806 switch (meta->map_ptr->map_type) { 4807 case BPF_MAP_TYPE_SOCKMAP: 4808 case BPF_MAP_TYPE_SOCKHASH: 4809 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 4810 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 4811 } else { 4812 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 4813 return -EINVAL; 4814 } 4815 break; 4816 4817 default: 4818 break; 4819 } 4820 return 0; 4821 } 4822 4823 struct bpf_reg_types { 4824 const enum bpf_reg_type types[10]; 4825 u32 *btf_id; 4826 }; 4827 4828 static const struct bpf_reg_types map_key_value_types = { 4829 .types = { 4830 PTR_TO_STACK, 4831 PTR_TO_PACKET, 4832 PTR_TO_PACKET_META, 4833 PTR_TO_MAP_KEY, 4834 PTR_TO_MAP_VALUE, 4835 }, 4836 }; 4837 4838 static const struct bpf_reg_types sock_types = { 4839 .types = { 4840 PTR_TO_SOCK_COMMON, 4841 PTR_TO_SOCKET, 4842 PTR_TO_TCP_SOCK, 4843 PTR_TO_XDP_SOCK, 4844 }, 4845 }; 4846 4847 #ifdef CONFIG_NET 4848 static const struct bpf_reg_types btf_id_sock_common_types = { 4849 .types = { 4850 PTR_TO_SOCK_COMMON, 4851 PTR_TO_SOCKET, 4852 PTR_TO_TCP_SOCK, 4853 PTR_TO_XDP_SOCK, 4854 PTR_TO_BTF_ID, 4855 }, 4856 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 4857 }; 4858 #endif 4859 4860 static const struct bpf_reg_types mem_types = { 4861 .types = { 4862 PTR_TO_STACK, 4863 PTR_TO_PACKET, 4864 PTR_TO_PACKET_META, 4865 PTR_TO_MAP_KEY, 4866 PTR_TO_MAP_VALUE, 4867 PTR_TO_MEM, 4868 PTR_TO_RDONLY_BUF, 4869 PTR_TO_RDWR_BUF, 4870 }, 4871 }; 4872 4873 static const struct bpf_reg_types int_ptr_types = { 4874 .types = { 4875 PTR_TO_STACK, 4876 PTR_TO_PACKET, 4877 PTR_TO_PACKET_META, 4878 PTR_TO_MAP_KEY, 4879 PTR_TO_MAP_VALUE, 4880 }, 4881 }; 4882 4883 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 4884 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 4885 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 4886 static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; 4887 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 4888 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; 4889 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; 4890 static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } }; 4891 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 4892 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 4893 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 4894 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 4895 4896 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 4897 [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, 4898 [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, 4899 [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, 4900 [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types, 4901 [ARG_CONST_SIZE] = &scalar_types, 4902 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 4903 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 4904 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 4905 [ARG_PTR_TO_CTX] = &context_types, 4906 [ARG_PTR_TO_CTX_OR_NULL] = &context_types, 4907 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 4908 #ifdef CONFIG_NET 4909 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 4910 #endif 4911 [ARG_PTR_TO_SOCKET] = &fullsock_types, 4912 [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, 4913 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 4914 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 4915 [ARG_PTR_TO_MEM] = &mem_types, 4916 [ARG_PTR_TO_MEM_OR_NULL] = &mem_types, 4917 [ARG_PTR_TO_UNINIT_MEM] = &mem_types, 4918 [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, 4919 [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types, 4920 [ARG_PTR_TO_INT] = &int_ptr_types, 4921 [ARG_PTR_TO_LONG] = &int_ptr_types, 4922 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 4923 [ARG_PTR_TO_FUNC] = &func_ptr_types, 4924 [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types, 4925 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 4926 [ARG_PTR_TO_TIMER] = &timer_types, 4927 }; 4928 4929 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 4930 enum bpf_arg_type arg_type, 4931 const u32 *arg_btf_id) 4932 { 4933 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4934 enum bpf_reg_type expected, type = reg->type; 4935 const struct bpf_reg_types *compatible; 4936 int i, j; 4937 4938 compatible = compatible_reg_types[arg_type]; 4939 if (!compatible) { 4940 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 4941 return -EFAULT; 4942 } 4943 4944 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 4945 expected = compatible->types[i]; 4946 if (expected == NOT_INIT) 4947 break; 4948 4949 if (type == expected) 4950 goto found; 4951 } 4952 4953 verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); 4954 for (j = 0; j + 1 < i; j++) 4955 verbose(env, "%s, ", reg_type_str[compatible->types[j]]); 4956 verbose(env, "%s\n", reg_type_str[compatible->types[j]]); 4957 return -EACCES; 4958 4959 found: 4960 if (type == PTR_TO_BTF_ID) { 4961 if (!arg_btf_id) { 4962 if (!compatible->btf_id) { 4963 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 4964 return -EFAULT; 4965 } 4966 arg_btf_id = compatible->btf_id; 4967 } 4968 4969 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 4970 btf_vmlinux, *arg_btf_id)) { 4971 verbose(env, "R%d is of type %s but %s is expected\n", 4972 regno, kernel_type_name(reg->btf, reg->btf_id), 4973 kernel_type_name(btf_vmlinux, *arg_btf_id)); 4974 return -EACCES; 4975 } 4976 4977 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4978 verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", 4979 regno); 4980 return -EACCES; 4981 } 4982 } 4983 4984 return 0; 4985 } 4986 4987 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 4988 struct bpf_call_arg_meta *meta, 4989 const struct bpf_func_proto *fn) 4990 { 4991 u32 regno = BPF_REG_1 + arg; 4992 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4993 enum bpf_arg_type arg_type = fn->arg_type[arg]; 4994 enum bpf_reg_type type = reg->type; 4995 int err = 0; 4996 4997 if (arg_type == ARG_DONTCARE) 4998 return 0; 4999 5000 err = check_reg_arg(env, regno, SRC_OP); 5001 if (err) 5002 return err; 5003 5004 if (arg_type == ARG_ANYTHING) { 5005 if (is_pointer_value(env, regno)) { 5006 verbose(env, "R%d leaks addr into helper function\n", 5007 regno); 5008 return -EACCES; 5009 } 5010 return 0; 5011 } 5012 5013 if (type_is_pkt_pointer(type) && 5014 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 5015 verbose(env, "helper access to the packet is not allowed\n"); 5016 return -EACCES; 5017 } 5018 5019 if (arg_type == ARG_PTR_TO_MAP_VALUE || 5020 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || 5021 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { 5022 err = resolve_map_arg_type(env, meta, &arg_type); 5023 if (err) 5024 return err; 5025 } 5026 5027 if (register_is_null(reg) && arg_type_may_be_null(arg_type)) 5028 /* A NULL register has a SCALAR_VALUE type, so skip 5029 * type checking. 5030 */ 5031 goto skip_type_check; 5032 5033 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); 5034 if (err) 5035 return err; 5036 5037 if (type == PTR_TO_CTX) { 5038 err = check_ctx_reg(env, reg, regno); 5039 if (err < 0) 5040 return err; 5041 } 5042 5043 skip_type_check: 5044 if (reg->ref_obj_id) { 5045 if (meta->ref_obj_id) { 5046 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 5047 regno, reg->ref_obj_id, 5048 meta->ref_obj_id); 5049 return -EFAULT; 5050 } 5051 meta->ref_obj_id = reg->ref_obj_id; 5052 } 5053 5054 if (arg_type == ARG_CONST_MAP_PTR) { 5055 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 5056 if (meta->map_ptr) { 5057 /* Use map_uid (which is unique id of inner map) to reject: 5058 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 5059 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 5060 * if (inner_map1 && inner_map2) { 5061 * timer = bpf_map_lookup_elem(inner_map1); 5062 * if (timer) 5063 * // mismatch would have been allowed 5064 * bpf_timer_init(timer, inner_map2); 5065 * } 5066 * 5067 * Comparing map_ptr is enough to distinguish normal and outer maps. 5068 */ 5069 if (meta->map_ptr != reg->map_ptr || 5070 meta->map_uid != reg->map_uid) { 5071 verbose(env, 5072 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 5073 meta->map_uid, reg->map_uid); 5074 return -EINVAL; 5075 } 5076 } 5077 meta->map_ptr = reg->map_ptr; 5078 meta->map_uid = reg->map_uid; 5079 } else if (arg_type == ARG_PTR_TO_MAP_KEY) { 5080 /* bpf_map_xxx(..., map_ptr, ..., key) call: 5081 * check that [key, key + map->key_size) are within 5082 * stack limits and initialized 5083 */ 5084 if (!meta->map_ptr) { 5085 /* in function declaration map_ptr must come before 5086 * map_key, so that it's verified and known before 5087 * we have to check map_key here. Otherwise it means 5088 * that kernel subsystem misconfigured verifier 5089 */ 5090 verbose(env, "invalid map_ptr to access map->key\n"); 5091 return -EACCES; 5092 } 5093 err = check_helper_mem_access(env, regno, 5094 meta->map_ptr->key_size, false, 5095 NULL); 5096 } else if (arg_type == ARG_PTR_TO_MAP_VALUE || 5097 (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && 5098 !register_is_null(reg)) || 5099 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { 5100 /* bpf_map_xxx(..., map_ptr, ..., value) call: 5101 * check [value, value + map->value_size) validity 5102 */ 5103 if (!meta->map_ptr) { 5104 /* kernel subsystem misconfigured verifier */ 5105 verbose(env, "invalid map_ptr to access map->value\n"); 5106 return -EACCES; 5107 } 5108 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); 5109 err = check_helper_mem_access(env, regno, 5110 meta->map_ptr->value_size, false, 5111 meta); 5112 } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { 5113 if (!reg->btf_id) { 5114 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 5115 return -EACCES; 5116 } 5117 meta->ret_btf = reg->btf; 5118 meta->ret_btf_id = reg->btf_id; 5119 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { 5120 if (meta->func_id == BPF_FUNC_spin_lock) { 5121 if (process_spin_lock(env, regno, true)) 5122 return -EACCES; 5123 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 5124 if (process_spin_lock(env, regno, false)) 5125 return -EACCES; 5126 } else { 5127 verbose(env, "verifier internal error\n"); 5128 return -EFAULT; 5129 } 5130 } else if (arg_type == ARG_PTR_TO_TIMER) { 5131 if (process_timer_func(env, regno, meta)) 5132 return -EACCES; 5133 } else if (arg_type == ARG_PTR_TO_FUNC) { 5134 meta->subprogno = reg->subprogno; 5135 } else if (arg_type_is_mem_ptr(arg_type)) { 5136 /* The access to this pointer is only checked when we hit the 5137 * next is_mem_size argument below. 5138 */ 5139 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM); 5140 } else if (arg_type_is_mem_size(arg_type)) { 5141 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); 5142 5143 /* This is used to refine r0 return value bounds for helpers 5144 * that enforce this value as an upper bound on return values. 5145 * See do_refine_retval_range() for helpers that can refine 5146 * the return value. C type of helper is u32 so we pull register 5147 * bound from umax_value however, if negative verifier errors 5148 * out. Only upper bounds can be learned because retval is an 5149 * int type and negative retvals are allowed. 5150 */ 5151 meta->msize_max_value = reg->umax_value; 5152 5153 /* The register is SCALAR_VALUE; the access check 5154 * happens using its boundaries. 5155 */ 5156 if (!tnum_is_const(reg->var_off)) 5157 /* For unprivileged variable accesses, disable raw 5158 * mode so that the program is required to 5159 * initialize all the memory that the helper could 5160 * just partially fill up. 5161 */ 5162 meta = NULL; 5163 5164 if (reg->smin_value < 0) { 5165 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 5166 regno); 5167 return -EACCES; 5168 } 5169 5170 if (reg->umin_value == 0) { 5171 err = check_helper_mem_access(env, regno - 1, 0, 5172 zero_size_allowed, 5173 meta); 5174 if (err) 5175 return err; 5176 } 5177 5178 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 5179 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 5180 regno); 5181 return -EACCES; 5182 } 5183 err = check_helper_mem_access(env, regno - 1, 5184 reg->umax_value, 5185 zero_size_allowed, meta); 5186 if (!err) 5187 err = mark_chain_precision(env, regno); 5188 } else if (arg_type_is_alloc_size(arg_type)) { 5189 if (!tnum_is_const(reg->var_off)) { 5190 verbose(env, "R%d is not a known constant'\n", 5191 regno); 5192 return -EACCES; 5193 } 5194 meta->mem_size = reg->var_off.value; 5195 } else if (arg_type_is_int_ptr(arg_type)) { 5196 int size = int_ptr_type_to_size(arg_type); 5197 5198 err = check_helper_mem_access(env, regno, size, false, meta); 5199 if (err) 5200 return err; 5201 err = check_ptr_alignment(env, reg, 0, size, true); 5202 } else if (arg_type == ARG_PTR_TO_CONST_STR) { 5203 struct bpf_map *map = reg->map_ptr; 5204 int map_off; 5205 u64 map_addr; 5206 char *str_ptr; 5207 5208 if (!bpf_map_is_rdonly(map)) { 5209 verbose(env, "R%d does not point to a readonly map'\n", regno); 5210 return -EACCES; 5211 } 5212 5213 if (!tnum_is_const(reg->var_off)) { 5214 verbose(env, "R%d is not a constant address'\n", regno); 5215 return -EACCES; 5216 } 5217 5218 if (!map->ops->map_direct_value_addr) { 5219 verbose(env, "no direct value access support for this map type\n"); 5220 return -EACCES; 5221 } 5222 5223 err = check_map_access(env, regno, reg->off, 5224 map->value_size - reg->off, false); 5225 if (err) 5226 return err; 5227 5228 map_off = reg->off + reg->var_off.value; 5229 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 5230 if (err) { 5231 verbose(env, "direct value access on string failed\n"); 5232 return err; 5233 } 5234 5235 str_ptr = (char *)(long)(map_addr); 5236 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 5237 verbose(env, "string is not zero-terminated\n"); 5238 return -EINVAL; 5239 } 5240 } 5241 5242 return err; 5243 } 5244 5245 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 5246 { 5247 enum bpf_attach_type eatype = env->prog->expected_attach_type; 5248 enum bpf_prog_type type = resolve_prog_type(env->prog); 5249 5250 if (func_id != BPF_FUNC_map_update_elem) 5251 return false; 5252 5253 /* It's not possible to get access to a locked struct sock in these 5254 * contexts, so updating is safe. 5255 */ 5256 switch (type) { 5257 case BPF_PROG_TYPE_TRACING: 5258 if (eatype == BPF_TRACE_ITER) 5259 return true; 5260 break; 5261 case BPF_PROG_TYPE_SOCKET_FILTER: 5262 case BPF_PROG_TYPE_SCHED_CLS: 5263 case BPF_PROG_TYPE_SCHED_ACT: 5264 case BPF_PROG_TYPE_XDP: 5265 case BPF_PROG_TYPE_SK_REUSEPORT: 5266 case BPF_PROG_TYPE_FLOW_DISSECTOR: 5267 case BPF_PROG_TYPE_SK_LOOKUP: 5268 return true; 5269 default: 5270 break; 5271 } 5272 5273 verbose(env, "cannot update sockmap in this context\n"); 5274 return false; 5275 } 5276 5277 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 5278 { 5279 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); 5280 } 5281 5282 static int check_map_func_compatibility(struct bpf_verifier_env *env, 5283 struct bpf_map *map, int func_id) 5284 { 5285 if (!map) 5286 return 0; 5287 5288 /* We need a two way check, first is from map perspective ... */ 5289 switch (map->map_type) { 5290 case BPF_MAP_TYPE_PROG_ARRAY: 5291 if (func_id != BPF_FUNC_tail_call) 5292 goto error; 5293 break; 5294 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 5295 if (func_id != BPF_FUNC_perf_event_read && 5296 func_id != BPF_FUNC_perf_event_output && 5297 func_id != BPF_FUNC_skb_output && 5298 func_id != BPF_FUNC_perf_event_read_value && 5299 func_id != BPF_FUNC_xdp_output) 5300 goto error; 5301 break; 5302 case BPF_MAP_TYPE_RINGBUF: 5303 if (func_id != BPF_FUNC_ringbuf_output && 5304 func_id != BPF_FUNC_ringbuf_reserve && 5305 func_id != BPF_FUNC_ringbuf_submit && 5306 func_id != BPF_FUNC_ringbuf_discard && 5307 func_id != BPF_FUNC_ringbuf_query) 5308 goto error; 5309 break; 5310 case BPF_MAP_TYPE_STACK_TRACE: 5311 if (func_id != BPF_FUNC_get_stackid) 5312 goto error; 5313 break; 5314 case BPF_MAP_TYPE_CGROUP_ARRAY: 5315 if (func_id != BPF_FUNC_skb_under_cgroup && 5316 func_id != BPF_FUNC_current_task_under_cgroup) 5317 goto error; 5318 break; 5319 case BPF_MAP_TYPE_CGROUP_STORAGE: 5320 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 5321 if (func_id != BPF_FUNC_get_local_storage) 5322 goto error; 5323 break; 5324 case BPF_MAP_TYPE_DEVMAP: 5325 case BPF_MAP_TYPE_DEVMAP_HASH: 5326 if (func_id != BPF_FUNC_redirect_map && 5327 func_id != BPF_FUNC_map_lookup_elem) 5328 goto error; 5329 break; 5330 /* Restrict bpf side of cpumap and xskmap, open when use-cases 5331 * appear. 5332 */ 5333 case BPF_MAP_TYPE_CPUMAP: 5334 if (func_id != BPF_FUNC_redirect_map) 5335 goto error; 5336 break; 5337 case BPF_MAP_TYPE_XSKMAP: 5338 if (func_id != BPF_FUNC_redirect_map && 5339 func_id != BPF_FUNC_map_lookup_elem) 5340 goto error; 5341 break; 5342 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 5343 case BPF_MAP_TYPE_HASH_OF_MAPS: 5344 if (func_id != BPF_FUNC_map_lookup_elem) 5345 goto error; 5346 break; 5347 case BPF_MAP_TYPE_SOCKMAP: 5348 if (func_id != BPF_FUNC_sk_redirect_map && 5349 func_id != BPF_FUNC_sock_map_update && 5350 func_id != BPF_FUNC_map_delete_elem && 5351 func_id != BPF_FUNC_msg_redirect_map && 5352 func_id != BPF_FUNC_sk_select_reuseport && 5353 func_id != BPF_FUNC_map_lookup_elem && 5354 !may_update_sockmap(env, func_id)) 5355 goto error; 5356 break; 5357 case BPF_MAP_TYPE_SOCKHASH: 5358 if (func_id != BPF_FUNC_sk_redirect_hash && 5359 func_id != BPF_FUNC_sock_hash_update && 5360 func_id != BPF_FUNC_map_delete_elem && 5361 func_id != BPF_FUNC_msg_redirect_hash && 5362 func_id != BPF_FUNC_sk_select_reuseport && 5363 func_id != BPF_FUNC_map_lookup_elem && 5364 !may_update_sockmap(env, func_id)) 5365 goto error; 5366 break; 5367 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 5368 if (func_id != BPF_FUNC_sk_select_reuseport) 5369 goto error; 5370 break; 5371 case BPF_MAP_TYPE_QUEUE: 5372 case BPF_MAP_TYPE_STACK: 5373 if (func_id != BPF_FUNC_map_peek_elem && 5374 func_id != BPF_FUNC_map_pop_elem && 5375 func_id != BPF_FUNC_map_push_elem) 5376 goto error; 5377 break; 5378 case BPF_MAP_TYPE_SK_STORAGE: 5379 if (func_id != BPF_FUNC_sk_storage_get && 5380 func_id != BPF_FUNC_sk_storage_delete) 5381 goto error; 5382 break; 5383 case BPF_MAP_TYPE_INODE_STORAGE: 5384 if (func_id != BPF_FUNC_inode_storage_get && 5385 func_id != BPF_FUNC_inode_storage_delete) 5386 goto error; 5387 break; 5388 case BPF_MAP_TYPE_TASK_STORAGE: 5389 if (func_id != BPF_FUNC_task_storage_get && 5390 func_id != BPF_FUNC_task_storage_delete) 5391 goto error; 5392 break; 5393 default: 5394 break; 5395 } 5396 5397 /* ... and second from the function itself. */ 5398 switch (func_id) { 5399 case BPF_FUNC_tail_call: 5400 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 5401 goto error; 5402 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 5403 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 5404 return -EINVAL; 5405 } 5406 break; 5407 case BPF_FUNC_perf_event_read: 5408 case BPF_FUNC_perf_event_output: 5409 case BPF_FUNC_perf_event_read_value: 5410 case BPF_FUNC_skb_output: 5411 case BPF_FUNC_xdp_output: 5412 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 5413 goto error; 5414 break; 5415 case BPF_FUNC_get_stackid: 5416 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 5417 goto error; 5418 break; 5419 case BPF_FUNC_current_task_under_cgroup: 5420 case BPF_FUNC_skb_under_cgroup: 5421 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 5422 goto error; 5423 break; 5424 case BPF_FUNC_redirect_map: 5425 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 5426 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 5427 map->map_type != BPF_MAP_TYPE_CPUMAP && 5428 map->map_type != BPF_MAP_TYPE_XSKMAP) 5429 goto error; 5430 break; 5431 case BPF_FUNC_sk_redirect_map: 5432 case BPF_FUNC_msg_redirect_map: 5433 case BPF_FUNC_sock_map_update: 5434 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 5435 goto error; 5436 break; 5437 case BPF_FUNC_sk_redirect_hash: 5438 case BPF_FUNC_msg_redirect_hash: 5439 case BPF_FUNC_sock_hash_update: 5440 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 5441 goto error; 5442 break; 5443 case BPF_FUNC_get_local_storage: 5444 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 5445 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 5446 goto error; 5447 break; 5448 case BPF_FUNC_sk_select_reuseport: 5449 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 5450 map->map_type != BPF_MAP_TYPE_SOCKMAP && 5451 map->map_type != BPF_MAP_TYPE_SOCKHASH) 5452 goto error; 5453 break; 5454 case BPF_FUNC_map_peek_elem: 5455 case BPF_FUNC_map_pop_elem: 5456 case BPF_FUNC_map_push_elem: 5457 if (map->map_type != BPF_MAP_TYPE_QUEUE && 5458 map->map_type != BPF_MAP_TYPE_STACK) 5459 goto error; 5460 break; 5461 case BPF_FUNC_sk_storage_get: 5462 case BPF_FUNC_sk_storage_delete: 5463 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 5464 goto error; 5465 break; 5466 case BPF_FUNC_inode_storage_get: 5467 case BPF_FUNC_inode_storage_delete: 5468 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 5469 goto error; 5470 break; 5471 case BPF_FUNC_task_storage_get: 5472 case BPF_FUNC_task_storage_delete: 5473 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 5474 goto error; 5475 break; 5476 default: 5477 break; 5478 } 5479 5480 return 0; 5481 error: 5482 verbose(env, "cannot pass map_type %d into func %s#%d\n", 5483 map->map_type, func_id_name(func_id), func_id); 5484 return -EINVAL; 5485 } 5486 5487 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 5488 { 5489 int count = 0; 5490 5491 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 5492 count++; 5493 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 5494 count++; 5495 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 5496 count++; 5497 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 5498 count++; 5499 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 5500 count++; 5501 5502 /* We only support one arg being in raw mode at the moment, 5503 * which is sufficient for the helper functions we have 5504 * right now. 5505 */ 5506 return count <= 1; 5507 } 5508 5509 static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, 5510 enum bpf_arg_type arg_next) 5511 { 5512 return (arg_type_is_mem_ptr(arg_curr) && 5513 !arg_type_is_mem_size(arg_next)) || 5514 (!arg_type_is_mem_ptr(arg_curr) && 5515 arg_type_is_mem_size(arg_next)); 5516 } 5517 5518 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 5519 { 5520 /* bpf_xxx(..., buf, len) call will access 'len' 5521 * bytes from memory 'buf'. Both arg types need 5522 * to be paired, so make sure there's no buggy 5523 * helper function specification. 5524 */ 5525 if (arg_type_is_mem_size(fn->arg1_type) || 5526 arg_type_is_mem_ptr(fn->arg5_type) || 5527 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || 5528 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || 5529 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || 5530 check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) 5531 return false; 5532 5533 return true; 5534 } 5535 5536 static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) 5537 { 5538 int count = 0; 5539 5540 if (arg_type_may_be_refcounted(fn->arg1_type)) 5541 count++; 5542 if (arg_type_may_be_refcounted(fn->arg2_type)) 5543 count++; 5544 if (arg_type_may_be_refcounted(fn->arg3_type)) 5545 count++; 5546 if (arg_type_may_be_refcounted(fn->arg4_type)) 5547 count++; 5548 if (arg_type_may_be_refcounted(fn->arg5_type)) 5549 count++; 5550 5551 /* A reference acquiring function cannot acquire 5552 * another refcounted ptr. 5553 */ 5554 if (may_be_acquire_function(func_id) && count) 5555 return false; 5556 5557 /* We only support one arg being unreferenced at the moment, 5558 * which is sufficient for the helper functions we have right now. 5559 */ 5560 return count <= 1; 5561 } 5562 5563 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 5564 { 5565 int i; 5566 5567 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 5568 if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) 5569 return false; 5570 5571 if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) 5572 return false; 5573 } 5574 5575 return true; 5576 } 5577 5578 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 5579 { 5580 return check_raw_mode_ok(fn) && 5581 check_arg_pair_ok(fn) && 5582 check_btf_id_ok(fn) && 5583 check_refcount_ok(fn, func_id) ? 0 : -EINVAL; 5584 } 5585 5586 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 5587 * are now invalid, so turn them into unknown SCALAR_VALUE. 5588 */ 5589 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, 5590 struct bpf_func_state *state) 5591 { 5592 struct bpf_reg_state *regs = state->regs, *reg; 5593 int i; 5594 5595 for (i = 0; i < MAX_BPF_REG; i++) 5596 if (reg_is_pkt_pointer_any(®s[i])) 5597 mark_reg_unknown(env, regs, i); 5598 5599 bpf_for_each_spilled_reg(i, state, reg) { 5600 if (!reg) 5601 continue; 5602 if (reg_is_pkt_pointer_any(reg)) 5603 __mark_reg_unknown(env, reg); 5604 } 5605 } 5606 5607 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 5608 { 5609 struct bpf_verifier_state *vstate = env->cur_state; 5610 int i; 5611 5612 for (i = 0; i <= vstate->curframe; i++) 5613 __clear_all_pkt_pointers(env, vstate->frame[i]); 5614 } 5615 5616 enum { 5617 AT_PKT_END = -1, 5618 BEYOND_PKT_END = -2, 5619 }; 5620 5621 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 5622 { 5623 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 5624 struct bpf_reg_state *reg = &state->regs[regn]; 5625 5626 if (reg->type != PTR_TO_PACKET) 5627 /* PTR_TO_PACKET_META is not supported yet */ 5628 return; 5629 5630 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 5631 * How far beyond pkt_end it goes is unknown. 5632 * if (!range_open) it's the case of pkt >= pkt_end 5633 * if (range_open) it's the case of pkt > pkt_end 5634 * hence this pointer is at least 1 byte bigger than pkt_end 5635 */ 5636 if (range_open) 5637 reg->range = BEYOND_PKT_END; 5638 else 5639 reg->range = AT_PKT_END; 5640 } 5641 5642 static void release_reg_references(struct bpf_verifier_env *env, 5643 struct bpf_func_state *state, 5644 int ref_obj_id) 5645 { 5646 struct bpf_reg_state *regs = state->regs, *reg; 5647 int i; 5648 5649 for (i = 0; i < MAX_BPF_REG; i++) 5650 if (regs[i].ref_obj_id == ref_obj_id) 5651 mark_reg_unknown(env, regs, i); 5652 5653 bpf_for_each_spilled_reg(i, state, reg) { 5654 if (!reg) 5655 continue; 5656 if (reg->ref_obj_id == ref_obj_id) 5657 __mark_reg_unknown(env, reg); 5658 } 5659 } 5660 5661 /* The pointer with the specified id has released its reference to kernel 5662 * resources. Identify all copies of the same pointer and clear the reference. 5663 */ 5664 static int release_reference(struct bpf_verifier_env *env, 5665 int ref_obj_id) 5666 { 5667 struct bpf_verifier_state *vstate = env->cur_state; 5668 int err; 5669 int i; 5670 5671 err = release_reference_state(cur_func(env), ref_obj_id); 5672 if (err) 5673 return err; 5674 5675 for (i = 0; i <= vstate->curframe; i++) 5676 release_reg_references(env, vstate->frame[i], ref_obj_id); 5677 5678 return 0; 5679 } 5680 5681 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 5682 struct bpf_reg_state *regs) 5683 { 5684 int i; 5685 5686 /* after the call registers r0 - r5 were scratched */ 5687 for (i = 0; i < CALLER_SAVED_REGS; i++) { 5688 mark_reg_not_init(env, regs, caller_saved[i]); 5689 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 5690 } 5691 } 5692 5693 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 5694 struct bpf_func_state *caller, 5695 struct bpf_func_state *callee, 5696 int insn_idx); 5697 5698 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5699 int *insn_idx, int subprog, 5700 set_callee_state_fn set_callee_state_cb) 5701 { 5702 struct bpf_verifier_state *state = env->cur_state; 5703 struct bpf_func_info_aux *func_info_aux; 5704 struct bpf_func_state *caller, *callee; 5705 int err; 5706 bool is_global = false; 5707 5708 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 5709 verbose(env, "the call stack of %d frames is too deep\n", 5710 state->curframe + 2); 5711 return -E2BIG; 5712 } 5713 5714 caller = state->frame[state->curframe]; 5715 if (state->frame[state->curframe + 1]) { 5716 verbose(env, "verifier bug. Frame %d already allocated\n", 5717 state->curframe + 1); 5718 return -EFAULT; 5719 } 5720 5721 func_info_aux = env->prog->aux->func_info_aux; 5722 if (func_info_aux) 5723 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 5724 err = btf_check_subprog_arg_match(env, subprog, caller->regs); 5725 if (err == -EFAULT) 5726 return err; 5727 if (is_global) { 5728 if (err) { 5729 verbose(env, "Caller passes invalid args into func#%d\n", 5730 subprog); 5731 return err; 5732 } else { 5733 if (env->log.level & BPF_LOG_LEVEL) 5734 verbose(env, 5735 "Func#%d is global and valid. Skipping.\n", 5736 subprog); 5737 clear_caller_saved_regs(env, caller->regs); 5738 5739 /* All global functions return a 64-bit SCALAR_VALUE */ 5740 mark_reg_unknown(env, caller->regs, BPF_REG_0); 5741 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 5742 5743 /* continue with next insn after call */ 5744 return 0; 5745 } 5746 } 5747 5748 if (insn->code == (BPF_JMP | BPF_CALL) && 5749 insn->imm == BPF_FUNC_timer_set_callback) { 5750 struct bpf_verifier_state *async_cb; 5751 5752 /* there is no real recursion here. timer callbacks are async */ 5753 env->subprog_info[subprog].is_async_cb = true; 5754 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 5755 *insn_idx, subprog); 5756 if (!async_cb) 5757 return -EFAULT; 5758 callee = async_cb->frame[0]; 5759 callee->async_entry_cnt = caller->async_entry_cnt + 1; 5760 5761 /* Convert bpf_timer_set_callback() args into timer callback args */ 5762 err = set_callee_state_cb(env, caller, callee, *insn_idx); 5763 if (err) 5764 return err; 5765 5766 clear_caller_saved_regs(env, caller->regs); 5767 mark_reg_unknown(env, caller->regs, BPF_REG_0); 5768 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 5769 /* continue with next insn after call */ 5770 return 0; 5771 } 5772 5773 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 5774 if (!callee) 5775 return -ENOMEM; 5776 state->frame[state->curframe + 1] = callee; 5777 5778 /* callee cannot access r0, r6 - r9 for reading and has to write 5779 * into its own stack before reading from it. 5780 * callee can read/write into caller's stack 5781 */ 5782 init_func_state(env, callee, 5783 /* remember the callsite, it will be used by bpf_exit */ 5784 *insn_idx /* callsite */, 5785 state->curframe + 1 /* frameno within this callchain */, 5786 subprog /* subprog number within this prog */); 5787 5788 /* Transfer references to the callee */ 5789 err = copy_reference_state(callee, caller); 5790 if (err) 5791 return err; 5792 5793 err = set_callee_state_cb(env, caller, callee, *insn_idx); 5794 if (err) 5795 return err; 5796 5797 clear_caller_saved_regs(env, caller->regs); 5798 5799 /* only increment it after check_reg_arg() finished */ 5800 state->curframe++; 5801 5802 /* and go analyze first insn of the callee */ 5803 *insn_idx = env->subprog_info[subprog].start - 1; 5804 5805 if (env->log.level & BPF_LOG_LEVEL) { 5806 verbose(env, "caller:\n"); 5807 print_verifier_state(env, caller); 5808 verbose(env, "callee:\n"); 5809 print_verifier_state(env, callee); 5810 } 5811 return 0; 5812 } 5813 5814 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 5815 struct bpf_func_state *caller, 5816 struct bpf_func_state *callee) 5817 { 5818 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 5819 * void *callback_ctx, u64 flags); 5820 * callback_fn(struct bpf_map *map, void *key, void *value, 5821 * void *callback_ctx); 5822 */ 5823 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 5824 5825 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 5826 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 5827 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 5828 5829 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 5830 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 5831 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 5832 5833 /* pointer to stack or null */ 5834 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 5835 5836 /* unused */ 5837 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 5838 return 0; 5839 } 5840 5841 static int set_callee_state(struct bpf_verifier_env *env, 5842 struct bpf_func_state *caller, 5843 struct bpf_func_state *callee, int insn_idx) 5844 { 5845 int i; 5846 5847 /* copy r1 - r5 args that callee can access. The copy includes parent 5848 * pointers, which connects us up to the liveness chain 5849 */ 5850 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 5851 callee->regs[i] = caller->regs[i]; 5852 return 0; 5853 } 5854 5855 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5856 int *insn_idx) 5857 { 5858 int subprog, target_insn; 5859 5860 target_insn = *insn_idx + insn->imm + 1; 5861 subprog = find_subprog(env, target_insn); 5862 if (subprog < 0) { 5863 verbose(env, "verifier bug. No program starts at insn %d\n", 5864 target_insn); 5865 return -EFAULT; 5866 } 5867 5868 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 5869 } 5870 5871 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 5872 struct bpf_func_state *caller, 5873 struct bpf_func_state *callee, 5874 int insn_idx) 5875 { 5876 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 5877 struct bpf_map *map; 5878 int err; 5879 5880 if (bpf_map_ptr_poisoned(insn_aux)) { 5881 verbose(env, "tail_call abusing map_ptr\n"); 5882 return -EINVAL; 5883 } 5884 5885 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 5886 if (!map->ops->map_set_for_each_callback_args || 5887 !map->ops->map_for_each_callback) { 5888 verbose(env, "callback function not allowed for map\n"); 5889 return -ENOTSUPP; 5890 } 5891 5892 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 5893 if (err) 5894 return err; 5895 5896 callee->in_callback_fn = true; 5897 return 0; 5898 } 5899 5900 static int set_timer_callback_state(struct bpf_verifier_env *env, 5901 struct bpf_func_state *caller, 5902 struct bpf_func_state *callee, 5903 int insn_idx) 5904 { 5905 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 5906 5907 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 5908 * callback_fn(struct bpf_map *map, void *key, void *value); 5909 */ 5910 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 5911 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 5912 callee->regs[BPF_REG_1].map_ptr = map_ptr; 5913 5914 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 5915 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 5916 callee->regs[BPF_REG_2].map_ptr = map_ptr; 5917 5918 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 5919 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 5920 callee->regs[BPF_REG_3].map_ptr = map_ptr; 5921 5922 /* unused */ 5923 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 5924 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 5925 callee->in_async_callback_fn = true; 5926 return 0; 5927 } 5928 5929 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 5930 { 5931 struct bpf_verifier_state *state = env->cur_state; 5932 struct bpf_func_state *caller, *callee; 5933 struct bpf_reg_state *r0; 5934 int err; 5935 5936 callee = state->frame[state->curframe]; 5937 r0 = &callee->regs[BPF_REG_0]; 5938 if (r0->type == PTR_TO_STACK) { 5939 /* technically it's ok to return caller's stack pointer 5940 * (or caller's caller's pointer) back to the caller, 5941 * since these pointers are valid. Only current stack 5942 * pointer will be invalid as soon as function exits, 5943 * but let's be conservative 5944 */ 5945 verbose(env, "cannot return stack pointer to the caller\n"); 5946 return -EINVAL; 5947 } 5948 5949 state->curframe--; 5950 caller = state->frame[state->curframe]; 5951 if (callee->in_callback_fn) { 5952 /* enforce R0 return value range [0, 1]. */ 5953 struct tnum range = tnum_range(0, 1); 5954 5955 if (r0->type != SCALAR_VALUE) { 5956 verbose(env, "R0 not a scalar value\n"); 5957 return -EACCES; 5958 } 5959 if (!tnum_in(range, r0->var_off)) { 5960 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 5961 return -EINVAL; 5962 } 5963 } else { 5964 /* return to the caller whatever r0 had in the callee */ 5965 caller->regs[BPF_REG_0] = *r0; 5966 } 5967 5968 /* Transfer references to the caller */ 5969 err = copy_reference_state(caller, callee); 5970 if (err) 5971 return err; 5972 5973 *insn_idx = callee->callsite + 1; 5974 if (env->log.level & BPF_LOG_LEVEL) { 5975 verbose(env, "returning from callee:\n"); 5976 print_verifier_state(env, callee); 5977 verbose(env, "to caller at %d:\n", *insn_idx); 5978 print_verifier_state(env, caller); 5979 } 5980 /* clear everything in the callee */ 5981 free_func_state(callee); 5982 state->frame[state->curframe + 1] = NULL; 5983 return 0; 5984 } 5985 5986 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 5987 int func_id, 5988 struct bpf_call_arg_meta *meta) 5989 { 5990 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 5991 5992 if (ret_type != RET_INTEGER || 5993 (func_id != BPF_FUNC_get_stack && 5994 func_id != BPF_FUNC_get_task_stack && 5995 func_id != BPF_FUNC_probe_read_str && 5996 func_id != BPF_FUNC_probe_read_kernel_str && 5997 func_id != BPF_FUNC_probe_read_user_str)) 5998 return; 5999 6000 ret_reg->smax_value = meta->msize_max_value; 6001 ret_reg->s32_max_value = meta->msize_max_value; 6002 ret_reg->smin_value = -MAX_ERRNO; 6003 ret_reg->s32_min_value = -MAX_ERRNO; 6004 __reg_deduce_bounds(ret_reg); 6005 __reg_bound_offset(ret_reg); 6006 __update_reg_bounds(ret_reg); 6007 } 6008 6009 static int 6010 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 6011 int func_id, int insn_idx) 6012 { 6013 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 6014 struct bpf_map *map = meta->map_ptr; 6015 6016 if (func_id != BPF_FUNC_tail_call && 6017 func_id != BPF_FUNC_map_lookup_elem && 6018 func_id != BPF_FUNC_map_update_elem && 6019 func_id != BPF_FUNC_map_delete_elem && 6020 func_id != BPF_FUNC_map_push_elem && 6021 func_id != BPF_FUNC_map_pop_elem && 6022 func_id != BPF_FUNC_map_peek_elem && 6023 func_id != BPF_FUNC_for_each_map_elem && 6024 func_id != BPF_FUNC_redirect_map) 6025 return 0; 6026 6027 if (map == NULL) { 6028 verbose(env, "kernel subsystem misconfigured verifier\n"); 6029 return -EINVAL; 6030 } 6031 6032 /* In case of read-only, some additional restrictions 6033 * need to be applied in order to prevent altering the 6034 * state of the map from program side. 6035 */ 6036 if ((map->map_flags & BPF_F_RDONLY_PROG) && 6037 (func_id == BPF_FUNC_map_delete_elem || 6038 func_id == BPF_FUNC_map_update_elem || 6039 func_id == BPF_FUNC_map_push_elem || 6040 func_id == BPF_FUNC_map_pop_elem)) { 6041 verbose(env, "write into map forbidden\n"); 6042 return -EACCES; 6043 } 6044 6045 if (!BPF_MAP_PTR(aux->map_ptr_state)) 6046 bpf_map_ptr_store(aux, meta->map_ptr, 6047 !meta->map_ptr->bypass_spec_v1); 6048 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 6049 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 6050 !meta->map_ptr->bypass_spec_v1); 6051 return 0; 6052 } 6053 6054 static int 6055 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 6056 int func_id, int insn_idx) 6057 { 6058 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 6059 struct bpf_reg_state *regs = cur_regs(env), *reg; 6060 struct bpf_map *map = meta->map_ptr; 6061 struct tnum range; 6062 u64 val; 6063 int err; 6064 6065 if (func_id != BPF_FUNC_tail_call) 6066 return 0; 6067 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 6068 verbose(env, "kernel subsystem misconfigured verifier\n"); 6069 return -EINVAL; 6070 } 6071 6072 range = tnum_range(0, map->max_entries - 1); 6073 reg = ®s[BPF_REG_3]; 6074 6075 if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { 6076 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 6077 return 0; 6078 } 6079 6080 err = mark_chain_precision(env, BPF_REG_3); 6081 if (err) 6082 return err; 6083 6084 val = reg->var_off.value; 6085 if (bpf_map_key_unseen(aux)) 6086 bpf_map_key_store(aux, val); 6087 else if (!bpf_map_key_poisoned(aux) && 6088 bpf_map_key_immediate(aux) != val) 6089 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 6090 return 0; 6091 } 6092 6093 static int check_reference_leak(struct bpf_verifier_env *env) 6094 { 6095 struct bpf_func_state *state = cur_func(env); 6096 int i; 6097 6098 for (i = 0; i < state->acquired_refs; i++) { 6099 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 6100 state->refs[i].id, state->refs[i].insn_idx); 6101 } 6102 return state->acquired_refs ? -EINVAL : 0; 6103 } 6104 6105 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 6106 struct bpf_reg_state *regs) 6107 { 6108 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 6109 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 6110 struct bpf_map *fmt_map = fmt_reg->map_ptr; 6111 int err, fmt_map_off, num_args; 6112 u64 fmt_addr; 6113 char *fmt; 6114 6115 /* data must be an array of u64 */ 6116 if (data_len_reg->var_off.value % 8) 6117 return -EINVAL; 6118 num_args = data_len_reg->var_off.value / 8; 6119 6120 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 6121 * and map_direct_value_addr is set. 6122 */ 6123 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 6124 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 6125 fmt_map_off); 6126 if (err) { 6127 verbose(env, "verifier bug\n"); 6128 return -EFAULT; 6129 } 6130 fmt = (char *)(long)fmt_addr + fmt_map_off; 6131 6132 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 6133 * can focus on validating the format specifiers. 6134 */ 6135 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args); 6136 if (err < 0) 6137 verbose(env, "Invalid format string\n"); 6138 6139 return err; 6140 } 6141 6142 static int check_get_func_ip(struct bpf_verifier_env *env) 6143 { 6144 enum bpf_attach_type eatype = env->prog->expected_attach_type; 6145 enum bpf_prog_type type = resolve_prog_type(env->prog); 6146 int func_id = BPF_FUNC_get_func_ip; 6147 6148 if (type == BPF_PROG_TYPE_TRACING) { 6149 if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT && 6150 eatype != BPF_MODIFY_RETURN) { 6151 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 6152 func_id_name(func_id), func_id); 6153 return -ENOTSUPP; 6154 } 6155 return 0; 6156 } else if (type == BPF_PROG_TYPE_KPROBE) { 6157 return 0; 6158 } 6159 6160 verbose(env, "func %s#%d not supported for program type %d\n", 6161 func_id_name(func_id), func_id, type); 6162 return -ENOTSUPP; 6163 } 6164 6165 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 6166 int *insn_idx_p) 6167 { 6168 const struct bpf_func_proto *fn = NULL; 6169 struct bpf_reg_state *regs; 6170 struct bpf_call_arg_meta meta; 6171 int insn_idx = *insn_idx_p; 6172 bool changes_data; 6173 int i, err, func_id; 6174 6175 /* find function prototype */ 6176 func_id = insn->imm; 6177 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 6178 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 6179 func_id); 6180 return -EINVAL; 6181 } 6182 6183 if (env->ops->get_func_proto) 6184 fn = env->ops->get_func_proto(func_id, env->prog); 6185 if (!fn) { 6186 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 6187 func_id); 6188 return -EINVAL; 6189 } 6190 6191 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 6192 if (!env->prog->gpl_compatible && fn->gpl_only) { 6193 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 6194 return -EINVAL; 6195 } 6196 6197 if (fn->allowed && !fn->allowed(env->prog)) { 6198 verbose(env, "helper call is not allowed in probe\n"); 6199 return -EINVAL; 6200 } 6201 6202 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 6203 changes_data = bpf_helper_changes_pkt_data(fn->func); 6204 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 6205 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 6206 func_id_name(func_id), func_id); 6207 return -EINVAL; 6208 } 6209 6210 memset(&meta, 0, sizeof(meta)); 6211 meta.pkt_access = fn->pkt_access; 6212 6213 err = check_func_proto(fn, func_id); 6214 if (err) { 6215 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 6216 func_id_name(func_id), func_id); 6217 return err; 6218 } 6219 6220 meta.func_id = func_id; 6221 /* check args */ 6222 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 6223 err = check_func_arg(env, i, &meta, fn); 6224 if (err) 6225 return err; 6226 } 6227 6228 err = record_func_map(env, &meta, func_id, insn_idx); 6229 if (err) 6230 return err; 6231 6232 err = record_func_key(env, &meta, func_id, insn_idx); 6233 if (err) 6234 return err; 6235 6236 /* Mark slots with STACK_MISC in case of raw mode, stack offset 6237 * is inferred from register state. 6238 */ 6239 for (i = 0; i < meta.access_size; i++) { 6240 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 6241 BPF_WRITE, -1, false); 6242 if (err) 6243 return err; 6244 } 6245 6246 if (func_id == BPF_FUNC_tail_call) { 6247 err = check_reference_leak(env); 6248 if (err) { 6249 verbose(env, "tail_call would lead to reference leak\n"); 6250 return err; 6251 } 6252 } else if (is_release_function(func_id)) { 6253 err = release_reference(env, meta.ref_obj_id); 6254 if (err) { 6255 verbose(env, "func %s#%d reference has not been acquired before\n", 6256 func_id_name(func_id), func_id); 6257 return err; 6258 } 6259 } 6260 6261 regs = cur_regs(env); 6262 6263 /* check that flags argument in get_local_storage(map, flags) is 0, 6264 * this is required because get_local_storage() can't return an error. 6265 */ 6266 if (func_id == BPF_FUNC_get_local_storage && 6267 !register_is_null(®s[BPF_REG_2])) { 6268 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 6269 return -EINVAL; 6270 } 6271 6272 if (func_id == BPF_FUNC_for_each_map_elem) { 6273 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 6274 set_map_elem_callback_state); 6275 if (err < 0) 6276 return -EINVAL; 6277 } 6278 6279 if (func_id == BPF_FUNC_timer_set_callback) { 6280 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 6281 set_timer_callback_state); 6282 if (err < 0) 6283 return -EINVAL; 6284 } 6285 6286 if (func_id == BPF_FUNC_snprintf) { 6287 err = check_bpf_snprintf_call(env, regs); 6288 if (err < 0) 6289 return err; 6290 } 6291 6292 /* reset caller saved regs */ 6293 for (i = 0; i < CALLER_SAVED_REGS; i++) { 6294 mark_reg_not_init(env, regs, caller_saved[i]); 6295 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 6296 } 6297 6298 /* helper call returns 64-bit value. */ 6299 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 6300 6301 /* update return register (already marked as written above) */ 6302 if (fn->ret_type == RET_INTEGER) { 6303 /* sets type to SCALAR_VALUE */ 6304 mark_reg_unknown(env, regs, BPF_REG_0); 6305 } else if (fn->ret_type == RET_VOID) { 6306 regs[BPF_REG_0].type = NOT_INIT; 6307 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || 6308 fn->ret_type == RET_PTR_TO_MAP_VALUE) { 6309 /* There is no offset yet applied, variable or fixed */ 6310 mark_reg_known_zero(env, regs, BPF_REG_0); 6311 /* remember map_ptr, so that check_map_access() 6312 * can check 'value_size' boundary of memory access 6313 * to map element returned from bpf_map_lookup_elem() 6314 */ 6315 if (meta.map_ptr == NULL) { 6316 verbose(env, 6317 "kernel subsystem misconfigured verifier\n"); 6318 return -EINVAL; 6319 } 6320 regs[BPF_REG_0].map_ptr = meta.map_ptr; 6321 regs[BPF_REG_0].map_uid = meta.map_uid; 6322 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { 6323 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; 6324 if (map_value_has_spin_lock(meta.map_ptr)) 6325 regs[BPF_REG_0].id = ++env->id_gen; 6326 } else { 6327 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; 6328 } 6329 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { 6330 mark_reg_known_zero(env, regs, BPF_REG_0); 6331 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; 6332 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { 6333 mark_reg_known_zero(env, regs, BPF_REG_0); 6334 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; 6335 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { 6336 mark_reg_known_zero(env, regs, BPF_REG_0); 6337 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; 6338 } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) { 6339 mark_reg_known_zero(env, regs, BPF_REG_0); 6340 regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; 6341 regs[BPF_REG_0].mem_size = meta.mem_size; 6342 } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL || 6343 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) { 6344 const struct btf_type *t; 6345 6346 mark_reg_known_zero(env, regs, BPF_REG_0); 6347 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 6348 if (!btf_type_is_struct(t)) { 6349 u32 tsize; 6350 const struct btf_type *ret; 6351 const char *tname; 6352 6353 /* resolve the type size of ksym. */ 6354 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 6355 if (IS_ERR(ret)) { 6356 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 6357 verbose(env, "unable to resolve the size of type '%s': %ld\n", 6358 tname, PTR_ERR(ret)); 6359 return -EINVAL; 6360 } 6361 regs[BPF_REG_0].type = 6362 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? 6363 PTR_TO_MEM : PTR_TO_MEM_OR_NULL; 6364 regs[BPF_REG_0].mem_size = tsize; 6365 } else { 6366 regs[BPF_REG_0].type = 6367 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? 6368 PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL; 6369 regs[BPF_REG_0].btf = meta.ret_btf; 6370 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 6371 } 6372 } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL || 6373 fn->ret_type == RET_PTR_TO_BTF_ID) { 6374 int ret_btf_id; 6375 6376 mark_reg_known_zero(env, regs, BPF_REG_0); 6377 regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ? 6378 PTR_TO_BTF_ID : 6379 PTR_TO_BTF_ID_OR_NULL; 6380 ret_btf_id = *fn->ret_btf_id; 6381 if (ret_btf_id == 0) { 6382 verbose(env, "invalid return type %d of func %s#%d\n", 6383 fn->ret_type, func_id_name(func_id), func_id); 6384 return -EINVAL; 6385 } 6386 /* current BPF helper definitions are only coming from 6387 * built-in code with type IDs from vmlinux BTF 6388 */ 6389 regs[BPF_REG_0].btf = btf_vmlinux; 6390 regs[BPF_REG_0].btf_id = ret_btf_id; 6391 } else { 6392 verbose(env, "unknown return type %d of func %s#%d\n", 6393 fn->ret_type, func_id_name(func_id), func_id); 6394 return -EINVAL; 6395 } 6396 6397 if (reg_type_may_be_null(regs[BPF_REG_0].type)) 6398 regs[BPF_REG_0].id = ++env->id_gen; 6399 6400 if (is_ptr_cast_function(func_id)) { 6401 /* For release_reference() */ 6402 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 6403 } else if (is_acquire_function(func_id, meta.map_ptr)) { 6404 int id = acquire_reference_state(env, insn_idx); 6405 6406 if (id < 0) 6407 return id; 6408 /* For mark_ptr_or_null_reg() */ 6409 regs[BPF_REG_0].id = id; 6410 /* For release_reference() */ 6411 regs[BPF_REG_0].ref_obj_id = id; 6412 } 6413 6414 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 6415 6416 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 6417 if (err) 6418 return err; 6419 6420 if ((func_id == BPF_FUNC_get_stack || 6421 func_id == BPF_FUNC_get_task_stack) && 6422 !env->prog->has_callchain_buf) { 6423 const char *err_str; 6424 6425 #ifdef CONFIG_PERF_EVENTS 6426 err = get_callchain_buffers(sysctl_perf_event_max_stack); 6427 err_str = "cannot get callchain buffer for func %s#%d\n"; 6428 #else 6429 err = -ENOTSUPP; 6430 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 6431 #endif 6432 if (err) { 6433 verbose(env, err_str, func_id_name(func_id), func_id); 6434 return err; 6435 } 6436 6437 env->prog->has_callchain_buf = true; 6438 } 6439 6440 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 6441 env->prog->call_get_stack = true; 6442 6443 if (func_id == BPF_FUNC_get_func_ip) { 6444 if (check_get_func_ip(env)) 6445 return -ENOTSUPP; 6446 env->prog->call_get_func_ip = true; 6447 } 6448 6449 if (changes_data) 6450 clear_all_pkt_pointers(env); 6451 return 0; 6452 } 6453 6454 /* mark_btf_func_reg_size() is used when the reg size is determined by 6455 * the BTF func_proto's return value size and argument. 6456 */ 6457 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 6458 size_t reg_size) 6459 { 6460 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 6461 6462 if (regno == BPF_REG_0) { 6463 /* Function return value */ 6464 reg->live |= REG_LIVE_WRITTEN; 6465 reg->subreg_def = reg_size == sizeof(u64) ? 6466 DEF_NOT_SUBREG : env->insn_idx + 1; 6467 } else { 6468 /* Function argument */ 6469 if (reg_size == sizeof(u64)) { 6470 mark_insn_zext(env, reg); 6471 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 6472 } else { 6473 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 6474 } 6475 } 6476 } 6477 6478 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn) 6479 { 6480 const struct btf_type *t, *func, *func_proto, *ptr_type; 6481 struct bpf_reg_state *regs = cur_regs(env); 6482 const char *func_name, *ptr_type_name; 6483 u32 i, nargs, func_id, ptr_type_id; 6484 const struct btf_param *args; 6485 int err; 6486 6487 func_id = insn->imm; 6488 func = btf_type_by_id(btf_vmlinux, func_id); 6489 func_name = btf_name_by_offset(btf_vmlinux, func->name_off); 6490 func_proto = btf_type_by_id(btf_vmlinux, func->type); 6491 6492 if (!env->ops->check_kfunc_call || 6493 !env->ops->check_kfunc_call(func_id)) { 6494 verbose(env, "calling kernel function %s is not allowed\n", 6495 func_name); 6496 return -EACCES; 6497 } 6498 6499 /* Check the arguments */ 6500 err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs); 6501 if (err) 6502 return err; 6503 6504 for (i = 0; i < CALLER_SAVED_REGS; i++) 6505 mark_reg_not_init(env, regs, caller_saved[i]); 6506 6507 /* Check return type */ 6508 t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL); 6509 if (btf_type_is_scalar(t)) { 6510 mark_reg_unknown(env, regs, BPF_REG_0); 6511 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 6512 } else if (btf_type_is_ptr(t)) { 6513 ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type, 6514 &ptr_type_id); 6515 if (!btf_type_is_struct(ptr_type)) { 6516 ptr_type_name = btf_name_by_offset(btf_vmlinux, 6517 ptr_type->name_off); 6518 verbose(env, "kernel function %s returns pointer type %s %s is not supported\n", 6519 func_name, btf_type_str(ptr_type), 6520 ptr_type_name); 6521 return -EINVAL; 6522 } 6523 mark_reg_known_zero(env, regs, BPF_REG_0); 6524 regs[BPF_REG_0].btf = btf_vmlinux; 6525 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 6526 regs[BPF_REG_0].btf_id = ptr_type_id; 6527 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 6528 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ 6529 6530 nargs = btf_type_vlen(func_proto); 6531 args = (const struct btf_param *)(func_proto + 1); 6532 for (i = 0; i < nargs; i++) { 6533 u32 regno = i + 1; 6534 6535 t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL); 6536 if (btf_type_is_ptr(t)) 6537 mark_btf_func_reg_size(env, regno, sizeof(void *)); 6538 else 6539 /* scalar. ensured by btf_check_kfunc_arg_match() */ 6540 mark_btf_func_reg_size(env, regno, t->size); 6541 } 6542 6543 return 0; 6544 } 6545 6546 static bool signed_add_overflows(s64 a, s64 b) 6547 { 6548 /* Do the add in u64, where overflow is well-defined */ 6549 s64 res = (s64)((u64)a + (u64)b); 6550 6551 if (b < 0) 6552 return res > a; 6553 return res < a; 6554 } 6555 6556 static bool signed_add32_overflows(s32 a, s32 b) 6557 { 6558 /* Do the add in u32, where overflow is well-defined */ 6559 s32 res = (s32)((u32)a + (u32)b); 6560 6561 if (b < 0) 6562 return res > a; 6563 return res < a; 6564 } 6565 6566 static bool signed_sub_overflows(s64 a, s64 b) 6567 { 6568 /* Do the sub in u64, where overflow is well-defined */ 6569 s64 res = (s64)((u64)a - (u64)b); 6570 6571 if (b < 0) 6572 return res < a; 6573 return res > a; 6574 } 6575 6576 static bool signed_sub32_overflows(s32 a, s32 b) 6577 { 6578 /* Do the sub in u32, where overflow is well-defined */ 6579 s32 res = (s32)((u32)a - (u32)b); 6580 6581 if (b < 0) 6582 return res < a; 6583 return res > a; 6584 } 6585 6586 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 6587 const struct bpf_reg_state *reg, 6588 enum bpf_reg_type type) 6589 { 6590 bool known = tnum_is_const(reg->var_off); 6591 s64 val = reg->var_off.value; 6592 s64 smin = reg->smin_value; 6593 6594 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 6595 verbose(env, "math between %s pointer and %lld is not allowed\n", 6596 reg_type_str[type], val); 6597 return false; 6598 } 6599 6600 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 6601 verbose(env, "%s pointer offset %d is not allowed\n", 6602 reg_type_str[type], reg->off); 6603 return false; 6604 } 6605 6606 if (smin == S64_MIN) { 6607 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 6608 reg_type_str[type]); 6609 return false; 6610 } 6611 6612 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 6613 verbose(env, "value %lld makes %s pointer be out of bounds\n", 6614 smin, reg_type_str[type]); 6615 return false; 6616 } 6617 6618 return true; 6619 } 6620 6621 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 6622 { 6623 return &env->insn_aux_data[env->insn_idx]; 6624 } 6625 6626 enum { 6627 REASON_BOUNDS = -1, 6628 REASON_TYPE = -2, 6629 REASON_PATHS = -3, 6630 REASON_LIMIT = -4, 6631 REASON_STACK = -5, 6632 }; 6633 6634 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 6635 u32 *alu_limit, bool mask_to_left) 6636 { 6637 u32 max = 0, ptr_limit = 0; 6638 6639 switch (ptr_reg->type) { 6640 case PTR_TO_STACK: 6641 /* Offset 0 is out-of-bounds, but acceptable start for the 6642 * left direction, see BPF_REG_FP. Also, unknown scalar 6643 * offset where we would need to deal with min/max bounds is 6644 * currently prohibited for unprivileged. 6645 */ 6646 max = MAX_BPF_STACK + mask_to_left; 6647 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 6648 break; 6649 case PTR_TO_MAP_VALUE: 6650 max = ptr_reg->map_ptr->value_size; 6651 ptr_limit = (mask_to_left ? 6652 ptr_reg->smin_value : 6653 ptr_reg->umax_value) + ptr_reg->off; 6654 break; 6655 default: 6656 return REASON_TYPE; 6657 } 6658 6659 if (ptr_limit >= max) 6660 return REASON_LIMIT; 6661 *alu_limit = ptr_limit; 6662 return 0; 6663 } 6664 6665 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 6666 const struct bpf_insn *insn) 6667 { 6668 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 6669 } 6670 6671 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 6672 u32 alu_state, u32 alu_limit) 6673 { 6674 /* If we arrived here from different branches with different 6675 * state or limits to sanitize, then this won't work. 6676 */ 6677 if (aux->alu_state && 6678 (aux->alu_state != alu_state || 6679 aux->alu_limit != alu_limit)) 6680 return REASON_PATHS; 6681 6682 /* Corresponding fixup done in do_misc_fixups(). */ 6683 aux->alu_state = alu_state; 6684 aux->alu_limit = alu_limit; 6685 return 0; 6686 } 6687 6688 static int sanitize_val_alu(struct bpf_verifier_env *env, 6689 struct bpf_insn *insn) 6690 { 6691 struct bpf_insn_aux_data *aux = cur_aux(env); 6692 6693 if (can_skip_alu_sanitation(env, insn)) 6694 return 0; 6695 6696 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 6697 } 6698 6699 static bool sanitize_needed(u8 opcode) 6700 { 6701 return opcode == BPF_ADD || opcode == BPF_SUB; 6702 } 6703 6704 struct bpf_sanitize_info { 6705 struct bpf_insn_aux_data aux; 6706 bool mask_to_left; 6707 }; 6708 6709 static struct bpf_verifier_state * 6710 sanitize_speculative_path(struct bpf_verifier_env *env, 6711 const struct bpf_insn *insn, 6712 u32 next_idx, u32 curr_idx) 6713 { 6714 struct bpf_verifier_state *branch; 6715 struct bpf_reg_state *regs; 6716 6717 branch = push_stack(env, next_idx, curr_idx, true); 6718 if (branch && insn) { 6719 regs = branch->frame[branch->curframe]->regs; 6720 if (BPF_SRC(insn->code) == BPF_K) { 6721 mark_reg_unknown(env, regs, insn->dst_reg); 6722 } else if (BPF_SRC(insn->code) == BPF_X) { 6723 mark_reg_unknown(env, regs, insn->dst_reg); 6724 mark_reg_unknown(env, regs, insn->src_reg); 6725 } 6726 } 6727 return branch; 6728 } 6729 6730 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 6731 struct bpf_insn *insn, 6732 const struct bpf_reg_state *ptr_reg, 6733 const struct bpf_reg_state *off_reg, 6734 struct bpf_reg_state *dst_reg, 6735 struct bpf_sanitize_info *info, 6736 const bool commit_window) 6737 { 6738 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 6739 struct bpf_verifier_state *vstate = env->cur_state; 6740 bool off_is_imm = tnum_is_const(off_reg->var_off); 6741 bool off_is_neg = off_reg->smin_value < 0; 6742 bool ptr_is_dst_reg = ptr_reg == dst_reg; 6743 u8 opcode = BPF_OP(insn->code); 6744 u32 alu_state, alu_limit; 6745 struct bpf_reg_state tmp; 6746 bool ret; 6747 int err; 6748 6749 if (can_skip_alu_sanitation(env, insn)) 6750 return 0; 6751 6752 /* We already marked aux for masking from non-speculative 6753 * paths, thus we got here in the first place. We only care 6754 * to explore bad access from here. 6755 */ 6756 if (vstate->speculative) 6757 goto do_sim; 6758 6759 if (!commit_window) { 6760 if (!tnum_is_const(off_reg->var_off) && 6761 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 6762 return REASON_BOUNDS; 6763 6764 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 6765 (opcode == BPF_SUB && !off_is_neg); 6766 } 6767 6768 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 6769 if (err < 0) 6770 return err; 6771 6772 if (commit_window) { 6773 /* In commit phase we narrow the masking window based on 6774 * the observed pointer move after the simulated operation. 6775 */ 6776 alu_state = info->aux.alu_state; 6777 alu_limit = abs(info->aux.alu_limit - alu_limit); 6778 } else { 6779 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 6780 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 6781 alu_state |= ptr_is_dst_reg ? 6782 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 6783 6784 /* Limit pruning on unknown scalars to enable deep search for 6785 * potential masking differences from other program paths. 6786 */ 6787 if (!off_is_imm) 6788 env->explore_alu_limits = true; 6789 } 6790 6791 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 6792 if (err < 0) 6793 return err; 6794 do_sim: 6795 /* If we're in commit phase, we're done here given we already 6796 * pushed the truncated dst_reg into the speculative verification 6797 * stack. 6798 * 6799 * Also, when register is a known constant, we rewrite register-based 6800 * operation to immediate-based, and thus do not need masking (and as 6801 * a consequence, do not need to simulate the zero-truncation either). 6802 */ 6803 if (commit_window || off_is_imm) 6804 return 0; 6805 6806 /* Simulate and find potential out-of-bounds access under 6807 * speculative execution from truncation as a result of 6808 * masking when off was not within expected range. If off 6809 * sits in dst, then we temporarily need to move ptr there 6810 * to simulate dst (== 0) +/-= ptr. Needed, for example, 6811 * for cases where we use K-based arithmetic in one direction 6812 * and truncated reg-based in the other in order to explore 6813 * bad access. 6814 */ 6815 if (!ptr_is_dst_reg) { 6816 tmp = *dst_reg; 6817 *dst_reg = *ptr_reg; 6818 } 6819 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 6820 env->insn_idx); 6821 if (!ptr_is_dst_reg && ret) 6822 *dst_reg = tmp; 6823 return !ret ? REASON_STACK : 0; 6824 } 6825 6826 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 6827 { 6828 struct bpf_verifier_state *vstate = env->cur_state; 6829 6830 /* If we simulate paths under speculation, we don't update the 6831 * insn as 'seen' such that when we verify unreachable paths in 6832 * the non-speculative domain, sanitize_dead_code() can still 6833 * rewrite/sanitize them. 6834 */ 6835 if (!vstate->speculative) 6836 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 6837 } 6838 6839 static int sanitize_err(struct bpf_verifier_env *env, 6840 const struct bpf_insn *insn, int reason, 6841 const struct bpf_reg_state *off_reg, 6842 const struct bpf_reg_state *dst_reg) 6843 { 6844 static const char *err = "pointer arithmetic with it prohibited for !root"; 6845 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 6846 u32 dst = insn->dst_reg, src = insn->src_reg; 6847 6848 switch (reason) { 6849 case REASON_BOUNDS: 6850 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 6851 off_reg == dst_reg ? dst : src, err); 6852 break; 6853 case REASON_TYPE: 6854 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 6855 off_reg == dst_reg ? src : dst, err); 6856 break; 6857 case REASON_PATHS: 6858 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 6859 dst, op, err); 6860 break; 6861 case REASON_LIMIT: 6862 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 6863 dst, op, err); 6864 break; 6865 case REASON_STACK: 6866 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 6867 dst, err); 6868 break; 6869 default: 6870 verbose(env, "verifier internal error: unknown reason (%d)\n", 6871 reason); 6872 break; 6873 } 6874 6875 return -EACCES; 6876 } 6877 6878 /* check that stack access falls within stack limits and that 'reg' doesn't 6879 * have a variable offset. 6880 * 6881 * Variable offset is prohibited for unprivileged mode for simplicity since it 6882 * requires corresponding support in Spectre masking for stack ALU. See also 6883 * retrieve_ptr_limit(). 6884 * 6885 * 6886 * 'off' includes 'reg->off'. 6887 */ 6888 static int check_stack_access_for_ptr_arithmetic( 6889 struct bpf_verifier_env *env, 6890 int regno, 6891 const struct bpf_reg_state *reg, 6892 int off) 6893 { 6894 if (!tnum_is_const(reg->var_off)) { 6895 char tn_buf[48]; 6896 6897 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6898 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 6899 regno, tn_buf, off); 6900 return -EACCES; 6901 } 6902 6903 if (off >= 0 || off < -MAX_BPF_STACK) { 6904 verbose(env, "R%d stack pointer arithmetic goes out of range, " 6905 "prohibited for !root; off=%d\n", regno, off); 6906 return -EACCES; 6907 } 6908 6909 return 0; 6910 } 6911 6912 static int sanitize_check_bounds(struct bpf_verifier_env *env, 6913 const struct bpf_insn *insn, 6914 const struct bpf_reg_state *dst_reg) 6915 { 6916 u32 dst = insn->dst_reg; 6917 6918 /* For unprivileged we require that resulting offset must be in bounds 6919 * in order to be able to sanitize access later on. 6920 */ 6921 if (env->bypass_spec_v1) 6922 return 0; 6923 6924 switch (dst_reg->type) { 6925 case PTR_TO_STACK: 6926 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 6927 dst_reg->off + dst_reg->var_off.value)) 6928 return -EACCES; 6929 break; 6930 case PTR_TO_MAP_VALUE: 6931 if (check_map_access(env, dst, dst_reg->off, 1, false)) { 6932 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 6933 "prohibited for !root\n", dst); 6934 return -EACCES; 6935 } 6936 break; 6937 default: 6938 break; 6939 } 6940 6941 return 0; 6942 } 6943 6944 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 6945 * Caller should also handle BPF_MOV case separately. 6946 * If we return -EACCES, caller may want to try again treating pointer as a 6947 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 6948 */ 6949 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 6950 struct bpf_insn *insn, 6951 const struct bpf_reg_state *ptr_reg, 6952 const struct bpf_reg_state *off_reg) 6953 { 6954 struct bpf_verifier_state *vstate = env->cur_state; 6955 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 6956 struct bpf_reg_state *regs = state->regs, *dst_reg; 6957 bool known = tnum_is_const(off_reg->var_off); 6958 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 6959 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 6960 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 6961 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 6962 struct bpf_sanitize_info info = {}; 6963 u8 opcode = BPF_OP(insn->code); 6964 u32 dst = insn->dst_reg; 6965 int ret; 6966 6967 dst_reg = ®s[dst]; 6968 6969 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 6970 smin_val > smax_val || umin_val > umax_val) { 6971 /* Taint dst register if offset had invalid bounds derived from 6972 * e.g. dead branches. 6973 */ 6974 __mark_reg_unknown(env, dst_reg); 6975 return 0; 6976 } 6977 6978 if (BPF_CLASS(insn->code) != BPF_ALU64) { 6979 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 6980 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 6981 __mark_reg_unknown(env, dst_reg); 6982 return 0; 6983 } 6984 6985 verbose(env, 6986 "R%d 32-bit pointer arithmetic prohibited\n", 6987 dst); 6988 return -EACCES; 6989 } 6990 6991 switch (ptr_reg->type) { 6992 case PTR_TO_MAP_VALUE_OR_NULL: 6993 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 6994 dst, reg_type_str[ptr_reg->type]); 6995 return -EACCES; 6996 case CONST_PTR_TO_MAP: 6997 /* smin_val represents the known value */ 6998 if (known && smin_val == 0 && opcode == BPF_ADD) 6999 break; 7000 fallthrough; 7001 case PTR_TO_PACKET_END: 7002 case PTR_TO_SOCKET: 7003 case PTR_TO_SOCKET_OR_NULL: 7004 case PTR_TO_SOCK_COMMON: 7005 case PTR_TO_SOCK_COMMON_OR_NULL: 7006 case PTR_TO_TCP_SOCK: 7007 case PTR_TO_TCP_SOCK_OR_NULL: 7008 case PTR_TO_XDP_SOCK: 7009 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 7010 dst, reg_type_str[ptr_reg->type]); 7011 return -EACCES; 7012 default: 7013 break; 7014 } 7015 7016 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 7017 * The id may be overwritten later if we create a new variable offset. 7018 */ 7019 dst_reg->type = ptr_reg->type; 7020 dst_reg->id = ptr_reg->id; 7021 7022 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 7023 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 7024 return -EINVAL; 7025 7026 /* pointer types do not carry 32-bit bounds at the moment. */ 7027 __mark_reg32_unbounded(dst_reg); 7028 7029 if (sanitize_needed(opcode)) { 7030 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 7031 &info, false); 7032 if (ret < 0) 7033 return sanitize_err(env, insn, ret, off_reg, dst_reg); 7034 } 7035 7036 switch (opcode) { 7037 case BPF_ADD: 7038 /* We can take a fixed offset as long as it doesn't overflow 7039 * the s32 'off' field 7040 */ 7041 if (known && (ptr_reg->off + smin_val == 7042 (s64)(s32)(ptr_reg->off + smin_val))) { 7043 /* pointer += K. Accumulate it into fixed offset */ 7044 dst_reg->smin_value = smin_ptr; 7045 dst_reg->smax_value = smax_ptr; 7046 dst_reg->umin_value = umin_ptr; 7047 dst_reg->umax_value = umax_ptr; 7048 dst_reg->var_off = ptr_reg->var_off; 7049 dst_reg->off = ptr_reg->off + smin_val; 7050 dst_reg->raw = ptr_reg->raw; 7051 break; 7052 } 7053 /* A new variable offset is created. Note that off_reg->off 7054 * == 0, since it's a scalar. 7055 * dst_reg gets the pointer type and since some positive 7056 * integer value was added to the pointer, give it a new 'id' 7057 * if it's a PTR_TO_PACKET. 7058 * this creates a new 'base' pointer, off_reg (variable) gets 7059 * added into the variable offset, and we copy the fixed offset 7060 * from ptr_reg. 7061 */ 7062 if (signed_add_overflows(smin_ptr, smin_val) || 7063 signed_add_overflows(smax_ptr, smax_val)) { 7064 dst_reg->smin_value = S64_MIN; 7065 dst_reg->smax_value = S64_MAX; 7066 } else { 7067 dst_reg->smin_value = smin_ptr + smin_val; 7068 dst_reg->smax_value = smax_ptr + smax_val; 7069 } 7070 if (umin_ptr + umin_val < umin_ptr || 7071 umax_ptr + umax_val < umax_ptr) { 7072 dst_reg->umin_value = 0; 7073 dst_reg->umax_value = U64_MAX; 7074 } else { 7075 dst_reg->umin_value = umin_ptr + umin_val; 7076 dst_reg->umax_value = umax_ptr + umax_val; 7077 } 7078 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 7079 dst_reg->off = ptr_reg->off; 7080 dst_reg->raw = ptr_reg->raw; 7081 if (reg_is_pkt_pointer(ptr_reg)) { 7082 dst_reg->id = ++env->id_gen; 7083 /* something was added to pkt_ptr, set range to zero */ 7084 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 7085 } 7086 break; 7087 case BPF_SUB: 7088 if (dst_reg == off_reg) { 7089 /* scalar -= pointer. Creates an unknown scalar */ 7090 verbose(env, "R%d tried to subtract pointer from scalar\n", 7091 dst); 7092 return -EACCES; 7093 } 7094 /* We don't allow subtraction from FP, because (according to 7095 * test_verifier.c test "invalid fp arithmetic", JITs might not 7096 * be able to deal with it. 7097 */ 7098 if (ptr_reg->type == PTR_TO_STACK) { 7099 verbose(env, "R%d subtraction from stack pointer prohibited\n", 7100 dst); 7101 return -EACCES; 7102 } 7103 if (known && (ptr_reg->off - smin_val == 7104 (s64)(s32)(ptr_reg->off - smin_val))) { 7105 /* pointer -= K. Subtract it from fixed offset */ 7106 dst_reg->smin_value = smin_ptr; 7107 dst_reg->smax_value = smax_ptr; 7108 dst_reg->umin_value = umin_ptr; 7109 dst_reg->umax_value = umax_ptr; 7110 dst_reg->var_off = ptr_reg->var_off; 7111 dst_reg->id = ptr_reg->id; 7112 dst_reg->off = ptr_reg->off - smin_val; 7113 dst_reg->raw = ptr_reg->raw; 7114 break; 7115 } 7116 /* A new variable offset is created. If the subtrahend is known 7117 * nonnegative, then any reg->range we had before is still good. 7118 */ 7119 if (signed_sub_overflows(smin_ptr, smax_val) || 7120 signed_sub_overflows(smax_ptr, smin_val)) { 7121 /* Overflow possible, we know nothing */ 7122 dst_reg->smin_value = S64_MIN; 7123 dst_reg->smax_value = S64_MAX; 7124 } else { 7125 dst_reg->smin_value = smin_ptr - smax_val; 7126 dst_reg->smax_value = smax_ptr - smin_val; 7127 } 7128 if (umin_ptr < umax_val) { 7129 /* Overflow possible, we know nothing */ 7130 dst_reg->umin_value = 0; 7131 dst_reg->umax_value = U64_MAX; 7132 } else { 7133 /* Cannot overflow (as long as bounds are consistent) */ 7134 dst_reg->umin_value = umin_ptr - umax_val; 7135 dst_reg->umax_value = umax_ptr - umin_val; 7136 } 7137 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 7138 dst_reg->off = ptr_reg->off; 7139 dst_reg->raw = ptr_reg->raw; 7140 if (reg_is_pkt_pointer(ptr_reg)) { 7141 dst_reg->id = ++env->id_gen; 7142 /* something was added to pkt_ptr, set range to zero */ 7143 if (smin_val < 0) 7144 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 7145 } 7146 break; 7147 case BPF_AND: 7148 case BPF_OR: 7149 case BPF_XOR: 7150 /* bitwise ops on pointers are troublesome, prohibit. */ 7151 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 7152 dst, bpf_alu_string[opcode >> 4]); 7153 return -EACCES; 7154 default: 7155 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 7156 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 7157 dst, bpf_alu_string[opcode >> 4]); 7158 return -EACCES; 7159 } 7160 7161 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 7162 return -EINVAL; 7163 7164 __update_reg_bounds(dst_reg); 7165 __reg_deduce_bounds(dst_reg); 7166 __reg_bound_offset(dst_reg); 7167 7168 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 7169 return -EACCES; 7170 if (sanitize_needed(opcode)) { 7171 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 7172 &info, true); 7173 if (ret < 0) 7174 return sanitize_err(env, insn, ret, off_reg, dst_reg); 7175 } 7176 7177 return 0; 7178 } 7179 7180 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 7181 struct bpf_reg_state *src_reg) 7182 { 7183 s32 smin_val = src_reg->s32_min_value; 7184 s32 smax_val = src_reg->s32_max_value; 7185 u32 umin_val = src_reg->u32_min_value; 7186 u32 umax_val = src_reg->u32_max_value; 7187 7188 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 7189 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 7190 dst_reg->s32_min_value = S32_MIN; 7191 dst_reg->s32_max_value = S32_MAX; 7192 } else { 7193 dst_reg->s32_min_value += smin_val; 7194 dst_reg->s32_max_value += smax_val; 7195 } 7196 if (dst_reg->u32_min_value + umin_val < umin_val || 7197 dst_reg->u32_max_value + umax_val < umax_val) { 7198 dst_reg->u32_min_value = 0; 7199 dst_reg->u32_max_value = U32_MAX; 7200 } else { 7201 dst_reg->u32_min_value += umin_val; 7202 dst_reg->u32_max_value += umax_val; 7203 } 7204 } 7205 7206 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 7207 struct bpf_reg_state *src_reg) 7208 { 7209 s64 smin_val = src_reg->smin_value; 7210 s64 smax_val = src_reg->smax_value; 7211 u64 umin_val = src_reg->umin_value; 7212 u64 umax_val = src_reg->umax_value; 7213 7214 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 7215 signed_add_overflows(dst_reg->smax_value, smax_val)) { 7216 dst_reg->smin_value = S64_MIN; 7217 dst_reg->smax_value = S64_MAX; 7218 } else { 7219 dst_reg->smin_value += smin_val; 7220 dst_reg->smax_value += smax_val; 7221 } 7222 if (dst_reg->umin_value + umin_val < umin_val || 7223 dst_reg->umax_value + umax_val < umax_val) { 7224 dst_reg->umin_value = 0; 7225 dst_reg->umax_value = U64_MAX; 7226 } else { 7227 dst_reg->umin_value += umin_val; 7228 dst_reg->umax_value += umax_val; 7229 } 7230 } 7231 7232 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 7233 struct bpf_reg_state *src_reg) 7234 { 7235 s32 smin_val = src_reg->s32_min_value; 7236 s32 smax_val = src_reg->s32_max_value; 7237 u32 umin_val = src_reg->u32_min_value; 7238 u32 umax_val = src_reg->u32_max_value; 7239 7240 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 7241 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 7242 /* Overflow possible, we know nothing */ 7243 dst_reg->s32_min_value = S32_MIN; 7244 dst_reg->s32_max_value = S32_MAX; 7245 } else { 7246 dst_reg->s32_min_value -= smax_val; 7247 dst_reg->s32_max_value -= smin_val; 7248 } 7249 if (dst_reg->u32_min_value < umax_val) { 7250 /* Overflow possible, we know nothing */ 7251 dst_reg->u32_min_value = 0; 7252 dst_reg->u32_max_value = U32_MAX; 7253 } else { 7254 /* Cannot overflow (as long as bounds are consistent) */ 7255 dst_reg->u32_min_value -= umax_val; 7256 dst_reg->u32_max_value -= umin_val; 7257 } 7258 } 7259 7260 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 7261 struct bpf_reg_state *src_reg) 7262 { 7263 s64 smin_val = src_reg->smin_value; 7264 s64 smax_val = src_reg->smax_value; 7265 u64 umin_val = src_reg->umin_value; 7266 u64 umax_val = src_reg->umax_value; 7267 7268 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 7269 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 7270 /* Overflow possible, we know nothing */ 7271 dst_reg->smin_value = S64_MIN; 7272 dst_reg->smax_value = S64_MAX; 7273 } else { 7274 dst_reg->smin_value -= smax_val; 7275 dst_reg->smax_value -= smin_val; 7276 } 7277 if (dst_reg->umin_value < umax_val) { 7278 /* Overflow possible, we know nothing */ 7279 dst_reg->umin_value = 0; 7280 dst_reg->umax_value = U64_MAX; 7281 } else { 7282 /* Cannot overflow (as long as bounds are consistent) */ 7283 dst_reg->umin_value -= umax_val; 7284 dst_reg->umax_value -= umin_val; 7285 } 7286 } 7287 7288 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 7289 struct bpf_reg_state *src_reg) 7290 { 7291 s32 smin_val = src_reg->s32_min_value; 7292 u32 umin_val = src_reg->u32_min_value; 7293 u32 umax_val = src_reg->u32_max_value; 7294 7295 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 7296 /* Ain't nobody got time to multiply that sign */ 7297 __mark_reg32_unbounded(dst_reg); 7298 return; 7299 } 7300 /* Both values are positive, so we can work with unsigned and 7301 * copy the result to signed (unless it exceeds S32_MAX). 7302 */ 7303 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 7304 /* Potential overflow, we know nothing */ 7305 __mark_reg32_unbounded(dst_reg); 7306 return; 7307 } 7308 dst_reg->u32_min_value *= umin_val; 7309 dst_reg->u32_max_value *= umax_val; 7310 if (dst_reg->u32_max_value > S32_MAX) { 7311 /* Overflow possible, we know nothing */ 7312 dst_reg->s32_min_value = S32_MIN; 7313 dst_reg->s32_max_value = S32_MAX; 7314 } else { 7315 dst_reg->s32_min_value = dst_reg->u32_min_value; 7316 dst_reg->s32_max_value = dst_reg->u32_max_value; 7317 } 7318 } 7319 7320 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 7321 struct bpf_reg_state *src_reg) 7322 { 7323 s64 smin_val = src_reg->smin_value; 7324 u64 umin_val = src_reg->umin_value; 7325 u64 umax_val = src_reg->umax_value; 7326 7327 if (smin_val < 0 || dst_reg->smin_value < 0) { 7328 /* Ain't nobody got time to multiply that sign */ 7329 __mark_reg64_unbounded(dst_reg); 7330 return; 7331 } 7332 /* Both values are positive, so we can work with unsigned and 7333 * copy the result to signed (unless it exceeds S64_MAX). 7334 */ 7335 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 7336 /* Potential overflow, we know nothing */ 7337 __mark_reg64_unbounded(dst_reg); 7338 return; 7339 } 7340 dst_reg->umin_value *= umin_val; 7341 dst_reg->umax_value *= umax_val; 7342 if (dst_reg->umax_value > S64_MAX) { 7343 /* Overflow possible, we know nothing */ 7344 dst_reg->smin_value = S64_MIN; 7345 dst_reg->smax_value = S64_MAX; 7346 } else { 7347 dst_reg->smin_value = dst_reg->umin_value; 7348 dst_reg->smax_value = dst_reg->umax_value; 7349 } 7350 } 7351 7352 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 7353 struct bpf_reg_state *src_reg) 7354 { 7355 bool src_known = tnum_subreg_is_const(src_reg->var_off); 7356 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 7357 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 7358 s32 smin_val = src_reg->s32_min_value; 7359 u32 umax_val = src_reg->u32_max_value; 7360 7361 if (src_known && dst_known) { 7362 __mark_reg32_known(dst_reg, var32_off.value); 7363 return; 7364 } 7365 7366 /* We get our minimum from the var_off, since that's inherently 7367 * bitwise. Our maximum is the minimum of the operands' maxima. 7368 */ 7369 dst_reg->u32_min_value = var32_off.value; 7370 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 7371 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 7372 /* Lose signed bounds when ANDing negative numbers, 7373 * ain't nobody got time for that. 7374 */ 7375 dst_reg->s32_min_value = S32_MIN; 7376 dst_reg->s32_max_value = S32_MAX; 7377 } else { 7378 /* ANDing two positives gives a positive, so safe to 7379 * cast result into s64. 7380 */ 7381 dst_reg->s32_min_value = dst_reg->u32_min_value; 7382 dst_reg->s32_max_value = dst_reg->u32_max_value; 7383 } 7384 } 7385 7386 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 7387 struct bpf_reg_state *src_reg) 7388 { 7389 bool src_known = tnum_is_const(src_reg->var_off); 7390 bool dst_known = tnum_is_const(dst_reg->var_off); 7391 s64 smin_val = src_reg->smin_value; 7392 u64 umax_val = src_reg->umax_value; 7393 7394 if (src_known && dst_known) { 7395 __mark_reg_known(dst_reg, dst_reg->var_off.value); 7396 return; 7397 } 7398 7399 /* We get our minimum from the var_off, since that's inherently 7400 * bitwise. Our maximum is the minimum of the operands' maxima. 7401 */ 7402 dst_reg->umin_value = dst_reg->var_off.value; 7403 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 7404 if (dst_reg->smin_value < 0 || smin_val < 0) { 7405 /* Lose signed bounds when ANDing negative numbers, 7406 * ain't nobody got time for that. 7407 */ 7408 dst_reg->smin_value = S64_MIN; 7409 dst_reg->smax_value = S64_MAX; 7410 } else { 7411 /* ANDing two positives gives a positive, so safe to 7412 * cast result into s64. 7413 */ 7414 dst_reg->smin_value = dst_reg->umin_value; 7415 dst_reg->smax_value = dst_reg->umax_value; 7416 } 7417 /* We may learn something more from the var_off */ 7418 __update_reg_bounds(dst_reg); 7419 } 7420 7421 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 7422 struct bpf_reg_state *src_reg) 7423 { 7424 bool src_known = tnum_subreg_is_const(src_reg->var_off); 7425 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 7426 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 7427 s32 smin_val = src_reg->s32_min_value; 7428 u32 umin_val = src_reg->u32_min_value; 7429 7430 if (src_known && dst_known) { 7431 __mark_reg32_known(dst_reg, var32_off.value); 7432 return; 7433 } 7434 7435 /* We get our maximum from the var_off, and our minimum is the 7436 * maximum of the operands' minima 7437 */ 7438 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 7439 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 7440 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 7441 /* Lose signed bounds when ORing negative numbers, 7442 * ain't nobody got time for that. 7443 */ 7444 dst_reg->s32_min_value = S32_MIN; 7445 dst_reg->s32_max_value = S32_MAX; 7446 } else { 7447 /* ORing two positives gives a positive, so safe to 7448 * cast result into s64. 7449 */ 7450 dst_reg->s32_min_value = dst_reg->u32_min_value; 7451 dst_reg->s32_max_value = dst_reg->u32_max_value; 7452 } 7453 } 7454 7455 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 7456 struct bpf_reg_state *src_reg) 7457 { 7458 bool src_known = tnum_is_const(src_reg->var_off); 7459 bool dst_known = tnum_is_const(dst_reg->var_off); 7460 s64 smin_val = src_reg->smin_value; 7461 u64 umin_val = src_reg->umin_value; 7462 7463 if (src_known && dst_known) { 7464 __mark_reg_known(dst_reg, dst_reg->var_off.value); 7465 return; 7466 } 7467 7468 /* We get our maximum from the var_off, and our minimum is the 7469 * maximum of the operands' minima 7470 */ 7471 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 7472 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 7473 if (dst_reg->smin_value < 0 || smin_val < 0) { 7474 /* Lose signed bounds when ORing negative numbers, 7475 * ain't nobody got time for that. 7476 */ 7477 dst_reg->smin_value = S64_MIN; 7478 dst_reg->smax_value = S64_MAX; 7479 } else { 7480 /* ORing two positives gives a positive, so safe to 7481 * cast result into s64. 7482 */ 7483 dst_reg->smin_value = dst_reg->umin_value; 7484 dst_reg->smax_value = dst_reg->umax_value; 7485 } 7486 /* We may learn something more from the var_off */ 7487 __update_reg_bounds(dst_reg); 7488 } 7489 7490 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 7491 struct bpf_reg_state *src_reg) 7492 { 7493 bool src_known = tnum_subreg_is_const(src_reg->var_off); 7494 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 7495 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 7496 s32 smin_val = src_reg->s32_min_value; 7497 7498 if (src_known && dst_known) { 7499 __mark_reg32_known(dst_reg, var32_off.value); 7500 return; 7501 } 7502 7503 /* We get both minimum and maximum from the var32_off. */ 7504 dst_reg->u32_min_value = var32_off.value; 7505 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 7506 7507 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 7508 /* XORing two positive sign numbers gives a positive, 7509 * so safe to cast u32 result into s32. 7510 */ 7511 dst_reg->s32_min_value = dst_reg->u32_min_value; 7512 dst_reg->s32_max_value = dst_reg->u32_max_value; 7513 } else { 7514 dst_reg->s32_min_value = S32_MIN; 7515 dst_reg->s32_max_value = S32_MAX; 7516 } 7517 } 7518 7519 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 7520 struct bpf_reg_state *src_reg) 7521 { 7522 bool src_known = tnum_is_const(src_reg->var_off); 7523 bool dst_known = tnum_is_const(dst_reg->var_off); 7524 s64 smin_val = src_reg->smin_value; 7525 7526 if (src_known && dst_known) { 7527 /* dst_reg->var_off.value has been updated earlier */ 7528 __mark_reg_known(dst_reg, dst_reg->var_off.value); 7529 return; 7530 } 7531 7532 /* We get both minimum and maximum from the var_off. */ 7533 dst_reg->umin_value = dst_reg->var_off.value; 7534 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 7535 7536 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 7537 /* XORing two positive sign numbers gives a positive, 7538 * so safe to cast u64 result into s64. 7539 */ 7540 dst_reg->smin_value = dst_reg->umin_value; 7541 dst_reg->smax_value = dst_reg->umax_value; 7542 } else { 7543 dst_reg->smin_value = S64_MIN; 7544 dst_reg->smax_value = S64_MAX; 7545 } 7546 7547 __update_reg_bounds(dst_reg); 7548 } 7549 7550 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 7551 u64 umin_val, u64 umax_val) 7552 { 7553 /* We lose all sign bit information (except what we can pick 7554 * up from var_off) 7555 */ 7556 dst_reg->s32_min_value = S32_MIN; 7557 dst_reg->s32_max_value = S32_MAX; 7558 /* If we might shift our top bit out, then we know nothing */ 7559 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 7560 dst_reg->u32_min_value = 0; 7561 dst_reg->u32_max_value = U32_MAX; 7562 } else { 7563 dst_reg->u32_min_value <<= umin_val; 7564 dst_reg->u32_max_value <<= umax_val; 7565 } 7566 } 7567 7568 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 7569 struct bpf_reg_state *src_reg) 7570 { 7571 u32 umax_val = src_reg->u32_max_value; 7572 u32 umin_val = src_reg->u32_min_value; 7573 /* u32 alu operation will zext upper bits */ 7574 struct tnum subreg = tnum_subreg(dst_reg->var_off); 7575 7576 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 7577 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 7578 /* Not required but being careful mark reg64 bounds as unknown so 7579 * that we are forced to pick them up from tnum and zext later and 7580 * if some path skips this step we are still safe. 7581 */ 7582 __mark_reg64_unbounded(dst_reg); 7583 __update_reg32_bounds(dst_reg); 7584 } 7585 7586 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 7587 u64 umin_val, u64 umax_val) 7588 { 7589 /* Special case <<32 because it is a common compiler pattern to sign 7590 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 7591 * positive we know this shift will also be positive so we can track 7592 * bounds correctly. Otherwise we lose all sign bit information except 7593 * what we can pick up from var_off. Perhaps we can generalize this 7594 * later to shifts of any length. 7595 */ 7596 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 7597 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 7598 else 7599 dst_reg->smax_value = S64_MAX; 7600 7601 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 7602 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 7603 else 7604 dst_reg->smin_value = S64_MIN; 7605 7606 /* If we might shift our top bit out, then we know nothing */ 7607 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 7608 dst_reg->umin_value = 0; 7609 dst_reg->umax_value = U64_MAX; 7610 } else { 7611 dst_reg->umin_value <<= umin_val; 7612 dst_reg->umax_value <<= umax_val; 7613 } 7614 } 7615 7616 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 7617 struct bpf_reg_state *src_reg) 7618 { 7619 u64 umax_val = src_reg->umax_value; 7620 u64 umin_val = src_reg->umin_value; 7621 7622 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 7623 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 7624 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 7625 7626 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 7627 /* We may learn something more from the var_off */ 7628 __update_reg_bounds(dst_reg); 7629 } 7630 7631 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 7632 struct bpf_reg_state *src_reg) 7633 { 7634 struct tnum subreg = tnum_subreg(dst_reg->var_off); 7635 u32 umax_val = src_reg->u32_max_value; 7636 u32 umin_val = src_reg->u32_min_value; 7637 7638 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 7639 * be negative, then either: 7640 * 1) src_reg might be zero, so the sign bit of the result is 7641 * unknown, so we lose our signed bounds 7642 * 2) it's known negative, thus the unsigned bounds capture the 7643 * signed bounds 7644 * 3) the signed bounds cross zero, so they tell us nothing 7645 * about the result 7646 * If the value in dst_reg is known nonnegative, then again the 7647 * unsigned bounds capture the signed bounds. 7648 * Thus, in all cases it suffices to blow away our signed bounds 7649 * and rely on inferring new ones from the unsigned bounds and 7650 * var_off of the result. 7651 */ 7652 dst_reg->s32_min_value = S32_MIN; 7653 dst_reg->s32_max_value = S32_MAX; 7654 7655 dst_reg->var_off = tnum_rshift(subreg, umin_val); 7656 dst_reg->u32_min_value >>= umax_val; 7657 dst_reg->u32_max_value >>= umin_val; 7658 7659 __mark_reg64_unbounded(dst_reg); 7660 __update_reg32_bounds(dst_reg); 7661 } 7662 7663 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 7664 struct bpf_reg_state *src_reg) 7665 { 7666 u64 umax_val = src_reg->umax_value; 7667 u64 umin_val = src_reg->umin_value; 7668 7669 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 7670 * be negative, then either: 7671 * 1) src_reg might be zero, so the sign bit of the result is 7672 * unknown, so we lose our signed bounds 7673 * 2) it's known negative, thus the unsigned bounds capture the 7674 * signed bounds 7675 * 3) the signed bounds cross zero, so they tell us nothing 7676 * about the result 7677 * If the value in dst_reg is known nonnegative, then again the 7678 * unsigned bounds capture the signed bounds. 7679 * Thus, in all cases it suffices to blow away our signed bounds 7680 * and rely on inferring new ones from the unsigned bounds and 7681 * var_off of the result. 7682 */ 7683 dst_reg->smin_value = S64_MIN; 7684 dst_reg->smax_value = S64_MAX; 7685 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 7686 dst_reg->umin_value >>= umax_val; 7687 dst_reg->umax_value >>= umin_val; 7688 7689 /* Its not easy to operate on alu32 bounds here because it depends 7690 * on bits being shifted in. Take easy way out and mark unbounded 7691 * so we can recalculate later from tnum. 7692 */ 7693 __mark_reg32_unbounded(dst_reg); 7694 __update_reg_bounds(dst_reg); 7695 } 7696 7697 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 7698 struct bpf_reg_state *src_reg) 7699 { 7700 u64 umin_val = src_reg->u32_min_value; 7701 7702 /* Upon reaching here, src_known is true and 7703 * umax_val is equal to umin_val. 7704 */ 7705 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 7706 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 7707 7708 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 7709 7710 /* blow away the dst_reg umin_value/umax_value and rely on 7711 * dst_reg var_off to refine the result. 7712 */ 7713 dst_reg->u32_min_value = 0; 7714 dst_reg->u32_max_value = U32_MAX; 7715 7716 __mark_reg64_unbounded(dst_reg); 7717 __update_reg32_bounds(dst_reg); 7718 } 7719 7720 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 7721 struct bpf_reg_state *src_reg) 7722 { 7723 u64 umin_val = src_reg->umin_value; 7724 7725 /* Upon reaching here, src_known is true and umax_val is equal 7726 * to umin_val. 7727 */ 7728 dst_reg->smin_value >>= umin_val; 7729 dst_reg->smax_value >>= umin_val; 7730 7731 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 7732 7733 /* blow away the dst_reg umin_value/umax_value and rely on 7734 * dst_reg var_off to refine the result. 7735 */ 7736 dst_reg->umin_value = 0; 7737 dst_reg->umax_value = U64_MAX; 7738 7739 /* Its not easy to operate on alu32 bounds here because it depends 7740 * on bits being shifted in from upper 32-bits. Take easy way out 7741 * and mark unbounded so we can recalculate later from tnum. 7742 */ 7743 __mark_reg32_unbounded(dst_reg); 7744 __update_reg_bounds(dst_reg); 7745 } 7746 7747 /* WARNING: This function does calculations on 64-bit values, but the actual 7748 * execution may occur on 32-bit values. Therefore, things like bitshifts 7749 * need extra checks in the 32-bit case. 7750 */ 7751 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 7752 struct bpf_insn *insn, 7753 struct bpf_reg_state *dst_reg, 7754 struct bpf_reg_state src_reg) 7755 { 7756 struct bpf_reg_state *regs = cur_regs(env); 7757 u8 opcode = BPF_OP(insn->code); 7758 bool src_known; 7759 s64 smin_val, smax_val; 7760 u64 umin_val, umax_val; 7761 s32 s32_min_val, s32_max_val; 7762 u32 u32_min_val, u32_max_val; 7763 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 7764 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 7765 int ret; 7766 7767 smin_val = src_reg.smin_value; 7768 smax_val = src_reg.smax_value; 7769 umin_val = src_reg.umin_value; 7770 umax_val = src_reg.umax_value; 7771 7772 s32_min_val = src_reg.s32_min_value; 7773 s32_max_val = src_reg.s32_max_value; 7774 u32_min_val = src_reg.u32_min_value; 7775 u32_max_val = src_reg.u32_max_value; 7776 7777 if (alu32) { 7778 src_known = tnum_subreg_is_const(src_reg.var_off); 7779 if ((src_known && 7780 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 7781 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 7782 /* Taint dst register if offset had invalid bounds 7783 * derived from e.g. dead branches. 7784 */ 7785 __mark_reg_unknown(env, dst_reg); 7786 return 0; 7787 } 7788 } else { 7789 src_known = tnum_is_const(src_reg.var_off); 7790 if ((src_known && 7791 (smin_val != smax_val || umin_val != umax_val)) || 7792 smin_val > smax_val || umin_val > umax_val) { 7793 /* Taint dst register if offset had invalid bounds 7794 * derived from e.g. dead branches. 7795 */ 7796 __mark_reg_unknown(env, dst_reg); 7797 return 0; 7798 } 7799 } 7800 7801 if (!src_known && 7802 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 7803 __mark_reg_unknown(env, dst_reg); 7804 return 0; 7805 } 7806 7807 if (sanitize_needed(opcode)) { 7808 ret = sanitize_val_alu(env, insn); 7809 if (ret < 0) 7810 return sanitize_err(env, insn, ret, NULL, NULL); 7811 } 7812 7813 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 7814 * There are two classes of instructions: The first class we track both 7815 * alu32 and alu64 sign/unsigned bounds independently this provides the 7816 * greatest amount of precision when alu operations are mixed with jmp32 7817 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 7818 * and BPF_OR. This is possible because these ops have fairly easy to 7819 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 7820 * See alu32 verifier tests for examples. The second class of 7821 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 7822 * with regards to tracking sign/unsigned bounds because the bits may 7823 * cross subreg boundaries in the alu64 case. When this happens we mark 7824 * the reg unbounded in the subreg bound space and use the resulting 7825 * tnum to calculate an approximation of the sign/unsigned bounds. 7826 */ 7827 switch (opcode) { 7828 case BPF_ADD: 7829 scalar32_min_max_add(dst_reg, &src_reg); 7830 scalar_min_max_add(dst_reg, &src_reg); 7831 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 7832 break; 7833 case BPF_SUB: 7834 scalar32_min_max_sub(dst_reg, &src_reg); 7835 scalar_min_max_sub(dst_reg, &src_reg); 7836 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 7837 break; 7838 case BPF_MUL: 7839 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 7840 scalar32_min_max_mul(dst_reg, &src_reg); 7841 scalar_min_max_mul(dst_reg, &src_reg); 7842 break; 7843 case BPF_AND: 7844 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 7845 scalar32_min_max_and(dst_reg, &src_reg); 7846 scalar_min_max_and(dst_reg, &src_reg); 7847 break; 7848 case BPF_OR: 7849 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 7850 scalar32_min_max_or(dst_reg, &src_reg); 7851 scalar_min_max_or(dst_reg, &src_reg); 7852 break; 7853 case BPF_XOR: 7854 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 7855 scalar32_min_max_xor(dst_reg, &src_reg); 7856 scalar_min_max_xor(dst_reg, &src_reg); 7857 break; 7858 case BPF_LSH: 7859 if (umax_val >= insn_bitness) { 7860 /* Shifts greater than 31 or 63 are undefined. 7861 * This includes shifts by a negative number. 7862 */ 7863 mark_reg_unknown(env, regs, insn->dst_reg); 7864 break; 7865 } 7866 if (alu32) 7867 scalar32_min_max_lsh(dst_reg, &src_reg); 7868 else 7869 scalar_min_max_lsh(dst_reg, &src_reg); 7870 break; 7871 case BPF_RSH: 7872 if (umax_val >= insn_bitness) { 7873 /* Shifts greater than 31 or 63 are undefined. 7874 * This includes shifts by a negative number. 7875 */ 7876 mark_reg_unknown(env, regs, insn->dst_reg); 7877 break; 7878 } 7879 if (alu32) 7880 scalar32_min_max_rsh(dst_reg, &src_reg); 7881 else 7882 scalar_min_max_rsh(dst_reg, &src_reg); 7883 break; 7884 case BPF_ARSH: 7885 if (umax_val >= insn_bitness) { 7886 /* Shifts greater than 31 or 63 are undefined. 7887 * This includes shifts by a negative number. 7888 */ 7889 mark_reg_unknown(env, regs, insn->dst_reg); 7890 break; 7891 } 7892 if (alu32) 7893 scalar32_min_max_arsh(dst_reg, &src_reg); 7894 else 7895 scalar_min_max_arsh(dst_reg, &src_reg); 7896 break; 7897 default: 7898 mark_reg_unknown(env, regs, insn->dst_reg); 7899 break; 7900 } 7901 7902 /* ALU32 ops are zero extended into 64bit register */ 7903 if (alu32) 7904 zext_32_to_64(dst_reg); 7905 7906 __update_reg_bounds(dst_reg); 7907 __reg_deduce_bounds(dst_reg); 7908 __reg_bound_offset(dst_reg); 7909 return 0; 7910 } 7911 7912 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 7913 * and var_off. 7914 */ 7915 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 7916 struct bpf_insn *insn) 7917 { 7918 struct bpf_verifier_state *vstate = env->cur_state; 7919 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7920 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 7921 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 7922 u8 opcode = BPF_OP(insn->code); 7923 int err; 7924 7925 dst_reg = ®s[insn->dst_reg]; 7926 src_reg = NULL; 7927 if (dst_reg->type != SCALAR_VALUE) 7928 ptr_reg = dst_reg; 7929 else 7930 /* Make sure ID is cleared otherwise dst_reg min/max could be 7931 * incorrectly propagated into other registers by find_equal_scalars() 7932 */ 7933 dst_reg->id = 0; 7934 if (BPF_SRC(insn->code) == BPF_X) { 7935 src_reg = ®s[insn->src_reg]; 7936 if (src_reg->type != SCALAR_VALUE) { 7937 if (dst_reg->type != SCALAR_VALUE) { 7938 /* Combining two pointers by any ALU op yields 7939 * an arbitrary scalar. Disallow all math except 7940 * pointer subtraction 7941 */ 7942 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 7943 mark_reg_unknown(env, regs, insn->dst_reg); 7944 return 0; 7945 } 7946 verbose(env, "R%d pointer %s pointer prohibited\n", 7947 insn->dst_reg, 7948 bpf_alu_string[opcode >> 4]); 7949 return -EACCES; 7950 } else { 7951 /* scalar += pointer 7952 * This is legal, but we have to reverse our 7953 * src/dest handling in computing the range 7954 */ 7955 err = mark_chain_precision(env, insn->dst_reg); 7956 if (err) 7957 return err; 7958 return adjust_ptr_min_max_vals(env, insn, 7959 src_reg, dst_reg); 7960 } 7961 } else if (ptr_reg) { 7962 /* pointer += scalar */ 7963 err = mark_chain_precision(env, insn->src_reg); 7964 if (err) 7965 return err; 7966 return adjust_ptr_min_max_vals(env, insn, 7967 dst_reg, src_reg); 7968 } 7969 } else { 7970 /* Pretend the src is a reg with a known value, since we only 7971 * need to be able to read from this state. 7972 */ 7973 off_reg.type = SCALAR_VALUE; 7974 __mark_reg_known(&off_reg, insn->imm); 7975 src_reg = &off_reg; 7976 if (ptr_reg) /* pointer += K */ 7977 return adjust_ptr_min_max_vals(env, insn, 7978 ptr_reg, src_reg); 7979 } 7980 7981 /* Got here implies adding two SCALAR_VALUEs */ 7982 if (WARN_ON_ONCE(ptr_reg)) { 7983 print_verifier_state(env, state); 7984 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 7985 return -EINVAL; 7986 } 7987 if (WARN_ON(!src_reg)) { 7988 print_verifier_state(env, state); 7989 verbose(env, "verifier internal error: no src_reg\n"); 7990 return -EINVAL; 7991 } 7992 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 7993 } 7994 7995 /* check validity of 32-bit and 64-bit arithmetic operations */ 7996 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 7997 { 7998 struct bpf_reg_state *regs = cur_regs(env); 7999 u8 opcode = BPF_OP(insn->code); 8000 int err; 8001 8002 if (opcode == BPF_END || opcode == BPF_NEG) { 8003 if (opcode == BPF_NEG) { 8004 if (BPF_SRC(insn->code) != 0 || 8005 insn->src_reg != BPF_REG_0 || 8006 insn->off != 0 || insn->imm != 0) { 8007 verbose(env, "BPF_NEG uses reserved fields\n"); 8008 return -EINVAL; 8009 } 8010 } else { 8011 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 8012 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 8013 BPF_CLASS(insn->code) == BPF_ALU64) { 8014 verbose(env, "BPF_END uses reserved fields\n"); 8015 return -EINVAL; 8016 } 8017 } 8018 8019 /* check src operand */ 8020 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 8021 if (err) 8022 return err; 8023 8024 if (is_pointer_value(env, insn->dst_reg)) { 8025 verbose(env, "R%d pointer arithmetic prohibited\n", 8026 insn->dst_reg); 8027 return -EACCES; 8028 } 8029 8030 /* check dest operand */ 8031 err = check_reg_arg(env, insn->dst_reg, DST_OP); 8032 if (err) 8033 return err; 8034 8035 } else if (opcode == BPF_MOV) { 8036 8037 if (BPF_SRC(insn->code) == BPF_X) { 8038 if (insn->imm != 0 || insn->off != 0) { 8039 verbose(env, "BPF_MOV uses reserved fields\n"); 8040 return -EINVAL; 8041 } 8042 8043 /* check src operand */ 8044 err = check_reg_arg(env, insn->src_reg, SRC_OP); 8045 if (err) 8046 return err; 8047 } else { 8048 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 8049 verbose(env, "BPF_MOV uses reserved fields\n"); 8050 return -EINVAL; 8051 } 8052 } 8053 8054 /* check dest operand, mark as required later */ 8055 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 8056 if (err) 8057 return err; 8058 8059 if (BPF_SRC(insn->code) == BPF_X) { 8060 struct bpf_reg_state *src_reg = regs + insn->src_reg; 8061 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 8062 8063 if (BPF_CLASS(insn->code) == BPF_ALU64) { 8064 /* case: R1 = R2 8065 * copy register state to dest reg 8066 */ 8067 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 8068 /* Assign src and dst registers the same ID 8069 * that will be used by find_equal_scalars() 8070 * to propagate min/max range. 8071 */ 8072 src_reg->id = ++env->id_gen; 8073 *dst_reg = *src_reg; 8074 dst_reg->live |= REG_LIVE_WRITTEN; 8075 dst_reg->subreg_def = DEF_NOT_SUBREG; 8076 } else { 8077 /* R1 = (u32) R2 */ 8078 if (is_pointer_value(env, insn->src_reg)) { 8079 verbose(env, 8080 "R%d partial copy of pointer\n", 8081 insn->src_reg); 8082 return -EACCES; 8083 } else if (src_reg->type == SCALAR_VALUE) { 8084 *dst_reg = *src_reg; 8085 /* Make sure ID is cleared otherwise 8086 * dst_reg min/max could be incorrectly 8087 * propagated into src_reg by find_equal_scalars() 8088 */ 8089 dst_reg->id = 0; 8090 dst_reg->live |= REG_LIVE_WRITTEN; 8091 dst_reg->subreg_def = env->insn_idx + 1; 8092 } else { 8093 mark_reg_unknown(env, regs, 8094 insn->dst_reg); 8095 } 8096 zext_32_to_64(dst_reg); 8097 } 8098 } else { 8099 /* case: R = imm 8100 * remember the value we stored into this reg 8101 */ 8102 /* clear any state __mark_reg_known doesn't set */ 8103 mark_reg_unknown(env, regs, insn->dst_reg); 8104 regs[insn->dst_reg].type = SCALAR_VALUE; 8105 if (BPF_CLASS(insn->code) == BPF_ALU64) { 8106 __mark_reg_known(regs + insn->dst_reg, 8107 insn->imm); 8108 } else { 8109 __mark_reg_known(regs + insn->dst_reg, 8110 (u32)insn->imm); 8111 } 8112 } 8113 8114 } else if (opcode > BPF_END) { 8115 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 8116 return -EINVAL; 8117 8118 } else { /* all other ALU ops: and, sub, xor, add, ... */ 8119 8120 if (BPF_SRC(insn->code) == BPF_X) { 8121 if (insn->imm != 0 || insn->off != 0) { 8122 verbose(env, "BPF_ALU uses reserved fields\n"); 8123 return -EINVAL; 8124 } 8125 /* check src1 operand */ 8126 err = check_reg_arg(env, insn->src_reg, SRC_OP); 8127 if (err) 8128 return err; 8129 } else { 8130 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 8131 verbose(env, "BPF_ALU uses reserved fields\n"); 8132 return -EINVAL; 8133 } 8134 } 8135 8136 /* check src2 operand */ 8137 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 8138 if (err) 8139 return err; 8140 8141 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 8142 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 8143 verbose(env, "div by zero\n"); 8144 return -EINVAL; 8145 } 8146 8147 if ((opcode == BPF_LSH || opcode == BPF_RSH || 8148 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 8149 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 8150 8151 if (insn->imm < 0 || insn->imm >= size) { 8152 verbose(env, "invalid shift %d\n", insn->imm); 8153 return -EINVAL; 8154 } 8155 } 8156 8157 /* check dest operand */ 8158 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 8159 if (err) 8160 return err; 8161 8162 return adjust_reg_min_max_vals(env, insn); 8163 } 8164 8165 return 0; 8166 } 8167 8168 static void __find_good_pkt_pointers(struct bpf_func_state *state, 8169 struct bpf_reg_state *dst_reg, 8170 enum bpf_reg_type type, int new_range) 8171 { 8172 struct bpf_reg_state *reg; 8173 int i; 8174 8175 for (i = 0; i < MAX_BPF_REG; i++) { 8176 reg = &state->regs[i]; 8177 if (reg->type == type && reg->id == dst_reg->id) 8178 /* keep the maximum range already checked */ 8179 reg->range = max(reg->range, new_range); 8180 } 8181 8182 bpf_for_each_spilled_reg(i, state, reg) { 8183 if (!reg) 8184 continue; 8185 if (reg->type == type && reg->id == dst_reg->id) 8186 reg->range = max(reg->range, new_range); 8187 } 8188 } 8189 8190 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 8191 struct bpf_reg_state *dst_reg, 8192 enum bpf_reg_type type, 8193 bool range_right_open) 8194 { 8195 int new_range, i; 8196 8197 if (dst_reg->off < 0 || 8198 (dst_reg->off == 0 && range_right_open)) 8199 /* This doesn't give us any range */ 8200 return; 8201 8202 if (dst_reg->umax_value > MAX_PACKET_OFF || 8203 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 8204 /* Risk of overflow. For instance, ptr + (1<<63) may be less 8205 * than pkt_end, but that's because it's also less than pkt. 8206 */ 8207 return; 8208 8209 new_range = dst_reg->off; 8210 if (range_right_open) 8211 new_range--; 8212 8213 /* Examples for register markings: 8214 * 8215 * pkt_data in dst register: 8216 * 8217 * r2 = r3; 8218 * r2 += 8; 8219 * if (r2 > pkt_end) goto <handle exception> 8220 * <access okay> 8221 * 8222 * r2 = r3; 8223 * r2 += 8; 8224 * if (r2 < pkt_end) goto <access okay> 8225 * <handle exception> 8226 * 8227 * Where: 8228 * r2 == dst_reg, pkt_end == src_reg 8229 * r2=pkt(id=n,off=8,r=0) 8230 * r3=pkt(id=n,off=0,r=0) 8231 * 8232 * pkt_data in src register: 8233 * 8234 * r2 = r3; 8235 * r2 += 8; 8236 * if (pkt_end >= r2) goto <access okay> 8237 * <handle exception> 8238 * 8239 * r2 = r3; 8240 * r2 += 8; 8241 * if (pkt_end <= r2) goto <handle exception> 8242 * <access okay> 8243 * 8244 * Where: 8245 * pkt_end == dst_reg, r2 == src_reg 8246 * r2=pkt(id=n,off=8,r=0) 8247 * r3=pkt(id=n,off=0,r=0) 8248 * 8249 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 8250 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 8251 * and [r3, r3 + 8-1) respectively is safe to access depending on 8252 * the check. 8253 */ 8254 8255 /* If our ids match, then we must have the same max_value. And we 8256 * don't care about the other reg's fixed offset, since if it's too big 8257 * the range won't allow anything. 8258 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 8259 */ 8260 for (i = 0; i <= vstate->curframe; i++) 8261 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, 8262 new_range); 8263 } 8264 8265 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 8266 { 8267 struct tnum subreg = tnum_subreg(reg->var_off); 8268 s32 sval = (s32)val; 8269 8270 switch (opcode) { 8271 case BPF_JEQ: 8272 if (tnum_is_const(subreg)) 8273 return !!tnum_equals_const(subreg, val); 8274 break; 8275 case BPF_JNE: 8276 if (tnum_is_const(subreg)) 8277 return !tnum_equals_const(subreg, val); 8278 break; 8279 case BPF_JSET: 8280 if ((~subreg.mask & subreg.value) & val) 8281 return 1; 8282 if (!((subreg.mask | subreg.value) & val)) 8283 return 0; 8284 break; 8285 case BPF_JGT: 8286 if (reg->u32_min_value > val) 8287 return 1; 8288 else if (reg->u32_max_value <= val) 8289 return 0; 8290 break; 8291 case BPF_JSGT: 8292 if (reg->s32_min_value > sval) 8293 return 1; 8294 else if (reg->s32_max_value <= sval) 8295 return 0; 8296 break; 8297 case BPF_JLT: 8298 if (reg->u32_max_value < val) 8299 return 1; 8300 else if (reg->u32_min_value >= val) 8301 return 0; 8302 break; 8303 case BPF_JSLT: 8304 if (reg->s32_max_value < sval) 8305 return 1; 8306 else if (reg->s32_min_value >= sval) 8307 return 0; 8308 break; 8309 case BPF_JGE: 8310 if (reg->u32_min_value >= val) 8311 return 1; 8312 else if (reg->u32_max_value < val) 8313 return 0; 8314 break; 8315 case BPF_JSGE: 8316 if (reg->s32_min_value >= sval) 8317 return 1; 8318 else if (reg->s32_max_value < sval) 8319 return 0; 8320 break; 8321 case BPF_JLE: 8322 if (reg->u32_max_value <= val) 8323 return 1; 8324 else if (reg->u32_min_value > val) 8325 return 0; 8326 break; 8327 case BPF_JSLE: 8328 if (reg->s32_max_value <= sval) 8329 return 1; 8330 else if (reg->s32_min_value > sval) 8331 return 0; 8332 break; 8333 } 8334 8335 return -1; 8336 } 8337 8338 8339 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 8340 { 8341 s64 sval = (s64)val; 8342 8343 switch (opcode) { 8344 case BPF_JEQ: 8345 if (tnum_is_const(reg->var_off)) 8346 return !!tnum_equals_const(reg->var_off, val); 8347 break; 8348 case BPF_JNE: 8349 if (tnum_is_const(reg->var_off)) 8350 return !tnum_equals_const(reg->var_off, val); 8351 break; 8352 case BPF_JSET: 8353 if ((~reg->var_off.mask & reg->var_off.value) & val) 8354 return 1; 8355 if (!((reg->var_off.mask | reg->var_off.value) & val)) 8356 return 0; 8357 break; 8358 case BPF_JGT: 8359 if (reg->umin_value > val) 8360 return 1; 8361 else if (reg->umax_value <= val) 8362 return 0; 8363 break; 8364 case BPF_JSGT: 8365 if (reg->smin_value > sval) 8366 return 1; 8367 else if (reg->smax_value <= sval) 8368 return 0; 8369 break; 8370 case BPF_JLT: 8371 if (reg->umax_value < val) 8372 return 1; 8373 else if (reg->umin_value >= val) 8374 return 0; 8375 break; 8376 case BPF_JSLT: 8377 if (reg->smax_value < sval) 8378 return 1; 8379 else if (reg->smin_value >= sval) 8380 return 0; 8381 break; 8382 case BPF_JGE: 8383 if (reg->umin_value >= val) 8384 return 1; 8385 else if (reg->umax_value < val) 8386 return 0; 8387 break; 8388 case BPF_JSGE: 8389 if (reg->smin_value >= sval) 8390 return 1; 8391 else if (reg->smax_value < sval) 8392 return 0; 8393 break; 8394 case BPF_JLE: 8395 if (reg->umax_value <= val) 8396 return 1; 8397 else if (reg->umin_value > val) 8398 return 0; 8399 break; 8400 case BPF_JSLE: 8401 if (reg->smax_value <= sval) 8402 return 1; 8403 else if (reg->smin_value > sval) 8404 return 0; 8405 break; 8406 } 8407 8408 return -1; 8409 } 8410 8411 /* compute branch direction of the expression "if (reg opcode val) goto target;" 8412 * and return: 8413 * 1 - branch will be taken and "goto target" will be executed 8414 * 0 - branch will not be taken and fall-through to next insn 8415 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 8416 * range [0,10] 8417 */ 8418 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 8419 bool is_jmp32) 8420 { 8421 if (__is_pointer_value(false, reg)) { 8422 if (!reg_type_not_null(reg->type)) 8423 return -1; 8424 8425 /* If pointer is valid tests against zero will fail so we can 8426 * use this to direct branch taken. 8427 */ 8428 if (val != 0) 8429 return -1; 8430 8431 switch (opcode) { 8432 case BPF_JEQ: 8433 return 0; 8434 case BPF_JNE: 8435 return 1; 8436 default: 8437 return -1; 8438 } 8439 } 8440 8441 if (is_jmp32) 8442 return is_branch32_taken(reg, val, opcode); 8443 return is_branch64_taken(reg, val, opcode); 8444 } 8445 8446 static int flip_opcode(u32 opcode) 8447 { 8448 /* How can we transform "a <op> b" into "b <op> a"? */ 8449 static const u8 opcode_flip[16] = { 8450 /* these stay the same */ 8451 [BPF_JEQ >> 4] = BPF_JEQ, 8452 [BPF_JNE >> 4] = BPF_JNE, 8453 [BPF_JSET >> 4] = BPF_JSET, 8454 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 8455 [BPF_JGE >> 4] = BPF_JLE, 8456 [BPF_JGT >> 4] = BPF_JLT, 8457 [BPF_JLE >> 4] = BPF_JGE, 8458 [BPF_JLT >> 4] = BPF_JGT, 8459 [BPF_JSGE >> 4] = BPF_JSLE, 8460 [BPF_JSGT >> 4] = BPF_JSLT, 8461 [BPF_JSLE >> 4] = BPF_JSGE, 8462 [BPF_JSLT >> 4] = BPF_JSGT 8463 }; 8464 return opcode_flip[opcode >> 4]; 8465 } 8466 8467 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 8468 struct bpf_reg_state *src_reg, 8469 u8 opcode) 8470 { 8471 struct bpf_reg_state *pkt; 8472 8473 if (src_reg->type == PTR_TO_PACKET_END) { 8474 pkt = dst_reg; 8475 } else if (dst_reg->type == PTR_TO_PACKET_END) { 8476 pkt = src_reg; 8477 opcode = flip_opcode(opcode); 8478 } else { 8479 return -1; 8480 } 8481 8482 if (pkt->range >= 0) 8483 return -1; 8484 8485 switch (opcode) { 8486 case BPF_JLE: 8487 /* pkt <= pkt_end */ 8488 fallthrough; 8489 case BPF_JGT: 8490 /* pkt > pkt_end */ 8491 if (pkt->range == BEYOND_PKT_END) 8492 /* pkt has at last one extra byte beyond pkt_end */ 8493 return opcode == BPF_JGT; 8494 break; 8495 case BPF_JLT: 8496 /* pkt < pkt_end */ 8497 fallthrough; 8498 case BPF_JGE: 8499 /* pkt >= pkt_end */ 8500 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 8501 return opcode == BPF_JGE; 8502 break; 8503 } 8504 return -1; 8505 } 8506 8507 /* Adjusts the register min/max values in the case that the dst_reg is the 8508 * variable register that we are working on, and src_reg is a constant or we're 8509 * simply doing a BPF_K check. 8510 * In JEQ/JNE cases we also adjust the var_off values. 8511 */ 8512 static void reg_set_min_max(struct bpf_reg_state *true_reg, 8513 struct bpf_reg_state *false_reg, 8514 u64 val, u32 val32, 8515 u8 opcode, bool is_jmp32) 8516 { 8517 struct tnum false_32off = tnum_subreg(false_reg->var_off); 8518 struct tnum false_64off = false_reg->var_off; 8519 struct tnum true_32off = tnum_subreg(true_reg->var_off); 8520 struct tnum true_64off = true_reg->var_off; 8521 s64 sval = (s64)val; 8522 s32 sval32 = (s32)val32; 8523 8524 /* If the dst_reg is a pointer, we can't learn anything about its 8525 * variable offset from the compare (unless src_reg were a pointer into 8526 * the same object, but we don't bother with that. 8527 * Since false_reg and true_reg have the same type by construction, we 8528 * only need to check one of them for pointerness. 8529 */ 8530 if (__is_pointer_value(false, false_reg)) 8531 return; 8532 8533 switch (opcode) { 8534 case BPF_JEQ: 8535 case BPF_JNE: 8536 { 8537 struct bpf_reg_state *reg = 8538 opcode == BPF_JEQ ? true_reg : false_reg; 8539 8540 /* JEQ/JNE comparison doesn't change the register equivalence. 8541 * r1 = r2; 8542 * if (r1 == 42) goto label; 8543 * ... 8544 * label: // here both r1 and r2 are known to be 42. 8545 * 8546 * Hence when marking register as known preserve it's ID. 8547 */ 8548 if (is_jmp32) 8549 __mark_reg32_known(reg, val32); 8550 else 8551 ___mark_reg_known(reg, val); 8552 break; 8553 } 8554 case BPF_JSET: 8555 if (is_jmp32) { 8556 false_32off = tnum_and(false_32off, tnum_const(~val32)); 8557 if (is_power_of_2(val32)) 8558 true_32off = tnum_or(true_32off, 8559 tnum_const(val32)); 8560 } else { 8561 false_64off = tnum_and(false_64off, tnum_const(~val)); 8562 if (is_power_of_2(val)) 8563 true_64off = tnum_or(true_64off, 8564 tnum_const(val)); 8565 } 8566 break; 8567 case BPF_JGE: 8568 case BPF_JGT: 8569 { 8570 if (is_jmp32) { 8571 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 8572 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 8573 8574 false_reg->u32_max_value = min(false_reg->u32_max_value, 8575 false_umax); 8576 true_reg->u32_min_value = max(true_reg->u32_min_value, 8577 true_umin); 8578 } else { 8579 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 8580 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 8581 8582 false_reg->umax_value = min(false_reg->umax_value, false_umax); 8583 true_reg->umin_value = max(true_reg->umin_value, true_umin); 8584 } 8585 break; 8586 } 8587 case BPF_JSGE: 8588 case BPF_JSGT: 8589 { 8590 if (is_jmp32) { 8591 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 8592 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 8593 8594 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 8595 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 8596 } else { 8597 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 8598 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 8599 8600 false_reg->smax_value = min(false_reg->smax_value, false_smax); 8601 true_reg->smin_value = max(true_reg->smin_value, true_smin); 8602 } 8603 break; 8604 } 8605 case BPF_JLE: 8606 case BPF_JLT: 8607 { 8608 if (is_jmp32) { 8609 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 8610 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 8611 8612 false_reg->u32_min_value = max(false_reg->u32_min_value, 8613 false_umin); 8614 true_reg->u32_max_value = min(true_reg->u32_max_value, 8615 true_umax); 8616 } else { 8617 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 8618 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 8619 8620 false_reg->umin_value = max(false_reg->umin_value, false_umin); 8621 true_reg->umax_value = min(true_reg->umax_value, true_umax); 8622 } 8623 break; 8624 } 8625 case BPF_JSLE: 8626 case BPF_JSLT: 8627 { 8628 if (is_jmp32) { 8629 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 8630 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 8631 8632 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 8633 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 8634 } else { 8635 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 8636 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 8637 8638 false_reg->smin_value = max(false_reg->smin_value, false_smin); 8639 true_reg->smax_value = min(true_reg->smax_value, true_smax); 8640 } 8641 break; 8642 } 8643 default: 8644 return; 8645 } 8646 8647 if (is_jmp32) { 8648 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 8649 tnum_subreg(false_32off)); 8650 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 8651 tnum_subreg(true_32off)); 8652 __reg_combine_32_into_64(false_reg); 8653 __reg_combine_32_into_64(true_reg); 8654 } else { 8655 false_reg->var_off = false_64off; 8656 true_reg->var_off = true_64off; 8657 __reg_combine_64_into_32(false_reg); 8658 __reg_combine_64_into_32(true_reg); 8659 } 8660 } 8661 8662 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 8663 * the variable reg. 8664 */ 8665 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 8666 struct bpf_reg_state *false_reg, 8667 u64 val, u32 val32, 8668 u8 opcode, bool is_jmp32) 8669 { 8670 opcode = flip_opcode(opcode); 8671 /* This uses zero as "not present in table"; luckily the zero opcode, 8672 * BPF_JA, can't get here. 8673 */ 8674 if (opcode) 8675 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 8676 } 8677 8678 /* Regs are known to be equal, so intersect their min/max/var_off */ 8679 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 8680 struct bpf_reg_state *dst_reg) 8681 { 8682 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 8683 dst_reg->umin_value); 8684 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 8685 dst_reg->umax_value); 8686 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 8687 dst_reg->smin_value); 8688 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 8689 dst_reg->smax_value); 8690 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 8691 dst_reg->var_off); 8692 /* We might have learned new bounds from the var_off. */ 8693 __update_reg_bounds(src_reg); 8694 __update_reg_bounds(dst_reg); 8695 /* We might have learned something about the sign bit. */ 8696 __reg_deduce_bounds(src_reg); 8697 __reg_deduce_bounds(dst_reg); 8698 /* We might have learned some bits from the bounds. */ 8699 __reg_bound_offset(src_reg); 8700 __reg_bound_offset(dst_reg); 8701 /* Intersecting with the old var_off might have improved our bounds 8702 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 8703 * then new var_off is (0; 0x7f...fc) which improves our umax. 8704 */ 8705 __update_reg_bounds(src_reg); 8706 __update_reg_bounds(dst_reg); 8707 } 8708 8709 static void reg_combine_min_max(struct bpf_reg_state *true_src, 8710 struct bpf_reg_state *true_dst, 8711 struct bpf_reg_state *false_src, 8712 struct bpf_reg_state *false_dst, 8713 u8 opcode) 8714 { 8715 switch (opcode) { 8716 case BPF_JEQ: 8717 __reg_combine_min_max(true_src, true_dst); 8718 break; 8719 case BPF_JNE: 8720 __reg_combine_min_max(false_src, false_dst); 8721 break; 8722 } 8723 } 8724 8725 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 8726 struct bpf_reg_state *reg, u32 id, 8727 bool is_null) 8728 { 8729 if (reg_type_may_be_null(reg->type) && reg->id == id && 8730 !WARN_ON_ONCE(!reg->id)) { 8731 /* Old offset (both fixed and variable parts) should 8732 * have been known-zero, because we don't allow pointer 8733 * arithmetic on pointers that might be NULL. 8734 */ 8735 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || 8736 !tnum_equals_const(reg->var_off, 0) || 8737 reg->off)) { 8738 __mark_reg_known_zero(reg); 8739 reg->off = 0; 8740 } 8741 if (is_null) { 8742 reg->type = SCALAR_VALUE; 8743 /* We don't need id and ref_obj_id from this point 8744 * onwards anymore, thus we should better reset it, 8745 * so that state pruning has chances to take effect. 8746 */ 8747 reg->id = 0; 8748 reg->ref_obj_id = 0; 8749 8750 return; 8751 } 8752 8753 mark_ptr_not_null_reg(reg); 8754 8755 if (!reg_may_point_to_spin_lock(reg)) { 8756 /* For not-NULL ptr, reg->ref_obj_id will be reset 8757 * in release_reg_references(). 8758 * 8759 * reg->id is still used by spin_lock ptr. Other 8760 * than spin_lock ptr type, reg->id can be reset. 8761 */ 8762 reg->id = 0; 8763 } 8764 } 8765 } 8766 8767 static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, 8768 bool is_null) 8769 { 8770 struct bpf_reg_state *reg; 8771 int i; 8772 8773 for (i = 0; i < MAX_BPF_REG; i++) 8774 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); 8775 8776 bpf_for_each_spilled_reg(i, state, reg) { 8777 if (!reg) 8778 continue; 8779 mark_ptr_or_null_reg(state, reg, id, is_null); 8780 } 8781 } 8782 8783 /* The logic is similar to find_good_pkt_pointers(), both could eventually 8784 * be folded together at some point. 8785 */ 8786 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 8787 bool is_null) 8788 { 8789 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 8790 struct bpf_reg_state *regs = state->regs; 8791 u32 ref_obj_id = regs[regno].ref_obj_id; 8792 u32 id = regs[regno].id; 8793 int i; 8794 8795 if (ref_obj_id && ref_obj_id == id && is_null) 8796 /* regs[regno] is in the " == NULL" branch. 8797 * No one could have freed the reference state before 8798 * doing the NULL check. 8799 */ 8800 WARN_ON_ONCE(release_reference_state(state, id)); 8801 8802 for (i = 0; i <= vstate->curframe; i++) 8803 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); 8804 } 8805 8806 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 8807 struct bpf_reg_state *dst_reg, 8808 struct bpf_reg_state *src_reg, 8809 struct bpf_verifier_state *this_branch, 8810 struct bpf_verifier_state *other_branch) 8811 { 8812 if (BPF_SRC(insn->code) != BPF_X) 8813 return false; 8814 8815 /* Pointers are always 64-bit. */ 8816 if (BPF_CLASS(insn->code) == BPF_JMP32) 8817 return false; 8818 8819 switch (BPF_OP(insn->code)) { 8820 case BPF_JGT: 8821 if ((dst_reg->type == PTR_TO_PACKET && 8822 src_reg->type == PTR_TO_PACKET_END) || 8823 (dst_reg->type == PTR_TO_PACKET_META && 8824 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8825 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 8826 find_good_pkt_pointers(this_branch, dst_reg, 8827 dst_reg->type, false); 8828 mark_pkt_end(other_branch, insn->dst_reg, true); 8829 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8830 src_reg->type == PTR_TO_PACKET) || 8831 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8832 src_reg->type == PTR_TO_PACKET_META)) { 8833 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 8834 find_good_pkt_pointers(other_branch, src_reg, 8835 src_reg->type, true); 8836 mark_pkt_end(this_branch, insn->src_reg, false); 8837 } else { 8838 return false; 8839 } 8840 break; 8841 case BPF_JLT: 8842 if ((dst_reg->type == PTR_TO_PACKET && 8843 src_reg->type == PTR_TO_PACKET_END) || 8844 (dst_reg->type == PTR_TO_PACKET_META && 8845 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8846 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 8847 find_good_pkt_pointers(other_branch, dst_reg, 8848 dst_reg->type, true); 8849 mark_pkt_end(this_branch, insn->dst_reg, false); 8850 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8851 src_reg->type == PTR_TO_PACKET) || 8852 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8853 src_reg->type == PTR_TO_PACKET_META)) { 8854 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 8855 find_good_pkt_pointers(this_branch, src_reg, 8856 src_reg->type, false); 8857 mark_pkt_end(other_branch, insn->src_reg, true); 8858 } else { 8859 return false; 8860 } 8861 break; 8862 case BPF_JGE: 8863 if ((dst_reg->type == PTR_TO_PACKET && 8864 src_reg->type == PTR_TO_PACKET_END) || 8865 (dst_reg->type == PTR_TO_PACKET_META && 8866 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8867 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 8868 find_good_pkt_pointers(this_branch, dst_reg, 8869 dst_reg->type, true); 8870 mark_pkt_end(other_branch, insn->dst_reg, false); 8871 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8872 src_reg->type == PTR_TO_PACKET) || 8873 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8874 src_reg->type == PTR_TO_PACKET_META)) { 8875 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 8876 find_good_pkt_pointers(other_branch, src_reg, 8877 src_reg->type, false); 8878 mark_pkt_end(this_branch, insn->src_reg, true); 8879 } else { 8880 return false; 8881 } 8882 break; 8883 case BPF_JLE: 8884 if ((dst_reg->type == PTR_TO_PACKET && 8885 src_reg->type == PTR_TO_PACKET_END) || 8886 (dst_reg->type == PTR_TO_PACKET_META && 8887 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 8888 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 8889 find_good_pkt_pointers(other_branch, dst_reg, 8890 dst_reg->type, false); 8891 mark_pkt_end(this_branch, insn->dst_reg, true); 8892 } else if ((dst_reg->type == PTR_TO_PACKET_END && 8893 src_reg->type == PTR_TO_PACKET) || 8894 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 8895 src_reg->type == PTR_TO_PACKET_META)) { 8896 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 8897 find_good_pkt_pointers(this_branch, src_reg, 8898 src_reg->type, true); 8899 mark_pkt_end(other_branch, insn->src_reg, false); 8900 } else { 8901 return false; 8902 } 8903 break; 8904 default: 8905 return false; 8906 } 8907 8908 return true; 8909 } 8910 8911 static void find_equal_scalars(struct bpf_verifier_state *vstate, 8912 struct bpf_reg_state *known_reg) 8913 { 8914 struct bpf_func_state *state; 8915 struct bpf_reg_state *reg; 8916 int i, j; 8917 8918 for (i = 0; i <= vstate->curframe; i++) { 8919 state = vstate->frame[i]; 8920 for (j = 0; j < MAX_BPF_REG; j++) { 8921 reg = &state->regs[j]; 8922 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 8923 *reg = *known_reg; 8924 } 8925 8926 bpf_for_each_spilled_reg(j, state, reg) { 8927 if (!reg) 8928 continue; 8929 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 8930 *reg = *known_reg; 8931 } 8932 } 8933 } 8934 8935 static int check_cond_jmp_op(struct bpf_verifier_env *env, 8936 struct bpf_insn *insn, int *insn_idx) 8937 { 8938 struct bpf_verifier_state *this_branch = env->cur_state; 8939 struct bpf_verifier_state *other_branch; 8940 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 8941 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 8942 u8 opcode = BPF_OP(insn->code); 8943 bool is_jmp32; 8944 int pred = -1; 8945 int err; 8946 8947 /* Only conditional jumps are expected to reach here. */ 8948 if (opcode == BPF_JA || opcode > BPF_JSLE) { 8949 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 8950 return -EINVAL; 8951 } 8952 8953 if (BPF_SRC(insn->code) == BPF_X) { 8954 if (insn->imm != 0) { 8955 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 8956 return -EINVAL; 8957 } 8958 8959 /* check src1 operand */ 8960 err = check_reg_arg(env, insn->src_reg, SRC_OP); 8961 if (err) 8962 return err; 8963 8964 if (is_pointer_value(env, insn->src_reg)) { 8965 verbose(env, "R%d pointer comparison prohibited\n", 8966 insn->src_reg); 8967 return -EACCES; 8968 } 8969 src_reg = ®s[insn->src_reg]; 8970 } else { 8971 if (insn->src_reg != BPF_REG_0) { 8972 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 8973 return -EINVAL; 8974 } 8975 } 8976 8977 /* check src2 operand */ 8978 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 8979 if (err) 8980 return err; 8981 8982 dst_reg = ®s[insn->dst_reg]; 8983 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 8984 8985 if (BPF_SRC(insn->code) == BPF_K) { 8986 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 8987 } else if (src_reg->type == SCALAR_VALUE && 8988 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 8989 pred = is_branch_taken(dst_reg, 8990 tnum_subreg(src_reg->var_off).value, 8991 opcode, 8992 is_jmp32); 8993 } else if (src_reg->type == SCALAR_VALUE && 8994 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 8995 pred = is_branch_taken(dst_reg, 8996 src_reg->var_off.value, 8997 opcode, 8998 is_jmp32); 8999 } else if (reg_is_pkt_pointer_any(dst_reg) && 9000 reg_is_pkt_pointer_any(src_reg) && 9001 !is_jmp32) { 9002 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 9003 } 9004 9005 if (pred >= 0) { 9006 /* If we get here with a dst_reg pointer type it is because 9007 * above is_branch_taken() special cased the 0 comparison. 9008 */ 9009 if (!__is_pointer_value(false, dst_reg)) 9010 err = mark_chain_precision(env, insn->dst_reg); 9011 if (BPF_SRC(insn->code) == BPF_X && !err && 9012 !__is_pointer_value(false, src_reg)) 9013 err = mark_chain_precision(env, insn->src_reg); 9014 if (err) 9015 return err; 9016 } 9017 9018 if (pred == 1) { 9019 /* Only follow the goto, ignore fall-through. If needed, push 9020 * the fall-through branch for simulation under speculative 9021 * execution. 9022 */ 9023 if (!env->bypass_spec_v1 && 9024 !sanitize_speculative_path(env, insn, *insn_idx + 1, 9025 *insn_idx)) 9026 return -EFAULT; 9027 *insn_idx += insn->off; 9028 return 0; 9029 } else if (pred == 0) { 9030 /* Only follow the fall-through branch, since that's where the 9031 * program will go. If needed, push the goto branch for 9032 * simulation under speculative execution. 9033 */ 9034 if (!env->bypass_spec_v1 && 9035 !sanitize_speculative_path(env, insn, 9036 *insn_idx + insn->off + 1, 9037 *insn_idx)) 9038 return -EFAULT; 9039 return 0; 9040 } 9041 9042 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 9043 false); 9044 if (!other_branch) 9045 return -EFAULT; 9046 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 9047 9048 /* detect if we are comparing against a constant value so we can adjust 9049 * our min/max values for our dst register. 9050 * this is only legit if both are scalars (or pointers to the same 9051 * object, I suppose, but we don't support that right now), because 9052 * otherwise the different base pointers mean the offsets aren't 9053 * comparable. 9054 */ 9055 if (BPF_SRC(insn->code) == BPF_X) { 9056 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 9057 9058 if (dst_reg->type == SCALAR_VALUE && 9059 src_reg->type == SCALAR_VALUE) { 9060 if (tnum_is_const(src_reg->var_off) || 9061 (is_jmp32 && 9062 tnum_is_const(tnum_subreg(src_reg->var_off)))) 9063 reg_set_min_max(&other_branch_regs[insn->dst_reg], 9064 dst_reg, 9065 src_reg->var_off.value, 9066 tnum_subreg(src_reg->var_off).value, 9067 opcode, is_jmp32); 9068 else if (tnum_is_const(dst_reg->var_off) || 9069 (is_jmp32 && 9070 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 9071 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 9072 src_reg, 9073 dst_reg->var_off.value, 9074 tnum_subreg(dst_reg->var_off).value, 9075 opcode, is_jmp32); 9076 else if (!is_jmp32 && 9077 (opcode == BPF_JEQ || opcode == BPF_JNE)) 9078 /* Comparing for equality, we can combine knowledge */ 9079 reg_combine_min_max(&other_branch_regs[insn->src_reg], 9080 &other_branch_regs[insn->dst_reg], 9081 src_reg, dst_reg, opcode); 9082 if (src_reg->id && 9083 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 9084 find_equal_scalars(this_branch, src_reg); 9085 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 9086 } 9087 9088 } 9089 } else if (dst_reg->type == SCALAR_VALUE) { 9090 reg_set_min_max(&other_branch_regs[insn->dst_reg], 9091 dst_reg, insn->imm, (u32)insn->imm, 9092 opcode, is_jmp32); 9093 } 9094 9095 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 9096 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 9097 find_equal_scalars(this_branch, dst_reg); 9098 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 9099 } 9100 9101 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 9102 * NOTE: these optimizations below are related with pointer comparison 9103 * which will never be JMP32. 9104 */ 9105 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 9106 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 9107 reg_type_may_be_null(dst_reg->type)) { 9108 /* Mark all identical registers in each branch as either 9109 * safe or unknown depending R == 0 or R != 0 conditional. 9110 */ 9111 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 9112 opcode == BPF_JNE); 9113 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 9114 opcode == BPF_JEQ); 9115 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 9116 this_branch, other_branch) && 9117 is_pointer_value(env, insn->dst_reg)) { 9118 verbose(env, "R%d pointer comparison prohibited\n", 9119 insn->dst_reg); 9120 return -EACCES; 9121 } 9122 if (env->log.level & BPF_LOG_LEVEL) 9123 print_verifier_state(env, this_branch->frame[this_branch->curframe]); 9124 return 0; 9125 } 9126 9127 /* verify BPF_LD_IMM64 instruction */ 9128 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 9129 { 9130 struct bpf_insn_aux_data *aux = cur_aux(env); 9131 struct bpf_reg_state *regs = cur_regs(env); 9132 struct bpf_reg_state *dst_reg; 9133 struct bpf_map *map; 9134 int err; 9135 9136 if (BPF_SIZE(insn->code) != BPF_DW) { 9137 verbose(env, "invalid BPF_LD_IMM insn\n"); 9138 return -EINVAL; 9139 } 9140 if (insn->off != 0) { 9141 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 9142 return -EINVAL; 9143 } 9144 9145 err = check_reg_arg(env, insn->dst_reg, DST_OP); 9146 if (err) 9147 return err; 9148 9149 dst_reg = ®s[insn->dst_reg]; 9150 if (insn->src_reg == 0) { 9151 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 9152 9153 dst_reg->type = SCALAR_VALUE; 9154 __mark_reg_known(®s[insn->dst_reg], imm); 9155 return 0; 9156 } 9157 9158 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 9159 mark_reg_known_zero(env, regs, insn->dst_reg); 9160 9161 dst_reg->type = aux->btf_var.reg_type; 9162 switch (dst_reg->type) { 9163 case PTR_TO_MEM: 9164 dst_reg->mem_size = aux->btf_var.mem_size; 9165 break; 9166 case PTR_TO_BTF_ID: 9167 case PTR_TO_PERCPU_BTF_ID: 9168 dst_reg->btf = aux->btf_var.btf; 9169 dst_reg->btf_id = aux->btf_var.btf_id; 9170 break; 9171 default: 9172 verbose(env, "bpf verifier is misconfigured\n"); 9173 return -EFAULT; 9174 } 9175 return 0; 9176 } 9177 9178 if (insn->src_reg == BPF_PSEUDO_FUNC) { 9179 struct bpf_prog_aux *aux = env->prog->aux; 9180 u32 subprogno = insn[1].imm; 9181 9182 if (!aux->func_info) { 9183 verbose(env, "missing btf func_info\n"); 9184 return -EINVAL; 9185 } 9186 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 9187 verbose(env, "callback function not static\n"); 9188 return -EINVAL; 9189 } 9190 9191 dst_reg->type = PTR_TO_FUNC; 9192 dst_reg->subprogno = subprogno; 9193 return 0; 9194 } 9195 9196 map = env->used_maps[aux->map_index]; 9197 mark_reg_known_zero(env, regs, insn->dst_reg); 9198 dst_reg->map_ptr = map; 9199 9200 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 9201 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 9202 dst_reg->type = PTR_TO_MAP_VALUE; 9203 dst_reg->off = aux->map_off; 9204 if (map_value_has_spin_lock(map)) 9205 dst_reg->id = ++env->id_gen; 9206 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 9207 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 9208 dst_reg->type = CONST_PTR_TO_MAP; 9209 } else { 9210 verbose(env, "bpf verifier is misconfigured\n"); 9211 return -EINVAL; 9212 } 9213 9214 return 0; 9215 } 9216 9217 static bool may_access_skb(enum bpf_prog_type type) 9218 { 9219 switch (type) { 9220 case BPF_PROG_TYPE_SOCKET_FILTER: 9221 case BPF_PROG_TYPE_SCHED_CLS: 9222 case BPF_PROG_TYPE_SCHED_ACT: 9223 return true; 9224 default: 9225 return false; 9226 } 9227 } 9228 9229 /* verify safety of LD_ABS|LD_IND instructions: 9230 * - they can only appear in the programs where ctx == skb 9231 * - since they are wrappers of function calls, they scratch R1-R5 registers, 9232 * preserve R6-R9, and store return value into R0 9233 * 9234 * Implicit input: 9235 * ctx == skb == R6 == CTX 9236 * 9237 * Explicit input: 9238 * SRC == any register 9239 * IMM == 32-bit immediate 9240 * 9241 * Output: 9242 * R0 - 8/16/32-bit skb data converted to cpu endianness 9243 */ 9244 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 9245 { 9246 struct bpf_reg_state *regs = cur_regs(env); 9247 static const int ctx_reg = BPF_REG_6; 9248 u8 mode = BPF_MODE(insn->code); 9249 int i, err; 9250 9251 if (!may_access_skb(resolve_prog_type(env->prog))) { 9252 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 9253 return -EINVAL; 9254 } 9255 9256 if (!env->ops->gen_ld_abs) { 9257 verbose(env, "bpf verifier is misconfigured\n"); 9258 return -EINVAL; 9259 } 9260 9261 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 9262 BPF_SIZE(insn->code) == BPF_DW || 9263 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 9264 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 9265 return -EINVAL; 9266 } 9267 9268 /* check whether implicit source operand (register R6) is readable */ 9269 err = check_reg_arg(env, ctx_reg, SRC_OP); 9270 if (err) 9271 return err; 9272 9273 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 9274 * gen_ld_abs() may terminate the program at runtime, leading to 9275 * reference leak. 9276 */ 9277 err = check_reference_leak(env); 9278 if (err) { 9279 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 9280 return err; 9281 } 9282 9283 if (env->cur_state->active_spin_lock) { 9284 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 9285 return -EINVAL; 9286 } 9287 9288 if (regs[ctx_reg].type != PTR_TO_CTX) { 9289 verbose(env, 9290 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 9291 return -EINVAL; 9292 } 9293 9294 if (mode == BPF_IND) { 9295 /* check explicit source operand */ 9296 err = check_reg_arg(env, insn->src_reg, SRC_OP); 9297 if (err) 9298 return err; 9299 } 9300 9301 err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); 9302 if (err < 0) 9303 return err; 9304 9305 /* reset caller saved regs to unreadable */ 9306 for (i = 0; i < CALLER_SAVED_REGS; i++) { 9307 mark_reg_not_init(env, regs, caller_saved[i]); 9308 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 9309 } 9310 9311 /* mark destination R0 register as readable, since it contains 9312 * the value fetched from the packet. 9313 * Already marked as written above. 9314 */ 9315 mark_reg_unknown(env, regs, BPF_REG_0); 9316 /* ld_abs load up to 32-bit skb data. */ 9317 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 9318 return 0; 9319 } 9320 9321 static int check_return_code(struct bpf_verifier_env *env) 9322 { 9323 struct tnum enforce_attach_type_range = tnum_unknown; 9324 const struct bpf_prog *prog = env->prog; 9325 struct bpf_reg_state *reg; 9326 struct tnum range = tnum_range(0, 1); 9327 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 9328 int err; 9329 struct bpf_func_state *frame = env->cur_state->frame[0]; 9330 const bool is_subprog = frame->subprogno; 9331 9332 /* LSM and struct_ops func-ptr's return type could be "void" */ 9333 if (!is_subprog && 9334 (prog_type == BPF_PROG_TYPE_STRUCT_OPS || 9335 prog_type == BPF_PROG_TYPE_LSM) && 9336 !prog->aux->attach_func_proto->type) 9337 return 0; 9338 9339 /* eBPF calling convention is such that R0 is used 9340 * to return the value from eBPF program. 9341 * Make sure that it's readable at this time 9342 * of bpf_exit, which means that program wrote 9343 * something into it earlier 9344 */ 9345 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 9346 if (err) 9347 return err; 9348 9349 if (is_pointer_value(env, BPF_REG_0)) { 9350 verbose(env, "R0 leaks addr as return value\n"); 9351 return -EACCES; 9352 } 9353 9354 reg = cur_regs(env) + BPF_REG_0; 9355 9356 if (frame->in_async_callback_fn) { 9357 /* enforce return zero from async callbacks like timer */ 9358 if (reg->type != SCALAR_VALUE) { 9359 verbose(env, "In async callback the register R0 is not a known value (%s)\n", 9360 reg_type_str[reg->type]); 9361 return -EINVAL; 9362 } 9363 9364 if (!tnum_in(tnum_const(0), reg->var_off)) { 9365 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); 9366 return -EINVAL; 9367 } 9368 return 0; 9369 } 9370 9371 if (is_subprog) { 9372 if (reg->type != SCALAR_VALUE) { 9373 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 9374 reg_type_str[reg->type]); 9375 return -EINVAL; 9376 } 9377 return 0; 9378 } 9379 9380 switch (prog_type) { 9381 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 9382 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 9383 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 9384 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 9385 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 9386 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 9387 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 9388 range = tnum_range(1, 1); 9389 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 9390 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 9391 range = tnum_range(0, 3); 9392 break; 9393 case BPF_PROG_TYPE_CGROUP_SKB: 9394 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 9395 range = tnum_range(0, 3); 9396 enforce_attach_type_range = tnum_range(2, 3); 9397 } 9398 break; 9399 case BPF_PROG_TYPE_CGROUP_SOCK: 9400 case BPF_PROG_TYPE_SOCK_OPS: 9401 case BPF_PROG_TYPE_CGROUP_DEVICE: 9402 case BPF_PROG_TYPE_CGROUP_SYSCTL: 9403 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 9404 break; 9405 case BPF_PROG_TYPE_RAW_TRACEPOINT: 9406 if (!env->prog->aux->attach_btf_id) 9407 return 0; 9408 range = tnum_const(0); 9409 break; 9410 case BPF_PROG_TYPE_TRACING: 9411 switch (env->prog->expected_attach_type) { 9412 case BPF_TRACE_FENTRY: 9413 case BPF_TRACE_FEXIT: 9414 range = tnum_const(0); 9415 break; 9416 case BPF_TRACE_RAW_TP: 9417 case BPF_MODIFY_RETURN: 9418 return 0; 9419 case BPF_TRACE_ITER: 9420 break; 9421 default: 9422 return -ENOTSUPP; 9423 } 9424 break; 9425 case BPF_PROG_TYPE_SK_LOOKUP: 9426 range = tnum_range(SK_DROP, SK_PASS); 9427 break; 9428 case BPF_PROG_TYPE_EXT: 9429 /* freplace program can return anything as its return value 9430 * depends on the to-be-replaced kernel func or bpf program. 9431 */ 9432 default: 9433 return 0; 9434 } 9435 9436 if (reg->type != SCALAR_VALUE) { 9437 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 9438 reg_type_str[reg->type]); 9439 return -EINVAL; 9440 } 9441 9442 if (!tnum_in(range, reg->var_off)) { 9443 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 9444 return -EINVAL; 9445 } 9446 9447 if (!tnum_is_unknown(enforce_attach_type_range) && 9448 tnum_in(enforce_attach_type_range, reg->var_off)) 9449 env->prog->enforce_expected_attach_type = 1; 9450 return 0; 9451 } 9452 9453 /* non-recursive DFS pseudo code 9454 * 1 procedure DFS-iterative(G,v): 9455 * 2 label v as discovered 9456 * 3 let S be a stack 9457 * 4 S.push(v) 9458 * 5 while S is not empty 9459 * 6 t <- S.pop() 9460 * 7 if t is what we're looking for: 9461 * 8 return t 9462 * 9 for all edges e in G.adjacentEdges(t) do 9463 * 10 if edge e is already labelled 9464 * 11 continue with the next edge 9465 * 12 w <- G.adjacentVertex(t,e) 9466 * 13 if vertex w is not discovered and not explored 9467 * 14 label e as tree-edge 9468 * 15 label w as discovered 9469 * 16 S.push(w) 9470 * 17 continue at 5 9471 * 18 else if vertex w is discovered 9472 * 19 label e as back-edge 9473 * 20 else 9474 * 21 // vertex w is explored 9475 * 22 label e as forward- or cross-edge 9476 * 23 label t as explored 9477 * 24 S.pop() 9478 * 9479 * convention: 9480 * 0x10 - discovered 9481 * 0x11 - discovered and fall-through edge labelled 9482 * 0x12 - discovered and fall-through and branch edges labelled 9483 * 0x20 - explored 9484 */ 9485 9486 enum { 9487 DISCOVERED = 0x10, 9488 EXPLORED = 0x20, 9489 FALLTHROUGH = 1, 9490 BRANCH = 2, 9491 }; 9492 9493 static u32 state_htab_size(struct bpf_verifier_env *env) 9494 { 9495 return env->prog->len; 9496 } 9497 9498 static struct bpf_verifier_state_list **explored_state( 9499 struct bpf_verifier_env *env, 9500 int idx) 9501 { 9502 struct bpf_verifier_state *cur = env->cur_state; 9503 struct bpf_func_state *state = cur->frame[cur->curframe]; 9504 9505 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 9506 } 9507 9508 static void init_explored_state(struct bpf_verifier_env *env, int idx) 9509 { 9510 env->insn_aux_data[idx].prune_point = true; 9511 } 9512 9513 enum { 9514 DONE_EXPLORING = 0, 9515 KEEP_EXPLORING = 1, 9516 }; 9517 9518 /* t, w, e - match pseudo-code above: 9519 * t - index of current instruction 9520 * w - next instruction 9521 * e - edge 9522 */ 9523 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 9524 bool loop_ok) 9525 { 9526 int *insn_stack = env->cfg.insn_stack; 9527 int *insn_state = env->cfg.insn_state; 9528 9529 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 9530 return DONE_EXPLORING; 9531 9532 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 9533 return DONE_EXPLORING; 9534 9535 if (w < 0 || w >= env->prog->len) { 9536 verbose_linfo(env, t, "%d: ", t); 9537 verbose(env, "jump out of range from insn %d to %d\n", t, w); 9538 return -EINVAL; 9539 } 9540 9541 if (e == BRANCH) 9542 /* mark branch target for state pruning */ 9543 init_explored_state(env, w); 9544 9545 if (insn_state[w] == 0) { 9546 /* tree-edge */ 9547 insn_state[t] = DISCOVERED | e; 9548 insn_state[w] = DISCOVERED; 9549 if (env->cfg.cur_stack >= env->prog->len) 9550 return -E2BIG; 9551 insn_stack[env->cfg.cur_stack++] = w; 9552 return KEEP_EXPLORING; 9553 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 9554 if (loop_ok && env->bpf_capable) 9555 return DONE_EXPLORING; 9556 verbose_linfo(env, t, "%d: ", t); 9557 verbose_linfo(env, w, "%d: ", w); 9558 verbose(env, "back-edge from insn %d to %d\n", t, w); 9559 return -EINVAL; 9560 } else if (insn_state[w] == EXPLORED) { 9561 /* forward- or cross-edge */ 9562 insn_state[t] = DISCOVERED | e; 9563 } else { 9564 verbose(env, "insn state internal bug\n"); 9565 return -EFAULT; 9566 } 9567 return DONE_EXPLORING; 9568 } 9569 9570 static int visit_func_call_insn(int t, int insn_cnt, 9571 struct bpf_insn *insns, 9572 struct bpf_verifier_env *env, 9573 bool visit_callee) 9574 { 9575 int ret; 9576 9577 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 9578 if (ret) 9579 return ret; 9580 9581 if (t + 1 < insn_cnt) 9582 init_explored_state(env, t + 1); 9583 if (visit_callee) { 9584 init_explored_state(env, t); 9585 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, 9586 /* It's ok to allow recursion from CFG point of 9587 * view. __check_func_call() will do the actual 9588 * check. 9589 */ 9590 bpf_pseudo_func(insns + t)); 9591 } 9592 return ret; 9593 } 9594 9595 /* Visits the instruction at index t and returns one of the following: 9596 * < 0 - an error occurred 9597 * DONE_EXPLORING - the instruction was fully explored 9598 * KEEP_EXPLORING - there is still work to be done before it is fully explored 9599 */ 9600 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) 9601 { 9602 struct bpf_insn *insns = env->prog->insnsi; 9603 int ret; 9604 9605 if (bpf_pseudo_func(insns + t)) 9606 return visit_func_call_insn(t, insn_cnt, insns, env, true); 9607 9608 /* All non-branch instructions have a single fall-through edge. */ 9609 if (BPF_CLASS(insns[t].code) != BPF_JMP && 9610 BPF_CLASS(insns[t].code) != BPF_JMP32) 9611 return push_insn(t, t + 1, FALLTHROUGH, env, false); 9612 9613 switch (BPF_OP(insns[t].code)) { 9614 case BPF_EXIT: 9615 return DONE_EXPLORING; 9616 9617 case BPF_CALL: 9618 if (insns[t].imm == BPF_FUNC_timer_set_callback) 9619 /* Mark this call insn to trigger is_state_visited() check 9620 * before call itself is processed by __check_func_call(). 9621 * Otherwise new async state will be pushed for further 9622 * exploration. 9623 */ 9624 init_explored_state(env, t); 9625 return visit_func_call_insn(t, insn_cnt, insns, env, 9626 insns[t].src_reg == BPF_PSEUDO_CALL); 9627 9628 case BPF_JA: 9629 if (BPF_SRC(insns[t].code) != BPF_K) 9630 return -EINVAL; 9631 9632 /* unconditional jump with single edge */ 9633 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 9634 true); 9635 if (ret) 9636 return ret; 9637 9638 /* unconditional jmp is not a good pruning point, 9639 * but it's marked, since backtracking needs 9640 * to record jmp history in is_state_visited(). 9641 */ 9642 init_explored_state(env, t + insns[t].off + 1); 9643 /* tell verifier to check for equivalent states 9644 * after every call and jump 9645 */ 9646 if (t + 1 < insn_cnt) 9647 init_explored_state(env, t + 1); 9648 9649 return ret; 9650 9651 default: 9652 /* conditional jump with two edges */ 9653 init_explored_state(env, t); 9654 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 9655 if (ret) 9656 return ret; 9657 9658 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 9659 } 9660 } 9661 9662 /* non-recursive depth-first-search to detect loops in BPF program 9663 * loop == back-edge in directed graph 9664 */ 9665 static int check_cfg(struct bpf_verifier_env *env) 9666 { 9667 int insn_cnt = env->prog->len; 9668 int *insn_stack, *insn_state; 9669 int ret = 0; 9670 int i; 9671 9672 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 9673 if (!insn_state) 9674 return -ENOMEM; 9675 9676 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 9677 if (!insn_stack) { 9678 kvfree(insn_state); 9679 return -ENOMEM; 9680 } 9681 9682 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 9683 insn_stack[0] = 0; /* 0 is the first instruction */ 9684 env->cfg.cur_stack = 1; 9685 9686 while (env->cfg.cur_stack > 0) { 9687 int t = insn_stack[env->cfg.cur_stack - 1]; 9688 9689 ret = visit_insn(t, insn_cnt, env); 9690 switch (ret) { 9691 case DONE_EXPLORING: 9692 insn_state[t] = EXPLORED; 9693 env->cfg.cur_stack--; 9694 break; 9695 case KEEP_EXPLORING: 9696 break; 9697 default: 9698 if (ret > 0) { 9699 verbose(env, "visit_insn internal bug\n"); 9700 ret = -EFAULT; 9701 } 9702 goto err_free; 9703 } 9704 } 9705 9706 if (env->cfg.cur_stack < 0) { 9707 verbose(env, "pop stack internal bug\n"); 9708 ret = -EFAULT; 9709 goto err_free; 9710 } 9711 9712 for (i = 0; i < insn_cnt; i++) { 9713 if (insn_state[i] != EXPLORED) { 9714 verbose(env, "unreachable insn %d\n", i); 9715 ret = -EINVAL; 9716 goto err_free; 9717 } 9718 } 9719 ret = 0; /* cfg looks good */ 9720 9721 err_free: 9722 kvfree(insn_state); 9723 kvfree(insn_stack); 9724 env->cfg.insn_state = env->cfg.insn_stack = NULL; 9725 return ret; 9726 } 9727 9728 static int check_abnormal_return(struct bpf_verifier_env *env) 9729 { 9730 int i; 9731 9732 for (i = 1; i < env->subprog_cnt; i++) { 9733 if (env->subprog_info[i].has_ld_abs) { 9734 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 9735 return -EINVAL; 9736 } 9737 if (env->subprog_info[i].has_tail_call) { 9738 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 9739 return -EINVAL; 9740 } 9741 } 9742 return 0; 9743 } 9744 9745 /* The minimum supported BTF func info size */ 9746 #define MIN_BPF_FUNCINFO_SIZE 8 9747 #define MAX_FUNCINFO_REC_SIZE 252 9748 9749 static int check_btf_func(struct bpf_verifier_env *env, 9750 const union bpf_attr *attr, 9751 bpfptr_t uattr) 9752 { 9753 const struct btf_type *type, *func_proto, *ret_type; 9754 u32 i, nfuncs, urec_size, min_size; 9755 u32 krec_size = sizeof(struct bpf_func_info); 9756 struct bpf_func_info *krecord; 9757 struct bpf_func_info_aux *info_aux = NULL; 9758 struct bpf_prog *prog; 9759 const struct btf *btf; 9760 bpfptr_t urecord; 9761 u32 prev_offset = 0; 9762 bool scalar_return; 9763 int ret = -ENOMEM; 9764 9765 nfuncs = attr->func_info_cnt; 9766 if (!nfuncs) { 9767 if (check_abnormal_return(env)) 9768 return -EINVAL; 9769 return 0; 9770 } 9771 9772 if (nfuncs != env->subprog_cnt) { 9773 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 9774 return -EINVAL; 9775 } 9776 9777 urec_size = attr->func_info_rec_size; 9778 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 9779 urec_size > MAX_FUNCINFO_REC_SIZE || 9780 urec_size % sizeof(u32)) { 9781 verbose(env, "invalid func info rec size %u\n", urec_size); 9782 return -EINVAL; 9783 } 9784 9785 prog = env->prog; 9786 btf = prog->aux->btf; 9787 9788 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 9789 min_size = min_t(u32, krec_size, urec_size); 9790 9791 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 9792 if (!krecord) 9793 return -ENOMEM; 9794 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 9795 if (!info_aux) 9796 goto err_free; 9797 9798 for (i = 0; i < nfuncs; i++) { 9799 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 9800 if (ret) { 9801 if (ret == -E2BIG) { 9802 verbose(env, "nonzero tailing record in func info"); 9803 /* set the size kernel expects so loader can zero 9804 * out the rest of the record. 9805 */ 9806 if (copy_to_bpfptr_offset(uattr, 9807 offsetof(union bpf_attr, func_info_rec_size), 9808 &min_size, sizeof(min_size))) 9809 ret = -EFAULT; 9810 } 9811 goto err_free; 9812 } 9813 9814 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 9815 ret = -EFAULT; 9816 goto err_free; 9817 } 9818 9819 /* check insn_off */ 9820 ret = -EINVAL; 9821 if (i == 0) { 9822 if (krecord[i].insn_off) { 9823 verbose(env, 9824 "nonzero insn_off %u for the first func info record", 9825 krecord[i].insn_off); 9826 goto err_free; 9827 } 9828 } else if (krecord[i].insn_off <= prev_offset) { 9829 verbose(env, 9830 "same or smaller insn offset (%u) than previous func info record (%u)", 9831 krecord[i].insn_off, prev_offset); 9832 goto err_free; 9833 } 9834 9835 if (env->subprog_info[i].start != krecord[i].insn_off) { 9836 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 9837 goto err_free; 9838 } 9839 9840 /* check type_id */ 9841 type = btf_type_by_id(btf, krecord[i].type_id); 9842 if (!type || !btf_type_is_func(type)) { 9843 verbose(env, "invalid type id %d in func info", 9844 krecord[i].type_id); 9845 goto err_free; 9846 } 9847 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 9848 9849 func_proto = btf_type_by_id(btf, type->type); 9850 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 9851 /* btf_func_check() already verified it during BTF load */ 9852 goto err_free; 9853 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 9854 scalar_return = 9855 btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); 9856 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 9857 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 9858 goto err_free; 9859 } 9860 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 9861 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 9862 goto err_free; 9863 } 9864 9865 prev_offset = krecord[i].insn_off; 9866 bpfptr_add(&urecord, urec_size); 9867 } 9868 9869 prog->aux->func_info = krecord; 9870 prog->aux->func_info_cnt = nfuncs; 9871 prog->aux->func_info_aux = info_aux; 9872 return 0; 9873 9874 err_free: 9875 kvfree(krecord); 9876 kfree(info_aux); 9877 return ret; 9878 } 9879 9880 static void adjust_btf_func(struct bpf_verifier_env *env) 9881 { 9882 struct bpf_prog_aux *aux = env->prog->aux; 9883 int i; 9884 9885 if (!aux->func_info) 9886 return; 9887 9888 for (i = 0; i < env->subprog_cnt; i++) 9889 aux->func_info[i].insn_off = env->subprog_info[i].start; 9890 } 9891 9892 #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ 9893 sizeof(((struct bpf_line_info *)(0))->line_col)) 9894 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 9895 9896 static int check_btf_line(struct bpf_verifier_env *env, 9897 const union bpf_attr *attr, 9898 bpfptr_t uattr) 9899 { 9900 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 9901 struct bpf_subprog_info *sub; 9902 struct bpf_line_info *linfo; 9903 struct bpf_prog *prog; 9904 const struct btf *btf; 9905 bpfptr_t ulinfo; 9906 int err; 9907 9908 nr_linfo = attr->line_info_cnt; 9909 if (!nr_linfo) 9910 return 0; 9911 9912 rec_size = attr->line_info_rec_size; 9913 if (rec_size < MIN_BPF_LINEINFO_SIZE || 9914 rec_size > MAX_LINEINFO_REC_SIZE || 9915 rec_size & (sizeof(u32) - 1)) 9916 return -EINVAL; 9917 9918 /* Need to zero it in case the userspace may 9919 * pass in a smaller bpf_line_info object. 9920 */ 9921 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 9922 GFP_KERNEL | __GFP_NOWARN); 9923 if (!linfo) 9924 return -ENOMEM; 9925 9926 prog = env->prog; 9927 btf = prog->aux->btf; 9928 9929 s = 0; 9930 sub = env->subprog_info; 9931 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 9932 expected_size = sizeof(struct bpf_line_info); 9933 ncopy = min_t(u32, expected_size, rec_size); 9934 for (i = 0; i < nr_linfo; i++) { 9935 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 9936 if (err) { 9937 if (err == -E2BIG) { 9938 verbose(env, "nonzero tailing record in line_info"); 9939 if (copy_to_bpfptr_offset(uattr, 9940 offsetof(union bpf_attr, line_info_rec_size), 9941 &expected_size, sizeof(expected_size))) 9942 err = -EFAULT; 9943 } 9944 goto err_free; 9945 } 9946 9947 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 9948 err = -EFAULT; 9949 goto err_free; 9950 } 9951 9952 /* 9953 * Check insn_off to ensure 9954 * 1) strictly increasing AND 9955 * 2) bounded by prog->len 9956 * 9957 * The linfo[0].insn_off == 0 check logically falls into 9958 * the later "missing bpf_line_info for func..." case 9959 * because the first linfo[0].insn_off must be the 9960 * first sub also and the first sub must have 9961 * subprog_info[0].start == 0. 9962 */ 9963 if ((i && linfo[i].insn_off <= prev_offset) || 9964 linfo[i].insn_off >= prog->len) { 9965 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 9966 i, linfo[i].insn_off, prev_offset, 9967 prog->len); 9968 err = -EINVAL; 9969 goto err_free; 9970 } 9971 9972 if (!prog->insnsi[linfo[i].insn_off].code) { 9973 verbose(env, 9974 "Invalid insn code at line_info[%u].insn_off\n", 9975 i); 9976 err = -EINVAL; 9977 goto err_free; 9978 } 9979 9980 if (!btf_name_by_offset(btf, linfo[i].line_off) || 9981 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 9982 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 9983 err = -EINVAL; 9984 goto err_free; 9985 } 9986 9987 if (s != env->subprog_cnt) { 9988 if (linfo[i].insn_off == sub[s].start) { 9989 sub[s].linfo_idx = i; 9990 s++; 9991 } else if (sub[s].start < linfo[i].insn_off) { 9992 verbose(env, "missing bpf_line_info for func#%u\n", s); 9993 err = -EINVAL; 9994 goto err_free; 9995 } 9996 } 9997 9998 prev_offset = linfo[i].insn_off; 9999 bpfptr_add(&ulinfo, rec_size); 10000 } 10001 10002 if (s != env->subprog_cnt) { 10003 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 10004 env->subprog_cnt - s, s); 10005 err = -EINVAL; 10006 goto err_free; 10007 } 10008 10009 prog->aux->linfo = linfo; 10010 prog->aux->nr_linfo = nr_linfo; 10011 10012 return 0; 10013 10014 err_free: 10015 kvfree(linfo); 10016 return err; 10017 } 10018 10019 static int check_btf_info(struct bpf_verifier_env *env, 10020 const union bpf_attr *attr, 10021 bpfptr_t uattr) 10022 { 10023 struct btf *btf; 10024 int err; 10025 10026 if (!attr->func_info_cnt && !attr->line_info_cnt) { 10027 if (check_abnormal_return(env)) 10028 return -EINVAL; 10029 return 0; 10030 } 10031 10032 btf = btf_get_by_fd(attr->prog_btf_fd); 10033 if (IS_ERR(btf)) 10034 return PTR_ERR(btf); 10035 if (btf_is_kernel(btf)) { 10036 btf_put(btf); 10037 return -EACCES; 10038 } 10039 env->prog->aux->btf = btf; 10040 10041 err = check_btf_func(env, attr, uattr); 10042 if (err) 10043 return err; 10044 10045 err = check_btf_line(env, attr, uattr); 10046 if (err) 10047 return err; 10048 10049 return 0; 10050 } 10051 10052 /* check %cur's range satisfies %old's */ 10053 static bool range_within(struct bpf_reg_state *old, 10054 struct bpf_reg_state *cur) 10055 { 10056 return old->umin_value <= cur->umin_value && 10057 old->umax_value >= cur->umax_value && 10058 old->smin_value <= cur->smin_value && 10059 old->smax_value >= cur->smax_value && 10060 old->u32_min_value <= cur->u32_min_value && 10061 old->u32_max_value >= cur->u32_max_value && 10062 old->s32_min_value <= cur->s32_min_value && 10063 old->s32_max_value >= cur->s32_max_value; 10064 } 10065 10066 /* If in the old state two registers had the same id, then they need to have 10067 * the same id in the new state as well. But that id could be different from 10068 * the old state, so we need to track the mapping from old to new ids. 10069 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 10070 * regs with old id 5 must also have new id 9 for the new state to be safe. But 10071 * regs with a different old id could still have new id 9, we don't care about 10072 * that. 10073 * So we look through our idmap to see if this old id has been seen before. If 10074 * so, we require the new id to match; otherwise, we add the id pair to the map. 10075 */ 10076 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 10077 { 10078 unsigned int i; 10079 10080 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 10081 if (!idmap[i].old) { 10082 /* Reached an empty slot; haven't seen this id before */ 10083 idmap[i].old = old_id; 10084 idmap[i].cur = cur_id; 10085 return true; 10086 } 10087 if (idmap[i].old == old_id) 10088 return idmap[i].cur == cur_id; 10089 } 10090 /* We ran out of idmap slots, which should be impossible */ 10091 WARN_ON_ONCE(1); 10092 return false; 10093 } 10094 10095 static void clean_func_state(struct bpf_verifier_env *env, 10096 struct bpf_func_state *st) 10097 { 10098 enum bpf_reg_liveness live; 10099 int i, j; 10100 10101 for (i = 0; i < BPF_REG_FP; i++) { 10102 live = st->regs[i].live; 10103 /* liveness must not touch this register anymore */ 10104 st->regs[i].live |= REG_LIVE_DONE; 10105 if (!(live & REG_LIVE_READ)) 10106 /* since the register is unused, clear its state 10107 * to make further comparison simpler 10108 */ 10109 __mark_reg_not_init(env, &st->regs[i]); 10110 } 10111 10112 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 10113 live = st->stack[i].spilled_ptr.live; 10114 /* liveness must not touch this stack slot anymore */ 10115 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 10116 if (!(live & REG_LIVE_READ)) { 10117 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 10118 for (j = 0; j < BPF_REG_SIZE; j++) 10119 st->stack[i].slot_type[j] = STACK_INVALID; 10120 } 10121 } 10122 } 10123 10124 static void clean_verifier_state(struct bpf_verifier_env *env, 10125 struct bpf_verifier_state *st) 10126 { 10127 int i; 10128 10129 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 10130 /* all regs in this state in all frames were already marked */ 10131 return; 10132 10133 for (i = 0; i <= st->curframe; i++) 10134 clean_func_state(env, st->frame[i]); 10135 } 10136 10137 /* the parentage chains form a tree. 10138 * the verifier states are added to state lists at given insn and 10139 * pushed into state stack for future exploration. 10140 * when the verifier reaches bpf_exit insn some of the verifer states 10141 * stored in the state lists have their final liveness state already, 10142 * but a lot of states will get revised from liveness point of view when 10143 * the verifier explores other branches. 10144 * Example: 10145 * 1: r0 = 1 10146 * 2: if r1 == 100 goto pc+1 10147 * 3: r0 = 2 10148 * 4: exit 10149 * when the verifier reaches exit insn the register r0 in the state list of 10150 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 10151 * of insn 2 and goes exploring further. At the insn 4 it will walk the 10152 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 10153 * 10154 * Since the verifier pushes the branch states as it sees them while exploring 10155 * the program the condition of walking the branch instruction for the second 10156 * time means that all states below this branch were already explored and 10157 * their final liveness marks are already propagated. 10158 * Hence when the verifier completes the search of state list in is_state_visited() 10159 * we can call this clean_live_states() function to mark all liveness states 10160 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 10161 * will not be used. 10162 * This function also clears the registers and stack for states that !READ 10163 * to simplify state merging. 10164 * 10165 * Important note here that walking the same branch instruction in the callee 10166 * doesn't meant that the states are DONE. The verifier has to compare 10167 * the callsites 10168 */ 10169 static void clean_live_states(struct bpf_verifier_env *env, int insn, 10170 struct bpf_verifier_state *cur) 10171 { 10172 struct bpf_verifier_state_list *sl; 10173 int i; 10174 10175 sl = *explored_state(env, insn); 10176 while (sl) { 10177 if (sl->state.branches) 10178 goto next; 10179 if (sl->state.insn_idx != insn || 10180 sl->state.curframe != cur->curframe) 10181 goto next; 10182 for (i = 0; i <= cur->curframe; i++) 10183 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 10184 goto next; 10185 clean_verifier_state(env, &sl->state); 10186 next: 10187 sl = sl->next; 10188 } 10189 } 10190 10191 /* Returns true if (rold safe implies rcur safe) */ 10192 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 10193 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) 10194 { 10195 bool equal; 10196 10197 if (!(rold->live & REG_LIVE_READ)) 10198 /* explored state didn't use this */ 10199 return true; 10200 10201 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 10202 10203 if (rold->type == PTR_TO_STACK) 10204 /* two stack pointers are equal only if they're pointing to 10205 * the same stack frame, since fp-8 in foo != fp-8 in bar 10206 */ 10207 return equal && rold->frameno == rcur->frameno; 10208 10209 if (equal) 10210 return true; 10211 10212 if (rold->type == NOT_INIT) 10213 /* explored state can't have used this */ 10214 return true; 10215 if (rcur->type == NOT_INIT) 10216 return false; 10217 switch (rold->type) { 10218 case SCALAR_VALUE: 10219 if (env->explore_alu_limits) 10220 return false; 10221 if (rcur->type == SCALAR_VALUE) { 10222 if (!rold->precise && !rcur->precise) 10223 return true; 10224 /* new val must satisfy old val knowledge */ 10225 return range_within(rold, rcur) && 10226 tnum_in(rold->var_off, rcur->var_off); 10227 } else { 10228 /* We're trying to use a pointer in place of a scalar. 10229 * Even if the scalar was unbounded, this could lead to 10230 * pointer leaks because scalars are allowed to leak 10231 * while pointers are not. We could make this safe in 10232 * special cases if root is calling us, but it's 10233 * probably not worth the hassle. 10234 */ 10235 return false; 10236 } 10237 case PTR_TO_MAP_KEY: 10238 case PTR_TO_MAP_VALUE: 10239 /* If the new min/max/var_off satisfy the old ones and 10240 * everything else matches, we are OK. 10241 * 'id' is not compared, since it's only used for maps with 10242 * bpf_spin_lock inside map element and in such cases if 10243 * the rest of the prog is valid for one map element then 10244 * it's valid for all map elements regardless of the key 10245 * used in bpf_map_lookup() 10246 */ 10247 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 10248 range_within(rold, rcur) && 10249 tnum_in(rold->var_off, rcur->var_off); 10250 case PTR_TO_MAP_VALUE_OR_NULL: 10251 /* a PTR_TO_MAP_VALUE could be safe to use as a 10252 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 10253 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 10254 * checked, doing so could have affected others with the same 10255 * id, and we can't check for that because we lost the id when 10256 * we converted to a PTR_TO_MAP_VALUE. 10257 */ 10258 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) 10259 return false; 10260 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 10261 return false; 10262 /* Check our ids match any regs they're supposed to */ 10263 return check_ids(rold->id, rcur->id, idmap); 10264 case PTR_TO_PACKET_META: 10265 case PTR_TO_PACKET: 10266 if (rcur->type != rold->type) 10267 return false; 10268 /* We must have at least as much range as the old ptr 10269 * did, so that any accesses which were safe before are 10270 * still safe. This is true even if old range < old off, 10271 * since someone could have accessed through (ptr - k), or 10272 * even done ptr -= k in a register, to get a safe access. 10273 */ 10274 if (rold->range > rcur->range) 10275 return false; 10276 /* If the offsets don't match, we can't trust our alignment; 10277 * nor can we be sure that we won't fall out of range. 10278 */ 10279 if (rold->off != rcur->off) 10280 return false; 10281 /* id relations must be preserved */ 10282 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 10283 return false; 10284 /* new val must satisfy old val knowledge */ 10285 return range_within(rold, rcur) && 10286 tnum_in(rold->var_off, rcur->var_off); 10287 case PTR_TO_CTX: 10288 case CONST_PTR_TO_MAP: 10289 case PTR_TO_PACKET_END: 10290 case PTR_TO_FLOW_KEYS: 10291 case PTR_TO_SOCKET: 10292 case PTR_TO_SOCKET_OR_NULL: 10293 case PTR_TO_SOCK_COMMON: 10294 case PTR_TO_SOCK_COMMON_OR_NULL: 10295 case PTR_TO_TCP_SOCK: 10296 case PTR_TO_TCP_SOCK_OR_NULL: 10297 case PTR_TO_XDP_SOCK: 10298 /* Only valid matches are exact, which memcmp() above 10299 * would have accepted 10300 */ 10301 default: 10302 /* Don't know what's going on, just say it's not safe */ 10303 return false; 10304 } 10305 10306 /* Shouldn't get here; if we do, say it's not safe */ 10307 WARN_ON_ONCE(1); 10308 return false; 10309 } 10310 10311 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 10312 struct bpf_func_state *cur, struct bpf_id_pair *idmap) 10313 { 10314 int i, spi; 10315 10316 /* walk slots of the explored stack and ignore any additional 10317 * slots in the current stack, since explored(safe) state 10318 * didn't use them 10319 */ 10320 for (i = 0; i < old->allocated_stack; i++) { 10321 spi = i / BPF_REG_SIZE; 10322 10323 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 10324 i += BPF_REG_SIZE - 1; 10325 /* explored state didn't use this */ 10326 continue; 10327 } 10328 10329 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 10330 continue; 10331 10332 /* explored stack has more populated slots than current stack 10333 * and these slots were used 10334 */ 10335 if (i >= cur->allocated_stack) 10336 return false; 10337 10338 /* if old state was safe with misc data in the stack 10339 * it will be safe with zero-initialized stack. 10340 * The opposite is not true 10341 */ 10342 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 10343 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 10344 continue; 10345 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 10346 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 10347 /* Ex: old explored (safe) state has STACK_SPILL in 10348 * this stack slot, but current has STACK_MISC -> 10349 * this verifier states are not equivalent, 10350 * return false to continue verification of this path 10351 */ 10352 return false; 10353 if (i % BPF_REG_SIZE) 10354 continue; 10355 if (old->stack[spi].slot_type[0] != STACK_SPILL) 10356 continue; 10357 if (!regsafe(env, &old->stack[spi].spilled_ptr, 10358 &cur->stack[spi].spilled_ptr, idmap)) 10359 /* when explored and current stack slot are both storing 10360 * spilled registers, check that stored pointers types 10361 * are the same as well. 10362 * Ex: explored safe path could have stored 10363 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 10364 * but current path has stored: 10365 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 10366 * such verifier states are not equivalent. 10367 * return false to continue verification of this path 10368 */ 10369 return false; 10370 } 10371 return true; 10372 } 10373 10374 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 10375 { 10376 if (old->acquired_refs != cur->acquired_refs) 10377 return false; 10378 return !memcmp(old->refs, cur->refs, 10379 sizeof(*old->refs) * old->acquired_refs); 10380 } 10381 10382 /* compare two verifier states 10383 * 10384 * all states stored in state_list are known to be valid, since 10385 * verifier reached 'bpf_exit' instruction through them 10386 * 10387 * this function is called when verifier exploring different branches of 10388 * execution popped from the state stack. If it sees an old state that has 10389 * more strict register state and more strict stack state then this execution 10390 * branch doesn't need to be explored further, since verifier already 10391 * concluded that more strict state leads to valid finish. 10392 * 10393 * Therefore two states are equivalent if register state is more conservative 10394 * and explored stack state is more conservative than the current one. 10395 * Example: 10396 * explored current 10397 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 10398 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 10399 * 10400 * In other words if current stack state (one being explored) has more 10401 * valid slots than old one that already passed validation, it means 10402 * the verifier can stop exploring and conclude that current state is valid too 10403 * 10404 * Similarly with registers. If explored state has register type as invalid 10405 * whereas register type in current state is meaningful, it means that 10406 * the current state will reach 'bpf_exit' instruction safely 10407 */ 10408 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 10409 struct bpf_func_state *cur) 10410 { 10411 int i; 10412 10413 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 10414 for (i = 0; i < MAX_BPF_REG; i++) 10415 if (!regsafe(env, &old->regs[i], &cur->regs[i], 10416 env->idmap_scratch)) 10417 return false; 10418 10419 if (!stacksafe(env, old, cur, env->idmap_scratch)) 10420 return false; 10421 10422 if (!refsafe(old, cur)) 10423 return false; 10424 10425 return true; 10426 } 10427 10428 static bool states_equal(struct bpf_verifier_env *env, 10429 struct bpf_verifier_state *old, 10430 struct bpf_verifier_state *cur) 10431 { 10432 int i; 10433 10434 if (old->curframe != cur->curframe) 10435 return false; 10436 10437 /* Verification state from speculative execution simulation 10438 * must never prune a non-speculative execution one. 10439 */ 10440 if (old->speculative && !cur->speculative) 10441 return false; 10442 10443 if (old->active_spin_lock != cur->active_spin_lock) 10444 return false; 10445 10446 /* for states to be equal callsites have to be the same 10447 * and all frame states need to be equivalent 10448 */ 10449 for (i = 0; i <= old->curframe; i++) { 10450 if (old->frame[i]->callsite != cur->frame[i]->callsite) 10451 return false; 10452 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 10453 return false; 10454 } 10455 return true; 10456 } 10457 10458 /* Return 0 if no propagation happened. Return negative error code if error 10459 * happened. Otherwise, return the propagated bit. 10460 */ 10461 static int propagate_liveness_reg(struct bpf_verifier_env *env, 10462 struct bpf_reg_state *reg, 10463 struct bpf_reg_state *parent_reg) 10464 { 10465 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 10466 u8 flag = reg->live & REG_LIVE_READ; 10467 int err; 10468 10469 /* When comes here, read flags of PARENT_REG or REG could be any of 10470 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 10471 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 10472 */ 10473 if (parent_flag == REG_LIVE_READ64 || 10474 /* Or if there is no read flag from REG. */ 10475 !flag || 10476 /* Or if the read flag from REG is the same as PARENT_REG. */ 10477 parent_flag == flag) 10478 return 0; 10479 10480 err = mark_reg_read(env, reg, parent_reg, flag); 10481 if (err) 10482 return err; 10483 10484 return flag; 10485 } 10486 10487 /* A write screens off any subsequent reads; but write marks come from the 10488 * straight-line code between a state and its parent. When we arrive at an 10489 * equivalent state (jump target or such) we didn't arrive by the straight-line 10490 * code, so read marks in the state must propagate to the parent regardless 10491 * of the state's write marks. That's what 'parent == state->parent' comparison 10492 * in mark_reg_read() is for. 10493 */ 10494 static int propagate_liveness(struct bpf_verifier_env *env, 10495 const struct bpf_verifier_state *vstate, 10496 struct bpf_verifier_state *vparent) 10497 { 10498 struct bpf_reg_state *state_reg, *parent_reg; 10499 struct bpf_func_state *state, *parent; 10500 int i, frame, err = 0; 10501 10502 if (vparent->curframe != vstate->curframe) { 10503 WARN(1, "propagate_live: parent frame %d current frame %d\n", 10504 vparent->curframe, vstate->curframe); 10505 return -EFAULT; 10506 } 10507 /* Propagate read liveness of registers... */ 10508 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 10509 for (frame = 0; frame <= vstate->curframe; frame++) { 10510 parent = vparent->frame[frame]; 10511 state = vstate->frame[frame]; 10512 parent_reg = parent->regs; 10513 state_reg = state->regs; 10514 /* We don't need to worry about FP liveness, it's read-only */ 10515 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 10516 err = propagate_liveness_reg(env, &state_reg[i], 10517 &parent_reg[i]); 10518 if (err < 0) 10519 return err; 10520 if (err == REG_LIVE_READ64) 10521 mark_insn_zext(env, &parent_reg[i]); 10522 } 10523 10524 /* Propagate stack slots. */ 10525 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 10526 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 10527 parent_reg = &parent->stack[i].spilled_ptr; 10528 state_reg = &state->stack[i].spilled_ptr; 10529 err = propagate_liveness_reg(env, state_reg, 10530 parent_reg); 10531 if (err < 0) 10532 return err; 10533 } 10534 } 10535 return 0; 10536 } 10537 10538 /* find precise scalars in the previous equivalent state and 10539 * propagate them into the current state 10540 */ 10541 static int propagate_precision(struct bpf_verifier_env *env, 10542 const struct bpf_verifier_state *old) 10543 { 10544 struct bpf_reg_state *state_reg; 10545 struct bpf_func_state *state; 10546 int i, err = 0; 10547 10548 state = old->frame[old->curframe]; 10549 state_reg = state->regs; 10550 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 10551 if (state_reg->type != SCALAR_VALUE || 10552 !state_reg->precise) 10553 continue; 10554 if (env->log.level & BPF_LOG_LEVEL2) 10555 verbose(env, "propagating r%d\n", i); 10556 err = mark_chain_precision(env, i); 10557 if (err < 0) 10558 return err; 10559 } 10560 10561 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 10562 if (state->stack[i].slot_type[0] != STACK_SPILL) 10563 continue; 10564 state_reg = &state->stack[i].spilled_ptr; 10565 if (state_reg->type != SCALAR_VALUE || 10566 !state_reg->precise) 10567 continue; 10568 if (env->log.level & BPF_LOG_LEVEL2) 10569 verbose(env, "propagating fp%d\n", 10570 (-i - 1) * BPF_REG_SIZE); 10571 err = mark_chain_precision_stack(env, i); 10572 if (err < 0) 10573 return err; 10574 } 10575 return 0; 10576 } 10577 10578 static bool states_maybe_looping(struct bpf_verifier_state *old, 10579 struct bpf_verifier_state *cur) 10580 { 10581 struct bpf_func_state *fold, *fcur; 10582 int i, fr = cur->curframe; 10583 10584 if (old->curframe != fr) 10585 return false; 10586 10587 fold = old->frame[fr]; 10588 fcur = cur->frame[fr]; 10589 for (i = 0; i < MAX_BPF_REG; i++) 10590 if (memcmp(&fold->regs[i], &fcur->regs[i], 10591 offsetof(struct bpf_reg_state, parent))) 10592 return false; 10593 return true; 10594 } 10595 10596 10597 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 10598 { 10599 struct bpf_verifier_state_list *new_sl; 10600 struct bpf_verifier_state_list *sl, **pprev; 10601 struct bpf_verifier_state *cur = env->cur_state, *new; 10602 int i, j, err, states_cnt = 0; 10603 bool add_new_state = env->test_state_freq ? true : false; 10604 10605 cur->last_insn_idx = env->prev_insn_idx; 10606 if (!env->insn_aux_data[insn_idx].prune_point) 10607 /* this 'insn_idx' instruction wasn't marked, so we will not 10608 * be doing state search here 10609 */ 10610 return 0; 10611 10612 /* bpf progs typically have pruning point every 4 instructions 10613 * http://vger.kernel.org/bpfconf2019.html#session-1 10614 * Do not add new state for future pruning if the verifier hasn't seen 10615 * at least 2 jumps and at least 8 instructions. 10616 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 10617 * In tests that amounts to up to 50% reduction into total verifier 10618 * memory consumption and 20% verifier time speedup. 10619 */ 10620 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 10621 env->insn_processed - env->prev_insn_processed >= 8) 10622 add_new_state = true; 10623 10624 pprev = explored_state(env, insn_idx); 10625 sl = *pprev; 10626 10627 clean_live_states(env, insn_idx, cur); 10628 10629 while (sl) { 10630 states_cnt++; 10631 if (sl->state.insn_idx != insn_idx) 10632 goto next; 10633 10634 if (sl->state.branches) { 10635 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 10636 10637 if (frame->in_async_callback_fn && 10638 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 10639 /* Different async_entry_cnt means that the verifier is 10640 * processing another entry into async callback. 10641 * Seeing the same state is not an indication of infinite 10642 * loop or infinite recursion. 10643 * But finding the same state doesn't mean that it's safe 10644 * to stop processing the current state. The previous state 10645 * hasn't yet reached bpf_exit, since state.branches > 0. 10646 * Checking in_async_callback_fn alone is not enough either. 10647 * Since the verifier still needs to catch infinite loops 10648 * inside async callbacks. 10649 */ 10650 } else if (states_maybe_looping(&sl->state, cur) && 10651 states_equal(env, &sl->state, cur)) { 10652 verbose_linfo(env, insn_idx, "; "); 10653 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 10654 return -EINVAL; 10655 } 10656 /* if the verifier is processing a loop, avoid adding new state 10657 * too often, since different loop iterations have distinct 10658 * states and may not help future pruning. 10659 * This threshold shouldn't be too low to make sure that 10660 * a loop with large bound will be rejected quickly. 10661 * The most abusive loop will be: 10662 * r1 += 1 10663 * if r1 < 1000000 goto pc-2 10664 * 1M insn_procssed limit / 100 == 10k peak states. 10665 * This threshold shouldn't be too high either, since states 10666 * at the end of the loop are likely to be useful in pruning. 10667 */ 10668 if (env->jmps_processed - env->prev_jmps_processed < 20 && 10669 env->insn_processed - env->prev_insn_processed < 100) 10670 add_new_state = false; 10671 goto miss; 10672 } 10673 if (states_equal(env, &sl->state, cur)) { 10674 sl->hit_cnt++; 10675 /* reached equivalent register/stack state, 10676 * prune the search. 10677 * Registers read by the continuation are read by us. 10678 * If we have any write marks in env->cur_state, they 10679 * will prevent corresponding reads in the continuation 10680 * from reaching our parent (an explored_state). Our 10681 * own state will get the read marks recorded, but 10682 * they'll be immediately forgotten as we're pruning 10683 * this state and will pop a new one. 10684 */ 10685 err = propagate_liveness(env, &sl->state, cur); 10686 10687 /* if previous state reached the exit with precision and 10688 * current state is equivalent to it (except precsion marks) 10689 * the precision needs to be propagated back in 10690 * the current state. 10691 */ 10692 err = err ? : push_jmp_history(env, cur); 10693 err = err ? : propagate_precision(env, &sl->state); 10694 if (err) 10695 return err; 10696 return 1; 10697 } 10698 miss: 10699 /* when new state is not going to be added do not increase miss count. 10700 * Otherwise several loop iterations will remove the state 10701 * recorded earlier. The goal of these heuristics is to have 10702 * states from some iterations of the loop (some in the beginning 10703 * and some at the end) to help pruning. 10704 */ 10705 if (add_new_state) 10706 sl->miss_cnt++; 10707 /* heuristic to determine whether this state is beneficial 10708 * to keep checking from state equivalence point of view. 10709 * Higher numbers increase max_states_per_insn and verification time, 10710 * but do not meaningfully decrease insn_processed. 10711 */ 10712 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 10713 /* the state is unlikely to be useful. Remove it to 10714 * speed up verification 10715 */ 10716 *pprev = sl->next; 10717 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 10718 u32 br = sl->state.branches; 10719 10720 WARN_ONCE(br, 10721 "BUG live_done but branches_to_explore %d\n", 10722 br); 10723 free_verifier_state(&sl->state, false); 10724 kfree(sl); 10725 env->peak_states--; 10726 } else { 10727 /* cannot free this state, since parentage chain may 10728 * walk it later. Add it for free_list instead to 10729 * be freed at the end of verification 10730 */ 10731 sl->next = env->free_list; 10732 env->free_list = sl; 10733 } 10734 sl = *pprev; 10735 continue; 10736 } 10737 next: 10738 pprev = &sl->next; 10739 sl = *pprev; 10740 } 10741 10742 if (env->max_states_per_insn < states_cnt) 10743 env->max_states_per_insn = states_cnt; 10744 10745 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 10746 return push_jmp_history(env, cur); 10747 10748 if (!add_new_state) 10749 return push_jmp_history(env, cur); 10750 10751 /* There were no equivalent states, remember the current one. 10752 * Technically the current state is not proven to be safe yet, 10753 * but it will either reach outer most bpf_exit (which means it's safe) 10754 * or it will be rejected. When there are no loops the verifier won't be 10755 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 10756 * again on the way to bpf_exit. 10757 * When looping the sl->state.branches will be > 0 and this state 10758 * will not be considered for equivalence until branches == 0. 10759 */ 10760 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 10761 if (!new_sl) 10762 return -ENOMEM; 10763 env->total_states++; 10764 env->peak_states++; 10765 env->prev_jmps_processed = env->jmps_processed; 10766 env->prev_insn_processed = env->insn_processed; 10767 10768 /* add new state to the head of linked list */ 10769 new = &new_sl->state; 10770 err = copy_verifier_state(new, cur); 10771 if (err) { 10772 free_verifier_state(new, false); 10773 kfree(new_sl); 10774 return err; 10775 } 10776 new->insn_idx = insn_idx; 10777 WARN_ONCE(new->branches != 1, 10778 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 10779 10780 cur->parent = new; 10781 cur->first_insn_idx = insn_idx; 10782 clear_jmp_history(cur); 10783 new_sl->next = *explored_state(env, insn_idx); 10784 *explored_state(env, insn_idx) = new_sl; 10785 /* connect new state to parentage chain. Current frame needs all 10786 * registers connected. Only r6 - r9 of the callers are alive (pushed 10787 * to the stack implicitly by JITs) so in callers' frames connect just 10788 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 10789 * the state of the call instruction (with WRITTEN set), and r0 comes 10790 * from callee with its full parentage chain, anyway. 10791 */ 10792 /* clear write marks in current state: the writes we did are not writes 10793 * our child did, so they don't screen off its reads from us. 10794 * (There are no read marks in current state, because reads always mark 10795 * their parent and current state never has children yet. Only 10796 * explored_states can get read marks.) 10797 */ 10798 for (j = 0; j <= cur->curframe; j++) { 10799 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 10800 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 10801 for (i = 0; i < BPF_REG_FP; i++) 10802 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 10803 } 10804 10805 /* all stack frames are accessible from callee, clear them all */ 10806 for (j = 0; j <= cur->curframe; j++) { 10807 struct bpf_func_state *frame = cur->frame[j]; 10808 struct bpf_func_state *newframe = new->frame[j]; 10809 10810 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 10811 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 10812 frame->stack[i].spilled_ptr.parent = 10813 &newframe->stack[i].spilled_ptr; 10814 } 10815 } 10816 return 0; 10817 } 10818 10819 /* Return true if it's OK to have the same insn return a different type. */ 10820 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 10821 { 10822 switch (type) { 10823 case PTR_TO_CTX: 10824 case PTR_TO_SOCKET: 10825 case PTR_TO_SOCKET_OR_NULL: 10826 case PTR_TO_SOCK_COMMON: 10827 case PTR_TO_SOCK_COMMON_OR_NULL: 10828 case PTR_TO_TCP_SOCK: 10829 case PTR_TO_TCP_SOCK_OR_NULL: 10830 case PTR_TO_XDP_SOCK: 10831 case PTR_TO_BTF_ID: 10832 case PTR_TO_BTF_ID_OR_NULL: 10833 return false; 10834 default: 10835 return true; 10836 } 10837 } 10838 10839 /* If an instruction was previously used with particular pointer types, then we 10840 * need to be careful to avoid cases such as the below, where it may be ok 10841 * for one branch accessing the pointer, but not ok for the other branch: 10842 * 10843 * R1 = sock_ptr 10844 * goto X; 10845 * ... 10846 * R1 = some_other_valid_ptr; 10847 * goto X; 10848 * ... 10849 * R2 = *(u32 *)(R1 + 0); 10850 */ 10851 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 10852 { 10853 return src != prev && (!reg_type_mismatch_ok(src) || 10854 !reg_type_mismatch_ok(prev)); 10855 } 10856 10857 static int do_check(struct bpf_verifier_env *env) 10858 { 10859 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 10860 struct bpf_verifier_state *state = env->cur_state; 10861 struct bpf_insn *insns = env->prog->insnsi; 10862 struct bpf_reg_state *regs; 10863 int insn_cnt = env->prog->len; 10864 bool do_print_state = false; 10865 int prev_insn_idx = -1; 10866 10867 for (;;) { 10868 struct bpf_insn *insn; 10869 u8 class; 10870 int err; 10871 10872 env->prev_insn_idx = prev_insn_idx; 10873 if (env->insn_idx >= insn_cnt) { 10874 verbose(env, "invalid insn idx %d insn_cnt %d\n", 10875 env->insn_idx, insn_cnt); 10876 return -EFAULT; 10877 } 10878 10879 insn = &insns[env->insn_idx]; 10880 class = BPF_CLASS(insn->code); 10881 10882 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 10883 verbose(env, 10884 "BPF program is too large. Processed %d insn\n", 10885 env->insn_processed); 10886 return -E2BIG; 10887 } 10888 10889 err = is_state_visited(env, env->insn_idx); 10890 if (err < 0) 10891 return err; 10892 if (err == 1) { 10893 /* found equivalent state, can prune the search */ 10894 if (env->log.level & BPF_LOG_LEVEL) { 10895 if (do_print_state) 10896 verbose(env, "\nfrom %d to %d%s: safe\n", 10897 env->prev_insn_idx, env->insn_idx, 10898 env->cur_state->speculative ? 10899 " (speculative execution)" : ""); 10900 else 10901 verbose(env, "%d: safe\n", env->insn_idx); 10902 } 10903 goto process_bpf_exit; 10904 } 10905 10906 if (signal_pending(current)) 10907 return -EAGAIN; 10908 10909 if (need_resched()) 10910 cond_resched(); 10911 10912 if (env->log.level & BPF_LOG_LEVEL2 || 10913 (env->log.level & BPF_LOG_LEVEL && do_print_state)) { 10914 if (env->log.level & BPF_LOG_LEVEL2) 10915 verbose(env, "%d:", env->insn_idx); 10916 else 10917 verbose(env, "\nfrom %d to %d%s:", 10918 env->prev_insn_idx, env->insn_idx, 10919 env->cur_state->speculative ? 10920 " (speculative execution)" : ""); 10921 print_verifier_state(env, state->frame[state->curframe]); 10922 do_print_state = false; 10923 } 10924 10925 if (env->log.level & BPF_LOG_LEVEL) { 10926 const struct bpf_insn_cbs cbs = { 10927 .cb_call = disasm_kfunc_name, 10928 .cb_print = verbose, 10929 .private_data = env, 10930 }; 10931 10932 verbose_linfo(env, env->insn_idx, "; "); 10933 verbose(env, "%d: ", env->insn_idx); 10934 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 10935 } 10936 10937 if (bpf_prog_is_dev_bound(env->prog->aux)) { 10938 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 10939 env->prev_insn_idx); 10940 if (err) 10941 return err; 10942 } 10943 10944 regs = cur_regs(env); 10945 sanitize_mark_insn_seen(env); 10946 prev_insn_idx = env->insn_idx; 10947 10948 if (class == BPF_ALU || class == BPF_ALU64) { 10949 err = check_alu_op(env, insn); 10950 if (err) 10951 return err; 10952 10953 } else if (class == BPF_LDX) { 10954 enum bpf_reg_type *prev_src_type, src_reg_type; 10955 10956 /* check for reserved fields is already done */ 10957 10958 /* check src operand */ 10959 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10960 if (err) 10961 return err; 10962 10963 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 10964 if (err) 10965 return err; 10966 10967 src_reg_type = regs[insn->src_reg].type; 10968 10969 /* check that memory (src_reg + off) is readable, 10970 * the state of dst_reg will be updated by this func 10971 */ 10972 err = check_mem_access(env, env->insn_idx, insn->src_reg, 10973 insn->off, BPF_SIZE(insn->code), 10974 BPF_READ, insn->dst_reg, false); 10975 if (err) 10976 return err; 10977 10978 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 10979 10980 if (*prev_src_type == NOT_INIT) { 10981 /* saw a valid insn 10982 * dst_reg = *(u32 *)(src_reg + off) 10983 * save type to validate intersecting paths 10984 */ 10985 *prev_src_type = src_reg_type; 10986 10987 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 10988 /* ABuser program is trying to use the same insn 10989 * dst_reg = *(u32*) (src_reg + off) 10990 * with different pointer types: 10991 * src_reg == ctx in one branch and 10992 * src_reg == stack|map in some other branch. 10993 * Reject it. 10994 */ 10995 verbose(env, "same insn cannot be used with different pointers\n"); 10996 return -EINVAL; 10997 } 10998 10999 } else if (class == BPF_STX) { 11000 enum bpf_reg_type *prev_dst_type, dst_reg_type; 11001 11002 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 11003 err = check_atomic(env, env->insn_idx, insn); 11004 if (err) 11005 return err; 11006 env->insn_idx++; 11007 continue; 11008 } 11009 11010 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 11011 verbose(env, "BPF_STX uses reserved fields\n"); 11012 return -EINVAL; 11013 } 11014 11015 /* check src1 operand */ 11016 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11017 if (err) 11018 return err; 11019 /* check src2 operand */ 11020 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11021 if (err) 11022 return err; 11023 11024 dst_reg_type = regs[insn->dst_reg].type; 11025 11026 /* check that memory (dst_reg + off) is writeable */ 11027 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 11028 insn->off, BPF_SIZE(insn->code), 11029 BPF_WRITE, insn->src_reg, false); 11030 if (err) 11031 return err; 11032 11033 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 11034 11035 if (*prev_dst_type == NOT_INIT) { 11036 *prev_dst_type = dst_reg_type; 11037 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 11038 verbose(env, "same insn cannot be used with different pointers\n"); 11039 return -EINVAL; 11040 } 11041 11042 } else if (class == BPF_ST) { 11043 if (BPF_MODE(insn->code) != BPF_MEM || 11044 insn->src_reg != BPF_REG_0) { 11045 verbose(env, "BPF_ST uses reserved fields\n"); 11046 return -EINVAL; 11047 } 11048 /* check src operand */ 11049 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11050 if (err) 11051 return err; 11052 11053 if (is_ctx_reg(env, insn->dst_reg)) { 11054 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 11055 insn->dst_reg, 11056 reg_type_str[reg_state(env, insn->dst_reg)->type]); 11057 return -EACCES; 11058 } 11059 11060 /* check that memory (dst_reg + off) is writeable */ 11061 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 11062 insn->off, BPF_SIZE(insn->code), 11063 BPF_WRITE, -1, false); 11064 if (err) 11065 return err; 11066 11067 } else if (class == BPF_JMP || class == BPF_JMP32) { 11068 u8 opcode = BPF_OP(insn->code); 11069 11070 env->jmps_processed++; 11071 if (opcode == BPF_CALL) { 11072 if (BPF_SRC(insn->code) != BPF_K || 11073 insn->off != 0 || 11074 (insn->src_reg != BPF_REG_0 && 11075 insn->src_reg != BPF_PSEUDO_CALL && 11076 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 11077 insn->dst_reg != BPF_REG_0 || 11078 class == BPF_JMP32) { 11079 verbose(env, "BPF_CALL uses reserved fields\n"); 11080 return -EINVAL; 11081 } 11082 11083 if (env->cur_state->active_spin_lock && 11084 (insn->src_reg == BPF_PSEUDO_CALL || 11085 insn->imm != BPF_FUNC_spin_unlock)) { 11086 verbose(env, "function calls are not allowed while holding a lock\n"); 11087 return -EINVAL; 11088 } 11089 if (insn->src_reg == BPF_PSEUDO_CALL) 11090 err = check_func_call(env, insn, &env->insn_idx); 11091 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 11092 err = check_kfunc_call(env, insn); 11093 else 11094 err = check_helper_call(env, insn, &env->insn_idx); 11095 if (err) 11096 return err; 11097 } else if (opcode == BPF_JA) { 11098 if (BPF_SRC(insn->code) != BPF_K || 11099 insn->imm != 0 || 11100 insn->src_reg != BPF_REG_0 || 11101 insn->dst_reg != BPF_REG_0 || 11102 class == BPF_JMP32) { 11103 verbose(env, "BPF_JA uses reserved fields\n"); 11104 return -EINVAL; 11105 } 11106 11107 env->insn_idx += insn->off + 1; 11108 continue; 11109 11110 } else if (opcode == BPF_EXIT) { 11111 if (BPF_SRC(insn->code) != BPF_K || 11112 insn->imm != 0 || 11113 insn->src_reg != BPF_REG_0 || 11114 insn->dst_reg != BPF_REG_0 || 11115 class == BPF_JMP32) { 11116 verbose(env, "BPF_EXIT uses reserved fields\n"); 11117 return -EINVAL; 11118 } 11119 11120 if (env->cur_state->active_spin_lock) { 11121 verbose(env, "bpf_spin_unlock is missing\n"); 11122 return -EINVAL; 11123 } 11124 11125 if (state->curframe) { 11126 /* exit from nested function */ 11127 err = prepare_func_exit(env, &env->insn_idx); 11128 if (err) 11129 return err; 11130 do_print_state = true; 11131 continue; 11132 } 11133 11134 err = check_reference_leak(env); 11135 if (err) 11136 return err; 11137 11138 err = check_return_code(env); 11139 if (err) 11140 return err; 11141 process_bpf_exit: 11142 update_branch_counts(env, env->cur_state); 11143 err = pop_stack(env, &prev_insn_idx, 11144 &env->insn_idx, pop_log); 11145 if (err < 0) { 11146 if (err != -ENOENT) 11147 return err; 11148 break; 11149 } else { 11150 do_print_state = true; 11151 continue; 11152 } 11153 } else { 11154 err = check_cond_jmp_op(env, insn, &env->insn_idx); 11155 if (err) 11156 return err; 11157 } 11158 } else if (class == BPF_LD) { 11159 u8 mode = BPF_MODE(insn->code); 11160 11161 if (mode == BPF_ABS || mode == BPF_IND) { 11162 err = check_ld_abs(env, insn); 11163 if (err) 11164 return err; 11165 11166 } else if (mode == BPF_IMM) { 11167 err = check_ld_imm(env, insn); 11168 if (err) 11169 return err; 11170 11171 env->insn_idx++; 11172 sanitize_mark_insn_seen(env); 11173 } else { 11174 verbose(env, "invalid BPF_LD mode\n"); 11175 return -EINVAL; 11176 } 11177 } else { 11178 verbose(env, "unknown insn class %d\n", class); 11179 return -EINVAL; 11180 } 11181 11182 env->insn_idx++; 11183 } 11184 11185 return 0; 11186 } 11187 11188 static int find_btf_percpu_datasec(struct btf *btf) 11189 { 11190 const struct btf_type *t; 11191 const char *tname; 11192 int i, n; 11193 11194 /* 11195 * Both vmlinux and module each have their own ".data..percpu" 11196 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 11197 * types to look at only module's own BTF types. 11198 */ 11199 n = btf_nr_types(btf); 11200 if (btf_is_module(btf)) 11201 i = btf_nr_types(btf_vmlinux); 11202 else 11203 i = 1; 11204 11205 for(; i < n; i++) { 11206 t = btf_type_by_id(btf, i); 11207 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 11208 continue; 11209 11210 tname = btf_name_by_offset(btf, t->name_off); 11211 if (!strcmp(tname, ".data..percpu")) 11212 return i; 11213 } 11214 11215 return -ENOENT; 11216 } 11217 11218 /* replace pseudo btf_id with kernel symbol address */ 11219 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 11220 struct bpf_insn *insn, 11221 struct bpf_insn_aux_data *aux) 11222 { 11223 const struct btf_var_secinfo *vsi; 11224 const struct btf_type *datasec; 11225 struct btf_mod_pair *btf_mod; 11226 const struct btf_type *t; 11227 const char *sym_name; 11228 bool percpu = false; 11229 u32 type, id = insn->imm; 11230 struct btf *btf; 11231 s32 datasec_id; 11232 u64 addr; 11233 int i, btf_fd, err; 11234 11235 btf_fd = insn[1].imm; 11236 if (btf_fd) { 11237 btf = btf_get_by_fd(btf_fd); 11238 if (IS_ERR(btf)) { 11239 verbose(env, "invalid module BTF object FD specified.\n"); 11240 return -EINVAL; 11241 } 11242 } else { 11243 if (!btf_vmlinux) { 11244 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 11245 return -EINVAL; 11246 } 11247 btf = btf_vmlinux; 11248 btf_get(btf); 11249 } 11250 11251 t = btf_type_by_id(btf, id); 11252 if (!t) { 11253 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 11254 err = -ENOENT; 11255 goto err_put; 11256 } 11257 11258 if (!btf_type_is_var(t)) { 11259 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); 11260 err = -EINVAL; 11261 goto err_put; 11262 } 11263 11264 sym_name = btf_name_by_offset(btf, t->name_off); 11265 addr = kallsyms_lookup_name(sym_name); 11266 if (!addr) { 11267 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 11268 sym_name); 11269 err = -ENOENT; 11270 goto err_put; 11271 } 11272 11273 datasec_id = find_btf_percpu_datasec(btf); 11274 if (datasec_id > 0) { 11275 datasec = btf_type_by_id(btf, datasec_id); 11276 for_each_vsi(i, datasec, vsi) { 11277 if (vsi->type == id) { 11278 percpu = true; 11279 break; 11280 } 11281 } 11282 } 11283 11284 insn[0].imm = (u32)addr; 11285 insn[1].imm = addr >> 32; 11286 11287 type = t->type; 11288 t = btf_type_skip_modifiers(btf, type, NULL); 11289 if (percpu) { 11290 aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID; 11291 aux->btf_var.btf = btf; 11292 aux->btf_var.btf_id = type; 11293 } else if (!btf_type_is_struct(t)) { 11294 const struct btf_type *ret; 11295 const char *tname; 11296 u32 tsize; 11297 11298 /* resolve the type size of ksym. */ 11299 ret = btf_resolve_size(btf, t, &tsize); 11300 if (IS_ERR(ret)) { 11301 tname = btf_name_by_offset(btf, t->name_off); 11302 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 11303 tname, PTR_ERR(ret)); 11304 err = -EINVAL; 11305 goto err_put; 11306 } 11307 aux->btf_var.reg_type = PTR_TO_MEM; 11308 aux->btf_var.mem_size = tsize; 11309 } else { 11310 aux->btf_var.reg_type = PTR_TO_BTF_ID; 11311 aux->btf_var.btf = btf; 11312 aux->btf_var.btf_id = type; 11313 } 11314 11315 /* check whether we recorded this BTF (and maybe module) already */ 11316 for (i = 0; i < env->used_btf_cnt; i++) { 11317 if (env->used_btfs[i].btf == btf) { 11318 btf_put(btf); 11319 return 0; 11320 } 11321 } 11322 11323 if (env->used_btf_cnt >= MAX_USED_BTFS) { 11324 err = -E2BIG; 11325 goto err_put; 11326 } 11327 11328 btf_mod = &env->used_btfs[env->used_btf_cnt]; 11329 btf_mod->btf = btf; 11330 btf_mod->module = NULL; 11331 11332 /* if we reference variables from kernel module, bump its refcount */ 11333 if (btf_is_module(btf)) { 11334 btf_mod->module = btf_try_get_module(btf); 11335 if (!btf_mod->module) { 11336 err = -ENXIO; 11337 goto err_put; 11338 } 11339 } 11340 11341 env->used_btf_cnt++; 11342 11343 return 0; 11344 err_put: 11345 btf_put(btf); 11346 return err; 11347 } 11348 11349 static int check_map_prealloc(struct bpf_map *map) 11350 { 11351 return (map->map_type != BPF_MAP_TYPE_HASH && 11352 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 11353 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || 11354 !(map->map_flags & BPF_F_NO_PREALLOC); 11355 } 11356 11357 static bool is_tracing_prog_type(enum bpf_prog_type type) 11358 { 11359 switch (type) { 11360 case BPF_PROG_TYPE_KPROBE: 11361 case BPF_PROG_TYPE_TRACEPOINT: 11362 case BPF_PROG_TYPE_PERF_EVENT: 11363 case BPF_PROG_TYPE_RAW_TRACEPOINT: 11364 return true; 11365 default: 11366 return false; 11367 } 11368 } 11369 11370 static bool is_preallocated_map(struct bpf_map *map) 11371 { 11372 if (!check_map_prealloc(map)) 11373 return false; 11374 if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) 11375 return false; 11376 return true; 11377 } 11378 11379 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 11380 struct bpf_map *map, 11381 struct bpf_prog *prog) 11382 11383 { 11384 enum bpf_prog_type prog_type = resolve_prog_type(prog); 11385 /* 11386 * Validate that trace type programs use preallocated hash maps. 11387 * 11388 * For programs attached to PERF events this is mandatory as the 11389 * perf NMI can hit any arbitrary code sequence. 11390 * 11391 * All other trace types using preallocated hash maps are unsafe as 11392 * well because tracepoint or kprobes can be inside locked regions 11393 * of the memory allocator or at a place where a recursion into the 11394 * memory allocator would see inconsistent state. 11395 * 11396 * On RT enabled kernels run-time allocation of all trace type 11397 * programs is strictly prohibited due to lock type constraints. On 11398 * !RT kernels it is allowed for backwards compatibility reasons for 11399 * now, but warnings are emitted so developers are made aware of 11400 * the unsafety and can fix their programs before this is enforced. 11401 */ 11402 if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { 11403 if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { 11404 verbose(env, "perf_event programs can only use preallocated hash map\n"); 11405 return -EINVAL; 11406 } 11407 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 11408 verbose(env, "trace type programs can only use preallocated hash map\n"); 11409 return -EINVAL; 11410 } 11411 WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); 11412 verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); 11413 } 11414 11415 if (map_value_has_spin_lock(map)) { 11416 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 11417 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 11418 return -EINVAL; 11419 } 11420 11421 if (is_tracing_prog_type(prog_type)) { 11422 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 11423 return -EINVAL; 11424 } 11425 11426 if (prog->aux->sleepable) { 11427 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 11428 return -EINVAL; 11429 } 11430 } 11431 11432 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 11433 !bpf_offload_prog_map_match(prog, map)) { 11434 verbose(env, "offload device mismatch between prog and map\n"); 11435 return -EINVAL; 11436 } 11437 11438 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 11439 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 11440 return -EINVAL; 11441 } 11442 11443 if (prog->aux->sleepable) 11444 switch (map->map_type) { 11445 case BPF_MAP_TYPE_HASH: 11446 case BPF_MAP_TYPE_LRU_HASH: 11447 case BPF_MAP_TYPE_ARRAY: 11448 case BPF_MAP_TYPE_PERCPU_HASH: 11449 case BPF_MAP_TYPE_PERCPU_ARRAY: 11450 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 11451 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 11452 case BPF_MAP_TYPE_HASH_OF_MAPS: 11453 if (!is_preallocated_map(map)) { 11454 verbose(env, 11455 "Sleepable programs can only use preallocated maps\n"); 11456 return -EINVAL; 11457 } 11458 break; 11459 case BPF_MAP_TYPE_RINGBUF: 11460 break; 11461 default: 11462 verbose(env, 11463 "Sleepable programs can only use array, hash, and ringbuf maps\n"); 11464 return -EINVAL; 11465 } 11466 11467 return 0; 11468 } 11469 11470 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 11471 { 11472 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 11473 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 11474 } 11475 11476 /* find and rewrite pseudo imm in ld_imm64 instructions: 11477 * 11478 * 1. if it accesses map FD, replace it with actual map pointer. 11479 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 11480 * 11481 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 11482 */ 11483 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 11484 { 11485 struct bpf_insn *insn = env->prog->insnsi; 11486 int insn_cnt = env->prog->len; 11487 int i, j, err; 11488 11489 err = bpf_prog_calc_tag(env->prog); 11490 if (err) 11491 return err; 11492 11493 for (i = 0; i < insn_cnt; i++, insn++) { 11494 if (BPF_CLASS(insn->code) == BPF_LDX && 11495 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 11496 verbose(env, "BPF_LDX uses reserved fields\n"); 11497 return -EINVAL; 11498 } 11499 11500 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 11501 struct bpf_insn_aux_data *aux; 11502 struct bpf_map *map; 11503 struct fd f; 11504 u64 addr; 11505 u32 fd; 11506 11507 if (i == insn_cnt - 1 || insn[1].code != 0 || 11508 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 11509 insn[1].off != 0) { 11510 verbose(env, "invalid bpf_ld_imm64 insn\n"); 11511 return -EINVAL; 11512 } 11513 11514 if (insn[0].src_reg == 0) 11515 /* valid generic load 64-bit imm */ 11516 goto next_insn; 11517 11518 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 11519 aux = &env->insn_aux_data[i]; 11520 err = check_pseudo_btf_id(env, insn, aux); 11521 if (err) 11522 return err; 11523 goto next_insn; 11524 } 11525 11526 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 11527 aux = &env->insn_aux_data[i]; 11528 aux->ptr_type = PTR_TO_FUNC; 11529 goto next_insn; 11530 } 11531 11532 /* In final convert_pseudo_ld_imm64() step, this is 11533 * converted into regular 64-bit imm load insn. 11534 */ 11535 switch (insn[0].src_reg) { 11536 case BPF_PSEUDO_MAP_VALUE: 11537 case BPF_PSEUDO_MAP_IDX_VALUE: 11538 break; 11539 case BPF_PSEUDO_MAP_FD: 11540 case BPF_PSEUDO_MAP_IDX: 11541 if (insn[1].imm == 0) 11542 break; 11543 fallthrough; 11544 default: 11545 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 11546 return -EINVAL; 11547 } 11548 11549 switch (insn[0].src_reg) { 11550 case BPF_PSEUDO_MAP_IDX_VALUE: 11551 case BPF_PSEUDO_MAP_IDX: 11552 if (bpfptr_is_null(env->fd_array)) { 11553 verbose(env, "fd_idx without fd_array is invalid\n"); 11554 return -EPROTO; 11555 } 11556 if (copy_from_bpfptr_offset(&fd, env->fd_array, 11557 insn[0].imm * sizeof(fd), 11558 sizeof(fd))) 11559 return -EFAULT; 11560 break; 11561 default: 11562 fd = insn[0].imm; 11563 break; 11564 } 11565 11566 f = fdget(fd); 11567 map = __bpf_map_get(f); 11568 if (IS_ERR(map)) { 11569 verbose(env, "fd %d is not pointing to valid bpf_map\n", 11570 insn[0].imm); 11571 return PTR_ERR(map); 11572 } 11573 11574 err = check_map_prog_compatibility(env, map, env->prog); 11575 if (err) { 11576 fdput(f); 11577 return err; 11578 } 11579 11580 aux = &env->insn_aux_data[i]; 11581 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 11582 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 11583 addr = (unsigned long)map; 11584 } else { 11585 u32 off = insn[1].imm; 11586 11587 if (off >= BPF_MAX_VAR_OFF) { 11588 verbose(env, "direct value offset of %u is not allowed\n", off); 11589 fdput(f); 11590 return -EINVAL; 11591 } 11592 11593 if (!map->ops->map_direct_value_addr) { 11594 verbose(env, "no direct value access support for this map type\n"); 11595 fdput(f); 11596 return -EINVAL; 11597 } 11598 11599 err = map->ops->map_direct_value_addr(map, &addr, off); 11600 if (err) { 11601 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 11602 map->value_size, off); 11603 fdput(f); 11604 return err; 11605 } 11606 11607 aux->map_off = off; 11608 addr += off; 11609 } 11610 11611 insn[0].imm = (u32)addr; 11612 insn[1].imm = addr >> 32; 11613 11614 /* check whether we recorded this map already */ 11615 for (j = 0; j < env->used_map_cnt; j++) { 11616 if (env->used_maps[j] == map) { 11617 aux->map_index = j; 11618 fdput(f); 11619 goto next_insn; 11620 } 11621 } 11622 11623 if (env->used_map_cnt >= MAX_USED_MAPS) { 11624 fdput(f); 11625 return -E2BIG; 11626 } 11627 11628 /* hold the map. If the program is rejected by verifier, 11629 * the map will be released by release_maps() or it 11630 * will be used by the valid program until it's unloaded 11631 * and all maps are released in free_used_maps() 11632 */ 11633 bpf_map_inc(map); 11634 11635 aux->map_index = env->used_map_cnt; 11636 env->used_maps[env->used_map_cnt++] = map; 11637 11638 if (bpf_map_is_cgroup_storage(map) && 11639 bpf_cgroup_storage_assign(env->prog->aux, map)) { 11640 verbose(env, "only one cgroup storage of each type is allowed\n"); 11641 fdput(f); 11642 return -EBUSY; 11643 } 11644 11645 fdput(f); 11646 next_insn: 11647 insn++; 11648 i++; 11649 continue; 11650 } 11651 11652 /* Basic sanity check before we invest more work here. */ 11653 if (!bpf_opcode_in_insntable(insn->code)) { 11654 verbose(env, "unknown opcode %02x\n", insn->code); 11655 return -EINVAL; 11656 } 11657 } 11658 11659 /* now all pseudo BPF_LD_IMM64 instructions load valid 11660 * 'struct bpf_map *' into a register instead of user map_fd. 11661 * These pointers will be used later by verifier to validate map access. 11662 */ 11663 return 0; 11664 } 11665 11666 /* drop refcnt of maps used by the rejected program */ 11667 static void release_maps(struct bpf_verifier_env *env) 11668 { 11669 __bpf_free_used_maps(env->prog->aux, env->used_maps, 11670 env->used_map_cnt); 11671 } 11672 11673 /* drop refcnt of maps used by the rejected program */ 11674 static void release_btfs(struct bpf_verifier_env *env) 11675 { 11676 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 11677 env->used_btf_cnt); 11678 } 11679 11680 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 11681 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 11682 { 11683 struct bpf_insn *insn = env->prog->insnsi; 11684 int insn_cnt = env->prog->len; 11685 int i; 11686 11687 for (i = 0; i < insn_cnt; i++, insn++) { 11688 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 11689 continue; 11690 if (insn->src_reg == BPF_PSEUDO_FUNC) 11691 continue; 11692 insn->src_reg = 0; 11693 } 11694 } 11695 11696 /* single env->prog->insni[off] instruction was replaced with the range 11697 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 11698 * [0, off) and [off, end) to new locations, so the patched range stays zero 11699 */ 11700 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 11701 struct bpf_insn_aux_data *new_data, 11702 struct bpf_prog *new_prog, u32 off, u32 cnt) 11703 { 11704 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 11705 struct bpf_insn *insn = new_prog->insnsi; 11706 u32 old_seen = old_data[off].seen; 11707 u32 prog_len; 11708 int i; 11709 11710 /* aux info at OFF always needs adjustment, no matter fast path 11711 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 11712 * original insn at old prog. 11713 */ 11714 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 11715 11716 if (cnt == 1) 11717 return; 11718 prog_len = new_prog->len; 11719 11720 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 11721 memcpy(new_data + off + cnt - 1, old_data + off, 11722 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 11723 for (i = off; i < off + cnt - 1; i++) { 11724 /* Expand insni[off]'s seen count to the patched range. */ 11725 new_data[i].seen = old_seen; 11726 new_data[i].zext_dst = insn_has_def32(env, insn + i); 11727 } 11728 env->insn_aux_data = new_data; 11729 vfree(old_data); 11730 } 11731 11732 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 11733 { 11734 int i; 11735 11736 if (len == 1) 11737 return; 11738 /* NOTE: fake 'exit' subprog should be updated as well. */ 11739 for (i = 0; i <= env->subprog_cnt; i++) { 11740 if (env->subprog_info[i].start <= off) 11741 continue; 11742 env->subprog_info[i].start += len - 1; 11743 } 11744 } 11745 11746 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 11747 { 11748 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 11749 int i, sz = prog->aux->size_poke_tab; 11750 struct bpf_jit_poke_descriptor *desc; 11751 11752 for (i = 0; i < sz; i++) { 11753 desc = &tab[i]; 11754 if (desc->insn_idx <= off) 11755 continue; 11756 desc->insn_idx += len - 1; 11757 } 11758 } 11759 11760 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 11761 const struct bpf_insn *patch, u32 len) 11762 { 11763 struct bpf_prog *new_prog; 11764 struct bpf_insn_aux_data *new_data = NULL; 11765 11766 if (len > 1) { 11767 new_data = vzalloc(array_size(env->prog->len + len - 1, 11768 sizeof(struct bpf_insn_aux_data))); 11769 if (!new_data) 11770 return NULL; 11771 } 11772 11773 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 11774 if (IS_ERR(new_prog)) { 11775 if (PTR_ERR(new_prog) == -ERANGE) 11776 verbose(env, 11777 "insn %d cannot be patched due to 16-bit range\n", 11778 env->insn_aux_data[off].orig_idx); 11779 vfree(new_data); 11780 return NULL; 11781 } 11782 adjust_insn_aux_data(env, new_data, new_prog, off, len); 11783 adjust_subprog_starts(env, off, len); 11784 adjust_poke_descs(new_prog, off, len); 11785 return new_prog; 11786 } 11787 11788 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 11789 u32 off, u32 cnt) 11790 { 11791 int i, j; 11792 11793 /* find first prog starting at or after off (first to remove) */ 11794 for (i = 0; i < env->subprog_cnt; i++) 11795 if (env->subprog_info[i].start >= off) 11796 break; 11797 /* find first prog starting at or after off + cnt (first to stay) */ 11798 for (j = i; j < env->subprog_cnt; j++) 11799 if (env->subprog_info[j].start >= off + cnt) 11800 break; 11801 /* if j doesn't start exactly at off + cnt, we are just removing 11802 * the front of previous prog 11803 */ 11804 if (env->subprog_info[j].start != off + cnt) 11805 j--; 11806 11807 if (j > i) { 11808 struct bpf_prog_aux *aux = env->prog->aux; 11809 int move; 11810 11811 /* move fake 'exit' subprog as well */ 11812 move = env->subprog_cnt + 1 - j; 11813 11814 memmove(env->subprog_info + i, 11815 env->subprog_info + j, 11816 sizeof(*env->subprog_info) * move); 11817 env->subprog_cnt -= j - i; 11818 11819 /* remove func_info */ 11820 if (aux->func_info) { 11821 move = aux->func_info_cnt - j; 11822 11823 memmove(aux->func_info + i, 11824 aux->func_info + j, 11825 sizeof(*aux->func_info) * move); 11826 aux->func_info_cnt -= j - i; 11827 /* func_info->insn_off is set after all code rewrites, 11828 * in adjust_btf_func() - no need to adjust 11829 */ 11830 } 11831 } else { 11832 /* convert i from "first prog to remove" to "first to adjust" */ 11833 if (env->subprog_info[i].start == off) 11834 i++; 11835 } 11836 11837 /* update fake 'exit' subprog as well */ 11838 for (; i <= env->subprog_cnt; i++) 11839 env->subprog_info[i].start -= cnt; 11840 11841 return 0; 11842 } 11843 11844 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 11845 u32 cnt) 11846 { 11847 struct bpf_prog *prog = env->prog; 11848 u32 i, l_off, l_cnt, nr_linfo; 11849 struct bpf_line_info *linfo; 11850 11851 nr_linfo = prog->aux->nr_linfo; 11852 if (!nr_linfo) 11853 return 0; 11854 11855 linfo = prog->aux->linfo; 11856 11857 /* find first line info to remove, count lines to be removed */ 11858 for (i = 0; i < nr_linfo; i++) 11859 if (linfo[i].insn_off >= off) 11860 break; 11861 11862 l_off = i; 11863 l_cnt = 0; 11864 for (; i < nr_linfo; i++) 11865 if (linfo[i].insn_off < off + cnt) 11866 l_cnt++; 11867 else 11868 break; 11869 11870 /* First live insn doesn't match first live linfo, it needs to "inherit" 11871 * last removed linfo. prog is already modified, so prog->len == off 11872 * means no live instructions after (tail of the program was removed). 11873 */ 11874 if (prog->len != off && l_cnt && 11875 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 11876 l_cnt--; 11877 linfo[--i].insn_off = off + cnt; 11878 } 11879 11880 /* remove the line info which refer to the removed instructions */ 11881 if (l_cnt) { 11882 memmove(linfo + l_off, linfo + i, 11883 sizeof(*linfo) * (nr_linfo - i)); 11884 11885 prog->aux->nr_linfo -= l_cnt; 11886 nr_linfo = prog->aux->nr_linfo; 11887 } 11888 11889 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 11890 for (i = l_off; i < nr_linfo; i++) 11891 linfo[i].insn_off -= cnt; 11892 11893 /* fix up all subprogs (incl. 'exit') which start >= off */ 11894 for (i = 0; i <= env->subprog_cnt; i++) 11895 if (env->subprog_info[i].linfo_idx > l_off) { 11896 /* program may have started in the removed region but 11897 * may not be fully removed 11898 */ 11899 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 11900 env->subprog_info[i].linfo_idx -= l_cnt; 11901 else 11902 env->subprog_info[i].linfo_idx = l_off; 11903 } 11904 11905 return 0; 11906 } 11907 11908 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 11909 { 11910 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11911 unsigned int orig_prog_len = env->prog->len; 11912 int err; 11913 11914 if (bpf_prog_is_dev_bound(env->prog->aux)) 11915 bpf_prog_offload_remove_insns(env, off, cnt); 11916 11917 err = bpf_remove_insns(env->prog, off, cnt); 11918 if (err) 11919 return err; 11920 11921 err = adjust_subprog_starts_after_remove(env, off, cnt); 11922 if (err) 11923 return err; 11924 11925 err = bpf_adj_linfo_after_remove(env, off, cnt); 11926 if (err) 11927 return err; 11928 11929 memmove(aux_data + off, aux_data + off + cnt, 11930 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 11931 11932 return 0; 11933 } 11934 11935 /* The verifier does more data flow analysis than llvm and will not 11936 * explore branches that are dead at run time. Malicious programs can 11937 * have dead code too. Therefore replace all dead at-run-time code 11938 * with 'ja -1'. 11939 * 11940 * Just nops are not optimal, e.g. if they would sit at the end of the 11941 * program and through another bug we would manage to jump there, then 11942 * we'd execute beyond program memory otherwise. Returning exception 11943 * code also wouldn't work since we can have subprogs where the dead 11944 * code could be located. 11945 */ 11946 static void sanitize_dead_code(struct bpf_verifier_env *env) 11947 { 11948 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11949 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 11950 struct bpf_insn *insn = env->prog->insnsi; 11951 const int insn_cnt = env->prog->len; 11952 int i; 11953 11954 for (i = 0; i < insn_cnt; i++) { 11955 if (aux_data[i].seen) 11956 continue; 11957 memcpy(insn + i, &trap, sizeof(trap)); 11958 aux_data[i].zext_dst = false; 11959 } 11960 } 11961 11962 static bool insn_is_cond_jump(u8 code) 11963 { 11964 u8 op; 11965 11966 if (BPF_CLASS(code) == BPF_JMP32) 11967 return true; 11968 11969 if (BPF_CLASS(code) != BPF_JMP) 11970 return false; 11971 11972 op = BPF_OP(code); 11973 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 11974 } 11975 11976 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 11977 { 11978 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 11979 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 11980 struct bpf_insn *insn = env->prog->insnsi; 11981 const int insn_cnt = env->prog->len; 11982 int i; 11983 11984 for (i = 0; i < insn_cnt; i++, insn++) { 11985 if (!insn_is_cond_jump(insn->code)) 11986 continue; 11987 11988 if (!aux_data[i + 1].seen) 11989 ja.off = insn->off; 11990 else if (!aux_data[i + 1 + insn->off].seen) 11991 ja.off = 0; 11992 else 11993 continue; 11994 11995 if (bpf_prog_is_dev_bound(env->prog->aux)) 11996 bpf_prog_offload_replace_insn(env, i, &ja); 11997 11998 memcpy(insn, &ja, sizeof(ja)); 11999 } 12000 } 12001 12002 static int opt_remove_dead_code(struct bpf_verifier_env *env) 12003 { 12004 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 12005 int insn_cnt = env->prog->len; 12006 int i, err; 12007 12008 for (i = 0; i < insn_cnt; i++) { 12009 int j; 12010 12011 j = 0; 12012 while (i + j < insn_cnt && !aux_data[i + j].seen) 12013 j++; 12014 if (!j) 12015 continue; 12016 12017 err = verifier_remove_insns(env, i, j); 12018 if (err) 12019 return err; 12020 insn_cnt = env->prog->len; 12021 } 12022 12023 return 0; 12024 } 12025 12026 static int opt_remove_nops(struct bpf_verifier_env *env) 12027 { 12028 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 12029 struct bpf_insn *insn = env->prog->insnsi; 12030 int insn_cnt = env->prog->len; 12031 int i, err; 12032 12033 for (i = 0; i < insn_cnt; i++) { 12034 if (memcmp(&insn[i], &ja, sizeof(ja))) 12035 continue; 12036 12037 err = verifier_remove_insns(env, i, 1); 12038 if (err) 12039 return err; 12040 insn_cnt--; 12041 i--; 12042 } 12043 12044 return 0; 12045 } 12046 12047 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 12048 const union bpf_attr *attr) 12049 { 12050 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 12051 struct bpf_insn_aux_data *aux = env->insn_aux_data; 12052 int i, patch_len, delta = 0, len = env->prog->len; 12053 struct bpf_insn *insns = env->prog->insnsi; 12054 struct bpf_prog *new_prog; 12055 bool rnd_hi32; 12056 12057 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 12058 zext_patch[1] = BPF_ZEXT_REG(0); 12059 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 12060 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 12061 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 12062 for (i = 0; i < len; i++) { 12063 int adj_idx = i + delta; 12064 struct bpf_insn insn; 12065 int load_reg; 12066 12067 insn = insns[adj_idx]; 12068 load_reg = insn_def_regno(&insn); 12069 if (!aux[adj_idx].zext_dst) { 12070 u8 code, class; 12071 u32 imm_rnd; 12072 12073 if (!rnd_hi32) 12074 continue; 12075 12076 code = insn.code; 12077 class = BPF_CLASS(code); 12078 if (load_reg == -1) 12079 continue; 12080 12081 /* NOTE: arg "reg" (the fourth one) is only used for 12082 * BPF_STX + SRC_OP, so it is safe to pass NULL 12083 * here. 12084 */ 12085 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 12086 if (class == BPF_LD && 12087 BPF_MODE(code) == BPF_IMM) 12088 i++; 12089 continue; 12090 } 12091 12092 /* ctx load could be transformed into wider load. */ 12093 if (class == BPF_LDX && 12094 aux[adj_idx].ptr_type == PTR_TO_CTX) 12095 continue; 12096 12097 imm_rnd = get_random_int(); 12098 rnd_hi32_patch[0] = insn; 12099 rnd_hi32_patch[1].imm = imm_rnd; 12100 rnd_hi32_patch[3].dst_reg = load_reg; 12101 patch = rnd_hi32_patch; 12102 patch_len = 4; 12103 goto apply_patch_buffer; 12104 } 12105 12106 /* Add in an zero-extend instruction if a) the JIT has requested 12107 * it or b) it's a CMPXCHG. 12108 * 12109 * The latter is because: BPF_CMPXCHG always loads a value into 12110 * R0, therefore always zero-extends. However some archs' 12111 * equivalent instruction only does this load when the 12112 * comparison is successful. This detail of CMPXCHG is 12113 * orthogonal to the general zero-extension behaviour of the 12114 * CPU, so it's treated independently of bpf_jit_needs_zext. 12115 */ 12116 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 12117 continue; 12118 12119 if (WARN_ON(load_reg == -1)) { 12120 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 12121 return -EFAULT; 12122 } 12123 12124 zext_patch[0] = insn; 12125 zext_patch[1].dst_reg = load_reg; 12126 zext_patch[1].src_reg = load_reg; 12127 patch = zext_patch; 12128 patch_len = 2; 12129 apply_patch_buffer: 12130 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 12131 if (!new_prog) 12132 return -ENOMEM; 12133 env->prog = new_prog; 12134 insns = new_prog->insnsi; 12135 aux = env->insn_aux_data; 12136 delta += patch_len - 1; 12137 } 12138 12139 return 0; 12140 } 12141 12142 /* convert load instructions that access fields of a context type into a 12143 * sequence of instructions that access fields of the underlying structure: 12144 * struct __sk_buff -> struct sk_buff 12145 * struct bpf_sock_ops -> struct sock 12146 */ 12147 static int convert_ctx_accesses(struct bpf_verifier_env *env) 12148 { 12149 const struct bpf_verifier_ops *ops = env->ops; 12150 int i, cnt, size, ctx_field_size, delta = 0; 12151 const int insn_cnt = env->prog->len; 12152 struct bpf_insn insn_buf[16], *insn; 12153 u32 target_size, size_default, off; 12154 struct bpf_prog *new_prog; 12155 enum bpf_access_type type; 12156 bool is_narrower_load; 12157 12158 if (ops->gen_prologue || env->seen_direct_write) { 12159 if (!ops->gen_prologue) { 12160 verbose(env, "bpf verifier is misconfigured\n"); 12161 return -EINVAL; 12162 } 12163 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 12164 env->prog); 12165 if (cnt >= ARRAY_SIZE(insn_buf)) { 12166 verbose(env, "bpf verifier is misconfigured\n"); 12167 return -EINVAL; 12168 } else if (cnt) { 12169 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 12170 if (!new_prog) 12171 return -ENOMEM; 12172 12173 env->prog = new_prog; 12174 delta += cnt - 1; 12175 } 12176 } 12177 12178 if (bpf_prog_is_dev_bound(env->prog->aux)) 12179 return 0; 12180 12181 insn = env->prog->insnsi + delta; 12182 12183 for (i = 0; i < insn_cnt; i++, insn++) { 12184 bpf_convert_ctx_access_t convert_ctx_access; 12185 bool ctx_access; 12186 12187 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 12188 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 12189 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 12190 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { 12191 type = BPF_READ; 12192 ctx_access = true; 12193 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 12194 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 12195 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 12196 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 12197 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 12198 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 12199 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 12200 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 12201 type = BPF_WRITE; 12202 ctx_access = BPF_CLASS(insn->code) == BPF_STX; 12203 } else { 12204 continue; 12205 } 12206 12207 if (type == BPF_WRITE && 12208 env->insn_aux_data[i + delta].sanitize_stack_spill) { 12209 struct bpf_insn patch[] = { 12210 *insn, 12211 BPF_ST_NOSPEC(), 12212 }; 12213 12214 cnt = ARRAY_SIZE(patch); 12215 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 12216 if (!new_prog) 12217 return -ENOMEM; 12218 12219 delta += cnt - 1; 12220 env->prog = new_prog; 12221 insn = new_prog->insnsi + i + delta; 12222 continue; 12223 } 12224 12225 if (!ctx_access) 12226 continue; 12227 12228 switch (env->insn_aux_data[i + delta].ptr_type) { 12229 case PTR_TO_CTX: 12230 if (!ops->convert_ctx_access) 12231 continue; 12232 convert_ctx_access = ops->convert_ctx_access; 12233 break; 12234 case PTR_TO_SOCKET: 12235 case PTR_TO_SOCK_COMMON: 12236 convert_ctx_access = bpf_sock_convert_ctx_access; 12237 break; 12238 case PTR_TO_TCP_SOCK: 12239 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 12240 break; 12241 case PTR_TO_XDP_SOCK: 12242 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 12243 break; 12244 case PTR_TO_BTF_ID: 12245 if (type == BPF_READ) { 12246 insn->code = BPF_LDX | BPF_PROBE_MEM | 12247 BPF_SIZE((insn)->code); 12248 env->prog->aux->num_exentries++; 12249 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { 12250 verbose(env, "Writes through BTF pointers are not allowed\n"); 12251 return -EINVAL; 12252 } 12253 continue; 12254 default: 12255 continue; 12256 } 12257 12258 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 12259 size = BPF_LDST_BYTES(insn); 12260 12261 /* If the read access is a narrower load of the field, 12262 * convert to a 4/8-byte load, to minimum program type specific 12263 * convert_ctx_access changes. If conversion is successful, 12264 * we will apply proper mask to the result. 12265 */ 12266 is_narrower_load = size < ctx_field_size; 12267 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 12268 off = insn->off; 12269 if (is_narrower_load) { 12270 u8 size_code; 12271 12272 if (type == BPF_WRITE) { 12273 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 12274 return -EINVAL; 12275 } 12276 12277 size_code = BPF_H; 12278 if (ctx_field_size == 4) 12279 size_code = BPF_W; 12280 else if (ctx_field_size == 8) 12281 size_code = BPF_DW; 12282 12283 insn->off = off & ~(size_default - 1); 12284 insn->code = BPF_LDX | BPF_MEM | size_code; 12285 } 12286 12287 target_size = 0; 12288 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 12289 &target_size); 12290 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 12291 (ctx_field_size && !target_size)) { 12292 verbose(env, "bpf verifier is misconfigured\n"); 12293 return -EINVAL; 12294 } 12295 12296 if (is_narrower_load && size < target_size) { 12297 u8 shift = bpf_ctx_narrow_access_offset( 12298 off, size, size_default) * 8; 12299 if (ctx_field_size <= 4) { 12300 if (shift) 12301 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 12302 insn->dst_reg, 12303 shift); 12304 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 12305 (1 << size * 8) - 1); 12306 } else { 12307 if (shift) 12308 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 12309 insn->dst_reg, 12310 shift); 12311 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 12312 (1ULL << size * 8) - 1); 12313 } 12314 } 12315 12316 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12317 if (!new_prog) 12318 return -ENOMEM; 12319 12320 delta += cnt - 1; 12321 12322 /* keep walking new program and skip insns we just inserted */ 12323 env->prog = new_prog; 12324 insn = new_prog->insnsi + i + delta; 12325 } 12326 12327 return 0; 12328 } 12329 12330 static int jit_subprogs(struct bpf_verifier_env *env) 12331 { 12332 struct bpf_prog *prog = env->prog, **func, *tmp; 12333 int i, j, subprog_start, subprog_end = 0, len, subprog; 12334 struct bpf_map *map_ptr; 12335 struct bpf_insn *insn; 12336 void *old_bpf_func; 12337 int err, num_exentries; 12338 12339 if (env->subprog_cnt <= 1) 12340 return 0; 12341 12342 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 12343 if (bpf_pseudo_func(insn)) { 12344 env->insn_aux_data[i].call_imm = insn->imm; 12345 /* subprog is encoded in insn[1].imm */ 12346 continue; 12347 } 12348 12349 if (!bpf_pseudo_call(insn)) 12350 continue; 12351 /* Upon error here we cannot fall back to interpreter but 12352 * need a hard reject of the program. Thus -EFAULT is 12353 * propagated in any case. 12354 */ 12355 subprog = find_subprog(env, i + insn->imm + 1); 12356 if (subprog < 0) { 12357 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 12358 i + insn->imm + 1); 12359 return -EFAULT; 12360 } 12361 /* temporarily remember subprog id inside insn instead of 12362 * aux_data, since next loop will split up all insns into funcs 12363 */ 12364 insn->off = subprog; 12365 /* remember original imm in case JIT fails and fallback 12366 * to interpreter will be needed 12367 */ 12368 env->insn_aux_data[i].call_imm = insn->imm; 12369 /* point imm to __bpf_call_base+1 from JITs point of view */ 12370 insn->imm = 1; 12371 } 12372 12373 err = bpf_prog_alloc_jited_linfo(prog); 12374 if (err) 12375 goto out_undo_insn; 12376 12377 err = -ENOMEM; 12378 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 12379 if (!func) 12380 goto out_undo_insn; 12381 12382 for (i = 0; i < env->subprog_cnt; i++) { 12383 subprog_start = subprog_end; 12384 subprog_end = env->subprog_info[i + 1].start; 12385 12386 len = subprog_end - subprog_start; 12387 /* BPF_PROG_RUN doesn't call subprogs directly, 12388 * hence main prog stats include the runtime of subprogs. 12389 * subprogs don't have IDs and not reachable via prog_get_next_id 12390 * func[i]->stats will never be accessed and stays NULL 12391 */ 12392 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 12393 if (!func[i]) 12394 goto out_free; 12395 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 12396 len * sizeof(struct bpf_insn)); 12397 func[i]->type = prog->type; 12398 func[i]->len = len; 12399 if (bpf_prog_calc_tag(func[i])) 12400 goto out_free; 12401 func[i]->is_func = 1; 12402 func[i]->aux->func_idx = i; 12403 /* Below members will be freed only at prog->aux */ 12404 func[i]->aux->btf = prog->aux->btf; 12405 func[i]->aux->func_info = prog->aux->func_info; 12406 func[i]->aux->poke_tab = prog->aux->poke_tab; 12407 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 12408 12409 for (j = 0; j < prog->aux->size_poke_tab; j++) { 12410 struct bpf_jit_poke_descriptor *poke; 12411 12412 poke = &prog->aux->poke_tab[j]; 12413 if (poke->insn_idx < subprog_end && 12414 poke->insn_idx >= subprog_start) 12415 poke->aux = func[i]->aux; 12416 } 12417 12418 /* Use bpf_prog_F_tag to indicate functions in stack traces. 12419 * Long term would need debug info to populate names 12420 */ 12421 func[i]->aux->name[0] = 'F'; 12422 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 12423 func[i]->jit_requested = 1; 12424 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 12425 func[i]->aux->linfo = prog->aux->linfo; 12426 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 12427 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 12428 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 12429 num_exentries = 0; 12430 insn = func[i]->insnsi; 12431 for (j = 0; j < func[i]->len; j++, insn++) { 12432 if (BPF_CLASS(insn->code) == BPF_LDX && 12433 BPF_MODE(insn->code) == BPF_PROBE_MEM) 12434 num_exentries++; 12435 } 12436 func[i]->aux->num_exentries = num_exentries; 12437 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 12438 func[i] = bpf_int_jit_compile(func[i]); 12439 if (!func[i]->jited) { 12440 err = -ENOTSUPP; 12441 goto out_free; 12442 } 12443 cond_resched(); 12444 } 12445 12446 /* at this point all bpf functions were successfully JITed 12447 * now populate all bpf_calls with correct addresses and 12448 * run last pass of JIT 12449 */ 12450 for (i = 0; i < env->subprog_cnt; i++) { 12451 insn = func[i]->insnsi; 12452 for (j = 0; j < func[i]->len; j++, insn++) { 12453 if (bpf_pseudo_func(insn)) { 12454 subprog = insn[1].imm; 12455 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 12456 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 12457 continue; 12458 } 12459 if (!bpf_pseudo_call(insn)) 12460 continue; 12461 subprog = insn->off; 12462 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - 12463 __bpf_call_base; 12464 } 12465 12466 /* we use the aux data to keep a list of the start addresses 12467 * of the JITed images for each function in the program 12468 * 12469 * for some architectures, such as powerpc64, the imm field 12470 * might not be large enough to hold the offset of the start 12471 * address of the callee's JITed image from __bpf_call_base 12472 * 12473 * in such cases, we can lookup the start address of a callee 12474 * by using its subprog id, available from the off field of 12475 * the call instruction, as an index for this list 12476 */ 12477 func[i]->aux->func = func; 12478 func[i]->aux->func_cnt = env->subprog_cnt; 12479 } 12480 for (i = 0; i < env->subprog_cnt; i++) { 12481 old_bpf_func = func[i]->bpf_func; 12482 tmp = bpf_int_jit_compile(func[i]); 12483 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 12484 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 12485 err = -ENOTSUPP; 12486 goto out_free; 12487 } 12488 cond_resched(); 12489 } 12490 12491 /* finally lock prog and jit images for all functions and 12492 * populate kallsysm 12493 */ 12494 for (i = 0; i < env->subprog_cnt; i++) { 12495 bpf_prog_lock_ro(func[i]); 12496 bpf_prog_kallsyms_add(func[i]); 12497 } 12498 12499 /* Last step: make now unused interpreter insns from main 12500 * prog consistent for later dump requests, so they can 12501 * later look the same as if they were interpreted only. 12502 */ 12503 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 12504 if (bpf_pseudo_func(insn)) { 12505 insn[0].imm = env->insn_aux_data[i].call_imm; 12506 insn[1].imm = find_subprog(env, i + insn[0].imm + 1); 12507 continue; 12508 } 12509 if (!bpf_pseudo_call(insn)) 12510 continue; 12511 insn->off = env->insn_aux_data[i].call_imm; 12512 subprog = find_subprog(env, i + insn->off + 1); 12513 insn->imm = subprog; 12514 } 12515 12516 prog->jited = 1; 12517 prog->bpf_func = func[0]->bpf_func; 12518 prog->aux->func = func; 12519 prog->aux->func_cnt = env->subprog_cnt; 12520 bpf_prog_jit_attempt_done(prog); 12521 return 0; 12522 out_free: 12523 /* We failed JIT'ing, so at this point we need to unregister poke 12524 * descriptors from subprogs, so that kernel is not attempting to 12525 * patch it anymore as we're freeing the subprog JIT memory. 12526 */ 12527 for (i = 0; i < prog->aux->size_poke_tab; i++) { 12528 map_ptr = prog->aux->poke_tab[i].tail_call.map; 12529 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 12530 } 12531 /* At this point we're guaranteed that poke descriptors are not 12532 * live anymore. We can just unlink its descriptor table as it's 12533 * released with the main prog. 12534 */ 12535 for (i = 0; i < env->subprog_cnt; i++) { 12536 if (!func[i]) 12537 continue; 12538 func[i]->aux->poke_tab = NULL; 12539 bpf_jit_free(func[i]); 12540 } 12541 kfree(func); 12542 out_undo_insn: 12543 /* cleanup main prog to be interpreted */ 12544 prog->jit_requested = 0; 12545 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 12546 if (!bpf_pseudo_call(insn)) 12547 continue; 12548 insn->off = 0; 12549 insn->imm = env->insn_aux_data[i].call_imm; 12550 } 12551 bpf_prog_jit_attempt_done(prog); 12552 return err; 12553 } 12554 12555 static int fixup_call_args(struct bpf_verifier_env *env) 12556 { 12557 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 12558 struct bpf_prog *prog = env->prog; 12559 struct bpf_insn *insn = prog->insnsi; 12560 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 12561 int i, depth; 12562 #endif 12563 int err = 0; 12564 12565 if (env->prog->jit_requested && 12566 !bpf_prog_is_dev_bound(env->prog->aux)) { 12567 err = jit_subprogs(env); 12568 if (err == 0) 12569 return 0; 12570 if (err == -EFAULT) 12571 return err; 12572 } 12573 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 12574 if (has_kfunc_call) { 12575 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 12576 return -EINVAL; 12577 } 12578 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 12579 /* When JIT fails the progs with bpf2bpf calls and tail_calls 12580 * have to be rejected, since interpreter doesn't support them yet. 12581 */ 12582 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 12583 return -EINVAL; 12584 } 12585 for (i = 0; i < prog->len; i++, insn++) { 12586 if (bpf_pseudo_func(insn)) { 12587 /* When JIT fails the progs with callback calls 12588 * have to be rejected, since interpreter doesn't support them yet. 12589 */ 12590 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 12591 return -EINVAL; 12592 } 12593 12594 if (!bpf_pseudo_call(insn)) 12595 continue; 12596 depth = get_callee_stack_depth(env, insn, i); 12597 if (depth < 0) 12598 return depth; 12599 bpf_patch_call_args(insn, depth); 12600 } 12601 err = 0; 12602 #endif 12603 return err; 12604 } 12605 12606 static int fixup_kfunc_call(struct bpf_verifier_env *env, 12607 struct bpf_insn *insn) 12608 { 12609 const struct bpf_kfunc_desc *desc; 12610 12611 /* insn->imm has the btf func_id. Replace it with 12612 * an address (relative to __bpf_base_call). 12613 */ 12614 desc = find_kfunc_desc(env->prog, insn->imm); 12615 if (!desc) { 12616 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 12617 insn->imm); 12618 return -EFAULT; 12619 } 12620 12621 insn->imm = desc->imm; 12622 12623 return 0; 12624 } 12625 12626 /* Do various post-verification rewrites in a single program pass. 12627 * These rewrites simplify JIT and interpreter implementations. 12628 */ 12629 static int do_misc_fixups(struct bpf_verifier_env *env) 12630 { 12631 struct bpf_prog *prog = env->prog; 12632 bool expect_blinding = bpf_jit_blinding_enabled(prog); 12633 enum bpf_prog_type prog_type = resolve_prog_type(prog); 12634 struct bpf_insn *insn = prog->insnsi; 12635 const struct bpf_func_proto *fn; 12636 const int insn_cnt = prog->len; 12637 const struct bpf_map_ops *ops; 12638 struct bpf_insn_aux_data *aux; 12639 struct bpf_insn insn_buf[16]; 12640 struct bpf_prog *new_prog; 12641 struct bpf_map *map_ptr; 12642 int i, ret, cnt, delta = 0; 12643 12644 for (i = 0; i < insn_cnt; i++, insn++) { 12645 /* Make divide-by-zero exceptions impossible. */ 12646 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 12647 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 12648 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 12649 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 12650 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 12651 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 12652 struct bpf_insn *patchlet; 12653 struct bpf_insn chk_and_div[] = { 12654 /* [R,W]x div 0 -> 0 */ 12655 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 12656 BPF_JNE | BPF_K, insn->src_reg, 12657 0, 2, 0), 12658 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 12659 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 12660 *insn, 12661 }; 12662 struct bpf_insn chk_and_mod[] = { 12663 /* [R,W]x mod 0 -> [R,W]x */ 12664 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 12665 BPF_JEQ | BPF_K, insn->src_reg, 12666 0, 1 + (is64 ? 0 : 1), 0), 12667 *insn, 12668 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 12669 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 12670 }; 12671 12672 patchlet = isdiv ? chk_and_div : chk_and_mod; 12673 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 12674 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 12675 12676 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 12677 if (!new_prog) 12678 return -ENOMEM; 12679 12680 delta += cnt - 1; 12681 env->prog = prog = new_prog; 12682 insn = new_prog->insnsi + i + delta; 12683 continue; 12684 } 12685 12686 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 12687 if (BPF_CLASS(insn->code) == BPF_LD && 12688 (BPF_MODE(insn->code) == BPF_ABS || 12689 BPF_MODE(insn->code) == BPF_IND)) { 12690 cnt = env->ops->gen_ld_abs(insn, insn_buf); 12691 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 12692 verbose(env, "bpf verifier is misconfigured\n"); 12693 return -EINVAL; 12694 } 12695 12696 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12697 if (!new_prog) 12698 return -ENOMEM; 12699 12700 delta += cnt - 1; 12701 env->prog = prog = new_prog; 12702 insn = new_prog->insnsi + i + delta; 12703 continue; 12704 } 12705 12706 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 12707 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 12708 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 12709 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 12710 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 12711 struct bpf_insn *patch = &insn_buf[0]; 12712 bool issrc, isneg, isimm; 12713 u32 off_reg; 12714 12715 aux = &env->insn_aux_data[i + delta]; 12716 if (!aux->alu_state || 12717 aux->alu_state == BPF_ALU_NON_POINTER) 12718 continue; 12719 12720 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 12721 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 12722 BPF_ALU_SANITIZE_SRC; 12723 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 12724 12725 off_reg = issrc ? insn->src_reg : insn->dst_reg; 12726 if (isimm) { 12727 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 12728 } else { 12729 if (isneg) 12730 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 12731 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 12732 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 12733 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 12734 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 12735 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 12736 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 12737 } 12738 if (!issrc) 12739 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 12740 insn->src_reg = BPF_REG_AX; 12741 if (isneg) 12742 insn->code = insn->code == code_add ? 12743 code_sub : code_add; 12744 *patch++ = *insn; 12745 if (issrc && isneg && !isimm) 12746 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 12747 cnt = patch - insn_buf; 12748 12749 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12750 if (!new_prog) 12751 return -ENOMEM; 12752 12753 delta += cnt - 1; 12754 env->prog = prog = new_prog; 12755 insn = new_prog->insnsi + i + delta; 12756 continue; 12757 } 12758 12759 if (insn->code != (BPF_JMP | BPF_CALL)) 12760 continue; 12761 if (insn->src_reg == BPF_PSEUDO_CALL) 12762 continue; 12763 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 12764 ret = fixup_kfunc_call(env, insn); 12765 if (ret) 12766 return ret; 12767 continue; 12768 } 12769 12770 if (insn->imm == BPF_FUNC_get_route_realm) 12771 prog->dst_needed = 1; 12772 if (insn->imm == BPF_FUNC_get_prandom_u32) 12773 bpf_user_rnd_init_once(); 12774 if (insn->imm == BPF_FUNC_override_return) 12775 prog->kprobe_override = 1; 12776 if (insn->imm == BPF_FUNC_tail_call) { 12777 /* If we tail call into other programs, we 12778 * cannot make any assumptions since they can 12779 * be replaced dynamically during runtime in 12780 * the program array. 12781 */ 12782 prog->cb_access = 1; 12783 if (!allow_tail_call_in_subprogs(env)) 12784 prog->aux->stack_depth = MAX_BPF_STACK; 12785 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 12786 12787 /* mark bpf_tail_call as different opcode to avoid 12788 * conditional branch in the interpreter for every normal 12789 * call and to prevent accidental JITing by JIT compiler 12790 * that doesn't support bpf_tail_call yet 12791 */ 12792 insn->imm = 0; 12793 insn->code = BPF_JMP | BPF_TAIL_CALL; 12794 12795 aux = &env->insn_aux_data[i + delta]; 12796 if (env->bpf_capable && !expect_blinding && 12797 prog->jit_requested && 12798 !bpf_map_key_poisoned(aux) && 12799 !bpf_map_ptr_poisoned(aux) && 12800 !bpf_map_ptr_unpriv(aux)) { 12801 struct bpf_jit_poke_descriptor desc = { 12802 .reason = BPF_POKE_REASON_TAIL_CALL, 12803 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 12804 .tail_call.key = bpf_map_key_immediate(aux), 12805 .insn_idx = i + delta, 12806 }; 12807 12808 ret = bpf_jit_add_poke_descriptor(prog, &desc); 12809 if (ret < 0) { 12810 verbose(env, "adding tail call poke descriptor failed\n"); 12811 return ret; 12812 } 12813 12814 insn->imm = ret + 1; 12815 continue; 12816 } 12817 12818 if (!bpf_map_ptr_unpriv(aux)) 12819 continue; 12820 12821 /* instead of changing every JIT dealing with tail_call 12822 * emit two extra insns: 12823 * if (index >= max_entries) goto out; 12824 * index &= array->index_mask; 12825 * to avoid out-of-bounds cpu speculation 12826 */ 12827 if (bpf_map_ptr_poisoned(aux)) { 12828 verbose(env, "tail_call abusing map_ptr\n"); 12829 return -EINVAL; 12830 } 12831 12832 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 12833 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 12834 map_ptr->max_entries, 2); 12835 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 12836 container_of(map_ptr, 12837 struct bpf_array, 12838 map)->index_mask); 12839 insn_buf[2] = *insn; 12840 cnt = 3; 12841 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12842 if (!new_prog) 12843 return -ENOMEM; 12844 12845 delta += cnt - 1; 12846 env->prog = prog = new_prog; 12847 insn = new_prog->insnsi + i + delta; 12848 continue; 12849 } 12850 12851 if (insn->imm == BPF_FUNC_timer_set_callback) { 12852 /* The verifier will process callback_fn as many times as necessary 12853 * with different maps and the register states prepared by 12854 * set_timer_callback_state will be accurate. 12855 * 12856 * The following use case is valid: 12857 * map1 is shared by prog1, prog2, prog3. 12858 * prog1 calls bpf_timer_init for some map1 elements 12859 * prog2 calls bpf_timer_set_callback for some map1 elements. 12860 * Those that were not bpf_timer_init-ed will return -EINVAL. 12861 * prog3 calls bpf_timer_start for some map1 elements. 12862 * Those that were not both bpf_timer_init-ed and 12863 * bpf_timer_set_callback-ed will return -EINVAL. 12864 */ 12865 struct bpf_insn ld_addrs[2] = { 12866 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 12867 }; 12868 12869 insn_buf[0] = ld_addrs[0]; 12870 insn_buf[1] = ld_addrs[1]; 12871 insn_buf[2] = *insn; 12872 cnt = 3; 12873 12874 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 12875 if (!new_prog) 12876 return -ENOMEM; 12877 12878 delta += cnt - 1; 12879 env->prog = prog = new_prog; 12880 insn = new_prog->insnsi + i + delta; 12881 goto patch_call_imm; 12882 } 12883 12884 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 12885 * and other inlining handlers are currently limited to 64 bit 12886 * only. 12887 */ 12888 if (prog->jit_requested && BITS_PER_LONG == 64 && 12889 (insn->imm == BPF_FUNC_map_lookup_elem || 12890 insn->imm == BPF_FUNC_map_update_elem || 12891 insn->imm == BPF_FUNC_map_delete_elem || 12892 insn->imm == BPF_FUNC_map_push_elem || 12893 insn->imm == BPF_FUNC_map_pop_elem || 12894 insn->imm == BPF_FUNC_map_peek_elem || 12895 insn->imm == BPF_FUNC_redirect_map)) { 12896 aux = &env->insn_aux_data[i + delta]; 12897 if (bpf_map_ptr_poisoned(aux)) 12898 goto patch_call_imm; 12899 12900 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 12901 ops = map_ptr->ops; 12902 if (insn->imm == BPF_FUNC_map_lookup_elem && 12903 ops->map_gen_lookup) { 12904 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 12905 if (cnt == -EOPNOTSUPP) 12906 goto patch_map_ops_generic; 12907 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 12908 verbose(env, "bpf verifier is misconfigured\n"); 12909 return -EINVAL; 12910 } 12911 12912 new_prog = bpf_patch_insn_data(env, i + delta, 12913 insn_buf, cnt); 12914 if (!new_prog) 12915 return -ENOMEM; 12916 12917 delta += cnt - 1; 12918 env->prog = prog = new_prog; 12919 insn = new_prog->insnsi + i + delta; 12920 continue; 12921 } 12922 12923 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 12924 (void *(*)(struct bpf_map *map, void *key))NULL)); 12925 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 12926 (int (*)(struct bpf_map *map, void *key))NULL)); 12927 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 12928 (int (*)(struct bpf_map *map, void *key, void *value, 12929 u64 flags))NULL)); 12930 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 12931 (int (*)(struct bpf_map *map, void *value, 12932 u64 flags))NULL)); 12933 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 12934 (int (*)(struct bpf_map *map, void *value))NULL)); 12935 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 12936 (int (*)(struct bpf_map *map, void *value))NULL)); 12937 BUILD_BUG_ON(!__same_type(ops->map_redirect, 12938 (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL)); 12939 12940 patch_map_ops_generic: 12941 switch (insn->imm) { 12942 case BPF_FUNC_map_lookup_elem: 12943 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - 12944 __bpf_call_base; 12945 continue; 12946 case BPF_FUNC_map_update_elem: 12947 insn->imm = BPF_CAST_CALL(ops->map_update_elem) - 12948 __bpf_call_base; 12949 continue; 12950 case BPF_FUNC_map_delete_elem: 12951 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - 12952 __bpf_call_base; 12953 continue; 12954 case BPF_FUNC_map_push_elem: 12955 insn->imm = BPF_CAST_CALL(ops->map_push_elem) - 12956 __bpf_call_base; 12957 continue; 12958 case BPF_FUNC_map_pop_elem: 12959 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - 12960 __bpf_call_base; 12961 continue; 12962 case BPF_FUNC_map_peek_elem: 12963 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - 12964 __bpf_call_base; 12965 continue; 12966 case BPF_FUNC_redirect_map: 12967 insn->imm = BPF_CAST_CALL(ops->map_redirect) - 12968 __bpf_call_base; 12969 continue; 12970 } 12971 12972 goto patch_call_imm; 12973 } 12974 12975 /* Implement bpf_jiffies64 inline. */ 12976 if (prog->jit_requested && BITS_PER_LONG == 64 && 12977 insn->imm == BPF_FUNC_jiffies64) { 12978 struct bpf_insn ld_jiffies_addr[2] = { 12979 BPF_LD_IMM64(BPF_REG_0, 12980 (unsigned long)&jiffies), 12981 }; 12982 12983 insn_buf[0] = ld_jiffies_addr[0]; 12984 insn_buf[1] = ld_jiffies_addr[1]; 12985 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 12986 BPF_REG_0, 0); 12987 cnt = 3; 12988 12989 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 12990 cnt); 12991 if (!new_prog) 12992 return -ENOMEM; 12993 12994 delta += cnt - 1; 12995 env->prog = prog = new_prog; 12996 insn = new_prog->insnsi + i + delta; 12997 continue; 12998 } 12999 13000 /* Implement bpf_get_func_ip inline. */ 13001 if (prog_type == BPF_PROG_TYPE_TRACING && 13002 insn->imm == BPF_FUNC_get_func_ip) { 13003 /* Load IP address from ctx - 8 */ 13004 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 13005 13006 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 13007 if (!new_prog) 13008 return -ENOMEM; 13009 13010 env->prog = prog = new_prog; 13011 insn = new_prog->insnsi + i + delta; 13012 continue; 13013 } 13014 13015 patch_call_imm: 13016 fn = env->ops->get_func_proto(insn->imm, env->prog); 13017 /* all functions that have prototype and verifier allowed 13018 * programs to call them, must be real in-kernel functions 13019 */ 13020 if (!fn->func) { 13021 verbose(env, 13022 "kernel subsystem misconfigured func %s#%d\n", 13023 func_id_name(insn->imm), insn->imm); 13024 return -EFAULT; 13025 } 13026 insn->imm = fn->func - __bpf_call_base; 13027 } 13028 13029 /* Since poke tab is now finalized, publish aux to tracker. */ 13030 for (i = 0; i < prog->aux->size_poke_tab; i++) { 13031 map_ptr = prog->aux->poke_tab[i].tail_call.map; 13032 if (!map_ptr->ops->map_poke_track || 13033 !map_ptr->ops->map_poke_untrack || 13034 !map_ptr->ops->map_poke_run) { 13035 verbose(env, "bpf verifier is misconfigured\n"); 13036 return -EINVAL; 13037 } 13038 13039 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 13040 if (ret < 0) { 13041 verbose(env, "tracking tail call prog failed\n"); 13042 return ret; 13043 } 13044 } 13045 13046 sort_kfunc_descs_by_imm(env->prog); 13047 13048 return 0; 13049 } 13050 13051 static void free_states(struct bpf_verifier_env *env) 13052 { 13053 struct bpf_verifier_state_list *sl, *sln; 13054 int i; 13055 13056 sl = env->free_list; 13057 while (sl) { 13058 sln = sl->next; 13059 free_verifier_state(&sl->state, false); 13060 kfree(sl); 13061 sl = sln; 13062 } 13063 env->free_list = NULL; 13064 13065 if (!env->explored_states) 13066 return; 13067 13068 for (i = 0; i < state_htab_size(env); i++) { 13069 sl = env->explored_states[i]; 13070 13071 while (sl) { 13072 sln = sl->next; 13073 free_verifier_state(&sl->state, false); 13074 kfree(sl); 13075 sl = sln; 13076 } 13077 env->explored_states[i] = NULL; 13078 } 13079 } 13080 13081 static int do_check_common(struct bpf_verifier_env *env, int subprog) 13082 { 13083 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 13084 struct bpf_verifier_state *state; 13085 struct bpf_reg_state *regs; 13086 int ret, i; 13087 13088 env->prev_linfo = NULL; 13089 env->pass_cnt++; 13090 13091 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 13092 if (!state) 13093 return -ENOMEM; 13094 state->curframe = 0; 13095 state->speculative = false; 13096 state->branches = 1; 13097 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 13098 if (!state->frame[0]) { 13099 kfree(state); 13100 return -ENOMEM; 13101 } 13102 env->cur_state = state; 13103 init_func_state(env, state->frame[0], 13104 BPF_MAIN_FUNC /* callsite */, 13105 0 /* frameno */, 13106 subprog); 13107 13108 regs = state->frame[state->curframe]->regs; 13109 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 13110 ret = btf_prepare_func_args(env, subprog, regs); 13111 if (ret) 13112 goto out; 13113 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 13114 if (regs[i].type == PTR_TO_CTX) 13115 mark_reg_known_zero(env, regs, i); 13116 else if (regs[i].type == SCALAR_VALUE) 13117 mark_reg_unknown(env, regs, i); 13118 else if (regs[i].type == PTR_TO_MEM_OR_NULL) { 13119 const u32 mem_size = regs[i].mem_size; 13120 13121 mark_reg_known_zero(env, regs, i); 13122 regs[i].mem_size = mem_size; 13123 regs[i].id = ++env->id_gen; 13124 } 13125 } 13126 } else { 13127 /* 1st arg to a function */ 13128 regs[BPF_REG_1].type = PTR_TO_CTX; 13129 mark_reg_known_zero(env, regs, BPF_REG_1); 13130 ret = btf_check_subprog_arg_match(env, subprog, regs); 13131 if (ret == -EFAULT) 13132 /* unlikely verifier bug. abort. 13133 * ret == 0 and ret < 0 are sadly acceptable for 13134 * main() function due to backward compatibility. 13135 * Like socket filter program may be written as: 13136 * int bpf_prog(struct pt_regs *ctx) 13137 * and never dereference that ctx in the program. 13138 * 'struct pt_regs' is a type mismatch for socket 13139 * filter that should be using 'struct __sk_buff'. 13140 */ 13141 goto out; 13142 } 13143 13144 ret = do_check(env); 13145 out: 13146 /* check for NULL is necessary, since cur_state can be freed inside 13147 * do_check() under memory pressure. 13148 */ 13149 if (env->cur_state) { 13150 free_verifier_state(env->cur_state, true); 13151 env->cur_state = NULL; 13152 } 13153 while (!pop_stack(env, NULL, NULL, false)); 13154 if (!ret && pop_log) 13155 bpf_vlog_reset(&env->log, 0); 13156 free_states(env); 13157 return ret; 13158 } 13159 13160 /* Verify all global functions in a BPF program one by one based on their BTF. 13161 * All global functions must pass verification. Otherwise the whole program is rejected. 13162 * Consider: 13163 * int bar(int); 13164 * int foo(int f) 13165 * { 13166 * return bar(f); 13167 * } 13168 * int bar(int b) 13169 * { 13170 * ... 13171 * } 13172 * foo() will be verified first for R1=any_scalar_value. During verification it 13173 * will be assumed that bar() already verified successfully and call to bar() 13174 * from foo() will be checked for type match only. Later bar() will be verified 13175 * independently to check that it's safe for R1=any_scalar_value. 13176 */ 13177 static int do_check_subprogs(struct bpf_verifier_env *env) 13178 { 13179 struct bpf_prog_aux *aux = env->prog->aux; 13180 int i, ret; 13181 13182 if (!aux->func_info) 13183 return 0; 13184 13185 for (i = 1; i < env->subprog_cnt; i++) { 13186 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 13187 continue; 13188 env->insn_idx = env->subprog_info[i].start; 13189 WARN_ON_ONCE(env->insn_idx == 0); 13190 ret = do_check_common(env, i); 13191 if (ret) { 13192 return ret; 13193 } else if (env->log.level & BPF_LOG_LEVEL) { 13194 verbose(env, 13195 "Func#%d is safe for any args that match its prototype\n", 13196 i); 13197 } 13198 } 13199 return 0; 13200 } 13201 13202 static int do_check_main(struct bpf_verifier_env *env) 13203 { 13204 int ret; 13205 13206 env->insn_idx = 0; 13207 ret = do_check_common(env, 0); 13208 if (!ret) 13209 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 13210 return ret; 13211 } 13212 13213 13214 static void print_verification_stats(struct bpf_verifier_env *env) 13215 { 13216 int i; 13217 13218 if (env->log.level & BPF_LOG_STATS) { 13219 verbose(env, "verification time %lld usec\n", 13220 div_u64(env->verification_time, 1000)); 13221 verbose(env, "stack depth "); 13222 for (i = 0; i < env->subprog_cnt; i++) { 13223 u32 depth = env->subprog_info[i].stack_depth; 13224 13225 verbose(env, "%d", depth); 13226 if (i + 1 < env->subprog_cnt) 13227 verbose(env, "+"); 13228 } 13229 verbose(env, "\n"); 13230 } 13231 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 13232 "total_states %d peak_states %d mark_read %d\n", 13233 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 13234 env->max_states_per_insn, env->total_states, 13235 env->peak_states, env->longest_mark_read_walk); 13236 } 13237 13238 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 13239 { 13240 const struct btf_type *t, *func_proto; 13241 const struct bpf_struct_ops *st_ops; 13242 const struct btf_member *member; 13243 struct bpf_prog *prog = env->prog; 13244 u32 btf_id, member_idx; 13245 const char *mname; 13246 13247 if (!prog->gpl_compatible) { 13248 verbose(env, "struct ops programs must have a GPL compatible license\n"); 13249 return -EINVAL; 13250 } 13251 13252 btf_id = prog->aux->attach_btf_id; 13253 st_ops = bpf_struct_ops_find(btf_id); 13254 if (!st_ops) { 13255 verbose(env, "attach_btf_id %u is not a supported struct\n", 13256 btf_id); 13257 return -ENOTSUPP; 13258 } 13259 13260 t = st_ops->type; 13261 member_idx = prog->expected_attach_type; 13262 if (member_idx >= btf_type_vlen(t)) { 13263 verbose(env, "attach to invalid member idx %u of struct %s\n", 13264 member_idx, st_ops->name); 13265 return -EINVAL; 13266 } 13267 13268 member = &btf_type_member(t)[member_idx]; 13269 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 13270 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 13271 NULL); 13272 if (!func_proto) { 13273 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 13274 mname, member_idx, st_ops->name); 13275 return -EINVAL; 13276 } 13277 13278 if (st_ops->check_member) { 13279 int err = st_ops->check_member(t, member); 13280 13281 if (err) { 13282 verbose(env, "attach to unsupported member %s of struct %s\n", 13283 mname, st_ops->name); 13284 return err; 13285 } 13286 } 13287 13288 prog->aux->attach_func_proto = func_proto; 13289 prog->aux->attach_func_name = mname; 13290 env->ops = st_ops->verifier_ops; 13291 13292 return 0; 13293 } 13294 #define SECURITY_PREFIX "security_" 13295 13296 static int check_attach_modify_return(unsigned long addr, const char *func_name) 13297 { 13298 if (within_error_injection_list(addr) || 13299 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 13300 return 0; 13301 13302 return -EINVAL; 13303 } 13304 13305 /* list of non-sleepable functions that are otherwise on 13306 * ALLOW_ERROR_INJECTION list 13307 */ 13308 BTF_SET_START(btf_non_sleepable_error_inject) 13309 /* Three functions below can be called from sleepable and non-sleepable context. 13310 * Assume non-sleepable from bpf safety point of view. 13311 */ 13312 BTF_ID(func, __add_to_page_cache_locked) 13313 BTF_ID(func, should_fail_alloc_page) 13314 BTF_ID(func, should_failslab) 13315 BTF_SET_END(btf_non_sleepable_error_inject) 13316 13317 static int check_non_sleepable_error_inject(u32 btf_id) 13318 { 13319 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 13320 } 13321 13322 int bpf_check_attach_target(struct bpf_verifier_log *log, 13323 const struct bpf_prog *prog, 13324 const struct bpf_prog *tgt_prog, 13325 u32 btf_id, 13326 struct bpf_attach_target_info *tgt_info) 13327 { 13328 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 13329 const char prefix[] = "btf_trace_"; 13330 int ret = 0, subprog = -1, i; 13331 const struct btf_type *t; 13332 bool conservative = true; 13333 const char *tname; 13334 struct btf *btf; 13335 long addr = 0; 13336 13337 if (!btf_id) { 13338 bpf_log(log, "Tracing programs must provide btf_id\n"); 13339 return -EINVAL; 13340 } 13341 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 13342 if (!btf) { 13343 bpf_log(log, 13344 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 13345 return -EINVAL; 13346 } 13347 t = btf_type_by_id(btf, btf_id); 13348 if (!t) { 13349 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 13350 return -EINVAL; 13351 } 13352 tname = btf_name_by_offset(btf, t->name_off); 13353 if (!tname) { 13354 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 13355 return -EINVAL; 13356 } 13357 if (tgt_prog) { 13358 struct bpf_prog_aux *aux = tgt_prog->aux; 13359 13360 for (i = 0; i < aux->func_info_cnt; i++) 13361 if (aux->func_info[i].type_id == btf_id) { 13362 subprog = i; 13363 break; 13364 } 13365 if (subprog == -1) { 13366 bpf_log(log, "Subprog %s doesn't exist\n", tname); 13367 return -EINVAL; 13368 } 13369 conservative = aux->func_info_aux[subprog].unreliable; 13370 if (prog_extension) { 13371 if (conservative) { 13372 bpf_log(log, 13373 "Cannot replace static functions\n"); 13374 return -EINVAL; 13375 } 13376 if (!prog->jit_requested) { 13377 bpf_log(log, 13378 "Extension programs should be JITed\n"); 13379 return -EINVAL; 13380 } 13381 } 13382 if (!tgt_prog->jited) { 13383 bpf_log(log, "Can attach to only JITed progs\n"); 13384 return -EINVAL; 13385 } 13386 if (tgt_prog->type == prog->type) { 13387 /* Cannot fentry/fexit another fentry/fexit program. 13388 * Cannot attach program extension to another extension. 13389 * It's ok to attach fentry/fexit to extension program. 13390 */ 13391 bpf_log(log, "Cannot recursively attach\n"); 13392 return -EINVAL; 13393 } 13394 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 13395 prog_extension && 13396 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 13397 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 13398 /* Program extensions can extend all program types 13399 * except fentry/fexit. The reason is the following. 13400 * The fentry/fexit programs are used for performance 13401 * analysis, stats and can be attached to any program 13402 * type except themselves. When extension program is 13403 * replacing XDP function it is necessary to allow 13404 * performance analysis of all functions. Both original 13405 * XDP program and its program extension. Hence 13406 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 13407 * allowed. If extending of fentry/fexit was allowed it 13408 * would be possible to create long call chain 13409 * fentry->extension->fentry->extension beyond 13410 * reasonable stack size. Hence extending fentry is not 13411 * allowed. 13412 */ 13413 bpf_log(log, "Cannot extend fentry/fexit\n"); 13414 return -EINVAL; 13415 } 13416 } else { 13417 if (prog_extension) { 13418 bpf_log(log, "Cannot replace kernel functions\n"); 13419 return -EINVAL; 13420 } 13421 } 13422 13423 switch (prog->expected_attach_type) { 13424 case BPF_TRACE_RAW_TP: 13425 if (tgt_prog) { 13426 bpf_log(log, 13427 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 13428 return -EINVAL; 13429 } 13430 if (!btf_type_is_typedef(t)) { 13431 bpf_log(log, "attach_btf_id %u is not a typedef\n", 13432 btf_id); 13433 return -EINVAL; 13434 } 13435 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 13436 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 13437 btf_id, tname); 13438 return -EINVAL; 13439 } 13440 tname += sizeof(prefix) - 1; 13441 t = btf_type_by_id(btf, t->type); 13442 if (!btf_type_is_ptr(t)) 13443 /* should never happen in valid vmlinux build */ 13444 return -EINVAL; 13445 t = btf_type_by_id(btf, t->type); 13446 if (!btf_type_is_func_proto(t)) 13447 /* should never happen in valid vmlinux build */ 13448 return -EINVAL; 13449 13450 break; 13451 case BPF_TRACE_ITER: 13452 if (!btf_type_is_func(t)) { 13453 bpf_log(log, "attach_btf_id %u is not a function\n", 13454 btf_id); 13455 return -EINVAL; 13456 } 13457 t = btf_type_by_id(btf, t->type); 13458 if (!btf_type_is_func_proto(t)) 13459 return -EINVAL; 13460 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 13461 if (ret) 13462 return ret; 13463 break; 13464 default: 13465 if (!prog_extension) 13466 return -EINVAL; 13467 fallthrough; 13468 case BPF_MODIFY_RETURN: 13469 case BPF_LSM_MAC: 13470 case BPF_TRACE_FENTRY: 13471 case BPF_TRACE_FEXIT: 13472 if (!btf_type_is_func(t)) { 13473 bpf_log(log, "attach_btf_id %u is not a function\n", 13474 btf_id); 13475 return -EINVAL; 13476 } 13477 if (prog_extension && 13478 btf_check_type_match(log, prog, btf, t)) 13479 return -EINVAL; 13480 t = btf_type_by_id(btf, t->type); 13481 if (!btf_type_is_func_proto(t)) 13482 return -EINVAL; 13483 13484 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 13485 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 13486 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 13487 return -EINVAL; 13488 13489 if (tgt_prog && conservative) 13490 t = NULL; 13491 13492 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 13493 if (ret < 0) 13494 return ret; 13495 13496 if (tgt_prog) { 13497 if (subprog == 0) 13498 addr = (long) tgt_prog->bpf_func; 13499 else 13500 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 13501 } else { 13502 addr = kallsyms_lookup_name(tname); 13503 if (!addr) { 13504 bpf_log(log, 13505 "The address of function %s cannot be found\n", 13506 tname); 13507 return -ENOENT; 13508 } 13509 } 13510 13511 if (prog->aux->sleepable) { 13512 ret = -EINVAL; 13513 switch (prog->type) { 13514 case BPF_PROG_TYPE_TRACING: 13515 /* fentry/fexit/fmod_ret progs can be sleepable only if they are 13516 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 13517 */ 13518 if (!check_non_sleepable_error_inject(btf_id) && 13519 within_error_injection_list(addr)) 13520 ret = 0; 13521 break; 13522 case BPF_PROG_TYPE_LSM: 13523 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 13524 * Only some of them are sleepable. 13525 */ 13526 if (bpf_lsm_is_sleepable_hook(btf_id)) 13527 ret = 0; 13528 break; 13529 default: 13530 break; 13531 } 13532 if (ret) { 13533 bpf_log(log, "%s is not sleepable\n", tname); 13534 return ret; 13535 } 13536 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 13537 if (tgt_prog) { 13538 bpf_log(log, "can't modify return codes of BPF programs\n"); 13539 return -EINVAL; 13540 } 13541 ret = check_attach_modify_return(addr, tname); 13542 if (ret) { 13543 bpf_log(log, "%s() is not modifiable\n", tname); 13544 return ret; 13545 } 13546 } 13547 13548 break; 13549 } 13550 tgt_info->tgt_addr = addr; 13551 tgt_info->tgt_name = tname; 13552 tgt_info->tgt_type = t; 13553 return 0; 13554 } 13555 13556 BTF_SET_START(btf_id_deny) 13557 BTF_ID_UNUSED 13558 #ifdef CONFIG_SMP 13559 BTF_ID(func, migrate_disable) 13560 BTF_ID(func, migrate_enable) 13561 #endif 13562 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 13563 BTF_ID(func, rcu_read_unlock_strict) 13564 #endif 13565 BTF_SET_END(btf_id_deny) 13566 13567 static int check_attach_btf_id(struct bpf_verifier_env *env) 13568 { 13569 struct bpf_prog *prog = env->prog; 13570 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 13571 struct bpf_attach_target_info tgt_info = {}; 13572 u32 btf_id = prog->aux->attach_btf_id; 13573 struct bpf_trampoline *tr; 13574 int ret; 13575 u64 key; 13576 13577 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 13578 if (prog->aux->sleepable) 13579 /* attach_btf_id checked to be zero already */ 13580 return 0; 13581 verbose(env, "Syscall programs can only be sleepable\n"); 13582 return -EINVAL; 13583 } 13584 13585 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && 13586 prog->type != BPF_PROG_TYPE_LSM) { 13587 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); 13588 return -EINVAL; 13589 } 13590 13591 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 13592 return check_struct_ops_btf_id(env); 13593 13594 if (prog->type != BPF_PROG_TYPE_TRACING && 13595 prog->type != BPF_PROG_TYPE_LSM && 13596 prog->type != BPF_PROG_TYPE_EXT) 13597 return 0; 13598 13599 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 13600 if (ret) 13601 return ret; 13602 13603 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 13604 /* to make freplace equivalent to their targets, they need to 13605 * inherit env->ops and expected_attach_type for the rest of the 13606 * verification 13607 */ 13608 env->ops = bpf_verifier_ops[tgt_prog->type]; 13609 prog->expected_attach_type = tgt_prog->expected_attach_type; 13610 } 13611 13612 /* store info about the attachment target that will be used later */ 13613 prog->aux->attach_func_proto = tgt_info.tgt_type; 13614 prog->aux->attach_func_name = tgt_info.tgt_name; 13615 13616 if (tgt_prog) { 13617 prog->aux->saved_dst_prog_type = tgt_prog->type; 13618 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 13619 } 13620 13621 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 13622 prog->aux->attach_btf_trace = true; 13623 return 0; 13624 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 13625 if (!bpf_iter_prog_supported(prog)) 13626 return -EINVAL; 13627 return 0; 13628 } 13629 13630 if (prog->type == BPF_PROG_TYPE_LSM) { 13631 ret = bpf_lsm_verify_prog(&env->log, prog); 13632 if (ret < 0) 13633 return ret; 13634 } else if (prog->type == BPF_PROG_TYPE_TRACING && 13635 btf_id_set_contains(&btf_id_deny, btf_id)) { 13636 return -EINVAL; 13637 } 13638 13639 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 13640 tr = bpf_trampoline_get(key, &tgt_info); 13641 if (!tr) 13642 return -ENOMEM; 13643 13644 prog->aux->dst_trampoline = tr; 13645 return 0; 13646 } 13647 13648 struct btf *bpf_get_btf_vmlinux(void) 13649 { 13650 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 13651 mutex_lock(&bpf_verifier_lock); 13652 if (!btf_vmlinux) 13653 btf_vmlinux = btf_parse_vmlinux(); 13654 mutex_unlock(&bpf_verifier_lock); 13655 } 13656 return btf_vmlinux; 13657 } 13658 13659 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) 13660 { 13661 u64 start_time = ktime_get_ns(); 13662 struct bpf_verifier_env *env; 13663 struct bpf_verifier_log *log; 13664 int i, len, ret = -EINVAL; 13665 bool is_priv; 13666 13667 /* no program is valid */ 13668 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 13669 return -EINVAL; 13670 13671 /* 'struct bpf_verifier_env' can be global, but since it's not small, 13672 * allocate/free it every time bpf_check() is called 13673 */ 13674 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 13675 if (!env) 13676 return -ENOMEM; 13677 log = &env->log; 13678 13679 len = (*prog)->len; 13680 env->insn_aux_data = 13681 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 13682 ret = -ENOMEM; 13683 if (!env->insn_aux_data) 13684 goto err_free_env; 13685 for (i = 0; i < len; i++) 13686 env->insn_aux_data[i].orig_idx = i; 13687 env->prog = *prog; 13688 env->ops = bpf_verifier_ops[env->prog->type]; 13689 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 13690 is_priv = bpf_capable(); 13691 13692 bpf_get_btf_vmlinux(); 13693 13694 /* grab the mutex to protect few globals used by verifier */ 13695 if (!is_priv) 13696 mutex_lock(&bpf_verifier_lock); 13697 13698 if (attr->log_level || attr->log_buf || attr->log_size) { 13699 /* user requested verbose verifier output 13700 * and supplied buffer to store the verification trace 13701 */ 13702 log->level = attr->log_level; 13703 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 13704 log->len_total = attr->log_size; 13705 13706 ret = -EINVAL; 13707 /* log attributes have to be sane */ 13708 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || 13709 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) 13710 goto err_unlock; 13711 } 13712 13713 if (IS_ERR(btf_vmlinux)) { 13714 /* Either gcc or pahole or kernel are broken. */ 13715 verbose(env, "in-kernel BTF is malformed\n"); 13716 ret = PTR_ERR(btf_vmlinux); 13717 goto skip_full_check; 13718 } 13719 13720 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 13721 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 13722 env->strict_alignment = true; 13723 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 13724 env->strict_alignment = false; 13725 13726 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 13727 env->allow_uninit_stack = bpf_allow_uninit_stack(); 13728 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); 13729 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 13730 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 13731 env->bpf_capable = bpf_capable(); 13732 13733 if (is_priv) 13734 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 13735 13736 env->explored_states = kvcalloc(state_htab_size(env), 13737 sizeof(struct bpf_verifier_state_list *), 13738 GFP_USER); 13739 ret = -ENOMEM; 13740 if (!env->explored_states) 13741 goto skip_full_check; 13742 13743 ret = add_subprog_and_kfunc(env); 13744 if (ret < 0) 13745 goto skip_full_check; 13746 13747 ret = check_subprogs(env); 13748 if (ret < 0) 13749 goto skip_full_check; 13750 13751 ret = check_btf_info(env, attr, uattr); 13752 if (ret < 0) 13753 goto skip_full_check; 13754 13755 ret = check_attach_btf_id(env); 13756 if (ret) 13757 goto skip_full_check; 13758 13759 ret = resolve_pseudo_ldimm64(env); 13760 if (ret < 0) 13761 goto skip_full_check; 13762 13763 if (bpf_prog_is_dev_bound(env->prog->aux)) { 13764 ret = bpf_prog_offload_verifier_prep(env->prog); 13765 if (ret) 13766 goto skip_full_check; 13767 } 13768 13769 ret = check_cfg(env); 13770 if (ret < 0) 13771 goto skip_full_check; 13772 13773 ret = do_check_subprogs(env); 13774 ret = ret ?: do_check_main(env); 13775 13776 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 13777 ret = bpf_prog_offload_finalize(env); 13778 13779 skip_full_check: 13780 kvfree(env->explored_states); 13781 13782 if (ret == 0) 13783 ret = check_max_stack_depth(env); 13784 13785 /* instruction rewrites happen after this point */ 13786 if (is_priv) { 13787 if (ret == 0) 13788 opt_hard_wire_dead_code_branches(env); 13789 if (ret == 0) 13790 ret = opt_remove_dead_code(env); 13791 if (ret == 0) 13792 ret = opt_remove_nops(env); 13793 } else { 13794 if (ret == 0) 13795 sanitize_dead_code(env); 13796 } 13797 13798 if (ret == 0) 13799 /* program is valid, convert *(u32*)(ctx + off) accesses */ 13800 ret = convert_ctx_accesses(env); 13801 13802 if (ret == 0) 13803 ret = do_misc_fixups(env); 13804 13805 /* do 32-bit optimization after insn patching has done so those patched 13806 * insns could be handled correctly. 13807 */ 13808 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { 13809 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 13810 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 13811 : false; 13812 } 13813 13814 if (ret == 0) 13815 ret = fixup_call_args(env); 13816 13817 env->verification_time = ktime_get_ns() - start_time; 13818 print_verification_stats(env); 13819 13820 if (log->level && bpf_verifier_log_full(log)) 13821 ret = -ENOSPC; 13822 if (log->level && !log->ubuf) { 13823 ret = -EFAULT; 13824 goto err_release_maps; 13825 } 13826 13827 if (ret) 13828 goto err_release_maps; 13829 13830 if (env->used_map_cnt) { 13831 /* if program passed verifier, update used_maps in bpf_prog_info */ 13832 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 13833 sizeof(env->used_maps[0]), 13834 GFP_KERNEL); 13835 13836 if (!env->prog->aux->used_maps) { 13837 ret = -ENOMEM; 13838 goto err_release_maps; 13839 } 13840 13841 memcpy(env->prog->aux->used_maps, env->used_maps, 13842 sizeof(env->used_maps[0]) * env->used_map_cnt); 13843 env->prog->aux->used_map_cnt = env->used_map_cnt; 13844 } 13845 if (env->used_btf_cnt) { 13846 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 13847 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 13848 sizeof(env->used_btfs[0]), 13849 GFP_KERNEL); 13850 if (!env->prog->aux->used_btfs) { 13851 ret = -ENOMEM; 13852 goto err_release_maps; 13853 } 13854 13855 memcpy(env->prog->aux->used_btfs, env->used_btfs, 13856 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 13857 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 13858 } 13859 if (env->used_map_cnt || env->used_btf_cnt) { 13860 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 13861 * bpf_ld_imm64 instructions 13862 */ 13863 convert_pseudo_ld_imm64(env); 13864 } 13865 13866 adjust_btf_func(env); 13867 13868 err_release_maps: 13869 if (!env->prog->aux->used_maps) 13870 /* if we didn't copy map pointers into bpf_prog_info, release 13871 * them now. Otherwise free_used_maps() will release them. 13872 */ 13873 release_maps(env); 13874 if (!env->prog->aux->used_btfs) 13875 release_btfs(env); 13876 13877 /* extension progs temporarily inherit the attach_type of their targets 13878 for verification purposes, so set it back to zero before returning 13879 */ 13880 if (env->prog->type == BPF_PROG_TYPE_EXT) 13881 env->prog->expected_attach_type = 0; 13882 13883 *prog = env->prog; 13884 err_unlock: 13885 if (!is_priv) 13886 mutex_unlock(&bpf_verifier_lock); 13887 vfree(env->insn_aux_data); 13888 err_free_env: 13889 kfree(env); 13890 return ret; 13891 } 13892