1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <uapi/linux/btf.h> 5 #include <uapi/linux/bpf.h> 6 #include <uapi/linux/bpf_perf_event.h> 7 #include <uapi/linux/types.h> 8 #include <linux/seq_file.h> 9 #include <linux/compiler.h> 10 #include <linux/ctype.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/anon_inodes.h> 14 #include <linux/file.h> 15 #include <linux/uaccess.h> 16 #include <linux/kernel.h> 17 #include <linux/idr.h> 18 #include <linux/sort.h> 19 #include <linux/bpf_verifier.h> 20 #include <linux/btf.h> 21 #include <linux/btf_ids.h> 22 #include <linux/bpf_lsm.h> 23 #include <linux/skmsg.h> 24 #include <linux/perf_event.h> 25 #include <linux/bsearch.h> 26 #include <linux/kobject.h> 27 #include <linux/sysfs.h> 28 29 #include <net/netfilter/nf_bpf_link.h> 30 31 #include <net/sock.h> 32 #include "../tools/lib/bpf/relo_core.h" 33 34 /* BTF (BPF Type Format) is the meta data format which describes 35 * the data types of BPF program/map. Hence, it basically focus 36 * on the C programming language which the modern BPF is primary 37 * using. 38 * 39 * ELF Section: 40 * ~~~~~~~~~~~ 41 * The BTF data is stored under the ".BTF" ELF section 42 * 43 * struct btf_type: 44 * ~~~~~~~~~~~~~~~ 45 * Each 'struct btf_type' object describes a C data type. 46 * Depending on the type it is describing, a 'struct btf_type' 47 * object may be followed by more data. F.e. 48 * To describe an array, 'struct btf_type' is followed by 49 * 'struct btf_array'. 50 * 51 * 'struct btf_type' and any extra data following it are 52 * 4 bytes aligned. 53 * 54 * Type section: 55 * ~~~~~~~~~~~~~ 56 * The BTF type section contains a list of 'struct btf_type' objects. 57 * Each one describes a C type. Recall from the above section 58 * that a 'struct btf_type' object could be immediately followed by extra 59 * data in order to describe some particular C types. 60 * 61 * type_id: 62 * ~~~~~~~ 63 * Each btf_type object is identified by a type_id. The type_id 64 * is implicitly implied by the location of the btf_type object in 65 * the BTF type section. The first one has type_id 1. The second 66 * one has type_id 2...etc. Hence, an earlier btf_type has 67 * a smaller type_id. 68 * 69 * A btf_type object may refer to another btf_type object by using 70 * type_id (i.e. the "type" in the "struct btf_type"). 71 * 72 * NOTE that we cannot assume any reference-order. 73 * A btf_type object can refer to an earlier btf_type object 74 * but it can also refer to a later btf_type object. 75 * 76 * For example, to describe "const void *". A btf_type 77 * object describing "const" may refer to another btf_type 78 * object describing "void *". This type-reference is done 79 * by specifying type_id: 80 * 81 * [1] CONST (anon) type_id=2 82 * [2] PTR (anon) type_id=0 83 * 84 * The above is the btf_verifier debug log: 85 * - Each line started with "[?]" is a btf_type object 86 * - [?] is the type_id of the btf_type object. 87 * - CONST/PTR is the BTF_KIND_XXX 88 * - "(anon)" is the name of the type. It just 89 * happens that CONST and PTR has no name. 90 * - type_id=XXX is the 'u32 type' in btf_type 91 * 92 * NOTE: "void" has type_id 0 93 * 94 * String section: 95 * ~~~~~~~~~~~~~~ 96 * The BTF string section contains the names used by the type section. 97 * Each string is referred by an "offset" from the beginning of the 98 * string section. 99 * 100 * Each string is '\0' terminated. 101 * 102 * The first character in the string section must be '\0' 103 * which is used to mean 'anonymous'. Some btf_type may not 104 * have a name. 105 */ 106 107 /* BTF verification: 108 * 109 * To verify BTF data, two passes are needed. 110 * 111 * Pass #1 112 * ~~~~~~~ 113 * The first pass is to collect all btf_type objects to 114 * an array: "btf->types". 115 * 116 * Depending on the C type that a btf_type is describing, 117 * a btf_type may be followed by extra data. We don't know 118 * how many btf_type is there, and more importantly we don't 119 * know where each btf_type is located in the type section. 120 * 121 * Without knowing the location of each type_id, most verifications 122 * cannot be done. e.g. an earlier btf_type may refer to a later 123 * btf_type (recall the "const void *" above), so we cannot 124 * check this type-reference in the first pass. 125 * 126 * In the first pass, it still does some verifications (e.g. 127 * checking the name is a valid offset to the string section). 128 * 129 * Pass #2 130 * ~~~~~~~ 131 * The main focus is to resolve a btf_type that is referring 132 * to another type. 133 * 134 * We have to ensure the referring type: 135 * 1) does exist in the BTF (i.e. in btf->types[]) 136 * 2) does not cause a loop: 137 * struct A { 138 * struct B b; 139 * }; 140 * 141 * struct B { 142 * struct A a; 143 * }; 144 * 145 * btf_type_needs_resolve() decides if a btf_type needs 146 * to be resolved. 147 * 148 * The needs_resolve type implements the "resolve()" ops which 149 * essentially does a DFS and detects backedge. 150 * 151 * During resolve (or DFS), different C types have different 152 * "RESOLVED" conditions. 153 * 154 * When resolving a BTF_KIND_STRUCT, we need to resolve all its 155 * members because a member is always referring to another 156 * type. A struct's member can be treated as "RESOLVED" if 157 * it is referring to a BTF_KIND_PTR. Otherwise, the 158 * following valid C struct would be rejected: 159 * 160 * struct A { 161 * int m; 162 * struct A *a; 163 * }; 164 * 165 * When resolving a BTF_KIND_PTR, it needs to keep resolving if 166 * it is referring to another BTF_KIND_PTR. Otherwise, we cannot 167 * detect a pointer loop, e.g.: 168 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR + 169 * ^ | 170 * +-----------------------------------------+ 171 * 172 */ 173 174 #define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2) 175 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) 176 #define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK) 177 #define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3) 178 #define BITS_ROUNDUP_BYTES(bits) \ 179 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) 180 181 #define BTF_INFO_MASK 0x9f00ffff 182 #define BTF_INT_MASK 0x0fffffff 183 #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE) 184 #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET) 185 186 /* 16MB for 64k structs and each has 16 members and 187 * a few MB spaces for the string section. 188 * The hard limit is S32_MAX. 189 */ 190 #define BTF_MAX_SIZE (16 * 1024 * 1024) 191 192 #define for_each_member_from(i, from, struct_type, member) \ 193 for (i = from, member = btf_type_member(struct_type) + from; \ 194 i < btf_type_vlen(struct_type); \ 195 i++, member++) 196 197 #define for_each_vsi_from(i, from, struct_type, member) \ 198 for (i = from, member = btf_type_var_secinfo(struct_type) + from; \ 199 i < btf_type_vlen(struct_type); \ 200 i++, member++) 201 202 DEFINE_IDR(btf_idr); 203 DEFINE_SPINLOCK(btf_idr_lock); 204 205 enum btf_kfunc_hook { 206 BTF_KFUNC_HOOK_COMMON, 207 BTF_KFUNC_HOOK_XDP, 208 BTF_KFUNC_HOOK_TC, 209 BTF_KFUNC_HOOK_STRUCT_OPS, 210 BTF_KFUNC_HOOK_TRACING, 211 BTF_KFUNC_HOOK_SYSCALL, 212 BTF_KFUNC_HOOK_FMODRET, 213 BTF_KFUNC_HOOK_CGROUP_SKB, 214 BTF_KFUNC_HOOK_SCHED_ACT, 215 BTF_KFUNC_HOOK_SK_SKB, 216 BTF_KFUNC_HOOK_SOCKET_FILTER, 217 BTF_KFUNC_HOOK_LWT, 218 BTF_KFUNC_HOOK_NETFILTER, 219 BTF_KFUNC_HOOK_MAX, 220 }; 221 222 enum { 223 BTF_KFUNC_SET_MAX_CNT = 256, 224 BTF_DTOR_KFUNC_MAX_CNT = 256, 225 BTF_KFUNC_FILTER_MAX_CNT = 16, 226 }; 227 228 struct btf_kfunc_hook_filter { 229 btf_kfunc_filter_t filters[BTF_KFUNC_FILTER_MAX_CNT]; 230 u32 nr_filters; 231 }; 232 233 struct btf_kfunc_set_tab { 234 struct btf_id_set8 *sets[BTF_KFUNC_HOOK_MAX]; 235 struct btf_kfunc_hook_filter hook_filters[BTF_KFUNC_HOOK_MAX]; 236 }; 237 238 struct btf_id_dtor_kfunc_tab { 239 u32 cnt; 240 struct btf_id_dtor_kfunc dtors[]; 241 }; 242 243 struct btf { 244 void *data; 245 struct btf_type **types; 246 u32 *resolved_ids; 247 u32 *resolved_sizes; 248 const char *strings; 249 void *nohdr_data; 250 struct btf_header hdr; 251 u32 nr_types; /* includes VOID for base BTF */ 252 u32 types_size; 253 u32 data_size; 254 refcount_t refcnt; 255 u32 id; 256 struct rcu_head rcu; 257 struct btf_kfunc_set_tab *kfunc_set_tab; 258 struct btf_id_dtor_kfunc_tab *dtor_kfunc_tab; 259 struct btf_struct_metas *struct_meta_tab; 260 261 /* split BTF support */ 262 struct btf *base_btf; 263 u32 start_id; /* first type ID in this BTF (0 for base BTF) */ 264 u32 start_str_off; /* first string offset (0 for base BTF) */ 265 char name[MODULE_NAME_LEN]; 266 bool kernel_btf; 267 }; 268 269 enum verifier_phase { 270 CHECK_META, 271 CHECK_TYPE, 272 }; 273 274 struct resolve_vertex { 275 const struct btf_type *t; 276 u32 type_id; 277 u16 next_member; 278 }; 279 280 enum visit_state { 281 NOT_VISITED, 282 VISITED, 283 RESOLVED, 284 }; 285 286 enum resolve_mode { 287 RESOLVE_TBD, /* To Be Determined */ 288 RESOLVE_PTR, /* Resolving for Pointer */ 289 RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union 290 * or array 291 */ 292 }; 293 294 #define MAX_RESOLVE_DEPTH 32 295 296 struct btf_sec_info { 297 u32 off; 298 u32 len; 299 }; 300 301 struct btf_verifier_env { 302 struct btf *btf; 303 u8 *visit_states; 304 struct resolve_vertex stack[MAX_RESOLVE_DEPTH]; 305 struct bpf_verifier_log log; 306 u32 log_type_id; 307 u32 top_stack; 308 enum verifier_phase phase; 309 enum resolve_mode resolve_mode; 310 }; 311 312 static const char * const btf_kind_str[NR_BTF_KINDS] = { 313 [BTF_KIND_UNKN] = "UNKNOWN", 314 [BTF_KIND_INT] = "INT", 315 [BTF_KIND_PTR] = "PTR", 316 [BTF_KIND_ARRAY] = "ARRAY", 317 [BTF_KIND_STRUCT] = "STRUCT", 318 [BTF_KIND_UNION] = "UNION", 319 [BTF_KIND_ENUM] = "ENUM", 320 [BTF_KIND_FWD] = "FWD", 321 [BTF_KIND_TYPEDEF] = "TYPEDEF", 322 [BTF_KIND_VOLATILE] = "VOLATILE", 323 [BTF_KIND_CONST] = "CONST", 324 [BTF_KIND_RESTRICT] = "RESTRICT", 325 [BTF_KIND_FUNC] = "FUNC", 326 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", 327 [BTF_KIND_VAR] = "VAR", 328 [BTF_KIND_DATASEC] = "DATASEC", 329 [BTF_KIND_FLOAT] = "FLOAT", 330 [BTF_KIND_DECL_TAG] = "DECL_TAG", 331 [BTF_KIND_TYPE_TAG] = "TYPE_TAG", 332 [BTF_KIND_ENUM64] = "ENUM64", 333 }; 334 335 const char *btf_type_str(const struct btf_type *t) 336 { 337 return btf_kind_str[BTF_INFO_KIND(t->info)]; 338 } 339 340 /* Chunk size we use in safe copy of data to be shown. */ 341 #define BTF_SHOW_OBJ_SAFE_SIZE 32 342 343 /* 344 * This is the maximum size of a base type value (equivalent to a 345 * 128-bit int); if we are at the end of our safe buffer and have 346 * less than 16 bytes space we can't be assured of being able 347 * to copy the next type safely, so in such cases we will initiate 348 * a new copy. 349 */ 350 #define BTF_SHOW_OBJ_BASE_TYPE_SIZE 16 351 352 /* Type name size */ 353 #define BTF_SHOW_NAME_SIZE 80 354 355 /* 356 * The suffix of a type that indicates it cannot alias another type when 357 * comparing BTF IDs for kfunc invocations. 358 */ 359 #define NOCAST_ALIAS_SUFFIX "___init" 360 361 /* 362 * Common data to all BTF show operations. Private show functions can add 363 * their own data to a structure containing a struct btf_show and consult it 364 * in the show callback. See btf_type_show() below. 365 * 366 * One challenge with showing nested data is we want to skip 0-valued 367 * data, but in order to figure out whether a nested object is all zeros 368 * we need to walk through it. As a result, we need to make two passes 369 * when handling structs, unions and arrays; the first path simply looks 370 * for nonzero data, while the second actually does the display. The first 371 * pass is signalled by show->state.depth_check being set, and if we 372 * encounter a non-zero value we set show->state.depth_to_show to 373 * the depth at which we encountered it. When we have completed the 374 * first pass, we will know if anything needs to be displayed if 375 * depth_to_show > depth. See btf_[struct,array]_show() for the 376 * implementation of this. 377 * 378 * Another problem is we want to ensure the data for display is safe to 379 * access. To support this, the anonymous "struct {} obj" tracks the data 380 * object and our safe copy of it. We copy portions of the data needed 381 * to the object "copy" buffer, but because its size is limited to 382 * BTF_SHOW_OBJ_COPY_LEN bytes, multiple copies may be required as we 383 * traverse larger objects for display. 384 * 385 * The various data type show functions all start with a call to 386 * btf_show_start_type() which returns a pointer to the safe copy 387 * of the data needed (or if BTF_SHOW_UNSAFE is specified, to the 388 * raw data itself). btf_show_obj_safe() is responsible for 389 * using copy_from_kernel_nofault() to update the safe data if necessary 390 * as we traverse the object's data. skbuff-like semantics are 391 * used: 392 * 393 * - obj.head points to the start of the toplevel object for display 394 * - obj.size is the size of the toplevel object 395 * - obj.data points to the current point in the original data at 396 * which our safe data starts. obj.data will advance as we copy 397 * portions of the data. 398 * 399 * In most cases a single copy will suffice, but larger data structures 400 * such as "struct task_struct" will require many copies. The logic in 401 * btf_show_obj_safe() handles the logic that determines if a new 402 * copy_from_kernel_nofault() is needed. 403 */ 404 struct btf_show { 405 u64 flags; 406 void *target; /* target of show operation (seq file, buffer) */ 407 void (*showfn)(struct btf_show *show, const char *fmt, va_list args); 408 const struct btf *btf; 409 /* below are used during iteration */ 410 struct { 411 u8 depth; 412 u8 depth_to_show; 413 u8 depth_check; 414 u8 array_member:1, 415 array_terminated:1; 416 u16 array_encoding; 417 u32 type_id; 418 int status; /* non-zero for error */ 419 const struct btf_type *type; 420 const struct btf_member *member; 421 char name[BTF_SHOW_NAME_SIZE]; /* space for member name/type */ 422 } state; 423 struct { 424 u32 size; 425 void *head; 426 void *data; 427 u8 safe[BTF_SHOW_OBJ_SAFE_SIZE]; 428 } obj; 429 }; 430 431 struct btf_kind_operations { 432 s32 (*check_meta)(struct btf_verifier_env *env, 433 const struct btf_type *t, 434 u32 meta_left); 435 int (*resolve)(struct btf_verifier_env *env, 436 const struct resolve_vertex *v); 437 int (*check_member)(struct btf_verifier_env *env, 438 const struct btf_type *struct_type, 439 const struct btf_member *member, 440 const struct btf_type *member_type); 441 int (*check_kflag_member)(struct btf_verifier_env *env, 442 const struct btf_type *struct_type, 443 const struct btf_member *member, 444 const struct btf_type *member_type); 445 void (*log_details)(struct btf_verifier_env *env, 446 const struct btf_type *t); 447 void (*show)(const struct btf *btf, const struct btf_type *t, 448 u32 type_id, void *data, u8 bits_offsets, 449 struct btf_show *show); 450 }; 451 452 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS]; 453 static struct btf_type btf_void; 454 455 static int btf_resolve(struct btf_verifier_env *env, 456 const struct btf_type *t, u32 type_id); 457 458 static int btf_func_check(struct btf_verifier_env *env, 459 const struct btf_type *t); 460 461 static bool btf_type_is_modifier(const struct btf_type *t) 462 { 463 /* Some of them is not strictly a C modifier 464 * but they are grouped into the same bucket 465 * for BTF concern: 466 * A type (t) that refers to another 467 * type through t->type AND its size cannot 468 * be determined without following the t->type. 469 * 470 * ptr does not fall into this bucket 471 * because its size is always sizeof(void *). 472 */ 473 switch (BTF_INFO_KIND(t->info)) { 474 case BTF_KIND_TYPEDEF: 475 case BTF_KIND_VOLATILE: 476 case BTF_KIND_CONST: 477 case BTF_KIND_RESTRICT: 478 case BTF_KIND_TYPE_TAG: 479 return true; 480 } 481 482 return false; 483 } 484 485 bool btf_type_is_void(const struct btf_type *t) 486 { 487 return t == &btf_void; 488 } 489 490 static bool btf_type_is_fwd(const struct btf_type *t) 491 { 492 return BTF_INFO_KIND(t->info) == BTF_KIND_FWD; 493 } 494 495 static bool btf_type_nosize(const struct btf_type *t) 496 { 497 return btf_type_is_void(t) || btf_type_is_fwd(t) || 498 btf_type_is_func(t) || btf_type_is_func_proto(t); 499 } 500 501 static bool btf_type_nosize_or_null(const struct btf_type *t) 502 { 503 return !t || btf_type_nosize(t); 504 } 505 506 static bool btf_type_is_datasec(const struct btf_type *t) 507 { 508 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; 509 } 510 511 static bool btf_type_is_decl_tag(const struct btf_type *t) 512 { 513 return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; 514 } 515 516 static bool btf_type_is_decl_tag_target(const struct btf_type *t) 517 { 518 return btf_type_is_func(t) || btf_type_is_struct(t) || 519 btf_type_is_var(t) || btf_type_is_typedef(t); 520 } 521 522 u32 btf_nr_types(const struct btf *btf) 523 { 524 u32 total = 0; 525 526 while (btf) { 527 total += btf->nr_types; 528 btf = btf->base_btf; 529 } 530 531 return total; 532 } 533 534 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind) 535 { 536 const struct btf_type *t; 537 const char *tname; 538 u32 i, total; 539 540 total = btf_nr_types(btf); 541 for (i = 1; i < total; i++) { 542 t = btf_type_by_id(btf, i); 543 if (BTF_INFO_KIND(t->info) != kind) 544 continue; 545 546 tname = btf_name_by_offset(btf, t->name_off); 547 if (!strcmp(tname, name)) 548 return i; 549 } 550 551 return -ENOENT; 552 } 553 554 static s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p) 555 { 556 struct btf *btf; 557 s32 ret; 558 int id; 559 560 btf = bpf_get_btf_vmlinux(); 561 if (IS_ERR(btf)) 562 return PTR_ERR(btf); 563 if (!btf) 564 return -EINVAL; 565 566 ret = btf_find_by_name_kind(btf, name, kind); 567 /* ret is never zero, since btf_find_by_name_kind returns 568 * positive btf_id or negative error. 569 */ 570 if (ret > 0) { 571 btf_get(btf); 572 *btf_p = btf; 573 return ret; 574 } 575 576 /* If name is not found in vmlinux's BTF then search in module's BTFs */ 577 spin_lock_bh(&btf_idr_lock); 578 idr_for_each_entry(&btf_idr, btf, id) { 579 if (!btf_is_module(btf)) 580 continue; 581 /* linear search could be slow hence unlock/lock 582 * the IDR to avoiding holding it for too long 583 */ 584 btf_get(btf); 585 spin_unlock_bh(&btf_idr_lock); 586 ret = btf_find_by_name_kind(btf, name, kind); 587 if (ret > 0) { 588 *btf_p = btf; 589 return ret; 590 } 591 btf_put(btf); 592 spin_lock_bh(&btf_idr_lock); 593 } 594 spin_unlock_bh(&btf_idr_lock); 595 return ret; 596 } 597 598 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf, 599 u32 id, u32 *res_id) 600 { 601 const struct btf_type *t = btf_type_by_id(btf, id); 602 603 while (btf_type_is_modifier(t)) { 604 id = t->type; 605 t = btf_type_by_id(btf, t->type); 606 } 607 608 if (res_id) 609 *res_id = id; 610 611 return t; 612 } 613 614 const struct btf_type *btf_type_resolve_ptr(const struct btf *btf, 615 u32 id, u32 *res_id) 616 { 617 const struct btf_type *t; 618 619 t = btf_type_skip_modifiers(btf, id, NULL); 620 if (!btf_type_is_ptr(t)) 621 return NULL; 622 623 return btf_type_skip_modifiers(btf, t->type, res_id); 624 } 625 626 const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, 627 u32 id, u32 *res_id) 628 { 629 const struct btf_type *ptype; 630 631 ptype = btf_type_resolve_ptr(btf, id, res_id); 632 if (ptype && btf_type_is_func_proto(ptype)) 633 return ptype; 634 635 return NULL; 636 } 637 638 /* Types that act only as a source, not sink or intermediate 639 * type when resolving. 640 */ 641 static bool btf_type_is_resolve_source_only(const struct btf_type *t) 642 { 643 return btf_type_is_var(t) || 644 btf_type_is_decl_tag(t) || 645 btf_type_is_datasec(t); 646 } 647 648 /* What types need to be resolved? 649 * 650 * btf_type_is_modifier() is an obvious one. 651 * 652 * btf_type_is_struct() because its member refers to 653 * another type (through member->type). 654 * 655 * btf_type_is_var() because the variable refers to 656 * another type. btf_type_is_datasec() holds multiple 657 * btf_type_is_var() types that need resolving. 658 * 659 * btf_type_is_array() because its element (array->type) 660 * refers to another type. Array can be thought of a 661 * special case of struct while array just has the same 662 * member-type repeated by array->nelems of times. 663 */ 664 static bool btf_type_needs_resolve(const struct btf_type *t) 665 { 666 return btf_type_is_modifier(t) || 667 btf_type_is_ptr(t) || 668 btf_type_is_struct(t) || 669 btf_type_is_array(t) || 670 btf_type_is_var(t) || 671 btf_type_is_func(t) || 672 btf_type_is_decl_tag(t) || 673 btf_type_is_datasec(t); 674 } 675 676 /* t->size can be used */ 677 static bool btf_type_has_size(const struct btf_type *t) 678 { 679 switch (BTF_INFO_KIND(t->info)) { 680 case BTF_KIND_INT: 681 case BTF_KIND_STRUCT: 682 case BTF_KIND_UNION: 683 case BTF_KIND_ENUM: 684 case BTF_KIND_DATASEC: 685 case BTF_KIND_FLOAT: 686 case BTF_KIND_ENUM64: 687 return true; 688 } 689 690 return false; 691 } 692 693 static const char *btf_int_encoding_str(u8 encoding) 694 { 695 if (encoding == 0) 696 return "(none)"; 697 else if (encoding == BTF_INT_SIGNED) 698 return "SIGNED"; 699 else if (encoding == BTF_INT_CHAR) 700 return "CHAR"; 701 else if (encoding == BTF_INT_BOOL) 702 return "BOOL"; 703 else 704 return "UNKN"; 705 } 706 707 static u32 btf_type_int(const struct btf_type *t) 708 { 709 return *(u32 *)(t + 1); 710 } 711 712 static const struct btf_array *btf_type_array(const struct btf_type *t) 713 { 714 return (const struct btf_array *)(t + 1); 715 } 716 717 static const struct btf_enum *btf_type_enum(const struct btf_type *t) 718 { 719 return (const struct btf_enum *)(t + 1); 720 } 721 722 static const struct btf_var *btf_type_var(const struct btf_type *t) 723 { 724 return (const struct btf_var *)(t + 1); 725 } 726 727 static const struct btf_decl_tag *btf_type_decl_tag(const struct btf_type *t) 728 { 729 return (const struct btf_decl_tag *)(t + 1); 730 } 731 732 static const struct btf_enum64 *btf_type_enum64(const struct btf_type *t) 733 { 734 return (const struct btf_enum64 *)(t + 1); 735 } 736 737 static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) 738 { 739 return kind_ops[BTF_INFO_KIND(t->info)]; 740 } 741 742 static bool btf_name_offset_valid(const struct btf *btf, u32 offset) 743 { 744 if (!BTF_STR_OFFSET_VALID(offset)) 745 return false; 746 747 while (offset < btf->start_str_off) 748 btf = btf->base_btf; 749 750 offset -= btf->start_str_off; 751 return offset < btf->hdr.str_len; 752 } 753 754 static bool __btf_name_char_ok(char c, bool first) 755 { 756 if ((first ? !isalpha(c) : 757 !isalnum(c)) && 758 c != '_' && 759 c != '.') 760 return false; 761 return true; 762 } 763 764 static const char *btf_str_by_offset(const struct btf *btf, u32 offset) 765 { 766 while (offset < btf->start_str_off) 767 btf = btf->base_btf; 768 769 offset -= btf->start_str_off; 770 if (offset < btf->hdr.str_len) 771 return &btf->strings[offset]; 772 773 return NULL; 774 } 775 776 static bool __btf_name_valid(const struct btf *btf, u32 offset) 777 { 778 /* offset must be valid */ 779 const char *src = btf_str_by_offset(btf, offset); 780 const char *src_limit; 781 782 if (!__btf_name_char_ok(*src, true)) 783 return false; 784 785 /* set a limit on identifier length */ 786 src_limit = src + KSYM_NAME_LEN; 787 src++; 788 while (*src && src < src_limit) { 789 if (!__btf_name_char_ok(*src, false)) 790 return false; 791 src++; 792 } 793 794 return !*src; 795 } 796 797 static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) 798 { 799 return __btf_name_valid(btf, offset); 800 } 801 802 static bool btf_name_valid_section(const struct btf *btf, u32 offset) 803 { 804 return __btf_name_valid(btf, offset); 805 } 806 807 static const char *__btf_name_by_offset(const struct btf *btf, u32 offset) 808 { 809 const char *name; 810 811 if (!offset) 812 return "(anon)"; 813 814 name = btf_str_by_offset(btf, offset); 815 return name ?: "(invalid-name-offset)"; 816 } 817 818 const char *btf_name_by_offset(const struct btf *btf, u32 offset) 819 { 820 return btf_str_by_offset(btf, offset); 821 } 822 823 const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) 824 { 825 while (type_id < btf->start_id) 826 btf = btf->base_btf; 827 828 type_id -= btf->start_id; 829 if (type_id >= btf->nr_types) 830 return NULL; 831 return btf->types[type_id]; 832 } 833 EXPORT_SYMBOL_GPL(btf_type_by_id); 834 835 /* 836 * Regular int is not a bit field and it must be either 837 * u8/u16/u32/u64 or __int128. 838 */ 839 static bool btf_type_int_is_regular(const struct btf_type *t) 840 { 841 u8 nr_bits, nr_bytes; 842 u32 int_data; 843 844 int_data = btf_type_int(t); 845 nr_bits = BTF_INT_BITS(int_data); 846 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits); 847 if (BITS_PER_BYTE_MASKED(nr_bits) || 848 BTF_INT_OFFSET(int_data) || 849 (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) && 850 nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) && 851 nr_bytes != (2 * sizeof(u64)))) { 852 return false; 853 } 854 855 return true; 856 } 857 858 /* 859 * Check that given struct member is a regular int with expected 860 * offset and size. 861 */ 862 bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, 863 const struct btf_member *m, 864 u32 expected_offset, u32 expected_size) 865 { 866 const struct btf_type *t; 867 u32 id, int_data; 868 u8 nr_bits; 869 870 id = m->type; 871 t = btf_type_id_size(btf, &id, NULL); 872 if (!t || !btf_type_is_int(t)) 873 return false; 874 875 int_data = btf_type_int(t); 876 nr_bits = BTF_INT_BITS(int_data); 877 if (btf_type_kflag(s)) { 878 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset); 879 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset); 880 881 /* if kflag set, int should be a regular int and 882 * bit offset should be at byte boundary. 883 */ 884 return !bitfield_size && 885 BITS_ROUNDUP_BYTES(bit_offset) == expected_offset && 886 BITS_ROUNDUP_BYTES(nr_bits) == expected_size; 887 } 888 889 if (BTF_INT_OFFSET(int_data) || 890 BITS_PER_BYTE_MASKED(m->offset) || 891 BITS_ROUNDUP_BYTES(m->offset) != expected_offset || 892 BITS_PER_BYTE_MASKED(nr_bits) || 893 BITS_ROUNDUP_BYTES(nr_bits) != expected_size) 894 return false; 895 896 return true; 897 } 898 899 /* Similar to btf_type_skip_modifiers() but does not skip typedefs. */ 900 static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, 901 u32 id) 902 { 903 const struct btf_type *t = btf_type_by_id(btf, id); 904 905 while (btf_type_is_modifier(t) && 906 BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) { 907 t = btf_type_by_id(btf, t->type); 908 } 909 910 return t; 911 } 912 913 #define BTF_SHOW_MAX_ITER 10 914 915 #define BTF_KIND_BIT(kind) (1ULL << kind) 916 917 /* 918 * Populate show->state.name with type name information. 919 * Format of type name is 920 * 921 * [.member_name = ] (type_name) 922 */ 923 static const char *btf_show_name(struct btf_show *show) 924 { 925 /* BTF_MAX_ITER array suffixes "[]" */ 926 const char *array_suffixes = "[][][][][][][][][][]"; 927 const char *array_suffix = &array_suffixes[strlen(array_suffixes)]; 928 /* BTF_MAX_ITER pointer suffixes "*" */ 929 const char *ptr_suffixes = "**********"; 930 const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)]; 931 const char *name = NULL, *prefix = "", *parens = ""; 932 const struct btf_member *m = show->state.member; 933 const struct btf_type *t; 934 const struct btf_array *array; 935 u32 id = show->state.type_id; 936 const char *member = NULL; 937 bool show_member = false; 938 u64 kinds = 0; 939 int i; 940 941 show->state.name[0] = '\0'; 942 943 /* 944 * Don't show type name if we're showing an array member; 945 * in that case we show the array type so don't need to repeat 946 * ourselves for each member. 947 */ 948 if (show->state.array_member) 949 return ""; 950 951 /* Retrieve member name, if any. */ 952 if (m) { 953 member = btf_name_by_offset(show->btf, m->name_off); 954 show_member = strlen(member) > 0; 955 id = m->type; 956 } 957 958 /* 959 * Start with type_id, as we have resolved the struct btf_type * 960 * via btf_modifier_show() past the parent typedef to the child 961 * struct, int etc it is defined as. In such cases, the type_id 962 * still represents the starting type while the struct btf_type * 963 * in our show->state points at the resolved type of the typedef. 964 */ 965 t = btf_type_by_id(show->btf, id); 966 if (!t) 967 return ""; 968 969 /* 970 * The goal here is to build up the right number of pointer and 971 * array suffixes while ensuring the type name for a typedef 972 * is represented. Along the way we accumulate a list of 973 * BTF kinds we have encountered, since these will inform later 974 * display; for example, pointer types will not require an 975 * opening "{" for struct, we will just display the pointer value. 976 * 977 * We also want to accumulate the right number of pointer or array 978 * indices in the format string while iterating until we get to 979 * the typedef/pointee/array member target type. 980 * 981 * We start by pointing at the end of pointer and array suffix 982 * strings; as we accumulate pointers and arrays we move the pointer 983 * or array string backwards so it will show the expected number of 984 * '*' or '[]' for the type. BTF_SHOW_MAX_ITER of nesting of pointers 985 * and/or arrays and typedefs are supported as a precaution. 986 * 987 * We also want to get typedef name while proceeding to resolve 988 * type it points to so that we can add parentheses if it is a 989 * "typedef struct" etc. 990 */ 991 for (i = 0; i < BTF_SHOW_MAX_ITER; i++) { 992 993 switch (BTF_INFO_KIND(t->info)) { 994 case BTF_KIND_TYPEDEF: 995 if (!name) 996 name = btf_name_by_offset(show->btf, 997 t->name_off); 998 kinds |= BTF_KIND_BIT(BTF_KIND_TYPEDEF); 999 id = t->type; 1000 break; 1001 case BTF_KIND_ARRAY: 1002 kinds |= BTF_KIND_BIT(BTF_KIND_ARRAY); 1003 parens = "["; 1004 if (!t) 1005 return ""; 1006 array = btf_type_array(t); 1007 if (array_suffix > array_suffixes) 1008 array_suffix -= 2; 1009 id = array->type; 1010 break; 1011 case BTF_KIND_PTR: 1012 kinds |= BTF_KIND_BIT(BTF_KIND_PTR); 1013 if (ptr_suffix > ptr_suffixes) 1014 ptr_suffix -= 1; 1015 id = t->type; 1016 break; 1017 default: 1018 id = 0; 1019 break; 1020 } 1021 if (!id) 1022 break; 1023 t = btf_type_skip_qualifiers(show->btf, id); 1024 } 1025 /* We may not be able to represent this type; bail to be safe */ 1026 if (i == BTF_SHOW_MAX_ITER) 1027 return ""; 1028 1029 if (!name) 1030 name = btf_name_by_offset(show->btf, t->name_off); 1031 1032 switch (BTF_INFO_KIND(t->info)) { 1033 case BTF_KIND_STRUCT: 1034 case BTF_KIND_UNION: 1035 prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ? 1036 "struct" : "union"; 1037 /* if it's an array of struct/union, parens is already set */ 1038 if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY)))) 1039 parens = "{"; 1040 break; 1041 case BTF_KIND_ENUM: 1042 case BTF_KIND_ENUM64: 1043 prefix = "enum"; 1044 break; 1045 default: 1046 break; 1047 } 1048 1049 /* pointer does not require parens */ 1050 if (kinds & BTF_KIND_BIT(BTF_KIND_PTR)) 1051 parens = ""; 1052 /* typedef does not require struct/union/enum prefix */ 1053 if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF)) 1054 prefix = ""; 1055 1056 if (!name) 1057 name = ""; 1058 1059 /* Even if we don't want type name info, we want parentheses etc */ 1060 if (show->flags & BTF_SHOW_NONAME) 1061 snprintf(show->state.name, sizeof(show->state.name), "%s", 1062 parens); 1063 else 1064 snprintf(show->state.name, sizeof(show->state.name), 1065 "%s%s%s(%s%s%s%s%s%s)%s", 1066 /* first 3 strings comprise ".member = " */ 1067 show_member ? "." : "", 1068 show_member ? member : "", 1069 show_member ? " = " : "", 1070 /* ...next is our prefix (struct, enum, etc) */ 1071 prefix, 1072 strlen(prefix) > 0 && strlen(name) > 0 ? " " : "", 1073 /* ...this is the type name itself */ 1074 name, 1075 /* ...suffixed by the appropriate '*', '[]' suffixes */ 1076 strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix, 1077 array_suffix, parens); 1078 1079 return show->state.name; 1080 } 1081 1082 static const char *__btf_show_indent(struct btf_show *show) 1083 { 1084 const char *indents = " "; 1085 const char *indent = &indents[strlen(indents)]; 1086 1087 if ((indent - show->state.depth) >= indents) 1088 return indent - show->state.depth; 1089 return indents; 1090 } 1091 1092 static const char *btf_show_indent(struct btf_show *show) 1093 { 1094 return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show); 1095 } 1096 1097 static const char *btf_show_newline(struct btf_show *show) 1098 { 1099 return show->flags & BTF_SHOW_COMPACT ? "" : "\n"; 1100 } 1101 1102 static const char *btf_show_delim(struct btf_show *show) 1103 { 1104 if (show->state.depth == 0) 1105 return ""; 1106 1107 if ((show->flags & BTF_SHOW_COMPACT) && show->state.type && 1108 BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION) 1109 return "|"; 1110 1111 return ","; 1112 } 1113 1114 __printf(2, 3) static void btf_show(struct btf_show *show, const char *fmt, ...) 1115 { 1116 va_list args; 1117 1118 if (!show->state.depth_check) { 1119 va_start(args, fmt); 1120 show->showfn(show, fmt, args); 1121 va_end(args); 1122 } 1123 } 1124 1125 /* Macros are used here as btf_show_type_value[s]() prepends and appends 1126 * format specifiers to the format specifier passed in; these do the work of 1127 * adding indentation, delimiters etc while the caller simply has to specify 1128 * the type value(s) in the format specifier + value(s). 1129 */ 1130 #define btf_show_type_value(show, fmt, value) \ 1131 do { \ 1132 if ((value) != (__typeof__(value))0 || \ 1133 (show->flags & BTF_SHOW_ZERO) || \ 1134 show->state.depth == 0) { \ 1135 btf_show(show, "%s%s" fmt "%s%s", \ 1136 btf_show_indent(show), \ 1137 btf_show_name(show), \ 1138 value, btf_show_delim(show), \ 1139 btf_show_newline(show)); \ 1140 if (show->state.depth > show->state.depth_to_show) \ 1141 show->state.depth_to_show = show->state.depth; \ 1142 } \ 1143 } while (0) 1144 1145 #define btf_show_type_values(show, fmt, ...) \ 1146 do { \ 1147 btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show), \ 1148 btf_show_name(show), \ 1149 __VA_ARGS__, btf_show_delim(show), \ 1150 btf_show_newline(show)); \ 1151 if (show->state.depth > show->state.depth_to_show) \ 1152 show->state.depth_to_show = show->state.depth; \ 1153 } while (0) 1154 1155 /* How much is left to copy to safe buffer after @data? */ 1156 static int btf_show_obj_size_left(struct btf_show *show, void *data) 1157 { 1158 return show->obj.head + show->obj.size - data; 1159 } 1160 1161 /* Is object pointed to by @data of @size already copied to our safe buffer? */ 1162 static bool btf_show_obj_is_safe(struct btf_show *show, void *data, int size) 1163 { 1164 return data >= show->obj.data && 1165 (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE); 1166 } 1167 1168 /* 1169 * If object pointed to by @data of @size falls within our safe buffer, return 1170 * the equivalent pointer to the same safe data. Assumes 1171 * copy_from_kernel_nofault() has already happened and our safe buffer is 1172 * populated. 1173 */ 1174 static void *__btf_show_obj_safe(struct btf_show *show, void *data, int size) 1175 { 1176 if (btf_show_obj_is_safe(show, data, size)) 1177 return show->obj.safe + (data - show->obj.data); 1178 return NULL; 1179 } 1180 1181 /* 1182 * Return a safe-to-access version of data pointed to by @data. 1183 * We do this by copying the relevant amount of information 1184 * to the struct btf_show obj.safe buffer using copy_from_kernel_nofault(). 1185 * 1186 * If BTF_SHOW_UNSAFE is specified, just return data as-is; no 1187 * safe copy is needed. 1188 * 1189 * Otherwise we need to determine if we have the required amount 1190 * of data (determined by the @data pointer and the size of the 1191 * largest base type we can encounter (represented by 1192 * BTF_SHOW_OBJ_BASE_TYPE_SIZE). Having that much data ensures 1193 * that we will be able to print some of the current object, 1194 * and if more is needed a copy will be triggered. 1195 * Some objects such as structs will not fit into the buffer; 1196 * in such cases additional copies when we iterate over their 1197 * members may be needed. 1198 * 1199 * btf_show_obj_safe() is used to return a safe buffer for 1200 * btf_show_start_type(); this ensures that as we recurse into 1201 * nested types we always have safe data for the given type. 1202 * This approach is somewhat wasteful; it's possible for example 1203 * that when iterating over a large union we'll end up copying the 1204 * same data repeatedly, but the goal is safety not performance. 1205 * We use stack data as opposed to per-CPU buffers because the 1206 * iteration over a type can take some time, and preemption handling 1207 * would greatly complicate use of the safe buffer. 1208 */ 1209 static void *btf_show_obj_safe(struct btf_show *show, 1210 const struct btf_type *t, 1211 void *data) 1212 { 1213 const struct btf_type *rt; 1214 int size_left, size; 1215 void *safe = NULL; 1216 1217 if (show->flags & BTF_SHOW_UNSAFE) 1218 return data; 1219 1220 rt = btf_resolve_size(show->btf, t, &size); 1221 if (IS_ERR(rt)) { 1222 show->state.status = PTR_ERR(rt); 1223 return NULL; 1224 } 1225 1226 /* 1227 * Is this toplevel object? If so, set total object size and 1228 * initialize pointers. Otherwise check if we still fall within 1229 * our safe object data. 1230 */ 1231 if (show->state.depth == 0) { 1232 show->obj.size = size; 1233 show->obj.head = data; 1234 } else { 1235 /* 1236 * If the size of the current object is > our remaining 1237 * safe buffer we _may_ need to do a new copy. However 1238 * consider the case of a nested struct; it's size pushes 1239 * us over the safe buffer limit, but showing any individual 1240 * struct members does not. In such cases, we don't need 1241 * to initiate a fresh copy yet; however we definitely need 1242 * at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes left 1243 * in our buffer, regardless of the current object size. 1244 * The logic here is that as we resolve types we will 1245 * hit a base type at some point, and we need to be sure 1246 * the next chunk of data is safely available to display 1247 * that type info safely. We cannot rely on the size of 1248 * the current object here because it may be much larger 1249 * than our current buffer (e.g. task_struct is 8k). 1250 * All we want to do here is ensure that we can print the 1251 * next basic type, which we can if either 1252 * - the current type size is within the safe buffer; or 1253 * - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in 1254 * the safe buffer. 1255 */ 1256 safe = __btf_show_obj_safe(show, data, 1257 min(size, 1258 BTF_SHOW_OBJ_BASE_TYPE_SIZE)); 1259 } 1260 1261 /* 1262 * We need a new copy to our safe object, either because we haven't 1263 * yet copied and are initializing safe data, or because the data 1264 * we want falls outside the boundaries of the safe object. 1265 */ 1266 if (!safe) { 1267 size_left = btf_show_obj_size_left(show, data); 1268 if (size_left > BTF_SHOW_OBJ_SAFE_SIZE) 1269 size_left = BTF_SHOW_OBJ_SAFE_SIZE; 1270 show->state.status = copy_from_kernel_nofault(show->obj.safe, 1271 data, size_left); 1272 if (!show->state.status) { 1273 show->obj.data = data; 1274 safe = show->obj.safe; 1275 } 1276 } 1277 1278 return safe; 1279 } 1280 1281 /* 1282 * Set the type we are starting to show and return a safe data pointer 1283 * to be used for showing the associated data. 1284 */ 1285 static void *btf_show_start_type(struct btf_show *show, 1286 const struct btf_type *t, 1287 u32 type_id, void *data) 1288 { 1289 show->state.type = t; 1290 show->state.type_id = type_id; 1291 show->state.name[0] = '\0'; 1292 1293 return btf_show_obj_safe(show, t, data); 1294 } 1295 1296 static void btf_show_end_type(struct btf_show *show) 1297 { 1298 show->state.type = NULL; 1299 show->state.type_id = 0; 1300 show->state.name[0] = '\0'; 1301 } 1302 1303 static void *btf_show_start_aggr_type(struct btf_show *show, 1304 const struct btf_type *t, 1305 u32 type_id, void *data) 1306 { 1307 void *safe_data = btf_show_start_type(show, t, type_id, data); 1308 1309 if (!safe_data) 1310 return safe_data; 1311 1312 btf_show(show, "%s%s%s", btf_show_indent(show), 1313 btf_show_name(show), 1314 btf_show_newline(show)); 1315 show->state.depth++; 1316 return safe_data; 1317 } 1318 1319 static void btf_show_end_aggr_type(struct btf_show *show, 1320 const char *suffix) 1321 { 1322 show->state.depth--; 1323 btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix, 1324 btf_show_delim(show), btf_show_newline(show)); 1325 btf_show_end_type(show); 1326 } 1327 1328 static void btf_show_start_member(struct btf_show *show, 1329 const struct btf_member *m) 1330 { 1331 show->state.member = m; 1332 } 1333 1334 static void btf_show_start_array_member(struct btf_show *show) 1335 { 1336 show->state.array_member = 1; 1337 btf_show_start_member(show, NULL); 1338 } 1339 1340 static void btf_show_end_member(struct btf_show *show) 1341 { 1342 show->state.member = NULL; 1343 } 1344 1345 static void btf_show_end_array_member(struct btf_show *show) 1346 { 1347 show->state.array_member = 0; 1348 btf_show_end_member(show); 1349 } 1350 1351 static void *btf_show_start_array_type(struct btf_show *show, 1352 const struct btf_type *t, 1353 u32 type_id, 1354 u16 array_encoding, 1355 void *data) 1356 { 1357 show->state.array_encoding = array_encoding; 1358 show->state.array_terminated = 0; 1359 return btf_show_start_aggr_type(show, t, type_id, data); 1360 } 1361 1362 static void btf_show_end_array_type(struct btf_show *show) 1363 { 1364 show->state.array_encoding = 0; 1365 show->state.array_terminated = 0; 1366 btf_show_end_aggr_type(show, "]"); 1367 } 1368 1369 static void *btf_show_start_struct_type(struct btf_show *show, 1370 const struct btf_type *t, 1371 u32 type_id, 1372 void *data) 1373 { 1374 return btf_show_start_aggr_type(show, t, type_id, data); 1375 } 1376 1377 static void btf_show_end_struct_type(struct btf_show *show) 1378 { 1379 btf_show_end_aggr_type(show, "}"); 1380 } 1381 1382 __printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log, 1383 const char *fmt, ...) 1384 { 1385 va_list args; 1386 1387 va_start(args, fmt); 1388 bpf_verifier_vlog(log, fmt, args); 1389 va_end(args); 1390 } 1391 1392 __printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env, 1393 const char *fmt, ...) 1394 { 1395 struct bpf_verifier_log *log = &env->log; 1396 va_list args; 1397 1398 if (!bpf_verifier_log_needed(log)) 1399 return; 1400 1401 va_start(args, fmt); 1402 bpf_verifier_vlog(log, fmt, args); 1403 va_end(args); 1404 } 1405 1406 __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, 1407 const struct btf_type *t, 1408 bool log_details, 1409 const char *fmt, ...) 1410 { 1411 struct bpf_verifier_log *log = &env->log; 1412 struct btf *btf = env->btf; 1413 va_list args; 1414 1415 if (!bpf_verifier_log_needed(log)) 1416 return; 1417 1418 if (log->level == BPF_LOG_KERNEL) { 1419 /* btf verifier prints all types it is processing via 1420 * btf_verifier_log_type(..., fmt = NULL). 1421 * Skip those prints for in-kernel BTF verification. 1422 */ 1423 if (!fmt) 1424 return; 1425 1426 /* Skip logging when loading module BTF with mismatches permitted */ 1427 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) 1428 return; 1429 } 1430 1431 __btf_verifier_log(log, "[%u] %s %s%s", 1432 env->log_type_id, 1433 btf_type_str(t), 1434 __btf_name_by_offset(btf, t->name_off), 1435 log_details ? " " : ""); 1436 1437 if (log_details) 1438 btf_type_ops(t)->log_details(env, t); 1439 1440 if (fmt && *fmt) { 1441 __btf_verifier_log(log, " "); 1442 va_start(args, fmt); 1443 bpf_verifier_vlog(log, fmt, args); 1444 va_end(args); 1445 } 1446 1447 __btf_verifier_log(log, "\n"); 1448 } 1449 1450 #define btf_verifier_log_type(env, t, ...) \ 1451 __btf_verifier_log_type((env), (t), true, __VA_ARGS__) 1452 #define btf_verifier_log_basic(env, t, ...) \ 1453 __btf_verifier_log_type((env), (t), false, __VA_ARGS__) 1454 1455 __printf(4, 5) 1456 static void btf_verifier_log_member(struct btf_verifier_env *env, 1457 const struct btf_type *struct_type, 1458 const struct btf_member *member, 1459 const char *fmt, ...) 1460 { 1461 struct bpf_verifier_log *log = &env->log; 1462 struct btf *btf = env->btf; 1463 va_list args; 1464 1465 if (!bpf_verifier_log_needed(log)) 1466 return; 1467 1468 if (log->level == BPF_LOG_KERNEL) { 1469 if (!fmt) 1470 return; 1471 1472 /* Skip logging when loading module BTF with mismatches permitted */ 1473 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) 1474 return; 1475 } 1476 1477 /* The CHECK_META phase already did a btf dump. 1478 * 1479 * If member is logged again, it must hit an error in 1480 * parsing this member. It is useful to print out which 1481 * struct this member belongs to. 1482 */ 1483 if (env->phase != CHECK_META) 1484 btf_verifier_log_type(env, struct_type, NULL); 1485 1486 if (btf_type_kflag(struct_type)) 1487 __btf_verifier_log(log, 1488 "\t%s type_id=%u bitfield_size=%u bits_offset=%u", 1489 __btf_name_by_offset(btf, member->name_off), 1490 member->type, 1491 BTF_MEMBER_BITFIELD_SIZE(member->offset), 1492 BTF_MEMBER_BIT_OFFSET(member->offset)); 1493 else 1494 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u", 1495 __btf_name_by_offset(btf, member->name_off), 1496 member->type, member->offset); 1497 1498 if (fmt && *fmt) { 1499 __btf_verifier_log(log, " "); 1500 va_start(args, fmt); 1501 bpf_verifier_vlog(log, fmt, args); 1502 va_end(args); 1503 } 1504 1505 __btf_verifier_log(log, "\n"); 1506 } 1507 1508 __printf(4, 5) 1509 static void btf_verifier_log_vsi(struct btf_verifier_env *env, 1510 const struct btf_type *datasec_type, 1511 const struct btf_var_secinfo *vsi, 1512 const char *fmt, ...) 1513 { 1514 struct bpf_verifier_log *log = &env->log; 1515 va_list args; 1516 1517 if (!bpf_verifier_log_needed(log)) 1518 return; 1519 if (log->level == BPF_LOG_KERNEL && !fmt) 1520 return; 1521 if (env->phase != CHECK_META) 1522 btf_verifier_log_type(env, datasec_type, NULL); 1523 1524 __btf_verifier_log(log, "\t type_id=%u offset=%u size=%u", 1525 vsi->type, vsi->offset, vsi->size); 1526 if (fmt && *fmt) { 1527 __btf_verifier_log(log, " "); 1528 va_start(args, fmt); 1529 bpf_verifier_vlog(log, fmt, args); 1530 va_end(args); 1531 } 1532 1533 __btf_verifier_log(log, "\n"); 1534 } 1535 1536 static void btf_verifier_log_hdr(struct btf_verifier_env *env, 1537 u32 btf_data_size) 1538 { 1539 struct bpf_verifier_log *log = &env->log; 1540 const struct btf *btf = env->btf; 1541 const struct btf_header *hdr; 1542 1543 if (!bpf_verifier_log_needed(log)) 1544 return; 1545 1546 if (log->level == BPF_LOG_KERNEL) 1547 return; 1548 hdr = &btf->hdr; 1549 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic); 1550 __btf_verifier_log(log, "version: %u\n", hdr->version); 1551 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags); 1552 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len); 1553 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off); 1554 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len); 1555 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off); 1556 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len); 1557 __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size); 1558 } 1559 1560 static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) 1561 { 1562 struct btf *btf = env->btf; 1563 1564 if (btf->types_size == btf->nr_types) { 1565 /* Expand 'types' array */ 1566 1567 struct btf_type **new_types; 1568 u32 expand_by, new_size; 1569 1570 if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { 1571 btf_verifier_log(env, "Exceeded max num of types"); 1572 return -E2BIG; 1573 } 1574 1575 expand_by = max_t(u32, btf->types_size >> 2, 16); 1576 new_size = min_t(u32, BTF_MAX_TYPE, 1577 btf->types_size + expand_by); 1578 1579 new_types = kvcalloc(new_size, sizeof(*new_types), 1580 GFP_KERNEL | __GFP_NOWARN); 1581 if (!new_types) 1582 return -ENOMEM; 1583 1584 if (btf->nr_types == 0) { 1585 if (!btf->base_btf) { 1586 /* lazily init VOID type */ 1587 new_types[0] = &btf_void; 1588 btf->nr_types++; 1589 } 1590 } else { 1591 memcpy(new_types, btf->types, 1592 sizeof(*btf->types) * btf->nr_types); 1593 } 1594 1595 kvfree(btf->types); 1596 btf->types = new_types; 1597 btf->types_size = new_size; 1598 } 1599 1600 btf->types[btf->nr_types++] = t; 1601 1602 return 0; 1603 } 1604 1605 static int btf_alloc_id(struct btf *btf) 1606 { 1607 int id; 1608 1609 idr_preload(GFP_KERNEL); 1610 spin_lock_bh(&btf_idr_lock); 1611 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC); 1612 if (id > 0) 1613 btf->id = id; 1614 spin_unlock_bh(&btf_idr_lock); 1615 idr_preload_end(); 1616 1617 if (WARN_ON_ONCE(!id)) 1618 return -ENOSPC; 1619 1620 return id > 0 ? 0 : id; 1621 } 1622 1623 static void btf_free_id(struct btf *btf) 1624 { 1625 unsigned long flags; 1626 1627 /* 1628 * In map-in-map, calling map_delete_elem() on outer 1629 * map will call bpf_map_put on the inner map. 1630 * It will then eventually call btf_free_id() 1631 * on the inner map. Some of the map_delete_elem() 1632 * implementation may have irq disabled, so 1633 * we need to use the _irqsave() version instead 1634 * of the _bh() version. 1635 */ 1636 spin_lock_irqsave(&btf_idr_lock, flags); 1637 idr_remove(&btf_idr, btf->id); 1638 spin_unlock_irqrestore(&btf_idr_lock, flags); 1639 } 1640 1641 static void btf_free_kfunc_set_tab(struct btf *btf) 1642 { 1643 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; 1644 int hook; 1645 1646 if (!tab) 1647 return; 1648 /* For module BTF, we directly assign the sets being registered, so 1649 * there is nothing to free except kfunc_set_tab. 1650 */ 1651 if (btf_is_module(btf)) 1652 goto free_tab; 1653 for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) 1654 kfree(tab->sets[hook]); 1655 free_tab: 1656 kfree(tab); 1657 btf->kfunc_set_tab = NULL; 1658 } 1659 1660 static void btf_free_dtor_kfunc_tab(struct btf *btf) 1661 { 1662 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; 1663 1664 if (!tab) 1665 return; 1666 kfree(tab); 1667 btf->dtor_kfunc_tab = NULL; 1668 } 1669 1670 static void btf_struct_metas_free(struct btf_struct_metas *tab) 1671 { 1672 int i; 1673 1674 if (!tab) 1675 return; 1676 for (i = 0; i < tab->cnt; i++) 1677 btf_record_free(tab->types[i].record); 1678 kfree(tab); 1679 } 1680 1681 static void btf_free_struct_meta_tab(struct btf *btf) 1682 { 1683 struct btf_struct_metas *tab = btf->struct_meta_tab; 1684 1685 btf_struct_metas_free(tab); 1686 btf->struct_meta_tab = NULL; 1687 } 1688 1689 static void btf_free(struct btf *btf) 1690 { 1691 btf_free_struct_meta_tab(btf); 1692 btf_free_dtor_kfunc_tab(btf); 1693 btf_free_kfunc_set_tab(btf); 1694 kvfree(btf->types); 1695 kvfree(btf->resolved_sizes); 1696 kvfree(btf->resolved_ids); 1697 kvfree(btf->data); 1698 kfree(btf); 1699 } 1700 1701 static void btf_free_rcu(struct rcu_head *rcu) 1702 { 1703 struct btf *btf = container_of(rcu, struct btf, rcu); 1704 1705 btf_free(btf); 1706 } 1707 1708 void btf_get(struct btf *btf) 1709 { 1710 refcount_inc(&btf->refcnt); 1711 } 1712 1713 void btf_put(struct btf *btf) 1714 { 1715 if (btf && refcount_dec_and_test(&btf->refcnt)) { 1716 btf_free_id(btf); 1717 call_rcu(&btf->rcu, btf_free_rcu); 1718 } 1719 } 1720 1721 static int env_resolve_init(struct btf_verifier_env *env) 1722 { 1723 struct btf *btf = env->btf; 1724 u32 nr_types = btf->nr_types; 1725 u32 *resolved_sizes = NULL; 1726 u32 *resolved_ids = NULL; 1727 u8 *visit_states = NULL; 1728 1729 resolved_sizes = kvcalloc(nr_types, sizeof(*resolved_sizes), 1730 GFP_KERNEL | __GFP_NOWARN); 1731 if (!resolved_sizes) 1732 goto nomem; 1733 1734 resolved_ids = kvcalloc(nr_types, sizeof(*resolved_ids), 1735 GFP_KERNEL | __GFP_NOWARN); 1736 if (!resolved_ids) 1737 goto nomem; 1738 1739 visit_states = kvcalloc(nr_types, sizeof(*visit_states), 1740 GFP_KERNEL | __GFP_NOWARN); 1741 if (!visit_states) 1742 goto nomem; 1743 1744 btf->resolved_sizes = resolved_sizes; 1745 btf->resolved_ids = resolved_ids; 1746 env->visit_states = visit_states; 1747 1748 return 0; 1749 1750 nomem: 1751 kvfree(resolved_sizes); 1752 kvfree(resolved_ids); 1753 kvfree(visit_states); 1754 return -ENOMEM; 1755 } 1756 1757 static void btf_verifier_env_free(struct btf_verifier_env *env) 1758 { 1759 kvfree(env->visit_states); 1760 kfree(env); 1761 } 1762 1763 static bool env_type_is_resolve_sink(const struct btf_verifier_env *env, 1764 const struct btf_type *next_type) 1765 { 1766 switch (env->resolve_mode) { 1767 case RESOLVE_TBD: 1768 /* int, enum or void is a sink */ 1769 return !btf_type_needs_resolve(next_type); 1770 case RESOLVE_PTR: 1771 /* int, enum, void, struct, array, func or func_proto is a sink 1772 * for ptr 1773 */ 1774 return !btf_type_is_modifier(next_type) && 1775 !btf_type_is_ptr(next_type); 1776 case RESOLVE_STRUCT_OR_ARRAY: 1777 /* int, enum, void, ptr, func or func_proto is a sink 1778 * for struct and array 1779 */ 1780 return !btf_type_is_modifier(next_type) && 1781 !btf_type_is_array(next_type) && 1782 !btf_type_is_struct(next_type); 1783 default: 1784 BUG(); 1785 } 1786 } 1787 1788 static bool env_type_is_resolved(const struct btf_verifier_env *env, 1789 u32 type_id) 1790 { 1791 /* base BTF types should be resolved by now */ 1792 if (type_id < env->btf->start_id) 1793 return true; 1794 1795 return env->visit_states[type_id - env->btf->start_id] == RESOLVED; 1796 } 1797 1798 static int env_stack_push(struct btf_verifier_env *env, 1799 const struct btf_type *t, u32 type_id) 1800 { 1801 const struct btf *btf = env->btf; 1802 struct resolve_vertex *v; 1803 1804 if (env->top_stack == MAX_RESOLVE_DEPTH) 1805 return -E2BIG; 1806 1807 if (type_id < btf->start_id 1808 || env->visit_states[type_id - btf->start_id] != NOT_VISITED) 1809 return -EEXIST; 1810 1811 env->visit_states[type_id - btf->start_id] = VISITED; 1812 1813 v = &env->stack[env->top_stack++]; 1814 v->t = t; 1815 v->type_id = type_id; 1816 v->next_member = 0; 1817 1818 if (env->resolve_mode == RESOLVE_TBD) { 1819 if (btf_type_is_ptr(t)) 1820 env->resolve_mode = RESOLVE_PTR; 1821 else if (btf_type_is_struct(t) || btf_type_is_array(t)) 1822 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY; 1823 } 1824 1825 return 0; 1826 } 1827 1828 static void env_stack_set_next_member(struct btf_verifier_env *env, 1829 u16 next_member) 1830 { 1831 env->stack[env->top_stack - 1].next_member = next_member; 1832 } 1833 1834 static void env_stack_pop_resolved(struct btf_verifier_env *env, 1835 u32 resolved_type_id, 1836 u32 resolved_size) 1837 { 1838 u32 type_id = env->stack[--(env->top_stack)].type_id; 1839 struct btf *btf = env->btf; 1840 1841 type_id -= btf->start_id; /* adjust to local type id */ 1842 btf->resolved_sizes[type_id] = resolved_size; 1843 btf->resolved_ids[type_id] = resolved_type_id; 1844 env->visit_states[type_id] = RESOLVED; 1845 } 1846 1847 static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) 1848 { 1849 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL; 1850 } 1851 1852 /* Resolve the size of a passed-in "type" 1853 * 1854 * type: is an array (e.g. u32 array[x][y]) 1855 * return type: type "u32[x][y]", i.e. BTF_KIND_ARRAY, 1856 * *type_size: (x * y * sizeof(u32)). Hence, *type_size always 1857 * corresponds to the return type. 1858 * *elem_type: u32 1859 * *elem_id: id of u32 1860 * *total_nelems: (x * y). Hence, individual elem size is 1861 * (*type_size / *total_nelems) 1862 * *type_id: id of type if it's changed within the function, 0 if not 1863 * 1864 * type: is not an array (e.g. const struct X) 1865 * return type: type "struct X" 1866 * *type_size: sizeof(struct X) 1867 * *elem_type: same as return type ("struct X") 1868 * *elem_id: 0 1869 * *total_nelems: 1 1870 * *type_id: id of type if it's changed within the function, 0 if not 1871 */ 1872 static const struct btf_type * 1873 __btf_resolve_size(const struct btf *btf, const struct btf_type *type, 1874 u32 *type_size, const struct btf_type **elem_type, 1875 u32 *elem_id, u32 *total_nelems, u32 *type_id) 1876 { 1877 const struct btf_type *array_type = NULL; 1878 const struct btf_array *array = NULL; 1879 u32 i, size, nelems = 1, id = 0; 1880 1881 for (i = 0; i < MAX_RESOLVE_DEPTH; i++) { 1882 switch (BTF_INFO_KIND(type->info)) { 1883 /* type->size can be used */ 1884 case BTF_KIND_INT: 1885 case BTF_KIND_STRUCT: 1886 case BTF_KIND_UNION: 1887 case BTF_KIND_ENUM: 1888 case BTF_KIND_FLOAT: 1889 case BTF_KIND_ENUM64: 1890 size = type->size; 1891 goto resolved; 1892 1893 case BTF_KIND_PTR: 1894 size = sizeof(void *); 1895 goto resolved; 1896 1897 /* Modifiers */ 1898 case BTF_KIND_TYPEDEF: 1899 case BTF_KIND_VOLATILE: 1900 case BTF_KIND_CONST: 1901 case BTF_KIND_RESTRICT: 1902 case BTF_KIND_TYPE_TAG: 1903 id = type->type; 1904 type = btf_type_by_id(btf, type->type); 1905 break; 1906 1907 case BTF_KIND_ARRAY: 1908 if (!array_type) 1909 array_type = type; 1910 array = btf_type_array(type); 1911 if (nelems && array->nelems > U32_MAX / nelems) 1912 return ERR_PTR(-EINVAL); 1913 nelems *= array->nelems; 1914 type = btf_type_by_id(btf, array->type); 1915 break; 1916 1917 /* type without size */ 1918 default: 1919 return ERR_PTR(-EINVAL); 1920 } 1921 } 1922 1923 return ERR_PTR(-EINVAL); 1924 1925 resolved: 1926 if (nelems && size > U32_MAX / nelems) 1927 return ERR_PTR(-EINVAL); 1928 1929 *type_size = nelems * size; 1930 if (total_nelems) 1931 *total_nelems = nelems; 1932 if (elem_type) 1933 *elem_type = type; 1934 if (elem_id) 1935 *elem_id = array ? array->type : 0; 1936 if (type_id && id) 1937 *type_id = id; 1938 1939 return array_type ? : type; 1940 } 1941 1942 const struct btf_type * 1943 btf_resolve_size(const struct btf *btf, const struct btf_type *type, 1944 u32 *type_size) 1945 { 1946 return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL); 1947 } 1948 1949 static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id) 1950 { 1951 while (type_id < btf->start_id) 1952 btf = btf->base_btf; 1953 1954 return btf->resolved_ids[type_id - btf->start_id]; 1955 } 1956 1957 /* The input param "type_id" must point to a needs_resolve type */ 1958 static const struct btf_type *btf_type_id_resolve(const struct btf *btf, 1959 u32 *type_id) 1960 { 1961 *type_id = btf_resolved_type_id(btf, *type_id); 1962 return btf_type_by_id(btf, *type_id); 1963 } 1964 1965 static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id) 1966 { 1967 while (type_id < btf->start_id) 1968 btf = btf->base_btf; 1969 1970 return btf->resolved_sizes[type_id - btf->start_id]; 1971 } 1972 1973 const struct btf_type *btf_type_id_size(const struct btf *btf, 1974 u32 *type_id, u32 *ret_size) 1975 { 1976 const struct btf_type *size_type; 1977 u32 size_type_id = *type_id; 1978 u32 size = 0; 1979 1980 size_type = btf_type_by_id(btf, size_type_id); 1981 if (btf_type_nosize_or_null(size_type)) 1982 return NULL; 1983 1984 if (btf_type_has_size(size_type)) { 1985 size = size_type->size; 1986 } else if (btf_type_is_array(size_type)) { 1987 size = btf_resolved_type_size(btf, size_type_id); 1988 } else if (btf_type_is_ptr(size_type)) { 1989 size = sizeof(void *); 1990 } else { 1991 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) && 1992 !btf_type_is_var(size_type))) 1993 return NULL; 1994 1995 size_type_id = btf_resolved_type_id(btf, size_type_id); 1996 size_type = btf_type_by_id(btf, size_type_id); 1997 if (btf_type_nosize_or_null(size_type)) 1998 return NULL; 1999 else if (btf_type_has_size(size_type)) 2000 size = size_type->size; 2001 else if (btf_type_is_array(size_type)) 2002 size = btf_resolved_type_size(btf, size_type_id); 2003 else if (btf_type_is_ptr(size_type)) 2004 size = sizeof(void *); 2005 else 2006 return NULL; 2007 } 2008 2009 *type_id = size_type_id; 2010 if (ret_size) 2011 *ret_size = size; 2012 2013 return size_type; 2014 } 2015 2016 static int btf_df_check_member(struct btf_verifier_env *env, 2017 const struct btf_type *struct_type, 2018 const struct btf_member *member, 2019 const struct btf_type *member_type) 2020 { 2021 btf_verifier_log_basic(env, struct_type, 2022 "Unsupported check_member"); 2023 return -EINVAL; 2024 } 2025 2026 static int btf_df_check_kflag_member(struct btf_verifier_env *env, 2027 const struct btf_type *struct_type, 2028 const struct btf_member *member, 2029 const struct btf_type *member_type) 2030 { 2031 btf_verifier_log_basic(env, struct_type, 2032 "Unsupported check_kflag_member"); 2033 return -EINVAL; 2034 } 2035 2036 /* Used for ptr, array struct/union and float type members. 2037 * int, enum and modifier types have their specific callback functions. 2038 */ 2039 static int btf_generic_check_kflag_member(struct btf_verifier_env *env, 2040 const struct btf_type *struct_type, 2041 const struct btf_member *member, 2042 const struct btf_type *member_type) 2043 { 2044 if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) { 2045 btf_verifier_log_member(env, struct_type, member, 2046 "Invalid member bitfield_size"); 2047 return -EINVAL; 2048 } 2049 2050 /* bitfield size is 0, so member->offset represents bit offset only. 2051 * It is safe to call non kflag check_member variants. 2052 */ 2053 return btf_type_ops(member_type)->check_member(env, struct_type, 2054 member, 2055 member_type); 2056 } 2057 2058 static int btf_df_resolve(struct btf_verifier_env *env, 2059 const struct resolve_vertex *v) 2060 { 2061 btf_verifier_log_basic(env, v->t, "Unsupported resolve"); 2062 return -EINVAL; 2063 } 2064 2065 static void btf_df_show(const struct btf *btf, const struct btf_type *t, 2066 u32 type_id, void *data, u8 bits_offsets, 2067 struct btf_show *show) 2068 { 2069 btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info)); 2070 } 2071 2072 static int btf_int_check_member(struct btf_verifier_env *env, 2073 const struct btf_type *struct_type, 2074 const struct btf_member *member, 2075 const struct btf_type *member_type) 2076 { 2077 u32 int_data = btf_type_int(member_type); 2078 u32 struct_bits_off = member->offset; 2079 u32 struct_size = struct_type->size; 2080 u32 nr_copy_bits; 2081 u32 bytes_offset; 2082 2083 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) { 2084 btf_verifier_log_member(env, struct_type, member, 2085 "bits_offset exceeds U32_MAX"); 2086 return -EINVAL; 2087 } 2088 2089 struct_bits_off += BTF_INT_OFFSET(int_data); 2090 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2091 nr_copy_bits = BTF_INT_BITS(int_data) + 2092 BITS_PER_BYTE_MASKED(struct_bits_off); 2093 2094 if (nr_copy_bits > BITS_PER_U128) { 2095 btf_verifier_log_member(env, struct_type, member, 2096 "nr_copy_bits exceeds 128"); 2097 return -EINVAL; 2098 } 2099 2100 if (struct_size < bytes_offset || 2101 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { 2102 btf_verifier_log_member(env, struct_type, member, 2103 "Member exceeds struct_size"); 2104 return -EINVAL; 2105 } 2106 2107 return 0; 2108 } 2109 2110 static int btf_int_check_kflag_member(struct btf_verifier_env *env, 2111 const struct btf_type *struct_type, 2112 const struct btf_member *member, 2113 const struct btf_type *member_type) 2114 { 2115 u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset; 2116 u32 int_data = btf_type_int(member_type); 2117 u32 struct_size = struct_type->size; 2118 u32 nr_copy_bits; 2119 2120 /* a regular int type is required for the kflag int member */ 2121 if (!btf_type_int_is_regular(member_type)) { 2122 btf_verifier_log_member(env, struct_type, member, 2123 "Invalid member base type"); 2124 return -EINVAL; 2125 } 2126 2127 /* check sanity of bitfield size */ 2128 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); 2129 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); 2130 nr_int_data_bits = BTF_INT_BITS(int_data); 2131 if (!nr_bits) { 2132 /* Not a bitfield member, member offset must be at byte 2133 * boundary. 2134 */ 2135 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 2136 btf_verifier_log_member(env, struct_type, member, 2137 "Invalid member offset"); 2138 return -EINVAL; 2139 } 2140 2141 nr_bits = nr_int_data_bits; 2142 } else if (nr_bits > nr_int_data_bits) { 2143 btf_verifier_log_member(env, struct_type, member, 2144 "Invalid member bitfield_size"); 2145 return -EINVAL; 2146 } 2147 2148 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2149 nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off); 2150 if (nr_copy_bits > BITS_PER_U128) { 2151 btf_verifier_log_member(env, struct_type, member, 2152 "nr_copy_bits exceeds 128"); 2153 return -EINVAL; 2154 } 2155 2156 if (struct_size < bytes_offset || 2157 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { 2158 btf_verifier_log_member(env, struct_type, member, 2159 "Member exceeds struct_size"); 2160 return -EINVAL; 2161 } 2162 2163 return 0; 2164 } 2165 2166 static s32 btf_int_check_meta(struct btf_verifier_env *env, 2167 const struct btf_type *t, 2168 u32 meta_left) 2169 { 2170 u32 int_data, nr_bits, meta_needed = sizeof(int_data); 2171 u16 encoding; 2172 2173 if (meta_left < meta_needed) { 2174 btf_verifier_log_basic(env, t, 2175 "meta_left:%u meta_needed:%u", 2176 meta_left, meta_needed); 2177 return -EINVAL; 2178 } 2179 2180 if (btf_type_vlen(t)) { 2181 btf_verifier_log_type(env, t, "vlen != 0"); 2182 return -EINVAL; 2183 } 2184 2185 if (btf_type_kflag(t)) { 2186 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 2187 return -EINVAL; 2188 } 2189 2190 int_data = btf_type_int(t); 2191 if (int_data & ~BTF_INT_MASK) { 2192 btf_verifier_log_basic(env, t, "Invalid int_data:%x", 2193 int_data); 2194 return -EINVAL; 2195 } 2196 2197 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data); 2198 2199 if (nr_bits > BITS_PER_U128) { 2200 btf_verifier_log_type(env, t, "nr_bits exceeds %zu", 2201 BITS_PER_U128); 2202 return -EINVAL; 2203 } 2204 2205 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) { 2206 btf_verifier_log_type(env, t, "nr_bits exceeds type_size"); 2207 return -EINVAL; 2208 } 2209 2210 /* 2211 * Only one of the encoding bits is allowed and it 2212 * should be sufficient for the pretty print purpose (i.e. decoding). 2213 * Multiple bits can be allowed later if it is found 2214 * to be insufficient. 2215 */ 2216 encoding = BTF_INT_ENCODING(int_data); 2217 if (encoding && 2218 encoding != BTF_INT_SIGNED && 2219 encoding != BTF_INT_CHAR && 2220 encoding != BTF_INT_BOOL) { 2221 btf_verifier_log_type(env, t, "Unsupported encoding"); 2222 return -ENOTSUPP; 2223 } 2224 2225 btf_verifier_log_type(env, t, NULL); 2226 2227 return meta_needed; 2228 } 2229 2230 static void btf_int_log(struct btf_verifier_env *env, 2231 const struct btf_type *t) 2232 { 2233 int int_data = btf_type_int(t); 2234 2235 btf_verifier_log(env, 2236 "size=%u bits_offset=%u nr_bits=%u encoding=%s", 2237 t->size, BTF_INT_OFFSET(int_data), 2238 BTF_INT_BITS(int_data), 2239 btf_int_encoding_str(BTF_INT_ENCODING(int_data))); 2240 } 2241 2242 static void btf_int128_print(struct btf_show *show, void *data) 2243 { 2244 /* data points to a __int128 number. 2245 * Suppose 2246 * int128_num = *(__int128 *)data; 2247 * The below formulas shows what upper_num and lower_num represents: 2248 * upper_num = int128_num >> 64; 2249 * lower_num = int128_num & 0xffffffffFFFFFFFFULL; 2250 */ 2251 u64 upper_num, lower_num; 2252 2253 #ifdef __BIG_ENDIAN_BITFIELD 2254 upper_num = *(u64 *)data; 2255 lower_num = *(u64 *)(data + 8); 2256 #else 2257 upper_num = *(u64 *)(data + 8); 2258 lower_num = *(u64 *)data; 2259 #endif 2260 if (upper_num == 0) 2261 btf_show_type_value(show, "0x%llx", lower_num); 2262 else 2263 btf_show_type_values(show, "0x%llx%016llx", upper_num, 2264 lower_num); 2265 } 2266 2267 static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, 2268 u16 right_shift_bits) 2269 { 2270 u64 upper_num, lower_num; 2271 2272 #ifdef __BIG_ENDIAN_BITFIELD 2273 upper_num = print_num[0]; 2274 lower_num = print_num[1]; 2275 #else 2276 upper_num = print_num[1]; 2277 lower_num = print_num[0]; 2278 #endif 2279 2280 /* shake out un-needed bits by shift/or operations */ 2281 if (left_shift_bits >= 64) { 2282 upper_num = lower_num << (left_shift_bits - 64); 2283 lower_num = 0; 2284 } else { 2285 upper_num = (upper_num << left_shift_bits) | 2286 (lower_num >> (64 - left_shift_bits)); 2287 lower_num = lower_num << left_shift_bits; 2288 } 2289 2290 if (right_shift_bits >= 64) { 2291 lower_num = upper_num >> (right_shift_bits - 64); 2292 upper_num = 0; 2293 } else { 2294 lower_num = (lower_num >> right_shift_bits) | 2295 (upper_num << (64 - right_shift_bits)); 2296 upper_num = upper_num >> right_shift_bits; 2297 } 2298 2299 #ifdef __BIG_ENDIAN_BITFIELD 2300 print_num[0] = upper_num; 2301 print_num[1] = lower_num; 2302 #else 2303 print_num[0] = lower_num; 2304 print_num[1] = upper_num; 2305 #endif 2306 } 2307 2308 static void btf_bitfield_show(void *data, u8 bits_offset, 2309 u8 nr_bits, struct btf_show *show) 2310 { 2311 u16 left_shift_bits, right_shift_bits; 2312 u8 nr_copy_bytes; 2313 u8 nr_copy_bits; 2314 u64 print_num[2] = {}; 2315 2316 nr_copy_bits = nr_bits + bits_offset; 2317 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits); 2318 2319 memcpy(print_num, data, nr_copy_bytes); 2320 2321 #ifdef __BIG_ENDIAN_BITFIELD 2322 left_shift_bits = bits_offset; 2323 #else 2324 left_shift_bits = BITS_PER_U128 - nr_copy_bits; 2325 #endif 2326 right_shift_bits = BITS_PER_U128 - nr_bits; 2327 2328 btf_int128_shift(print_num, left_shift_bits, right_shift_bits); 2329 btf_int128_print(show, print_num); 2330 } 2331 2332 2333 static void btf_int_bits_show(const struct btf *btf, 2334 const struct btf_type *t, 2335 void *data, u8 bits_offset, 2336 struct btf_show *show) 2337 { 2338 u32 int_data = btf_type_int(t); 2339 u8 nr_bits = BTF_INT_BITS(int_data); 2340 u8 total_bits_offset; 2341 2342 /* 2343 * bits_offset is at most 7. 2344 * BTF_INT_OFFSET() cannot exceed 128 bits. 2345 */ 2346 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data); 2347 data += BITS_ROUNDDOWN_BYTES(total_bits_offset); 2348 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset); 2349 btf_bitfield_show(data, bits_offset, nr_bits, show); 2350 } 2351 2352 static void btf_int_show(const struct btf *btf, const struct btf_type *t, 2353 u32 type_id, void *data, u8 bits_offset, 2354 struct btf_show *show) 2355 { 2356 u32 int_data = btf_type_int(t); 2357 u8 encoding = BTF_INT_ENCODING(int_data); 2358 bool sign = encoding & BTF_INT_SIGNED; 2359 u8 nr_bits = BTF_INT_BITS(int_data); 2360 void *safe_data; 2361 2362 safe_data = btf_show_start_type(show, t, type_id, data); 2363 if (!safe_data) 2364 return; 2365 2366 if (bits_offset || BTF_INT_OFFSET(int_data) || 2367 BITS_PER_BYTE_MASKED(nr_bits)) { 2368 btf_int_bits_show(btf, t, safe_data, bits_offset, show); 2369 goto out; 2370 } 2371 2372 switch (nr_bits) { 2373 case 128: 2374 btf_int128_print(show, safe_data); 2375 break; 2376 case 64: 2377 if (sign) 2378 btf_show_type_value(show, "%lld", *(s64 *)safe_data); 2379 else 2380 btf_show_type_value(show, "%llu", *(u64 *)safe_data); 2381 break; 2382 case 32: 2383 if (sign) 2384 btf_show_type_value(show, "%d", *(s32 *)safe_data); 2385 else 2386 btf_show_type_value(show, "%u", *(u32 *)safe_data); 2387 break; 2388 case 16: 2389 if (sign) 2390 btf_show_type_value(show, "%d", *(s16 *)safe_data); 2391 else 2392 btf_show_type_value(show, "%u", *(u16 *)safe_data); 2393 break; 2394 case 8: 2395 if (show->state.array_encoding == BTF_INT_CHAR) { 2396 /* check for null terminator */ 2397 if (show->state.array_terminated) 2398 break; 2399 if (*(char *)data == '\0') { 2400 show->state.array_terminated = 1; 2401 break; 2402 } 2403 if (isprint(*(char *)data)) { 2404 btf_show_type_value(show, "'%c'", 2405 *(char *)safe_data); 2406 break; 2407 } 2408 } 2409 if (sign) 2410 btf_show_type_value(show, "%d", *(s8 *)safe_data); 2411 else 2412 btf_show_type_value(show, "%u", *(u8 *)safe_data); 2413 break; 2414 default: 2415 btf_int_bits_show(btf, t, safe_data, bits_offset, show); 2416 break; 2417 } 2418 out: 2419 btf_show_end_type(show); 2420 } 2421 2422 static const struct btf_kind_operations int_ops = { 2423 .check_meta = btf_int_check_meta, 2424 .resolve = btf_df_resolve, 2425 .check_member = btf_int_check_member, 2426 .check_kflag_member = btf_int_check_kflag_member, 2427 .log_details = btf_int_log, 2428 .show = btf_int_show, 2429 }; 2430 2431 static int btf_modifier_check_member(struct btf_verifier_env *env, 2432 const struct btf_type *struct_type, 2433 const struct btf_member *member, 2434 const struct btf_type *member_type) 2435 { 2436 const struct btf_type *resolved_type; 2437 u32 resolved_type_id = member->type; 2438 struct btf_member resolved_member; 2439 struct btf *btf = env->btf; 2440 2441 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); 2442 if (!resolved_type) { 2443 btf_verifier_log_member(env, struct_type, member, 2444 "Invalid member"); 2445 return -EINVAL; 2446 } 2447 2448 resolved_member = *member; 2449 resolved_member.type = resolved_type_id; 2450 2451 return btf_type_ops(resolved_type)->check_member(env, struct_type, 2452 &resolved_member, 2453 resolved_type); 2454 } 2455 2456 static int btf_modifier_check_kflag_member(struct btf_verifier_env *env, 2457 const struct btf_type *struct_type, 2458 const struct btf_member *member, 2459 const struct btf_type *member_type) 2460 { 2461 const struct btf_type *resolved_type; 2462 u32 resolved_type_id = member->type; 2463 struct btf_member resolved_member; 2464 struct btf *btf = env->btf; 2465 2466 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); 2467 if (!resolved_type) { 2468 btf_verifier_log_member(env, struct_type, member, 2469 "Invalid member"); 2470 return -EINVAL; 2471 } 2472 2473 resolved_member = *member; 2474 resolved_member.type = resolved_type_id; 2475 2476 return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type, 2477 &resolved_member, 2478 resolved_type); 2479 } 2480 2481 static int btf_ptr_check_member(struct btf_verifier_env *env, 2482 const struct btf_type *struct_type, 2483 const struct btf_member *member, 2484 const struct btf_type *member_type) 2485 { 2486 u32 struct_size, struct_bits_off, bytes_offset; 2487 2488 struct_size = struct_type->size; 2489 struct_bits_off = member->offset; 2490 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2491 2492 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 2493 btf_verifier_log_member(env, struct_type, member, 2494 "Member is not byte aligned"); 2495 return -EINVAL; 2496 } 2497 2498 if (struct_size - bytes_offset < sizeof(void *)) { 2499 btf_verifier_log_member(env, struct_type, member, 2500 "Member exceeds struct_size"); 2501 return -EINVAL; 2502 } 2503 2504 return 0; 2505 } 2506 2507 static int btf_ref_type_check_meta(struct btf_verifier_env *env, 2508 const struct btf_type *t, 2509 u32 meta_left) 2510 { 2511 const char *value; 2512 2513 if (btf_type_vlen(t)) { 2514 btf_verifier_log_type(env, t, "vlen != 0"); 2515 return -EINVAL; 2516 } 2517 2518 if (btf_type_kflag(t)) { 2519 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 2520 return -EINVAL; 2521 } 2522 2523 if (!BTF_TYPE_ID_VALID(t->type)) { 2524 btf_verifier_log_type(env, t, "Invalid type_id"); 2525 return -EINVAL; 2526 } 2527 2528 /* typedef/type_tag type must have a valid name, and other ref types, 2529 * volatile, const, restrict, should have a null name. 2530 */ 2531 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { 2532 if (!t->name_off || 2533 !btf_name_valid_identifier(env->btf, t->name_off)) { 2534 btf_verifier_log_type(env, t, "Invalid name"); 2535 return -EINVAL; 2536 } 2537 } else if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPE_TAG) { 2538 value = btf_name_by_offset(env->btf, t->name_off); 2539 if (!value || !value[0]) { 2540 btf_verifier_log_type(env, t, "Invalid name"); 2541 return -EINVAL; 2542 } 2543 } else { 2544 if (t->name_off) { 2545 btf_verifier_log_type(env, t, "Invalid name"); 2546 return -EINVAL; 2547 } 2548 } 2549 2550 btf_verifier_log_type(env, t, NULL); 2551 2552 return 0; 2553 } 2554 2555 static int btf_modifier_resolve(struct btf_verifier_env *env, 2556 const struct resolve_vertex *v) 2557 { 2558 const struct btf_type *t = v->t; 2559 const struct btf_type *next_type; 2560 u32 next_type_id = t->type; 2561 struct btf *btf = env->btf; 2562 2563 next_type = btf_type_by_id(btf, next_type_id); 2564 if (!next_type || btf_type_is_resolve_source_only(next_type)) { 2565 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2566 return -EINVAL; 2567 } 2568 2569 if (!env_type_is_resolve_sink(env, next_type) && 2570 !env_type_is_resolved(env, next_type_id)) 2571 return env_stack_push(env, next_type, next_type_id); 2572 2573 /* Figure out the resolved next_type_id with size. 2574 * They will be stored in the current modifier's 2575 * resolved_ids and resolved_sizes such that it can 2576 * save us a few type-following when we use it later (e.g. in 2577 * pretty print). 2578 */ 2579 if (!btf_type_id_size(btf, &next_type_id, NULL)) { 2580 if (env_type_is_resolved(env, next_type_id)) 2581 next_type = btf_type_id_resolve(btf, &next_type_id); 2582 2583 /* "typedef void new_void", "const void"...etc */ 2584 if (!btf_type_is_void(next_type) && 2585 !btf_type_is_fwd(next_type) && 2586 !btf_type_is_func_proto(next_type)) { 2587 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2588 return -EINVAL; 2589 } 2590 } 2591 2592 env_stack_pop_resolved(env, next_type_id, 0); 2593 2594 return 0; 2595 } 2596 2597 static int btf_var_resolve(struct btf_verifier_env *env, 2598 const struct resolve_vertex *v) 2599 { 2600 const struct btf_type *next_type; 2601 const struct btf_type *t = v->t; 2602 u32 next_type_id = t->type; 2603 struct btf *btf = env->btf; 2604 2605 next_type = btf_type_by_id(btf, next_type_id); 2606 if (!next_type || btf_type_is_resolve_source_only(next_type)) { 2607 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2608 return -EINVAL; 2609 } 2610 2611 if (!env_type_is_resolve_sink(env, next_type) && 2612 !env_type_is_resolved(env, next_type_id)) 2613 return env_stack_push(env, next_type, next_type_id); 2614 2615 if (btf_type_is_modifier(next_type)) { 2616 const struct btf_type *resolved_type; 2617 u32 resolved_type_id; 2618 2619 resolved_type_id = next_type_id; 2620 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); 2621 2622 if (btf_type_is_ptr(resolved_type) && 2623 !env_type_is_resolve_sink(env, resolved_type) && 2624 !env_type_is_resolved(env, resolved_type_id)) 2625 return env_stack_push(env, resolved_type, 2626 resolved_type_id); 2627 } 2628 2629 /* We must resolve to something concrete at this point, no 2630 * forward types or similar that would resolve to size of 2631 * zero is allowed. 2632 */ 2633 if (!btf_type_id_size(btf, &next_type_id, NULL)) { 2634 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2635 return -EINVAL; 2636 } 2637 2638 env_stack_pop_resolved(env, next_type_id, 0); 2639 2640 return 0; 2641 } 2642 2643 static int btf_ptr_resolve(struct btf_verifier_env *env, 2644 const struct resolve_vertex *v) 2645 { 2646 const struct btf_type *next_type; 2647 const struct btf_type *t = v->t; 2648 u32 next_type_id = t->type; 2649 struct btf *btf = env->btf; 2650 2651 next_type = btf_type_by_id(btf, next_type_id); 2652 if (!next_type || btf_type_is_resolve_source_only(next_type)) { 2653 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2654 return -EINVAL; 2655 } 2656 2657 if (!env_type_is_resolve_sink(env, next_type) && 2658 !env_type_is_resolved(env, next_type_id)) 2659 return env_stack_push(env, next_type, next_type_id); 2660 2661 /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY, 2662 * the modifier may have stopped resolving when it was resolved 2663 * to a ptr (last-resolved-ptr). 2664 * 2665 * We now need to continue from the last-resolved-ptr to 2666 * ensure the last-resolved-ptr will not referring back to 2667 * the current ptr (t). 2668 */ 2669 if (btf_type_is_modifier(next_type)) { 2670 const struct btf_type *resolved_type; 2671 u32 resolved_type_id; 2672 2673 resolved_type_id = next_type_id; 2674 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); 2675 2676 if (btf_type_is_ptr(resolved_type) && 2677 !env_type_is_resolve_sink(env, resolved_type) && 2678 !env_type_is_resolved(env, resolved_type_id)) 2679 return env_stack_push(env, resolved_type, 2680 resolved_type_id); 2681 } 2682 2683 if (!btf_type_id_size(btf, &next_type_id, NULL)) { 2684 if (env_type_is_resolved(env, next_type_id)) 2685 next_type = btf_type_id_resolve(btf, &next_type_id); 2686 2687 if (!btf_type_is_void(next_type) && 2688 !btf_type_is_fwd(next_type) && 2689 !btf_type_is_func_proto(next_type)) { 2690 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2691 return -EINVAL; 2692 } 2693 } 2694 2695 env_stack_pop_resolved(env, next_type_id, 0); 2696 2697 return 0; 2698 } 2699 2700 static void btf_modifier_show(const struct btf *btf, 2701 const struct btf_type *t, 2702 u32 type_id, void *data, 2703 u8 bits_offset, struct btf_show *show) 2704 { 2705 if (btf->resolved_ids) 2706 t = btf_type_id_resolve(btf, &type_id); 2707 else 2708 t = btf_type_skip_modifiers(btf, type_id, NULL); 2709 2710 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); 2711 } 2712 2713 static void btf_var_show(const struct btf *btf, const struct btf_type *t, 2714 u32 type_id, void *data, u8 bits_offset, 2715 struct btf_show *show) 2716 { 2717 t = btf_type_id_resolve(btf, &type_id); 2718 2719 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); 2720 } 2721 2722 static void btf_ptr_show(const struct btf *btf, const struct btf_type *t, 2723 u32 type_id, void *data, u8 bits_offset, 2724 struct btf_show *show) 2725 { 2726 void *safe_data; 2727 2728 safe_data = btf_show_start_type(show, t, type_id, data); 2729 if (!safe_data) 2730 return; 2731 2732 /* It is a hashed value unless BTF_SHOW_PTR_RAW is specified */ 2733 if (show->flags & BTF_SHOW_PTR_RAW) 2734 btf_show_type_value(show, "0x%px", *(void **)safe_data); 2735 else 2736 btf_show_type_value(show, "0x%p", *(void **)safe_data); 2737 btf_show_end_type(show); 2738 } 2739 2740 static void btf_ref_type_log(struct btf_verifier_env *env, 2741 const struct btf_type *t) 2742 { 2743 btf_verifier_log(env, "type_id=%u", t->type); 2744 } 2745 2746 static struct btf_kind_operations modifier_ops = { 2747 .check_meta = btf_ref_type_check_meta, 2748 .resolve = btf_modifier_resolve, 2749 .check_member = btf_modifier_check_member, 2750 .check_kflag_member = btf_modifier_check_kflag_member, 2751 .log_details = btf_ref_type_log, 2752 .show = btf_modifier_show, 2753 }; 2754 2755 static struct btf_kind_operations ptr_ops = { 2756 .check_meta = btf_ref_type_check_meta, 2757 .resolve = btf_ptr_resolve, 2758 .check_member = btf_ptr_check_member, 2759 .check_kflag_member = btf_generic_check_kflag_member, 2760 .log_details = btf_ref_type_log, 2761 .show = btf_ptr_show, 2762 }; 2763 2764 static s32 btf_fwd_check_meta(struct btf_verifier_env *env, 2765 const struct btf_type *t, 2766 u32 meta_left) 2767 { 2768 if (btf_type_vlen(t)) { 2769 btf_verifier_log_type(env, t, "vlen != 0"); 2770 return -EINVAL; 2771 } 2772 2773 if (t->type) { 2774 btf_verifier_log_type(env, t, "type != 0"); 2775 return -EINVAL; 2776 } 2777 2778 /* fwd type must have a valid name */ 2779 if (!t->name_off || 2780 !btf_name_valid_identifier(env->btf, t->name_off)) { 2781 btf_verifier_log_type(env, t, "Invalid name"); 2782 return -EINVAL; 2783 } 2784 2785 btf_verifier_log_type(env, t, NULL); 2786 2787 return 0; 2788 } 2789 2790 static void btf_fwd_type_log(struct btf_verifier_env *env, 2791 const struct btf_type *t) 2792 { 2793 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct"); 2794 } 2795 2796 static struct btf_kind_operations fwd_ops = { 2797 .check_meta = btf_fwd_check_meta, 2798 .resolve = btf_df_resolve, 2799 .check_member = btf_df_check_member, 2800 .check_kflag_member = btf_df_check_kflag_member, 2801 .log_details = btf_fwd_type_log, 2802 .show = btf_df_show, 2803 }; 2804 2805 static int btf_array_check_member(struct btf_verifier_env *env, 2806 const struct btf_type *struct_type, 2807 const struct btf_member *member, 2808 const struct btf_type *member_type) 2809 { 2810 u32 struct_bits_off = member->offset; 2811 u32 struct_size, bytes_offset; 2812 u32 array_type_id, array_size; 2813 struct btf *btf = env->btf; 2814 2815 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 2816 btf_verifier_log_member(env, struct_type, member, 2817 "Member is not byte aligned"); 2818 return -EINVAL; 2819 } 2820 2821 array_type_id = member->type; 2822 btf_type_id_size(btf, &array_type_id, &array_size); 2823 struct_size = struct_type->size; 2824 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2825 if (struct_size - bytes_offset < array_size) { 2826 btf_verifier_log_member(env, struct_type, member, 2827 "Member exceeds struct_size"); 2828 return -EINVAL; 2829 } 2830 2831 return 0; 2832 } 2833 2834 static s32 btf_array_check_meta(struct btf_verifier_env *env, 2835 const struct btf_type *t, 2836 u32 meta_left) 2837 { 2838 const struct btf_array *array = btf_type_array(t); 2839 u32 meta_needed = sizeof(*array); 2840 2841 if (meta_left < meta_needed) { 2842 btf_verifier_log_basic(env, t, 2843 "meta_left:%u meta_needed:%u", 2844 meta_left, meta_needed); 2845 return -EINVAL; 2846 } 2847 2848 /* array type should not have a name */ 2849 if (t->name_off) { 2850 btf_verifier_log_type(env, t, "Invalid name"); 2851 return -EINVAL; 2852 } 2853 2854 if (btf_type_vlen(t)) { 2855 btf_verifier_log_type(env, t, "vlen != 0"); 2856 return -EINVAL; 2857 } 2858 2859 if (btf_type_kflag(t)) { 2860 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 2861 return -EINVAL; 2862 } 2863 2864 if (t->size) { 2865 btf_verifier_log_type(env, t, "size != 0"); 2866 return -EINVAL; 2867 } 2868 2869 /* Array elem type and index type cannot be in type void, 2870 * so !array->type and !array->index_type are not allowed. 2871 */ 2872 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) { 2873 btf_verifier_log_type(env, t, "Invalid elem"); 2874 return -EINVAL; 2875 } 2876 2877 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) { 2878 btf_verifier_log_type(env, t, "Invalid index"); 2879 return -EINVAL; 2880 } 2881 2882 btf_verifier_log_type(env, t, NULL); 2883 2884 return meta_needed; 2885 } 2886 2887 static int btf_array_resolve(struct btf_verifier_env *env, 2888 const struct resolve_vertex *v) 2889 { 2890 const struct btf_array *array = btf_type_array(v->t); 2891 const struct btf_type *elem_type, *index_type; 2892 u32 elem_type_id, index_type_id; 2893 struct btf *btf = env->btf; 2894 u32 elem_size; 2895 2896 /* Check array->index_type */ 2897 index_type_id = array->index_type; 2898 index_type = btf_type_by_id(btf, index_type_id); 2899 if (btf_type_nosize_or_null(index_type) || 2900 btf_type_is_resolve_source_only(index_type)) { 2901 btf_verifier_log_type(env, v->t, "Invalid index"); 2902 return -EINVAL; 2903 } 2904 2905 if (!env_type_is_resolve_sink(env, index_type) && 2906 !env_type_is_resolved(env, index_type_id)) 2907 return env_stack_push(env, index_type, index_type_id); 2908 2909 index_type = btf_type_id_size(btf, &index_type_id, NULL); 2910 if (!index_type || !btf_type_is_int(index_type) || 2911 !btf_type_int_is_regular(index_type)) { 2912 btf_verifier_log_type(env, v->t, "Invalid index"); 2913 return -EINVAL; 2914 } 2915 2916 /* Check array->type */ 2917 elem_type_id = array->type; 2918 elem_type = btf_type_by_id(btf, elem_type_id); 2919 if (btf_type_nosize_or_null(elem_type) || 2920 btf_type_is_resolve_source_only(elem_type)) { 2921 btf_verifier_log_type(env, v->t, 2922 "Invalid elem"); 2923 return -EINVAL; 2924 } 2925 2926 if (!env_type_is_resolve_sink(env, elem_type) && 2927 !env_type_is_resolved(env, elem_type_id)) 2928 return env_stack_push(env, elem_type, elem_type_id); 2929 2930 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); 2931 if (!elem_type) { 2932 btf_verifier_log_type(env, v->t, "Invalid elem"); 2933 return -EINVAL; 2934 } 2935 2936 if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) { 2937 btf_verifier_log_type(env, v->t, "Invalid array of int"); 2938 return -EINVAL; 2939 } 2940 2941 if (array->nelems && elem_size > U32_MAX / array->nelems) { 2942 btf_verifier_log_type(env, v->t, 2943 "Array size overflows U32_MAX"); 2944 return -EINVAL; 2945 } 2946 2947 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems); 2948 2949 return 0; 2950 } 2951 2952 static void btf_array_log(struct btf_verifier_env *env, 2953 const struct btf_type *t) 2954 { 2955 const struct btf_array *array = btf_type_array(t); 2956 2957 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u", 2958 array->type, array->index_type, array->nelems); 2959 } 2960 2961 static void __btf_array_show(const struct btf *btf, const struct btf_type *t, 2962 u32 type_id, void *data, u8 bits_offset, 2963 struct btf_show *show) 2964 { 2965 const struct btf_array *array = btf_type_array(t); 2966 const struct btf_kind_operations *elem_ops; 2967 const struct btf_type *elem_type; 2968 u32 i, elem_size = 0, elem_type_id; 2969 u16 encoding = 0; 2970 2971 elem_type_id = array->type; 2972 elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL); 2973 if (elem_type && btf_type_has_size(elem_type)) 2974 elem_size = elem_type->size; 2975 2976 if (elem_type && btf_type_is_int(elem_type)) { 2977 u32 int_type = btf_type_int(elem_type); 2978 2979 encoding = BTF_INT_ENCODING(int_type); 2980 2981 /* 2982 * BTF_INT_CHAR encoding never seems to be set for 2983 * char arrays, so if size is 1 and element is 2984 * printable as a char, we'll do that. 2985 */ 2986 if (elem_size == 1) 2987 encoding = BTF_INT_CHAR; 2988 } 2989 2990 if (!btf_show_start_array_type(show, t, type_id, encoding, data)) 2991 return; 2992 2993 if (!elem_type) 2994 goto out; 2995 elem_ops = btf_type_ops(elem_type); 2996 2997 for (i = 0; i < array->nelems; i++) { 2998 2999 btf_show_start_array_member(show); 3000 3001 elem_ops->show(btf, elem_type, elem_type_id, data, 3002 bits_offset, show); 3003 data += elem_size; 3004 3005 btf_show_end_array_member(show); 3006 3007 if (show->state.array_terminated) 3008 break; 3009 } 3010 out: 3011 btf_show_end_array_type(show); 3012 } 3013 3014 static void btf_array_show(const struct btf *btf, const struct btf_type *t, 3015 u32 type_id, void *data, u8 bits_offset, 3016 struct btf_show *show) 3017 { 3018 const struct btf_member *m = show->state.member; 3019 3020 /* 3021 * First check if any members would be shown (are non-zero). 3022 * See comments above "struct btf_show" definition for more 3023 * details on how this works at a high-level. 3024 */ 3025 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { 3026 if (!show->state.depth_check) { 3027 show->state.depth_check = show->state.depth + 1; 3028 show->state.depth_to_show = 0; 3029 } 3030 __btf_array_show(btf, t, type_id, data, bits_offset, show); 3031 show->state.member = m; 3032 3033 if (show->state.depth_check != show->state.depth + 1) 3034 return; 3035 show->state.depth_check = 0; 3036 3037 if (show->state.depth_to_show <= show->state.depth) 3038 return; 3039 /* 3040 * Reaching here indicates we have recursed and found 3041 * non-zero array member(s). 3042 */ 3043 } 3044 __btf_array_show(btf, t, type_id, data, bits_offset, show); 3045 } 3046 3047 static struct btf_kind_operations array_ops = { 3048 .check_meta = btf_array_check_meta, 3049 .resolve = btf_array_resolve, 3050 .check_member = btf_array_check_member, 3051 .check_kflag_member = btf_generic_check_kflag_member, 3052 .log_details = btf_array_log, 3053 .show = btf_array_show, 3054 }; 3055 3056 static int btf_struct_check_member(struct btf_verifier_env *env, 3057 const struct btf_type *struct_type, 3058 const struct btf_member *member, 3059 const struct btf_type *member_type) 3060 { 3061 u32 struct_bits_off = member->offset; 3062 u32 struct_size, bytes_offset; 3063 3064 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 3065 btf_verifier_log_member(env, struct_type, member, 3066 "Member is not byte aligned"); 3067 return -EINVAL; 3068 } 3069 3070 struct_size = struct_type->size; 3071 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 3072 if (struct_size - bytes_offset < member_type->size) { 3073 btf_verifier_log_member(env, struct_type, member, 3074 "Member exceeds struct_size"); 3075 return -EINVAL; 3076 } 3077 3078 return 0; 3079 } 3080 3081 static s32 btf_struct_check_meta(struct btf_verifier_env *env, 3082 const struct btf_type *t, 3083 u32 meta_left) 3084 { 3085 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION; 3086 const struct btf_member *member; 3087 u32 meta_needed, last_offset; 3088 struct btf *btf = env->btf; 3089 u32 struct_size = t->size; 3090 u32 offset; 3091 u16 i; 3092 3093 meta_needed = btf_type_vlen(t) * sizeof(*member); 3094 if (meta_left < meta_needed) { 3095 btf_verifier_log_basic(env, t, 3096 "meta_left:%u meta_needed:%u", 3097 meta_left, meta_needed); 3098 return -EINVAL; 3099 } 3100 3101 /* struct type either no name or a valid one */ 3102 if (t->name_off && 3103 !btf_name_valid_identifier(env->btf, t->name_off)) { 3104 btf_verifier_log_type(env, t, "Invalid name"); 3105 return -EINVAL; 3106 } 3107 3108 btf_verifier_log_type(env, t, NULL); 3109 3110 last_offset = 0; 3111 for_each_member(i, t, member) { 3112 if (!btf_name_offset_valid(btf, member->name_off)) { 3113 btf_verifier_log_member(env, t, member, 3114 "Invalid member name_offset:%u", 3115 member->name_off); 3116 return -EINVAL; 3117 } 3118 3119 /* struct member either no name or a valid one */ 3120 if (member->name_off && 3121 !btf_name_valid_identifier(btf, member->name_off)) { 3122 btf_verifier_log_member(env, t, member, "Invalid name"); 3123 return -EINVAL; 3124 } 3125 /* A member cannot be in type void */ 3126 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) { 3127 btf_verifier_log_member(env, t, member, 3128 "Invalid type_id"); 3129 return -EINVAL; 3130 } 3131 3132 offset = __btf_member_bit_offset(t, member); 3133 if (is_union && offset) { 3134 btf_verifier_log_member(env, t, member, 3135 "Invalid member bits_offset"); 3136 return -EINVAL; 3137 } 3138 3139 /* 3140 * ">" instead of ">=" because the last member could be 3141 * "char a[0];" 3142 */ 3143 if (last_offset > offset) { 3144 btf_verifier_log_member(env, t, member, 3145 "Invalid member bits_offset"); 3146 return -EINVAL; 3147 } 3148 3149 if (BITS_ROUNDUP_BYTES(offset) > struct_size) { 3150 btf_verifier_log_member(env, t, member, 3151 "Member bits_offset exceeds its struct size"); 3152 return -EINVAL; 3153 } 3154 3155 btf_verifier_log_member(env, t, member, NULL); 3156 last_offset = offset; 3157 } 3158 3159 return meta_needed; 3160 } 3161 3162 static int btf_struct_resolve(struct btf_verifier_env *env, 3163 const struct resolve_vertex *v) 3164 { 3165 const struct btf_member *member; 3166 int err; 3167 u16 i; 3168 3169 /* Before continue resolving the next_member, 3170 * ensure the last member is indeed resolved to a 3171 * type with size info. 3172 */ 3173 if (v->next_member) { 3174 const struct btf_type *last_member_type; 3175 const struct btf_member *last_member; 3176 u32 last_member_type_id; 3177 3178 last_member = btf_type_member(v->t) + v->next_member - 1; 3179 last_member_type_id = last_member->type; 3180 if (WARN_ON_ONCE(!env_type_is_resolved(env, 3181 last_member_type_id))) 3182 return -EINVAL; 3183 3184 last_member_type = btf_type_by_id(env->btf, 3185 last_member_type_id); 3186 if (btf_type_kflag(v->t)) 3187 err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t, 3188 last_member, 3189 last_member_type); 3190 else 3191 err = btf_type_ops(last_member_type)->check_member(env, v->t, 3192 last_member, 3193 last_member_type); 3194 if (err) 3195 return err; 3196 } 3197 3198 for_each_member_from(i, v->next_member, v->t, member) { 3199 u32 member_type_id = member->type; 3200 const struct btf_type *member_type = btf_type_by_id(env->btf, 3201 member_type_id); 3202 3203 if (btf_type_nosize_or_null(member_type) || 3204 btf_type_is_resolve_source_only(member_type)) { 3205 btf_verifier_log_member(env, v->t, member, 3206 "Invalid member"); 3207 return -EINVAL; 3208 } 3209 3210 if (!env_type_is_resolve_sink(env, member_type) && 3211 !env_type_is_resolved(env, member_type_id)) { 3212 env_stack_set_next_member(env, i + 1); 3213 return env_stack_push(env, member_type, member_type_id); 3214 } 3215 3216 if (btf_type_kflag(v->t)) 3217 err = btf_type_ops(member_type)->check_kflag_member(env, v->t, 3218 member, 3219 member_type); 3220 else 3221 err = btf_type_ops(member_type)->check_member(env, v->t, 3222 member, 3223 member_type); 3224 if (err) 3225 return err; 3226 } 3227 3228 env_stack_pop_resolved(env, 0, 0); 3229 3230 return 0; 3231 } 3232 3233 static void btf_struct_log(struct btf_verifier_env *env, 3234 const struct btf_type *t) 3235 { 3236 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 3237 } 3238 3239 enum { 3240 BTF_FIELD_IGNORE = 0, 3241 BTF_FIELD_FOUND = 1, 3242 }; 3243 3244 struct btf_field_info { 3245 enum btf_field_type type; 3246 u32 off; 3247 union { 3248 struct { 3249 u32 type_id; 3250 } kptr; 3251 struct { 3252 const char *node_name; 3253 u32 value_btf_id; 3254 } graph_root; 3255 }; 3256 }; 3257 3258 static int btf_find_struct(const struct btf *btf, const struct btf_type *t, 3259 u32 off, int sz, enum btf_field_type field_type, 3260 struct btf_field_info *info) 3261 { 3262 if (!__btf_type_is_struct(t)) 3263 return BTF_FIELD_IGNORE; 3264 if (t->size != sz) 3265 return BTF_FIELD_IGNORE; 3266 info->type = field_type; 3267 info->off = off; 3268 return BTF_FIELD_FOUND; 3269 } 3270 3271 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, 3272 u32 off, int sz, struct btf_field_info *info) 3273 { 3274 enum btf_field_type type; 3275 u32 res_id; 3276 3277 /* Permit modifiers on the pointer itself */ 3278 if (btf_type_is_volatile(t)) 3279 t = btf_type_by_id(btf, t->type); 3280 /* For PTR, sz is always == 8 */ 3281 if (!btf_type_is_ptr(t)) 3282 return BTF_FIELD_IGNORE; 3283 t = btf_type_by_id(btf, t->type); 3284 3285 if (!btf_type_is_type_tag(t)) 3286 return BTF_FIELD_IGNORE; 3287 /* Reject extra tags */ 3288 if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) 3289 return -EINVAL; 3290 if (!strcmp("kptr_untrusted", __btf_name_by_offset(btf, t->name_off))) 3291 type = BPF_KPTR_UNREF; 3292 else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off))) 3293 type = BPF_KPTR_REF; 3294 else 3295 return -EINVAL; 3296 3297 /* Get the base type */ 3298 t = btf_type_skip_modifiers(btf, t->type, &res_id); 3299 /* Only pointer to struct is allowed */ 3300 if (!__btf_type_is_struct(t)) 3301 return -EINVAL; 3302 3303 info->type = type; 3304 info->off = off; 3305 info->kptr.type_id = res_id; 3306 return BTF_FIELD_FOUND; 3307 } 3308 3309 static const char *btf_find_decl_tag_value(const struct btf *btf, 3310 const struct btf_type *pt, 3311 int comp_idx, const char *tag_key) 3312 { 3313 int i; 3314 3315 for (i = 1; i < btf_nr_types(btf); i++) { 3316 const struct btf_type *t = btf_type_by_id(btf, i); 3317 int len = strlen(tag_key); 3318 3319 if (!btf_type_is_decl_tag(t)) 3320 continue; 3321 if (pt != btf_type_by_id(btf, t->type) || 3322 btf_type_decl_tag(t)->component_idx != comp_idx) 3323 continue; 3324 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) 3325 continue; 3326 return __btf_name_by_offset(btf, t->name_off) + len; 3327 } 3328 return NULL; 3329 } 3330 3331 static int 3332 btf_find_graph_root(const struct btf *btf, const struct btf_type *pt, 3333 const struct btf_type *t, int comp_idx, u32 off, 3334 int sz, struct btf_field_info *info, 3335 enum btf_field_type head_type) 3336 { 3337 const char *node_field_name; 3338 const char *value_type; 3339 s32 id; 3340 3341 if (!__btf_type_is_struct(t)) 3342 return BTF_FIELD_IGNORE; 3343 if (t->size != sz) 3344 return BTF_FIELD_IGNORE; 3345 value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:"); 3346 if (!value_type) 3347 return -EINVAL; 3348 node_field_name = strstr(value_type, ":"); 3349 if (!node_field_name) 3350 return -EINVAL; 3351 value_type = kstrndup(value_type, node_field_name - value_type, GFP_KERNEL | __GFP_NOWARN); 3352 if (!value_type) 3353 return -ENOMEM; 3354 id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT); 3355 kfree(value_type); 3356 if (id < 0) 3357 return id; 3358 node_field_name++; 3359 if (str_is_empty(node_field_name)) 3360 return -EINVAL; 3361 info->type = head_type; 3362 info->off = off; 3363 info->graph_root.value_btf_id = id; 3364 info->graph_root.node_name = node_field_name; 3365 return BTF_FIELD_FOUND; 3366 } 3367 3368 #define field_mask_test_name(field_type, field_type_str) \ 3369 if (field_mask & field_type && !strcmp(name, field_type_str)) { \ 3370 type = field_type; \ 3371 goto end; \ 3372 } 3373 3374 static int btf_get_field_type(const char *name, u32 field_mask, u32 *seen_mask, 3375 int *align, int *sz) 3376 { 3377 int type = 0; 3378 3379 if (field_mask & BPF_SPIN_LOCK) { 3380 if (!strcmp(name, "bpf_spin_lock")) { 3381 if (*seen_mask & BPF_SPIN_LOCK) 3382 return -E2BIG; 3383 *seen_mask |= BPF_SPIN_LOCK; 3384 type = BPF_SPIN_LOCK; 3385 goto end; 3386 } 3387 } 3388 if (field_mask & BPF_TIMER) { 3389 if (!strcmp(name, "bpf_timer")) { 3390 if (*seen_mask & BPF_TIMER) 3391 return -E2BIG; 3392 *seen_mask |= BPF_TIMER; 3393 type = BPF_TIMER; 3394 goto end; 3395 } 3396 } 3397 field_mask_test_name(BPF_LIST_HEAD, "bpf_list_head"); 3398 field_mask_test_name(BPF_LIST_NODE, "bpf_list_node"); 3399 field_mask_test_name(BPF_RB_ROOT, "bpf_rb_root"); 3400 field_mask_test_name(BPF_RB_NODE, "bpf_rb_node"); 3401 field_mask_test_name(BPF_REFCOUNT, "bpf_refcount"); 3402 3403 /* Only return BPF_KPTR when all other types with matchable names fail */ 3404 if (field_mask & BPF_KPTR) { 3405 type = BPF_KPTR_REF; 3406 goto end; 3407 } 3408 return 0; 3409 end: 3410 *sz = btf_field_type_size(type); 3411 *align = btf_field_type_align(type); 3412 return type; 3413 } 3414 3415 #undef field_mask_test_name 3416 3417 static int btf_find_struct_field(const struct btf *btf, 3418 const struct btf_type *t, u32 field_mask, 3419 struct btf_field_info *info, int info_cnt) 3420 { 3421 int ret, idx = 0, align, sz, field_type; 3422 const struct btf_member *member; 3423 struct btf_field_info tmp; 3424 u32 i, off, seen_mask = 0; 3425 3426 for_each_member(i, t, member) { 3427 const struct btf_type *member_type = btf_type_by_id(btf, 3428 member->type); 3429 3430 field_type = btf_get_field_type(__btf_name_by_offset(btf, member_type->name_off), 3431 field_mask, &seen_mask, &align, &sz); 3432 if (field_type == 0) 3433 continue; 3434 if (field_type < 0) 3435 return field_type; 3436 3437 off = __btf_member_bit_offset(t, member); 3438 if (off % 8) 3439 /* valid C code cannot generate such BTF */ 3440 return -EINVAL; 3441 off /= 8; 3442 if (off % align) 3443 continue; 3444 3445 switch (field_type) { 3446 case BPF_SPIN_LOCK: 3447 case BPF_TIMER: 3448 case BPF_LIST_NODE: 3449 case BPF_RB_NODE: 3450 case BPF_REFCOUNT: 3451 ret = btf_find_struct(btf, member_type, off, sz, field_type, 3452 idx < info_cnt ? &info[idx] : &tmp); 3453 if (ret < 0) 3454 return ret; 3455 break; 3456 case BPF_KPTR_UNREF: 3457 case BPF_KPTR_REF: 3458 ret = btf_find_kptr(btf, member_type, off, sz, 3459 idx < info_cnt ? &info[idx] : &tmp); 3460 if (ret < 0) 3461 return ret; 3462 break; 3463 case BPF_LIST_HEAD: 3464 case BPF_RB_ROOT: 3465 ret = btf_find_graph_root(btf, t, member_type, 3466 i, off, sz, 3467 idx < info_cnt ? &info[idx] : &tmp, 3468 field_type); 3469 if (ret < 0) 3470 return ret; 3471 break; 3472 default: 3473 return -EFAULT; 3474 } 3475 3476 if (ret == BTF_FIELD_IGNORE) 3477 continue; 3478 if (idx >= info_cnt) 3479 return -E2BIG; 3480 ++idx; 3481 } 3482 return idx; 3483 } 3484 3485 static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, 3486 u32 field_mask, struct btf_field_info *info, 3487 int info_cnt) 3488 { 3489 int ret, idx = 0, align, sz, field_type; 3490 const struct btf_var_secinfo *vsi; 3491 struct btf_field_info tmp; 3492 u32 i, off, seen_mask = 0; 3493 3494 for_each_vsi(i, t, vsi) { 3495 const struct btf_type *var = btf_type_by_id(btf, vsi->type); 3496 const struct btf_type *var_type = btf_type_by_id(btf, var->type); 3497 3498 field_type = btf_get_field_type(__btf_name_by_offset(btf, var_type->name_off), 3499 field_mask, &seen_mask, &align, &sz); 3500 if (field_type == 0) 3501 continue; 3502 if (field_type < 0) 3503 return field_type; 3504 3505 off = vsi->offset; 3506 if (vsi->size != sz) 3507 continue; 3508 if (off % align) 3509 continue; 3510 3511 switch (field_type) { 3512 case BPF_SPIN_LOCK: 3513 case BPF_TIMER: 3514 case BPF_LIST_NODE: 3515 case BPF_RB_NODE: 3516 case BPF_REFCOUNT: 3517 ret = btf_find_struct(btf, var_type, off, sz, field_type, 3518 idx < info_cnt ? &info[idx] : &tmp); 3519 if (ret < 0) 3520 return ret; 3521 break; 3522 case BPF_KPTR_UNREF: 3523 case BPF_KPTR_REF: 3524 ret = btf_find_kptr(btf, var_type, off, sz, 3525 idx < info_cnt ? &info[idx] : &tmp); 3526 if (ret < 0) 3527 return ret; 3528 break; 3529 case BPF_LIST_HEAD: 3530 case BPF_RB_ROOT: 3531 ret = btf_find_graph_root(btf, var, var_type, 3532 -1, off, sz, 3533 idx < info_cnt ? &info[idx] : &tmp, 3534 field_type); 3535 if (ret < 0) 3536 return ret; 3537 break; 3538 default: 3539 return -EFAULT; 3540 } 3541 3542 if (ret == BTF_FIELD_IGNORE) 3543 continue; 3544 if (idx >= info_cnt) 3545 return -E2BIG; 3546 ++idx; 3547 } 3548 return idx; 3549 } 3550 3551 static int btf_find_field(const struct btf *btf, const struct btf_type *t, 3552 u32 field_mask, struct btf_field_info *info, 3553 int info_cnt) 3554 { 3555 if (__btf_type_is_struct(t)) 3556 return btf_find_struct_field(btf, t, field_mask, info, info_cnt); 3557 else if (btf_type_is_datasec(t)) 3558 return btf_find_datasec_var(btf, t, field_mask, info, info_cnt); 3559 return -EINVAL; 3560 } 3561 3562 static int btf_parse_kptr(const struct btf *btf, struct btf_field *field, 3563 struct btf_field_info *info) 3564 { 3565 struct module *mod = NULL; 3566 const struct btf_type *t; 3567 /* If a matching btf type is found in kernel or module BTFs, kptr_ref 3568 * is that BTF, otherwise it's program BTF 3569 */ 3570 struct btf *kptr_btf; 3571 int ret; 3572 s32 id; 3573 3574 /* Find type in map BTF, and use it to look up the matching type 3575 * in vmlinux or module BTFs, by name and kind. 3576 */ 3577 t = btf_type_by_id(btf, info->kptr.type_id); 3578 id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info), 3579 &kptr_btf); 3580 if (id == -ENOENT) { 3581 /* btf_parse_kptr should only be called w/ btf = program BTF */ 3582 WARN_ON_ONCE(btf_is_kernel(btf)); 3583 3584 /* Type exists only in program BTF. Assume that it's a MEM_ALLOC 3585 * kptr allocated via bpf_obj_new 3586 */ 3587 field->kptr.dtor = NULL; 3588 id = info->kptr.type_id; 3589 kptr_btf = (struct btf *)btf; 3590 btf_get(kptr_btf); 3591 goto found_dtor; 3592 } 3593 if (id < 0) 3594 return id; 3595 3596 /* Find and stash the function pointer for the destruction function that 3597 * needs to be eventually invoked from the map free path. 3598 */ 3599 if (info->type == BPF_KPTR_REF) { 3600 const struct btf_type *dtor_func; 3601 const char *dtor_func_name; 3602 unsigned long addr; 3603 s32 dtor_btf_id; 3604 3605 /* This call also serves as a whitelist of allowed objects that 3606 * can be used as a referenced pointer and be stored in a map at 3607 * the same time. 3608 */ 3609 dtor_btf_id = btf_find_dtor_kfunc(kptr_btf, id); 3610 if (dtor_btf_id < 0) { 3611 ret = dtor_btf_id; 3612 goto end_btf; 3613 } 3614 3615 dtor_func = btf_type_by_id(kptr_btf, dtor_btf_id); 3616 if (!dtor_func) { 3617 ret = -ENOENT; 3618 goto end_btf; 3619 } 3620 3621 if (btf_is_module(kptr_btf)) { 3622 mod = btf_try_get_module(kptr_btf); 3623 if (!mod) { 3624 ret = -ENXIO; 3625 goto end_btf; 3626 } 3627 } 3628 3629 /* We already verified dtor_func to be btf_type_is_func 3630 * in register_btf_id_dtor_kfuncs. 3631 */ 3632 dtor_func_name = __btf_name_by_offset(kptr_btf, dtor_func->name_off); 3633 addr = kallsyms_lookup_name(dtor_func_name); 3634 if (!addr) { 3635 ret = -EINVAL; 3636 goto end_mod; 3637 } 3638 field->kptr.dtor = (void *)addr; 3639 } 3640 3641 found_dtor: 3642 field->kptr.btf_id = id; 3643 field->kptr.btf = kptr_btf; 3644 field->kptr.module = mod; 3645 return 0; 3646 end_mod: 3647 module_put(mod); 3648 end_btf: 3649 btf_put(kptr_btf); 3650 return ret; 3651 } 3652 3653 static int btf_parse_graph_root(const struct btf *btf, 3654 struct btf_field *field, 3655 struct btf_field_info *info, 3656 const char *node_type_name, 3657 size_t node_type_align) 3658 { 3659 const struct btf_type *t, *n = NULL; 3660 const struct btf_member *member; 3661 u32 offset; 3662 int i; 3663 3664 t = btf_type_by_id(btf, info->graph_root.value_btf_id); 3665 /* We've already checked that value_btf_id is a struct type. We 3666 * just need to figure out the offset of the list_node, and 3667 * verify its type. 3668 */ 3669 for_each_member(i, t, member) { 3670 if (strcmp(info->graph_root.node_name, 3671 __btf_name_by_offset(btf, member->name_off))) 3672 continue; 3673 /* Invalid BTF, two members with same name */ 3674 if (n) 3675 return -EINVAL; 3676 n = btf_type_by_id(btf, member->type); 3677 if (!__btf_type_is_struct(n)) 3678 return -EINVAL; 3679 if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off))) 3680 return -EINVAL; 3681 offset = __btf_member_bit_offset(n, member); 3682 if (offset % 8) 3683 return -EINVAL; 3684 offset /= 8; 3685 if (offset % node_type_align) 3686 return -EINVAL; 3687 3688 field->graph_root.btf = (struct btf *)btf; 3689 field->graph_root.value_btf_id = info->graph_root.value_btf_id; 3690 field->graph_root.node_offset = offset; 3691 } 3692 if (!n) 3693 return -ENOENT; 3694 return 0; 3695 } 3696 3697 static int btf_parse_list_head(const struct btf *btf, struct btf_field *field, 3698 struct btf_field_info *info) 3699 { 3700 return btf_parse_graph_root(btf, field, info, "bpf_list_node", 3701 __alignof__(struct bpf_list_node)); 3702 } 3703 3704 static int btf_parse_rb_root(const struct btf *btf, struct btf_field *field, 3705 struct btf_field_info *info) 3706 { 3707 return btf_parse_graph_root(btf, field, info, "bpf_rb_node", 3708 __alignof__(struct bpf_rb_node)); 3709 } 3710 3711 static int btf_field_cmp(const void *_a, const void *_b, const void *priv) 3712 { 3713 const struct btf_field *a = (const struct btf_field *)_a; 3714 const struct btf_field *b = (const struct btf_field *)_b; 3715 3716 if (a->offset < b->offset) 3717 return -1; 3718 else if (a->offset > b->offset) 3719 return 1; 3720 return 0; 3721 } 3722 3723 struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t, 3724 u32 field_mask, u32 value_size) 3725 { 3726 struct btf_field_info info_arr[BTF_FIELDS_MAX]; 3727 u32 next_off = 0, field_type_size; 3728 struct btf_record *rec; 3729 int ret, i, cnt; 3730 3731 ret = btf_find_field(btf, t, field_mask, info_arr, ARRAY_SIZE(info_arr)); 3732 if (ret < 0) 3733 return ERR_PTR(ret); 3734 if (!ret) 3735 return NULL; 3736 3737 cnt = ret; 3738 /* This needs to be kzalloc to zero out padding and unused fields, see 3739 * comment in btf_record_equal. 3740 */ 3741 rec = kzalloc(offsetof(struct btf_record, fields[cnt]), GFP_KERNEL | __GFP_NOWARN); 3742 if (!rec) 3743 return ERR_PTR(-ENOMEM); 3744 3745 rec->spin_lock_off = -EINVAL; 3746 rec->timer_off = -EINVAL; 3747 rec->refcount_off = -EINVAL; 3748 for (i = 0; i < cnt; i++) { 3749 field_type_size = btf_field_type_size(info_arr[i].type); 3750 if (info_arr[i].off + field_type_size > value_size) { 3751 WARN_ONCE(1, "verifier bug off %d size %d", info_arr[i].off, value_size); 3752 ret = -EFAULT; 3753 goto end; 3754 } 3755 if (info_arr[i].off < next_off) { 3756 ret = -EEXIST; 3757 goto end; 3758 } 3759 next_off = info_arr[i].off + field_type_size; 3760 3761 rec->field_mask |= info_arr[i].type; 3762 rec->fields[i].offset = info_arr[i].off; 3763 rec->fields[i].type = info_arr[i].type; 3764 rec->fields[i].size = field_type_size; 3765 3766 switch (info_arr[i].type) { 3767 case BPF_SPIN_LOCK: 3768 WARN_ON_ONCE(rec->spin_lock_off >= 0); 3769 /* Cache offset for faster lookup at runtime */ 3770 rec->spin_lock_off = rec->fields[i].offset; 3771 break; 3772 case BPF_TIMER: 3773 WARN_ON_ONCE(rec->timer_off >= 0); 3774 /* Cache offset for faster lookup at runtime */ 3775 rec->timer_off = rec->fields[i].offset; 3776 break; 3777 case BPF_REFCOUNT: 3778 WARN_ON_ONCE(rec->refcount_off >= 0); 3779 /* Cache offset for faster lookup at runtime */ 3780 rec->refcount_off = rec->fields[i].offset; 3781 break; 3782 case BPF_KPTR_UNREF: 3783 case BPF_KPTR_REF: 3784 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); 3785 if (ret < 0) 3786 goto end; 3787 break; 3788 case BPF_LIST_HEAD: 3789 ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]); 3790 if (ret < 0) 3791 goto end; 3792 break; 3793 case BPF_RB_ROOT: 3794 ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]); 3795 if (ret < 0) 3796 goto end; 3797 break; 3798 case BPF_LIST_NODE: 3799 case BPF_RB_NODE: 3800 break; 3801 default: 3802 ret = -EFAULT; 3803 goto end; 3804 } 3805 rec->cnt++; 3806 } 3807 3808 /* bpf_{list_head, rb_node} require bpf_spin_lock */ 3809 if ((btf_record_has_field(rec, BPF_LIST_HEAD) || 3810 btf_record_has_field(rec, BPF_RB_ROOT)) && rec->spin_lock_off < 0) { 3811 ret = -EINVAL; 3812 goto end; 3813 } 3814 3815 if (rec->refcount_off < 0 && 3816 btf_record_has_field(rec, BPF_LIST_NODE) && 3817 btf_record_has_field(rec, BPF_RB_NODE)) { 3818 ret = -EINVAL; 3819 goto end; 3820 } 3821 3822 sort_r(rec->fields, rec->cnt, sizeof(struct btf_field), btf_field_cmp, 3823 NULL, rec); 3824 3825 return rec; 3826 end: 3827 btf_record_free(rec); 3828 return ERR_PTR(ret); 3829 } 3830 3831 #define GRAPH_ROOT_MASK (BPF_LIST_HEAD | BPF_RB_ROOT) 3832 #define GRAPH_NODE_MASK (BPF_LIST_NODE | BPF_RB_NODE) 3833 3834 int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) 3835 { 3836 int i; 3837 3838 /* There are three types that signify ownership of some other type: 3839 * kptr_ref, bpf_list_head, bpf_rb_root. 3840 * kptr_ref only supports storing kernel types, which can't store 3841 * references to program allocated local types. 3842 * 3843 * Hence we only need to ensure that bpf_{list_head,rb_root} ownership 3844 * does not form cycles. 3845 */ 3846 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & GRAPH_ROOT_MASK)) 3847 return 0; 3848 for (i = 0; i < rec->cnt; i++) { 3849 struct btf_struct_meta *meta; 3850 u32 btf_id; 3851 3852 if (!(rec->fields[i].type & GRAPH_ROOT_MASK)) 3853 continue; 3854 btf_id = rec->fields[i].graph_root.value_btf_id; 3855 meta = btf_find_struct_meta(btf, btf_id); 3856 if (!meta) 3857 return -EFAULT; 3858 rec->fields[i].graph_root.value_rec = meta->record; 3859 3860 /* We need to set value_rec for all root types, but no need 3861 * to check ownership cycle for a type unless it's also a 3862 * node type. 3863 */ 3864 if (!(rec->field_mask & GRAPH_NODE_MASK)) 3865 continue; 3866 3867 /* We need to ensure ownership acyclicity among all types. The 3868 * proper way to do it would be to topologically sort all BTF 3869 * IDs based on the ownership edges, since there can be multiple 3870 * bpf_{list_head,rb_node} in a type. Instead, we use the 3871 * following resaoning: 3872 * 3873 * - A type can only be owned by another type in user BTF if it 3874 * has a bpf_{list,rb}_node. Let's call these node types. 3875 * - A type can only _own_ another type in user BTF if it has a 3876 * bpf_{list_head,rb_root}. Let's call these root types. 3877 * 3878 * We ensure that if a type is both a root and node, its 3879 * element types cannot be root types. 3880 * 3881 * To ensure acyclicity: 3882 * 3883 * When A is an root type but not a node, its ownership 3884 * chain can be: 3885 * A -> B -> C 3886 * Where: 3887 * - A is an root, e.g. has bpf_rb_root. 3888 * - B is both a root and node, e.g. has bpf_rb_node and 3889 * bpf_list_head. 3890 * - C is only an root, e.g. has bpf_list_node 3891 * 3892 * When A is both a root and node, some other type already 3893 * owns it in the BTF domain, hence it can not own 3894 * another root type through any of the ownership edges. 3895 * A -> B 3896 * Where: 3897 * - A is both an root and node. 3898 * - B is only an node. 3899 */ 3900 if (meta->record->field_mask & GRAPH_ROOT_MASK) 3901 return -ELOOP; 3902 } 3903 return 0; 3904 } 3905 3906 static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, 3907 u32 type_id, void *data, u8 bits_offset, 3908 struct btf_show *show) 3909 { 3910 const struct btf_member *member; 3911 void *safe_data; 3912 u32 i; 3913 3914 safe_data = btf_show_start_struct_type(show, t, type_id, data); 3915 if (!safe_data) 3916 return; 3917 3918 for_each_member(i, t, member) { 3919 const struct btf_type *member_type = btf_type_by_id(btf, 3920 member->type); 3921 const struct btf_kind_operations *ops; 3922 u32 member_offset, bitfield_size; 3923 u32 bytes_offset; 3924 u8 bits8_offset; 3925 3926 btf_show_start_member(show, member); 3927 3928 member_offset = __btf_member_bit_offset(t, member); 3929 bitfield_size = __btf_member_bitfield_size(t, member); 3930 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset); 3931 bits8_offset = BITS_PER_BYTE_MASKED(member_offset); 3932 if (bitfield_size) { 3933 safe_data = btf_show_start_type(show, member_type, 3934 member->type, 3935 data + bytes_offset); 3936 if (safe_data) 3937 btf_bitfield_show(safe_data, 3938 bits8_offset, 3939 bitfield_size, show); 3940 btf_show_end_type(show); 3941 } else { 3942 ops = btf_type_ops(member_type); 3943 ops->show(btf, member_type, member->type, 3944 data + bytes_offset, bits8_offset, show); 3945 } 3946 3947 btf_show_end_member(show); 3948 } 3949 3950 btf_show_end_struct_type(show); 3951 } 3952 3953 static void btf_struct_show(const struct btf *btf, const struct btf_type *t, 3954 u32 type_id, void *data, u8 bits_offset, 3955 struct btf_show *show) 3956 { 3957 const struct btf_member *m = show->state.member; 3958 3959 /* 3960 * First check if any members would be shown (are non-zero). 3961 * See comments above "struct btf_show" definition for more 3962 * details on how this works at a high-level. 3963 */ 3964 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { 3965 if (!show->state.depth_check) { 3966 show->state.depth_check = show->state.depth + 1; 3967 show->state.depth_to_show = 0; 3968 } 3969 __btf_struct_show(btf, t, type_id, data, bits_offset, show); 3970 /* Restore saved member data here */ 3971 show->state.member = m; 3972 if (show->state.depth_check != show->state.depth + 1) 3973 return; 3974 show->state.depth_check = 0; 3975 3976 if (show->state.depth_to_show <= show->state.depth) 3977 return; 3978 /* 3979 * Reaching here indicates we have recursed and found 3980 * non-zero child values. 3981 */ 3982 } 3983 3984 __btf_struct_show(btf, t, type_id, data, bits_offset, show); 3985 } 3986 3987 static struct btf_kind_operations struct_ops = { 3988 .check_meta = btf_struct_check_meta, 3989 .resolve = btf_struct_resolve, 3990 .check_member = btf_struct_check_member, 3991 .check_kflag_member = btf_generic_check_kflag_member, 3992 .log_details = btf_struct_log, 3993 .show = btf_struct_show, 3994 }; 3995 3996 static int btf_enum_check_member(struct btf_verifier_env *env, 3997 const struct btf_type *struct_type, 3998 const struct btf_member *member, 3999 const struct btf_type *member_type) 4000 { 4001 u32 struct_bits_off = member->offset; 4002 u32 struct_size, bytes_offset; 4003 4004 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 4005 btf_verifier_log_member(env, struct_type, member, 4006 "Member is not byte aligned"); 4007 return -EINVAL; 4008 } 4009 4010 struct_size = struct_type->size; 4011 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 4012 if (struct_size - bytes_offset < member_type->size) { 4013 btf_verifier_log_member(env, struct_type, member, 4014 "Member exceeds struct_size"); 4015 return -EINVAL; 4016 } 4017 4018 return 0; 4019 } 4020 4021 static int btf_enum_check_kflag_member(struct btf_verifier_env *env, 4022 const struct btf_type *struct_type, 4023 const struct btf_member *member, 4024 const struct btf_type *member_type) 4025 { 4026 u32 struct_bits_off, nr_bits, bytes_end, struct_size; 4027 u32 int_bitsize = sizeof(int) * BITS_PER_BYTE; 4028 4029 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); 4030 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); 4031 if (!nr_bits) { 4032 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 4033 btf_verifier_log_member(env, struct_type, member, 4034 "Member is not byte aligned"); 4035 return -EINVAL; 4036 } 4037 4038 nr_bits = int_bitsize; 4039 } else if (nr_bits > int_bitsize) { 4040 btf_verifier_log_member(env, struct_type, member, 4041 "Invalid member bitfield_size"); 4042 return -EINVAL; 4043 } 4044 4045 struct_size = struct_type->size; 4046 bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits); 4047 if (struct_size < bytes_end) { 4048 btf_verifier_log_member(env, struct_type, member, 4049 "Member exceeds struct_size"); 4050 return -EINVAL; 4051 } 4052 4053 return 0; 4054 } 4055 4056 static s32 btf_enum_check_meta(struct btf_verifier_env *env, 4057 const struct btf_type *t, 4058 u32 meta_left) 4059 { 4060 const struct btf_enum *enums = btf_type_enum(t); 4061 struct btf *btf = env->btf; 4062 const char *fmt_str; 4063 u16 i, nr_enums; 4064 u32 meta_needed; 4065 4066 nr_enums = btf_type_vlen(t); 4067 meta_needed = nr_enums * sizeof(*enums); 4068 4069 if (meta_left < meta_needed) { 4070 btf_verifier_log_basic(env, t, 4071 "meta_left:%u meta_needed:%u", 4072 meta_left, meta_needed); 4073 return -EINVAL; 4074 } 4075 4076 if (t->size > 8 || !is_power_of_2(t->size)) { 4077 btf_verifier_log_type(env, t, "Unexpected size"); 4078 return -EINVAL; 4079 } 4080 4081 /* enum type either no name or a valid one */ 4082 if (t->name_off && 4083 !btf_name_valid_identifier(env->btf, t->name_off)) { 4084 btf_verifier_log_type(env, t, "Invalid name"); 4085 return -EINVAL; 4086 } 4087 4088 btf_verifier_log_type(env, t, NULL); 4089 4090 for (i = 0; i < nr_enums; i++) { 4091 if (!btf_name_offset_valid(btf, enums[i].name_off)) { 4092 btf_verifier_log(env, "\tInvalid name_offset:%u", 4093 enums[i].name_off); 4094 return -EINVAL; 4095 } 4096 4097 /* enum member must have a valid name */ 4098 if (!enums[i].name_off || 4099 !btf_name_valid_identifier(btf, enums[i].name_off)) { 4100 btf_verifier_log_type(env, t, "Invalid name"); 4101 return -EINVAL; 4102 } 4103 4104 if (env->log.level == BPF_LOG_KERNEL) 4105 continue; 4106 fmt_str = btf_type_kflag(t) ? "\t%s val=%d\n" : "\t%s val=%u\n"; 4107 btf_verifier_log(env, fmt_str, 4108 __btf_name_by_offset(btf, enums[i].name_off), 4109 enums[i].val); 4110 } 4111 4112 return meta_needed; 4113 } 4114 4115 static void btf_enum_log(struct btf_verifier_env *env, 4116 const struct btf_type *t) 4117 { 4118 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 4119 } 4120 4121 static void btf_enum_show(const struct btf *btf, const struct btf_type *t, 4122 u32 type_id, void *data, u8 bits_offset, 4123 struct btf_show *show) 4124 { 4125 const struct btf_enum *enums = btf_type_enum(t); 4126 u32 i, nr_enums = btf_type_vlen(t); 4127 void *safe_data; 4128 int v; 4129 4130 safe_data = btf_show_start_type(show, t, type_id, data); 4131 if (!safe_data) 4132 return; 4133 4134 v = *(int *)safe_data; 4135 4136 for (i = 0; i < nr_enums; i++) { 4137 if (v != enums[i].val) 4138 continue; 4139 4140 btf_show_type_value(show, "%s", 4141 __btf_name_by_offset(btf, 4142 enums[i].name_off)); 4143 4144 btf_show_end_type(show); 4145 return; 4146 } 4147 4148 if (btf_type_kflag(t)) 4149 btf_show_type_value(show, "%d", v); 4150 else 4151 btf_show_type_value(show, "%u", v); 4152 btf_show_end_type(show); 4153 } 4154 4155 static struct btf_kind_operations enum_ops = { 4156 .check_meta = btf_enum_check_meta, 4157 .resolve = btf_df_resolve, 4158 .check_member = btf_enum_check_member, 4159 .check_kflag_member = btf_enum_check_kflag_member, 4160 .log_details = btf_enum_log, 4161 .show = btf_enum_show, 4162 }; 4163 4164 static s32 btf_enum64_check_meta(struct btf_verifier_env *env, 4165 const struct btf_type *t, 4166 u32 meta_left) 4167 { 4168 const struct btf_enum64 *enums = btf_type_enum64(t); 4169 struct btf *btf = env->btf; 4170 const char *fmt_str; 4171 u16 i, nr_enums; 4172 u32 meta_needed; 4173 4174 nr_enums = btf_type_vlen(t); 4175 meta_needed = nr_enums * sizeof(*enums); 4176 4177 if (meta_left < meta_needed) { 4178 btf_verifier_log_basic(env, t, 4179 "meta_left:%u meta_needed:%u", 4180 meta_left, meta_needed); 4181 return -EINVAL; 4182 } 4183 4184 if (t->size > 8 || !is_power_of_2(t->size)) { 4185 btf_verifier_log_type(env, t, "Unexpected size"); 4186 return -EINVAL; 4187 } 4188 4189 /* enum type either no name or a valid one */ 4190 if (t->name_off && 4191 !btf_name_valid_identifier(env->btf, t->name_off)) { 4192 btf_verifier_log_type(env, t, "Invalid name"); 4193 return -EINVAL; 4194 } 4195 4196 btf_verifier_log_type(env, t, NULL); 4197 4198 for (i = 0; i < nr_enums; i++) { 4199 if (!btf_name_offset_valid(btf, enums[i].name_off)) { 4200 btf_verifier_log(env, "\tInvalid name_offset:%u", 4201 enums[i].name_off); 4202 return -EINVAL; 4203 } 4204 4205 /* enum member must have a valid name */ 4206 if (!enums[i].name_off || 4207 !btf_name_valid_identifier(btf, enums[i].name_off)) { 4208 btf_verifier_log_type(env, t, "Invalid name"); 4209 return -EINVAL; 4210 } 4211 4212 if (env->log.level == BPF_LOG_KERNEL) 4213 continue; 4214 4215 fmt_str = btf_type_kflag(t) ? "\t%s val=%lld\n" : "\t%s val=%llu\n"; 4216 btf_verifier_log(env, fmt_str, 4217 __btf_name_by_offset(btf, enums[i].name_off), 4218 btf_enum64_value(enums + i)); 4219 } 4220 4221 return meta_needed; 4222 } 4223 4224 static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, 4225 u32 type_id, void *data, u8 bits_offset, 4226 struct btf_show *show) 4227 { 4228 const struct btf_enum64 *enums = btf_type_enum64(t); 4229 u32 i, nr_enums = btf_type_vlen(t); 4230 void *safe_data; 4231 s64 v; 4232 4233 safe_data = btf_show_start_type(show, t, type_id, data); 4234 if (!safe_data) 4235 return; 4236 4237 v = *(u64 *)safe_data; 4238 4239 for (i = 0; i < nr_enums; i++) { 4240 if (v != btf_enum64_value(enums + i)) 4241 continue; 4242 4243 btf_show_type_value(show, "%s", 4244 __btf_name_by_offset(btf, 4245 enums[i].name_off)); 4246 4247 btf_show_end_type(show); 4248 return; 4249 } 4250 4251 if (btf_type_kflag(t)) 4252 btf_show_type_value(show, "%lld", v); 4253 else 4254 btf_show_type_value(show, "%llu", v); 4255 btf_show_end_type(show); 4256 } 4257 4258 static struct btf_kind_operations enum64_ops = { 4259 .check_meta = btf_enum64_check_meta, 4260 .resolve = btf_df_resolve, 4261 .check_member = btf_enum_check_member, 4262 .check_kflag_member = btf_enum_check_kflag_member, 4263 .log_details = btf_enum_log, 4264 .show = btf_enum64_show, 4265 }; 4266 4267 static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, 4268 const struct btf_type *t, 4269 u32 meta_left) 4270 { 4271 u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param); 4272 4273 if (meta_left < meta_needed) { 4274 btf_verifier_log_basic(env, t, 4275 "meta_left:%u meta_needed:%u", 4276 meta_left, meta_needed); 4277 return -EINVAL; 4278 } 4279 4280 if (t->name_off) { 4281 btf_verifier_log_type(env, t, "Invalid name"); 4282 return -EINVAL; 4283 } 4284 4285 if (btf_type_kflag(t)) { 4286 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4287 return -EINVAL; 4288 } 4289 4290 btf_verifier_log_type(env, t, NULL); 4291 4292 return meta_needed; 4293 } 4294 4295 static void btf_func_proto_log(struct btf_verifier_env *env, 4296 const struct btf_type *t) 4297 { 4298 const struct btf_param *args = (const struct btf_param *)(t + 1); 4299 u16 nr_args = btf_type_vlen(t), i; 4300 4301 btf_verifier_log(env, "return=%u args=(", t->type); 4302 if (!nr_args) { 4303 btf_verifier_log(env, "void"); 4304 goto done; 4305 } 4306 4307 if (nr_args == 1 && !args[0].type) { 4308 /* Only one vararg */ 4309 btf_verifier_log(env, "vararg"); 4310 goto done; 4311 } 4312 4313 btf_verifier_log(env, "%u %s", args[0].type, 4314 __btf_name_by_offset(env->btf, 4315 args[0].name_off)); 4316 for (i = 1; i < nr_args - 1; i++) 4317 btf_verifier_log(env, ", %u %s", args[i].type, 4318 __btf_name_by_offset(env->btf, 4319 args[i].name_off)); 4320 4321 if (nr_args > 1) { 4322 const struct btf_param *last_arg = &args[nr_args - 1]; 4323 4324 if (last_arg->type) 4325 btf_verifier_log(env, ", %u %s", last_arg->type, 4326 __btf_name_by_offset(env->btf, 4327 last_arg->name_off)); 4328 else 4329 btf_verifier_log(env, ", vararg"); 4330 } 4331 4332 done: 4333 btf_verifier_log(env, ")"); 4334 } 4335 4336 static struct btf_kind_operations func_proto_ops = { 4337 .check_meta = btf_func_proto_check_meta, 4338 .resolve = btf_df_resolve, 4339 /* 4340 * BTF_KIND_FUNC_PROTO cannot be directly referred by 4341 * a struct's member. 4342 * 4343 * It should be a function pointer instead. 4344 * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO) 4345 * 4346 * Hence, there is no btf_func_check_member(). 4347 */ 4348 .check_member = btf_df_check_member, 4349 .check_kflag_member = btf_df_check_kflag_member, 4350 .log_details = btf_func_proto_log, 4351 .show = btf_df_show, 4352 }; 4353 4354 static s32 btf_func_check_meta(struct btf_verifier_env *env, 4355 const struct btf_type *t, 4356 u32 meta_left) 4357 { 4358 if (!t->name_off || 4359 !btf_name_valid_identifier(env->btf, t->name_off)) { 4360 btf_verifier_log_type(env, t, "Invalid name"); 4361 return -EINVAL; 4362 } 4363 4364 if (btf_type_vlen(t) > BTF_FUNC_GLOBAL) { 4365 btf_verifier_log_type(env, t, "Invalid func linkage"); 4366 return -EINVAL; 4367 } 4368 4369 if (btf_type_kflag(t)) { 4370 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4371 return -EINVAL; 4372 } 4373 4374 btf_verifier_log_type(env, t, NULL); 4375 4376 return 0; 4377 } 4378 4379 static int btf_func_resolve(struct btf_verifier_env *env, 4380 const struct resolve_vertex *v) 4381 { 4382 const struct btf_type *t = v->t; 4383 u32 next_type_id = t->type; 4384 int err; 4385 4386 err = btf_func_check(env, t); 4387 if (err) 4388 return err; 4389 4390 env_stack_pop_resolved(env, next_type_id, 0); 4391 return 0; 4392 } 4393 4394 static struct btf_kind_operations func_ops = { 4395 .check_meta = btf_func_check_meta, 4396 .resolve = btf_func_resolve, 4397 .check_member = btf_df_check_member, 4398 .check_kflag_member = btf_df_check_kflag_member, 4399 .log_details = btf_ref_type_log, 4400 .show = btf_df_show, 4401 }; 4402 4403 static s32 btf_var_check_meta(struct btf_verifier_env *env, 4404 const struct btf_type *t, 4405 u32 meta_left) 4406 { 4407 const struct btf_var *var; 4408 u32 meta_needed = sizeof(*var); 4409 4410 if (meta_left < meta_needed) { 4411 btf_verifier_log_basic(env, t, 4412 "meta_left:%u meta_needed:%u", 4413 meta_left, meta_needed); 4414 return -EINVAL; 4415 } 4416 4417 if (btf_type_vlen(t)) { 4418 btf_verifier_log_type(env, t, "vlen != 0"); 4419 return -EINVAL; 4420 } 4421 4422 if (btf_type_kflag(t)) { 4423 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4424 return -EINVAL; 4425 } 4426 4427 if (!t->name_off || 4428 !__btf_name_valid(env->btf, t->name_off)) { 4429 btf_verifier_log_type(env, t, "Invalid name"); 4430 return -EINVAL; 4431 } 4432 4433 /* A var cannot be in type void */ 4434 if (!t->type || !BTF_TYPE_ID_VALID(t->type)) { 4435 btf_verifier_log_type(env, t, "Invalid type_id"); 4436 return -EINVAL; 4437 } 4438 4439 var = btf_type_var(t); 4440 if (var->linkage != BTF_VAR_STATIC && 4441 var->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 4442 btf_verifier_log_type(env, t, "Linkage not supported"); 4443 return -EINVAL; 4444 } 4445 4446 btf_verifier_log_type(env, t, NULL); 4447 4448 return meta_needed; 4449 } 4450 4451 static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t) 4452 { 4453 const struct btf_var *var = btf_type_var(t); 4454 4455 btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage); 4456 } 4457 4458 static const struct btf_kind_operations var_ops = { 4459 .check_meta = btf_var_check_meta, 4460 .resolve = btf_var_resolve, 4461 .check_member = btf_df_check_member, 4462 .check_kflag_member = btf_df_check_kflag_member, 4463 .log_details = btf_var_log, 4464 .show = btf_var_show, 4465 }; 4466 4467 static s32 btf_datasec_check_meta(struct btf_verifier_env *env, 4468 const struct btf_type *t, 4469 u32 meta_left) 4470 { 4471 const struct btf_var_secinfo *vsi; 4472 u64 last_vsi_end_off = 0, sum = 0; 4473 u32 i, meta_needed; 4474 4475 meta_needed = btf_type_vlen(t) * sizeof(*vsi); 4476 if (meta_left < meta_needed) { 4477 btf_verifier_log_basic(env, t, 4478 "meta_left:%u meta_needed:%u", 4479 meta_left, meta_needed); 4480 return -EINVAL; 4481 } 4482 4483 if (!t->size) { 4484 btf_verifier_log_type(env, t, "size == 0"); 4485 return -EINVAL; 4486 } 4487 4488 if (btf_type_kflag(t)) { 4489 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4490 return -EINVAL; 4491 } 4492 4493 if (!t->name_off || 4494 !btf_name_valid_section(env->btf, t->name_off)) { 4495 btf_verifier_log_type(env, t, "Invalid name"); 4496 return -EINVAL; 4497 } 4498 4499 btf_verifier_log_type(env, t, NULL); 4500 4501 for_each_vsi(i, t, vsi) { 4502 /* A var cannot be in type void */ 4503 if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) { 4504 btf_verifier_log_vsi(env, t, vsi, 4505 "Invalid type_id"); 4506 return -EINVAL; 4507 } 4508 4509 if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) { 4510 btf_verifier_log_vsi(env, t, vsi, 4511 "Invalid offset"); 4512 return -EINVAL; 4513 } 4514 4515 if (!vsi->size || vsi->size > t->size) { 4516 btf_verifier_log_vsi(env, t, vsi, 4517 "Invalid size"); 4518 return -EINVAL; 4519 } 4520 4521 last_vsi_end_off = vsi->offset + vsi->size; 4522 if (last_vsi_end_off > t->size) { 4523 btf_verifier_log_vsi(env, t, vsi, 4524 "Invalid offset+size"); 4525 return -EINVAL; 4526 } 4527 4528 btf_verifier_log_vsi(env, t, vsi, NULL); 4529 sum += vsi->size; 4530 } 4531 4532 if (t->size < sum) { 4533 btf_verifier_log_type(env, t, "Invalid btf_info size"); 4534 return -EINVAL; 4535 } 4536 4537 return meta_needed; 4538 } 4539 4540 static int btf_datasec_resolve(struct btf_verifier_env *env, 4541 const struct resolve_vertex *v) 4542 { 4543 const struct btf_var_secinfo *vsi; 4544 struct btf *btf = env->btf; 4545 u16 i; 4546 4547 env->resolve_mode = RESOLVE_TBD; 4548 for_each_vsi_from(i, v->next_member, v->t, vsi) { 4549 u32 var_type_id = vsi->type, type_id, type_size = 0; 4550 const struct btf_type *var_type = btf_type_by_id(env->btf, 4551 var_type_id); 4552 if (!var_type || !btf_type_is_var(var_type)) { 4553 btf_verifier_log_vsi(env, v->t, vsi, 4554 "Not a VAR kind member"); 4555 return -EINVAL; 4556 } 4557 4558 if (!env_type_is_resolve_sink(env, var_type) && 4559 !env_type_is_resolved(env, var_type_id)) { 4560 env_stack_set_next_member(env, i + 1); 4561 return env_stack_push(env, var_type, var_type_id); 4562 } 4563 4564 type_id = var_type->type; 4565 if (!btf_type_id_size(btf, &type_id, &type_size)) { 4566 btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); 4567 return -EINVAL; 4568 } 4569 4570 if (vsi->size < type_size) { 4571 btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); 4572 return -EINVAL; 4573 } 4574 } 4575 4576 env_stack_pop_resolved(env, 0, 0); 4577 return 0; 4578 } 4579 4580 static void btf_datasec_log(struct btf_verifier_env *env, 4581 const struct btf_type *t) 4582 { 4583 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 4584 } 4585 4586 static void btf_datasec_show(const struct btf *btf, 4587 const struct btf_type *t, u32 type_id, 4588 void *data, u8 bits_offset, 4589 struct btf_show *show) 4590 { 4591 const struct btf_var_secinfo *vsi; 4592 const struct btf_type *var; 4593 u32 i; 4594 4595 if (!btf_show_start_type(show, t, type_id, data)) 4596 return; 4597 4598 btf_show_type_value(show, "section (\"%s\") = {", 4599 __btf_name_by_offset(btf, t->name_off)); 4600 for_each_vsi(i, t, vsi) { 4601 var = btf_type_by_id(btf, vsi->type); 4602 if (i) 4603 btf_show(show, ","); 4604 btf_type_ops(var)->show(btf, var, vsi->type, 4605 data + vsi->offset, bits_offset, show); 4606 } 4607 btf_show_end_type(show); 4608 } 4609 4610 static const struct btf_kind_operations datasec_ops = { 4611 .check_meta = btf_datasec_check_meta, 4612 .resolve = btf_datasec_resolve, 4613 .check_member = btf_df_check_member, 4614 .check_kflag_member = btf_df_check_kflag_member, 4615 .log_details = btf_datasec_log, 4616 .show = btf_datasec_show, 4617 }; 4618 4619 static s32 btf_float_check_meta(struct btf_verifier_env *env, 4620 const struct btf_type *t, 4621 u32 meta_left) 4622 { 4623 if (btf_type_vlen(t)) { 4624 btf_verifier_log_type(env, t, "vlen != 0"); 4625 return -EINVAL; 4626 } 4627 4628 if (btf_type_kflag(t)) { 4629 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4630 return -EINVAL; 4631 } 4632 4633 if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 && 4634 t->size != 16) { 4635 btf_verifier_log_type(env, t, "Invalid type_size"); 4636 return -EINVAL; 4637 } 4638 4639 btf_verifier_log_type(env, t, NULL); 4640 4641 return 0; 4642 } 4643 4644 static int btf_float_check_member(struct btf_verifier_env *env, 4645 const struct btf_type *struct_type, 4646 const struct btf_member *member, 4647 const struct btf_type *member_type) 4648 { 4649 u64 start_offset_bytes; 4650 u64 end_offset_bytes; 4651 u64 misalign_bits; 4652 u64 align_bytes; 4653 u64 align_bits; 4654 4655 /* Different architectures have different alignment requirements, so 4656 * here we check only for the reasonable minimum. This way we ensure 4657 * that types after CO-RE can pass the kernel BTF verifier. 4658 */ 4659 align_bytes = min_t(u64, sizeof(void *), member_type->size); 4660 align_bits = align_bytes * BITS_PER_BYTE; 4661 div64_u64_rem(member->offset, align_bits, &misalign_bits); 4662 if (misalign_bits) { 4663 btf_verifier_log_member(env, struct_type, member, 4664 "Member is not properly aligned"); 4665 return -EINVAL; 4666 } 4667 4668 start_offset_bytes = member->offset / BITS_PER_BYTE; 4669 end_offset_bytes = start_offset_bytes + member_type->size; 4670 if (end_offset_bytes > struct_type->size) { 4671 btf_verifier_log_member(env, struct_type, member, 4672 "Member exceeds struct_size"); 4673 return -EINVAL; 4674 } 4675 4676 return 0; 4677 } 4678 4679 static void btf_float_log(struct btf_verifier_env *env, 4680 const struct btf_type *t) 4681 { 4682 btf_verifier_log(env, "size=%u", t->size); 4683 } 4684 4685 static const struct btf_kind_operations float_ops = { 4686 .check_meta = btf_float_check_meta, 4687 .resolve = btf_df_resolve, 4688 .check_member = btf_float_check_member, 4689 .check_kflag_member = btf_generic_check_kflag_member, 4690 .log_details = btf_float_log, 4691 .show = btf_df_show, 4692 }; 4693 4694 static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env, 4695 const struct btf_type *t, 4696 u32 meta_left) 4697 { 4698 const struct btf_decl_tag *tag; 4699 u32 meta_needed = sizeof(*tag); 4700 s32 component_idx; 4701 const char *value; 4702 4703 if (meta_left < meta_needed) { 4704 btf_verifier_log_basic(env, t, 4705 "meta_left:%u meta_needed:%u", 4706 meta_left, meta_needed); 4707 return -EINVAL; 4708 } 4709 4710 value = btf_name_by_offset(env->btf, t->name_off); 4711 if (!value || !value[0]) { 4712 btf_verifier_log_type(env, t, "Invalid value"); 4713 return -EINVAL; 4714 } 4715 4716 if (btf_type_vlen(t)) { 4717 btf_verifier_log_type(env, t, "vlen != 0"); 4718 return -EINVAL; 4719 } 4720 4721 if (btf_type_kflag(t)) { 4722 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4723 return -EINVAL; 4724 } 4725 4726 component_idx = btf_type_decl_tag(t)->component_idx; 4727 if (component_idx < -1) { 4728 btf_verifier_log_type(env, t, "Invalid component_idx"); 4729 return -EINVAL; 4730 } 4731 4732 btf_verifier_log_type(env, t, NULL); 4733 4734 return meta_needed; 4735 } 4736 4737 static int btf_decl_tag_resolve(struct btf_verifier_env *env, 4738 const struct resolve_vertex *v) 4739 { 4740 const struct btf_type *next_type; 4741 const struct btf_type *t = v->t; 4742 u32 next_type_id = t->type; 4743 struct btf *btf = env->btf; 4744 s32 component_idx; 4745 u32 vlen; 4746 4747 next_type = btf_type_by_id(btf, next_type_id); 4748 if (!next_type || !btf_type_is_decl_tag_target(next_type)) { 4749 btf_verifier_log_type(env, v->t, "Invalid type_id"); 4750 return -EINVAL; 4751 } 4752 4753 if (!env_type_is_resolve_sink(env, next_type) && 4754 !env_type_is_resolved(env, next_type_id)) 4755 return env_stack_push(env, next_type, next_type_id); 4756 4757 component_idx = btf_type_decl_tag(t)->component_idx; 4758 if (component_idx != -1) { 4759 if (btf_type_is_var(next_type) || btf_type_is_typedef(next_type)) { 4760 btf_verifier_log_type(env, v->t, "Invalid component_idx"); 4761 return -EINVAL; 4762 } 4763 4764 if (btf_type_is_struct(next_type)) { 4765 vlen = btf_type_vlen(next_type); 4766 } else { 4767 /* next_type should be a function */ 4768 next_type = btf_type_by_id(btf, next_type->type); 4769 vlen = btf_type_vlen(next_type); 4770 } 4771 4772 if ((u32)component_idx >= vlen) { 4773 btf_verifier_log_type(env, v->t, "Invalid component_idx"); 4774 return -EINVAL; 4775 } 4776 } 4777 4778 env_stack_pop_resolved(env, next_type_id, 0); 4779 4780 return 0; 4781 } 4782 4783 static void btf_decl_tag_log(struct btf_verifier_env *env, const struct btf_type *t) 4784 { 4785 btf_verifier_log(env, "type=%u component_idx=%d", t->type, 4786 btf_type_decl_tag(t)->component_idx); 4787 } 4788 4789 static const struct btf_kind_operations decl_tag_ops = { 4790 .check_meta = btf_decl_tag_check_meta, 4791 .resolve = btf_decl_tag_resolve, 4792 .check_member = btf_df_check_member, 4793 .check_kflag_member = btf_df_check_kflag_member, 4794 .log_details = btf_decl_tag_log, 4795 .show = btf_df_show, 4796 }; 4797 4798 static int btf_func_proto_check(struct btf_verifier_env *env, 4799 const struct btf_type *t) 4800 { 4801 const struct btf_type *ret_type; 4802 const struct btf_param *args; 4803 const struct btf *btf; 4804 u16 nr_args, i; 4805 int err; 4806 4807 btf = env->btf; 4808 args = (const struct btf_param *)(t + 1); 4809 nr_args = btf_type_vlen(t); 4810 4811 /* Check func return type which could be "void" (t->type == 0) */ 4812 if (t->type) { 4813 u32 ret_type_id = t->type; 4814 4815 ret_type = btf_type_by_id(btf, ret_type_id); 4816 if (!ret_type) { 4817 btf_verifier_log_type(env, t, "Invalid return type"); 4818 return -EINVAL; 4819 } 4820 4821 if (btf_type_is_resolve_source_only(ret_type)) { 4822 btf_verifier_log_type(env, t, "Invalid return type"); 4823 return -EINVAL; 4824 } 4825 4826 if (btf_type_needs_resolve(ret_type) && 4827 !env_type_is_resolved(env, ret_type_id)) { 4828 err = btf_resolve(env, ret_type, ret_type_id); 4829 if (err) 4830 return err; 4831 } 4832 4833 /* Ensure the return type is a type that has a size */ 4834 if (!btf_type_id_size(btf, &ret_type_id, NULL)) { 4835 btf_verifier_log_type(env, t, "Invalid return type"); 4836 return -EINVAL; 4837 } 4838 } 4839 4840 if (!nr_args) 4841 return 0; 4842 4843 /* Last func arg type_id could be 0 if it is a vararg */ 4844 if (!args[nr_args - 1].type) { 4845 if (args[nr_args - 1].name_off) { 4846 btf_verifier_log_type(env, t, "Invalid arg#%u", 4847 nr_args); 4848 return -EINVAL; 4849 } 4850 nr_args--; 4851 } 4852 4853 for (i = 0; i < nr_args; i++) { 4854 const struct btf_type *arg_type; 4855 u32 arg_type_id; 4856 4857 arg_type_id = args[i].type; 4858 arg_type = btf_type_by_id(btf, arg_type_id); 4859 if (!arg_type) { 4860 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 4861 return -EINVAL; 4862 } 4863 4864 if (btf_type_is_resolve_source_only(arg_type)) { 4865 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 4866 return -EINVAL; 4867 } 4868 4869 if (args[i].name_off && 4870 (!btf_name_offset_valid(btf, args[i].name_off) || 4871 !btf_name_valid_identifier(btf, args[i].name_off))) { 4872 btf_verifier_log_type(env, t, 4873 "Invalid arg#%u", i + 1); 4874 return -EINVAL; 4875 } 4876 4877 if (btf_type_needs_resolve(arg_type) && 4878 !env_type_is_resolved(env, arg_type_id)) { 4879 err = btf_resolve(env, arg_type, arg_type_id); 4880 if (err) 4881 return err; 4882 } 4883 4884 if (!btf_type_id_size(btf, &arg_type_id, NULL)) { 4885 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 4886 return -EINVAL; 4887 } 4888 } 4889 4890 return 0; 4891 } 4892 4893 static int btf_func_check(struct btf_verifier_env *env, 4894 const struct btf_type *t) 4895 { 4896 const struct btf_type *proto_type; 4897 const struct btf_param *args; 4898 const struct btf *btf; 4899 u16 nr_args, i; 4900 4901 btf = env->btf; 4902 proto_type = btf_type_by_id(btf, t->type); 4903 4904 if (!proto_type || !btf_type_is_func_proto(proto_type)) { 4905 btf_verifier_log_type(env, t, "Invalid type_id"); 4906 return -EINVAL; 4907 } 4908 4909 args = (const struct btf_param *)(proto_type + 1); 4910 nr_args = btf_type_vlen(proto_type); 4911 for (i = 0; i < nr_args; i++) { 4912 if (!args[i].name_off && args[i].type) { 4913 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 4914 return -EINVAL; 4915 } 4916 } 4917 4918 return 0; 4919 } 4920 4921 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { 4922 [BTF_KIND_INT] = &int_ops, 4923 [BTF_KIND_PTR] = &ptr_ops, 4924 [BTF_KIND_ARRAY] = &array_ops, 4925 [BTF_KIND_STRUCT] = &struct_ops, 4926 [BTF_KIND_UNION] = &struct_ops, 4927 [BTF_KIND_ENUM] = &enum_ops, 4928 [BTF_KIND_FWD] = &fwd_ops, 4929 [BTF_KIND_TYPEDEF] = &modifier_ops, 4930 [BTF_KIND_VOLATILE] = &modifier_ops, 4931 [BTF_KIND_CONST] = &modifier_ops, 4932 [BTF_KIND_RESTRICT] = &modifier_ops, 4933 [BTF_KIND_FUNC] = &func_ops, 4934 [BTF_KIND_FUNC_PROTO] = &func_proto_ops, 4935 [BTF_KIND_VAR] = &var_ops, 4936 [BTF_KIND_DATASEC] = &datasec_ops, 4937 [BTF_KIND_FLOAT] = &float_ops, 4938 [BTF_KIND_DECL_TAG] = &decl_tag_ops, 4939 [BTF_KIND_TYPE_TAG] = &modifier_ops, 4940 [BTF_KIND_ENUM64] = &enum64_ops, 4941 }; 4942 4943 static s32 btf_check_meta(struct btf_verifier_env *env, 4944 const struct btf_type *t, 4945 u32 meta_left) 4946 { 4947 u32 saved_meta_left = meta_left; 4948 s32 var_meta_size; 4949 4950 if (meta_left < sizeof(*t)) { 4951 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu", 4952 env->log_type_id, meta_left, sizeof(*t)); 4953 return -EINVAL; 4954 } 4955 meta_left -= sizeof(*t); 4956 4957 if (t->info & ~BTF_INFO_MASK) { 4958 btf_verifier_log(env, "[%u] Invalid btf_info:%x", 4959 env->log_type_id, t->info); 4960 return -EINVAL; 4961 } 4962 4963 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || 4964 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { 4965 btf_verifier_log(env, "[%u] Invalid kind:%u", 4966 env->log_type_id, BTF_INFO_KIND(t->info)); 4967 return -EINVAL; 4968 } 4969 4970 if (!btf_name_offset_valid(env->btf, t->name_off)) { 4971 btf_verifier_log(env, "[%u] Invalid name_offset:%u", 4972 env->log_type_id, t->name_off); 4973 return -EINVAL; 4974 } 4975 4976 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left); 4977 if (var_meta_size < 0) 4978 return var_meta_size; 4979 4980 meta_left -= var_meta_size; 4981 4982 return saved_meta_left - meta_left; 4983 } 4984 4985 static int btf_check_all_metas(struct btf_verifier_env *env) 4986 { 4987 struct btf *btf = env->btf; 4988 struct btf_header *hdr; 4989 void *cur, *end; 4990 4991 hdr = &btf->hdr; 4992 cur = btf->nohdr_data + hdr->type_off; 4993 end = cur + hdr->type_len; 4994 4995 env->log_type_id = btf->base_btf ? btf->start_id : 1; 4996 while (cur < end) { 4997 struct btf_type *t = cur; 4998 s32 meta_size; 4999 5000 meta_size = btf_check_meta(env, t, end - cur); 5001 if (meta_size < 0) 5002 return meta_size; 5003 5004 btf_add_type(env, t); 5005 cur += meta_size; 5006 env->log_type_id++; 5007 } 5008 5009 return 0; 5010 } 5011 5012 static bool btf_resolve_valid(struct btf_verifier_env *env, 5013 const struct btf_type *t, 5014 u32 type_id) 5015 { 5016 struct btf *btf = env->btf; 5017 5018 if (!env_type_is_resolved(env, type_id)) 5019 return false; 5020 5021 if (btf_type_is_struct(t) || btf_type_is_datasec(t)) 5022 return !btf_resolved_type_id(btf, type_id) && 5023 !btf_resolved_type_size(btf, type_id); 5024 5025 if (btf_type_is_decl_tag(t) || btf_type_is_func(t)) 5026 return btf_resolved_type_id(btf, type_id) && 5027 !btf_resolved_type_size(btf, type_id); 5028 5029 if (btf_type_is_modifier(t) || btf_type_is_ptr(t) || 5030 btf_type_is_var(t)) { 5031 t = btf_type_id_resolve(btf, &type_id); 5032 return t && 5033 !btf_type_is_modifier(t) && 5034 !btf_type_is_var(t) && 5035 !btf_type_is_datasec(t); 5036 } 5037 5038 if (btf_type_is_array(t)) { 5039 const struct btf_array *array = btf_type_array(t); 5040 const struct btf_type *elem_type; 5041 u32 elem_type_id = array->type; 5042 u32 elem_size; 5043 5044 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); 5045 return elem_type && !btf_type_is_modifier(elem_type) && 5046 (array->nelems * elem_size == 5047 btf_resolved_type_size(btf, type_id)); 5048 } 5049 5050 return false; 5051 } 5052 5053 static int btf_resolve(struct btf_verifier_env *env, 5054 const struct btf_type *t, u32 type_id) 5055 { 5056 u32 save_log_type_id = env->log_type_id; 5057 const struct resolve_vertex *v; 5058 int err = 0; 5059 5060 env->resolve_mode = RESOLVE_TBD; 5061 env_stack_push(env, t, type_id); 5062 while (!err && (v = env_stack_peak(env))) { 5063 env->log_type_id = v->type_id; 5064 err = btf_type_ops(v->t)->resolve(env, v); 5065 } 5066 5067 env->log_type_id = type_id; 5068 if (err == -E2BIG) { 5069 btf_verifier_log_type(env, t, 5070 "Exceeded max resolving depth:%u", 5071 MAX_RESOLVE_DEPTH); 5072 } else if (err == -EEXIST) { 5073 btf_verifier_log_type(env, t, "Loop detected"); 5074 } 5075 5076 /* Final sanity check */ 5077 if (!err && !btf_resolve_valid(env, t, type_id)) { 5078 btf_verifier_log_type(env, t, "Invalid resolve state"); 5079 err = -EINVAL; 5080 } 5081 5082 env->log_type_id = save_log_type_id; 5083 return err; 5084 } 5085 5086 static int btf_check_all_types(struct btf_verifier_env *env) 5087 { 5088 struct btf *btf = env->btf; 5089 const struct btf_type *t; 5090 u32 type_id, i; 5091 int err; 5092 5093 err = env_resolve_init(env); 5094 if (err) 5095 return err; 5096 5097 env->phase++; 5098 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) { 5099 type_id = btf->start_id + i; 5100 t = btf_type_by_id(btf, type_id); 5101 5102 env->log_type_id = type_id; 5103 if (btf_type_needs_resolve(t) && 5104 !env_type_is_resolved(env, type_id)) { 5105 err = btf_resolve(env, t, type_id); 5106 if (err) 5107 return err; 5108 } 5109 5110 if (btf_type_is_func_proto(t)) { 5111 err = btf_func_proto_check(env, t); 5112 if (err) 5113 return err; 5114 } 5115 } 5116 5117 return 0; 5118 } 5119 5120 static int btf_parse_type_sec(struct btf_verifier_env *env) 5121 { 5122 const struct btf_header *hdr = &env->btf->hdr; 5123 int err; 5124 5125 /* Type section must align to 4 bytes */ 5126 if (hdr->type_off & (sizeof(u32) - 1)) { 5127 btf_verifier_log(env, "Unaligned type_off"); 5128 return -EINVAL; 5129 } 5130 5131 if (!env->btf->base_btf && !hdr->type_len) { 5132 btf_verifier_log(env, "No type found"); 5133 return -EINVAL; 5134 } 5135 5136 err = btf_check_all_metas(env); 5137 if (err) 5138 return err; 5139 5140 return btf_check_all_types(env); 5141 } 5142 5143 static int btf_parse_str_sec(struct btf_verifier_env *env) 5144 { 5145 const struct btf_header *hdr; 5146 struct btf *btf = env->btf; 5147 const char *start, *end; 5148 5149 hdr = &btf->hdr; 5150 start = btf->nohdr_data + hdr->str_off; 5151 end = start + hdr->str_len; 5152 5153 if (end != btf->data + btf->data_size) { 5154 btf_verifier_log(env, "String section is not at the end"); 5155 return -EINVAL; 5156 } 5157 5158 btf->strings = start; 5159 5160 if (btf->base_btf && !hdr->str_len) 5161 return 0; 5162 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) { 5163 btf_verifier_log(env, "Invalid string section"); 5164 return -EINVAL; 5165 } 5166 if (!btf->base_btf && start[0]) { 5167 btf_verifier_log(env, "Invalid string section"); 5168 return -EINVAL; 5169 } 5170 5171 return 0; 5172 } 5173 5174 static const size_t btf_sec_info_offset[] = { 5175 offsetof(struct btf_header, type_off), 5176 offsetof(struct btf_header, str_off), 5177 }; 5178 5179 static int btf_sec_info_cmp(const void *a, const void *b) 5180 { 5181 const struct btf_sec_info *x = a; 5182 const struct btf_sec_info *y = b; 5183 5184 return (int)(x->off - y->off) ? : (int)(x->len - y->len); 5185 } 5186 5187 static int btf_check_sec_info(struct btf_verifier_env *env, 5188 u32 btf_data_size) 5189 { 5190 struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)]; 5191 u32 total, expected_total, i; 5192 const struct btf_header *hdr; 5193 const struct btf *btf; 5194 5195 btf = env->btf; 5196 hdr = &btf->hdr; 5197 5198 /* Populate the secs from hdr */ 5199 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) 5200 secs[i] = *(struct btf_sec_info *)((void *)hdr + 5201 btf_sec_info_offset[i]); 5202 5203 sort(secs, ARRAY_SIZE(btf_sec_info_offset), 5204 sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL); 5205 5206 /* Check for gaps and overlap among sections */ 5207 total = 0; 5208 expected_total = btf_data_size - hdr->hdr_len; 5209 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) { 5210 if (expected_total < secs[i].off) { 5211 btf_verifier_log(env, "Invalid section offset"); 5212 return -EINVAL; 5213 } 5214 if (total < secs[i].off) { 5215 /* gap */ 5216 btf_verifier_log(env, "Unsupported section found"); 5217 return -EINVAL; 5218 } 5219 if (total > secs[i].off) { 5220 btf_verifier_log(env, "Section overlap found"); 5221 return -EINVAL; 5222 } 5223 if (expected_total - total < secs[i].len) { 5224 btf_verifier_log(env, 5225 "Total section length too long"); 5226 return -EINVAL; 5227 } 5228 total += secs[i].len; 5229 } 5230 5231 /* There is data other than hdr and known sections */ 5232 if (expected_total != total) { 5233 btf_verifier_log(env, "Unsupported section found"); 5234 return -EINVAL; 5235 } 5236 5237 return 0; 5238 } 5239 5240 static int btf_parse_hdr(struct btf_verifier_env *env) 5241 { 5242 u32 hdr_len, hdr_copy, btf_data_size; 5243 const struct btf_header *hdr; 5244 struct btf *btf; 5245 5246 btf = env->btf; 5247 btf_data_size = btf->data_size; 5248 5249 if (btf_data_size < offsetofend(struct btf_header, hdr_len)) { 5250 btf_verifier_log(env, "hdr_len not found"); 5251 return -EINVAL; 5252 } 5253 5254 hdr = btf->data; 5255 hdr_len = hdr->hdr_len; 5256 if (btf_data_size < hdr_len) { 5257 btf_verifier_log(env, "btf_header not found"); 5258 return -EINVAL; 5259 } 5260 5261 /* Ensure the unsupported header fields are zero */ 5262 if (hdr_len > sizeof(btf->hdr)) { 5263 u8 *expected_zero = btf->data + sizeof(btf->hdr); 5264 u8 *end = btf->data + hdr_len; 5265 5266 for (; expected_zero < end; expected_zero++) { 5267 if (*expected_zero) { 5268 btf_verifier_log(env, "Unsupported btf_header"); 5269 return -E2BIG; 5270 } 5271 } 5272 } 5273 5274 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); 5275 memcpy(&btf->hdr, btf->data, hdr_copy); 5276 5277 hdr = &btf->hdr; 5278 5279 btf_verifier_log_hdr(env, btf_data_size); 5280 5281 if (hdr->magic != BTF_MAGIC) { 5282 btf_verifier_log(env, "Invalid magic"); 5283 return -EINVAL; 5284 } 5285 5286 if (hdr->version != BTF_VERSION) { 5287 btf_verifier_log(env, "Unsupported version"); 5288 return -ENOTSUPP; 5289 } 5290 5291 if (hdr->flags) { 5292 btf_verifier_log(env, "Unsupported flags"); 5293 return -ENOTSUPP; 5294 } 5295 5296 if (!btf->base_btf && btf_data_size == hdr->hdr_len) { 5297 btf_verifier_log(env, "No data"); 5298 return -EINVAL; 5299 } 5300 5301 return btf_check_sec_info(env, btf_data_size); 5302 } 5303 5304 static const char *alloc_obj_fields[] = { 5305 "bpf_spin_lock", 5306 "bpf_list_head", 5307 "bpf_list_node", 5308 "bpf_rb_root", 5309 "bpf_rb_node", 5310 "bpf_refcount", 5311 }; 5312 5313 static struct btf_struct_metas * 5314 btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) 5315 { 5316 union { 5317 struct btf_id_set set; 5318 struct { 5319 u32 _cnt; 5320 u32 _ids[ARRAY_SIZE(alloc_obj_fields)]; 5321 } _arr; 5322 } aof; 5323 struct btf_struct_metas *tab = NULL; 5324 int i, n, id, ret; 5325 5326 BUILD_BUG_ON(offsetof(struct btf_id_set, cnt) != 0); 5327 BUILD_BUG_ON(sizeof(struct btf_id_set) != sizeof(u32)); 5328 5329 memset(&aof, 0, sizeof(aof)); 5330 for (i = 0; i < ARRAY_SIZE(alloc_obj_fields); i++) { 5331 /* Try to find whether this special type exists in user BTF, and 5332 * if so remember its ID so we can easily find it among members 5333 * of structs that we iterate in the next loop. 5334 */ 5335 id = btf_find_by_name_kind(btf, alloc_obj_fields[i], BTF_KIND_STRUCT); 5336 if (id < 0) 5337 continue; 5338 aof.set.ids[aof.set.cnt++] = id; 5339 } 5340 5341 if (!aof.set.cnt) 5342 return NULL; 5343 sort(&aof.set.ids, aof.set.cnt, sizeof(aof.set.ids[0]), btf_id_cmp_func, NULL); 5344 5345 n = btf_nr_types(btf); 5346 for (i = 1; i < n; i++) { 5347 struct btf_struct_metas *new_tab; 5348 const struct btf_member *member; 5349 struct btf_struct_meta *type; 5350 struct btf_record *record; 5351 const struct btf_type *t; 5352 int j, tab_cnt; 5353 5354 t = btf_type_by_id(btf, i); 5355 if (!t) { 5356 ret = -EINVAL; 5357 goto free; 5358 } 5359 if (!__btf_type_is_struct(t)) 5360 continue; 5361 5362 cond_resched(); 5363 5364 for_each_member(j, t, member) { 5365 if (btf_id_set_contains(&aof.set, member->type)) 5366 goto parse; 5367 } 5368 continue; 5369 parse: 5370 tab_cnt = tab ? tab->cnt : 0; 5371 new_tab = krealloc(tab, offsetof(struct btf_struct_metas, types[tab_cnt + 1]), 5372 GFP_KERNEL | __GFP_NOWARN); 5373 if (!new_tab) { 5374 ret = -ENOMEM; 5375 goto free; 5376 } 5377 if (!tab) 5378 new_tab->cnt = 0; 5379 tab = new_tab; 5380 5381 type = &tab->types[tab->cnt]; 5382 type->btf_id = i; 5383 record = btf_parse_fields(btf, t, BPF_SPIN_LOCK | BPF_LIST_HEAD | BPF_LIST_NODE | 5384 BPF_RB_ROOT | BPF_RB_NODE | BPF_REFCOUNT, t->size); 5385 /* The record cannot be unset, treat it as an error if so */ 5386 if (IS_ERR_OR_NULL(record)) { 5387 ret = PTR_ERR_OR_ZERO(record) ?: -EFAULT; 5388 goto free; 5389 } 5390 type->record = record; 5391 tab->cnt++; 5392 } 5393 return tab; 5394 free: 5395 btf_struct_metas_free(tab); 5396 return ERR_PTR(ret); 5397 } 5398 5399 struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id) 5400 { 5401 struct btf_struct_metas *tab; 5402 5403 BUILD_BUG_ON(offsetof(struct btf_struct_meta, btf_id) != 0); 5404 tab = btf->struct_meta_tab; 5405 if (!tab) 5406 return NULL; 5407 return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func); 5408 } 5409 5410 static int btf_check_type_tags(struct btf_verifier_env *env, 5411 struct btf *btf, int start_id) 5412 { 5413 int i, n, good_id = start_id - 1; 5414 bool in_tags; 5415 5416 n = btf_nr_types(btf); 5417 for (i = start_id; i < n; i++) { 5418 const struct btf_type *t; 5419 int chain_limit = 32; 5420 u32 cur_id = i; 5421 5422 t = btf_type_by_id(btf, i); 5423 if (!t) 5424 return -EINVAL; 5425 if (!btf_type_is_modifier(t)) 5426 continue; 5427 5428 cond_resched(); 5429 5430 in_tags = btf_type_is_type_tag(t); 5431 while (btf_type_is_modifier(t)) { 5432 if (!chain_limit--) { 5433 btf_verifier_log(env, "Max chain length or cycle detected"); 5434 return -ELOOP; 5435 } 5436 if (btf_type_is_type_tag(t)) { 5437 if (!in_tags) { 5438 btf_verifier_log(env, "Type tags don't precede modifiers"); 5439 return -EINVAL; 5440 } 5441 } else if (in_tags) { 5442 in_tags = false; 5443 } 5444 if (cur_id <= good_id) 5445 break; 5446 /* Move to next type */ 5447 cur_id = t->type; 5448 t = btf_type_by_id(btf, cur_id); 5449 if (!t) 5450 return -EINVAL; 5451 } 5452 good_id = i; 5453 } 5454 return 0; 5455 } 5456 5457 static int finalize_log(struct bpf_verifier_log *log, bpfptr_t uattr, u32 uattr_size) 5458 { 5459 u32 log_true_size; 5460 int err; 5461 5462 err = bpf_vlog_finalize(log, &log_true_size); 5463 5464 if (uattr_size >= offsetofend(union bpf_attr, btf_log_true_size) && 5465 copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, btf_log_true_size), 5466 &log_true_size, sizeof(log_true_size))) 5467 err = -EFAULT; 5468 5469 return err; 5470 } 5471 5472 static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) 5473 { 5474 bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); 5475 char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf); 5476 struct btf_struct_metas *struct_meta_tab; 5477 struct btf_verifier_env *env = NULL; 5478 struct btf *btf = NULL; 5479 u8 *data; 5480 int err, ret; 5481 5482 if (attr->btf_size > BTF_MAX_SIZE) 5483 return ERR_PTR(-E2BIG); 5484 5485 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN); 5486 if (!env) 5487 return ERR_PTR(-ENOMEM); 5488 5489 /* user could have requested verbose verifier output 5490 * and supplied buffer to store the verification trace 5491 */ 5492 err = bpf_vlog_init(&env->log, attr->btf_log_level, 5493 log_ubuf, attr->btf_log_size); 5494 if (err) 5495 goto errout_free; 5496 5497 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); 5498 if (!btf) { 5499 err = -ENOMEM; 5500 goto errout; 5501 } 5502 env->btf = btf; 5503 5504 data = kvmalloc(attr->btf_size, GFP_KERNEL | __GFP_NOWARN); 5505 if (!data) { 5506 err = -ENOMEM; 5507 goto errout; 5508 } 5509 5510 btf->data = data; 5511 btf->data_size = attr->btf_size; 5512 5513 if (copy_from_bpfptr(data, btf_data, attr->btf_size)) { 5514 err = -EFAULT; 5515 goto errout; 5516 } 5517 5518 err = btf_parse_hdr(env); 5519 if (err) 5520 goto errout; 5521 5522 btf->nohdr_data = btf->data + btf->hdr.hdr_len; 5523 5524 err = btf_parse_str_sec(env); 5525 if (err) 5526 goto errout; 5527 5528 err = btf_parse_type_sec(env); 5529 if (err) 5530 goto errout; 5531 5532 err = btf_check_type_tags(env, btf, 1); 5533 if (err) 5534 goto errout; 5535 5536 struct_meta_tab = btf_parse_struct_metas(&env->log, btf); 5537 if (IS_ERR(struct_meta_tab)) { 5538 err = PTR_ERR(struct_meta_tab); 5539 goto errout; 5540 } 5541 btf->struct_meta_tab = struct_meta_tab; 5542 5543 if (struct_meta_tab) { 5544 int i; 5545 5546 for (i = 0; i < struct_meta_tab->cnt; i++) { 5547 err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record); 5548 if (err < 0) 5549 goto errout_meta; 5550 } 5551 } 5552 5553 err = finalize_log(&env->log, uattr, uattr_size); 5554 if (err) 5555 goto errout_free; 5556 5557 btf_verifier_env_free(env); 5558 refcount_set(&btf->refcnt, 1); 5559 return btf; 5560 5561 errout_meta: 5562 btf_free_struct_meta_tab(btf); 5563 errout: 5564 /* overwrite err with -ENOSPC or -EFAULT */ 5565 ret = finalize_log(&env->log, uattr, uattr_size); 5566 if (ret) 5567 err = ret; 5568 errout_free: 5569 btf_verifier_env_free(env); 5570 if (btf) 5571 btf_free(btf); 5572 return ERR_PTR(err); 5573 } 5574 5575 extern char __weak __start_BTF[]; 5576 extern char __weak __stop_BTF[]; 5577 extern struct btf *btf_vmlinux; 5578 5579 #define BPF_MAP_TYPE(_id, _ops) 5580 #define BPF_LINK_TYPE(_id, _name) 5581 static union { 5582 struct bpf_ctx_convert { 5583 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 5584 prog_ctx_type _id##_prog; \ 5585 kern_ctx_type _id##_kern; 5586 #include <linux/bpf_types.h> 5587 #undef BPF_PROG_TYPE 5588 } *__t; 5589 /* 't' is written once under lock. Read many times. */ 5590 const struct btf_type *t; 5591 } bpf_ctx_convert; 5592 enum { 5593 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 5594 __ctx_convert##_id, 5595 #include <linux/bpf_types.h> 5596 #undef BPF_PROG_TYPE 5597 __ctx_convert_unused, /* to avoid empty enum in extreme .config */ 5598 }; 5599 static u8 bpf_ctx_convert_map[] = { 5600 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 5601 [_id] = __ctx_convert##_id, 5602 #include <linux/bpf_types.h> 5603 #undef BPF_PROG_TYPE 5604 0, /* avoid empty array */ 5605 }; 5606 #undef BPF_MAP_TYPE 5607 #undef BPF_LINK_TYPE 5608 5609 const struct btf_member * 5610 btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, 5611 const struct btf_type *t, enum bpf_prog_type prog_type, 5612 int arg) 5613 { 5614 const struct btf_type *conv_struct; 5615 const struct btf_type *ctx_struct; 5616 const struct btf_member *ctx_type; 5617 const char *tname, *ctx_tname; 5618 5619 conv_struct = bpf_ctx_convert.t; 5620 if (!conv_struct) { 5621 bpf_log(log, "btf_vmlinux is malformed\n"); 5622 return NULL; 5623 } 5624 t = btf_type_by_id(btf, t->type); 5625 while (btf_type_is_modifier(t)) 5626 t = btf_type_by_id(btf, t->type); 5627 if (!btf_type_is_struct(t)) { 5628 /* Only pointer to struct is supported for now. 5629 * That means that BPF_PROG_TYPE_TRACEPOINT with BTF 5630 * is not supported yet. 5631 * BPF_PROG_TYPE_RAW_TRACEPOINT is fine. 5632 */ 5633 return NULL; 5634 } 5635 tname = btf_name_by_offset(btf, t->name_off); 5636 if (!tname) { 5637 bpf_log(log, "arg#%d struct doesn't have a name\n", arg); 5638 return NULL; 5639 } 5640 /* prog_type is valid bpf program type. No need for bounds check. */ 5641 ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2; 5642 /* ctx_struct is a pointer to prog_ctx_type in vmlinux. 5643 * Like 'struct __sk_buff' 5644 */ 5645 ctx_struct = btf_type_by_id(btf_vmlinux, ctx_type->type); 5646 if (!ctx_struct) 5647 /* should not happen */ 5648 return NULL; 5649 again: 5650 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_struct->name_off); 5651 if (!ctx_tname) { 5652 /* should not happen */ 5653 bpf_log(log, "Please fix kernel include/linux/bpf_types.h\n"); 5654 return NULL; 5655 } 5656 /* only compare that prog's ctx type name is the same as 5657 * kernel expects. No need to compare field by field. 5658 * It's ok for bpf prog to do: 5659 * struct __sk_buff {}; 5660 * int socket_filter_bpf_prog(struct __sk_buff *skb) 5661 * { // no fields of skb are ever used } 5662 */ 5663 if (strcmp(ctx_tname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0) 5664 return ctx_type; 5665 if (strcmp(ctx_tname, "xdp_md") == 0 && strcmp(tname, "xdp_buff") == 0) 5666 return ctx_type; 5667 if (strcmp(ctx_tname, tname)) { 5668 /* bpf_user_pt_regs_t is a typedef, so resolve it to 5669 * underlying struct and check name again 5670 */ 5671 if (!btf_type_is_modifier(ctx_struct)) 5672 return NULL; 5673 while (btf_type_is_modifier(ctx_struct)) 5674 ctx_struct = btf_type_by_id(btf_vmlinux, ctx_struct->type); 5675 goto again; 5676 } 5677 return ctx_type; 5678 } 5679 5680 static int btf_translate_to_vmlinux(struct bpf_verifier_log *log, 5681 struct btf *btf, 5682 const struct btf_type *t, 5683 enum bpf_prog_type prog_type, 5684 int arg) 5685 { 5686 const struct btf_member *prog_ctx_type, *kern_ctx_type; 5687 5688 prog_ctx_type = btf_get_prog_ctx_type(log, btf, t, prog_type, arg); 5689 if (!prog_ctx_type) 5690 return -ENOENT; 5691 kern_ctx_type = prog_ctx_type + 1; 5692 return kern_ctx_type->type; 5693 } 5694 5695 int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type) 5696 { 5697 const struct btf_member *kctx_member; 5698 const struct btf_type *conv_struct; 5699 const struct btf_type *kctx_type; 5700 u32 kctx_type_id; 5701 5702 conv_struct = bpf_ctx_convert.t; 5703 /* get member for kernel ctx type */ 5704 kctx_member = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2 + 1; 5705 kctx_type_id = kctx_member->type; 5706 kctx_type = btf_type_by_id(btf_vmlinux, kctx_type_id); 5707 if (!btf_type_is_struct(kctx_type)) { 5708 bpf_log(log, "kern ctx type id %u is not a struct\n", kctx_type_id); 5709 return -EINVAL; 5710 } 5711 5712 return kctx_type_id; 5713 } 5714 5715 BTF_ID_LIST(bpf_ctx_convert_btf_id) 5716 BTF_ID(struct, bpf_ctx_convert) 5717 5718 struct btf *btf_parse_vmlinux(void) 5719 { 5720 struct btf_verifier_env *env = NULL; 5721 struct bpf_verifier_log *log; 5722 struct btf *btf = NULL; 5723 int err; 5724 5725 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN); 5726 if (!env) 5727 return ERR_PTR(-ENOMEM); 5728 5729 log = &env->log; 5730 log->level = BPF_LOG_KERNEL; 5731 5732 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); 5733 if (!btf) { 5734 err = -ENOMEM; 5735 goto errout; 5736 } 5737 env->btf = btf; 5738 5739 btf->data = __start_BTF; 5740 btf->data_size = __stop_BTF - __start_BTF; 5741 btf->kernel_btf = true; 5742 snprintf(btf->name, sizeof(btf->name), "vmlinux"); 5743 5744 err = btf_parse_hdr(env); 5745 if (err) 5746 goto errout; 5747 5748 btf->nohdr_data = btf->data + btf->hdr.hdr_len; 5749 5750 err = btf_parse_str_sec(env); 5751 if (err) 5752 goto errout; 5753 5754 err = btf_check_all_metas(env); 5755 if (err) 5756 goto errout; 5757 5758 err = btf_check_type_tags(env, btf, 1); 5759 if (err) 5760 goto errout; 5761 5762 /* btf_parse_vmlinux() runs under bpf_verifier_lock */ 5763 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); 5764 5765 bpf_struct_ops_init(btf, log); 5766 5767 refcount_set(&btf->refcnt, 1); 5768 5769 err = btf_alloc_id(btf); 5770 if (err) 5771 goto errout; 5772 5773 btf_verifier_env_free(env); 5774 return btf; 5775 5776 errout: 5777 btf_verifier_env_free(env); 5778 if (btf) { 5779 kvfree(btf->types); 5780 kfree(btf); 5781 } 5782 return ERR_PTR(err); 5783 } 5784 5785 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 5786 5787 static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size) 5788 { 5789 struct btf_verifier_env *env = NULL; 5790 struct bpf_verifier_log *log; 5791 struct btf *btf = NULL, *base_btf; 5792 int err; 5793 5794 base_btf = bpf_get_btf_vmlinux(); 5795 if (IS_ERR(base_btf)) 5796 return base_btf; 5797 if (!base_btf) 5798 return ERR_PTR(-EINVAL); 5799 5800 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN); 5801 if (!env) 5802 return ERR_PTR(-ENOMEM); 5803 5804 log = &env->log; 5805 log->level = BPF_LOG_KERNEL; 5806 5807 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); 5808 if (!btf) { 5809 err = -ENOMEM; 5810 goto errout; 5811 } 5812 env->btf = btf; 5813 5814 btf->base_btf = base_btf; 5815 btf->start_id = base_btf->nr_types; 5816 btf->start_str_off = base_btf->hdr.str_len; 5817 btf->kernel_btf = true; 5818 snprintf(btf->name, sizeof(btf->name), "%s", module_name); 5819 5820 btf->data = kvmalloc(data_size, GFP_KERNEL | __GFP_NOWARN); 5821 if (!btf->data) { 5822 err = -ENOMEM; 5823 goto errout; 5824 } 5825 memcpy(btf->data, data, data_size); 5826 btf->data_size = data_size; 5827 5828 err = btf_parse_hdr(env); 5829 if (err) 5830 goto errout; 5831 5832 btf->nohdr_data = btf->data + btf->hdr.hdr_len; 5833 5834 err = btf_parse_str_sec(env); 5835 if (err) 5836 goto errout; 5837 5838 err = btf_check_all_metas(env); 5839 if (err) 5840 goto errout; 5841 5842 err = btf_check_type_tags(env, btf, btf_nr_types(base_btf)); 5843 if (err) 5844 goto errout; 5845 5846 btf_verifier_env_free(env); 5847 refcount_set(&btf->refcnt, 1); 5848 return btf; 5849 5850 errout: 5851 btf_verifier_env_free(env); 5852 if (btf) { 5853 kvfree(btf->data); 5854 kvfree(btf->types); 5855 kfree(btf); 5856 } 5857 return ERR_PTR(err); 5858 } 5859 5860 #endif /* CONFIG_DEBUG_INFO_BTF_MODULES */ 5861 5862 struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) 5863 { 5864 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 5865 5866 if (tgt_prog) 5867 return tgt_prog->aux->btf; 5868 else 5869 return prog->aux->attach_btf; 5870 } 5871 5872 static bool is_int_ptr(struct btf *btf, const struct btf_type *t) 5873 { 5874 /* skip modifiers */ 5875 t = btf_type_skip_modifiers(btf, t->type, NULL); 5876 5877 return btf_type_is_int(t); 5878 } 5879 5880 static u32 get_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto, 5881 int off) 5882 { 5883 const struct btf_param *args; 5884 const struct btf_type *t; 5885 u32 offset = 0, nr_args; 5886 int i; 5887 5888 if (!func_proto) 5889 return off / 8; 5890 5891 nr_args = btf_type_vlen(func_proto); 5892 args = (const struct btf_param *)(func_proto + 1); 5893 for (i = 0; i < nr_args; i++) { 5894 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 5895 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); 5896 if (off < offset) 5897 return i; 5898 } 5899 5900 t = btf_type_skip_modifiers(btf, func_proto->type, NULL); 5901 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); 5902 if (off < offset) 5903 return nr_args; 5904 5905 return nr_args + 1; 5906 } 5907 5908 static bool prog_args_trusted(const struct bpf_prog *prog) 5909 { 5910 enum bpf_attach_type atype = prog->expected_attach_type; 5911 5912 switch (prog->type) { 5913 case BPF_PROG_TYPE_TRACING: 5914 return atype == BPF_TRACE_RAW_TP || atype == BPF_TRACE_ITER; 5915 case BPF_PROG_TYPE_LSM: 5916 return bpf_lsm_is_trusted(prog); 5917 case BPF_PROG_TYPE_STRUCT_OPS: 5918 return true; 5919 default: 5920 return false; 5921 } 5922 } 5923 5924 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 5925 const struct bpf_prog *prog, 5926 struct bpf_insn_access_aux *info) 5927 { 5928 const struct btf_type *t = prog->aux->attach_func_proto; 5929 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 5930 struct btf *btf = bpf_prog_get_target_btf(prog); 5931 const char *tname = prog->aux->attach_func_name; 5932 struct bpf_verifier_log *log = info->log; 5933 const struct btf_param *args; 5934 const char *tag_value; 5935 u32 nr_args, arg; 5936 int i, ret; 5937 5938 if (off % 8) { 5939 bpf_log(log, "func '%s' offset %d is not multiple of 8\n", 5940 tname, off); 5941 return false; 5942 } 5943 arg = get_ctx_arg_idx(btf, t, off); 5944 args = (const struct btf_param *)(t + 1); 5945 /* if (t == NULL) Fall back to default BPF prog with 5946 * MAX_BPF_FUNC_REG_ARGS u64 arguments. 5947 */ 5948 nr_args = t ? btf_type_vlen(t) : MAX_BPF_FUNC_REG_ARGS; 5949 if (prog->aux->attach_btf_trace) { 5950 /* skip first 'void *__data' argument in btf_trace_##name typedef */ 5951 args++; 5952 nr_args--; 5953 } 5954 5955 if (arg > nr_args) { 5956 bpf_log(log, "func '%s' doesn't have %d-th argument\n", 5957 tname, arg + 1); 5958 return false; 5959 } 5960 5961 if (arg == nr_args) { 5962 switch (prog->expected_attach_type) { 5963 case BPF_LSM_CGROUP: 5964 case BPF_LSM_MAC: 5965 case BPF_TRACE_FEXIT: 5966 /* When LSM programs are attached to void LSM hooks 5967 * they use FEXIT trampolines and when attached to 5968 * int LSM hooks, they use MODIFY_RETURN trampolines. 5969 * 5970 * While the LSM programs are BPF_MODIFY_RETURN-like 5971 * the check: 5972 * 5973 * if (ret_type != 'int') 5974 * return -EINVAL; 5975 * 5976 * is _not_ done here. This is still safe as LSM hooks 5977 * have only void and int return types. 5978 */ 5979 if (!t) 5980 return true; 5981 t = btf_type_by_id(btf, t->type); 5982 break; 5983 case BPF_MODIFY_RETURN: 5984 /* For now the BPF_MODIFY_RETURN can only be attached to 5985 * functions that return an int. 5986 */ 5987 if (!t) 5988 return false; 5989 5990 t = btf_type_skip_modifiers(btf, t->type, NULL); 5991 if (!btf_type_is_small_int(t)) { 5992 bpf_log(log, 5993 "ret type %s not allowed for fmod_ret\n", 5994 btf_type_str(t)); 5995 return false; 5996 } 5997 break; 5998 default: 5999 bpf_log(log, "func '%s' doesn't have %d-th argument\n", 6000 tname, arg + 1); 6001 return false; 6002 } 6003 } else { 6004 if (!t) 6005 /* Default prog with MAX_BPF_FUNC_REG_ARGS args */ 6006 return true; 6007 t = btf_type_by_id(btf, args[arg].type); 6008 } 6009 6010 /* skip modifiers */ 6011 while (btf_type_is_modifier(t)) 6012 t = btf_type_by_id(btf, t->type); 6013 if (btf_type_is_small_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t)) 6014 /* accessing a scalar */ 6015 return true; 6016 if (!btf_type_is_ptr(t)) { 6017 bpf_log(log, 6018 "func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n", 6019 tname, arg, 6020 __btf_name_by_offset(btf, t->name_off), 6021 btf_type_str(t)); 6022 return false; 6023 } 6024 6025 /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */ 6026 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { 6027 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; 6028 u32 type, flag; 6029 6030 type = base_type(ctx_arg_info->reg_type); 6031 flag = type_flag(ctx_arg_info->reg_type); 6032 if (ctx_arg_info->offset == off && type == PTR_TO_BUF && 6033 (flag & PTR_MAYBE_NULL)) { 6034 info->reg_type = ctx_arg_info->reg_type; 6035 return true; 6036 } 6037 } 6038 6039 if (t->type == 0) 6040 /* This is a pointer to void. 6041 * It is the same as scalar from the verifier safety pov. 6042 * No further pointer walking is allowed. 6043 */ 6044 return true; 6045 6046 if (is_int_ptr(btf, t)) 6047 return true; 6048 6049 /* this is a pointer to another type */ 6050 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { 6051 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; 6052 6053 if (ctx_arg_info->offset == off) { 6054 if (!ctx_arg_info->btf_id) { 6055 bpf_log(log,"invalid btf_id for context argument offset %u\n", off); 6056 return false; 6057 } 6058 6059 info->reg_type = ctx_arg_info->reg_type; 6060 info->btf = btf_vmlinux; 6061 info->btf_id = ctx_arg_info->btf_id; 6062 return true; 6063 } 6064 } 6065 6066 info->reg_type = PTR_TO_BTF_ID; 6067 if (prog_args_trusted(prog)) 6068 info->reg_type |= PTR_TRUSTED; 6069 6070 if (tgt_prog) { 6071 enum bpf_prog_type tgt_type; 6072 6073 if (tgt_prog->type == BPF_PROG_TYPE_EXT) 6074 tgt_type = tgt_prog->aux->saved_dst_prog_type; 6075 else 6076 tgt_type = tgt_prog->type; 6077 6078 ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg); 6079 if (ret > 0) { 6080 info->btf = btf_vmlinux; 6081 info->btf_id = ret; 6082 return true; 6083 } else { 6084 return false; 6085 } 6086 } 6087 6088 info->btf = btf; 6089 info->btf_id = t->type; 6090 t = btf_type_by_id(btf, t->type); 6091 6092 if (btf_type_is_type_tag(t)) { 6093 tag_value = __btf_name_by_offset(btf, t->name_off); 6094 if (strcmp(tag_value, "user") == 0) 6095 info->reg_type |= MEM_USER; 6096 if (strcmp(tag_value, "percpu") == 0) 6097 info->reg_type |= MEM_PERCPU; 6098 } 6099 6100 /* skip modifiers */ 6101 while (btf_type_is_modifier(t)) { 6102 info->btf_id = t->type; 6103 t = btf_type_by_id(btf, t->type); 6104 } 6105 if (!btf_type_is_struct(t)) { 6106 bpf_log(log, 6107 "func '%s' arg%d type %s is not a struct\n", 6108 tname, arg, btf_type_str(t)); 6109 return false; 6110 } 6111 bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n", 6112 tname, arg, info->btf_id, btf_type_str(t), 6113 __btf_name_by_offset(btf, t->name_off)); 6114 return true; 6115 } 6116 6117 enum bpf_struct_walk_result { 6118 /* < 0 error */ 6119 WALK_SCALAR = 0, 6120 WALK_PTR, 6121 WALK_STRUCT, 6122 }; 6123 6124 static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, 6125 const struct btf_type *t, int off, int size, 6126 u32 *next_btf_id, enum bpf_type_flag *flag, 6127 const char **field_name) 6128 { 6129 u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; 6130 const struct btf_type *mtype, *elem_type = NULL; 6131 const struct btf_member *member; 6132 const char *tname, *mname, *tag_value; 6133 u32 vlen, elem_id, mid; 6134 6135 *flag = 0; 6136 again: 6137 tname = __btf_name_by_offset(btf, t->name_off); 6138 if (!btf_type_is_struct(t)) { 6139 bpf_log(log, "Type '%s' is not a struct\n", tname); 6140 return -EINVAL; 6141 } 6142 6143 vlen = btf_type_vlen(t); 6144 if (off + size > t->size) { 6145 /* If the last element is a variable size array, we may 6146 * need to relax the rule. 6147 */ 6148 struct btf_array *array_elem; 6149 6150 if (vlen == 0) 6151 goto error; 6152 6153 member = btf_type_member(t) + vlen - 1; 6154 mtype = btf_type_skip_modifiers(btf, member->type, 6155 NULL); 6156 if (!btf_type_is_array(mtype)) 6157 goto error; 6158 6159 array_elem = (struct btf_array *)(mtype + 1); 6160 if (array_elem->nelems != 0) 6161 goto error; 6162 6163 moff = __btf_member_bit_offset(t, member) / 8; 6164 if (off < moff) 6165 goto error; 6166 6167 /* allow structure and integer */ 6168 t = btf_type_skip_modifiers(btf, array_elem->type, 6169 NULL); 6170 6171 if (btf_type_is_int(t)) 6172 return WALK_SCALAR; 6173 6174 if (!btf_type_is_struct(t)) 6175 goto error; 6176 6177 off = (off - moff) % t->size; 6178 goto again; 6179 6180 error: 6181 bpf_log(log, "access beyond struct %s at off %u size %u\n", 6182 tname, off, size); 6183 return -EACCES; 6184 } 6185 6186 for_each_member(i, t, member) { 6187 /* offset of the field in bytes */ 6188 moff = __btf_member_bit_offset(t, member) / 8; 6189 if (off + size <= moff) 6190 /* won't find anything, field is already too far */ 6191 break; 6192 6193 if (__btf_member_bitfield_size(t, member)) { 6194 u32 end_bit = __btf_member_bit_offset(t, member) + 6195 __btf_member_bitfield_size(t, member); 6196 6197 /* off <= moff instead of off == moff because clang 6198 * does not generate a BTF member for anonymous 6199 * bitfield like the ":16" here: 6200 * struct { 6201 * int :16; 6202 * int x:8; 6203 * }; 6204 */ 6205 if (off <= moff && 6206 BITS_ROUNDUP_BYTES(end_bit) <= off + size) 6207 return WALK_SCALAR; 6208 6209 /* off may be accessing a following member 6210 * 6211 * or 6212 * 6213 * Doing partial access at either end of this 6214 * bitfield. Continue on this case also to 6215 * treat it as not accessing this bitfield 6216 * and eventually error out as field not 6217 * found to keep it simple. 6218 * It could be relaxed if there was a legit 6219 * partial access case later. 6220 */ 6221 continue; 6222 } 6223 6224 /* In case of "off" is pointing to holes of a struct */ 6225 if (off < moff) 6226 break; 6227 6228 /* type of the field */ 6229 mid = member->type; 6230 mtype = btf_type_by_id(btf, member->type); 6231 mname = __btf_name_by_offset(btf, member->name_off); 6232 6233 mtype = __btf_resolve_size(btf, mtype, &msize, 6234 &elem_type, &elem_id, &total_nelems, 6235 &mid); 6236 if (IS_ERR(mtype)) { 6237 bpf_log(log, "field %s doesn't have size\n", mname); 6238 return -EFAULT; 6239 } 6240 6241 mtrue_end = moff + msize; 6242 if (off >= mtrue_end) 6243 /* no overlap with member, keep iterating */ 6244 continue; 6245 6246 if (btf_type_is_array(mtype)) { 6247 u32 elem_idx; 6248 6249 /* __btf_resolve_size() above helps to 6250 * linearize a multi-dimensional array. 6251 * 6252 * The logic here is treating an array 6253 * in a struct as the following way: 6254 * 6255 * struct outer { 6256 * struct inner array[2][2]; 6257 * }; 6258 * 6259 * looks like: 6260 * 6261 * struct outer { 6262 * struct inner array_elem0; 6263 * struct inner array_elem1; 6264 * struct inner array_elem2; 6265 * struct inner array_elem3; 6266 * }; 6267 * 6268 * When accessing outer->array[1][0], it moves 6269 * moff to "array_elem2", set mtype to 6270 * "struct inner", and msize also becomes 6271 * sizeof(struct inner). Then most of the 6272 * remaining logic will fall through without 6273 * caring the current member is an array or 6274 * not. 6275 * 6276 * Unlike mtype/msize/moff, mtrue_end does not 6277 * change. The naming difference ("_true") tells 6278 * that it is not always corresponding to 6279 * the current mtype/msize/moff. 6280 * It is the true end of the current 6281 * member (i.e. array in this case). That 6282 * will allow an int array to be accessed like 6283 * a scratch space, 6284 * i.e. allow access beyond the size of 6285 * the array's element as long as it is 6286 * within the mtrue_end boundary. 6287 */ 6288 6289 /* skip empty array */ 6290 if (moff == mtrue_end) 6291 continue; 6292 6293 msize /= total_nelems; 6294 elem_idx = (off - moff) / msize; 6295 moff += elem_idx * msize; 6296 mtype = elem_type; 6297 mid = elem_id; 6298 } 6299 6300 /* the 'off' we're looking for is either equal to start 6301 * of this field or inside of this struct 6302 */ 6303 if (btf_type_is_struct(mtype)) { 6304 if (BTF_INFO_KIND(mtype->info) == BTF_KIND_UNION && 6305 btf_type_vlen(mtype) != 1) 6306 /* 6307 * walking unions yields untrusted pointers 6308 * with exception of __bpf_md_ptr and other 6309 * unions with a single member 6310 */ 6311 *flag |= PTR_UNTRUSTED; 6312 6313 /* our field must be inside that union or struct */ 6314 t = mtype; 6315 6316 /* return if the offset matches the member offset */ 6317 if (off == moff) { 6318 *next_btf_id = mid; 6319 return WALK_STRUCT; 6320 } 6321 6322 /* adjust offset we're looking for */ 6323 off -= moff; 6324 goto again; 6325 } 6326 6327 if (btf_type_is_ptr(mtype)) { 6328 const struct btf_type *stype, *t; 6329 enum bpf_type_flag tmp_flag = 0; 6330 u32 id; 6331 6332 if (msize != size || off != moff) { 6333 bpf_log(log, 6334 "cannot access ptr member %s with moff %u in struct %s with off %u size %u\n", 6335 mname, moff, tname, off, size); 6336 return -EACCES; 6337 } 6338 6339 /* check type tag */ 6340 t = btf_type_by_id(btf, mtype->type); 6341 if (btf_type_is_type_tag(t)) { 6342 tag_value = __btf_name_by_offset(btf, t->name_off); 6343 /* check __user tag */ 6344 if (strcmp(tag_value, "user") == 0) 6345 tmp_flag = MEM_USER; 6346 /* check __percpu tag */ 6347 if (strcmp(tag_value, "percpu") == 0) 6348 tmp_flag = MEM_PERCPU; 6349 /* check __rcu tag */ 6350 if (strcmp(tag_value, "rcu") == 0) 6351 tmp_flag = MEM_RCU; 6352 } 6353 6354 stype = btf_type_skip_modifiers(btf, mtype->type, &id); 6355 if (btf_type_is_struct(stype)) { 6356 *next_btf_id = id; 6357 *flag |= tmp_flag; 6358 if (field_name) 6359 *field_name = mname; 6360 return WALK_PTR; 6361 } 6362 } 6363 6364 /* Allow more flexible access within an int as long as 6365 * it is within mtrue_end. 6366 * Since mtrue_end could be the end of an array, 6367 * that also allows using an array of int as a scratch 6368 * space. e.g. skb->cb[]. 6369 */ 6370 if (off + size > mtrue_end) { 6371 bpf_log(log, 6372 "access beyond the end of member %s (mend:%u) in struct %s with off %u size %u\n", 6373 mname, mtrue_end, tname, off, size); 6374 return -EACCES; 6375 } 6376 6377 return WALK_SCALAR; 6378 } 6379 bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off); 6380 return -EINVAL; 6381 } 6382 6383 int btf_struct_access(struct bpf_verifier_log *log, 6384 const struct bpf_reg_state *reg, 6385 int off, int size, enum bpf_access_type atype __maybe_unused, 6386 u32 *next_btf_id, enum bpf_type_flag *flag, 6387 const char **field_name) 6388 { 6389 const struct btf *btf = reg->btf; 6390 enum bpf_type_flag tmp_flag = 0; 6391 const struct btf_type *t; 6392 u32 id = reg->btf_id; 6393 int err; 6394 6395 while (type_is_alloc(reg->type)) { 6396 struct btf_struct_meta *meta; 6397 struct btf_record *rec; 6398 int i; 6399 6400 meta = btf_find_struct_meta(btf, id); 6401 if (!meta) 6402 break; 6403 rec = meta->record; 6404 for (i = 0; i < rec->cnt; i++) { 6405 struct btf_field *field = &rec->fields[i]; 6406 u32 offset = field->offset; 6407 if (off < offset + btf_field_type_size(field->type) && offset < off + size) { 6408 bpf_log(log, 6409 "direct access to %s is disallowed\n", 6410 btf_field_type_name(field->type)); 6411 return -EACCES; 6412 } 6413 } 6414 break; 6415 } 6416 6417 t = btf_type_by_id(btf, id); 6418 do { 6419 err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); 6420 6421 switch (err) { 6422 case WALK_PTR: 6423 /* For local types, the destination register cannot 6424 * become a pointer again. 6425 */ 6426 if (type_is_alloc(reg->type)) 6427 return SCALAR_VALUE; 6428 /* If we found the pointer or scalar on t+off, 6429 * we're done. 6430 */ 6431 *next_btf_id = id; 6432 *flag = tmp_flag; 6433 return PTR_TO_BTF_ID; 6434 case WALK_SCALAR: 6435 return SCALAR_VALUE; 6436 case WALK_STRUCT: 6437 /* We found nested struct, so continue the search 6438 * by diving in it. At this point the offset is 6439 * aligned with the new type, so set it to 0. 6440 */ 6441 t = btf_type_by_id(btf, id); 6442 off = 0; 6443 break; 6444 default: 6445 /* It's either error or unknown return value.. 6446 * scream and leave. 6447 */ 6448 if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value")) 6449 return -EINVAL; 6450 return err; 6451 } 6452 } while (t); 6453 6454 return -EINVAL; 6455 } 6456 6457 /* Check that two BTF types, each specified as an BTF object + id, are exactly 6458 * the same. Trivial ID check is not enough due to module BTFs, because we can 6459 * end up with two different module BTFs, but IDs point to the common type in 6460 * vmlinux BTF. 6461 */ 6462 bool btf_types_are_same(const struct btf *btf1, u32 id1, 6463 const struct btf *btf2, u32 id2) 6464 { 6465 if (id1 != id2) 6466 return false; 6467 if (btf1 == btf2) 6468 return true; 6469 return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2); 6470 } 6471 6472 bool btf_struct_ids_match(struct bpf_verifier_log *log, 6473 const struct btf *btf, u32 id, int off, 6474 const struct btf *need_btf, u32 need_type_id, 6475 bool strict) 6476 { 6477 const struct btf_type *type; 6478 enum bpf_type_flag flag; 6479 int err; 6480 6481 /* Are we already done? */ 6482 if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id)) 6483 return true; 6484 /* In case of strict type match, we do not walk struct, the top level 6485 * type match must succeed. When strict is true, off should have already 6486 * been 0. 6487 */ 6488 if (strict) 6489 return false; 6490 again: 6491 type = btf_type_by_id(btf, id); 6492 if (!type) 6493 return false; 6494 err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); 6495 if (err != WALK_STRUCT) 6496 return false; 6497 6498 /* We found nested struct object. If it matches 6499 * the requested ID, we're done. Otherwise let's 6500 * continue the search with offset 0 in the new 6501 * type. 6502 */ 6503 if (!btf_types_are_same(btf, id, need_btf, need_type_id)) { 6504 off = 0; 6505 goto again; 6506 } 6507 6508 return true; 6509 } 6510 6511 static int __get_type_size(struct btf *btf, u32 btf_id, 6512 const struct btf_type **ret_type) 6513 { 6514 const struct btf_type *t; 6515 6516 *ret_type = btf_type_by_id(btf, 0); 6517 if (!btf_id) 6518 /* void */ 6519 return 0; 6520 t = btf_type_by_id(btf, btf_id); 6521 while (t && btf_type_is_modifier(t)) 6522 t = btf_type_by_id(btf, t->type); 6523 if (!t) 6524 return -EINVAL; 6525 *ret_type = t; 6526 if (btf_type_is_ptr(t)) 6527 /* kernel size of pointer. Not BPF's size of pointer*/ 6528 return sizeof(void *); 6529 if (btf_type_is_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t)) 6530 return t->size; 6531 return -EINVAL; 6532 } 6533 6534 static u8 __get_type_fmodel_flags(const struct btf_type *t) 6535 { 6536 u8 flags = 0; 6537 6538 if (__btf_type_is_struct(t)) 6539 flags |= BTF_FMODEL_STRUCT_ARG; 6540 if (btf_type_is_signed_int(t)) 6541 flags |= BTF_FMODEL_SIGNED_ARG; 6542 6543 return flags; 6544 } 6545 6546 int btf_distill_func_proto(struct bpf_verifier_log *log, 6547 struct btf *btf, 6548 const struct btf_type *func, 6549 const char *tname, 6550 struct btf_func_model *m) 6551 { 6552 const struct btf_param *args; 6553 const struct btf_type *t; 6554 u32 i, nargs; 6555 int ret; 6556 6557 if (!func) { 6558 /* BTF function prototype doesn't match the verifier types. 6559 * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args. 6560 */ 6561 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 6562 m->arg_size[i] = 8; 6563 m->arg_flags[i] = 0; 6564 } 6565 m->ret_size = 8; 6566 m->ret_flags = 0; 6567 m->nr_args = MAX_BPF_FUNC_REG_ARGS; 6568 return 0; 6569 } 6570 args = (const struct btf_param *)(func + 1); 6571 nargs = btf_type_vlen(func); 6572 if (nargs > MAX_BPF_FUNC_ARGS) { 6573 bpf_log(log, 6574 "The function %s has %d arguments. Too many.\n", 6575 tname, nargs); 6576 return -EINVAL; 6577 } 6578 ret = __get_type_size(btf, func->type, &t); 6579 if (ret < 0 || __btf_type_is_struct(t)) { 6580 bpf_log(log, 6581 "The function %s return type %s is unsupported.\n", 6582 tname, btf_type_str(t)); 6583 return -EINVAL; 6584 } 6585 m->ret_size = ret; 6586 m->ret_flags = __get_type_fmodel_flags(t); 6587 6588 for (i = 0; i < nargs; i++) { 6589 if (i == nargs - 1 && args[i].type == 0) { 6590 bpf_log(log, 6591 "The function %s with variable args is unsupported.\n", 6592 tname); 6593 return -EINVAL; 6594 } 6595 ret = __get_type_size(btf, args[i].type, &t); 6596 6597 /* No support of struct argument size greater than 16 bytes */ 6598 if (ret < 0 || ret > 16) { 6599 bpf_log(log, 6600 "The function %s arg%d type %s is unsupported.\n", 6601 tname, i, btf_type_str(t)); 6602 return -EINVAL; 6603 } 6604 if (ret == 0) { 6605 bpf_log(log, 6606 "The function %s has malformed void argument.\n", 6607 tname); 6608 return -EINVAL; 6609 } 6610 m->arg_size[i] = ret; 6611 m->arg_flags[i] = __get_type_fmodel_flags(t); 6612 } 6613 m->nr_args = nargs; 6614 return 0; 6615 } 6616 6617 /* Compare BTFs of two functions assuming only scalars and pointers to context. 6618 * t1 points to BTF_KIND_FUNC in btf1 6619 * t2 points to BTF_KIND_FUNC in btf2 6620 * Returns: 6621 * EINVAL - function prototype mismatch 6622 * EFAULT - verifier bug 6623 * 0 - 99% match. The last 1% is validated by the verifier. 6624 */ 6625 static int btf_check_func_type_match(struct bpf_verifier_log *log, 6626 struct btf *btf1, const struct btf_type *t1, 6627 struct btf *btf2, const struct btf_type *t2) 6628 { 6629 const struct btf_param *args1, *args2; 6630 const char *fn1, *fn2, *s1, *s2; 6631 u32 nargs1, nargs2, i; 6632 6633 fn1 = btf_name_by_offset(btf1, t1->name_off); 6634 fn2 = btf_name_by_offset(btf2, t2->name_off); 6635 6636 if (btf_func_linkage(t1) != BTF_FUNC_GLOBAL) { 6637 bpf_log(log, "%s() is not a global function\n", fn1); 6638 return -EINVAL; 6639 } 6640 if (btf_func_linkage(t2) != BTF_FUNC_GLOBAL) { 6641 bpf_log(log, "%s() is not a global function\n", fn2); 6642 return -EINVAL; 6643 } 6644 6645 t1 = btf_type_by_id(btf1, t1->type); 6646 if (!t1 || !btf_type_is_func_proto(t1)) 6647 return -EFAULT; 6648 t2 = btf_type_by_id(btf2, t2->type); 6649 if (!t2 || !btf_type_is_func_proto(t2)) 6650 return -EFAULT; 6651 6652 args1 = (const struct btf_param *)(t1 + 1); 6653 nargs1 = btf_type_vlen(t1); 6654 args2 = (const struct btf_param *)(t2 + 1); 6655 nargs2 = btf_type_vlen(t2); 6656 6657 if (nargs1 != nargs2) { 6658 bpf_log(log, "%s() has %d args while %s() has %d args\n", 6659 fn1, nargs1, fn2, nargs2); 6660 return -EINVAL; 6661 } 6662 6663 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); 6664 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); 6665 if (t1->info != t2->info) { 6666 bpf_log(log, 6667 "Return type %s of %s() doesn't match type %s of %s()\n", 6668 btf_type_str(t1), fn1, 6669 btf_type_str(t2), fn2); 6670 return -EINVAL; 6671 } 6672 6673 for (i = 0; i < nargs1; i++) { 6674 t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); 6675 t2 = btf_type_skip_modifiers(btf2, args2[i].type, NULL); 6676 6677 if (t1->info != t2->info) { 6678 bpf_log(log, "arg%d in %s() is %s while %s() has %s\n", 6679 i, fn1, btf_type_str(t1), 6680 fn2, btf_type_str(t2)); 6681 return -EINVAL; 6682 } 6683 if (btf_type_has_size(t1) && t1->size != t2->size) { 6684 bpf_log(log, 6685 "arg%d in %s() has size %d while %s() has %d\n", 6686 i, fn1, t1->size, 6687 fn2, t2->size); 6688 return -EINVAL; 6689 } 6690 6691 /* global functions are validated with scalars and pointers 6692 * to context only. And only global functions can be replaced. 6693 * Hence type check only those types. 6694 */ 6695 if (btf_type_is_int(t1) || btf_is_any_enum(t1)) 6696 continue; 6697 if (!btf_type_is_ptr(t1)) { 6698 bpf_log(log, 6699 "arg%d in %s() has unrecognized type\n", 6700 i, fn1); 6701 return -EINVAL; 6702 } 6703 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); 6704 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); 6705 if (!btf_type_is_struct(t1)) { 6706 bpf_log(log, 6707 "arg%d in %s() is not a pointer to context\n", 6708 i, fn1); 6709 return -EINVAL; 6710 } 6711 if (!btf_type_is_struct(t2)) { 6712 bpf_log(log, 6713 "arg%d in %s() is not a pointer to context\n", 6714 i, fn2); 6715 return -EINVAL; 6716 } 6717 /* This is an optional check to make program writing easier. 6718 * Compare names of structs and report an error to the user. 6719 * btf_prepare_func_args() already checked that t2 struct 6720 * is a context type. btf_prepare_func_args() will check 6721 * later that t1 struct is a context type as well. 6722 */ 6723 s1 = btf_name_by_offset(btf1, t1->name_off); 6724 s2 = btf_name_by_offset(btf2, t2->name_off); 6725 if (strcmp(s1, s2)) { 6726 bpf_log(log, 6727 "arg%d %s(struct %s *) doesn't match %s(struct %s *)\n", 6728 i, fn1, s1, fn2, s2); 6729 return -EINVAL; 6730 } 6731 } 6732 return 0; 6733 } 6734 6735 /* Compare BTFs of given program with BTF of target program */ 6736 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 6737 struct btf *btf2, const struct btf_type *t2) 6738 { 6739 struct btf *btf1 = prog->aux->btf; 6740 const struct btf_type *t1; 6741 u32 btf_id = 0; 6742 6743 if (!prog->aux->func_info) { 6744 bpf_log(log, "Program extension requires BTF\n"); 6745 return -EINVAL; 6746 } 6747 6748 btf_id = prog->aux->func_info[0].type_id; 6749 if (!btf_id) 6750 return -EFAULT; 6751 6752 t1 = btf_type_by_id(btf1, btf_id); 6753 if (!t1 || !btf_type_is_func(t1)) 6754 return -EFAULT; 6755 6756 return btf_check_func_type_match(log, btf1, t1, btf2, t2); 6757 } 6758 6759 static int btf_check_func_arg_match(struct bpf_verifier_env *env, 6760 const struct btf *btf, u32 func_id, 6761 struct bpf_reg_state *regs, 6762 bool ptr_to_mem_ok, 6763 bool processing_call) 6764 { 6765 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 6766 struct bpf_verifier_log *log = &env->log; 6767 const char *func_name, *ref_tname; 6768 const struct btf_type *t, *ref_t; 6769 const struct btf_param *args; 6770 u32 i, nargs, ref_id; 6771 int ret; 6772 6773 t = btf_type_by_id(btf, func_id); 6774 if (!t || !btf_type_is_func(t)) { 6775 /* These checks were already done by the verifier while loading 6776 * struct bpf_func_info or in add_kfunc_call(). 6777 */ 6778 bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n", 6779 func_id); 6780 return -EFAULT; 6781 } 6782 func_name = btf_name_by_offset(btf, t->name_off); 6783 6784 t = btf_type_by_id(btf, t->type); 6785 if (!t || !btf_type_is_func_proto(t)) { 6786 bpf_log(log, "Invalid BTF of func %s\n", func_name); 6787 return -EFAULT; 6788 } 6789 args = (const struct btf_param *)(t + 1); 6790 nargs = btf_type_vlen(t); 6791 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 6792 bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs, 6793 MAX_BPF_FUNC_REG_ARGS); 6794 return -EINVAL; 6795 } 6796 6797 /* check that BTF function arguments match actual types that the 6798 * verifier sees. 6799 */ 6800 for (i = 0; i < nargs; i++) { 6801 enum bpf_arg_type arg_type = ARG_DONTCARE; 6802 u32 regno = i + 1; 6803 struct bpf_reg_state *reg = ®s[regno]; 6804 6805 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 6806 if (btf_type_is_scalar(t)) { 6807 if (reg->type == SCALAR_VALUE) 6808 continue; 6809 bpf_log(log, "R%d is not a scalar\n", regno); 6810 return -EINVAL; 6811 } 6812 6813 if (!btf_type_is_ptr(t)) { 6814 bpf_log(log, "Unrecognized arg#%d type %s\n", 6815 i, btf_type_str(t)); 6816 return -EINVAL; 6817 } 6818 6819 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); 6820 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 6821 6822 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 6823 if (ret < 0) 6824 return ret; 6825 6826 if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { 6827 /* If function expects ctx type in BTF check that caller 6828 * is passing PTR_TO_CTX. 6829 */ 6830 if (reg->type != PTR_TO_CTX) { 6831 bpf_log(log, 6832 "arg#%d expected pointer to ctx, but got %s\n", 6833 i, btf_type_str(t)); 6834 return -EINVAL; 6835 } 6836 } else if (ptr_to_mem_ok && processing_call) { 6837 const struct btf_type *resolve_ret; 6838 u32 type_size; 6839 6840 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); 6841 if (IS_ERR(resolve_ret)) { 6842 bpf_log(log, 6843 "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 6844 i, btf_type_str(ref_t), ref_tname, 6845 PTR_ERR(resolve_ret)); 6846 return -EINVAL; 6847 } 6848 6849 if (check_mem_reg(env, reg, regno, type_size)) 6850 return -EINVAL; 6851 } else { 6852 bpf_log(log, "reg type unsupported for arg#%d function %s#%d\n", i, 6853 func_name, func_id); 6854 return -EINVAL; 6855 } 6856 } 6857 6858 return 0; 6859 } 6860 6861 /* Compare BTF of a function declaration with given bpf_reg_state. 6862 * Returns: 6863 * EFAULT - there is a verifier bug. Abort verification. 6864 * EINVAL - there is a type mismatch or BTF is not available. 6865 * 0 - BTF matches with what bpf_reg_state expects. 6866 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. 6867 */ 6868 int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, 6869 struct bpf_reg_state *regs) 6870 { 6871 struct bpf_prog *prog = env->prog; 6872 struct btf *btf = prog->aux->btf; 6873 bool is_global; 6874 u32 btf_id; 6875 int err; 6876 6877 if (!prog->aux->func_info) 6878 return -EINVAL; 6879 6880 btf_id = prog->aux->func_info[subprog].type_id; 6881 if (!btf_id) 6882 return -EFAULT; 6883 6884 if (prog->aux->func_info_aux[subprog].unreliable) 6885 return -EINVAL; 6886 6887 is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 6888 err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, false); 6889 6890 /* Compiler optimizations can remove arguments from static functions 6891 * or mismatched type can be passed into a global function. 6892 * In such cases mark the function as unreliable from BTF point of view. 6893 */ 6894 if (err) 6895 prog->aux->func_info_aux[subprog].unreliable = true; 6896 return err; 6897 } 6898 6899 /* Compare BTF of a function call with given bpf_reg_state. 6900 * Returns: 6901 * EFAULT - there is a verifier bug. Abort verification. 6902 * EINVAL - there is a type mismatch or BTF is not available. 6903 * 0 - BTF matches with what bpf_reg_state expects. 6904 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. 6905 * 6906 * NOTE: the code is duplicated from btf_check_subprog_arg_match() 6907 * because btf_check_func_arg_match() is still doing both. Once that 6908 * function is split in 2, we can call from here btf_check_subprog_arg_match() 6909 * first, and then treat the calling part in a new code path. 6910 */ 6911 int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, 6912 struct bpf_reg_state *regs) 6913 { 6914 struct bpf_prog *prog = env->prog; 6915 struct btf *btf = prog->aux->btf; 6916 bool is_global; 6917 u32 btf_id; 6918 int err; 6919 6920 if (!prog->aux->func_info) 6921 return -EINVAL; 6922 6923 btf_id = prog->aux->func_info[subprog].type_id; 6924 if (!btf_id) 6925 return -EFAULT; 6926 6927 if (prog->aux->func_info_aux[subprog].unreliable) 6928 return -EINVAL; 6929 6930 is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 6931 err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, true); 6932 6933 /* Compiler optimizations can remove arguments from static functions 6934 * or mismatched type can be passed into a global function. 6935 * In such cases mark the function as unreliable from BTF point of view. 6936 */ 6937 if (err) 6938 prog->aux->func_info_aux[subprog].unreliable = true; 6939 return err; 6940 } 6941 6942 /* Convert BTF of a function into bpf_reg_state if possible 6943 * Returns: 6944 * EFAULT - there is a verifier bug. Abort verification. 6945 * EINVAL - cannot convert BTF. 6946 * 0 - Successfully converted BTF into bpf_reg_state 6947 * (either PTR_TO_CTX or SCALAR_VALUE). 6948 */ 6949 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 6950 struct bpf_reg_state *regs) 6951 { 6952 struct bpf_verifier_log *log = &env->log; 6953 struct bpf_prog *prog = env->prog; 6954 enum bpf_prog_type prog_type = prog->type; 6955 struct btf *btf = prog->aux->btf; 6956 const struct btf_param *args; 6957 const struct btf_type *t, *ref_t; 6958 u32 i, nargs, btf_id; 6959 const char *tname; 6960 6961 if (!prog->aux->func_info || 6962 prog->aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL) { 6963 bpf_log(log, "Verifier bug\n"); 6964 return -EFAULT; 6965 } 6966 6967 btf_id = prog->aux->func_info[subprog].type_id; 6968 if (!btf_id) { 6969 bpf_log(log, "Global functions need valid BTF\n"); 6970 return -EFAULT; 6971 } 6972 6973 t = btf_type_by_id(btf, btf_id); 6974 if (!t || !btf_type_is_func(t)) { 6975 /* These checks were already done by the verifier while loading 6976 * struct bpf_func_info 6977 */ 6978 bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n", 6979 subprog); 6980 return -EFAULT; 6981 } 6982 tname = btf_name_by_offset(btf, t->name_off); 6983 6984 if (log->level & BPF_LOG_LEVEL) 6985 bpf_log(log, "Validating %s() func#%d...\n", 6986 tname, subprog); 6987 6988 if (prog->aux->func_info_aux[subprog].unreliable) { 6989 bpf_log(log, "Verifier bug in function %s()\n", tname); 6990 return -EFAULT; 6991 } 6992 if (prog_type == BPF_PROG_TYPE_EXT) 6993 prog_type = prog->aux->dst_prog->type; 6994 6995 t = btf_type_by_id(btf, t->type); 6996 if (!t || !btf_type_is_func_proto(t)) { 6997 bpf_log(log, "Invalid type of function %s()\n", tname); 6998 return -EFAULT; 6999 } 7000 args = (const struct btf_param *)(t + 1); 7001 nargs = btf_type_vlen(t); 7002 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 7003 bpf_log(log, "Global function %s() with %d > %d args. Buggy compiler.\n", 7004 tname, nargs, MAX_BPF_FUNC_REG_ARGS); 7005 return -EINVAL; 7006 } 7007 /* check that function returns int */ 7008 t = btf_type_by_id(btf, t->type); 7009 while (btf_type_is_modifier(t)) 7010 t = btf_type_by_id(btf, t->type); 7011 if (!btf_type_is_int(t) && !btf_is_any_enum(t)) { 7012 bpf_log(log, 7013 "Global function %s() doesn't return scalar. Only those are supported.\n", 7014 tname); 7015 return -EINVAL; 7016 } 7017 /* Convert BTF function arguments into verifier types. 7018 * Only PTR_TO_CTX and SCALAR are supported atm. 7019 */ 7020 for (i = 0; i < nargs; i++) { 7021 struct bpf_reg_state *reg = ®s[i + 1]; 7022 7023 t = btf_type_by_id(btf, args[i].type); 7024 while (btf_type_is_modifier(t)) 7025 t = btf_type_by_id(btf, t->type); 7026 if (btf_type_is_int(t) || btf_is_any_enum(t)) { 7027 reg->type = SCALAR_VALUE; 7028 continue; 7029 } 7030 if (btf_type_is_ptr(t)) { 7031 if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { 7032 reg->type = PTR_TO_CTX; 7033 continue; 7034 } 7035 7036 t = btf_type_skip_modifiers(btf, t->type, NULL); 7037 7038 ref_t = btf_resolve_size(btf, t, ®->mem_size); 7039 if (IS_ERR(ref_t)) { 7040 bpf_log(log, 7041 "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 7042 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), 7043 PTR_ERR(ref_t)); 7044 return -EINVAL; 7045 } 7046 7047 reg->type = PTR_TO_MEM | PTR_MAYBE_NULL; 7048 reg->id = ++env->id_gen; 7049 7050 continue; 7051 } 7052 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", 7053 i, btf_type_str(t), tname); 7054 return -EINVAL; 7055 } 7056 return 0; 7057 } 7058 7059 static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, 7060 struct btf_show *show) 7061 { 7062 const struct btf_type *t = btf_type_by_id(btf, type_id); 7063 7064 show->btf = btf; 7065 memset(&show->state, 0, sizeof(show->state)); 7066 memset(&show->obj, 0, sizeof(show->obj)); 7067 7068 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); 7069 } 7070 7071 static void btf_seq_show(struct btf_show *show, const char *fmt, 7072 va_list args) 7073 { 7074 seq_vprintf((struct seq_file *)show->target, fmt, args); 7075 } 7076 7077 int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, 7078 void *obj, struct seq_file *m, u64 flags) 7079 { 7080 struct btf_show sseq; 7081 7082 sseq.target = m; 7083 sseq.showfn = btf_seq_show; 7084 sseq.flags = flags; 7085 7086 btf_type_show(btf, type_id, obj, &sseq); 7087 7088 return sseq.state.status; 7089 } 7090 7091 void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, 7092 struct seq_file *m) 7093 { 7094 (void) btf_type_seq_show_flags(btf, type_id, obj, m, 7095 BTF_SHOW_NONAME | BTF_SHOW_COMPACT | 7096 BTF_SHOW_ZERO | BTF_SHOW_UNSAFE); 7097 } 7098 7099 struct btf_show_snprintf { 7100 struct btf_show show; 7101 int len_left; /* space left in string */ 7102 int len; /* length we would have written */ 7103 }; 7104 7105 static void btf_snprintf_show(struct btf_show *show, const char *fmt, 7106 va_list args) 7107 { 7108 struct btf_show_snprintf *ssnprintf = (struct btf_show_snprintf *)show; 7109 int len; 7110 7111 len = vsnprintf(show->target, ssnprintf->len_left, fmt, args); 7112 7113 if (len < 0) { 7114 ssnprintf->len_left = 0; 7115 ssnprintf->len = len; 7116 } else if (len >= ssnprintf->len_left) { 7117 /* no space, drive on to get length we would have written */ 7118 ssnprintf->len_left = 0; 7119 ssnprintf->len += len; 7120 } else { 7121 ssnprintf->len_left -= len; 7122 ssnprintf->len += len; 7123 show->target += len; 7124 } 7125 } 7126 7127 int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, 7128 char *buf, int len, u64 flags) 7129 { 7130 struct btf_show_snprintf ssnprintf; 7131 7132 ssnprintf.show.target = buf; 7133 ssnprintf.show.flags = flags; 7134 ssnprintf.show.showfn = btf_snprintf_show; 7135 ssnprintf.len_left = len; 7136 ssnprintf.len = 0; 7137 7138 btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf); 7139 7140 /* If we encountered an error, return it. */ 7141 if (ssnprintf.show.state.status) 7142 return ssnprintf.show.state.status; 7143 7144 /* Otherwise return length we would have written */ 7145 return ssnprintf.len; 7146 } 7147 7148 #ifdef CONFIG_PROC_FS 7149 static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) 7150 { 7151 const struct btf *btf = filp->private_data; 7152 7153 seq_printf(m, "btf_id:\t%u\n", btf->id); 7154 } 7155 #endif 7156 7157 static int btf_release(struct inode *inode, struct file *filp) 7158 { 7159 btf_put(filp->private_data); 7160 return 0; 7161 } 7162 7163 const struct file_operations btf_fops = { 7164 #ifdef CONFIG_PROC_FS 7165 .show_fdinfo = bpf_btf_show_fdinfo, 7166 #endif 7167 .release = btf_release, 7168 }; 7169 7170 static int __btf_new_fd(struct btf *btf) 7171 { 7172 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); 7173 } 7174 7175 int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) 7176 { 7177 struct btf *btf; 7178 int ret; 7179 7180 btf = btf_parse(attr, uattr, uattr_size); 7181 if (IS_ERR(btf)) 7182 return PTR_ERR(btf); 7183 7184 ret = btf_alloc_id(btf); 7185 if (ret) { 7186 btf_free(btf); 7187 return ret; 7188 } 7189 7190 /* 7191 * The BTF ID is published to the userspace. 7192 * All BTF free must go through call_rcu() from 7193 * now on (i.e. free by calling btf_put()). 7194 */ 7195 7196 ret = __btf_new_fd(btf); 7197 if (ret < 0) 7198 btf_put(btf); 7199 7200 return ret; 7201 } 7202 7203 struct btf *btf_get_by_fd(int fd) 7204 { 7205 struct btf *btf; 7206 struct fd f; 7207 7208 f = fdget(fd); 7209 7210 if (!f.file) 7211 return ERR_PTR(-EBADF); 7212 7213 if (f.file->f_op != &btf_fops) { 7214 fdput(f); 7215 return ERR_PTR(-EINVAL); 7216 } 7217 7218 btf = f.file->private_data; 7219 refcount_inc(&btf->refcnt); 7220 fdput(f); 7221 7222 return btf; 7223 } 7224 7225 int btf_get_info_by_fd(const struct btf *btf, 7226 const union bpf_attr *attr, 7227 union bpf_attr __user *uattr) 7228 { 7229 struct bpf_btf_info __user *uinfo; 7230 struct bpf_btf_info info; 7231 u32 info_copy, btf_copy; 7232 void __user *ubtf; 7233 char __user *uname; 7234 u32 uinfo_len, uname_len, name_len; 7235 int ret = 0; 7236 7237 uinfo = u64_to_user_ptr(attr->info.info); 7238 uinfo_len = attr->info.info_len; 7239 7240 info_copy = min_t(u32, uinfo_len, sizeof(info)); 7241 memset(&info, 0, sizeof(info)); 7242 if (copy_from_user(&info, uinfo, info_copy)) 7243 return -EFAULT; 7244 7245 info.id = btf->id; 7246 ubtf = u64_to_user_ptr(info.btf); 7247 btf_copy = min_t(u32, btf->data_size, info.btf_size); 7248 if (copy_to_user(ubtf, btf->data, btf_copy)) 7249 return -EFAULT; 7250 info.btf_size = btf->data_size; 7251 7252 info.kernel_btf = btf->kernel_btf; 7253 7254 uname = u64_to_user_ptr(info.name); 7255 uname_len = info.name_len; 7256 if (!uname ^ !uname_len) 7257 return -EINVAL; 7258 7259 name_len = strlen(btf->name); 7260 info.name_len = name_len; 7261 7262 if (uname) { 7263 if (uname_len >= name_len + 1) { 7264 if (copy_to_user(uname, btf->name, name_len + 1)) 7265 return -EFAULT; 7266 } else { 7267 char zero = '\0'; 7268 7269 if (copy_to_user(uname, btf->name, uname_len - 1)) 7270 return -EFAULT; 7271 if (put_user(zero, uname + uname_len - 1)) 7272 return -EFAULT; 7273 /* let user-space know about too short buffer */ 7274 ret = -ENOSPC; 7275 } 7276 } 7277 7278 if (copy_to_user(uinfo, &info, info_copy) || 7279 put_user(info_copy, &uattr->info.info_len)) 7280 return -EFAULT; 7281 7282 return ret; 7283 } 7284 7285 int btf_get_fd_by_id(u32 id) 7286 { 7287 struct btf *btf; 7288 int fd; 7289 7290 rcu_read_lock(); 7291 btf = idr_find(&btf_idr, id); 7292 if (!btf || !refcount_inc_not_zero(&btf->refcnt)) 7293 btf = ERR_PTR(-ENOENT); 7294 rcu_read_unlock(); 7295 7296 if (IS_ERR(btf)) 7297 return PTR_ERR(btf); 7298 7299 fd = __btf_new_fd(btf); 7300 if (fd < 0) 7301 btf_put(btf); 7302 7303 return fd; 7304 } 7305 7306 u32 btf_obj_id(const struct btf *btf) 7307 { 7308 return btf->id; 7309 } 7310 7311 bool btf_is_kernel(const struct btf *btf) 7312 { 7313 return btf->kernel_btf; 7314 } 7315 7316 bool btf_is_module(const struct btf *btf) 7317 { 7318 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; 7319 } 7320 7321 enum { 7322 BTF_MODULE_F_LIVE = (1 << 0), 7323 }; 7324 7325 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 7326 struct btf_module { 7327 struct list_head list; 7328 struct module *module; 7329 struct btf *btf; 7330 struct bin_attribute *sysfs_attr; 7331 int flags; 7332 }; 7333 7334 static LIST_HEAD(btf_modules); 7335 static DEFINE_MUTEX(btf_module_mutex); 7336 7337 static ssize_t 7338 btf_module_read(struct file *file, struct kobject *kobj, 7339 struct bin_attribute *bin_attr, 7340 char *buf, loff_t off, size_t len) 7341 { 7342 const struct btf *btf = bin_attr->private; 7343 7344 memcpy(buf, btf->data + off, len); 7345 return len; 7346 } 7347 7348 static void purge_cand_cache(struct btf *btf); 7349 7350 static int btf_module_notify(struct notifier_block *nb, unsigned long op, 7351 void *module) 7352 { 7353 struct btf_module *btf_mod, *tmp; 7354 struct module *mod = module; 7355 struct btf *btf; 7356 int err = 0; 7357 7358 if (mod->btf_data_size == 0 || 7359 (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && 7360 op != MODULE_STATE_GOING)) 7361 goto out; 7362 7363 switch (op) { 7364 case MODULE_STATE_COMING: 7365 btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL); 7366 if (!btf_mod) { 7367 err = -ENOMEM; 7368 goto out; 7369 } 7370 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size); 7371 if (IS_ERR(btf)) { 7372 kfree(btf_mod); 7373 if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) { 7374 pr_warn("failed to validate module [%s] BTF: %ld\n", 7375 mod->name, PTR_ERR(btf)); 7376 err = PTR_ERR(btf); 7377 } else { 7378 pr_warn_once("Kernel module BTF mismatch detected, BTF debug info may be unavailable for some modules\n"); 7379 } 7380 goto out; 7381 } 7382 err = btf_alloc_id(btf); 7383 if (err) { 7384 btf_free(btf); 7385 kfree(btf_mod); 7386 goto out; 7387 } 7388 7389 purge_cand_cache(NULL); 7390 mutex_lock(&btf_module_mutex); 7391 btf_mod->module = module; 7392 btf_mod->btf = btf; 7393 list_add(&btf_mod->list, &btf_modules); 7394 mutex_unlock(&btf_module_mutex); 7395 7396 if (IS_ENABLED(CONFIG_SYSFS)) { 7397 struct bin_attribute *attr; 7398 7399 attr = kzalloc(sizeof(*attr), GFP_KERNEL); 7400 if (!attr) 7401 goto out; 7402 7403 sysfs_bin_attr_init(attr); 7404 attr->attr.name = btf->name; 7405 attr->attr.mode = 0444; 7406 attr->size = btf->data_size; 7407 attr->private = btf; 7408 attr->read = btf_module_read; 7409 7410 err = sysfs_create_bin_file(btf_kobj, attr); 7411 if (err) { 7412 pr_warn("failed to register module [%s] BTF in sysfs: %d\n", 7413 mod->name, err); 7414 kfree(attr); 7415 err = 0; 7416 goto out; 7417 } 7418 7419 btf_mod->sysfs_attr = attr; 7420 } 7421 7422 break; 7423 case MODULE_STATE_LIVE: 7424 mutex_lock(&btf_module_mutex); 7425 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 7426 if (btf_mod->module != module) 7427 continue; 7428 7429 btf_mod->flags |= BTF_MODULE_F_LIVE; 7430 break; 7431 } 7432 mutex_unlock(&btf_module_mutex); 7433 break; 7434 case MODULE_STATE_GOING: 7435 mutex_lock(&btf_module_mutex); 7436 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 7437 if (btf_mod->module != module) 7438 continue; 7439 7440 list_del(&btf_mod->list); 7441 if (btf_mod->sysfs_attr) 7442 sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); 7443 purge_cand_cache(btf_mod->btf); 7444 btf_put(btf_mod->btf); 7445 kfree(btf_mod->sysfs_attr); 7446 kfree(btf_mod); 7447 break; 7448 } 7449 mutex_unlock(&btf_module_mutex); 7450 break; 7451 } 7452 out: 7453 return notifier_from_errno(err); 7454 } 7455 7456 static struct notifier_block btf_module_nb = { 7457 .notifier_call = btf_module_notify, 7458 }; 7459 7460 static int __init btf_module_init(void) 7461 { 7462 register_module_notifier(&btf_module_nb); 7463 return 0; 7464 } 7465 7466 fs_initcall(btf_module_init); 7467 #endif /* CONFIG_DEBUG_INFO_BTF_MODULES */ 7468 7469 struct module *btf_try_get_module(const struct btf *btf) 7470 { 7471 struct module *res = NULL; 7472 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 7473 struct btf_module *btf_mod, *tmp; 7474 7475 mutex_lock(&btf_module_mutex); 7476 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 7477 if (btf_mod->btf != btf) 7478 continue; 7479 7480 /* We must only consider module whose __init routine has 7481 * finished, hence we must check for BTF_MODULE_F_LIVE flag, 7482 * which is set from the notifier callback for 7483 * MODULE_STATE_LIVE. 7484 */ 7485 if ((btf_mod->flags & BTF_MODULE_F_LIVE) && try_module_get(btf_mod->module)) 7486 res = btf_mod->module; 7487 7488 break; 7489 } 7490 mutex_unlock(&btf_module_mutex); 7491 #endif 7492 7493 return res; 7494 } 7495 7496 /* Returns struct btf corresponding to the struct module. 7497 * This function can return NULL or ERR_PTR. 7498 */ 7499 static struct btf *btf_get_module_btf(const struct module *module) 7500 { 7501 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 7502 struct btf_module *btf_mod, *tmp; 7503 #endif 7504 struct btf *btf = NULL; 7505 7506 if (!module) { 7507 btf = bpf_get_btf_vmlinux(); 7508 if (!IS_ERR_OR_NULL(btf)) 7509 btf_get(btf); 7510 return btf; 7511 } 7512 7513 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 7514 mutex_lock(&btf_module_mutex); 7515 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 7516 if (btf_mod->module != module) 7517 continue; 7518 7519 btf_get(btf_mod->btf); 7520 btf = btf_mod->btf; 7521 break; 7522 } 7523 mutex_unlock(&btf_module_mutex); 7524 #endif 7525 7526 return btf; 7527 } 7528 7529 BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags) 7530 { 7531 struct btf *btf = NULL; 7532 int btf_obj_fd = 0; 7533 long ret; 7534 7535 if (flags) 7536 return -EINVAL; 7537 7538 if (name_sz <= 1 || name[name_sz - 1]) 7539 return -EINVAL; 7540 7541 ret = bpf_find_btf_id(name, kind, &btf); 7542 if (ret > 0 && btf_is_module(btf)) { 7543 btf_obj_fd = __btf_new_fd(btf); 7544 if (btf_obj_fd < 0) { 7545 btf_put(btf); 7546 return btf_obj_fd; 7547 } 7548 return ret | (((u64)btf_obj_fd) << 32); 7549 } 7550 if (ret > 0) 7551 btf_put(btf); 7552 return ret; 7553 } 7554 7555 const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { 7556 .func = bpf_btf_find_by_name_kind, 7557 .gpl_only = false, 7558 .ret_type = RET_INTEGER, 7559 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, 7560 .arg2_type = ARG_CONST_SIZE, 7561 .arg3_type = ARG_ANYTHING, 7562 .arg4_type = ARG_ANYTHING, 7563 }; 7564 7565 BTF_ID_LIST_GLOBAL(btf_tracing_ids, MAX_BTF_TRACING_TYPE) 7566 #define BTF_TRACING_TYPE(name, type) BTF_ID(struct, type) 7567 BTF_TRACING_TYPE_xxx 7568 #undef BTF_TRACING_TYPE 7569 7570 static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, 7571 const struct btf_type *func, u32 func_flags) 7572 { 7573 u32 flags = func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); 7574 const char *name, *sfx, *iter_name; 7575 const struct btf_param *arg; 7576 const struct btf_type *t; 7577 char exp_name[128]; 7578 u32 nr_args; 7579 7580 /* exactly one of KF_ITER_{NEW,NEXT,DESTROY} can be set */ 7581 if (!flags || (flags & (flags - 1))) 7582 return -EINVAL; 7583 7584 /* any BPF iter kfunc should have `struct bpf_iter_<type> *` first arg */ 7585 nr_args = btf_type_vlen(func); 7586 if (nr_args < 1) 7587 return -EINVAL; 7588 7589 arg = &btf_params(func)[0]; 7590 t = btf_type_skip_modifiers(btf, arg->type, NULL); 7591 if (!t || !btf_type_is_ptr(t)) 7592 return -EINVAL; 7593 t = btf_type_skip_modifiers(btf, t->type, NULL); 7594 if (!t || !__btf_type_is_struct(t)) 7595 return -EINVAL; 7596 7597 name = btf_name_by_offset(btf, t->name_off); 7598 if (!name || strncmp(name, ITER_PREFIX, sizeof(ITER_PREFIX) - 1)) 7599 return -EINVAL; 7600 7601 /* sizeof(struct bpf_iter_<type>) should be a multiple of 8 to 7602 * fit nicely in stack slots 7603 */ 7604 if (t->size == 0 || (t->size % 8)) 7605 return -EINVAL; 7606 7607 /* validate bpf_iter_<type>_{new,next,destroy}(struct bpf_iter_<type> *) 7608 * naming pattern 7609 */ 7610 iter_name = name + sizeof(ITER_PREFIX) - 1; 7611 if (flags & KF_ITER_NEW) 7612 sfx = "new"; 7613 else if (flags & KF_ITER_NEXT) 7614 sfx = "next"; 7615 else /* (flags & KF_ITER_DESTROY) */ 7616 sfx = "destroy"; 7617 7618 snprintf(exp_name, sizeof(exp_name), "bpf_iter_%s_%s", iter_name, sfx); 7619 if (strcmp(func_name, exp_name)) 7620 return -EINVAL; 7621 7622 /* only iter constructor should have extra arguments */ 7623 if (!(flags & KF_ITER_NEW) && nr_args != 1) 7624 return -EINVAL; 7625 7626 if (flags & KF_ITER_NEXT) { 7627 /* bpf_iter_<type>_next() should return pointer */ 7628 t = btf_type_skip_modifiers(btf, func->type, NULL); 7629 if (!t || !btf_type_is_ptr(t)) 7630 return -EINVAL; 7631 } 7632 7633 if (flags & KF_ITER_DESTROY) { 7634 /* bpf_iter_<type>_destroy() should return void */ 7635 t = btf_type_by_id(btf, func->type); 7636 if (!t || !btf_type_is_void(t)) 7637 return -EINVAL; 7638 } 7639 7640 return 0; 7641 } 7642 7643 static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) 7644 { 7645 const struct btf_type *func; 7646 const char *func_name; 7647 int err; 7648 7649 /* any kfunc should be FUNC -> FUNC_PROTO */ 7650 func = btf_type_by_id(btf, func_id); 7651 if (!func || !btf_type_is_func(func)) 7652 return -EINVAL; 7653 7654 /* sanity check kfunc name */ 7655 func_name = btf_name_by_offset(btf, func->name_off); 7656 if (!func_name || !func_name[0]) 7657 return -EINVAL; 7658 7659 func = btf_type_by_id(btf, func->type); 7660 if (!func || !btf_type_is_func_proto(func)) 7661 return -EINVAL; 7662 7663 if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)) { 7664 err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); 7665 if (err) 7666 return err; 7667 } 7668 7669 return 0; 7670 } 7671 7672 /* Kernel Function (kfunc) BTF ID set registration API */ 7673 7674 static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, 7675 const struct btf_kfunc_id_set *kset) 7676 { 7677 struct btf_kfunc_hook_filter *hook_filter; 7678 struct btf_id_set8 *add_set = kset->set; 7679 bool vmlinux_set = !btf_is_module(btf); 7680 bool add_filter = !!kset->filter; 7681 struct btf_kfunc_set_tab *tab; 7682 struct btf_id_set8 *set; 7683 u32 set_cnt; 7684 int ret; 7685 7686 if (hook >= BTF_KFUNC_HOOK_MAX) { 7687 ret = -EINVAL; 7688 goto end; 7689 } 7690 7691 if (!add_set->cnt) 7692 return 0; 7693 7694 tab = btf->kfunc_set_tab; 7695 7696 if (tab && add_filter) { 7697 u32 i; 7698 7699 hook_filter = &tab->hook_filters[hook]; 7700 for (i = 0; i < hook_filter->nr_filters; i++) { 7701 if (hook_filter->filters[i] == kset->filter) { 7702 add_filter = false; 7703 break; 7704 } 7705 } 7706 7707 if (add_filter && hook_filter->nr_filters == BTF_KFUNC_FILTER_MAX_CNT) { 7708 ret = -E2BIG; 7709 goto end; 7710 } 7711 } 7712 7713 if (!tab) { 7714 tab = kzalloc(sizeof(*tab), GFP_KERNEL | __GFP_NOWARN); 7715 if (!tab) 7716 return -ENOMEM; 7717 btf->kfunc_set_tab = tab; 7718 } 7719 7720 set = tab->sets[hook]; 7721 /* Warn when register_btf_kfunc_id_set is called twice for the same hook 7722 * for module sets. 7723 */ 7724 if (WARN_ON_ONCE(set && !vmlinux_set)) { 7725 ret = -EINVAL; 7726 goto end; 7727 } 7728 7729 /* We don't need to allocate, concatenate, and sort module sets, because 7730 * only one is allowed per hook. Hence, we can directly assign the 7731 * pointer and return. 7732 */ 7733 if (!vmlinux_set) { 7734 tab->sets[hook] = add_set; 7735 goto do_add_filter; 7736 } 7737 7738 /* In case of vmlinux sets, there may be more than one set being 7739 * registered per hook. To create a unified set, we allocate a new set 7740 * and concatenate all individual sets being registered. While each set 7741 * is individually sorted, they may become unsorted when concatenated, 7742 * hence re-sorting the final set again is required to make binary 7743 * searching the set using btf_id_set8_contains function work. 7744 */ 7745 set_cnt = set ? set->cnt : 0; 7746 7747 if (set_cnt > U32_MAX - add_set->cnt) { 7748 ret = -EOVERFLOW; 7749 goto end; 7750 } 7751 7752 if (set_cnt + add_set->cnt > BTF_KFUNC_SET_MAX_CNT) { 7753 ret = -E2BIG; 7754 goto end; 7755 } 7756 7757 /* Grow set */ 7758 set = krealloc(tab->sets[hook], 7759 offsetof(struct btf_id_set8, pairs[set_cnt + add_set->cnt]), 7760 GFP_KERNEL | __GFP_NOWARN); 7761 if (!set) { 7762 ret = -ENOMEM; 7763 goto end; 7764 } 7765 7766 /* For newly allocated set, initialize set->cnt to 0 */ 7767 if (!tab->sets[hook]) 7768 set->cnt = 0; 7769 tab->sets[hook] = set; 7770 7771 /* Concatenate the two sets */ 7772 memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0])); 7773 set->cnt += add_set->cnt; 7774 7775 sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL); 7776 7777 do_add_filter: 7778 if (add_filter) { 7779 hook_filter = &tab->hook_filters[hook]; 7780 hook_filter->filters[hook_filter->nr_filters++] = kset->filter; 7781 } 7782 return 0; 7783 end: 7784 btf_free_kfunc_set_tab(btf); 7785 return ret; 7786 } 7787 7788 static u32 *__btf_kfunc_id_set_contains(const struct btf *btf, 7789 enum btf_kfunc_hook hook, 7790 u32 kfunc_btf_id, 7791 const struct bpf_prog *prog) 7792 { 7793 struct btf_kfunc_hook_filter *hook_filter; 7794 struct btf_id_set8 *set; 7795 u32 *id, i; 7796 7797 if (hook >= BTF_KFUNC_HOOK_MAX) 7798 return NULL; 7799 if (!btf->kfunc_set_tab) 7800 return NULL; 7801 hook_filter = &btf->kfunc_set_tab->hook_filters[hook]; 7802 for (i = 0; i < hook_filter->nr_filters; i++) { 7803 if (hook_filter->filters[i](prog, kfunc_btf_id)) 7804 return NULL; 7805 } 7806 set = btf->kfunc_set_tab->sets[hook]; 7807 if (!set) 7808 return NULL; 7809 id = btf_id_set8_contains(set, kfunc_btf_id); 7810 if (!id) 7811 return NULL; 7812 /* The flags for BTF ID are located next to it */ 7813 return id + 1; 7814 } 7815 7816 static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) 7817 { 7818 switch (prog_type) { 7819 case BPF_PROG_TYPE_UNSPEC: 7820 return BTF_KFUNC_HOOK_COMMON; 7821 case BPF_PROG_TYPE_XDP: 7822 return BTF_KFUNC_HOOK_XDP; 7823 case BPF_PROG_TYPE_SCHED_CLS: 7824 return BTF_KFUNC_HOOK_TC; 7825 case BPF_PROG_TYPE_STRUCT_OPS: 7826 return BTF_KFUNC_HOOK_STRUCT_OPS; 7827 case BPF_PROG_TYPE_TRACING: 7828 case BPF_PROG_TYPE_LSM: 7829 return BTF_KFUNC_HOOK_TRACING; 7830 case BPF_PROG_TYPE_SYSCALL: 7831 return BTF_KFUNC_HOOK_SYSCALL; 7832 case BPF_PROG_TYPE_CGROUP_SKB: 7833 return BTF_KFUNC_HOOK_CGROUP_SKB; 7834 case BPF_PROG_TYPE_SCHED_ACT: 7835 return BTF_KFUNC_HOOK_SCHED_ACT; 7836 case BPF_PROG_TYPE_SK_SKB: 7837 return BTF_KFUNC_HOOK_SK_SKB; 7838 case BPF_PROG_TYPE_SOCKET_FILTER: 7839 return BTF_KFUNC_HOOK_SOCKET_FILTER; 7840 case BPF_PROG_TYPE_LWT_OUT: 7841 case BPF_PROG_TYPE_LWT_IN: 7842 case BPF_PROG_TYPE_LWT_XMIT: 7843 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 7844 return BTF_KFUNC_HOOK_LWT; 7845 case BPF_PROG_TYPE_NETFILTER: 7846 return BTF_KFUNC_HOOK_NETFILTER; 7847 default: 7848 return BTF_KFUNC_HOOK_MAX; 7849 } 7850 } 7851 7852 /* Caution: 7853 * Reference to the module (obtained using btf_try_get_module) corresponding to 7854 * the struct btf *MUST* be held when calling this function from verifier 7855 * context. This is usually true as we stash references in prog's kfunc_btf_tab; 7856 * keeping the reference for the duration of the call provides the necessary 7857 * protection for looking up a well-formed btf->kfunc_set_tab. 7858 */ 7859 u32 *btf_kfunc_id_set_contains(const struct btf *btf, 7860 u32 kfunc_btf_id, 7861 const struct bpf_prog *prog) 7862 { 7863 enum bpf_prog_type prog_type = resolve_prog_type(prog); 7864 enum btf_kfunc_hook hook; 7865 u32 *kfunc_flags; 7866 7867 kfunc_flags = __btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id, prog); 7868 if (kfunc_flags) 7869 return kfunc_flags; 7870 7871 hook = bpf_prog_type_to_kfunc_hook(prog_type); 7872 return __btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id, prog); 7873 } 7874 7875 u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id, 7876 const struct bpf_prog *prog) 7877 { 7878 return __btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id, prog); 7879 } 7880 7881 static int __register_btf_kfunc_id_set(enum btf_kfunc_hook hook, 7882 const struct btf_kfunc_id_set *kset) 7883 { 7884 struct btf *btf; 7885 int ret, i; 7886 7887 btf = btf_get_module_btf(kset->owner); 7888 if (!btf) { 7889 if (!kset->owner && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 7890 pr_err("missing vmlinux BTF, cannot register kfuncs\n"); 7891 return -ENOENT; 7892 } 7893 if (kset->owner && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) { 7894 pr_err("missing module BTF, cannot register kfuncs\n"); 7895 return -ENOENT; 7896 } 7897 return 0; 7898 } 7899 if (IS_ERR(btf)) 7900 return PTR_ERR(btf); 7901 7902 for (i = 0; i < kset->set->cnt; i++) { 7903 ret = btf_check_kfunc_protos(btf, kset->set->pairs[i].id, 7904 kset->set->pairs[i].flags); 7905 if (ret) 7906 goto err_out; 7907 } 7908 7909 ret = btf_populate_kfunc_set(btf, hook, kset); 7910 7911 err_out: 7912 btf_put(btf); 7913 return ret; 7914 } 7915 7916 /* This function must be invoked only from initcalls/module init functions */ 7917 int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, 7918 const struct btf_kfunc_id_set *kset) 7919 { 7920 enum btf_kfunc_hook hook; 7921 7922 hook = bpf_prog_type_to_kfunc_hook(prog_type); 7923 return __register_btf_kfunc_id_set(hook, kset); 7924 } 7925 EXPORT_SYMBOL_GPL(register_btf_kfunc_id_set); 7926 7927 /* This function must be invoked only from initcalls/module init functions */ 7928 int register_btf_fmodret_id_set(const struct btf_kfunc_id_set *kset) 7929 { 7930 return __register_btf_kfunc_id_set(BTF_KFUNC_HOOK_FMODRET, kset); 7931 } 7932 EXPORT_SYMBOL_GPL(register_btf_fmodret_id_set); 7933 7934 s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id) 7935 { 7936 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; 7937 struct btf_id_dtor_kfunc *dtor; 7938 7939 if (!tab) 7940 return -ENOENT; 7941 /* Even though the size of tab->dtors[0] is > sizeof(u32), we only need 7942 * to compare the first u32 with btf_id, so we can reuse btf_id_cmp_func. 7943 */ 7944 BUILD_BUG_ON(offsetof(struct btf_id_dtor_kfunc, btf_id) != 0); 7945 dtor = bsearch(&btf_id, tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func); 7946 if (!dtor) 7947 return -ENOENT; 7948 return dtor->kfunc_btf_id; 7949 } 7950 7951 static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt) 7952 { 7953 const struct btf_type *dtor_func, *dtor_func_proto, *t; 7954 const struct btf_param *args; 7955 s32 dtor_btf_id; 7956 u32 nr_args, i; 7957 7958 for (i = 0; i < cnt; i++) { 7959 dtor_btf_id = dtors[i].kfunc_btf_id; 7960 7961 dtor_func = btf_type_by_id(btf, dtor_btf_id); 7962 if (!dtor_func || !btf_type_is_func(dtor_func)) 7963 return -EINVAL; 7964 7965 dtor_func_proto = btf_type_by_id(btf, dtor_func->type); 7966 if (!dtor_func_proto || !btf_type_is_func_proto(dtor_func_proto)) 7967 return -EINVAL; 7968 7969 /* Make sure the prototype of the destructor kfunc is 'void func(type *)' */ 7970 t = btf_type_by_id(btf, dtor_func_proto->type); 7971 if (!t || !btf_type_is_void(t)) 7972 return -EINVAL; 7973 7974 nr_args = btf_type_vlen(dtor_func_proto); 7975 if (nr_args != 1) 7976 return -EINVAL; 7977 args = btf_params(dtor_func_proto); 7978 t = btf_type_by_id(btf, args[0].type); 7979 /* Allow any pointer type, as width on targets Linux supports 7980 * will be same for all pointer types (i.e. sizeof(void *)) 7981 */ 7982 if (!t || !btf_type_is_ptr(t)) 7983 return -EINVAL; 7984 } 7985 return 0; 7986 } 7987 7988 /* This function must be invoked only from initcalls/module init functions */ 7989 int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt, 7990 struct module *owner) 7991 { 7992 struct btf_id_dtor_kfunc_tab *tab; 7993 struct btf *btf; 7994 u32 tab_cnt; 7995 int ret; 7996 7997 btf = btf_get_module_btf(owner); 7998 if (!btf) { 7999 if (!owner && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 8000 pr_err("missing vmlinux BTF, cannot register dtor kfuncs\n"); 8001 return -ENOENT; 8002 } 8003 if (owner && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) { 8004 pr_err("missing module BTF, cannot register dtor kfuncs\n"); 8005 return -ENOENT; 8006 } 8007 return 0; 8008 } 8009 if (IS_ERR(btf)) 8010 return PTR_ERR(btf); 8011 8012 if (add_cnt >= BTF_DTOR_KFUNC_MAX_CNT) { 8013 pr_err("cannot register more than %d kfunc destructors\n", BTF_DTOR_KFUNC_MAX_CNT); 8014 ret = -E2BIG; 8015 goto end; 8016 } 8017 8018 /* Ensure that the prototype of dtor kfuncs being registered is sane */ 8019 ret = btf_check_dtor_kfuncs(btf, dtors, add_cnt); 8020 if (ret < 0) 8021 goto end; 8022 8023 tab = btf->dtor_kfunc_tab; 8024 /* Only one call allowed for modules */ 8025 if (WARN_ON_ONCE(tab && btf_is_module(btf))) { 8026 ret = -EINVAL; 8027 goto end; 8028 } 8029 8030 tab_cnt = tab ? tab->cnt : 0; 8031 if (tab_cnt > U32_MAX - add_cnt) { 8032 ret = -EOVERFLOW; 8033 goto end; 8034 } 8035 if (tab_cnt + add_cnt >= BTF_DTOR_KFUNC_MAX_CNT) { 8036 pr_err("cannot register more than %d kfunc destructors\n", BTF_DTOR_KFUNC_MAX_CNT); 8037 ret = -E2BIG; 8038 goto end; 8039 } 8040 8041 tab = krealloc(btf->dtor_kfunc_tab, 8042 offsetof(struct btf_id_dtor_kfunc_tab, dtors[tab_cnt + add_cnt]), 8043 GFP_KERNEL | __GFP_NOWARN); 8044 if (!tab) { 8045 ret = -ENOMEM; 8046 goto end; 8047 } 8048 8049 if (!btf->dtor_kfunc_tab) 8050 tab->cnt = 0; 8051 btf->dtor_kfunc_tab = tab; 8052 8053 memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0])); 8054 tab->cnt += add_cnt; 8055 8056 sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL); 8057 8058 end: 8059 if (ret) 8060 btf_free_dtor_kfunc_tab(btf); 8061 btf_put(btf); 8062 return ret; 8063 } 8064 EXPORT_SYMBOL_GPL(register_btf_id_dtor_kfuncs); 8065 8066 #define MAX_TYPES_ARE_COMPAT_DEPTH 2 8067 8068 /* Check local and target types for compatibility. This check is used for 8069 * type-based CO-RE relocations and follow slightly different rules than 8070 * field-based relocations. This function assumes that root types were already 8071 * checked for name match. Beyond that initial root-level name check, names 8072 * are completely ignored. Compatibility rules are as follows: 8073 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but 8074 * kind should match for local and target types (i.e., STRUCT is not 8075 * compatible with UNION); 8076 * - for ENUMs/ENUM64s, the size is ignored; 8077 * - for INT, size and signedness are ignored; 8078 * - for ARRAY, dimensionality is ignored, element types are checked for 8079 * compatibility recursively; 8080 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 8081 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 8082 * - FUNC_PROTOs are compatible if they have compatible signature: same 8083 * number of input args and compatible return and argument types. 8084 * These rules are not set in stone and probably will be adjusted as we get 8085 * more experience with using BPF CO-RE relocations. 8086 */ 8087 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 8088 const struct btf *targ_btf, __u32 targ_id) 8089 { 8090 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 8091 MAX_TYPES_ARE_COMPAT_DEPTH); 8092 } 8093 8094 #define MAX_TYPES_MATCH_DEPTH 2 8095 8096 int bpf_core_types_match(const struct btf *local_btf, u32 local_id, 8097 const struct btf *targ_btf, u32 targ_id) 8098 { 8099 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 8100 MAX_TYPES_MATCH_DEPTH); 8101 } 8102 8103 static bool bpf_core_is_flavor_sep(const char *s) 8104 { 8105 /* check X___Y name pattern, where X and Y are not underscores */ 8106 return s[0] != '_' && /* X */ 8107 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 8108 s[4] != '_'; /* Y */ 8109 } 8110 8111 size_t bpf_core_essential_name_len(const char *name) 8112 { 8113 size_t n = strlen(name); 8114 int i; 8115 8116 for (i = n - 5; i >= 0; i--) { 8117 if (bpf_core_is_flavor_sep(name + i)) 8118 return i + 1; 8119 } 8120 return n; 8121 } 8122 8123 struct bpf_cand_cache { 8124 const char *name; 8125 u32 name_len; 8126 u16 kind; 8127 u16 cnt; 8128 struct { 8129 const struct btf *btf; 8130 u32 id; 8131 } cands[]; 8132 }; 8133 8134 static void bpf_free_cands(struct bpf_cand_cache *cands) 8135 { 8136 if (!cands->cnt) 8137 /* empty candidate array was allocated on stack */ 8138 return; 8139 kfree(cands); 8140 } 8141 8142 static void bpf_free_cands_from_cache(struct bpf_cand_cache *cands) 8143 { 8144 kfree(cands->name); 8145 kfree(cands); 8146 } 8147 8148 #define VMLINUX_CAND_CACHE_SIZE 31 8149 static struct bpf_cand_cache *vmlinux_cand_cache[VMLINUX_CAND_CACHE_SIZE]; 8150 8151 #define MODULE_CAND_CACHE_SIZE 31 8152 static struct bpf_cand_cache *module_cand_cache[MODULE_CAND_CACHE_SIZE]; 8153 8154 static DEFINE_MUTEX(cand_cache_mutex); 8155 8156 static void __print_cand_cache(struct bpf_verifier_log *log, 8157 struct bpf_cand_cache **cache, 8158 int cache_size) 8159 { 8160 struct bpf_cand_cache *cc; 8161 int i, j; 8162 8163 for (i = 0; i < cache_size; i++) { 8164 cc = cache[i]; 8165 if (!cc) 8166 continue; 8167 bpf_log(log, "[%d]%s(", i, cc->name); 8168 for (j = 0; j < cc->cnt; j++) { 8169 bpf_log(log, "%d", cc->cands[j].id); 8170 if (j < cc->cnt - 1) 8171 bpf_log(log, " "); 8172 } 8173 bpf_log(log, "), "); 8174 } 8175 } 8176 8177 static void print_cand_cache(struct bpf_verifier_log *log) 8178 { 8179 mutex_lock(&cand_cache_mutex); 8180 bpf_log(log, "vmlinux_cand_cache:"); 8181 __print_cand_cache(log, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE); 8182 bpf_log(log, "\nmodule_cand_cache:"); 8183 __print_cand_cache(log, module_cand_cache, MODULE_CAND_CACHE_SIZE); 8184 bpf_log(log, "\n"); 8185 mutex_unlock(&cand_cache_mutex); 8186 } 8187 8188 static u32 hash_cands(struct bpf_cand_cache *cands) 8189 { 8190 return jhash(cands->name, cands->name_len, 0); 8191 } 8192 8193 static struct bpf_cand_cache *check_cand_cache(struct bpf_cand_cache *cands, 8194 struct bpf_cand_cache **cache, 8195 int cache_size) 8196 { 8197 struct bpf_cand_cache *cc = cache[hash_cands(cands) % cache_size]; 8198 8199 if (cc && cc->name_len == cands->name_len && 8200 !strncmp(cc->name, cands->name, cands->name_len)) 8201 return cc; 8202 return NULL; 8203 } 8204 8205 static size_t sizeof_cands(int cnt) 8206 { 8207 return offsetof(struct bpf_cand_cache, cands[cnt]); 8208 } 8209 8210 static struct bpf_cand_cache *populate_cand_cache(struct bpf_cand_cache *cands, 8211 struct bpf_cand_cache **cache, 8212 int cache_size) 8213 { 8214 struct bpf_cand_cache **cc = &cache[hash_cands(cands) % cache_size], *new_cands; 8215 8216 if (*cc) { 8217 bpf_free_cands_from_cache(*cc); 8218 *cc = NULL; 8219 } 8220 new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL); 8221 if (!new_cands) { 8222 bpf_free_cands(cands); 8223 return ERR_PTR(-ENOMEM); 8224 } 8225 /* strdup the name, since it will stay in cache. 8226 * the cands->name points to strings in prog's BTF and the prog can be unloaded. 8227 */ 8228 new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL); 8229 bpf_free_cands(cands); 8230 if (!new_cands->name) { 8231 kfree(new_cands); 8232 return ERR_PTR(-ENOMEM); 8233 } 8234 *cc = new_cands; 8235 return new_cands; 8236 } 8237 8238 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 8239 static void __purge_cand_cache(struct btf *btf, struct bpf_cand_cache **cache, 8240 int cache_size) 8241 { 8242 struct bpf_cand_cache *cc; 8243 int i, j; 8244 8245 for (i = 0; i < cache_size; i++) { 8246 cc = cache[i]; 8247 if (!cc) 8248 continue; 8249 if (!btf) { 8250 /* when new module is loaded purge all of module_cand_cache, 8251 * since new module might have candidates with the name 8252 * that matches cached cands. 8253 */ 8254 bpf_free_cands_from_cache(cc); 8255 cache[i] = NULL; 8256 continue; 8257 } 8258 /* when module is unloaded purge cache entries 8259 * that match module's btf 8260 */ 8261 for (j = 0; j < cc->cnt; j++) 8262 if (cc->cands[j].btf == btf) { 8263 bpf_free_cands_from_cache(cc); 8264 cache[i] = NULL; 8265 break; 8266 } 8267 } 8268 8269 } 8270 8271 static void purge_cand_cache(struct btf *btf) 8272 { 8273 mutex_lock(&cand_cache_mutex); 8274 __purge_cand_cache(btf, module_cand_cache, MODULE_CAND_CACHE_SIZE); 8275 mutex_unlock(&cand_cache_mutex); 8276 } 8277 #endif 8278 8279 static struct bpf_cand_cache * 8280 bpf_core_add_cands(struct bpf_cand_cache *cands, const struct btf *targ_btf, 8281 int targ_start_id) 8282 { 8283 struct bpf_cand_cache *new_cands; 8284 const struct btf_type *t; 8285 const char *targ_name; 8286 size_t targ_essent_len; 8287 int n, i; 8288 8289 n = btf_nr_types(targ_btf); 8290 for (i = targ_start_id; i < n; i++) { 8291 t = btf_type_by_id(targ_btf, i); 8292 if (btf_kind(t) != cands->kind) 8293 continue; 8294 8295 targ_name = btf_name_by_offset(targ_btf, t->name_off); 8296 if (!targ_name) 8297 continue; 8298 8299 /* the resched point is before strncmp to make sure that search 8300 * for non-existing name will have a chance to schedule(). 8301 */ 8302 cond_resched(); 8303 8304 if (strncmp(cands->name, targ_name, cands->name_len) != 0) 8305 continue; 8306 8307 targ_essent_len = bpf_core_essential_name_len(targ_name); 8308 if (targ_essent_len != cands->name_len) 8309 continue; 8310 8311 /* most of the time there is only one candidate for a given kind+name pair */ 8312 new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL); 8313 if (!new_cands) { 8314 bpf_free_cands(cands); 8315 return ERR_PTR(-ENOMEM); 8316 } 8317 8318 memcpy(new_cands, cands, sizeof_cands(cands->cnt)); 8319 bpf_free_cands(cands); 8320 cands = new_cands; 8321 cands->cands[cands->cnt].btf = targ_btf; 8322 cands->cands[cands->cnt].id = i; 8323 cands->cnt++; 8324 } 8325 return cands; 8326 } 8327 8328 static struct bpf_cand_cache * 8329 bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id) 8330 { 8331 struct bpf_cand_cache *cands, *cc, local_cand = {}; 8332 const struct btf *local_btf = ctx->btf; 8333 const struct btf_type *local_type; 8334 const struct btf *main_btf; 8335 size_t local_essent_len; 8336 struct btf *mod_btf; 8337 const char *name; 8338 int id; 8339 8340 main_btf = bpf_get_btf_vmlinux(); 8341 if (IS_ERR(main_btf)) 8342 return ERR_CAST(main_btf); 8343 if (!main_btf) 8344 return ERR_PTR(-EINVAL); 8345 8346 local_type = btf_type_by_id(local_btf, local_type_id); 8347 if (!local_type) 8348 return ERR_PTR(-EINVAL); 8349 8350 name = btf_name_by_offset(local_btf, local_type->name_off); 8351 if (str_is_empty(name)) 8352 return ERR_PTR(-EINVAL); 8353 local_essent_len = bpf_core_essential_name_len(name); 8354 8355 cands = &local_cand; 8356 cands->name = name; 8357 cands->kind = btf_kind(local_type); 8358 cands->name_len = local_essent_len; 8359 8360 cc = check_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE); 8361 /* cands is a pointer to stack here */ 8362 if (cc) { 8363 if (cc->cnt) 8364 return cc; 8365 goto check_modules; 8366 } 8367 8368 /* Attempt to find target candidates in vmlinux BTF first */ 8369 cands = bpf_core_add_cands(cands, main_btf, 1); 8370 if (IS_ERR(cands)) 8371 return ERR_CAST(cands); 8372 8373 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 */ 8374 8375 /* populate cache even when cands->cnt == 0 */ 8376 cc = populate_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE); 8377 if (IS_ERR(cc)) 8378 return ERR_CAST(cc); 8379 8380 /* if vmlinux BTF has any candidate, don't go for module BTFs */ 8381 if (cc->cnt) 8382 return cc; 8383 8384 check_modules: 8385 /* cands is a pointer to stack here and cands->cnt == 0 */ 8386 cc = check_cand_cache(cands, module_cand_cache, MODULE_CAND_CACHE_SIZE); 8387 if (cc) 8388 /* if cache has it return it even if cc->cnt == 0 */ 8389 return cc; 8390 8391 /* If candidate is not found in vmlinux's BTF then search in module's BTFs */ 8392 spin_lock_bh(&btf_idr_lock); 8393 idr_for_each_entry(&btf_idr, mod_btf, id) { 8394 if (!btf_is_module(mod_btf)) 8395 continue; 8396 /* linear search could be slow hence unlock/lock 8397 * the IDR to avoiding holding it for too long 8398 */ 8399 btf_get(mod_btf); 8400 spin_unlock_bh(&btf_idr_lock); 8401 cands = bpf_core_add_cands(cands, mod_btf, btf_nr_types(main_btf)); 8402 btf_put(mod_btf); 8403 if (IS_ERR(cands)) 8404 return ERR_CAST(cands); 8405 spin_lock_bh(&btf_idr_lock); 8406 } 8407 spin_unlock_bh(&btf_idr_lock); 8408 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 8409 * or pointer to stack if cands->cnd == 0. 8410 * Copy it into the cache even when cands->cnt == 0 and 8411 * return the result. 8412 */ 8413 return populate_cand_cache(cands, module_cand_cache, MODULE_CAND_CACHE_SIZE); 8414 } 8415 8416 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, 8417 int relo_idx, void *insn) 8418 { 8419 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL; 8420 struct bpf_core_cand_list cands = {}; 8421 struct bpf_core_relo_res targ_res; 8422 struct bpf_core_spec *specs; 8423 int err; 8424 8425 /* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5" 8426 * into arrays of btf_ids of struct fields and array indices. 8427 */ 8428 specs = kcalloc(3, sizeof(*specs), GFP_KERNEL); 8429 if (!specs) 8430 return -ENOMEM; 8431 8432 if (need_cands) { 8433 struct bpf_cand_cache *cc; 8434 int i; 8435 8436 mutex_lock(&cand_cache_mutex); 8437 cc = bpf_core_find_cands(ctx, relo->type_id); 8438 if (IS_ERR(cc)) { 8439 bpf_log(ctx->log, "target candidate search failed for %d\n", 8440 relo->type_id); 8441 err = PTR_ERR(cc); 8442 goto out; 8443 } 8444 if (cc->cnt) { 8445 cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL); 8446 if (!cands.cands) { 8447 err = -ENOMEM; 8448 goto out; 8449 } 8450 } 8451 for (i = 0; i < cc->cnt; i++) { 8452 bpf_log(ctx->log, 8453 "CO-RE relocating %s %s: found target candidate [%d]\n", 8454 btf_kind_str[cc->kind], cc->name, cc->cands[i].id); 8455 cands.cands[i].btf = cc->cands[i].btf; 8456 cands.cands[i].id = cc->cands[i].id; 8457 } 8458 cands.len = cc->cnt; 8459 /* cand_cache_mutex needs to span the cache lookup and 8460 * copy of btf pointer into bpf_core_cand_list, 8461 * since module can be unloaded while bpf_core_calc_relo_insn 8462 * is working with module's btf. 8463 */ 8464 } 8465 8466 err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs, 8467 &targ_res); 8468 if (err) 8469 goto out; 8470 8471 err = bpf_core_patch_insn((void *)ctx->log, insn, relo->insn_off / 8, relo, relo_idx, 8472 &targ_res); 8473 8474 out: 8475 kfree(specs); 8476 if (need_cands) { 8477 kfree(cands.cands); 8478 mutex_unlock(&cand_cache_mutex); 8479 if (ctx->log->level & BPF_LOG_LEVEL2) 8480 print_cand_cache(ctx->log); 8481 } 8482 return err; 8483 } 8484 8485 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log, 8486 const struct bpf_reg_state *reg, 8487 const char *field_name, u32 btf_id, const char *suffix) 8488 { 8489 struct btf *btf = reg->btf; 8490 const struct btf_type *walk_type, *safe_type; 8491 const char *tname; 8492 char safe_tname[64]; 8493 long ret, safe_id; 8494 const struct btf_member *member; 8495 u32 i; 8496 8497 walk_type = btf_type_by_id(btf, reg->btf_id); 8498 if (!walk_type) 8499 return false; 8500 8501 tname = btf_name_by_offset(btf, walk_type->name_off); 8502 8503 ret = snprintf(safe_tname, sizeof(safe_tname), "%s%s", tname, suffix); 8504 if (ret < 0) 8505 return false; 8506 8507 safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info)); 8508 if (safe_id < 0) 8509 return false; 8510 8511 safe_type = btf_type_by_id(btf, safe_id); 8512 if (!safe_type) 8513 return false; 8514 8515 for_each_member(i, safe_type, member) { 8516 const char *m_name = __btf_name_by_offset(btf, member->name_off); 8517 const struct btf_type *mtype = btf_type_by_id(btf, member->type); 8518 u32 id; 8519 8520 if (!btf_type_is_ptr(mtype)) 8521 continue; 8522 8523 btf_type_skip_modifiers(btf, mtype->type, &id); 8524 /* If we match on both type and name, the field is considered trusted. */ 8525 if (btf_id == id && !strcmp(field_name, m_name)) 8526 return true; 8527 } 8528 8529 return false; 8530 } 8531 8532 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log, 8533 const struct btf *reg_btf, u32 reg_id, 8534 const struct btf *arg_btf, u32 arg_id) 8535 { 8536 const char *reg_name, *arg_name, *search_needle; 8537 const struct btf_type *reg_type, *arg_type; 8538 int reg_len, arg_len, cmp_len; 8539 size_t pattern_len = sizeof(NOCAST_ALIAS_SUFFIX) - sizeof(char); 8540 8541 reg_type = btf_type_by_id(reg_btf, reg_id); 8542 if (!reg_type) 8543 return false; 8544 8545 arg_type = btf_type_by_id(arg_btf, arg_id); 8546 if (!arg_type) 8547 return false; 8548 8549 reg_name = btf_name_by_offset(reg_btf, reg_type->name_off); 8550 arg_name = btf_name_by_offset(arg_btf, arg_type->name_off); 8551 8552 reg_len = strlen(reg_name); 8553 arg_len = strlen(arg_name); 8554 8555 /* Exactly one of the two type names may be suffixed with ___init, so 8556 * if the strings are the same size, they can't possibly be no-cast 8557 * aliases of one another. If you have two of the same type names, e.g. 8558 * they're both nf_conn___init, it would be improper to return true 8559 * because they are _not_ no-cast aliases, they are the same type. 8560 */ 8561 if (reg_len == arg_len) 8562 return false; 8563 8564 /* Either of the two names must be the other name, suffixed with ___init. */ 8565 if ((reg_len != arg_len + pattern_len) && 8566 (arg_len != reg_len + pattern_len)) 8567 return false; 8568 8569 if (reg_len < arg_len) { 8570 search_needle = strstr(arg_name, NOCAST_ALIAS_SUFFIX); 8571 cmp_len = reg_len; 8572 } else { 8573 search_needle = strstr(reg_name, NOCAST_ALIAS_SUFFIX); 8574 cmp_len = arg_len; 8575 } 8576 8577 if (!search_needle) 8578 return false; 8579 8580 /* ___init suffix must come at the end of the name */ 8581 if (*(search_needle + pattern_len) != '\0') 8582 return false; 8583 8584 return !strncmp(reg_name, arg_name, cmp_len); 8585 } 8586