1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/list.h> 35 #include <linux/limits.h> 36 #include <linux/perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <linux/version.h> 39 #include <sys/epoll.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <sys/stat.h> 43 #include <sys/types.h> 44 #include <sys/vfs.h> 45 #include <sys/utsname.h> 46 #include <sys/resource.h> 47 #include <libelf.h> 48 #include <gelf.h> 49 #include <zlib.h> 50 51 #include "libbpf.h" 52 #include "bpf.h" 53 #include "btf.h" 54 #include "str_error.h" 55 #include "libbpf_internal.h" 56 #include "hashmap.h" 57 58 #ifndef BPF_FS_MAGIC 59 #define BPF_FS_MAGIC 0xcafe4a11 60 #endif 61 62 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 63 64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 65 * compilation if user enables corresponding warning. Disable it explicitly. 66 */ 67 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 68 69 #define __printf(a, b) __attribute__((format(printf, a, b))) 70 71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 73 74 static int __base_pr(enum libbpf_print_level level, const char *format, 75 va_list args) 76 { 77 if (level == LIBBPF_DEBUG) 78 return 0; 79 80 return vfprintf(stderr, format, args); 81 } 82 83 static libbpf_print_fn_t __libbpf_pr = __base_pr; 84 85 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 86 { 87 libbpf_print_fn_t old_print_fn = __libbpf_pr; 88 89 __libbpf_pr = fn; 90 return old_print_fn; 91 } 92 93 __printf(2, 3) 94 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 95 { 96 va_list args; 97 98 if (!__libbpf_pr) 99 return; 100 101 va_start(args, format); 102 __libbpf_pr(level, format, args); 103 va_end(args); 104 } 105 106 static void pr_perm_msg(int err) 107 { 108 struct rlimit limit; 109 char buf[100]; 110 111 if (err != -EPERM || geteuid() != 0) 112 return; 113 114 err = getrlimit(RLIMIT_MEMLOCK, &limit); 115 if (err) 116 return; 117 118 if (limit.rlim_cur == RLIM_INFINITY) 119 return; 120 121 if (limit.rlim_cur < 1024) 122 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 123 else if (limit.rlim_cur < 1024*1024) 124 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 125 else 126 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 127 128 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 129 buf); 130 } 131 132 #define STRERR_BUFSIZE 128 133 134 /* Copied from tools/perf/util/util.h */ 135 #ifndef zfree 136 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 137 #endif 138 139 #ifndef zclose 140 # define zclose(fd) ({ \ 141 int ___err = 0; \ 142 if ((fd) >= 0) \ 143 ___err = close((fd)); \ 144 fd = -1; \ 145 ___err; }) 146 #endif 147 148 static inline __u64 ptr_to_u64(const void *ptr) 149 { 150 return (__u64) (unsigned long) ptr; 151 } 152 153 enum kern_feature_id { 154 /* v4.14: kernel support for program & map names. */ 155 FEAT_PROG_NAME, 156 /* v5.2: kernel support for global data sections. */ 157 FEAT_GLOBAL_DATA, 158 /* BTF support */ 159 FEAT_BTF, 160 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 161 FEAT_BTF_FUNC, 162 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 163 FEAT_BTF_DATASEC, 164 /* BTF_FUNC_GLOBAL is supported */ 165 FEAT_BTF_GLOBAL_FUNC, 166 /* BPF_F_MMAPABLE is supported for arrays */ 167 FEAT_ARRAY_MMAP, 168 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 169 FEAT_EXP_ATTACH_TYPE, 170 /* bpf_probe_read_{kernel,user}[_str] helpers */ 171 FEAT_PROBE_READ_KERN, 172 /* BPF_PROG_BIND_MAP is supported */ 173 FEAT_PROG_BIND_MAP, 174 /* Kernel support for module BTFs */ 175 FEAT_MODULE_BTF, 176 /* BTF_KIND_FLOAT support */ 177 FEAT_BTF_FLOAT, 178 __FEAT_CNT, 179 }; 180 181 static bool kernel_supports(enum kern_feature_id feat_id); 182 183 enum reloc_type { 184 RELO_LD64, 185 RELO_CALL, 186 RELO_DATA, 187 RELO_EXTERN_VAR, 188 RELO_EXTERN_FUNC, 189 RELO_SUBPROG_ADDR, 190 }; 191 192 struct reloc_desc { 193 enum reloc_type type; 194 int insn_idx; 195 int map_idx; 196 int sym_off; 197 }; 198 199 struct bpf_sec_def; 200 201 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec, 202 struct bpf_program *prog); 203 204 struct bpf_sec_def { 205 const char *sec; 206 size_t len; 207 enum bpf_prog_type prog_type; 208 enum bpf_attach_type expected_attach_type; 209 bool is_exp_attach_type_optional; 210 bool is_attachable; 211 bool is_attach_btf; 212 bool is_sleepable; 213 attach_fn_t attach_fn; 214 }; 215 216 /* 217 * bpf_prog should be a better name but it has been used in 218 * linux/filter.h. 219 */ 220 struct bpf_program { 221 const struct bpf_sec_def *sec_def; 222 char *sec_name; 223 size_t sec_idx; 224 /* this program's instruction offset (in number of instructions) 225 * within its containing ELF section 226 */ 227 size_t sec_insn_off; 228 /* number of original instructions in ELF section belonging to this 229 * program, not taking into account subprogram instructions possible 230 * appended later during relocation 231 */ 232 size_t sec_insn_cnt; 233 /* Offset (in number of instructions) of the start of instruction 234 * belonging to this BPF program within its containing main BPF 235 * program. For the entry-point (main) BPF program, this is always 236 * zero. For a sub-program, this gets reset before each of main BPF 237 * programs are processed and relocated and is used to determined 238 * whether sub-program was already appended to the main program, and 239 * if yes, at which instruction offset. 240 */ 241 size_t sub_insn_off; 242 243 char *name; 244 /* sec_name with / replaced by _; makes recursive pinning 245 * in bpf_object__pin_programs easier 246 */ 247 char *pin_name; 248 249 /* instructions that belong to BPF program; insns[0] is located at 250 * sec_insn_off instruction within its ELF section in ELF file, so 251 * when mapping ELF file instruction index to the local instruction, 252 * one needs to subtract sec_insn_off; and vice versa. 253 */ 254 struct bpf_insn *insns; 255 /* actual number of instruction in this BPF program's image; for 256 * entry-point BPF programs this includes the size of main program 257 * itself plus all the used sub-programs, appended at the end 258 */ 259 size_t insns_cnt; 260 261 struct reloc_desc *reloc_desc; 262 int nr_reloc; 263 int log_level; 264 265 struct { 266 int nr; 267 int *fds; 268 } instances; 269 bpf_program_prep_t preprocessor; 270 271 struct bpf_object *obj; 272 void *priv; 273 bpf_program_clear_priv_t clear_priv; 274 275 bool load; 276 bool mark_btf_static; 277 enum bpf_prog_type type; 278 enum bpf_attach_type expected_attach_type; 279 int prog_ifindex; 280 __u32 attach_btf_obj_fd; 281 __u32 attach_btf_id; 282 __u32 attach_prog_fd; 283 void *func_info; 284 __u32 func_info_rec_size; 285 __u32 func_info_cnt; 286 287 void *line_info; 288 __u32 line_info_rec_size; 289 __u32 line_info_cnt; 290 __u32 prog_flags; 291 }; 292 293 struct bpf_struct_ops { 294 const char *tname; 295 const struct btf_type *type; 296 struct bpf_program **progs; 297 __u32 *kern_func_off; 298 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 299 void *data; 300 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 301 * btf_vmlinux's format. 302 * struct bpf_struct_ops_tcp_congestion_ops { 303 * [... some other kernel fields ...] 304 * struct tcp_congestion_ops data; 305 * } 306 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 307 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 308 * from "data". 309 */ 310 void *kern_vdata; 311 __u32 type_id; 312 }; 313 314 #define DATA_SEC ".data" 315 #define BSS_SEC ".bss" 316 #define RODATA_SEC ".rodata" 317 #define KCONFIG_SEC ".kconfig" 318 #define KSYMS_SEC ".ksyms" 319 #define STRUCT_OPS_SEC ".struct_ops" 320 321 enum libbpf_map_type { 322 LIBBPF_MAP_UNSPEC, 323 LIBBPF_MAP_DATA, 324 LIBBPF_MAP_BSS, 325 LIBBPF_MAP_RODATA, 326 LIBBPF_MAP_KCONFIG, 327 }; 328 329 static const char * const libbpf_type_to_btf_name[] = { 330 [LIBBPF_MAP_DATA] = DATA_SEC, 331 [LIBBPF_MAP_BSS] = BSS_SEC, 332 [LIBBPF_MAP_RODATA] = RODATA_SEC, 333 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, 334 }; 335 336 struct bpf_map { 337 char *name; 338 int fd; 339 int sec_idx; 340 size_t sec_offset; 341 int map_ifindex; 342 int inner_map_fd; 343 struct bpf_map_def def; 344 __u32 numa_node; 345 __u32 btf_var_idx; 346 __u32 btf_key_type_id; 347 __u32 btf_value_type_id; 348 __u32 btf_vmlinux_value_type_id; 349 void *priv; 350 bpf_map_clear_priv_t clear_priv; 351 enum libbpf_map_type libbpf_type; 352 void *mmaped; 353 struct bpf_struct_ops *st_ops; 354 struct bpf_map *inner_map; 355 void **init_slots; 356 int init_slots_sz; 357 char *pin_path; 358 bool pinned; 359 bool reused; 360 }; 361 362 enum extern_type { 363 EXT_UNKNOWN, 364 EXT_KCFG, 365 EXT_KSYM, 366 }; 367 368 enum kcfg_type { 369 KCFG_UNKNOWN, 370 KCFG_CHAR, 371 KCFG_BOOL, 372 KCFG_INT, 373 KCFG_TRISTATE, 374 KCFG_CHAR_ARR, 375 }; 376 377 struct extern_desc { 378 enum extern_type type; 379 int sym_idx; 380 int btf_id; 381 int sec_btf_id; 382 const char *name; 383 bool is_set; 384 bool is_weak; 385 union { 386 struct { 387 enum kcfg_type type; 388 int sz; 389 int align; 390 int data_off; 391 bool is_signed; 392 } kcfg; 393 struct { 394 unsigned long long addr; 395 396 /* target btf_id of the corresponding kernel var. */ 397 int kernel_btf_obj_fd; 398 int kernel_btf_id; 399 400 /* local btf_id of the ksym extern's type. */ 401 __u32 type_id; 402 } ksym; 403 }; 404 }; 405 406 static LIST_HEAD(bpf_objects_list); 407 408 struct module_btf { 409 struct btf *btf; 410 char *name; 411 __u32 id; 412 int fd; 413 }; 414 415 struct bpf_object { 416 char name[BPF_OBJ_NAME_LEN]; 417 char license[64]; 418 __u32 kern_version; 419 420 struct bpf_program *programs; 421 size_t nr_programs; 422 struct bpf_map *maps; 423 size_t nr_maps; 424 size_t maps_cap; 425 426 char *kconfig; 427 struct extern_desc *externs; 428 int nr_extern; 429 int kconfig_map_idx; 430 int rodata_map_idx; 431 432 bool loaded; 433 bool has_subcalls; 434 435 /* 436 * Information when doing elf related work. Only valid if fd 437 * is valid. 438 */ 439 struct { 440 int fd; 441 const void *obj_buf; 442 size_t obj_buf_sz; 443 Elf *elf; 444 GElf_Ehdr ehdr; 445 Elf_Data *symbols; 446 Elf_Data *data; 447 Elf_Data *rodata; 448 Elf_Data *bss; 449 Elf_Data *st_ops_data; 450 size_t shstrndx; /* section index for section name strings */ 451 size_t strtabidx; 452 struct { 453 GElf_Shdr shdr; 454 Elf_Data *data; 455 } *reloc_sects; 456 int nr_reloc_sects; 457 int maps_shndx; 458 int btf_maps_shndx; 459 __u32 btf_maps_sec_btf_id; 460 int text_shndx; 461 int symbols_shndx; 462 int data_shndx; 463 int rodata_shndx; 464 int bss_shndx; 465 int st_ops_shndx; 466 } efile; 467 /* 468 * All loaded bpf_object is linked in a list, which is 469 * hidden to caller. bpf_objects__<func> handlers deal with 470 * all objects. 471 */ 472 struct list_head list; 473 474 struct btf *btf; 475 struct btf_ext *btf_ext; 476 477 /* Parse and load BTF vmlinux if any of the programs in the object need 478 * it at load time. 479 */ 480 struct btf *btf_vmlinux; 481 /* vmlinux BTF override for CO-RE relocations */ 482 struct btf *btf_vmlinux_override; 483 /* Lazily initialized kernel module BTFs */ 484 struct module_btf *btf_modules; 485 bool btf_modules_loaded; 486 size_t btf_module_cnt; 487 size_t btf_module_cap; 488 489 void *priv; 490 bpf_object_clear_priv_t clear_priv; 491 492 char path[]; 493 }; 494 #define obj_elf_valid(o) ((o)->efile.elf) 495 496 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 497 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 498 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 499 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 500 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr); 501 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 502 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 503 504 void bpf_program__unload(struct bpf_program *prog) 505 { 506 int i; 507 508 if (!prog) 509 return; 510 511 /* 512 * If the object is opened but the program was never loaded, 513 * it is possible that prog->instances.nr == -1. 514 */ 515 if (prog->instances.nr > 0) { 516 for (i = 0; i < prog->instances.nr; i++) 517 zclose(prog->instances.fds[i]); 518 } else if (prog->instances.nr != -1) { 519 pr_warn("Internal error: instances.nr is %d\n", 520 prog->instances.nr); 521 } 522 523 prog->instances.nr = -1; 524 zfree(&prog->instances.fds); 525 526 zfree(&prog->func_info); 527 zfree(&prog->line_info); 528 } 529 530 static void bpf_program__exit(struct bpf_program *prog) 531 { 532 if (!prog) 533 return; 534 535 if (prog->clear_priv) 536 prog->clear_priv(prog, prog->priv); 537 538 prog->priv = NULL; 539 prog->clear_priv = NULL; 540 541 bpf_program__unload(prog); 542 zfree(&prog->name); 543 zfree(&prog->sec_name); 544 zfree(&prog->pin_name); 545 zfree(&prog->insns); 546 zfree(&prog->reloc_desc); 547 548 prog->nr_reloc = 0; 549 prog->insns_cnt = 0; 550 prog->sec_idx = -1; 551 } 552 553 static char *__bpf_program__pin_name(struct bpf_program *prog) 554 { 555 char *name, *p; 556 557 name = p = strdup(prog->sec_name); 558 while ((p = strchr(p, '/'))) 559 *p = '_'; 560 561 return name; 562 } 563 564 static bool insn_is_subprog_call(const struct bpf_insn *insn) 565 { 566 return BPF_CLASS(insn->code) == BPF_JMP && 567 BPF_OP(insn->code) == BPF_CALL && 568 BPF_SRC(insn->code) == BPF_K && 569 insn->src_reg == BPF_PSEUDO_CALL && 570 insn->dst_reg == 0 && 571 insn->off == 0; 572 } 573 574 static bool is_ldimm64_insn(struct bpf_insn *insn) 575 { 576 return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 577 } 578 579 static bool is_call_insn(const struct bpf_insn *insn) 580 { 581 return insn->code == (BPF_JMP | BPF_CALL); 582 } 583 584 static bool insn_is_pseudo_func(struct bpf_insn *insn) 585 { 586 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 587 } 588 589 static int 590 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 591 const char *name, size_t sec_idx, const char *sec_name, 592 size_t sec_off, void *insn_data, size_t insn_data_sz) 593 { 594 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 595 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 596 sec_name, name, sec_off, insn_data_sz); 597 return -EINVAL; 598 } 599 600 memset(prog, 0, sizeof(*prog)); 601 prog->obj = obj; 602 603 prog->sec_idx = sec_idx; 604 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 605 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 606 /* insns_cnt can later be increased by appending used subprograms */ 607 prog->insns_cnt = prog->sec_insn_cnt; 608 609 prog->type = BPF_PROG_TYPE_UNSPEC; 610 prog->load = true; 611 612 prog->instances.fds = NULL; 613 prog->instances.nr = -1; 614 615 prog->sec_name = strdup(sec_name); 616 if (!prog->sec_name) 617 goto errout; 618 619 prog->name = strdup(name); 620 if (!prog->name) 621 goto errout; 622 623 prog->pin_name = __bpf_program__pin_name(prog); 624 if (!prog->pin_name) 625 goto errout; 626 627 prog->insns = malloc(insn_data_sz); 628 if (!prog->insns) 629 goto errout; 630 memcpy(prog->insns, insn_data, insn_data_sz); 631 632 return 0; 633 errout: 634 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 635 bpf_program__exit(prog); 636 return -ENOMEM; 637 } 638 639 static int 640 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 641 const char *sec_name, int sec_idx) 642 { 643 Elf_Data *symbols = obj->efile.symbols; 644 struct bpf_program *prog, *progs; 645 void *data = sec_data->d_buf; 646 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 647 int nr_progs, err, i; 648 const char *name; 649 GElf_Sym sym; 650 651 progs = obj->programs; 652 nr_progs = obj->nr_programs; 653 nr_syms = symbols->d_size / sizeof(GElf_Sym); 654 sec_off = 0; 655 656 for (i = 0; i < nr_syms; i++) { 657 if (!gelf_getsym(symbols, i, &sym)) 658 continue; 659 if (sym.st_shndx != sec_idx) 660 continue; 661 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC) 662 continue; 663 664 prog_sz = sym.st_size; 665 sec_off = sym.st_value; 666 667 name = elf_sym_str(obj, sym.st_name); 668 if (!name) { 669 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 670 sec_name, sec_off); 671 return -LIBBPF_ERRNO__FORMAT; 672 } 673 674 if (sec_off + prog_sz > sec_sz) { 675 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 676 sec_name, sec_off); 677 return -LIBBPF_ERRNO__FORMAT; 678 } 679 680 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 681 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 682 683 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 684 if (!progs) { 685 /* 686 * In this case the original obj->programs 687 * is still valid, so don't need special treat for 688 * bpf_close_object(). 689 */ 690 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 691 sec_name, name); 692 return -ENOMEM; 693 } 694 obj->programs = progs; 695 696 prog = &progs[nr_progs]; 697 698 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 699 sec_off, data + sec_off, prog_sz); 700 if (err) 701 return err; 702 703 /* if function is a global/weak symbol, but has hidden 704 * visibility (STV_HIDDEN), mark its BTF FUNC as static to 705 * enable more permissive BPF verification mode with more 706 * outside context available to BPF verifier 707 */ 708 if (GELF_ST_BIND(sym.st_info) != STB_LOCAL 709 && GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN) 710 prog->mark_btf_static = true; 711 712 nr_progs++; 713 obj->nr_programs = nr_progs; 714 } 715 716 return 0; 717 } 718 719 static __u32 get_kernel_version(void) 720 { 721 __u32 major, minor, patch; 722 struct utsname info; 723 724 uname(&info); 725 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) 726 return 0; 727 return KERNEL_VERSION(major, minor, patch); 728 } 729 730 static const struct btf_member * 731 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 732 { 733 struct btf_member *m; 734 int i; 735 736 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 737 if (btf_member_bit_offset(t, i) == bit_offset) 738 return m; 739 } 740 741 return NULL; 742 } 743 744 static const struct btf_member * 745 find_member_by_name(const struct btf *btf, const struct btf_type *t, 746 const char *name) 747 { 748 struct btf_member *m; 749 int i; 750 751 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 752 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 753 return m; 754 } 755 756 return NULL; 757 } 758 759 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 760 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 761 const char *name, __u32 kind); 762 763 static int 764 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 765 const struct btf_type **type, __u32 *type_id, 766 const struct btf_type **vtype, __u32 *vtype_id, 767 const struct btf_member **data_member) 768 { 769 const struct btf_type *kern_type, *kern_vtype; 770 const struct btf_member *kern_data_member; 771 __s32 kern_vtype_id, kern_type_id; 772 __u32 i; 773 774 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 775 if (kern_type_id < 0) { 776 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 777 tname); 778 return kern_type_id; 779 } 780 kern_type = btf__type_by_id(btf, kern_type_id); 781 782 /* Find the corresponding "map_value" type that will be used 783 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 784 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 785 * btf_vmlinux. 786 */ 787 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 788 tname, BTF_KIND_STRUCT); 789 if (kern_vtype_id < 0) { 790 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 791 STRUCT_OPS_VALUE_PREFIX, tname); 792 return kern_vtype_id; 793 } 794 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 795 796 /* Find "struct tcp_congestion_ops" from 797 * struct bpf_struct_ops_tcp_congestion_ops { 798 * [ ... ] 799 * struct tcp_congestion_ops data; 800 * } 801 */ 802 kern_data_member = btf_members(kern_vtype); 803 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 804 if (kern_data_member->type == kern_type_id) 805 break; 806 } 807 if (i == btf_vlen(kern_vtype)) { 808 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 809 tname, STRUCT_OPS_VALUE_PREFIX, tname); 810 return -EINVAL; 811 } 812 813 *type = kern_type; 814 *type_id = kern_type_id; 815 *vtype = kern_vtype; 816 *vtype_id = kern_vtype_id; 817 *data_member = kern_data_member; 818 819 return 0; 820 } 821 822 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 823 { 824 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 825 } 826 827 /* Init the map's fields that depend on kern_btf */ 828 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 829 const struct btf *btf, 830 const struct btf *kern_btf) 831 { 832 const struct btf_member *member, *kern_member, *kern_data_member; 833 const struct btf_type *type, *kern_type, *kern_vtype; 834 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 835 struct bpf_struct_ops *st_ops; 836 void *data, *kern_data; 837 const char *tname; 838 int err; 839 840 st_ops = map->st_ops; 841 type = st_ops->type; 842 tname = st_ops->tname; 843 err = find_struct_ops_kern_types(kern_btf, tname, 844 &kern_type, &kern_type_id, 845 &kern_vtype, &kern_vtype_id, 846 &kern_data_member); 847 if (err) 848 return err; 849 850 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 851 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 852 853 map->def.value_size = kern_vtype->size; 854 map->btf_vmlinux_value_type_id = kern_vtype_id; 855 856 st_ops->kern_vdata = calloc(1, kern_vtype->size); 857 if (!st_ops->kern_vdata) 858 return -ENOMEM; 859 860 data = st_ops->data; 861 kern_data_off = kern_data_member->offset / 8; 862 kern_data = st_ops->kern_vdata + kern_data_off; 863 864 member = btf_members(type); 865 for (i = 0; i < btf_vlen(type); i++, member++) { 866 const struct btf_type *mtype, *kern_mtype; 867 __u32 mtype_id, kern_mtype_id; 868 void *mdata, *kern_mdata; 869 __s64 msize, kern_msize; 870 __u32 moff, kern_moff; 871 __u32 kern_member_idx; 872 const char *mname; 873 874 mname = btf__name_by_offset(btf, member->name_off); 875 kern_member = find_member_by_name(kern_btf, kern_type, mname); 876 if (!kern_member) { 877 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 878 map->name, mname); 879 return -ENOTSUP; 880 } 881 882 kern_member_idx = kern_member - btf_members(kern_type); 883 if (btf_member_bitfield_size(type, i) || 884 btf_member_bitfield_size(kern_type, kern_member_idx)) { 885 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 886 map->name, mname); 887 return -ENOTSUP; 888 } 889 890 moff = member->offset / 8; 891 kern_moff = kern_member->offset / 8; 892 893 mdata = data + moff; 894 kern_mdata = kern_data + kern_moff; 895 896 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 897 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 898 &kern_mtype_id); 899 if (BTF_INFO_KIND(mtype->info) != 900 BTF_INFO_KIND(kern_mtype->info)) { 901 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 902 map->name, mname, BTF_INFO_KIND(mtype->info), 903 BTF_INFO_KIND(kern_mtype->info)); 904 return -ENOTSUP; 905 } 906 907 if (btf_is_ptr(mtype)) { 908 struct bpf_program *prog; 909 910 prog = st_ops->progs[i]; 911 if (!prog) 912 continue; 913 914 kern_mtype = skip_mods_and_typedefs(kern_btf, 915 kern_mtype->type, 916 &kern_mtype_id); 917 918 /* mtype->type must be a func_proto which was 919 * guaranteed in bpf_object__collect_st_ops_relos(), 920 * so only check kern_mtype for func_proto here. 921 */ 922 if (!btf_is_func_proto(kern_mtype)) { 923 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 924 map->name, mname); 925 return -ENOTSUP; 926 } 927 928 prog->attach_btf_id = kern_type_id; 929 prog->expected_attach_type = kern_member_idx; 930 931 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 932 933 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 934 map->name, mname, prog->name, moff, 935 kern_moff); 936 937 continue; 938 } 939 940 msize = btf__resolve_size(btf, mtype_id); 941 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 942 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 943 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 944 map->name, mname, (ssize_t)msize, 945 (ssize_t)kern_msize); 946 return -ENOTSUP; 947 } 948 949 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 950 map->name, mname, (unsigned int)msize, 951 moff, kern_moff); 952 memcpy(kern_mdata, mdata, msize); 953 } 954 955 return 0; 956 } 957 958 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 959 { 960 struct bpf_map *map; 961 size_t i; 962 int err; 963 964 for (i = 0; i < obj->nr_maps; i++) { 965 map = &obj->maps[i]; 966 967 if (!bpf_map__is_struct_ops(map)) 968 continue; 969 970 err = bpf_map__init_kern_struct_ops(map, obj->btf, 971 obj->btf_vmlinux); 972 if (err) 973 return err; 974 } 975 976 return 0; 977 } 978 979 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) 980 { 981 const struct btf_type *type, *datasec; 982 const struct btf_var_secinfo *vsi; 983 struct bpf_struct_ops *st_ops; 984 const char *tname, *var_name; 985 __s32 type_id, datasec_id; 986 const struct btf *btf; 987 struct bpf_map *map; 988 __u32 i; 989 990 if (obj->efile.st_ops_shndx == -1) 991 return 0; 992 993 btf = obj->btf; 994 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, 995 BTF_KIND_DATASEC); 996 if (datasec_id < 0) { 997 pr_warn("struct_ops init: DATASEC %s not found\n", 998 STRUCT_OPS_SEC); 999 return -EINVAL; 1000 } 1001 1002 datasec = btf__type_by_id(btf, datasec_id); 1003 vsi = btf_var_secinfos(datasec); 1004 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1005 type = btf__type_by_id(obj->btf, vsi->type); 1006 var_name = btf__name_by_offset(obj->btf, type->name_off); 1007 1008 type_id = btf__resolve_type(obj->btf, vsi->type); 1009 if (type_id < 0) { 1010 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1011 vsi->type, STRUCT_OPS_SEC); 1012 return -EINVAL; 1013 } 1014 1015 type = btf__type_by_id(obj->btf, type_id); 1016 tname = btf__name_by_offset(obj->btf, type->name_off); 1017 if (!tname[0]) { 1018 pr_warn("struct_ops init: anonymous type is not supported\n"); 1019 return -ENOTSUP; 1020 } 1021 if (!btf_is_struct(type)) { 1022 pr_warn("struct_ops init: %s is not a struct\n", tname); 1023 return -EINVAL; 1024 } 1025 1026 map = bpf_object__add_map(obj); 1027 if (IS_ERR(map)) 1028 return PTR_ERR(map); 1029 1030 map->sec_idx = obj->efile.st_ops_shndx; 1031 map->sec_offset = vsi->offset; 1032 map->name = strdup(var_name); 1033 if (!map->name) 1034 return -ENOMEM; 1035 1036 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1037 map->def.key_size = sizeof(int); 1038 map->def.value_size = type->size; 1039 map->def.max_entries = 1; 1040 1041 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1042 if (!map->st_ops) 1043 return -ENOMEM; 1044 st_ops = map->st_ops; 1045 st_ops->data = malloc(type->size); 1046 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1047 st_ops->kern_func_off = malloc(btf_vlen(type) * 1048 sizeof(*st_ops->kern_func_off)); 1049 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1050 return -ENOMEM; 1051 1052 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { 1053 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1054 var_name, STRUCT_OPS_SEC); 1055 return -EINVAL; 1056 } 1057 1058 memcpy(st_ops->data, 1059 obj->efile.st_ops_data->d_buf + vsi->offset, 1060 type->size); 1061 st_ops->tname = tname; 1062 st_ops->type = type; 1063 st_ops->type_id = type_id; 1064 1065 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1066 tname, type_id, var_name, vsi->offset); 1067 } 1068 1069 return 0; 1070 } 1071 1072 static struct bpf_object *bpf_object__new(const char *path, 1073 const void *obj_buf, 1074 size_t obj_buf_sz, 1075 const char *obj_name) 1076 { 1077 struct bpf_object *obj; 1078 char *end; 1079 1080 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1081 if (!obj) { 1082 pr_warn("alloc memory failed for %s\n", path); 1083 return ERR_PTR(-ENOMEM); 1084 } 1085 1086 strcpy(obj->path, path); 1087 if (obj_name) { 1088 strncpy(obj->name, obj_name, sizeof(obj->name) - 1); 1089 obj->name[sizeof(obj->name) - 1] = 0; 1090 } else { 1091 /* Using basename() GNU version which doesn't modify arg. */ 1092 strncpy(obj->name, basename((void *)path), 1093 sizeof(obj->name) - 1); 1094 end = strchr(obj->name, '.'); 1095 if (end) 1096 *end = 0; 1097 } 1098 1099 obj->efile.fd = -1; 1100 /* 1101 * Caller of this function should also call 1102 * bpf_object__elf_finish() after data collection to return 1103 * obj_buf to user. If not, we should duplicate the buffer to 1104 * avoid user freeing them before elf finish. 1105 */ 1106 obj->efile.obj_buf = obj_buf; 1107 obj->efile.obj_buf_sz = obj_buf_sz; 1108 obj->efile.maps_shndx = -1; 1109 obj->efile.btf_maps_shndx = -1; 1110 obj->efile.data_shndx = -1; 1111 obj->efile.rodata_shndx = -1; 1112 obj->efile.bss_shndx = -1; 1113 obj->efile.st_ops_shndx = -1; 1114 obj->kconfig_map_idx = -1; 1115 obj->rodata_map_idx = -1; 1116 1117 obj->kern_version = get_kernel_version(); 1118 obj->loaded = false; 1119 1120 INIT_LIST_HEAD(&obj->list); 1121 list_add(&obj->list, &bpf_objects_list); 1122 return obj; 1123 } 1124 1125 static void bpf_object__elf_finish(struct bpf_object *obj) 1126 { 1127 if (!obj_elf_valid(obj)) 1128 return; 1129 1130 if (obj->efile.elf) { 1131 elf_end(obj->efile.elf); 1132 obj->efile.elf = NULL; 1133 } 1134 obj->efile.symbols = NULL; 1135 obj->efile.data = NULL; 1136 obj->efile.rodata = NULL; 1137 obj->efile.bss = NULL; 1138 obj->efile.st_ops_data = NULL; 1139 1140 zfree(&obj->efile.reloc_sects); 1141 obj->efile.nr_reloc_sects = 0; 1142 zclose(obj->efile.fd); 1143 obj->efile.obj_buf = NULL; 1144 obj->efile.obj_buf_sz = 0; 1145 } 1146 1147 static int bpf_object__elf_init(struct bpf_object *obj) 1148 { 1149 int err = 0; 1150 GElf_Ehdr *ep; 1151 1152 if (obj_elf_valid(obj)) { 1153 pr_warn("elf: init internal error\n"); 1154 return -LIBBPF_ERRNO__LIBELF; 1155 } 1156 1157 if (obj->efile.obj_buf_sz > 0) { 1158 /* 1159 * obj_buf should have been validated by 1160 * bpf_object__open_buffer(). 1161 */ 1162 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, 1163 obj->efile.obj_buf_sz); 1164 } else { 1165 obj->efile.fd = open(obj->path, O_RDONLY); 1166 if (obj->efile.fd < 0) { 1167 char errmsg[STRERR_BUFSIZE], *cp; 1168 1169 err = -errno; 1170 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1171 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1172 return err; 1173 } 1174 1175 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1176 } 1177 1178 if (!obj->efile.elf) { 1179 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1180 err = -LIBBPF_ERRNO__LIBELF; 1181 goto errout; 1182 } 1183 1184 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 1185 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1186 err = -LIBBPF_ERRNO__FORMAT; 1187 goto errout; 1188 } 1189 ep = &obj->efile.ehdr; 1190 1191 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) { 1192 pr_warn("elf: failed to get section names section index for %s: %s\n", 1193 obj->path, elf_errmsg(-1)); 1194 err = -LIBBPF_ERRNO__FORMAT; 1195 goto errout; 1196 } 1197 1198 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1199 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) { 1200 pr_warn("elf: failed to get section names strings from %s: %s\n", 1201 obj->path, elf_errmsg(-1)); 1202 err = -LIBBPF_ERRNO__FORMAT; 1203 goto errout; 1204 } 1205 1206 /* Old LLVM set e_machine to EM_NONE */ 1207 if (ep->e_type != ET_REL || 1208 (ep->e_machine && ep->e_machine != EM_BPF)) { 1209 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1210 err = -LIBBPF_ERRNO__FORMAT; 1211 goto errout; 1212 } 1213 1214 return 0; 1215 errout: 1216 bpf_object__elf_finish(obj); 1217 return err; 1218 } 1219 1220 static int bpf_object__check_endianness(struct bpf_object *obj) 1221 { 1222 #if __BYTE_ORDER == __LITTLE_ENDIAN 1223 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 1224 return 0; 1225 #elif __BYTE_ORDER == __BIG_ENDIAN 1226 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 1227 return 0; 1228 #else 1229 # error "Unrecognized __BYTE_ORDER__" 1230 #endif 1231 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1232 return -LIBBPF_ERRNO__ENDIAN; 1233 } 1234 1235 static int 1236 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1237 { 1238 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 1239 pr_debug("license of %s is %s\n", obj->path, obj->license); 1240 return 0; 1241 } 1242 1243 static int 1244 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1245 { 1246 __u32 kver; 1247 1248 if (size != sizeof(kver)) { 1249 pr_warn("invalid kver section in %s\n", obj->path); 1250 return -LIBBPF_ERRNO__FORMAT; 1251 } 1252 memcpy(&kver, data, sizeof(kver)); 1253 obj->kern_version = kver; 1254 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1255 return 0; 1256 } 1257 1258 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1259 { 1260 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1261 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1262 return true; 1263 return false; 1264 } 1265 1266 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 1267 __u32 *size) 1268 { 1269 int ret = -ENOENT; 1270 1271 *size = 0; 1272 if (!name) { 1273 return -EINVAL; 1274 } else if (!strcmp(name, DATA_SEC)) { 1275 if (obj->efile.data) 1276 *size = obj->efile.data->d_size; 1277 } else if (!strcmp(name, BSS_SEC)) { 1278 if (obj->efile.bss) 1279 *size = obj->efile.bss->d_size; 1280 } else if (!strcmp(name, RODATA_SEC)) { 1281 if (obj->efile.rodata) 1282 *size = obj->efile.rodata->d_size; 1283 } else if (!strcmp(name, STRUCT_OPS_SEC)) { 1284 if (obj->efile.st_ops_data) 1285 *size = obj->efile.st_ops_data->d_size; 1286 } else { 1287 Elf_Scn *scn = elf_sec_by_name(obj, name); 1288 Elf_Data *data = elf_sec_data(obj, scn); 1289 1290 if (data) { 1291 ret = 0; /* found it */ 1292 *size = data->d_size; 1293 } 1294 } 1295 1296 return *size ? 0 : ret; 1297 } 1298 1299 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 1300 __u32 *off) 1301 { 1302 Elf_Data *symbols = obj->efile.symbols; 1303 const char *sname; 1304 size_t si; 1305 1306 if (!name || !off) 1307 return -EINVAL; 1308 1309 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 1310 GElf_Sym sym; 1311 1312 if (!gelf_getsym(symbols, si, &sym)) 1313 continue; 1314 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 1315 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 1316 continue; 1317 1318 sname = elf_sym_str(obj, sym.st_name); 1319 if (!sname) { 1320 pr_warn("failed to get sym name string for var %s\n", 1321 name); 1322 return -EIO; 1323 } 1324 if (strcmp(name, sname) == 0) { 1325 *off = sym.st_value; 1326 return 0; 1327 } 1328 } 1329 1330 return -ENOENT; 1331 } 1332 1333 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1334 { 1335 struct bpf_map *new_maps; 1336 size_t new_cap; 1337 int i; 1338 1339 if (obj->nr_maps < obj->maps_cap) 1340 return &obj->maps[obj->nr_maps++]; 1341 1342 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 1343 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps)); 1344 if (!new_maps) { 1345 pr_warn("alloc maps for object failed\n"); 1346 return ERR_PTR(-ENOMEM); 1347 } 1348 1349 obj->maps_cap = new_cap; 1350 obj->maps = new_maps; 1351 1352 /* zero out new maps */ 1353 memset(obj->maps + obj->nr_maps, 0, 1354 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 1355 /* 1356 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 1357 * when failure (zclose won't close negative fd)). 1358 */ 1359 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 1360 obj->maps[i].fd = -1; 1361 obj->maps[i].inner_map_fd = -1; 1362 } 1363 1364 return &obj->maps[obj->nr_maps++]; 1365 } 1366 1367 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1368 { 1369 long page_sz = sysconf(_SC_PAGE_SIZE); 1370 size_t map_sz; 1371 1372 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; 1373 map_sz = roundup(map_sz, page_sz); 1374 return map_sz; 1375 } 1376 1377 static char *internal_map_name(struct bpf_object *obj, 1378 enum libbpf_map_type type) 1379 { 1380 char map_name[BPF_OBJ_NAME_LEN], *p; 1381 const char *sfx = libbpf_type_to_btf_name[type]; 1382 int sfx_len = max((size_t)7, strlen(sfx)); 1383 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, 1384 strlen(obj->name)); 1385 1386 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1387 sfx_len, libbpf_type_to_btf_name[type]); 1388 1389 /* sanitise map name to characters allowed by kernel */ 1390 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1391 if (!isalnum(*p) && *p != '_' && *p != '.') 1392 *p = '_'; 1393 1394 return strdup(map_name); 1395 } 1396 1397 static int 1398 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1399 int sec_idx, void *data, size_t data_sz) 1400 { 1401 struct bpf_map_def *def; 1402 struct bpf_map *map; 1403 int err; 1404 1405 map = bpf_object__add_map(obj); 1406 if (IS_ERR(map)) 1407 return PTR_ERR(map); 1408 1409 map->libbpf_type = type; 1410 map->sec_idx = sec_idx; 1411 map->sec_offset = 0; 1412 map->name = internal_map_name(obj, type); 1413 if (!map->name) { 1414 pr_warn("failed to alloc map name\n"); 1415 return -ENOMEM; 1416 } 1417 1418 def = &map->def; 1419 def->type = BPF_MAP_TYPE_ARRAY; 1420 def->key_size = sizeof(int); 1421 def->value_size = data_sz; 1422 def->max_entries = 1; 1423 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1424 ? BPF_F_RDONLY_PROG : 0; 1425 def->map_flags |= BPF_F_MMAPABLE; 1426 1427 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1428 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1429 1430 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 1431 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1432 if (map->mmaped == MAP_FAILED) { 1433 err = -errno; 1434 map->mmaped = NULL; 1435 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1436 map->name, err); 1437 zfree(&map->name); 1438 return err; 1439 } 1440 1441 if (data) 1442 memcpy(map->mmaped, data, data_sz); 1443 1444 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1445 return 0; 1446 } 1447 1448 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1449 { 1450 int err; 1451 1452 /* 1453 * Populate obj->maps with libbpf internal maps. 1454 */ 1455 if (obj->efile.data_shndx >= 0) { 1456 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1457 obj->efile.data_shndx, 1458 obj->efile.data->d_buf, 1459 obj->efile.data->d_size); 1460 if (err) 1461 return err; 1462 } 1463 if (obj->efile.rodata_shndx >= 0) { 1464 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1465 obj->efile.rodata_shndx, 1466 obj->efile.rodata->d_buf, 1467 obj->efile.rodata->d_size); 1468 if (err) 1469 return err; 1470 1471 obj->rodata_map_idx = obj->nr_maps - 1; 1472 } 1473 if (obj->efile.bss_shndx >= 0) { 1474 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1475 obj->efile.bss_shndx, 1476 NULL, 1477 obj->efile.bss->d_size); 1478 if (err) 1479 return err; 1480 } 1481 return 0; 1482 } 1483 1484 1485 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1486 const void *name) 1487 { 1488 int i; 1489 1490 for (i = 0; i < obj->nr_extern; i++) { 1491 if (strcmp(obj->externs[i].name, name) == 0) 1492 return &obj->externs[i]; 1493 } 1494 return NULL; 1495 } 1496 1497 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1498 char value) 1499 { 1500 switch (ext->kcfg.type) { 1501 case KCFG_BOOL: 1502 if (value == 'm') { 1503 pr_warn("extern (kcfg) %s=%c should be tristate or char\n", 1504 ext->name, value); 1505 return -EINVAL; 1506 } 1507 *(bool *)ext_val = value == 'y' ? true : false; 1508 break; 1509 case KCFG_TRISTATE: 1510 if (value == 'y') 1511 *(enum libbpf_tristate *)ext_val = TRI_YES; 1512 else if (value == 'm') 1513 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1514 else /* value == 'n' */ 1515 *(enum libbpf_tristate *)ext_val = TRI_NO; 1516 break; 1517 case KCFG_CHAR: 1518 *(char *)ext_val = value; 1519 break; 1520 case KCFG_UNKNOWN: 1521 case KCFG_INT: 1522 case KCFG_CHAR_ARR: 1523 default: 1524 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", 1525 ext->name, value); 1526 return -EINVAL; 1527 } 1528 ext->is_set = true; 1529 return 0; 1530 } 1531 1532 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1533 const char *value) 1534 { 1535 size_t len; 1536 1537 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1538 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); 1539 return -EINVAL; 1540 } 1541 1542 len = strlen(value); 1543 if (value[len - 1] != '"') { 1544 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1545 ext->name, value); 1546 return -EINVAL; 1547 } 1548 1549 /* strip quotes */ 1550 len -= 2; 1551 if (len >= ext->kcfg.sz) { 1552 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", 1553 ext->name, value, len, ext->kcfg.sz - 1); 1554 len = ext->kcfg.sz - 1; 1555 } 1556 memcpy(ext_val, value + 1, len); 1557 ext_val[len] = '\0'; 1558 ext->is_set = true; 1559 return 0; 1560 } 1561 1562 static int parse_u64(const char *value, __u64 *res) 1563 { 1564 char *value_end; 1565 int err; 1566 1567 errno = 0; 1568 *res = strtoull(value, &value_end, 0); 1569 if (errno) { 1570 err = -errno; 1571 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1572 return err; 1573 } 1574 if (*value_end) { 1575 pr_warn("failed to parse '%s' as integer completely\n", value); 1576 return -EINVAL; 1577 } 1578 return 0; 1579 } 1580 1581 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1582 { 1583 int bit_sz = ext->kcfg.sz * 8; 1584 1585 if (ext->kcfg.sz == 8) 1586 return true; 1587 1588 /* Validate that value stored in u64 fits in integer of `ext->sz` 1589 * bytes size without any loss of information. If the target integer 1590 * is signed, we rely on the following limits of integer type of 1591 * Y bits and subsequent transformation: 1592 * 1593 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1594 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1595 * 0 <= X + 2^(Y-1) < 2^Y 1596 * 1597 * For unsigned target integer, check that all the (64 - Y) bits are 1598 * zero. 1599 */ 1600 if (ext->kcfg.is_signed) 1601 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1602 else 1603 return (v >> bit_sz) == 0; 1604 } 1605 1606 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1607 __u64 value) 1608 { 1609 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 1610 pr_warn("extern (kcfg) %s=%llu should be integer\n", 1611 ext->name, (unsigned long long)value); 1612 return -EINVAL; 1613 } 1614 if (!is_kcfg_value_in_range(ext, value)) { 1615 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", 1616 ext->name, (unsigned long long)value, ext->kcfg.sz); 1617 return -ERANGE; 1618 } 1619 switch (ext->kcfg.sz) { 1620 case 1: *(__u8 *)ext_val = value; break; 1621 case 2: *(__u16 *)ext_val = value; break; 1622 case 4: *(__u32 *)ext_val = value; break; 1623 case 8: *(__u64 *)ext_val = value; break; 1624 default: 1625 return -EINVAL; 1626 } 1627 ext->is_set = true; 1628 return 0; 1629 } 1630 1631 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1632 char *buf, void *data) 1633 { 1634 struct extern_desc *ext; 1635 char *sep, *value; 1636 int len, err = 0; 1637 void *ext_val; 1638 __u64 num; 1639 1640 if (strncmp(buf, "CONFIG_", 7)) 1641 return 0; 1642 1643 sep = strchr(buf, '='); 1644 if (!sep) { 1645 pr_warn("failed to parse '%s': no separator\n", buf); 1646 return -EINVAL; 1647 } 1648 1649 /* Trim ending '\n' */ 1650 len = strlen(buf); 1651 if (buf[len - 1] == '\n') 1652 buf[len - 1] = '\0'; 1653 /* Split on '=' and ensure that a value is present. */ 1654 *sep = '\0'; 1655 if (!sep[1]) { 1656 *sep = '='; 1657 pr_warn("failed to parse '%s': no value\n", buf); 1658 return -EINVAL; 1659 } 1660 1661 ext = find_extern_by_name(obj, buf); 1662 if (!ext || ext->is_set) 1663 return 0; 1664 1665 ext_val = data + ext->kcfg.data_off; 1666 value = sep + 1; 1667 1668 switch (*value) { 1669 case 'y': case 'n': case 'm': 1670 err = set_kcfg_value_tri(ext, ext_val, *value); 1671 break; 1672 case '"': 1673 err = set_kcfg_value_str(ext, ext_val, value); 1674 break; 1675 default: 1676 /* assume integer */ 1677 err = parse_u64(value, &num); 1678 if (err) { 1679 pr_warn("extern (kcfg) %s=%s should be integer\n", 1680 ext->name, value); 1681 return err; 1682 } 1683 err = set_kcfg_value_num(ext, ext_val, num); 1684 break; 1685 } 1686 if (err) 1687 return err; 1688 pr_debug("extern (kcfg) %s=%s\n", ext->name, value); 1689 return 0; 1690 } 1691 1692 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1693 { 1694 char buf[PATH_MAX]; 1695 struct utsname uts; 1696 int len, err = 0; 1697 gzFile file; 1698 1699 uname(&uts); 1700 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1701 if (len < 0) 1702 return -EINVAL; 1703 else if (len >= PATH_MAX) 1704 return -ENAMETOOLONG; 1705 1706 /* gzopen also accepts uncompressed files. */ 1707 file = gzopen(buf, "r"); 1708 if (!file) 1709 file = gzopen("/proc/config.gz", "r"); 1710 1711 if (!file) { 1712 pr_warn("failed to open system Kconfig\n"); 1713 return -ENOENT; 1714 } 1715 1716 while (gzgets(file, buf, sizeof(buf))) { 1717 err = bpf_object__process_kconfig_line(obj, buf, data); 1718 if (err) { 1719 pr_warn("error parsing system Kconfig line '%s': %d\n", 1720 buf, err); 1721 goto out; 1722 } 1723 } 1724 1725 out: 1726 gzclose(file); 1727 return err; 1728 } 1729 1730 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 1731 const char *config, void *data) 1732 { 1733 char buf[PATH_MAX]; 1734 int err = 0; 1735 FILE *file; 1736 1737 file = fmemopen((void *)config, strlen(config), "r"); 1738 if (!file) { 1739 err = -errno; 1740 pr_warn("failed to open in-memory Kconfig: %d\n", err); 1741 return err; 1742 } 1743 1744 while (fgets(buf, sizeof(buf), file)) { 1745 err = bpf_object__process_kconfig_line(obj, buf, data); 1746 if (err) { 1747 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 1748 buf, err); 1749 break; 1750 } 1751 } 1752 1753 fclose(file); 1754 return err; 1755 } 1756 1757 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 1758 { 1759 struct extern_desc *last_ext = NULL, *ext; 1760 size_t map_sz; 1761 int i, err; 1762 1763 for (i = 0; i < obj->nr_extern; i++) { 1764 ext = &obj->externs[i]; 1765 if (ext->type == EXT_KCFG) 1766 last_ext = ext; 1767 } 1768 1769 if (!last_ext) 1770 return 0; 1771 1772 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 1773 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 1774 obj->efile.symbols_shndx, 1775 NULL, map_sz); 1776 if (err) 1777 return err; 1778 1779 obj->kconfig_map_idx = obj->nr_maps - 1; 1780 1781 return 0; 1782 } 1783 1784 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 1785 { 1786 Elf_Data *symbols = obj->efile.symbols; 1787 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 1788 Elf_Data *data = NULL; 1789 Elf_Scn *scn; 1790 1791 if (obj->efile.maps_shndx < 0) 1792 return 0; 1793 1794 if (!symbols) 1795 return -EINVAL; 1796 1797 1798 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx); 1799 data = elf_sec_data(obj, scn); 1800 if (!scn || !data) { 1801 pr_warn("elf: failed to get legacy map definitions for %s\n", 1802 obj->path); 1803 return -EINVAL; 1804 } 1805 1806 /* 1807 * Count number of maps. Each map has a name. 1808 * Array of maps is not supported: only the first element is 1809 * considered. 1810 * 1811 * TODO: Detect array of map and report error. 1812 */ 1813 nr_syms = symbols->d_size / sizeof(GElf_Sym); 1814 for (i = 0; i < nr_syms; i++) { 1815 GElf_Sym sym; 1816 1817 if (!gelf_getsym(symbols, i, &sym)) 1818 continue; 1819 if (sym.st_shndx != obj->efile.maps_shndx) 1820 continue; 1821 nr_maps++; 1822 } 1823 /* Assume equally sized map definitions */ 1824 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n", 1825 nr_maps, data->d_size, obj->path); 1826 1827 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { 1828 pr_warn("elf: unable to determine legacy map definition size in %s\n", 1829 obj->path); 1830 return -EINVAL; 1831 } 1832 map_def_sz = data->d_size / nr_maps; 1833 1834 /* Fill obj->maps using data in "maps" section. */ 1835 for (i = 0; i < nr_syms; i++) { 1836 GElf_Sym sym; 1837 const char *map_name; 1838 struct bpf_map_def *def; 1839 struct bpf_map *map; 1840 1841 if (!gelf_getsym(symbols, i, &sym)) 1842 continue; 1843 if (sym.st_shndx != obj->efile.maps_shndx) 1844 continue; 1845 1846 map = bpf_object__add_map(obj); 1847 if (IS_ERR(map)) 1848 return PTR_ERR(map); 1849 1850 map_name = elf_sym_str(obj, sym.st_name); 1851 if (!map_name) { 1852 pr_warn("failed to get map #%d name sym string for obj %s\n", 1853 i, obj->path); 1854 return -LIBBPF_ERRNO__FORMAT; 1855 } 1856 1857 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1858 map->sec_idx = sym.st_shndx; 1859 map->sec_offset = sym.st_value; 1860 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 1861 map_name, map->sec_idx, map->sec_offset); 1862 if (sym.st_value + map_def_sz > data->d_size) { 1863 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", 1864 obj->path, map_name); 1865 return -EINVAL; 1866 } 1867 1868 map->name = strdup(map_name); 1869 if (!map->name) { 1870 pr_warn("failed to alloc map name\n"); 1871 return -ENOMEM; 1872 } 1873 pr_debug("map %d is \"%s\"\n", i, map->name); 1874 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 1875 /* 1876 * If the definition of the map in the object file fits in 1877 * bpf_map_def, copy it. Any extra fields in our version 1878 * of bpf_map_def will default to zero as a result of the 1879 * calloc above. 1880 */ 1881 if (map_def_sz <= sizeof(struct bpf_map_def)) { 1882 memcpy(&map->def, def, map_def_sz); 1883 } else { 1884 /* 1885 * Here the map structure being read is bigger than what 1886 * we expect, truncate if the excess bits are all zero. 1887 * If they are not zero, reject this map as 1888 * incompatible. 1889 */ 1890 char *b; 1891 1892 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1893 b < ((char *)def) + map_def_sz; b++) { 1894 if (*b != 0) { 1895 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", 1896 obj->path, map_name); 1897 if (strict) 1898 return -EINVAL; 1899 } 1900 } 1901 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1902 } 1903 } 1904 return 0; 1905 } 1906 1907 const struct btf_type * 1908 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 1909 { 1910 const struct btf_type *t = btf__type_by_id(btf, id); 1911 1912 if (res_id) 1913 *res_id = id; 1914 1915 while (btf_is_mod(t) || btf_is_typedef(t)) { 1916 if (res_id) 1917 *res_id = t->type; 1918 t = btf__type_by_id(btf, t->type); 1919 } 1920 1921 return t; 1922 } 1923 1924 static const struct btf_type * 1925 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 1926 { 1927 const struct btf_type *t; 1928 1929 t = skip_mods_and_typedefs(btf, id, NULL); 1930 if (!btf_is_ptr(t)) 1931 return NULL; 1932 1933 t = skip_mods_and_typedefs(btf, t->type, res_id); 1934 1935 return btf_is_func_proto(t) ? t : NULL; 1936 } 1937 1938 static const char *__btf_kind_str(__u16 kind) 1939 { 1940 switch (kind) { 1941 case BTF_KIND_UNKN: return "void"; 1942 case BTF_KIND_INT: return "int"; 1943 case BTF_KIND_PTR: return "ptr"; 1944 case BTF_KIND_ARRAY: return "array"; 1945 case BTF_KIND_STRUCT: return "struct"; 1946 case BTF_KIND_UNION: return "union"; 1947 case BTF_KIND_ENUM: return "enum"; 1948 case BTF_KIND_FWD: return "fwd"; 1949 case BTF_KIND_TYPEDEF: return "typedef"; 1950 case BTF_KIND_VOLATILE: return "volatile"; 1951 case BTF_KIND_CONST: return "const"; 1952 case BTF_KIND_RESTRICT: return "restrict"; 1953 case BTF_KIND_FUNC: return "func"; 1954 case BTF_KIND_FUNC_PROTO: return "func_proto"; 1955 case BTF_KIND_VAR: return "var"; 1956 case BTF_KIND_DATASEC: return "datasec"; 1957 case BTF_KIND_FLOAT: return "float"; 1958 default: return "unknown"; 1959 } 1960 } 1961 1962 const char *btf_kind_str(const struct btf_type *t) 1963 { 1964 return __btf_kind_str(btf_kind(t)); 1965 } 1966 1967 /* 1968 * Fetch integer attribute of BTF map definition. Such attributes are 1969 * represented using a pointer to an array, in which dimensionality of array 1970 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 1971 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 1972 * type definition, while using only sizeof(void *) space in ELF data section. 1973 */ 1974 static bool get_map_field_int(const char *map_name, const struct btf *btf, 1975 const struct btf_member *m, __u32 *res) 1976 { 1977 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 1978 const char *name = btf__name_by_offset(btf, m->name_off); 1979 const struct btf_array *arr_info; 1980 const struct btf_type *arr_t; 1981 1982 if (!btf_is_ptr(t)) { 1983 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 1984 map_name, name, btf_kind_str(t)); 1985 return false; 1986 } 1987 1988 arr_t = btf__type_by_id(btf, t->type); 1989 if (!arr_t) { 1990 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 1991 map_name, name, t->type); 1992 return false; 1993 } 1994 if (!btf_is_array(arr_t)) { 1995 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 1996 map_name, name, btf_kind_str(arr_t)); 1997 return false; 1998 } 1999 arr_info = btf_array(arr_t); 2000 *res = arr_info->nelems; 2001 return true; 2002 } 2003 2004 static int build_map_pin_path(struct bpf_map *map, const char *path) 2005 { 2006 char buf[PATH_MAX]; 2007 int len; 2008 2009 if (!path) 2010 path = "/sys/fs/bpf"; 2011 2012 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); 2013 if (len < 0) 2014 return -EINVAL; 2015 else if (len >= PATH_MAX) 2016 return -ENAMETOOLONG; 2017 2018 return bpf_map__set_pin_path(map, buf); 2019 } 2020 2021 int parse_btf_map_def(const char *map_name, struct btf *btf, 2022 const struct btf_type *def_t, bool strict, 2023 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2024 { 2025 const struct btf_type *t; 2026 const struct btf_member *m; 2027 bool is_inner = inner_def == NULL; 2028 int vlen, i; 2029 2030 vlen = btf_vlen(def_t); 2031 m = btf_members(def_t); 2032 for (i = 0; i < vlen; i++, m++) { 2033 const char *name = btf__name_by_offset(btf, m->name_off); 2034 2035 if (!name) { 2036 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2037 return -EINVAL; 2038 } 2039 if (strcmp(name, "type") == 0) { 2040 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2041 return -EINVAL; 2042 map_def->parts |= MAP_DEF_MAP_TYPE; 2043 } else if (strcmp(name, "max_entries") == 0) { 2044 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2045 return -EINVAL; 2046 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2047 } else if (strcmp(name, "map_flags") == 0) { 2048 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2049 return -EINVAL; 2050 map_def->parts |= MAP_DEF_MAP_FLAGS; 2051 } else if (strcmp(name, "numa_node") == 0) { 2052 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2053 return -EINVAL; 2054 map_def->parts |= MAP_DEF_NUMA_NODE; 2055 } else if (strcmp(name, "key_size") == 0) { 2056 __u32 sz; 2057 2058 if (!get_map_field_int(map_name, btf, m, &sz)) 2059 return -EINVAL; 2060 if (map_def->key_size && map_def->key_size != sz) { 2061 pr_warn("map '%s': conflicting key size %u != %u.\n", 2062 map_name, map_def->key_size, sz); 2063 return -EINVAL; 2064 } 2065 map_def->key_size = sz; 2066 map_def->parts |= MAP_DEF_KEY_SIZE; 2067 } else if (strcmp(name, "key") == 0) { 2068 __s64 sz; 2069 2070 t = btf__type_by_id(btf, m->type); 2071 if (!t) { 2072 pr_warn("map '%s': key type [%d] not found.\n", 2073 map_name, m->type); 2074 return -EINVAL; 2075 } 2076 if (!btf_is_ptr(t)) { 2077 pr_warn("map '%s': key spec is not PTR: %s.\n", 2078 map_name, btf_kind_str(t)); 2079 return -EINVAL; 2080 } 2081 sz = btf__resolve_size(btf, t->type); 2082 if (sz < 0) { 2083 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2084 map_name, t->type, (ssize_t)sz); 2085 return sz; 2086 } 2087 if (map_def->key_size && map_def->key_size != sz) { 2088 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2089 map_name, map_def->key_size, (ssize_t)sz); 2090 return -EINVAL; 2091 } 2092 map_def->key_size = sz; 2093 map_def->key_type_id = t->type; 2094 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2095 } else if (strcmp(name, "value_size") == 0) { 2096 __u32 sz; 2097 2098 if (!get_map_field_int(map_name, btf, m, &sz)) 2099 return -EINVAL; 2100 if (map_def->value_size && map_def->value_size != sz) { 2101 pr_warn("map '%s': conflicting value size %u != %u.\n", 2102 map_name, map_def->value_size, sz); 2103 return -EINVAL; 2104 } 2105 map_def->value_size = sz; 2106 map_def->parts |= MAP_DEF_VALUE_SIZE; 2107 } else if (strcmp(name, "value") == 0) { 2108 __s64 sz; 2109 2110 t = btf__type_by_id(btf, m->type); 2111 if (!t) { 2112 pr_warn("map '%s': value type [%d] not found.\n", 2113 map_name, m->type); 2114 return -EINVAL; 2115 } 2116 if (!btf_is_ptr(t)) { 2117 pr_warn("map '%s': value spec is not PTR: %s.\n", 2118 map_name, btf_kind_str(t)); 2119 return -EINVAL; 2120 } 2121 sz = btf__resolve_size(btf, t->type); 2122 if (sz < 0) { 2123 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2124 map_name, t->type, (ssize_t)sz); 2125 return sz; 2126 } 2127 if (map_def->value_size && map_def->value_size != sz) { 2128 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2129 map_name, map_def->value_size, (ssize_t)sz); 2130 return -EINVAL; 2131 } 2132 map_def->value_size = sz; 2133 map_def->value_type_id = t->type; 2134 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2135 } 2136 else if (strcmp(name, "values") == 0) { 2137 char inner_map_name[128]; 2138 int err; 2139 2140 if (is_inner) { 2141 pr_warn("map '%s': multi-level inner maps not supported.\n", 2142 map_name); 2143 return -ENOTSUP; 2144 } 2145 if (i != vlen - 1) { 2146 pr_warn("map '%s': '%s' member should be last.\n", 2147 map_name, name); 2148 return -EINVAL; 2149 } 2150 if (!bpf_map_type__is_map_in_map(map_def->map_type)) { 2151 pr_warn("map '%s': should be map-in-map.\n", 2152 map_name); 2153 return -ENOTSUP; 2154 } 2155 if (map_def->value_size && map_def->value_size != 4) { 2156 pr_warn("map '%s': conflicting value size %u != 4.\n", 2157 map_name, map_def->value_size); 2158 return -EINVAL; 2159 } 2160 map_def->value_size = 4; 2161 t = btf__type_by_id(btf, m->type); 2162 if (!t) { 2163 pr_warn("map '%s': map-in-map inner type [%d] not found.\n", 2164 map_name, m->type); 2165 return -EINVAL; 2166 } 2167 if (!btf_is_array(t) || btf_array(t)->nelems) { 2168 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n", 2169 map_name); 2170 return -EINVAL; 2171 } 2172 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2173 if (!btf_is_ptr(t)) { 2174 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2175 map_name, btf_kind_str(t)); 2176 return -EINVAL; 2177 } 2178 t = skip_mods_and_typedefs(btf, t->type, NULL); 2179 if (!btf_is_struct(t)) { 2180 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2181 map_name, btf_kind_str(t)); 2182 return -EINVAL; 2183 } 2184 2185 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2186 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2187 if (err) 2188 return err; 2189 2190 map_def->parts |= MAP_DEF_INNER_MAP; 2191 } else if (strcmp(name, "pinning") == 0) { 2192 __u32 val; 2193 2194 if (is_inner) { 2195 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2196 return -EINVAL; 2197 } 2198 if (!get_map_field_int(map_name, btf, m, &val)) 2199 return -EINVAL; 2200 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2201 pr_warn("map '%s': invalid pinning value %u.\n", 2202 map_name, val); 2203 return -EINVAL; 2204 } 2205 map_def->pinning = val; 2206 map_def->parts |= MAP_DEF_PINNING; 2207 } else { 2208 if (strict) { 2209 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2210 return -ENOTSUP; 2211 } 2212 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2213 } 2214 } 2215 2216 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2217 pr_warn("map '%s': map type isn't specified.\n", map_name); 2218 return -EINVAL; 2219 } 2220 2221 return 0; 2222 } 2223 2224 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2225 { 2226 map->def.type = def->map_type; 2227 map->def.key_size = def->key_size; 2228 map->def.value_size = def->value_size; 2229 map->def.max_entries = def->max_entries; 2230 map->def.map_flags = def->map_flags; 2231 2232 map->numa_node = def->numa_node; 2233 map->btf_key_type_id = def->key_type_id; 2234 map->btf_value_type_id = def->value_type_id; 2235 2236 if (def->parts & MAP_DEF_MAP_TYPE) 2237 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2238 2239 if (def->parts & MAP_DEF_KEY_TYPE) 2240 pr_debug("map '%s': found key [%u], sz = %u.\n", 2241 map->name, def->key_type_id, def->key_size); 2242 else if (def->parts & MAP_DEF_KEY_SIZE) 2243 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2244 2245 if (def->parts & MAP_DEF_VALUE_TYPE) 2246 pr_debug("map '%s': found value [%u], sz = %u.\n", 2247 map->name, def->value_type_id, def->value_size); 2248 else if (def->parts & MAP_DEF_VALUE_SIZE) 2249 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2250 2251 if (def->parts & MAP_DEF_MAX_ENTRIES) 2252 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2253 if (def->parts & MAP_DEF_MAP_FLAGS) 2254 pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags); 2255 if (def->parts & MAP_DEF_PINNING) 2256 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2257 if (def->parts & MAP_DEF_NUMA_NODE) 2258 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2259 2260 if (def->parts & MAP_DEF_INNER_MAP) 2261 pr_debug("map '%s': found inner map definition.\n", map->name); 2262 } 2263 2264 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2265 const struct btf_type *sec, 2266 int var_idx, int sec_idx, 2267 const Elf_Data *data, bool strict, 2268 const char *pin_root_path) 2269 { 2270 struct btf_map_def map_def = {}, inner_def = {}; 2271 const struct btf_type *var, *def; 2272 const struct btf_var_secinfo *vi; 2273 const struct btf_var *var_extra; 2274 const char *map_name; 2275 struct bpf_map *map; 2276 int err; 2277 2278 vi = btf_var_secinfos(sec) + var_idx; 2279 var = btf__type_by_id(obj->btf, vi->type); 2280 var_extra = btf_var(var); 2281 map_name = btf__name_by_offset(obj->btf, var->name_off); 2282 2283 if (map_name == NULL || map_name[0] == '\0') { 2284 pr_warn("map #%d: empty name.\n", var_idx); 2285 return -EINVAL; 2286 } 2287 if ((__u64)vi->offset + vi->size > data->d_size) { 2288 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2289 return -EINVAL; 2290 } 2291 if (!btf_is_var(var)) { 2292 pr_warn("map '%s': unexpected var kind %s.\n", 2293 map_name, btf_kind_str(var)); 2294 return -EINVAL; 2295 } 2296 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && 2297 var_extra->linkage != BTF_VAR_STATIC) { 2298 pr_warn("map '%s': unsupported var linkage %u.\n", 2299 map_name, var_extra->linkage); 2300 return -EOPNOTSUPP; 2301 } 2302 2303 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2304 if (!btf_is_struct(def)) { 2305 pr_warn("map '%s': unexpected def kind %s.\n", 2306 map_name, btf_kind_str(var)); 2307 return -EINVAL; 2308 } 2309 if (def->size > vi->size) { 2310 pr_warn("map '%s': invalid def size.\n", map_name); 2311 return -EINVAL; 2312 } 2313 2314 map = bpf_object__add_map(obj); 2315 if (IS_ERR(map)) 2316 return PTR_ERR(map); 2317 map->name = strdup(map_name); 2318 if (!map->name) { 2319 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2320 return -ENOMEM; 2321 } 2322 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2323 map->def.type = BPF_MAP_TYPE_UNSPEC; 2324 map->sec_idx = sec_idx; 2325 map->sec_offset = vi->offset; 2326 map->btf_var_idx = var_idx; 2327 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2328 map_name, map->sec_idx, map->sec_offset); 2329 2330 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2331 if (err) 2332 return err; 2333 2334 fill_map_from_def(map, &map_def); 2335 2336 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2337 err = build_map_pin_path(map, pin_root_path); 2338 if (err) { 2339 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2340 return err; 2341 } 2342 } 2343 2344 if (map_def.parts & MAP_DEF_INNER_MAP) { 2345 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2346 if (!map->inner_map) 2347 return -ENOMEM; 2348 map->inner_map->fd = -1; 2349 map->inner_map->sec_idx = sec_idx; 2350 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2351 if (!map->inner_map->name) 2352 return -ENOMEM; 2353 sprintf(map->inner_map->name, "%s.inner", map_name); 2354 2355 fill_map_from_def(map->inner_map, &inner_def); 2356 } 2357 2358 return 0; 2359 } 2360 2361 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2362 const char *pin_root_path) 2363 { 2364 const struct btf_type *sec = NULL; 2365 int nr_types, i, vlen, err; 2366 const struct btf_type *t; 2367 const char *name; 2368 Elf_Data *data; 2369 Elf_Scn *scn; 2370 2371 if (obj->efile.btf_maps_shndx < 0) 2372 return 0; 2373 2374 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2375 data = elf_sec_data(obj, scn); 2376 if (!scn || !data) { 2377 pr_warn("elf: failed to get %s map definitions for %s\n", 2378 MAPS_ELF_SEC, obj->path); 2379 return -EINVAL; 2380 } 2381 2382 nr_types = btf__get_nr_types(obj->btf); 2383 for (i = 1; i <= nr_types; i++) { 2384 t = btf__type_by_id(obj->btf, i); 2385 if (!btf_is_datasec(t)) 2386 continue; 2387 name = btf__name_by_offset(obj->btf, t->name_off); 2388 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2389 sec = t; 2390 obj->efile.btf_maps_sec_btf_id = i; 2391 break; 2392 } 2393 } 2394 2395 if (!sec) { 2396 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2397 return -ENOENT; 2398 } 2399 2400 vlen = btf_vlen(sec); 2401 for (i = 0; i < vlen; i++) { 2402 err = bpf_object__init_user_btf_map(obj, sec, i, 2403 obj->efile.btf_maps_shndx, 2404 data, strict, 2405 pin_root_path); 2406 if (err) 2407 return err; 2408 } 2409 2410 return 0; 2411 } 2412 2413 static int bpf_object__init_maps(struct bpf_object *obj, 2414 const struct bpf_object_open_opts *opts) 2415 { 2416 const char *pin_root_path; 2417 bool strict; 2418 int err; 2419 2420 strict = !OPTS_GET(opts, relaxed_maps, false); 2421 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2422 2423 err = bpf_object__init_user_maps(obj, strict); 2424 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2425 err = err ?: bpf_object__init_global_data_maps(obj); 2426 err = err ?: bpf_object__init_kconfig_map(obj); 2427 err = err ?: bpf_object__init_struct_ops_maps(obj); 2428 if (err) 2429 return err; 2430 2431 return 0; 2432 } 2433 2434 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2435 { 2436 GElf_Shdr sh; 2437 2438 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh)) 2439 return false; 2440 2441 return sh.sh_flags & SHF_EXECINSTR; 2442 } 2443 2444 static bool btf_needs_sanitization(struct bpf_object *obj) 2445 { 2446 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC); 2447 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC); 2448 bool has_float = kernel_supports(FEAT_BTF_FLOAT); 2449 bool has_func = kernel_supports(FEAT_BTF_FUNC); 2450 2451 return !has_func || !has_datasec || !has_func_global || !has_float; 2452 } 2453 2454 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2455 { 2456 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC); 2457 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC); 2458 bool has_float = kernel_supports(FEAT_BTF_FLOAT); 2459 bool has_func = kernel_supports(FEAT_BTF_FUNC); 2460 struct btf_type *t; 2461 int i, j, vlen; 2462 2463 for (i = 1; i <= btf__get_nr_types(btf); i++) { 2464 t = (struct btf_type *)btf__type_by_id(btf, i); 2465 2466 if (!has_datasec && btf_is_var(t)) { 2467 /* replace VAR with INT */ 2468 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2469 /* 2470 * using size = 1 is the safest choice, 4 will be too 2471 * big and cause kernel BTF validation failure if 2472 * original variable took less than 4 bytes 2473 */ 2474 t->size = 1; 2475 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2476 } else if (!has_datasec && btf_is_datasec(t)) { 2477 /* replace DATASEC with STRUCT */ 2478 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2479 struct btf_member *m = btf_members(t); 2480 struct btf_type *vt; 2481 char *name; 2482 2483 name = (char *)btf__name_by_offset(btf, t->name_off); 2484 while (*name) { 2485 if (*name == '.') 2486 *name = '_'; 2487 name++; 2488 } 2489 2490 vlen = btf_vlen(t); 2491 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2492 for (j = 0; j < vlen; j++, v++, m++) { 2493 /* order of field assignments is important */ 2494 m->offset = v->offset * 8; 2495 m->type = v->type; 2496 /* preserve variable name as member name */ 2497 vt = (void *)btf__type_by_id(btf, v->type); 2498 m->name_off = vt->name_off; 2499 } 2500 } else if (!has_func && btf_is_func_proto(t)) { 2501 /* replace FUNC_PROTO with ENUM */ 2502 vlen = btf_vlen(t); 2503 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2504 t->size = sizeof(__u32); /* kernel enforced */ 2505 } else if (!has_func && btf_is_func(t)) { 2506 /* replace FUNC with TYPEDEF */ 2507 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2508 } else if (!has_func_global && btf_is_func(t)) { 2509 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2510 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2511 } else if (!has_float && btf_is_float(t)) { 2512 /* replace FLOAT with an equally-sized empty STRUCT; 2513 * since C compilers do not accept e.g. "float" as a 2514 * valid struct name, make it anonymous 2515 */ 2516 t->name_off = 0; 2517 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2518 } 2519 } 2520 } 2521 2522 static bool libbpf_needs_btf(const struct bpf_object *obj) 2523 { 2524 return obj->efile.btf_maps_shndx >= 0 || 2525 obj->efile.st_ops_shndx >= 0 || 2526 obj->nr_extern > 0; 2527 } 2528 2529 static bool kernel_needs_btf(const struct bpf_object *obj) 2530 { 2531 return obj->efile.st_ops_shndx >= 0; 2532 } 2533 2534 static int bpf_object__init_btf(struct bpf_object *obj, 2535 Elf_Data *btf_data, 2536 Elf_Data *btf_ext_data) 2537 { 2538 int err = -ENOENT; 2539 2540 if (btf_data) { 2541 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2542 if (IS_ERR(obj->btf)) { 2543 err = PTR_ERR(obj->btf); 2544 obj->btf = NULL; 2545 pr_warn("Error loading ELF section %s: %d.\n", 2546 BTF_ELF_SEC, err); 2547 goto out; 2548 } 2549 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2550 btf__set_pointer_size(obj->btf, 8); 2551 err = 0; 2552 } 2553 if (btf_ext_data) { 2554 if (!obj->btf) { 2555 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2556 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2557 goto out; 2558 } 2559 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 2560 btf_ext_data->d_size); 2561 if (IS_ERR(obj->btf_ext)) { 2562 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n", 2563 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); 2564 obj->btf_ext = NULL; 2565 goto out; 2566 } 2567 } 2568 out: 2569 if (err && libbpf_needs_btf(obj)) { 2570 pr_warn("BTF is required, but is missing or corrupted.\n"); 2571 return err; 2572 } 2573 return 0; 2574 } 2575 2576 static int bpf_object__finalize_btf(struct bpf_object *obj) 2577 { 2578 int err; 2579 2580 if (!obj->btf) 2581 return 0; 2582 2583 err = btf__finalize_data(obj, obj->btf); 2584 if (err) { 2585 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); 2586 return err; 2587 } 2588 2589 return 0; 2590 } 2591 2592 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 2593 { 2594 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 2595 prog->type == BPF_PROG_TYPE_LSM) 2596 return true; 2597 2598 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 2599 * also need vmlinux BTF 2600 */ 2601 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 2602 return true; 2603 2604 return false; 2605 } 2606 2607 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 2608 { 2609 struct bpf_program *prog; 2610 int i; 2611 2612 /* CO-RE relocations need kernel BTF */ 2613 if (obj->btf_ext && obj->btf_ext->core_relo_info.len) 2614 return true; 2615 2616 /* Support for typed ksyms needs kernel BTF */ 2617 for (i = 0; i < obj->nr_extern; i++) { 2618 const struct extern_desc *ext; 2619 2620 ext = &obj->externs[i]; 2621 if (ext->type == EXT_KSYM && ext->ksym.type_id) 2622 return true; 2623 } 2624 2625 bpf_object__for_each_program(prog, obj) { 2626 if (!prog->load) 2627 continue; 2628 if (prog_needs_vmlinux_btf(prog)) 2629 return true; 2630 } 2631 2632 return false; 2633 } 2634 2635 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 2636 { 2637 int err; 2638 2639 /* btf_vmlinux could be loaded earlier */ 2640 if (obj->btf_vmlinux) 2641 return 0; 2642 2643 if (!force && !obj_needs_vmlinux_btf(obj)) 2644 return 0; 2645 2646 obj->btf_vmlinux = libbpf_find_kernel_btf(); 2647 if (IS_ERR(obj->btf_vmlinux)) { 2648 err = PTR_ERR(obj->btf_vmlinux); 2649 pr_warn("Error loading vmlinux BTF: %d\n", err); 2650 obj->btf_vmlinux = NULL; 2651 return err; 2652 } 2653 return 0; 2654 } 2655 2656 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 2657 { 2658 struct btf *kern_btf = obj->btf; 2659 bool btf_mandatory, sanitize; 2660 int i, err = 0; 2661 2662 if (!obj->btf) 2663 return 0; 2664 2665 if (!kernel_supports(FEAT_BTF)) { 2666 if (kernel_needs_btf(obj)) { 2667 err = -EOPNOTSUPP; 2668 goto report; 2669 } 2670 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 2671 return 0; 2672 } 2673 2674 /* Even though some subprogs are global/weak, user might prefer more 2675 * permissive BPF verification process that BPF verifier performs for 2676 * static functions, taking into account more context from the caller 2677 * functions. In such case, they need to mark such subprogs with 2678 * __attribute__((visibility("hidden"))) and libbpf will adjust 2679 * corresponding FUNC BTF type to be marked as static and trigger more 2680 * involved BPF verification process. 2681 */ 2682 for (i = 0; i < obj->nr_programs; i++) { 2683 struct bpf_program *prog = &obj->programs[i]; 2684 struct btf_type *t; 2685 const char *name; 2686 int j, n; 2687 2688 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 2689 continue; 2690 2691 n = btf__get_nr_types(obj->btf); 2692 for (j = 1; j <= n; j++) { 2693 t = btf_type_by_id(obj->btf, j); 2694 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 2695 continue; 2696 2697 name = btf__str_by_offset(obj->btf, t->name_off); 2698 if (strcmp(name, prog->name) != 0) 2699 continue; 2700 2701 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 2702 break; 2703 } 2704 } 2705 2706 sanitize = btf_needs_sanitization(obj); 2707 if (sanitize) { 2708 const void *raw_data; 2709 __u32 sz; 2710 2711 /* clone BTF to sanitize a copy and leave the original intact */ 2712 raw_data = btf__get_raw_data(obj->btf, &sz); 2713 kern_btf = btf__new(raw_data, sz); 2714 if (IS_ERR(kern_btf)) 2715 return PTR_ERR(kern_btf); 2716 2717 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2718 btf__set_pointer_size(obj->btf, 8); 2719 bpf_object__sanitize_btf(obj, kern_btf); 2720 } 2721 2722 err = btf__load(kern_btf); 2723 if (sanitize) { 2724 if (!err) { 2725 /* move fd to libbpf's BTF */ 2726 btf__set_fd(obj->btf, btf__fd(kern_btf)); 2727 btf__set_fd(kern_btf, -1); 2728 } 2729 btf__free(kern_btf); 2730 } 2731 report: 2732 if (err) { 2733 btf_mandatory = kernel_needs_btf(obj); 2734 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 2735 btf_mandatory ? "BTF is mandatory, can't proceed." 2736 : "BTF is optional, ignoring."); 2737 if (!btf_mandatory) 2738 err = 0; 2739 } 2740 return err; 2741 } 2742 2743 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 2744 { 2745 const char *name; 2746 2747 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 2748 if (!name) { 2749 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2750 off, obj->path, elf_errmsg(-1)); 2751 return NULL; 2752 } 2753 2754 return name; 2755 } 2756 2757 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 2758 { 2759 const char *name; 2760 2761 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 2762 if (!name) { 2763 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2764 off, obj->path, elf_errmsg(-1)); 2765 return NULL; 2766 } 2767 2768 return name; 2769 } 2770 2771 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 2772 { 2773 Elf_Scn *scn; 2774 2775 scn = elf_getscn(obj->efile.elf, idx); 2776 if (!scn) { 2777 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 2778 idx, obj->path, elf_errmsg(-1)); 2779 return NULL; 2780 } 2781 return scn; 2782 } 2783 2784 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 2785 { 2786 Elf_Scn *scn = NULL; 2787 Elf *elf = obj->efile.elf; 2788 const char *sec_name; 2789 2790 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2791 sec_name = elf_sec_name(obj, scn); 2792 if (!sec_name) 2793 return NULL; 2794 2795 if (strcmp(sec_name, name) != 0) 2796 continue; 2797 2798 return scn; 2799 } 2800 return NULL; 2801 } 2802 2803 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr) 2804 { 2805 if (!scn) 2806 return -EINVAL; 2807 2808 if (gelf_getshdr(scn, hdr) != hdr) { 2809 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 2810 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2811 return -EINVAL; 2812 } 2813 2814 return 0; 2815 } 2816 2817 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 2818 { 2819 const char *name; 2820 GElf_Shdr sh; 2821 2822 if (!scn) 2823 return NULL; 2824 2825 if (elf_sec_hdr(obj, scn, &sh)) 2826 return NULL; 2827 2828 name = elf_sec_str(obj, sh.sh_name); 2829 if (!name) { 2830 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 2831 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2832 return NULL; 2833 } 2834 2835 return name; 2836 } 2837 2838 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 2839 { 2840 Elf_Data *data; 2841 2842 if (!scn) 2843 return NULL; 2844 2845 data = elf_getdata(scn, 0); 2846 if (!data) { 2847 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 2848 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 2849 obj->path, elf_errmsg(-1)); 2850 return NULL; 2851 } 2852 2853 return data; 2854 } 2855 2856 static bool is_sec_name_dwarf(const char *name) 2857 { 2858 /* approximation, but the actual list is too long */ 2859 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 2860 } 2861 2862 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name) 2863 { 2864 /* no special handling of .strtab */ 2865 if (hdr->sh_type == SHT_STRTAB) 2866 return true; 2867 2868 /* ignore .llvm_addrsig section as well */ 2869 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 2870 return true; 2871 2872 /* no subprograms will lead to an empty .text section, ignore it */ 2873 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 2874 strcmp(name, ".text") == 0) 2875 return true; 2876 2877 /* DWARF sections */ 2878 if (is_sec_name_dwarf(name)) 2879 return true; 2880 2881 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 2882 name += sizeof(".rel") - 1; 2883 /* DWARF section relocations */ 2884 if (is_sec_name_dwarf(name)) 2885 return true; 2886 2887 /* .BTF and .BTF.ext don't need relocations */ 2888 if (strcmp(name, BTF_ELF_SEC) == 0 || 2889 strcmp(name, BTF_EXT_ELF_SEC) == 0) 2890 return true; 2891 } 2892 2893 return false; 2894 } 2895 2896 static int cmp_progs(const void *_a, const void *_b) 2897 { 2898 const struct bpf_program *a = _a; 2899 const struct bpf_program *b = _b; 2900 2901 if (a->sec_idx != b->sec_idx) 2902 return a->sec_idx < b->sec_idx ? -1 : 1; 2903 2904 /* sec_insn_off can't be the same within the section */ 2905 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 2906 } 2907 2908 static int bpf_object__elf_collect(struct bpf_object *obj) 2909 { 2910 Elf *elf = obj->efile.elf; 2911 Elf_Data *btf_ext_data = NULL; 2912 Elf_Data *btf_data = NULL; 2913 int idx = 0, err = 0; 2914 const char *name; 2915 Elf_Data *data; 2916 Elf_Scn *scn; 2917 GElf_Shdr sh; 2918 2919 /* a bunch of ELF parsing functionality depends on processing symbols, 2920 * so do the first pass and find the symbol table 2921 */ 2922 scn = NULL; 2923 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2924 if (elf_sec_hdr(obj, scn, &sh)) 2925 return -LIBBPF_ERRNO__FORMAT; 2926 2927 if (sh.sh_type == SHT_SYMTAB) { 2928 if (obj->efile.symbols) { 2929 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 2930 return -LIBBPF_ERRNO__FORMAT; 2931 } 2932 2933 data = elf_sec_data(obj, scn); 2934 if (!data) 2935 return -LIBBPF_ERRNO__FORMAT; 2936 2937 obj->efile.symbols = data; 2938 obj->efile.symbols_shndx = elf_ndxscn(scn); 2939 obj->efile.strtabidx = sh.sh_link; 2940 } 2941 } 2942 2943 scn = NULL; 2944 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2945 idx++; 2946 2947 if (elf_sec_hdr(obj, scn, &sh)) 2948 return -LIBBPF_ERRNO__FORMAT; 2949 2950 name = elf_sec_str(obj, sh.sh_name); 2951 if (!name) 2952 return -LIBBPF_ERRNO__FORMAT; 2953 2954 if (ignore_elf_section(&sh, name)) 2955 continue; 2956 2957 data = elf_sec_data(obj, scn); 2958 if (!data) 2959 return -LIBBPF_ERRNO__FORMAT; 2960 2961 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 2962 idx, name, (unsigned long)data->d_size, 2963 (int)sh.sh_link, (unsigned long)sh.sh_flags, 2964 (int)sh.sh_type); 2965 2966 if (strcmp(name, "license") == 0) { 2967 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 2968 if (err) 2969 return err; 2970 } else if (strcmp(name, "version") == 0) { 2971 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 2972 if (err) 2973 return err; 2974 } else if (strcmp(name, "maps") == 0) { 2975 obj->efile.maps_shndx = idx; 2976 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 2977 obj->efile.btf_maps_shndx = idx; 2978 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 2979 btf_data = data; 2980 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 2981 btf_ext_data = data; 2982 } else if (sh.sh_type == SHT_SYMTAB) { 2983 /* already processed during the first pass above */ 2984 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 2985 if (sh.sh_flags & SHF_EXECINSTR) { 2986 if (strcmp(name, ".text") == 0) 2987 obj->efile.text_shndx = idx; 2988 err = bpf_object__add_programs(obj, data, name, idx); 2989 if (err) 2990 return err; 2991 } else if (strcmp(name, DATA_SEC) == 0) { 2992 obj->efile.data = data; 2993 obj->efile.data_shndx = idx; 2994 } else if (strcmp(name, RODATA_SEC) == 0) { 2995 obj->efile.rodata = data; 2996 obj->efile.rodata_shndx = idx; 2997 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 2998 obj->efile.st_ops_data = data; 2999 obj->efile.st_ops_shndx = idx; 3000 } else { 3001 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3002 idx, name); 3003 } 3004 } else if (sh.sh_type == SHT_REL) { 3005 int nr_sects = obj->efile.nr_reloc_sects; 3006 void *sects = obj->efile.reloc_sects; 3007 int sec = sh.sh_info; /* points to other section */ 3008 3009 /* Only do relo for section with exec instructions */ 3010 if (!section_have_execinstr(obj, sec) && 3011 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3012 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3013 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3014 idx, name, sec, 3015 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>"); 3016 continue; 3017 } 3018 3019 sects = libbpf_reallocarray(sects, nr_sects + 1, 3020 sizeof(*obj->efile.reloc_sects)); 3021 if (!sects) 3022 return -ENOMEM; 3023 3024 obj->efile.reloc_sects = sects; 3025 obj->efile.nr_reloc_sects++; 3026 3027 obj->efile.reloc_sects[nr_sects].shdr = sh; 3028 obj->efile.reloc_sects[nr_sects].data = data; 3029 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) { 3030 obj->efile.bss = data; 3031 obj->efile.bss_shndx = idx; 3032 } else { 3033 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3034 (size_t)sh.sh_size); 3035 } 3036 } 3037 3038 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3039 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3040 return -LIBBPF_ERRNO__FORMAT; 3041 } 3042 3043 /* sort BPF programs by section name and in-section instruction offset 3044 * for faster search */ 3045 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3046 3047 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3048 } 3049 3050 static bool sym_is_extern(const GElf_Sym *sym) 3051 { 3052 int bind = GELF_ST_BIND(sym->st_info); 3053 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3054 return sym->st_shndx == SHN_UNDEF && 3055 (bind == STB_GLOBAL || bind == STB_WEAK) && 3056 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 3057 } 3058 3059 static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) 3060 { 3061 int bind = GELF_ST_BIND(sym->st_info); 3062 int type = GELF_ST_TYPE(sym->st_info); 3063 3064 /* in .text section */ 3065 if (sym->st_shndx != text_shndx) 3066 return false; 3067 3068 /* local function */ 3069 if (bind == STB_LOCAL && type == STT_SECTION) 3070 return true; 3071 3072 /* global function */ 3073 return bind == STB_GLOBAL && type == STT_FUNC; 3074 } 3075 3076 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3077 { 3078 const struct btf_type *t; 3079 const char *tname; 3080 int i, n; 3081 3082 if (!btf) 3083 return -ESRCH; 3084 3085 n = btf__get_nr_types(btf); 3086 for (i = 1; i <= n; i++) { 3087 t = btf__type_by_id(btf, i); 3088 3089 if (!btf_is_var(t) && !btf_is_func(t)) 3090 continue; 3091 3092 tname = btf__name_by_offset(btf, t->name_off); 3093 if (strcmp(tname, ext_name)) 3094 continue; 3095 3096 if (btf_is_var(t) && 3097 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3098 return -EINVAL; 3099 3100 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3101 return -EINVAL; 3102 3103 return i; 3104 } 3105 3106 return -ENOENT; 3107 } 3108 3109 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 3110 const struct btf_var_secinfo *vs; 3111 const struct btf_type *t; 3112 int i, j, n; 3113 3114 if (!btf) 3115 return -ESRCH; 3116 3117 n = btf__get_nr_types(btf); 3118 for (i = 1; i <= n; i++) { 3119 t = btf__type_by_id(btf, i); 3120 3121 if (!btf_is_datasec(t)) 3122 continue; 3123 3124 vs = btf_var_secinfos(t); 3125 for (j = 0; j < btf_vlen(t); j++, vs++) { 3126 if (vs->type == ext_btf_id) 3127 return i; 3128 } 3129 } 3130 3131 return -ENOENT; 3132 } 3133 3134 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 3135 bool *is_signed) 3136 { 3137 const struct btf_type *t; 3138 const char *name; 3139 3140 t = skip_mods_and_typedefs(btf, id, NULL); 3141 name = btf__name_by_offset(btf, t->name_off); 3142 3143 if (is_signed) 3144 *is_signed = false; 3145 switch (btf_kind(t)) { 3146 case BTF_KIND_INT: { 3147 int enc = btf_int_encoding(t); 3148 3149 if (enc & BTF_INT_BOOL) 3150 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 3151 if (is_signed) 3152 *is_signed = enc & BTF_INT_SIGNED; 3153 if (t->size == 1) 3154 return KCFG_CHAR; 3155 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 3156 return KCFG_UNKNOWN; 3157 return KCFG_INT; 3158 } 3159 case BTF_KIND_ENUM: 3160 if (t->size != 4) 3161 return KCFG_UNKNOWN; 3162 if (strcmp(name, "libbpf_tristate")) 3163 return KCFG_UNKNOWN; 3164 return KCFG_TRISTATE; 3165 case BTF_KIND_ARRAY: 3166 if (btf_array(t)->nelems == 0) 3167 return KCFG_UNKNOWN; 3168 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 3169 return KCFG_UNKNOWN; 3170 return KCFG_CHAR_ARR; 3171 default: 3172 return KCFG_UNKNOWN; 3173 } 3174 } 3175 3176 static int cmp_externs(const void *_a, const void *_b) 3177 { 3178 const struct extern_desc *a = _a; 3179 const struct extern_desc *b = _b; 3180 3181 if (a->type != b->type) 3182 return a->type < b->type ? -1 : 1; 3183 3184 if (a->type == EXT_KCFG) { 3185 /* descending order by alignment requirements */ 3186 if (a->kcfg.align != b->kcfg.align) 3187 return a->kcfg.align > b->kcfg.align ? -1 : 1; 3188 /* ascending order by size, within same alignment class */ 3189 if (a->kcfg.sz != b->kcfg.sz) 3190 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 3191 } 3192 3193 /* resolve ties by name */ 3194 return strcmp(a->name, b->name); 3195 } 3196 3197 static int find_int_btf_id(const struct btf *btf) 3198 { 3199 const struct btf_type *t; 3200 int i, n; 3201 3202 n = btf__get_nr_types(btf); 3203 for (i = 1; i <= n; i++) { 3204 t = btf__type_by_id(btf, i); 3205 3206 if (btf_is_int(t) && btf_int_bits(t) == 32) 3207 return i; 3208 } 3209 3210 return 0; 3211 } 3212 3213 static int add_dummy_ksym_var(struct btf *btf) 3214 { 3215 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 3216 const struct btf_var_secinfo *vs; 3217 const struct btf_type *sec; 3218 3219 if (!btf) 3220 return 0; 3221 3222 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 3223 BTF_KIND_DATASEC); 3224 if (sec_btf_id < 0) 3225 return 0; 3226 3227 sec = btf__type_by_id(btf, sec_btf_id); 3228 vs = btf_var_secinfos(sec); 3229 for (i = 0; i < btf_vlen(sec); i++, vs++) { 3230 const struct btf_type *vt; 3231 3232 vt = btf__type_by_id(btf, vs->type); 3233 if (btf_is_func(vt)) 3234 break; 3235 } 3236 3237 /* No func in ksyms sec. No need to add dummy var. */ 3238 if (i == btf_vlen(sec)) 3239 return 0; 3240 3241 int_btf_id = find_int_btf_id(btf); 3242 dummy_var_btf_id = btf__add_var(btf, 3243 "dummy_ksym", 3244 BTF_VAR_GLOBAL_ALLOCATED, 3245 int_btf_id); 3246 if (dummy_var_btf_id < 0) 3247 pr_warn("cannot create a dummy_ksym var\n"); 3248 3249 return dummy_var_btf_id; 3250 } 3251 3252 static int bpf_object__collect_externs(struct bpf_object *obj) 3253 { 3254 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 3255 const struct btf_type *t; 3256 struct extern_desc *ext; 3257 int i, n, off, dummy_var_btf_id; 3258 const char *ext_name, *sec_name; 3259 Elf_Scn *scn; 3260 GElf_Shdr sh; 3261 3262 if (!obj->efile.symbols) 3263 return 0; 3264 3265 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 3266 if (elf_sec_hdr(obj, scn, &sh)) 3267 return -LIBBPF_ERRNO__FORMAT; 3268 3269 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 3270 if (dummy_var_btf_id < 0) 3271 return dummy_var_btf_id; 3272 3273 n = sh.sh_size / sh.sh_entsize; 3274 pr_debug("looking for externs among %d symbols...\n", n); 3275 3276 for (i = 0; i < n; i++) { 3277 GElf_Sym sym; 3278 3279 if (!gelf_getsym(obj->efile.symbols, i, &sym)) 3280 return -LIBBPF_ERRNO__FORMAT; 3281 if (!sym_is_extern(&sym)) 3282 continue; 3283 ext_name = elf_sym_str(obj, sym.st_name); 3284 if (!ext_name || !ext_name[0]) 3285 continue; 3286 3287 ext = obj->externs; 3288 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 3289 if (!ext) 3290 return -ENOMEM; 3291 obj->externs = ext; 3292 ext = &ext[obj->nr_extern]; 3293 memset(ext, 0, sizeof(*ext)); 3294 obj->nr_extern++; 3295 3296 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 3297 if (ext->btf_id <= 0) { 3298 pr_warn("failed to find BTF for extern '%s': %d\n", 3299 ext_name, ext->btf_id); 3300 return ext->btf_id; 3301 } 3302 t = btf__type_by_id(obj->btf, ext->btf_id); 3303 ext->name = btf__name_by_offset(obj->btf, t->name_off); 3304 ext->sym_idx = i; 3305 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; 3306 3307 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 3308 if (ext->sec_btf_id <= 0) { 3309 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 3310 ext_name, ext->btf_id, ext->sec_btf_id); 3311 return ext->sec_btf_id; 3312 } 3313 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 3314 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 3315 3316 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 3317 if (btf_is_func(t)) { 3318 pr_warn("extern function %s is unsupported under %s section\n", 3319 ext->name, KCONFIG_SEC); 3320 return -ENOTSUP; 3321 } 3322 kcfg_sec = sec; 3323 ext->type = EXT_KCFG; 3324 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 3325 if (ext->kcfg.sz <= 0) { 3326 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 3327 ext_name, ext->kcfg.sz); 3328 return ext->kcfg.sz; 3329 } 3330 ext->kcfg.align = btf__align_of(obj->btf, t->type); 3331 if (ext->kcfg.align <= 0) { 3332 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 3333 ext_name, ext->kcfg.align); 3334 return -EINVAL; 3335 } 3336 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 3337 &ext->kcfg.is_signed); 3338 if (ext->kcfg.type == KCFG_UNKNOWN) { 3339 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); 3340 return -ENOTSUP; 3341 } 3342 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 3343 if (btf_is_func(t) && ext->is_weak) { 3344 pr_warn("extern weak function %s is unsupported\n", 3345 ext->name); 3346 return -ENOTSUP; 3347 } 3348 ksym_sec = sec; 3349 ext->type = EXT_KSYM; 3350 skip_mods_and_typedefs(obj->btf, t->type, 3351 &ext->ksym.type_id); 3352 } else { 3353 pr_warn("unrecognized extern section '%s'\n", sec_name); 3354 return -ENOTSUP; 3355 } 3356 } 3357 pr_debug("collected %d externs total\n", obj->nr_extern); 3358 3359 if (!obj->nr_extern) 3360 return 0; 3361 3362 /* sort externs by type, for kcfg ones also by (align, size, name) */ 3363 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 3364 3365 /* for .ksyms section, we need to turn all externs into allocated 3366 * variables in BTF to pass kernel verification; we do this by 3367 * pretending that each extern is a 8-byte variable 3368 */ 3369 if (ksym_sec) { 3370 /* find existing 4-byte integer type in BTF to use for fake 3371 * extern variables in DATASEC 3372 */ 3373 int int_btf_id = find_int_btf_id(obj->btf); 3374 /* For extern function, a dummy_var added earlier 3375 * will be used to replace the vs->type and 3376 * its name string will be used to refill 3377 * the missing param's name. 3378 */ 3379 const struct btf_type *dummy_var; 3380 3381 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 3382 for (i = 0; i < obj->nr_extern; i++) { 3383 ext = &obj->externs[i]; 3384 if (ext->type != EXT_KSYM) 3385 continue; 3386 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 3387 i, ext->sym_idx, ext->name); 3388 } 3389 3390 sec = ksym_sec; 3391 n = btf_vlen(sec); 3392 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 3393 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3394 struct btf_type *vt; 3395 3396 vt = (void *)btf__type_by_id(obj->btf, vs->type); 3397 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 3398 ext = find_extern_by_name(obj, ext_name); 3399 if (!ext) { 3400 pr_warn("failed to find extern definition for BTF %s '%s'\n", 3401 btf_kind_str(vt), ext_name); 3402 return -ESRCH; 3403 } 3404 if (btf_is_func(vt)) { 3405 const struct btf_type *func_proto; 3406 struct btf_param *param; 3407 int j; 3408 3409 func_proto = btf__type_by_id(obj->btf, 3410 vt->type); 3411 param = btf_params(func_proto); 3412 /* Reuse the dummy_var string if the 3413 * func proto does not have param name. 3414 */ 3415 for (j = 0; j < btf_vlen(func_proto); j++) 3416 if (param[j].type && !param[j].name_off) 3417 param[j].name_off = 3418 dummy_var->name_off; 3419 vs->type = dummy_var_btf_id; 3420 vt->info &= ~0xffff; 3421 vt->info |= BTF_FUNC_GLOBAL; 3422 } else { 3423 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3424 vt->type = int_btf_id; 3425 } 3426 vs->offset = off; 3427 vs->size = sizeof(int); 3428 } 3429 sec->size = off; 3430 } 3431 3432 if (kcfg_sec) { 3433 sec = kcfg_sec; 3434 /* for kcfg externs calculate their offsets within a .kconfig map */ 3435 off = 0; 3436 for (i = 0; i < obj->nr_extern; i++) { 3437 ext = &obj->externs[i]; 3438 if (ext->type != EXT_KCFG) 3439 continue; 3440 3441 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 3442 off = ext->kcfg.data_off + ext->kcfg.sz; 3443 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 3444 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 3445 } 3446 sec->size = off; 3447 n = btf_vlen(sec); 3448 for (i = 0; i < n; i++) { 3449 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3450 3451 t = btf__type_by_id(obj->btf, vs->type); 3452 ext_name = btf__name_by_offset(obj->btf, t->name_off); 3453 ext = find_extern_by_name(obj, ext_name); 3454 if (!ext) { 3455 pr_warn("failed to find extern definition for BTF var '%s'\n", 3456 ext_name); 3457 return -ESRCH; 3458 } 3459 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3460 vs->offset = ext->kcfg.data_off; 3461 } 3462 } 3463 return 0; 3464 } 3465 3466 struct bpf_program * 3467 bpf_object__find_program_by_title(const struct bpf_object *obj, 3468 const char *title) 3469 { 3470 struct bpf_program *pos; 3471 3472 bpf_object__for_each_program(pos, obj) { 3473 if (pos->sec_name && !strcmp(pos->sec_name, title)) 3474 return pos; 3475 } 3476 return NULL; 3477 } 3478 3479 static bool prog_is_subprog(const struct bpf_object *obj, 3480 const struct bpf_program *prog) 3481 { 3482 /* For legacy reasons, libbpf supports an entry-point BPF programs 3483 * without SEC() attribute, i.e., those in the .text section. But if 3484 * there are 2 or more such programs in the .text section, they all 3485 * must be subprograms called from entry-point BPF programs in 3486 * designated SEC()'tions, otherwise there is no way to distinguish 3487 * which of those programs should be loaded vs which are a subprogram. 3488 * Similarly, if there is a function/program in .text and at least one 3489 * other BPF program with custom SEC() attribute, then we just assume 3490 * .text programs are subprograms (even if they are not called from 3491 * other programs), because libbpf never explicitly supported mixing 3492 * SEC()-designated BPF programs and .text entry-point BPF programs. 3493 */ 3494 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 3495 } 3496 3497 struct bpf_program * 3498 bpf_object__find_program_by_name(const struct bpf_object *obj, 3499 const char *name) 3500 { 3501 struct bpf_program *prog; 3502 3503 bpf_object__for_each_program(prog, obj) { 3504 if (prog_is_subprog(obj, prog)) 3505 continue; 3506 if (!strcmp(prog->name, name)) 3507 return prog; 3508 } 3509 return NULL; 3510 } 3511 3512 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 3513 int shndx) 3514 { 3515 return shndx == obj->efile.data_shndx || 3516 shndx == obj->efile.bss_shndx || 3517 shndx == obj->efile.rodata_shndx; 3518 } 3519 3520 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 3521 int shndx) 3522 { 3523 return shndx == obj->efile.maps_shndx || 3524 shndx == obj->efile.btf_maps_shndx; 3525 } 3526 3527 static enum libbpf_map_type 3528 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 3529 { 3530 if (shndx == obj->efile.data_shndx) 3531 return LIBBPF_MAP_DATA; 3532 else if (shndx == obj->efile.bss_shndx) 3533 return LIBBPF_MAP_BSS; 3534 else if (shndx == obj->efile.rodata_shndx) 3535 return LIBBPF_MAP_RODATA; 3536 else if (shndx == obj->efile.symbols_shndx) 3537 return LIBBPF_MAP_KCONFIG; 3538 else 3539 return LIBBPF_MAP_UNSPEC; 3540 } 3541 3542 static int bpf_program__record_reloc(struct bpf_program *prog, 3543 struct reloc_desc *reloc_desc, 3544 __u32 insn_idx, const char *sym_name, 3545 const GElf_Sym *sym, const GElf_Rel *rel) 3546 { 3547 struct bpf_insn *insn = &prog->insns[insn_idx]; 3548 size_t map_idx, nr_maps = prog->obj->nr_maps; 3549 struct bpf_object *obj = prog->obj; 3550 __u32 shdr_idx = sym->st_shndx; 3551 enum libbpf_map_type type; 3552 const char *sym_sec_name; 3553 struct bpf_map *map; 3554 3555 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 3556 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 3557 prog->name, sym_name, insn_idx, insn->code); 3558 return -LIBBPF_ERRNO__RELOC; 3559 } 3560 3561 if (sym_is_extern(sym)) { 3562 int sym_idx = GELF_R_SYM(rel->r_info); 3563 int i, n = obj->nr_extern; 3564 struct extern_desc *ext; 3565 3566 for (i = 0; i < n; i++) { 3567 ext = &obj->externs[i]; 3568 if (ext->sym_idx == sym_idx) 3569 break; 3570 } 3571 if (i >= n) { 3572 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 3573 prog->name, sym_name, sym_idx); 3574 return -LIBBPF_ERRNO__RELOC; 3575 } 3576 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 3577 prog->name, i, ext->name, ext->sym_idx, insn_idx); 3578 if (insn->code == (BPF_JMP | BPF_CALL)) 3579 reloc_desc->type = RELO_EXTERN_FUNC; 3580 else 3581 reloc_desc->type = RELO_EXTERN_VAR; 3582 reloc_desc->insn_idx = insn_idx; 3583 reloc_desc->sym_off = i; /* sym_off stores extern index */ 3584 return 0; 3585 } 3586 3587 /* sub-program call relocation */ 3588 if (is_call_insn(insn)) { 3589 if (insn->src_reg != BPF_PSEUDO_CALL) { 3590 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 3591 return -LIBBPF_ERRNO__RELOC; 3592 } 3593 /* text_shndx can be 0, if no default "main" program exists */ 3594 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 3595 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3596 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 3597 prog->name, sym_name, sym_sec_name); 3598 return -LIBBPF_ERRNO__RELOC; 3599 } 3600 if (sym->st_value % BPF_INSN_SZ) { 3601 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 3602 prog->name, sym_name, (size_t)sym->st_value); 3603 return -LIBBPF_ERRNO__RELOC; 3604 } 3605 reloc_desc->type = RELO_CALL; 3606 reloc_desc->insn_idx = insn_idx; 3607 reloc_desc->sym_off = sym->st_value; 3608 return 0; 3609 } 3610 3611 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 3612 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 3613 prog->name, sym_name, shdr_idx); 3614 return -LIBBPF_ERRNO__RELOC; 3615 } 3616 3617 /* loading subprog addresses */ 3618 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 3619 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 3620 * local_func: sym->st_value = 0, insn->imm = offset in the section. 3621 */ 3622 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 3623 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 3624 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 3625 return -LIBBPF_ERRNO__RELOC; 3626 } 3627 3628 reloc_desc->type = RELO_SUBPROG_ADDR; 3629 reloc_desc->insn_idx = insn_idx; 3630 reloc_desc->sym_off = sym->st_value; 3631 return 0; 3632 } 3633 3634 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 3635 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3636 3637 /* generic map reference relocation */ 3638 if (type == LIBBPF_MAP_UNSPEC) { 3639 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 3640 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 3641 prog->name, sym_name, sym_sec_name); 3642 return -LIBBPF_ERRNO__RELOC; 3643 } 3644 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3645 map = &obj->maps[map_idx]; 3646 if (map->libbpf_type != type || 3647 map->sec_idx != sym->st_shndx || 3648 map->sec_offset != sym->st_value) 3649 continue; 3650 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 3651 prog->name, map_idx, map->name, map->sec_idx, 3652 map->sec_offset, insn_idx); 3653 break; 3654 } 3655 if (map_idx >= nr_maps) { 3656 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 3657 prog->name, sym_sec_name, (size_t)sym->st_value); 3658 return -LIBBPF_ERRNO__RELOC; 3659 } 3660 reloc_desc->type = RELO_LD64; 3661 reloc_desc->insn_idx = insn_idx; 3662 reloc_desc->map_idx = map_idx; 3663 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 3664 return 0; 3665 } 3666 3667 /* global data map relocation */ 3668 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 3669 pr_warn("prog '%s': bad data relo against section '%s'\n", 3670 prog->name, sym_sec_name); 3671 return -LIBBPF_ERRNO__RELOC; 3672 } 3673 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3674 map = &obj->maps[map_idx]; 3675 if (map->libbpf_type != type) 3676 continue; 3677 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 3678 prog->name, map_idx, map->name, map->sec_idx, 3679 map->sec_offset, insn_idx); 3680 break; 3681 } 3682 if (map_idx >= nr_maps) { 3683 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 3684 prog->name, sym_sec_name); 3685 return -LIBBPF_ERRNO__RELOC; 3686 } 3687 3688 reloc_desc->type = RELO_DATA; 3689 reloc_desc->insn_idx = insn_idx; 3690 reloc_desc->map_idx = map_idx; 3691 reloc_desc->sym_off = sym->st_value; 3692 return 0; 3693 } 3694 3695 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 3696 { 3697 return insn_idx >= prog->sec_insn_off && 3698 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 3699 } 3700 3701 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 3702 size_t sec_idx, size_t insn_idx) 3703 { 3704 int l = 0, r = obj->nr_programs - 1, m; 3705 struct bpf_program *prog; 3706 3707 while (l < r) { 3708 m = l + (r - l + 1) / 2; 3709 prog = &obj->programs[m]; 3710 3711 if (prog->sec_idx < sec_idx || 3712 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 3713 l = m; 3714 else 3715 r = m - 1; 3716 } 3717 /* matching program could be at index l, but it still might be the 3718 * wrong one, so we need to double check conditions for the last time 3719 */ 3720 prog = &obj->programs[l]; 3721 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 3722 return prog; 3723 return NULL; 3724 } 3725 3726 static int 3727 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data) 3728 { 3729 Elf_Data *symbols = obj->efile.symbols; 3730 const char *relo_sec_name, *sec_name; 3731 size_t sec_idx = shdr->sh_info; 3732 struct bpf_program *prog; 3733 struct reloc_desc *relos; 3734 int err, i, nrels; 3735 const char *sym_name; 3736 __u32 insn_idx; 3737 Elf_Scn *scn; 3738 Elf_Data *scn_data; 3739 GElf_Sym sym; 3740 GElf_Rel rel; 3741 3742 scn = elf_sec_by_idx(obj, sec_idx); 3743 scn_data = elf_sec_data(obj, scn); 3744 3745 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 3746 sec_name = elf_sec_name(obj, scn); 3747 if (!relo_sec_name || !sec_name) 3748 return -EINVAL; 3749 3750 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 3751 relo_sec_name, sec_idx, sec_name); 3752 nrels = shdr->sh_size / shdr->sh_entsize; 3753 3754 for (i = 0; i < nrels; i++) { 3755 if (!gelf_getrel(data, i, &rel)) { 3756 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 3757 return -LIBBPF_ERRNO__FORMAT; 3758 } 3759 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 3760 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n", 3761 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3762 return -LIBBPF_ERRNO__FORMAT; 3763 } 3764 3765 if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) { 3766 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 3767 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3768 return -LIBBPF_ERRNO__FORMAT; 3769 } 3770 3771 insn_idx = rel.r_offset / BPF_INSN_SZ; 3772 /* relocations against static functions are recorded as 3773 * relocations against the section that contains a function; 3774 * in such case, symbol will be STT_SECTION and sym.st_name 3775 * will point to empty string (0), so fetch section name 3776 * instead 3777 */ 3778 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0) 3779 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx)); 3780 else 3781 sym_name = elf_sym_str(obj, sym.st_name); 3782 sym_name = sym_name ?: "<?"; 3783 3784 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 3785 relo_sec_name, i, insn_idx, sym_name); 3786 3787 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 3788 if (!prog) { 3789 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 3790 relo_sec_name, i, sec_name, insn_idx); 3791 continue; 3792 } 3793 3794 relos = libbpf_reallocarray(prog->reloc_desc, 3795 prog->nr_reloc + 1, sizeof(*relos)); 3796 if (!relos) 3797 return -ENOMEM; 3798 prog->reloc_desc = relos; 3799 3800 /* adjust insn_idx to local BPF program frame of reference */ 3801 insn_idx -= prog->sec_insn_off; 3802 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 3803 insn_idx, sym_name, &sym, &rel); 3804 if (err) 3805 return err; 3806 3807 prog->nr_reloc++; 3808 } 3809 return 0; 3810 } 3811 3812 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 3813 { 3814 struct bpf_map_def *def = &map->def; 3815 __u32 key_type_id = 0, value_type_id = 0; 3816 int ret; 3817 3818 /* if it's BTF-defined map, we don't need to search for type IDs. 3819 * For struct_ops map, it does not need btf_key_type_id and 3820 * btf_value_type_id. 3821 */ 3822 if (map->sec_idx == obj->efile.btf_maps_shndx || 3823 bpf_map__is_struct_ops(map)) 3824 return 0; 3825 3826 if (!bpf_map__is_internal(map)) { 3827 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 3828 def->value_size, &key_type_id, 3829 &value_type_id); 3830 } else { 3831 /* 3832 * LLVM annotates global data differently in BTF, that is, 3833 * only as '.data', '.bss' or '.rodata'. 3834 */ 3835 ret = btf__find_by_name(obj->btf, 3836 libbpf_type_to_btf_name[map->libbpf_type]); 3837 } 3838 if (ret < 0) 3839 return ret; 3840 3841 map->btf_key_type_id = key_type_id; 3842 map->btf_value_type_id = bpf_map__is_internal(map) ? 3843 ret : value_type_id; 3844 return 0; 3845 } 3846 3847 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 3848 { 3849 struct bpf_map_info info = {}; 3850 __u32 len = sizeof(info); 3851 int new_fd, err; 3852 char *new_name; 3853 3854 err = bpf_obj_get_info_by_fd(fd, &info, &len); 3855 if (err) 3856 return err; 3857 3858 new_name = strdup(info.name); 3859 if (!new_name) 3860 return -errno; 3861 3862 new_fd = open("/", O_RDONLY | O_CLOEXEC); 3863 if (new_fd < 0) { 3864 err = -errno; 3865 goto err_free_new_name; 3866 } 3867 3868 new_fd = dup3(fd, new_fd, O_CLOEXEC); 3869 if (new_fd < 0) { 3870 err = -errno; 3871 goto err_close_new_fd; 3872 } 3873 3874 err = zclose(map->fd); 3875 if (err) { 3876 err = -errno; 3877 goto err_close_new_fd; 3878 } 3879 free(map->name); 3880 3881 map->fd = new_fd; 3882 map->name = new_name; 3883 map->def.type = info.type; 3884 map->def.key_size = info.key_size; 3885 map->def.value_size = info.value_size; 3886 map->def.max_entries = info.max_entries; 3887 map->def.map_flags = info.map_flags; 3888 map->btf_key_type_id = info.btf_key_type_id; 3889 map->btf_value_type_id = info.btf_value_type_id; 3890 map->reused = true; 3891 3892 return 0; 3893 3894 err_close_new_fd: 3895 close(new_fd); 3896 err_free_new_name: 3897 free(new_name); 3898 return err; 3899 } 3900 3901 __u32 bpf_map__max_entries(const struct bpf_map *map) 3902 { 3903 return map->def.max_entries; 3904 } 3905 3906 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 3907 { 3908 if (!bpf_map_type__is_map_in_map(map->def.type)) 3909 return NULL; 3910 3911 return map->inner_map; 3912 } 3913 3914 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 3915 { 3916 if (map->fd >= 0) 3917 return -EBUSY; 3918 map->def.max_entries = max_entries; 3919 return 0; 3920 } 3921 3922 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 3923 { 3924 if (!map || !max_entries) 3925 return -EINVAL; 3926 3927 return bpf_map__set_max_entries(map, max_entries); 3928 } 3929 3930 static int 3931 bpf_object__probe_loading(struct bpf_object *obj) 3932 { 3933 struct bpf_load_program_attr attr; 3934 char *cp, errmsg[STRERR_BUFSIZE]; 3935 struct bpf_insn insns[] = { 3936 BPF_MOV64_IMM(BPF_REG_0, 0), 3937 BPF_EXIT_INSN(), 3938 }; 3939 int ret; 3940 3941 /* make sure basic loading works */ 3942 3943 memset(&attr, 0, sizeof(attr)); 3944 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3945 attr.insns = insns; 3946 attr.insns_cnt = ARRAY_SIZE(insns); 3947 attr.license = "GPL"; 3948 3949 ret = bpf_load_program_xattr(&attr, NULL, 0); 3950 if (ret < 0) { 3951 ret = errno; 3952 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 3953 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 3954 "program. Make sure your kernel supports BPF " 3955 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 3956 "set to big enough value.\n", __func__, cp, ret); 3957 return -ret; 3958 } 3959 close(ret); 3960 3961 return 0; 3962 } 3963 3964 static int probe_fd(int fd) 3965 { 3966 if (fd >= 0) 3967 close(fd); 3968 return fd >= 0; 3969 } 3970 3971 static int probe_kern_prog_name(void) 3972 { 3973 struct bpf_load_program_attr attr; 3974 struct bpf_insn insns[] = { 3975 BPF_MOV64_IMM(BPF_REG_0, 0), 3976 BPF_EXIT_INSN(), 3977 }; 3978 int ret; 3979 3980 /* make sure loading with name works */ 3981 3982 memset(&attr, 0, sizeof(attr)); 3983 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3984 attr.insns = insns; 3985 attr.insns_cnt = ARRAY_SIZE(insns); 3986 attr.license = "GPL"; 3987 attr.name = "test"; 3988 ret = bpf_load_program_xattr(&attr, NULL, 0); 3989 return probe_fd(ret); 3990 } 3991 3992 static int probe_kern_global_data(void) 3993 { 3994 struct bpf_load_program_attr prg_attr; 3995 struct bpf_create_map_attr map_attr; 3996 char *cp, errmsg[STRERR_BUFSIZE]; 3997 struct bpf_insn insns[] = { 3998 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 3999 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 4000 BPF_MOV64_IMM(BPF_REG_0, 0), 4001 BPF_EXIT_INSN(), 4002 }; 4003 int ret, map; 4004 4005 memset(&map_attr, 0, sizeof(map_attr)); 4006 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4007 map_attr.key_size = sizeof(int); 4008 map_attr.value_size = 32; 4009 map_attr.max_entries = 1; 4010 4011 map = bpf_create_map_xattr(&map_attr); 4012 if (map < 0) { 4013 ret = -errno; 4014 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4015 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4016 __func__, cp, -ret); 4017 return ret; 4018 } 4019 4020 insns[0].imm = map; 4021 4022 memset(&prg_attr, 0, sizeof(prg_attr)); 4023 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4024 prg_attr.insns = insns; 4025 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4026 prg_attr.license = "GPL"; 4027 4028 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 4029 close(map); 4030 return probe_fd(ret); 4031 } 4032 4033 static int probe_kern_btf(void) 4034 { 4035 static const char strs[] = "\0int"; 4036 __u32 types[] = { 4037 /* int */ 4038 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4039 }; 4040 4041 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4042 strs, sizeof(strs))); 4043 } 4044 4045 static int probe_kern_btf_func(void) 4046 { 4047 static const char strs[] = "\0int\0x\0a"; 4048 /* void x(int a) {} */ 4049 __u32 types[] = { 4050 /* int */ 4051 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4052 /* FUNC_PROTO */ /* [2] */ 4053 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4054 BTF_PARAM_ENC(7, 1), 4055 /* FUNC x */ /* [3] */ 4056 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 4057 }; 4058 4059 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4060 strs, sizeof(strs))); 4061 } 4062 4063 static int probe_kern_btf_func_global(void) 4064 { 4065 static const char strs[] = "\0int\0x\0a"; 4066 /* static void x(int a) {} */ 4067 __u32 types[] = { 4068 /* int */ 4069 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4070 /* FUNC_PROTO */ /* [2] */ 4071 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4072 BTF_PARAM_ENC(7, 1), 4073 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 4074 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 4075 }; 4076 4077 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4078 strs, sizeof(strs))); 4079 } 4080 4081 static int probe_kern_btf_datasec(void) 4082 { 4083 static const char strs[] = "\0x\0.data"; 4084 /* static int a; */ 4085 __u32 types[] = { 4086 /* int */ 4087 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4088 /* VAR x */ /* [2] */ 4089 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4090 BTF_VAR_STATIC, 4091 /* DATASEC val */ /* [3] */ 4092 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 4093 BTF_VAR_SECINFO_ENC(2, 0, 4), 4094 }; 4095 4096 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4097 strs, sizeof(strs))); 4098 } 4099 4100 static int probe_kern_btf_float(void) 4101 { 4102 static const char strs[] = "\0float"; 4103 __u32 types[] = { 4104 /* float */ 4105 BTF_TYPE_FLOAT_ENC(1, 4), 4106 }; 4107 4108 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4109 strs, sizeof(strs))); 4110 } 4111 4112 static int probe_kern_array_mmap(void) 4113 { 4114 struct bpf_create_map_attr attr = { 4115 .map_type = BPF_MAP_TYPE_ARRAY, 4116 .map_flags = BPF_F_MMAPABLE, 4117 .key_size = sizeof(int), 4118 .value_size = sizeof(int), 4119 .max_entries = 1, 4120 }; 4121 4122 return probe_fd(bpf_create_map_xattr(&attr)); 4123 } 4124 4125 static int probe_kern_exp_attach_type(void) 4126 { 4127 struct bpf_load_program_attr attr; 4128 struct bpf_insn insns[] = { 4129 BPF_MOV64_IMM(BPF_REG_0, 0), 4130 BPF_EXIT_INSN(), 4131 }; 4132 4133 memset(&attr, 0, sizeof(attr)); 4134 /* use any valid combination of program type and (optional) 4135 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 4136 * to see if kernel supports expected_attach_type field for 4137 * BPF_PROG_LOAD command 4138 */ 4139 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 4140 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; 4141 attr.insns = insns; 4142 attr.insns_cnt = ARRAY_SIZE(insns); 4143 attr.license = "GPL"; 4144 4145 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4146 } 4147 4148 static int probe_kern_probe_read_kernel(void) 4149 { 4150 struct bpf_load_program_attr attr; 4151 struct bpf_insn insns[] = { 4152 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 4153 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 4154 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 4155 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 4156 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 4157 BPF_EXIT_INSN(), 4158 }; 4159 4160 memset(&attr, 0, sizeof(attr)); 4161 attr.prog_type = BPF_PROG_TYPE_KPROBE; 4162 attr.insns = insns; 4163 attr.insns_cnt = ARRAY_SIZE(insns); 4164 attr.license = "GPL"; 4165 4166 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4167 } 4168 4169 static int probe_prog_bind_map(void) 4170 { 4171 struct bpf_load_program_attr prg_attr; 4172 struct bpf_create_map_attr map_attr; 4173 char *cp, errmsg[STRERR_BUFSIZE]; 4174 struct bpf_insn insns[] = { 4175 BPF_MOV64_IMM(BPF_REG_0, 0), 4176 BPF_EXIT_INSN(), 4177 }; 4178 int ret, map, prog; 4179 4180 memset(&map_attr, 0, sizeof(map_attr)); 4181 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4182 map_attr.key_size = sizeof(int); 4183 map_attr.value_size = 32; 4184 map_attr.max_entries = 1; 4185 4186 map = bpf_create_map_xattr(&map_attr); 4187 if (map < 0) { 4188 ret = -errno; 4189 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4190 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4191 __func__, cp, -ret); 4192 return ret; 4193 } 4194 4195 memset(&prg_attr, 0, sizeof(prg_attr)); 4196 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4197 prg_attr.insns = insns; 4198 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4199 prg_attr.license = "GPL"; 4200 4201 prog = bpf_load_program_xattr(&prg_attr, NULL, 0); 4202 if (prog < 0) { 4203 close(map); 4204 return 0; 4205 } 4206 4207 ret = bpf_prog_bind_map(prog, map, NULL); 4208 4209 close(map); 4210 close(prog); 4211 4212 return ret >= 0; 4213 } 4214 4215 static int probe_module_btf(void) 4216 { 4217 static const char strs[] = "\0int"; 4218 __u32 types[] = { 4219 /* int */ 4220 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4221 }; 4222 struct bpf_btf_info info; 4223 __u32 len = sizeof(info); 4224 char name[16]; 4225 int fd, err; 4226 4227 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); 4228 if (fd < 0) 4229 return 0; /* BTF not supported at all */ 4230 4231 memset(&info, 0, sizeof(info)); 4232 info.name = ptr_to_u64(name); 4233 info.name_len = sizeof(name); 4234 4235 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 4236 * kernel's module BTF support coincides with support for 4237 * name/name_len fields in struct bpf_btf_info. 4238 */ 4239 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4240 close(fd); 4241 return !err; 4242 } 4243 4244 enum kern_feature_result { 4245 FEAT_UNKNOWN = 0, 4246 FEAT_SUPPORTED = 1, 4247 FEAT_MISSING = 2, 4248 }; 4249 4250 typedef int (*feature_probe_fn)(void); 4251 4252 static struct kern_feature_desc { 4253 const char *desc; 4254 feature_probe_fn probe; 4255 enum kern_feature_result res; 4256 } feature_probes[__FEAT_CNT] = { 4257 [FEAT_PROG_NAME] = { 4258 "BPF program name", probe_kern_prog_name, 4259 }, 4260 [FEAT_GLOBAL_DATA] = { 4261 "global variables", probe_kern_global_data, 4262 }, 4263 [FEAT_BTF] = { 4264 "minimal BTF", probe_kern_btf, 4265 }, 4266 [FEAT_BTF_FUNC] = { 4267 "BTF functions", probe_kern_btf_func, 4268 }, 4269 [FEAT_BTF_GLOBAL_FUNC] = { 4270 "BTF global function", probe_kern_btf_func_global, 4271 }, 4272 [FEAT_BTF_DATASEC] = { 4273 "BTF data section and variable", probe_kern_btf_datasec, 4274 }, 4275 [FEAT_ARRAY_MMAP] = { 4276 "ARRAY map mmap()", probe_kern_array_mmap, 4277 }, 4278 [FEAT_EXP_ATTACH_TYPE] = { 4279 "BPF_PROG_LOAD expected_attach_type attribute", 4280 probe_kern_exp_attach_type, 4281 }, 4282 [FEAT_PROBE_READ_KERN] = { 4283 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 4284 }, 4285 [FEAT_PROG_BIND_MAP] = { 4286 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 4287 }, 4288 [FEAT_MODULE_BTF] = { 4289 "module BTF support", probe_module_btf, 4290 }, 4291 [FEAT_BTF_FLOAT] = { 4292 "BTF_KIND_FLOAT support", probe_kern_btf_float, 4293 }, 4294 }; 4295 4296 static bool kernel_supports(enum kern_feature_id feat_id) 4297 { 4298 struct kern_feature_desc *feat = &feature_probes[feat_id]; 4299 int ret; 4300 4301 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { 4302 ret = feat->probe(); 4303 if (ret > 0) { 4304 WRITE_ONCE(feat->res, FEAT_SUPPORTED); 4305 } else if (ret == 0) { 4306 WRITE_ONCE(feat->res, FEAT_MISSING); 4307 } else { 4308 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); 4309 WRITE_ONCE(feat->res, FEAT_MISSING); 4310 } 4311 } 4312 4313 return READ_ONCE(feat->res) == FEAT_SUPPORTED; 4314 } 4315 4316 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 4317 { 4318 struct bpf_map_info map_info = {}; 4319 char msg[STRERR_BUFSIZE]; 4320 __u32 map_info_len; 4321 4322 map_info_len = sizeof(map_info); 4323 4324 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) { 4325 pr_warn("failed to get map info for map FD %d: %s\n", 4326 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg))); 4327 return false; 4328 } 4329 4330 return (map_info.type == map->def.type && 4331 map_info.key_size == map->def.key_size && 4332 map_info.value_size == map->def.value_size && 4333 map_info.max_entries == map->def.max_entries && 4334 map_info.map_flags == map->def.map_flags); 4335 } 4336 4337 static int 4338 bpf_object__reuse_map(struct bpf_map *map) 4339 { 4340 char *cp, errmsg[STRERR_BUFSIZE]; 4341 int err, pin_fd; 4342 4343 pin_fd = bpf_obj_get(map->pin_path); 4344 if (pin_fd < 0) { 4345 err = -errno; 4346 if (err == -ENOENT) { 4347 pr_debug("found no pinned map to reuse at '%s'\n", 4348 map->pin_path); 4349 return 0; 4350 } 4351 4352 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 4353 pr_warn("couldn't retrieve pinned map '%s': %s\n", 4354 map->pin_path, cp); 4355 return err; 4356 } 4357 4358 if (!map_is_reuse_compat(map, pin_fd)) { 4359 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 4360 map->pin_path); 4361 close(pin_fd); 4362 return -EINVAL; 4363 } 4364 4365 err = bpf_map__reuse_fd(map, pin_fd); 4366 if (err) { 4367 close(pin_fd); 4368 return err; 4369 } 4370 map->pinned = true; 4371 pr_debug("reused pinned map at '%s'\n", map->pin_path); 4372 4373 return 0; 4374 } 4375 4376 static int 4377 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 4378 { 4379 enum libbpf_map_type map_type = map->libbpf_type; 4380 char *cp, errmsg[STRERR_BUFSIZE]; 4381 int err, zero = 0; 4382 4383 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 4384 if (err) { 4385 err = -errno; 4386 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4387 pr_warn("Error setting initial map(%s) contents: %s\n", 4388 map->name, cp); 4389 return err; 4390 } 4391 4392 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 4393 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 4394 err = bpf_map_freeze(map->fd); 4395 if (err) { 4396 err = -errno; 4397 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4398 pr_warn("Error freezing map(%s) as read-only: %s\n", 4399 map->name, cp); 4400 return err; 4401 } 4402 } 4403 return 0; 4404 } 4405 4406 static void bpf_map__destroy(struct bpf_map *map); 4407 4408 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map) 4409 { 4410 struct bpf_create_map_attr create_attr; 4411 struct bpf_map_def *def = &map->def; 4412 4413 memset(&create_attr, 0, sizeof(create_attr)); 4414 4415 if (kernel_supports(FEAT_PROG_NAME)) 4416 create_attr.name = map->name; 4417 create_attr.map_ifindex = map->map_ifindex; 4418 create_attr.map_type = def->type; 4419 create_attr.map_flags = def->map_flags; 4420 create_attr.key_size = def->key_size; 4421 create_attr.value_size = def->value_size; 4422 create_attr.numa_node = map->numa_node; 4423 4424 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) { 4425 int nr_cpus; 4426 4427 nr_cpus = libbpf_num_possible_cpus(); 4428 if (nr_cpus < 0) { 4429 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 4430 map->name, nr_cpus); 4431 return nr_cpus; 4432 } 4433 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 4434 create_attr.max_entries = nr_cpus; 4435 } else { 4436 create_attr.max_entries = def->max_entries; 4437 } 4438 4439 if (bpf_map__is_struct_ops(map)) 4440 create_attr.btf_vmlinux_value_type_id = 4441 map->btf_vmlinux_value_type_id; 4442 4443 create_attr.btf_fd = 0; 4444 create_attr.btf_key_type_id = 0; 4445 create_attr.btf_value_type_id = 0; 4446 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) { 4447 create_attr.btf_fd = btf__fd(obj->btf); 4448 create_attr.btf_key_type_id = map->btf_key_type_id; 4449 create_attr.btf_value_type_id = map->btf_value_type_id; 4450 } 4451 4452 if (bpf_map_type__is_map_in_map(def->type)) { 4453 if (map->inner_map) { 4454 int err; 4455 4456 err = bpf_object__create_map(obj, map->inner_map); 4457 if (err) { 4458 pr_warn("map '%s': failed to create inner map: %d\n", 4459 map->name, err); 4460 return err; 4461 } 4462 map->inner_map_fd = bpf_map__fd(map->inner_map); 4463 } 4464 if (map->inner_map_fd >= 0) 4465 create_attr.inner_map_fd = map->inner_map_fd; 4466 } 4467 4468 map->fd = bpf_create_map_xattr(&create_attr); 4469 if (map->fd < 0 && (create_attr.btf_key_type_id || 4470 create_attr.btf_value_type_id)) { 4471 char *cp, errmsg[STRERR_BUFSIZE]; 4472 int err = -errno; 4473 4474 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4475 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 4476 map->name, cp, err); 4477 create_attr.btf_fd = 0; 4478 create_attr.btf_key_type_id = 0; 4479 create_attr.btf_value_type_id = 0; 4480 map->btf_key_type_id = 0; 4481 map->btf_value_type_id = 0; 4482 map->fd = bpf_create_map_xattr(&create_attr); 4483 } 4484 4485 if (map->fd < 0) 4486 return -errno; 4487 4488 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 4489 bpf_map__destroy(map->inner_map); 4490 zfree(&map->inner_map); 4491 } 4492 4493 return 0; 4494 } 4495 4496 static int init_map_slots(struct bpf_map *map) 4497 { 4498 const struct bpf_map *targ_map; 4499 unsigned int i; 4500 int fd, err; 4501 4502 for (i = 0; i < map->init_slots_sz; i++) { 4503 if (!map->init_slots[i]) 4504 continue; 4505 4506 targ_map = map->init_slots[i]; 4507 fd = bpf_map__fd(targ_map); 4508 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 4509 if (err) { 4510 err = -errno; 4511 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 4512 map->name, i, targ_map->name, 4513 fd, err); 4514 return err; 4515 } 4516 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 4517 map->name, i, targ_map->name, fd); 4518 } 4519 4520 zfree(&map->init_slots); 4521 map->init_slots_sz = 0; 4522 4523 return 0; 4524 } 4525 4526 static int 4527 bpf_object__create_maps(struct bpf_object *obj) 4528 { 4529 struct bpf_map *map; 4530 char *cp, errmsg[STRERR_BUFSIZE]; 4531 unsigned int i, j; 4532 int err; 4533 4534 for (i = 0; i < obj->nr_maps; i++) { 4535 map = &obj->maps[i]; 4536 4537 if (map->pin_path) { 4538 err = bpf_object__reuse_map(map); 4539 if (err) { 4540 pr_warn("map '%s': error reusing pinned map\n", 4541 map->name); 4542 goto err_out; 4543 } 4544 } 4545 4546 if (map->fd >= 0) { 4547 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 4548 map->name, map->fd); 4549 } else { 4550 err = bpf_object__create_map(obj, map); 4551 if (err) 4552 goto err_out; 4553 4554 pr_debug("map '%s': created successfully, fd=%d\n", 4555 map->name, map->fd); 4556 4557 if (bpf_map__is_internal(map)) { 4558 err = bpf_object__populate_internal_map(obj, map); 4559 if (err < 0) { 4560 zclose(map->fd); 4561 goto err_out; 4562 } 4563 } 4564 4565 if (map->init_slots_sz) { 4566 err = init_map_slots(map); 4567 if (err < 0) { 4568 zclose(map->fd); 4569 goto err_out; 4570 } 4571 } 4572 } 4573 4574 if (map->pin_path && !map->pinned) { 4575 err = bpf_map__pin(map, NULL); 4576 if (err) { 4577 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 4578 map->name, map->pin_path, err); 4579 zclose(map->fd); 4580 goto err_out; 4581 } 4582 } 4583 } 4584 4585 return 0; 4586 4587 err_out: 4588 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4589 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 4590 pr_perm_msg(err); 4591 for (j = 0; j < i; j++) 4592 zclose(obj->maps[j].fd); 4593 return err; 4594 } 4595 4596 #define BPF_CORE_SPEC_MAX_LEN 64 4597 4598 /* represents BPF CO-RE field or array element accessor */ 4599 struct bpf_core_accessor { 4600 __u32 type_id; /* struct/union type or array element type */ 4601 __u32 idx; /* field index or array index */ 4602 const char *name; /* field name or NULL for array accessor */ 4603 }; 4604 4605 struct bpf_core_spec { 4606 const struct btf *btf; 4607 /* high-level spec: named fields and array indices only */ 4608 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; 4609 /* original unresolved (no skip_mods_or_typedefs) root type ID */ 4610 __u32 root_type_id; 4611 /* CO-RE relocation kind */ 4612 enum bpf_core_relo_kind relo_kind; 4613 /* high-level spec length */ 4614 int len; 4615 /* raw, low-level spec: 1-to-1 with accessor spec string */ 4616 int raw_spec[BPF_CORE_SPEC_MAX_LEN]; 4617 /* raw spec length */ 4618 int raw_len; 4619 /* field bit offset represented by spec */ 4620 __u32 bit_offset; 4621 }; 4622 4623 static bool str_is_empty(const char *s) 4624 { 4625 return !s || !s[0]; 4626 } 4627 4628 static bool is_flex_arr(const struct btf *btf, 4629 const struct bpf_core_accessor *acc, 4630 const struct btf_array *arr) 4631 { 4632 const struct btf_type *t; 4633 4634 /* not a flexible array, if not inside a struct or has non-zero size */ 4635 if (!acc->name || arr->nelems > 0) 4636 return false; 4637 4638 /* has to be the last member of enclosing struct */ 4639 t = btf__type_by_id(btf, acc->type_id); 4640 return acc->idx == btf_vlen(t) - 1; 4641 } 4642 4643 static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) 4644 { 4645 switch (kind) { 4646 case BPF_FIELD_BYTE_OFFSET: return "byte_off"; 4647 case BPF_FIELD_BYTE_SIZE: return "byte_sz"; 4648 case BPF_FIELD_EXISTS: return "field_exists"; 4649 case BPF_FIELD_SIGNED: return "signed"; 4650 case BPF_FIELD_LSHIFT_U64: return "lshift_u64"; 4651 case BPF_FIELD_RSHIFT_U64: return "rshift_u64"; 4652 case BPF_TYPE_ID_LOCAL: return "local_type_id"; 4653 case BPF_TYPE_ID_TARGET: return "target_type_id"; 4654 case BPF_TYPE_EXISTS: return "type_exists"; 4655 case BPF_TYPE_SIZE: return "type_size"; 4656 case BPF_ENUMVAL_EXISTS: return "enumval_exists"; 4657 case BPF_ENUMVAL_VALUE: return "enumval_value"; 4658 default: return "unknown"; 4659 } 4660 } 4661 4662 static bool core_relo_is_field_based(enum bpf_core_relo_kind kind) 4663 { 4664 switch (kind) { 4665 case BPF_FIELD_BYTE_OFFSET: 4666 case BPF_FIELD_BYTE_SIZE: 4667 case BPF_FIELD_EXISTS: 4668 case BPF_FIELD_SIGNED: 4669 case BPF_FIELD_LSHIFT_U64: 4670 case BPF_FIELD_RSHIFT_U64: 4671 return true; 4672 default: 4673 return false; 4674 } 4675 } 4676 4677 static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) 4678 { 4679 switch (kind) { 4680 case BPF_TYPE_ID_LOCAL: 4681 case BPF_TYPE_ID_TARGET: 4682 case BPF_TYPE_EXISTS: 4683 case BPF_TYPE_SIZE: 4684 return true; 4685 default: 4686 return false; 4687 } 4688 } 4689 4690 static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) 4691 { 4692 switch (kind) { 4693 case BPF_ENUMVAL_EXISTS: 4694 case BPF_ENUMVAL_VALUE: 4695 return true; 4696 default: 4697 return false; 4698 } 4699 } 4700 4701 /* 4702 * Turn bpf_core_relo into a low- and high-level spec representation, 4703 * validating correctness along the way, as well as calculating resulting 4704 * field bit offset, specified by accessor string. Low-level spec captures 4705 * every single level of nestedness, including traversing anonymous 4706 * struct/union members. High-level one only captures semantically meaningful 4707 * "turning points": named fields and array indicies. 4708 * E.g., for this case: 4709 * 4710 * struct sample { 4711 * int __unimportant; 4712 * struct { 4713 * int __1; 4714 * int __2; 4715 * int a[7]; 4716 * }; 4717 * }; 4718 * 4719 * struct sample *s = ...; 4720 * 4721 * int x = &s->a[3]; // access string = '0:1:2:3' 4722 * 4723 * Low-level spec has 1:1 mapping with each element of access string (it's 4724 * just a parsed access string representation): [0, 1, 2, 3]. 4725 * 4726 * High-level spec will capture only 3 points: 4727 * - intial zero-index access by pointer (&s->... is the same as &s[0]...); 4728 * - field 'a' access (corresponds to '2' in low-level spec); 4729 * - array element #3 access (corresponds to '3' in low-level spec). 4730 * 4731 * Type-based relocations (TYPE_EXISTS/TYPE_SIZE, 4732 * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their 4733 * spec and raw_spec are kept empty. 4734 * 4735 * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access 4736 * string to specify enumerator's value index that need to be relocated. 4737 */ 4738 static int bpf_core_parse_spec(const struct btf *btf, 4739 __u32 type_id, 4740 const char *spec_str, 4741 enum bpf_core_relo_kind relo_kind, 4742 struct bpf_core_spec *spec) 4743 { 4744 int access_idx, parsed_len, i; 4745 struct bpf_core_accessor *acc; 4746 const struct btf_type *t; 4747 const char *name; 4748 __u32 id; 4749 __s64 sz; 4750 4751 if (str_is_empty(spec_str) || *spec_str == ':') 4752 return -EINVAL; 4753 4754 memset(spec, 0, sizeof(*spec)); 4755 spec->btf = btf; 4756 spec->root_type_id = type_id; 4757 spec->relo_kind = relo_kind; 4758 4759 /* type-based relocations don't have a field access string */ 4760 if (core_relo_is_type_based(relo_kind)) { 4761 if (strcmp(spec_str, "0")) 4762 return -EINVAL; 4763 return 0; 4764 } 4765 4766 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ 4767 while (*spec_str) { 4768 if (*spec_str == ':') 4769 ++spec_str; 4770 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) 4771 return -EINVAL; 4772 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 4773 return -E2BIG; 4774 spec_str += parsed_len; 4775 spec->raw_spec[spec->raw_len++] = access_idx; 4776 } 4777 4778 if (spec->raw_len == 0) 4779 return -EINVAL; 4780 4781 t = skip_mods_and_typedefs(btf, type_id, &id); 4782 if (!t) 4783 return -EINVAL; 4784 4785 access_idx = spec->raw_spec[0]; 4786 acc = &spec->spec[0]; 4787 acc->type_id = id; 4788 acc->idx = access_idx; 4789 spec->len++; 4790 4791 if (core_relo_is_enumval_based(relo_kind)) { 4792 if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t)) 4793 return -EINVAL; 4794 4795 /* record enumerator name in a first accessor */ 4796 acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off); 4797 return 0; 4798 } 4799 4800 if (!core_relo_is_field_based(relo_kind)) 4801 return -EINVAL; 4802 4803 sz = btf__resolve_size(btf, id); 4804 if (sz < 0) 4805 return sz; 4806 spec->bit_offset = access_idx * sz * 8; 4807 4808 for (i = 1; i < spec->raw_len; i++) { 4809 t = skip_mods_and_typedefs(btf, id, &id); 4810 if (!t) 4811 return -EINVAL; 4812 4813 access_idx = spec->raw_spec[i]; 4814 acc = &spec->spec[spec->len]; 4815 4816 if (btf_is_composite(t)) { 4817 const struct btf_member *m; 4818 __u32 bit_offset; 4819 4820 if (access_idx >= btf_vlen(t)) 4821 return -EINVAL; 4822 4823 bit_offset = btf_member_bit_offset(t, access_idx); 4824 spec->bit_offset += bit_offset; 4825 4826 m = btf_members(t) + access_idx; 4827 if (m->name_off) { 4828 name = btf__name_by_offset(btf, m->name_off); 4829 if (str_is_empty(name)) 4830 return -EINVAL; 4831 4832 acc->type_id = id; 4833 acc->idx = access_idx; 4834 acc->name = name; 4835 spec->len++; 4836 } 4837 4838 id = m->type; 4839 } else if (btf_is_array(t)) { 4840 const struct btf_array *a = btf_array(t); 4841 bool flex; 4842 4843 t = skip_mods_and_typedefs(btf, a->type, &id); 4844 if (!t) 4845 return -EINVAL; 4846 4847 flex = is_flex_arr(btf, acc - 1, a); 4848 if (!flex && access_idx >= a->nelems) 4849 return -EINVAL; 4850 4851 spec->spec[spec->len].type_id = id; 4852 spec->spec[spec->len].idx = access_idx; 4853 spec->len++; 4854 4855 sz = btf__resolve_size(btf, id); 4856 if (sz < 0) 4857 return sz; 4858 spec->bit_offset += access_idx * sz * 8; 4859 } else { 4860 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", 4861 type_id, spec_str, i, id, btf_kind_str(t)); 4862 return -EINVAL; 4863 } 4864 } 4865 4866 return 0; 4867 } 4868 4869 static bool bpf_core_is_flavor_sep(const char *s) 4870 { 4871 /* check X___Y name pattern, where X and Y are not underscores */ 4872 return s[0] != '_' && /* X */ 4873 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 4874 s[4] != '_'; /* Y */ 4875 } 4876 4877 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 4878 * before last triple underscore. Struct name part after last triple 4879 * underscore is ignored by BPF CO-RE relocation during relocation matching. 4880 */ 4881 static size_t bpf_core_essential_name_len(const char *name) 4882 { 4883 size_t n = strlen(name); 4884 int i; 4885 4886 for (i = n - 5; i >= 0; i--) { 4887 if (bpf_core_is_flavor_sep(name + i)) 4888 return i + 1; 4889 } 4890 return n; 4891 } 4892 4893 struct core_cand 4894 { 4895 const struct btf *btf; 4896 const struct btf_type *t; 4897 const char *name; 4898 __u32 id; 4899 }; 4900 4901 /* dynamically sized list of type IDs and its associated struct btf */ 4902 struct core_cand_list { 4903 struct core_cand *cands; 4904 int len; 4905 }; 4906 4907 static void bpf_core_free_cands(struct core_cand_list *cands) 4908 { 4909 free(cands->cands); 4910 free(cands); 4911 } 4912 4913 static int bpf_core_add_cands(struct core_cand *local_cand, 4914 size_t local_essent_len, 4915 const struct btf *targ_btf, 4916 const char *targ_btf_name, 4917 int targ_start_id, 4918 struct core_cand_list *cands) 4919 { 4920 struct core_cand *new_cands, *cand; 4921 const struct btf_type *t; 4922 const char *targ_name; 4923 size_t targ_essent_len; 4924 int n, i; 4925 4926 n = btf__get_nr_types(targ_btf); 4927 for (i = targ_start_id; i <= n; i++) { 4928 t = btf__type_by_id(targ_btf, i); 4929 if (btf_kind(t) != btf_kind(local_cand->t)) 4930 continue; 4931 4932 targ_name = btf__name_by_offset(targ_btf, t->name_off); 4933 if (str_is_empty(targ_name)) 4934 continue; 4935 4936 targ_essent_len = bpf_core_essential_name_len(targ_name); 4937 if (targ_essent_len != local_essent_len) 4938 continue; 4939 4940 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0) 4941 continue; 4942 4943 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 4944 local_cand->id, btf_kind_str(local_cand->t), 4945 local_cand->name, i, btf_kind_str(t), targ_name, 4946 targ_btf_name); 4947 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 4948 sizeof(*cands->cands)); 4949 if (!new_cands) 4950 return -ENOMEM; 4951 4952 cand = &new_cands[cands->len]; 4953 cand->btf = targ_btf; 4954 cand->t = t; 4955 cand->name = targ_name; 4956 cand->id = i; 4957 4958 cands->cands = new_cands; 4959 cands->len++; 4960 } 4961 return 0; 4962 } 4963 4964 static int load_module_btfs(struct bpf_object *obj) 4965 { 4966 struct bpf_btf_info info; 4967 struct module_btf *mod_btf; 4968 struct btf *btf; 4969 char name[64]; 4970 __u32 id = 0, len; 4971 int err, fd; 4972 4973 if (obj->btf_modules_loaded) 4974 return 0; 4975 4976 /* don't do this again, even if we find no module BTFs */ 4977 obj->btf_modules_loaded = true; 4978 4979 /* kernel too old to support module BTFs */ 4980 if (!kernel_supports(FEAT_MODULE_BTF)) 4981 return 0; 4982 4983 while (true) { 4984 err = bpf_btf_get_next_id(id, &id); 4985 if (err && errno == ENOENT) 4986 return 0; 4987 if (err) { 4988 err = -errno; 4989 pr_warn("failed to iterate BTF objects: %d\n", err); 4990 return err; 4991 } 4992 4993 fd = bpf_btf_get_fd_by_id(id); 4994 if (fd < 0) { 4995 if (errno == ENOENT) 4996 continue; /* expected race: BTF was unloaded */ 4997 err = -errno; 4998 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 4999 return err; 5000 } 5001 5002 len = sizeof(info); 5003 memset(&info, 0, sizeof(info)); 5004 info.name = ptr_to_u64(name); 5005 info.name_len = sizeof(name); 5006 5007 err = bpf_obj_get_info_by_fd(fd, &info, &len); 5008 if (err) { 5009 err = -errno; 5010 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 5011 goto err_out; 5012 } 5013 5014 /* ignore non-module BTFs */ 5015 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 5016 close(fd); 5017 continue; 5018 } 5019 5020 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 5021 if (IS_ERR(btf)) { 5022 pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n", 5023 name, id, PTR_ERR(btf)); 5024 err = PTR_ERR(btf); 5025 goto err_out; 5026 } 5027 5028 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 5029 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 5030 if (err) 5031 goto err_out; 5032 5033 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 5034 5035 mod_btf->btf = btf; 5036 mod_btf->id = id; 5037 mod_btf->fd = fd; 5038 mod_btf->name = strdup(name); 5039 if (!mod_btf->name) { 5040 err = -ENOMEM; 5041 goto err_out; 5042 } 5043 continue; 5044 5045 err_out: 5046 close(fd); 5047 return err; 5048 } 5049 5050 return 0; 5051 } 5052 5053 static struct core_cand_list * 5054 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 5055 { 5056 struct core_cand local_cand = {}; 5057 struct core_cand_list *cands; 5058 const struct btf *main_btf; 5059 size_t local_essent_len; 5060 int err, i; 5061 5062 local_cand.btf = local_btf; 5063 local_cand.t = btf__type_by_id(local_btf, local_type_id); 5064 if (!local_cand.t) 5065 return ERR_PTR(-EINVAL); 5066 5067 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off); 5068 if (str_is_empty(local_cand.name)) 5069 return ERR_PTR(-EINVAL); 5070 local_essent_len = bpf_core_essential_name_len(local_cand.name); 5071 5072 cands = calloc(1, sizeof(*cands)); 5073 if (!cands) 5074 return ERR_PTR(-ENOMEM); 5075 5076 /* Attempt to find target candidates in vmlinux BTF first */ 5077 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 5078 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 5079 if (err) 5080 goto err_out; 5081 5082 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 5083 if (cands->len) 5084 return cands; 5085 5086 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 5087 if (obj->btf_vmlinux_override) 5088 return cands; 5089 5090 /* now look through module BTFs, trying to still find candidates */ 5091 err = load_module_btfs(obj); 5092 if (err) 5093 goto err_out; 5094 5095 for (i = 0; i < obj->btf_module_cnt; i++) { 5096 err = bpf_core_add_cands(&local_cand, local_essent_len, 5097 obj->btf_modules[i].btf, 5098 obj->btf_modules[i].name, 5099 btf__get_nr_types(obj->btf_vmlinux) + 1, 5100 cands); 5101 if (err) 5102 goto err_out; 5103 } 5104 5105 return cands; 5106 err_out: 5107 bpf_core_free_cands(cands); 5108 return ERR_PTR(err); 5109 } 5110 5111 /* Check two types for compatibility for the purpose of field access 5112 * relocation. const/volatile/restrict and typedefs are skipped to ensure we 5113 * are relocating semantically compatible entities: 5114 * - any two STRUCTs/UNIONs are compatible and can be mixed; 5115 * - any two FWDs are compatible, if their names match (modulo flavor suffix); 5116 * - any two PTRs are always compatible; 5117 * - for ENUMs, names should be the same (ignoring flavor suffix) or at 5118 * least one of enums should be anonymous; 5119 * - for ENUMs, check sizes, names are ignored; 5120 * - for INT, size and signedness are ignored; 5121 * - any two FLOATs are always compatible; 5122 * - for ARRAY, dimensionality is ignored, element types are checked for 5123 * compatibility recursively; 5124 * - everything else shouldn't be ever a target of relocation. 5125 * These rules are not set in stone and probably will be adjusted as we get 5126 * more experience with using BPF CO-RE relocations. 5127 */ 5128 static int bpf_core_fields_are_compat(const struct btf *local_btf, 5129 __u32 local_id, 5130 const struct btf *targ_btf, 5131 __u32 targ_id) 5132 { 5133 const struct btf_type *local_type, *targ_type; 5134 5135 recur: 5136 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 5137 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 5138 if (!local_type || !targ_type) 5139 return -EINVAL; 5140 5141 if (btf_is_composite(local_type) && btf_is_composite(targ_type)) 5142 return 1; 5143 if (btf_kind(local_type) != btf_kind(targ_type)) 5144 return 0; 5145 5146 switch (btf_kind(local_type)) { 5147 case BTF_KIND_PTR: 5148 case BTF_KIND_FLOAT: 5149 return 1; 5150 case BTF_KIND_FWD: 5151 case BTF_KIND_ENUM: { 5152 const char *local_name, *targ_name; 5153 size_t local_len, targ_len; 5154 5155 local_name = btf__name_by_offset(local_btf, 5156 local_type->name_off); 5157 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); 5158 local_len = bpf_core_essential_name_len(local_name); 5159 targ_len = bpf_core_essential_name_len(targ_name); 5160 /* one of them is anonymous or both w/ same flavor-less names */ 5161 return local_len == 0 || targ_len == 0 || 5162 (local_len == targ_len && 5163 strncmp(local_name, targ_name, local_len) == 0); 5164 } 5165 case BTF_KIND_INT: 5166 /* just reject deprecated bitfield-like integers; all other 5167 * integers are by default compatible between each other 5168 */ 5169 return btf_int_offset(local_type) == 0 && 5170 btf_int_offset(targ_type) == 0; 5171 case BTF_KIND_ARRAY: 5172 local_id = btf_array(local_type)->type; 5173 targ_id = btf_array(targ_type)->type; 5174 goto recur; 5175 default: 5176 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", 5177 btf_kind(local_type), local_id, targ_id); 5178 return 0; 5179 } 5180 } 5181 5182 /* 5183 * Given single high-level named field accessor in local type, find 5184 * corresponding high-level accessor for a target type. Along the way, 5185 * maintain low-level spec for target as well. Also keep updating target 5186 * bit offset. 5187 * 5188 * Searching is performed through recursive exhaustive enumeration of all 5189 * fields of a struct/union. If there are any anonymous (embedded) 5190 * structs/unions, they are recursively searched as well. If field with 5191 * desired name is found, check compatibility between local and target types, 5192 * before returning result. 5193 * 5194 * 1 is returned, if field is found. 5195 * 0 is returned if no compatible field is found. 5196 * <0 is returned on error. 5197 */ 5198 static int bpf_core_match_member(const struct btf *local_btf, 5199 const struct bpf_core_accessor *local_acc, 5200 const struct btf *targ_btf, 5201 __u32 targ_id, 5202 struct bpf_core_spec *spec, 5203 __u32 *next_targ_id) 5204 { 5205 const struct btf_type *local_type, *targ_type; 5206 const struct btf_member *local_member, *m; 5207 const char *local_name, *targ_name; 5208 __u32 local_id; 5209 int i, n, found; 5210 5211 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 5212 if (!targ_type) 5213 return -EINVAL; 5214 if (!btf_is_composite(targ_type)) 5215 return 0; 5216 5217 local_id = local_acc->type_id; 5218 local_type = btf__type_by_id(local_btf, local_id); 5219 local_member = btf_members(local_type) + local_acc->idx; 5220 local_name = btf__name_by_offset(local_btf, local_member->name_off); 5221 5222 n = btf_vlen(targ_type); 5223 m = btf_members(targ_type); 5224 for (i = 0; i < n; i++, m++) { 5225 __u32 bit_offset; 5226 5227 bit_offset = btf_member_bit_offset(targ_type, i); 5228 5229 /* too deep struct/union/array nesting */ 5230 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 5231 return -E2BIG; 5232 5233 /* speculate this member will be the good one */ 5234 spec->bit_offset += bit_offset; 5235 spec->raw_spec[spec->raw_len++] = i; 5236 5237 targ_name = btf__name_by_offset(targ_btf, m->name_off); 5238 if (str_is_empty(targ_name)) { 5239 /* embedded struct/union, we need to go deeper */ 5240 found = bpf_core_match_member(local_btf, local_acc, 5241 targ_btf, m->type, 5242 spec, next_targ_id); 5243 if (found) /* either found or error */ 5244 return found; 5245 } else if (strcmp(local_name, targ_name) == 0) { 5246 /* matching named field */ 5247 struct bpf_core_accessor *targ_acc; 5248 5249 targ_acc = &spec->spec[spec->len++]; 5250 targ_acc->type_id = targ_id; 5251 targ_acc->idx = i; 5252 targ_acc->name = targ_name; 5253 5254 *next_targ_id = m->type; 5255 found = bpf_core_fields_are_compat(local_btf, 5256 local_member->type, 5257 targ_btf, m->type); 5258 if (!found) 5259 spec->len--; /* pop accessor */ 5260 return found; 5261 } 5262 /* member turned out not to be what we looked for */ 5263 spec->bit_offset -= bit_offset; 5264 spec->raw_len--; 5265 } 5266 5267 return 0; 5268 } 5269 5270 /* Check local and target types for compatibility. This check is used for 5271 * type-based CO-RE relocations and follow slightly different rules than 5272 * field-based relocations. This function assumes that root types were already 5273 * checked for name match. Beyond that initial root-level name check, names 5274 * are completely ignored. Compatibility rules are as follows: 5275 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 5276 * kind should match for local and target types (i.e., STRUCT is not 5277 * compatible with UNION); 5278 * - for ENUMs, the size is ignored; 5279 * - for INT, size and signedness are ignored; 5280 * - for ARRAY, dimensionality is ignored, element types are checked for 5281 * compatibility recursively; 5282 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 5283 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 5284 * - FUNC_PROTOs are compatible if they have compatible signature: same 5285 * number of input args and compatible return and argument types. 5286 * These rules are not set in stone and probably will be adjusted as we get 5287 * more experience with using BPF CO-RE relocations. 5288 */ 5289 static int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 5290 const struct btf *targ_btf, __u32 targ_id) 5291 { 5292 const struct btf_type *local_type, *targ_type; 5293 int depth = 32; /* max recursion depth */ 5294 5295 /* caller made sure that names match (ignoring flavor suffix) */ 5296 local_type = btf__type_by_id(local_btf, local_id); 5297 targ_type = btf__type_by_id(targ_btf, targ_id); 5298 if (btf_kind(local_type) != btf_kind(targ_type)) 5299 return 0; 5300 5301 recur: 5302 depth--; 5303 if (depth < 0) 5304 return -EINVAL; 5305 5306 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 5307 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 5308 if (!local_type || !targ_type) 5309 return -EINVAL; 5310 5311 if (btf_kind(local_type) != btf_kind(targ_type)) 5312 return 0; 5313 5314 switch (btf_kind(local_type)) { 5315 case BTF_KIND_UNKN: 5316 case BTF_KIND_STRUCT: 5317 case BTF_KIND_UNION: 5318 case BTF_KIND_ENUM: 5319 case BTF_KIND_FWD: 5320 return 1; 5321 case BTF_KIND_INT: 5322 /* just reject deprecated bitfield-like integers; all other 5323 * integers are by default compatible between each other 5324 */ 5325 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; 5326 case BTF_KIND_PTR: 5327 local_id = local_type->type; 5328 targ_id = targ_type->type; 5329 goto recur; 5330 case BTF_KIND_ARRAY: 5331 local_id = btf_array(local_type)->type; 5332 targ_id = btf_array(targ_type)->type; 5333 goto recur; 5334 case BTF_KIND_FUNC_PROTO: { 5335 struct btf_param *local_p = btf_params(local_type); 5336 struct btf_param *targ_p = btf_params(targ_type); 5337 __u16 local_vlen = btf_vlen(local_type); 5338 __u16 targ_vlen = btf_vlen(targ_type); 5339 int i, err; 5340 5341 if (local_vlen != targ_vlen) 5342 return 0; 5343 5344 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { 5345 skip_mods_and_typedefs(local_btf, local_p->type, &local_id); 5346 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); 5347 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id); 5348 if (err <= 0) 5349 return err; 5350 } 5351 5352 /* tail recurse for return type check */ 5353 skip_mods_and_typedefs(local_btf, local_type->type, &local_id); 5354 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); 5355 goto recur; 5356 } 5357 default: 5358 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", 5359 btf_kind_str(local_type), local_id, targ_id); 5360 return 0; 5361 } 5362 } 5363 5364 /* 5365 * Try to match local spec to a target type and, if successful, produce full 5366 * target spec (high-level, low-level + bit offset). 5367 */ 5368 static int bpf_core_spec_match(struct bpf_core_spec *local_spec, 5369 const struct btf *targ_btf, __u32 targ_id, 5370 struct bpf_core_spec *targ_spec) 5371 { 5372 const struct btf_type *targ_type; 5373 const struct bpf_core_accessor *local_acc; 5374 struct bpf_core_accessor *targ_acc; 5375 int i, sz, matched; 5376 5377 memset(targ_spec, 0, sizeof(*targ_spec)); 5378 targ_spec->btf = targ_btf; 5379 targ_spec->root_type_id = targ_id; 5380 targ_spec->relo_kind = local_spec->relo_kind; 5381 5382 if (core_relo_is_type_based(local_spec->relo_kind)) { 5383 return bpf_core_types_are_compat(local_spec->btf, 5384 local_spec->root_type_id, 5385 targ_btf, targ_id); 5386 } 5387 5388 local_acc = &local_spec->spec[0]; 5389 targ_acc = &targ_spec->spec[0]; 5390 5391 if (core_relo_is_enumval_based(local_spec->relo_kind)) { 5392 size_t local_essent_len, targ_essent_len; 5393 const struct btf_enum *e; 5394 const char *targ_name; 5395 5396 /* has to resolve to an enum */ 5397 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id); 5398 if (!btf_is_enum(targ_type)) 5399 return 0; 5400 5401 local_essent_len = bpf_core_essential_name_len(local_acc->name); 5402 5403 for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) { 5404 targ_name = btf__name_by_offset(targ_spec->btf, e->name_off); 5405 targ_essent_len = bpf_core_essential_name_len(targ_name); 5406 if (targ_essent_len != local_essent_len) 5407 continue; 5408 if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) { 5409 targ_acc->type_id = targ_id; 5410 targ_acc->idx = i; 5411 targ_acc->name = targ_name; 5412 targ_spec->len++; 5413 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; 5414 targ_spec->raw_len++; 5415 return 1; 5416 } 5417 } 5418 return 0; 5419 } 5420 5421 if (!core_relo_is_field_based(local_spec->relo_kind)) 5422 return -EINVAL; 5423 5424 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { 5425 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, 5426 &targ_id); 5427 if (!targ_type) 5428 return -EINVAL; 5429 5430 if (local_acc->name) { 5431 matched = bpf_core_match_member(local_spec->btf, 5432 local_acc, 5433 targ_btf, targ_id, 5434 targ_spec, &targ_id); 5435 if (matched <= 0) 5436 return matched; 5437 } else { 5438 /* for i=0, targ_id is already treated as array element 5439 * type (because it's the original struct), for others 5440 * we should find array element type first 5441 */ 5442 if (i > 0) { 5443 const struct btf_array *a; 5444 bool flex; 5445 5446 if (!btf_is_array(targ_type)) 5447 return 0; 5448 5449 a = btf_array(targ_type); 5450 flex = is_flex_arr(targ_btf, targ_acc - 1, a); 5451 if (!flex && local_acc->idx >= a->nelems) 5452 return 0; 5453 if (!skip_mods_and_typedefs(targ_btf, a->type, 5454 &targ_id)) 5455 return -EINVAL; 5456 } 5457 5458 /* too deep struct/union/array nesting */ 5459 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 5460 return -E2BIG; 5461 5462 targ_acc->type_id = targ_id; 5463 targ_acc->idx = local_acc->idx; 5464 targ_acc->name = NULL; 5465 targ_spec->len++; 5466 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; 5467 targ_spec->raw_len++; 5468 5469 sz = btf__resolve_size(targ_btf, targ_id); 5470 if (sz < 0) 5471 return sz; 5472 targ_spec->bit_offset += local_acc->idx * sz * 8; 5473 } 5474 } 5475 5476 return 1; 5477 } 5478 5479 static int bpf_core_calc_field_relo(const struct bpf_program *prog, 5480 const struct bpf_core_relo *relo, 5481 const struct bpf_core_spec *spec, 5482 __u32 *val, __u32 *field_sz, __u32 *type_id, 5483 bool *validate) 5484 { 5485 const struct bpf_core_accessor *acc; 5486 const struct btf_type *t; 5487 __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id; 5488 const struct btf_member *m; 5489 const struct btf_type *mt; 5490 bool bitfield; 5491 __s64 sz; 5492 5493 *field_sz = 0; 5494 5495 if (relo->kind == BPF_FIELD_EXISTS) { 5496 *val = spec ? 1 : 0; 5497 return 0; 5498 } 5499 5500 if (!spec) 5501 return -EUCLEAN; /* request instruction poisoning */ 5502 5503 acc = &spec->spec[spec->len - 1]; 5504 t = btf__type_by_id(spec->btf, acc->type_id); 5505 5506 /* a[n] accessor needs special handling */ 5507 if (!acc->name) { 5508 if (relo->kind == BPF_FIELD_BYTE_OFFSET) { 5509 *val = spec->bit_offset / 8; 5510 /* remember field size for load/store mem size */ 5511 sz = btf__resolve_size(spec->btf, acc->type_id); 5512 if (sz < 0) 5513 return -EINVAL; 5514 *field_sz = sz; 5515 *type_id = acc->type_id; 5516 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) { 5517 sz = btf__resolve_size(spec->btf, acc->type_id); 5518 if (sz < 0) 5519 return -EINVAL; 5520 *val = sz; 5521 } else { 5522 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", 5523 prog->name, relo->kind, relo->insn_off / 8); 5524 return -EINVAL; 5525 } 5526 if (validate) 5527 *validate = true; 5528 return 0; 5529 } 5530 5531 m = btf_members(t) + acc->idx; 5532 mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id); 5533 bit_off = spec->bit_offset; 5534 bit_sz = btf_member_bitfield_size(t, acc->idx); 5535 5536 bitfield = bit_sz > 0; 5537 if (bitfield) { 5538 byte_sz = mt->size; 5539 byte_off = bit_off / 8 / byte_sz * byte_sz; 5540 /* figure out smallest int size necessary for bitfield load */ 5541 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { 5542 if (byte_sz >= 8) { 5543 /* bitfield can't be read with 64-bit read */ 5544 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", 5545 prog->name, relo->kind, relo->insn_off / 8); 5546 return -E2BIG; 5547 } 5548 byte_sz *= 2; 5549 byte_off = bit_off / 8 / byte_sz * byte_sz; 5550 } 5551 } else { 5552 sz = btf__resolve_size(spec->btf, field_type_id); 5553 if (sz < 0) 5554 return -EINVAL; 5555 byte_sz = sz; 5556 byte_off = spec->bit_offset / 8; 5557 bit_sz = byte_sz * 8; 5558 } 5559 5560 /* for bitfields, all the relocatable aspects are ambiguous and we 5561 * might disagree with compiler, so turn off validation of expected 5562 * value, except for signedness 5563 */ 5564 if (validate) 5565 *validate = !bitfield; 5566 5567 switch (relo->kind) { 5568 case BPF_FIELD_BYTE_OFFSET: 5569 *val = byte_off; 5570 if (!bitfield) { 5571 *field_sz = byte_sz; 5572 *type_id = field_type_id; 5573 } 5574 break; 5575 case BPF_FIELD_BYTE_SIZE: 5576 *val = byte_sz; 5577 break; 5578 case BPF_FIELD_SIGNED: 5579 /* enums will be assumed unsigned */ 5580 *val = btf_is_enum(mt) || 5581 (btf_int_encoding(mt) & BTF_INT_SIGNED); 5582 if (validate) 5583 *validate = true; /* signedness is never ambiguous */ 5584 break; 5585 case BPF_FIELD_LSHIFT_U64: 5586 #if __BYTE_ORDER == __LITTLE_ENDIAN 5587 *val = 64 - (bit_off + bit_sz - byte_off * 8); 5588 #else 5589 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); 5590 #endif 5591 break; 5592 case BPF_FIELD_RSHIFT_U64: 5593 *val = 64 - bit_sz; 5594 if (validate) 5595 *validate = true; /* right shift is never ambiguous */ 5596 break; 5597 case BPF_FIELD_EXISTS: 5598 default: 5599 return -EOPNOTSUPP; 5600 } 5601 5602 return 0; 5603 } 5604 5605 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, 5606 const struct bpf_core_spec *spec, 5607 __u32 *val) 5608 { 5609 __s64 sz; 5610 5611 /* type-based relos return zero when target type is not found */ 5612 if (!spec) { 5613 *val = 0; 5614 return 0; 5615 } 5616 5617 switch (relo->kind) { 5618 case BPF_TYPE_ID_TARGET: 5619 *val = spec->root_type_id; 5620 break; 5621 case BPF_TYPE_EXISTS: 5622 *val = 1; 5623 break; 5624 case BPF_TYPE_SIZE: 5625 sz = btf__resolve_size(spec->btf, spec->root_type_id); 5626 if (sz < 0) 5627 return -EINVAL; 5628 *val = sz; 5629 break; 5630 case BPF_TYPE_ID_LOCAL: 5631 /* BPF_TYPE_ID_LOCAL is handled specially and shouldn't get here */ 5632 default: 5633 return -EOPNOTSUPP; 5634 } 5635 5636 return 0; 5637 } 5638 5639 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, 5640 const struct bpf_core_spec *spec, 5641 __u32 *val) 5642 { 5643 const struct btf_type *t; 5644 const struct btf_enum *e; 5645 5646 switch (relo->kind) { 5647 case BPF_ENUMVAL_EXISTS: 5648 *val = spec ? 1 : 0; 5649 break; 5650 case BPF_ENUMVAL_VALUE: 5651 if (!spec) 5652 return -EUCLEAN; /* request instruction poisoning */ 5653 t = btf__type_by_id(spec->btf, spec->spec[0].type_id); 5654 e = btf_enum(t) + spec->spec[0].idx; 5655 *val = e->val; 5656 break; 5657 default: 5658 return -EOPNOTSUPP; 5659 } 5660 5661 return 0; 5662 } 5663 5664 struct bpf_core_relo_res 5665 { 5666 /* expected value in the instruction, unless validate == false */ 5667 __u32 orig_val; 5668 /* new value that needs to be patched up to */ 5669 __u32 new_val; 5670 /* relocation unsuccessful, poison instruction, but don't fail load */ 5671 bool poison; 5672 /* some relocations can't be validated against orig_val */ 5673 bool validate; 5674 /* for field byte offset relocations or the forms: 5675 * *(T *)(rX + <off>) = rY 5676 * rX = *(T *)(rY + <off>), 5677 * we remember original and resolved field size to adjust direct 5678 * memory loads of pointers and integers; this is necessary for 32-bit 5679 * host kernel architectures, but also allows to automatically 5680 * relocate fields that were resized from, e.g., u32 to u64, etc. 5681 */ 5682 bool fail_memsz_adjust; 5683 __u32 orig_sz; 5684 __u32 orig_type_id; 5685 __u32 new_sz; 5686 __u32 new_type_id; 5687 }; 5688 5689 /* Calculate original and target relocation values, given local and target 5690 * specs and relocation kind. These values are calculated for each candidate. 5691 * If there are multiple candidates, resulting values should all be consistent 5692 * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity. 5693 * If instruction has to be poisoned, *poison will be set to true. 5694 */ 5695 static int bpf_core_calc_relo(const struct bpf_program *prog, 5696 const struct bpf_core_relo *relo, 5697 int relo_idx, 5698 const struct bpf_core_spec *local_spec, 5699 const struct bpf_core_spec *targ_spec, 5700 struct bpf_core_relo_res *res) 5701 { 5702 int err = -EOPNOTSUPP; 5703 5704 res->orig_val = 0; 5705 res->new_val = 0; 5706 res->poison = false; 5707 res->validate = true; 5708 res->fail_memsz_adjust = false; 5709 res->orig_sz = res->new_sz = 0; 5710 res->orig_type_id = res->new_type_id = 0; 5711 5712 if (core_relo_is_field_based(relo->kind)) { 5713 err = bpf_core_calc_field_relo(prog, relo, local_spec, 5714 &res->orig_val, &res->orig_sz, 5715 &res->orig_type_id, &res->validate); 5716 err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec, 5717 &res->new_val, &res->new_sz, 5718 &res->new_type_id, NULL); 5719 if (err) 5720 goto done; 5721 /* Validate if it's safe to adjust load/store memory size. 5722 * Adjustments are performed only if original and new memory 5723 * sizes differ. 5724 */ 5725 res->fail_memsz_adjust = false; 5726 if (res->orig_sz != res->new_sz) { 5727 const struct btf_type *orig_t, *new_t; 5728 5729 orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id); 5730 new_t = btf__type_by_id(targ_spec->btf, res->new_type_id); 5731 5732 /* There are two use cases in which it's safe to 5733 * adjust load/store's mem size: 5734 * - reading a 32-bit kernel pointer, while on BPF 5735 * size pointers are always 64-bit; in this case 5736 * it's safe to "downsize" instruction size due to 5737 * pointer being treated as unsigned integer with 5738 * zero-extended upper 32-bits; 5739 * - reading unsigned integers, again due to 5740 * zero-extension is preserving the value correctly. 5741 * 5742 * In all other cases it's incorrect to attempt to 5743 * load/store field because read value will be 5744 * incorrect, so we poison relocated instruction. 5745 */ 5746 if (btf_is_ptr(orig_t) && btf_is_ptr(new_t)) 5747 goto done; 5748 if (btf_is_int(orig_t) && btf_is_int(new_t) && 5749 btf_int_encoding(orig_t) != BTF_INT_SIGNED && 5750 btf_int_encoding(new_t) != BTF_INT_SIGNED) 5751 goto done; 5752 5753 /* mark as invalid mem size adjustment, but this will 5754 * only be checked for LDX/STX/ST insns 5755 */ 5756 res->fail_memsz_adjust = true; 5757 } 5758 } else if (core_relo_is_type_based(relo->kind)) { 5759 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val); 5760 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val); 5761 } else if (core_relo_is_enumval_based(relo->kind)) { 5762 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val); 5763 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val); 5764 } 5765 5766 done: 5767 if (err == -EUCLEAN) { 5768 /* EUCLEAN is used to signal instruction poisoning request */ 5769 res->poison = true; 5770 err = 0; 5771 } else if (err == -EOPNOTSUPP) { 5772 /* EOPNOTSUPP means unknown/unsupported relocation */ 5773 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", 5774 prog->name, relo_idx, core_relo_kind_str(relo->kind), 5775 relo->kind, relo->insn_off / 8); 5776 } 5777 5778 return err; 5779 } 5780 5781 /* 5782 * Turn instruction for which CO_RE relocation failed into invalid one with 5783 * distinct signature. 5784 */ 5785 static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx, 5786 int insn_idx, struct bpf_insn *insn) 5787 { 5788 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", 5789 prog->name, relo_idx, insn_idx); 5790 insn->code = BPF_JMP | BPF_CALL; 5791 insn->dst_reg = 0; 5792 insn->src_reg = 0; 5793 insn->off = 0; 5794 /* if this instruction is reachable (not a dead code), 5795 * verifier will complain with the following message: 5796 * invalid func unknown#195896080 5797 */ 5798 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ 5799 } 5800 5801 static int insn_bpf_size_to_bytes(struct bpf_insn *insn) 5802 { 5803 switch (BPF_SIZE(insn->code)) { 5804 case BPF_DW: return 8; 5805 case BPF_W: return 4; 5806 case BPF_H: return 2; 5807 case BPF_B: return 1; 5808 default: return -1; 5809 } 5810 } 5811 5812 static int insn_bytes_to_bpf_size(__u32 sz) 5813 { 5814 switch (sz) { 5815 case 8: return BPF_DW; 5816 case 4: return BPF_W; 5817 case 2: return BPF_H; 5818 case 1: return BPF_B; 5819 default: return -1; 5820 } 5821 } 5822 5823 /* 5824 * Patch relocatable BPF instruction. 5825 * 5826 * Patched value is determined by relocation kind and target specification. 5827 * For existence relocations target spec will be NULL if field/type is not found. 5828 * Expected insn->imm value is determined using relocation kind and local 5829 * spec, and is checked before patching instruction. If actual insn->imm value 5830 * is wrong, bail out with error. 5831 * 5832 * Currently supported classes of BPF instruction are: 5833 * 1. rX = <imm> (assignment with immediate operand); 5834 * 2. rX += <imm> (arithmetic operations with immediate operand); 5835 * 3. rX = <imm64> (load with 64-bit immediate value); 5836 * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64}; 5837 * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64}; 5838 * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}. 5839 */ 5840 static int bpf_core_patch_insn(struct bpf_program *prog, 5841 const struct bpf_core_relo *relo, 5842 int relo_idx, 5843 const struct bpf_core_relo_res *res) 5844 { 5845 __u32 orig_val, new_val; 5846 struct bpf_insn *insn; 5847 int insn_idx; 5848 __u8 class; 5849 5850 if (relo->insn_off % BPF_INSN_SZ) 5851 return -EINVAL; 5852 insn_idx = relo->insn_off / BPF_INSN_SZ; 5853 /* adjust insn_idx from section frame of reference to the local 5854 * program's frame of reference; (sub-)program code is not yet 5855 * relocated, so it's enough to just subtract in-section offset 5856 */ 5857 insn_idx = insn_idx - prog->sec_insn_off; 5858 insn = &prog->insns[insn_idx]; 5859 class = BPF_CLASS(insn->code); 5860 5861 if (res->poison) { 5862 poison: 5863 /* poison second part of ldimm64 to avoid confusing error from 5864 * verifier about "unknown opcode 00" 5865 */ 5866 if (is_ldimm64_insn(insn)) 5867 bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1); 5868 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn); 5869 return 0; 5870 } 5871 5872 orig_val = res->orig_val; 5873 new_val = res->new_val; 5874 5875 switch (class) { 5876 case BPF_ALU: 5877 case BPF_ALU64: 5878 if (BPF_SRC(insn->code) != BPF_K) 5879 return -EINVAL; 5880 if (res->validate && insn->imm != orig_val) { 5881 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n", 5882 prog->name, relo_idx, 5883 insn_idx, insn->imm, orig_val, new_val); 5884 return -EINVAL; 5885 } 5886 orig_val = insn->imm; 5887 insn->imm = new_val; 5888 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n", 5889 prog->name, relo_idx, insn_idx, 5890 orig_val, new_val); 5891 break; 5892 case BPF_LDX: 5893 case BPF_ST: 5894 case BPF_STX: 5895 if (res->validate && insn->off != orig_val) { 5896 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n", 5897 prog->name, relo_idx, insn_idx, insn->off, orig_val, new_val); 5898 return -EINVAL; 5899 } 5900 if (new_val > SHRT_MAX) { 5901 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n", 5902 prog->name, relo_idx, insn_idx, new_val); 5903 return -ERANGE; 5904 } 5905 if (res->fail_memsz_adjust) { 5906 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. " 5907 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n", 5908 prog->name, relo_idx, insn_idx); 5909 goto poison; 5910 } 5911 5912 orig_val = insn->off; 5913 insn->off = new_val; 5914 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n", 5915 prog->name, relo_idx, insn_idx, orig_val, new_val); 5916 5917 if (res->new_sz != res->orig_sz) { 5918 int insn_bytes_sz, insn_bpf_sz; 5919 5920 insn_bytes_sz = insn_bpf_size_to_bytes(insn); 5921 if (insn_bytes_sz != res->orig_sz) { 5922 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n", 5923 prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz); 5924 return -EINVAL; 5925 } 5926 5927 insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz); 5928 if (insn_bpf_sz < 0) { 5929 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n", 5930 prog->name, relo_idx, insn_idx, res->new_sz); 5931 return -EINVAL; 5932 } 5933 5934 insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code); 5935 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n", 5936 prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz); 5937 } 5938 break; 5939 case BPF_LD: { 5940 __u64 imm; 5941 5942 if (!is_ldimm64_insn(insn) || 5943 insn[0].src_reg != 0 || insn[0].off != 0 || 5944 insn_idx + 1 >= prog->insns_cnt || 5945 insn[1].code != 0 || insn[1].dst_reg != 0 || 5946 insn[1].src_reg != 0 || insn[1].off != 0) { 5947 pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n", 5948 prog->name, relo_idx, insn_idx); 5949 return -EINVAL; 5950 } 5951 5952 imm = insn[0].imm + ((__u64)insn[1].imm << 32); 5953 if (res->validate && imm != orig_val) { 5954 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n", 5955 prog->name, relo_idx, 5956 insn_idx, (unsigned long long)imm, 5957 orig_val, new_val); 5958 return -EINVAL; 5959 } 5960 5961 insn[0].imm = new_val; 5962 insn[1].imm = 0; /* currently only 32-bit values are supported */ 5963 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n", 5964 prog->name, relo_idx, insn_idx, 5965 (unsigned long long)imm, new_val); 5966 break; 5967 } 5968 default: 5969 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", 5970 prog->name, relo_idx, insn_idx, insn->code, 5971 insn->src_reg, insn->dst_reg, insn->off, insn->imm); 5972 return -EINVAL; 5973 } 5974 5975 return 0; 5976 } 5977 5978 /* Output spec definition in the format: 5979 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, 5980 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b 5981 */ 5982 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) 5983 { 5984 const struct btf_type *t; 5985 const struct btf_enum *e; 5986 const char *s; 5987 __u32 type_id; 5988 int i; 5989 5990 type_id = spec->root_type_id; 5991 t = btf__type_by_id(spec->btf, type_id); 5992 s = btf__name_by_offset(spec->btf, t->name_off); 5993 5994 libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s); 5995 5996 if (core_relo_is_type_based(spec->relo_kind)) 5997 return; 5998 5999 if (core_relo_is_enumval_based(spec->relo_kind)) { 6000 t = skip_mods_and_typedefs(spec->btf, type_id, NULL); 6001 e = btf_enum(t) + spec->raw_spec[0]; 6002 s = btf__name_by_offset(spec->btf, e->name_off); 6003 6004 libbpf_print(level, "::%s = %u", s, e->val); 6005 return; 6006 } 6007 6008 if (core_relo_is_field_based(spec->relo_kind)) { 6009 for (i = 0; i < spec->len; i++) { 6010 if (spec->spec[i].name) 6011 libbpf_print(level, ".%s", spec->spec[i].name); 6012 else if (i > 0 || spec->spec[i].idx > 0) 6013 libbpf_print(level, "[%u]", spec->spec[i].idx); 6014 } 6015 6016 libbpf_print(level, " ("); 6017 for (i = 0; i < spec->raw_len; i++) 6018 libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]); 6019 6020 if (spec->bit_offset % 8) 6021 libbpf_print(level, " @ offset %u.%u)", 6022 spec->bit_offset / 8, spec->bit_offset % 8); 6023 else 6024 libbpf_print(level, " @ offset %u)", spec->bit_offset / 8); 6025 return; 6026 } 6027 } 6028 6029 static size_t bpf_core_hash_fn(const void *key, void *ctx) 6030 { 6031 return (size_t)key; 6032 } 6033 6034 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) 6035 { 6036 return k1 == k2; 6037 } 6038 6039 static void *u32_as_hash_key(__u32 x) 6040 { 6041 return (void *)(uintptr_t)x; 6042 } 6043 6044 /* 6045 * CO-RE relocate single instruction. 6046 * 6047 * The outline and important points of the algorithm: 6048 * 1. For given local type, find corresponding candidate target types. 6049 * Candidate type is a type with the same "essential" name, ignoring 6050 * everything after last triple underscore (___). E.g., `sample`, 6051 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates 6052 * for each other. Names with triple underscore are referred to as 6053 * "flavors" and are useful, among other things, to allow to 6054 * specify/support incompatible variations of the same kernel struct, which 6055 * might differ between different kernel versions and/or build 6056 * configurations. 6057 * 6058 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C 6059 * converter, when deduplicated BTF of a kernel still contains more than 6060 * one different types with the same name. In that case, ___2, ___3, etc 6061 * are appended starting from second name conflict. But start flavors are 6062 * also useful to be defined "locally", in BPF program, to extract same 6063 * data from incompatible changes between different kernel 6064 * versions/configurations. For instance, to handle field renames between 6065 * kernel versions, one can use two flavors of the struct name with the 6066 * same common name and use conditional relocations to extract that field, 6067 * depending on target kernel version. 6068 * 2. For each candidate type, try to match local specification to this 6069 * candidate target type. Matching involves finding corresponding 6070 * high-level spec accessors, meaning that all named fields should match, 6071 * as well as all array accesses should be within the actual bounds. Also, 6072 * types should be compatible (see bpf_core_fields_are_compat for details). 6073 * 3. It is supported and expected that there might be multiple flavors 6074 * matching the spec. As long as all the specs resolve to the same set of 6075 * offsets across all candidates, there is no error. If there is any 6076 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate 6077 * imprefection of BTF deduplication, which can cause slight duplication of 6078 * the same BTF type, if some directly or indirectly referenced (by 6079 * pointer) type gets resolved to different actual types in different 6080 * object files. If such situation occurs, deduplicated BTF will end up 6081 * with two (or more) structurally identical types, which differ only in 6082 * types they refer to through pointer. This should be OK in most cases and 6083 * is not an error. 6084 * 4. Candidate types search is performed by linearly scanning through all 6085 * types in target BTF. It is anticipated that this is overall more 6086 * efficient memory-wise and not significantly worse (if not better) 6087 * CPU-wise compared to prebuilding a map from all local type names to 6088 * a list of candidate type names. It's also sped up by caching resolved 6089 * list of matching candidates per each local "root" type ID, that has at 6090 * least one bpf_core_relo associated with it. This list is shared 6091 * between multiple relocations for the same type ID and is updated as some 6092 * of the candidates are pruned due to structural incompatibility. 6093 */ 6094 static int bpf_core_apply_relo(struct bpf_program *prog, 6095 const struct bpf_core_relo *relo, 6096 int relo_idx, 6097 const struct btf *local_btf, 6098 struct hashmap *cand_cache) 6099 { 6100 struct bpf_core_spec local_spec, cand_spec, targ_spec = {}; 6101 const void *type_key = u32_as_hash_key(relo->type_id); 6102 struct bpf_core_relo_res cand_res, targ_res; 6103 const struct btf_type *local_type; 6104 const char *local_name; 6105 struct core_cand_list *cands = NULL; 6106 __u32 local_id; 6107 const char *spec_str; 6108 int i, j, err; 6109 6110 local_id = relo->type_id; 6111 local_type = btf__type_by_id(local_btf, local_id); 6112 if (!local_type) 6113 return -EINVAL; 6114 6115 local_name = btf__name_by_offset(local_btf, local_type->name_off); 6116 if (!local_name) 6117 return -EINVAL; 6118 6119 spec_str = btf__name_by_offset(local_btf, relo->access_str_off); 6120 if (str_is_empty(spec_str)) 6121 return -EINVAL; 6122 6123 err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec); 6124 if (err) { 6125 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", 6126 prog->name, relo_idx, local_id, btf_kind_str(local_type), 6127 str_is_empty(local_name) ? "<anon>" : local_name, 6128 spec_str, err); 6129 return -EINVAL; 6130 } 6131 6132 pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog->name, 6133 relo_idx, core_relo_kind_str(relo->kind), relo->kind); 6134 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); 6135 libbpf_print(LIBBPF_DEBUG, "\n"); 6136 6137 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ 6138 if (relo->kind == BPF_TYPE_ID_LOCAL) { 6139 targ_res.validate = true; 6140 targ_res.poison = false; 6141 targ_res.orig_val = local_spec.root_type_id; 6142 targ_res.new_val = local_spec.root_type_id; 6143 goto patch_insn; 6144 } 6145 6146 /* libbpf doesn't support candidate search for anonymous types */ 6147 if (str_is_empty(spec_str)) { 6148 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", 6149 prog->name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); 6150 return -EOPNOTSUPP; 6151 } 6152 6153 if (!hashmap__find(cand_cache, type_key, (void **)&cands)) { 6154 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 6155 if (IS_ERR(cands)) { 6156 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 6157 prog->name, relo_idx, local_id, btf_kind_str(local_type), 6158 local_name, PTR_ERR(cands)); 6159 return PTR_ERR(cands); 6160 } 6161 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL); 6162 if (err) { 6163 bpf_core_free_cands(cands); 6164 return err; 6165 } 6166 } 6167 6168 for (i = 0, j = 0; i < cands->len; i++) { 6169 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf, 6170 cands->cands[i].id, &cand_spec); 6171 if (err < 0) { 6172 pr_warn("prog '%s': relo #%d: error matching candidate #%d ", 6173 prog->name, relo_idx, i); 6174 bpf_core_dump_spec(LIBBPF_WARN, &cand_spec); 6175 libbpf_print(LIBBPF_WARN, ": %d\n", err); 6176 return err; 6177 } 6178 6179 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog->name, 6180 relo_idx, err == 0 ? "non-matching" : "matching", i); 6181 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); 6182 libbpf_print(LIBBPF_DEBUG, "\n"); 6183 6184 if (err == 0) 6185 continue; 6186 6187 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, &cand_spec, &cand_res); 6188 if (err) 6189 return err; 6190 6191 if (j == 0) { 6192 targ_res = cand_res; 6193 targ_spec = cand_spec; 6194 } else if (cand_spec.bit_offset != targ_spec.bit_offset) { 6195 /* if there are many field relo candidates, they 6196 * should all resolve to the same bit offset 6197 */ 6198 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n", 6199 prog->name, relo_idx, cand_spec.bit_offset, 6200 targ_spec.bit_offset); 6201 return -EINVAL; 6202 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) { 6203 /* all candidates should result in the same relocation 6204 * decision and value, otherwise it's dangerous to 6205 * proceed due to ambiguity 6206 */ 6207 pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n", 6208 prog->name, relo_idx, 6209 cand_res.poison ? "failure" : "success", cand_res.new_val, 6210 targ_res.poison ? "failure" : "success", targ_res.new_val); 6211 return -EINVAL; 6212 } 6213 6214 cands->cands[j++] = cands->cands[i]; 6215 } 6216 6217 /* 6218 * For BPF_FIELD_EXISTS relo or when used BPF program has field 6219 * existence checks or kernel version/config checks, it's expected 6220 * that we might not find any candidates. In this case, if field 6221 * wasn't found in any candidate, the list of candidates shouldn't 6222 * change at all, we'll just handle relocating appropriately, 6223 * depending on relo's kind. 6224 */ 6225 if (j > 0) 6226 cands->len = j; 6227 6228 /* 6229 * If no candidates were found, it might be both a programmer error, 6230 * as well as expected case, depending whether instruction w/ 6231 * relocation is guarded in some way that makes it unreachable (dead 6232 * code) if relocation can't be resolved. This is handled in 6233 * bpf_core_patch_insn() uniformly by replacing that instruction with 6234 * BPF helper call insn (using invalid helper ID). If that instruction 6235 * is indeed unreachable, then it will be ignored and eliminated by 6236 * verifier. If it was an error, then verifier will complain and point 6237 * to a specific instruction number in its log. 6238 */ 6239 if (j == 0) { 6240 pr_debug("prog '%s': relo #%d: no matching targets found\n", 6241 prog->name, relo_idx); 6242 6243 /* calculate single target relo result explicitly */ 6244 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, NULL, &targ_res); 6245 if (err) 6246 return err; 6247 } 6248 6249 patch_insn: 6250 /* bpf_core_patch_insn() should know how to handle missing targ_spec */ 6251 err = bpf_core_patch_insn(prog, relo, relo_idx, &targ_res); 6252 if (err) { 6253 pr_warn("prog '%s': relo #%d: failed to patch insn #%zu: %d\n", 6254 prog->name, relo_idx, relo->insn_off / BPF_INSN_SZ, err); 6255 return -EINVAL; 6256 } 6257 6258 return 0; 6259 } 6260 6261 static int 6262 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 6263 { 6264 const struct btf_ext_info_sec *sec; 6265 const struct bpf_core_relo *rec; 6266 const struct btf_ext_info *seg; 6267 struct hashmap_entry *entry; 6268 struct hashmap *cand_cache = NULL; 6269 struct bpf_program *prog; 6270 const char *sec_name; 6271 int i, err = 0, insn_idx, sec_idx; 6272 6273 if (obj->btf_ext->core_relo_info.len == 0) 6274 return 0; 6275 6276 if (targ_btf_path) { 6277 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 6278 if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) { 6279 err = PTR_ERR(obj->btf_vmlinux_override); 6280 pr_warn("failed to parse target BTF: %d\n", err); 6281 return err; 6282 } 6283 } 6284 6285 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 6286 if (IS_ERR(cand_cache)) { 6287 err = PTR_ERR(cand_cache); 6288 goto out; 6289 } 6290 6291 seg = &obj->btf_ext->core_relo_info; 6292 for_each_btf_ext_sec(seg, sec) { 6293 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 6294 if (str_is_empty(sec_name)) { 6295 err = -EINVAL; 6296 goto out; 6297 } 6298 /* bpf_object's ELF is gone by now so it's not easy to find 6299 * section index by section name, but we can find *any* 6300 * bpf_program within desired section name and use it's 6301 * prog->sec_idx to do a proper search by section index and 6302 * instruction offset 6303 */ 6304 prog = NULL; 6305 for (i = 0; i < obj->nr_programs; i++) { 6306 prog = &obj->programs[i]; 6307 if (strcmp(prog->sec_name, sec_name) == 0) 6308 break; 6309 } 6310 if (!prog) { 6311 pr_warn("sec '%s': failed to find a BPF program\n", sec_name); 6312 return -ENOENT; 6313 } 6314 sec_idx = prog->sec_idx; 6315 6316 pr_debug("sec '%s': found %d CO-RE relocations\n", 6317 sec_name, sec->num_info); 6318 6319 for_each_btf_ext_rec(seg, sec, i, rec) { 6320 insn_idx = rec->insn_off / BPF_INSN_SZ; 6321 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 6322 if (!prog) { 6323 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n", 6324 sec_name, insn_idx, i); 6325 err = -EINVAL; 6326 goto out; 6327 } 6328 /* no need to apply CO-RE relocation if the program is 6329 * not going to be loaded 6330 */ 6331 if (!prog->load) 6332 continue; 6333 6334 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache); 6335 if (err) { 6336 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 6337 prog->name, i, err); 6338 goto out; 6339 } 6340 } 6341 } 6342 6343 out: 6344 /* obj->btf_vmlinux and module BTFs are freed after object load */ 6345 btf__free(obj->btf_vmlinux_override); 6346 obj->btf_vmlinux_override = NULL; 6347 6348 if (!IS_ERR_OR_NULL(cand_cache)) { 6349 hashmap__for_each_entry(cand_cache, entry, i) { 6350 bpf_core_free_cands(entry->value); 6351 } 6352 hashmap__free(cand_cache); 6353 } 6354 return err; 6355 } 6356 6357 /* Relocate data references within program code: 6358 * - map references; 6359 * - global variable references; 6360 * - extern references. 6361 */ 6362 static int 6363 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 6364 { 6365 int i; 6366 6367 for (i = 0; i < prog->nr_reloc; i++) { 6368 struct reloc_desc *relo = &prog->reloc_desc[i]; 6369 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 6370 struct extern_desc *ext; 6371 6372 switch (relo->type) { 6373 case RELO_LD64: 6374 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 6375 insn[0].imm = obj->maps[relo->map_idx].fd; 6376 break; 6377 case RELO_DATA: 6378 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6379 insn[1].imm = insn[0].imm + relo->sym_off; 6380 insn[0].imm = obj->maps[relo->map_idx].fd; 6381 break; 6382 case RELO_EXTERN_VAR: 6383 ext = &obj->externs[relo->sym_off]; 6384 if (ext->type == EXT_KCFG) { 6385 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6386 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 6387 insn[1].imm = ext->kcfg.data_off; 6388 } else /* EXT_KSYM */ { 6389 if (ext->ksym.type_id) { /* typed ksyms */ 6390 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 6391 insn[0].imm = ext->ksym.kernel_btf_id; 6392 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 6393 } else { /* typeless ksyms */ 6394 insn[0].imm = (__u32)ext->ksym.addr; 6395 insn[1].imm = ext->ksym.addr >> 32; 6396 } 6397 } 6398 break; 6399 case RELO_EXTERN_FUNC: 6400 ext = &obj->externs[relo->sym_off]; 6401 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 6402 insn[0].imm = ext->ksym.kernel_btf_id; 6403 break; 6404 case RELO_SUBPROG_ADDR: 6405 insn[0].src_reg = BPF_PSEUDO_FUNC; 6406 /* will be handled as a follow up pass */ 6407 break; 6408 case RELO_CALL: 6409 /* will be handled as a follow up pass */ 6410 break; 6411 default: 6412 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 6413 prog->name, i, relo->type); 6414 return -EINVAL; 6415 } 6416 } 6417 6418 return 0; 6419 } 6420 6421 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 6422 const struct bpf_program *prog, 6423 const struct btf_ext_info *ext_info, 6424 void **prog_info, __u32 *prog_rec_cnt, 6425 __u32 *prog_rec_sz) 6426 { 6427 void *copy_start = NULL, *copy_end = NULL; 6428 void *rec, *rec_end, *new_prog_info; 6429 const struct btf_ext_info_sec *sec; 6430 size_t old_sz, new_sz; 6431 const char *sec_name; 6432 int i, off_adj; 6433 6434 for_each_btf_ext_sec(ext_info, sec) { 6435 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 6436 if (!sec_name) 6437 return -EINVAL; 6438 if (strcmp(sec_name, prog->sec_name) != 0) 6439 continue; 6440 6441 for_each_btf_ext_rec(ext_info, sec, i, rec) { 6442 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 6443 6444 if (insn_off < prog->sec_insn_off) 6445 continue; 6446 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 6447 break; 6448 6449 if (!copy_start) 6450 copy_start = rec; 6451 copy_end = rec + ext_info->rec_size; 6452 } 6453 6454 if (!copy_start) 6455 return -ENOENT; 6456 6457 /* append func/line info of a given (sub-)program to the main 6458 * program func/line info 6459 */ 6460 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 6461 new_sz = old_sz + (copy_end - copy_start); 6462 new_prog_info = realloc(*prog_info, new_sz); 6463 if (!new_prog_info) 6464 return -ENOMEM; 6465 *prog_info = new_prog_info; 6466 *prog_rec_cnt = new_sz / ext_info->rec_size; 6467 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 6468 6469 /* Kernel instruction offsets are in units of 8-byte 6470 * instructions, while .BTF.ext instruction offsets generated 6471 * by Clang are in units of bytes. So convert Clang offsets 6472 * into kernel offsets and adjust offset according to program 6473 * relocated position. 6474 */ 6475 off_adj = prog->sub_insn_off - prog->sec_insn_off; 6476 rec = new_prog_info + old_sz; 6477 rec_end = new_prog_info + new_sz; 6478 for (; rec < rec_end; rec += ext_info->rec_size) { 6479 __u32 *insn_off = rec; 6480 6481 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 6482 } 6483 *prog_rec_sz = ext_info->rec_size; 6484 return 0; 6485 } 6486 6487 return -ENOENT; 6488 } 6489 6490 static int 6491 reloc_prog_func_and_line_info(const struct bpf_object *obj, 6492 struct bpf_program *main_prog, 6493 const struct bpf_program *prog) 6494 { 6495 int err; 6496 6497 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 6498 * supprot func/line info 6499 */ 6500 if (!obj->btf_ext || !kernel_supports(FEAT_BTF_FUNC)) 6501 return 0; 6502 6503 /* only attempt func info relocation if main program's func_info 6504 * relocation was successful 6505 */ 6506 if (main_prog != prog && !main_prog->func_info) 6507 goto line_info; 6508 6509 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 6510 &main_prog->func_info, 6511 &main_prog->func_info_cnt, 6512 &main_prog->func_info_rec_size); 6513 if (err) { 6514 if (err != -ENOENT) { 6515 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 6516 prog->name, err); 6517 return err; 6518 } 6519 if (main_prog->func_info) { 6520 /* 6521 * Some info has already been found but has problem 6522 * in the last btf_ext reloc. Must have to error out. 6523 */ 6524 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 6525 return err; 6526 } 6527 /* Have problem loading the very first info. Ignore the rest. */ 6528 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 6529 prog->name); 6530 } 6531 6532 line_info: 6533 /* don't relocate line info if main program's relocation failed */ 6534 if (main_prog != prog && !main_prog->line_info) 6535 return 0; 6536 6537 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 6538 &main_prog->line_info, 6539 &main_prog->line_info_cnt, 6540 &main_prog->line_info_rec_size); 6541 if (err) { 6542 if (err != -ENOENT) { 6543 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 6544 prog->name, err); 6545 return err; 6546 } 6547 if (main_prog->line_info) { 6548 /* 6549 * Some info has already been found but has problem 6550 * in the last btf_ext reloc. Must have to error out. 6551 */ 6552 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 6553 return err; 6554 } 6555 /* Have problem loading the very first info. Ignore the rest. */ 6556 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 6557 prog->name); 6558 } 6559 return 0; 6560 } 6561 6562 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 6563 { 6564 size_t insn_idx = *(const size_t *)key; 6565 const struct reloc_desc *relo = elem; 6566 6567 if (insn_idx == relo->insn_idx) 6568 return 0; 6569 return insn_idx < relo->insn_idx ? -1 : 1; 6570 } 6571 6572 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 6573 { 6574 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 6575 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 6576 } 6577 6578 static int 6579 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 6580 struct bpf_program *prog) 6581 { 6582 size_t sub_insn_idx, insn_idx, new_cnt; 6583 struct bpf_program *subprog; 6584 struct bpf_insn *insns, *insn; 6585 struct reloc_desc *relo; 6586 int err; 6587 6588 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 6589 if (err) 6590 return err; 6591 6592 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 6593 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6594 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 6595 continue; 6596 6597 relo = find_prog_insn_relo(prog, insn_idx); 6598 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 6599 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 6600 prog->name, insn_idx, relo->type); 6601 return -LIBBPF_ERRNO__RELOC; 6602 } 6603 if (relo) { 6604 /* sub-program instruction index is a combination of 6605 * an offset of a symbol pointed to by relocation and 6606 * call instruction's imm field; for global functions, 6607 * call always has imm = -1, but for static functions 6608 * relocation is against STT_SECTION and insn->imm 6609 * points to a start of a static function 6610 * 6611 * for subprog addr relocation, the relo->sym_off + insn->imm is 6612 * the byte offset in the corresponding section. 6613 */ 6614 if (relo->type == RELO_CALL) 6615 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6616 else 6617 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 6618 } else if (insn_is_pseudo_func(insn)) { 6619 /* 6620 * RELO_SUBPROG_ADDR relo is always emitted even if both 6621 * functions are in the same section, so it shouldn't reach here. 6622 */ 6623 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 6624 prog->name, insn_idx); 6625 return -LIBBPF_ERRNO__RELOC; 6626 } else { 6627 /* if subprogram call is to a static function within 6628 * the same ELF section, there won't be any relocation 6629 * emitted, but it also means there is no additional 6630 * offset necessary, insns->imm is relative to 6631 * instruction's original position within the section 6632 */ 6633 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 6634 } 6635 6636 /* we enforce that sub-programs should be in .text section */ 6637 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 6638 if (!subprog) { 6639 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 6640 prog->name); 6641 return -LIBBPF_ERRNO__RELOC; 6642 } 6643 6644 /* if it's the first call instruction calling into this 6645 * subprogram (meaning this subprog hasn't been processed 6646 * yet) within the context of current main program: 6647 * - append it at the end of main program's instructions blog; 6648 * - process is recursively, while current program is put on hold; 6649 * - if that subprogram calls some other not yet processes 6650 * subprogram, same thing will happen recursively until 6651 * there are no more unprocesses subprograms left to append 6652 * and relocate. 6653 */ 6654 if (subprog->sub_insn_off == 0) { 6655 subprog->sub_insn_off = main_prog->insns_cnt; 6656 6657 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 6658 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 6659 if (!insns) { 6660 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 6661 return -ENOMEM; 6662 } 6663 main_prog->insns = insns; 6664 main_prog->insns_cnt = new_cnt; 6665 6666 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 6667 subprog->insns_cnt * sizeof(*insns)); 6668 6669 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 6670 main_prog->name, subprog->insns_cnt, subprog->name); 6671 6672 err = bpf_object__reloc_code(obj, main_prog, subprog); 6673 if (err) 6674 return err; 6675 } 6676 6677 /* main_prog->insns memory could have been re-allocated, so 6678 * calculate pointer again 6679 */ 6680 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6681 /* calculate correct instruction position within current main 6682 * prog; each main prog can have a different set of 6683 * subprograms appended (potentially in different order as 6684 * well), so position of any subprog can be different for 6685 * different main programs */ 6686 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 6687 6688 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 6689 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 6690 } 6691 6692 return 0; 6693 } 6694 6695 /* 6696 * Relocate sub-program calls. 6697 * 6698 * Algorithm operates as follows. Each entry-point BPF program (referred to as 6699 * main prog) is processed separately. For each subprog (non-entry functions, 6700 * that can be called from either entry progs or other subprogs) gets their 6701 * sub_insn_off reset to zero. This serves as indicator that this subprogram 6702 * hasn't been yet appended and relocated within current main prog. Once its 6703 * relocated, sub_insn_off will point at the position within current main prog 6704 * where given subprog was appended. This will further be used to relocate all 6705 * the call instructions jumping into this subprog. 6706 * 6707 * We start with main program and process all call instructions. If the call 6708 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 6709 * is zero), subprog instructions are appended at the end of main program's 6710 * instruction array. Then main program is "put on hold" while we recursively 6711 * process newly appended subprogram. If that subprogram calls into another 6712 * subprogram that hasn't been appended, new subprogram is appended again to 6713 * the *main* prog's instructions (subprog's instructions are always left 6714 * untouched, as they need to be in unmodified state for subsequent main progs 6715 * and subprog instructions are always sent only as part of a main prog) and 6716 * the process continues recursively. Once all the subprogs called from a main 6717 * prog or any of its subprogs are appended (and relocated), all their 6718 * positions within finalized instructions array are known, so it's easy to 6719 * rewrite call instructions with correct relative offsets, corresponding to 6720 * desired target subprog. 6721 * 6722 * Its important to realize that some subprogs might not be called from some 6723 * main prog and any of its called/used subprogs. Those will keep their 6724 * subprog->sub_insn_off as zero at all times and won't be appended to current 6725 * main prog and won't be relocated within the context of current main prog. 6726 * They might still be used from other main progs later. 6727 * 6728 * Visually this process can be shown as below. Suppose we have two main 6729 * programs mainA and mainB and BPF object contains three subprogs: subA, 6730 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 6731 * subC both call subB: 6732 * 6733 * +--------+ +-------+ 6734 * | v v | 6735 * +--+---+ +--+-+-+ +---+--+ 6736 * | subA | | subB | | subC | 6737 * +--+---+ +------+ +---+--+ 6738 * ^ ^ 6739 * | | 6740 * +---+-------+ +------+----+ 6741 * | mainA | | mainB | 6742 * +-----------+ +-----------+ 6743 * 6744 * We'll start relocating mainA, will find subA, append it and start 6745 * processing sub A recursively: 6746 * 6747 * +-----------+------+ 6748 * | mainA | subA | 6749 * +-----------+------+ 6750 * 6751 * At this point we notice that subB is used from subA, so we append it and 6752 * relocate (there are no further subcalls from subB): 6753 * 6754 * +-----------+------+------+ 6755 * | mainA | subA | subB | 6756 * +-----------+------+------+ 6757 * 6758 * At this point, we relocate subA calls, then go one level up and finish with 6759 * relocatin mainA calls. mainA is done. 6760 * 6761 * For mainB process is similar but results in different order. We start with 6762 * mainB and skip subA and subB, as mainB never calls them (at least 6763 * directly), but we see subC is needed, so we append and start processing it: 6764 * 6765 * +-----------+------+ 6766 * | mainB | subC | 6767 * +-----------+------+ 6768 * Now we see subC needs subB, so we go back to it, append and relocate it: 6769 * 6770 * +-----------+------+------+ 6771 * | mainB | subC | subB | 6772 * +-----------+------+------+ 6773 * 6774 * At this point we unwind recursion, relocate calls in subC, then in mainB. 6775 */ 6776 static int 6777 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 6778 { 6779 struct bpf_program *subprog; 6780 int i, err; 6781 6782 /* mark all subprogs as not relocated (yet) within the context of 6783 * current main program 6784 */ 6785 for (i = 0; i < obj->nr_programs; i++) { 6786 subprog = &obj->programs[i]; 6787 if (!prog_is_subprog(obj, subprog)) 6788 continue; 6789 6790 subprog->sub_insn_off = 0; 6791 } 6792 6793 err = bpf_object__reloc_code(obj, prog, prog); 6794 if (err) 6795 return err; 6796 6797 6798 return 0; 6799 } 6800 6801 static int 6802 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 6803 { 6804 struct bpf_program *prog; 6805 size_t i; 6806 int err; 6807 6808 if (obj->btf_ext) { 6809 err = bpf_object__relocate_core(obj, targ_btf_path); 6810 if (err) { 6811 pr_warn("failed to perform CO-RE relocations: %d\n", 6812 err); 6813 return err; 6814 } 6815 } 6816 /* relocate data references first for all programs and sub-programs, 6817 * as they don't change relative to code locations, so subsequent 6818 * subprogram processing won't need to re-calculate any of them 6819 */ 6820 for (i = 0; i < obj->nr_programs; i++) { 6821 prog = &obj->programs[i]; 6822 err = bpf_object__relocate_data(obj, prog); 6823 if (err) { 6824 pr_warn("prog '%s': failed to relocate data references: %d\n", 6825 prog->name, err); 6826 return err; 6827 } 6828 } 6829 /* now relocate subprogram calls and append used subprograms to main 6830 * programs; each copy of subprogram code needs to be relocated 6831 * differently for each main program, because its code location might 6832 * have changed 6833 */ 6834 for (i = 0; i < obj->nr_programs; i++) { 6835 prog = &obj->programs[i]; 6836 /* sub-program's sub-calls are relocated within the context of 6837 * its main program only 6838 */ 6839 if (prog_is_subprog(obj, prog)) 6840 continue; 6841 6842 err = bpf_object__relocate_calls(obj, prog); 6843 if (err) { 6844 pr_warn("prog '%s': failed to relocate calls: %d\n", 6845 prog->name, err); 6846 return err; 6847 } 6848 } 6849 /* free up relocation descriptors */ 6850 for (i = 0; i < obj->nr_programs; i++) { 6851 prog = &obj->programs[i]; 6852 zfree(&prog->reloc_desc); 6853 prog->nr_reloc = 0; 6854 } 6855 return 0; 6856 } 6857 6858 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 6859 GElf_Shdr *shdr, Elf_Data *data); 6860 6861 static int bpf_object__collect_map_relos(struct bpf_object *obj, 6862 GElf_Shdr *shdr, Elf_Data *data) 6863 { 6864 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 6865 int i, j, nrels, new_sz; 6866 const struct btf_var_secinfo *vi = NULL; 6867 const struct btf_type *sec, *var, *def; 6868 struct bpf_map *map = NULL, *targ_map; 6869 const struct btf_member *member; 6870 const char *name, *mname; 6871 Elf_Data *symbols; 6872 unsigned int moff; 6873 GElf_Sym sym; 6874 GElf_Rel rel; 6875 void *tmp; 6876 6877 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 6878 return -EINVAL; 6879 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 6880 if (!sec) 6881 return -EINVAL; 6882 6883 symbols = obj->efile.symbols; 6884 nrels = shdr->sh_size / shdr->sh_entsize; 6885 for (i = 0; i < nrels; i++) { 6886 if (!gelf_getrel(data, i, &rel)) { 6887 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 6888 return -LIBBPF_ERRNO__FORMAT; 6889 } 6890 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 6891 pr_warn(".maps relo #%d: symbol %zx not found\n", 6892 i, (size_t)GELF_R_SYM(rel.r_info)); 6893 return -LIBBPF_ERRNO__FORMAT; 6894 } 6895 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 6896 if (sym.st_shndx != obj->efile.btf_maps_shndx) { 6897 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 6898 i, name); 6899 return -LIBBPF_ERRNO__RELOC; 6900 } 6901 6902 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n", 6903 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value, 6904 (size_t)rel.r_offset, sym.st_name, name); 6905 6906 for (j = 0; j < obj->nr_maps; j++) { 6907 map = &obj->maps[j]; 6908 if (map->sec_idx != obj->efile.btf_maps_shndx) 6909 continue; 6910 6911 vi = btf_var_secinfos(sec) + map->btf_var_idx; 6912 if (vi->offset <= rel.r_offset && 6913 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size) 6914 break; 6915 } 6916 if (j == obj->nr_maps) { 6917 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n", 6918 i, name, (size_t)rel.r_offset); 6919 return -EINVAL; 6920 } 6921 6922 if (!bpf_map_type__is_map_in_map(map->def.type)) 6923 return -EINVAL; 6924 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 6925 map->def.key_size != sizeof(int)) { 6926 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 6927 i, map->name, sizeof(int)); 6928 return -EINVAL; 6929 } 6930 6931 targ_map = bpf_object__find_map_by_name(obj, name); 6932 if (!targ_map) 6933 return -ESRCH; 6934 6935 var = btf__type_by_id(obj->btf, vi->type); 6936 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 6937 if (btf_vlen(def) == 0) 6938 return -EINVAL; 6939 member = btf_members(def) + btf_vlen(def) - 1; 6940 mname = btf__name_by_offset(obj->btf, member->name_off); 6941 if (strcmp(mname, "values")) 6942 return -EINVAL; 6943 6944 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 6945 if (rel.r_offset - vi->offset < moff) 6946 return -EINVAL; 6947 6948 moff = rel.r_offset - vi->offset - moff; 6949 /* here we use BPF pointer size, which is always 64 bit, as we 6950 * are parsing ELF that was built for BPF target 6951 */ 6952 if (moff % bpf_ptr_sz) 6953 return -EINVAL; 6954 moff /= bpf_ptr_sz; 6955 if (moff >= map->init_slots_sz) { 6956 new_sz = moff + 1; 6957 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 6958 if (!tmp) 6959 return -ENOMEM; 6960 map->init_slots = tmp; 6961 memset(map->init_slots + map->init_slots_sz, 0, 6962 (new_sz - map->init_slots_sz) * host_ptr_sz); 6963 map->init_slots_sz = new_sz; 6964 } 6965 map->init_slots[moff] = targ_map; 6966 6967 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n", 6968 i, map->name, moff, name); 6969 } 6970 6971 return 0; 6972 } 6973 6974 static int cmp_relocs(const void *_a, const void *_b) 6975 { 6976 const struct reloc_desc *a = _a; 6977 const struct reloc_desc *b = _b; 6978 6979 if (a->insn_idx != b->insn_idx) 6980 return a->insn_idx < b->insn_idx ? -1 : 1; 6981 6982 /* no two relocations should have the same insn_idx, but ... */ 6983 if (a->type != b->type) 6984 return a->type < b->type ? -1 : 1; 6985 6986 return 0; 6987 } 6988 6989 static int bpf_object__collect_relos(struct bpf_object *obj) 6990 { 6991 int i, err; 6992 6993 for (i = 0; i < obj->efile.nr_reloc_sects; i++) { 6994 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; 6995 Elf_Data *data = obj->efile.reloc_sects[i].data; 6996 int idx = shdr->sh_info; 6997 6998 if (shdr->sh_type != SHT_REL) { 6999 pr_warn("internal error at %d\n", __LINE__); 7000 return -LIBBPF_ERRNO__INTERNAL; 7001 } 7002 7003 if (idx == obj->efile.st_ops_shndx) 7004 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 7005 else if (idx == obj->efile.btf_maps_shndx) 7006 err = bpf_object__collect_map_relos(obj, shdr, data); 7007 else 7008 err = bpf_object__collect_prog_relos(obj, shdr, data); 7009 if (err) 7010 return err; 7011 } 7012 7013 for (i = 0; i < obj->nr_programs; i++) { 7014 struct bpf_program *p = &obj->programs[i]; 7015 7016 if (!p->nr_reloc) 7017 continue; 7018 7019 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 7020 } 7021 return 0; 7022 } 7023 7024 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 7025 { 7026 if (BPF_CLASS(insn->code) == BPF_JMP && 7027 BPF_OP(insn->code) == BPF_CALL && 7028 BPF_SRC(insn->code) == BPF_K && 7029 insn->src_reg == 0 && 7030 insn->dst_reg == 0) { 7031 *func_id = insn->imm; 7032 return true; 7033 } 7034 return false; 7035 } 7036 7037 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 7038 { 7039 struct bpf_insn *insn = prog->insns; 7040 enum bpf_func_id func_id; 7041 int i; 7042 7043 for (i = 0; i < prog->insns_cnt; i++, insn++) { 7044 if (!insn_is_helper_call(insn, &func_id)) 7045 continue; 7046 7047 /* on kernels that don't yet support 7048 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 7049 * to bpf_probe_read() which works well for old kernels 7050 */ 7051 switch (func_id) { 7052 case BPF_FUNC_probe_read_kernel: 7053 case BPF_FUNC_probe_read_user: 7054 if (!kernel_supports(FEAT_PROBE_READ_KERN)) 7055 insn->imm = BPF_FUNC_probe_read; 7056 break; 7057 case BPF_FUNC_probe_read_kernel_str: 7058 case BPF_FUNC_probe_read_user_str: 7059 if (!kernel_supports(FEAT_PROBE_READ_KERN)) 7060 insn->imm = BPF_FUNC_probe_read_str; 7061 break; 7062 default: 7063 break; 7064 } 7065 } 7066 return 0; 7067 } 7068 7069 static int 7070 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 7071 char *license, __u32 kern_version, int *pfd) 7072 { 7073 struct bpf_prog_load_params load_attr = {}; 7074 char *cp, errmsg[STRERR_BUFSIZE]; 7075 size_t log_buf_size = 0; 7076 char *log_buf = NULL; 7077 int btf_fd, ret; 7078 7079 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 7080 /* 7081 * The program type must be set. Most likely we couldn't find a proper 7082 * section definition at load time, and thus we didn't infer the type. 7083 */ 7084 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 7085 prog->name, prog->sec_name); 7086 return -EINVAL; 7087 } 7088 7089 if (!insns || !insns_cnt) 7090 return -EINVAL; 7091 7092 load_attr.prog_type = prog->type; 7093 /* old kernels might not support specifying expected_attach_type */ 7094 if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def && 7095 prog->sec_def->is_exp_attach_type_optional) 7096 load_attr.expected_attach_type = 0; 7097 else 7098 load_attr.expected_attach_type = prog->expected_attach_type; 7099 if (kernel_supports(FEAT_PROG_NAME)) 7100 load_attr.name = prog->name; 7101 load_attr.insns = insns; 7102 load_attr.insn_cnt = insns_cnt; 7103 load_attr.license = license; 7104 load_attr.attach_btf_id = prog->attach_btf_id; 7105 if (prog->attach_prog_fd) 7106 load_attr.attach_prog_fd = prog->attach_prog_fd; 7107 else 7108 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 7109 load_attr.attach_btf_id = prog->attach_btf_id; 7110 load_attr.kern_version = kern_version; 7111 load_attr.prog_ifindex = prog->prog_ifindex; 7112 7113 /* specify func_info/line_info only if kernel supports them */ 7114 btf_fd = bpf_object__btf_fd(prog->obj); 7115 if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) { 7116 load_attr.prog_btf_fd = btf_fd; 7117 load_attr.func_info = prog->func_info; 7118 load_attr.func_info_rec_size = prog->func_info_rec_size; 7119 load_attr.func_info_cnt = prog->func_info_cnt; 7120 load_attr.line_info = prog->line_info; 7121 load_attr.line_info_rec_size = prog->line_info_rec_size; 7122 load_attr.line_info_cnt = prog->line_info_cnt; 7123 } 7124 load_attr.log_level = prog->log_level; 7125 load_attr.prog_flags = prog->prog_flags; 7126 7127 retry_load: 7128 if (log_buf_size) { 7129 log_buf = malloc(log_buf_size); 7130 if (!log_buf) 7131 return -ENOMEM; 7132 7133 *log_buf = 0; 7134 } 7135 7136 load_attr.log_buf = log_buf; 7137 load_attr.log_buf_sz = log_buf_size; 7138 ret = libbpf__bpf_prog_load(&load_attr); 7139 7140 if (ret >= 0) { 7141 if (log_buf && load_attr.log_level) 7142 pr_debug("verifier log:\n%s", log_buf); 7143 7144 if (prog->obj->rodata_map_idx >= 0 && 7145 kernel_supports(FEAT_PROG_BIND_MAP)) { 7146 struct bpf_map *rodata_map = 7147 &prog->obj->maps[prog->obj->rodata_map_idx]; 7148 7149 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) { 7150 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7151 pr_warn("prog '%s': failed to bind .rodata map: %s\n", 7152 prog->name, cp); 7153 /* Don't fail hard if can't bind rodata. */ 7154 } 7155 } 7156 7157 *pfd = ret; 7158 ret = 0; 7159 goto out; 7160 } 7161 7162 if (!log_buf || errno == ENOSPC) { 7163 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, 7164 log_buf_size << 1); 7165 7166 free(log_buf); 7167 goto retry_load; 7168 } 7169 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD; 7170 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7171 pr_warn("load bpf program failed: %s\n", cp); 7172 pr_perm_msg(ret); 7173 7174 if (log_buf && log_buf[0] != '\0') { 7175 ret = -LIBBPF_ERRNO__VERIFY; 7176 pr_warn("-- BEGIN DUMP LOG ---\n"); 7177 pr_warn("\n%s\n", log_buf); 7178 pr_warn("-- END LOG --\n"); 7179 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) { 7180 pr_warn("Program too large (%zu insns), at most %d insns\n", 7181 load_attr.insn_cnt, BPF_MAXINSNS); 7182 ret = -LIBBPF_ERRNO__PROG2BIG; 7183 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 7184 /* Wrong program type? */ 7185 int fd; 7186 7187 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 7188 load_attr.expected_attach_type = 0; 7189 load_attr.log_buf = NULL; 7190 load_attr.log_buf_sz = 0; 7191 fd = libbpf__bpf_prog_load(&load_attr); 7192 if (fd >= 0) { 7193 close(fd); 7194 ret = -LIBBPF_ERRNO__PROGTYPE; 7195 goto out; 7196 } 7197 } 7198 7199 out: 7200 free(log_buf); 7201 return ret; 7202 } 7203 7204 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id); 7205 7206 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) 7207 { 7208 int err = 0, fd, i; 7209 7210 if (prog->obj->loaded) { 7211 pr_warn("prog '%s': can't load after object was loaded\n", prog->name); 7212 return -EINVAL; 7213 } 7214 7215 if ((prog->type == BPF_PROG_TYPE_TRACING || 7216 prog->type == BPF_PROG_TYPE_LSM || 7217 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { 7218 int btf_obj_fd = 0, btf_type_id = 0; 7219 7220 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id); 7221 if (err) 7222 return err; 7223 7224 prog->attach_btf_obj_fd = btf_obj_fd; 7225 prog->attach_btf_id = btf_type_id; 7226 } 7227 7228 if (prog->instances.nr < 0 || !prog->instances.fds) { 7229 if (prog->preprocessor) { 7230 pr_warn("Internal error: can't load program '%s'\n", 7231 prog->name); 7232 return -LIBBPF_ERRNO__INTERNAL; 7233 } 7234 7235 prog->instances.fds = malloc(sizeof(int)); 7236 if (!prog->instances.fds) { 7237 pr_warn("Not enough memory for BPF fds\n"); 7238 return -ENOMEM; 7239 } 7240 prog->instances.nr = 1; 7241 prog->instances.fds[0] = -1; 7242 } 7243 7244 if (!prog->preprocessor) { 7245 if (prog->instances.nr != 1) { 7246 pr_warn("prog '%s': inconsistent nr(%d) != 1\n", 7247 prog->name, prog->instances.nr); 7248 } 7249 err = load_program(prog, prog->insns, prog->insns_cnt, 7250 license, kern_ver, &fd); 7251 if (!err) 7252 prog->instances.fds[0] = fd; 7253 goto out; 7254 } 7255 7256 for (i = 0; i < prog->instances.nr; i++) { 7257 struct bpf_prog_prep_result result; 7258 bpf_program_prep_t preprocessor = prog->preprocessor; 7259 7260 memset(&result, 0, sizeof(result)); 7261 err = preprocessor(prog, i, prog->insns, 7262 prog->insns_cnt, &result); 7263 if (err) { 7264 pr_warn("Preprocessing the %dth instance of program '%s' failed\n", 7265 i, prog->name); 7266 goto out; 7267 } 7268 7269 if (!result.new_insn_ptr || !result.new_insn_cnt) { 7270 pr_debug("Skip loading the %dth instance of program '%s'\n", 7271 i, prog->name); 7272 prog->instances.fds[i] = -1; 7273 if (result.pfd) 7274 *result.pfd = -1; 7275 continue; 7276 } 7277 7278 err = load_program(prog, result.new_insn_ptr, 7279 result.new_insn_cnt, license, kern_ver, &fd); 7280 if (err) { 7281 pr_warn("Loading the %dth instance of program '%s' failed\n", 7282 i, prog->name); 7283 goto out; 7284 } 7285 7286 if (result.pfd) 7287 *result.pfd = fd; 7288 prog->instances.fds[i] = fd; 7289 } 7290 out: 7291 if (err) 7292 pr_warn("failed to load program '%s'\n", prog->name); 7293 zfree(&prog->insns); 7294 prog->insns_cnt = 0; 7295 return err; 7296 } 7297 7298 static int 7299 bpf_object__load_progs(struct bpf_object *obj, int log_level) 7300 { 7301 struct bpf_program *prog; 7302 size_t i; 7303 int err; 7304 7305 for (i = 0; i < obj->nr_programs; i++) { 7306 prog = &obj->programs[i]; 7307 err = bpf_object__sanitize_prog(obj, prog); 7308 if (err) 7309 return err; 7310 } 7311 7312 for (i = 0; i < obj->nr_programs; i++) { 7313 prog = &obj->programs[i]; 7314 if (prog_is_subprog(obj, prog)) 7315 continue; 7316 if (!prog->load) { 7317 pr_debug("prog '%s': skipped loading\n", prog->name); 7318 continue; 7319 } 7320 prog->log_level |= log_level; 7321 err = bpf_program__load(prog, obj->license, obj->kern_version); 7322 if (err) 7323 return err; 7324 } 7325 return 0; 7326 } 7327 7328 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 7329 7330 static struct bpf_object * 7331 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, 7332 const struct bpf_object_open_opts *opts) 7333 { 7334 const char *obj_name, *kconfig; 7335 struct bpf_program *prog; 7336 struct bpf_object *obj; 7337 char tmp_name[64]; 7338 int err; 7339 7340 if (elf_version(EV_CURRENT) == EV_NONE) { 7341 pr_warn("failed to init libelf for %s\n", 7342 path ? : "(mem buf)"); 7343 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 7344 } 7345 7346 if (!OPTS_VALID(opts, bpf_object_open_opts)) 7347 return ERR_PTR(-EINVAL); 7348 7349 obj_name = OPTS_GET(opts, object_name, NULL); 7350 if (obj_buf) { 7351 if (!obj_name) { 7352 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 7353 (unsigned long)obj_buf, 7354 (unsigned long)obj_buf_sz); 7355 obj_name = tmp_name; 7356 } 7357 path = obj_name; 7358 pr_debug("loading object '%s' from buffer\n", obj_name); 7359 } 7360 7361 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 7362 if (IS_ERR(obj)) 7363 return obj; 7364 7365 kconfig = OPTS_GET(opts, kconfig, NULL); 7366 if (kconfig) { 7367 obj->kconfig = strdup(kconfig); 7368 if (!obj->kconfig) 7369 return ERR_PTR(-ENOMEM); 7370 } 7371 7372 err = bpf_object__elf_init(obj); 7373 err = err ? : bpf_object__check_endianness(obj); 7374 err = err ? : bpf_object__elf_collect(obj); 7375 err = err ? : bpf_object__collect_externs(obj); 7376 err = err ? : bpf_object__finalize_btf(obj); 7377 err = err ? : bpf_object__init_maps(obj, opts); 7378 err = err ? : bpf_object__collect_relos(obj); 7379 if (err) 7380 goto out; 7381 bpf_object__elf_finish(obj); 7382 7383 bpf_object__for_each_program(prog, obj) { 7384 prog->sec_def = find_sec_def(prog->sec_name); 7385 if (!prog->sec_def) { 7386 /* couldn't guess, but user might manually specify */ 7387 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 7388 prog->name, prog->sec_name); 7389 continue; 7390 } 7391 7392 if (prog->sec_def->is_sleepable) 7393 prog->prog_flags |= BPF_F_SLEEPABLE; 7394 bpf_program__set_type(prog, prog->sec_def->prog_type); 7395 bpf_program__set_expected_attach_type(prog, 7396 prog->sec_def->expected_attach_type); 7397 7398 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || 7399 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) 7400 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); 7401 } 7402 7403 return obj; 7404 out: 7405 bpf_object__close(obj); 7406 return ERR_PTR(err); 7407 } 7408 7409 static struct bpf_object * 7410 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) 7411 { 7412 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 7413 .relaxed_maps = flags & MAPS_RELAX_COMPAT, 7414 ); 7415 7416 /* param validation */ 7417 if (!attr->file) 7418 return NULL; 7419 7420 pr_debug("loading %s\n", attr->file); 7421 return __bpf_object__open(attr->file, NULL, 0, &opts); 7422 } 7423 7424 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 7425 { 7426 return __bpf_object__open_xattr(attr, 0); 7427 } 7428 7429 struct bpf_object *bpf_object__open(const char *path) 7430 { 7431 struct bpf_object_open_attr attr = { 7432 .file = path, 7433 .prog_type = BPF_PROG_TYPE_UNSPEC, 7434 }; 7435 7436 return bpf_object__open_xattr(&attr); 7437 } 7438 7439 struct bpf_object * 7440 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 7441 { 7442 if (!path) 7443 return ERR_PTR(-EINVAL); 7444 7445 pr_debug("loading %s\n", path); 7446 7447 return __bpf_object__open(path, NULL, 0, opts); 7448 } 7449 7450 struct bpf_object * 7451 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 7452 const struct bpf_object_open_opts *opts) 7453 { 7454 if (!obj_buf || obj_buf_sz == 0) 7455 return ERR_PTR(-EINVAL); 7456 7457 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts); 7458 } 7459 7460 struct bpf_object * 7461 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, 7462 const char *name) 7463 { 7464 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 7465 .object_name = name, 7466 /* wrong default, but backwards-compatible */ 7467 .relaxed_maps = true, 7468 ); 7469 7470 /* returning NULL is wrong, but backwards-compatible */ 7471 if (!obj_buf || obj_buf_sz == 0) 7472 return NULL; 7473 7474 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts); 7475 } 7476 7477 int bpf_object__unload(struct bpf_object *obj) 7478 { 7479 size_t i; 7480 7481 if (!obj) 7482 return -EINVAL; 7483 7484 for (i = 0; i < obj->nr_maps; i++) { 7485 zclose(obj->maps[i].fd); 7486 if (obj->maps[i].st_ops) 7487 zfree(&obj->maps[i].st_ops->kern_vdata); 7488 } 7489 7490 for (i = 0; i < obj->nr_programs; i++) 7491 bpf_program__unload(&obj->programs[i]); 7492 7493 return 0; 7494 } 7495 7496 static int bpf_object__sanitize_maps(struct bpf_object *obj) 7497 { 7498 struct bpf_map *m; 7499 7500 bpf_object__for_each_map(m, obj) { 7501 if (!bpf_map__is_internal(m)) 7502 continue; 7503 if (!kernel_supports(FEAT_GLOBAL_DATA)) { 7504 pr_warn("kernel doesn't support global data\n"); 7505 return -ENOTSUP; 7506 } 7507 if (!kernel_supports(FEAT_ARRAY_MMAP)) 7508 m->def.map_flags ^= BPF_F_MMAPABLE; 7509 } 7510 7511 return 0; 7512 } 7513 7514 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 7515 { 7516 char sym_type, sym_name[500]; 7517 unsigned long long sym_addr; 7518 const struct btf_type *t; 7519 struct extern_desc *ext; 7520 int ret, err = 0; 7521 FILE *f; 7522 7523 f = fopen("/proc/kallsyms", "r"); 7524 if (!f) { 7525 err = -errno; 7526 pr_warn("failed to open /proc/kallsyms: %d\n", err); 7527 return err; 7528 } 7529 7530 while (true) { 7531 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 7532 &sym_addr, &sym_type, sym_name); 7533 if (ret == EOF && feof(f)) 7534 break; 7535 if (ret != 3) { 7536 pr_warn("failed to read kallsyms entry: %d\n", ret); 7537 err = -EINVAL; 7538 goto out; 7539 } 7540 7541 ext = find_extern_by_name(obj, sym_name); 7542 if (!ext || ext->type != EXT_KSYM) 7543 continue; 7544 7545 t = btf__type_by_id(obj->btf, ext->btf_id); 7546 if (!btf_is_var(t)) 7547 continue; 7548 7549 if (ext->is_set && ext->ksym.addr != sym_addr) { 7550 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", 7551 sym_name, ext->ksym.addr, sym_addr); 7552 err = -EINVAL; 7553 goto out; 7554 } 7555 if (!ext->is_set) { 7556 ext->is_set = true; 7557 ext->ksym.addr = sym_addr; 7558 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); 7559 } 7560 } 7561 7562 out: 7563 fclose(f); 7564 return err; 7565 } 7566 7567 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 7568 __u16 kind, struct btf **res_btf, 7569 int *res_btf_fd) 7570 { 7571 int i, id, btf_fd, err; 7572 struct btf *btf; 7573 7574 btf = obj->btf_vmlinux; 7575 btf_fd = 0; 7576 id = btf__find_by_name_kind(btf, ksym_name, kind); 7577 7578 if (id == -ENOENT) { 7579 err = load_module_btfs(obj); 7580 if (err) 7581 return err; 7582 7583 for (i = 0; i < obj->btf_module_cnt; i++) { 7584 btf = obj->btf_modules[i].btf; 7585 /* we assume module BTF FD is always >0 */ 7586 btf_fd = obj->btf_modules[i].fd; 7587 id = btf__find_by_name_kind(btf, ksym_name, kind); 7588 if (id != -ENOENT) 7589 break; 7590 } 7591 } 7592 if (id <= 0) { 7593 pr_warn("extern (%s ksym) '%s': failed to find BTF ID in kernel BTF(s).\n", 7594 __btf_kind_str(kind), ksym_name); 7595 return -ESRCH; 7596 } 7597 7598 *res_btf = btf; 7599 *res_btf_fd = btf_fd; 7600 return id; 7601 } 7602 7603 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 7604 struct extern_desc *ext) 7605 { 7606 const struct btf_type *targ_var, *targ_type; 7607 __u32 targ_type_id, local_type_id; 7608 const char *targ_var_name; 7609 int id, btf_fd = 0, err; 7610 struct btf *btf = NULL; 7611 7612 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd); 7613 if (id < 0) 7614 return id; 7615 7616 /* find local type_id */ 7617 local_type_id = ext->ksym.type_id; 7618 7619 /* find target type_id */ 7620 targ_var = btf__type_by_id(btf, id); 7621 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 7622 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 7623 7624 err = bpf_core_types_are_compat(obj->btf, local_type_id, 7625 btf, targ_type_id); 7626 if (err <= 0) { 7627 const struct btf_type *local_type; 7628 const char *targ_name, *local_name; 7629 7630 local_type = btf__type_by_id(obj->btf, local_type_id); 7631 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 7632 targ_name = btf__name_by_offset(btf, targ_type->name_off); 7633 7634 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 7635 ext->name, local_type_id, 7636 btf_kind_str(local_type), local_name, targ_type_id, 7637 btf_kind_str(targ_type), targ_name); 7638 return -EINVAL; 7639 } 7640 7641 ext->is_set = true; 7642 ext->ksym.kernel_btf_obj_fd = btf_fd; 7643 ext->ksym.kernel_btf_id = id; 7644 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 7645 ext->name, id, btf_kind_str(targ_var), targ_var_name); 7646 7647 return 0; 7648 } 7649 7650 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 7651 struct extern_desc *ext) 7652 { 7653 int local_func_proto_id, kfunc_proto_id, kfunc_id; 7654 const struct btf_type *kern_func; 7655 struct btf *kern_btf = NULL; 7656 int ret, kern_btf_fd = 0; 7657 7658 local_func_proto_id = ext->ksym.type_id; 7659 7660 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, 7661 &kern_btf, &kern_btf_fd); 7662 if (kfunc_id < 0) { 7663 pr_warn("extern (func ksym) '%s': not found in kernel BTF\n", 7664 ext->name); 7665 return kfunc_id; 7666 } 7667 7668 if (kern_btf != obj->btf_vmlinux) { 7669 pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n", 7670 ext->name); 7671 return -ENOTSUP; 7672 } 7673 7674 kern_func = btf__type_by_id(kern_btf, kfunc_id); 7675 kfunc_proto_id = kern_func->type; 7676 7677 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 7678 kern_btf, kfunc_proto_id); 7679 if (ret <= 0) { 7680 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n", 7681 ext->name, local_func_proto_id, kfunc_proto_id); 7682 return -EINVAL; 7683 } 7684 7685 ext->is_set = true; 7686 ext->ksym.kernel_btf_obj_fd = kern_btf_fd; 7687 ext->ksym.kernel_btf_id = kfunc_id; 7688 pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n", 7689 ext->name, kfunc_id); 7690 7691 return 0; 7692 } 7693 7694 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 7695 { 7696 const struct btf_type *t; 7697 struct extern_desc *ext; 7698 int i, err; 7699 7700 for (i = 0; i < obj->nr_extern; i++) { 7701 ext = &obj->externs[i]; 7702 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 7703 continue; 7704 7705 t = btf__type_by_id(obj->btf, ext->btf_id); 7706 if (btf_is_var(t)) 7707 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 7708 else 7709 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 7710 if (err) 7711 return err; 7712 } 7713 return 0; 7714 } 7715 7716 static int bpf_object__resolve_externs(struct bpf_object *obj, 7717 const char *extra_kconfig) 7718 { 7719 bool need_config = false, need_kallsyms = false; 7720 bool need_vmlinux_btf = false; 7721 struct extern_desc *ext; 7722 void *kcfg_data = NULL; 7723 int err, i; 7724 7725 if (obj->nr_extern == 0) 7726 return 0; 7727 7728 if (obj->kconfig_map_idx >= 0) 7729 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 7730 7731 for (i = 0; i < obj->nr_extern; i++) { 7732 ext = &obj->externs[i]; 7733 7734 if (ext->type == EXT_KCFG && 7735 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 7736 void *ext_val = kcfg_data + ext->kcfg.data_off; 7737 __u32 kver = get_kernel_version(); 7738 7739 if (!kver) { 7740 pr_warn("failed to get kernel version\n"); 7741 return -EINVAL; 7742 } 7743 err = set_kcfg_value_num(ext, ext_val, kver); 7744 if (err) 7745 return err; 7746 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); 7747 } else if (ext->type == EXT_KCFG && 7748 strncmp(ext->name, "CONFIG_", 7) == 0) { 7749 need_config = true; 7750 } else if (ext->type == EXT_KSYM) { 7751 if (ext->ksym.type_id) 7752 need_vmlinux_btf = true; 7753 else 7754 need_kallsyms = true; 7755 } else { 7756 pr_warn("unrecognized extern '%s'\n", ext->name); 7757 return -EINVAL; 7758 } 7759 } 7760 if (need_config && extra_kconfig) { 7761 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 7762 if (err) 7763 return -EINVAL; 7764 need_config = false; 7765 for (i = 0; i < obj->nr_extern; i++) { 7766 ext = &obj->externs[i]; 7767 if (ext->type == EXT_KCFG && !ext->is_set) { 7768 need_config = true; 7769 break; 7770 } 7771 } 7772 } 7773 if (need_config) { 7774 err = bpf_object__read_kconfig_file(obj, kcfg_data); 7775 if (err) 7776 return -EINVAL; 7777 } 7778 if (need_kallsyms) { 7779 err = bpf_object__read_kallsyms_file(obj); 7780 if (err) 7781 return -EINVAL; 7782 } 7783 if (need_vmlinux_btf) { 7784 err = bpf_object__resolve_ksyms_btf_id(obj); 7785 if (err) 7786 return -EINVAL; 7787 } 7788 for (i = 0; i < obj->nr_extern; i++) { 7789 ext = &obj->externs[i]; 7790 7791 if (!ext->is_set && !ext->is_weak) { 7792 pr_warn("extern %s (strong) not resolved\n", ext->name); 7793 return -ESRCH; 7794 } else if (!ext->is_set) { 7795 pr_debug("extern %s (weak) not resolved, defaulting to zero\n", 7796 ext->name); 7797 } 7798 } 7799 7800 return 0; 7801 } 7802 7803 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 7804 { 7805 struct bpf_object *obj; 7806 int err, i; 7807 7808 if (!attr) 7809 return -EINVAL; 7810 obj = attr->obj; 7811 if (!obj) 7812 return -EINVAL; 7813 7814 if (obj->loaded) { 7815 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 7816 return -EINVAL; 7817 } 7818 7819 err = bpf_object__probe_loading(obj); 7820 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 7821 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 7822 err = err ? : bpf_object__sanitize_and_load_btf(obj); 7823 err = err ? : bpf_object__sanitize_maps(obj); 7824 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 7825 err = err ? : bpf_object__create_maps(obj); 7826 err = err ? : bpf_object__relocate(obj, attr->target_btf_path); 7827 err = err ? : bpf_object__load_progs(obj, attr->log_level); 7828 7829 /* clean up module BTFs */ 7830 for (i = 0; i < obj->btf_module_cnt; i++) { 7831 close(obj->btf_modules[i].fd); 7832 btf__free(obj->btf_modules[i].btf); 7833 free(obj->btf_modules[i].name); 7834 } 7835 free(obj->btf_modules); 7836 7837 /* clean up vmlinux BTF */ 7838 btf__free(obj->btf_vmlinux); 7839 obj->btf_vmlinux = NULL; 7840 7841 obj->loaded = true; /* doesn't matter if successfully or not */ 7842 7843 if (err) 7844 goto out; 7845 7846 return 0; 7847 out: 7848 /* unpin any maps that were auto-pinned during load */ 7849 for (i = 0; i < obj->nr_maps; i++) 7850 if (obj->maps[i].pinned && !obj->maps[i].reused) 7851 bpf_map__unpin(&obj->maps[i], NULL); 7852 7853 bpf_object__unload(obj); 7854 pr_warn("failed to load object '%s'\n", obj->path); 7855 return err; 7856 } 7857 7858 int bpf_object__load(struct bpf_object *obj) 7859 { 7860 struct bpf_object_load_attr attr = { 7861 .obj = obj, 7862 }; 7863 7864 return bpf_object__load_xattr(&attr); 7865 } 7866 7867 static int make_parent_dir(const char *path) 7868 { 7869 char *cp, errmsg[STRERR_BUFSIZE]; 7870 char *dname, *dir; 7871 int err = 0; 7872 7873 dname = strdup(path); 7874 if (dname == NULL) 7875 return -ENOMEM; 7876 7877 dir = dirname(dname); 7878 if (mkdir(dir, 0700) && errno != EEXIST) 7879 err = -errno; 7880 7881 free(dname); 7882 if (err) { 7883 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 7884 pr_warn("failed to mkdir %s: %s\n", path, cp); 7885 } 7886 return err; 7887 } 7888 7889 static int check_path(const char *path) 7890 { 7891 char *cp, errmsg[STRERR_BUFSIZE]; 7892 struct statfs st_fs; 7893 char *dname, *dir; 7894 int err = 0; 7895 7896 if (path == NULL) 7897 return -EINVAL; 7898 7899 dname = strdup(path); 7900 if (dname == NULL) 7901 return -ENOMEM; 7902 7903 dir = dirname(dname); 7904 if (statfs(dir, &st_fs)) { 7905 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7906 pr_warn("failed to statfs %s: %s\n", dir, cp); 7907 err = -errno; 7908 } 7909 free(dname); 7910 7911 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 7912 pr_warn("specified path %s is not on BPF FS\n", path); 7913 err = -EINVAL; 7914 } 7915 7916 return err; 7917 } 7918 7919 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 7920 int instance) 7921 { 7922 char *cp, errmsg[STRERR_BUFSIZE]; 7923 int err; 7924 7925 err = make_parent_dir(path); 7926 if (err) 7927 return err; 7928 7929 err = check_path(path); 7930 if (err) 7931 return err; 7932 7933 if (prog == NULL) { 7934 pr_warn("invalid program pointer\n"); 7935 return -EINVAL; 7936 } 7937 7938 if (instance < 0 || instance >= prog->instances.nr) { 7939 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7940 instance, prog->name, prog->instances.nr); 7941 return -EINVAL; 7942 } 7943 7944 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 7945 err = -errno; 7946 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 7947 pr_warn("failed to pin program: %s\n", cp); 7948 return err; 7949 } 7950 pr_debug("pinned program '%s'\n", path); 7951 7952 return 0; 7953 } 7954 7955 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 7956 int instance) 7957 { 7958 int err; 7959 7960 err = check_path(path); 7961 if (err) 7962 return err; 7963 7964 if (prog == NULL) { 7965 pr_warn("invalid program pointer\n"); 7966 return -EINVAL; 7967 } 7968 7969 if (instance < 0 || instance >= prog->instances.nr) { 7970 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7971 instance, prog->name, prog->instances.nr); 7972 return -EINVAL; 7973 } 7974 7975 err = unlink(path); 7976 if (err != 0) 7977 return -errno; 7978 pr_debug("unpinned program '%s'\n", path); 7979 7980 return 0; 7981 } 7982 7983 int bpf_program__pin(struct bpf_program *prog, const char *path) 7984 { 7985 int i, err; 7986 7987 err = make_parent_dir(path); 7988 if (err) 7989 return err; 7990 7991 err = check_path(path); 7992 if (err) 7993 return err; 7994 7995 if (prog == NULL) { 7996 pr_warn("invalid program pointer\n"); 7997 return -EINVAL; 7998 } 7999 8000 if (prog->instances.nr <= 0) { 8001 pr_warn("no instances of prog %s to pin\n", prog->name); 8002 return -EINVAL; 8003 } 8004 8005 if (prog->instances.nr == 1) { 8006 /* don't create subdirs when pinning single instance */ 8007 return bpf_program__pin_instance(prog, path, 0); 8008 } 8009 8010 for (i = 0; i < prog->instances.nr; i++) { 8011 char buf[PATH_MAX]; 8012 int len; 8013 8014 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 8015 if (len < 0) { 8016 err = -EINVAL; 8017 goto err_unpin; 8018 } else if (len >= PATH_MAX) { 8019 err = -ENAMETOOLONG; 8020 goto err_unpin; 8021 } 8022 8023 err = bpf_program__pin_instance(prog, buf, i); 8024 if (err) 8025 goto err_unpin; 8026 } 8027 8028 return 0; 8029 8030 err_unpin: 8031 for (i = i - 1; i >= 0; i--) { 8032 char buf[PATH_MAX]; 8033 int len; 8034 8035 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 8036 if (len < 0) 8037 continue; 8038 else if (len >= PATH_MAX) 8039 continue; 8040 8041 bpf_program__unpin_instance(prog, buf, i); 8042 } 8043 8044 rmdir(path); 8045 8046 return err; 8047 } 8048 8049 int bpf_program__unpin(struct bpf_program *prog, const char *path) 8050 { 8051 int i, err; 8052 8053 err = check_path(path); 8054 if (err) 8055 return err; 8056 8057 if (prog == NULL) { 8058 pr_warn("invalid program pointer\n"); 8059 return -EINVAL; 8060 } 8061 8062 if (prog->instances.nr <= 0) { 8063 pr_warn("no instances of prog %s to pin\n", prog->name); 8064 return -EINVAL; 8065 } 8066 8067 if (prog->instances.nr == 1) { 8068 /* don't create subdirs when pinning single instance */ 8069 return bpf_program__unpin_instance(prog, path, 0); 8070 } 8071 8072 for (i = 0; i < prog->instances.nr; i++) { 8073 char buf[PATH_MAX]; 8074 int len; 8075 8076 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 8077 if (len < 0) 8078 return -EINVAL; 8079 else if (len >= PATH_MAX) 8080 return -ENAMETOOLONG; 8081 8082 err = bpf_program__unpin_instance(prog, buf, i); 8083 if (err) 8084 return err; 8085 } 8086 8087 err = rmdir(path); 8088 if (err) 8089 return -errno; 8090 8091 return 0; 8092 } 8093 8094 int bpf_map__pin(struct bpf_map *map, const char *path) 8095 { 8096 char *cp, errmsg[STRERR_BUFSIZE]; 8097 int err; 8098 8099 if (map == NULL) { 8100 pr_warn("invalid map pointer\n"); 8101 return -EINVAL; 8102 } 8103 8104 if (map->pin_path) { 8105 if (path && strcmp(path, map->pin_path)) { 8106 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8107 bpf_map__name(map), map->pin_path, path); 8108 return -EINVAL; 8109 } else if (map->pinned) { 8110 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 8111 bpf_map__name(map), map->pin_path); 8112 return 0; 8113 } 8114 } else { 8115 if (!path) { 8116 pr_warn("missing a path to pin map '%s' at\n", 8117 bpf_map__name(map)); 8118 return -EINVAL; 8119 } else if (map->pinned) { 8120 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 8121 return -EEXIST; 8122 } 8123 8124 map->pin_path = strdup(path); 8125 if (!map->pin_path) { 8126 err = -errno; 8127 goto out_err; 8128 } 8129 } 8130 8131 err = make_parent_dir(map->pin_path); 8132 if (err) 8133 return err; 8134 8135 err = check_path(map->pin_path); 8136 if (err) 8137 return err; 8138 8139 if (bpf_obj_pin(map->fd, map->pin_path)) { 8140 err = -errno; 8141 goto out_err; 8142 } 8143 8144 map->pinned = true; 8145 pr_debug("pinned map '%s'\n", map->pin_path); 8146 8147 return 0; 8148 8149 out_err: 8150 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8151 pr_warn("failed to pin map: %s\n", cp); 8152 return err; 8153 } 8154 8155 int bpf_map__unpin(struct bpf_map *map, const char *path) 8156 { 8157 int err; 8158 8159 if (map == NULL) { 8160 pr_warn("invalid map pointer\n"); 8161 return -EINVAL; 8162 } 8163 8164 if (map->pin_path) { 8165 if (path && strcmp(path, map->pin_path)) { 8166 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8167 bpf_map__name(map), map->pin_path, path); 8168 return -EINVAL; 8169 } 8170 path = map->pin_path; 8171 } else if (!path) { 8172 pr_warn("no path to unpin map '%s' from\n", 8173 bpf_map__name(map)); 8174 return -EINVAL; 8175 } 8176 8177 err = check_path(path); 8178 if (err) 8179 return err; 8180 8181 err = unlink(path); 8182 if (err != 0) 8183 return -errno; 8184 8185 map->pinned = false; 8186 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 8187 8188 return 0; 8189 } 8190 8191 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 8192 { 8193 char *new = NULL; 8194 8195 if (path) { 8196 new = strdup(path); 8197 if (!new) 8198 return -errno; 8199 } 8200 8201 free(map->pin_path); 8202 map->pin_path = new; 8203 return 0; 8204 } 8205 8206 const char *bpf_map__get_pin_path(const struct bpf_map *map) 8207 { 8208 return map->pin_path; 8209 } 8210 8211 bool bpf_map__is_pinned(const struct bpf_map *map) 8212 { 8213 return map->pinned; 8214 } 8215 8216 static void sanitize_pin_path(char *s) 8217 { 8218 /* bpffs disallows periods in path names */ 8219 while (*s) { 8220 if (*s == '.') 8221 *s = '_'; 8222 s++; 8223 } 8224 } 8225 8226 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 8227 { 8228 struct bpf_map *map; 8229 int err; 8230 8231 if (!obj) 8232 return -ENOENT; 8233 8234 if (!obj->loaded) { 8235 pr_warn("object not yet loaded; load it first\n"); 8236 return -ENOENT; 8237 } 8238 8239 bpf_object__for_each_map(map, obj) { 8240 char *pin_path = NULL; 8241 char buf[PATH_MAX]; 8242 8243 if (path) { 8244 int len; 8245 8246 len = snprintf(buf, PATH_MAX, "%s/%s", path, 8247 bpf_map__name(map)); 8248 if (len < 0) { 8249 err = -EINVAL; 8250 goto err_unpin_maps; 8251 } else if (len >= PATH_MAX) { 8252 err = -ENAMETOOLONG; 8253 goto err_unpin_maps; 8254 } 8255 sanitize_pin_path(buf); 8256 pin_path = buf; 8257 } else if (!map->pin_path) { 8258 continue; 8259 } 8260 8261 err = bpf_map__pin(map, pin_path); 8262 if (err) 8263 goto err_unpin_maps; 8264 } 8265 8266 return 0; 8267 8268 err_unpin_maps: 8269 while ((map = bpf_map__prev(map, obj))) { 8270 if (!map->pin_path) 8271 continue; 8272 8273 bpf_map__unpin(map, NULL); 8274 } 8275 8276 return err; 8277 } 8278 8279 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 8280 { 8281 struct bpf_map *map; 8282 int err; 8283 8284 if (!obj) 8285 return -ENOENT; 8286 8287 bpf_object__for_each_map(map, obj) { 8288 char *pin_path = NULL; 8289 char buf[PATH_MAX]; 8290 8291 if (path) { 8292 int len; 8293 8294 len = snprintf(buf, PATH_MAX, "%s/%s", path, 8295 bpf_map__name(map)); 8296 if (len < 0) 8297 return -EINVAL; 8298 else if (len >= PATH_MAX) 8299 return -ENAMETOOLONG; 8300 sanitize_pin_path(buf); 8301 pin_path = buf; 8302 } else if (!map->pin_path) { 8303 continue; 8304 } 8305 8306 err = bpf_map__unpin(map, pin_path); 8307 if (err) 8308 return err; 8309 } 8310 8311 return 0; 8312 } 8313 8314 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 8315 { 8316 struct bpf_program *prog; 8317 int err; 8318 8319 if (!obj) 8320 return -ENOENT; 8321 8322 if (!obj->loaded) { 8323 pr_warn("object not yet loaded; load it first\n"); 8324 return -ENOENT; 8325 } 8326 8327 bpf_object__for_each_program(prog, obj) { 8328 char buf[PATH_MAX]; 8329 int len; 8330 8331 len = snprintf(buf, PATH_MAX, "%s/%s", path, 8332 prog->pin_name); 8333 if (len < 0) { 8334 err = -EINVAL; 8335 goto err_unpin_programs; 8336 } else if (len >= PATH_MAX) { 8337 err = -ENAMETOOLONG; 8338 goto err_unpin_programs; 8339 } 8340 8341 err = bpf_program__pin(prog, buf); 8342 if (err) 8343 goto err_unpin_programs; 8344 } 8345 8346 return 0; 8347 8348 err_unpin_programs: 8349 while ((prog = bpf_program__prev(prog, obj))) { 8350 char buf[PATH_MAX]; 8351 int len; 8352 8353 len = snprintf(buf, PATH_MAX, "%s/%s", path, 8354 prog->pin_name); 8355 if (len < 0) 8356 continue; 8357 else if (len >= PATH_MAX) 8358 continue; 8359 8360 bpf_program__unpin(prog, buf); 8361 } 8362 8363 return err; 8364 } 8365 8366 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 8367 { 8368 struct bpf_program *prog; 8369 int err; 8370 8371 if (!obj) 8372 return -ENOENT; 8373 8374 bpf_object__for_each_program(prog, obj) { 8375 char buf[PATH_MAX]; 8376 int len; 8377 8378 len = snprintf(buf, PATH_MAX, "%s/%s", path, 8379 prog->pin_name); 8380 if (len < 0) 8381 return -EINVAL; 8382 else if (len >= PATH_MAX) 8383 return -ENAMETOOLONG; 8384 8385 err = bpf_program__unpin(prog, buf); 8386 if (err) 8387 return err; 8388 } 8389 8390 return 0; 8391 } 8392 8393 int bpf_object__pin(struct bpf_object *obj, const char *path) 8394 { 8395 int err; 8396 8397 err = bpf_object__pin_maps(obj, path); 8398 if (err) 8399 return err; 8400 8401 err = bpf_object__pin_programs(obj, path); 8402 if (err) { 8403 bpf_object__unpin_maps(obj, path); 8404 return err; 8405 } 8406 8407 return 0; 8408 } 8409 8410 static void bpf_map__destroy(struct bpf_map *map) 8411 { 8412 if (map->clear_priv) 8413 map->clear_priv(map, map->priv); 8414 map->priv = NULL; 8415 map->clear_priv = NULL; 8416 8417 if (map->inner_map) { 8418 bpf_map__destroy(map->inner_map); 8419 zfree(&map->inner_map); 8420 } 8421 8422 zfree(&map->init_slots); 8423 map->init_slots_sz = 0; 8424 8425 if (map->mmaped) { 8426 munmap(map->mmaped, bpf_map_mmap_sz(map)); 8427 map->mmaped = NULL; 8428 } 8429 8430 if (map->st_ops) { 8431 zfree(&map->st_ops->data); 8432 zfree(&map->st_ops->progs); 8433 zfree(&map->st_ops->kern_func_off); 8434 zfree(&map->st_ops); 8435 } 8436 8437 zfree(&map->name); 8438 zfree(&map->pin_path); 8439 8440 if (map->fd >= 0) 8441 zclose(map->fd); 8442 } 8443 8444 void bpf_object__close(struct bpf_object *obj) 8445 { 8446 size_t i; 8447 8448 if (IS_ERR_OR_NULL(obj)) 8449 return; 8450 8451 if (obj->clear_priv) 8452 obj->clear_priv(obj, obj->priv); 8453 8454 bpf_object__elf_finish(obj); 8455 bpf_object__unload(obj); 8456 btf__free(obj->btf); 8457 btf_ext__free(obj->btf_ext); 8458 8459 for (i = 0; i < obj->nr_maps; i++) 8460 bpf_map__destroy(&obj->maps[i]); 8461 8462 zfree(&obj->kconfig); 8463 zfree(&obj->externs); 8464 obj->nr_extern = 0; 8465 8466 zfree(&obj->maps); 8467 obj->nr_maps = 0; 8468 8469 if (obj->programs && obj->nr_programs) { 8470 for (i = 0; i < obj->nr_programs; i++) 8471 bpf_program__exit(&obj->programs[i]); 8472 } 8473 zfree(&obj->programs); 8474 8475 list_del(&obj->list); 8476 free(obj); 8477 } 8478 8479 struct bpf_object * 8480 bpf_object__next(struct bpf_object *prev) 8481 { 8482 struct bpf_object *next; 8483 8484 if (!prev) 8485 next = list_first_entry(&bpf_objects_list, 8486 struct bpf_object, 8487 list); 8488 else 8489 next = list_next_entry(prev, list); 8490 8491 /* Empty list is noticed here so don't need checking on entry. */ 8492 if (&next->list == &bpf_objects_list) 8493 return NULL; 8494 8495 return next; 8496 } 8497 8498 const char *bpf_object__name(const struct bpf_object *obj) 8499 { 8500 return obj ? obj->name : ERR_PTR(-EINVAL); 8501 } 8502 8503 unsigned int bpf_object__kversion(const struct bpf_object *obj) 8504 { 8505 return obj ? obj->kern_version : 0; 8506 } 8507 8508 struct btf *bpf_object__btf(const struct bpf_object *obj) 8509 { 8510 return obj ? obj->btf : NULL; 8511 } 8512 8513 int bpf_object__btf_fd(const struct bpf_object *obj) 8514 { 8515 return obj->btf ? btf__fd(obj->btf) : -1; 8516 } 8517 8518 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 8519 { 8520 if (obj->loaded) 8521 return -EINVAL; 8522 8523 obj->kern_version = kern_version; 8524 8525 return 0; 8526 } 8527 8528 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 8529 bpf_object_clear_priv_t clear_priv) 8530 { 8531 if (obj->priv && obj->clear_priv) 8532 obj->clear_priv(obj, obj->priv); 8533 8534 obj->priv = priv; 8535 obj->clear_priv = clear_priv; 8536 return 0; 8537 } 8538 8539 void *bpf_object__priv(const struct bpf_object *obj) 8540 { 8541 return obj ? obj->priv : ERR_PTR(-EINVAL); 8542 } 8543 8544 static struct bpf_program * 8545 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 8546 bool forward) 8547 { 8548 size_t nr_programs = obj->nr_programs; 8549 ssize_t idx; 8550 8551 if (!nr_programs) 8552 return NULL; 8553 8554 if (!p) 8555 /* Iter from the beginning */ 8556 return forward ? &obj->programs[0] : 8557 &obj->programs[nr_programs - 1]; 8558 8559 if (p->obj != obj) { 8560 pr_warn("error: program handler doesn't match object\n"); 8561 return NULL; 8562 } 8563 8564 idx = (p - obj->programs) + (forward ? 1 : -1); 8565 if (idx >= obj->nr_programs || idx < 0) 8566 return NULL; 8567 return &obj->programs[idx]; 8568 } 8569 8570 struct bpf_program * 8571 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 8572 { 8573 struct bpf_program *prog = prev; 8574 8575 do { 8576 prog = __bpf_program__iter(prog, obj, true); 8577 } while (prog && prog_is_subprog(obj, prog)); 8578 8579 return prog; 8580 } 8581 8582 struct bpf_program * 8583 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 8584 { 8585 struct bpf_program *prog = next; 8586 8587 do { 8588 prog = __bpf_program__iter(prog, obj, false); 8589 } while (prog && prog_is_subprog(obj, prog)); 8590 8591 return prog; 8592 } 8593 8594 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 8595 bpf_program_clear_priv_t clear_priv) 8596 { 8597 if (prog->priv && prog->clear_priv) 8598 prog->clear_priv(prog, prog->priv); 8599 8600 prog->priv = priv; 8601 prog->clear_priv = clear_priv; 8602 return 0; 8603 } 8604 8605 void *bpf_program__priv(const struct bpf_program *prog) 8606 { 8607 return prog ? prog->priv : ERR_PTR(-EINVAL); 8608 } 8609 8610 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 8611 { 8612 prog->prog_ifindex = ifindex; 8613 } 8614 8615 const char *bpf_program__name(const struct bpf_program *prog) 8616 { 8617 return prog->name; 8618 } 8619 8620 const char *bpf_program__section_name(const struct bpf_program *prog) 8621 { 8622 return prog->sec_name; 8623 } 8624 8625 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 8626 { 8627 const char *title; 8628 8629 title = prog->sec_name; 8630 if (needs_copy) { 8631 title = strdup(title); 8632 if (!title) { 8633 pr_warn("failed to strdup program title\n"); 8634 return ERR_PTR(-ENOMEM); 8635 } 8636 } 8637 8638 return title; 8639 } 8640 8641 bool bpf_program__autoload(const struct bpf_program *prog) 8642 { 8643 return prog->load; 8644 } 8645 8646 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 8647 { 8648 if (prog->obj->loaded) 8649 return -EINVAL; 8650 8651 prog->load = autoload; 8652 return 0; 8653 } 8654 8655 int bpf_program__fd(const struct bpf_program *prog) 8656 { 8657 return bpf_program__nth_fd(prog, 0); 8658 } 8659 8660 size_t bpf_program__size(const struct bpf_program *prog) 8661 { 8662 return prog->insns_cnt * BPF_INSN_SZ; 8663 } 8664 8665 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 8666 bpf_program_prep_t prep) 8667 { 8668 int *instances_fds; 8669 8670 if (nr_instances <= 0 || !prep) 8671 return -EINVAL; 8672 8673 if (prog->instances.nr > 0 || prog->instances.fds) { 8674 pr_warn("Can't set pre-processor after loading\n"); 8675 return -EINVAL; 8676 } 8677 8678 instances_fds = malloc(sizeof(int) * nr_instances); 8679 if (!instances_fds) { 8680 pr_warn("alloc memory failed for fds\n"); 8681 return -ENOMEM; 8682 } 8683 8684 /* fill all fd with -1 */ 8685 memset(instances_fds, -1, sizeof(int) * nr_instances); 8686 8687 prog->instances.nr = nr_instances; 8688 prog->instances.fds = instances_fds; 8689 prog->preprocessor = prep; 8690 return 0; 8691 } 8692 8693 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 8694 { 8695 int fd; 8696 8697 if (!prog) 8698 return -EINVAL; 8699 8700 if (n >= prog->instances.nr || n < 0) { 8701 pr_warn("Can't get the %dth fd from program %s: only %d instances\n", 8702 n, prog->name, prog->instances.nr); 8703 return -EINVAL; 8704 } 8705 8706 fd = prog->instances.fds[n]; 8707 if (fd < 0) { 8708 pr_warn("%dth instance of program '%s' is invalid\n", 8709 n, prog->name); 8710 return -ENOENT; 8711 } 8712 8713 return fd; 8714 } 8715 8716 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog) 8717 { 8718 return prog->type; 8719 } 8720 8721 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 8722 { 8723 prog->type = type; 8724 } 8725 8726 static bool bpf_program__is_type(const struct bpf_program *prog, 8727 enum bpf_prog_type type) 8728 { 8729 return prog ? (prog->type == type) : false; 8730 } 8731 8732 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 8733 int bpf_program__set_##NAME(struct bpf_program *prog) \ 8734 { \ 8735 if (!prog) \ 8736 return -EINVAL; \ 8737 bpf_program__set_type(prog, TYPE); \ 8738 return 0; \ 8739 } \ 8740 \ 8741 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 8742 { \ 8743 return bpf_program__is_type(prog, TYPE); \ 8744 } \ 8745 8746 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 8747 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); 8748 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 8749 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 8750 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 8751 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 8752 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 8753 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 8754 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 8755 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); 8756 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); 8757 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); 8758 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP); 8759 8760 enum bpf_attach_type 8761 bpf_program__get_expected_attach_type(const struct bpf_program *prog) 8762 { 8763 return prog->expected_attach_type; 8764 } 8765 8766 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 8767 enum bpf_attach_type type) 8768 { 8769 prog->expected_attach_type = type; 8770 } 8771 8772 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \ 8773 attachable, attach_btf) \ 8774 { \ 8775 .sec = string, \ 8776 .len = sizeof(string) - 1, \ 8777 .prog_type = ptype, \ 8778 .expected_attach_type = eatype, \ 8779 .is_exp_attach_type_optional = eatype_optional, \ 8780 .is_attachable = attachable, \ 8781 .is_attach_btf = attach_btf, \ 8782 } 8783 8784 /* Programs that can NOT be attached. */ 8785 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) 8786 8787 /* Programs that can be attached. */ 8788 #define BPF_APROG_SEC(string, ptype, atype) \ 8789 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0) 8790 8791 /* Programs that must specify expected attach type at load time. */ 8792 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 8793 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0) 8794 8795 /* Programs that use BTF to identify attach point */ 8796 #define BPF_PROG_BTF(string, ptype, eatype) \ 8797 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1) 8798 8799 /* Programs that can be attached but attach type can't be identified by section 8800 * name. Kept for backward compatibility. 8801 */ 8802 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 8803 8804 #define SEC_DEF(sec_pfx, ptype, ...) { \ 8805 .sec = sec_pfx, \ 8806 .len = sizeof(sec_pfx) - 1, \ 8807 .prog_type = BPF_PROG_TYPE_##ptype, \ 8808 __VA_ARGS__ \ 8809 } 8810 8811 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 8812 struct bpf_program *prog); 8813 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 8814 struct bpf_program *prog); 8815 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 8816 struct bpf_program *prog); 8817 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 8818 struct bpf_program *prog); 8819 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 8820 struct bpf_program *prog); 8821 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 8822 struct bpf_program *prog); 8823 8824 static const struct bpf_sec_def section_defs[] = { 8825 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 8826 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT), 8827 SEC_DEF("kprobe/", KPROBE, 8828 .attach_fn = attach_kprobe), 8829 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), 8830 SEC_DEF("kretprobe/", KPROBE, 8831 .attach_fn = attach_kprobe), 8832 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), 8833 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 8834 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 8835 SEC_DEF("tracepoint/", TRACEPOINT, 8836 .attach_fn = attach_tp), 8837 SEC_DEF("tp/", TRACEPOINT, 8838 .attach_fn = attach_tp), 8839 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 8840 .attach_fn = attach_raw_tp), 8841 SEC_DEF("raw_tp/", RAW_TRACEPOINT, 8842 .attach_fn = attach_raw_tp), 8843 SEC_DEF("tp_btf/", TRACING, 8844 .expected_attach_type = BPF_TRACE_RAW_TP, 8845 .is_attach_btf = true, 8846 .attach_fn = attach_trace), 8847 SEC_DEF("fentry/", TRACING, 8848 .expected_attach_type = BPF_TRACE_FENTRY, 8849 .is_attach_btf = true, 8850 .attach_fn = attach_trace), 8851 SEC_DEF("fmod_ret/", TRACING, 8852 .expected_attach_type = BPF_MODIFY_RETURN, 8853 .is_attach_btf = true, 8854 .attach_fn = attach_trace), 8855 SEC_DEF("fexit/", TRACING, 8856 .expected_attach_type = BPF_TRACE_FEXIT, 8857 .is_attach_btf = true, 8858 .attach_fn = attach_trace), 8859 SEC_DEF("fentry.s/", TRACING, 8860 .expected_attach_type = BPF_TRACE_FENTRY, 8861 .is_attach_btf = true, 8862 .is_sleepable = true, 8863 .attach_fn = attach_trace), 8864 SEC_DEF("fmod_ret.s/", TRACING, 8865 .expected_attach_type = BPF_MODIFY_RETURN, 8866 .is_attach_btf = true, 8867 .is_sleepable = true, 8868 .attach_fn = attach_trace), 8869 SEC_DEF("fexit.s/", TRACING, 8870 .expected_attach_type = BPF_TRACE_FEXIT, 8871 .is_attach_btf = true, 8872 .is_sleepable = true, 8873 .attach_fn = attach_trace), 8874 SEC_DEF("freplace/", EXT, 8875 .is_attach_btf = true, 8876 .attach_fn = attach_trace), 8877 SEC_DEF("lsm/", LSM, 8878 .is_attach_btf = true, 8879 .expected_attach_type = BPF_LSM_MAC, 8880 .attach_fn = attach_lsm), 8881 SEC_DEF("lsm.s/", LSM, 8882 .is_attach_btf = true, 8883 .is_sleepable = true, 8884 .expected_attach_type = BPF_LSM_MAC, 8885 .attach_fn = attach_lsm), 8886 SEC_DEF("iter/", TRACING, 8887 .expected_attach_type = BPF_TRACE_ITER, 8888 .is_attach_btf = true, 8889 .attach_fn = attach_iter), 8890 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP, 8891 BPF_XDP_DEVMAP), 8892 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP, 8893 BPF_XDP_CPUMAP), 8894 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP, 8895 BPF_XDP), 8896 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 8897 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 8898 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 8899 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 8900 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 8901 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 8902 BPF_CGROUP_INET_INGRESS), 8903 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 8904 BPF_CGROUP_INET_EGRESS), 8905 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 8906 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK, 8907 BPF_CGROUP_INET_SOCK_CREATE), 8908 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK, 8909 BPF_CGROUP_INET_SOCK_RELEASE), 8910 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 8911 BPF_CGROUP_INET_SOCK_CREATE), 8912 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 8913 BPF_CGROUP_INET4_POST_BIND), 8914 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 8915 BPF_CGROUP_INET6_POST_BIND), 8916 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 8917 BPF_CGROUP_DEVICE), 8918 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 8919 BPF_CGROUP_SOCK_OPS), 8920 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 8921 BPF_SK_SKB_STREAM_PARSER), 8922 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 8923 BPF_SK_SKB_STREAM_VERDICT), 8924 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 8925 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 8926 BPF_SK_MSG_VERDICT), 8927 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 8928 BPF_LIRC_MODE2), 8929 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 8930 BPF_FLOW_DISSECTOR), 8931 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8932 BPF_CGROUP_INET4_BIND), 8933 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8934 BPF_CGROUP_INET6_BIND), 8935 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8936 BPF_CGROUP_INET4_CONNECT), 8937 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8938 BPF_CGROUP_INET6_CONNECT), 8939 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8940 BPF_CGROUP_UDP4_SENDMSG), 8941 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8942 BPF_CGROUP_UDP6_SENDMSG), 8943 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8944 BPF_CGROUP_UDP4_RECVMSG), 8945 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8946 BPF_CGROUP_UDP6_RECVMSG), 8947 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8948 BPF_CGROUP_INET4_GETPEERNAME), 8949 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8950 BPF_CGROUP_INET6_GETPEERNAME), 8951 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8952 BPF_CGROUP_INET4_GETSOCKNAME), 8953 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8954 BPF_CGROUP_INET6_GETSOCKNAME), 8955 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 8956 BPF_CGROUP_SYSCTL), 8957 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8958 BPF_CGROUP_GETSOCKOPT), 8959 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8960 BPF_CGROUP_SETSOCKOPT), 8961 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), 8962 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP, 8963 BPF_SK_LOOKUP), 8964 }; 8965 8966 #undef BPF_PROG_SEC_IMPL 8967 #undef BPF_PROG_SEC 8968 #undef BPF_APROG_SEC 8969 #undef BPF_EAPROG_SEC 8970 #undef BPF_APROG_COMPAT 8971 #undef SEC_DEF 8972 8973 #define MAX_TYPE_NAME_SIZE 32 8974 8975 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 8976 { 8977 int i, n = ARRAY_SIZE(section_defs); 8978 8979 for (i = 0; i < n; i++) { 8980 if (strncmp(sec_name, 8981 section_defs[i].sec, section_defs[i].len)) 8982 continue; 8983 return §ion_defs[i]; 8984 } 8985 return NULL; 8986 } 8987 8988 static char *libbpf_get_type_names(bool attach_type) 8989 { 8990 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 8991 char *buf; 8992 8993 buf = malloc(len); 8994 if (!buf) 8995 return NULL; 8996 8997 buf[0] = '\0'; 8998 /* Forge string buf with all available names */ 8999 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 9000 if (attach_type && !section_defs[i].is_attachable) 9001 continue; 9002 9003 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 9004 free(buf); 9005 return NULL; 9006 } 9007 strcat(buf, " "); 9008 strcat(buf, section_defs[i].sec); 9009 } 9010 9011 return buf; 9012 } 9013 9014 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 9015 enum bpf_attach_type *expected_attach_type) 9016 { 9017 const struct bpf_sec_def *sec_def; 9018 char *type_names; 9019 9020 if (!name) 9021 return -EINVAL; 9022 9023 sec_def = find_sec_def(name); 9024 if (sec_def) { 9025 *prog_type = sec_def->prog_type; 9026 *expected_attach_type = sec_def->expected_attach_type; 9027 return 0; 9028 } 9029 9030 pr_debug("failed to guess program type from ELF section '%s'\n", name); 9031 type_names = libbpf_get_type_names(false); 9032 if (type_names != NULL) { 9033 pr_debug("supported section(type) names are:%s\n", type_names); 9034 free(type_names); 9035 } 9036 9037 return -ESRCH; 9038 } 9039 9040 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 9041 size_t offset) 9042 { 9043 struct bpf_map *map; 9044 size_t i; 9045 9046 for (i = 0; i < obj->nr_maps; i++) { 9047 map = &obj->maps[i]; 9048 if (!bpf_map__is_struct_ops(map)) 9049 continue; 9050 if (map->sec_offset <= offset && 9051 offset - map->sec_offset < map->def.value_size) 9052 return map; 9053 } 9054 9055 return NULL; 9056 } 9057 9058 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 9059 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 9060 GElf_Shdr *shdr, Elf_Data *data) 9061 { 9062 const struct btf_member *member; 9063 struct bpf_struct_ops *st_ops; 9064 struct bpf_program *prog; 9065 unsigned int shdr_idx; 9066 const struct btf *btf; 9067 struct bpf_map *map; 9068 Elf_Data *symbols; 9069 unsigned int moff, insn_idx; 9070 const char *name; 9071 __u32 member_idx; 9072 GElf_Sym sym; 9073 GElf_Rel rel; 9074 int i, nrels; 9075 9076 symbols = obj->efile.symbols; 9077 btf = obj->btf; 9078 nrels = shdr->sh_size / shdr->sh_entsize; 9079 for (i = 0; i < nrels; i++) { 9080 if (!gelf_getrel(data, i, &rel)) { 9081 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 9082 return -LIBBPF_ERRNO__FORMAT; 9083 } 9084 9085 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 9086 pr_warn("struct_ops reloc: symbol %zx not found\n", 9087 (size_t)GELF_R_SYM(rel.r_info)); 9088 return -LIBBPF_ERRNO__FORMAT; 9089 } 9090 9091 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 9092 map = find_struct_ops_map_by_offset(obj, rel.r_offset); 9093 if (!map) { 9094 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", 9095 (size_t)rel.r_offset); 9096 return -EINVAL; 9097 } 9098 9099 moff = rel.r_offset - map->sec_offset; 9100 shdr_idx = sym.st_shndx; 9101 st_ops = map->st_ops; 9102 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 9103 map->name, 9104 (long long)(rel.r_info >> 32), 9105 (long long)sym.st_value, 9106 shdr_idx, (size_t)rel.r_offset, 9107 map->sec_offset, sym.st_name, name); 9108 9109 if (shdr_idx >= SHN_LORESERVE) { 9110 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", 9111 map->name, (size_t)rel.r_offset, shdr_idx); 9112 return -LIBBPF_ERRNO__RELOC; 9113 } 9114 if (sym.st_value % BPF_INSN_SZ) { 9115 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 9116 map->name, (unsigned long long)sym.st_value); 9117 return -LIBBPF_ERRNO__FORMAT; 9118 } 9119 insn_idx = sym.st_value / BPF_INSN_SZ; 9120 9121 member = find_member_by_offset(st_ops->type, moff * 8); 9122 if (!member) { 9123 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 9124 map->name, moff); 9125 return -EINVAL; 9126 } 9127 member_idx = member - btf_members(st_ops->type); 9128 name = btf__name_by_offset(btf, member->name_off); 9129 9130 if (!resolve_func_ptr(btf, member->type, NULL)) { 9131 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 9132 map->name, name); 9133 return -EINVAL; 9134 } 9135 9136 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 9137 if (!prog) { 9138 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 9139 map->name, shdr_idx, name); 9140 return -EINVAL; 9141 } 9142 9143 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 9144 const struct bpf_sec_def *sec_def; 9145 9146 sec_def = find_sec_def(prog->sec_name); 9147 if (sec_def && 9148 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) { 9149 /* for pr_warn */ 9150 prog->type = sec_def->prog_type; 9151 goto invalid_prog; 9152 } 9153 9154 prog->type = BPF_PROG_TYPE_STRUCT_OPS; 9155 prog->attach_btf_id = st_ops->type_id; 9156 prog->expected_attach_type = member_idx; 9157 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || 9158 prog->attach_btf_id != st_ops->type_id || 9159 prog->expected_attach_type != member_idx) { 9160 goto invalid_prog; 9161 } 9162 st_ops->progs[member_idx] = prog; 9163 } 9164 9165 return 0; 9166 9167 invalid_prog: 9168 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 9169 map->name, prog->name, prog->sec_name, prog->type, 9170 prog->attach_btf_id, prog->expected_attach_type, name); 9171 return -EINVAL; 9172 } 9173 9174 #define BTF_TRACE_PREFIX "btf_trace_" 9175 #define BTF_LSM_PREFIX "bpf_lsm_" 9176 #define BTF_ITER_PREFIX "bpf_iter_" 9177 #define BTF_MAX_NAME_SIZE 128 9178 9179 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 9180 const char *name, __u32 kind) 9181 { 9182 char btf_type_name[BTF_MAX_NAME_SIZE]; 9183 int ret; 9184 9185 ret = snprintf(btf_type_name, sizeof(btf_type_name), 9186 "%s%s", prefix, name); 9187 /* snprintf returns the number of characters written excluding the 9188 * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 9189 * indicates truncation. 9190 */ 9191 if (ret < 0 || ret >= sizeof(btf_type_name)) 9192 return -ENAMETOOLONG; 9193 return btf__find_by_name_kind(btf, btf_type_name, kind); 9194 } 9195 9196 static inline int find_attach_btf_id(struct btf *btf, const char *name, 9197 enum bpf_attach_type attach_type) 9198 { 9199 int err; 9200 9201 if (attach_type == BPF_TRACE_RAW_TP) 9202 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name, 9203 BTF_KIND_TYPEDEF); 9204 else if (attach_type == BPF_LSM_MAC) 9205 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name, 9206 BTF_KIND_FUNC); 9207 else if (attach_type == BPF_TRACE_ITER) 9208 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name, 9209 BTF_KIND_FUNC); 9210 else 9211 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 9212 9213 return err; 9214 } 9215 9216 int libbpf_find_vmlinux_btf_id(const char *name, 9217 enum bpf_attach_type attach_type) 9218 { 9219 struct btf *btf; 9220 int err; 9221 9222 btf = libbpf_find_kernel_btf(); 9223 if (IS_ERR(btf)) { 9224 pr_warn("vmlinux BTF is not found\n"); 9225 return -EINVAL; 9226 } 9227 9228 err = find_attach_btf_id(btf, name, attach_type); 9229 if (err <= 0) 9230 pr_warn("%s is not found in vmlinux BTF\n", name); 9231 9232 btf__free(btf); 9233 return err; 9234 } 9235 9236 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 9237 { 9238 struct bpf_prog_info_linear *info_linear; 9239 struct bpf_prog_info *info; 9240 struct btf *btf = NULL; 9241 int err = -EINVAL; 9242 9243 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); 9244 if (IS_ERR_OR_NULL(info_linear)) { 9245 pr_warn("failed get_prog_info_linear for FD %d\n", 9246 attach_prog_fd); 9247 return -EINVAL; 9248 } 9249 info = &info_linear->info; 9250 if (!info->btf_id) { 9251 pr_warn("The target program doesn't have BTF\n"); 9252 goto out; 9253 } 9254 if (btf__get_from_id(info->btf_id, &btf)) { 9255 pr_warn("Failed to get BTF of the program\n"); 9256 goto out; 9257 } 9258 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 9259 btf__free(btf); 9260 if (err <= 0) { 9261 pr_warn("%s is not found in prog's BTF\n", name); 9262 goto out; 9263 } 9264 out: 9265 free(info_linear); 9266 return err; 9267 } 9268 9269 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 9270 enum bpf_attach_type attach_type, 9271 int *btf_obj_fd, int *btf_type_id) 9272 { 9273 int ret, i; 9274 9275 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 9276 if (ret > 0) { 9277 *btf_obj_fd = 0; /* vmlinux BTF */ 9278 *btf_type_id = ret; 9279 return 0; 9280 } 9281 if (ret != -ENOENT) 9282 return ret; 9283 9284 ret = load_module_btfs(obj); 9285 if (ret) 9286 return ret; 9287 9288 for (i = 0; i < obj->btf_module_cnt; i++) { 9289 const struct module_btf *mod = &obj->btf_modules[i]; 9290 9291 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 9292 if (ret > 0) { 9293 *btf_obj_fd = mod->fd; 9294 *btf_type_id = ret; 9295 return 0; 9296 } 9297 if (ret == -ENOENT) 9298 continue; 9299 9300 return ret; 9301 } 9302 9303 return -ESRCH; 9304 } 9305 9306 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id) 9307 { 9308 enum bpf_attach_type attach_type = prog->expected_attach_type; 9309 __u32 attach_prog_fd = prog->attach_prog_fd; 9310 const char *name = prog->sec_name, *attach_name; 9311 const struct bpf_sec_def *sec = NULL; 9312 int i, err; 9313 9314 if (!name) 9315 return -EINVAL; 9316 9317 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 9318 if (!section_defs[i].is_attach_btf) 9319 continue; 9320 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 9321 continue; 9322 9323 sec = §ion_defs[i]; 9324 break; 9325 } 9326 9327 if (!sec) { 9328 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name); 9329 return -ESRCH; 9330 } 9331 attach_name = name + sec->len; 9332 9333 /* BPF program's BTF ID */ 9334 if (attach_prog_fd) { 9335 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 9336 if (err < 0) { 9337 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 9338 attach_prog_fd, attach_name, err); 9339 return err; 9340 } 9341 *btf_obj_fd = 0; 9342 *btf_type_id = err; 9343 return 0; 9344 } 9345 9346 /* kernel/module BTF ID */ 9347 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); 9348 if (err) { 9349 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err); 9350 return err; 9351 } 9352 return 0; 9353 } 9354 9355 int libbpf_attach_type_by_name(const char *name, 9356 enum bpf_attach_type *attach_type) 9357 { 9358 char *type_names; 9359 int i; 9360 9361 if (!name) 9362 return -EINVAL; 9363 9364 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 9365 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 9366 continue; 9367 if (!section_defs[i].is_attachable) 9368 return -EINVAL; 9369 *attach_type = section_defs[i].expected_attach_type; 9370 return 0; 9371 } 9372 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 9373 type_names = libbpf_get_type_names(true); 9374 if (type_names != NULL) { 9375 pr_debug("attachable section(type) names are:%s\n", type_names); 9376 free(type_names); 9377 } 9378 9379 return -EINVAL; 9380 } 9381 9382 int bpf_map__fd(const struct bpf_map *map) 9383 { 9384 return map ? map->fd : -EINVAL; 9385 } 9386 9387 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 9388 { 9389 return map ? &map->def : ERR_PTR(-EINVAL); 9390 } 9391 9392 const char *bpf_map__name(const struct bpf_map *map) 9393 { 9394 return map ? map->name : NULL; 9395 } 9396 9397 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 9398 { 9399 return map->def.type; 9400 } 9401 9402 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 9403 { 9404 if (map->fd >= 0) 9405 return -EBUSY; 9406 map->def.type = type; 9407 return 0; 9408 } 9409 9410 __u32 bpf_map__map_flags(const struct bpf_map *map) 9411 { 9412 return map->def.map_flags; 9413 } 9414 9415 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 9416 { 9417 if (map->fd >= 0) 9418 return -EBUSY; 9419 map->def.map_flags = flags; 9420 return 0; 9421 } 9422 9423 __u32 bpf_map__numa_node(const struct bpf_map *map) 9424 { 9425 return map->numa_node; 9426 } 9427 9428 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 9429 { 9430 if (map->fd >= 0) 9431 return -EBUSY; 9432 map->numa_node = numa_node; 9433 return 0; 9434 } 9435 9436 __u32 bpf_map__key_size(const struct bpf_map *map) 9437 { 9438 return map->def.key_size; 9439 } 9440 9441 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 9442 { 9443 if (map->fd >= 0) 9444 return -EBUSY; 9445 map->def.key_size = size; 9446 return 0; 9447 } 9448 9449 __u32 bpf_map__value_size(const struct bpf_map *map) 9450 { 9451 return map->def.value_size; 9452 } 9453 9454 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 9455 { 9456 if (map->fd >= 0) 9457 return -EBUSY; 9458 map->def.value_size = size; 9459 return 0; 9460 } 9461 9462 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 9463 { 9464 return map ? map->btf_key_type_id : 0; 9465 } 9466 9467 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 9468 { 9469 return map ? map->btf_value_type_id : 0; 9470 } 9471 9472 int bpf_map__set_priv(struct bpf_map *map, void *priv, 9473 bpf_map_clear_priv_t clear_priv) 9474 { 9475 if (!map) 9476 return -EINVAL; 9477 9478 if (map->priv) { 9479 if (map->clear_priv) 9480 map->clear_priv(map, map->priv); 9481 } 9482 9483 map->priv = priv; 9484 map->clear_priv = clear_priv; 9485 return 0; 9486 } 9487 9488 void *bpf_map__priv(const struct bpf_map *map) 9489 { 9490 return map ? map->priv : ERR_PTR(-EINVAL); 9491 } 9492 9493 int bpf_map__set_initial_value(struct bpf_map *map, 9494 const void *data, size_t size) 9495 { 9496 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 9497 size != map->def.value_size || map->fd >= 0) 9498 return -EINVAL; 9499 9500 memcpy(map->mmaped, data, size); 9501 return 0; 9502 } 9503 9504 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 9505 { 9506 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 9507 } 9508 9509 bool bpf_map__is_internal(const struct bpf_map *map) 9510 { 9511 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 9512 } 9513 9514 __u32 bpf_map__ifindex(const struct bpf_map *map) 9515 { 9516 return map->map_ifindex; 9517 } 9518 9519 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 9520 { 9521 if (map->fd >= 0) 9522 return -EBUSY; 9523 map->map_ifindex = ifindex; 9524 return 0; 9525 } 9526 9527 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 9528 { 9529 if (!bpf_map_type__is_map_in_map(map->def.type)) { 9530 pr_warn("error: unsupported map type\n"); 9531 return -EINVAL; 9532 } 9533 if (map->inner_map_fd != -1) { 9534 pr_warn("error: inner_map_fd already specified\n"); 9535 return -EINVAL; 9536 } 9537 zfree(&map->inner_map); 9538 map->inner_map_fd = fd; 9539 return 0; 9540 } 9541 9542 static struct bpf_map * 9543 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 9544 { 9545 ssize_t idx; 9546 struct bpf_map *s, *e; 9547 9548 if (!obj || !obj->maps) 9549 return NULL; 9550 9551 s = obj->maps; 9552 e = obj->maps + obj->nr_maps; 9553 9554 if ((m < s) || (m >= e)) { 9555 pr_warn("error in %s: map handler doesn't belong to object\n", 9556 __func__); 9557 return NULL; 9558 } 9559 9560 idx = (m - obj->maps) + i; 9561 if (idx >= obj->nr_maps || idx < 0) 9562 return NULL; 9563 return &obj->maps[idx]; 9564 } 9565 9566 struct bpf_map * 9567 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 9568 { 9569 if (prev == NULL) 9570 return obj->maps; 9571 9572 return __bpf_map__iter(prev, obj, 1); 9573 } 9574 9575 struct bpf_map * 9576 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 9577 { 9578 if (next == NULL) { 9579 if (!obj->nr_maps) 9580 return NULL; 9581 return obj->maps + obj->nr_maps - 1; 9582 } 9583 9584 return __bpf_map__iter(next, obj, -1); 9585 } 9586 9587 struct bpf_map * 9588 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 9589 { 9590 struct bpf_map *pos; 9591 9592 bpf_object__for_each_map(pos, obj) { 9593 if (pos->name && !strcmp(pos->name, name)) 9594 return pos; 9595 } 9596 return NULL; 9597 } 9598 9599 int 9600 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 9601 { 9602 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 9603 } 9604 9605 struct bpf_map * 9606 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 9607 { 9608 return ERR_PTR(-ENOTSUP); 9609 } 9610 9611 long libbpf_get_error(const void *ptr) 9612 { 9613 return PTR_ERR_OR_ZERO(ptr); 9614 } 9615 9616 int bpf_prog_load(const char *file, enum bpf_prog_type type, 9617 struct bpf_object **pobj, int *prog_fd) 9618 { 9619 struct bpf_prog_load_attr attr; 9620 9621 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 9622 attr.file = file; 9623 attr.prog_type = type; 9624 attr.expected_attach_type = 0; 9625 9626 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 9627 } 9628 9629 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 9630 struct bpf_object **pobj, int *prog_fd) 9631 { 9632 struct bpf_object_open_attr open_attr = {}; 9633 struct bpf_program *prog, *first_prog = NULL; 9634 struct bpf_object *obj; 9635 struct bpf_map *map; 9636 int err; 9637 9638 if (!attr) 9639 return -EINVAL; 9640 if (!attr->file) 9641 return -EINVAL; 9642 9643 open_attr.file = attr->file; 9644 open_attr.prog_type = attr->prog_type; 9645 9646 obj = bpf_object__open_xattr(&open_attr); 9647 if (IS_ERR_OR_NULL(obj)) 9648 return -ENOENT; 9649 9650 bpf_object__for_each_program(prog, obj) { 9651 enum bpf_attach_type attach_type = attr->expected_attach_type; 9652 /* 9653 * to preserve backwards compatibility, bpf_prog_load treats 9654 * attr->prog_type, if specified, as an override to whatever 9655 * bpf_object__open guessed 9656 */ 9657 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { 9658 bpf_program__set_type(prog, attr->prog_type); 9659 bpf_program__set_expected_attach_type(prog, 9660 attach_type); 9661 } 9662 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { 9663 /* 9664 * we haven't guessed from section name and user 9665 * didn't provide a fallback type, too bad... 9666 */ 9667 bpf_object__close(obj); 9668 return -EINVAL; 9669 } 9670 9671 prog->prog_ifindex = attr->ifindex; 9672 prog->log_level = attr->log_level; 9673 prog->prog_flags |= attr->prog_flags; 9674 if (!first_prog) 9675 first_prog = prog; 9676 } 9677 9678 bpf_object__for_each_map(map, obj) { 9679 if (!bpf_map__is_offload_neutral(map)) 9680 map->map_ifindex = attr->ifindex; 9681 } 9682 9683 if (!first_prog) { 9684 pr_warn("object file doesn't contain bpf program\n"); 9685 bpf_object__close(obj); 9686 return -ENOENT; 9687 } 9688 9689 err = bpf_object__load(obj); 9690 if (err) { 9691 bpf_object__close(obj); 9692 return err; 9693 } 9694 9695 *pobj = obj; 9696 *prog_fd = bpf_program__fd(first_prog); 9697 return 0; 9698 } 9699 9700 struct bpf_link { 9701 int (*detach)(struct bpf_link *link); 9702 int (*destroy)(struct bpf_link *link); 9703 char *pin_path; /* NULL, if not pinned */ 9704 int fd; /* hook FD, -1 if not applicable */ 9705 bool disconnected; 9706 }; 9707 9708 /* Replace link's underlying BPF program with the new one */ 9709 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 9710 { 9711 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 9712 } 9713 9714 /* Release "ownership" of underlying BPF resource (typically, BPF program 9715 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 9716 * link, when destructed through bpf_link__destroy() call won't attempt to 9717 * detach/unregisted that BPF resource. This is useful in situations where, 9718 * say, attached BPF program has to outlive userspace program that attached it 9719 * in the system. Depending on type of BPF program, though, there might be 9720 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 9721 * exit of userspace program doesn't trigger automatic detachment and clean up 9722 * inside the kernel. 9723 */ 9724 void bpf_link__disconnect(struct bpf_link *link) 9725 { 9726 link->disconnected = true; 9727 } 9728 9729 int bpf_link__destroy(struct bpf_link *link) 9730 { 9731 int err = 0; 9732 9733 if (IS_ERR_OR_NULL(link)) 9734 return 0; 9735 9736 if (!link->disconnected && link->detach) 9737 err = link->detach(link); 9738 if (link->destroy) 9739 link->destroy(link); 9740 if (link->pin_path) 9741 free(link->pin_path); 9742 free(link); 9743 9744 return err; 9745 } 9746 9747 int bpf_link__fd(const struct bpf_link *link) 9748 { 9749 return link->fd; 9750 } 9751 9752 const char *bpf_link__pin_path(const struct bpf_link *link) 9753 { 9754 return link->pin_path; 9755 } 9756 9757 static int bpf_link__detach_fd(struct bpf_link *link) 9758 { 9759 return close(link->fd); 9760 } 9761 9762 struct bpf_link *bpf_link__open(const char *path) 9763 { 9764 struct bpf_link *link; 9765 int fd; 9766 9767 fd = bpf_obj_get(path); 9768 if (fd < 0) { 9769 fd = -errno; 9770 pr_warn("failed to open link at %s: %d\n", path, fd); 9771 return ERR_PTR(fd); 9772 } 9773 9774 link = calloc(1, sizeof(*link)); 9775 if (!link) { 9776 close(fd); 9777 return ERR_PTR(-ENOMEM); 9778 } 9779 link->detach = &bpf_link__detach_fd; 9780 link->fd = fd; 9781 9782 link->pin_path = strdup(path); 9783 if (!link->pin_path) { 9784 bpf_link__destroy(link); 9785 return ERR_PTR(-ENOMEM); 9786 } 9787 9788 return link; 9789 } 9790 9791 int bpf_link__detach(struct bpf_link *link) 9792 { 9793 return bpf_link_detach(link->fd) ? -errno : 0; 9794 } 9795 9796 int bpf_link__pin(struct bpf_link *link, const char *path) 9797 { 9798 int err; 9799 9800 if (link->pin_path) 9801 return -EBUSY; 9802 err = make_parent_dir(path); 9803 if (err) 9804 return err; 9805 err = check_path(path); 9806 if (err) 9807 return err; 9808 9809 link->pin_path = strdup(path); 9810 if (!link->pin_path) 9811 return -ENOMEM; 9812 9813 if (bpf_obj_pin(link->fd, link->pin_path)) { 9814 err = -errno; 9815 zfree(&link->pin_path); 9816 return err; 9817 } 9818 9819 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 9820 return 0; 9821 } 9822 9823 int bpf_link__unpin(struct bpf_link *link) 9824 { 9825 int err; 9826 9827 if (!link->pin_path) 9828 return -EINVAL; 9829 9830 err = unlink(link->pin_path); 9831 if (err != 0) 9832 return -errno; 9833 9834 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 9835 zfree(&link->pin_path); 9836 return 0; 9837 } 9838 9839 static int bpf_link__detach_perf_event(struct bpf_link *link) 9840 { 9841 int err; 9842 9843 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0); 9844 if (err) 9845 err = -errno; 9846 9847 close(link->fd); 9848 return err; 9849 } 9850 9851 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, 9852 int pfd) 9853 { 9854 char errmsg[STRERR_BUFSIZE]; 9855 struct bpf_link *link; 9856 int prog_fd, err; 9857 9858 if (pfd < 0) { 9859 pr_warn("prog '%s': invalid perf event FD %d\n", 9860 prog->name, pfd); 9861 return ERR_PTR(-EINVAL); 9862 } 9863 prog_fd = bpf_program__fd(prog); 9864 if (prog_fd < 0) { 9865 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 9866 prog->name); 9867 return ERR_PTR(-EINVAL); 9868 } 9869 9870 link = calloc(1, sizeof(*link)); 9871 if (!link) 9872 return ERR_PTR(-ENOMEM); 9873 link->detach = &bpf_link__detach_perf_event; 9874 link->fd = pfd; 9875 9876 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 9877 err = -errno; 9878 free(link); 9879 pr_warn("prog '%s': failed to attach to pfd %d: %s\n", 9880 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9881 if (err == -EPROTO) 9882 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 9883 prog->name, pfd); 9884 return ERR_PTR(err); 9885 } 9886 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 9887 err = -errno; 9888 free(link); 9889 pr_warn("prog '%s': failed to enable pfd %d: %s\n", 9890 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9891 return ERR_PTR(err); 9892 } 9893 return link; 9894 } 9895 9896 /* 9897 * this function is expected to parse integer in the range of [0, 2^31-1] from 9898 * given file using scanf format string fmt. If actual parsed value is 9899 * negative, the result might be indistinguishable from error 9900 */ 9901 static int parse_uint_from_file(const char *file, const char *fmt) 9902 { 9903 char buf[STRERR_BUFSIZE]; 9904 int err, ret; 9905 FILE *f; 9906 9907 f = fopen(file, "r"); 9908 if (!f) { 9909 err = -errno; 9910 pr_debug("failed to open '%s': %s\n", file, 9911 libbpf_strerror_r(err, buf, sizeof(buf))); 9912 return err; 9913 } 9914 err = fscanf(f, fmt, &ret); 9915 if (err != 1) { 9916 err = err == EOF ? -EIO : -errno; 9917 pr_debug("failed to parse '%s': %s\n", file, 9918 libbpf_strerror_r(err, buf, sizeof(buf))); 9919 fclose(f); 9920 return err; 9921 } 9922 fclose(f); 9923 return ret; 9924 } 9925 9926 static int determine_kprobe_perf_type(void) 9927 { 9928 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 9929 9930 return parse_uint_from_file(file, "%d\n"); 9931 } 9932 9933 static int determine_uprobe_perf_type(void) 9934 { 9935 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 9936 9937 return parse_uint_from_file(file, "%d\n"); 9938 } 9939 9940 static int determine_kprobe_retprobe_bit(void) 9941 { 9942 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 9943 9944 return parse_uint_from_file(file, "config:%d\n"); 9945 } 9946 9947 static int determine_uprobe_retprobe_bit(void) 9948 { 9949 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 9950 9951 return parse_uint_from_file(file, "config:%d\n"); 9952 } 9953 9954 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 9955 uint64_t offset, int pid) 9956 { 9957 struct perf_event_attr attr = {}; 9958 char errmsg[STRERR_BUFSIZE]; 9959 int type, pfd, err; 9960 9961 type = uprobe ? determine_uprobe_perf_type() 9962 : determine_kprobe_perf_type(); 9963 if (type < 0) { 9964 pr_warn("failed to determine %s perf type: %s\n", 9965 uprobe ? "uprobe" : "kprobe", 9966 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 9967 return type; 9968 } 9969 if (retprobe) { 9970 int bit = uprobe ? determine_uprobe_retprobe_bit() 9971 : determine_kprobe_retprobe_bit(); 9972 9973 if (bit < 0) { 9974 pr_warn("failed to determine %s retprobe bit: %s\n", 9975 uprobe ? "uprobe" : "kprobe", 9976 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 9977 return bit; 9978 } 9979 attr.config |= 1 << bit; 9980 } 9981 attr.size = sizeof(attr); 9982 attr.type = type; 9983 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 9984 attr.config2 = offset; /* kprobe_addr or probe_offset */ 9985 9986 /* pid filter is meaningful only for uprobes */ 9987 pfd = syscall(__NR_perf_event_open, &attr, 9988 pid < 0 ? -1 : pid /* pid */, 9989 pid == -1 ? 0 : -1 /* cpu */, 9990 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9991 if (pfd < 0) { 9992 err = -errno; 9993 pr_warn("%s perf_event_open() failed: %s\n", 9994 uprobe ? "uprobe" : "kprobe", 9995 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9996 return err; 9997 } 9998 return pfd; 9999 } 10000 10001 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 10002 bool retprobe, 10003 const char *func_name) 10004 { 10005 char errmsg[STRERR_BUFSIZE]; 10006 struct bpf_link *link; 10007 int pfd, err; 10008 10009 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 10010 0 /* offset */, -1 /* pid */); 10011 if (pfd < 0) { 10012 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n", 10013 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 10014 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 10015 return ERR_PTR(pfd); 10016 } 10017 link = bpf_program__attach_perf_event(prog, pfd); 10018 if (IS_ERR(link)) { 10019 close(pfd); 10020 err = PTR_ERR(link); 10021 pr_warn("prog '%s': failed to attach to %s '%s': %s\n", 10022 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 10023 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10024 return link; 10025 } 10026 return link; 10027 } 10028 10029 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 10030 struct bpf_program *prog) 10031 { 10032 const char *func_name; 10033 bool retprobe; 10034 10035 func_name = prog->sec_name + sec->len; 10036 retprobe = strcmp(sec->sec, "kretprobe/") == 0; 10037 10038 return bpf_program__attach_kprobe(prog, retprobe, func_name); 10039 } 10040 10041 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 10042 bool retprobe, pid_t pid, 10043 const char *binary_path, 10044 size_t func_offset) 10045 { 10046 char errmsg[STRERR_BUFSIZE]; 10047 struct bpf_link *link; 10048 int pfd, err; 10049 10050 pfd = perf_event_open_probe(true /* uprobe */, retprobe, 10051 binary_path, func_offset, pid); 10052 if (pfd < 0) { 10053 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 10054 prog->name, retprobe ? "uretprobe" : "uprobe", 10055 binary_path, func_offset, 10056 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 10057 return ERR_PTR(pfd); 10058 } 10059 link = bpf_program__attach_perf_event(prog, pfd); 10060 if (IS_ERR(link)) { 10061 close(pfd); 10062 err = PTR_ERR(link); 10063 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 10064 prog->name, retprobe ? "uretprobe" : "uprobe", 10065 binary_path, func_offset, 10066 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10067 return link; 10068 } 10069 return link; 10070 } 10071 10072 static int determine_tracepoint_id(const char *tp_category, 10073 const char *tp_name) 10074 { 10075 char file[PATH_MAX]; 10076 int ret; 10077 10078 ret = snprintf(file, sizeof(file), 10079 "/sys/kernel/debug/tracing/events/%s/%s/id", 10080 tp_category, tp_name); 10081 if (ret < 0) 10082 return -errno; 10083 if (ret >= sizeof(file)) { 10084 pr_debug("tracepoint %s/%s path is too long\n", 10085 tp_category, tp_name); 10086 return -E2BIG; 10087 } 10088 return parse_uint_from_file(file, "%d\n"); 10089 } 10090 10091 static int perf_event_open_tracepoint(const char *tp_category, 10092 const char *tp_name) 10093 { 10094 struct perf_event_attr attr = {}; 10095 char errmsg[STRERR_BUFSIZE]; 10096 int tp_id, pfd, err; 10097 10098 tp_id = determine_tracepoint_id(tp_category, tp_name); 10099 if (tp_id < 0) { 10100 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 10101 tp_category, tp_name, 10102 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 10103 return tp_id; 10104 } 10105 10106 attr.type = PERF_TYPE_TRACEPOINT; 10107 attr.size = sizeof(attr); 10108 attr.config = tp_id; 10109 10110 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 10111 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10112 if (pfd < 0) { 10113 err = -errno; 10114 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 10115 tp_category, tp_name, 10116 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10117 return err; 10118 } 10119 return pfd; 10120 } 10121 10122 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 10123 const char *tp_category, 10124 const char *tp_name) 10125 { 10126 char errmsg[STRERR_BUFSIZE]; 10127 struct bpf_link *link; 10128 int pfd, err; 10129 10130 pfd = perf_event_open_tracepoint(tp_category, tp_name); 10131 if (pfd < 0) { 10132 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 10133 prog->name, tp_category, tp_name, 10134 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 10135 return ERR_PTR(pfd); 10136 } 10137 link = bpf_program__attach_perf_event(prog, pfd); 10138 if (IS_ERR(link)) { 10139 close(pfd); 10140 err = PTR_ERR(link); 10141 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 10142 prog->name, tp_category, tp_name, 10143 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10144 return link; 10145 } 10146 return link; 10147 } 10148 10149 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 10150 struct bpf_program *prog) 10151 { 10152 char *sec_name, *tp_cat, *tp_name; 10153 struct bpf_link *link; 10154 10155 sec_name = strdup(prog->sec_name); 10156 if (!sec_name) 10157 return ERR_PTR(-ENOMEM); 10158 10159 /* extract "tp/<category>/<name>" */ 10160 tp_cat = sec_name + sec->len; 10161 tp_name = strchr(tp_cat, '/'); 10162 if (!tp_name) { 10163 link = ERR_PTR(-EINVAL); 10164 goto out; 10165 } 10166 *tp_name = '\0'; 10167 tp_name++; 10168 10169 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 10170 out: 10171 free(sec_name); 10172 return link; 10173 } 10174 10175 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 10176 const char *tp_name) 10177 { 10178 char errmsg[STRERR_BUFSIZE]; 10179 struct bpf_link *link; 10180 int prog_fd, pfd; 10181 10182 prog_fd = bpf_program__fd(prog); 10183 if (prog_fd < 0) { 10184 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 10185 return ERR_PTR(-EINVAL); 10186 } 10187 10188 link = calloc(1, sizeof(*link)); 10189 if (!link) 10190 return ERR_PTR(-ENOMEM); 10191 link->detach = &bpf_link__detach_fd; 10192 10193 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 10194 if (pfd < 0) { 10195 pfd = -errno; 10196 free(link); 10197 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 10198 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 10199 return ERR_PTR(pfd); 10200 } 10201 link->fd = pfd; 10202 return link; 10203 } 10204 10205 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 10206 struct bpf_program *prog) 10207 { 10208 const char *tp_name = prog->sec_name + sec->len; 10209 10210 return bpf_program__attach_raw_tracepoint(prog, tp_name); 10211 } 10212 10213 /* Common logic for all BPF program types that attach to a btf_id */ 10214 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog) 10215 { 10216 char errmsg[STRERR_BUFSIZE]; 10217 struct bpf_link *link; 10218 int prog_fd, pfd; 10219 10220 prog_fd = bpf_program__fd(prog); 10221 if (prog_fd < 0) { 10222 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 10223 return ERR_PTR(-EINVAL); 10224 } 10225 10226 link = calloc(1, sizeof(*link)); 10227 if (!link) 10228 return ERR_PTR(-ENOMEM); 10229 link->detach = &bpf_link__detach_fd; 10230 10231 pfd = bpf_raw_tracepoint_open(NULL, prog_fd); 10232 if (pfd < 0) { 10233 pfd = -errno; 10234 free(link); 10235 pr_warn("prog '%s': failed to attach: %s\n", 10236 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 10237 return ERR_PTR(pfd); 10238 } 10239 link->fd = pfd; 10240 return (struct bpf_link *)link; 10241 } 10242 10243 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog) 10244 { 10245 return bpf_program__attach_btf_id(prog); 10246 } 10247 10248 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog) 10249 { 10250 return bpf_program__attach_btf_id(prog); 10251 } 10252 10253 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 10254 struct bpf_program *prog) 10255 { 10256 return bpf_program__attach_trace(prog); 10257 } 10258 10259 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 10260 struct bpf_program *prog) 10261 { 10262 return bpf_program__attach_lsm(prog); 10263 } 10264 10265 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 10266 struct bpf_program *prog) 10267 { 10268 return bpf_program__attach_iter(prog, NULL); 10269 } 10270 10271 static struct bpf_link * 10272 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id, 10273 const char *target_name) 10274 { 10275 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, 10276 .target_btf_id = btf_id); 10277 enum bpf_attach_type attach_type; 10278 char errmsg[STRERR_BUFSIZE]; 10279 struct bpf_link *link; 10280 int prog_fd, link_fd; 10281 10282 prog_fd = bpf_program__fd(prog); 10283 if (prog_fd < 0) { 10284 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 10285 return ERR_PTR(-EINVAL); 10286 } 10287 10288 link = calloc(1, sizeof(*link)); 10289 if (!link) 10290 return ERR_PTR(-ENOMEM); 10291 link->detach = &bpf_link__detach_fd; 10292 10293 attach_type = bpf_program__get_expected_attach_type(prog); 10294 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); 10295 if (link_fd < 0) { 10296 link_fd = -errno; 10297 free(link); 10298 pr_warn("prog '%s': failed to attach to %s: %s\n", 10299 prog->name, target_name, 10300 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 10301 return ERR_PTR(link_fd); 10302 } 10303 link->fd = link_fd; 10304 return link; 10305 } 10306 10307 struct bpf_link * 10308 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd) 10309 { 10310 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup"); 10311 } 10312 10313 struct bpf_link * 10314 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd) 10315 { 10316 return bpf_program__attach_fd(prog, netns_fd, 0, "netns"); 10317 } 10318 10319 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex) 10320 { 10321 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 10322 return bpf_program__attach_fd(prog, ifindex, 0, "xdp"); 10323 } 10324 10325 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog, 10326 int target_fd, 10327 const char *attach_func_name) 10328 { 10329 int btf_id; 10330 10331 if (!!target_fd != !!attach_func_name) { 10332 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 10333 prog->name); 10334 return ERR_PTR(-EINVAL); 10335 } 10336 10337 if (prog->type != BPF_PROG_TYPE_EXT) { 10338 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 10339 prog->name); 10340 return ERR_PTR(-EINVAL); 10341 } 10342 10343 if (target_fd) { 10344 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 10345 if (btf_id < 0) 10346 return ERR_PTR(btf_id); 10347 10348 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace"); 10349 } else { 10350 /* no target, so use raw_tracepoint_open for compatibility 10351 * with old kernels 10352 */ 10353 return bpf_program__attach_trace(prog); 10354 } 10355 } 10356 10357 struct bpf_link * 10358 bpf_program__attach_iter(struct bpf_program *prog, 10359 const struct bpf_iter_attach_opts *opts) 10360 { 10361 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 10362 char errmsg[STRERR_BUFSIZE]; 10363 struct bpf_link *link; 10364 int prog_fd, link_fd; 10365 __u32 target_fd = 0; 10366 10367 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 10368 return ERR_PTR(-EINVAL); 10369 10370 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 10371 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 10372 10373 prog_fd = bpf_program__fd(prog); 10374 if (prog_fd < 0) { 10375 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 10376 return ERR_PTR(-EINVAL); 10377 } 10378 10379 link = calloc(1, sizeof(*link)); 10380 if (!link) 10381 return ERR_PTR(-ENOMEM); 10382 link->detach = &bpf_link__detach_fd; 10383 10384 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 10385 &link_create_opts); 10386 if (link_fd < 0) { 10387 link_fd = -errno; 10388 free(link); 10389 pr_warn("prog '%s': failed to attach to iterator: %s\n", 10390 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 10391 return ERR_PTR(link_fd); 10392 } 10393 link->fd = link_fd; 10394 return link; 10395 } 10396 10397 struct bpf_link *bpf_program__attach(struct bpf_program *prog) 10398 { 10399 const struct bpf_sec_def *sec_def; 10400 10401 sec_def = find_sec_def(prog->sec_name); 10402 if (!sec_def || !sec_def->attach_fn) 10403 return ERR_PTR(-ESRCH); 10404 10405 return sec_def->attach_fn(sec_def, prog); 10406 } 10407 10408 static int bpf_link__detach_struct_ops(struct bpf_link *link) 10409 { 10410 __u32 zero = 0; 10411 10412 if (bpf_map_delete_elem(link->fd, &zero)) 10413 return -errno; 10414 10415 return 0; 10416 } 10417 10418 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map) 10419 { 10420 struct bpf_struct_ops *st_ops; 10421 struct bpf_link *link; 10422 __u32 i, zero = 0; 10423 int err; 10424 10425 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 10426 return ERR_PTR(-EINVAL); 10427 10428 link = calloc(1, sizeof(*link)); 10429 if (!link) 10430 return ERR_PTR(-EINVAL); 10431 10432 st_ops = map->st_ops; 10433 for (i = 0; i < btf_vlen(st_ops->type); i++) { 10434 struct bpf_program *prog = st_ops->progs[i]; 10435 void *kern_data; 10436 int prog_fd; 10437 10438 if (!prog) 10439 continue; 10440 10441 prog_fd = bpf_program__fd(prog); 10442 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 10443 *(unsigned long *)kern_data = prog_fd; 10444 } 10445 10446 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); 10447 if (err) { 10448 err = -errno; 10449 free(link); 10450 return ERR_PTR(err); 10451 } 10452 10453 link->detach = bpf_link__detach_struct_ops; 10454 link->fd = map->fd; 10455 10456 return link; 10457 } 10458 10459 enum bpf_perf_event_ret 10460 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 10461 void **copy_mem, size_t *copy_size, 10462 bpf_perf_event_print_t fn, void *private_data) 10463 { 10464 struct perf_event_mmap_page *header = mmap_mem; 10465 __u64 data_head = ring_buffer_read_head(header); 10466 __u64 data_tail = header->data_tail; 10467 void *base = ((__u8 *)header) + page_size; 10468 int ret = LIBBPF_PERF_EVENT_CONT; 10469 struct perf_event_header *ehdr; 10470 size_t ehdr_size; 10471 10472 while (data_head != data_tail) { 10473 ehdr = base + (data_tail & (mmap_size - 1)); 10474 ehdr_size = ehdr->size; 10475 10476 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 10477 void *copy_start = ehdr; 10478 size_t len_first = base + mmap_size - copy_start; 10479 size_t len_secnd = ehdr_size - len_first; 10480 10481 if (*copy_size < ehdr_size) { 10482 free(*copy_mem); 10483 *copy_mem = malloc(ehdr_size); 10484 if (!*copy_mem) { 10485 *copy_size = 0; 10486 ret = LIBBPF_PERF_EVENT_ERROR; 10487 break; 10488 } 10489 *copy_size = ehdr_size; 10490 } 10491 10492 memcpy(*copy_mem, copy_start, len_first); 10493 memcpy(*copy_mem + len_first, base, len_secnd); 10494 ehdr = *copy_mem; 10495 } 10496 10497 ret = fn(ehdr, private_data); 10498 data_tail += ehdr_size; 10499 if (ret != LIBBPF_PERF_EVENT_CONT) 10500 break; 10501 } 10502 10503 ring_buffer_write_tail(header, data_tail); 10504 return ret; 10505 } 10506 10507 struct perf_buffer; 10508 10509 struct perf_buffer_params { 10510 struct perf_event_attr *attr; 10511 /* if event_cb is specified, it takes precendence */ 10512 perf_buffer_event_fn event_cb; 10513 /* sample_cb and lost_cb are higher-level common-case callbacks */ 10514 perf_buffer_sample_fn sample_cb; 10515 perf_buffer_lost_fn lost_cb; 10516 void *ctx; 10517 int cpu_cnt; 10518 int *cpus; 10519 int *map_keys; 10520 }; 10521 10522 struct perf_cpu_buf { 10523 struct perf_buffer *pb; 10524 void *base; /* mmap()'ed memory */ 10525 void *buf; /* for reconstructing segmented data */ 10526 size_t buf_size; 10527 int fd; 10528 int cpu; 10529 int map_key; 10530 }; 10531 10532 struct perf_buffer { 10533 perf_buffer_event_fn event_cb; 10534 perf_buffer_sample_fn sample_cb; 10535 perf_buffer_lost_fn lost_cb; 10536 void *ctx; /* passed into callbacks */ 10537 10538 size_t page_size; 10539 size_t mmap_size; 10540 struct perf_cpu_buf **cpu_bufs; 10541 struct epoll_event *events; 10542 int cpu_cnt; /* number of allocated CPU buffers */ 10543 int epoll_fd; /* perf event FD */ 10544 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 10545 }; 10546 10547 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 10548 struct perf_cpu_buf *cpu_buf) 10549 { 10550 if (!cpu_buf) 10551 return; 10552 if (cpu_buf->base && 10553 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 10554 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 10555 if (cpu_buf->fd >= 0) { 10556 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 10557 close(cpu_buf->fd); 10558 } 10559 free(cpu_buf->buf); 10560 free(cpu_buf); 10561 } 10562 10563 void perf_buffer__free(struct perf_buffer *pb) 10564 { 10565 int i; 10566 10567 if (IS_ERR_OR_NULL(pb)) 10568 return; 10569 if (pb->cpu_bufs) { 10570 for (i = 0; i < pb->cpu_cnt; i++) { 10571 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 10572 10573 if (!cpu_buf) 10574 continue; 10575 10576 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 10577 perf_buffer__free_cpu_buf(pb, cpu_buf); 10578 } 10579 free(pb->cpu_bufs); 10580 } 10581 if (pb->epoll_fd >= 0) 10582 close(pb->epoll_fd); 10583 free(pb->events); 10584 free(pb); 10585 } 10586 10587 static struct perf_cpu_buf * 10588 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 10589 int cpu, int map_key) 10590 { 10591 struct perf_cpu_buf *cpu_buf; 10592 char msg[STRERR_BUFSIZE]; 10593 int err; 10594 10595 cpu_buf = calloc(1, sizeof(*cpu_buf)); 10596 if (!cpu_buf) 10597 return ERR_PTR(-ENOMEM); 10598 10599 cpu_buf->pb = pb; 10600 cpu_buf->cpu = cpu; 10601 cpu_buf->map_key = map_key; 10602 10603 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 10604 -1, PERF_FLAG_FD_CLOEXEC); 10605 if (cpu_buf->fd < 0) { 10606 err = -errno; 10607 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 10608 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 10609 goto error; 10610 } 10611 10612 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 10613 PROT_READ | PROT_WRITE, MAP_SHARED, 10614 cpu_buf->fd, 0); 10615 if (cpu_buf->base == MAP_FAILED) { 10616 cpu_buf->base = NULL; 10617 err = -errno; 10618 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 10619 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 10620 goto error; 10621 } 10622 10623 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 10624 err = -errno; 10625 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 10626 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 10627 goto error; 10628 } 10629 10630 return cpu_buf; 10631 10632 error: 10633 perf_buffer__free_cpu_buf(pb, cpu_buf); 10634 return (struct perf_cpu_buf *)ERR_PTR(err); 10635 } 10636 10637 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 10638 struct perf_buffer_params *p); 10639 10640 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 10641 const struct perf_buffer_opts *opts) 10642 { 10643 struct perf_buffer_params p = {}; 10644 struct perf_event_attr attr = { 0, }; 10645 10646 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 10647 attr.type = PERF_TYPE_SOFTWARE; 10648 attr.sample_type = PERF_SAMPLE_RAW; 10649 attr.sample_period = 1; 10650 attr.wakeup_events = 1; 10651 10652 p.attr = &attr; 10653 p.sample_cb = opts ? opts->sample_cb : NULL; 10654 p.lost_cb = opts ? opts->lost_cb : NULL; 10655 p.ctx = opts ? opts->ctx : NULL; 10656 10657 return __perf_buffer__new(map_fd, page_cnt, &p); 10658 } 10659 10660 struct perf_buffer * 10661 perf_buffer__new_raw(int map_fd, size_t page_cnt, 10662 const struct perf_buffer_raw_opts *opts) 10663 { 10664 struct perf_buffer_params p = {}; 10665 10666 p.attr = opts->attr; 10667 p.event_cb = opts->event_cb; 10668 p.ctx = opts->ctx; 10669 p.cpu_cnt = opts->cpu_cnt; 10670 p.cpus = opts->cpus; 10671 p.map_keys = opts->map_keys; 10672 10673 return __perf_buffer__new(map_fd, page_cnt, &p); 10674 } 10675 10676 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 10677 struct perf_buffer_params *p) 10678 { 10679 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 10680 struct bpf_map_info map; 10681 char msg[STRERR_BUFSIZE]; 10682 struct perf_buffer *pb; 10683 bool *online = NULL; 10684 __u32 map_info_len; 10685 int err, i, j, n; 10686 10687 if (page_cnt & (page_cnt - 1)) { 10688 pr_warn("page count should be power of two, but is %zu\n", 10689 page_cnt); 10690 return ERR_PTR(-EINVAL); 10691 } 10692 10693 /* best-effort sanity checks */ 10694 memset(&map, 0, sizeof(map)); 10695 map_info_len = sizeof(map); 10696 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 10697 if (err) { 10698 err = -errno; 10699 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 10700 * -EBADFD, -EFAULT, or -E2BIG on real error 10701 */ 10702 if (err != -EINVAL) { 10703 pr_warn("failed to get map info for map FD %d: %s\n", 10704 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 10705 return ERR_PTR(err); 10706 } 10707 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 10708 map_fd); 10709 } else { 10710 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 10711 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 10712 map.name); 10713 return ERR_PTR(-EINVAL); 10714 } 10715 } 10716 10717 pb = calloc(1, sizeof(*pb)); 10718 if (!pb) 10719 return ERR_PTR(-ENOMEM); 10720 10721 pb->event_cb = p->event_cb; 10722 pb->sample_cb = p->sample_cb; 10723 pb->lost_cb = p->lost_cb; 10724 pb->ctx = p->ctx; 10725 10726 pb->page_size = getpagesize(); 10727 pb->mmap_size = pb->page_size * page_cnt; 10728 pb->map_fd = map_fd; 10729 10730 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 10731 if (pb->epoll_fd < 0) { 10732 err = -errno; 10733 pr_warn("failed to create epoll instance: %s\n", 10734 libbpf_strerror_r(err, msg, sizeof(msg))); 10735 goto error; 10736 } 10737 10738 if (p->cpu_cnt > 0) { 10739 pb->cpu_cnt = p->cpu_cnt; 10740 } else { 10741 pb->cpu_cnt = libbpf_num_possible_cpus(); 10742 if (pb->cpu_cnt < 0) { 10743 err = pb->cpu_cnt; 10744 goto error; 10745 } 10746 if (map.max_entries && map.max_entries < pb->cpu_cnt) 10747 pb->cpu_cnt = map.max_entries; 10748 } 10749 10750 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 10751 if (!pb->events) { 10752 err = -ENOMEM; 10753 pr_warn("failed to allocate events: out of memory\n"); 10754 goto error; 10755 } 10756 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 10757 if (!pb->cpu_bufs) { 10758 err = -ENOMEM; 10759 pr_warn("failed to allocate buffers: out of memory\n"); 10760 goto error; 10761 } 10762 10763 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 10764 if (err) { 10765 pr_warn("failed to get online CPU mask: %d\n", err); 10766 goto error; 10767 } 10768 10769 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 10770 struct perf_cpu_buf *cpu_buf; 10771 int cpu, map_key; 10772 10773 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 10774 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 10775 10776 /* in case user didn't explicitly requested particular CPUs to 10777 * be attached to, skip offline/not present CPUs 10778 */ 10779 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 10780 continue; 10781 10782 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 10783 if (IS_ERR(cpu_buf)) { 10784 err = PTR_ERR(cpu_buf); 10785 goto error; 10786 } 10787 10788 pb->cpu_bufs[j] = cpu_buf; 10789 10790 err = bpf_map_update_elem(pb->map_fd, &map_key, 10791 &cpu_buf->fd, 0); 10792 if (err) { 10793 err = -errno; 10794 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 10795 cpu, map_key, cpu_buf->fd, 10796 libbpf_strerror_r(err, msg, sizeof(msg))); 10797 goto error; 10798 } 10799 10800 pb->events[j].events = EPOLLIN; 10801 pb->events[j].data.ptr = cpu_buf; 10802 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 10803 &pb->events[j]) < 0) { 10804 err = -errno; 10805 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 10806 cpu, cpu_buf->fd, 10807 libbpf_strerror_r(err, msg, sizeof(msg))); 10808 goto error; 10809 } 10810 j++; 10811 } 10812 pb->cpu_cnt = j; 10813 free(online); 10814 10815 return pb; 10816 10817 error: 10818 free(online); 10819 if (pb) 10820 perf_buffer__free(pb); 10821 return ERR_PTR(err); 10822 } 10823 10824 struct perf_sample_raw { 10825 struct perf_event_header header; 10826 uint32_t size; 10827 char data[]; 10828 }; 10829 10830 struct perf_sample_lost { 10831 struct perf_event_header header; 10832 uint64_t id; 10833 uint64_t lost; 10834 uint64_t sample_id; 10835 }; 10836 10837 static enum bpf_perf_event_ret 10838 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 10839 { 10840 struct perf_cpu_buf *cpu_buf = ctx; 10841 struct perf_buffer *pb = cpu_buf->pb; 10842 void *data = e; 10843 10844 /* user wants full control over parsing perf event */ 10845 if (pb->event_cb) 10846 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 10847 10848 switch (e->type) { 10849 case PERF_RECORD_SAMPLE: { 10850 struct perf_sample_raw *s = data; 10851 10852 if (pb->sample_cb) 10853 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 10854 break; 10855 } 10856 case PERF_RECORD_LOST: { 10857 struct perf_sample_lost *s = data; 10858 10859 if (pb->lost_cb) 10860 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 10861 break; 10862 } 10863 default: 10864 pr_warn("unknown perf sample type %d\n", e->type); 10865 return LIBBPF_PERF_EVENT_ERROR; 10866 } 10867 return LIBBPF_PERF_EVENT_CONT; 10868 } 10869 10870 static int perf_buffer__process_records(struct perf_buffer *pb, 10871 struct perf_cpu_buf *cpu_buf) 10872 { 10873 enum bpf_perf_event_ret ret; 10874 10875 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 10876 pb->page_size, &cpu_buf->buf, 10877 &cpu_buf->buf_size, 10878 perf_buffer__process_record, cpu_buf); 10879 if (ret != LIBBPF_PERF_EVENT_CONT) 10880 return ret; 10881 return 0; 10882 } 10883 10884 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 10885 { 10886 return pb->epoll_fd; 10887 } 10888 10889 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 10890 { 10891 int i, cnt, err; 10892 10893 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 10894 for (i = 0; i < cnt; i++) { 10895 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 10896 10897 err = perf_buffer__process_records(pb, cpu_buf); 10898 if (err) { 10899 pr_warn("error while processing records: %d\n", err); 10900 return err; 10901 } 10902 } 10903 return cnt < 0 ? -errno : cnt; 10904 } 10905 10906 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 10907 * manager. 10908 */ 10909 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 10910 { 10911 return pb->cpu_cnt; 10912 } 10913 10914 /* 10915 * Return perf_event FD of a ring buffer in *buf_idx* slot of 10916 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 10917 * select()/poll()/epoll() Linux syscalls. 10918 */ 10919 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 10920 { 10921 struct perf_cpu_buf *cpu_buf; 10922 10923 if (buf_idx >= pb->cpu_cnt) 10924 return -EINVAL; 10925 10926 cpu_buf = pb->cpu_bufs[buf_idx]; 10927 if (!cpu_buf) 10928 return -ENOENT; 10929 10930 return cpu_buf->fd; 10931 } 10932 10933 /* 10934 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 10935 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 10936 * consume, do nothing and return success. 10937 * Returns: 10938 * - 0 on success; 10939 * - <0 on failure. 10940 */ 10941 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 10942 { 10943 struct perf_cpu_buf *cpu_buf; 10944 10945 if (buf_idx >= pb->cpu_cnt) 10946 return -EINVAL; 10947 10948 cpu_buf = pb->cpu_bufs[buf_idx]; 10949 if (!cpu_buf) 10950 return -ENOENT; 10951 10952 return perf_buffer__process_records(pb, cpu_buf); 10953 } 10954 10955 int perf_buffer__consume(struct perf_buffer *pb) 10956 { 10957 int i, err; 10958 10959 for (i = 0; i < pb->cpu_cnt; i++) { 10960 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 10961 10962 if (!cpu_buf) 10963 continue; 10964 10965 err = perf_buffer__process_records(pb, cpu_buf); 10966 if (err) { 10967 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 10968 return err; 10969 } 10970 } 10971 return 0; 10972 } 10973 10974 struct bpf_prog_info_array_desc { 10975 int array_offset; /* e.g. offset of jited_prog_insns */ 10976 int count_offset; /* e.g. offset of jited_prog_len */ 10977 int size_offset; /* > 0: offset of rec size, 10978 * < 0: fix size of -size_offset 10979 */ 10980 }; 10981 10982 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 10983 [BPF_PROG_INFO_JITED_INSNS] = { 10984 offsetof(struct bpf_prog_info, jited_prog_insns), 10985 offsetof(struct bpf_prog_info, jited_prog_len), 10986 -1, 10987 }, 10988 [BPF_PROG_INFO_XLATED_INSNS] = { 10989 offsetof(struct bpf_prog_info, xlated_prog_insns), 10990 offsetof(struct bpf_prog_info, xlated_prog_len), 10991 -1, 10992 }, 10993 [BPF_PROG_INFO_MAP_IDS] = { 10994 offsetof(struct bpf_prog_info, map_ids), 10995 offsetof(struct bpf_prog_info, nr_map_ids), 10996 -(int)sizeof(__u32), 10997 }, 10998 [BPF_PROG_INFO_JITED_KSYMS] = { 10999 offsetof(struct bpf_prog_info, jited_ksyms), 11000 offsetof(struct bpf_prog_info, nr_jited_ksyms), 11001 -(int)sizeof(__u64), 11002 }, 11003 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 11004 offsetof(struct bpf_prog_info, jited_func_lens), 11005 offsetof(struct bpf_prog_info, nr_jited_func_lens), 11006 -(int)sizeof(__u32), 11007 }, 11008 [BPF_PROG_INFO_FUNC_INFO] = { 11009 offsetof(struct bpf_prog_info, func_info), 11010 offsetof(struct bpf_prog_info, nr_func_info), 11011 offsetof(struct bpf_prog_info, func_info_rec_size), 11012 }, 11013 [BPF_PROG_INFO_LINE_INFO] = { 11014 offsetof(struct bpf_prog_info, line_info), 11015 offsetof(struct bpf_prog_info, nr_line_info), 11016 offsetof(struct bpf_prog_info, line_info_rec_size), 11017 }, 11018 [BPF_PROG_INFO_JITED_LINE_INFO] = { 11019 offsetof(struct bpf_prog_info, jited_line_info), 11020 offsetof(struct bpf_prog_info, nr_jited_line_info), 11021 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 11022 }, 11023 [BPF_PROG_INFO_PROG_TAGS] = { 11024 offsetof(struct bpf_prog_info, prog_tags), 11025 offsetof(struct bpf_prog_info, nr_prog_tags), 11026 -(int)sizeof(__u8) * BPF_TAG_SIZE, 11027 }, 11028 11029 }; 11030 11031 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 11032 int offset) 11033 { 11034 __u32 *array = (__u32 *)info; 11035 11036 if (offset >= 0) 11037 return array[offset / sizeof(__u32)]; 11038 return -(int)offset; 11039 } 11040 11041 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 11042 int offset) 11043 { 11044 __u64 *array = (__u64 *)info; 11045 11046 if (offset >= 0) 11047 return array[offset / sizeof(__u64)]; 11048 return -(int)offset; 11049 } 11050 11051 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 11052 __u32 val) 11053 { 11054 __u32 *array = (__u32 *)info; 11055 11056 if (offset >= 0) 11057 array[offset / sizeof(__u32)] = val; 11058 } 11059 11060 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 11061 __u64 val) 11062 { 11063 __u64 *array = (__u64 *)info; 11064 11065 if (offset >= 0) 11066 array[offset / sizeof(__u64)] = val; 11067 } 11068 11069 struct bpf_prog_info_linear * 11070 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 11071 { 11072 struct bpf_prog_info_linear *info_linear; 11073 struct bpf_prog_info info = {}; 11074 __u32 info_len = sizeof(info); 11075 __u32 data_len = 0; 11076 int i, err; 11077 void *ptr; 11078 11079 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 11080 return ERR_PTR(-EINVAL); 11081 11082 /* step 1: get array dimensions */ 11083 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 11084 if (err) { 11085 pr_debug("can't get prog info: %s", strerror(errno)); 11086 return ERR_PTR(-EFAULT); 11087 } 11088 11089 /* step 2: calculate total size of all arrays */ 11090 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 11091 bool include_array = (arrays & (1UL << i)) > 0; 11092 struct bpf_prog_info_array_desc *desc; 11093 __u32 count, size; 11094 11095 desc = bpf_prog_info_array_desc + i; 11096 11097 /* kernel is too old to support this field */ 11098 if (info_len < desc->array_offset + sizeof(__u32) || 11099 info_len < desc->count_offset + sizeof(__u32) || 11100 (desc->size_offset > 0 && info_len < desc->size_offset)) 11101 include_array = false; 11102 11103 if (!include_array) { 11104 arrays &= ~(1UL << i); /* clear the bit */ 11105 continue; 11106 } 11107 11108 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 11109 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 11110 11111 data_len += count * size; 11112 } 11113 11114 /* step 3: allocate continuous memory */ 11115 data_len = roundup(data_len, sizeof(__u64)); 11116 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 11117 if (!info_linear) 11118 return ERR_PTR(-ENOMEM); 11119 11120 /* step 4: fill data to info_linear->info */ 11121 info_linear->arrays = arrays; 11122 memset(&info_linear->info, 0, sizeof(info)); 11123 ptr = info_linear->data; 11124 11125 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 11126 struct bpf_prog_info_array_desc *desc; 11127 __u32 count, size; 11128 11129 if ((arrays & (1UL << i)) == 0) 11130 continue; 11131 11132 desc = bpf_prog_info_array_desc + i; 11133 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 11134 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 11135 bpf_prog_info_set_offset_u32(&info_linear->info, 11136 desc->count_offset, count); 11137 bpf_prog_info_set_offset_u32(&info_linear->info, 11138 desc->size_offset, size); 11139 bpf_prog_info_set_offset_u64(&info_linear->info, 11140 desc->array_offset, 11141 ptr_to_u64(ptr)); 11142 ptr += count * size; 11143 } 11144 11145 /* step 5: call syscall again to get required arrays */ 11146 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 11147 if (err) { 11148 pr_debug("can't get prog info: %s", strerror(errno)); 11149 free(info_linear); 11150 return ERR_PTR(-EFAULT); 11151 } 11152 11153 /* step 6: verify the data */ 11154 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 11155 struct bpf_prog_info_array_desc *desc; 11156 __u32 v1, v2; 11157 11158 if ((arrays & (1UL << i)) == 0) 11159 continue; 11160 11161 desc = bpf_prog_info_array_desc + i; 11162 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 11163 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 11164 desc->count_offset); 11165 if (v1 != v2) 11166 pr_warn("%s: mismatch in element count\n", __func__); 11167 11168 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 11169 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 11170 desc->size_offset); 11171 if (v1 != v2) 11172 pr_warn("%s: mismatch in rec size\n", __func__); 11173 } 11174 11175 /* step 7: update info_len and data_len */ 11176 info_linear->info_len = sizeof(struct bpf_prog_info); 11177 info_linear->data_len = data_len; 11178 11179 return info_linear; 11180 } 11181 11182 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 11183 { 11184 int i; 11185 11186 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 11187 struct bpf_prog_info_array_desc *desc; 11188 __u64 addr, offs; 11189 11190 if ((info_linear->arrays & (1UL << i)) == 0) 11191 continue; 11192 11193 desc = bpf_prog_info_array_desc + i; 11194 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 11195 desc->array_offset); 11196 offs = addr - ptr_to_u64(info_linear->data); 11197 bpf_prog_info_set_offset_u64(&info_linear->info, 11198 desc->array_offset, offs); 11199 } 11200 } 11201 11202 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 11203 { 11204 int i; 11205 11206 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 11207 struct bpf_prog_info_array_desc *desc; 11208 __u64 addr, offs; 11209 11210 if ((info_linear->arrays & (1UL << i)) == 0) 11211 continue; 11212 11213 desc = bpf_prog_info_array_desc + i; 11214 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 11215 desc->array_offset); 11216 addr = offs + ptr_to_u64(info_linear->data); 11217 bpf_prog_info_set_offset_u64(&info_linear->info, 11218 desc->array_offset, addr); 11219 } 11220 } 11221 11222 int bpf_program__set_attach_target(struct bpf_program *prog, 11223 int attach_prog_fd, 11224 const char *attach_func_name) 11225 { 11226 int btf_obj_fd = 0, btf_id = 0, err; 11227 11228 if (!prog || attach_prog_fd < 0 || !attach_func_name) 11229 return -EINVAL; 11230 11231 if (prog->obj->loaded) 11232 return -EINVAL; 11233 11234 if (attach_prog_fd) { 11235 btf_id = libbpf_find_prog_btf_id(attach_func_name, 11236 attach_prog_fd); 11237 if (btf_id < 0) 11238 return btf_id; 11239 } else { 11240 /* load btf_vmlinux, if not yet */ 11241 err = bpf_object__load_vmlinux_btf(prog->obj, true); 11242 if (err) 11243 return err; 11244 err = find_kernel_btf_id(prog->obj, attach_func_name, 11245 prog->expected_attach_type, 11246 &btf_obj_fd, &btf_id); 11247 if (err) 11248 return err; 11249 } 11250 11251 prog->attach_btf_id = btf_id; 11252 prog->attach_btf_obj_fd = btf_obj_fd; 11253 prog->attach_prog_fd = attach_prog_fd; 11254 return 0; 11255 } 11256 11257 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 11258 { 11259 int err = 0, n, len, start, end = -1; 11260 bool *tmp; 11261 11262 *mask = NULL; 11263 *mask_sz = 0; 11264 11265 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 11266 while (*s) { 11267 if (*s == ',' || *s == '\n') { 11268 s++; 11269 continue; 11270 } 11271 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 11272 if (n <= 0 || n > 2) { 11273 pr_warn("Failed to get CPU range %s: %d\n", s, n); 11274 err = -EINVAL; 11275 goto cleanup; 11276 } else if (n == 1) { 11277 end = start; 11278 } 11279 if (start < 0 || start > end) { 11280 pr_warn("Invalid CPU range [%d,%d] in %s\n", 11281 start, end, s); 11282 err = -EINVAL; 11283 goto cleanup; 11284 } 11285 tmp = realloc(*mask, end + 1); 11286 if (!tmp) { 11287 err = -ENOMEM; 11288 goto cleanup; 11289 } 11290 *mask = tmp; 11291 memset(tmp + *mask_sz, 0, start - *mask_sz); 11292 memset(tmp + start, 1, end - start + 1); 11293 *mask_sz = end + 1; 11294 s += len; 11295 } 11296 if (!*mask_sz) { 11297 pr_warn("Empty CPU range\n"); 11298 return -EINVAL; 11299 } 11300 return 0; 11301 cleanup: 11302 free(*mask); 11303 *mask = NULL; 11304 return err; 11305 } 11306 11307 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 11308 { 11309 int fd, err = 0, len; 11310 char buf[128]; 11311 11312 fd = open(fcpu, O_RDONLY); 11313 if (fd < 0) { 11314 err = -errno; 11315 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 11316 return err; 11317 } 11318 len = read(fd, buf, sizeof(buf)); 11319 close(fd); 11320 if (len <= 0) { 11321 err = len ? -errno : -EINVAL; 11322 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 11323 return err; 11324 } 11325 if (len >= sizeof(buf)) { 11326 pr_warn("CPU mask is too big in file %s\n", fcpu); 11327 return -E2BIG; 11328 } 11329 buf[len] = '\0'; 11330 11331 return parse_cpu_mask_str(buf, mask, mask_sz); 11332 } 11333 11334 int libbpf_num_possible_cpus(void) 11335 { 11336 static const char *fcpu = "/sys/devices/system/cpu/possible"; 11337 static int cpus; 11338 int err, n, i, tmp_cpus; 11339 bool *mask; 11340 11341 tmp_cpus = READ_ONCE(cpus); 11342 if (tmp_cpus > 0) 11343 return tmp_cpus; 11344 11345 err = parse_cpu_mask_file(fcpu, &mask, &n); 11346 if (err) 11347 return err; 11348 11349 tmp_cpus = 0; 11350 for (i = 0; i < n; i++) { 11351 if (mask[i]) 11352 tmp_cpus++; 11353 } 11354 free(mask); 11355 11356 WRITE_ONCE(cpus, tmp_cpus); 11357 return tmp_cpus; 11358 } 11359 11360 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 11361 const struct bpf_object_open_opts *opts) 11362 { 11363 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 11364 .object_name = s->name, 11365 ); 11366 struct bpf_object *obj; 11367 int i; 11368 11369 /* Attempt to preserve opts->object_name, unless overriden by user 11370 * explicitly. Overwriting object name for skeletons is discouraged, 11371 * as it breaks global data maps, because they contain object name 11372 * prefix as their own map name prefix. When skeleton is generated, 11373 * bpftool is making an assumption that this name will stay the same. 11374 */ 11375 if (opts) { 11376 memcpy(&skel_opts, opts, sizeof(*opts)); 11377 if (!opts->object_name) 11378 skel_opts.object_name = s->name; 11379 } 11380 11381 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 11382 if (IS_ERR(obj)) { 11383 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n", 11384 s->name, PTR_ERR(obj)); 11385 return PTR_ERR(obj); 11386 } 11387 11388 *s->obj = obj; 11389 11390 for (i = 0; i < s->map_cnt; i++) { 11391 struct bpf_map **map = s->maps[i].map; 11392 const char *name = s->maps[i].name; 11393 void **mmaped = s->maps[i].mmaped; 11394 11395 *map = bpf_object__find_map_by_name(obj, name); 11396 if (!*map) { 11397 pr_warn("failed to find skeleton map '%s'\n", name); 11398 return -ESRCH; 11399 } 11400 11401 /* externs shouldn't be pre-setup from user code */ 11402 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 11403 *mmaped = (*map)->mmaped; 11404 } 11405 11406 for (i = 0; i < s->prog_cnt; i++) { 11407 struct bpf_program **prog = s->progs[i].prog; 11408 const char *name = s->progs[i].name; 11409 11410 *prog = bpf_object__find_program_by_name(obj, name); 11411 if (!*prog) { 11412 pr_warn("failed to find skeleton program '%s'\n", name); 11413 return -ESRCH; 11414 } 11415 } 11416 11417 return 0; 11418 } 11419 11420 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 11421 { 11422 int i, err; 11423 11424 err = bpf_object__load(*s->obj); 11425 if (err) { 11426 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 11427 return err; 11428 } 11429 11430 for (i = 0; i < s->map_cnt; i++) { 11431 struct bpf_map *map = *s->maps[i].map; 11432 size_t mmap_sz = bpf_map_mmap_sz(map); 11433 int prot, map_fd = bpf_map__fd(map); 11434 void **mmaped = s->maps[i].mmaped; 11435 11436 if (!mmaped) 11437 continue; 11438 11439 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 11440 *mmaped = NULL; 11441 continue; 11442 } 11443 11444 if (map->def.map_flags & BPF_F_RDONLY_PROG) 11445 prot = PROT_READ; 11446 else 11447 prot = PROT_READ | PROT_WRITE; 11448 11449 /* Remap anonymous mmap()-ed "map initialization image" as 11450 * a BPF map-backed mmap()-ed memory, but preserving the same 11451 * memory address. This will cause kernel to change process' 11452 * page table to point to a different piece of kernel memory, 11453 * but from userspace point of view memory address (and its 11454 * contents, being identical at this point) will stay the 11455 * same. This mapping will be released by bpf_object__close() 11456 * as per normal clean up procedure, so we don't need to worry 11457 * about it from skeleton's clean up perspective. 11458 */ 11459 *mmaped = mmap(map->mmaped, mmap_sz, prot, 11460 MAP_SHARED | MAP_FIXED, map_fd, 0); 11461 if (*mmaped == MAP_FAILED) { 11462 err = -errno; 11463 *mmaped = NULL; 11464 pr_warn("failed to re-mmap() map '%s': %d\n", 11465 bpf_map__name(map), err); 11466 return err; 11467 } 11468 } 11469 11470 return 0; 11471 } 11472 11473 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 11474 { 11475 int i; 11476 11477 for (i = 0; i < s->prog_cnt; i++) { 11478 struct bpf_program *prog = *s->progs[i].prog; 11479 struct bpf_link **link = s->progs[i].link; 11480 const struct bpf_sec_def *sec_def; 11481 11482 if (!prog->load) 11483 continue; 11484 11485 sec_def = find_sec_def(prog->sec_name); 11486 if (!sec_def || !sec_def->attach_fn) 11487 continue; 11488 11489 *link = sec_def->attach_fn(sec_def, prog); 11490 if (IS_ERR(*link)) { 11491 pr_warn("failed to auto-attach program '%s': %ld\n", 11492 bpf_program__name(prog), PTR_ERR(*link)); 11493 return PTR_ERR(*link); 11494 } 11495 } 11496 11497 return 0; 11498 } 11499 11500 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 11501 { 11502 int i; 11503 11504 for (i = 0; i < s->prog_cnt; i++) { 11505 struct bpf_link **link = s->progs[i].link; 11506 11507 bpf_link__destroy(*link); 11508 *link = NULL; 11509 } 11510 } 11511 11512 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 11513 { 11514 if (s->progs) 11515 bpf_object__detach_skeleton(s); 11516 if (s->obj) 11517 bpf_object__close(*s->obj); 11518 free(s->maps); 11519 free(s->progs); 11520 free(s); 11521 } 11522