1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/list.h> 35 #include <linux/limits.h> 36 #include <linux/perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <linux/version.h> 39 #include <sys/epoll.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <sys/stat.h> 43 #include <sys/types.h> 44 #include <sys/vfs.h> 45 #include <sys/utsname.h> 46 #include <sys/resource.h> 47 #include <libelf.h> 48 #include <gelf.h> 49 #include <zlib.h> 50 51 #include "libbpf.h" 52 #include "bpf.h" 53 #include "btf.h" 54 #include "str_error.h" 55 #include "libbpf_internal.h" 56 #include "hashmap.h" 57 #include "bpf_gen_internal.h" 58 59 #ifndef BPF_FS_MAGIC 60 #define BPF_FS_MAGIC 0xcafe4a11 61 #endif 62 63 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 64 65 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 66 * compilation if user enables corresponding warning. Disable it explicitly. 67 */ 68 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 69 70 #define __printf(a, b) __attribute__((format(printf, a, b))) 71 72 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 73 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 74 75 static int __base_pr(enum libbpf_print_level level, const char *format, 76 va_list args) 77 { 78 if (level == LIBBPF_DEBUG) 79 return 0; 80 81 return vfprintf(stderr, format, args); 82 } 83 84 static libbpf_print_fn_t __libbpf_pr = __base_pr; 85 86 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 87 { 88 libbpf_print_fn_t old_print_fn = __libbpf_pr; 89 90 __libbpf_pr = fn; 91 return old_print_fn; 92 } 93 94 __printf(2, 3) 95 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 96 { 97 va_list args; 98 99 if (!__libbpf_pr) 100 return; 101 102 va_start(args, format); 103 __libbpf_pr(level, format, args); 104 va_end(args); 105 } 106 107 static void pr_perm_msg(int err) 108 { 109 struct rlimit limit; 110 char buf[100]; 111 112 if (err != -EPERM || geteuid() != 0) 113 return; 114 115 err = getrlimit(RLIMIT_MEMLOCK, &limit); 116 if (err) 117 return; 118 119 if (limit.rlim_cur == RLIM_INFINITY) 120 return; 121 122 if (limit.rlim_cur < 1024) 123 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 124 else if (limit.rlim_cur < 1024*1024) 125 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 126 else 127 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 128 129 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 130 buf); 131 } 132 133 #define STRERR_BUFSIZE 128 134 135 /* Copied from tools/perf/util/util.h */ 136 #ifndef zfree 137 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 138 #endif 139 140 #ifndef zclose 141 # define zclose(fd) ({ \ 142 int ___err = 0; \ 143 if ((fd) >= 0) \ 144 ___err = close((fd)); \ 145 fd = -1; \ 146 ___err; }) 147 #endif 148 149 static inline __u64 ptr_to_u64(const void *ptr) 150 { 151 return (__u64) (unsigned long) ptr; 152 } 153 154 /* this goes away in libbpf 1.0 */ 155 enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE; 156 157 int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 158 { 159 /* __LIBBPF_STRICT_LAST is the last power-of-2 value used + 1, so to 160 * get all possible values we compensate last +1, and then (2*x - 1) 161 * to get the bit mask 162 */ 163 if (mode != LIBBPF_STRICT_ALL 164 && (mode & ~((__LIBBPF_STRICT_LAST - 1) * 2 - 1))) 165 return errno = EINVAL, -EINVAL; 166 167 libbpf_mode = mode; 168 return 0; 169 } 170 171 enum kern_feature_id { 172 /* v4.14: kernel support for program & map names. */ 173 FEAT_PROG_NAME, 174 /* v5.2: kernel support for global data sections. */ 175 FEAT_GLOBAL_DATA, 176 /* BTF support */ 177 FEAT_BTF, 178 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 179 FEAT_BTF_FUNC, 180 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 181 FEAT_BTF_DATASEC, 182 /* BTF_FUNC_GLOBAL is supported */ 183 FEAT_BTF_GLOBAL_FUNC, 184 /* BPF_F_MMAPABLE is supported for arrays */ 185 FEAT_ARRAY_MMAP, 186 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 187 FEAT_EXP_ATTACH_TYPE, 188 /* bpf_probe_read_{kernel,user}[_str] helpers */ 189 FEAT_PROBE_READ_KERN, 190 /* BPF_PROG_BIND_MAP is supported */ 191 FEAT_PROG_BIND_MAP, 192 /* Kernel support for module BTFs */ 193 FEAT_MODULE_BTF, 194 /* BTF_KIND_FLOAT support */ 195 FEAT_BTF_FLOAT, 196 /* BPF perf link support */ 197 FEAT_PERF_LINK, 198 /* BTF_KIND_TAG support */ 199 FEAT_BTF_TAG, 200 __FEAT_CNT, 201 }; 202 203 static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); 204 205 enum reloc_type { 206 RELO_LD64, 207 RELO_CALL, 208 RELO_DATA, 209 RELO_EXTERN_VAR, 210 RELO_EXTERN_FUNC, 211 RELO_SUBPROG_ADDR, 212 }; 213 214 struct reloc_desc { 215 enum reloc_type type; 216 int insn_idx; 217 int map_idx; 218 int sym_off; 219 }; 220 221 struct bpf_sec_def; 222 223 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_program *prog); 224 225 struct bpf_sec_def { 226 const char *sec; 227 size_t len; 228 enum bpf_prog_type prog_type; 229 enum bpf_attach_type expected_attach_type; 230 bool is_exp_attach_type_optional; 231 bool is_attachable; 232 bool is_attach_btf; 233 bool is_sleepable; 234 attach_fn_t attach_fn; 235 }; 236 237 /* 238 * bpf_prog should be a better name but it has been used in 239 * linux/filter.h. 240 */ 241 struct bpf_program { 242 const struct bpf_sec_def *sec_def; 243 char *sec_name; 244 size_t sec_idx; 245 /* this program's instruction offset (in number of instructions) 246 * within its containing ELF section 247 */ 248 size_t sec_insn_off; 249 /* number of original instructions in ELF section belonging to this 250 * program, not taking into account subprogram instructions possible 251 * appended later during relocation 252 */ 253 size_t sec_insn_cnt; 254 /* Offset (in number of instructions) of the start of instruction 255 * belonging to this BPF program within its containing main BPF 256 * program. For the entry-point (main) BPF program, this is always 257 * zero. For a sub-program, this gets reset before each of main BPF 258 * programs are processed and relocated and is used to determined 259 * whether sub-program was already appended to the main program, and 260 * if yes, at which instruction offset. 261 */ 262 size_t sub_insn_off; 263 264 char *name; 265 /* sec_name with / replaced by _; makes recursive pinning 266 * in bpf_object__pin_programs easier 267 */ 268 char *pin_name; 269 270 /* instructions that belong to BPF program; insns[0] is located at 271 * sec_insn_off instruction within its ELF section in ELF file, so 272 * when mapping ELF file instruction index to the local instruction, 273 * one needs to subtract sec_insn_off; and vice versa. 274 */ 275 struct bpf_insn *insns; 276 /* actual number of instruction in this BPF program's image; for 277 * entry-point BPF programs this includes the size of main program 278 * itself plus all the used sub-programs, appended at the end 279 */ 280 size_t insns_cnt; 281 282 struct reloc_desc *reloc_desc; 283 int nr_reloc; 284 int log_level; 285 286 struct { 287 int nr; 288 int *fds; 289 } instances; 290 bpf_program_prep_t preprocessor; 291 292 struct bpf_object *obj; 293 void *priv; 294 bpf_program_clear_priv_t clear_priv; 295 296 bool load; 297 bool mark_btf_static; 298 enum bpf_prog_type type; 299 enum bpf_attach_type expected_attach_type; 300 int prog_ifindex; 301 __u32 attach_btf_obj_fd; 302 __u32 attach_btf_id; 303 __u32 attach_prog_fd; 304 void *func_info; 305 __u32 func_info_rec_size; 306 __u32 func_info_cnt; 307 308 void *line_info; 309 __u32 line_info_rec_size; 310 __u32 line_info_cnt; 311 __u32 prog_flags; 312 }; 313 314 struct bpf_struct_ops { 315 const char *tname; 316 const struct btf_type *type; 317 struct bpf_program **progs; 318 __u32 *kern_func_off; 319 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 320 void *data; 321 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 322 * btf_vmlinux's format. 323 * struct bpf_struct_ops_tcp_congestion_ops { 324 * [... some other kernel fields ...] 325 * struct tcp_congestion_ops data; 326 * } 327 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 328 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 329 * from "data". 330 */ 331 void *kern_vdata; 332 __u32 type_id; 333 }; 334 335 #define DATA_SEC ".data" 336 #define BSS_SEC ".bss" 337 #define RODATA_SEC ".rodata" 338 #define KCONFIG_SEC ".kconfig" 339 #define KSYMS_SEC ".ksyms" 340 #define STRUCT_OPS_SEC ".struct_ops" 341 342 enum libbpf_map_type { 343 LIBBPF_MAP_UNSPEC, 344 LIBBPF_MAP_DATA, 345 LIBBPF_MAP_BSS, 346 LIBBPF_MAP_RODATA, 347 LIBBPF_MAP_KCONFIG, 348 }; 349 350 static const char * const libbpf_type_to_btf_name[] = { 351 [LIBBPF_MAP_DATA] = DATA_SEC, 352 [LIBBPF_MAP_BSS] = BSS_SEC, 353 [LIBBPF_MAP_RODATA] = RODATA_SEC, 354 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, 355 }; 356 357 struct bpf_map { 358 char *name; 359 int fd; 360 int sec_idx; 361 size_t sec_offset; 362 int map_ifindex; 363 int inner_map_fd; 364 struct bpf_map_def def; 365 __u32 numa_node; 366 __u32 btf_var_idx; 367 __u32 btf_key_type_id; 368 __u32 btf_value_type_id; 369 __u32 btf_vmlinux_value_type_id; 370 void *priv; 371 bpf_map_clear_priv_t clear_priv; 372 enum libbpf_map_type libbpf_type; 373 void *mmaped; 374 struct bpf_struct_ops *st_ops; 375 struct bpf_map *inner_map; 376 void **init_slots; 377 int init_slots_sz; 378 char *pin_path; 379 bool pinned; 380 bool reused; 381 }; 382 383 enum extern_type { 384 EXT_UNKNOWN, 385 EXT_KCFG, 386 EXT_KSYM, 387 }; 388 389 enum kcfg_type { 390 KCFG_UNKNOWN, 391 KCFG_CHAR, 392 KCFG_BOOL, 393 KCFG_INT, 394 KCFG_TRISTATE, 395 KCFG_CHAR_ARR, 396 }; 397 398 struct extern_desc { 399 enum extern_type type; 400 int sym_idx; 401 int btf_id; 402 int sec_btf_id; 403 const char *name; 404 bool is_set; 405 bool is_weak; 406 union { 407 struct { 408 enum kcfg_type type; 409 int sz; 410 int align; 411 int data_off; 412 bool is_signed; 413 } kcfg; 414 struct { 415 unsigned long long addr; 416 417 /* target btf_id of the corresponding kernel var. */ 418 int kernel_btf_obj_fd; 419 int kernel_btf_id; 420 421 /* local btf_id of the ksym extern's type. */ 422 __u32 type_id; 423 } ksym; 424 }; 425 }; 426 427 static LIST_HEAD(bpf_objects_list); 428 429 struct module_btf { 430 struct btf *btf; 431 char *name; 432 __u32 id; 433 int fd; 434 }; 435 436 struct bpf_object { 437 char name[BPF_OBJ_NAME_LEN]; 438 char license[64]; 439 __u32 kern_version; 440 441 struct bpf_program *programs; 442 size_t nr_programs; 443 struct bpf_map *maps; 444 size_t nr_maps; 445 size_t maps_cap; 446 447 char *kconfig; 448 struct extern_desc *externs; 449 int nr_extern; 450 int kconfig_map_idx; 451 int rodata_map_idx; 452 453 bool loaded; 454 bool has_subcalls; 455 456 struct bpf_gen *gen_loader; 457 458 /* 459 * Information when doing elf related work. Only valid if fd 460 * is valid. 461 */ 462 struct { 463 int fd; 464 const void *obj_buf; 465 size_t obj_buf_sz; 466 Elf *elf; 467 GElf_Ehdr ehdr; 468 Elf_Data *symbols; 469 Elf_Data *data; 470 Elf_Data *rodata; 471 Elf_Data *bss; 472 Elf_Data *st_ops_data; 473 size_t shstrndx; /* section index for section name strings */ 474 size_t strtabidx; 475 struct { 476 GElf_Shdr shdr; 477 Elf_Data *data; 478 } *reloc_sects; 479 int nr_reloc_sects; 480 int maps_shndx; 481 int btf_maps_shndx; 482 __u32 btf_maps_sec_btf_id; 483 int text_shndx; 484 int symbols_shndx; 485 int data_shndx; 486 int rodata_shndx; 487 int bss_shndx; 488 int st_ops_shndx; 489 } efile; 490 /* 491 * All loaded bpf_object is linked in a list, which is 492 * hidden to caller. bpf_objects__<func> handlers deal with 493 * all objects. 494 */ 495 struct list_head list; 496 497 struct btf *btf; 498 struct btf_ext *btf_ext; 499 500 /* Parse and load BTF vmlinux if any of the programs in the object need 501 * it at load time. 502 */ 503 struct btf *btf_vmlinux; 504 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 505 * override for vmlinux BTF. 506 */ 507 char *btf_custom_path; 508 /* vmlinux BTF override for CO-RE relocations */ 509 struct btf *btf_vmlinux_override; 510 /* Lazily initialized kernel module BTFs */ 511 struct module_btf *btf_modules; 512 bool btf_modules_loaded; 513 size_t btf_module_cnt; 514 size_t btf_module_cap; 515 516 void *priv; 517 bpf_object_clear_priv_t clear_priv; 518 519 char path[]; 520 }; 521 #define obj_elf_valid(o) ((o)->efile.elf) 522 523 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 524 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 525 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 526 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 527 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr); 528 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 529 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 530 531 void bpf_program__unload(struct bpf_program *prog) 532 { 533 int i; 534 535 if (!prog) 536 return; 537 538 /* 539 * If the object is opened but the program was never loaded, 540 * it is possible that prog->instances.nr == -1. 541 */ 542 if (prog->instances.nr > 0) { 543 for (i = 0; i < prog->instances.nr; i++) 544 zclose(prog->instances.fds[i]); 545 } else if (prog->instances.nr != -1) { 546 pr_warn("Internal error: instances.nr is %d\n", 547 prog->instances.nr); 548 } 549 550 prog->instances.nr = -1; 551 zfree(&prog->instances.fds); 552 553 zfree(&prog->func_info); 554 zfree(&prog->line_info); 555 } 556 557 static void bpf_program__exit(struct bpf_program *prog) 558 { 559 if (!prog) 560 return; 561 562 if (prog->clear_priv) 563 prog->clear_priv(prog, prog->priv); 564 565 prog->priv = NULL; 566 prog->clear_priv = NULL; 567 568 bpf_program__unload(prog); 569 zfree(&prog->name); 570 zfree(&prog->sec_name); 571 zfree(&prog->pin_name); 572 zfree(&prog->insns); 573 zfree(&prog->reloc_desc); 574 575 prog->nr_reloc = 0; 576 prog->insns_cnt = 0; 577 prog->sec_idx = -1; 578 } 579 580 static char *__bpf_program__pin_name(struct bpf_program *prog) 581 { 582 char *name, *p; 583 584 name = p = strdup(prog->sec_name); 585 while ((p = strchr(p, '/'))) 586 *p = '_'; 587 588 return name; 589 } 590 591 static bool insn_is_subprog_call(const struct bpf_insn *insn) 592 { 593 return BPF_CLASS(insn->code) == BPF_JMP && 594 BPF_OP(insn->code) == BPF_CALL && 595 BPF_SRC(insn->code) == BPF_K && 596 insn->src_reg == BPF_PSEUDO_CALL && 597 insn->dst_reg == 0 && 598 insn->off == 0; 599 } 600 601 static bool is_call_insn(const struct bpf_insn *insn) 602 { 603 return insn->code == (BPF_JMP | BPF_CALL); 604 } 605 606 static bool insn_is_pseudo_func(struct bpf_insn *insn) 607 { 608 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 609 } 610 611 static int 612 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 613 const char *name, size_t sec_idx, const char *sec_name, 614 size_t sec_off, void *insn_data, size_t insn_data_sz) 615 { 616 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 617 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 618 sec_name, name, sec_off, insn_data_sz); 619 return -EINVAL; 620 } 621 622 memset(prog, 0, sizeof(*prog)); 623 prog->obj = obj; 624 625 prog->sec_idx = sec_idx; 626 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 627 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 628 /* insns_cnt can later be increased by appending used subprograms */ 629 prog->insns_cnt = prog->sec_insn_cnt; 630 631 prog->type = BPF_PROG_TYPE_UNSPEC; 632 prog->load = true; 633 634 prog->instances.fds = NULL; 635 prog->instances.nr = -1; 636 637 prog->sec_name = strdup(sec_name); 638 if (!prog->sec_name) 639 goto errout; 640 641 prog->name = strdup(name); 642 if (!prog->name) 643 goto errout; 644 645 prog->pin_name = __bpf_program__pin_name(prog); 646 if (!prog->pin_name) 647 goto errout; 648 649 prog->insns = malloc(insn_data_sz); 650 if (!prog->insns) 651 goto errout; 652 memcpy(prog->insns, insn_data, insn_data_sz); 653 654 return 0; 655 errout: 656 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 657 bpf_program__exit(prog); 658 return -ENOMEM; 659 } 660 661 static int 662 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 663 const char *sec_name, int sec_idx) 664 { 665 Elf_Data *symbols = obj->efile.symbols; 666 struct bpf_program *prog, *progs; 667 void *data = sec_data->d_buf; 668 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 669 int nr_progs, err, i; 670 const char *name; 671 GElf_Sym sym; 672 673 progs = obj->programs; 674 nr_progs = obj->nr_programs; 675 nr_syms = symbols->d_size / sizeof(GElf_Sym); 676 sec_off = 0; 677 678 for (i = 0; i < nr_syms; i++) { 679 if (!gelf_getsym(symbols, i, &sym)) 680 continue; 681 if (sym.st_shndx != sec_idx) 682 continue; 683 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC) 684 continue; 685 686 prog_sz = sym.st_size; 687 sec_off = sym.st_value; 688 689 name = elf_sym_str(obj, sym.st_name); 690 if (!name) { 691 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 692 sec_name, sec_off); 693 return -LIBBPF_ERRNO__FORMAT; 694 } 695 696 if (sec_off + prog_sz > sec_sz) { 697 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 698 sec_name, sec_off); 699 return -LIBBPF_ERRNO__FORMAT; 700 } 701 702 if (sec_idx != obj->efile.text_shndx && GELF_ST_BIND(sym.st_info) == STB_LOCAL) { 703 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 704 return -ENOTSUP; 705 } 706 707 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 708 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 709 710 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 711 if (!progs) { 712 /* 713 * In this case the original obj->programs 714 * is still valid, so don't need special treat for 715 * bpf_close_object(). 716 */ 717 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 718 sec_name, name); 719 return -ENOMEM; 720 } 721 obj->programs = progs; 722 723 prog = &progs[nr_progs]; 724 725 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 726 sec_off, data + sec_off, prog_sz); 727 if (err) 728 return err; 729 730 /* if function is a global/weak symbol, but has restricted 731 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 732 * as static to enable more permissive BPF verification mode 733 * with more outside context available to BPF verifier 734 */ 735 if (GELF_ST_BIND(sym.st_info) != STB_LOCAL 736 && (GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN 737 || GELF_ST_VISIBILITY(sym.st_other) == STV_INTERNAL)) 738 prog->mark_btf_static = true; 739 740 nr_progs++; 741 obj->nr_programs = nr_progs; 742 } 743 744 return 0; 745 } 746 747 static __u32 get_kernel_version(void) 748 { 749 __u32 major, minor, patch; 750 struct utsname info; 751 752 uname(&info); 753 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) 754 return 0; 755 return KERNEL_VERSION(major, minor, patch); 756 } 757 758 static const struct btf_member * 759 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 760 { 761 struct btf_member *m; 762 int i; 763 764 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 765 if (btf_member_bit_offset(t, i) == bit_offset) 766 return m; 767 } 768 769 return NULL; 770 } 771 772 static const struct btf_member * 773 find_member_by_name(const struct btf *btf, const struct btf_type *t, 774 const char *name) 775 { 776 struct btf_member *m; 777 int i; 778 779 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 780 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 781 return m; 782 } 783 784 return NULL; 785 } 786 787 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 788 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 789 const char *name, __u32 kind); 790 791 static int 792 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 793 const struct btf_type **type, __u32 *type_id, 794 const struct btf_type **vtype, __u32 *vtype_id, 795 const struct btf_member **data_member) 796 { 797 const struct btf_type *kern_type, *kern_vtype; 798 const struct btf_member *kern_data_member; 799 __s32 kern_vtype_id, kern_type_id; 800 __u32 i; 801 802 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 803 if (kern_type_id < 0) { 804 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 805 tname); 806 return kern_type_id; 807 } 808 kern_type = btf__type_by_id(btf, kern_type_id); 809 810 /* Find the corresponding "map_value" type that will be used 811 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 812 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 813 * btf_vmlinux. 814 */ 815 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 816 tname, BTF_KIND_STRUCT); 817 if (kern_vtype_id < 0) { 818 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 819 STRUCT_OPS_VALUE_PREFIX, tname); 820 return kern_vtype_id; 821 } 822 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 823 824 /* Find "struct tcp_congestion_ops" from 825 * struct bpf_struct_ops_tcp_congestion_ops { 826 * [ ... ] 827 * struct tcp_congestion_ops data; 828 * } 829 */ 830 kern_data_member = btf_members(kern_vtype); 831 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 832 if (kern_data_member->type == kern_type_id) 833 break; 834 } 835 if (i == btf_vlen(kern_vtype)) { 836 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 837 tname, STRUCT_OPS_VALUE_PREFIX, tname); 838 return -EINVAL; 839 } 840 841 *type = kern_type; 842 *type_id = kern_type_id; 843 *vtype = kern_vtype; 844 *vtype_id = kern_vtype_id; 845 *data_member = kern_data_member; 846 847 return 0; 848 } 849 850 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 851 { 852 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 853 } 854 855 /* Init the map's fields that depend on kern_btf */ 856 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 857 const struct btf *btf, 858 const struct btf *kern_btf) 859 { 860 const struct btf_member *member, *kern_member, *kern_data_member; 861 const struct btf_type *type, *kern_type, *kern_vtype; 862 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 863 struct bpf_struct_ops *st_ops; 864 void *data, *kern_data; 865 const char *tname; 866 int err; 867 868 st_ops = map->st_ops; 869 type = st_ops->type; 870 tname = st_ops->tname; 871 err = find_struct_ops_kern_types(kern_btf, tname, 872 &kern_type, &kern_type_id, 873 &kern_vtype, &kern_vtype_id, 874 &kern_data_member); 875 if (err) 876 return err; 877 878 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 879 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 880 881 map->def.value_size = kern_vtype->size; 882 map->btf_vmlinux_value_type_id = kern_vtype_id; 883 884 st_ops->kern_vdata = calloc(1, kern_vtype->size); 885 if (!st_ops->kern_vdata) 886 return -ENOMEM; 887 888 data = st_ops->data; 889 kern_data_off = kern_data_member->offset / 8; 890 kern_data = st_ops->kern_vdata + kern_data_off; 891 892 member = btf_members(type); 893 for (i = 0; i < btf_vlen(type); i++, member++) { 894 const struct btf_type *mtype, *kern_mtype; 895 __u32 mtype_id, kern_mtype_id; 896 void *mdata, *kern_mdata; 897 __s64 msize, kern_msize; 898 __u32 moff, kern_moff; 899 __u32 kern_member_idx; 900 const char *mname; 901 902 mname = btf__name_by_offset(btf, member->name_off); 903 kern_member = find_member_by_name(kern_btf, kern_type, mname); 904 if (!kern_member) { 905 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 906 map->name, mname); 907 return -ENOTSUP; 908 } 909 910 kern_member_idx = kern_member - btf_members(kern_type); 911 if (btf_member_bitfield_size(type, i) || 912 btf_member_bitfield_size(kern_type, kern_member_idx)) { 913 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 914 map->name, mname); 915 return -ENOTSUP; 916 } 917 918 moff = member->offset / 8; 919 kern_moff = kern_member->offset / 8; 920 921 mdata = data + moff; 922 kern_mdata = kern_data + kern_moff; 923 924 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 925 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 926 &kern_mtype_id); 927 if (BTF_INFO_KIND(mtype->info) != 928 BTF_INFO_KIND(kern_mtype->info)) { 929 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 930 map->name, mname, BTF_INFO_KIND(mtype->info), 931 BTF_INFO_KIND(kern_mtype->info)); 932 return -ENOTSUP; 933 } 934 935 if (btf_is_ptr(mtype)) { 936 struct bpf_program *prog; 937 938 prog = st_ops->progs[i]; 939 if (!prog) 940 continue; 941 942 kern_mtype = skip_mods_and_typedefs(kern_btf, 943 kern_mtype->type, 944 &kern_mtype_id); 945 946 /* mtype->type must be a func_proto which was 947 * guaranteed in bpf_object__collect_st_ops_relos(), 948 * so only check kern_mtype for func_proto here. 949 */ 950 if (!btf_is_func_proto(kern_mtype)) { 951 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 952 map->name, mname); 953 return -ENOTSUP; 954 } 955 956 prog->attach_btf_id = kern_type_id; 957 prog->expected_attach_type = kern_member_idx; 958 959 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 960 961 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 962 map->name, mname, prog->name, moff, 963 kern_moff); 964 965 continue; 966 } 967 968 msize = btf__resolve_size(btf, mtype_id); 969 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 970 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 971 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 972 map->name, mname, (ssize_t)msize, 973 (ssize_t)kern_msize); 974 return -ENOTSUP; 975 } 976 977 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 978 map->name, mname, (unsigned int)msize, 979 moff, kern_moff); 980 memcpy(kern_mdata, mdata, msize); 981 } 982 983 return 0; 984 } 985 986 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 987 { 988 struct bpf_map *map; 989 size_t i; 990 int err; 991 992 for (i = 0; i < obj->nr_maps; i++) { 993 map = &obj->maps[i]; 994 995 if (!bpf_map__is_struct_ops(map)) 996 continue; 997 998 err = bpf_map__init_kern_struct_ops(map, obj->btf, 999 obj->btf_vmlinux); 1000 if (err) 1001 return err; 1002 } 1003 1004 return 0; 1005 } 1006 1007 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) 1008 { 1009 const struct btf_type *type, *datasec; 1010 const struct btf_var_secinfo *vsi; 1011 struct bpf_struct_ops *st_ops; 1012 const char *tname, *var_name; 1013 __s32 type_id, datasec_id; 1014 const struct btf *btf; 1015 struct bpf_map *map; 1016 __u32 i; 1017 1018 if (obj->efile.st_ops_shndx == -1) 1019 return 0; 1020 1021 btf = obj->btf; 1022 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, 1023 BTF_KIND_DATASEC); 1024 if (datasec_id < 0) { 1025 pr_warn("struct_ops init: DATASEC %s not found\n", 1026 STRUCT_OPS_SEC); 1027 return -EINVAL; 1028 } 1029 1030 datasec = btf__type_by_id(btf, datasec_id); 1031 vsi = btf_var_secinfos(datasec); 1032 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1033 type = btf__type_by_id(obj->btf, vsi->type); 1034 var_name = btf__name_by_offset(obj->btf, type->name_off); 1035 1036 type_id = btf__resolve_type(obj->btf, vsi->type); 1037 if (type_id < 0) { 1038 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1039 vsi->type, STRUCT_OPS_SEC); 1040 return -EINVAL; 1041 } 1042 1043 type = btf__type_by_id(obj->btf, type_id); 1044 tname = btf__name_by_offset(obj->btf, type->name_off); 1045 if (!tname[0]) { 1046 pr_warn("struct_ops init: anonymous type is not supported\n"); 1047 return -ENOTSUP; 1048 } 1049 if (!btf_is_struct(type)) { 1050 pr_warn("struct_ops init: %s is not a struct\n", tname); 1051 return -EINVAL; 1052 } 1053 1054 map = bpf_object__add_map(obj); 1055 if (IS_ERR(map)) 1056 return PTR_ERR(map); 1057 1058 map->sec_idx = obj->efile.st_ops_shndx; 1059 map->sec_offset = vsi->offset; 1060 map->name = strdup(var_name); 1061 if (!map->name) 1062 return -ENOMEM; 1063 1064 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1065 map->def.key_size = sizeof(int); 1066 map->def.value_size = type->size; 1067 map->def.max_entries = 1; 1068 1069 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1070 if (!map->st_ops) 1071 return -ENOMEM; 1072 st_ops = map->st_ops; 1073 st_ops->data = malloc(type->size); 1074 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1075 st_ops->kern_func_off = malloc(btf_vlen(type) * 1076 sizeof(*st_ops->kern_func_off)); 1077 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1078 return -ENOMEM; 1079 1080 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { 1081 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1082 var_name, STRUCT_OPS_SEC); 1083 return -EINVAL; 1084 } 1085 1086 memcpy(st_ops->data, 1087 obj->efile.st_ops_data->d_buf + vsi->offset, 1088 type->size); 1089 st_ops->tname = tname; 1090 st_ops->type = type; 1091 st_ops->type_id = type_id; 1092 1093 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1094 tname, type_id, var_name, vsi->offset); 1095 } 1096 1097 return 0; 1098 } 1099 1100 static struct bpf_object *bpf_object__new(const char *path, 1101 const void *obj_buf, 1102 size_t obj_buf_sz, 1103 const char *obj_name) 1104 { 1105 struct bpf_object *obj; 1106 char *end; 1107 1108 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1109 if (!obj) { 1110 pr_warn("alloc memory failed for %s\n", path); 1111 return ERR_PTR(-ENOMEM); 1112 } 1113 1114 strcpy(obj->path, path); 1115 if (obj_name) { 1116 strncpy(obj->name, obj_name, sizeof(obj->name) - 1); 1117 obj->name[sizeof(obj->name) - 1] = 0; 1118 } else { 1119 /* Using basename() GNU version which doesn't modify arg. */ 1120 strncpy(obj->name, basename((void *)path), 1121 sizeof(obj->name) - 1); 1122 end = strchr(obj->name, '.'); 1123 if (end) 1124 *end = 0; 1125 } 1126 1127 obj->efile.fd = -1; 1128 /* 1129 * Caller of this function should also call 1130 * bpf_object__elf_finish() after data collection to return 1131 * obj_buf to user. If not, we should duplicate the buffer to 1132 * avoid user freeing them before elf finish. 1133 */ 1134 obj->efile.obj_buf = obj_buf; 1135 obj->efile.obj_buf_sz = obj_buf_sz; 1136 obj->efile.maps_shndx = -1; 1137 obj->efile.btf_maps_shndx = -1; 1138 obj->efile.data_shndx = -1; 1139 obj->efile.rodata_shndx = -1; 1140 obj->efile.bss_shndx = -1; 1141 obj->efile.st_ops_shndx = -1; 1142 obj->kconfig_map_idx = -1; 1143 obj->rodata_map_idx = -1; 1144 1145 obj->kern_version = get_kernel_version(); 1146 obj->loaded = false; 1147 1148 INIT_LIST_HEAD(&obj->list); 1149 list_add(&obj->list, &bpf_objects_list); 1150 return obj; 1151 } 1152 1153 static void bpf_object__elf_finish(struct bpf_object *obj) 1154 { 1155 if (!obj_elf_valid(obj)) 1156 return; 1157 1158 if (obj->efile.elf) { 1159 elf_end(obj->efile.elf); 1160 obj->efile.elf = NULL; 1161 } 1162 obj->efile.symbols = NULL; 1163 obj->efile.data = NULL; 1164 obj->efile.rodata = NULL; 1165 obj->efile.bss = NULL; 1166 obj->efile.st_ops_data = NULL; 1167 1168 zfree(&obj->efile.reloc_sects); 1169 obj->efile.nr_reloc_sects = 0; 1170 zclose(obj->efile.fd); 1171 obj->efile.obj_buf = NULL; 1172 obj->efile.obj_buf_sz = 0; 1173 } 1174 1175 static int bpf_object__elf_init(struct bpf_object *obj) 1176 { 1177 int err = 0; 1178 GElf_Ehdr *ep; 1179 1180 if (obj_elf_valid(obj)) { 1181 pr_warn("elf: init internal error\n"); 1182 return -LIBBPF_ERRNO__LIBELF; 1183 } 1184 1185 if (obj->efile.obj_buf_sz > 0) { 1186 /* 1187 * obj_buf should have been validated by 1188 * bpf_object__open_buffer(). 1189 */ 1190 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, 1191 obj->efile.obj_buf_sz); 1192 } else { 1193 obj->efile.fd = open(obj->path, O_RDONLY); 1194 if (obj->efile.fd < 0) { 1195 char errmsg[STRERR_BUFSIZE], *cp; 1196 1197 err = -errno; 1198 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1199 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1200 return err; 1201 } 1202 1203 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1204 } 1205 1206 if (!obj->efile.elf) { 1207 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1208 err = -LIBBPF_ERRNO__LIBELF; 1209 goto errout; 1210 } 1211 1212 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 1213 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1214 err = -LIBBPF_ERRNO__FORMAT; 1215 goto errout; 1216 } 1217 ep = &obj->efile.ehdr; 1218 1219 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) { 1220 pr_warn("elf: failed to get section names section index for %s: %s\n", 1221 obj->path, elf_errmsg(-1)); 1222 err = -LIBBPF_ERRNO__FORMAT; 1223 goto errout; 1224 } 1225 1226 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1227 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) { 1228 pr_warn("elf: failed to get section names strings from %s: %s\n", 1229 obj->path, elf_errmsg(-1)); 1230 err = -LIBBPF_ERRNO__FORMAT; 1231 goto errout; 1232 } 1233 1234 /* Old LLVM set e_machine to EM_NONE */ 1235 if (ep->e_type != ET_REL || 1236 (ep->e_machine && ep->e_machine != EM_BPF)) { 1237 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1238 err = -LIBBPF_ERRNO__FORMAT; 1239 goto errout; 1240 } 1241 1242 return 0; 1243 errout: 1244 bpf_object__elf_finish(obj); 1245 return err; 1246 } 1247 1248 static int bpf_object__check_endianness(struct bpf_object *obj) 1249 { 1250 #if __BYTE_ORDER == __LITTLE_ENDIAN 1251 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 1252 return 0; 1253 #elif __BYTE_ORDER == __BIG_ENDIAN 1254 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 1255 return 0; 1256 #else 1257 # error "Unrecognized __BYTE_ORDER__" 1258 #endif 1259 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1260 return -LIBBPF_ERRNO__ENDIAN; 1261 } 1262 1263 static int 1264 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1265 { 1266 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 1267 pr_debug("license of %s is %s\n", obj->path, obj->license); 1268 return 0; 1269 } 1270 1271 static int 1272 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1273 { 1274 __u32 kver; 1275 1276 if (size != sizeof(kver)) { 1277 pr_warn("invalid kver section in %s\n", obj->path); 1278 return -LIBBPF_ERRNO__FORMAT; 1279 } 1280 memcpy(&kver, data, sizeof(kver)); 1281 obj->kern_version = kver; 1282 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1283 return 0; 1284 } 1285 1286 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1287 { 1288 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1289 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1290 return true; 1291 return false; 1292 } 1293 1294 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 1295 __u32 *size) 1296 { 1297 int ret = -ENOENT; 1298 1299 *size = 0; 1300 if (!name) { 1301 return -EINVAL; 1302 } else if (!strcmp(name, DATA_SEC)) { 1303 if (obj->efile.data) 1304 *size = obj->efile.data->d_size; 1305 } else if (!strcmp(name, BSS_SEC)) { 1306 if (obj->efile.bss) 1307 *size = obj->efile.bss->d_size; 1308 } else if (!strcmp(name, RODATA_SEC)) { 1309 if (obj->efile.rodata) 1310 *size = obj->efile.rodata->d_size; 1311 } else if (!strcmp(name, STRUCT_OPS_SEC)) { 1312 if (obj->efile.st_ops_data) 1313 *size = obj->efile.st_ops_data->d_size; 1314 } else { 1315 Elf_Scn *scn = elf_sec_by_name(obj, name); 1316 Elf_Data *data = elf_sec_data(obj, scn); 1317 1318 if (data) { 1319 ret = 0; /* found it */ 1320 *size = data->d_size; 1321 } 1322 } 1323 1324 return *size ? 0 : ret; 1325 } 1326 1327 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 1328 __u32 *off) 1329 { 1330 Elf_Data *symbols = obj->efile.symbols; 1331 const char *sname; 1332 size_t si; 1333 1334 if (!name || !off) 1335 return -EINVAL; 1336 1337 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 1338 GElf_Sym sym; 1339 1340 if (!gelf_getsym(symbols, si, &sym)) 1341 continue; 1342 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 1343 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 1344 continue; 1345 1346 sname = elf_sym_str(obj, sym.st_name); 1347 if (!sname) { 1348 pr_warn("failed to get sym name string for var %s\n", 1349 name); 1350 return -EIO; 1351 } 1352 if (strcmp(name, sname) == 0) { 1353 *off = sym.st_value; 1354 return 0; 1355 } 1356 } 1357 1358 return -ENOENT; 1359 } 1360 1361 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1362 { 1363 struct bpf_map *new_maps; 1364 size_t new_cap; 1365 int i; 1366 1367 if (obj->nr_maps < obj->maps_cap) 1368 return &obj->maps[obj->nr_maps++]; 1369 1370 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 1371 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps)); 1372 if (!new_maps) { 1373 pr_warn("alloc maps for object failed\n"); 1374 return ERR_PTR(-ENOMEM); 1375 } 1376 1377 obj->maps_cap = new_cap; 1378 obj->maps = new_maps; 1379 1380 /* zero out new maps */ 1381 memset(obj->maps + obj->nr_maps, 0, 1382 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 1383 /* 1384 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 1385 * when failure (zclose won't close negative fd)). 1386 */ 1387 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 1388 obj->maps[i].fd = -1; 1389 obj->maps[i].inner_map_fd = -1; 1390 } 1391 1392 return &obj->maps[obj->nr_maps++]; 1393 } 1394 1395 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1396 { 1397 long page_sz = sysconf(_SC_PAGE_SIZE); 1398 size_t map_sz; 1399 1400 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; 1401 map_sz = roundup(map_sz, page_sz); 1402 return map_sz; 1403 } 1404 1405 static char *internal_map_name(struct bpf_object *obj, 1406 enum libbpf_map_type type) 1407 { 1408 char map_name[BPF_OBJ_NAME_LEN], *p; 1409 const char *sfx = libbpf_type_to_btf_name[type]; 1410 int sfx_len = max((size_t)7, strlen(sfx)); 1411 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, 1412 strlen(obj->name)); 1413 1414 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1415 sfx_len, libbpf_type_to_btf_name[type]); 1416 1417 /* sanitise map name to characters allowed by kernel */ 1418 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1419 if (!isalnum(*p) && *p != '_' && *p != '.') 1420 *p = '_'; 1421 1422 return strdup(map_name); 1423 } 1424 1425 static int 1426 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1427 int sec_idx, void *data, size_t data_sz) 1428 { 1429 struct bpf_map_def *def; 1430 struct bpf_map *map; 1431 int err; 1432 1433 map = bpf_object__add_map(obj); 1434 if (IS_ERR(map)) 1435 return PTR_ERR(map); 1436 1437 map->libbpf_type = type; 1438 map->sec_idx = sec_idx; 1439 map->sec_offset = 0; 1440 map->name = internal_map_name(obj, type); 1441 if (!map->name) { 1442 pr_warn("failed to alloc map name\n"); 1443 return -ENOMEM; 1444 } 1445 1446 def = &map->def; 1447 def->type = BPF_MAP_TYPE_ARRAY; 1448 def->key_size = sizeof(int); 1449 def->value_size = data_sz; 1450 def->max_entries = 1; 1451 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1452 ? BPF_F_RDONLY_PROG : 0; 1453 def->map_flags |= BPF_F_MMAPABLE; 1454 1455 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1456 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1457 1458 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 1459 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1460 if (map->mmaped == MAP_FAILED) { 1461 err = -errno; 1462 map->mmaped = NULL; 1463 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1464 map->name, err); 1465 zfree(&map->name); 1466 return err; 1467 } 1468 1469 if (data) 1470 memcpy(map->mmaped, data, data_sz); 1471 1472 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1473 return 0; 1474 } 1475 1476 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1477 { 1478 int err; 1479 1480 /* 1481 * Populate obj->maps with libbpf internal maps. 1482 */ 1483 if (obj->efile.data_shndx >= 0) { 1484 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1485 obj->efile.data_shndx, 1486 obj->efile.data->d_buf, 1487 obj->efile.data->d_size); 1488 if (err) 1489 return err; 1490 } 1491 if (obj->efile.rodata_shndx >= 0) { 1492 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1493 obj->efile.rodata_shndx, 1494 obj->efile.rodata->d_buf, 1495 obj->efile.rodata->d_size); 1496 if (err) 1497 return err; 1498 1499 obj->rodata_map_idx = obj->nr_maps - 1; 1500 } 1501 if (obj->efile.bss_shndx >= 0) { 1502 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1503 obj->efile.bss_shndx, 1504 NULL, 1505 obj->efile.bss->d_size); 1506 if (err) 1507 return err; 1508 } 1509 return 0; 1510 } 1511 1512 1513 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1514 const void *name) 1515 { 1516 int i; 1517 1518 for (i = 0; i < obj->nr_extern; i++) { 1519 if (strcmp(obj->externs[i].name, name) == 0) 1520 return &obj->externs[i]; 1521 } 1522 return NULL; 1523 } 1524 1525 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1526 char value) 1527 { 1528 switch (ext->kcfg.type) { 1529 case KCFG_BOOL: 1530 if (value == 'm') { 1531 pr_warn("extern (kcfg) %s=%c should be tristate or char\n", 1532 ext->name, value); 1533 return -EINVAL; 1534 } 1535 *(bool *)ext_val = value == 'y' ? true : false; 1536 break; 1537 case KCFG_TRISTATE: 1538 if (value == 'y') 1539 *(enum libbpf_tristate *)ext_val = TRI_YES; 1540 else if (value == 'm') 1541 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1542 else /* value == 'n' */ 1543 *(enum libbpf_tristate *)ext_val = TRI_NO; 1544 break; 1545 case KCFG_CHAR: 1546 *(char *)ext_val = value; 1547 break; 1548 case KCFG_UNKNOWN: 1549 case KCFG_INT: 1550 case KCFG_CHAR_ARR: 1551 default: 1552 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", 1553 ext->name, value); 1554 return -EINVAL; 1555 } 1556 ext->is_set = true; 1557 return 0; 1558 } 1559 1560 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1561 const char *value) 1562 { 1563 size_t len; 1564 1565 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1566 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); 1567 return -EINVAL; 1568 } 1569 1570 len = strlen(value); 1571 if (value[len - 1] != '"') { 1572 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1573 ext->name, value); 1574 return -EINVAL; 1575 } 1576 1577 /* strip quotes */ 1578 len -= 2; 1579 if (len >= ext->kcfg.sz) { 1580 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", 1581 ext->name, value, len, ext->kcfg.sz - 1); 1582 len = ext->kcfg.sz - 1; 1583 } 1584 memcpy(ext_val, value + 1, len); 1585 ext_val[len] = '\0'; 1586 ext->is_set = true; 1587 return 0; 1588 } 1589 1590 static int parse_u64(const char *value, __u64 *res) 1591 { 1592 char *value_end; 1593 int err; 1594 1595 errno = 0; 1596 *res = strtoull(value, &value_end, 0); 1597 if (errno) { 1598 err = -errno; 1599 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1600 return err; 1601 } 1602 if (*value_end) { 1603 pr_warn("failed to parse '%s' as integer completely\n", value); 1604 return -EINVAL; 1605 } 1606 return 0; 1607 } 1608 1609 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1610 { 1611 int bit_sz = ext->kcfg.sz * 8; 1612 1613 if (ext->kcfg.sz == 8) 1614 return true; 1615 1616 /* Validate that value stored in u64 fits in integer of `ext->sz` 1617 * bytes size without any loss of information. If the target integer 1618 * is signed, we rely on the following limits of integer type of 1619 * Y bits and subsequent transformation: 1620 * 1621 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1622 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1623 * 0 <= X + 2^(Y-1) < 2^Y 1624 * 1625 * For unsigned target integer, check that all the (64 - Y) bits are 1626 * zero. 1627 */ 1628 if (ext->kcfg.is_signed) 1629 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1630 else 1631 return (v >> bit_sz) == 0; 1632 } 1633 1634 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1635 __u64 value) 1636 { 1637 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 1638 pr_warn("extern (kcfg) %s=%llu should be integer\n", 1639 ext->name, (unsigned long long)value); 1640 return -EINVAL; 1641 } 1642 if (!is_kcfg_value_in_range(ext, value)) { 1643 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", 1644 ext->name, (unsigned long long)value, ext->kcfg.sz); 1645 return -ERANGE; 1646 } 1647 switch (ext->kcfg.sz) { 1648 case 1: *(__u8 *)ext_val = value; break; 1649 case 2: *(__u16 *)ext_val = value; break; 1650 case 4: *(__u32 *)ext_val = value; break; 1651 case 8: *(__u64 *)ext_val = value; break; 1652 default: 1653 return -EINVAL; 1654 } 1655 ext->is_set = true; 1656 return 0; 1657 } 1658 1659 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1660 char *buf, void *data) 1661 { 1662 struct extern_desc *ext; 1663 char *sep, *value; 1664 int len, err = 0; 1665 void *ext_val; 1666 __u64 num; 1667 1668 if (strncmp(buf, "CONFIG_", 7)) 1669 return 0; 1670 1671 sep = strchr(buf, '='); 1672 if (!sep) { 1673 pr_warn("failed to parse '%s': no separator\n", buf); 1674 return -EINVAL; 1675 } 1676 1677 /* Trim ending '\n' */ 1678 len = strlen(buf); 1679 if (buf[len - 1] == '\n') 1680 buf[len - 1] = '\0'; 1681 /* Split on '=' and ensure that a value is present. */ 1682 *sep = '\0'; 1683 if (!sep[1]) { 1684 *sep = '='; 1685 pr_warn("failed to parse '%s': no value\n", buf); 1686 return -EINVAL; 1687 } 1688 1689 ext = find_extern_by_name(obj, buf); 1690 if (!ext || ext->is_set) 1691 return 0; 1692 1693 ext_val = data + ext->kcfg.data_off; 1694 value = sep + 1; 1695 1696 switch (*value) { 1697 case 'y': case 'n': case 'm': 1698 err = set_kcfg_value_tri(ext, ext_val, *value); 1699 break; 1700 case '"': 1701 err = set_kcfg_value_str(ext, ext_val, value); 1702 break; 1703 default: 1704 /* assume integer */ 1705 err = parse_u64(value, &num); 1706 if (err) { 1707 pr_warn("extern (kcfg) %s=%s should be integer\n", 1708 ext->name, value); 1709 return err; 1710 } 1711 err = set_kcfg_value_num(ext, ext_val, num); 1712 break; 1713 } 1714 if (err) 1715 return err; 1716 pr_debug("extern (kcfg) %s=%s\n", ext->name, value); 1717 return 0; 1718 } 1719 1720 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1721 { 1722 char buf[PATH_MAX]; 1723 struct utsname uts; 1724 int len, err = 0; 1725 gzFile file; 1726 1727 uname(&uts); 1728 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1729 if (len < 0) 1730 return -EINVAL; 1731 else if (len >= PATH_MAX) 1732 return -ENAMETOOLONG; 1733 1734 /* gzopen also accepts uncompressed files. */ 1735 file = gzopen(buf, "r"); 1736 if (!file) 1737 file = gzopen("/proc/config.gz", "r"); 1738 1739 if (!file) { 1740 pr_warn("failed to open system Kconfig\n"); 1741 return -ENOENT; 1742 } 1743 1744 while (gzgets(file, buf, sizeof(buf))) { 1745 err = bpf_object__process_kconfig_line(obj, buf, data); 1746 if (err) { 1747 pr_warn("error parsing system Kconfig line '%s': %d\n", 1748 buf, err); 1749 goto out; 1750 } 1751 } 1752 1753 out: 1754 gzclose(file); 1755 return err; 1756 } 1757 1758 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 1759 const char *config, void *data) 1760 { 1761 char buf[PATH_MAX]; 1762 int err = 0; 1763 FILE *file; 1764 1765 file = fmemopen((void *)config, strlen(config), "r"); 1766 if (!file) { 1767 err = -errno; 1768 pr_warn("failed to open in-memory Kconfig: %d\n", err); 1769 return err; 1770 } 1771 1772 while (fgets(buf, sizeof(buf), file)) { 1773 err = bpf_object__process_kconfig_line(obj, buf, data); 1774 if (err) { 1775 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 1776 buf, err); 1777 break; 1778 } 1779 } 1780 1781 fclose(file); 1782 return err; 1783 } 1784 1785 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 1786 { 1787 struct extern_desc *last_ext = NULL, *ext; 1788 size_t map_sz; 1789 int i, err; 1790 1791 for (i = 0; i < obj->nr_extern; i++) { 1792 ext = &obj->externs[i]; 1793 if (ext->type == EXT_KCFG) 1794 last_ext = ext; 1795 } 1796 1797 if (!last_ext) 1798 return 0; 1799 1800 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 1801 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 1802 obj->efile.symbols_shndx, 1803 NULL, map_sz); 1804 if (err) 1805 return err; 1806 1807 obj->kconfig_map_idx = obj->nr_maps - 1; 1808 1809 return 0; 1810 } 1811 1812 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 1813 { 1814 Elf_Data *symbols = obj->efile.symbols; 1815 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 1816 Elf_Data *data = NULL; 1817 Elf_Scn *scn; 1818 1819 if (obj->efile.maps_shndx < 0) 1820 return 0; 1821 1822 if (!symbols) 1823 return -EINVAL; 1824 1825 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx); 1826 data = elf_sec_data(obj, scn); 1827 if (!scn || !data) { 1828 pr_warn("elf: failed to get legacy map definitions for %s\n", 1829 obj->path); 1830 return -EINVAL; 1831 } 1832 1833 /* 1834 * Count number of maps. Each map has a name. 1835 * Array of maps is not supported: only the first element is 1836 * considered. 1837 * 1838 * TODO: Detect array of map and report error. 1839 */ 1840 nr_syms = symbols->d_size / sizeof(GElf_Sym); 1841 for (i = 0; i < nr_syms; i++) { 1842 GElf_Sym sym; 1843 1844 if (!gelf_getsym(symbols, i, &sym)) 1845 continue; 1846 if (sym.st_shndx != obj->efile.maps_shndx) 1847 continue; 1848 nr_maps++; 1849 } 1850 /* Assume equally sized map definitions */ 1851 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n", 1852 nr_maps, data->d_size, obj->path); 1853 1854 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { 1855 pr_warn("elf: unable to determine legacy map definition size in %s\n", 1856 obj->path); 1857 return -EINVAL; 1858 } 1859 map_def_sz = data->d_size / nr_maps; 1860 1861 /* Fill obj->maps using data in "maps" section. */ 1862 for (i = 0; i < nr_syms; i++) { 1863 GElf_Sym sym; 1864 const char *map_name; 1865 struct bpf_map_def *def; 1866 struct bpf_map *map; 1867 1868 if (!gelf_getsym(symbols, i, &sym)) 1869 continue; 1870 if (sym.st_shndx != obj->efile.maps_shndx) 1871 continue; 1872 1873 map = bpf_object__add_map(obj); 1874 if (IS_ERR(map)) 1875 return PTR_ERR(map); 1876 1877 map_name = elf_sym_str(obj, sym.st_name); 1878 if (!map_name) { 1879 pr_warn("failed to get map #%d name sym string for obj %s\n", 1880 i, obj->path); 1881 return -LIBBPF_ERRNO__FORMAT; 1882 } 1883 1884 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION 1885 || GELF_ST_BIND(sym.st_info) == STB_LOCAL) { 1886 pr_warn("map '%s' (legacy): static maps are not supported\n", map_name); 1887 return -ENOTSUP; 1888 } 1889 1890 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1891 map->sec_idx = sym.st_shndx; 1892 map->sec_offset = sym.st_value; 1893 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 1894 map_name, map->sec_idx, map->sec_offset); 1895 if (sym.st_value + map_def_sz > data->d_size) { 1896 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", 1897 obj->path, map_name); 1898 return -EINVAL; 1899 } 1900 1901 map->name = strdup(map_name); 1902 if (!map->name) { 1903 pr_warn("failed to alloc map name\n"); 1904 return -ENOMEM; 1905 } 1906 pr_debug("map %d is \"%s\"\n", i, map->name); 1907 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 1908 /* 1909 * If the definition of the map in the object file fits in 1910 * bpf_map_def, copy it. Any extra fields in our version 1911 * of bpf_map_def will default to zero as a result of the 1912 * calloc above. 1913 */ 1914 if (map_def_sz <= sizeof(struct bpf_map_def)) { 1915 memcpy(&map->def, def, map_def_sz); 1916 } else { 1917 /* 1918 * Here the map structure being read is bigger than what 1919 * we expect, truncate if the excess bits are all zero. 1920 * If they are not zero, reject this map as 1921 * incompatible. 1922 */ 1923 char *b; 1924 1925 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1926 b < ((char *)def) + map_def_sz; b++) { 1927 if (*b != 0) { 1928 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", 1929 obj->path, map_name); 1930 if (strict) 1931 return -EINVAL; 1932 } 1933 } 1934 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1935 } 1936 } 1937 return 0; 1938 } 1939 1940 const struct btf_type * 1941 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 1942 { 1943 const struct btf_type *t = btf__type_by_id(btf, id); 1944 1945 if (res_id) 1946 *res_id = id; 1947 1948 while (btf_is_mod(t) || btf_is_typedef(t)) { 1949 if (res_id) 1950 *res_id = t->type; 1951 t = btf__type_by_id(btf, t->type); 1952 } 1953 1954 return t; 1955 } 1956 1957 static const struct btf_type * 1958 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 1959 { 1960 const struct btf_type *t; 1961 1962 t = skip_mods_and_typedefs(btf, id, NULL); 1963 if (!btf_is_ptr(t)) 1964 return NULL; 1965 1966 t = skip_mods_and_typedefs(btf, t->type, res_id); 1967 1968 return btf_is_func_proto(t) ? t : NULL; 1969 } 1970 1971 static const char *__btf_kind_str(__u16 kind) 1972 { 1973 switch (kind) { 1974 case BTF_KIND_UNKN: return "void"; 1975 case BTF_KIND_INT: return "int"; 1976 case BTF_KIND_PTR: return "ptr"; 1977 case BTF_KIND_ARRAY: return "array"; 1978 case BTF_KIND_STRUCT: return "struct"; 1979 case BTF_KIND_UNION: return "union"; 1980 case BTF_KIND_ENUM: return "enum"; 1981 case BTF_KIND_FWD: return "fwd"; 1982 case BTF_KIND_TYPEDEF: return "typedef"; 1983 case BTF_KIND_VOLATILE: return "volatile"; 1984 case BTF_KIND_CONST: return "const"; 1985 case BTF_KIND_RESTRICT: return "restrict"; 1986 case BTF_KIND_FUNC: return "func"; 1987 case BTF_KIND_FUNC_PROTO: return "func_proto"; 1988 case BTF_KIND_VAR: return "var"; 1989 case BTF_KIND_DATASEC: return "datasec"; 1990 case BTF_KIND_FLOAT: return "float"; 1991 case BTF_KIND_TAG: return "tag"; 1992 default: return "unknown"; 1993 } 1994 } 1995 1996 const char *btf_kind_str(const struct btf_type *t) 1997 { 1998 return __btf_kind_str(btf_kind(t)); 1999 } 2000 2001 /* 2002 * Fetch integer attribute of BTF map definition. Such attributes are 2003 * represented using a pointer to an array, in which dimensionality of array 2004 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2005 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2006 * type definition, while using only sizeof(void *) space in ELF data section. 2007 */ 2008 static bool get_map_field_int(const char *map_name, const struct btf *btf, 2009 const struct btf_member *m, __u32 *res) 2010 { 2011 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2012 const char *name = btf__name_by_offset(btf, m->name_off); 2013 const struct btf_array *arr_info; 2014 const struct btf_type *arr_t; 2015 2016 if (!btf_is_ptr(t)) { 2017 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2018 map_name, name, btf_kind_str(t)); 2019 return false; 2020 } 2021 2022 arr_t = btf__type_by_id(btf, t->type); 2023 if (!arr_t) { 2024 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2025 map_name, name, t->type); 2026 return false; 2027 } 2028 if (!btf_is_array(arr_t)) { 2029 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2030 map_name, name, btf_kind_str(arr_t)); 2031 return false; 2032 } 2033 arr_info = btf_array(arr_t); 2034 *res = arr_info->nelems; 2035 return true; 2036 } 2037 2038 static int build_map_pin_path(struct bpf_map *map, const char *path) 2039 { 2040 char buf[PATH_MAX]; 2041 int len; 2042 2043 if (!path) 2044 path = "/sys/fs/bpf"; 2045 2046 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); 2047 if (len < 0) 2048 return -EINVAL; 2049 else if (len >= PATH_MAX) 2050 return -ENAMETOOLONG; 2051 2052 return bpf_map__set_pin_path(map, buf); 2053 } 2054 2055 int parse_btf_map_def(const char *map_name, struct btf *btf, 2056 const struct btf_type *def_t, bool strict, 2057 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2058 { 2059 const struct btf_type *t; 2060 const struct btf_member *m; 2061 bool is_inner = inner_def == NULL; 2062 int vlen, i; 2063 2064 vlen = btf_vlen(def_t); 2065 m = btf_members(def_t); 2066 for (i = 0; i < vlen; i++, m++) { 2067 const char *name = btf__name_by_offset(btf, m->name_off); 2068 2069 if (!name) { 2070 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2071 return -EINVAL; 2072 } 2073 if (strcmp(name, "type") == 0) { 2074 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2075 return -EINVAL; 2076 map_def->parts |= MAP_DEF_MAP_TYPE; 2077 } else if (strcmp(name, "max_entries") == 0) { 2078 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2079 return -EINVAL; 2080 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2081 } else if (strcmp(name, "map_flags") == 0) { 2082 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2083 return -EINVAL; 2084 map_def->parts |= MAP_DEF_MAP_FLAGS; 2085 } else if (strcmp(name, "numa_node") == 0) { 2086 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2087 return -EINVAL; 2088 map_def->parts |= MAP_DEF_NUMA_NODE; 2089 } else if (strcmp(name, "key_size") == 0) { 2090 __u32 sz; 2091 2092 if (!get_map_field_int(map_name, btf, m, &sz)) 2093 return -EINVAL; 2094 if (map_def->key_size && map_def->key_size != sz) { 2095 pr_warn("map '%s': conflicting key size %u != %u.\n", 2096 map_name, map_def->key_size, sz); 2097 return -EINVAL; 2098 } 2099 map_def->key_size = sz; 2100 map_def->parts |= MAP_DEF_KEY_SIZE; 2101 } else if (strcmp(name, "key") == 0) { 2102 __s64 sz; 2103 2104 t = btf__type_by_id(btf, m->type); 2105 if (!t) { 2106 pr_warn("map '%s': key type [%d] not found.\n", 2107 map_name, m->type); 2108 return -EINVAL; 2109 } 2110 if (!btf_is_ptr(t)) { 2111 pr_warn("map '%s': key spec is not PTR: %s.\n", 2112 map_name, btf_kind_str(t)); 2113 return -EINVAL; 2114 } 2115 sz = btf__resolve_size(btf, t->type); 2116 if (sz < 0) { 2117 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2118 map_name, t->type, (ssize_t)sz); 2119 return sz; 2120 } 2121 if (map_def->key_size && map_def->key_size != sz) { 2122 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2123 map_name, map_def->key_size, (ssize_t)sz); 2124 return -EINVAL; 2125 } 2126 map_def->key_size = sz; 2127 map_def->key_type_id = t->type; 2128 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2129 } else if (strcmp(name, "value_size") == 0) { 2130 __u32 sz; 2131 2132 if (!get_map_field_int(map_name, btf, m, &sz)) 2133 return -EINVAL; 2134 if (map_def->value_size && map_def->value_size != sz) { 2135 pr_warn("map '%s': conflicting value size %u != %u.\n", 2136 map_name, map_def->value_size, sz); 2137 return -EINVAL; 2138 } 2139 map_def->value_size = sz; 2140 map_def->parts |= MAP_DEF_VALUE_SIZE; 2141 } else if (strcmp(name, "value") == 0) { 2142 __s64 sz; 2143 2144 t = btf__type_by_id(btf, m->type); 2145 if (!t) { 2146 pr_warn("map '%s': value type [%d] not found.\n", 2147 map_name, m->type); 2148 return -EINVAL; 2149 } 2150 if (!btf_is_ptr(t)) { 2151 pr_warn("map '%s': value spec is not PTR: %s.\n", 2152 map_name, btf_kind_str(t)); 2153 return -EINVAL; 2154 } 2155 sz = btf__resolve_size(btf, t->type); 2156 if (sz < 0) { 2157 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2158 map_name, t->type, (ssize_t)sz); 2159 return sz; 2160 } 2161 if (map_def->value_size && map_def->value_size != sz) { 2162 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2163 map_name, map_def->value_size, (ssize_t)sz); 2164 return -EINVAL; 2165 } 2166 map_def->value_size = sz; 2167 map_def->value_type_id = t->type; 2168 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2169 } 2170 else if (strcmp(name, "values") == 0) { 2171 char inner_map_name[128]; 2172 int err; 2173 2174 if (is_inner) { 2175 pr_warn("map '%s': multi-level inner maps not supported.\n", 2176 map_name); 2177 return -ENOTSUP; 2178 } 2179 if (i != vlen - 1) { 2180 pr_warn("map '%s': '%s' member should be last.\n", 2181 map_name, name); 2182 return -EINVAL; 2183 } 2184 if (!bpf_map_type__is_map_in_map(map_def->map_type)) { 2185 pr_warn("map '%s': should be map-in-map.\n", 2186 map_name); 2187 return -ENOTSUP; 2188 } 2189 if (map_def->value_size && map_def->value_size != 4) { 2190 pr_warn("map '%s': conflicting value size %u != 4.\n", 2191 map_name, map_def->value_size); 2192 return -EINVAL; 2193 } 2194 map_def->value_size = 4; 2195 t = btf__type_by_id(btf, m->type); 2196 if (!t) { 2197 pr_warn("map '%s': map-in-map inner type [%d] not found.\n", 2198 map_name, m->type); 2199 return -EINVAL; 2200 } 2201 if (!btf_is_array(t) || btf_array(t)->nelems) { 2202 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n", 2203 map_name); 2204 return -EINVAL; 2205 } 2206 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2207 if (!btf_is_ptr(t)) { 2208 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2209 map_name, btf_kind_str(t)); 2210 return -EINVAL; 2211 } 2212 t = skip_mods_and_typedefs(btf, t->type, NULL); 2213 if (!btf_is_struct(t)) { 2214 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2215 map_name, btf_kind_str(t)); 2216 return -EINVAL; 2217 } 2218 2219 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2220 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2221 if (err) 2222 return err; 2223 2224 map_def->parts |= MAP_DEF_INNER_MAP; 2225 } else if (strcmp(name, "pinning") == 0) { 2226 __u32 val; 2227 2228 if (is_inner) { 2229 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2230 return -EINVAL; 2231 } 2232 if (!get_map_field_int(map_name, btf, m, &val)) 2233 return -EINVAL; 2234 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2235 pr_warn("map '%s': invalid pinning value %u.\n", 2236 map_name, val); 2237 return -EINVAL; 2238 } 2239 map_def->pinning = val; 2240 map_def->parts |= MAP_DEF_PINNING; 2241 } else { 2242 if (strict) { 2243 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2244 return -ENOTSUP; 2245 } 2246 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2247 } 2248 } 2249 2250 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2251 pr_warn("map '%s': map type isn't specified.\n", map_name); 2252 return -EINVAL; 2253 } 2254 2255 return 0; 2256 } 2257 2258 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2259 { 2260 map->def.type = def->map_type; 2261 map->def.key_size = def->key_size; 2262 map->def.value_size = def->value_size; 2263 map->def.max_entries = def->max_entries; 2264 map->def.map_flags = def->map_flags; 2265 2266 map->numa_node = def->numa_node; 2267 map->btf_key_type_id = def->key_type_id; 2268 map->btf_value_type_id = def->value_type_id; 2269 2270 if (def->parts & MAP_DEF_MAP_TYPE) 2271 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2272 2273 if (def->parts & MAP_DEF_KEY_TYPE) 2274 pr_debug("map '%s': found key [%u], sz = %u.\n", 2275 map->name, def->key_type_id, def->key_size); 2276 else if (def->parts & MAP_DEF_KEY_SIZE) 2277 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2278 2279 if (def->parts & MAP_DEF_VALUE_TYPE) 2280 pr_debug("map '%s': found value [%u], sz = %u.\n", 2281 map->name, def->value_type_id, def->value_size); 2282 else if (def->parts & MAP_DEF_VALUE_SIZE) 2283 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2284 2285 if (def->parts & MAP_DEF_MAX_ENTRIES) 2286 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2287 if (def->parts & MAP_DEF_MAP_FLAGS) 2288 pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags); 2289 if (def->parts & MAP_DEF_PINNING) 2290 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2291 if (def->parts & MAP_DEF_NUMA_NODE) 2292 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2293 2294 if (def->parts & MAP_DEF_INNER_MAP) 2295 pr_debug("map '%s': found inner map definition.\n", map->name); 2296 } 2297 2298 static const char *btf_var_linkage_str(__u32 linkage) 2299 { 2300 switch (linkage) { 2301 case BTF_VAR_STATIC: return "static"; 2302 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2303 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2304 default: return "unknown"; 2305 } 2306 } 2307 2308 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2309 const struct btf_type *sec, 2310 int var_idx, int sec_idx, 2311 const Elf_Data *data, bool strict, 2312 const char *pin_root_path) 2313 { 2314 struct btf_map_def map_def = {}, inner_def = {}; 2315 const struct btf_type *var, *def; 2316 const struct btf_var_secinfo *vi; 2317 const struct btf_var *var_extra; 2318 const char *map_name; 2319 struct bpf_map *map; 2320 int err; 2321 2322 vi = btf_var_secinfos(sec) + var_idx; 2323 var = btf__type_by_id(obj->btf, vi->type); 2324 var_extra = btf_var(var); 2325 map_name = btf__name_by_offset(obj->btf, var->name_off); 2326 2327 if (map_name == NULL || map_name[0] == '\0') { 2328 pr_warn("map #%d: empty name.\n", var_idx); 2329 return -EINVAL; 2330 } 2331 if ((__u64)vi->offset + vi->size > data->d_size) { 2332 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2333 return -EINVAL; 2334 } 2335 if (!btf_is_var(var)) { 2336 pr_warn("map '%s': unexpected var kind %s.\n", 2337 map_name, btf_kind_str(var)); 2338 return -EINVAL; 2339 } 2340 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2341 pr_warn("map '%s': unsupported map linkage %s.\n", 2342 map_name, btf_var_linkage_str(var_extra->linkage)); 2343 return -EOPNOTSUPP; 2344 } 2345 2346 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2347 if (!btf_is_struct(def)) { 2348 pr_warn("map '%s': unexpected def kind %s.\n", 2349 map_name, btf_kind_str(var)); 2350 return -EINVAL; 2351 } 2352 if (def->size > vi->size) { 2353 pr_warn("map '%s': invalid def size.\n", map_name); 2354 return -EINVAL; 2355 } 2356 2357 map = bpf_object__add_map(obj); 2358 if (IS_ERR(map)) 2359 return PTR_ERR(map); 2360 map->name = strdup(map_name); 2361 if (!map->name) { 2362 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2363 return -ENOMEM; 2364 } 2365 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2366 map->def.type = BPF_MAP_TYPE_UNSPEC; 2367 map->sec_idx = sec_idx; 2368 map->sec_offset = vi->offset; 2369 map->btf_var_idx = var_idx; 2370 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2371 map_name, map->sec_idx, map->sec_offset); 2372 2373 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2374 if (err) 2375 return err; 2376 2377 fill_map_from_def(map, &map_def); 2378 2379 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2380 err = build_map_pin_path(map, pin_root_path); 2381 if (err) { 2382 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2383 return err; 2384 } 2385 } 2386 2387 if (map_def.parts & MAP_DEF_INNER_MAP) { 2388 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2389 if (!map->inner_map) 2390 return -ENOMEM; 2391 map->inner_map->fd = -1; 2392 map->inner_map->sec_idx = sec_idx; 2393 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2394 if (!map->inner_map->name) 2395 return -ENOMEM; 2396 sprintf(map->inner_map->name, "%s.inner", map_name); 2397 2398 fill_map_from_def(map->inner_map, &inner_def); 2399 } 2400 2401 return 0; 2402 } 2403 2404 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2405 const char *pin_root_path) 2406 { 2407 const struct btf_type *sec = NULL; 2408 int nr_types, i, vlen, err; 2409 const struct btf_type *t; 2410 const char *name; 2411 Elf_Data *data; 2412 Elf_Scn *scn; 2413 2414 if (obj->efile.btf_maps_shndx < 0) 2415 return 0; 2416 2417 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2418 data = elf_sec_data(obj, scn); 2419 if (!scn || !data) { 2420 pr_warn("elf: failed to get %s map definitions for %s\n", 2421 MAPS_ELF_SEC, obj->path); 2422 return -EINVAL; 2423 } 2424 2425 nr_types = btf__get_nr_types(obj->btf); 2426 for (i = 1; i <= nr_types; i++) { 2427 t = btf__type_by_id(obj->btf, i); 2428 if (!btf_is_datasec(t)) 2429 continue; 2430 name = btf__name_by_offset(obj->btf, t->name_off); 2431 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2432 sec = t; 2433 obj->efile.btf_maps_sec_btf_id = i; 2434 break; 2435 } 2436 } 2437 2438 if (!sec) { 2439 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2440 return -ENOENT; 2441 } 2442 2443 vlen = btf_vlen(sec); 2444 for (i = 0; i < vlen; i++) { 2445 err = bpf_object__init_user_btf_map(obj, sec, i, 2446 obj->efile.btf_maps_shndx, 2447 data, strict, 2448 pin_root_path); 2449 if (err) 2450 return err; 2451 } 2452 2453 return 0; 2454 } 2455 2456 static int bpf_object__init_maps(struct bpf_object *obj, 2457 const struct bpf_object_open_opts *opts) 2458 { 2459 const char *pin_root_path; 2460 bool strict; 2461 int err; 2462 2463 strict = !OPTS_GET(opts, relaxed_maps, false); 2464 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2465 2466 err = bpf_object__init_user_maps(obj, strict); 2467 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2468 err = err ?: bpf_object__init_global_data_maps(obj); 2469 err = err ?: bpf_object__init_kconfig_map(obj); 2470 err = err ?: bpf_object__init_struct_ops_maps(obj); 2471 2472 return err; 2473 } 2474 2475 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2476 { 2477 GElf_Shdr sh; 2478 2479 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh)) 2480 return false; 2481 2482 return sh.sh_flags & SHF_EXECINSTR; 2483 } 2484 2485 static bool btf_needs_sanitization(struct bpf_object *obj) 2486 { 2487 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2488 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2489 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2490 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2491 bool has_tag = kernel_supports(obj, FEAT_BTF_TAG); 2492 2493 return !has_func || !has_datasec || !has_func_global || !has_float || !has_tag; 2494 } 2495 2496 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2497 { 2498 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2499 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2500 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2501 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2502 bool has_tag = kernel_supports(obj, FEAT_BTF_TAG); 2503 struct btf_type *t; 2504 int i, j, vlen; 2505 2506 for (i = 1; i <= btf__get_nr_types(btf); i++) { 2507 t = (struct btf_type *)btf__type_by_id(btf, i); 2508 2509 if ((!has_datasec && btf_is_var(t)) || (!has_tag && btf_is_tag(t))) { 2510 /* replace VAR/TAG with INT */ 2511 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2512 /* 2513 * using size = 1 is the safest choice, 4 will be too 2514 * big and cause kernel BTF validation failure if 2515 * original variable took less than 4 bytes 2516 */ 2517 t->size = 1; 2518 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2519 } else if (!has_datasec && btf_is_datasec(t)) { 2520 /* replace DATASEC with STRUCT */ 2521 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2522 struct btf_member *m = btf_members(t); 2523 struct btf_type *vt; 2524 char *name; 2525 2526 name = (char *)btf__name_by_offset(btf, t->name_off); 2527 while (*name) { 2528 if (*name == '.') 2529 *name = '_'; 2530 name++; 2531 } 2532 2533 vlen = btf_vlen(t); 2534 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2535 for (j = 0; j < vlen; j++, v++, m++) { 2536 /* order of field assignments is important */ 2537 m->offset = v->offset * 8; 2538 m->type = v->type; 2539 /* preserve variable name as member name */ 2540 vt = (void *)btf__type_by_id(btf, v->type); 2541 m->name_off = vt->name_off; 2542 } 2543 } else if (!has_func && btf_is_func_proto(t)) { 2544 /* replace FUNC_PROTO with ENUM */ 2545 vlen = btf_vlen(t); 2546 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2547 t->size = sizeof(__u32); /* kernel enforced */ 2548 } else if (!has_func && btf_is_func(t)) { 2549 /* replace FUNC with TYPEDEF */ 2550 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2551 } else if (!has_func_global && btf_is_func(t)) { 2552 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2553 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2554 } else if (!has_float && btf_is_float(t)) { 2555 /* replace FLOAT with an equally-sized empty STRUCT; 2556 * since C compilers do not accept e.g. "float" as a 2557 * valid struct name, make it anonymous 2558 */ 2559 t->name_off = 0; 2560 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2561 } 2562 } 2563 } 2564 2565 static bool libbpf_needs_btf(const struct bpf_object *obj) 2566 { 2567 return obj->efile.btf_maps_shndx >= 0 || 2568 obj->efile.st_ops_shndx >= 0 || 2569 obj->nr_extern > 0; 2570 } 2571 2572 static bool kernel_needs_btf(const struct bpf_object *obj) 2573 { 2574 return obj->efile.st_ops_shndx >= 0; 2575 } 2576 2577 static int bpf_object__init_btf(struct bpf_object *obj, 2578 Elf_Data *btf_data, 2579 Elf_Data *btf_ext_data) 2580 { 2581 int err = -ENOENT; 2582 2583 if (btf_data) { 2584 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2585 err = libbpf_get_error(obj->btf); 2586 if (err) { 2587 obj->btf = NULL; 2588 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); 2589 goto out; 2590 } 2591 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2592 btf__set_pointer_size(obj->btf, 8); 2593 } 2594 if (btf_ext_data) { 2595 if (!obj->btf) { 2596 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2597 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2598 goto out; 2599 } 2600 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 2601 err = libbpf_get_error(obj->btf_ext); 2602 if (err) { 2603 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", 2604 BTF_EXT_ELF_SEC, err); 2605 obj->btf_ext = NULL; 2606 goto out; 2607 } 2608 } 2609 out: 2610 if (err && libbpf_needs_btf(obj)) { 2611 pr_warn("BTF is required, but is missing or corrupted.\n"); 2612 return err; 2613 } 2614 return 0; 2615 } 2616 2617 static int bpf_object__finalize_btf(struct bpf_object *obj) 2618 { 2619 int err; 2620 2621 if (!obj->btf) 2622 return 0; 2623 2624 err = btf__finalize_data(obj, obj->btf); 2625 if (err) { 2626 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); 2627 return err; 2628 } 2629 2630 return 0; 2631 } 2632 2633 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 2634 { 2635 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 2636 prog->type == BPF_PROG_TYPE_LSM) 2637 return true; 2638 2639 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 2640 * also need vmlinux BTF 2641 */ 2642 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 2643 return true; 2644 2645 return false; 2646 } 2647 2648 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 2649 { 2650 struct bpf_program *prog; 2651 int i; 2652 2653 /* CO-RE relocations need kernel BTF, only when btf_custom_path 2654 * is not specified 2655 */ 2656 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 2657 return true; 2658 2659 /* Support for typed ksyms needs kernel BTF */ 2660 for (i = 0; i < obj->nr_extern; i++) { 2661 const struct extern_desc *ext; 2662 2663 ext = &obj->externs[i]; 2664 if (ext->type == EXT_KSYM && ext->ksym.type_id) 2665 return true; 2666 } 2667 2668 bpf_object__for_each_program(prog, obj) { 2669 if (!prog->load) 2670 continue; 2671 if (prog_needs_vmlinux_btf(prog)) 2672 return true; 2673 } 2674 2675 return false; 2676 } 2677 2678 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 2679 { 2680 int err; 2681 2682 /* btf_vmlinux could be loaded earlier */ 2683 if (obj->btf_vmlinux || obj->gen_loader) 2684 return 0; 2685 2686 if (!force && !obj_needs_vmlinux_btf(obj)) 2687 return 0; 2688 2689 obj->btf_vmlinux = btf__load_vmlinux_btf(); 2690 err = libbpf_get_error(obj->btf_vmlinux); 2691 if (err) { 2692 pr_warn("Error loading vmlinux BTF: %d\n", err); 2693 obj->btf_vmlinux = NULL; 2694 return err; 2695 } 2696 return 0; 2697 } 2698 2699 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 2700 { 2701 struct btf *kern_btf = obj->btf; 2702 bool btf_mandatory, sanitize; 2703 int i, err = 0; 2704 2705 if (!obj->btf) 2706 return 0; 2707 2708 if (!kernel_supports(obj, FEAT_BTF)) { 2709 if (kernel_needs_btf(obj)) { 2710 err = -EOPNOTSUPP; 2711 goto report; 2712 } 2713 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 2714 return 0; 2715 } 2716 2717 /* Even though some subprogs are global/weak, user might prefer more 2718 * permissive BPF verification process that BPF verifier performs for 2719 * static functions, taking into account more context from the caller 2720 * functions. In such case, they need to mark such subprogs with 2721 * __attribute__((visibility("hidden"))) and libbpf will adjust 2722 * corresponding FUNC BTF type to be marked as static and trigger more 2723 * involved BPF verification process. 2724 */ 2725 for (i = 0; i < obj->nr_programs; i++) { 2726 struct bpf_program *prog = &obj->programs[i]; 2727 struct btf_type *t; 2728 const char *name; 2729 int j, n; 2730 2731 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 2732 continue; 2733 2734 n = btf__get_nr_types(obj->btf); 2735 for (j = 1; j <= n; j++) { 2736 t = btf_type_by_id(obj->btf, j); 2737 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 2738 continue; 2739 2740 name = btf__str_by_offset(obj->btf, t->name_off); 2741 if (strcmp(name, prog->name) != 0) 2742 continue; 2743 2744 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 2745 break; 2746 } 2747 } 2748 2749 sanitize = btf_needs_sanitization(obj); 2750 if (sanitize) { 2751 const void *raw_data; 2752 __u32 sz; 2753 2754 /* clone BTF to sanitize a copy and leave the original intact */ 2755 raw_data = btf__get_raw_data(obj->btf, &sz); 2756 kern_btf = btf__new(raw_data, sz); 2757 err = libbpf_get_error(kern_btf); 2758 if (err) 2759 return err; 2760 2761 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2762 btf__set_pointer_size(obj->btf, 8); 2763 bpf_object__sanitize_btf(obj, kern_btf); 2764 } 2765 2766 if (obj->gen_loader) { 2767 __u32 raw_size = 0; 2768 const void *raw_data = btf__get_raw_data(kern_btf, &raw_size); 2769 2770 if (!raw_data) 2771 return -ENOMEM; 2772 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 2773 /* Pretend to have valid FD to pass various fd >= 0 checks. 2774 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 2775 */ 2776 btf__set_fd(kern_btf, 0); 2777 } else { 2778 err = btf__load_into_kernel(kern_btf); 2779 } 2780 if (sanitize) { 2781 if (!err) { 2782 /* move fd to libbpf's BTF */ 2783 btf__set_fd(obj->btf, btf__fd(kern_btf)); 2784 btf__set_fd(kern_btf, -1); 2785 } 2786 btf__free(kern_btf); 2787 } 2788 report: 2789 if (err) { 2790 btf_mandatory = kernel_needs_btf(obj); 2791 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 2792 btf_mandatory ? "BTF is mandatory, can't proceed." 2793 : "BTF is optional, ignoring."); 2794 if (!btf_mandatory) 2795 err = 0; 2796 } 2797 return err; 2798 } 2799 2800 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 2801 { 2802 const char *name; 2803 2804 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 2805 if (!name) { 2806 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2807 off, obj->path, elf_errmsg(-1)); 2808 return NULL; 2809 } 2810 2811 return name; 2812 } 2813 2814 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 2815 { 2816 const char *name; 2817 2818 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 2819 if (!name) { 2820 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2821 off, obj->path, elf_errmsg(-1)); 2822 return NULL; 2823 } 2824 2825 return name; 2826 } 2827 2828 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 2829 { 2830 Elf_Scn *scn; 2831 2832 scn = elf_getscn(obj->efile.elf, idx); 2833 if (!scn) { 2834 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 2835 idx, obj->path, elf_errmsg(-1)); 2836 return NULL; 2837 } 2838 return scn; 2839 } 2840 2841 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 2842 { 2843 Elf_Scn *scn = NULL; 2844 Elf *elf = obj->efile.elf; 2845 const char *sec_name; 2846 2847 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2848 sec_name = elf_sec_name(obj, scn); 2849 if (!sec_name) 2850 return NULL; 2851 2852 if (strcmp(sec_name, name) != 0) 2853 continue; 2854 2855 return scn; 2856 } 2857 return NULL; 2858 } 2859 2860 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr) 2861 { 2862 if (!scn) 2863 return -EINVAL; 2864 2865 if (gelf_getshdr(scn, hdr) != hdr) { 2866 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 2867 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2868 return -EINVAL; 2869 } 2870 2871 return 0; 2872 } 2873 2874 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 2875 { 2876 const char *name; 2877 GElf_Shdr sh; 2878 2879 if (!scn) 2880 return NULL; 2881 2882 if (elf_sec_hdr(obj, scn, &sh)) 2883 return NULL; 2884 2885 name = elf_sec_str(obj, sh.sh_name); 2886 if (!name) { 2887 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 2888 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2889 return NULL; 2890 } 2891 2892 return name; 2893 } 2894 2895 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 2896 { 2897 Elf_Data *data; 2898 2899 if (!scn) 2900 return NULL; 2901 2902 data = elf_getdata(scn, 0); 2903 if (!data) { 2904 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 2905 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 2906 obj->path, elf_errmsg(-1)); 2907 return NULL; 2908 } 2909 2910 return data; 2911 } 2912 2913 static bool is_sec_name_dwarf(const char *name) 2914 { 2915 /* approximation, but the actual list is too long */ 2916 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 2917 } 2918 2919 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name) 2920 { 2921 /* no special handling of .strtab */ 2922 if (hdr->sh_type == SHT_STRTAB) 2923 return true; 2924 2925 /* ignore .llvm_addrsig section as well */ 2926 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 2927 return true; 2928 2929 /* no subprograms will lead to an empty .text section, ignore it */ 2930 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 2931 strcmp(name, ".text") == 0) 2932 return true; 2933 2934 /* DWARF sections */ 2935 if (is_sec_name_dwarf(name)) 2936 return true; 2937 2938 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 2939 name += sizeof(".rel") - 1; 2940 /* DWARF section relocations */ 2941 if (is_sec_name_dwarf(name)) 2942 return true; 2943 2944 /* .BTF and .BTF.ext don't need relocations */ 2945 if (strcmp(name, BTF_ELF_SEC) == 0 || 2946 strcmp(name, BTF_EXT_ELF_SEC) == 0) 2947 return true; 2948 } 2949 2950 return false; 2951 } 2952 2953 static int cmp_progs(const void *_a, const void *_b) 2954 { 2955 const struct bpf_program *a = _a; 2956 const struct bpf_program *b = _b; 2957 2958 if (a->sec_idx != b->sec_idx) 2959 return a->sec_idx < b->sec_idx ? -1 : 1; 2960 2961 /* sec_insn_off can't be the same within the section */ 2962 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 2963 } 2964 2965 static int bpf_object__elf_collect(struct bpf_object *obj) 2966 { 2967 Elf *elf = obj->efile.elf; 2968 Elf_Data *btf_ext_data = NULL; 2969 Elf_Data *btf_data = NULL; 2970 int idx = 0, err = 0; 2971 const char *name; 2972 Elf_Data *data; 2973 Elf_Scn *scn; 2974 GElf_Shdr sh; 2975 2976 /* a bunch of ELF parsing functionality depends on processing symbols, 2977 * so do the first pass and find the symbol table 2978 */ 2979 scn = NULL; 2980 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2981 if (elf_sec_hdr(obj, scn, &sh)) 2982 return -LIBBPF_ERRNO__FORMAT; 2983 2984 if (sh.sh_type == SHT_SYMTAB) { 2985 if (obj->efile.symbols) { 2986 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 2987 return -LIBBPF_ERRNO__FORMAT; 2988 } 2989 2990 data = elf_sec_data(obj, scn); 2991 if (!data) 2992 return -LIBBPF_ERRNO__FORMAT; 2993 2994 obj->efile.symbols = data; 2995 obj->efile.symbols_shndx = elf_ndxscn(scn); 2996 obj->efile.strtabidx = sh.sh_link; 2997 } 2998 } 2999 3000 if (!obj->efile.symbols) { 3001 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n", 3002 obj->path); 3003 return -ENOENT; 3004 } 3005 3006 scn = NULL; 3007 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3008 idx++; 3009 3010 if (elf_sec_hdr(obj, scn, &sh)) 3011 return -LIBBPF_ERRNO__FORMAT; 3012 3013 name = elf_sec_str(obj, sh.sh_name); 3014 if (!name) 3015 return -LIBBPF_ERRNO__FORMAT; 3016 3017 if (ignore_elf_section(&sh, name)) 3018 continue; 3019 3020 data = elf_sec_data(obj, scn); 3021 if (!data) 3022 return -LIBBPF_ERRNO__FORMAT; 3023 3024 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 3025 idx, name, (unsigned long)data->d_size, 3026 (int)sh.sh_link, (unsigned long)sh.sh_flags, 3027 (int)sh.sh_type); 3028 3029 if (strcmp(name, "license") == 0) { 3030 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 3031 if (err) 3032 return err; 3033 } else if (strcmp(name, "version") == 0) { 3034 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 3035 if (err) 3036 return err; 3037 } else if (strcmp(name, "maps") == 0) { 3038 obj->efile.maps_shndx = idx; 3039 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 3040 obj->efile.btf_maps_shndx = idx; 3041 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 3042 btf_data = data; 3043 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 3044 btf_ext_data = data; 3045 } else if (sh.sh_type == SHT_SYMTAB) { 3046 /* already processed during the first pass above */ 3047 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 3048 if (sh.sh_flags & SHF_EXECINSTR) { 3049 if (strcmp(name, ".text") == 0) 3050 obj->efile.text_shndx = idx; 3051 err = bpf_object__add_programs(obj, data, name, idx); 3052 if (err) 3053 return err; 3054 } else if (strcmp(name, DATA_SEC) == 0) { 3055 obj->efile.data = data; 3056 obj->efile.data_shndx = idx; 3057 } else if (strcmp(name, RODATA_SEC) == 0) { 3058 obj->efile.rodata = data; 3059 obj->efile.rodata_shndx = idx; 3060 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 3061 obj->efile.st_ops_data = data; 3062 obj->efile.st_ops_shndx = idx; 3063 } else { 3064 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3065 idx, name); 3066 } 3067 } else if (sh.sh_type == SHT_REL) { 3068 int nr_sects = obj->efile.nr_reloc_sects; 3069 void *sects = obj->efile.reloc_sects; 3070 int sec = sh.sh_info; /* points to other section */ 3071 3072 /* Only do relo for section with exec instructions */ 3073 if (!section_have_execinstr(obj, sec) && 3074 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3075 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3076 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3077 idx, name, sec, 3078 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>"); 3079 continue; 3080 } 3081 3082 sects = libbpf_reallocarray(sects, nr_sects + 1, 3083 sizeof(*obj->efile.reloc_sects)); 3084 if (!sects) 3085 return -ENOMEM; 3086 3087 obj->efile.reloc_sects = sects; 3088 obj->efile.nr_reloc_sects++; 3089 3090 obj->efile.reloc_sects[nr_sects].shdr = sh; 3091 obj->efile.reloc_sects[nr_sects].data = data; 3092 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) { 3093 obj->efile.bss = data; 3094 obj->efile.bss_shndx = idx; 3095 } else { 3096 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3097 (size_t)sh.sh_size); 3098 } 3099 } 3100 3101 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3102 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3103 return -LIBBPF_ERRNO__FORMAT; 3104 } 3105 3106 /* sort BPF programs by section name and in-section instruction offset 3107 * for faster search */ 3108 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3109 3110 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3111 } 3112 3113 static bool sym_is_extern(const GElf_Sym *sym) 3114 { 3115 int bind = GELF_ST_BIND(sym->st_info); 3116 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3117 return sym->st_shndx == SHN_UNDEF && 3118 (bind == STB_GLOBAL || bind == STB_WEAK) && 3119 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 3120 } 3121 3122 static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) 3123 { 3124 int bind = GELF_ST_BIND(sym->st_info); 3125 int type = GELF_ST_TYPE(sym->st_info); 3126 3127 /* in .text section */ 3128 if (sym->st_shndx != text_shndx) 3129 return false; 3130 3131 /* local function */ 3132 if (bind == STB_LOCAL && type == STT_SECTION) 3133 return true; 3134 3135 /* global function */ 3136 return bind == STB_GLOBAL && type == STT_FUNC; 3137 } 3138 3139 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3140 { 3141 const struct btf_type *t; 3142 const char *tname; 3143 int i, n; 3144 3145 if (!btf) 3146 return -ESRCH; 3147 3148 n = btf__get_nr_types(btf); 3149 for (i = 1; i <= n; i++) { 3150 t = btf__type_by_id(btf, i); 3151 3152 if (!btf_is_var(t) && !btf_is_func(t)) 3153 continue; 3154 3155 tname = btf__name_by_offset(btf, t->name_off); 3156 if (strcmp(tname, ext_name)) 3157 continue; 3158 3159 if (btf_is_var(t) && 3160 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3161 return -EINVAL; 3162 3163 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3164 return -EINVAL; 3165 3166 return i; 3167 } 3168 3169 return -ENOENT; 3170 } 3171 3172 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 3173 const struct btf_var_secinfo *vs; 3174 const struct btf_type *t; 3175 int i, j, n; 3176 3177 if (!btf) 3178 return -ESRCH; 3179 3180 n = btf__get_nr_types(btf); 3181 for (i = 1; i <= n; i++) { 3182 t = btf__type_by_id(btf, i); 3183 3184 if (!btf_is_datasec(t)) 3185 continue; 3186 3187 vs = btf_var_secinfos(t); 3188 for (j = 0; j < btf_vlen(t); j++, vs++) { 3189 if (vs->type == ext_btf_id) 3190 return i; 3191 } 3192 } 3193 3194 return -ENOENT; 3195 } 3196 3197 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 3198 bool *is_signed) 3199 { 3200 const struct btf_type *t; 3201 const char *name; 3202 3203 t = skip_mods_and_typedefs(btf, id, NULL); 3204 name = btf__name_by_offset(btf, t->name_off); 3205 3206 if (is_signed) 3207 *is_signed = false; 3208 switch (btf_kind(t)) { 3209 case BTF_KIND_INT: { 3210 int enc = btf_int_encoding(t); 3211 3212 if (enc & BTF_INT_BOOL) 3213 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 3214 if (is_signed) 3215 *is_signed = enc & BTF_INT_SIGNED; 3216 if (t->size == 1) 3217 return KCFG_CHAR; 3218 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 3219 return KCFG_UNKNOWN; 3220 return KCFG_INT; 3221 } 3222 case BTF_KIND_ENUM: 3223 if (t->size != 4) 3224 return KCFG_UNKNOWN; 3225 if (strcmp(name, "libbpf_tristate")) 3226 return KCFG_UNKNOWN; 3227 return KCFG_TRISTATE; 3228 case BTF_KIND_ARRAY: 3229 if (btf_array(t)->nelems == 0) 3230 return KCFG_UNKNOWN; 3231 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 3232 return KCFG_UNKNOWN; 3233 return KCFG_CHAR_ARR; 3234 default: 3235 return KCFG_UNKNOWN; 3236 } 3237 } 3238 3239 static int cmp_externs(const void *_a, const void *_b) 3240 { 3241 const struct extern_desc *a = _a; 3242 const struct extern_desc *b = _b; 3243 3244 if (a->type != b->type) 3245 return a->type < b->type ? -1 : 1; 3246 3247 if (a->type == EXT_KCFG) { 3248 /* descending order by alignment requirements */ 3249 if (a->kcfg.align != b->kcfg.align) 3250 return a->kcfg.align > b->kcfg.align ? -1 : 1; 3251 /* ascending order by size, within same alignment class */ 3252 if (a->kcfg.sz != b->kcfg.sz) 3253 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 3254 } 3255 3256 /* resolve ties by name */ 3257 return strcmp(a->name, b->name); 3258 } 3259 3260 static int find_int_btf_id(const struct btf *btf) 3261 { 3262 const struct btf_type *t; 3263 int i, n; 3264 3265 n = btf__get_nr_types(btf); 3266 for (i = 1; i <= n; i++) { 3267 t = btf__type_by_id(btf, i); 3268 3269 if (btf_is_int(t) && btf_int_bits(t) == 32) 3270 return i; 3271 } 3272 3273 return 0; 3274 } 3275 3276 static int add_dummy_ksym_var(struct btf *btf) 3277 { 3278 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 3279 const struct btf_var_secinfo *vs; 3280 const struct btf_type *sec; 3281 3282 if (!btf) 3283 return 0; 3284 3285 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 3286 BTF_KIND_DATASEC); 3287 if (sec_btf_id < 0) 3288 return 0; 3289 3290 sec = btf__type_by_id(btf, sec_btf_id); 3291 vs = btf_var_secinfos(sec); 3292 for (i = 0; i < btf_vlen(sec); i++, vs++) { 3293 const struct btf_type *vt; 3294 3295 vt = btf__type_by_id(btf, vs->type); 3296 if (btf_is_func(vt)) 3297 break; 3298 } 3299 3300 /* No func in ksyms sec. No need to add dummy var. */ 3301 if (i == btf_vlen(sec)) 3302 return 0; 3303 3304 int_btf_id = find_int_btf_id(btf); 3305 dummy_var_btf_id = btf__add_var(btf, 3306 "dummy_ksym", 3307 BTF_VAR_GLOBAL_ALLOCATED, 3308 int_btf_id); 3309 if (dummy_var_btf_id < 0) 3310 pr_warn("cannot create a dummy_ksym var\n"); 3311 3312 return dummy_var_btf_id; 3313 } 3314 3315 static int bpf_object__collect_externs(struct bpf_object *obj) 3316 { 3317 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 3318 const struct btf_type *t; 3319 struct extern_desc *ext; 3320 int i, n, off, dummy_var_btf_id; 3321 const char *ext_name, *sec_name; 3322 Elf_Scn *scn; 3323 GElf_Shdr sh; 3324 3325 if (!obj->efile.symbols) 3326 return 0; 3327 3328 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 3329 if (elf_sec_hdr(obj, scn, &sh)) 3330 return -LIBBPF_ERRNO__FORMAT; 3331 3332 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 3333 if (dummy_var_btf_id < 0) 3334 return dummy_var_btf_id; 3335 3336 n = sh.sh_size / sh.sh_entsize; 3337 pr_debug("looking for externs among %d symbols...\n", n); 3338 3339 for (i = 0; i < n; i++) { 3340 GElf_Sym sym; 3341 3342 if (!gelf_getsym(obj->efile.symbols, i, &sym)) 3343 return -LIBBPF_ERRNO__FORMAT; 3344 if (!sym_is_extern(&sym)) 3345 continue; 3346 ext_name = elf_sym_str(obj, sym.st_name); 3347 if (!ext_name || !ext_name[0]) 3348 continue; 3349 3350 ext = obj->externs; 3351 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 3352 if (!ext) 3353 return -ENOMEM; 3354 obj->externs = ext; 3355 ext = &ext[obj->nr_extern]; 3356 memset(ext, 0, sizeof(*ext)); 3357 obj->nr_extern++; 3358 3359 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 3360 if (ext->btf_id <= 0) { 3361 pr_warn("failed to find BTF for extern '%s': %d\n", 3362 ext_name, ext->btf_id); 3363 return ext->btf_id; 3364 } 3365 t = btf__type_by_id(obj->btf, ext->btf_id); 3366 ext->name = btf__name_by_offset(obj->btf, t->name_off); 3367 ext->sym_idx = i; 3368 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; 3369 3370 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 3371 if (ext->sec_btf_id <= 0) { 3372 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 3373 ext_name, ext->btf_id, ext->sec_btf_id); 3374 return ext->sec_btf_id; 3375 } 3376 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 3377 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 3378 3379 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 3380 if (btf_is_func(t)) { 3381 pr_warn("extern function %s is unsupported under %s section\n", 3382 ext->name, KCONFIG_SEC); 3383 return -ENOTSUP; 3384 } 3385 kcfg_sec = sec; 3386 ext->type = EXT_KCFG; 3387 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 3388 if (ext->kcfg.sz <= 0) { 3389 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 3390 ext_name, ext->kcfg.sz); 3391 return ext->kcfg.sz; 3392 } 3393 ext->kcfg.align = btf__align_of(obj->btf, t->type); 3394 if (ext->kcfg.align <= 0) { 3395 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 3396 ext_name, ext->kcfg.align); 3397 return -EINVAL; 3398 } 3399 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 3400 &ext->kcfg.is_signed); 3401 if (ext->kcfg.type == KCFG_UNKNOWN) { 3402 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); 3403 return -ENOTSUP; 3404 } 3405 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 3406 if (btf_is_func(t) && ext->is_weak) { 3407 pr_warn("extern weak function %s is unsupported\n", 3408 ext->name); 3409 return -ENOTSUP; 3410 } 3411 ksym_sec = sec; 3412 ext->type = EXT_KSYM; 3413 skip_mods_and_typedefs(obj->btf, t->type, 3414 &ext->ksym.type_id); 3415 } else { 3416 pr_warn("unrecognized extern section '%s'\n", sec_name); 3417 return -ENOTSUP; 3418 } 3419 } 3420 pr_debug("collected %d externs total\n", obj->nr_extern); 3421 3422 if (!obj->nr_extern) 3423 return 0; 3424 3425 /* sort externs by type, for kcfg ones also by (align, size, name) */ 3426 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 3427 3428 /* for .ksyms section, we need to turn all externs into allocated 3429 * variables in BTF to pass kernel verification; we do this by 3430 * pretending that each extern is a 8-byte variable 3431 */ 3432 if (ksym_sec) { 3433 /* find existing 4-byte integer type in BTF to use for fake 3434 * extern variables in DATASEC 3435 */ 3436 int int_btf_id = find_int_btf_id(obj->btf); 3437 /* For extern function, a dummy_var added earlier 3438 * will be used to replace the vs->type and 3439 * its name string will be used to refill 3440 * the missing param's name. 3441 */ 3442 const struct btf_type *dummy_var; 3443 3444 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 3445 for (i = 0; i < obj->nr_extern; i++) { 3446 ext = &obj->externs[i]; 3447 if (ext->type != EXT_KSYM) 3448 continue; 3449 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 3450 i, ext->sym_idx, ext->name); 3451 } 3452 3453 sec = ksym_sec; 3454 n = btf_vlen(sec); 3455 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 3456 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3457 struct btf_type *vt; 3458 3459 vt = (void *)btf__type_by_id(obj->btf, vs->type); 3460 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 3461 ext = find_extern_by_name(obj, ext_name); 3462 if (!ext) { 3463 pr_warn("failed to find extern definition for BTF %s '%s'\n", 3464 btf_kind_str(vt), ext_name); 3465 return -ESRCH; 3466 } 3467 if (btf_is_func(vt)) { 3468 const struct btf_type *func_proto; 3469 struct btf_param *param; 3470 int j; 3471 3472 func_proto = btf__type_by_id(obj->btf, 3473 vt->type); 3474 param = btf_params(func_proto); 3475 /* Reuse the dummy_var string if the 3476 * func proto does not have param name. 3477 */ 3478 for (j = 0; j < btf_vlen(func_proto); j++) 3479 if (param[j].type && !param[j].name_off) 3480 param[j].name_off = 3481 dummy_var->name_off; 3482 vs->type = dummy_var_btf_id; 3483 vt->info &= ~0xffff; 3484 vt->info |= BTF_FUNC_GLOBAL; 3485 } else { 3486 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3487 vt->type = int_btf_id; 3488 } 3489 vs->offset = off; 3490 vs->size = sizeof(int); 3491 } 3492 sec->size = off; 3493 } 3494 3495 if (kcfg_sec) { 3496 sec = kcfg_sec; 3497 /* for kcfg externs calculate their offsets within a .kconfig map */ 3498 off = 0; 3499 for (i = 0; i < obj->nr_extern; i++) { 3500 ext = &obj->externs[i]; 3501 if (ext->type != EXT_KCFG) 3502 continue; 3503 3504 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 3505 off = ext->kcfg.data_off + ext->kcfg.sz; 3506 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 3507 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 3508 } 3509 sec->size = off; 3510 n = btf_vlen(sec); 3511 for (i = 0; i < n; i++) { 3512 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3513 3514 t = btf__type_by_id(obj->btf, vs->type); 3515 ext_name = btf__name_by_offset(obj->btf, t->name_off); 3516 ext = find_extern_by_name(obj, ext_name); 3517 if (!ext) { 3518 pr_warn("failed to find extern definition for BTF var '%s'\n", 3519 ext_name); 3520 return -ESRCH; 3521 } 3522 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3523 vs->offset = ext->kcfg.data_off; 3524 } 3525 } 3526 return 0; 3527 } 3528 3529 struct bpf_program * 3530 bpf_object__find_program_by_title(const struct bpf_object *obj, 3531 const char *title) 3532 { 3533 struct bpf_program *pos; 3534 3535 bpf_object__for_each_program(pos, obj) { 3536 if (pos->sec_name && !strcmp(pos->sec_name, title)) 3537 return pos; 3538 } 3539 return errno = ENOENT, NULL; 3540 } 3541 3542 static bool prog_is_subprog(const struct bpf_object *obj, 3543 const struct bpf_program *prog) 3544 { 3545 /* For legacy reasons, libbpf supports an entry-point BPF programs 3546 * without SEC() attribute, i.e., those in the .text section. But if 3547 * there are 2 or more such programs in the .text section, they all 3548 * must be subprograms called from entry-point BPF programs in 3549 * designated SEC()'tions, otherwise there is no way to distinguish 3550 * which of those programs should be loaded vs which are a subprogram. 3551 * Similarly, if there is a function/program in .text and at least one 3552 * other BPF program with custom SEC() attribute, then we just assume 3553 * .text programs are subprograms (even if they are not called from 3554 * other programs), because libbpf never explicitly supported mixing 3555 * SEC()-designated BPF programs and .text entry-point BPF programs. 3556 */ 3557 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 3558 } 3559 3560 struct bpf_program * 3561 bpf_object__find_program_by_name(const struct bpf_object *obj, 3562 const char *name) 3563 { 3564 struct bpf_program *prog; 3565 3566 bpf_object__for_each_program(prog, obj) { 3567 if (prog_is_subprog(obj, prog)) 3568 continue; 3569 if (!strcmp(prog->name, name)) 3570 return prog; 3571 } 3572 return errno = ENOENT, NULL; 3573 } 3574 3575 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 3576 int shndx) 3577 { 3578 return shndx == obj->efile.data_shndx || 3579 shndx == obj->efile.bss_shndx || 3580 shndx == obj->efile.rodata_shndx; 3581 } 3582 3583 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 3584 int shndx) 3585 { 3586 return shndx == obj->efile.maps_shndx || 3587 shndx == obj->efile.btf_maps_shndx; 3588 } 3589 3590 static enum libbpf_map_type 3591 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 3592 { 3593 if (shndx == obj->efile.data_shndx) 3594 return LIBBPF_MAP_DATA; 3595 else if (shndx == obj->efile.bss_shndx) 3596 return LIBBPF_MAP_BSS; 3597 else if (shndx == obj->efile.rodata_shndx) 3598 return LIBBPF_MAP_RODATA; 3599 else if (shndx == obj->efile.symbols_shndx) 3600 return LIBBPF_MAP_KCONFIG; 3601 else 3602 return LIBBPF_MAP_UNSPEC; 3603 } 3604 3605 static int bpf_program__record_reloc(struct bpf_program *prog, 3606 struct reloc_desc *reloc_desc, 3607 __u32 insn_idx, const char *sym_name, 3608 const GElf_Sym *sym, const GElf_Rel *rel) 3609 { 3610 struct bpf_insn *insn = &prog->insns[insn_idx]; 3611 size_t map_idx, nr_maps = prog->obj->nr_maps; 3612 struct bpf_object *obj = prog->obj; 3613 __u32 shdr_idx = sym->st_shndx; 3614 enum libbpf_map_type type; 3615 const char *sym_sec_name; 3616 struct bpf_map *map; 3617 3618 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 3619 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 3620 prog->name, sym_name, insn_idx, insn->code); 3621 return -LIBBPF_ERRNO__RELOC; 3622 } 3623 3624 if (sym_is_extern(sym)) { 3625 int sym_idx = GELF_R_SYM(rel->r_info); 3626 int i, n = obj->nr_extern; 3627 struct extern_desc *ext; 3628 3629 for (i = 0; i < n; i++) { 3630 ext = &obj->externs[i]; 3631 if (ext->sym_idx == sym_idx) 3632 break; 3633 } 3634 if (i >= n) { 3635 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 3636 prog->name, sym_name, sym_idx); 3637 return -LIBBPF_ERRNO__RELOC; 3638 } 3639 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 3640 prog->name, i, ext->name, ext->sym_idx, insn_idx); 3641 if (insn->code == (BPF_JMP | BPF_CALL)) 3642 reloc_desc->type = RELO_EXTERN_FUNC; 3643 else 3644 reloc_desc->type = RELO_EXTERN_VAR; 3645 reloc_desc->insn_idx = insn_idx; 3646 reloc_desc->sym_off = i; /* sym_off stores extern index */ 3647 return 0; 3648 } 3649 3650 /* sub-program call relocation */ 3651 if (is_call_insn(insn)) { 3652 if (insn->src_reg != BPF_PSEUDO_CALL) { 3653 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 3654 return -LIBBPF_ERRNO__RELOC; 3655 } 3656 /* text_shndx can be 0, if no default "main" program exists */ 3657 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 3658 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3659 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 3660 prog->name, sym_name, sym_sec_name); 3661 return -LIBBPF_ERRNO__RELOC; 3662 } 3663 if (sym->st_value % BPF_INSN_SZ) { 3664 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 3665 prog->name, sym_name, (size_t)sym->st_value); 3666 return -LIBBPF_ERRNO__RELOC; 3667 } 3668 reloc_desc->type = RELO_CALL; 3669 reloc_desc->insn_idx = insn_idx; 3670 reloc_desc->sym_off = sym->st_value; 3671 return 0; 3672 } 3673 3674 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 3675 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 3676 prog->name, sym_name, shdr_idx); 3677 return -LIBBPF_ERRNO__RELOC; 3678 } 3679 3680 /* loading subprog addresses */ 3681 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 3682 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 3683 * local_func: sym->st_value = 0, insn->imm = offset in the section. 3684 */ 3685 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 3686 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 3687 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 3688 return -LIBBPF_ERRNO__RELOC; 3689 } 3690 3691 reloc_desc->type = RELO_SUBPROG_ADDR; 3692 reloc_desc->insn_idx = insn_idx; 3693 reloc_desc->sym_off = sym->st_value; 3694 return 0; 3695 } 3696 3697 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 3698 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3699 3700 /* generic map reference relocation */ 3701 if (type == LIBBPF_MAP_UNSPEC) { 3702 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 3703 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 3704 prog->name, sym_name, sym_sec_name); 3705 return -LIBBPF_ERRNO__RELOC; 3706 } 3707 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3708 map = &obj->maps[map_idx]; 3709 if (map->libbpf_type != type || 3710 map->sec_idx != sym->st_shndx || 3711 map->sec_offset != sym->st_value) 3712 continue; 3713 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 3714 prog->name, map_idx, map->name, map->sec_idx, 3715 map->sec_offset, insn_idx); 3716 break; 3717 } 3718 if (map_idx >= nr_maps) { 3719 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 3720 prog->name, sym_sec_name, (size_t)sym->st_value); 3721 return -LIBBPF_ERRNO__RELOC; 3722 } 3723 reloc_desc->type = RELO_LD64; 3724 reloc_desc->insn_idx = insn_idx; 3725 reloc_desc->map_idx = map_idx; 3726 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 3727 return 0; 3728 } 3729 3730 /* global data map relocation */ 3731 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 3732 pr_warn("prog '%s': bad data relo against section '%s'\n", 3733 prog->name, sym_sec_name); 3734 return -LIBBPF_ERRNO__RELOC; 3735 } 3736 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3737 map = &obj->maps[map_idx]; 3738 if (map->libbpf_type != type) 3739 continue; 3740 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 3741 prog->name, map_idx, map->name, map->sec_idx, 3742 map->sec_offset, insn_idx); 3743 break; 3744 } 3745 if (map_idx >= nr_maps) { 3746 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 3747 prog->name, sym_sec_name); 3748 return -LIBBPF_ERRNO__RELOC; 3749 } 3750 3751 reloc_desc->type = RELO_DATA; 3752 reloc_desc->insn_idx = insn_idx; 3753 reloc_desc->map_idx = map_idx; 3754 reloc_desc->sym_off = sym->st_value; 3755 return 0; 3756 } 3757 3758 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 3759 { 3760 return insn_idx >= prog->sec_insn_off && 3761 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 3762 } 3763 3764 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 3765 size_t sec_idx, size_t insn_idx) 3766 { 3767 int l = 0, r = obj->nr_programs - 1, m; 3768 struct bpf_program *prog; 3769 3770 while (l < r) { 3771 m = l + (r - l + 1) / 2; 3772 prog = &obj->programs[m]; 3773 3774 if (prog->sec_idx < sec_idx || 3775 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 3776 l = m; 3777 else 3778 r = m - 1; 3779 } 3780 /* matching program could be at index l, but it still might be the 3781 * wrong one, so we need to double check conditions for the last time 3782 */ 3783 prog = &obj->programs[l]; 3784 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 3785 return prog; 3786 return NULL; 3787 } 3788 3789 static int 3790 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data) 3791 { 3792 Elf_Data *symbols = obj->efile.symbols; 3793 const char *relo_sec_name, *sec_name; 3794 size_t sec_idx = shdr->sh_info; 3795 struct bpf_program *prog; 3796 struct reloc_desc *relos; 3797 int err, i, nrels; 3798 const char *sym_name; 3799 __u32 insn_idx; 3800 Elf_Scn *scn; 3801 Elf_Data *scn_data; 3802 GElf_Sym sym; 3803 GElf_Rel rel; 3804 3805 scn = elf_sec_by_idx(obj, sec_idx); 3806 scn_data = elf_sec_data(obj, scn); 3807 3808 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 3809 sec_name = elf_sec_name(obj, scn); 3810 if (!relo_sec_name || !sec_name) 3811 return -EINVAL; 3812 3813 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 3814 relo_sec_name, sec_idx, sec_name); 3815 nrels = shdr->sh_size / shdr->sh_entsize; 3816 3817 for (i = 0; i < nrels; i++) { 3818 if (!gelf_getrel(data, i, &rel)) { 3819 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 3820 return -LIBBPF_ERRNO__FORMAT; 3821 } 3822 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 3823 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n", 3824 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3825 return -LIBBPF_ERRNO__FORMAT; 3826 } 3827 3828 if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) { 3829 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 3830 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3831 return -LIBBPF_ERRNO__FORMAT; 3832 } 3833 3834 insn_idx = rel.r_offset / BPF_INSN_SZ; 3835 /* relocations against static functions are recorded as 3836 * relocations against the section that contains a function; 3837 * in such case, symbol will be STT_SECTION and sym.st_name 3838 * will point to empty string (0), so fetch section name 3839 * instead 3840 */ 3841 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0) 3842 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx)); 3843 else 3844 sym_name = elf_sym_str(obj, sym.st_name); 3845 sym_name = sym_name ?: "<?"; 3846 3847 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 3848 relo_sec_name, i, insn_idx, sym_name); 3849 3850 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 3851 if (!prog) { 3852 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 3853 relo_sec_name, i, sec_name, insn_idx); 3854 continue; 3855 } 3856 3857 relos = libbpf_reallocarray(prog->reloc_desc, 3858 prog->nr_reloc + 1, sizeof(*relos)); 3859 if (!relos) 3860 return -ENOMEM; 3861 prog->reloc_desc = relos; 3862 3863 /* adjust insn_idx to local BPF program frame of reference */ 3864 insn_idx -= prog->sec_insn_off; 3865 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 3866 insn_idx, sym_name, &sym, &rel); 3867 if (err) 3868 return err; 3869 3870 prog->nr_reloc++; 3871 } 3872 return 0; 3873 } 3874 3875 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 3876 { 3877 struct bpf_map_def *def = &map->def; 3878 __u32 key_type_id = 0, value_type_id = 0; 3879 int ret; 3880 3881 /* if it's BTF-defined map, we don't need to search for type IDs. 3882 * For struct_ops map, it does not need btf_key_type_id and 3883 * btf_value_type_id. 3884 */ 3885 if (map->sec_idx == obj->efile.btf_maps_shndx || 3886 bpf_map__is_struct_ops(map)) 3887 return 0; 3888 3889 if (!bpf_map__is_internal(map)) { 3890 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 3891 def->value_size, &key_type_id, 3892 &value_type_id); 3893 } else { 3894 /* 3895 * LLVM annotates global data differently in BTF, that is, 3896 * only as '.data', '.bss' or '.rodata'. 3897 */ 3898 ret = btf__find_by_name(obj->btf, 3899 libbpf_type_to_btf_name[map->libbpf_type]); 3900 } 3901 if (ret < 0) 3902 return ret; 3903 3904 map->btf_key_type_id = key_type_id; 3905 map->btf_value_type_id = bpf_map__is_internal(map) ? 3906 ret : value_type_id; 3907 return 0; 3908 } 3909 3910 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 3911 { 3912 char file[PATH_MAX], buff[4096]; 3913 FILE *fp; 3914 __u32 val; 3915 int err; 3916 3917 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 3918 memset(info, 0, sizeof(*info)); 3919 3920 fp = fopen(file, "r"); 3921 if (!fp) { 3922 err = -errno; 3923 pr_warn("failed to open %s: %d. No procfs support?\n", file, 3924 err); 3925 return err; 3926 } 3927 3928 while (fgets(buff, sizeof(buff), fp)) { 3929 if (sscanf(buff, "map_type:\t%u", &val) == 1) 3930 info->type = val; 3931 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 3932 info->key_size = val; 3933 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 3934 info->value_size = val; 3935 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 3936 info->max_entries = val; 3937 else if (sscanf(buff, "map_flags:\t%i", &val) == 1) 3938 info->map_flags = val; 3939 } 3940 3941 fclose(fp); 3942 3943 return 0; 3944 } 3945 3946 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 3947 { 3948 struct bpf_map_info info = {}; 3949 __u32 len = sizeof(info); 3950 int new_fd, err; 3951 char *new_name; 3952 3953 err = bpf_obj_get_info_by_fd(fd, &info, &len); 3954 if (err && errno == EINVAL) 3955 err = bpf_get_map_info_from_fdinfo(fd, &info); 3956 if (err) 3957 return libbpf_err(err); 3958 3959 new_name = strdup(info.name); 3960 if (!new_name) 3961 return libbpf_err(-errno); 3962 3963 new_fd = open("/", O_RDONLY | O_CLOEXEC); 3964 if (new_fd < 0) { 3965 err = -errno; 3966 goto err_free_new_name; 3967 } 3968 3969 new_fd = dup3(fd, new_fd, O_CLOEXEC); 3970 if (new_fd < 0) { 3971 err = -errno; 3972 goto err_close_new_fd; 3973 } 3974 3975 err = zclose(map->fd); 3976 if (err) { 3977 err = -errno; 3978 goto err_close_new_fd; 3979 } 3980 free(map->name); 3981 3982 map->fd = new_fd; 3983 map->name = new_name; 3984 map->def.type = info.type; 3985 map->def.key_size = info.key_size; 3986 map->def.value_size = info.value_size; 3987 map->def.max_entries = info.max_entries; 3988 map->def.map_flags = info.map_flags; 3989 map->btf_key_type_id = info.btf_key_type_id; 3990 map->btf_value_type_id = info.btf_value_type_id; 3991 map->reused = true; 3992 3993 return 0; 3994 3995 err_close_new_fd: 3996 close(new_fd); 3997 err_free_new_name: 3998 free(new_name); 3999 return libbpf_err(err); 4000 } 4001 4002 __u32 bpf_map__max_entries(const struct bpf_map *map) 4003 { 4004 return map->def.max_entries; 4005 } 4006 4007 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 4008 { 4009 if (!bpf_map_type__is_map_in_map(map->def.type)) 4010 return errno = EINVAL, NULL; 4011 4012 return map->inner_map; 4013 } 4014 4015 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 4016 { 4017 if (map->fd >= 0) 4018 return libbpf_err(-EBUSY); 4019 map->def.max_entries = max_entries; 4020 return 0; 4021 } 4022 4023 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 4024 { 4025 if (!map || !max_entries) 4026 return libbpf_err(-EINVAL); 4027 4028 return bpf_map__set_max_entries(map, max_entries); 4029 } 4030 4031 static int 4032 bpf_object__probe_loading(struct bpf_object *obj) 4033 { 4034 struct bpf_load_program_attr attr; 4035 char *cp, errmsg[STRERR_BUFSIZE]; 4036 struct bpf_insn insns[] = { 4037 BPF_MOV64_IMM(BPF_REG_0, 0), 4038 BPF_EXIT_INSN(), 4039 }; 4040 int ret; 4041 4042 if (obj->gen_loader) 4043 return 0; 4044 4045 /* make sure basic loading works */ 4046 4047 memset(&attr, 0, sizeof(attr)); 4048 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4049 attr.insns = insns; 4050 attr.insns_cnt = ARRAY_SIZE(insns); 4051 attr.license = "GPL"; 4052 4053 ret = bpf_load_program_xattr(&attr, NULL, 0); 4054 if (ret < 0) { 4055 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; 4056 ret = bpf_load_program_xattr(&attr, NULL, 0); 4057 } 4058 if (ret < 0) { 4059 ret = errno; 4060 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4061 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 4062 "program. Make sure your kernel supports BPF " 4063 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 4064 "set to big enough value.\n", __func__, cp, ret); 4065 return -ret; 4066 } 4067 close(ret); 4068 4069 return 0; 4070 } 4071 4072 static int probe_fd(int fd) 4073 { 4074 if (fd >= 0) 4075 close(fd); 4076 return fd >= 0; 4077 } 4078 4079 static int probe_kern_prog_name(void) 4080 { 4081 struct bpf_load_program_attr attr; 4082 struct bpf_insn insns[] = { 4083 BPF_MOV64_IMM(BPF_REG_0, 0), 4084 BPF_EXIT_INSN(), 4085 }; 4086 int ret; 4087 4088 /* make sure loading with name works */ 4089 4090 memset(&attr, 0, sizeof(attr)); 4091 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4092 attr.insns = insns; 4093 attr.insns_cnt = ARRAY_SIZE(insns); 4094 attr.license = "GPL"; 4095 attr.name = "test"; 4096 ret = bpf_load_program_xattr(&attr, NULL, 0); 4097 return probe_fd(ret); 4098 } 4099 4100 static int probe_kern_global_data(void) 4101 { 4102 struct bpf_load_program_attr prg_attr; 4103 struct bpf_create_map_attr map_attr; 4104 char *cp, errmsg[STRERR_BUFSIZE]; 4105 struct bpf_insn insns[] = { 4106 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 4107 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 4108 BPF_MOV64_IMM(BPF_REG_0, 0), 4109 BPF_EXIT_INSN(), 4110 }; 4111 int ret, map; 4112 4113 memset(&map_attr, 0, sizeof(map_attr)); 4114 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4115 map_attr.key_size = sizeof(int); 4116 map_attr.value_size = 32; 4117 map_attr.max_entries = 1; 4118 4119 map = bpf_create_map_xattr(&map_attr); 4120 if (map < 0) { 4121 ret = -errno; 4122 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4123 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4124 __func__, cp, -ret); 4125 return ret; 4126 } 4127 4128 insns[0].imm = map; 4129 4130 memset(&prg_attr, 0, sizeof(prg_attr)); 4131 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4132 prg_attr.insns = insns; 4133 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4134 prg_attr.license = "GPL"; 4135 4136 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 4137 close(map); 4138 return probe_fd(ret); 4139 } 4140 4141 static int probe_kern_btf(void) 4142 { 4143 static const char strs[] = "\0int"; 4144 __u32 types[] = { 4145 /* int */ 4146 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4147 }; 4148 4149 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4150 strs, sizeof(strs))); 4151 } 4152 4153 static int probe_kern_btf_func(void) 4154 { 4155 static const char strs[] = "\0int\0x\0a"; 4156 /* void x(int a) {} */ 4157 __u32 types[] = { 4158 /* int */ 4159 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4160 /* FUNC_PROTO */ /* [2] */ 4161 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4162 BTF_PARAM_ENC(7, 1), 4163 /* FUNC x */ /* [3] */ 4164 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 4165 }; 4166 4167 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4168 strs, sizeof(strs))); 4169 } 4170 4171 static int probe_kern_btf_func_global(void) 4172 { 4173 static const char strs[] = "\0int\0x\0a"; 4174 /* static void x(int a) {} */ 4175 __u32 types[] = { 4176 /* int */ 4177 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4178 /* FUNC_PROTO */ /* [2] */ 4179 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4180 BTF_PARAM_ENC(7, 1), 4181 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 4182 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 4183 }; 4184 4185 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4186 strs, sizeof(strs))); 4187 } 4188 4189 static int probe_kern_btf_datasec(void) 4190 { 4191 static const char strs[] = "\0x\0.data"; 4192 /* static int a; */ 4193 __u32 types[] = { 4194 /* int */ 4195 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4196 /* VAR x */ /* [2] */ 4197 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4198 BTF_VAR_STATIC, 4199 /* DATASEC val */ /* [3] */ 4200 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 4201 BTF_VAR_SECINFO_ENC(2, 0, 4), 4202 }; 4203 4204 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4205 strs, sizeof(strs))); 4206 } 4207 4208 static int probe_kern_btf_float(void) 4209 { 4210 static const char strs[] = "\0float"; 4211 __u32 types[] = { 4212 /* float */ 4213 BTF_TYPE_FLOAT_ENC(1, 4), 4214 }; 4215 4216 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4217 strs, sizeof(strs))); 4218 } 4219 4220 static int probe_kern_btf_tag(void) 4221 { 4222 static const char strs[] = "\0tag"; 4223 __u32 types[] = { 4224 /* int */ 4225 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4226 /* VAR x */ /* [2] */ 4227 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4228 BTF_VAR_STATIC, 4229 /* attr */ 4230 BTF_TYPE_TAG_ENC(1, 2, -1), 4231 }; 4232 4233 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4234 strs, sizeof(strs))); 4235 } 4236 4237 static int probe_kern_array_mmap(void) 4238 { 4239 struct bpf_create_map_attr attr = { 4240 .map_type = BPF_MAP_TYPE_ARRAY, 4241 .map_flags = BPF_F_MMAPABLE, 4242 .key_size = sizeof(int), 4243 .value_size = sizeof(int), 4244 .max_entries = 1, 4245 }; 4246 4247 return probe_fd(bpf_create_map_xattr(&attr)); 4248 } 4249 4250 static int probe_kern_exp_attach_type(void) 4251 { 4252 struct bpf_load_program_attr attr; 4253 struct bpf_insn insns[] = { 4254 BPF_MOV64_IMM(BPF_REG_0, 0), 4255 BPF_EXIT_INSN(), 4256 }; 4257 4258 memset(&attr, 0, sizeof(attr)); 4259 /* use any valid combination of program type and (optional) 4260 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 4261 * to see if kernel supports expected_attach_type field for 4262 * BPF_PROG_LOAD command 4263 */ 4264 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 4265 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; 4266 attr.insns = insns; 4267 attr.insns_cnt = ARRAY_SIZE(insns); 4268 attr.license = "GPL"; 4269 4270 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4271 } 4272 4273 static int probe_kern_probe_read_kernel(void) 4274 { 4275 struct bpf_load_program_attr attr; 4276 struct bpf_insn insns[] = { 4277 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 4278 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 4279 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 4280 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 4281 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 4282 BPF_EXIT_INSN(), 4283 }; 4284 4285 memset(&attr, 0, sizeof(attr)); 4286 attr.prog_type = BPF_PROG_TYPE_KPROBE; 4287 attr.insns = insns; 4288 attr.insns_cnt = ARRAY_SIZE(insns); 4289 attr.license = "GPL"; 4290 4291 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4292 } 4293 4294 static int probe_prog_bind_map(void) 4295 { 4296 struct bpf_load_program_attr prg_attr; 4297 struct bpf_create_map_attr map_attr; 4298 char *cp, errmsg[STRERR_BUFSIZE]; 4299 struct bpf_insn insns[] = { 4300 BPF_MOV64_IMM(BPF_REG_0, 0), 4301 BPF_EXIT_INSN(), 4302 }; 4303 int ret, map, prog; 4304 4305 memset(&map_attr, 0, sizeof(map_attr)); 4306 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4307 map_attr.key_size = sizeof(int); 4308 map_attr.value_size = 32; 4309 map_attr.max_entries = 1; 4310 4311 map = bpf_create_map_xattr(&map_attr); 4312 if (map < 0) { 4313 ret = -errno; 4314 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4315 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4316 __func__, cp, -ret); 4317 return ret; 4318 } 4319 4320 memset(&prg_attr, 0, sizeof(prg_attr)); 4321 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4322 prg_attr.insns = insns; 4323 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4324 prg_attr.license = "GPL"; 4325 4326 prog = bpf_load_program_xattr(&prg_attr, NULL, 0); 4327 if (prog < 0) { 4328 close(map); 4329 return 0; 4330 } 4331 4332 ret = bpf_prog_bind_map(prog, map, NULL); 4333 4334 close(map); 4335 close(prog); 4336 4337 return ret >= 0; 4338 } 4339 4340 static int probe_module_btf(void) 4341 { 4342 static const char strs[] = "\0int"; 4343 __u32 types[] = { 4344 /* int */ 4345 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4346 }; 4347 struct bpf_btf_info info; 4348 __u32 len = sizeof(info); 4349 char name[16]; 4350 int fd, err; 4351 4352 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); 4353 if (fd < 0) 4354 return 0; /* BTF not supported at all */ 4355 4356 memset(&info, 0, sizeof(info)); 4357 info.name = ptr_to_u64(name); 4358 info.name_len = sizeof(name); 4359 4360 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 4361 * kernel's module BTF support coincides with support for 4362 * name/name_len fields in struct bpf_btf_info. 4363 */ 4364 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4365 close(fd); 4366 return !err; 4367 } 4368 4369 static int probe_perf_link(void) 4370 { 4371 struct bpf_load_program_attr attr; 4372 struct bpf_insn insns[] = { 4373 BPF_MOV64_IMM(BPF_REG_0, 0), 4374 BPF_EXIT_INSN(), 4375 }; 4376 int prog_fd, link_fd, err; 4377 4378 memset(&attr, 0, sizeof(attr)); 4379 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; 4380 attr.insns = insns; 4381 attr.insns_cnt = ARRAY_SIZE(insns); 4382 attr.license = "GPL"; 4383 prog_fd = bpf_load_program_xattr(&attr, NULL, 0); 4384 if (prog_fd < 0) 4385 return -errno; 4386 4387 /* use invalid perf_event FD to get EBADF, if link is supported; 4388 * otherwise EINVAL should be returned 4389 */ 4390 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); 4391 err = -errno; /* close() can clobber errno */ 4392 4393 if (link_fd >= 0) 4394 close(link_fd); 4395 close(prog_fd); 4396 4397 return link_fd < 0 && err == -EBADF; 4398 } 4399 4400 enum kern_feature_result { 4401 FEAT_UNKNOWN = 0, 4402 FEAT_SUPPORTED = 1, 4403 FEAT_MISSING = 2, 4404 }; 4405 4406 typedef int (*feature_probe_fn)(void); 4407 4408 static struct kern_feature_desc { 4409 const char *desc; 4410 feature_probe_fn probe; 4411 enum kern_feature_result res; 4412 } feature_probes[__FEAT_CNT] = { 4413 [FEAT_PROG_NAME] = { 4414 "BPF program name", probe_kern_prog_name, 4415 }, 4416 [FEAT_GLOBAL_DATA] = { 4417 "global variables", probe_kern_global_data, 4418 }, 4419 [FEAT_BTF] = { 4420 "minimal BTF", probe_kern_btf, 4421 }, 4422 [FEAT_BTF_FUNC] = { 4423 "BTF functions", probe_kern_btf_func, 4424 }, 4425 [FEAT_BTF_GLOBAL_FUNC] = { 4426 "BTF global function", probe_kern_btf_func_global, 4427 }, 4428 [FEAT_BTF_DATASEC] = { 4429 "BTF data section and variable", probe_kern_btf_datasec, 4430 }, 4431 [FEAT_ARRAY_MMAP] = { 4432 "ARRAY map mmap()", probe_kern_array_mmap, 4433 }, 4434 [FEAT_EXP_ATTACH_TYPE] = { 4435 "BPF_PROG_LOAD expected_attach_type attribute", 4436 probe_kern_exp_attach_type, 4437 }, 4438 [FEAT_PROBE_READ_KERN] = { 4439 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 4440 }, 4441 [FEAT_PROG_BIND_MAP] = { 4442 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 4443 }, 4444 [FEAT_MODULE_BTF] = { 4445 "module BTF support", probe_module_btf, 4446 }, 4447 [FEAT_BTF_FLOAT] = { 4448 "BTF_KIND_FLOAT support", probe_kern_btf_float, 4449 }, 4450 [FEAT_PERF_LINK] = { 4451 "BPF perf link support", probe_perf_link, 4452 }, 4453 [FEAT_BTF_TAG] = { 4454 "BTF_KIND_TAG support", probe_kern_btf_tag, 4455 }, 4456 }; 4457 4458 static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 4459 { 4460 struct kern_feature_desc *feat = &feature_probes[feat_id]; 4461 int ret; 4462 4463 if (obj->gen_loader) 4464 /* To generate loader program assume the latest kernel 4465 * to avoid doing extra prog_load, map_create syscalls. 4466 */ 4467 return true; 4468 4469 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { 4470 ret = feat->probe(); 4471 if (ret > 0) { 4472 WRITE_ONCE(feat->res, FEAT_SUPPORTED); 4473 } else if (ret == 0) { 4474 WRITE_ONCE(feat->res, FEAT_MISSING); 4475 } else { 4476 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); 4477 WRITE_ONCE(feat->res, FEAT_MISSING); 4478 } 4479 } 4480 4481 return READ_ONCE(feat->res) == FEAT_SUPPORTED; 4482 } 4483 4484 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 4485 { 4486 struct bpf_map_info map_info = {}; 4487 char msg[STRERR_BUFSIZE]; 4488 __u32 map_info_len; 4489 int err; 4490 4491 map_info_len = sizeof(map_info); 4492 4493 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len); 4494 if (err && errno == EINVAL) 4495 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 4496 if (err) { 4497 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 4498 libbpf_strerror_r(errno, msg, sizeof(msg))); 4499 return false; 4500 } 4501 4502 return (map_info.type == map->def.type && 4503 map_info.key_size == map->def.key_size && 4504 map_info.value_size == map->def.value_size && 4505 map_info.max_entries == map->def.max_entries && 4506 map_info.map_flags == map->def.map_flags); 4507 } 4508 4509 static int 4510 bpf_object__reuse_map(struct bpf_map *map) 4511 { 4512 char *cp, errmsg[STRERR_BUFSIZE]; 4513 int err, pin_fd; 4514 4515 pin_fd = bpf_obj_get(map->pin_path); 4516 if (pin_fd < 0) { 4517 err = -errno; 4518 if (err == -ENOENT) { 4519 pr_debug("found no pinned map to reuse at '%s'\n", 4520 map->pin_path); 4521 return 0; 4522 } 4523 4524 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 4525 pr_warn("couldn't retrieve pinned map '%s': %s\n", 4526 map->pin_path, cp); 4527 return err; 4528 } 4529 4530 if (!map_is_reuse_compat(map, pin_fd)) { 4531 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 4532 map->pin_path); 4533 close(pin_fd); 4534 return -EINVAL; 4535 } 4536 4537 err = bpf_map__reuse_fd(map, pin_fd); 4538 if (err) { 4539 close(pin_fd); 4540 return err; 4541 } 4542 map->pinned = true; 4543 pr_debug("reused pinned map at '%s'\n", map->pin_path); 4544 4545 return 0; 4546 } 4547 4548 static int 4549 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 4550 { 4551 enum libbpf_map_type map_type = map->libbpf_type; 4552 char *cp, errmsg[STRERR_BUFSIZE]; 4553 int err, zero = 0; 4554 4555 if (obj->gen_loader) { 4556 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 4557 map->mmaped, map->def.value_size); 4558 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 4559 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 4560 return 0; 4561 } 4562 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 4563 if (err) { 4564 err = -errno; 4565 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4566 pr_warn("Error setting initial map(%s) contents: %s\n", 4567 map->name, cp); 4568 return err; 4569 } 4570 4571 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 4572 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 4573 err = bpf_map_freeze(map->fd); 4574 if (err) { 4575 err = -errno; 4576 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4577 pr_warn("Error freezing map(%s) as read-only: %s\n", 4578 map->name, cp); 4579 return err; 4580 } 4581 } 4582 return 0; 4583 } 4584 4585 static void bpf_map__destroy(struct bpf_map *map); 4586 4587 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 4588 { 4589 struct bpf_create_map_attr create_attr; 4590 struct bpf_map_def *def = &map->def; 4591 int err = 0; 4592 4593 memset(&create_attr, 0, sizeof(create_attr)); 4594 4595 if (kernel_supports(obj, FEAT_PROG_NAME)) 4596 create_attr.name = map->name; 4597 create_attr.map_ifindex = map->map_ifindex; 4598 create_attr.map_type = def->type; 4599 create_attr.map_flags = def->map_flags; 4600 create_attr.key_size = def->key_size; 4601 create_attr.value_size = def->value_size; 4602 create_attr.numa_node = map->numa_node; 4603 4604 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) { 4605 int nr_cpus; 4606 4607 nr_cpus = libbpf_num_possible_cpus(); 4608 if (nr_cpus < 0) { 4609 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 4610 map->name, nr_cpus); 4611 return nr_cpus; 4612 } 4613 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 4614 create_attr.max_entries = nr_cpus; 4615 } else { 4616 create_attr.max_entries = def->max_entries; 4617 } 4618 4619 if (bpf_map__is_struct_ops(map)) 4620 create_attr.btf_vmlinux_value_type_id = 4621 map->btf_vmlinux_value_type_id; 4622 4623 create_attr.btf_fd = 0; 4624 create_attr.btf_key_type_id = 0; 4625 create_attr.btf_value_type_id = 0; 4626 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) { 4627 create_attr.btf_fd = btf__fd(obj->btf); 4628 create_attr.btf_key_type_id = map->btf_key_type_id; 4629 create_attr.btf_value_type_id = map->btf_value_type_id; 4630 } 4631 4632 if (bpf_map_type__is_map_in_map(def->type)) { 4633 if (map->inner_map) { 4634 err = bpf_object__create_map(obj, map->inner_map, true); 4635 if (err) { 4636 pr_warn("map '%s': failed to create inner map: %d\n", 4637 map->name, err); 4638 return err; 4639 } 4640 map->inner_map_fd = bpf_map__fd(map->inner_map); 4641 } 4642 if (map->inner_map_fd >= 0) 4643 create_attr.inner_map_fd = map->inner_map_fd; 4644 } 4645 4646 if (obj->gen_loader) { 4647 bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps); 4648 /* Pretend to have valid FD to pass various fd >= 0 checks. 4649 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 4650 */ 4651 map->fd = 0; 4652 } else { 4653 map->fd = bpf_create_map_xattr(&create_attr); 4654 } 4655 if (map->fd < 0 && (create_attr.btf_key_type_id || 4656 create_attr.btf_value_type_id)) { 4657 char *cp, errmsg[STRERR_BUFSIZE]; 4658 4659 err = -errno; 4660 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4661 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 4662 map->name, cp, err); 4663 create_attr.btf_fd = 0; 4664 create_attr.btf_key_type_id = 0; 4665 create_attr.btf_value_type_id = 0; 4666 map->btf_key_type_id = 0; 4667 map->btf_value_type_id = 0; 4668 map->fd = bpf_create_map_xattr(&create_attr); 4669 } 4670 4671 err = map->fd < 0 ? -errno : 0; 4672 4673 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 4674 if (obj->gen_loader) 4675 map->inner_map->fd = -1; 4676 bpf_map__destroy(map->inner_map); 4677 zfree(&map->inner_map); 4678 } 4679 4680 return err; 4681 } 4682 4683 static int init_map_slots(struct bpf_object *obj, struct bpf_map *map) 4684 { 4685 const struct bpf_map *targ_map; 4686 unsigned int i; 4687 int fd, err = 0; 4688 4689 for (i = 0; i < map->init_slots_sz; i++) { 4690 if (!map->init_slots[i]) 4691 continue; 4692 4693 targ_map = map->init_slots[i]; 4694 fd = bpf_map__fd(targ_map); 4695 if (obj->gen_loader) { 4696 pr_warn("// TODO map_update_elem: idx %td key %d value==map_idx %td\n", 4697 map - obj->maps, i, targ_map - obj->maps); 4698 return -ENOTSUP; 4699 } else { 4700 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 4701 } 4702 if (err) { 4703 err = -errno; 4704 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 4705 map->name, i, targ_map->name, 4706 fd, err); 4707 return err; 4708 } 4709 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 4710 map->name, i, targ_map->name, fd); 4711 } 4712 4713 zfree(&map->init_slots); 4714 map->init_slots_sz = 0; 4715 4716 return 0; 4717 } 4718 4719 static int 4720 bpf_object__create_maps(struct bpf_object *obj) 4721 { 4722 struct bpf_map *map; 4723 char *cp, errmsg[STRERR_BUFSIZE]; 4724 unsigned int i, j; 4725 int err; 4726 bool retried; 4727 4728 for (i = 0; i < obj->nr_maps; i++) { 4729 map = &obj->maps[i]; 4730 4731 retried = false; 4732 retry: 4733 if (map->pin_path) { 4734 err = bpf_object__reuse_map(map); 4735 if (err) { 4736 pr_warn("map '%s': error reusing pinned map\n", 4737 map->name); 4738 goto err_out; 4739 } 4740 if (retried && map->fd < 0) { 4741 pr_warn("map '%s': cannot find pinned map\n", 4742 map->name); 4743 err = -ENOENT; 4744 goto err_out; 4745 } 4746 } 4747 4748 if (map->fd >= 0) { 4749 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 4750 map->name, map->fd); 4751 } else { 4752 err = bpf_object__create_map(obj, map, false); 4753 if (err) 4754 goto err_out; 4755 4756 pr_debug("map '%s': created successfully, fd=%d\n", 4757 map->name, map->fd); 4758 4759 if (bpf_map__is_internal(map)) { 4760 err = bpf_object__populate_internal_map(obj, map); 4761 if (err < 0) { 4762 zclose(map->fd); 4763 goto err_out; 4764 } 4765 } 4766 4767 if (map->init_slots_sz) { 4768 err = init_map_slots(obj, map); 4769 if (err < 0) { 4770 zclose(map->fd); 4771 goto err_out; 4772 } 4773 } 4774 } 4775 4776 if (map->pin_path && !map->pinned) { 4777 err = bpf_map__pin(map, NULL); 4778 if (err) { 4779 zclose(map->fd); 4780 if (!retried && err == -EEXIST) { 4781 retried = true; 4782 goto retry; 4783 } 4784 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 4785 map->name, map->pin_path, err); 4786 goto err_out; 4787 } 4788 } 4789 } 4790 4791 return 0; 4792 4793 err_out: 4794 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4795 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 4796 pr_perm_msg(err); 4797 for (j = 0; j < i; j++) 4798 zclose(obj->maps[j].fd); 4799 return err; 4800 } 4801 4802 static bool bpf_core_is_flavor_sep(const char *s) 4803 { 4804 /* check X___Y name pattern, where X and Y are not underscores */ 4805 return s[0] != '_' && /* X */ 4806 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 4807 s[4] != '_'; /* Y */ 4808 } 4809 4810 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 4811 * before last triple underscore. Struct name part after last triple 4812 * underscore is ignored by BPF CO-RE relocation during relocation matching. 4813 */ 4814 size_t bpf_core_essential_name_len(const char *name) 4815 { 4816 size_t n = strlen(name); 4817 int i; 4818 4819 for (i = n - 5; i >= 0; i--) { 4820 if (bpf_core_is_flavor_sep(name + i)) 4821 return i + 1; 4822 } 4823 return n; 4824 } 4825 4826 static void bpf_core_free_cands(struct bpf_core_cand_list *cands) 4827 { 4828 free(cands->cands); 4829 free(cands); 4830 } 4831 4832 static int bpf_core_add_cands(struct bpf_core_cand *local_cand, 4833 size_t local_essent_len, 4834 const struct btf *targ_btf, 4835 const char *targ_btf_name, 4836 int targ_start_id, 4837 struct bpf_core_cand_list *cands) 4838 { 4839 struct bpf_core_cand *new_cands, *cand; 4840 const struct btf_type *t; 4841 const char *targ_name; 4842 size_t targ_essent_len; 4843 int n, i; 4844 4845 n = btf__get_nr_types(targ_btf); 4846 for (i = targ_start_id; i <= n; i++) { 4847 t = btf__type_by_id(targ_btf, i); 4848 if (btf_kind(t) != btf_kind(local_cand->t)) 4849 continue; 4850 4851 targ_name = btf__name_by_offset(targ_btf, t->name_off); 4852 if (str_is_empty(targ_name)) 4853 continue; 4854 4855 targ_essent_len = bpf_core_essential_name_len(targ_name); 4856 if (targ_essent_len != local_essent_len) 4857 continue; 4858 4859 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0) 4860 continue; 4861 4862 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 4863 local_cand->id, btf_kind_str(local_cand->t), 4864 local_cand->name, i, btf_kind_str(t), targ_name, 4865 targ_btf_name); 4866 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 4867 sizeof(*cands->cands)); 4868 if (!new_cands) 4869 return -ENOMEM; 4870 4871 cand = &new_cands[cands->len]; 4872 cand->btf = targ_btf; 4873 cand->t = t; 4874 cand->name = targ_name; 4875 cand->id = i; 4876 4877 cands->cands = new_cands; 4878 cands->len++; 4879 } 4880 return 0; 4881 } 4882 4883 static int load_module_btfs(struct bpf_object *obj) 4884 { 4885 struct bpf_btf_info info; 4886 struct module_btf *mod_btf; 4887 struct btf *btf; 4888 char name[64]; 4889 __u32 id = 0, len; 4890 int err, fd; 4891 4892 if (obj->btf_modules_loaded) 4893 return 0; 4894 4895 if (obj->gen_loader) 4896 return 0; 4897 4898 /* don't do this again, even if we find no module BTFs */ 4899 obj->btf_modules_loaded = true; 4900 4901 /* kernel too old to support module BTFs */ 4902 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 4903 return 0; 4904 4905 while (true) { 4906 err = bpf_btf_get_next_id(id, &id); 4907 if (err && errno == ENOENT) 4908 return 0; 4909 if (err) { 4910 err = -errno; 4911 pr_warn("failed to iterate BTF objects: %d\n", err); 4912 return err; 4913 } 4914 4915 fd = bpf_btf_get_fd_by_id(id); 4916 if (fd < 0) { 4917 if (errno == ENOENT) 4918 continue; /* expected race: BTF was unloaded */ 4919 err = -errno; 4920 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 4921 return err; 4922 } 4923 4924 len = sizeof(info); 4925 memset(&info, 0, sizeof(info)); 4926 info.name = ptr_to_u64(name); 4927 info.name_len = sizeof(name); 4928 4929 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4930 if (err) { 4931 err = -errno; 4932 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 4933 goto err_out; 4934 } 4935 4936 /* ignore non-module BTFs */ 4937 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 4938 close(fd); 4939 continue; 4940 } 4941 4942 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 4943 err = libbpf_get_error(btf); 4944 if (err) { 4945 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", 4946 name, id, err); 4947 goto err_out; 4948 } 4949 4950 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 4951 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 4952 if (err) 4953 goto err_out; 4954 4955 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 4956 4957 mod_btf->btf = btf; 4958 mod_btf->id = id; 4959 mod_btf->fd = fd; 4960 mod_btf->name = strdup(name); 4961 if (!mod_btf->name) { 4962 err = -ENOMEM; 4963 goto err_out; 4964 } 4965 continue; 4966 4967 err_out: 4968 close(fd); 4969 return err; 4970 } 4971 4972 return 0; 4973 } 4974 4975 static struct bpf_core_cand_list * 4976 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 4977 { 4978 struct bpf_core_cand local_cand = {}; 4979 struct bpf_core_cand_list *cands; 4980 const struct btf *main_btf; 4981 size_t local_essent_len; 4982 int err, i; 4983 4984 local_cand.btf = local_btf; 4985 local_cand.t = btf__type_by_id(local_btf, local_type_id); 4986 if (!local_cand.t) 4987 return ERR_PTR(-EINVAL); 4988 4989 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off); 4990 if (str_is_empty(local_cand.name)) 4991 return ERR_PTR(-EINVAL); 4992 local_essent_len = bpf_core_essential_name_len(local_cand.name); 4993 4994 cands = calloc(1, sizeof(*cands)); 4995 if (!cands) 4996 return ERR_PTR(-ENOMEM); 4997 4998 /* Attempt to find target candidates in vmlinux BTF first */ 4999 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 5000 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 5001 if (err) 5002 goto err_out; 5003 5004 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 5005 if (cands->len) 5006 return cands; 5007 5008 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 5009 if (obj->btf_vmlinux_override) 5010 return cands; 5011 5012 /* now look through module BTFs, trying to still find candidates */ 5013 err = load_module_btfs(obj); 5014 if (err) 5015 goto err_out; 5016 5017 for (i = 0; i < obj->btf_module_cnt; i++) { 5018 err = bpf_core_add_cands(&local_cand, local_essent_len, 5019 obj->btf_modules[i].btf, 5020 obj->btf_modules[i].name, 5021 btf__get_nr_types(obj->btf_vmlinux) + 1, 5022 cands); 5023 if (err) 5024 goto err_out; 5025 } 5026 5027 return cands; 5028 err_out: 5029 bpf_core_free_cands(cands); 5030 return ERR_PTR(err); 5031 } 5032 5033 /* Check local and target types for compatibility. This check is used for 5034 * type-based CO-RE relocations and follow slightly different rules than 5035 * field-based relocations. This function assumes that root types were already 5036 * checked for name match. Beyond that initial root-level name check, names 5037 * are completely ignored. Compatibility rules are as follows: 5038 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 5039 * kind should match for local and target types (i.e., STRUCT is not 5040 * compatible with UNION); 5041 * - for ENUMs, the size is ignored; 5042 * - for INT, size and signedness are ignored; 5043 * - for ARRAY, dimensionality is ignored, element types are checked for 5044 * compatibility recursively; 5045 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 5046 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 5047 * - FUNC_PROTOs are compatible if they have compatible signature: same 5048 * number of input args and compatible return and argument types. 5049 * These rules are not set in stone and probably will be adjusted as we get 5050 * more experience with using BPF CO-RE relocations. 5051 */ 5052 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 5053 const struct btf *targ_btf, __u32 targ_id) 5054 { 5055 const struct btf_type *local_type, *targ_type; 5056 int depth = 32; /* max recursion depth */ 5057 5058 /* caller made sure that names match (ignoring flavor suffix) */ 5059 local_type = btf__type_by_id(local_btf, local_id); 5060 targ_type = btf__type_by_id(targ_btf, targ_id); 5061 if (btf_kind(local_type) != btf_kind(targ_type)) 5062 return 0; 5063 5064 recur: 5065 depth--; 5066 if (depth < 0) 5067 return -EINVAL; 5068 5069 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 5070 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 5071 if (!local_type || !targ_type) 5072 return -EINVAL; 5073 5074 if (btf_kind(local_type) != btf_kind(targ_type)) 5075 return 0; 5076 5077 switch (btf_kind(local_type)) { 5078 case BTF_KIND_UNKN: 5079 case BTF_KIND_STRUCT: 5080 case BTF_KIND_UNION: 5081 case BTF_KIND_ENUM: 5082 case BTF_KIND_FWD: 5083 return 1; 5084 case BTF_KIND_INT: 5085 /* just reject deprecated bitfield-like integers; all other 5086 * integers are by default compatible between each other 5087 */ 5088 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; 5089 case BTF_KIND_PTR: 5090 local_id = local_type->type; 5091 targ_id = targ_type->type; 5092 goto recur; 5093 case BTF_KIND_ARRAY: 5094 local_id = btf_array(local_type)->type; 5095 targ_id = btf_array(targ_type)->type; 5096 goto recur; 5097 case BTF_KIND_FUNC_PROTO: { 5098 struct btf_param *local_p = btf_params(local_type); 5099 struct btf_param *targ_p = btf_params(targ_type); 5100 __u16 local_vlen = btf_vlen(local_type); 5101 __u16 targ_vlen = btf_vlen(targ_type); 5102 int i, err; 5103 5104 if (local_vlen != targ_vlen) 5105 return 0; 5106 5107 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { 5108 skip_mods_and_typedefs(local_btf, local_p->type, &local_id); 5109 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); 5110 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id); 5111 if (err <= 0) 5112 return err; 5113 } 5114 5115 /* tail recurse for return type check */ 5116 skip_mods_and_typedefs(local_btf, local_type->type, &local_id); 5117 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); 5118 goto recur; 5119 } 5120 default: 5121 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", 5122 btf_kind_str(local_type), local_id, targ_id); 5123 return 0; 5124 } 5125 } 5126 5127 static size_t bpf_core_hash_fn(const void *key, void *ctx) 5128 { 5129 return (size_t)key; 5130 } 5131 5132 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) 5133 { 5134 return k1 == k2; 5135 } 5136 5137 static void *u32_as_hash_key(__u32 x) 5138 { 5139 return (void *)(uintptr_t)x; 5140 } 5141 5142 static int bpf_core_apply_relo(struct bpf_program *prog, 5143 const struct bpf_core_relo *relo, 5144 int relo_idx, 5145 const struct btf *local_btf, 5146 struct hashmap *cand_cache) 5147 { 5148 const void *type_key = u32_as_hash_key(relo->type_id); 5149 struct bpf_core_cand_list *cands = NULL; 5150 const char *prog_name = prog->name; 5151 const struct btf_type *local_type; 5152 const char *local_name; 5153 __u32 local_id = relo->type_id; 5154 struct bpf_insn *insn; 5155 int insn_idx, err; 5156 5157 if (relo->insn_off % BPF_INSN_SZ) 5158 return -EINVAL; 5159 insn_idx = relo->insn_off / BPF_INSN_SZ; 5160 /* adjust insn_idx from section frame of reference to the local 5161 * program's frame of reference; (sub-)program code is not yet 5162 * relocated, so it's enough to just subtract in-section offset 5163 */ 5164 insn_idx = insn_idx - prog->sec_insn_off; 5165 if (insn_idx > prog->insns_cnt) 5166 return -EINVAL; 5167 insn = &prog->insns[insn_idx]; 5168 5169 local_type = btf__type_by_id(local_btf, local_id); 5170 if (!local_type) 5171 return -EINVAL; 5172 5173 local_name = btf__name_by_offset(local_btf, local_type->name_off); 5174 if (!local_name) 5175 return -EINVAL; 5176 5177 if (prog->obj->gen_loader) { 5178 pr_warn("// TODO core_relo: prog %td insn[%d] %s kind %d\n", 5179 prog - prog->obj->programs, relo->insn_off / 8, 5180 local_name, relo->kind); 5181 return -ENOTSUP; 5182 } 5183 5184 if (relo->kind != BPF_TYPE_ID_LOCAL && 5185 !hashmap__find(cand_cache, type_key, (void **)&cands)) { 5186 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 5187 if (IS_ERR(cands)) { 5188 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 5189 prog_name, relo_idx, local_id, btf_kind_str(local_type), 5190 local_name, PTR_ERR(cands)); 5191 return PTR_ERR(cands); 5192 } 5193 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL); 5194 if (err) { 5195 bpf_core_free_cands(cands); 5196 return err; 5197 } 5198 } 5199 5200 return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands); 5201 } 5202 5203 static int 5204 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 5205 { 5206 const struct btf_ext_info_sec *sec; 5207 const struct bpf_core_relo *rec; 5208 const struct btf_ext_info *seg; 5209 struct hashmap_entry *entry; 5210 struct hashmap *cand_cache = NULL; 5211 struct bpf_program *prog; 5212 const char *sec_name; 5213 int i, err = 0, insn_idx, sec_idx; 5214 5215 if (obj->btf_ext->core_relo_info.len == 0) 5216 return 0; 5217 5218 if (targ_btf_path) { 5219 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 5220 err = libbpf_get_error(obj->btf_vmlinux_override); 5221 if (err) { 5222 pr_warn("failed to parse target BTF: %d\n", err); 5223 return err; 5224 } 5225 } 5226 5227 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 5228 if (IS_ERR(cand_cache)) { 5229 err = PTR_ERR(cand_cache); 5230 goto out; 5231 } 5232 5233 seg = &obj->btf_ext->core_relo_info; 5234 for_each_btf_ext_sec(seg, sec) { 5235 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5236 if (str_is_empty(sec_name)) { 5237 err = -EINVAL; 5238 goto out; 5239 } 5240 /* bpf_object's ELF is gone by now so it's not easy to find 5241 * section index by section name, but we can find *any* 5242 * bpf_program within desired section name and use it's 5243 * prog->sec_idx to do a proper search by section index and 5244 * instruction offset 5245 */ 5246 prog = NULL; 5247 for (i = 0; i < obj->nr_programs; i++) { 5248 prog = &obj->programs[i]; 5249 if (strcmp(prog->sec_name, sec_name) == 0) 5250 break; 5251 } 5252 if (!prog) { 5253 pr_warn("sec '%s': failed to find a BPF program\n", sec_name); 5254 return -ENOENT; 5255 } 5256 sec_idx = prog->sec_idx; 5257 5258 pr_debug("sec '%s': found %d CO-RE relocations\n", 5259 sec_name, sec->num_info); 5260 5261 for_each_btf_ext_rec(seg, sec, i, rec) { 5262 insn_idx = rec->insn_off / BPF_INSN_SZ; 5263 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 5264 if (!prog) { 5265 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n", 5266 sec_name, insn_idx, i); 5267 err = -EINVAL; 5268 goto out; 5269 } 5270 /* no need to apply CO-RE relocation if the program is 5271 * not going to be loaded 5272 */ 5273 if (!prog->load) 5274 continue; 5275 5276 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache); 5277 if (err) { 5278 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 5279 prog->name, i, err); 5280 goto out; 5281 } 5282 } 5283 } 5284 5285 out: 5286 /* obj->btf_vmlinux and module BTFs are freed after object load */ 5287 btf__free(obj->btf_vmlinux_override); 5288 obj->btf_vmlinux_override = NULL; 5289 5290 if (!IS_ERR_OR_NULL(cand_cache)) { 5291 hashmap__for_each_entry(cand_cache, entry, i) { 5292 bpf_core_free_cands(entry->value); 5293 } 5294 hashmap__free(cand_cache); 5295 } 5296 return err; 5297 } 5298 5299 /* Relocate data references within program code: 5300 * - map references; 5301 * - global variable references; 5302 * - extern references. 5303 */ 5304 static int 5305 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 5306 { 5307 int i; 5308 5309 for (i = 0; i < prog->nr_reloc; i++) { 5310 struct reloc_desc *relo = &prog->reloc_desc[i]; 5311 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5312 struct extern_desc *ext; 5313 5314 switch (relo->type) { 5315 case RELO_LD64: 5316 if (obj->gen_loader) { 5317 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 5318 insn[0].imm = relo->map_idx; 5319 } else { 5320 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 5321 insn[0].imm = obj->maps[relo->map_idx].fd; 5322 } 5323 break; 5324 case RELO_DATA: 5325 insn[1].imm = insn[0].imm + relo->sym_off; 5326 if (obj->gen_loader) { 5327 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5328 insn[0].imm = relo->map_idx; 5329 } else { 5330 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5331 insn[0].imm = obj->maps[relo->map_idx].fd; 5332 } 5333 break; 5334 case RELO_EXTERN_VAR: 5335 ext = &obj->externs[relo->sym_off]; 5336 if (ext->type == EXT_KCFG) { 5337 if (obj->gen_loader) { 5338 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5339 insn[0].imm = obj->kconfig_map_idx; 5340 } else { 5341 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5342 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 5343 } 5344 insn[1].imm = ext->kcfg.data_off; 5345 } else /* EXT_KSYM */ { 5346 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 5347 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 5348 insn[0].imm = ext->ksym.kernel_btf_id; 5349 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 5350 } else { /* typeless ksyms or unresolved typed ksyms */ 5351 insn[0].imm = (__u32)ext->ksym.addr; 5352 insn[1].imm = ext->ksym.addr >> 32; 5353 } 5354 } 5355 break; 5356 case RELO_EXTERN_FUNC: 5357 ext = &obj->externs[relo->sym_off]; 5358 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 5359 insn[0].imm = ext->ksym.kernel_btf_id; 5360 break; 5361 case RELO_SUBPROG_ADDR: 5362 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 5363 pr_warn("prog '%s': relo #%d: bad insn\n", 5364 prog->name, i); 5365 return -EINVAL; 5366 } 5367 /* handled already */ 5368 break; 5369 case RELO_CALL: 5370 /* handled already */ 5371 break; 5372 default: 5373 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 5374 prog->name, i, relo->type); 5375 return -EINVAL; 5376 } 5377 } 5378 5379 return 0; 5380 } 5381 5382 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 5383 const struct bpf_program *prog, 5384 const struct btf_ext_info *ext_info, 5385 void **prog_info, __u32 *prog_rec_cnt, 5386 __u32 *prog_rec_sz) 5387 { 5388 void *copy_start = NULL, *copy_end = NULL; 5389 void *rec, *rec_end, *new_prog_info; 5390 const struct btf_ext_info_sec *sec; 5391 size_t old_sz, new_sz; 5392 const char *sec_name; 5393 int i, off_adj; 5394 5395 for_each_btf_ext_sec(ext_info, sec) { 5396 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5397 if (!sec_name) 5398 return -EINVAL; 5399 if (strcmp(sec_name, prog->sec_name) != 0) 5400 continue; 5401 5402 for_each_btf_ext_rec(ext_info, sec, i, rec) { 5403 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 5404 5405 if (insn_off < prog->sec_insn_off) 5406 continue; 5407 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 5408 break; 5409 5410 if (!copy_start) 5411 copy_start = rec; 5412 copy_end = rec + ext_info->rec_size; 5413 } 5414 5415 if (!copy_start) 5416 return -ENOENT; 5417 5418 /* append func/line info of a given (sub-)program to the main 5419 * program func/line info 5420 */ 5421 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 5422 new_sz = old_sz + (copy_end - copy_start); 5423 new_prog_info = realloc(*prog_info, new_sz); 5424 if (!new_prog_info) 5425 return -ENOMEM; 5426 *prog_info = new_prog_info; 5427 *prog_rec_cnt = new_sz / ext_info->rec_size; 5428 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 5429 5430 /* Kernel instruction offsets are in units of 8-byte 5431 * instructions, while .BTF.ext instruction offsets generated 5432 * by Clang are in units of bytes. So convert Clang offsets 5433 * into kernel offsets and adjust offset according to program 5434 * relocated position. 5435 */ 5436 off_adj = prog->sub_insn_off - prog->sec_insn_off; 5437 rec = new_prog_info + old_sz; 5438 rec_end = new_prog_info + new_sz; 5439 for (; rec < rec_end; rec += ext_info->rec_size) { 5440 __u32 *insn_off = rec; 5441 5442 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 5443 } 5444 *prog_rec_sz = ext_info->rec_size; 5445 return 0; 5446 } 5447 5448 return -ENOENT; 5449 } 5450 5451 static int 5452 reloc_prog_func_and_line_info(const struct bpf_object *obj, 5453 struct bpf_program *main_prog, 5454 const struct bpf_program *prog) 5455 { 5456 int err; 5457 5458 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 5459 * supprot func/line info 5460 */ 5461 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 5462 return 0; 5463 5464 /* only attempt func info relocation if main program's func_info 5465 * relocation was successful 5466 */ 5467 if (main_prog != prog && !main_prog->func_info) 5468 goto line_info; 5469 5470 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 5471 &main_prog->func_info, 5472 &main_prog->func_info_cnt, 5473 &main_prog->func_info_rec_size); 5474 if (err) { 5475 if (err != -ENOENT) { 5476 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 5477 prog->name, err); 5478 return err; 5479 } 5480 if (main_prog->func_info) { 5481 /* 5482 * Some info has already been found but has problem 5483 * in the last btf_ext reloc. Must have to error out. 5484 */ 5485 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 5486 return err; 5487 } 5488 /* Have problem loading the very first info. Ignore the rest. */ 5489 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 5490 prog->name); 5491 } 5492 5493 line_info: 5494 /* don't relocate line info if main program's relocation failed */ 5495 if (main_prog != prog && !main_prog->line_info) 5496 return 0; 5497 5498 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 5499 &main_prog->line_info, 5500 &main_prog->line_info_cnt, 5501 &main_prog->line_info_rec_size); 5502 if (err) { 5503 if (err != -ENOENT) { 5504 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 5505 prog->name, err); 5506 return err; 5507 } 5508 if (main_prog->line_info) { 5509 /* 5510 * Some info has already been found but has problem 5511 * in the last btf_ext reloc. Must have to error out. 5512 */ 5513 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 5514 return err; 5515 } 5516 /* Have problem loading the very first info. Ignore the rest. */ 5517 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 5518 prog->name); 5519 } 5520 return 0; 5521 } 5522 5523 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 5524 { 5525 size_t insn_idx = *(const size_t *)key; 5526 const struct reloc_desc *relo = elem; 5527 5528 if (insn_idx == relo->insn_idx) 5529 return 0; 5530 return insn_idx < relo->insn_idx ? -1 : 1; 5531 } 5532 5533 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 5534 { 5535 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 5536 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 5537 } 5538 5539 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 5540 { 5541 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 5542 struct reloc_desc *relos; 5543 int i; 5544 5545 if (main_prog == subprog) 5546 return 0; 5547 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 5548 if (!relos) 5549 return -ENOMEM; 5550 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 5551 sizeof(*relos) * subprog->nr_reloc); 5552 5553 for (i = main_prog->nr_reloc; i < new_cnt; i++) 5554 relos[i].insn_idx += subprog->sub_insn_off; 5555 /* After insn_idx adjustment the 'relos' array is still sorted 5556 * by insn_idx and doesn't break bsearch. 5557 */ 5558 main_prog->reloc_desc = relos; 5559 main_prog->nr_reloc = new_cnt; 5560 return 0; 5561 } 5562 5563 static int 5564 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 5565 struct bpf_program *prog) 5566 { 5567 size_t sub_insn_idx, insn_idx, new_cnt; 5568 struct bpf_program *subprog; 5569 struct bpf_insn *insns, *insn; 5570 struct reloc_desc *relo; 5571 int err; 5572 5573 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 5574 if (err) 5575 return err; 5576 5577 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 5578 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 5579 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 5580 continue; 5581 5582 relo = find_prog_insn_relo(prog, insn_idx); 5583 if (relo && relo->type == RELO_EXTERN_FUNC) 5584 /* kfunc relocations will be handled later 5585 * in bpf_object__relocate_data() 5586 */ 5587 continue; 5588 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 5589 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 5590 prog->name, insn_idx, relo->type); 5591 return -LIBBPF_ERRNO__RELOC; 5592 } 5593 if (relo) { 5594 /* sub-program instruction index is a combination of 5595 * an offset of a symbol pointed to by relocation and 5596 * call instruction's imm field; for global functions, 5597 * call always has imm = -1, but for static functions 5598 * relocation is against STT_SECTION and insn->imm 5599 * points to a start of a static function 5600 * 5601 * for subprog addr relocation, the relo->sym_off + insn->imm is 5602 * the byte offset in the corresponding section. 5603 */ 5604 if (relo->type == RELO_CALL) 5605 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 5606 else 5607 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 5608 } else if (insn_is_pseudo_func(insn)) { 5609 /* 5610 * RELO_SUBPROG_ADDR relo is always emitted even if both 5611 * functions are in the same section, so it shouldn't reach here. 5612 */ 5613 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 5614 prog->name, insn_idx); 5615 return -LIBBPF_ERRNO__RELOC; 5616 } else { 5617 /* if subprogram call is to a static function within 5618 * the same ELF section, there won't be any relocation 5619 * emitted, but it also means there is no additional 5620 * offset necessary, insns->imm is relative to 5621 * instruction's original position within the section 5622 */ 5623 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 5624 } 5625 5626 /* we enforce that sub-programs should be in .text section */ 5627 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 5628 if (!subprog) { 5629 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 5630 prog->name); 5631 return -LIBBPF_ERRNO__RELOC; 5632 } 5633 5634 /* if it's the first call instruction calling into this 5635 * subprogram (meaning this subprog hasn't been processed 5636 * yet) within the context of current main program: 5637 * - append it at the end of main program's instructions blog; 5638 * - process is recursively, while current program is put on hold; 5639 * - if that subprogram calls some other not yet processes 5640 * subprogram, same thing will happen recursively until 5641 * there are no more unprocesses subprograms left to append 5642 * and relocate. 5643 */ 5644 if (subprog->sub_insn_off == 0) { 5645 subprog->sub_insn_off = main_prog->insns_cnt; 5646 5647 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 5648 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 5649 if (!insns) { 5650 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 5651 return -ENOMEM; 5652 } 5653 main_prog->insns = insns; 5654 main_prog->insns_cnt = new_cnt; 5655 5656 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 5657 subprog->insns_cnt * sizeof(*insns)); 5658 5659 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 5660 main_prog->name, subprog->insns_cnt, subprog->name); 5661 5662 /* The subprog insns are now appended. Append its relos too. */ 5663 err = append_subprog_relos(main_prog, subprog); 5664 if (err) 5665 return err; 5666 err = bpf_object__reloc_code(obj, main_prog, subprog); 5667 if (err) 5668 return err; 5669 } 5670 5671 /* main_prog->insns memory could have been re-allocated, so 5672 * calculate pointer again 5673 */ 5674 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 5675 /* calculate correct instruction position within current main 5676 * prog; each main prog can have a different set of 5677 * subprograms appended (potentially in different order as 5678 * well), so position of any subprog can be different for 5679 * different main programs */ 5680 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 5681 5682 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 5683 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 5684 } 5685 5686 return 0; 5687 } 5688 5689 /* 5690 * Relocate sub-program calls. 5691 * 5692 * Algorithm operates as follows. Each entry-point BPF program (referred to as 5693 * main prog) is processed separately. For each subprog (non-entry functions, 5694 * that can be called from either entry progs or other subprogs) gets their 5695 * sub_insn_off reset to zero. This serves as indicator that this subprogram 5696 * hasn't been yet appended and relocated within current main prog. Once its 5697 * relocated, sub_insn_off will point at the position within current main prog 5698 * where given subprog was appended. This will further be used to relocate all 5699 * the call instructions jumping into this subprog. 5700 * 5701 * We start with main program and process all call instructions. If the call 5702 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 5703 * is zero), subprog instructions are appended at the end of main program's 5704 * instruction array. Then main program is "put on hold" while we recursively 5705 * process newly appended subprogram. If that subprogram calls into another 5706 * subprogram that hasn't been appended, new subprogram is appended again to 5707 * the *main* prog's instructions (subprog's instructions are always left 5708 * untouched, as they need to be in unmodified state for subsequent main progs 5709 * and subprog instructions are always sent only as part of a main prog) and 5710 * the process continues recursively. Once all the subprogs called from a main 5711 * prog or any of its subprogs are appended (and relocated), all their 5712 * positions within finalized instructions array are known, so it's easy to 5713 * rewrite call instructions with correct relative offsets, corresponding to 5714 * desired target subprog. 5715 * 5716 * Its important to realize that some subprogs might not be called from some 5717 * main prog and any of its called/used subprogs. Those will keep their 5718 * subprog->sub_insn_off as zero at all times and won't be appended to current 5719 * main prog and won't be relocated within the context of current main prog. 5720 * They might still be used from other main progs later. 5721 * 5722 * Visually this process can be shown as below. Suppose we have two main 5723 * programs mainA and mainB and BPF object contains three subprogs: subA, 5724 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 5725 * subC both call subB: 5726 * 5727 * +--------+ +-------+ 5728 * | v v | 5729 * +--+---+ +--+-+-+ +---+--+ 5730 * | subA | | subB | | subC | 5731 * +--+---+ +------+ +---+--+ 5732 * ^ ^ 5733 * | | 5734 * +---+-------+ +------+----+ 5735 * | mainA | | mainB | 5736 * +-----------+ +-----------+ 5737 * 5738 * We'll start relocating mainA, will find subA, append it and start 5739 * processing sub A recursively: 5740 * 5741 * +-----------+------+ 5742 * | mainA | subA | 5743 * +-----------+------+ 5744 * 5745 * At this point we notice that subB is used from subA, so we append it and 5746 * relocate (there are no further subcalls from subB): 5747 * 5748 * +-----------+------+------+ 5749 * | mainA | subA | subB | 5750 * +-----------+------+------+ 5751 * 5752 * At this point, we relocate subA calls, then go one level up and finish with 5753 * relocatin mainA calls. mainA is done. 5754 * 5755 * For mainB process is similar but results in different order. We start with 5756 * mainB and skip subA and subB, as mainB never calls them (at least 5757 * directly), but we see subC is needed, so we append and start processing it: 5758 * 5759 * +-----------+------+ 5760 * | mainB | subC | 5761 * +-----------+------+ 5762 * Now we see subC needs subB, so we go back to it, append and relocate it: 5763 * 5764 * +-----------+------+------+ 5765 * | mainB | subC | subB | 5766 * +-----------+------+------+ 5767 * 5768 * At this point we unwind recursion, relocate calls in subC, then in mainB. 5769 */ 5770 static int 5771 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 5772 { 5773 struct bpf_program *subprog; 5774 int i, err; 5775 5776 /* mark all subprogs as not relocated (yet) within the context of 5777 * current main program 5778 */ 5779 for (i = 0; i < obj->nr_programs; i++) { 5780 subprog = &obj->programs[i]; 5781 if (!prog_is_subprog(obj, subprog)) 5782 continue; 5783 5784 subprog->sub_insn_off = 0; 5785 } 5786 5787 err = bpf_object__reloc_code(obj, prog, prog); 5788 if (err) 5789 return err; 5790 5791 5792 return 0; 5793 } 5794 5795 static void 5796 bpf_object__free_relocs(struct bpf_object *obj) 5797 { 5798 struct bpf_program *prog; 5799 int i; 5800 5801 /* free up relocation descriptors */ 5802 for (i = 0; i < obj->nr_programs; i++) { 5803 prog = &obj->programs[i]; 5804 zfree(&prog->reloc_desc); 5805 prog->nr_reloc = 0; 5806 } 5807 } 5808 5809 static int 5810 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 5811 { 5812 struct bpf_program *prog; 5813 size_t i, j; 5814 int err; 5815 5816 if (obj->btf_ext) { 5817 err = bpf_object__relocate_core(obj, targ_btf_path); 5818 if (err) { 5819 pr_warn("failed to perform CO-RE relocations: %d\n", 5820 err); 5821 return err; 5822 } 5823 } 5824 5825 /* Before relocating calls pre-process relocations and mark 5826 * few ld_imm64 instructions that points to subprogs. 5827 * Otherwise bpf_object__reloc_code() later would have to consider 5828 * all ld_imm64 insns as relocation candidates. That would 5829 * reduce relocation speed, since amount of find_prog_insn_relo() 5830 * would increase and most of them will fail to find a relo. 5831 */ 5832 for (i = 0; i < obj->nr_programs; i++) { 5833 prog = &obj->programs[i]; 5834 for (j = 0; j < prog->nr_reloc; j++) { 5835 struct reloc_desc *relo = &prog->reloc_desc[j]; 5836 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5837 5838 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 5839 if (relo->type == RELO_SUBPROG_ADDR) 5840 insn[0].src_reg = BPF_PSEUDO_FUNC; 5841 } 5842 } 5843 5844 /* relocate subprogram calls and append used subprograms to main 5845 * programs; each copy of subprogram code needs to be relocated 5846 * differently for each main program, because its code location might 5847 * have changed. 5848 * Append subprog relos to main programs to allow data relos to be 5849 * processed after text is completely relocated. 5850 */ 5851 for (i = 0; i < obj->nr_programs; i++) { 5852 prog = &obj->programs[i]; 5853 /* sub-program's sub-calls are relocated within the context of 5854 * its main program only 5855 */ 5856 if (prog_is_subprog(obj, prog)) 5857 continue; 5858 5859 err = bpf_object__relocate_calls(obj, prog); 5860 if (err) { 5861 pr_warn("prog '%s': failed to relocate calls: %d\n", 5862 prog->name, err); 5863 return err; 5864 } 5865 } 5866 /* Process data relos for main programs */ 5867 for (i = 0; i < obj->nr_programs; i++) { 5868 prog = &obj->programs[i]; 5869 if (prog_is_subprog(obj, prog)) 5870 continue; 5871 err = bpf_object__relocate_data(obj, prog); 5872 if (err) { 5873 pr_warn("prog '%s': failed to relocate data references: %d\n", 5874 prog->name, err); 5875 return err; 5876 } 5877 } 5878 if (!obj->gen_loader) 5879 bpf_object__free_relocs(obj); 5880 return 0; 5881 } 5882 5883 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 5884 GElf_Shdr *shdr, Elf_Data *data); 5885 5886 static int bpf_object__collect_map_relos(struct bpf_object *obj, 5887 GElf_Shdr *shdr, Elf_Data *data) 5888 { 5889 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 5890 int i, j, nrels, new_sz; 5891 const struct btf_var_secinfo *vi = NULL; 5892 const struct btf_type *sec, *var, *def; 5893 struct bpf_map *map = NULL, *targ_map; 5894 const struct btf_member *member; 5895 const char *name, *mname; 5896 Elf_Data *symbols; 5897 unsigned int moff; 5898 GElf_Sym sym; 5899 GElf_Rel rel; 5900 void *tmp; 5901 5902 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 5903 return -EINVAL; 5904 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 5905 if (!sec) 5906 return -EINVAL; 5907 5908 symbols = obj->efile.symbols; 5909 nrels = shdr->sh_size / shdr->sh_entsize; 5910 for (i = 0; i < nrels; i++) { 5911 if (!gelf_getrel(data, i, &rel)) { 5912 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 5913 return -LIBBPF_ERRNO__FORMAT; 5914 } 5915 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 5916 pr_warn(".maps relo #%d: symbol %zx not found\n", 5917 i, (size_t)GELF_R_SYM(rel.r_info)); 5918 return -LIBBPF_ERRNO__FORMAT; 5919 } 5920 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 5921 if (sym.st_shndx != obj->efile.btf_maps_shndx) { 5922 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 5923 i, name); 5924 return -LIBBPF_ERRNO__RELOC; 5925 } 5926 5927 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n", 5928 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value, 5929 (size_t)rel.r_offset, sym.st_name, name); 5930 5931 for (j = 0; j < obj->nr_maps; j++) { 5932 map = &obj->maps[j]; 5933 if (map->sec_idx != obj->efile.btf_maps_shndx) 5934 continue; 5935 5936 vi = btf_var_secinfos(sec) + map->btf_var_idx; 5937 if (vi->offset <= rel.r_offset && 5938 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size) 5939 break; 5940 } 5941 if (j == obj->nr_maps) { 5942 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n", 5943 i, name, (size_t)rel.r_offset); 5944 return -EINVAL; 5945 } 5946 5947 if (!bpf_map_type__is_map_in_map(map->def.type)) 5948 return -EINVAL; 5949 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 5950 map->def.key_size != sizeof(int)) { 5951 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 5952 i, map->name, sizeof(int)); 5953 return -EINVAL; 5954 } 5955 5956 targ_map = bpf_object__find_map_by_name(obj, name); 5957 if (!targ_map) 5958 return -ESRCH; 5959 5960 var = btf__type_by_id(obj->btf, vi->type); 5961 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 5962 if (btf_vlen(def) == 0) 5963 return -EINVAL; 5964 member = btf_members(def) + btf_vlen(def) - 1; 5965 mname = btf__name_by_offset(obj->btf, member->name_off); 5966 if (strcmp(mname, "values")) 5967 return -EINVAL; 5968 5969 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 5970 if (rel.r_offset - vi->offset < moff) 5971 return -EINVAL; 5972 5973 moff = rel.r_offset - vi->offset - moff; 5974 /* here we use BPF pointer size, which is always 64 bit, as we 5975 * are parsing ELF that was built for BPF target 5976 */ 5977 if (moff % bpf_ptr_sz) 5978 return -EINVAL; 5979 moff /= bpf_ptr_sz; 5980 if (moff >= map->init_slots_sz) { 5981 new_sz = moff + 1; 5982 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 5983 if (!tmp) 5984 return -ENOMEM; 5985 map->init_slots = tmp; 5986 memset(map->init_slots + map->init_slots_sz, 0, 5987 (new_sz - map->init_slots_sz) * host_ptr_sz); 5988 map->init_slots_sz = new_sz; 5989 } 5990 map->init_slots[moff] = targ_map; 5991 5992 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n", 5993 i, map->name, moff, name); 5994 } 5995 5996 return 0; 5997 } 5998 5999 static int cmp_relocs(const void *_a, const void *_b) 6000 { 6001 const struct reloc_desc *a = _a; 6002 const struct reloc_desc *b = _b; 6003 6004 if (a->insn_idx != b->insn_idx) 6005 return a->insn_idx < b->insn_idx ? -1 : 1; 6006 6007 /* no two relocations should have the same insn_idx, but ... */ 6008 if (a->type != b->type) 6009 return a->type < b->type ? -1 : 1; 6010 6011 return 0; 6012 } 6013 6014 static int bpf_object__collect_relos(struct bpf_object *obj) 6015 { 6016 int i, err; 6017 6018 for (i = 0; i < obj->efile.nr_reloc_sects; i++) { 6019 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; 6020 Elf_Data *data = obj->efile.reloc_sects[i].data; 6021 int idx = shdr->sh_info; 6022 6023 if (shdr->sh_type != SHT_REL) { 6024 pr_warn("internal error at %d\n", __LINE__); 6025 return -LIBBPF_ERRNO__INTERNAL; 6026 } 6027 6028 if (idx == obj->efile.st_ops_shndx) 6029 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 6030 else if (idx == obj->efile.btf_maps_shndx) 6031 err = bpf_object__collect_map_relos(obj, shdr, data); 6032 else 6033 err = bpf_object__collect_prog_relos(obj, shdr, data); 6034 if (err) 6035 return err; 6036 } 6037 6038 for (i = 0; i < obj->nr_programs; i++) { 6039 struct bpf_program *p = &obj->programs[i]; 6040 6041 if (!p->nr_reloc) 6042 continue; 6043 6044 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 6045 } 6046 return 0; 6047 } 6048 6049 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 6050 { 6051 if (BPF_CLASS(insn->code) == BPF_JMP && 6052 BPF_OP(insn->code) == BPF_CALL && 6053 BPF_SRC(insn->code) == BPF_K && 6054 insn->src_reg == 0 && 6055 insn->dst_reg == 0) { 6056 *func_id = insn->imm; 6057 return true; 6058 } 6059 return false; 6060 } 6061 6062 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 6063 { 6064 struct bpf_insn *insn = prog->insns; 6065 enum bpf_func_id func_id; 6066 int i; 6067 6068 if (obj->gen_loader) 6069 return 0; 6070 6071 for (i = 0; i < prog->insns_cnt; i++, insn++) { 6072 if (!insn_is_helper_call(insn, &func_id)) 6073 continue; 6074 6075 /* on kernels that don't yet support 6076 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 6077 * to bpf_probe_read() which works well for old kernels 6078 */ 6079 switch (func_id) { 6080 case BPF_FUNC_probe_read_kernel: 6081 case BPF_FUNC_probe_read_user: 6082 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6083 insn->imm = BPF_FUNC_probe_read; 6084 break; 6085 case BPF_FUNC_probe_read_kernel_str: 6086 case BPF_FUNC_probe_read_user_str: 6087 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6088 insn->imm = BPF_FUNC_probe_read_str; 6089 break; 6090 default: 6091 break; 6092 } 6093 } 6094 return 0; 6095 } 6096 6097 static int 6098 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 6099 char *license, __u32 kern_version, int *pfd) 6100 { 6101 struct bpf_prog_load_params load_attr = {}; 6102 char *cp, errmsg[STRERR_BUFSIZE]; 6103 size_t log_buf_size = 0; 6104 char *log_buf = NULL; 6105 int btf_fd, ret; 6106 6107 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 6108 /* 6109 * The program type must be set. Most likely we couldn't find a proper 6110 * section definition at load time, and thus we didn't infer the type. 6111 */ 6112 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 6113 prog->name, prog->sec_name); 6114 return -EINVAL; 6115 } 6116 6117 if (!insns || !insns_cnt) 6118 return -EINVAL; 6119 6120 load_attr.prog_type = prog->type; 6121 /* old kernels might not support specifying expected_attach_type */ 6122 if (!kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE) && prog->sec_def && 6123 prog->sec_def->is_exp_attach_type_optional) 6124 load_attr.expected_attach_type = 0; 6125 else 6126 load_attr.expected_attach_type = prog->expected_attach_type; 6127 if (kernel_supports(prog->obj, FEAT_PROG_NAME)) 6128 load_attr.name = prog->name; 6129 load_attr.insns = insns; 6130 load_attr.insn_cnt = insns_cnt; 6131 load_attr.license = license; 6132 load_attr.attach_btf_id = prog->attach_btf_id; 6133 if (prog->attach_prog_fd) 6134 load_attr.attach_prog_fd = prog->attach_prog_fd; 6135 else 6136 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 6137 load_attr.attach_btf_id = prog->attach_btf_id; 6138 load_attr.kern_version = kern_version; 6139 load_attr.prog_ifindex = prog->prog_ifindex; 6140 6141 /* specify func_info/line_info only if kernel supports them */ 6142 btf_fd = bpf_object__btf_fd(prog->obj); 6143 if (btf_fd >= 0 && kernel_supports(prog->obj, FEAT_BTF_FUNC)) { 6144 load_attr.prog_btf_fd = btf_fd; 6145 load_attr.func_info = prog->func_info; 6146 load_attr.func_info_rec_size = prog->func_info_rec_size; 6147 load_attr.func_info_cnt = prog->func_info_cnt; 6148 load_attr.line_info = prog->line_info; 6149 load_attr.line_info_rec_size = prog->line_info_rec_size; 6150 load_attr.line_info_cnt = prog->line_info_cnt; 6151 } 6152 load_attr.log_level = prog->log_level; 6153 load_attr.prog_flags = prog->prog_flags; 6154 6155 if (prog->obj->gen_loader) { 6156 bpf_gen__prog_load(prog->obj->gen_loader, &load_attr, 6157 prog - prog->obj->programs); 6158 *pfd = -1; 6159 return 0; 6160 } 6161 retry_load: 6162 if (log_buf_size) { 6163 log_buf = malloc(log_buf_size); 6164 if (!log_buf) 6165 return -ENOMEM; 6166 6167 *log_buf = 0; 6168 } 6169 6170 load_attr.log_buf = log_buf; 6171 load_attr.log_buf_sz = log_buf_size; 6172 ret = libbpf__bpf_prog_load(&load_attr); 6173 6174 if (ret >= 0) { 6175 if (log_buf && load_attr.log_level) 6176 pr_debug("verifier log:\n%s", log_buf); 6177 6178 if (prog->obj->rodata_map_idx >= 0 && 6179 kernel_supports(prog->obj, FEAT_PROG_BIND_MAP)) { 6180 struct bpf_map *rodata_map = 6181 &prog->obj->maps[prog->obj->rodata_map_idx]; 6182 6183 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) { 6184 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6185 pr_warn("prog '%s': failed to bind .rodata map: %s\n", 6186 prog->name, cp); 6187 /* Don't fail hard if can't bind rodata. */ 6188 } 6189 } 6190 6191 *pfd = ret; 6192 ret = 0; 6193 goto out; 6194 } 6195 6196 if (!log_buf || errno == ENOSPC) { 6197 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, 6198 log_buf_size << 1); 6199 6200 free(log_buf); 6201 goto retry_load; 6202 } 6203 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD; 6204 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6205 pr_warn("load bpf program failed: %s\n", cp); 6206 pr_perm_msg(ret); 6207 6208 if (log_buf && log_buf[0] != '\0') { 6209 ret = -LIBBPF_ERRNO__VERIFY; 6210 pr_warn("-- BEGIN DUMP LOG ---\n"); 6211 pr_warn("\n%s\n", log_buf); 6212 pr_warn("-- END LOG --\n"); 6213 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) { 6214 pr_warn("Program too large (%zu insns), at most %d insns\n", 6215 load_attr.insn_cnt, BPF_MAXINSNS); 6216 ret = -LIBBPF_ERRNO__PROG2BIG; 6217 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 6218 /* Wrong program type? */ 6219 int fd; 6220 6221 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 6222 load_attr.expected_attach_type = 0; 6223 load_attr.log_buf = NULL; 6224 load_attr.log_buf_sz = 0; 6225 fd = libbpf__bpf_prog_load(&load_attr); 6226 if (fd >= 0) { 6227 close(fd); 6228 ret = -LIBBPF_ERRNO__PROGTYPE; 6229 goto out; 6230 } 6231 } 6232 6233 out: 6234 free(log_buf); 6235 return ret; 6236 } 6237 6238 static int bpf_program__record_externs(struct bpf_program *prog) 6239 { 6240 struct bpf_object *obj = prog->obj; 6241 int i; 6242 6243 for (i = 0; i < prog->nr_reloc; i++) { 6244 struct reloc_desc *relo = &prog->reloc_desc[i]; 6245 struct extern_desc *ext = &obj->externs[relo->sym_off]; 6246 6247 switch (relo->type) { 6248 case RELO_EXTERN_VAR: 6249 if (ext->type != EXT_KSYM) 6250 continue; 6251 if (!ext->ksym.type_id) { 6252 pr_warn("typeless ksym %s is not supported yet\n", 6253 ext->name); 6254 return -ENOTSUP; 6255 } 6256 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR, 6257 relo->insn_idx); 6258 break; 6259 case RELO_EXTERN_FUNC: 6260 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC, 6261 relo->insn_idx); 6262 break; 6263 default: 6264 continue; 6265 } 6266 } 6267 return 0; 6268 } 6269 6270 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id); 6271 6272 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) 6273 { 6274 int err = 0, fd, i; 6275 6276 if (prog->obj->loaded) { 6277 pr_warn("prog '%s': can't load after object was loaded\n", prog->name); 6278 return libbpf_err(-EINVAL); 6279 } 6280 6281 if ((prog->type == BPF_PROG_TYPE_TRACING || 6282 prog->type == BPF_PROG_TYPE_LSM || 6283 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { 6284 int btf_obj_fd = 0, btf_type_id = 0; 6285 6286 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id); 6287 if (err) 6288 return libbpf_err(err); 6289 6290 prog->attach_btf_obj_fd = btf_obj_fd; 6291 prog->attach_btf_id = btf_type_id; 6292 } 6293 6294 if (prog->instances.nr < 0 || !prog->instances.fds) { 6295 if (prog->preprocessor) { 6296 pr_warn("Internal error: can't load program '%s'\n", 6297 prog->name); 6298 return libbpf_err(-LIBBPF_ERRNO__INTERNAL); 6299 } 6300 6301 prog->instances.fds = malloc(sizeof(int)); 6302 if (!prog->instances.fds) { 6303 pr_warn("Not enough memory for BPF fds\n"); 6304 return libbpf_err(-ENOMEM); 6305 } 6306 prog->instances.nr = 1; 6307 prog->instances.fds[0] = -1; 6308 } 6309 6310 if (!prog->preprocessor) { 6311 if (prog->instances.nr != 1) { 6312 pr_warn("prog '%s': inconsistent nr(%d) != 1\n", 6313 prog->name, prog->instances.nr); 6314 } 6315 if (prog->obj->gen_loader) 6316 bpf_program__record_externs(prog); 6317 err = load_program(prog, prog->insns, prog->insns_cnt, 6318 license, kern_ver, &fd); 6319 if (!err) 6320 prog->instances.fds[0] = fd; 6321 goto out; 6322 } 6323 6324 for (i = 0; i < prog->instances.nr; i++) { 6325 struct bpf_prog_prep_result result; 6326 bpf_program_prep_t preprocessor = prog->preprocessor; 6327 6328 memset(&result, 0, sizeof(result)); 6329 err = preprocessor(prog, i, prog->insns, 6330 prog->insns_cnt, &result); 6331 if (err) { 6332 pr_warn("Preprocessing the %dth instance of program '%s' failed\n", 6333 i, prog->name); 6334 goto out; 6335 } 6336 6337 if (!result.new_insn_ptr || !result.new_insn_cnt) { 6338 pr_debug("Skip loading the %dth instance of program '%s'\n", 6339 i, prog->name); 6340 prog->instances.fds[i] = -1; 6341 if (result.pfd) 6342 *result.pfd = -1; 6343 continue; 6344 } 6345 6346 err = load_program(prog, result.new_insn_ptr, 6347 result.new_insn_cnt, license, kern_ver, &fd); 6348 if (err) { 6349 pr_warn("Loading the %dth instance of program '%s' failed\n", 6350 i, prog->name); 6351 goto out; 6352 } 6353 6354 if (result.pfd) 6355 *result.pfd = fd; 6356 prog->instances.fds[i] = fd; 6357 } 6358 out: 6359 if (err) 6360 pr_warn("failed to load program '%s'\n", prog->name); 6361 zfree(&prog->insns); 6362 prog->insns_cnt = 0; 6363 return libbpf_err(err); 6364 } 6365 6366 static int 6367 bpf_object__load_progs(struct bpf_object *obj, int log_level) 6368 { 6369 struct bpf_program *prog; 6370 size_t i; 6371 int err; 6372 6373 for (i = 0; i < obj->nr_programs; i++) { 6374 prog = &obj->programs[i]; 6375 err = bpf_object__sanitize_prog(obj, prog); 6376 if (err) 6377 return err; 6378 } 6379 6380 for (i = 0; i < obj->nr_programs; i++) { 6381 prog = &obj->programs[i]; 6382 if (prog_is_subprog(obj, prog)) 6383 continue; 6384 if (!prog->load) { 6385 pr_debug("prog '%s': skipped loading\n", prog->name); 6386 continue; 6387 } 6388 prog->log_level |= log_level; 6389 err = bpf_program__load(prog, obj->license, obj->kern_version); 6390 if (err) 6391 return err; 6392 } 6393 if (obj->gen_loader) 6394 bpf_object__free_relocs(obj); 6395 return 0; 6396 } 6397 6398 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 6399 6400 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) 6401 { 6402 struct bpf_program *prog; 6403 6404 bpf_object__for_each_program(prog, obj) { 6405 prog->sec_def = find_sec_def(prog->sec_name); 6406 if (!prog->sec_def) { 6407 /* couldn't guess, but user might manually specify */ 6408 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 6409 prog->name, prog->sec_name); 6410 continue; 6411 } 6412 6413 if (prog->sec_def->is_sleepable) 6414 prog->prog_flags |= BPF_F_SLEEPABLE; 6415 bpf_program__set_type(prog, prog->sec_def->prog_type); 6416 bpf_program__set_expected_attach_type(prog, prog->sec_def->expected_attach_type); 6417 6418 #pragma GCC diagnostic push 6419 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 6420 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || 6421 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) 6422 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); 6423 #pragma GCC diagnostic pop 6424 } 6425 6426 return 0; 6427 } 6428 6429 static struct bpf_object * 6430 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, 6431 const struct bpf_object_open_opts *opts) 6432 { 6433 const char *obj_name, *kconfig, *btf_tmp_path; 6434 struct bpf_object *obj; 6435 char tmp_name[64]; 6436 int err; 6437 6438 if (elf_version(EV_CURRENT) == EV_NONE) { 6439 pr_warn("failed to init libelf for %s\n", 6440 path ? : "(mem buf)"); 6441 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 6442 } 6443 6444 if (!OPTS_VALID(opts, bpf_object_open_opts)) 6445 return ERR_PTR(-EINVAL); 6446 6447 obj_name = OPTS_GET(opts, object_name, NULL); 6448 if (obj_buf) { 6449 if (!obj_name) { 6450 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 6451 (unsigned long)obj_buf, 6452 (unsigned long)obj_buf_sz); 6453 obj_name = tmp_name; 6454 } 6455 path = obj_name; 6456 pr_debug("loading object '%s' from buffer\n", obj_name); 6457 } 6458 6459 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 6460 if (IS_ERR(obj)) 6461 return obj; 6462 6463 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 6464 if (btf_tmp_path) { 6465 if (strlen(btf_tmp_path) >= PATH_MAX) { 6466 err = -ENAMETOOLONG; 6467 goto out; 6468 } 6469 obj->btf_custom_path = strdup(btf_tmp_path); 6470 if (!obj->btf_custom_path) { 6471 err = -ENOMEM; 6472 goto out; 6473 } 6474 } 6475 6476 kconfig = OPTS_GET(opts, kconfig, NULL); 6477 if (kconfig) { 6478 obj->kconfig = strdup(kconfig); 6479 if (!obj->kconfig) { 6480 err = -ENOMEM; 6481 goto out; 6482 } 6483 } 6484 6485 err = bpf_object__elf_init(obj); 6486 err = err ? : bpf_object__check_endianness(obj); 6487 err = err ? : bpf_object__elf_collect(obj); 6488 err = err ? : bpf_object__collect_externs(obj); 6489 err = err ? : bpf_object__finalize_btf(obj); 6490 err = err ? : bpf_object__init_maps(obj, opts); 6491 err = err ? : bpf_object_init_progs(obj, opts); 6492 err = err ? : bpf_object__collect_relos(obj); 6493 if (err) 6494 goto out; 6495 6496 bpf_object__elf_finish(obj); 6497 6498 return obj; 6499 out: 6500 bpf_object__close(obj); 6501 return ERR_PTR(err); 6502 } 6503 6504 static struct bpf_object * 6505 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) 6506 { 6507 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 6508 .relaxed_maps = flags & MAPS_RELAX_COMPAT, 6509 ); 6510 6511 /* param validation */ 6512 if (!attr->file) 6513 return NULL; 6514 6515 pr_debug("loading %s\n", attr->file); 6516 return __bpf_object__open(attr->file, NULL, 0, &opts); 6517 } 6518 6519 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 6520 { 6521 return libbpf_ptr(__bpf_object__open_xattr(attr, 0)); 6522 } 6523 6524 struct bpf_object *bpf_object__open(const char *path) 6525 { 6526 struct bpf_object_open_attr attr = { 6527 .file = path, 6528 .prog_type = BPF_PROG_TYPE_UNSPEC, 6529 }; 6530 6531 return libbpf_ptr(__bpf_object__open_xattr(&attr, 0)); 6532 } 6533 6534 struct bpf_object * 6535 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 6536 { 6537 if (!path) 6538 return libbpf_err_ptr(-EINVAL); 6539 6540 pr_debug("loading %s\n", path); 6541 6542 return libbpf_ptr(__bpf_object__open(path, NULL, 0, opts)); 6543 } 6544 6545 struct bpf_object * 6546 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 6547 const struct bpf_object_open_opts *opts) 6548 { 6549 if (!obj_buf || obj_buf_sz == 0) 6550 return libbpf_err_ptr(-EINVAL); 6551 6552 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, opts)); 6553 } 6554 6555 struct bpf_object * 6556 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, 6557 const char *name) 6558 { 6559 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 6560 .object_name = name, 6561 /* wrong default, but backwards-compatible */ 6562 .relaxed_maps = true, 6563 ); 6564 6565 /* returning NULL is wrong, but backwards-compatible */ 6566 if (!obj_buf || obj_buf_sz == 0) 6567 return errno = EINVAL, NULL; 6568 6569 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, &opts)); 6570 } 6571 6572 int bpf_object__unload(struct bpf_object *obj) 6573 { 6574 size_t i; 6575 6576 if (!obj) 6577 return libbpf_err(-EINVAL); 6578 6579 for (i = 0; i < obj->nr_maps; i++) { 6580 zclose(obj->maps[i].fd); 6581 if (obj->maps[i].st_ops) 6582 zfree(&obj->maps[i].st_ops->kern_vdata); 6583 } 6584 6585 for (i = 0; i < obj->nr_programs; i++) 6586 bpf_program__unload(&obj->programs[i]); 6587 6588 return 0; 6589 } 6590 6591 static int bpf_object__sanitize_maps(struct bpf_object *obj) 6592 { 6593 struct bpf_map *m; 6594 6595 bpf_object__for_each_map(m, obj) { 6596 if (!bpf_map__is_internal(m)) 6597 continue; 6598 if (!kernel_supports(obj, FEAT_GLOBAL_DATA)) { 6599 pr_warn("kernel doesn't support global data\n"); 6600 return -ENOTSUP; 6601 } 6602 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 6603 m->def.map_flags ^= BPF_F_MMAPABLE; 6604 } 6605 6606 return 0; 6607 } 6608 6609 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 6610 { 6611 char sym_type, sym_name[500]; 6612 unsigned long long sym_addr; 6613 const struct btf_type *t; 6614 struct extern_desc *ext; 6615 int ret, err = 0; 6616 FILE *f; 6617 6618 f = fopen("/proc/kallsyms", "r"); 6619 if (!f) { 6620 err = -errno; 6621 pr_warn("failed to open /proc/kallsyms: %d\n", err); 6622 return err; 6623 } 6624 6625 while (true) { 6626 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 6627 &sym_addr, &sym_type, sym_name); 6628 if (ret == EOF && feof(f)) 6629 break; 6630 if (ret != 3) { 6631 pr_warn("failed to read kallsyms entry: %d\n", ret); 6632 err = -EINVAL; 6633 goto out; 6634 } 6635 6636 ext = find_extern_by_name(obj, sym_name); 6637 if (!ext || ext->type != EXT_KSYM) 6638 continue; 6639 6640 t = btf__type_by_id(obj->btf, ext->btf_id); 6641 if (!btf_is_var(t)) 6642 continue; 6643 6644 if (ext->is_set && ext->ksym.addr != sym_addr) { 6645 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", 6646 sym_name, ext->ksym.addr, sym_addr); 6647 err = -EINVAL; 6648 goto out; 6649 } 6650 if (!ext->is_set) { 6651 ext->is_set = true; 6652 ext->ksym.addr = sym_addr; 6653 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); 6654 } 6655 } 6656 6657 out: 6658 fclose(f); 6659 return err; 6660 } 6661 6662 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 6663 __u16 kind, struct btf **res_btf, 6664 int *res_btf_fd) 6665 { 6666 int i, id, btf_fd, err; 6667 struct btf *btf; 6668 6669 btf = obj->btf_vmlinux; 6670 btf_fd = 0; 6671 id = btf__find_by_name_kind(btf, ksym_name, kind); 6672 6673 if (id == -ENOENT) { 6674 err = load_module_btfs(obj); 6675 if (err) 6676 return err; 6677 6678 for (i = 0; i < obj->btf_module_cnt; i++) { 6679 btf = obj->btf_modules[i].btf; 6680 /* we assume module BTF FD is always >0 */ 6681 btf_fd = obj->btf_modules[i].fd; 6682 id = btf__find_by_name_kind(btf, ksym_name, kind); 6683 if (id != -ENOENT) 6684 break; 6685 } 6686 } 6687 if (id <= 0) 6688 return -ESRCH; 6689 6690 *res_btf = btf; 6691 *res_btf_fd = btf_fd; 6692 return id; 6693 } 6694 6695 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 6696 struct extern_desc *ext) 6697 { 6698 const struct btf_type *targ_var, *targ_type; 6699 __u32 targ_type_id, local_type_id; 6700 const char *targ_var_name; 6701 int id, btf_fd = 0, err; 6702 struct btf *btf = NULL; 6703 6704 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd); 6705 if (id == -ESRCH && ext->is_weak) { 6706 return 0; 6707 } else if (id < 0) { 6708 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 6709 ext->name); 6710 return id; 6711 } 6712 6713 /* find local type_id */ 6714 local_type_id = ext->ksym.type_id; 6715 6716 /* find target type_id */ 6717 targ_var = btf__type_by_id(btf, id); 6718 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 6719 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 6720 6721 err = bpf_core_types_are_compat(obj->btf, local_type_id, 6722 btf, targ_type_id); 6723 if (err <= 0) { 6724 const struct btf_type *local_type; 6725 const char *targ_name, *local_name; 6726 6727 local_type = btf__type_by_id(obj->btf, local_type_id); 6728 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 6729 targ_name = btf__name_by_offset(btf, targ_type->name_off); 6730 6731 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 6732 ext->name, local_type_id, 6733 btf_kind_str(local_type), local_name, targ_type_id, 6734 btf_kind_str(targ_type), targ_name); 6735 return -EINVAL; 6736 } 6737 6738 ext->is_set = true; 6739 ext->ksym.kernel_btf_obj_fd = btf_fd; 6740 ext->ksym.kernel_btf_id = id; 6741 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 6742 ext->name, id, btf_kind_str(targ_var), targ_var_name); 6743 6744 return 0; 6745 } 6746 6747 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 6748 struct extern_desc *ext) 6749 { 6750 int local_func_proto_id, kfunc_proto_id, kfunc_id; 6751 const struct btf_type *kern_func; 6752 struct btf *kern_btf = NULL; 6753 int ret, kern_btf_fd = 0; 6754 6755 local_func_proto_id = ext->ksym.type_id; 6756 6757 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, 6758 &kern_btf, &kern_btf_fd); 6759 if (kfunc_id < 0) { 6760 pr_warn("extern (func ksym) '%s': not found in kernel BTF\n", 6761 ext->name); 6762 return kfunc_id; 6763 } 6764 6765 if (kern_btf != obj->btf_vmlinux) { 6766 pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n", 6767 ext->name); 6768 return -ENOTSUP; 6769 } 6770 6771 kern_func = btf__type_by_id(kern_btf, kfunc_id); 6772 kfunc_proto_id = kern_func->type; 6773 6774 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 6775 kern_btf, kfunc_proto_id); 6776 if (ret <= 0) { 6777 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n", 6778 ext->name, local_func_proto_id, kfunc_proto_id); 6779 return -EINVAL; 6780 } 6781 6782 ext->is_set = true; 6783 ext->ksym.kernel_btf_obj_fd = kern_btf_fd; 6784 ext->ksym.kernel_btf_id = kfunc_id; 6785 pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n", 6786 ext->name, kfunc_id); 6787 6788 return 0; 6789 } 6790 6791 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 6792 { 6793 const struct btf_type *t; 6794 struct extern_desc *ext; 6795 int i, err; 6796 6797 for (i = 0; i < obj->nr_extern; i++) { 6798 ext = &obj->externs[i]; 6799 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 6800 continue; 6801 6802 if (obj->gen_loader) { 6803 ext->is_set = true; 6804 ext->ksym.kernel_btf_obj_fd = 0; 6805 ext->ksym.kernel_btf_id = 0; 6806 continue; 6807 } 6808 t = btf__type_by_id(obj->btf, ext->btf_id); 6809 if (btf_is_var(t)) 6810 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 6811 else 6812 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 6813 if (err) 6814 return err; 6815 } 6816 return 0; 6817 } 6818 6819 static int bpf_object__resolve_externs(struct bpf_object *obj, 6820 const char *extra_kconfig) 6821 { 6822 bool need_config = false, need_kallsyms = false; 6823 bool need_vmlinux_btf = false; 6824 struct extern_desc *ext; 6825 void *kcfg_data = NULL; 6826 int err, i; 6827 6828 if (obj->nr_extern == 0) 6829 return 0; 6830 6831 if (obj->kconfig_map_idx >= 0) 6832 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 6833 6834 for (i = 0; i < obj->nr_extern; i++) { 6835 ext = &obj->externs[i]; 6836 6837 if (ext->type == EXT_KCFG && 6838 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 6839 void *ext_val = kcfg_data + ext->kcfg.data_off; 6840 __u32 kver = get_kernel_version(); 6841 6842 if (!kver) { 6843 pr_warn("failed to get kernel version\n"); 6844 return -EINVAL; 6845 } 6846 err = set_kcfg_value_num(ext, ext_val, kver); 6847 if (err) 6848 return err; 6849 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); 6850 } else if (ext->type == EXT_KCFG && 6851 strncmp(ext->name, "CONFIG_", 7) == 0) { 6852 need_config = true; 6853 } else if (ext->type == EXT_KSYM) { 6854 if (ext->ksym.type_id) 6855 need_vmlinux_btf = true; 6856 else 6857 need_kallsyms = true; 6858 } else { 6859 pr_warn("unrecognized extern '%s'\n", ext->name); 6860 return -EINVAL; 6861 } 6862 } 6863 if (need_config && extra_kconfig) { 6864 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 6865 if (err) 6866 return -EINVAL; 6867 need_config = false; 6868 for (i = 0; i < obj->nr_extern; i++) { 6869 ext = &obj->externs[i]; 6870 if (ext->type == EXT_KCFG && !ext->is_set) { 6871 need_config = true; 6872 break; 6873 } 6874 } 6875 } 6876 if (need_config) { 6877 err = bpf_object__read_kconfig_file(obj, kcfg_data); 6878 if (err) 6879 return -EINVAL; 6880 } 6881 if (need_kallsyms) { 6882 err = bpf_object__read_kallsyms_file(obj); 6883 if (err) 6884 return -EINVAL; 6885 } 6886 if (need_vmlinux_btf) { 6887 err = bpf_object__resolve_ksyms_btf_id(obj); 6888 if (err) 6889 return -EINVAL; 6890 } 6891 for (i = 0; i < obj->nr_extern; i++) { 6892 ext = &obj->externs[i]; 6893 6894 if (!ext->is_set && !ext->is_weak) { 6895 pr_warn("extern %s (strong) not resolved\n", ext->name); 6896 return -ESRCH; 6897 } else if (!ext->is_set) { 6898 pr_debug("extern %s (weak) not resolved, defaulting to zero\n", 6899 ext->name); 6900 } 6901 } 6902 6903 return 0; 6904 } 6905 6906 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 6907 { 6908 struct bpf_object *obj; 6909 int err, i; 6910 6911 if (!attr) 6912 return libbpf_err(-EINVAL); 6913 obj = attr->obj; 6914 if (!obj) 6915 return libbpf_err(-EINVAL); 6916 6917 if (obj->loaded) { 6918 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 6919 return libbpf_err(-EINVAL); 6920 } 6921 6922 if (obj->gen_loader) 6923 bpf_gen__init(obj->gen_loader, attr->log_level); 6924 6925 err = bpf_object__probe_loading(obj); 6926 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 6927 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 6928 err = err ? : bpf_object__sanitize_and_load_btf(obj); 6929 err = err ? : bpf_object__sanitize_maps(obj); 6930 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 6931 err = err ? : bpf_object__create_maps(obj); 6932 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : attr->target_btf_path); 6933 err = err ? : bpf_object__load_progs(obj, attr->log_level); 6934 6935 if (obj->gen_loader) { 6936 /* reset FDs */ 6937 btf__set_fd(obj->btf, -1); 6938 for (i = 0; i < obj->nr_maps; i++) 6939 obj->maps[i].fd = -1; 6940 if (!err) 6941 err = bpf_gen__finish(obj->gen_loader); 6942 } 6943 6944 /* clean up module BTFs */ 6945 for (i = 0; i < obj->btf_module_cnt; i++) { 6946 close(obj->btf_modules[i].fd); 6947 btf__free(obj->btf_modules[i].btf); 6948 free(obj->btf_modules[i].name); 6949 } 6950 free(obj->btf_modules); 6951 6952 /* clean up vmlinux BTF */ 6953 btf__free(obj->btf_vmlinux); 6954 obj->btf_vmlinux = NULL; 6955 6956 obj->loaded = true; /* doesn't matter if successfully or not */ 6957 6958 if (err) 6959 goto out; 6960 6961 return 0; 6962 out: 6963 /* unpin any maps that were auto-pinned during load */ 6964 for (i = 0; i < obj->nr_maps; i++) 6965 if (obj->maps[i].pinned && !obj->maps[i].reused) 6966 bpf_map__unpin(&obj->maps[i], NULL); 6967 6968 bpf_object__unload(obj); 6969 pr_warn("failed to load object '%s'\n", obj->path); 6970 return libbpf_err(err); 6971 } 6972 6973 int bpf_object__load(struct bpf_object *obj) 6974 { 6975 struct bpf_object_load_attr attr = { 6976 .obj = obj, 6977 }; 6978 6979 return bpf_object__load_xattr(&attr); 6980 } 6981 6982 static int make_parent_dir(const char *path) 6983 { 6984 char *cp, errmsg[STRERR_BUFSIZE]; 6985 char *dname, *dir; 6986 int err = 0; 6987 6988 dname = strdup(path); 6989 if (dname == NULL) 6990 return -ENOMEM; 6991 6992 dir = dirname(dname); 6993 if (mkdir(dir, 0700) && errno != EEXIST) 6994 err = -errno; 6995 6996 free(dname); 6997 if (err) { 6998 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 6999 pr_warn("failed to mkdir %s: %s\n", path, cp); 7000 } 7001 return err; 7002 } 7003 7004 static int check_path(const char *path) 7005 { 7006 char *cp, errmsg[STRERR_BUFSIZE]; 7007 struct statfs st_fs; 7008 char *dname, *dir; 7009 int err = 0; 7010 7011 if (path == NULL) 7012 return -EINVAL; 7013 7014 dname = strdup(path); 7015 if (dname == NULL) 7016 return -ENOMEM; 7017 7018 dir = dirname(dname); 7019 if (statfs(dir, &st_fs)) { 7020 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7021 pr_warn("failed to statfs %s: %s\n", dir, cp); 7022 err = -errno; 7023 } 7024 free(dname); 7025 7026 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 7027 pr_warn("specified path %s is not on BPF FS\n", path); 7028 err = -EINVAL; 7029 } 7030 7031 return err; 7032 } 7033 7034 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 7035 int instance) 7036 { 7037 char *cp, errmsg[STRERR_BUFSIZE]; 7038 int err; 7039 7040 err = make_parent_dir(path); 7041 if (err) 7042 return libbpf_err(err); 7043 7044 err = check_path(path); 7045 if (err) 7046 return libbpf_err(err); 7047 7048 if (prog == NULL) { 7049 pr_warn("invalid program pointer\n"); 7050 return libbpf_err(-EINVAL); 7051 } 7052 7053 if (instance < 0 || instance >= prog->instances.nr) { 7054 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7055 instance, prog->name, prog->instances.nr); 7056 return libbpf_err(-EINVAL); 7057 } 7058 7059 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 7060 err = -errno; 7061 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 7062 pr_warn("failed to pin program: %s\n", cp); 7063 return libbpf_err(err); 7064 } 7065 pr_debug("pinned program '%s'\n", path); 7066 7067 return 0; 7068 } 7069 7070 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 7071 int instance) 7072 { 7073 int err; 7074 7075 err = check_path(path); 7076 if (err) 7077 return libbpf_err(err); 7078 7079 if (prog == NULL) { 7080 pr_warn("invalid program pointer\n"); 7081 return libbpf_err(-EINVAL); 7082 } 7083 7084 if (instance < 0 || instance >= prog->instances.nr) { 7085 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7086 instance, prog->name, prog->instances.nr); 7087 return libbpf_err(-EINVAL); 7088 } 7089 7090 err = unlink(path); 7091 if (err != 0) 7092 return libbpf_err(-errno); 7093 7094 pr_debug("unpinned program '%s'\n", path); 7095 7096 return 0; 7097 } 7098 7099 int bpf_program__pin(struct bpf_program *prog, const char *path) 7100 { 7101 int i, err; 7102 7103 err = make_parent_dir(path); 7104 if (err) 7105 return libbpf_err(err); 7106 7107 err = check_path(path); 7108 if (err) 7109 return libbpf_err(err); 7110 7111 if (prog == NULL) { 7112 pr_warn("invalid program pointer\n"); 7113 return libbpf_err(-EINVAL); 7114 } 7115 7116 if (prog->instances.nr <= 0) { 7117 pr_warn("no instances of prog %s to pin\n", prog->name); 7118 return libbpf_err(-EINVAL); 7119 } 7120 7121 if (prog->instances.nr == 1) { 7122 /* don't create subdirs when pinning single instance */ 7123 return bpf_program__pin_instance(prog, path, 0); 7124 } 7125 7126 for (i = 0; i < prog->instances.nr; i++) { 7127 char buf[PATH_MAX]; 7128 int len; 7129 7130 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7131 if (len < 0) { 7132 err = -EINVAL; 7133 goto err_unpin; 7134 } else if (len >= PATH_MAX) { 7135 err = -ENAMETOOLONG; 7136 goto err_unpin; 7137 } 7138 7139 err = bpf_program__pin_instance(prog, buf, i); 7140 if (err) 7141 goto err_unpin; 7142 } 7143 7144 return 0; 7145 7146 err_unpin: 7147 for (i = i - 1; i >= 0; i--) { 7148 char buf[PATH_MAX]; 7149 int len; 7150 7151 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7152 if (len < 0) 7153 continue; 7154 else if (len >= PATH_MAX) 7155 continue; 7156 7157 bpf_program__unpin_instance(prog, buf, i); 7158 } 7159 7160 rmdir(path); 7161 7162 return libbpf_err(err); 7163 } 7164 7165 int bpf_program__unpin(struct bpf_program *prog, const char *path) 7166 { 7167 int i, err; 7168 7169 err = check_path(path); 7170 if (err) 7171 return libbpf_err(err); 7172 7173 if (prog == NULL) { 7174 pr_warn("invalid program pointer\n"); 7175 return libbpf_err(-EINVAL); 7176 } 7177 7178 if (prog->instances.nr <= 0) { 7179 pr_warn("no instances of prog %s to pin\n", prog->name); 7180 return libbpf_err(-EINVAL); 7181 } 7182 7183 if (prog->instances.nr == 1) { 7184 /* don't create subdirs when pinning single instance */ 7185 return bpf_program__unpin_instance(prog, path, 0); 7186 } 7187 7188 for (i = 0; i < prog->instances.nr; i++) { 7189 char buf[PATH_MAX]; 7190 int len; 7191 7192 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7193 if (len < 0) 7194 return libbpf_err(-EINVAL); 7195 else if (len >= PATH_MAX) 7196 return libbpf_err(-ENAMETOOLONG); 7197 7198 err = bpf_program__unpin_instance(prog, buf, i); 7199 if (err) 7200 return err; 7201 } 7202 7203 err = rmdir(path); 7204 if (err) 7205 return libbpf_err(-errno); 7206 7207 return 0; 7208 } 7209 7210 int bpf_map__pin(struct bpf_map *map, const char *path) 7211 { 7212 char *cp, errmsg[STRERR_BUFSIZE]; 7213 int err; 7214 7215 if (map == NULL) { 7216 pr_warn("invalid map pointer\n"); 7217 return libbpf_err(-EINVAL); 7218 } 7219 7220 if (map->pin_path) { 7221 if (path && strcmp(path, map->pin_path)) { 7222 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 7223 bpf_map__name(map), map->pin_path, path); 7224 return libbpf_err(-EINVAL); 7225 } else if (map->pinned) { 7226 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 7227 bpf_map__name(map), map->pin_path); 7228 return 0; 7229 } 7230 } else { 7231 if (!path) { 7232 pr_warn("missing a path to pin map '%s' at\n", 7233 bpf_map__name(map)); 7234 return libbpf_err(-EINVAL); 7235 } else if (map->pinned) { 7236 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 7237 return libbpf_err(-EEXIST); 7238 } 7239 7240 map->pin_path = strdup(path); 7241 if (!map->pin_path) { 7242 err = -errno; 7243 goto out_err; 7244 } 7245 } 7246 7247 err = make_parent_dir(map->pin_path); 7248 if (err) 7249 return libbpf_err(err); 7250 7251 err = check_path(map->pin_path); 7252 if (err) 7253 return libbpf_err(err); 7254 7255 if (bpf_obj_pin(map->fd, map->pin_path)) { 7256 err = -errno; 7257 goto out_err; 7258 } 7259 7260 map->pinned = true; 7261 pr_debug("pinned map '%s'\n", map->pin_path); 7262 7263 return 0; 7264 7265 out_err: 7266 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 7267 pr_warn("failed to pin map: %s\n", cp); 7268 return libbpf_err(err); 7269 } 7270 7271 int bpf_map__unpin(struct bpf_map *map, const char *path) 7272 { 7273 int err; 7274 7275 if (map == NULL) { 7276 pr_warn("invalid map pointer\n"); 7277 return libbpf_err(-EINVAL); 7278 } 7279 7280 if (map->pin_path) { 7281 if (path && strcmp(path, map->pin_path)) { 7282 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 7283 bpf_map__name(map), map->pin_path, path); 7284 return libbpf_err(-EINVAL); 7285 } 7286 path = map->pin_path; 7287 } else if (!path) { 7288 pr_warn("no path to unpin map '%s' from\n", 7289 bpf_map__name(map)); 7290 return libbpf_err(-EINVAL); 7291 } 7292 7293 err = check_path(path); 7294 if (err) 7295 return libbpf_err(err); 7296 7297 err = unlink(path); 7298 if (err != 0) 7299 return libbpf_err(-errno); 7300 7301 map->pinned = false; 7302 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 7303 7304 return 0; 7305 } 7306 7307 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 7308 { 7309 char *new = NULL; 7310 7311 if (path) { 7312 new = strdup(path); 7313 if (!new) 7314 return libbpf_err(-errno); 7315 } 7316 7317 free(map->pin_path); 7318 map->pin_path = new; 7319 return 0; 7320 } 7321 7322 const char *bpf_map__get_pin_path(const struct bpf_map *map) 7323 { 7324 return map->pin_path; 7325 } 7326 7327 const char *bpf_map__pin_path(const struct bpf_map *map) 7328 { 7329 return map->pin_path; 7330 } 7331 7332 bool bpf_map__is_pinned(const struct bpf_map *map) 7333 { 7334 return map->pinned; 7335 } 7336 7337 static void sanitize_pin_path(char *s) 7338 { 7339 /* bpffs disallows periods in path names */ 7340 while (*s) { 7341 if (*s == '.') 7342 *s = '_'; 7343 s++; 7344 } 7345 } 7346 7347 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 7348 { 7349 struct bpf_map *map; 7350 int err; 7351 7352 if (!obj) 7353 return libbpf_err(-ENOENT); 7354 7355 if (!obj->loaded) { 7356 pr_warn("object not yet loaded; load it first\n"); 7357 return libbpf_err(-ENOENT); 7358 } 7359 7360 bpf_object__for_each_map(map, obj) { 7361 char *pin_path = NULL; 7362 char buf[PATH_MAX]; 7363 7364 if (path) { 7365 int len; 7366 7367 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7368 bpf_map__name(map)); 7369 if (len < 0) { 7370 err = -EINVAL; 7371 goto err_unpin_maps; 7372 } else if (len >= PATH_MAX) { 7373 err = -ENAMETOOLONG; 7374 goto err_unpin_maps; 7375 } 7376 sanitize_pin_path(buf); 7377 pin_path = buf; 7378 } else if (!map->pin_path) { 7379 continue; 7380 } 7381 7382 err = bpf_map__pin(map, pin_path); 7383 if (err) 7384 goto err_unpin_maps; 7385 } 7386 7387 return 0; 7388 7389 err_unpin_maps: 7390 while ((map = bpf_map__prev(map, obj))) { 7391 if (!map->pin_path) 7392 continue; 7393 7394 bpf_map__unpin(map, NULL); 7395 } 7396 7397 return libbpf_err(err); 7398 } 7399 7400 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 7401 { 7402 struct bpf_map *map; 7403 int err; 7404 7405 if (!obj) 7406 return libbpf_err(-ENOENT); 7407 7408 bpf_object__for_each_map(map, obj) { 7409 char *pin_path = NULL; 7410 char buf[PATH_MAX]; 7411 7412 if (path) { 7413 int len; 7414 7415 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7416 bpf_map__name(map)); 7417 if (len < 0) 7418 return libbpf_err(-EINVAL); 7419 else if (len >= PATH_MAX) 7420 return libbpf_err(-ENAMETOOLONG); 7421 sanitize_pin_path(buf); 7422 pin_path = buf; 7423 } else if (!map->pin_path) { 7424 continue; 7425 } 7426 7427 err = bpf_map__unpin(map, pin_path); 7428 if (err) 7429 return libbpf_err(err); 7430 } 7431 7432 return 0; 7433 } 7434 7435 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 7436 { 7437 struct bpf_program *prog; 7438 int err; 7439 7440 if (!obj) 7441 return libbpf_err(-ENOENT); 7442 7443 if (!obj->loaded) { 7444 pr_warn("object not yet loaded; load it first\n"); 7445 return libbpf_err(-ENOENT); 7446 } 7447 7448 bpf_object__for_each_program(prog, obj) { 7449 char buf[PATH_MAX]; 7450 int len; 7451 7452 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7453 prog->pin_name); 7454 if (len < 0) { 7455 err = -EINVAL; 7456 goto err_unpin_programs; 7457 } else if (len >= PATH_MAX) { 7458 err = -ENAMETOOLONG; 7459 goto err_unpin_programs; 7460 } 7461 7462 err = bpf_program__pin(prog, buf); 7463 if (err) 7464 goto err_unpin_programs; 7465 } 7466 7467 return 0; 7468 7469 err_unpin_programs: 7470 while ((prog = bpf_program__prev(prog, obj))) { 7471 char buf[PATH_MAX]; 7472 int len; 7473 7474 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7475 prog->pin_name); 7476 if (len < 0) 7477 continue; 7478 else if (len >= PATH_MAX) 7479 continue; 7480 7481 bpf_program__unpin(prog, buf); 7482 } 7483 7484 return libbpf_err(err); 7485 } 7486 7487 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 7488 { 7489 struct bpf_program *prog; 7490 int err; 7491 7492 if (!obj) 7493 return libbpf_err(-ENOENT); 7494 7495 bpf_object__for_each_program(prog, obj) { 7496 char buf[PATH_MAX]; 7497 int len; 7498 7499 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7500 prog->pin_name); 7501 if (len < 0) 7502 return libbpf_err(-EINVAL); 7503 else if (len >= PATH_MAX) 7504 return libbpf_err(-ENAMETOOLONG); 7505 7506 err = bpf_program__unpin(prog, buf); 7507 if (err) 7508 return libbpf_err(err); 7509 } 7510 7511 return 0; 7512 } 7513 7514 int bpf_object__pin(struct bpf_object *obj, const char *path) 7515 { 7516 int err; 7517 7518 err = bpf_object__pin_maps(obj, path); 7519 if (err) 7520 return libbpf_err(err); 7521 7522 err = bpf_object__pin_programs(obj, path); 7523 if (err) { 7524 bpf_object__unpin_maps(obj, path); 7525 return libbpf_err(err); 7526 } 7527 7528 return 0; 7529 } 7530 7531 static void bpf_map__destroy(struct bpf_map *map) 7532 { 7533 if (map->clear_priv) 7534 map->clear_priv(map, map->priv); 7535 map->priv = NULL; 7536 map->clear_priv = NULL; 7537 7538 if (map->inner_map) { 7539 bpf_map__destroy(map->inner_map); 7540 zfree(&map->inner_map); 7541 } 7542 7543 zfree(&map->init_slots); 7544 map->init_slots_sz = 0; 7545 7546 if (map->mmaped) { 7547 munmap(map->mmaped, bpf_map_mmap_sz(map)); 7548 map->mmaped = NULL; 7549 } 7550 7551 if (map->st_ops) { 7552 zfree(&map->st_ops->data); 7553 zfree(&map->st_ops->progs); 7554 zfree(&map->st_ops->kern_func_off); 7555 zfree(&map->st_ops); 7556 } 7557 7558 zfree(&map->name); 7559 zfree(&map->pin_path); 7560 7561 if (map->fd >= 0) 7562 zclose(map->fd); 7563 } 7564 7565 void bpf_object__close(struct bpf_object *obj) 7566 { 7567 size_t i; 7568 7569 if (IS_ERR_OR_NULL(obj)) 7570 return; 7571 7572 if (obj->clear_priv) 7573 obj->clear_priv(obj, obj->priv); 7574 7575 bpf_gen__free(obj->gen_loader); 7576 bpf_object__elf_finish(obj); 7577 bpf_object__unload(obj); 7578 btf__free(obj->btf); 7579 btf_ext__free(obj->btf_ext); 7580 7581 for (i = 0; i < obj->nr_maps; i++) 7582 bpf_map__destroy(&obj->maps[i]); 7583 7584 zfree(&obj->btf_custom_path); 7585 zfree(&obj->kconfig); 7586 zfree(&obj->externs); 7587 obj->nr_extern = 0; 7588 7589 zfree(&obj->maps); 7590 obj->nr_maps = 0; 7591 7592 if (obj->programs && obj->nr_programs) { 7593 for (i = 0; i < obj->nr_programs; i++) 7594 bpf_program__exit(&obj->programs[i]); 7595 } 7596 zfree(&obj->programs); 7597 7598 list_del(&obj->list); 7599 free(obj); 7600 } 7601 7602 struct bpf_object * 7603 bpf_object__next(struct bpf_object *prev) 7604 { 7605 struct bpf_object *next; 7606 7607 if (!prev) 7608 next = list_first_entry(&bpf_objects_list, 7609 struct bpf_object, 7610 list); 7611 else 7612 next = list_next_entry(prev, list); 7613 7614 /* Empty list is noticed here so don't need checking on entry. */ 7615 if (&next->list == &bpf_objects_list) 7616 return NULL; 7617 7618 return next; 7619 } 7620 7621 const char *bpf_object__name(const struct bpf_object *obj) 7622 { 7623 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 7624 } 7625 7626 unsigned int bpf_object__kversion(const struct bpf_object *obj) 7627 { 7628 return obj ? obj->kern_version : 0; 7629 } 7630 7631 struct btf *bpf_object__btf(const struct bpf_object *obj) 7632 { 7633 return obj ? obj->btf : NULL; 7634 } 7635 7636 int bpf_object__btf_fd(const struct bpf_object *obj) 7637 { 7638 return obj->btf ? btf__fd(obj->btf) : -1; 7639 } 7640 7641 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 7642 { 7643 if (obj->loaded) 7644 return libbpf_err(-EINVAL); 7645 7646 obj->kern_version = kern_version; 7647 7648 return 0; 7649 } 7650 7651 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 7652 bpf_object_clear_priv_t clear_priv) 7653 { 7654 if (obj->priv && obj->clear_priv) 7655 obj->clear_priv(obj, obj->priv); 7656 7657 obj->priv = priv; 7658 obj->clear_priv = clear_priv; 7659 return 0; 7660 } 7661 7662 void *bpf_object__priv(const struct bpf_object *obj) 7663 { 7664 return obj ? obj->priv : libbpf_err_ptr(-EINVAL); 7665 } 7666 7667 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 7668 { 7669 struct bpf_gen *gen; 7670 7671 if (!opts) 7672 return -EFAULT; 7673 if (!OPTS_VALID(opts, gen_loader_opts)) 7674 return -EINVAL; 7675 gen = calloc(sizeof(*gen), 1); 7676 if (!gen) 7677 return -ENOMEM; 7678 gen->opts = opts; 7679 obj->gen_loader = gen; 7680 return 0; 7681 } 7682 7683 static struct bpf_program * 7684 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 7685 bool forward) 7686 { 7687 size_t nr_programs = obj->nr_programs; 7688 ssize_t idx; 7689 7690 if (!nr_programs) 7691 return NULL; 7692 7693 if (!p) 7694 /* Iter from the beginning */ 7695 return forward ? &obj->programs[0] : 7696 &obj->programs[nr_programs - 1]; 7697 7698 if (p->obj != obj) { 7699 pr_warn("error: program handler doesn't match object\n"); 7700 return errno = EINVAL, NULL; 7701 } 7702 7703 idx = (p - obj->programs) + (forward ? 1 : -1); 7704 if (idx >= obj->nr_programs || idx < 0) 7705 return NULL; 7706 return &obj->programs[idx]; 7707 } 7708 7709 struct bpf_program * 7710 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 7711 { 7712 struct bpf_program *prog = prev; 7713 7714 do { 7715 prog = __bpf_program__iter(prog, obj, true); 7716 } while (prog && prog_is_subprog(obj, prog)); 7717 7718 return prog; 7719 } 7720 7721 struct bpf_program * 7722 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 7723 { 7724 struct bpf_program *prog = next; 7725 7726 do { 7727 prog = __bpf_program__iter(prog, obj, false); 7728 } while (prog && prog_is_subprog(obj, prog)); 7729 7730 return prog; 7731 } 7732 7733 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 7734 bpf_program_clear_priv_t clear_priv) 7735 { 7736 if (prog->priv && prog->clear_priv) 7737 prog->clear_priv(prog, prog->priv); 7738 7739 prog->priv = priv; 7740 prog->clear_priv = clear_priv; 7741 return 0; 7742 } 7743 7744 void *bpf_program__priv(const struct bpf_program *prog) 7745 { 7746 return prog ? prog->priv : libbpf_err_ptr(-EINVAL); 7747 } 7748 7749 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 7750 { 7751 prog->prog_ifindex = ifindex; 7752 } 7753 7754 const char *bpf_program__name(const struct bpf_program *prog) 7755 { 7756 return prog->name; 7757 } 7758 7759 const char *bpf_program__section_name(const struct bpf_program *prog) 7760 { 7761 return prog->sec_name; 7762 } 7763 7764 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 7765 { 7766 const char *title; 7767 7768 title = prog->sec_name; 7769 if (needs_copy) { 7770 title = strdup(title); 7771 if (!title) { 7772 pr_warn("failed to strdup program title\n"); 7773 return libbpf_err_ptr(-ENOMEM); 7774 } 7775 } 7776 7777 return title; 7778 } 7779 7780 bool bpf_program__autoload(const struct bpf_program *prog) 7781 { 7782 return prog->load; 7783 } 7784 7785 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 7786 { 7787 if (prog->obj->loaded) 7788 return libbpf_err(-EINVAL); 7789 7790 prog->load = autoload; 7791 return 0; 7792 } 7793 7794 int bpf_program__fd(const struct bpf_program *prog) 7795 { 7796 return bpf_program__nth_fd(prog, 0); 7797 } 7798 7799 size_t bpf_program__size(const struct bpf_program *prog) 7800 { 7801 return prog->insns_cnt * BPF_INSN_SZ; 7802 } 7803 7804 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 7805 bpf_program_prep_t prep) 7806 { 7807 int *instances_fds; 7808 7809 if (nr_instances <= 0 || !prep) 7810 return libbpf_err(-EINVAL); 7811 7812 if (prog->instances.nr > 0 || prog->instances.fds) { 7813 pr_warn("Can't set pre-processor after loading\n"); 7814 return libbpf_err(-EINVAL); 7815 } 7816 7817 instances_fds = malloc(sizeof(int) * nr_instances); 7818 if (!instances_fds) { 7819 pr_warn("alloc memory failed for fds\n"); 7820 return libbpf_err(-ENOMEM); 7821 } 7822 7823 /* fill all fd with -1 */ 7824 memset(instances_fds, -1, sizeof(int) * nr_instances); 7825 7826 prog->instances.nr = nr_instances; 7827 prog->instances.fds = instances_fds; 7828 prog->preprocessor = prep; 7829 return 0; 7830 } 7831 7832 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 7833 { 7834 int fd; 7835 7836 if (!prog) 7837 return libbpf_err(-EINVAL); 7838 7839 if (n >= prog->instances.nr || n < 0) { 7840 pr_warn("Can't get the %dth fd from program %s: only %d instances\n", 7841 n, prog->name, prog->instances.nr); 7842 return libbpf_err(-EINVAL); 7843 } 7844 7845 fd = prog->instances.fds[n]; 7846 if (fd < 0) { 7847 pr_warn("%dth instance of program '%s' is invalid\n", 7848 n, prog->name); 7849 return libbpf_err(-ENOENT); 7850 } 7851 7852 return fd; 7853 } 7854 7855 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog) 7856 { 7857 return prog->type; 7858 } 7859 7860 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 7861 { 7862 prog->type = type; 7863 } 7864 7865 static bool bpf_program__is_type(const struct bpf_program *prog, 7866 enum bpf_prog_type type) 7867 { 7868 return prog ? (prog->type == type) : false; 7869 } 7870 7871 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 7872 int bpf_program__set_##NAME(struct bpf_program *prog) \ 7873 { \ 7874 if (!prog) \ 7875 return libbpf_err(-EINVAL); \ 7876 bpf_program__set_type(prog, TYPE); \ 7877 return 0; \ 7878 } \ 7879 \ 7880 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 7881 { \ 7882 return bpf_program__is_type(prog, TYPE); \ 7883 } \ 7884 7885 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 7886 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); 7887 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 7888 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 7889 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 7890 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 7891 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 7892 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 7893 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 7894 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); 7895 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); 7896 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); 7897 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP); 7898 7899 enum bpf_attach_type 7900 bpf_program__get_expected_attach_type(const struct bpf_program *prog) 7901 { 7902 return prog->expected_attach_type; 7903 } 7904 7905 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 7906 enum bpf_attach_type type) 7907 { 7908 prog->expected_attach_type = type; 7909 } 7910 7911 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \ 7912 attachable, attach_btf) \ 7913 { \ 7914 .sec = string, \ 7915 .len = sizeof(string) - 1, \ 7916 .prog_type = ptype, \ 7917 .expected_attach_type = eatype, \ 7918 .is_exp_attach_type_optional = eatype_optional, \ 7919 .is_attachable = attachable, \ 7920 .is_attach_btf = attach_btf, \ 7921 } 7922 7923 /* Programs that can NOT be attached. */ 7924 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) 7925 7926 /* Programs that can be attached. */ 7927 #define BPF_APROG_SEC(string, ptype, atype) \ 7928 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0) 7929 7930 /* Programs that must specify expected attach type at load time. */ 7931 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 7932 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0) 7933 7934 /* Programs that use BTF to identify attach point */ 7935 #define BPF_PROG_BTF(string, ptype, eatype) \ 7936 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1) 7937 7938 /* Programs that can be attached but attach type can't be identified by section 7939 * name. Kept for backward compatibility. 7940 */ 7941 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 7942 7943 #define SEC_DEF(sec_pfx, ptype, ...) { \ 7944 .sec = sec_pfx, \ 7945 .len = sizeof(sec_pfx) - 1, \ 7946 .prog_type = BPF_PROG_TYPE_##ptype, \ 7947 __VA_ARGS__ \ 7948 } 7949 7950 static struct bpf_link *attach_kprobe(const struct bpf_program *prog); 7951 static struct bpf_link *attach_tp(const struct bpf_program *prog); 7952 static struct bpf_link *attach_raw_tp(const struct bpf_program *prog); 7953 static struct bpf_link *attach_trace(const struct bpf_program *prog); 7954 static struct bpf_link *attach_lsm(const struct bpf_program *prog); 7955 static struct bpf_link *attach_iter(const struct bpf_program *prog); 7956 7957 static const struct bpf_sec_def section_defs[] = { 7958 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 7959 BPF_EAPROG_SEC("sk_reuseport/migrate", BPF_PROG_TYPE_SK_REUSEPORT, 7960 BPF_SK_REUSEPORT_SELECT_OR_MIGRATE), 7961 BPF_EAPROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT, 7962 BPF_SK_REUSEPORT_SELECT), 7963 SEC_DEF("kprobe/", KPROBE, 7964 .attach_fn = attach_kprobe), 7965 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), 7966 SEC_DEF("kretprobe/", KPROBE, 7967 .attach_fn = attach_kprobe), 7968 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), 7969 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 7970 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 7971 SEC_DEF("tracepoint/", TRACEPOINT, 7972 .attach_fn = attach_tp), 7973 SEC_DEF("tp/", TRACEPOINT, 7974 .attach_fn = attach_tp), 7975 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 7976 .attach_fn = attach_raw_tp), 7977 SEC_DEF("raw_tp/", RAW_TRACEPOINT, 7978 .attach_fn = attach_raw_tp), 7979 SEC_DEF("tp_btf/", TRACING, 7980 .expected_attach_type = BPF_TRACE_RAW_TP, 7981 .is_attach_btf = true, 7982 .attach_fn = attach_trace), 7983 SEC_DEF("fentry/", TRACING, 7984 .expected_attach_type = BPF_TRACE_FENTRY, 7985 .is_attach_btf = true, 7986 .attach_fn = attach_trace), 7987 SEC_DEF("fmod_ret/", TRACING, 7988 .expected_attach_type = BPF_MODIFY_RETURN, 7989 .is_attach_btf = true, 7990 .attach_fn = attach_trace), 7991 SEC_DEF("fexit/", TRACING, 7992 .expected_attach_type = BPF_TRACE_FEXIT, 7993 .is_attach_btf = true, 7994 .attach_fn = attach_trace), 7995 SEC_DEF("fentry.s/", TRACING, 7996 .expected_attach_type = BPF_TRACE_FENTRY, 7997 .is_attach_btf = true, 7998 .is_sleepable = true, 7999 .attach_fn = attach_trace), 8000 SEC_DEF("fmod_ret.s/", TRACING, 8001 .expected_attach_type = BPF_MODIFY_RETURN, 8002 .is_attach_btf = true, 8003 .is_sleepable = true, 8004 .attach_fn = attach_trace), 8005 SEC_DEF("fexit.s/", TRACING, 8006 .expected_attach_type = BPF_TRACE_FEXIT, 8007 .is_attach_btf = true, 8008 .is_sleepable = true, 8009 .attach_fn = attach_trace), 8010 SEC_DEF("freplace/", EXT, 8011 .is_attach_btf = true, 8012 .attach_fn = attach_trace), 8013 SEC_DEF("lsm/", LSM, 8014 .is_attach_btf = true, 8015 .expected_attach_type = BPF_LSM_MAC, 8016 .attach_fn = attach_lsm), 8017 SEC_DEF("lsm.s/", LSM, 8018 .is_attach_btf = true, 8019 .is_sleepable = true, 8020 .expected_attach_type = BPF_LSM_MAC, 8021 .attach_fn = attach_lsm), 8022 SEC_DEF("iter/", TRACING, 8023 .expected_attach_type = BPF_TRACE_ITER, 8024 .is_attach_btf = true, 8025 .attach_fn = attach_iter), 8026 SEC_DEF("syscall", SYSCALL, 8027 .is_sleepable = true), 8028 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP, 8029 BPF_XDP_DEVMAP), 8030 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP, 8031 BPF_XDP_CPUMAP), 8032 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP, 8033 BPF_XDP), 8034 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 8035 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 8036 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 8037 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 8038 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 8039 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 8040 BPF_CGROUP_INET_INGRESS), 8041 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 8042 BPF_CGROUP_INET_EGRESS), 8043 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 8044 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK, 8045 BPF_CGROUP_INET_SOCK_CREATE), 8046 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK, 8047 BPF_CGROUP_INET_SOCK_RELEASE), 8048 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 8049 BPF_CGROUP_INET_SOCK_CREATE), 8050 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 8051 BPF_CGROUP_INET4_POST_BIND), 8052 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 8053 BPF_CGROUP_INET6_POST_BIND), 8054 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 8055 BPF_CGROUP_DEVICE), 8056 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 8057 BPF_CGROUP_SOCK_OPS), 8058 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 8059 BPF_SK_SKB_STREAM_PARSER), 8060 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 8061 BPF_SK_SKB_STREAM_VERDICT), 8062 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 8063 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 8064 BPF_SK_MSG_VERDICT), 8065 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 8066 BPF_LIRC_MODE2), 8067 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 8068 BPF_FLOW_DISSECTOR), 8069 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8070 BPF_CGROUP_INET4_BIND), 8071 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8072 BPF_CGROUP_INET6_BIND), 8073 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8074 BPF_CGROUP_INET4_CONNECT), 8075 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8076 BPF_CGROUP_INET6_CONNECT), 8077 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8078 BPF_CGROUP_UDP4_SENDMSG), 8079 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8080 BPF_CGROUP_UDP6_SENDMSG), 8081 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8082 BPF_CGROUP_UDP4_RECVMSG), 8083 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8084 BPF_CGROUP_UDP6_RECVMSG), 8085 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8086 BPF_CGROUP_INET4_GETPEERNAME), 8087 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8088 BPF_CGROUP_INET6_GETPEERNAME), 8089 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8090 BPF_CGROUP_INET4_GETSOCKNAME), 8091 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8092 BPF_CGROUP_INET6_GETSOCKNAME), 8093 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 8094 BPF_CGROUP_SYSCTL), 8095 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8096 BPF_CGROUP_GETSOCKOPT), 8097 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8098 BPF_CGROUP_SETSOCKOPT), 8099 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), 8100 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP, 8101 BPF_SK_LOOKUP), 8102 }; 8103 8104 #undef BPF_PROG_SEC_IMPL 8105 #undef BPF_PROG_SEC 8106 #undef BPF_APROG_SEC 8107 #undef BPF_EAPROG_SEC 8108 #undef BPF_APROG_COMPAT 8109 #undef SEC_DEF 8110 8111 #define MAX_TYPE_NAME_SIZE 32 8112 8113 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 8114 { 8115 int i, n = ARRAY_SIZE(section_defs); 8116 8117 for (i = 0; i < n; i++) { 8118 if (strncmp(sec_name, 8119 section_defs[i].sec, section_defs[i].len)) 8120 continue; 8121 return §ion_defs[i]; 8122 } 8123 return NULL; 8124 } 8125 8126 static char *libbpf_get_type_names(bool attach_type) 8127 { 8128 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 8129 char *buf; 8130 8131 buf = malloc(len); 8132 if (!buf) 8133 return NULL; 8134 8135 buf[0] = '\0'; 8136 /* Forge string buf with all available names */ 8137 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8138 if (attach_type && !section_defs[i].is_attachable) 8139 continue; 8140 8141 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 8142 free(buf); 8143 return NULL; 8144 } 8145 strcat(buf, " "); 8146 strcat(buf, section_defs[i].sec); 8147 } 8148 8149 return buf; 8150 } 8151 8152 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 8153 enum bpf_attach_type *expected_attach_type) 8154 { 8155 const struct bpf_sec_def *sec_def; 8156 char *type_names; 8157 8158 if (!name) 8159 return libbpf_err(-EINVAL); 8160 8161 sec_def = find_sec_def(name); 8162 if (sec_def) { 8163 *prog_type = sec_def->prog_type; 8164 *expected_attach_type = sec_def->expected_attach_type; 8165 return 0; 8166 } 8167 8168 pr_debug("failed to guess program type from ELF section '%s'\n", name); 8169 type_names = libbpf_get_type_names(false); 8170 if (type_names != NULL) { 8171 pr_debug("supported section(type) names are:%s\n", type_names); 8172 free(type_names); 8173 } 8174 8175 return libbpf_err(-ESRCH); 8176 } 8177 8178 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 8179 size_t offset) 8180 { 8181 struct bpf_map *map; 8182 size_t i; 8183 8184 for (i = 0; i < obj->nr_maps; i++) { 8185 map = &obj->maps[i]; 8186 if (!bpf_map__is_struct_ops(map)) 8187 continue; 8188 if (map->sec_offset <= offset && 8189 offset - map->sec_offset < map->def.value_size) 8190 return map; 8191 } 8192 8193 return NULL; 8194 } 8195 8196 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 8197 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 8198 GElf_Shdr *shdr, Elf_Data *data) 8199 { 8200 const struct btf_member *member; 8201 struct bpf_struct_ops *st_ops; 8202 struct bpf_program *prog; 8203 unsigned int shdr_idx; 8204 const struct btf *btf; 8205 struct bpf_map *map; 8206 Elf_Data *symbols; 8207 unsigned int moff, insn_idx; 8208 const char *name; 8209 __u32 member_idx; 8210 GElf_Sym sym; 8211 GElf_Rel rel; 8212 int i, nrels; 8213 8214 symbols = obj->efile.symbols; 8215 btf = obj->btf; 8216 nrels = shdr->sh_size / shdr->sh_entsize; 8217 for (i = 0; i < nrels; i++) { 8218 if (!gelf_getrel(data, i, &rel)) { 8219 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 8220 return -LIBBPF_ERRNO__FORMAT; 8221 } 8222 8223 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 8224 pr_warn("struct_ops reloc: symbol %zx not found\n", 8225 (size_t)GELF_R_SYM(rel.r_info)); 8226 return -LIBBPF_ERRNO__FORMAT; 8227 } 8228 8229 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 8230 map = find_struct_ops_map_by_offset(obj, rel.r_offset); 8231 if (!map) { 8232 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", 8233 (size_t)rel.r_offset); 8234 return -EINVAL; 8235 } 8236 8237 moff = rel.r_offset - map->sec_offset; 8238 shdr_idx = sym.st_shndx; 8239 st_ops = map->st_ops; 8240 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 8241 map->name, 8242 (long long)(rel.r_info >> 32), 8243 (long long)sym.st_value, 8244 shdr_idx, (size_t)rel.r_offset, 8245 map->sec_offset, sym.st_name, name); 8246 8247 if (shdr_idx >= SHN_LORESERVE) { 8248 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", 8249 map->name, (size_t)rel.r_offset, shdr_idx); 8250 return -LIBBPF_ERRNO__RELOC; 8251 } 8252 if (sym.st_value % BPF_INSN_SZ) { 8253 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 8254 map->name, (unsigned long long)sym.st_value); 8255 return -LIBBPF_ERRNO__FORMAT; 8256 } 8257 insn_idx = sym.st_value / BPF_INSN_SZ; 8258 8259 member = find_member_by_offset(st_ops->type, moff * 8); 8260 if (!member) { 8261 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 8262 map->name, moff); 8263 return -EINVAL; 8264 } 8265 member_idx = member - btf_members(st_ops->type); 8266 name = btf__name_by_offset(btf, member->name_off); 8267 8268 if (!resolve_func_ptr(btf, member->type, NULL)) { 8269 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 8270 map->name, name); 8271 return -EINVAL; 8272 } 8273 8274 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 8275 if (!prog) { 8276 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 8277 map->name, shdr_idx, name); 8278 return -EINVAL; 8279 } 8280 8281 /* prevent the use of BPF prog with invalid type */ 8282 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { 8283 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n", 8284 map->name, prog->name); 8285 return -EINVAL; 8286 } 8287 8288 /* if we haven't yet processed this BPF program, record proper 8289 * attach_btf_id and member_idx 8290 */ 8291 if (!prog->attach_btf_id) { 8292 prog->attach_btf_id = st_ops->type_id; 8293 prog->expected_attach_type = member_idx; 8294 } 8295 8296 /* struct_ops BPF prog can be re-used between multiple 8297 * .struct_ops as long as it's the same struct_ops struct 8298 * definition and the same function pointer field 8299 */ 8300 if (prog->attach_btf_id != st_ops->type_id || 8301 prog->expected_attach_type != member_idx) { 8302 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 8303 map->name, prog->name, prog->sec_name, prog->type, 8304 prog->attach_btf_id, prog->expected_attach_type, name); 8305 return -EINVAL; 8306 } 8307 8308 st_ops->progs[member_idx] = prog; 8309 } 8310 8311 return 0; 8312 } 8313 8314 #define BTF_TRACE_PREFIX "btf_trace_" 8315 #define BTF_LSM_PREFIX "bpf_lsm_" 8316 #define BTF_ITER_PREFIX "bpf_iter_" 8317 #define BTF_MAX_NAME_SIZE 128 8318 8319 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 8320 const char **prefix, int *kind) 8321 { 8322 switch (attach_type) { 8323 case BPF_TRACE_RAW_TP: 8324 *prefix = BTF_TRACE_PREFIX; 8325 *kind = BTF_KIND_TYPEDEF; 8326 break; 8327 case BPF_LSM_MAC: 8328 *prefix = BTF_LSM_PREFIX; 8329 *kind = BTF_KIND_FUNC; 8330 break; 8331 case BPF_TRACE_ITER: 8332 *prefix = BTF_ITER_PREFIX; 8333 *kind = BTF_KIND_FUNC; 8334 break; 8335 default: 8336 *prefix = ""; 8337 *kind = BTF_KIND_FUNC; 8338 } 8339 } 8340 8341 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 8342 const char *name, __u32 kind) 8343 { 8344 char btf_type_name[BTF_MAX_NAME_SIZE]; 8345 int ret; 8346 8347 ret = snprintf(btf_type_name, sizeof(btf_type_name), 8348 "%s%s", prefix, name); 8349 /* snprintf returns the number of characters written excluding the 8350 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 8351 * indicates truncation. 8352 */ 8353 if (ret < 0 || ret >= sizeof(btf_type_name)) 8354 return -ENAMETOOLONG; 8355 return btf__find_by_name_kind(btf, btf_type_name, kind); 8356 } 8357 8358 static inline int find_attach_btf_id(struct btf *btf, const char *name, 8359 enum bpf_attach_type attach_type) 8360 { 8361 const char *prefix; 8362 int kind; 8363 8364 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 8365 return find_btf_by_prefix_kind(btf, prefix, name, kind); 8366 } 8367 8368 int libbpf_find_vmlinux_btf_id(const char *name, 8369 enum bpf_attach_type attach_type) 8370 { 8371 struct btf *btf; 8372 int err; 8373 8374 btf = btf__load_vmlinux_btf(); 8375 err = libbpf_get_error(btf); 8376 if (err) { 8377 pr_warn("vmlinux BTF is not found\n"); 8378 return libbpf_err(err); 8379 } 8380 8381 err = find_attach_btf_id(btf, name, attach_type); 8382 if (err <= 0) 8383 pr_warn("%s is not found in vmlinux BTF\n", name); 8384 8385 btf__free(btf); 8386 return libbpf_err(err); 8387 } 8388 8389 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 8390 { 8391 struct bpf_prog_info_linear *info_linear; 8392 struct bpf_prog_info *info; 8393 struct btf *btf; 8394 int err; 8395 8396 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); 8397 err = libbpf_get_error(info_linear); 8398 if (err) { 8399 pr_warn("failed get_prog_info_linear for FD %d\n", 8400 attach_prog_fd); 8401 return err; 8402 } 8403 8404 err = -EINVAL; 8405 info = &info_linear->info; 8406 if (!info->btf_id) { 8407 pr_warn("The target program doesn't have BTF\n"); 8408 goto out; 8409 } 8410 btf = btf__load_from_kernel_by_id(info->btf_id); 8411 if (libbpf_get_error(btf)) { 8412 pr_warn("Failed to get BTF of the program\n"); 8413 goto out; 8414 } 8415 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 8416 btf__free(btf); 8417 if (err <= 0) { 8418 pr_warn("%s is not found in prog's BTF\n", name); 8419 goto out; 8420 } 8421 out: 8422 free(info_linear); 8423 return err; 8424 } 8425 8426 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 8427 enum bpf_attach_type attach_type, 8428 int *btf_obj_fd, int *btf_type_id) 8429 { 8430 int ret, i; 8431 8432 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 8433 if (ret > 0) { 8434 *btf_obj_fd = 0; /* vmlinux BTF */ 8435 *btf_type_id = ret; 8436 return 0; 8437 } 8438 if (ret != -ENOENT) 8439 return ret; 8440 8441 ret = load_module_btfs(obj); 8442 if (ret) 8443 return ret; 8444 8445 for (i = 0; i < obj->btf_module_cnt; i++) { 8446 const struct module_btf *mod = &obj->btf_modules[i]; 8447 8448 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 8449 if (ret > 0) { 8450 *btf_obj_fd = mod->fd; 8451 *btf_type_id = ret; 8452 return 0; 8453 } 8454 if (ret == -ENOENT) 8455 continue; 8456 8457 return ret; 8458 } 8459 8460 return -ESRCH; 8461 } 8462 8463 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id) 8464 { 8465 enum bpf_attach_type attach_type = prog->expected_attach_type; 8466 __u32 attach_prog_fd = prog->attach_prog_fd; 8467 const char *attach_name; 8468 int err = 0; 8469 8470 if (!prog->sec_def || !prog->sec_def->is_attach_btf) { 8471 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", 8472 prog->sec_name); 8473 return -ESRCH; 8474 } 8475 attach_name = prog->sec_name + prog->sec_def->len; 8476 8477 /* BPF program's BTF ID */ 8478 if (attach_prog_fd) { 8479 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 8480 if (err < 0) { 8481 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 8482 attach_prog_fd, attach_name, err); 8483 return err; 8484 } 8485 *btf_obj_fd = 0; 8486 *btf_type_id = err; 8487 return 0; 8488 } 8489 8490 /* kernel/module BTF ID */ 8491 if (prog->obj->gen_loader) { 8492 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 8493 *btf_obj_fd = 0; 8494 *btf_type_id = 1; 8495 } else { 8496 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); 8497 } 8498 if (err) { 8499 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err); 8500 return err; 8501 } 8502 return 0; 8503 } 8504 8505 int libbpf_attach_type_by_name(const char *name, 8506 enum bpf_attach_type *attach_type) 8507 { 8508 char *type_names; 8509 const struct bpf_sec_def *sec_def; 8510 8511 if (!name) 8512 return libbpf_err(-EINVAL); 8513 8514 sec_def = find_sec_def(name); 8515 if (!sec_def) { 8516 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 8517 type_names = libbpf_get_type_names(true); 8518 if (type_names != NULL) { 8519 pr_debug("attachable section(type) names are:%s\n", type_names); 8520 free(type_names); 8521 } 8522 8523 return libbpf_err(-EINVAL); 8524 } 8525 8526 if (!sec_def->is_attachable) 8527 return libbpf_err(-EINVAL); 8528 8529 *attach_type = sec_def->expected_attach_type; 8530 return 0; 8531 } 8532 8533 int bpf_map__fd(const struct bpf_map *map) 8534 { 8535 return map ? map->fd : libbpf_err(-EINVAL); 8536 } 8537 8538 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 8539 { 8540 return map ? &map->def : libbpf_err_ptr(-EINVAL); 8541 } 8542 8543 const char *bpf_map__name(const struct bpf_map *map) 8544 { 8545 return map ? map->name : NULL; 8546 } 8547 8548 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 8549 { 8550 return map->def.type; 8551 } 8552 8553 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 8554 { 8555 if (map->fd >= 0) 8556 return libbpf_err(-EBUSY); 8557 map->def.type = type; 8558 return 0; 8559 } 8560 8561 __u32 bpf_map__map_flags(const struct bpf_map *map) 8562 { 8563 return map->def.map_flags; 8564 } 8565 8566 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 8567 { 8568 if (map->fd >= 0) 8569 return libbpf_err(-EBUSY); 8570 map->def.map_flags = flags; 8571 return 0; 8572 } 8573 8574 __u32 bpf_map__numa_node(const struct bpf_map *map) 8575 { 8576 return map->numa_node; 8577 } 8578 8579 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 8580 { 8581 if (map->fd >= 0) 8582 return libbpf_err(-EBUSY); 8583 map->numa_node = numa_node; 8584 return 0; 8585 } 8586 8587 __u32 bpf_map__key_size(const struct bpf_map *map) 8588 { 8589 return map->def.key_size; 8590 } 8591 8592 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 8593 { 8594 if (map->fd >= 0) 8595 return libbpf_err(-EBUSY); 8596 map->def.key_size = size; 8597 return 0; 8598 } 8599 8600 __u32 bpf_map__value_size(const struct bpf_map *map) 8601 { 8602 return map->def.value_size; 8603 } 8604 8605 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 8606 { 8607 if (map->fd >= 0) 8608 return libbpf_err(-EBUSY); 8609 map->def.value_size = size; 8610 return 0; 8611 } 8612 8613 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 8614 { 8615 return map ? map->btf_key_type_id : 0; 8616 } 8617 8618 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 8619 { 8620 return map ? map->btf_value_type_id : 0; 8621 } 8622 8623 int bpf_map__set_priv(struct bpf_map *map, void *priv, 8624 bpf_map_clear_priv_t clear_priv) 8625 { 8626 if (!map) 8627 return libbpf_err(-EINVAL); 8628 8629 if (map->priv) { 8630 if (map->clear_priv) 8631 map->clear_priv(map, map->priv); 8632 } 8633 8634 map->priv = priv; 8635 map->clear_priv = clear_priv; 8636 return 0; 8637 } 8638 8639 void *bpf_map__priv(const struct bpf_map *map) 8640 { 8641 return map ? map->priv : libbpf_err_ptr(-EINVAL); 8642 } 8643 8644 int bpf_map__set_initial_value(struct bpf_map *map, 8645 const void *data, size_t size) 8646 { 8647 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 8648 size != map->def.value_size || map->fd >= 0) 8649 return libbpf_err(-EINVAL); 8650 8651 memcpy(map->mmaped, data, size); 8652 return 0; 8653 } 8654 8655 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) 8656 { 8657 if (!map->mmaped) 8658 return NULL; 8659 *psize = map->def.value_size; 8660 return map->mmaped; 8661 } 8662 8663 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 8664 { 8665 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 8666 } 8667 8668 bool bpf_map__is_internal(const struct bpf_map *map) 8669 { 8670 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 8671 } 8672 8673 __u32 bpf_map__ifindex(const struct bpf_map *map) 8674 { 8675 return map->map_ifindex; 8676 } 8677 8678 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 8679 { 8680 if (map->fd >= 0) 8681 return libbpf_err(-EBUSY); 8682 map->map_ifindex = ifindex; 8683 return 0; 8684 } 8685 8686 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 8687 { 8688 if (!bpf_map_type__is_map_in_map(map->def.type)) { 8689 pr_warn("error: unsupported map type\n"); 8690 return libbpf_err(-EINVAL); 8691 } 8692 if (map->inner_map_fd != -1) { 8693 pr_warn("error: inner_map_fd already specified\n"); 8694 return libbpf_err(-EINVAL); 8695 } 8696 zfree(&map->inner_map); 8697 map->inner_map_fd = fd; 8698 return 0; 8699 } 8700 8701 static struct bpf_map * 8702 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 8703 { 8704 ssize_t idx; 8705 struct bpf_map *s, *e; 8706 8707 if (!obj || !obj->maps) 8708 return errno = EINVAL, NULL; 8709 8710 s = obj->maps; 8711 e = obj->maps + obj->nr_maps; 8712 8713 if ((m < s) || (m >= e)) { 8714 pr_warn("error in %s: map handler doesn't belong to object\n", 8715 __func__); 8716 return errno = EINVAL, NULL; 8717 } 8718 8719 idx = (m - obj->maps) + i; 8720 if (idx >= obj->nr_maps || idx < 0) 8721 return NULL; 8722 return &obj->maps[idx]; 8723 } 8724 8725 struct bpf_map * 8726 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 8727 { 8728 if (prev == NULL) 8729 return obj->maps; 8730 8731 return __bpf_map__iter(prev, obj, 1); 8732 } 8733 8734 struct bpf_map * 8735 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 8736 { 8737 if (next == NULL) { 8738 if (!obj->nr_maps) 8739 return NULL; 8740 return obj->maps + obj->nr_maps - 1; 8741 } 8742 8743 return __bpf_map__iter(next, obj, -1); 8744 } 8745 8746 struct bpf_map * 8747 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 8748 { 8749 struct bpf_map *pos; 8750 8751 bpf_object__for_each_map(pos, obj) { 8752 if (pos->name && !strcmp(pos->name, name)) 8753 return pos; 8754 } 8755 return errno = ENOENT, NULL; 8756 } 8757 8758 int 8759 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 8760 { 8761 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 8762 } 8763 8764 struct bpf_map * 8765 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 8766 { 8767 return libbpf_err_ptr(-ENOTSUP); 8768 } 8769 8770 long libbpf_get_error(const void *ptr) 8771 { 8772 if (!IS_ERR_OR_NULL(ptr)) 8773 return 0; 8774 8775 if (IS_ERR(ptr)) 8776 errno = -PTR_ERR(ptr); 8777 8778 /* If ptr == NULL, then errno should be already set by the failing 8779 * API, because libbpf never returns NULL on success and it now always 8780 * sets errno on error. So no extra errno handling for ptr == NULL 8781 * case. 8782 */ 8783 return -errno; 8784 } 8785 8786 int bpf_prog_load(const char *file, enum bpf_prog_type type, 8787 struct bpf_object **pobj, int *prog_fd) 8788 { 8789 struct bpf_prog_load_attr attr; 8790 8791 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 8792 attr.file = file; 8793 attr.prog_type = type; 8794 attr.expected_attach_type = 0; 8795 8796 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 8797 } 8798 8799 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 8800 struct bpf_object **pobj, int *prog_fd) 8801 { 8802 struct bpf_object_open_attr open_attr = {}; 8803 struct bpf_program *prog, *first_prog = NULL; 8804 struct bpf_object *obj; 8805 struct bpf_map *map; 8806 int err; 8807 8808 if (!attr) 8809 return libbpf_err(-EINVAL); 8810 if (!attr->file) 8811 return libbpf_err(-EINVAL); 8812 8813 open_attr.file = attr->file; 8814 open_attr.prog_type = attr->prog_type; 8815 8816 obj = bpf_object__open_xattr(&open_attr); 8817 err = libbpf_get_error(obj); 8818 if (err) 8819 return libbpf_err(-ENOENT); 8820 8821 bpf_object__for_each_program(prog, obj) { 8822 enum bpf_attach_type attach_type = attr->expected_attach_type; 8823 /* 8824 * to preserve backwards compatibility, bpf_prog_load treats 8825 * attr->prog_type, if specified, as an override to whatever 8826 * bpf_object__open guessed 8827 */ 8828 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { 8829 bpf_program__set_type(prog, attr->prog_type); 8830 bpf_program__set_expected_attach_type(prog, 8831 attach_type); 8832 } 8833 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { 8834 /* 8835 * we haven't guessed from section name and user 8836 * didn't provide a fallback type, too bad... 8837 */ 8838 bpf_object__close(obj); 8839 return libbpf_err(-EINVAL); 8840 } 8841 8842 prog->prog_ifindex = attr->ifindex; 8843 prog->log_level = attr->log_level; 8844 prog->prog_flags |= attr->prog_flags; 8845 if (!first_prog) 8846 first_prog = prog; 8847 } 8848 8849 bpf_object__for_each_map(map, obj) { 8850 if (!bpf_map__is_offload_neutral(map)) 8851 map->map_ifindex = attr->ifindex; 8852 } 8853 8854 if (!first_prog) { 8855 pr_warn("object file doesn't contain bpf program\n"); 8856 bpf_object__close(obj); 8857 return libbpf_err(-ENOENT); 8858 } 8859 8860 err = bpf_object__load(obj); 8861 if (err) { 8862 bpf_object__close(obj); 8863 return libbpf_err(err); 8864 } 8865 8866 *pobj = obj; 8867 *prog_fd = bpf_program__fd(first_prog); 8868 return 0; 8869 } 8870 8871 struct bpf_link { 8872 int (*detach)(struct bpf_link *link); 8873 void (*dealloc)(struct bpf_link *link); 8874 char *pin_path; /* NULL, if not pinned */ 8875 int fd; /* hook FD, -1 if not applicable */ 8876 bool disconnected; 8877 }; 8878 8879 /* Replace link's underlying BPF program with the new one */ 8880 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 8881 { 8882 int ret; 8883 8884 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 8885 return libbpf_err_errno(ret); 8886 } 8887 8888 /* Release "ownership" of underlying BPF resource (typically, BPF program 8889 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 8890 * link, when destructed through bpf_link__destroy() call won't attempt to 8891 * detach/unregisted that BPF resource. This is useful in situations where, 8892 * say, attached BPF program has to outlive userspace program that attached it 8893 * in the system. Depending on type of BPF program, though, there might be 8894 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 8895 * exit of userspace program doesn't trigger automatic detachment and clean up 8896 * inside the kernel. 8897 */ 8898 void bpf_link__disconnect(struct bpf_link *link) 8899 { 8900 link->disconnected = true; 8901 } 8902 8903 int bpf_link__destroy(struct bpf_link *link) 8904 { 8905 int err = 0; 8906 8907 if (IS_ERR_OR_NULL(link)) 8908 return 0; 8909 8910 if (!link->disconnected && link->detach) 8911 err = link->detach(link); 8912 if (link->pin_path) 8913 free(link->pin_path); 8914 if (link->dealloc) 8915 link->dealloc(link); 8916 else 8917 free(link); 8918 8919 return libbpf_err(err); 8920 } 8921 8922 int bpf_link__fd(const struct bpf_link *link) 8923 { 8924 return link->fd; 8925 } 8926 8927 const char *bpf_link__pin_path(const struct bpf_link *link) 8928 { 8929 return link->pin_path; 8930 } 8931 8932 static int bpf_link__detach_fd(struct bpf_link *link) 8933 { 8934 return libbpf_err_errno(close(link->fd)); 8935 } 8936 8937 struct bpf_link *bpf_link__open(const char *path) 8938 { 8939 struct bpf_link *link; 8940 int fd; 8941 8942 fd = bpf_obj_get(path); 8943 if (fd < 0) { 8944 fd = -errno; 8945 pr_warn("failed to open link at %s: %d\n", path, fd); 8946 return libbpf_err_ptr(fd); 8947 } 8948 8949 link = calloc(1, sizeof(*link)); 8950 if (!link) { 8951 close(fd); 8952 return libbpf_err_ptr(-ENOMEM); 8953 } 8954 link->detach = &bpf_link__detach_fd; 8955 link->fd = fd; 8956 8957 link->pin_path = strdup(path); 8958 if (!link->pin_path) { 8959 bpf_link__destroy(link); 8960 return libbpf_err_ptr(-ENOMEM); 8961 } 8962 8963 return link; 8964 } 8965 8966 int bpf_link__detach(struct bpf_link *link) 8967 { 8968 return bpf_link_detach(link->fd) ? -errno : 0; 8969 } 8970 8971 int bpf_link__pin(struct bpf_link *link, const char *path) 8972 { 8973 int err; 8974 8975 if (link->pin_path) 8976 return libbpf_err(-EBUSY); 8977 err = make_parent_dir(path); 8978 if (err) 8979 return libbpf_err(err); 8980 err = check_path(path); 8981 if (err) 8982 return libbpf_err(err); 8983 8984 link->pin_path = strdup(path); 8985 if (!link->pin_path) 8986 return libbpf_err(-ENOMEM); 8987 8988 if (bpf_obj_pin(link->fd, link->pin_path)) { 8989 err = -errno; 8990 zfree(&link->pin_path); 8991 return libbpf_err(err); 8992 } 8993 8994 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 8995 return 0; 8996 } 8997 8998 int bpf_link__unpin(struct bpf_link *link) 8999 { 9000 int err; 9001 9002 if (!link->pin_path) 9003 return libbpf_err(-EINVAL); 9004 9005 err = unlink(link->pin_path); 9006 if (err != 0) 9007 return -errno; 9008 9009 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 9010 zfree(&link->pin_path); 9011 return 0; 9012 } 9013 9014 static int poke_kprobe_events(bool add, const char *name, bool retprobe, uint64_t offset) 9015 { 9016 int fd, ret = 0; 9017 pid_t p = getpid(); 9018 char cmd[260], probename[128], probefunc[128]; 9019 const char *file = "/sys/kernel/debug/tracing/kprobe_events"; 9020 9021 if (retprobe) 9022 snprintf(probename, sizeof(probename), "kretprobes/%s_libbpf_%u", name, p); 9023 else 9024 snprintf(probename, sizeof(probename), "kprobes/%s_libbpf_%u", name, p); 9025 9026 if (offset) 9027 snprintf(probefunc, sizeof(probefunc), "%s+%zu", name, (size_t)offset); 9028 9029 if (add) { 9030 snprintf(cmd, sizeof(cmd), "%c:%s %s", 9031 retprobe ? 'r' : 'p', 9032 probename, 9033 offset ? probefunc : name); 9034 } else { 9035 snprintf(cmd, sizeof(cmd), "-:%s", probename); 9036 } 9037 9038 fd = open(file, O_WRONLY | O_APPEND, 0); 9039 if (!fd) 9040 return -errno; 9041 ret = write(fd, cmd, strlen(cmd)); 9042 if (ret < 0) 9043 ret = -errno; 9044 close(fd); 9045 9046 return ret; 9047 } 9048 9049 static inline int add_kprobe_event_legacy(const char *name, bool retprobe, uint64_t offset) 9050 { 9051 return poke_kprobe_events(true, name, retprobe, offset); 9052 } 9053 9054 static inline int remove_kprobe_event_legacy(const char *name, bool retprobe) 9055 { 9056 return poke_kprobe_events(false, name, retprobe, 0); 9057 } 9058 9059 struct bpf_link_perf { 9060 struct bpf_link link; 9061 int perf_event_fd; 9062 /* legacy kprobe support: keep track of probe identifier and type */ 9063 char *legacy_probe_name; 9064 bool legacy_is_retprobe; 9065 }; 9066 9067 static int bpf_link_perf_detach(struct bpf_link *link) 9068 { 9069 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 9070 int err = 0; 9071 9072 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 9073 err = -errno; 9074 9075 if (perf_link->perf_event_fd != link->fd) 9076 close(perf_link->perf_event_fd); 9077 close(link->fd); 9078 9079 /* legacy kprobe needs to be removed after perf event fd closure */ 9080 if (perf_link->legacy_probe_name) 9081 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, 9082 perf_link->legacy_is_retprobe); 9083 9084 return err; 9085 } 9086 9087 static void bpf_link_perf_dealloc(struct bpf_link *link) 9088 { 9089 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 9090 9091 free(perf_link->legacy_probe_name); 9092 free(perf_link); 9093 } 9094 9095 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, 9096 const struct bpf_perf_event_opts *opts) 9097 { 9098 char errmsg[STRERR_BUFSIZE]; 9099 struct bpf_link_perf *link; 9100 int prog_fd, link_fd = -1, err; 9101 9102 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 9103 return libbpf_err_ptr(-EINVAL); 9104 9105 if (pfd < 0) { 9106 pr_warn("prog '%s': invalid perf event FD %d\n", 9107 prog->name, pfd); 9108 return libbpf_err_ptr(-EINVAL); 9109 } 9110 prog_fd = bpf_program__fd(prog); 9111 if (prog_fd < 0) { 9112 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 9113 prog->name); 9114 return libbpf_err_ptr(-EINVAL); 9115 } 9116 9117 link = calloc(1, sizeof(*link)); 9118 if (!link) 9119 return libbpf_err_ptr(-ENOMEM); 9120 link->link.detach = &bpf_link_perf_detach; 9121 link->link.dealloc = &bpf_link_perf_dealloc; 9122 link->perf_event_fd = pfd; 9123 9124 if (kernel_supports(prog->obj, FEAT_PERF_LINK)) { 9125 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 9126 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 9127 9128 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 9129 if (link_fd < 0) { 9130 err = -errno; 9131 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", 9132 prog->name, pfd, 9133 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9134 goto err_out; 9135 } 9136 link->link.fd = link_fd; 9137 } else { 9138 if (OPTS_GET(opts, bpf_cookie, 0)) { 9139 pr_warn("prog '%s': user context value is not supported\n", prog->name); 9140 err = -EOPNOTSUPP; 9141 goto err_out; 9142 } 9143 9144 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 9145 err = -errno; 9146 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 9147 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9148 if (err == -EPROTO) 9149 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 9150 prog->name, pfd); 9151 goto err_out; 9152 } 9153 link->link.fd = pfd; 9154 } 9155 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 9156 err = -errno; 9157 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 9158 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9159 goto err_out; 9160 } 9161 9162 return &link->link; 9163 err_out: 9164 if (link_fd >= 0) 9165 close(link_fd); 9166 free(link); 9167 return libbpf_err_ptr(err); 9168 } 9169 9170 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) 9171 { 9172 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 9173 } 9174 9175 /* 9176 * this function is expected to parse integer in the range of [0, 2^31-1] from 9177 * given file using scanf format string fmt. If actual parsed value is 9178 * negative, the result might be indistinguishable from error 9179 */ 9180 static int parse_uint_from_file(const char *file, const char *fmt) 9181 { 9182 char buf[STRERR_BUFSIZE]; 9183 int err, ret; 9184 FILE *f; 9185 9186 f = fopen(file, "r"); 9187 if (!f) { 9188 err = -errno; 9189 pr_debug("failed to open '%s': %s\n", file, 9190 libbpf_strerror_r(err, buf, sizeof(buf))); 9191 return err; 9192 } 9193 err = fscanf(f, fmt, &ret); 9194 if (err != 1) { 9195 err = err == EOF ? -EIO : -errno; 9196 pr_debug("failed to parse '%s': %s\n", file, 9197 libbpf_strerror_r(err, buf, sizeof(buf))); 9198 fclose(f); 9199 return err; 9200 } 9201 fclose(f); 9202 return ret; 9203 } 9204 9205 static int determine_kprobe_perf_type_legacy(const char *func_name, bool is_retprobe) 9206 { 9207 char file[192]; 9208 9209 snprintf(file, sizeof(file), 9210 "/sys/kernel/debug/tracing/events/%s/%s_libbpf_%d/id", 9211 is_retprobe ? "kretprobes" : "kprobes", 9212 func_name, getpid()); 9213 9214 return parse_uint_from_file(file, "%d\n"); 9215 } 9216 9217 static int determine_kprobe_perf_type(void) 9218 { 9219 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 9220 9221 return parse_uint_from_file(file, "%d\n"); 9222 } 9223 9224 static int determine_uprobe_perf_type(void) 9225 { 9226 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 9227 9228 return parse_uint_from_file(file, "%d\n"); 9229 } 9230 9231 static int determine_kprobe_retprobe_bit(void) 9232 { 9233 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 9234 9235 return parse_uint_from_file(file, "config:%d\n"); 9236 } 9237 9238 static int determine_uprobe_retprobe_bit(void) 9239 { 9240 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 9241 9242 return parse_uint_from_file(file, "config:%d\n"); 9243 } 9244 9245 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 9246 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 9247 9248 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 9249 uint64_t offset, int pid, size_t ref_ctr_off) 9250 { 9251 struct perf_event_attr attr = {}; 9252 char errmsg[STRERR_BUFSIZE]; 9253 int type, pfd, err; 9254 9255 if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 9256 return -EINVAL; 9257 9258 type = uprobe ? determine_uprobe_perf_type() 9259 : determine_kprobe_perf_type(); 9260 if (type < 0) { 9261 pr_warn("failed to determine %s perf type: %s\n", 9262 uprobe ? "uprobe" : "kprobe", 9263 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 9264 return type; 9265 } 9266 if (retprobe) { 9267 int bit = uprobe ? determine_uprobe_retprobe_bit() 9268 : determine_kprobe_retprobe_bit(); 9269 9270 if (bit < 0) { 9271 pr_warn("failed to determine %s retprobe bit: %s\n", 9272 uprobe ? "uprobe" : "kprobe", 9273 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 9274 return bit; 9275 } 9276 attr.config |= 1 << bit; 9277 } 9278 attr.size = sizeof(attr); 9279 attr.type = type; 9280 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 9281 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 9282 attr.config2 = offset; /* kprobe_addr or probe_offset */ 9283 9284 /* pid filter is meaningful only for uprobes */ 9285 pfd = syscall(__NR_perf_event_open, &attr, 9286 pid < 0 ? -1 : pid /* pid */, 9287 pid == -1 ? 0 : -1 /* cpu */, 9288 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9289 if (pfd < 0) { 9290 err = -errno; 9291 pr_warn("%s perf_event_open() failed: %s\n", 9292 uprobe ? "uprobe" : "kprobe", 9293 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9294 return err; 9295 } 9296 return pfd; 9297 } 9298 9299 static int perf_event_kprobe_open_legacy(bool retprobe, const char *name, uint64_t offset, int pid) 9300 { 9301 struct perf_event_attr attr = {}; 9302 char errmsg[STRERR_BUFSIZE]; 9303 int type, pfd, err; 9304 9305 err = add_kprobe_event_legacy(name, retprobe, offset); 9306 if (err < 0) { 9307 pr_warn("failed to add legacy kprobe event: %s\n", 9308 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9309 return err; 9310 } 9311 type = determine_kprobe_perf_type_legacy(name, retprobe); 9312 if (type < 0) { 9313 pr_warn("failed to determine legacy kprobe event id: %s\n", 9314 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 9315 return type; 9316 } 9317 attr.size = sizeof(attr); 9318 attr.config = type; 9319 attr.type = PERF_TYPE_TRACEPOINT; 9320 9321 pfd = syscall(__NR_perf_event_open, &attr, 9322 pid < 0 ? -1 : pid, /* pid */ 9323 pid == -1 ? 0 : -1, /* cpu */ 9324 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9325 if (pfd < 0) { 9326 err = -errno; 9327 pr_warn("legacy kprobe perf_event_open() failed: %s\n", 9328 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9329 return err; 9330 } 9331 return pfd; 9332 } 9333 9334 struct bpf_link * 9335 bpf_program__attach_kprobe_opts(const struct bpf_program *prog, 9336 const char *func_name, 9337 const struct bpf_kprobe_opts *opts) 9338 { 9339 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9340 char errmsg[STRERR_BUFSIZE]; 9341 char *legacy_probe = NULL; 9342 struct bpf_link *link; 9343 unsigned long offset; 9344 bool retprobe, legacy; 9345 int pfd, err; 9346 9347 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 9348 return libbpf_err_ptr(-EINVAL); 9349 9350 retprobe = OPTS_GET(opts, retprobe, false); 9351 offset = OPTS_GET(opts, offset, 0); 9352 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9353 9354 legacy = determine_kprobe_perf_type() < 0; 9355 if (!legacy) { 9356 pfd = perf_event_open_probe(false /* uprobe */, retprobe, 9357 func_name, offset, 9358 -1 /* pid */, 0 /* ref_ctr_off */); 9359 } else { 9360 legacy_probe = strdup(func_name); 9361 if (!legacy_probe) 9362 return libbpf_err_ptr(-ENOMEM); 9363 9364 pfd = perf_event_kprobe_open_legacy(retprobe, func_name, 9365 offset, -1 /* pid */); 9366 } 9367 if (pfd < 0) { 9368 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n", 9369 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 9370 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9371 return libbpf_err_ptr(pfd); 9372 } 9373 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9374 err = libbpf_get_error(link); 9375 if (err) { 9376 close(pfd); 9377 pr_warn("prog '%s': failed to attach to %s '%s': %s\n", 9378 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 9379 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9380 return libbpf_err_ptr(err); 9381 } 9382 if (legacy) { 9383 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 9384 9385 perf_link->legacy_probe_name = legacy_probe; 9386 perf_link->legacy_is_retprobe = retprobe; 9387 } 9388 9389 return link; 9390 } 9391 9392 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, 9393 bool retprobe, 9394 const char *func_name) 9395 { 9396 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 9397 .retprobe = retprobe, 9398 ); 9399 9400 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 9401 } 9402 9403 static struct bpf_link *attach_kprobe(const struct bpf_program *prog) 9404 { 9405 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 9406 unsigned long offset = 0; 9407 struct bpf_link *link; 9408 const char *func_name; 9409 char *func; 9410 int n, err; 9411 9412 func_name = prog->sec_name + prog->sec_def->len; 9413 opts.retprobe = strcmp(prog->sec_def->sec, "kretprobe/") == 0; 9414 9415 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 9416 if (n < 1) { 9417 err = -EINVAL; 9418 pr_warn("kprobe name is invalid: %s\n", func_name); 9419 return libbpf_err_ptr(err); 9420 } 9421 if (opts.retprobe && offset != 0) { 9422 free(func); 9423 err = -EINVAL; 9424 pr_warn("kretprobes do not support offset specification\n"); 9425 return libbpf_err_ptr(err); 9426 } 9427 9428 opts.offset = offset; 9429 link = bpf_program__attach_kprobe_opts(prog, func, &opts); 9430 free(func); 9431 return link; 9432 } 9433 9434 LIBBPF_API struct bpf_link * 9435 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, 9436 const char *binary_path, size_t func_offset, 9437 const struct bpf_uprobe_opts *opts) 9438 { 9439 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9440 char errmsg[STRERR_BUFSIZE]; 9441 struct bpf_link *link; 9442 size_t ref_ctr_off; 9443 int pfd, err; 9444 bool retprobe; 9445 9446 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 9447 return libbpf_err_ptr(-EINVAL); 9448 9449 retprobe = OPTS_GET(opts, retprobe, false); 9450 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 9451 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9452 9453 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 9454 func_offset, pid, ref_ctr_off); 9455 if (pfd < 0) { 9456 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 9457 prog->name, retprobe ? "uretprobe" : "uprobe", 9458 binary_path, func_offset, 9459 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9460 return libbpf_err_ptr(pfd); 9461 } 9462 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9463 err = libbpf_get_error(link); 9464 if (err) { 9465 close(pfd); 9466 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 9467 prog->name, retprobe ? "uretprobe" : "uprobe", 9468 binary_path, func_offset, 9469 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9470 return libbpf_err_ptr(err); 9471 } 9472 return link; 9473 } 9474 9475 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, 9476 bool retprobe, pid_t pid, 9477 const char *binary_path, 9478 size_t func_offset) 9479 { 9480 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 9481 9482 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 9483 } 9484 9485 static int determine_tracepoint_id(const char *tp_category, 9486 const char *tp_name) 9487 { 9488 char file[PATH_MAX]; 9489 int ret; 9490 9491 ret = snprintf(file, sizeof(file), 9492 "/sys/kernel/debug/tracing/events/%s/%s/id", 9493 tp_category, tp_name); 9494 if (ret < 0) 9495 return -errno; 9496 if (ret >= sizeof(file)) { 9497 pr_debug("tracepoint %s/%s path is too long\n", 9498 tp_category, tp_name); 9499 return -E2BIG; 9500 } 9501 return parse_uint_from_file(file, "%d\n"); 9502 } 9503 9504 static int perf_event_open_tracepoint(const char *tp_category, 9505 const char *tp_name) 9506 { 9507 struct perf_event_attr attr = {}; 9508 char errmsg[STRERR_BUFSIZE]; 9509 int tp_id, pfd, err; 9510 9511 tp_id = determine_tracepoint_id(tp_category, tp_name); 9512 if (tp_id < 0) { 9513 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 9514 tp_category, tp_name, 9515 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 9516 return tp_id; 9517 } 9518 9519 attr.type = PERF_TYPE_TRACEPOINT; 9520 attr.size = sizeof(attr); 9521 attr.config = tp_id; 9522 9523 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 9524 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9525 if (pfd < 0) { 9526 err = -errno; 9527 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 9528 tp_category, tp_name, 9529 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9530 return err; 9531 } 9532 return pfd; 9533 } 9534 9535 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, 9536 const char *tp_category, 9537 const char *tp_name, 9538 const struct bpf_tracepoint_opts *opts) 9539 { 9540 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9541 char errmsg[STRERR_BUFSIZE]; 9542 struct bpf_link *link; 9543 int pfd, err; 9544 9545 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 9546 return libbpf_err_ptr(-EINVAL); 9547 9548 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9549 9550 pfd = perf_event_open_tracepoint(tp_category, tp_name); 9551 if (pfd < 0) { 9552 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 9553 prog->name, tp_category, tp_name, 9554 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9555 return libbpf_err_ptr(pfd); 9556 } 9557 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9558 err = libbpf_get_error(link); 9559 if (err) { 9560 close(pfd); 9561 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 9562 prog->name, tp_category, tp_name, 9563 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9564 return libbpf_err_ptr(err); 9565 } 9566 return link; 9567 } 9568 9569 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, 9570 const char *tp_category, 9571 const char *tp_name) 9572 { 9573 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 9574 } 9575 9576 static struct bpf_link *attach_tp(const struct bpf_program *prog) 9577 { 9578 char *sec_name, *tp_cat, *tp_name; 9579 struct bpf_link *link; 9580 9581 sec_name = strdup(prog->sec_name); 9582 if (!sec_name) 9583 return libbpf_err_ptr(-ENOMEM); 9584 9585 /* extract "tp/<category>/<name>" */ 9586 tp_cat = sec_name + prog->sec_def->len; 9587 tp_name = strchr(tp_cat, '/'); 9588 if (!tp_name) { 9589 free(sec_name); 9590 return libbpf_err_ptr(-EINVAL); 9591 } 9592 *tp_name = '\0'; 9593 tp_name++; 9594 9595 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 9596 free(sec_name); 9597 return link; 9598 } 9599 9600 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, 9601 const char *tp_name) 9602 { 9603 char errmsg[STRERR_BUFSIZE]; 9604 struct bpf_link *link; 9605 int prog_fd, pfd; 9606 9607 prog_fd = bpf_program__fd(prog); 9608 if (prog_fd < 0) { 9609 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9610 return libbpf_err_ptr(-EINVAL); 9611 } 9612 9613 link = calloc(1, sizeof(*link)); 9614 if (!link) 9615 return libbpf_err_ptr(-ENOMEM); 9616 link->detach = &bpf_link__detach_fd; 9617 9618 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 9619 if (pfd < 0) { 9620 pfd = -errno; 9621 free(link); 9622 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 9623 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9624 return libbpf_err_ptr(pfd); 9625 } 9626 link->fd = pfd; 9627 return link; 9628 } 9629 9630 static struct bpf_link *attach_raw_tp(const struct bpf_program *prog) 9631 { 9632 const char *tp_name = prog->sec_name + prog->sec_def->len; 9633 9634 return bpf_program__attach_raw_tracepoint(prog, tp_name); 9635 } 9636 9637 /* Common logic for all BPF program types that attach to a btf_id */ 9638 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog) 9639 { 9640 char errmsg[STRERR_BUFSIZE]; 9641 struct bpf_link *link; 9642 int prog_fd, pfd; 9643 9644 prog_fd = bpf_program__fd(prog); 9645 if (prog_fd < 0) { 9646 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9647 return libbpf_err_ptr(-EINVAL); 9648 } 9649 9650 link = calloc(1, sizeof(*link)); 9651 if (!link) 9652 return libbpf_err_ptr(-ENOMEM); 9653 link->detach = &bpf_link__detach_fd; 9654 9655 pfd = bpf_raw_tracepoint_open(NULL, prog_fd); 9656 if (pfd < 0) { 9657 pfd = -errno; 9658 free(link); 9659 pr_warn("prog '%s': failed to attach: %s\n", 9660 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9661 return libbpf_err_ptr(pfd); 9662 } 9663 link->fd = pfd; 9664 return (struct bpf_link *)link; 9665 } 9666 9667 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) 9668 { 9669 return bpf_program__attach_btf_id(prog); 9670 } 9671 9672 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) 9673 { 9674 return bpf_program__attach_btf_id(prog); 9675 } 9676 9677 static struct bpf_link *attach_trace(const struct bpf_program *prog) 9678 { 9679 return bpf_program__attach_trace(prog); 9680 } 9681 9682 static struct bpf_link *attach_lsm(const struct bpf_program *prog) 9683 { 9684 return bpf_program__attach_lsm(prog); 9685 } 9686 9687 static struct bpf_link * 9688 bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id, 9689 const char *target_name) 9690 { 9691 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, 9692 .target_btf_id = btf_id); 9693 enum bpf_attach_type attach_type; 9694 char errmsg[STRERR_BUFSIZE]; 9695 struct bpf_link *link; 9696 int prog_fd, link_fd; 9697 9698 prog_fd = bpf_program__fd(prog); 9699 if (prog_fd < 0) { 9700 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9701 return libbpf_err_ptr(-EINVAL); 9702 } 9703 9704 link = calloc(1, sizeof(*link)); 9705 if (!link) 9706 return libbpf_err_ptr(-ENOMEM); 9707 link->detach = &bpf_link__detach_fd; 9708 9709 attach_type = bpf_program__get_expected_attach_type(prog); 9710 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); 9711 if (link_fd < 0) { 9712 link_fd = -errno; 9713 free(link); 9714 pr_warn("prog '%s': failed to attach to %s: %s\n", 9715 prog->name, target_name, 9716 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 9717 return libbpf_err_ptr(link_fd); 9718 } 9719 link->fd = link_fd; 9720 return link; 9721 } 9722 9723 struct bpf_link * 9724 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) 9725 { 9726 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup"); 9727 } 9728 9729 struct bpf_link * 9730 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) 9731 { 9732 return bpf_program__attach_fd(prog, netns_fd, 0, "netns"); 9733 } 9734 9735 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) 9736 { 9737 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 9738 return bpf_program__attach_fd(prog, ifindex, 0, "xdp"); 9739 } 9740 9741 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, 9742 int target_fd, 9743 const char *attach_func_name) 9744 { 9745 int btf_id; 9746 9747 if (!!target_fd != !!attach_func_name) { 9748 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 9749 prog->name); 9750 return libbpf_err_ptr(-EINVAL); 9751 } 9752 9753 if (prog->type != BPF_PROG_TYPE_EXT) { 9754 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 9755 prog->name); 9756 return libbpf_err_ptr(-EINVAL); 9757 } 9758 9759 if (target_fd) { 9760 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 9761 if (btf_id < 0) 9762 return libbpf_err_ptr(btf_id); 9763 9764 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace"); 9765 } else { 9766 /* no target, so use raw_tracepoint_open for compatibility 9767 * with old kernels 9768 */ 9769 return bpf_program__attach_trace(prog); 9770 } 9771 } 9772 9773 struct bpf_link * 9774 bpf_program__attach_iter(const struct bpf_program *prog, 9775 const struct bpf_iter_attach_opts *opts) 9776 { 9777 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 9778 char errmsg[STRERR_BUFSIZE]; 9779 struct bpf_link *link; 9780 int prog_fd, link_fd; 9781 __u32 target_fd = 0; 9782 9783 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 9784 return libbpf_err_ptr(-EINVAL); 9785 9786 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 9787 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 9788 9789 prog_fd = bpf_program__fd(prog); 9790 if (prog_fd < 0) { 9791 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9792 return libbpf_err_ptr(-EINVAL); 9793 } 9794 9795 link = calloc(1, sizeof(*link)); 9796 if (!link) 9797 return libbpf_err_ptr(-ENOMEM); 9798 link->detach = &bpf_link__detach_fd; 9799 9800 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 9801 &link_create_opts); 9802 if (link_fd < 0) { 9803 link_fd = -errno; 9804 free(link); 9805 pr_warn("prog '%s': failed to attach to iterator: %s\n", 9806 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 9807 return libbpf_err_ptr(link_fd); 9808 } 9809 link->fd = link_fd; 9810 return link; 9811 } 9812 9813 static struct bpf_link *attach_iter(const struct bpf_program *prog) 9814 { 9815 return bpf_program__attach_iter(prog, NULL); 9816 } 9817 9818 struct bpf_link *bpf_program__attach(const struct bpf_program *prog) 9819 { 9820 if (!prog->sec_def || !prog->sec_def->attach_fn) 9821 return libbpf_err_ptr(-ESRCH); 9822 9823 return prog->sec_def->attach_fn(prog); 9824 } 9825 9826 static int bpf_link__detach_struct_ops(struct bpf_link *link) 9827 { 9828 __u32 zero = 0; 9829 9830 if (bpf_map_delete_elem(link->fd, &zero)) 9831 return -errno; 9832 9833 return 0; 9834 } 9835 9836 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) 9837 { 9838 struct bpf_struct_ops *st_ops; 9839 struct bpf_link *link; 9840 __u32 i, zero = 0; 9841 int err; 9842 9843 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 9844 return libbpf_err_ptr(-EINVAL); 9845 9846 link = calloc(1, sizeof(*link)); 9847 if (!link) 9848 return libbpf_err_ptr(-EINVAL); 9849 9850 st_ops = map->st_ops; 9851 for (i = 0; i < btf_vlen(st_ops->type); i++) { 9852 struct bpf_program *prog = st_ops->progs[i]; 9853 void *kern_data; 9854 int prog_fd; 9855 9856 if (!prog) 9857 continue; 9858 9859 prog_fd = bpf_program__fd(prog); 9860 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 9861 *(unsigned long *)kern_data = prog_fd; 9862 } 9863 9864 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); 9865 if (err) { 9866 err = -errno; 9867 free(link); 9868 return libbpf_err_ptr(err); 9869 } 9870 9871 link->detach = bpf_link__detach_struct_ops; 9872 link->fd = map->fd; 9873 9874 return link; 9875 } 9876 9877 enum bpf_perf_event_ret 9878 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 9879 void **copy_mem, size_t *copy_size, 9880 bpf_perf_event_print_t fn, void *private_data) 9881 { 9882 struct perf_event_mmap_page *header = mmap_mem; 9883 __u64 data_head = ring_buffer_read_head(header); 9884 __u64 data_tail = header->data_tail; 9885 void *base = ((__u8 *)header) + page_size; 9886 int ret = LIBBPF_PERF_EVENT_CONT; 9887 struct perf_event_header *ehdr; 9888 size_t ehdr_size; 9889 9890 while (data_head != data_tail) { 9891 ehdr = base + (data_tail & (mmap_size - 1)); 9892 ehdr_size = ehdr->size; 9893 9894 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 9895 void *copy_start = ehdr; 9896 size_t len_first = base + mmap_size - copy_start; 9897 size_t len_secnd = ehdr_size - len_first; 9898 9899 if (*copy_size < ehdr_size) { 9900 free(*copy_mem); 9901 *copy_mem = malloc(ehdr_size); 9902 if (!*copy_mem) { 9903 *copy_size = 0; 9904 ret = LIBBPF_PERF_EVENT_ERROR; 9905 break; 9906 } 9907 *copy_size = ehdr_size; 9908 } 9909 9910 memcpy(*copy_mem, copy_start, len_first); 9911 memcpy(*copy_mem + len_first, base, len_secnd); 9912 ehdr = *copy_mem; 9913 } 9914 9915 ret = fn(ehdr, private_data); 9916 data_tail += ehdr_size; 9917 if (ret != LIBBPF_PERF_EVENT_CONT) 9918 break; 9919 } 9920 9921 ring_buffer_write_tail(header, data_tail); 9922 return libbpf_err(ret); 9923 } 9924 9925 struct perf_buffer; 9926 9927 struct perf_buffer_params { 9928 struct perf_event_attr *attr; 9929 /* if event_cb is specified, it takes precendence */ 9930 perf_buffer_event_fn event_cb; 9931 /* sample_cb and lost_cb are higher-level common-case callbacks */ 9932 perf_buffer_sample_fn sample_cb; 9933 perf_buffer_lost_fn lost_cb; 9934 void *ctx; 9935 int cpu_cnt; 9936 int *cpus; 9937 int *map_keys; 9938 }; 9939 9940 struct perf_cpu_buf { 9941 struct perf_buffer *pb; 9942 void *base; /* mmap()'ed memory */ 9943 void *buf; /* for reconstructing segmented data */ 9944 size_t buf_size; 9945 int fd; 9946 int cpu; 9947 int map_key; 9948 }; 9949 9950 struct perf_buffer { 9951 perf_buffer_event_fn event_cb; 9952 perf_buffer_sample_fn sample_cb; 9953 perf_buffer_lost_fn lost_cb; 9954 void *ctx; /* passed into callbacks */ 9955 9956 size_t page_size; 9957 size_t mmap_size; 9958 struct perf_cpu_buf **cpu_bufs; 9959 struct epoll_event *events; 9960 int cpu_cnt; /* number of allocated CPU buffers */ 9961 int epoll_fd; /* perf event FD */ 9962 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 9963 }; 9964 9965 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 9966 struct perf_cpu_buf *cpu_buf) 9967 { 9968 if (!cpu_buf) 9969 return; 9970 if (cpu_buf->base && 9971 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 9972 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 9973 if (cpu_buf->fd >= 0) { 9974 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 9975 close(cpu_buf->fd); 9976 } 9977 free(cpu_buf->buf); 9978 free(cpu_buf); 9979 } 9980 9981 void perf_buffer__free(struct perf_buffer *pb) 9982 { 9983 int i; 9984 9985 if (IS_ERR_OR_NULL(pb)) 9986 return; 9987 if (pb->cpu_bufs) { 9988 for (i = 0; i < pb->cpu_cnt; i++) { 9989 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 9990 9991 if (!cpu_buf) 9992 continue; 9993 9994 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 9995 perf_buffer__free_cpu_buf(pb, cpu_buf); 9996 } 9997 free(pb->cpu_bufs); 9998 } 9999 if (pb->epoll_fd >= 0) 10000 close(pb->epoll_fd); 10001 free(pb->events); 10002 free(pb); 10003 } 10004 10005 static struct perf_cpu_buf * 10006 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 10007 int cpu, int map_key) 10008 { 10009 struct perf_cpu_buf *cpu_buf; 10010 char msg[STRERR_BUFSIZE]; 10011 int err; 10012 10013 cpu_buf = calloc(1, sizeof(*cpu_buf)); 10014 if (!cpu_buf) 10015 return ERR_PTR(-ENOMEM); 10016 10017 cpu_buf->pb = pb; 10018 cpu_buf->cpu = cpu; 10019 cpu_buf->map_key = map_key; 10020 10021 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 10022 -1, PERF_FLAG_FD_CLOEXEC); 10023 if (cpu_buf->fd < 0) { 10024 err = -errno; 10025 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 10026 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 10027 goto error; 10028 } 10029 10030 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 10031 PROT_READ | PROT_WRITE, MAP_SHARED, 10032 cpu_buf->fd, 0); 10033 if (cpu_buf->base == MAP_FAILED) { 10034 cpu_buf->base = NULL; 10035 err = -errno; 10036 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 10037 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 10038 goto error; 10039 } 10040 10041 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 10042 err = -errno; 10043 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 10044 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 10045 goto error; 10046 } 10047 10048 return cpu_buf; 10049 10050 error: 10051 perf_buffer__free_cpu_buf(pb, cpu_buf); 10052 return (struct perf_cpu_buf *)ERR_PTR(err); 10053 } 10054 10055 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 10056 struct perf_buffer_params *p); 10057 10058 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 10059 const struct perf_buffer_opts *opts) 10060 { 10061 struct perf_buffer_params p = {}; 10062 struct perf_event_attr attr = { 0, }; 10063 10064 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 10065 attr.type = PERF_TYPE_SOFTWARE; 10066 attr.sample_type = PERF_SAMPLE_RAW; 10067 attr.sample_period = 1; 10068 attr.wakeup_events = 1; 10069 10070 p.attr = &attr; 10071 p.sample_cb = opts ? opts->sample_cb : NULL; 10072 p.lost_cb = opts ? opts->lost_cb : NULL; 10073 p.ctx = opts ? opts->ctx : NULL; 10074 10075 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 10076 } 10077 10078 struct perf_buffer * 10079 perf_buffer__new_raw(int map_fd, size_t page_cnt, 10080 const struct perf_buffer_raw_opts *opts) 10081 { 10082 struct perf_buffer_params p = {}; 10083 10084 p.attr = opts->attr; 10085 p.event_cb = opts->event_cb; 10086 p.ctx = opts->ctx; 10087 p.cpu_cnt = opts->cpu_cnt; 10088 p.cpus = opts->cpus; 10089 p.map_keys = opts->map_keys; 10090 10091 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 10092 } 10093 10094 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 10095 struct perf_buffer_params *p) 10096 { 10097 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 10098 struct bpf_map_info map; 10099 char msg[STRERR_BUFSIZE]; 10100 struct perf_buffer *pb; 10101 bool *online = NULL; 10102 __u32 map_info_len; 10103 int err, i, j, n; 10104 10105 if (page_cnt & (page_cnt - 1)) { 10106 pr_warn("page count should be power of two, but is %zu\n", 10107 page_cnt); 10108 return ERR_PTR(-EINVAL); 10109 } 10110 10111 /* best-effort sanity checks */ 10112 memset(&map, 0, sizeof(map)); 10113 map_info_len = sizeof(map); 10114 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 10115 if (err) { 10116 err = -errno; 10117 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 10118 * -EBADFD, -EFAULT, or -E2BIG on real error 10119 */ 10120 if (err != -EINVAL) { 10121 pr_warn("failed to get map info for map FD %d: %s\n", 10122 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 10123 return ERR_PTR(err); 10124 } 10125 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 10126 map_fd); 10127 } else { 10128 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 10129 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 10130 map.name); 10131 return ERR_PTR(-EINVAL); 10132 } 10133 } 10134 10135 pb = calloc(1, sizeof(*pb)); 10136 if (!pb) 10137 return ERR_PTR(-ENOMEM); 10138 10139 pb->event_cb = p->event_cb; 10140 pb->sample_cb = p->sample_cb; 10141 pb->lost_cb = p->lost_cb; 10142 pb->ctx = p->ctx; 10143 10144 pb->page_size = getpagesize(); 10145 pb->mmap_size = pb->page_size * page_cnt; 10146 pb->map_fd = map_fd; 10147 10148 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 10149 if (pb->epoll_fd < 0) { 10150 err = -errno; 10151 pr_warn("failed to create epoll instance: %s\n", 10152 libbpf_strerror_r(err, msg, sizeof(msg))); 10153 goto error; 10154 } 10155 10156 if (p->cpu_cnt > 0) { 10157 pb->cpu_cnt = p->cpu_cnt; 10158 } else { 10159 pb->cpu_cnt = libbpf_num_possible_cpus(); 10160 if (pb->cpu_cnt < 0) { 10161 err = pb->cpu_cnt; 10162 goto error; 10163 } 10164 if (map.max_entries && map.max_entries < pb->cpu_cnt) 10165 pb->cpu_cnt = map.max_entries; 10166 } 10167 10168 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 10169 if (!pb->events) { 10170 err = -ENOMEM; 10171 pr_warn("failed to allocate events: out of memory\n"); 10172 goto error; 10173 } 10174 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 10175 if (!pb->cpu_bufs) { 10176 err = -ENOMEM; 10177 pr_warn("failed to allocate buffers: out of memory\n"); 10178 goto error; 10179 } 10180 10181 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 10182 if (err) { 10183 pr_warn("failed to get online CPU mask: %d\n", err); 10184 goto error; 10185 } 10186 10187 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 10188 struct perf_cpu_buf *cpu_buf; 10189 int cpu, map_key; 10190 10191 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 10192 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 10193 10194 /* in case user didn't explicitly requested particular CPUs to 10195 * be attached to, skip offline/not present CPUs 10196 */ 10197 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 10198 continue; 10199 10200 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 10201 if (IS_ERR(cpu_buf)) { 10202 err = PTR_ERR(cpu_buf); 10203 goto error; 10204 } 10205 10206 pb->cpu_bufs[j] = cpu_buf; 10207 10208 err = bpf_map_update_elem(pb->map_fd, &map_key, 10209 &cpu_buf->fd, 0); 10210 if (err) { 10211 err = -errno; 10212 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 10213 cpu, map_key, cpu_buf->fd, 10214 libbpf_strerror_r(err, msg, sizeof(msg))); 10215 goto error; 10216 } 10217 10218 pb->events[j].events = EPOLLIN; 10219 pb->events[j].data.ptr = cpu_buf; 10220 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 10221 &pb->events[j]) < 0) { 10222 err = -errno; 10223 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 10224 cpu, cpu_buf->fd, 10225 libbpf_strerror_r(err, msg, sizeof(msg))); 10226 goto error; 10227 } 10228 j++; 10229 } 10230 pb->cpu_cnt = j; 10231 free(online); 10232 10233 return pb; 10234 10235 error: 10236 free(online); 10237 if (pb) 10238 perf_buffer__free(pb); 10239 return ERR_PTR(err); 10240 } 10241 10242 struct perf_sample_raw { 10243 struct perf_event_header header; 10244 uint32_t size; 10245 char data[]; 10246 }; 10247 10248 struct perf_sample_lost { 10249 struct perf_event_header header; 10250 uint64_t id; 10251 uint64_t lost; 10252 uint64_t sample_id; 10253 }; 10254 10255 static enum bpf_perf_event_ret 10256 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 10257 { 10258 struct perf_cpu_buf *cpu_buf = ctx; 10259 struct perf_buffer *pb = cpu_buf->pb; 10260 void *data = e; 10261 10262 /* user wants full control over parsing perf event */ 10263 if (pb->event_cb) 10264 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 10265 10266 switch (e->type) { 10267 case PERF_RECORD_SAMPLE: { 10268 struct perf_sample_raw *s = data; 10269 10270 if (pb->sample_cb) 10271 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 10272 break; 10273 } 10274 case PERF_RECORD_LOST: { 10275 struct perf_sample_lost *s = data; 10276 10277 if (pb->lost_cb) 10278 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 10279 break; 10280 } 10281 default: 10282 pr_warn("unknown perf sample type %d\n", e->type); 10283 return LIBBPF_PERF_EVENT_ERROR; 10284 } 10285 return LIBBPF_PERF_EVENT_CONT; 10286 } 10287 10288 static int perf_buffer__process_records(struct perf_buffer *pb, 10289 struct perf_cpu_buf *cpu_buf) 10290 { 10291 enum bpf_perf_event_ret ret; 10292 10293 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 10294 pb->page_size, &cpu_buf->buf, 10295 &cpu_buf->buf_size, 10296 perf_buffer__process_record, cpu_buf); 10297 if (ret != LIBBPF_PERF_EVENT_CONT) 10298 return ret; 10299 return 0; 10300 } 10301 10302 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 10303 { 10304 return pb->epoll_fd; 10305 } 10306 10307 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 10308 { 10309 int i, cnt, err; 10310 10311 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 10312 if (cnt < 0) 10313 return -errno; 10314 10315 for (i = 0; i < cnt; i++) { 10316 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 10317 10318 err = perf_buffer__process_records(pb, cpu_buf); 10319 if (err) { 10320 pr_warn("error while processing records: %d\n", err); 10321 return libbpf_err(err); 10322 } 10323 } 10324 return cnt; 10325 } 10326 10327 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 10328 * manager. 10329 */ 10330 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 10331 { 10332 return pb->cpu_cnt; 10333 } 10334 10335 /* 10336 * Return perf_event FD of a ring buffer in *buf_idx* slot of 10337 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 10338 * select()/poll()/epoll() Linux syscalls. 10339 */ 10340 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 10341 { 10342 struct perf_cpu_buf *cpu_buf; 10343 10344 if (buf_idx >= pb->cpu_cnt) 10345 return libbpf_err(-EINVAL); 10346 10347 cpu_buf = pb->cpu_bufs[buf_idx]; 10348 if (!cpu_buf) 10349 return libbpf_err(-ENOENT); 10350 10351 return cpu_buf->fd; 10352 } 10353 10354 /* 10355 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 10356 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 10357 * consume, do nothing and return success. 10358 * Returns: 10359 * - 0 on success; 10360 * - <0 on failure. 10361 */ 10362 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 10363 { 10364 struct perf_cpu_buf *cpu_buf; 10365 10366 if (buf_idx >= pb->cpu_cnt) 10367 return libbpf_err(-EINVAL); 10368 10369 cpu_buf = pb->cpu_bufs[buf_idx]; 10370 if (!cpu_buf) 10371 return libbpf_err(-ENOENT); 10372 10373 return perf_buffer__process_records(pb, cpu_buf); 10374 } 10375 10376 int perf_buffer__consume(struct perf_buffer *pb) 10377 { 10378 int i, err; 10379 10380 for (i = 0; i < pb->cpu_cnt; i++) { 10381 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 10382 10383 if (!cpu_buf) 10384 continue; 10385 10386 err = perf_buffer__process_records(pb, cpu_buf); 10387 if (err) { 10388 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 10389 return libbpf_err(err); 10390 } 10391 } 10392 return 0; 10393 } 10394 10395 struct bpf_prog_info_array_desc { 10396 int array_offset; /* e.g. offset of jited_prog_insns */ 10397 int count_offset; /* e.g. offset of jited_prog_len */ 10398 int size_offset; /* > 0: offset of rec size, 10399 * < 0: fix size of -size_offset 10400 */ 10401 }; 10402 10403 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 10404 [BPF_PROG_INFO_JITED_INSNS] = { 10405 offsetof(struct bpf_prog_info, jited_prog_insns), 10406 offsetof(struct bpf_prog_info, jited_prog_len), 10407 -1, 10408 }, 10409 [BPF_PROG_INFO_XLATED_INSNS] = { 10410 offsetof(struct bpf_prog_info, xlated_prog_insns), 10411 offsetof(struct bpf_prog_info, xlated_prog_len), 10412 -1, 10413 }, 10414 [BPF_PROG_INFO_MAP_IDS] = { 10415 offsetof(struct bpf_prog_info, map_ids), 10416 offsetof(struct bpf_prog_info, nr_map_ids), 10417 -(int)sizeof(__u32), 10418 }, 10419 [BPF_PROG_INFO_JITED_KSYMS] = { 10420 offsetof(struct bpf_prog_info, jited_ksyms), 10421 offsetof(struct bpf_prog_info, nr_jited_ksyms), 10422 -(int)sizeof(__u64), 10423 }, 10424 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 10425 offsetof(struct bpf_prog_info, jited_func_lens), 10426 offsetof(struct bpf_prog_info, nr_jited_func_lens), 10427 -(int)sizeof(__u32), 10428 }, 10429 [BPF_PROG_INFO_FUNC_INFO] = { 10430 offsetof(struct bpf_prog_info, func_info), 10431 offsetof(struct bpf_prog_info, nr_func_info), 10432 offsetof(struct bpf_prog_info, func_info_rec_size), 10433 }, 10434 [BPF_PROG_INFO_LINE_INFO] = { 10435 offsetof(struct bpf_prog_info, line_info), 10436 offsetof(struct bpf_prog_info, nr_line_info), 10437 offsetof(struct bpf_prog_info, line_info_rec_size), 10438 }, 10439 [BPF_PROG_INFO_JITED_LINE_INFO] = { 10440 offsetof(struct bpf_prog_info, jited_line_info), 10441 offsetof(struct bpf_prog_info, nr_jited_line_info), 10442 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 10443 }, 10444 [BPF_PROG_INFO_PROG_TAGS] = { 10445 offsetof(struct bpf_prog_info, prog_tags), 10446 offsetof(struct bpf_prog_info, nr_prog_tags), 10447 -(int)sizeof(__u8) * BPF_TAG_SIZE, 10448 }, 10449 10450 }; 10451 10452 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 10453 int offset) 10454 { 10455 __u32 *array = (__u32 *)info; 10456 10457 if (offset >= 0) 10458 return array[offset / sizeof(__u32)]; 10459 return -(int)offset; 10460 } 10461 10462 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 10463 int offset) 10464 { 10465 __u64 *array = (__u64 *)info; 10466 10467 if (offset >= 0) 10468 return array[offset / sizeof(__u64)]; 10469 return -(int)offset; 10470 } 10471 10472 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 10473 __u32 val) 10474 { 10475 __u32 *array = (__u32 *)info; 10476 10477 if (offset >= 0) 10478 array[offset / sizeof(__u32)] = val; 10479 } 10480 10481 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 10482 __u64 val) 10483 { 10484 __u64 *array = (__u64 *)info; 10485 10486 if (offset >= 0) 10487 array[offset / sizeof(__u64)] = val; 10488 } 10489 10490 struct bpf_prog_info_linear * 10491 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 10492 { 10493 struct bpf_prog_info_linear *info_linear; 10494 struct bpf_prog_info info = {}; 10495 __u32 info_len = sizeof(info); 10496 __u32 data_len = 0; 10497 int i, err; 10498 void *ptr; 10499 10500 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 10501 return libbpf_err_ptr(-EINVAL); 10502 10503 /* step 1: get array dimensions */ 10504 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 10505 if (err) { 10506 pr_debug("can't get prog info: %s", strerror(errno)); 10507 return libbpf_err_ptr(-EFAULT); 10508 } 10509 10510 /* step 2: calculate total size of all arrays */ 10511 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10512 bool include_array = (arrays & (1UL << i)) > 0; 10513 struct bpf_prog_info_array_desc *desc; 10514 __u32 count, size; 10515 10516 desc = bpf_prog_info_array_desc + i; 10517 10518 /* kernel is too old to support this field */ 10519 if (info_len < desc->array_offset + sizeof(__u32) || 10520 info_len < desc->count_offset + sizeof(__u32) || 10521 (desc->size_offset > 0 && info_len < desc->size_offset)) 10522 include_array = false; 10523 10524 if (!include_array) { 10525 arrays &= ~(1UL << i); /* clear the bit */ 10526 continue; 10527 } 10528 10529 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10530 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10531 10532 data_len += count * size; 10533 } 10534 10535 /* step 3: allocate continuous memory */ 10536 data_len = roundup(data_len, sizeof(__u64)); 10537 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 10538 if (!info_linear) 10539 return libbpf_err_ptr(-ENOMEM); 10540 10541 /* step 4: fill data to info_linear->info */ 10542 info_linear->arrays = arrays; 10543 memset(&info_linear->info, 0, sizeof(info)); 10544 ptr = info_linear->data; 10545 10546 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10547 struct bpf_prog_info_array_desc *desc; 10548 __u32 count, size; 10549 10550 if ((arrays & (1UL << i)) == 0) 10551 continue; 10552 10553 desc = bpf_prog_info_array_desc + i; 10554 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10555 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10556 bpf_prog_info_set_offset_u32(&info_linear->info, 10557 desc->count_offset, count); 10558 bpf_prog_info_set_offset_u32(&info_linear->info, 10559 desc->size_offset, size); 10560 bpf_prog_info_set_offset_u64(&info_linear->info, 10561 desc->array_offset, 10562 ptr_to_u64(ptr)); 10563 ptr += count * size; 10564 } 10565 10566 /* step 5: call syscall again to get required arrays */ 10567 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 10568 if (err) { 10569 pr_debug("can't get prog info: %s", strerror(errno)); 10570 free(info_linear); 10571 return libbpf_err_ptr(-EFAULT); 10572 } 10573 10574 /* step 6: verify the data */ 10575 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10576 struct bpf_prog_info_array_desc *desc; 10577 __u32 v1, v2; 10578 10579 if ((arrays & (1UL << i)) == 0) 10580 continue; 10581 10582 desc = bpf_prog_info_array_desc + i; 10583 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10584 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 10585 desc->count_offset); 10586 if (v1 != v2) 10587 pr_warn("%s: mismatch in element count\n", __func__); 10588 10589 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10590 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 10591 desc->size_offset); 10592 if (v1 != v2) 10593 pr_warn("%s: mismatch in rec size\n", __func__); 10594 } 10595 10596 /* step 7: update info_len and data_len */ 10597 info_linear->info_len = sizeof(struct bpf_prog_info); 10598 info_linear->data_len = data_len; 10599 10600 return info_linear; 10601 } 10602 10603 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 10604 { 10605 int i; 10606 10607 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10608 struct bpf_prog_info_array_desc *desc; 10609 __u64 addr, offs; 10610 10611 if ((info_linear->arrays & (1UL << i)) == 0) 10612 continue; 10613 10614 desc = bpf_prog_info_array_desc + i; 10615 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 10616 desc->array_offset); 10617 offs = addr - ptr_to_u64(info_linear->data); 10618 bpf_prog_info_set_offset_u64(&info_linear->info, 10619 desc->array_offset, offs); 10620 } 10621 } 10622 10623 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 10624 { 10625 int i; 10626 10627 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10628 struct bpf_prog_info_array_desc *desc; 10629 __u64 addr, offs; 10630 10631 if ((info_linear->arrays & (1UL << i)) == 0) 10632 continue; 10633 10634 desc = bpf_prog_info_array_desc + i; 10635 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 10636 desc->array_offset); 10637 addr = offs + ptr_to_u64(info_linear->data); 10638 bpf_prog_info_set_offset_u64(&info_linear->info, 10639 desc->array_offset, addr); 10640 } 10641 } 10642 10643 int bpf_program__set_attach_target(struct bpf_program *prog, 10644 int attach_prog_fd, 10645 const char *attach_func_name) 10646 { 10647 int btf_obj_fd = 0, btf_id = 0, err; 10648 10649 if (!prog || attach_prog_fd < 0) 10650 return libbpf_err(-EINVAL); 10651 10652 if (prog->obj->loaded) 10653 return libbpf_err(-EINVAL); 10654 10655 if (attach_prog_fd && !attach_func_name) { 10656 /* remember attach_prog_fd and let bpf_program__load() find 10657 * BTF ID during the program load 10658 */ 10659 prog->attach_prog_fd = attach_prog_fd; 10660 return 0; 10661 } 10662 10663 if (attach_prog_fd) { 10664 btf_id = libbpf_find_prog_btf_id(attach_func_name, 10665 attach_prog_fd); 10666 if (btf_id < 0) 10667 return libbpf_err(btf_id); 10668 } else { 10669 if (!attach_func_name) 10670 return libbpf_err(-EINVAL); 10671 10672 /* load btf_vmlinux, if not yet */ 10673 err = bpf_object__load_vmlinux_btf(prog->obj, true); 10674 if (err) 10675 return libbpf_err(err); 10676 err = find_kernel_btf_id(prog->obj, attach_func_name, 10677 prog->expected_attach_type, 10678 &btf_obj_fd, &btf_id); 10679 if (err) 10680 return libbpf_err(err); 10681 } 10682 10683 prog->attach_btf_id = btf_id; 10684 prog->attach_btf_obj_fd = btf_obj_fd; 10685 prog->attach_prog_fd = attach_prog_fd; 10686 return 0; 10687 } 10688 10689 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 10690 { 10691 int err = 0, n, len, start, end = -1; 10692 bool *tmp; 10693 10694 *mask = NULL; 10695 *mask_sz = 0; 10696 10697 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 10698 while (*s) { 10699 if (*s == ',' || *s == '\n') { 10700 s++; 10701 continue; 10702 } 10703 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 10704 if (n <= 0 || n > 2) { 10705 pr_warn("Failed to get CPU range %s: %d\n", s, n); 10706 err = -EINVAL; 10707 goto cleanup; 10708 } else if (n == 1) { 10709 end = start; 10710 } 10711 if (start < 0 || start > end) { 10712 pr_warn("Invalid CPU range [%d,%d] in %s\n", 10713 start, end, s); 10714 err = -EINVAL; 10715 goto cleanup; 10716 } 10717 tmp = realloc(*mask, end + 1); 10718 if (!tmp) { 10719 err = -ENOMEM; 10720 goto cleanup; 10721 } 10722 *mask = tmp; 10723 memset(tmp + *mask_sz, 0, start - *mask_sz); 10724 memset(tmp + start, 1, end - start + 1); 10725 *mask_sz = end + 1; 10726 s += len; 10727 } 10728 if (!*mask_sz) { 10729 pr_warn("Empty CPU range\n"); 10730 return -EINVAL; 10731 } 10732 return 0; 10733 cleanup: 10734 free(*mask); 10735 *mask = NULL; 10736 return err; 10737 } 10738 10739 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 10740 { 10741 int fd, err = 0, len; 10742 char buf[128]; 10743 10744 fd = open(fcpu, O_RDONLY); 10745 if (fd < 0) { 10746 err = -errno; 10747 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 10748 return err; 10749 } 10750 len = read(fd, buf, sizeof(buf)); 10751 close(fd); 10752 if (len <= 0) { 10753 err = len ? -errno : -EINVAL; 10754 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 10755 return err; 10756 } 10757 if (len >= sizeof(buf)) { 10758 pr_warn("CPU mask is too big in file %s\n", fcpu); 10759 return -E2BIG; 10760 } 10761 buf[len] = '\0'; 10762 10763 return parse_cpu_mask_str(buf, mask, mask_sz); 10764 } 10765 10766 int libbpf_num_possible_cpus(void) 10767 { 10768 static const char *fcpu = "/sys/devices/system/cpu/possible"; 10769 static int cpus; 10770 int err, n, i, tmp_cpus; 10771 bool *mask; 10772 10773 tmp_cpus = READ_ONCE(cpus); 10774 if (tmp_cpus > 0) 10775 return tmp_cpus; 10776 10777 err = parse_cpu_mask_file(fcpu, &mask, &n); 10778 if (err) 10779 return libbpf_err(err); 10780 10781 tmp_cpus = 0; 10782 for (i = 0; i < n; i++) { 10783 if (mask[i]) 10784 tmp_cpus++; 10785 } 10786 free(mask); 10787 10788 WRITE_ONCE(cpus, tmp_cpus); 10789 return tmp_cpus; 10790 } 10791 10792 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 10793 const struct bpf_object_open_opts *opts) 10794 { 10795 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 10796 .object_name = s->name, 10797 ); 10798 struct bpf_object *obj; 10799 int i, err; 10800 10801 /* Attempt to preserve opts->object_name, unless overriden by user 10802 * explicitly. Overwriting object name for skeletons is discouraged, 10803 * as it breaks global data maps, because they contain object name 10804 * prefix as their own map name prefix. When skeleton is generated, 10805 * bpftool is making an assumption that this name will stay the same. 10806 */ 10807 if (opts) { 10808 memcpy(&skel_opts, opts, sizeof(*opts)); 10809 if (!opts->object_name) 10810 skel_opts.object_name = s->name; 10811 } 10812 10813 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 10814 err = libbpf_get_error(obj); 10815 if (err) { 10816 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", 10817 s->name, err); 10818 return libbpf_err(err); 10819 } 10820 10821 *s->obj = obj; 10822 10823 for (i = 0; i < s->map_cnt; i++) { 10824 struct bpf_map **map = s->maps[i].map; 10825 const char *name = s->maps[i].name; 10826 void **mmaped = s->maps[i].mmaped; 10827 10828 *map = bpf_object__find_map_by_name(obj, name); 10829 if (!*map) { 10830 pr_warn("failed to find skeleton map '%s'\n", name); 10831 return libbpf_err(-ESRCH); 10832 } 10833 10834 /* externs shouldn't be pre-setup from user code */ 10835 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 10836 *mmaped = (*map)->mmaped; 10837 } 10838 10839 for (i = 0; i < s->prog_cnt; i++) { 10840 struct bpf_program **prog = s->progs[i].prog; 10841 const char *name = s->progs[i].name; 10842 10843 *prog = bpf_object__find_program_by_name(obj, name); 10844 if (!*prog) { 10845 pr_warn("failed to find skeleton program '%s'\n", name); 10846 return libbpf_err(-ESRCH); 10847 } 10848 } 10849 10850 return 0; 10851 } 10852 10853 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 10854 { 10855 int i, err; 10856 10857 err = bpf_object__load(*s->obj); 10858 if (err) { 10859 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 10860 return libbpf_err(err); 10861 } 10862 10863 for (i = 0; i < s->map_cnt; i++) { 10864 struct bpf_map *map = *s->maps[i].map; 10865 size_t mmap_sz = bpf_map_mmap_sz(map); 10866 int prot, map_fd = bpf_map__fd(map); 10867 void **mmaped = s->maps[i].mmaped; 10868 10869 if (!mmaped) 10870 continue; 10871 10872 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 10873 *mmaped = NULL; 10874 continue; 10875 } 10876 10877 if (map->def.map_flags & BPF_F_RDONLY_PROG) 10878 prot = PROT_READ; 10879 else 10880 prot = PROT_READ | PROT_WRITE; 10881 10882 /* Remap anonymous mmap()-ed "map initialization image" as 10883 * a BPF map-backed mmap()-ed memory, but preserving the same 10884 * memory address. This will cause kernel to change process' 10885 * page table to point to a different piece of kernel memory, 10886 * but from userspace point of view memory address (and its 10887 * contents, being identical at this point) will stay the 10888 * same. This mapping will be released by bpf_object__close() 10889 * as per normal clean up procedure, so we don't need to worry 10890 * about it from skeleton's clean up perspective. 10891 */ 10892 *mmaped = mmap(map->mmaped, mmap_sz, prot, 10893 MAP_SHARED | MAP_FIXED, map_fd, 0); 10894 if (*mmaped == MAP_FAILED) { 10895 err = -errno; 10896 *mmaped = NULL; 10897 pr_warn("failed to re-mmap() map '%s': %d\n", 10898 bpf_map__name(map), err); 10899 return libbpf_err(err); 10900 } 10901 } 10902 10903 return 0; 10904 } 10905 10906 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 10907 { 10908 int i, err; 10909 10910 for (i = 0; i < s->prog_cnt; i++) { 10911 struct bpf_program *prog = *s->progs[i].prog; 10912 struct bpf_link **link = s->progs[i].link; 10913 10914 if (!prog->load) 10915 continue; 10916 10917 /* auto-attaching not supported for this program */ 10918 if (!prog->sec_def || !prog->sec_def->attach_fn) 10919 continue; 10920 10921 *link = bpf_program__attach(prog); 10922 err = libbpf_get_error(*link); 10923 if (err) { 10924 pr_warn("failed to auto-attach program '%s': %d\n", 10925 bpf_program__name(prog), err); 10926 return libbpf_err(err); 10927 } 10928 } 10929 10930 return 0; 10931 } 10932 10933 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 10934 { 10935 int i; 10936 10937 for (i = 0; i < s->prog_cnt; i++) { 10938 struct bpf_link **link = s->progs[i].link; 10939 10940 bpf_link__destroy(*link); 10941 *link = NULL; 10942 } 10943 } 10944 10945 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 10946 { 10947 if (s->progs) 10948 bpf_object__detach_skeleton(s); 10949 if (s->obj) 10950 bpf_object__close(*s->obj); 10951 free(s->maps); 10952 free(s->progs); 10953 free(s); 10954 } 10955