1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/list.h> 35 #include <linux/limits.h> 36 #include <linux/perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <linux/version.h> 39 #include <sys/epoll.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <sys/stat.h> 43 #include <sys/types.h> 44 #include <sys/vfs.h> 45 #include <sys/utsname.h> 46 #include <sys/resource.h> 47 #include <libelf.h> 48 #include <gelf.h> 49 #include <zlib.h> 50 51 #include "libbpf.h" 52 #include "bpf.h" 53 #include "btf.h" 54 #include "str_error.h" 55 #include "libbpf_internal.h" 56 #include "hashmap.h" 57 #include "bpf_gen_internal.h" 58 59 #ifndef BPF_FS_MAGIC 60 #define BPF_FS_MAGIC 0xcafe4a11 61 #endif 62 63 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 64 65 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 66 * compilation if user enables corresponding warning. Disable it explicitly. 67 */ 68 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 69 70 #define __printf(a, b) __attribute__((format(printf, a, b))) 71 72 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 73 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 74 75 static int __base_pr(enum libbpf_print_level level, const char *format, 76 va_list args) 77 { 78 if (level == LIBBPF_DEBUG) 79 return 0; 80 81 return vfprintf(stderr, format, args); 82 } 83 84 static libbpf_print_fn_t __libbpf_pr = __base_pr; 85 86 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 87 { 88 libbpf_print_fn_t old_print_fn = __libbpf_pr; 89 90 __libbpf_pr = fn; 91 return old_print_fn; 92 } 93 94 __printf(2, 3) 95 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 96 { 97 va_list args; 98 99 if (!__libbpf_pr) 100 return; 101 102 va_start(args, format); 103 __libbpf_pr(level, format, args); 104 va_end(args); 105 } 106 107 static void pr_perm_msg(int err) 108 { 109 struct rlimit limit; 110 char buf[100]; 111 112 if (err != -EPERM || geteuid() != 0) 113 return; 114 115 err = getrlimit(RLIMIT_MEMLOCK, &limit); 116 if (err) 117 return; 118 119 if (limit.rlim_cur == RLIM_INFINITY) 120 return; 121 122 if (limit.rlim_cur < 1024) 123 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 124 else if (limit.rlim_cur < 1024*1024) 125 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 126 else 127 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 128 129 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 130 buf); 131 } 132 133 #define STRERR_BUFSIZE 128 134 135 /* Copied from tools/perf/util/util.h */ 136 #ifndef zfree 137 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 138 #endif 139 140 #ifndef zclose 141 # define zclose(fd) ({ \ 142 int ___err = 0; \ 143 if ((fd) >= 0) \ 144 ___err = close((fd)); \ 145 fd = -1; \ 146 ___err; }) 147 #endif 148 149 static inline __u64 ptr_to_u64(const void *ptr) 150 { 151 return (__u64) (unsigned long) ptr; 152 } 153 154 /* this goes away in libbpf 1.0 */ 155 enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE; 156 157 int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 158 { 159 /* __LIBBPF_STRICT_LAST is the last power-of-2 value used + 1, so to 160 * get all possible values we compensate last +1, and then (2*x - 1) 161 * to get the bit mask 162 */ 163 if (mode != LIBBPF_STRICT_ALL 164 && (mode & ~((__LIBBPF_STRICT_LAST - 1) * 2 - 1))) 165 return errno = EINVAL, -EINVAL; 166 167 libbpf_mode = mode; 168 return 0; 169 } 170 171 enum kern_feature_id { 172 /* v4.14: kernel support for program & map names. */ 173 FEAT_PROG_NAME, 174 /* v5.2: kernel support for global data sections. */ 175 FEAT_GLOBAL_DATA, 176 /* BTF support */ 177 FEAT_BTF, 178 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 179 FEAT_BTF_FUNC, 180 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 181 FEAT_BTF_DATASEC, 182 /* BTF_FUNC_GLOBAL is supported */ 183 FEAT_BTF_GLOBAL_FUNC, 184 /* BPF_F_MMAPABLE is supported for arrays */ 185 FEAT_ARRAY_MMAP, 186 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 187 FEAT_EXP_ATTACH_TYPE, 188 /* bpf_probe_read_{kernel,user}[_str] helpers */ 189 FEAT_PROBE_READ_KERN, 190 /* BPF_PROG_BIND_MAP is supported */ 191 FEAT_PROG_BIND_MAP, 192 /* Kernel support for module BTFs */ 193 FEAT_MODULE_BTF, 194 /* BTF_KIND_FLOAT support */ 195 FEAT_BTF_FLOAT, 196 /* BPF perf link support */ 197 FEAT_PERF_LINK, 198 __FEAT_CNT, 199 }; 200 201 static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); 202 203 enum reloc_type { 204 RELO_LD64, 205 RELO_CALL, 206 RELO_DATA, 207 RELO_EXTERN_VAR, 208 RELO_EXTERN_FUNC, 209 RELO_SUBPROG_ADDR, 210 }; 211 212 struct reloc_desc { 213 enum reloc_type type; 214 int insn_idx; 215 int map_idx; 216 int sym_off; 217 }; 218 219 struct bpf_sec_def; 220 221 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec, 222 struct bpf_program *prog); 223 224 struct bpf_sec_def { 225 const char *sec; 226 size_t len; 227 enum bpf_prog_type prog_type; 228 enum bpf_attach_type expected_attach_type; 229 bool is_exp_attach_type_optional; 230 bool is_attachable; 231 bool is_attach_btf; 232 bool is_sleepable; 233 attach_fn_t attach_fn; 234 }; 235 236 /* 237 * bpf_prog should be a better name but it has been used in 238 * linux/filter.h. 239 */ 240 struct bpf_program { 241 const struct bpf_sec_def *sec_def; 242 char *sec_name; 243 size_t sec_idx; 244 /* this program's instruction offset (in number of instructions) 245 * within its containing ELF section 246 */ 247 size_t sec_insn_off; 248 /* number of original instructions in ELF section belonging to this 249 * program, not taking into account subprogram instructions possible 250 * appended later during relocation 251 */ 252 size_t sec_insn_cnt; 253 /* Offset (in number of instructions) of the start of instruction 254 * belonging to this BPF program within its containing main BPF 255 * program. For the entry-point (main) BPF program, this is always 256 * zero. For a sub-program, this gets reset before each of main BPF 257 * programs are processed and relocated and is used to determined 258 * whether sub-program was already appended to the main program, and 259 * if yes, at which instruction offset. 260 */ 261 size_t sub_insn_off; 262 263 char *name; 264 /* sec_name with / replaced by _; makes recursive pinning 265 * in bpf_object__pin_programs easier 266 */ 267 char *pin_name; 268 269 /* instructions that belong to BPF program; insns[0] is located at 270 * sec_insn_off instruction within its ELF section in ELF file, so 271 * when mapping ELF file instruction index to the local instruction, 272 * one needs to subtract sec_insn_off; and vice versa. 273 */ 274 struct bpf_insn *insns; 275 /* actual number of instruction in this BPF program's image; for 276 * entry-point BPF programs this includes the size of main program 277 * itself plus all the used sub-programs, appended at the end 278 */ 279 size_t insns_cnt; 280 281 struct reloc_desc *reloc_desc; 282 int nr_reloc; 283 int log_level; 284 285 struct { 286 int nr; 287 int *fds; 288 } instances; 289 bpf_program_prep_t preprocessor; 290 291 struct bpf_object *obj; 292 void *priv; 293 bpf_program_clear_priv_t clear_priv; 294 295 bool load; 296 bool mark_btf_static; 297 enum bpf_prog_type type; 298 enum bpf_attach_type expected_attach_type; 299 int prog_ifindex; 300 __u32 attach_btf_obj_fd; 301 __u32 attach_btf_id; 302 __u32 attach_prog_fd; 303 void *func_info; 304 __u32 func_info_rec_size; 305 __u32 func_info_cnt; 306 307 void *line_info; 308 __u32 line_info_rec_size; 309 __u32 line_info_cnt; 310 __u32 prog_flags; 311 }; 312 313 struct bpf_struct_ops { 314 const char *tname; 315 const struct btf_type *type; 316 struct bpf_program **progs; 317 __u32 *kern_func_off; 318 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 319 void *data; 320 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 321 * btf_vmlinux's format. 322 * struct bpf_struct_ops_tcp_congestion_ops { 323 * [... some other kernel fields ...] 324 * struct tcp_congestion_ops data; 325 * } 326 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 327 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 328 * from "data". 329 */ 330 void *kern_vdata; 331 __u32 type_id; 332 }; 333 334 #define DATA_SEC ".data" 335 #define BSS_SEC ".bss" 336 #define RODATA_SEC ".rodata" 337 #define KCONFIG_SEC ".kconfig" 338 #define KSYMS_SEC ".ksyms" 339 #define STRUCT_OPS_SEC ".struct_ops" 340 341 enum libbpf_map_type { 342 LIBBPF_MAP_UNSPEC, 343 LIBBPF_MAP_DATA, 344 LIBBPF_MAP_BSS, 345 LIBBPF_MAP_RODATA, 346 LIBBPF_MAP_KCONFIG, 347 }; 348 349 static const char * const libbpf_type_to_btf_name[] = { 350 [LIBBPF_MAP_DATA] = DATA_SEC, 351 [LIBBPF_MAP_BSS] = BSS_SEC, 352 [LIBBPF_MAP_RODATA] = RODATA_SEC, 353 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, 354 }; 355 356 struct bpf_map { 357 char *name; 358 int fd; 359 int sec_idx; 360 size_t sec_offset; 361 int map_ifindex; 362 int inner_map_fd; 363 struct bpf_map_def def; 364 __u32 numa_node; 365 __u32 btf_var_idx; 366 __u32 btf_key_type_id; 367 __u32 btf_value_type_id; 368 __u32 btf_vmlinux_value_type_id; 369 void *priv; 370 bpf_map_clear_priv_t clear_priv; 371 enum libbpf_map_type libbpf_type; 372 void *mmaped; 373 struct bpf_struct_ops *st_ops; 374 struct bpf_map *inner_map; 375 void **init_slots; 376 int init_slots_sz; 377 char *pin_path; 378 bool pinned; 379 bool reused; 380 }; 381 382 enum extern_type { 383 EXT_UNKNOWN, 384 EXT_KCFG, 385 EXT_KSYM, 386 }; 387 388 enum kcfg_type { 389 KCFG_UNKNOWN, 390 KCFG_CHAR, 391 KCFG_BOOL, 392 KCFG_INT, 393 KCFG_TRISTATE, 394 KCFG_CHAR_ARR, 395 }; 396 397 struct extern_desc { 398 enum extern_type type; 399 int sym_idx; 400 int btf_id; 401 int sec_btf_id; 402 const char *name; 403 bool is_set; 404 bool is_weak; 405 union { 406 struct { 407 enum kcfg_type type; 408 int sz; 409 int align; 410 int data_off; 411 bool is_signed; 412 } kcfg; 413 struct { 414 unsigned long long addr; 415 416 /* target btf_id of the corresponding kernel var. */ 417 int kernel_btf_obj_fd; 418 int kernel_btf_id; 419 420 /* local btf_id of the ksym extern's type. */ 421 __u32 type_id; 422 } ksym; 423 }; 424 }; 425 426 static LIST_HEAD(bpf_objects_list); 427 428 struct module_btf { 429 struct btf *btf; 430 char *name; 431 __u32 id; 432 int fd; 433 }; 434 435 struct bpf_object { 436 char name[BPF_OBJ_NAME_LEN]; 437 char license[64]; 438 __u32 kern_version; 439 440 struct bpf_program *programs; 441 size_t nr_programs; 442 struct bpf_map *maps; 443 size_t nr_maps; 444 size_t maps_cap; 445 446 char *kconfig; 447 struct extern_desc *externs; 448 int nr_extern; 449 int kconfig_map_idx; 450 int rodata_map_idx; 451 452 bool loaded; 453 bool has_subcalls; 454 455 struct bpf_gen *gen_loader; 456 457 /* 458 * Information when doing elf related work. Only valid if fd 459 * is valid. 460 */ 461 struct { 462 int fd; 463 const void *obj_buf; 464 size_t obj_buf_sz; 465 Elf *elf; 466 GElf_Ehdr ehdr; 467 Elf_Data *symbols; 468 Elf_Data *data; 469 Elf_Data *rodata; 470 Elf_Data *bss; 471 Elf_Data *st_ops_data; 472 size_t shstrndx; /* section index for section name strings */ 473 size_t strtabidx; 474 struct { 475 GElf_Shdr shdr; 476 Elf_Data *data; 477 } *reloc_sects; 478 int nr_reloc_sects; 479 int maps_shndx; 480 int btf_maps_shndx; 481 __u32 btf_maps_sec_btf_id; 482 int text_shndx; 483 int symbols_shndx; 484 int data_shndx; 485 int rodata_shndx; 486 int bss_shndx; 487 int st_ops_shndx; 488 } efile; 489 /* 490 * All loaded bpf_object is linked in a list, which is 491 * hidden to caller. bpf_objects__<func> handlers deal with 492 * all objects. 493 */ 494 struct list_head list; 495 496 struct btf *btf; 497 struct btf_ext *btf_ext; 498 499 /* Parse and load BTF vmlinux if any of the programs in the object need 500 * it at load time. 501 */ 502 struct btf *btf_vmlinux; 503 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 504 * override for vmlinux BTF. 505 */ 506 char *btf_custom_path; 507 /* vmlinux BTF override for CO-RE relocations */ 508 struct btf *btf_vmlinux_override; 509 /* Lazily initialized kernel module BTFs */ 510 struct module_btf *btf_modules; 511 bool btf_modules_loaded; 512 size_t btf_module_cnt; 513 size_t btf_module_cap; 514 515 void *priv; 516 bpf_object_clear_priv_t clear_priv; 517 518 char path[]; 519 }; 520 #define obj_elf_valid(o) ((o)->efile.elf) 521 522 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 523 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 524 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 525 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 526 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr); 527 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 528 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 529 530 void bpf_program__unload(struct bpf_program *prog) 531 { 532 int i; 533 534 if (!prog) 535 return; 536 537 /* 538 * If the object is opened but the program was never loaded, 539 * it is possible that prog->instances.nr == -1. 540 */ 541 if (prog->instances.nr > 0) { 542 for (i = 0; i < prog->instances.nr; i++) 543 zclose(prog->instances.fds[i]); 544 } else if (prog->instances.nr != -1) { 545 pr_warn("Internal error: instances.nr is %d\n", 546 prog->instances.nr); 547 } 548 549 prog->instances.nr = -1; 550 zfree(&prog->instances.fds); 551 552 zfree(&prog->func_info); 553 zfree(&prog->line_info); 554 } 555 556 static void bpf_program__exit(struct bpf_program *prog) 557 { 558 if (!prog) 559 return; 560 561 if (prog->clear_priv) 562 prog->clear_priv(prog, prog->priv); 563 564 prog->priv = NULL; 565 prog->clear_priv = NULL; 566 567 bpf_program__unload(prog); 568 zfree(&prog->name); 569 zfree(&prog->sec_name); 570 zfree(&prog->pin_name); 571 zfree(&prog->insns); 572 zfree(&prog->reloc_desc); 573 574 prog->nr_reloc = 0; 575 prog->insns_cnt = 0; 576 prog->sec_idx = -1; 577 } 578 579 static char *__bpf_program__pin_name(struct bpf_program *prog) 580 { 581 char *name, *p; 582 583 name = p = strdup(prog->sec_name); 584 while ((p = strchr(p, '/'))) 585 *p = '_'; 586 587 return name; 588 } 589 590 static bool insn_is_subprog_call(const struct bpf_insn *insn) 591 { 592 return BPF_CLASS(insn->code) == BPF_JMP && 593 BPF_OP(insn->code) == BPF_CALL && 594 BPF_SRC(insn->code) == BPF_K && 595 insn->src_reg == BPF_PSEUDO_CALL && 596 insn->dst_reg == 0 && 597 insn->off == 0; 598 } 599 600 static bool is_call_insn(const struct bpf_insn *insn) 601 { 602 return insn->code == (BPF_JMP | BPF_CALL); 603 } 604 605 static bool insn_is_pseudo_func(struct bpf_insn *insn) 606 { 607 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 608 } 609 610 static int 611 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 612 const char *name, size_t sec_idx, const char *sec_name, 613 size_t sec_off, void *insn_data, size_t insn_data_sz) 614 { 615 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 616 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 617 sec_name, name, sec_off, insn_data_sz); 618 return -EINVAL; 619 } 620 621 memset(prog, 0, sizeof(*prog)); 622 prog->obj = obj; 623 624 prog->sec_idx = sec_idx; 625 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 626 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 627 /* insns_cnt can later be increased by appending used subprograms */ 628 prog->insns_cnt = prog->sec_insn_cnt; 629 630 prog->type = BPF_PROG_TYPE_UNSPEC; 631 prog->load = true; 632 633 prog->instances.fds = NULL; 634 prog->instances.nr = -1; 635 636 prog->sec_name = strdup(sec_name); 637 if (!prog->sec_name) 638 goto errout; 639 640 prog->name = strdup(name); 641 if (!prog->name) 642 goto errout; 643 644 prog->pin_name = __bpf_program__pin_name(prog); 645 if (!prog->pin_name) 646 goto errout; 647 648 prog->insns = malloc(insn_data_sz); 649 if (!prog->insns) 650 goto errout; 651 memcpy(prog->insns, insn_data, insn_data_sz); 652 653 return 0; 654 errout: 655 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 656 bpf_program__exit(prog); 657 return -ENOMEM; 658 } 659 660 static int 661 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 662 const char *sec_name, int sec_idx) 663 { 664 Elf_Data *symbols = obj->efile.symbols; 665 struct bpf_program *prog, *progs; 666 void *data = sec_data->d_buf; 667 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 668 int nr_progs, err, i; 669 const char *name; 670 GElf_Sym sym; 671 672 progs = obj->programs; 673 nr_progs = obj->nr_programs; 674 nr_syms = symbols->d_size / sizeof(GElf_Sym); 675 sec_off = 0; 676 677 for (i = 0; i < nr_syms; i++) { 678 if (!gelf_getsym(symbols, i, &sym)) 679 continue; 680 if (sym.st_shndx != sec_idx) 681 continue; 682 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC) 683 continue; 684 685 prog_sz = sym.st_size; 686 sec_off = sym.st_value; 687 688 name = elf_sym_str(obj, sym.st_name); 689 if (!name) { 690 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 691 sec_name, sec_off); 692 return -LIBBPF_ERRNO__FORMAT; 693 } 694 695 if (sec_off + prog_sz > sec_sz) { 696 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 697 sec_name, sec_off); 698 return -LIBBPF_ERRNO__FORMAT; 699 } 700 701 if (sec_idx != obj->efile.text_shndx && GELF_ST_BIND(sym.st_info) == STB_LOCAL) { 702 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 703 return -ENOTSUP; 704 } 705 706 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 707 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 708 709 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 710 if (!progs) { 711 /* 712 * In this case the original obj->programs 713 * is still valid, so don't need special treat for 714 * bpf_close_object(). 715 */ 716 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 717 sec_name, name); 718 return -ENOMEM; 719 } 720 obj->programs = progs; 721 722 prog = &progs[nr_progs]; 723 724 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 725 sec_off, data + sec_off, prog_sz); 726 if (err) 727 return err; 728 729 /* if function is a global/weak symbol, but has restricted 730 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 731 * as static to enable more permissive BPF verification mode 732 * with more outside context available to BPF verifier 733 */ 734 if (GELF_ST_BIND(sym.st_info) != STB_LOCAL 735 && (GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN 736 || GELF_ST_VISIBILITY(sym.st_other) == STV_INTERNAL)) 737 prog->mark_btf_static = true; 738 739 nr_progs++; 740 obj->nr_programs = nr_progs; 741 } 742 743 return 0; 744 } 745 746 static __u32 get_kernel_version(void) 747 { 748 __u32 major, minor, patch; 749 struct utsname info; 750 751 uname(&info); 752 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) 753 return 0; 754 return KERNEL_VERSION(major, minor, patch); 755 } 756 757 static const struct btf_member * 758 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 759 { 760 struct btf_member *m; 761 int i; 762 763 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 764 if (btf_member_bit_offset(t, i) == bit_offset) 765 return m; 766 } 767 768 return NULL; 769 } 770 771 static const struct btf_member * 772 find_member_by_name(const struct btf *btf, const struct btf_type *t, 773 const char *name) 774 { 775 struct btf_member *m; 776 int i; 777 778 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 779 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 780 return m; 781 } 782 783 return NULL; 784 } 785 786 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 787 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 788 const char *name, __u32 kind); 789 790 static int 791 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 792 const struct btf_type **type, __u32 *type_id, 793 const struct btf_type **vtype, __u32 *vtype_id, 794 const struct btf_member **data_member) 795 { 796 const struct btf_type *kern_type, *kern_vtype; 797 const struct btf_member *kern_data_member; 798 __s32 kern_vtype_id, kern_type_id; 799 __u32 i; 800 801 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 802 if (kern_type_id < 0) { 803 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 804 tname); 805 return kern_type_id; 806 } 807 kern_type = btf__type_by_id(btf, kern_type_id); 808 809 /* Find the corresponding "map_value" type that will be used 810 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 811 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 812 * btf_vmlinux. 813 */ 814 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 815 tname, BTF_KIND_STRUCT); 816 if (kern_vtype_id < 0) { 817 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 818 STRUCT_OPS_VALUE_PREFIX, tname); 819 return kern_vtype_id; 820 } 821 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 822 823 /* Find "struct tcp_congestion_ops" from 824 * struct bpf_struct_ops_tcp_congestion_ops { 825 * [ ... ] 826 * struct tcp_congestion_ops data; 827 * } 828 */ 829 kern_data_member = btf_members(kern_vtype); 830 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 831 if (kern_data_member->type == kern_type_id) 832 break; 833 } 834 if (i == btf_vlen(kern_vtype)) { 835 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 836 tname, STRUCT_OPS_VALUE_PREFIX, tname); 837 return -EINVAL; 838 } 839 840 *type = kern_type; 841 *type_id = kern_type_id; 842 *vtype = kern_vtype; 843 *vtype_id = kern_vtype_id; 844 *data_member = kern_data_member; 845 846 return 0; 847 } 848 849 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 850 { 851 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 852 } 853 854 /* Init the map's fields that depend on kern_btf */ 855 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 856 const struct btf *btf, 857 const struct btf *kern_btf) 858 { 859 const struct btf_member *member, *kern_member, *kern_data_member; 860 const struct btf_type *type, *kern_type, *kern_vtype; 861 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 862 struct bpf_struct_ops *st_ops; 863 void *data, *kern_data; 864 const char *tname; 865 int err; 866 867 st_ops = map->st_ops; 868 type = st_ops->type; 869 tname = st_ops->tname; 870 err = find_struct_ops_kern_types(kern_btf, tname, 871 &kern_type, &kern_type_id, 872 &kern_vtype, &kern_vtype_id, 873 &kern_data_member); 874 if (err) 875 return err; 876 877 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 878 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 879 880 map->def.value_size = kern_vtype->size; 881 map->btf_vmlinux_value_type_id = kern_vtype_id; 882 883 st_ops->kern_vdata = calloc(1, kern_vtype->size); 884 if (!st_ops->kern_vdata) 885 return -ENOMEM; 886 887 data = st_ops->data; 888 kern_data_off = kern_data_member->offset / 8; 889 kern_data = st_ops->kern_vdata + kern_data_off; 890 891 member = btf_members(type); 892 for (i = 0; i < btf_vlen(type); i++, member++) { 893 const struct btf_type *mtype, *kern_mtype; 894 __u32 mtype_id, kern_mtype_id; 895 void *mdata, *kern_mdata; 896 __s64 msize, kern_msize; 897 __u32 moff, kern_moff; 898 __u32 kern_member_idx; 899 const char *mname; 900 901 mname = btf__name_by_offset(btf, member->name_off); 902 kern_member = find_member_by_name(kern_btf, kern_type, mname); 903 if (!kern_member) { 904 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 905 map->name, mname); 906 return -ENOTSUP; 907 } 908 909 kern_member_idx = kern_member - btf_members(kern_type); 910 if (btf_member_bitfield_size(type, i) || 911 btf_member_bitfield_size(kern_type, kern_member_idx)) { 912 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 913 map->name, mname); 914 return -ENOTSUP; 915 } 916 917 moff = member->offset / 8; 918 kern_moff = kern_member->offset / 8; 919 920 mdata = data + moff; 921 kern_mdata = kern_data + kern_moff; 922 923 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 924 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 925 &kern_mtype_id); 926 if (BTF_INFO_KIND(mtype->info) != 927 BTF_INFO_KIND(kern_mtype->info)) { 928 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 929 map->name, mname, BTF_INFO_KIND(mtype->info), 930 BTF_INFO_KIND(kern_mtype->info)); 931 return -ENOTSUP; 932 } 933 934 if (btf_is_ptr(mtype)) { 935 struct bpf_program *prog; 936 937 prog = st_ops->progs[i]; 938 if (!prog) 939 continue; 940 941 kern_mtype = skip_mods_and_typedefs(kern_btf, 942 kern_mtype->type, 943 &kern_mtype_id); 944 945 /* mtype->type must be a func_proto which was 946 * guaranteed in bpf_object__collect_st_ops_relos(), 947 * so only check kern_mtype for func_proto here. 948 */ 949 if (!btf_is_func_proto(kern_mtype)) { 950 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 951 map->name, mname); 952 return -ENOTSUP; 953 } 954 955 prog->attach_btf_id = kern_type_id; 956 prog->expected_attach_type = kern_member_idx; 957 958 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 959 960 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 961 map->name, mname, prog->name, moff, 962 kern_moff); 963 964 continue; 965 } 966 967 msize = btf__resolve_size(btf, mtype_id); 968 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 969 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 970 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 971 map->name, mname, (ssize_t)msize, 972 (ssize_t)kern_msize); 973 return -ENOTSUP; 974 } 975 976 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 977 map->name, mname, (unsigned int)msize, 978 moff, kern_moff); 979 memcpy(kern_mdata, mdata, msize); 980 } 981 982 return 0; 983 } 984 985 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 986 { 987 struct bpf_map *map; 988 size_t i; 989 int err; 990 991 for (i = 0; i < obj->nr_maps; i++) { 992 map = &obj->maps[i]; 993 994 if (!bpf_map__is_struct_ops(map)) 995 continue; 996 997 err = bpf_map__init_kern_struct_ops(map, obj->btf, 998 obj->btf_vmlinux); 999 if (err) 1000 return err; 1001 } 1002 1003 return 0; 1004 } 1005 1006 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) 1007 { 1008 const struct btf_type *type, *datasec; 1009 const struct btf_var_secinfo *vsi; 1010 struct bpf_struct_ops *st_ops; 1011 const char *tname, *var_name; 1012 __s32 type_id, datasec_id; 1013 const struct btf *btf; 1014 struct bpf_map *map; 1015 __u32 i; 1016 1017 if (obj->efile.st_ops_shndx == -1) 1018 return 0; 1019 1020 btf = obj->btf; 1021 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, 1022 BTF_KIND_DATASEC); 1023 if (datasec_id < 0) { 1024 pr_warn("struct_ops init: DATASEC %s not found\n", 1025 STRUCT_OPS_SEC); 1026 return -EINVAL; 1027 } 1028 1029 datasec = btf__type_by_id(btf, datasec_id); 1030 vsi = btf_var_secinfos(datasec); 1031 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1032 type = btf__type_by_id(obj->btf, vsi->type); 1033 var_name = btf__name_by_offset(obj->btf, type->name_off); 1034 1035 type_id = btf__resolve_type(obj->btf, vsi->type); 1036 if (type_id < 0) { 1037 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1038 vsi->type, STRUCT_OPS_SEC); 1039 return -EINVAL; 1040 } 1041 1042 type = btf__type_by_id(obj->btf, type_id); 1043 tname = btf__name_by_offset(obj->btf, type->name_off); 1044 if (!tname[0]) { 1045 pr_warn("struct_ops init: anonymous type is not supported\n"); 1046 return -ENOTSUP; 1047 } 1048 if (!btf_is_struct(type)) { 1049 pr_warn("struct_ops init: %s is not a struct\n", tname); 1050 return -EINVAL; 1051 } 1052 1053 map = bpf_object__add_map(obj); 1054 if (IS_ERR(map)) 1055 return PTR_ERR(map); 1056 1057 map->sec_idx = obj->efile.st_ops_shndx; 1058 map->sec_offset = vsi->offset; 1059 map->name = strdup(var_name); 1060 if (!map->name) 1061 return -ENOMEM; 1062 1063 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1064 map->def.key_size = sizeof(int); 1065 map->def.value_size = type->size; 1066 map->def.max_entries = 1; 1067 1068 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1069 if (!map->st_ops) 1070 return -ENOMEM; 1071 st_ops = map->st_ops; 1072 st_ops->data = malloc(type->size); 1073 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1074 st_ops->kern_func_off = malloc(btf_vlen(type) * 1075 sizeof(*st_ops->kern_func_off)); 1076 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1077 return -ENOMEM; 1078 1079 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { 1080 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1081 var_name, STRUCT_OPS_SEC); 1082 return -EINVAL; 1083 } 1084 1085 memcpy(st_ops->data, 1086 obj->efile.st_ops_data->d_buf + vsi->offset, 1087 type->size); 1088 st_ops->tname = tname; 1089 st_ops->type = type; 1090 st_ops->type_id = type_id; 1091 1092 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1093 tname, type_id, var_name, vsi->offset); 1094 } 1095 1096 return 0; 1097 } 1098 1099 static struct bpf_object *bpf_object__new(const char *path, 1100 const void *obj_buf, 1101 size_t obj_buf_sz, 1102 const char *obj_name) 1103 { 1104 struct bpf_object *obj; 1105 char *end; 1106 1107 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1108 if (!obj) { 1109 pr_warn("alloc memory failed for %s\n", path); 1110 return ERR_PTR(-ENOMEM); 1111 } 1112 1113 strcpy(obj->path, path); 1114 if (obj_name) { 1115 strncpy(obj->name, obj_name, sizeof(obj->name) - 1); 1116 obj->name[sizeof(obj->name) - 1] = 0; 1117 } else { 1118 /* Using basename() GNU version which doesn't modify arg. */ 1119 strncpy(obj->name, basename((void *)path), 1120 sizeof(obj->name) - 1); 1121 end = strchr(obj->name, '.'); 1122 if (end) 1123 *end = 0; 1124 } 1125 1126 obj->efile.fd = -1; 1127 /* 1128 * Caller of this function should also call 1129 * bpf_object__elf_finish() after data collection to return 1130 * obj_buf to user. If not, we should duplicate the buffer to 1131 * avoid user freeing them before elf finish. 1132 */ 1133 obj->efile.obj_buf = obj_buf; 1134 obj->efile.obj_buf_sz = obj_buf_sz; 1135 obj->efile.maps_shndx = -1; 1136 obj->efile.btf_maps_shndx = -1; 1137 obj->efile.data_shndx = -1; 1138 obj->efile.rodata_shndx = -1; 1139 obj->efile.bss_shndx = -1; 1140 obj->efile.st_ops_shndx = -1; 1141 obj->kconfig_map_idx = -1; 1142 obj->rodata_map_idx = -1; 1143 1144 obj->kern_version = get_kernel_version(); 1145 obj->loaded = false; 1146 1147 INIT_LIST_HEAD(&obj->list); 1148 list_add(&obj->list, &bpf_objects_list); 1149 return obj; 1150 } 1151 1152 static void bpf_object__elf_finish(struct bpf_object *obj) 1153 { 1154 if (!obj_elf_valid(obj)) 1155 return; 1156 1157 if (obj->efile.elf) { 1158 elf_end(obj->efile.elf); 1159 obj->efile.elf = NULL; 1160 } 1161 obj->efile.symbols = NULL; 1162 obj->efile.data = NULL; 1163 obj->efile.rodata = NULL; 1164 obj->efile.bss = NULL; 1165 obj->efile.st_ops_data = NULL; 1166 1167 zfree(&obj->efile.reloc_sects); 1168 obj->efile.nr_reloc_sects = 0; 1169 zclose(obj->efile.fd); 1170 obj->efile.obj_buf = NULL; 1171 obj->efile.obj_buf_sz = 0; 1172 } 1173 1174 static int bpf_object__elf_init(struct bpf_object *obj) 1175 { 1176 int err = 0; 1177 GElf_Ehdr *ep; 1178 1179 if (obj_elf_valid(obj)) { 1180 pr_warn("elf: init internal error\n"); 1181 return -LIBBPF_ERRNO__LIBELF; 1182 } 1183 1184 if (obj->efile.obj_buf_sz > 0) { 1185 /* 1186 * obj_buf should have been validated by 1187 * bpf_object__open_buffer(). 1188 */ 1189 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, 1190 obj->efile.obj_buf_sz); 1191 } else { 1192 obj->efile.fd = open(obj->path, O_RDONLY); 1193 if (obj->efile.fd < 0) { 1194 char errmsg[STRERR_BUFSIZE], *cp; 1195 1196 err = -errno; 1197 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1198 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1199 return err; 1200 } 1201 1202 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1203 } 1204 1205 if (!obj->efile.elf) { 1206 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1207 err = -LIBBPF_ERRNO__LIBELF; 1208 goto errout; 1209 } 1210 1211 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 1212 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1213 err = -LIBBPF_ERRNO__FORMAT; 1214 goto errout; 1215 } 1216 ep = &obj->efile.ehdr; 1217 1218 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) { 1219 pr_warn("elf: failed to get section names section index for %s: %s\n", 1220 obj->path, elf_errmsg(-1)); 1221 err = -LIBBPF_ERRNO__FORMAT; 1222 goto errout; 1223 } 1224 1225 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1226 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) { 1227 pr_warn("elf: failed to get section names strings from %s: %s\n", 1228 obj->path, elf_errmsg(-1)); 1229 err = -LIBBPF_ERRNO__FORMAT; 1230 goto errout; 1231 } 1232 1233 /* Old LLVM set e_machine to EM_NONE */ 1234 if (ep->e_type != ET_REL || 1235 (ep->e_machine && ep->e_machine != EM_BPF)) { 1236 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1237 err = -LIBBPF_ERRNO__FORMAT; 1238 goto errout; 1239 } 1240 1241 return 0; 1242 errout: 1243 bpf_object__elf_finish(obj); 1244 return err; 1245 } 1246 1247 static int bpf_object__check_endianness(struct bpf_object *obj) 1248 { 1249 #if __BYTE_ORDER == __LITTLE_ENDIAN 1250 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 1251 return 0; 1252 #elif __BYTE_ORDER == __BIG_ENDIAN 1253 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 1254 return 0; 1255 #else 1256 # error "Unrecognized __BYTE_ORDER__" 1257 #endif 1258 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1259 return -LIBBPF_ERRNO__ENDIAN; 1260 } 1261 1262 static int 1263 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1264 { 1265 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 1266 pr_debug("license of %s is %s\n", obj->path, obj->license); 1267 return 0; 1268 } 1269 1270 static int 1271 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1272 { 1273 __u32 kver; 1274 1275 if (size != sizeof(kver)) { 1276 pr_warn("invalid kver section in %s\n", obj->path); 1277 return -LIBBPF_ERRNO__FORMAT; 1278 } 1279 memcpy(&kver, data, sizeof(kver)); 1280 obj->kern_version = kver; 1281 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1282 return 0; 1283 } 1284 1285 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1286 { 1287 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1288 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1289 return true; 1290 return false; 1291 } 1292 1293 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 1294 __u32 *size) 1295 { 1296 int ret = -ENOENT; 1297 1298 *size = 0; 1299 if (!name) { 1300 return -EINVAL; 1301 } else if (!strcmp(name, DATA_SEC)) { 1302 if (obj->efile.data) 1303 *size = obj->efile.data->d_size; 1304 } else if (!strcmp(name, BSS_SEC)) { 1305 if (obj->efile.bss) 1306 *size = obj->efile.bss->d_size; 1307 } else if (!strcmp(name, RODATA_SEC)) { 1308 if (obj->efile.rodata) 1309 *size = obj->efile.rodata->d_size; 1310 } else if (!strcmp(name, STRUCT_OPS_SEC)) { 1311 if (obj->efile.st_ops_data) 1312 *size = obj->efile.st_ops_data->d_size; 1313 } else { 1314 Elf_Scn *scn = elf_sec_by_name(obj, name); 1315 Elf_Data *data = elf_sec_data(obj, scn); 1316 1317 if (data) { 1318 ret = 0; /* found it */ 1319 *size = data->d_size; 1320 } 1321 } 1322 1323 return *size ? 0 : ret; 1324 } 1325 1326 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 1327 __u32 *off) 1328 { 1329 Elf_Data *symbols = obj->efile.symbols; 1330 const char *sname; 1331 size_t si; 1332 1333 if (!name || !off) 1334 return -EINVAL; 1335 1336 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 1337 GElf_Sym sym; 1338 1339 if (!gelf_getsym(symbols, si, &sym)) 1340 continue; 1341 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 1342 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 1343 continue; 1344 1345 sname = elf_sym_str(obj, sym.st_name); 1346 if (!sname) { 1347 pr_warn("failed to get sym name string for var %s\n", 1348 name); 1349 return -EIO; 1350 } 1351 if (strcmp(name, sname) == 0) { 1352 *off = sym.st_value; 1353 return 0; 1354 } 1355 } 1356 1357 return -ENOENT; 1358 } 1359 1360 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1361 { 1362 struct bpf_map *new_maps; 1363 size_t new_cap; 1364 int i; 1365 1366 if (obj->nr_maps < obj->maps_cap) 1367 return &obj->maps[obj->nr_maps++]; 1368 1369 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 1370 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps)); 1371 if (!new_maps) { 1372 pr_warn("alloc maps for object failed\n"); 1373 return ERR_PTR(-ENOMEM); 1374 } 1375 1376 obj->maps_cap = new_cap; 1377 obj->maps = new_maps; 1378 1379 /* zero out new maps */ 1380 memset(obj->maps + obj->nr_maps, 0, 1381 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 1382 /* 1383 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 1384 * when failure (zclose won't close negative fd)). 1385 */ 1386 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 1387 obj->maps[i].fd = -1; 1388 obj->maps[i].inner_map_fd = -1; 1389 } 1390 1391 return &obj->maps[obj->nr_maps++]; 1392 } 1393 1394 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1395 { 1396 long page_sz = sysconf(_SC_PAGE_SIZE); 1397 size_t map_sz; 1398 1399 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; 1400 map_sz = roundup(map_sz, page_sz); 1401 return map_sz; 1402 } 1403 1404 static char *internal_map_name(struct bpf_object *obj, 1405 enum libbpf_map_type type) 1406 { 1407 char map_name[BPF_OBJ_NAME_LEN], *p; 1408 const char *sfx = libbpf_type_to_btf_name[type]; 1409 int sfx_len = max((size_t)7, strlen(sfx)); 1410 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, 1411 strlen(obj->name)); 1412 1413 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1414 sfx_len, libbpf_type_to_btf_name[type]); 1415 1416 /* sanitise map name to characters allowed by kernel */ 1417 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1418 if (!isalnum(*p) && *p != '_' && *p != '.') 1419 *p = '_'; 1420 1421 return strdup(map_name); 1422 } 1423 1424 static int 1425 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1426 int sec_idx, void *data, size_t data_sz) 1427 { 1428 struct bpf_map_def *def; 1429 struct bpf_map *map; 1430 int err; 1431 1432 map = bpf_object__add_map(obj); 1433 if (IS_ERR(map)) 1434 return PTR_ERR(map); 1435 1436 map->libbpf_type = type; 1437 map->sec_idx = sec_idx; 1438 map->sec_offset = 0; 1439 map->name = internal_map_name(obj, type); 1440 if (!map->name) { 1441 pr_warn("failed to alloc map name\n"); 1442 return -ENOMEM; 1443 } 1444 1445 def = &map->def; 1446 def->type = BPF_MAP_TYPE_ARRAY; 1447 def->key_size = sizeof(int); 1448 def->value_size = data_sz; 1449 def->max_entries = 1; 1450 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1451 ? BPF_F_RDONLY_PROG : 0; 1452 def->map_flags |= BPF_F_MMAPABLE; 1453 1454 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1455 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1456 1457 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 1458 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1459 if (map->mmaped == MAP_FAILED) { 1460 err = -errno; 1461 map->mmaped = NULL; 1462 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1463 map->name, err); 1464 zfree(&map->name); 1465 return err; 1466 } 1467 1468 if (data) 1469 memcpy(map->mmaped, data, data_sz); 1470 1471 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1472 return 0; 1473 } 1474 1475 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1476 { 1477 int err; 1478 1479 /* 1480 * Populate obj->maps with libbpf internal maps. 1481 */ 1482 if (obj->efile.data_shndx >= 0) { 1483 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1484 obj->efile.data_shndx, 1485 obj->efile.data->d_buf, 1486 obj->efile.data->d_size); 1487 if (err) 1488 return err; 1489 } 1490 if (obj->efile.rodata_shndx >= 0) { 1491 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1492 obj->efile.rodata_shndx, 1493 obj->efile.rodata->d_buf, 1494 obj->efile.rodata->d_size); 1495 if (err) 1496 return err; 1497 1498 obj->rodata_map_idx = obj->nr_maps - 1; 1499 } 1500 if (obj->efile.bss_shndx >= 0) { 1501 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1502 obj->efile.bss_shndx, 1503 NULL, 1504 obj->efile.bss->d_size); 1505 if (err) 1506 return err; 1507 } 1508 return 0; 1509 } 1510 1511 1512 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1513 const void *name) 1514 { 1515 int i; 1516 1517 for (i = 0; i < obj->nr_extern; i++) { 1518 if (strcmp(obj->externs[i].name, name) == 0) 1519 return &obj->externs[i]; 1520 } 1521 return NULL; 1522 } 1523 1524 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1525 char value) 1526 { 1527 switch (ext->kcfg.type) { 1528 case KCFG_BOOL: 1529 if (value == 'm') { 1530 pr_warn("extern (kcfg) %s=%c should be tristate or char\n", 1531 ext->name, value); 1532 return -EINVAL; 1533 } 1534 *(bool *)ext_val = value == 'y' ? true : false; 1535 break; 1536 case KCFG_TRISTATE: 1537 if (value == 'y') 1538 *(enum libbpf_tristate *)ext_val = TRI_YES; 1539 else if (value == 'm') 1540 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1541 else /* value == 'n' */ 1542 *(enum libbpf_tristate *)ext_val = TRI_NO; 1543 break; 1544 case KCFG_CHAR: 1545 *(char *)ext_val = value; 1546 break; 1547 case KCFG_UNKNOWN: 1548 case KCFG_INT: 1549 case KCFG_CHAR_ARR: 1550 default: 1551 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", 1552 ext->name, value); 1553 return -EINVAL; 1554 } 1555 ext->is_set = true; 1556 return 0; 1557 } 1558 1559 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1560 const char *value) 1561 { 1562 size_t len; 1563 1564 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1565 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); 1566 return -EINVAL; 1567 } 1568 1569 len = strlen(value); 1570 if (value[len - 1] != '"') { 1571 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1572 ext->name, value); 1573 return -EINVAL; 1574 } 1575 1576 /* strip quotes */ 1577 len -= 2; 1578 if (len >= ext->kcfg.sz) { 1579 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", 1580 ext->name, value, len, ext->kcfg.sz - 1); 1581 len = ext->kcfg.sz - 1; 1582 } 1583 memcpy(ext_val, value + 1, len); 1584 ext_val[len] = '\0'; 1585 ext->is_set = true; 1586 return 0; 1587 } 1588 1589 static int parse_u64(const char *value, __u64 *res) 1590 { 1591 char *value_end; 1592 int err; 1593 1594 errno = 0; 1595 *res = strtoull(value, &value_end, 0); 1596 if (errno) { 1597 err = -errno; 1598 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1599 return err; 1600 } 1601 if (*value_end) { 1602 pr_warn("failed to parse '%s' as integer completely\n", value); 1603 return -EINVAL; 1604 } 1605 return 0; 1606 } 1607 1608 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1609 { 1610 int bit_sz = ext->kcfg.sz * 8; 1611 1612 if (ext->kcfg.sz == 8) 1613 return true; 1614 1615 /* Validate that value stored in u64 fits in integer of `ext->sz` 1616 * bytes size without any loss of information. If the target integer 1617 * is signed, we rely on the following limits of integer type of 1618 * Y bits and subsequent transformation: 1619 * 1620 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1621 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1622 * 0 <= X + 2^(Y-1) < 2^Y 1623 * 1624 * For unsigned target integer, check that all the (64 - Y) bits are 1625 * zero. 1626 */ 1627 if (ext->kcfg.is_signed) 1628 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1629 else 1630 return (v >> bit_sz) == 0; 1631 } 1632 1633 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1634 __u64 value) 1635 { 1636 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 1637 pr_warn("extern (kcfg) %s=%llu should be integer\n", 1638 ext->name, (unsigned long long)value); 1639 return -EINVAL; 1640 } 1641 if (!is_kcfg_value_in_range(ext, value)) { 1642 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", 1643 ext->name, (unsigned long long)value, ext->kcfg.sz); 1644 return -ERANGE; 1645 } 1646 switch (ext->kcfg.sz) { 1647 case 1: *(__u8 *)ext_val = value; break; 1648 case 2: *(__u16 *)ext_val = value; break; 1649 case 4: *(__u32 *)ext_val = value; break; 1650 case 8: *(__u64 *)ext_val = value; break; 1651 default: 1652 return -EINVAL; 1653 } 1654 ext->is_set = true; 1655 return 0; 1656 } 1657 1658 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1659 char *buf, void *data) 1660 { 1661 struct extern_desc *ext; 1662 char *sep, *value; 1663 int len, err = 0; 1664 void *ext_val; 1665 __u64 num; 1666 1667 if (strncmp(buf, "CONFIG_", 7)) 1668 return 0; 1669 1670 sep = strchr(buf, '='); 1671 if (!sep) { 1672 pr_warn("failed to parse '%s': no separator\n", buf); 1673 return -EINVAL; 1674 } 1675 1676 /* Trim ending '\n' */ 1677 len = strlen(buf); 1678 if (buf[len - 1] == '\n') 1679 buf[len - 1] = '\0'; 1680 /* Split on '=' and ensure that a value is present. */ 1681 *sep = '\0'; 1682 if (!sep[1]) { 1683 *sep = '='; 1684 pr_warn("failed to parse '%s': no value\n", buf); 1685 return -EINVAL; 1686 } 1687 1688 ext = find_extern_by_name(obj, buf); 1689 if (!ext || ext->is_set) 1690 return 0; 1691 1692 ext_val = data + ext->kcfg.data_off; 1693 value = sep + 1; 1694 1695 switch (*value) { 1696 case 'y': case 'n': case 'm': 1697 err = set_kcfg_value_tri(ext, ext_val, *value); 1698 break; 1699 case '"': 1700 err = set_kcfg_value_str(ext, ext_val, value); 1701 break; 1702 default: 1703 /* assume integer */ 1704 err = parse_u64(value, &num); 1705 if (err) { 1706 pr_warn("extern (kcfg) %s=%s should be integer\n", 1707 ext->name, value); 1708 return err; 1709 } 1710 err = set_kcfg_value_num(ext, ext_val, num); 1711 break; 1712 } 1713 if (err) 1714 return err; 1715 pr_debug("extern (kcfg) %s=%s\n", ext->name, value); 1716 return 0; 1717 } 1718 1719 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1720 { 1721 char buf[PATH_MAX]; 1722 struct utsname uts; 1723 int len, err = 0; 1724 gzFile file; 1725 1726 uname(&uts); 1727 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1728 if (len < 0) 1729 return -EINVAL; 1730 else if (len >= PATH_MAX) 1731 return -ENAMETOOLONG; 1732 1733 /* gzopen also accepts uncompressed files. */ 1734 file = gzopen(buf, "r"); 1735 if (!file) 1736 file = gzopen("/proc/config.gz", "r"); 1737 1738 if (!file) { 1739 pr_warn("failed to open system Kconfig\n"); 1740 return -ENOENT; 1741 } 1742 1743 while (gzgets(file, buf, sizeof(buf))) { 1744 err = bpf_object__process_kconfig_line(obj, buf, data); 1745 if (err) { 1746 pr_warn("error parsing system Kconfig line '%s': %d\n", 1747 buf, err); 1748 goto out; 1749 } 1750 } 1751 1752 out: 1753 gzclose(file); 1754 return err; 1755 } 1756 1757 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 1758 const char *config, void *data) 1759 { 1760 char buf[PATH_MAX]; 1761 int err = 0; 1762 FILE *file; 1763 1764 file = fmemopen((void *)config, strlen(config), "r"); 1765 if (!file) { 1766 err = -errno; 1767 pr_warn("failed to open in-memory Kconfig: %d\n", err); 1768 return err; 1769 } 1770 1771 while (fgets(buf, sizeof(buf), file)) { 1772 err = bpf_object__process_kconfig_line(obj, buf, data); 1773 if (err) { 1774 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 1775 buf, err); 1776 break; 1777 } 1778 } 1779 1780 fclose(file); 1781 return err; 1782 } 1783 1784 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 1785 { 1786 struct extern_desc *last_ext = NULL, *ext; 1787 size_t map_sz; 1788 int i, err; 1789 1790 for (i = 0; i < obj->nr_extern; i++) { 1791 ext = &obj->externs[i]; 1792 if (ext->type == EXT_KCFG) 1793 last_ext = ext; 1794 } 1795 1796 if (!last_ext) 1797 return 0; 1798 1799 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 1800 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 1801 obj->efile.symbols_shndx, 1802 NULL, map_sz); 1803 if (err) 1804 return err; 1805 1806 obj->kconfig_map_idx = obj->nr_maps - 1; 1807 1808 return 0; 1809 } 1810 1811 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 1812 { 1813 Elf_Data *symbols = obj->efile.symbols; 1814 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 1815 Elf_Data *data = NULL; 1816 Elf_Scn *scn; 1817 1818 if (obj->efile.maps_shndx < 0) 1819 return 0; 1820 1821 if (!symbols) 1822 return -EINVAL; 1823 1824 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx); 1825 data = elf_sec_data(obj, scn); 1826 if (!scn || !data) { 1827 pr_warn("elf: failed to get legacy map definitions for %s\n", 1828 obj->path); 1829 return -EINVAL; 1830 } 1831 1832 /* 1833 * Count number of maps. Each map has a name. 1834 * Array of maps is not supported: only the first element is 1835 * considered. 1836 * 1837 * TODO: Detect array of map and report error. 1838 */ 1839 nr_syms = symbols->d_size / sizeof(GElf_Sym); 1840 for (i = 0; i < nr_syms; i++) { 1841 GElf_Sym sym; 1842 1843 if (!gelf_getsym(symbols, i, &sym)) 1844 continue; 1845 if (sym.st_shndx != obj->efile.maps_shndx) 1846 continue; 1847 nr_maps++; 1848 } 1849 /* Assume equally sized map definitions */ 1850 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n", 1851 nr_maps, data->d_size, obj->path); 1852 1853 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { 1854 pr_warn("elf: unable to determine legacy map definition size in %s\n", 1855 obj->path); 1856 return -EINVAL; 1857 } 1858 map_def_sz = data->d_size / nr_maps; 1859 1860 /* Fill obj->maps using data in "maps" section. */ 1861 for (i = 0; i < nr_syms; i++) { 1862 GElf_Sym sym; 1863 const char *map_name; 1864 struct bpf_map_def *def; 1865 struct bpf_map *map; 1866 1867 if (!gelf_getsym(symbols, i, &sym)) 1868 continue; 1869 if (sym.st_shndx != obj->efile.maps_shndx) 1870 continue; 1871 1872 map = bpf_object__add_map(obj); 1873 if (IS_ERR(map)) 1874 return PTR_ERR(map); 1875 1876 map_name = elf_sym_str(obj, sym.st_name); 1877 if (!map_name) { 1878 pr_warn("failed to get map #%d name sym string for obj %s\n", 1879 i, obj->path); 1880 return -LIBBPF_ERRNO__FORMAT; 1881 } 1882 1883 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION 1884 || GELF_ST_BIND(sym.st_info) == STB_LOCAL) { 1885 pr_warn("map '%s' (legacy): static maps are not supported\n", map_name); 1886 return -ENOTSUP; 1887 } 1888 1889 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1890 map->sec_idx = sym.st_shndx; 1891 map->sec_offset = sym.st_value; 1892 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 1893 map_name, map->sec_idx, map->sec_offset); 1894 if (sym.st_value + map_def_sz > data->d_size) { 1895 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", 1896 obj->path, map_name); 1897 return -EINVAL; 1898 } 1899 1900 map->name = strdup(map_name); 1901 if (!map->name) { 1902 pr_warn("failed to alloc map name\n"); 1903 return -ENOMEM; 1904 } 1905 pr_debug("map %d is \"%s\"\n", i, map->name); 1906 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 1907 /* 1908 * If the definition of the map in the object file fits in 1909 * bpf_map_def, copy it. Any extra fields in our version 1910 * of bpf_map_def will default to zero as a result of the 1911 * calloc above. 1912 */ 1913 if (map_def_sz <= sizeof(struct bpf_map_def)) { 1914 memcpy(&map->def, def, map_def_sz); 1915 } else { 1916 /* 1917 * Here the map structure being read is bigger than what 1918 * we expect, truncate if the excess bits are all zero. 1919 * If they are not zero, reject this map as 1920 * incompatible. 1921 */ 1922 char *b; 1923 1924 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1925 b < ((char *)def) + map_def_sz; b++) { 1926 if (*b != 0) { 1927 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", 1928 obj->path, map_name); 1929 if (strict) 1930 return -EINVAL; 1931 } 1932 } 1933 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1934 } 1935 } 1936 return 0; 1937 } 1938 1939 const struct btf_type * 1940 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 1941 { 1942 const struct btf_type *t = btf__type_by_id(btf, id); 1943 1944 if (res_id) 1945 *res_id = id; 1946 1947 while (btf_is_mod(t) || btf_is_typedef(t)) { 1948 if (res_id) 1949 *res_id = t->type; 1950 t = btf__type_by_id(btf, t->type); 1951 } 1952 1953 return t; 1954 } 1955 1956 static const struct btf_type * 1957 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 1958 { 1959 const struct btf_type *t; 1960 1961 t = skip_mods_and_typedefs(btf, id, NULL); 1962 if (!btf_is_ptr(t)) 1963 return NULL; 1964 1965 t = skip_mods_and_typedefs(btf, t->type, res_id); 1966 1967 return btf_is_func_proto(t) ? t : NULL; 1968 } 1969 1970 static const char *__btf_kind_str(__u16 kind) 1971 { 1972 switch (kind) { 1973 case BTF_KIND_UNKN: return "void"; 1974 case BTF_KIND_INT: return "int"; 1975 case BTF_KIND_PTR: return "ptr"; 1976 case BTF_KIND_ARRAY: return "array"; 1977 case BTF_KIND_STRUCT: return "struct"; 1978 case BTF_KIND_UNION: return "union"; 1979 case BTF_KIND_ENUM: return "enum"; 1980 case BTF_KIND_FWD: return "fwd"; 1981 case BTF_KIND_TYPEDEF: return "typedef"; 1982 case BTF_KIND_VOLATILE: return "volatile"; 1983 case BTF_KIND_CONST: return "const"; 1984 case BTF_KIND_RESTRICT: return "restrict"; 1985 case BTF_KIND_FUNC: return "func"; 1986 case BTF_KIND_FUNC_PROTO: return "func_proto"; 1987 case BTF_KIND_VAR: return "var"; 1988 case BTF_KIND_DATASEC: return "datasec"; 1989 case BTF_KIND_FLOAT: return "float"; 1990 default: return "unknown"; 1991 } 1992 } 1993 1994 const char *btf_kind_str(const struct btf_type *t) 1995 { 1996 return __btf_kind_str(btf_kind(t)); 1997 } 1998 1999 /* 2000 * Fetch integer attribute of BTF map definition. Such attributes are 2001 * represented using a pointer to an array, in which dimensionality of array 2002 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2003 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2004 * type definition, while using only sizeof(void *) space in ELF data section. 2005 */ 2006 static bool get_map_field_int(const char *map_name, const struct btf *btf, 2007 const struct btf_member *m, __u32 *res) 2008 { 2009 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2010 const char *name = btf__name_by_offset(btf, m->name_off); 2011 const struct btf_array *arr_info; 2012 const struct btf_type *arr_t; 2013 2014 if (!btf_is_ptr(t)) { 2015 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2016 map_name, name, btf_kind_str(t)); 2017 return false; 2018 } 2019 2020 arr_t = btf__type_by_id(btf, t->type); 2021 if (!arr_t) { 2022 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2023 map_name, name, t->type); 2024 return false; 2025 } 2026 if (!btf_is_array(arr_t)) { 2027 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2028 map_name, name, btf_kind_str(arr_t)); 2029 return false; 2030 } 2031 arr_info = btf_array(arr_t); 2032 *res = arr_info->nelems; 2033 return true; 2034 } 2035 2036 static int build_map_pin_path(struct bpf_map *map, const char *path) 2037 { 2038 char buf[PATH_MAX]; 2039 int len; 2040 2041 if (!path) 2042 path = "/sys/fs/bpf"; 2043 2044 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); 2045 if (len < 0) 2046 return -EINVAL; 2047 else if (len >= PATH_MAX) 2048 return -ENAMETOOLONG; 2049 2050 return bpf_map__set_pin_path(map, buf); 2051 } 2052 2053 int parse_btf_map_def(const char *map_name, struct btf *btf, 2054 const struct btf_type *def_t, bool strict, 2055 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2056 { 2057 const struct btf_type *t; 2058 const struct btf_member *m; 2059 bool is_inner = inner_def == NULL; 2060 int vlen, i; 2061 2062 vlen = btf_vlen(def_t); 2063 m = btf_members(def_t); 2064 for (i = 0; i < vlen; i++, m++) { 2065 const char *name = btf__name_by_offset(btf, m->name_off); 2066 2067 if (!name) { 2068 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2069 return -EINVAL; 2070 } 2071 if (strcmp(name, "type") == 0) { 2072 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2073 return -EINVAL; 2074 map_def->parts |= MAP_DEF_MAP_TYPE; 2075 } else if (strcmp(name, "max_entries") == 0) { 2076 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2077 return -EINVAL; 2078 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2079 } else if (strcmp(name, "map_flags") == 0) { 2080 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2081 return -EINVAL; 2082 map_def->parts |= MAP_DEF_MAP_FLAGS; 2083 } else if (strcmp(name, "numa_node") == 0) { 2084 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2085 return -EINVAL; 2086 map_def->parts |= MAP_DEF_NUMA_NODE; 2087 } else if (strcmp(name, "key_size") == 0) { 2088 __u32 sz; 2089 2090 if (!get_map_field_int(map_name, btf, m, &sz)) 2091 return -EINVAL; 2092 if (map_def->key_size && map_def->key_size != sz) { 2093 pr_warn("map '%s': conflicting key size %u != %u.\n", 2094 map_name, map_def->key_size, sz); 2095 return -EINVAL; 2096 } 2097 map_def->key_size = sz; 2098 map_def->parts |= MAP_DEF_KEY_SIZE; 2099 } else if (strcmp(name, "key") == 0) { 2100 __s64 sz; 2101 2102 t = btf__type_by_id(btf, m->type); 2103 if (!t) { 2104 pr_warn("map '%s': key type [%d] not found.\n", 2105 map_name, m->type); 2106 return -EINVAL; 2107 } 2108 if (!btf_is_ptr(t)) { 2109 pr_warn("map '%s': key spec is not PTR: %s.\n", 2110 map_name, btf_kind_str(t)); 2111 return -EINVAL; 2112 } 2113 sz = btf__resolve_size(btf, t->type); 2114 if (sz < 0) { 2115 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2116 map_name, t->type, (ssize_t)sz); 2117 return sz; 2118 } 2119 if (map_def->key_size && map_def->key_size != sz) { 2120 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2121 map_name, map_def->key_size, (ssize_t)sz); 2122 return -EINVAL; 2123 } 2124 map_def->key_size = sz; 2125 map_def->key_type_id = t->type; 2126 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2127 } else if (strcmp(name, "value_size") == 0) { 2128 __u32 sz; 2129 2130 if (!get_map_field_int(map_name, btf, m, &sz)) 2131 return -EINVAL; 2132 if (map_def->value_size && map_def->value_size != sz) { 2133 pr_warn("map '%s': conflicting value size %u != %u.\n", 2134 map_name, map_def->value_size, sz); 2135 return -EINVAL; 2136 } 2137 map_def->value_size = sz; 2138 map_def->parts |= MAP_DEF_VALUE_SIZE; 2139 } else if (strcmp(name, "value") == 0) { 2140 __s64 sz; 2141 2142 t = btf__type_by_id(btf, m->type); 2143 if (!t) { 2144 pr_warn("map '%s': value type [%d] not found.\n", 2145 map_name, m->type); 2146 return -EINVAL; 2147 } 2148 if (!btf_is_ptr(t)) { 2149 pr_warn("map '%s': value spec is not PTR: %s.\n", 2150 map_name, btf_kind_str(t)); 2151 return -EINVAL; 2152 } 2153 sz = btf__resolve_size(btf, t->type); 2154 if (sz < 0) { 2155 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2156 map_name, t->type, (ssize_t)sz); 2157 return sz; 2158 } 2159 if (map_def->value_size && map_def->value_size != sz) { 2160 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2161 map_name, map_def->value_size, (ssize_t)sz); 2162 return -EINVAL; 2163 } 2164 map_def->value_size = sz; 2165 map_def->value_type_id = t->type; 2166 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2167 } 2168 else if (strcmp(name, "values") == 0) { 2169 char inner_map_name[128]; 2170 int err; 2171 2172 if (is_inner) { 2173 pr_warn("map '%s': multi-level inner maps not supported.\n", 2174 map_name); 2175 return -ENOTSUP; 2176 } 2177 if (i != vlen - 1) { 2178 pr_warn("map '%s': '%s' member should be last.\n", 2179 map_name, name); 2180 return -EINVAL; 2181 } 2182 if (!bpf_map_type__is_map_in_map(map_def->map_type)) { 2183 pr_warn("map '%s': should be map-in-map.\n", 2184 map_name); 2185 return -ENOTSUP; 2186 } 2187 if (map_def->value_size && map_def->value_size != 4) { 2188 pr_warn("map '%s': conflicting value size %u != 4.\n", 2189 map_name, map_def->value_size); 2190 return -EINVAL; 2191 } 2192 map_def->value_size = 4; 2193 t = btf__type_by_id(btf, m->type); 2194 if (!t) { 2195 pr_warn("map '%s': map-in-map inner type [%d] not found.\n", 2196 map_name, m->type); 2197 return -EINVAL; 2198 } 2199 if (!btf_is_array(t) || btf_array(t)->nelems) { 2200 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n", 2201 map_name); 2202 return -EINVAL; 2203 } 2204 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2205 if (!btf_is_ptr(t)) { 2206 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2207 map_name, btf_kind_str(t)); 2208 return -EINVAL; 2209 } 2210 t = skip_mods_and_typedefs(btf, t->type, NULL); 2211 if (!btf_is_struct(t)) { 2212 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2213 map_name, btf_kind_str(t)); 2214 return -EINVAL; 2215 } 2216 2217 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2218 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2219 if (err) 2220 return err; 2221 2222 map_def->parts |= MAP_DEF_INNER_MAP; 2223 } else if (strcmp(name, "pinning") == 0) { 2224 __u32 val; 2225 2226 if (is_inner) { 2227 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2228 return -EINVAL; 2229 } 2230 if (!get_map_field_int(map_name, btf, m, &val)) 2231 return -EINVAL; 2232 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2233 pr_warn("map '%s': invalid pinning value %u.\n", 2234 map_name, val); 2235 return -EINVAL; 2236 } 2237 map_def->pinning = val; 2238 map_def->parts |= MAP_DEF_PINNING; 2239 } else { 2240 if (strict) { 2241 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2242 return -ENOTSUP; 2243 } 2244 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2245 } 2246 } 2247 2248 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2249 pr_warn("map '%s': map type isn't specified.\n", map_name); 2250 return -EINVAL; 2251 } 2252 2253 return 0; 2254 } 2255 2256 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2257 { 2258 map->def.type = def->map_type; 2259 map->def.key_size = def->key_size; 2260 map->def.value_size = def->value_size; 2261 map->def.max_entries = def->max_entries; 2262 map->def.map_flags = def->map_flags; 2263 2264 map->numa_node = def->numa_node; 2265 map->btf_key_type_id = def->key_type_id; 2266 map->btf_value_type_id = def->value_type_id; 2267 2268 if (def->parts & MAP_DEF_MAP_TYPE) 2269 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2270 2271 if (def->parts & MAP_DEF_KEY_TYPE) 2272 pr_debug("map '%s': found key [%u], sz = %u.\n", 2273 map->name, def->key_type_id, def->key_size); 2274 else if (def->parts & MAP_DEF_KEY_SIZE) 2275 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2276 2277 if (def->parts & MAP_DEF_VALUE_TYPE) 2278 pr_debug("map '%s': found value [%u], sz = %u.\n", 2279 map->name, def->value_type_id, def->value_size); 2280 else if (def->parts & MAP_DEF_VALUE_SIZE) 2281 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2282 2283 if (def->parts & MAP_DEF_MAX_ENTRIES) 2284 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2285 if (def->parts & MAP_DEF_MAP_FLAGS) 2286 pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags); 2287 if (def->parts & MAP_DEF_PINNING) 2288 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2289 if (def->parts & MAP_DEF_NUMA_NODE) 2290 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2291 2292 if (def->parts & MAP_DEF_INNER_MAP) 2293 pr_debug("map '%s': found inner map definition.\n", map->name); 2294 } 2295 2296 static const char *btf_var_linkage_str(__u32 linkage) 2297 { 2298 switch (linkage) { 2299 case BTF_VAR_STATIC: return "static"; 2300 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2301 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2302 default: return "unknown"; 2303 } 2304 } 2305 2306 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2307 const struct btf_type *sec, 2308 int var_idx, int sec_idx, 2309 const Elf_Data *data, bool strict, 2310 const char *pin_root_path) 2311 { 2312 struct btf_map_def map_def = {}, inner_def = {}; 2313 const struct btf_type *var, *def; 2314 const struct btf_var_secinfo *vi; 2315 const struct btf_var *var_extra; 2316 const char *map_name; 2317 struct bpf_map *map; 2318 int err; 2319 2320 vi = btf_var_secinfos(sec) + var_idx; 2321 var = btf__type_by_id(obj->btf, vi->type); 2322 var_extra = btf_var(var); 2323 map_name = btf__name_by_offset(obj->btf, var->name_off); 2324 2325 if (map_name == NULL || map_name[0] == '\0') { 2326 pr_warn("map #%d: empty name.\n", var_idx); 2327 return -EINVAL; 2328 } 2329 if ((__u64)vi->offset + vi->size > data->d_size) { 2330 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2331 return -EINVAL; 2332 } 2333 if (!btf_is_var(var)) { 2334 pr_warn("map '%s': unexpected var kind %s.\n", 2335 map_name, btf_kind_str(var)); 2336 return -EINVAL; 2337 } 2338 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2339 pr_warn("map '%s': unsupported map linkage %s.\n", 2340 map_name, btf_var_linkage_str(var_extra->linkage)); 2341 return -EOPNOTSUPP; 2342 } 2343 2344 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2345 if (!btf_is_struct(def)) { 2346 pr_warn("map '%s': unexpected def kind %s.\n", 2347 map_name, btf_kind_str(var)); 2348 return -EINVAL; 2349 } 2350 if (def->size > vi->size) { 2351 pr_warn("map '%s': invalid def size.\n", map_name); 2352 return -EINVAL; 2353 } 2354 2355 map = bpf_object__add_map(obj); 2356 if (IS_ERR(map)) 2357 return PTR_ERR(map); 2358 map->name = strdup(map_name); 2359 if (!map->name) { 2360 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2361 return -ENOMEM; 2362 } 2363 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2364 map->def.type = BPF_MAP_TYPE_UNSPEC; 2365 map->sec_idx = sec_idx; 2366 map->sec_offset = vi->offset; 2367 map->btf_var_idx = var_idx; 2368 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2369 map_name, map->sec_idx, map->sec_offset); 2370 2371 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2372 if (err) 2373 return err; 2374 2375 fill_map_from_def(map, &map_def); 2376 2377 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2378 err = build_map_pin_path(map, pin_root_path); 2379 if (err) { 2380 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2381 return err; 2382 } 2383 } 2384 2385 if (map_def.parts & MAP_DEF_INNER_MAP) { 2386 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2387 if (!map->inner_map) 2388 return -ENOMEM; 2389 map->inner_map->fd = -1; 2390 map->inner_map->sec_idx = sec_idx; 2391 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2392 if (!map->inner_map->name) 2393 return -ENOMEM; 2394 sprintf(map->inner_map->name, "%s.inner", map_name); 2395 2396 fill_map_from_def(map->inner_map, &inner_def); 2397 } 2398 2399 return 0; 2400 } 2401 2402 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2403 const char *pin_root_path) 2404 { 2405 const struct btf_type *sec = NULL; 2406 int nr_types, i, vlen, err; 2407 const struct btf_type *t; 2408 const char *name; 2409 Elf_Data *data; 2410 Elf_Scn *scn; 2411 2412 if (obj->efile.btf_maps_shndx < 0) 2413 return 0; 2414 2415 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2416 data = elf_sec_data(obj, scn); 2417 if (!scn || !data) { 2418 pr_warn("elf: failed to get %s map definitions for %s\n", 2419 MAPS_ELF_SEC, obj->path); 2420 return -EINVAL; 2421 } 2422 2423 nr_types = btf__get_nr_types(obj->btf); 2424 for (i = 1; i <= nr_types; i++) { 2425 t = btf__type_by_id(obj->btf, i); 2426 if (!btf_is_datasec(t)) 2427 continue; 2428 name = btf__name_by_offset(obj->btf, t->name_off); 2429 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2430 sec = t; 2431 obj->efile.btf_maps_sec_btf_id = i; 2432 break; 2433 } 2434 } 2435 2436 if (!sec) { 2437 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2438 return -ENOENT; 2439 } 2440 2441 vlen = btf_vlen(sec); 2442 for (i = 0; i < vlen; i++) { 2443 err = bpf_object__init_user_btf_map(obj, sec, i, 2444 obj->efile.btf_maps_shndx, 2445 data, strict, 2446 pin_root_path); 2447 if (err) 2448 return err; 2449 } 2450 2451 return 0; 2452 } 2453 2454 static int bpf_object__init_maps(struct bpf_object *obj, 2455 const struct bpf_object_open_opts *opts) 2456 { 2457 const char *pin_root_path; 2458 bool strict; 2459 int err; 2460 2461 strict = !OPTS_GET(opts, relaxed_maps, false); 2462 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2463 2464 err = bpf_object__init_user_maps(obj, strict); 2465 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2466 err = err ?: bpf_object__init_global_data_maps(obj); 2467 err = err ?: bpf_object__init_kconfig_map(obj); 2468 err = err ?: bpf_object__init_struct_ops_maps(obj); 2469 2470 return err; 2471 } 2472 2473 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2474 { 2475 GElf_Shdr sh; 2476 2477 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh)) 2478 return false; 2479 2480 return sh.sh_flags & SHF_EXECINSTR; 2481 } 2482 2483 static bool btf_needs_sanitization(struct bpf_object *obj) 2484 { 2485 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2486 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2487 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2488 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2489 2490 return !has_func || !has_datasec || !has_func_global || !has_float; 2491 } 2492 2493 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2494 { 2495 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2496 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2497 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2498 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2499 struct btf_type *t; 2500 int i, j, vlen; 2501 2502 for (i = 1; i <= btf__get_nr_types(btf); i++) { 2503 t = (struct btf_type *)btf__type_by_id(btf, i); 2504 2505 if (!has_datasec && btf_is_var(t)) { 2506 /* replace VAR with INT */ 2507 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2508 /* 2509 * using size = 1 is the safest choice, 4 will be too 2510 * big and cause kernel BTF validation failure if 2511 * original variable took less than 4 bytes 2512 */ 2513 t->size = 1; 2514 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2515 } else if (!has_datasec && btf_is_datasec(t)) { 2516 /* replace DATASEC with STRUCT */ 2517 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2518 struct btf_member *m = btf_members(t); 2519 struct btf_type *vt; 2520 char *name; 2521 2522 name = (char *)btf__name_by_offset(btf, t->name_off); 2523 while (*name) { 2524 if (*name == '.') 2525 *name = '_'; 2526 name++; 2527 } 2528 2529 vlen = btf_vlen(t); 2530 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2531 for (j = 0; j < vlen; j++, v++, m++) { 2532 /* order of field assignments is important */ 2533 m->offset = v->offset * 8; 2534 m->type = v->type; 2535 /* preserve variable name as member name */ 2536 vt = (void *)btf__type_by_id(btf, v->type); 2537 m->name_off = vt->name_off; 2538 } 2539 } else if (!has_func && btf_is_func_proto(t)) { 2540 /* replace FUNC_PROTO with ENUM */ 2541 vlen = btf_vlen(t); 2542 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2543 t->size = sizeof(__u32); /* kernel enforced */ 2544 } else if (!has_func && btf_is_func(t)) { 2545 /* replace FUNC with TYPEDEF */ 2546 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2547 } else if (!has_func_global && btf_is_func(t)) { 2548 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2549 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2550 } else if (!has_float && btf_is_float(t)) { 2551 /* replace FLOAT with an equally-sized empty STRUCT; 2552 * since C compilers do not accept e.g. "float" as a 2553 * valid struct name, make it anonymous 2554 */ 2555 t->name_off = 0; 2556 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2557 } 2558 } 2559 } 2560 2561 static bool libbpf_needs_btf(const struct bpf_object *obj) 2562 { 2563 return obj->efile.btf_maps_shndx >= 0 || 2564 obj->efile.st_ops_shndx >= 0 || 2565 obj->nr_extern > 0; 2566 } 2567 2568 static bool kernel_needs_btf(const struct bpf_object *obj) 2569 { 2570 return obj->efile.st_ops_shndx >= 0; 2571 } 2572 2573 static int bpf_object__init_btf(struct bpf_object *obj, 2574 Elf_Data *btf_data, 2575 Elf_Data *btf_ext_data) 2576 { 2577 int err = -ENOENT; 2578 2579 if (btf_data) { 2580 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2581 err = libbpf_get_error(obj->btf); 2582 if (err) { 2583 obj->btf = NULL; 2584 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); 2585 goto out; 2586 } 2587 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2588 btf__set_pointer_size(obj->btf, 8); 2589 } 2590 if (btf_ext_data) { 2591 if (!obj->btf) { 2592 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2593 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2594 goto out; 2595 } 2596 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 2597 err = libbpf_get_error(obj->btf_ext); 2598 if (err) { 2599 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", 2600 BTF_EXT_ELF_SEC, err); 2601 obj->btf_ext = NULL; 2602 goto out; 2603 } 2604 } 2605 out: 2606 if (err && libbpf_needs_btf(obj)) { 2607 pr_warn("BTF is required, but is missing or corrupted.\n"); 2608 return err; 2609 } 2610 return 0; 2611 } 2612 2613 static int bpf_object__finalize_btf(struct bpf_object *obj) 2614 { 2615 int err; 2616 2617 if (!obj->btf) 2618 return 0; 2619 2620 err = btf__finalize_data(obj, obj->btf); 2621 if (err) { 2622 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); 2623 return err; 2624 } 2625 2626 return 0; 2627 } 2628 2629 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 2630 { 2631 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 2632 prog->type == BPF_PROG_TYPE_LSM) 2633 return true; 2634 2635 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 2636 * also need vmlinux BTF 2637 */ 2638 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 2639 return true; 2640 2641 return false; 2642 } 2643 2644 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 2645 { 2646 struct bpf_program *prog; 2647 int i; 2648 2649 /* CO-RE relocations need kernel BTF, only when btf_custom_path 2650 * is not specified 2651 */ 2652 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 2653 return true; 2654 2655 /* Support for typed ksyms needs kernel BTF */ 2656 for (i = 0; i < obj->nr_extern; i++) { 2657 const struct extern_desc *ext; 2658 2659 ext = &obj->externs[i]; 2660 if (ext->type == EXT_KSYM && ext->ksym.type_id) 2661 return true; 2662 } 2663 2664 bpf_object__for_each_program(prog, obj) { 2665 if (!prog->load) 2666 continue; 2667 if (prog_needs_vmlinux_btf(prog)) 2668 return true; 2669 } 2670 2671 return false; 2672 } 2673 2674 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 2675 { 2676 int err; 2677 2678 /* btf_vmlinux could be loaded earlier */ 2679 if (obj->btf_vmlinux || obj->gen_loader) 2680 return 0; 2681 2682 if (!force && !obj_needs_vmlinux_btf(obj)) 2683 return 0; 2684 2685 obj->btf_vmlinux = btf__load_vmlinux_btf(); 2686 err = libbpf_get_error(obj->btf_vmlinux); 2687 if (err) { 2688 pr_warn("Error loading vmlinux BTF: %d\n", err); 2689 obj->btf_vmlinux = NULL; 2690 return err; 2691 } 2692 return 0; 2693 } 2694 2695 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 2696 { 2697 struct btf *kern_btf = obj->btf; 2698 bool btf_mandatory, sanitize; 2699 int i, err = 0; 2700 2701 if (!obj->btf) 2702 return 0; 2703 2704 if (!kernel_supports(obj, FEAT_BTF)) { 2705 if (kernel_needs_btf(obj)) { 2706 err = -EOPNOTSUPP; 2707 goto report; 2708 } 2709 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 2710 return 0; 2711 } 2712 2713 /* Even though some subprogs are global/weak, user might prefer more 2714 * permissive BPF verification process that BPF verifier performs for 2715 * static functions, taking into account more context from the caller 2716 * functions. In such case, they need to mark such subprogs with 2717 * __attribute__((visibility("hidden"))) and libbpf will adjust 2718 * corresponding FUNC BTF type to be marked as static and trigger more 2719 * involved BPF verification process. 2720 */ 2721 for (i = 0; i < obj->nr_programs; i++) { 2722 struct bpf_program *prog = &obj->programs[i]; 2723 struct btf_type *t; 2724 const char *name; 2725 int j, n; 2726 2727 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 2728 continue; 2729 2730 n = btf__get_nr_types(obj->btf); 2731 for (j = 1; j <= n; j++) { 2732 t = btf_type_by_id(obj->btf, j); 2733 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 2734 continue; 2735 2736 name = btf__str_by_offset(obj->btf, t->name_off); 2737 if (strcmp(name, prog->name) != 0) 2738 continue; 2739 2740 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 2741 break; 2742 } 2743 } 2744 2745 sanitize = btf_needs_sanitization(obj); 2746 if (sanitize) { 2747 const void *raw_data; 2748 __u32 sz; 2749 2750 /* clone BTF to sanitize a copy and leave the original intact */ 2751 raw_data = btf__get_raw_data(obj->btf, &sz); 2752 kern_btf = btf__new(raw_data, sz); 2753 err = libbpf_get_error(kern_btf); 2754 if (err) 2755 return err; 2756 2757 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2758 btf__set_pointer_size(obj->btf, 8); 2759 bpf_object__sanitize_btf(obj, kern_btf); 2760 } 2761 2762 if (obj->gen_loader) { 2763 __u32 raw_size = 0; 2764 const void *raw_data = btf__get_raw_data(kern_btf, &raw_size); 2765 2766 if (!raw_data) 2767 return -ENOMEM; 2768 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 2769 /* Pretend to have valid FD to pass various fd >= 0 checks. 2770 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 2771 */ 2772 btf__set_fd(kern_btf, 0); 2773 } else { 2774 err = btf__load_into_kernel(kern_btf); 2775 } 2776 if (sanitize) { 2777 if (!err) { 2778 /* move fd to libbpf's BTF */ 2779 btf__set_fd(obj->btf, btf__fd(kern_btf)); 2780 btf__set_fd(kern_btf, -1); 2781 } 2782 btf__free(kern_btf); 2783 } 2784 report: 2785 if (err) { 2786 btf_mandatory = kernel_needs_btf(obj); 2787 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 2788 btf_mandatory ? "BTF is mandatory, can't proceed." 2789 : "BTF is optional, ignoring."); 2790 if (!btf_mandatory) 2791 err = 0; 2792 } 2793 return err; 2794 } 2795 2796 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 2797 { 2798 const char *name; 2799 2800 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 2801 if (!name) { 2802 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2803 off, obj->path, elf_errmsg(-1)); 2804 return NULL; 2805 } 2806 2807 return name; 2808 } 2809 2810 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 2811 { 2812 const char *name; 2813 2814 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 2815 if (!name) { 2816 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2817 off, obj->path, elf_errmsg(-1)); 2818 return NULL; 2819 } 2820 2821 return name; 2822 } 2823 2824 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 2825 { 2826 Elf_Scn *scn; 2827 2828 scn = elf_getscn(obj->efile.elf, idx); 2829 if (!scn) { 2830 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 2831 idx, obj->path, elf_errmsg(-1)); 2832 return NULL; 2833 } 2834 return scn; 2835 } 2836 2837 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 2838 { 2839 Elf_Scn *scn = NULL; 2840 Elf *elf = obj->efile.elf; 2841 const char *sec_name; 2842 2843 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2844 sec_name = elf_sec_name(obj, scn); 2845 if (!sec_name) 2846 return NULL; 2847 2848 if (strcmp(sec_name, name) != 0) 2849 continue; 2850 2851 return scn; 2852 } 2853 return NULL; 2854 } 2855 2856 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr) 2857 { 2858 if (!scn) 2859 return -EINVAL; 2860 2861 if (gelf_getshdr(scn, hdr) != hdr) { 2862 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 2863 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2864 return -EINVAL; 2865 } 2866 2867 return 0; 2868 } 2869 2870 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 2871 { 2872 const char *name; 2873 GElf_Shdr sh; 2874 2875 if (!scn) 2876 return NULL; 2877 2878 if (elf_sec_hdr(obj, scn, &sh)) 2879 return NULL; 2880 2881 name = elf_sec_str(obj, sh.sh_name); 2882 if (!name) { 2883 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 2884 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2885 return NULL; 2886 } 2887 2888 return name; 2889 } 2890 2891 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 2892 { 2893 Elf_Data *data; 2894 2895 if (!scn) 2896 return NULL; 2897 2898 data = elf_getdata(scn, 0); 2899 if (!data) { 2900 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 2901 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 2902 obj->path, elf_errmsg(-1)); 2903 return NULL; 2904 } 2905 2906 return data; 2907 } 2908 2909 static bool is_sec_name_dwarf(const char *name) 2910 { 2911 /* approximation, but the actual list is too long */ 2912 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 2913 } 2914 2915 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name) 2916 { 2917 /* no special handling of .strtab */ 2918 if (hdr->sh_type == SHT_STRTAB) 2919 return true; 2920 2921 /* ignore .llvm_addrsig section as well */ 2922 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 2923 return true; 2924 2925 /* no subprograms will lead to an empty .text section, ignore it */ 2926 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 2927 strcmp(name, ".text") == 0) 2928 return true; 2929 2930 /* DWARF sections */ 2931 if (is_sec_name_dwarf(name)) 2932 return true; 2933 2934 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 2935 name += sizeof(".rel") - 1; 2936 /* DWARF section relocations */ 2937 if (is_sec_name_dwarf(name)) 2938 return true; 2939 2940 /* .BTF and .BTF.ext don't need relocations */ 2941 if (strcmp(name, BTF_ELF_SEC) == 0 || 2942 strcmp(name, BTF_EXT_ELF_SEC) == 0) 2943 return true; 2944 } 2945 2946 return false; 2947 } 2948 2949 static int cmp_progs(const void *_a, const void *_b) 2950 { 2951 const struct bpf_program *a = _a; 2952 const struct bpf_program *b = _b; 2953 2954 if (a->sec_idx != b->sec_idx) 2955 return a->sec_idx < b->sec_idx ? -1 : 1; 2956 2957 /* sec_insn_off can't be the same within the section */ 2958 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 2959 } 2960 2961 static int bpf_object__elf_collect(struct bpf_object *obj) 2962 { 2963 Elf *elf = obj->efile.elf; 2964 Elf_Data *btf_ext_data = NULL; 2965 Elf_Data *btf_data = NULL; 2966 int idx = 0, err = 0; 2967 const char *name; 2968 Elf_Data *data; 2969 Elf_Scn *scn; 2970 GElf_Shdr sh; 2971 2972 /* a bunch of ELF parsing functionality depends on processing symbols, 2973 * so do the first pass and find the symbol table 2974 */ 2975 scn = NULL; 2976 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2977 if (elf_sec_hdr(obj, scn, &sh)) 2978 return -LIBBPF_ERRNO__FORMAT; 2979 2980 if (sh.sh_type == SHT_SYMTAB) { 2981 if (obj->efile.symbols) { 2982 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 2983 return -LIBBPF_ERRNO__FORMAT; 2984 } 2985 2986 data = elf_sec_data(obj, scn); 2987 if (!data) 2988 return -LIBBPF_ERRNO__FORMAT; 2989 2990 obj->efile.symbols = data; 2991 obj->efile.symbols_shndx = elf_ndxscn(scn); 2992 obj->efile.strtabidx = sh.sh_link; 2993 } 2994 } 2995 2996 scn = NULL; 2997 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2998 idx++; 2999 3000 if (elf_sec_hdr(obj, scn, &sh)) 3001 return -LIBBPF_ERRNO__FORMAT; 3002 3003 name = elf_sec_str(obj, sh.sh_name); 3004 if (!name) 3005 return -LIBBPF_ERRNO__FORMAT; 3006 3007 if (ignore_elf_section(&sh, name)) 3008 continue; 3009 3010 data = elf_sec_data(obj, scn); 3011 if (!data) 3012 return -LIBBPF_ERRNO__FORMAT; 3013 3014 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 3015 idx, name, (unsigned long)data->d_size, 3016 (int)sh.sh_link, (unsigned long)sh.sh_flags, 3017 (int)sh.sh_type); 3018 3019 if (strcmp(name, "license") == 0) { 3020 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 3021 if (err) 3022 return err; 3023 } else if (strcmp(name, "version") == 0) { 3024 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 3025 if (err) 3026 return err; 3027 } else if (strcmp(name, "maps") == 0) { 3028 obj->efile.maps_shndx = idx; 3029 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 3030 obj->efile.btf_maps_shndx = idx; 3031 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 3032 btf_data = data; 3033 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 3034 btf_ext_data = data; 3035 } else if (sh.sh_type == SHT_SYMTAB) { 3036 /* already processed during the first pass above */ 3037 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 3038 if (sh.sh_flags & SHF_EXECINSTR) { 3039 if (strcmp(name, ".text") == 0) 3040 obj->efile.text_shndx = idx; 3041 err = bpf_object__add_programs(obj, data, name, idx); 3042 if (err) 3043 return err; 3044 } else if (strcmp(name, DATA_SEC) == 0) { 3045 obj->efile.data = data; 3046 obj->efile.data_shndx = idx; 3047 } else if (strcmp(name, RODATA_SEC) == 0) { 3048 obj->efile.rodata = data; 3049 obj->efile.rodata_shndx = idx; 3050 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 3051 obj->efile.st_ops_data = data; 3052 obj->efile.st_ops_shndx = idx; 3053 } else { 3054 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3055 idx, name); 3056 } 3057 } else if (sh.sh_type == SHT_REL) { 3058 int nr_sects = obj->efile.nr_reloc_sects; 3059 void *sects = obj->efile.reloc_sects; 3060 int sec = sh.sh_info; /* points to other section */ 3061 3062 /* Only do relo for section with exec instructions */ 3063 if (!section_have_execinstr(obj, sec) && 3064 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3065 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3066 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3067 idx, name, sec, 3068 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>"); 3069 continue; 3070 } 3071 3072 sects = libbpf_reallocarray(sects, nr_sects + 1, 3073 sizeof(*obj->efile.reloc_sects)); 3074 if (!sects) 3075 return -ENOMEM; 3076 3077 obj->efile.reloc_sects = sects; 3078 obj->efile.nr_reloc_sects++; 3079 3080 obj->efile.reloc_sects[nr_sects].shdr = sh; 3081 obj->efile.reloc_sects[nr_sects].data = data; 3082 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) { 3083 obj->efile.bss = data; 3084 obj->efile.bss_shndx = idx; 3085 } else { 3086 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3087 (size_t)sh.sh_size); 3088 } 3089 } 3090 3091 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3092 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3093 return -LIBBPF_ERRNO__FORMAT; 3094 } 3095 3096 /* sort BPF programs by section name and in-section instruction offset 3097 * for faster search */ 3098 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3099 3100 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3101 } 3102 3103 static bool sym_is_extern(const GElf_Sym *sym) 3104 { 3105 int bind = GELF_ST_BIND(sym->st_info); 3106 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3107 return sym->st_shndx == SHN_UNDEF && 3108 (bind == STB_GLOBAL || bind == STB_WEAK) && 3109 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 3110 } 3111 3112 static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) 3113 { 3114 int bind = GELF_ST_BIND(sym->st_info); 3115 int type = GELF_ST_TYPE(sym->st_info); 3116 3117 /* in .text section */ 3118 if (sym->st_shndx != text_shndx) 3119 return false; 3120 3121 /* local function */ 3122 if (bind == STB_LOCAL && type == STT_SECTION) 3123 return true; 3124 3125 /* global function */ 3126 return bind == STB_GLOBAL && type == STT_FUNC; 3127 } 3128 3129 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3130 { 3131 const struct btf_type *t; 3132 const char *tname; 3133 int i, n; 3134 3135 if (!btf) 3136 return -ESRCH; 3137 3138 n = btf__get_nr_types(btf); 3139 for (i = 1; i <= n; i++) { 3140 t = btf__type_by_id(btf, i); 3141 3142 if (!btf_is_var(t) && !btf_is_func(t)) 3143 continue; 3144 3145 tname = btf__name_by_offset(btf, t->name_off); 3146 if (strcmp(tname, ext_name)) 3147 continue; 3148 3149 if (btf_is_var(t) && 3150 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3151 return -EINVAL; 3152 3153 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3154 return -EINVAL; 3155 3156 return i; 3157 } 3158 3159 return -ENOENT; 3160 } 3161 3162 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 3163 const struct btf_var_secinfo *vs; 3164 const struct btf_type *t; 3165 int i, j, n; 3166 3167 if (!btf) 3168 return -ESRCH; 3169 3170 n = btf__get_nr_types(btf); 3171 for (i = 1; i <= n; i++) { 3172 t = btf__type_by_id(btf, i); 3173 3174 if (!btf_is_datasec(t)) 3175 continue; 3176 3177 vs = btf_var_secinfos(t); 3178 for (j = 0; j < btf_vlen(t); j++, vs++) { 3179 if (vs->type == ext_btf_id) 3180 return i; 3181 } 3182 } 3183 3184 return -ENOENT; 3185 } 3186 3187 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 3188 bool *is_signed) 3189 { 3190 const struct btf_type *t; 3191 const char *name; 3192 3193 t = skip_mods_and_typedefs(btf, id, NULL); 3194 name = btf__name_by_offset(btf, t->name_off); 3195 3196 if (is_signed) 3197 *is_signed = false; 3198 switch (btf_kind(t)) { 3199 case BTF_KIND_INT: { 3200 int enc = btf_int_encoding(t); 3201 3202 if (enc & BTF_INT_BOOL) 3203 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 3204 if (is_signed) 3205 *is_signed = enc & BTF_INT_SIGNED; 3206 if (t->size == 1) 3207 return KCFG_CHAR; 3208 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 3209 return KCFG_UNKNOWN; 3210 return KCFG_INT; 3211 } 3212 case BTF_KIND_ENUM: 3213 if (t->size != 4) 3214 return KCFG_UNKNOWN; 3215 if (strcmp(name, "libbpf_tristate")) 3216 return KCFG_UNKNOWN; 3217 return KCFG_TRISTATE; 3218 case BTF_KIND_ARRAY: 3219 if (btf_array(t)->nelems == 0) 3220 return KCFG_UNKNOWN; 3221 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 3222 return KCFG_UNKNOWN; 3223 return KCFG_CHAR_ARR; 3224 default: 3225 return KCFG_UNKNOWN; 3226 } 3227 } 3228 3229 static int cmp_externs(const void *_a, const void *_b) 3230 { 3231 const struct extern_desc *a = _a; 3232 const struct extern_desc *b = _b; 3233 3234 if (a->type != b->type) 3235 return a->type < b->type ? -1 : 1; 3236 3237 if (a->type == EXT_KCFG) { 3238 /* descending order by alignment requirements */ 3239 if (a->kcfg.align != b->kcfg.align) 3240 return a->kcfg.align > b->kcfg.align ? -1 : 1; 3241 /* ascending order by size, within same alignment class */ 3242 if (a->kcfg.sz != b->kcfg.sz) 3243 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 3244 } 3245 3246 /* resolve ties by name */ 3247 return strcmp(a->name, b->name); 3248 } 3249 3250 static int find_int_btf_id(const struct btf *btf) 3251 { 3252 const struct btf_type *t; 3253 int i, n; 3254 3255 n = btf__get_nr_types(btf); 3256 for (i = 1; i <= n; i++) { 3257 t = btf__type_by_id(btf, i); 3258 3259 if (btf_is_int(t) && btf_int_bits(t) == 32) 3260 return i; 3261 } 3262 3263 return 0; 3264 } 3265 3266 static int add_dummy_ksym_var(struct btf *btf) 3267 { 3268 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 3269 const struct btf_var_secinfo *vs; 3270 const struct btf_type *sec; 3271 3272 if (!btf) 3273 return 0; 3274 3275 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 3276 BTF_KIND_DATASEC); 3277 if (sec_btf_id < 0) 3278 return 0; 3279 3280 sec = btf__type_by_id(btf, sec_btf_id); 3281 vs = btf_var_secinfos(sec); 3282 for (i = 0; i < btf_vlen(sec); i++, vs++) { 3283 const struct btf_type *vt; 3284 3285 vt = btf__type_by_id(btf, vs->type); 3286 if (btf_is_func(vt)) 3287 break; 3288 } 3289 3290 /* No func in ksyms sec. No need to add dummy var. */ 3291 if (i == btf_vlen(sec)) 3292 return 0; 3293 3294 int_btf_id = find_int_btf_id(btf); 3295 dummy_var_btf_id = btf__add_var(btf, 3296 "dummy_ksym", 3297 BTF_VAR_GLOBAL_ALLOCATED, 3298 int_btf_id); 3299 if (dummy_var_btf_id < 0) 3300 pr_warn("cannot create a dummy_ksym var\n"); 3301 3302 return dummy_var_btf_id; 3303 } 3304 3305 static int bpf_object__collect_externs(struct bpf_object *obj) 3306 { 3307 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 3308 const struct btf_type *t; 3309 struct extern_desc *ext; 3310 int i, n, off, dummy_var_btf_id; 3311 const char *ext_name, *sec_name; 3312 Elf_Scn *scn; 3313 GElf_Shdr sh; 3314 3315 if (!obj->efile.symbols) 3316 return 0; 3317 3318 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 3319 if (elf_sec_hdr(obj, scn, &sh)) 3320 return -LIBBPF_ERRNO__FORMAT; 3321 3322 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 3323 if (dummy_var_btf_id < 0) 3324 return dummy_var_btf_id; 3325 3326 n = sh.sh_size / sh.sh_entsize; 3327 pr_debug("looking for externs among %d symbols...\n", n); 3328 3329 for (i = 0; i < n; i++) { 3330 GElf_Sym sym; 3331 3332 if (!gelf_getsym(obj->efile.symbols, i, &sym)) 3333 return -LIBBPF_ERRNO__FORMAT; 3334 if (!sym_is_extern(&sym)) 3335 continue; 3336 ext_name = elf_sym_str(obj, sym.st_name); 3337 if (!ext_name || !ext_name[0]) 3338 continue; 3339 3340 ext = obj->externs; 3341 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 3342 if (!ext) 3343 return -ENOMEM; 3344 obj->externs = ext; 3345 ext = &ext[obj->nr_extern]; 3346 memset(ext, 0, sizeof(*ext)); 3347 obj->nr_extern++; 3348 3349 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 3350 if (ext->btf_id <= 0) { 3351 pr_warn("failed to find BTF for extern '%s': %d\n", 3352 ext_name, ext->btf_id); 3353 return ext->btf_id; 3354 } 3355 t = btf__type_by_id(obj->btf, ext->btf_id); 3356 ext->name = btf__name_by_offset(obj->btf, t->name_off); 3357 ext->sym_idx = i; 3358 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; 3359 3360 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 3361 if (ext->sec_btf_id <= 0) { 3362 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 3363 ext_name, ext->btf_id, ext->sec_btf_id); 3364 return ext->sec_btf_id; 3365 } 3366 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 3367 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 3368 3369 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 3370 if (btf_is_func(t)) { 3371 pr_warn("extern function %s is unsupported under %s section\n", 3372 ext->name, KCONFIG_SEC); 3373 return -ENOTSUP; 3374 } 3375 kcfg_sec = sec; 3376 ext->type = EXT_KCFG; 3377 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 3378 if (ext->kcfg.sz <= 0) { 3379 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 3380 ext_name, ext->kcfg.sz); 3381 return ext->kcfg.sz; 3382 } 3383 ext->kcfg.align = btf__align_of(obj->btf, t->type); 3384 if (ext->kcfg.align <= 0) { 3385 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 3386 ext_name, ext->kcfg.align); 3387 return -EINVAL; 3388 } 3389 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 3390 &ext->kcfg.is_signed); 3391 if (ext->kcfg.type == KCFG_UNKNOWN) { 3392 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); 3393 return -ENOTSUP; 3394 } 3395 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 3396 if (btf_is_func(t) && ext->is_weak) { 3397 pr_warn("extern weak function %s is unsupported\n", 3398 ext->name); 3399 return -ENOTSUP; 3400 } 3401 ksym_sec = sec; 3402 ext->type = EXT_KSYM; 3403 skip_mods_and_typedefs(obj->btf, t->type, 3404 &ext->ksym.type_id); 3405 } else { 3406 pr_warn("unrecognized extern section '%s'\n", sec_name); 3407 return -ENOTSUP; 3408 } 3409 } 3410 pr_debug("collected %d externs total\n", obj->nr_extern); 3411 3412 if (!obj->nr_extern) 3413 return 0; 3414 3415 /* sort externs by type, for kcfg ones also by (align, size, name) */ 3416 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 3417 3418 /* for .ksyms section, we need to turn all externs into allocated 3419 * variables in BTF to pass kernel verification; we do this by 3420 * pretending that each extern is a 8-byte variable 3421 */ 3422 if (ksym_sec) { 3423 /* find existing 4-byte integer type in BTF to use for fake 3424 * extern variables in DATASEC 3425 */ 3426 int int_btf_id = find_int_btf_id(obj->btf); 3427 /* For extern function, a dummy_var added earlier 3428 * will be used to replace the vs->type and 3429 * its name string will be used to refill 3430 * the missing param's name. 3431 */ 3432 const struct btf_type *dummy_var; 3433 3434 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 3435 for (i = 0; i < obj->nr_extern; i++) { 3436 ext = &obj->externs[i]; 3437 if (ext->type != EXT_KSYM) 3438 continue; 3439 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 3440 i, ext->sym_idx, ext->name); 3441 } 3442 3443 sec = ksym_sec; 3444 n = btf_vlen(sec); 3445 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 3446 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3447 struct btf_type *vt; 3448 3449 vt = (void *)btf__type_by_id(obj->btf, vs->type); 3450 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 3451 ext = find_extern_by_name(obj, ext_name); 3452 if (!ext) { 3453 pr_warn("failed to find extern definition for BTF %s '%s'\n", 3454 btf_kind_str(vt), ext_name); 3455 return -ESRCH; 3456 } 3457 if (btf_is_func(vt)) { 3458 const struct btf_type *func_proto; 3459 struct btf_param *param; 3460 int j; 3461 3462 func_proto = btf__type_by_id(obj->btf, 3463 vt->type); 3464 param = btf_params(func_proto); 3465 /* Reuse the dummy_var string if the 3466 * func proto does not have param name. 3467 */ 3468 for (j = 0; j < btf_vlen(func_proto); j++) 3469 if (param[j].type && !param[j].name_off) 3470 param[j].name_off = 3471 dummy_var->name_off; 3472 vs->type = dummy_var_btf_id; 3473 vt->info &= ~0xffff; 3474 vt->info |= BTF_FUNC_GLOBAL; 3475 } else { 3476 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3477 vt->type = int_btf_id; 3478 } 3479 vs->offset = off; 3480 vs->size = sizeof(int); 3481 } 3482 sec->size = off; 3483 } 3484 3485 if (kcfg_sec) { 3486 sec = kcfg_sec; 3487 /* for kcfg externs calculate their offsets within a .kconfig map */ 3488 off = 0; 3489 for (i = 0; i < obj->nr_extern; i++) { 3490 ext = &obj->externs[i]; 3491 if (ext->type != EXT_KCFG) 3492 continue; 3493 3494 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 3495 off = ext->kcfg.data_off + ext->kcfg.sz; 3496 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 3497 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 3498 } 3499 sec->size = off; 3500 n = btf_vlen(sec); 3501 for (i = 0; i < n; i++) { 3502 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3503 3504 t = btf__type_by_id(obj->btf, vs->type); 3505 ext_name = btf__name_by_offset(obj->btf, t->name_off); 3506 ext = find_extern_by_name(obj, ext_name); 3507 if (!ext) { 3508 pr_warn("failed to find extern definition for BTF var '%s'\n", 3509 ext_name); 3510 return -ESRCH; 3511 } 3512 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3513 vs->offset = ext->kcfg.data_off; 3514 } 3515 } 3516 return 0; 3517 } 3518 3519 struct bpf_program * 3520 bpf_object__find_program_by_title(const struct bpf_object *obj, 3521 const char *title) 3522 { 3523 struct bpf_program *pos; 3524 3525 bpf_object__for_each_program(pos, obj) { 3526 if (pos->sec_name && !strcmp(pos->sec_name, title)) 3527 return pos; 3528 } 3529 return errno = ENOENT, NULL; 3530 } 3531 3532 static bool prog_is_subprog(const struct bpf_object *obj, 3533 const struct bpf_program *prog) 3534 { 3535 /* For legacy reasons, libbpf supports an entry-point BPF programs 3536 * without SEC() attribute, i.e., those in the .text section. But if 3537 * there are 2 or more such programs in the .text section, they all 3538 * must be subprograms called from entry-point BPF programs in 3539 * designated SEC()'tions, otherwise there is no way to distinguish 3540 * which of those programs should be loaded vs which are a subprogram. 3541 * Similarly, if there is a function/program in .text and at least one 3542 * other BPF program with custom SEC() attribute, then we just assume 3543 * .text programs are subprograms (even if they are not called from 3544 * other programs), because libbpf never explicitly supported mixing 3545 * SEC()-designated BPF programs and .text entry-point BPF programs. 3546 */ 3547 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 3548 } 3549 3550 struct bpf_program * 3551 bpf_object__find_program_by_name(const struct bpf_object *obj, 3552 const char *name) 3553 { 3554 struct bpf_program *prog; 3555 3556 bpf_object__for_each_program(prog, obj) { 3557 if (prog_is_subprog(obj, prog)) 3558 continue; 3559 if (!strcmp(prog->name, name)) 3560 return prog; 3561 } 3562 return errno = ENOENT, NULL; 3563 } 3564 3565 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 3566 int shndx) 3567 { 3568 return shndx == obj->efile.data_shndx || 3569 shndx == obj->efile.bss_shndx || 3570 shndx == obj->efile.rodata_shndx; 3571 } 3572 3573 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 3574 int shndx) 3575 { 3576 return shndx == obj->efile.maps_shndx || 3577 shndx == obj->efile.btf_maps_shndx; 3578 } 3579 3580 static enum libbpf_map_type 3581 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 3582 { 3583 if (shndx == obj->efile.data_shndx) 3584 return LIBBPF_MAP_DATA; 3585 else if (shndx == obj->efile.bss_shndx) 3586 return LIBBPF_MAP_BSS; 3587 else if (shndx == obj->efile.rodata_shndx) 3588 return LIBBPF_MAP_RODATA; 3589 else if (shndx == obj->efile.symbols_shndx) 3590 return LIBBPF_MAP_KCONFIG; 3591 else 3592 return LIBBPF_MAP_UNSPEC; 3593 } 3594 3595 static int bpf_program__record_reloc(struct bpf_program *prog, 3596 struct reloc_desc *reloc_desc, 3597 __u32 insn_idx, const char *sym_name, 3598 const GElf_Sym *sym, const GElf_Rel *rel) 3599 { 3600 struct bpf_insn *insn = &prog->insns[insn_idx]; 3601 size_t map_idx, nr_maps = prog->obj->nr_maps; 3602 struct bpf_object *obj = prog->obj; 3603 __u32 shdr_idx = sym->st_shndx; 3604 enum libbpf_map_type type; 3605 const char *sym_sec_name; 3606 struct bpf_map *map; 3607 3608 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 3609 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 3610 prog->name, sym_name, insn_idx, insn->code); 3611 return -LIBBPF_ERRNO__RELOC; 3612 } 3613 3614 if (sym_is_extern(sym)) { 3615 int sym_idx = GELF_R_SYM(rel->r_info); 3616 int i, n = obj->nr_extern; 3617 struct extern_desc *ext; 3618 3619 for (i = 0; i < n; i++) { 3620 ext = &obj->externs[i]; 3621 if (ext->sym_idx == sym_idx) 3622 break; 3623 } 3624 if (i >= n) { 3625 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 3626 prog->name, sym_name, sym_idx); 3627 return -LIBBPF_ERRNO__RELOC; 3628 } 3629 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 3630 prog->name, i, ext->name, ext->sym_idx, insn_idx); 3631 if (insn->code == (BPF_JMP | BPF_CALL)) 3632 reloc_desc->type = RELO_EXTERN_FUNC; 3633 else 3634 reloc_desc->type = RELO_EXTERN_VAR; 3635 reloc_desc->insn_idx = insn_idx; 3636 reloc_desc->sym_off = i; /* sym_off stores extern index */ 3637 return 0; 3638 } 3639 3640 /* sub-program call relocation */ 3641 if (is_call_insn(insn)) { 3642 if (insn->src_reg != BPF_PSEUDO_CALL) { 3643 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 3644 return -LIBBPF_ERRNO__RELOC; 3645 } 3646 /* text_shndx can be 0, if no default "main" program exists */ 3647 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 3648 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3649 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 3650 prog->name, sym_name, sym_sec_name); 3651 return -LIBBPF_ERRNO__RELOC; 3652 } 3653 if (sym->st_value % BPF_INSN_SZ) { 3654 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 3655 prog->name, sym_name, (size_t)sym->st_value); 3656 return -LIBBPF_ERRNO__RELOC; 3657 } 3658 reloc_desc->type = RELO_CALL; 3659 reloc_desc->insn_idx = insn_idx; 3660 reloc_desc->sym_off = sym->st_value; 3661 return 0; 3662 } 3663 3664 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 3665 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 3666 prog->name, sym_name, shdr_idx); 3667 return -LIBBPF_ERRNO__RELOC; 3668 } 3669 3670 /* loading subprog addresses */ 3671 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 3672 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 3673 * local_func: sym->st_value = 0, insn->imm = offset in the section. 3674 */ 3675 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 3676 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 3677 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 3678 return -LIBBPF_ERRNO__RELOC; 3679 } 3680 3681 reloc_desc->type = RELO_SUBPROG_ADDR; 3682 reloc_desc->insn_idx = insn_idx; 3683 reloc_desc->sym_off = sym->st_value; 3684 return 0; 3685 } 3686 3687 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 3688 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3689 3690 /* generic map reference relocation */ 3691 if (type == LIBBPF_MAP_UNSPEC) { 3692 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 3693 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 3694 prog->name, sym_name, sym_sec_name); 3695 return -LIBBPF_ERRNO__RELOC; 3696 } 3697 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3698 map = &obj->maps[map_idx]; 3699 if (map->libbpf_type != type || 3700 map->sec_idx != sym->st_shndx || 3701 map->sec_offset != sym->st_value) 3702 continue; 3703 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 3704 prog->name, map_idx, map->name, map->sec_idx, 3705 map->sec_offset, insn_idx); 3706 break; 3707 } 3708 if (map_idx >= nr_maps) { 3709 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 3710 prog->name, sym_sec_name, (size_t)sym->st_value); 3711 return -LIBBPF_ERRNO__RELOC; 3712 } 3713 reloc_desc->type = RELO_LD64; 3714 reloc_desc->insn_idx = insn_idx; 3715 reloc_desc->map_idx = map_idx; 3716 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 3717 return 0; 3718 } 3719 3720 /* global data map relocation */ 3721 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 3722 pr_warn("prog '%s': bad data relo against section '%s'\n", 3723 prog->name, sym_sec_name); 3724 return -LIBBPF_ERRNO__RELOC; 3725 } 3726 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3727 map = &obj->maps[map_idx]; 3728 if (map->libbpf_type != type) 3729 continue; 3730 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 3731 prog->name, map_idx, map->name, map->sec_idx, 3732 map->sec_offset, insn_idx); 3733 break; 3734 } 3735 if (map_idx >= nr_maps) { 3736 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 3737 prog->name, sym_sec_name); 3738 return -LIBBPF_ERRNO__RELOC; 3739 } 3740 3741 reloc_desc->type = RELO_DATA; 3742 reloc_desc->insn_idx = insn_idx; 3743 reloc_desc->map_idx = map_idx; 3744 reloc_desc->sym_off = sym->st_value; 3745 return 0; 3746 } 3747 3748 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 3749 { 3750 return insn_idx >= prog->sec_insn_off && 3751 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 3752 } 3753 3754 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 3755 size_t sec_idx, size_t insn_idx) 3756 { 3757 int l = 0, r = obj->nr_programs - 1, m; 3758 struct bpf_program *prog; 3759 3760 while (l < r) { 3761 m = l + (r - l + 1) / 2; 3762 prog = &obj->programs[m]; 3763 3764 if (prog->sec_idx < sec_idx || 3765 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 3766 l = m; 3767 else 3768 r = m - 1; 3769 } 3770 /* matching program could be at index l, but it still might be the 3771 * wrong one, so we need to double check conditions for the last time 3772 */ 3773 prog = &obj->programs[l]; 3774 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 3775 return prog; 3776 return NULL; 3777 } 3778 3779 static int 3780 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data) 3781 { 3782 Elf_Data *symbols = obj->efile.symbols; 3783 const char *relo_sec_name, *sec_name; 3784 size_t sec_idx = shdr->sh_info; 3785 struct bpf_program *prog; 3786 struct reloc_desc *relos; 3787 int err, i, nrels; 3788 const char *sym_name; 3789 __u32 insn_idx; 3790 Elf_Scn *scn; 3791 Elf_Data *scn_data; 3792 GElf_Sym sym; 3793 GElf_Rel rel; 3794 3795 scn = elf_sec_by_idx(obj, sec_idx); 3796 scn_data = elf_sec_data(obj, scn); 3797 3798 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 3799 sec_name = elf_sec_name(obj, scn); 3800 if (!relo_sec_name || !sec_name) 3801 return -EINVAL; 3802 3803 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 3804 relo_sec_name, sec_idx, sec_name); 3805 nrels = shdr->sh_size / shdr->sh_entsize; 3806 3807 for (i = 0; i < nrels; i++) { 3808 if (!gelf_getrel(data, i, &rel)) { 3809 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 3810 return -LIBBPF_ERRNO__FORMAT; 3811 } 3812 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 3813 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n", 3814 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3815 return -LIBBPF_ERRNO__FORMAT; 3816 } 3817 3818 if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) { 3819 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 3820 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3821 return -LIBBPF_ERRNO__FORMAT; 3822 } 3823 3824 insn_idx = rel.r_offset / BPF_INSN_SZ; 3825 /* relocations against static functions are recorded as 3826 * relocations against the section that contains a function; 3827 * in such case, symbol will be STT_SECTION and sym.st_name 3828 * will point to empty string (0), so fetch section name 3829 * instead 3830 */ 3831 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0) 3832 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx)); 3833 else 3834 sym_name = elf_sym_str(obj, sym.st_name); 3835 sym_name = sym_name ?: "<?"; 3836 3837 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 3838 relo_sec_name, i, insn_idx, sym_name); 3839 3840 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 3841 if (!prog) { 3842 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 3843 relo_sec_name, i, sec_name, insn_idx); 3844 continue; 3845 } 3846 3847 relos = libbpf_reallocarray(prog->reloc_desc, 3848 prog->nr_reloc + 1, sizeof(*relos)); 3849 if (!relos) 3850 return -ENOMEM; 3851 prog->reloc_desc = relos; 3852 3853 /* adjust insn_idx to local BPF program frame of reference */ 3854 insn_idx -= prog->sec_insn_off; 3855 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 3856 insn_idx, sym_name, &sym, &rel); 3857 if (err) 3858 return err; 3859 3860 prog->nr_reloc++; 3861 } 3862 return 0; 3863 } 3864 3865 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 3866 { 3867 struct bpf_map_def *def = &map->def; 3868 __u32 key_type_id = 0, value_type_id = 0; 3869 int ret; 3870 3871 /* if it's BTF-defined map, we don't need to search for type IDs. 3872 * For struct_ops map, it does not need btf_key_type_id and 3873 * btf_value_type_id. 3874 */ 3875 if (map->sec_idx == obj->efile.btf_maps_shndx || 3876 bpf_map__is_struct_ops(map)) 3877 return 0; 3878 3879 if (!bpf_map__is_internal(map)) { 3880 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 3881 def->value_size, &key_type_id, 3882 &value_type_id); 3883 } else { 3884 /* 3885 * LLVM annotates global data differently in BTF, that is, 3886 * only as '.data', '.bss' or '.rodata'. 3887 */ 3888 ret = btf__find_by_name(obj->btf, 3889 libbpf_type_to_btf_name[map->libbpf_type]); 3890 } 3891 if (ret < 0) 3892 return ret; 3893 3894 map->btf_key_type_id = key_type_id; 3895 map->btf_value_type_id = bpf_map__is_internal(map) ? 3896 ret : value_type_id; 3897 return 0; 3898 } 3899 3900 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 3901 { 3902 char file[PATH_MAX], buff[4096]; 3903 FILE *fp; 3904 __u32 val; 3905 int err; 3906 3907 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 3908 memset(info, 0, sizeof(*info)); 3909 3910 fp = fopen(file, "r"); 3911 if (!fp) { 3912 err = -errno; 3913 pr_warn("failed to open %s: %d. No procfs support?\n", file, 3914 err); 3915 return err; 3916 } 3917 3918 while (fgets(buff, sizeof(buff), fp)) { 3919 if (sscanf(buff, "map_type:\t%u", &val) == 1) 3920 info->type = val; 3921 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 3922 info->key_size = val; 3923 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 3924 info->value_size = val; 3925 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 3926 info->max_entries = val; 3927 else if (sscanf(buff, "map_flags:\t%i", &val) == 1) 3928 info->map_flags = val; 3929 } 3930 3931 fclose(fp); 3932 3933 return 0; 3934 } 3935 3936 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 3937 { 3938 struct bpf_map_info info = {}; 3939 __u32 len = sizeof(info); 3940 int new_fd, err; 3941 char *new_name; 3942 3943 err = bpf_obj_get_info_by_fd(fd, &info, &len); 3944 if (err && errno == EINVAL) 3945 err = bpf_get_map_info_from_fdinfo(fd, &info); 3946 if (err) 3947 return libbpf_err(err); 3948 3949 new_name = strdup(info.name); 3950 if (!new_name) 3951 return libbpf_err(-errno); 3952 3953 new_fd = open("/", O_RDONLY | O_CLOEXEC); 3954 if (new_fd < 0) { 3955 err = -errno; 3956 goto err_free_new_name; 3957 } 3958 3959 new_fd = dup3(fd, new_fd, O_CLOEXEC); 3960 if (new_fd < 0) { 3961 err = -errno; 3962 goto err_close_new_fd; 3963 } 3964 3965 err = zclose(map->fd); 3966 if (err) { 3967 err = -errno; 3968 goto err_close_new_fd; 3969 } 3970 free(map->name); 3971 3972 map->fd = new_fd; 3973 map->name = new_name; 3974 map->def.type = info.type; 3975 map->def.key_size = info.key_size; 3976 map->def.value_size = info.value_size; 3977 map->def.max_entries = info.max_entries; 3978 map->def.map_flags = info.map_flags; 3979 map->btf_key_type_id = info.btf_key_type_id; 3980 map->btf_value_type_id = info.btf_value_type_id; 3981 map->reused = true; 3982 3983 return 0; 3984 3985 err_close_new_fd: 3986 close(new_fd); 3987 err_free_new_name: 3988 free(new_name); 3989 return libbpf_err(err); 3990 } 3991 3992 __u32 bpf_map__max_entries(const struct bpf_map *map) 3993 { 3994 return map->def.max_entries; 3995 } 3996 3997 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 3998 { 3999 if (!bpf_map_type__is_map_in_map(map->def.type)) 4000 return errno = EINVAL, NULL; 4001 4002 return map->inner_map; 4003 } 4004 4005 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 4006 { 4007 if (map->fd >= 0) 4008 return libbpf_err(-EBUSY); 4009 map->def.max_entries = max_entries; 4010 return 0; 4011 } 4012 4013 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 4014 { 4015 if (!map || !max_entries) 4016 return libbpf_err(-EINVAL); 4017 4018 return bpf_map__set_max_entries(map, max_entries); 4019 } 4020 4021 static int 4022 bpf_object__probe_loading(struct bpf_object *obj) 4023 { 4024 struct bpf_load_program_attr attr; 4025 char *cp, errmsg[STRERR_BUFSIZE]; 4026 struct bpf_insn insns[] = { 4027 BPF_MOV64_IMM(BPF_REG_0, 0), 4028 BPF_EXIT_INSN(), 4029 }; 4030 int ret; 4031 4032 if (obj->gen_loader) 4033 return 0; 4034 4035 /* make sure basic loading works */ 4036 4037 memset(&attr, 0, sizeof(attr)); 4038 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4039 attr.insns = insns; 4040 attr.insns_cnt = ARRAY_SIZE(insns); 4041 attr.license = "GPL"; 4042 4043 ret = bpf_load_program_xattr(&attr, NULL, 0); 4044 if (ret < 0) { 4045 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; 4046 ret = bpf_load_program_xattr(&attr, NULL, 0); 4047 } 4048 if (ret < 0) { 4049 ret = errno; 4050 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4051 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 4052 "program. Make sure your kernel supports BPF " 4053 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 4054 "set to big enough value.\n", __func__, cp, ret); 4055 return -ret; 4056 } 4057 close(ret); 4058 4059 return 0; 4060 } 4061 4062 static int probe_fd(int fd) 4063 { 4064 if (fd >= 0) 4065 close(fd); 4066 return fd >= 0; 4067 } 4068 4069 static int probe_kern_prog_name(void) 4070 { 4071 struct bpf_load_program_attr attr; 4072 struct bpf_insn insns[] = { 4073 BPF_MOV64_IMM(BPF_REG_0, 0), 4074 BPF_EXIT_INSN(), 4075 }; 4076 int ret; 4077 4078 /* make sure loading with name works */ 4079 4080 memset(&attr, 0, sizeof(attr)); 4081 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4082 attr.insns = insns; 4083 attr.insns_cnt = ARRAY_SIZE(insns); 4084 attr.license = "GPL"; 4085 attr.name = "test"; 4086 ret = bpf_load_program_xattr(&attr, NULL, 0); 4087 return probe_fd(ret); 4088 } 4089 4090 static int probe_kern_global_data(void) 4091 { 4092 struct bpf_load_program_attr prg_attr; 4093 struct bpf_create_map_attr map_attr; 4094 char *cp, errmsg[STRERR_BUFSIZE]; 4095 struct bpf_insn insns[] = { 4096 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 4097 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 4098 BPF_MOV64_IMM(BPF_REG_0, 0), 4099 BPF_EXIT_INSN(), 4100 }; 4101 int ret, map; 4102 4103 memset(&map_attr, 0, sizeof(map_attr)); 4104 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4105 map_attr.key_size = sizeof(int); 4106 map_attr.value_size = 32; 4107 map_attr.max_entries = 1; 4108 4109 map = bpf_create_map_xattr(&map_attr); 4110 if (map < 0) { 4111 ret = -errno; 4112 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4113 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4114 __func__, cp, -ret); 4115 return ret; 4116 } 4117 4118 insns[0].imm = map; 4119 4120 memset(&prg_attr, 0, sizeof(prg_attr)); 4121 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4122 prg_attr.insns = insns; 4123 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4124 prg_attr.license = "GPL"; 4125 4126 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 4127 close(map); 4128 return probe_fd(ret); 4129 } 4130 4131 static int probe_kern_btf(void) 4132 { 4133 static const char strs[] = "\0int"; 4134 __u32 types[] = { 4135 /* int */ 4136 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4137 }; 4138 4139 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4140 strs, sizeof(strs))); 4141 } 4142 4143 static int probe_kern_btf_func(void) 4144 { 4145 static const char strs[] = "\0int\0x\0a"; 4146 /* void x(int a) {} */ 4147 __u32 types[] = { 4148 /* int */ 4149 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4150 /* FUNC_PROTO */ /* [2] */ 4151 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4152 BTF_PARAM_ENC(7, 1), 4153 /* FUNC x */ /* [3] */ 4154 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 4155 }; 4156 4157 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4158 strs, sizeof(strs))); 4159 } 4160 4161 static int probe_kern_btf_func_global(void) 4162 { 4163 static const char strs[] = "\0int\0x\0a"; 4164 /* static void x(int a) {} */ 4165 __u32 types[] = { 4166 /* int */ 4167 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4168 /* FUNC_PROTO */ /* [2] */ 4169 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4170 BTF_PARAM_ENC(7, 1), 4171 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 4172 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 4173 }; 4174 4175 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4176 strs, sizeof(strs))); 4177 } 4178 4179 static int probe_kern_btf_datasec(void) 4180 { 4181 static const char strs[] = "\0x\0.data"; 4182 /* static int a; */ 4183 __u32 types[] = { 4184 /* int */ 4185 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4186 /* VAR x */ /* [2] */ 4187 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4188 BTF_VAR_STATIC, 4189 /* DATASEC val */ /* [3] */ 4190 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 4191 BTF_VAR_SECINFO_ENC(2, 0, 4), 4192 }; 4193 4194 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4195 strs, sizeof(strs))); 4196 } 4197 4198 static int probe_kern_btf_float(void) 4199 { 4200 static const char strs[] = "\0float"; 4201 __u32 types[] = { 4202 /* float */ 4203 BTF_TYPE_FLOAT_ENC(1, 4), 4204 }; 4205 4206 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4207 strs, sizeof(strs))); 4208 } 4209 4210 static int probe_kern_array_mmap(void) 4211 { 4212 struct bpf_create_map_attr attr = { 4213 .map_type = BPF_MAP_TYPE_ARRAY, 4214 .map_flags = BPF_F_MMAPABLE, 4215 .key_size = sizeof(int), 4216 .value_size = sizeof(int), 4217 .max_entries = 1, 4218 }; 4219 4220 return probe_fd(bpf_create_map_xattr(&attr)); 4221 } 4222 4223 static int probe_kern_exp_attach_type(void) 4224 { 4225 struct bpf_load_program_attr attr; 4226 struct bpf_insn insns[] = { 4227 BPF_MOV64_IMM(BPF_REG_0, 0), 4228 BPF_EXIT_INSN(), 4229 }; 4230 4231 memset(&attr, 0, sizeof(attr)); 4232 /* use any valid combination of program type and (optional) 4233 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 4234 * to see if kernel supports expected_attach_type field for 4235 * BPF_PROG_LOAD command 4236 */ 4237 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 4238 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; 4239 attr.insns = insns; 4240 attr.insns_cnt = ARRAY_SIZE(insns); 4241 attr.license = "GPL"; 4242 4243 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4244 } 4245 4246 static int probe_kern_probe_read_kernel(void) 4247 { 4248 struct bpf_load_program_attr attr; 4249 struct bpf_insn insns[] = { 4250 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 4251 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 4252 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 4253 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 4254 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 4255 BPF_EXIT_INSN(), 4256 }; 4257 4258 memset(&attr, 0, sizeof(attr)); 4259 attr.prog_type = BPF_PROG_TYPE_KPROBE; 4260 attr.insns = insns; 4261 attr.insns_cnt = ARRAY_SIZE(insns); 4262 attr.license = "GPL"; 4263 4264 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4265 } 4266 4267 static int probe_prog_bind_map(void) 4268 { 4269 struct bpf_load_program_attr prg_attr; 4270 struct bpf_create_map_attr map_attr; 4271 char *cp, errmsg[STRERR_BUFSIZE]; 4272 struct bpf_insn insns[] = { 4273 BPF_MOV64_IMM(BPF_REG_0, 0), 4274 BPF_EXIT_INSN(), 4275 }; 4276 int ret, map, prog; 4277 4278 memset(&map_attr, 0, sizeof(map_attr)); 4279 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4280 map_attr.key_size = sizeof(int); 4281 map_attr.value_size = 32; 4282 map_attr.max_entries = 1; 4283 4284 map = bpf_create_map_xattr(&map_attr); 4285 if (map < 0) { 4286 ret = -errno; 4287 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4288 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4289 __func__, cp, -ret); 4290 return ret; 4291 } 4292 4293 memset(&prg_attr, 0, sizeof(prg_attr)); 4294 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4295 prg_attr.insns = insns; 4296 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4297 prg_attr.license = "GPL"; 4298 4299 prog = bpf_load_program_xattr(&prg_attr, NULL, 0); 4300 if (prog < 0) { 4301 close(map); 4302 return 0; 4303 } 4304 4305 ret = bpf_prog_bind_map(prog, map, NULL); 4306 4307 close(map); 4308 close(prog); 4309 4310 return ret >= 0; 4311 } 4312 4313 static int probe_module_btf(void) 4314 { 4315 static const char strs[] = "\0int"; 4316 __u32 types[] = { 4317 /* int */ 4318 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4319 }; 4320 struct bpf_btf_info info; 4321 __u32 len = sizeof(info); 4322 char name[16]; 4323 int fd, err; 4324 4325 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); 4326 if (fd < 0) 4327 return 0; /* BTF not supported at all */ 4328 4329 memset(&info, 0, sizeof(info)); 4330 info.name = ptr_to_u64(name); 4331 info.name_len = sizeof(name); 4332 4333 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 4334 * kernel's module BTF support coincides with support for 4335 * name/name_len fields in struct bpf_btf_info. 4336 */ 4337 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4338 close(fd); 4339 return !err; 4340 } 4341 4342 static int probe_perf_link(void) 4343 { 4344 struct bpf_load_program_attr attr; 4345 struct bpf_insn insns[] = { 4346 BPF_MOV64_IMM(BPF_REG_0, 0), 4347 BPF_EXIT_INSN(), 4348 }; 4349 int prog_fd, link_fd, err; 4350 4351 memset(&attr, 0, sizeof(attr)); 4352 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; 4353 attr.insns = insns; 4354 attr.insns_cnt = ARRAY_SIZE(insns); 4355 attr.license = "GPL"; 4356 prog_fd = bpf_load_program_xattr(&attr, NULL, 0); 4357 if (prog_fd < 0) 4358 return -errno; 4359 4360 /* use invalid perf_event FD to get EBADF, if link is supported; 4361 * otherwise EINVAL should be returned 4362 */ 4363 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); 4364 err = -errno; /* close() can clobber errno */ 4365 4366 if (link_fd >= 0) 4367 close(link_fd); 4368 close(prog_fd); 4369 4370 return link_fd < 0 && err == -EBADF; 4371 } 4372 4373 enum kern_feature_result { 4374 FEAT_UNKNOWN = 0, 4375 FEAT_SUPPORTED = 1, 4376 FEAT_MISSING = 2, 4377 }; 4378 4379 typedef int (*feature_probe_fn)(void); 4380 4381 static struct kern_feature_desc { 4382 const char *desc; 4383 feature_probe_fn probe; 4384 enum kern_feature_result res; 4385 } feature_probes[__FEAT_CNT] = { 4386 [FEAT_PROG_NAME] = { 4387 "BPF program name", probe_kern_prog_name, 4388 }, 4389 [FEAT_GLOBAL_DATA] = { 4390 "global variables", probe_kern_global_data, 4391 }, 4392 [FEAT_BTF] = { 4393 "minimal BTF", probe_kern_btf, 4394 }, 4395 [FEAT_BTF_FUNC] = { 4396 "BTF functions", probe_kern_btf_func, 4397 }, 4398 [FEAT_BTF_GLOBAL_FUNC] = { 4399 "BTF global function", probe_kern_btf_func_global, 4400 }, 4401 [FEAT_BTF_DATASEC] = { 4402 "BTF data section and variable", probe_kern_btf_datasec, 4403 }, 4404 [FEAT_ARRAY_MMAP] = { 4405 "ARRAY map mmap()", probe_kern_array_mmap, 4406 }, 4407 [FEAT_EXP_ATTACH_TYPE] = { 4408 "BPF_PROG_LOAD expected_attach_type attribute", 4409 probe_kern_exp_attach_type, 4410 }, 4411 [FEAT_PROBE_READ_KERN] = { 4412 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 4413 }, 4414 [FEAT_PROG_BIND_MAP] = { 4415 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 4416 }, 4417 [FEAT_MODULE_BTF] = { 4418 "module BTF support", probe_module_btf, 4419 }, 4420 [FEAT_BTF_FLOAT] = { 4421 "BTF_KIND_FLOAT support", probe_kern_btf_float, 4422 }, 4423 [FEAT_PERF_LINK] = { 4424 "BPF perf link support", probe_perf_link, 4425 }, 4426 }; 4427 4428 static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 4429 { 4430 struct kern_feature_desc *feat = &feature_probes[feat_id]; 4431 int ret; 4432 4433 if (obj->gen_loader) 4434 /* To generate loader program assume the latest kernel 4435 * to avoid doing extra prog_load, map_create syscalls. 4436 */ 4437 return true; 4438 4439 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { 4440 ret = feat->probe(); 4441 if (ret > 0) { 4442 WRITE_ONCE(feat->res, FEAT_SUPPORTED); 4443 } else if (ret == 0) { 4444 WRITE_ONCE(feat->res, FEAT_MISSING); 4445 } else { 4446 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); 4447 WRITE_ONCE(feat->res, FEAT_MISSING); 4448 } 4449 } 4450 4451 return READ_ONCE(feat->res) == FEAT_SUPPORTED; 4452 } 4453 4454 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 4455 { 4456 struct bpf_map_info map_info = {}; 4457 char msg[STRERR_BUFSIZE]; 4458 __u32 map_info_len; 4459 int err; 4460 4461 map_info_len = sizeof(map_info); 4462 4463 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len); 4464 if (err && errno == EINVAL) 4465 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 4466 if (err) { 4467 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 4468 libbpf_strerror_r(errno, msg, sizeof(msg))); 4469 return false; 4470 } 4471 4472 return (map_info.type == map->def.type && 4473 map_info.key_size == map->def.key_size && 4474 map_info.value_size == map->def.value_size && 4475 map_info.max_entries == map->def.max_entries && 4476 map_info.map_flags == map->def.map_flags); 4477 } 4478 4479 static int 4480 bpf_object__reuse_map(struct bpf_map *map) 4481 { 4482 char *cp, errmsg[STRERR_BUFSIZE]; 4483 int err, pin_fd; 4484 4485 pin_fd = bpf_obj_get(map->pin_path); 4486 if (pin_fd < 0) { 4487 err = -errno; 4488 if (err == -ENOENT) { 4489 pr_debug("found no pinned map to reuse at '%s'\n", 4490 map->pin_path); 4491 return 0; 4492 } 4493 4494 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 4495 pr_warn("couldn't retrieve pinned map '%s': %s\n", 4496 map->pin_path, cp); 4497 return err; 4498 } 4499 4500 if (!map_is_reuse_compat(map, pin_fd)) { 4501 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 4502 map->pin_path); 4503 close(pin_fd); 4504 return -EINVAL; 4505 } 4506 4507 err = bpf_map__reuse_fd(map, pin_fd); 4508 if (err) { 4509 close(pin_fd); 4510 return err; 4511 } 4512 map->pinned = true; 4513 pr_debug("reused pinned map at '%s'\n", map->pin_path); 4514 4515 return 0; 4516 } 4517 4518 static int 4519 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 4520 { 4521 enum libbpf_map_type map_type = map->libbpf_type; 4522 char *cp, errmsg[STRERR_BUFSIZE]; 4523 int err, zero = 0; 4524 4525 if (obj->gen_loader) { 4526 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 4527 map->mmaped, map->def.value_size); 4528 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 4529 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 4530 return 0; 4531 } 4532 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 4533 if (err) { 4534 err = -errno; 4535 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4536 pr_warn("Error setting initial map(%s) contents: %s\n", 4537 map->name, cp); 4538 return err; 4539 } 4540 4541 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 4542 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 4543 err = bpf_map_freeze(map->fd); 4544 if (err) { 4545 err = -errno; 4546 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4547 pr_warn("Error freezing map(%s) as read-only: %s\n", 4548 map->name, cp); 4549 return err; 4550 } 4551 } 4552 return 0; 4553 } 4554 4555 static void bpf_map__destroy(struct bpf_map *map); 4556 4557 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 4558 { 4559 struct bpf_create_map_attr create_attr; 4560 struct bpf_map_def *def = &map->def; 4561 int err = 0; 4562 4563 memset(&create_attr, 0, sizeof(create_attr)); 4564 4565 if (kernel_supports(obj, FEAT_PROG_NAME)) 4566 create_attr.name = map->name; 4567 create_attr.map_ifindex = map->map_ifindex; 4568 create_attr.map_type = def->type; 4569 create_attr.map_flags = def->map_flags; 4570 create_attr.key_size = def->key_size; 4571 create_attr.value_size = def->value_size; 4572 create_attr.numa_node = map->numa_node; 4573 4574 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) { 4575 int nr_cpus; 4576 4577 nr_cpus = libbpf_num_possible_cpus(); 4578 if (nr_cpus < 0) { 4579 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 4580 map->name, nr_cpus); 4581 return nr_cpus; 4582 } 4583 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 4584 create_attr.max_entries = nr_cpus; 4585 } else { 4586 create_attr.max_entries = def->max_entries; 4587 } 4588 4589 if (bpf_map__is_struct_ops(map)) 4590 create_attr.btf_vmlinux_value_type_id = 4591 map->btf_vmlinux_value_type_id; 4592 4593 create_attr.btf_fd = 0; 4594 create_attr.btf_key_type_id = 0; 4595 create_attr.btf_value_type_id = 0; 4596 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) { 4597 create_attr.btf_fd = btf__fd(obj->btf); 4598 create_attr.btf_key_type_id = map->btf_key_type_id; 4599 create_attr.btf_value_type_id = map->btf_value_type_id; 4600 } 4601 4602 if (bpf_map_type__is_map_in_map(def->type)) { 4603 if (map->inner_map) { 4604 err = bpf_object__create_map(obj, map->inner_map, true); 4605 if (err) { 4606 pr_warn("map '%s': failed to create inner map: %d\n", 4607 map->name, err); 4608 return err; 4609 } 4610 map->inner_map_fd = bpf_map__fd(map->inner_map); 4611 } 4612 if (map->inner_map_fd >= 0) 4613 create_attr.inner_map_fd = map->inner_map_fd; 4614 } 4615 4616 if (obj->gen_loader) { 4617 bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps); 4618 /* Pretend to have valid FD to pass various fd >= 0 checks. 4619 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 4620 */ 4621 map->fd = 0; 4622 } else { 4623 map->fd = bpf_create_map_xattr(&create_attr); 4624 } 4625 if (map->fd < 0 && (create_attr.btf_key_type_id || 4626 create_attr.btf_value_type_id)) { 4627 char *cp, errmsg[STRERR_BUFSIZE]; 4628 4629 err = -errno; 4630 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4631 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 4632 map->name, cp, err); 4633 create_attr.btf_fd = 0; 4634 create_attr.btf_key_type_id = 0; 4635 create_attr.btf_value_type_id = 0; 4636 map->btf_key_type_id = 0; 4637 map->btf_value_type_id = 0; 4638 map->fd = bpf_create_map_xattr(&create_attr); 4639 } 4640 4641 err = map->fd < 0 ? -errno : 0; 4642 4643 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 4644 if (obj->gen_loader) 4645 map->inner_map->fd = -1; 4646 bpf_map__destroy(map->inner_map); 4647 zfree(&map->inner_map); 4648 } 4649 4650 return err; 4651 } 4652 4653 static int init_map_slots(struct bpf_object *obj, struct bpf_map *map) 4654 { 4655 const struct bpf_map *targ_map; 4656 unsigned int i; 4657 int fd, err = 0; 4658 4659 for (i = 0; i < map->init_slots_sz; i++) { 4660 if (!map->init_slots[i]) 4661 continue; 4662 4663 targ_map = map->init_slots[i]; 4664 fd = bpf_map__fd(targ_map); 4665 if (obj->gen_loader) { 4666 pr_warn("// TODO map_update_elem: idx %td key %d value==map_idx %td\n", 4667 map - obj->maps, i, targ_map - obj->maps); 4668 return -ENOTSUP; 4669 } else { 4670 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 4671 } 4672 if (err) { 4673 err = -errno; 4674 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 4675 map->name, i, targ_map->name, 4676 fd, err); 4677 return err; 4678 } 4679 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 4680 map->name, i, targ_map->name, fd); 4681 } 4682 4683 zfree(&map->init_slots); 4684 map->init_slots_sz = 0; 4685 4686 return 0; 4687 } 4688 4689 static int 4690 bpf_object__create_maps(struct bpf_object *obj) 4691 { 4692 struct bpf_map *map; 4693 char *cp, errmsg[STRERR_BUFSIZE]; 4694 unsigned int i, j; 4695 int err; 4696 bool retried; 4697 4698 for (i = 0; i < obj->nr_maps; i++) { 4699 map = &obj->maps[i]; 4700 4701 retried = false; 4702 retry: 4703 if (map->pin_path) { 4704 err = bpf_object__reuse_map(map); 4705 if (err) { 4706 pr_warn("map '%s': error reusing pinned map\n", 4707 map->name); 4708 goto err_out; 4709 } 4710 if (retried && map->fd < 0) { 4711 pr_warn("map '%s': cannot find pinned map\n", 4712 map->name); 4713 err = -ENOENT; 4714 goto err_out; 4715 } 4716 } 4717 4718 if (map->fd >= 0) { 4719 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 4720 map->name, map->fd); 4721 } else { 4722 err = bpf_object__create_map(obj, map, false); 4723 if (err) 4724 goto err_out; 4725 4726 pr_debug("map '%s': created successfully, fd=%d\n", 4727 map->name, map->fd); 4728 4729 if (bpf_map__is_internal(map)) { 4730 err = bpf_object__populate_internal_map(obj, map); 4731 if (err < 0) { 4732 zclose(map->fd); 4733 goto err_out; 4734 } 4735 } 4736 4737 if (map->init_slots_sz) { 4738 err = init_map_slots(obj, map); 4739 if (err < 0) { 4740 zclose(map->fd); 4741 goto err_out; 4742 } 4743 } 4744 } 4745 4746 if (map->pin_path && !map->pinned) { 4747 err = bpf_map__pin(map, NULL); 4748 if (err) { 4749 zclose(map->fd); 4750 if (!retried && err == -EEXIST) { 4751 retried = true; 4752 goto retry; 4753 } 4754 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 4755 map->name, map->pin_path, err); 4756 goto err_out; 4757 } 4758 } 4759 } 4760 4761 return 0; 4762 4763 err_out: 4764 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4765 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 4766 pr_perm_msg(err); 4767 for (j = 0; j < i; j++) 4768 zclose(obj->maps[j].fd); 4769 return err; 4770 } 4771 4772 static bool bpf_core_is_flavor_sep(const char *s) 4773 { 4774 /* check X___Y name pattern, where X and Y are not underscores */ 4775 return s[0] != '_' && /* X */ 4776 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 4777 s[4] != '_'; /* Y */ 4778 } 4779 4780 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 4781 * before last triple underscore. Struct name part after last triple 4782 * underscore is ignored by BPF CO-RE relocation during relocation matching. 4783 */ 4784 size_t bpf_core_essential_name_len(const char *name) 4785 { 4786 size_t n = strlen(name); 4787 int i; 4788 4789 for (i = n - 5; i >= 0; i--) { 4790 if (bpf_core_is_flavor_sep(name + i)) 4791 return i + 1; 4792 } 4793 return n; 4794 } 4795 4796 static void bpf_core_free_cands(struct bpf_core_cand_list *cands) 4797 { 4798 free(cands->cands); 4799 free(cands); 4800 } 4801 4802 static int bpf_core_add_cands(struct bpf_core_cand *local_cand, 4803 size_t local_essent_len, 4804 const struct btf *targ_btf, 4805 const char *targ_btf_name, 4806 int targ_start_id, 4807 struct bpf_core_cand_list *cands) 4808 { 4809 struct bpf_core_cand *new_cands, *cand; 4810 const struct btf_type *t; 4811 const char *targ_name; 4812 size_t targ_essent_len; 4813 int n, i; 4814 4815 n = btf__get_nr_types(targ_btf); 4816 for (i = targ_start_id; i <= n; i++) { 4817 t = btf__type_by_id(targ_btf, i); 4818 if (btf_kind(t) != btf_kind(local_cand->t)) 4819 continue; 4820 4821 targ_name = btf__name_by_offset(targ_btf, t->name_off); 4822 if (str_is_empty(targ_name)) 4823 continue; 4824 4825 targ_essent_len = bpf_core_essential_name_len(targ_name); 4826 if (targ_essent_len != local_essent_len) 4827 continue; 4828 4829 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0) 4830 continue; 4831 4832 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 4833 local_cand->id, btf_kind_str(local_cand->t), 4834 local_cand->name, i, btf_kind_str(t), targ_name, 4835 targ_btf_name); 4836 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 4837 sizeof(*cands->cands)); 4838 if (!new_cands) 4839 return -ENOMEM; 4840 4841 cand = &new_cands[cands->len]; 4842 cand->btf = targ_btf; 4843 cand->t = t; 4844 cand->name = targ_name; 4845 cand->id = i; 4846 4847 cands->cands = new_cands; 4848 cands->len++; 4849 } 4850 return 0; 4851 } 4852 4853 static int load_module_btfs(struct bpf_object *obj) 4854 { 4855 struct bpf_btf_info info; 4856 struct module_btf *mod_btf; 4857 struct btf *btf; 4858 char name[64]; 4859 __u32 id = 0, len; 4860 int err, fd; 4861 4862 if (obj->btf_modules_loaded) 4863 return 0; 4864 4865 if (obj->gen_loader) 4866 return 0; 4867 4868 /* don't do this again, even if we find no module BTFs */ 4869 obj->btf_modules_loaded = true; 4870 4871 /* kernel too old to support module BTFs */ 4872 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 4873 return 0; 4874 4875 while (true) { 4876 err = bpf_btf_get_next_id(id, &id); 4877 if (err && errno == ENOENT) 4878 return 0; 4879 if (err) { 4880 err = -errno; 4881 pr_warn("failed to iterate BTF objects: %d\n", err); 4882 return err; 4883 } 4884 4885 fd = bpf_btf_get_fd_by_id(id); 4886 if (fd < 0) { 4887 if (errno == ENOENT) 4888 continue; /* expected race: BTF was unloaded */ 4889 err = -errno; 4890 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 4891 return err; 4892 } 4893 4894 len = sizeof(info); 4895 memset(&info, 0, sizeof(info)); 4896 info.name = ptr_to_u64(name); 4897 info.name_len = sizeof(name); 4898 4899 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4900 if (err) { 4901 err = -errno; 4902 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 4903 goto err_out; 4904 } 4905 4906 /* ignore non-module BTFs */ 4907 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 4908 close(fd); 4909 continue; 4910 } 4911 4912 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 4913 err = libbpf_get_error(btf); 4914 if (err) { 4915 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", 4916 name, id, err); 4917 goto err_out; 4918 } 4919 4920 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 4921 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 4922 if (err) 4923 goto err_out; 4924 4925 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 4926 4927 mod_btf->btf = btf; 4928 mod_btf->id = id; 4929 mod_btf->fd = fd; 4930 mod_btf->name = strdup(name); 4931 if (!mod_btf->name) { 4932 err = -ENOMEM; 4933 goto err_out; 4934 } 4935 continue; 4936 4937 err_out: 4938 close(fd); 4939 return err; 4940 } 4941 4942 return 0; 4943 } 4944 4945 static struct bpf_core_cand_list * 4946 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 4947 { 4948 struct bpf_core_cand local_cand = {}; 4949 struct bpf_core_cand_list *cands; 4950 const struct btf *main_btf; 4951 size_t local_essent_len; 4952 int err, i; 4953 4954 local_cand.btf = local_btf; 4955 local_cand.t = btf__type_by_id(local_btf, local_type_id); 4956 if (!local_cand.t) 4957 return ERR_PTR(-EINVAL); 4958 4959 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off); 4960 if (str_is_empty(local_cand.name)) 4961 return ERR_PTR(-EINVAL); 4962 local_essent_len = bpf_core_essential_name_len(local_cand.name); 4963 4964 cands = calloc(1, sizeof(*cands)); 4965 if (!cands) 4966 return ERR_PTR(-ENOMEM); 4967 4968 /* Attempt to find target candidates in vmlinux BTF first */ 4969 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 4970 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 4971 if (err) 4972 goto err_out; 4973 4974 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 4975 if (cands->len) 4976 return cands; 4977 4978 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 4979 if (obj->btf_vmlinux_override) 4980 return cands; 4981 4982 /* now look through module BTFs, trying to still find candidates */ 4983 err = load_module_btfs(obj); 4984 if (err) 4985 goto err_out; 4986 4987 for (i = 0; i < obj->btf_module_cnt; i++) { 4988 err = bpf_core_add_cands(&local_cand, local_essent_len, 4989 obj->btf_modules[i].btf, 4990 obj->btf_modules[i].name, 4991 btf__get_nr_types(obj->btf_vmlinux) + 1, 4992 cands); 4993 if (err) 4994 goto err_out; 4995 } 4996 4997 return cands; 4998 err_out: 4999 bpf_core_free_cands(cands); 5000 return ERR_PTR(err); 5001 } 5002 5003 /* Check local and target types for compatibility. This check is used for 5004 * type-based CO-RE relocations and follow slightly different rules than 5005 * field-based relocations. This function assumes that root types were already 5006 * checked for name match. Beyond that initial root-level name check, names 5007 * are completely ignored. Compatibility rules are as follows: 5008 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 5009 * kind should match for local and target types (i.e., STRUCT is not 5010 * compatible with UNION); 5011 * - for ENUMs, the size is ignored; 5012 * - for INT, size and signedness are ignored; 5013 * - for ARRAY, dimensionality is ignored, element types are checked for 5014 * compatibility recursively; 5015 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 5016 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 5017 * - FUNC_PROTOs are compatible if they have compatible signature: same 5018 * number of input args and compatible return and argument types. 5019 * These rules are not set in stone and probably will be adjusted as we get 5020 * more experience with using BPF CO-RE relocations. 5021 */ 5022 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 5023 const struct btf *targ_btf, __u32 targ_id) 5024 { 5025 const struct btf_type *local_type, *targ_type; 5026 int depth = 32; /* max recursion depth */ 5027 5028 /* caller made sure that names match (ignoring flavor suffix) */ 5029 local_type = btf__type_by_id(local_btf, local_id); 5030 targ_type = btf__type_by_id(targ_btf, targ_id); 5031 if (btf_kind(local_type) != btf_kind(targ_type)) 5032 return 0; 5033 5034 recur: 5035 depth--; 5036 if (depth < 0) 5037 return -EINVAL; 5038 5039 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 5040 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 5041 if (!local_type || !targ_type) 5042 return -EINVAL; 5043 5044 if (btf_kind(local_type) != btf_kind(targ_type)) 5045 return 0; 5046 5047 switch (btf_kind(local_type)) { 5048 case BTF_KIND_UNKN: 5049 case BTF_KIND_STRUCT: 5050 case BTF_KIND_UNION: 5051 case BTF_KIND_ENUM: 5052 case BTF_KIND_FWD: 5053 return 1; 5054 case BTF_KIND_INT: 5055 /* just reject deprecated bitfield-like integers; all other 5056 * integers are by default compatible between each other 5057 */ 5058 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; 5059 case BTF_KIND_PTR: 5060 local_id = local_type->type; 5061 targ_id = targ_type->type; 5062 goto recur; 5063 case BTF_KIND_ARRAY: 5064 local_id = btf_array(local_type)->type; 5065 targ_id = btf_array(targ_type)->type; 5066 goto recur; 5067 case BTF_KIND_FUNC_PROTO: { 5068 struct btf_param *local_p = btf_params(local_type); 5069 struct btf_param *targ_p = btf_params(targ_type); 5070 __u16 local_vlen = btf_vlen(local_type); 5071 __u16 targ_vlen = btf_vlen(targ_type); 5072 int i, err; 5073 5074 if (local_vlen != targ_vlen) 5075 return 0; 5076 5077 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { 5078 skip_mods_and_typedefs(local_btf, local_p->type, &local_id); 5079 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); 5080 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id); 5081 if (err <= 0) 5082 return err; 5083 } 5084 5085 /* tail recurse for return type check */ 5086 skip_mods_and_typedefs(local_btf, local_type->type, &local_id); 5087 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); 5088 goto recur; 5089 } 5090 default: 5091 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", 5092 btf_kind_str(local_type), local_id, targ_id); 5093 return 0; 5094 } 5095 } 5096 5097 static size_t bpf_core_hash_fn(const void *key, void *ctx) 5098 { 5099 return (size_t)key; 5100 } 5101 5102 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) 5103 { 5104 return k1 == k2; 5105 } 5106 5107 static void *u32_as_hash_key(__u32 x) 5108 { 5109 return (void *)(uintptr_t)x; 5110 } 5111 5112 static int bpf_core_apply_relo(struct bpf_program *prog, 5113 const struct bpf_core_relo *relo, 5114 int relo_idx, 5115 const struct btf *local_btf, 5116 struct hashmap *cand_cache) 5117 { 5118 const void *type_key = u32_as_hash_key(relo->type_id); 5119 struct bpf_core_cand_list *cands = NULL; 5120 const char *prog_name = prog->name; 5121 const struct btf_type *local_type; 5122 const char *local_name; 5123 __u32 local_id = relo->type_id; 5124 struct bpf_insn *insn; 5125 int insn_idx, err; 5126 5127 if (relo->insn_off % BPF_INSN_SZ) 5128 return -EINVAL; 5129 insn_idx = relo->insn_off / BPF_INSN_SZ; 5130 /* adjust insn_idx from section frame of reference to the local 5131 * program's frame of reference; (sub-)program code is not yet 5132 * relocated, so it's enough to just subtract in-section offset 5133 */ 5134 insn_idx = insn_idx - prog->sec_insn_off; 5135 if (insn_idx > prog->insns_cnt) 5136 return -EINVAL; 5137 insn = &prog->insns[insn_idx]; 5138 5139 local_type = btf__type_by_id(local_btf, local_id); 5140 if (!local_type) 5141 return -EINVAL; 5142 5143 local_name = btf__name_by_offset(local_btf, local_type->name_off); 5144 if (!local_name) 5145 return -EINVAL; 5146 5147 if (prog->obj->gen_loader) { 5148 pr_warn("// TODO core_relo: prog %td insn[%d] %s kind %d\n", 5149 prog - prog->obj->programs, relo->insn_off / 8, 5150 local_name, relo->kind); 5151 return -ENOTSUP; 5152 } 5153 5154 if (relo->kind != BPF_TYPE_ID_LOCAL && 5155 !hashmap__find(cand_cache, type_key, (void **)&cands)) { 5156 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 5157 if (IS_ERR(cands)) { 5158 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 5159 prog_name, relo_idx, local_id, btf_kind_str(local_type), 5160 local_name, PTR_ERR(cands)); 5161 return PTR_ERR(cands); 5162 } 5163 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL); 5164 if (err) { 5165 bpf_core_free_cands(cands); 5166 return err; 5167 } 5168 } 5169 5170 return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands); 5171 } 5172 5173 static int 5174 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 5175 { 5176 const struct btf_ext_info_sec *sec; 5177 const struct bpf_core_relo *rec; 5178 const struct btf_ext_info *seg; 5179 struct hashmap_entry *entry; 5180 struct hashmap *cand_cache = NULL; 5181 struct bpf_program *prog; 5182 const char *sec_name; 5183 int i, err = 0, insn_idx, sec_idx; 5184 5185 if (obj->btf_ext->core_relo_info.len == 0) 5186 return 0; 5187 5188 if (targ_btf_path) { 5189 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 5190 err = libbpf_get_error(obj->btf_vmlinux_override); 5191 if (err) { 5192 pr_warn("failed to parse target BTF: %d\n", err); 5193 return err; 5194 } 5195 } 5196 5197 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 5198 if (IS_ERR(cand_cache)) { 5199 err = PTR_ERR(cand_cache); 5200 goto out; 5201 } 5202 5203 seg = &obj->btf_ext->core_relo_info; 5204 for_each_btf_ext_sec(seg, sec) { 5205 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5206 if (str_is_empty(sec_name)) { 5207 err = -EINVAL; 5208 goto out; 5209 } 5210 /* bpf_object's ELF is gone by now so it's not easy to find 5211 * section index by section name, but we can find *any* 5212 * bpf_program within desired section name and use it's 5213 * prog->sec_idx to do a proper search by section index and 5214 * instruction offset 5215 */ 5216 prog = NULL; 5217 for (i = 0; i < obj->nr_programs; i++) { 5218 prog = &obj->programs[i]; 5219 if (strcmp(prog->sec_name, sec_name) == 0) 5220 break; 5221 } 5222 if (!prog) { 5223 pr_warn("sec '%s': failed to find a BPF program\n", sec_name); 5224 return -ENOENT; 5225 } 5226 sec_idx = prog->sec_idx; 5227 5228 pr_debug("sec '%s': found %d CO-RE relocations\n", 5229 sec_name, sec->num_info); 5230 5231 for_each_btf_ext_rec(seg, sec, i, rec) { 5232 insn_idx = rec->insn_off / BPF_INSN_SZ; 5233 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 5234 if (!prog) { 5235 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n", 5236 sec_name, insn_idx, i); 5237 err = -EINVAL; 5238 goto out; 5239 } 5240 /* no need to apply CO-RE relocation if the program is 5241 * not going to be loaded 5242 */ 5243 if (!prog->load) 5244 continue; 5245 5246 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache); 5247 if (err) { 5248 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 5249 prog->name, i, err); 5250 goto out; 5251 } 5252 } 5253 } 5254 5255 out: 5256 /* obj->btf_vmlinux and module BTFs are freed after object load */ 5257 btf__free(obj->btf_vmlinux_override); 5258 obj->btf_vmlinux_override = NULL; 5259 5260 if (!IS_ERR_OR_NULL(cand_cache)) { 5261 hashmap__for_each_entry(cand_cache, entry, i) { 5262 bpf_core_free_cands(entry->value); 5263 } 5264 hashmap__free(cand_cache); 5265 } 5266 return err; 5267 } 5268 5269 /* Relocate data references within program code: 5270 * - map references; 5271 * - global variable references; 5272 * - extern references. 5273 */ 5274 static int 5275 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 5276 { 5277 int i; 5278 5279 for (i = 0; i < prog->nr_reloc; i++) { 5280 struct reloc_desc *relo = &prog->reloc_desc[i]; 5281 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5282 struct extern_desc *ext; 5283 5284 switch (relo->type) { 5285 case RELO_LD64: 5286 if (obj->gen_loader) { 5287 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 5288 insn[0].imm = relo->map_idx; 5289 } else { 5290 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 5291 insn[0].imm = obj->maps[relo->map_idx].fd; 5292 } 5293 break; 5294 case RELO_DATA: 5295 insn[1].imm = insn[0].imm + relo->sym_off; 5296 if (obj->gen_loader) { 5297 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5298 insn[0].imm = relo->map_idx; 5299 } else { 5300 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5301 insn[0].imm = obj->maps[relo->map_idx].fd; 5302 } 5303 break; 5304 case RELO_EXTERN_VAR: 5305 ext = &obj->externs[relo->sym_off]; 5306 if (ext->type == EXT_KCFG) { 5307 if (obj->gen_loader) { 5308 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5309 insn[0].imm = obj->kconfig_map_idx; 5310 } else { 5311 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5312 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 5313 } 5314 insn[1].imm = ext->kcfg.data_off; 5315 } else /* EXT_KSYM */ { 5316 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 5317 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 5318 insn[0].imm = ext->ksym.kernel_btf_id; 5319 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 5320 } else { /* typeless ksyms or unresolved typed ksyms */ 5321 insn[0].imm = (__u32)ext->ksym.addr; 5322 insn[1].imm = ext->ksym.addr >> 32; 5323 } 5324 } 5325 break; 5326 case RELO_EXTERN_FUNC: 5327 ext = &obj->externs[relo->sym_off]; 5328 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 5329 insn[0].imm = ext->ksym.kernel_btf_id; 5330 break; 5331 case RELO_SUBPROG_ADDR: 5332 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 5333 pr_warn("prog '%s': relo #%d: bad insn\n", 5334 prog->name, i); 5335 return -EINVAL; 5336 } 5337 /* handled already */ 5338 break; 5339 case RELO_CALL: 5340 /* handled already */ 5341 break; 5342 default: 5343 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 5344 prog->name, i, relo->type); 5345 return -EINVAL; 5346 } 5347 } 5348 5349 return 0; 5350 } 5351 5352 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 5353 const struct bpf_program *prog, 5354 const struct btf_ext_info *ext_info, 5355 void **prog_info, __u32 *prog_rec_cnt, 5356 __u32 *prog_rec_sz) 5357 { 5358 void *copy_start = NULL, *copy_end = NULL; 5359 void *rec, *rec_end, *new_prog_info; 5360 const struct btf_ext_info_sec *sec; 5361 size_t old_sz, new_sz; 5362 const char *sec_name; 5363 int i, off_adj; 5364 5365 for_each_btf_ext_sec(ext_info, sec) { 5366 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5367 if (!sec_name) 5368 return -EINVAL; 5369 if (strcmp(sec_name, prog->sec_name) != 0) 5370 continue; 5371 5372 for_each_btf_ext_rec(ext_info, sec, i, rec) { 5373 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 5374 5375 if (insn_off < prog->sec_insn_off) 5376 continue; 5377 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 5378 break; 5379 5380 if (!copy_start) 5381 copy_start = rec; 5382 copy_end = rec + ext_info->rec_size; 5383 } 5384 5385 if (!copy_start) 5386 return -ENOENT; 5387 5388 /* append func/line info of a given (sub-)program to the main 5389 * program func/line info 5390 */ 5391 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 5392 new_sz = old_sz + (copy_end - copy_start); 5393 new_prog_info = realloc(*prog_info, new_sz); 5394 if (!new_prog_info) 5395 return -ENOMEM; 5396 *prog_info = new_prog_info; 5397 *prog_rec_cnt = new_sz / ext_info->rec_size; 5398 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 5399 5400 /* Kernel instruction offsets are in units of 8-byte 5401 * instructions, while .BTF.ext instruction offsets generated 5402 * by Clang are in units of bytes. So convert Clang offsets 5403 * into kernel offsets and adjust offset according to program 5404 * relocated position. 5405 */ 5406 off_adj = prog->sub_insn_off - prog->sec_insn_off; 5407 rec = new_prog_info + old_sz; 5408 rec_end = new_prog_info + new_sz; 5409 for (; rec < rec_end; rec += ext_info->rec_size) { 5410 __u32 *insn_off = rec; 5411 5412 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 5413 } 5414 *prog_rec_sz = ext_info->rec_size; 5415 return 0; 5416 } 5417 5418 return -ENOENT; 5419 } 5420 5421 static int 5422 reloc_prog_func_and_line_info(const struct bpf_object *obj, 5423 struct bpf_program *main_prog, 5424 const struct bpf_program *prog) 5425 { 5426 int err; 5427 5428 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 5429 * supprot func/line info 5430 */ 5431 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 5432 return 0; 5433 5434 /* only attempt func info relocation if main program's func_info 5435 * relocation was successful 5436 */ 5437 if (main_prog != prog && !main_prog->func_info) 5438 goto line_info; 5439 5440 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 5441 &main_prog->func_info, 5442 &main_prog->func_info_cnt, 5443 &main_prog->func_info_rec_size); 5444 if (err) { 5445 if (err != -ENOENT) { 5446 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 5447 prog->name, err); 5448 return err; 5449 } 5450 if (main_prog->func_info) { 5451 /* 5452 * Some info has already been found but has problem 5453 * in the last btf_ext reloc. Must have to error out. 5454 */ 5455 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 5456 return err; 5457 } 5458 /* Have problem loading the very first info. Ignore the rest. */ 5459 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 5460 prog->name); 5461 } 5462 5463 line_info: 5464 /* don't relocate line info if main program's relocation failed */ 5465 if (main_prog != prog && !main_prog->line_info) 5466 return 0; 5467 5468 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 5469 &main_prog->line_info, 5470 &main_prog->line_info_cnt, 5471 &main_prog->line_info_rec_size); 5472 if (err) { 5473 if (err != -ENOENT) { 5474 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 5475 prog->name, err); 5476 return err; 5477 } 5478 if (main_prog->line_info) { 5479 /* 5480 * Some info has already been found but has problem 5481 * in the last btf_ext reloc. Must have to error out. 5482 */ 5483 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 5484 return err; 5485 } 5486 /* Have problem loading the very first info. Ignore the rest. */ 5487 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 5488 prog->name); 5489 } 5490 return 0; 5491 } 5492 5493 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 5494 { 5495 size_t insn_idx = *(const size_t *)key; 5496 const struct reloc_desc *relo = elem; 5497 5498 if (insn_idx == relo->insn_idx) 5499 return 0; 5500 return insn_idx < relo->insn_idx ? -1 : 1; 5501 } 5502 5503 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 5504 { 5505 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 5506 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 5507 } 5508 5509 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 5510 { 5511 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 5512 struct reloc_desc *relos; 5513 int i; 5514 5515 if (main_prog == subprog) 5516 return 0; 5517 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 5518 if (!relos) 5519 return -ENOMEM; 5520 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 5521 sizeof(*relos) * subprog->nr_reloc); 5522 5523 for (i = main_prog->nr_reloc; i < new_cnt; i++) 5524 relos[i].insn_idx += subprog->sub_insn_off; 5525 /* After insn_idx adjustment the 'relos' array is still sorted 5526 * by insn_idx and doesn't break bsearch. 5527 */ 5528 main_prog->reloc_desc = relos; 5529 main_prog->nr_reloc = new_cnt; 5530 return 0; 5531 } 5532 5533 static int 5534 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 5535 struct bpf_program *prog) 5536 { 5537 size_t sub_insn_idx, insn_idx, new_cnt; 5538 struct bpf_program *subprog; 5539 struct bpf_insn *insns, *insn; 5540 struct reloc_desc *relo; 5541 int err; 5542 5543 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 5544 if (err) 5545 return err; 5546 5547 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 5548 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 5549 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 5550 continue; 5551 5552 relo = find_prog_insn_relo(prog, insn_idx); 5553 if (relo && relo->type == RELO_EXTERN_FUNC) 5554 /* kfunc relocations will be handled later 5555 * in bpf_object__relocate_data() 5556 */ 5557 continue; 5558 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 5559 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 5560 prog->name, insn_idx, relo->type); 5561 return -LIBBPF_ERRNO__RELOC; 5562 } 5563 if (relo) { 5564 /* sub-program instruction index is a combination of 5565 * an offset of a symbol pointed to by relocation and 5566 * call instruction's imm field; for global functions, 5567 * call always has imm = -1, but for static functions 5568 * relocation is against STT_SECTION and insn->imm 5569 * points to a start of a static function 5570 * 5571 * for subprog addr relocation, the relo->sym_off + insn->imm is 5572 * the byte offset in the corresponding section. 5573 */ 5574 if (relo->type == RELO_CALL) 5575 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 5576 else 5577 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 5578 } else if (insn_is_pseudo_func(insn)) { 5579 /* 5580 * RELO_SUBPROG_ADDR relo is always emitted even if both 5581 * functions are in the same section, so it shouldn't reach here. 5582 */ 5583 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 5584 prog->name, insn_idx); 5585 return -LIBBPF_ERRNO__RELOC; 5586 } else { 5587 /* if subprogram call is to a static function within 5588 * the same ELF section, there won't be any relocation 5589 * emitted, but it also means there is no additional 5590 * offset necessary, insns->imm is relative to 5591 * instruction's original position within the section 5592 */ 5593 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 5594 } 5595 5596 /* we enforce that sub-programs should be in .text section */ 5597 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 5598 if (!subprog) { 5599 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 5600 prog->name); 5601 return -LIBBPF_ERRNO__RELOC; 5602 } 5603 5604 /* if it's the first call instruction calling into this 5605 * subprogram (meaning this subprog hasn't been processed 5606 * yet) within the context of current main program: 5607 * - append it at the end of main program's instructions blog; 5608 * - process is recursively, while current program is put on hold; 5609 * - if that subprogram calls some other not yet processes 5610 * subprogram, same thing will happen recursively until 5611 * there are no more unprocesses subprograms left to append 5612 * and relocate. 5613 */ 5614 if (subprog->sub_insn_off == 0) { 5615 subprog->sub_insn_off = main_prog->insns_cnt; 5616 5617 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 5618 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 5619 if (!insns) { 5620 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 5621 return -ENOMEM; 5622 } 5623 main_prog->insns = insns; 5624 main_prog->insns_cnt = new_cnt; 5625 5626 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 5627 subprog->insns_cnt * sizeof(*insns)); 5628 5629 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 5630 main_prog->name, subprog->insns_cnt, subprog->name); 5631 5632 /* The subprog insns are now appended. Append its relos too. */ 5633 err = append_subprog_relos(main_prog, subprog); 5634 if (err) 5635 return err; 5636 err = bpf_object__reloc_code(obj, main_prog, subprog); 5637 if (err) 5638 return err; 5639 } 5640 5641 /* main_prog->insns memory could have been re-allocated, so 5642 * calculate pointer again 5643 */ 5644 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 5645 /* calculate correct instruction position within current main 5646 * prog; each main prog can have a different set of 5647 * subprograms appended (potentially in different order as 5648 * well), so position of any subprog can be different for 5649 * different main programs */ 5650 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 5651 5652 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 5653 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 5654 } 5655 5656 return 0; 5657 } 5658 5659 /* 5660 * Relocate sub-program calls. 5661 * 5662 * Algorithm operates as follows. Each entry-point BPF program (referred to as 5663 * main prog) is processed separately. For each subprog (non-entry functions, 5664 * that can be called from either entry progs or other subprogs) gets their 5665 * sub_insn_off reset to zero. This serves as indicator that this subprogram 5666 * hasn't been yet appended and relocated within current main prog. Once its 5667 * relocated, sub_insn_off will point at the position within current main prog 5668 * where given subprog was appended. This will further be used to relocate all 5669 * the call instructions jumping into this subprog. 5670 * 5671 * We start with main program and process all call instructions. If the call 5672 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 5673 * is zero), subprog instructions are appended at the end of main program's 5674 * instruction array. Then main program is "put on hold" while we recursively 5675 * process newly appended subprogram. If that subprogram calls into another 5676 * subprogram that hasn't been appended, new subprogram is appended again to 5677 * the *main* prog's instructions (subprog's instructions are always left 5678 * untouched, as they need to be in unmodified state for subsequent main progs 5679 * and subprog instructions are always sent only as part of a main prog) and 5680 * the process continues recursively. Once all the subprogs called from a main 5681 * prog or any of its subprogs are appended (and relocated), all their 5682 * positions within finalized instructions array are known, so it's easy to 5683 * rewrite call instructions with correct relative offsets, corresponding to 5684 * desired target subprog. 5685 * 5686 * Its important to realize that some subprogs might not be called from some 5687 * main prog and any of its called/used subprogs. Those will keep their 5688 * subprog->sub_insn_off as zero at all times and won't be appended to current 5689 * main prog and won't be relocated within the context of current main prog. 5690 * They might still be used from other main progs later. 5691 * 5692 * Visually this process can be shown as below. Suppose we have two main 5693 * programs mainA and mainB and BPF object contains three subprogs: subA, 5694 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 5695 * subC both call subB: 5696 * 5697 * +--------+ +-------+ 5698 * | v v | 5699 * +--+---+ +--+-+-+ +---+--+ 5700 * | subA | | subB | | subC | 5701 * +--+---+ +------+ +---+--+ 5702 * ^ ^ 5703 * | | 5704 * +---+-------+ +------+----+ 5705 * | mainA | | mainB | 5706 * +-----------+ +-----------+ 5707 * 5708 * We'll start relocating mainA, will find subA, append it and start 5709 * processing sub A recursively: 5710 * 5711 * +-----------+------+ 5712 * | mainA | subA | 5713 * +-----------+------+ 5714 * 5715 * At this point we notice that subB is used from subA, so we append it and 5716 * relocate (there are no further subcalls from subB): 5717 * 5718 * +-----------+------+------+ 5719 * | mainA | subA | subB | 5720 * +-----------+------+------+ 5721 * 5722 * At this point, we relocate subA calls, then go one level up and finish with 5723 * relocatin mainA calls. mainA is done. 5724 * 5725 * For mainB process is similar but results in different order. We start with 5726 * mainB and skip subA and subB, as mainB never calls them (at least 5727 * directly), but we see subC is needed, so we append and start processing it: 5728 * 5729 * +-----------+------+ 5730 * | mainB | subC | 5731 * +-----------+------+ 5732 * Now we see subC needs subB, so we go back to it, append and relocate it: 5733 * 5734 * +-----------+------+------+ 5735 * | mainB | subC | subB | 5736 * +-----------+------+------+ 5737 * 5738 * At this point we unwind recursion, relocate calls in subC, then in mainB. 5739 */ 5740 static int 5741 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 5742 { 5743 struct bpf_program *subprog; 5744 int i, err; 5745 5746 /* mark all subprogs as not relocated (yet) within the context of 5747 * current main program 5748 */ 5749 for (i = 0; i < obj->nr_programs; i++) { 5750 subprog = &obj->programs[i]; 5751 if (!prog_is_subprog(obj, subprog)) 5752 continue; 5753 5754 subprog->sub_insn_off = 0; 5755 } 5756 5757 err = bpf_object__reloc_code(obj, prog, prog); 5758 if (err) 5759 return err; 5760 5761 5762 return 0; 5763 } 5764 5765 static void 5766 bpf_object__free_relocs(struct bpf_object *obj) 5767 { 5768 struct bpf_program *prog; 5769 int i; 5770 5771 /* free up relocation descriptors */ 5772 for (i = 0; i < obj->nr_programs; i++) { 5773 prog = &obj->programs[i]; 5774 zfree(&prog->reloc_desc); 5775 prog->nr_reloc = 0; 5776 } 5777 } 5778 5779 static int 5780 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 5781 { 5782 struct bpf_program *prog; 5783 size_t i, j; 5784 int err; 5785 5786 if (obj->btf_ext) { 5787 err = bpf_object__relocate_core(obj, targ_btf_path); 5788 if (err) { 5789 pr_warn("failed to perform CO-RE relocations: %d\n", 5790 err); 5791 return err; 5792 } 5793 } 5794 5795 /* Before relocating calls pre-process relocations and mark 5796 * few ld_imm64 instructions that points to subprogs. 5797 * Otherwise bpf_object__reloc_code() later would have to consider 5798 * all ld_imm64 insns as relocation candidates. That would 5799 * reduce relocation speed, since amount of find_prog_insn_relo() 5800 * would increase and most of them will fail to find a relo. 5801 */ 5802 for (i = 0; i < obj->nr_programs; i++) { 5803 prog = &obj->programs[i]; 5804 for (j = 0; j < prog->nr_reloc; j++) { 5805 struct reloc_desc *relo = &prog->reloc_desc[j]; 5806 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5807 5808 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 5809 if (relo->type == RELO_SUBPROG_ADDR) 5810 insn[0].src_reg = BPF_PSEUDO_FUNC; 5811 } 5812 } 5813 5814 /* relocate subprogram calls and append used subprograms to main 5815 * programs; each copy of subprogram code needs to be relocated 5816 * differently for each main program, because its code location might 5817 * have changed. 5818 * Append subprog relos to main programs to allow data relos to be 5819 * processed after text is completely relocated. 5820 */ 5821 for (i = 0; i < obj->nr_programs; i++) { 5822 prog = &obj->programs[i]; 5823 /* sub-program's sub-calls are relocated within the context of 5824 * its main program only 5825 */ 5826 if (prog_is_subprog(obj, prog)) 5827 continue; 5828 5829 err = bpf_object__relocate_calls(obj, prog); 5830 if (err) { 5831 pr_warn("prog '%s': failed to relocate calls: %d\n", 5832 prog->name, err); 5833 return err; 5834 } 5835 } 5836 /* Process data relos for main programs */ 5837 for (i = 0; i < obj->nr_programs; i++) { 5838 prog = &obj->programs[i]; 5839 if (prog_is_subprog(obj, prog)) 5840 continue; 5841 err = bpf_object__relocate_data(obj, prog); 5842 if (err) { 5843 pr_warn("prog '%s': failed to relocate data references: %d\n", 5844 prog->name, err); 5845 return err; 5846 } 5847 } 5848 if (!obj->gen_loader) 5849 bpf_object__free_relocs(obj); 5850 return 0; 5851 } 5852 5853 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 5854 GElf_Shdr *shdr, Elf_Data *data); 5855 5856 static int bpf_object__collect_map_relos(struct bpf_object *obj, 5857 GElf_Shdr *shdr, Elf_Data *data) 5858 { 5859 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 5860 int i, j, nrels, new_sz; 5861 const struct btf_var_secinfo *vi = NULL; 5862 const struct btf_type *sec, *var, *def; 5863 struct bpf_map *map = NULL, *targ_map; 5864 const struct btf_member *member; 5865 const char *name, *mname; 5866 Elf_Data *symbols; 5867 unsigned int moff; 5868 GElf_Sym sym; 5869 GElf_Rel rel; 5870 void *tmp; 5871 5872 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 5873 return -EINVAL; 5874 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 5875 if (!sec) 5876 return -EINVAL; 5877 5878 symbols = obj->efile.symbols; 5879 nrels = shdr->sh_size / shdr->sh_entsize; 5880 for (i = 0; i < nrels; i++) { 5881 if (!gelf_getrel(data, i, &rel)) { 5882 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 5883 return -LIBBPF_ERRNO__FORMAT; 5884 } 5885 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 5886 pr_warn(".maps relo #%d: symbol %zx not found\n", 5887 i, (size_t)GELF_R_SYM(rel.r_info)); 5888 return -LIBBPF_ERRNO__FORMAT; 5889 } 5890 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 5891 if (sym.st_shndx != obj->efile.btf_maps_shndx) { 5892 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 5893 i, name); 5894 return -LIBBPF_ERRNO__RELOC; 5895 } 5896 5897 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n", 5898 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value, 5899 (size_t)rel.r_offset, sym.st_name, name); 5900 5901 for (j = 0; j < obj->nr_maps; j++) { 5902 map = &obj->maps[j]; 5903 if (map->sec_idx != obj->efile.btf_maps_shndx) 5904 continue; 5905 5906 vi = btf_var_secinfos(sec) + map->btf_var_idx; 5907 if (vi->offset <= rel.r_offset && 5908 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size) 5909 break; 5910 } 5911 if (j == obj->nr_maps) { 5912 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n", 5913 i, name, (size_t)rel.r_offset); 5914 return -EINVAL; 5915 } 5916 5917 if (!bpf_map_type__is_map_in_map(map->def.type)) 5918 return -EINVAL; 5919 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 5920 map->def.key_size != sizeof(int)) { 5921 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 5922 i, map->name, sizeof(int)); 5923 return -EINVAL; 5924 } 5925 5926 targ_map = bpf_object__find_map_by_name(obj, name); 5927 if (!targ_map) 5928 return -ESRCH; 5929 5930 var = btf__type_by_id(obj->btf, vi->type); 5931 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 5932 if (btf_vlen(def) == 0) 5933 return -EINVAL; 5934 member = btf_members(def) + btf_vlen(def) - 1; 5935 mname = btf__name_by_offset(obj->btf, member->name_off); 5936 if (strcmp(mname, "values")) 5937 return -EINVAL; 5938 5939 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 5940 if (rel.r_offset - vi->offset < moff) 5941 return -EINVAL; 5942 5943 moff = rel.r_offset - vi->offset - moff; 5944 /* here we use BPF pointer size, which is always 64 bit, as we 5945 * are parsing ELF that was built for BPF target 5946 */ 5947 if (moff % bpf_ptr_sz) 5948 return -EINVAL; 5949 moff /= bpf_ptr_sz; 5950 if (moff >= map->init_slots_sz) { 5951 new_sz = moff + 1; 5952 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 5953 if (!tmp) 5954 return -ENOMEM; 5955 map->init_slots = tmp; 5956 memset(map->init_slots + map->init_slots_sz, 0, 5957 (new_sz - map->init_slots_sz) * host_ptr_sz); 5958 map->init_slots_sz = new_sz; 5959 } 5960 map->init_slots[moff] = targ_map; 5961 5962 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n", 5963 i, map->name, moff, name); 5964 } 5965 5966 return 0; 5967 } 5968 5969 static int cmp_relocs(const void *_a, const void *_b) 5970 { 5971 const struct reloc_desc *a = _a; 5972 const struct reloc_desc *b = _b; 5973 5974 if (a->insn_idx != b->insn_idx) 5975 return a->insn_idx < b->insn_idx ? -1 : 1; 5976 5977 /* no two relocations should have the same insn_idx, but ... */ 5978 if (a->type != b->type) 5979 return a->type < b->type ? -1 : 1; 5980 5981 return 0; 5982 } 5983 5984 static int bpf_object__collect_relos(struct bpf_object *obj) 5985 { 5986 int i, err; 5987 5988 for (i = 0; i < obj->efile.nr_reloc_sects; i++) { 5989 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; 5990 Elf_Data *data = obj->efile.reloc_sects[i].data; 5991 int idx = shdr->sh_info; 5992 5993 if (shdr->sh_type != SHT_REL) { 5994 pr_warn("internal error at %d\n", __LINE__); 5995 return -LIBBPF_ERRNO__INTERNAL; 5996 } 5997 5998 if (idx == obj->efile.st_ops_shndx) 5999 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 6000 else if (idx == obj->efile.btf_maps_shndx) 6001 err = bpf_object__collect_map_relos(obj, shdr, data); 6002 else 6003 err = bpf_object__collect_prog_relos(obj, shdr, data); 6004 if (err) 6005 return err; 6006 } 6007 6008 for (i = 0; i < obj->nr_programs; i++) { 6009 struct bpf_program *p = &obj->programs[i]; 6010 6011 if (!p->nr_reloc) 6012 continue; 6013 6014 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 6015 } 6016 return 0; 6017 } 6018 6019 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 6020 { 6021 if (BPF_CLASS(insn->code) == BPF_JMP && 6022 BPF_OP(insn->code) == BPF_CALL && 6023 BPF_SRC(insn->code) == BPF_K && 6024 insn->src_reg == 0 && 6025 insn->dst_reg == 0) { 6026 *func_id = insn->imm; 6027 return true; 6028 } 6029 return false; 6030 } 6031 6032 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 6033 { 6034 struct bpf_insn *insn = prog->insns; 6035 enum bpf_func_id func_id; 6036 int i; 6037 6038 if (obj->gen_loader) 6039 return 0; 6040 6041 for (i = 0; i < prog->insns_cnt; i++, insn++) { 6042 if (!insn_is_helper_call(insn, &func_id)) 6043 continue; 6044 6045 /* on kernels that don't yet support 6046 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 6047 * to bpf_probe_read() which works well for old kernels 6048 */ 6049 switch (func_id) { 6050 case BPF_FUNC_probe_read_kernel: 6051 case BPF_FUNC_probe_read_user: 6052 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6053 insn->imm = BPF_FUNC_probe_read; 6054 break; 6055 case BPF_FUNC_probe_read_kernel_str: 6056 case BPF_FUNC_probe_read_user_str: 6057 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6058 insn->imm = BPF_FUNC_probe_read_str; 6059 break; 6060 default: 6061 break; 6062 } 6063 } 6064 return 0; 6065 } 6066 6067 static int 6068 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 6069 char *license, __u32 kern_version, int *pfd) 6070 { 6071 struct bpf_prog_load_params load_attr = {}; 6072 char *cp, errmsg[STRERR_BUFSIZE]; 6073 size_t log_buf_size = 0; 6074 char *log_buf = NULL; 6075 int btf_fd, ret; 6076 6077 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 6078 /* 6079 * The program type must be set. Most likely we couldn't find a proper 6080 * section definition at load time, and thus we didn't infer the type. 6081 */ 6082 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 6083 prog->name, prog->sec_name); 6084 return -EINVAL; 6085 } 6086 6087 if (!insns || !insns_cnt) 6088 return -EINVAL; 6089 6090 load_attr.prog_type = prog->type; 6091 /* old kernels might not support specifying expected_attach_type */ 6092 if (!kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE) && prog->sec_def && 6093 prog->sec_def->is_exp_attach_type_optional) 6094 load_attr.expected_attach_type = 0; 6095 else 6096 load_attr.expected_attach_type = prog->expected_attach_type; 6097 if (kernel_supports(prog->obj, FEAT_PROG_NAME)) 6098 load_attr.name = prog->name; 6099 load_attr.insns = insns; 6100 load_attr.insn_cnt = insns_cnt; 6101 load_attr.license = license; 6102 load_attr.attach_btf_id = prog->attach_btf_id; 6103 if (prog->attach_prog_fd) 6104 load_attr.attach_prog_fd = prog->attach_prog_fd; 6105 else 6106 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 6107 load_attr.attach_btf_id = prog->attach_btf_id; 6108 load_attr.kern_version = kern_version; 6109 load_attr.prog_ifindex = prog->prog_ifindex; 6110 6111 /* specify func_info/line_info only if kernel supports them */ 6112 btf_fd = bpf_object__btf_fd(prog->obj); 6113 if (btf_fd >= 0 && kernel_supports(prog->obj, FEAT_BTF_FUNC)) { 6114 load_attr.prog_btf_fd = btf_fd; 6115 load_attr.func_info = prog->func_info; 6116 load_attr.func_info_rec_size = prog->func_info_rec_size; 6117 load_attr.func_info_cnt = prog->func_info_cnt; 6118 load_attr.line_info = prog->line_info; 6119 load_attr.line_info_rec_size = prog->line_info_rec_size; 6120 load_attr.line_info_cnt = prog->line_info_cnt; 6121 } 6122 load_attr.log_level = prog->log_level; 6123 load_attr.prog_flags = prog->prog_flags; 6124 6125 if (prog->obj->gen_loader) { 6126 bpf_gen__prog_load(prog->obj->gen_loader, &load_attr, 6127 prog - prog->obj->programs); 6128 *pfd = -1; 6129 return 0; 6130 } 6131 retry_load: 6132 if (log_buf_size) { 6133 log_buf = malloc(log_buf_size); 6134 if (!log_buf) 6135 return -ENOMEM; 6136 6137 *log_buf = 0; 6138 } 6139 6140 load_attr.log_buf = log_buf; 6141 load_attr.log_buf_sz = log_buf_size; 6142 ret = libbpf__bpf_prog_load(&load_attr); 6143 6144 if (ret >= 0) { 6145 if (log_buf && load_attr.log_level) 6146 pr_debug("verifier log:\n%s", log_buf); 6147 6148 if (prog->obj->rodata_map_idx >= 0 && 6149 kernel_supports(prog->obj, FEAT_PROG_BIND_MAP)) { 6150 struct bpf_map *rodata_map = 6151 &prog->obj->maps[prog->obj->rodata_map_idx]; 6152 6153 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) { 6154 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6155 pr_warn("prog '%s': failed to bind .rodata map: %s\n", 6156 prog->name, cp); 6157 /* Don't fail hard if can't bind rodata. */ 6158 } 6159 } 6160 6161 *pfd = ret; 6162 ret = 0; 6163 goto out; 6164 } 6165 6166 if (!log_buf || errno == ENOSPC) { 6167 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, 6168 log_buf_size << 1); 6169 6170 free(log_buf); 6171 goto retry_load; 6172 } 6173 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD; 6174 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6175 pr_warn("load bpf program failed: %s\n", cp); 6176 pr_perm_msg(ret); 6177 6178 if (log_buf && log_buf[0] != '\0') { 6179 ret = -LIBBPF_ERRNO__VERIFY; 6180 pr_warn("-- BEGIN DUMP LOG ---\n"); 6181 pr_warn("\n%s\n", log_buf); 6182 pr_warn("-- END LOG --\n"); 6183 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) { 6184 pr_warn("Program too large (%zu insns), at most %d insns\n", 6185 load_attr.insn_cnt, BPF_MAXINSNS); 6186 ret = -LIBBPF_ERRNO__PROG2BIG; 6187 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 6188 /* Wrong program type? */ 6189 int fd; 6190 6191 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 6192 load_attr.expected_attach_type = 0; 6193 load_attr.log_buf = NULL; 6194 load_attr.log_buf_sz = 0; 6195 fd = libbpf__bpf_prog_load(&load_attr); 6196 if (fd >= 0) { 6197 close(fd); 6198 ret = -LIBBPF_ERRNO__PROGTYPE; 6199 goto out; 6200 } 6201 } 6202 6203 out: 6204 free(log_buf); 6205 return ret; 6206 } 6207 6208 static int bpf_program__record_externs(struct bpf_program *prog) 6209 { 6210 struct bpf_object *obj = prog->obj; 6211 int i; 6212 6213 for (i = 0; i < prog->nr_reloc; i++) { 6214 struct reloc_desc *relo = &prog->reloc_desc[i]; 6215 struct extern_desc *ext = &obj->externs[relo->sym_off]; 6216 6217 switch (relo->type) { 6218 case RELO_EXTERN_VAR: 6219 if (ext->type != EXT_KSYM) 6220 continue; 6221 if (!ext->ksym.type_id) { 6222 pr_warn("typeless ksym %s is not supported yet\n", 6223 ext->name); 6224 return -ENOTSUP; 6225 } 6226 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR, 6227 relo->insn_idx); 6228 break; 6229 case RELO_EXTERN_FUNC: 6230 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC, 6231 relo->insn_idx); 6232 break; 6233 default: 6234 continue; 6235 } 6236 } 6237 return 0; 6238 } 6239 6240 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id); 6241 6242 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) 6243 { 6244 int err = 0, fd, i; 6245 6246 if (prog->obj->loaded) { 6247 pr_warn("prog '%s': can't load after object was loaded\n", prog->name); 6248 return libbpf_err(-EINVAL); 6249 } 6250 6251 if ((prog->type == BPF_PROG_TYPE_TRACING || 6252 prog->type == BPF_PROG_TYPE_LSM || 6253 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { 6254 int btf_obj_fd = 0, btf_type_id = 0; 6255 6256 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id); 6257 if (err) 6258 return libbpf_err(err); 6259 6260 prog->attach_btf_obj_fd = btf_obj_fd; 6261 prog->attach_btf_id = btf_type_id; 6262 } 6263 6264 if (prog->instances.nr < 0 || !prog->instances.fds) { 6265 if (prog->preprocessor) { 6266 pr_warn("Internal error: can't load program '%s'\n", 6267 prog->name); 6268 return libbpf_err(-LIBBPF_ERRNO__INTERNAL); 6269 } 6270 6271 prog->instances.fds = malloc(sizeof(int)); 6272 if (!prog->instances.fds) { 6273 pr_warn("Not enough memory for BPF fds\n"); 6274 return libbpf_err(-ENOMEM); 6275 } 6276 prog->instances.nr = 1; 6277 prog->instances.fds[0] = -1; 6278 } 6279 6280 if (!prog->preprocessor) { 6281 if (prog->instances.nr != 1) { 6282 pr_warn("prog '%s': inconsistent nr(%d) != 1\n", 6283 prog->name, prog->instances.nr); 6284 } 6285 if (prog->obj->gen_loader) 6286 bpf_program__record_externs(prog); 6287 err = load_program(prog, prog->insns, prog->insns_cnt, 6288 license, kern_ver, &fd); 6289 if (!err) 6290 prog->instances.fds[0] = fd; 6291 goto out; 6292 } 6293 6294 for (i = 0; i < prog->instances.nr; i++) { 6295 struct bpf_prog_prep_result result; 6296 bpf_program_prep_t preprocessor = prog->preprocessor; 6297 6298 memset(&result, 0, sizeof(result)); 6299 err = preprocessor(prog, i, prog->insns, 6300 prog->insns_cnt, &result); 6301 if (err) { 6302 pr_warn("Preprocessing the %dth instance of program '%s' failed\n", 6303 i, prog->name); 6304 goto out; 6305 } 6306 6307 if (!result.new_insn_ptr || !result.new_insn_cnt) { 6308 pr_debug("Skip loading the %dth instance of program '%s'\n", 6309 i, prog->name); 6310 prog->instances.fds[i] = -1; 6311 if (result.pfd) 6312 *result.pfd = -1; 6313 continue; 6314 } 6315 6316 err = load_program(prog, result.new_insn_ptr, 6317 result.new_insn_cnt, license, kern_ver, &fd); 6318 if (err) { 6319 pr_warn("Loading the %dth instance of program '%s' failed\n", 6320 i, prog->name); 6321 goto out; 6322 } 6323 6324 if (result.pfd) 6325 *result.pfd = fd; 6326 prog->instances.fds[i] = fd; 6327 } 6328 out: 6329 if (err) 6330 pr_warn("failed to load program '%s'\n", prog->name); 6331 zfree(&prog->insns); 6332 prog->insns_cnt = 0; 6333 return libbpf_err(err); 6334 } 6335 6336 static int 6337 bpf_object__load_progs(struct bpf_object *obj, int log_level) 6338 { 6339 struct bpf_program *prog; 6340 size_t i; 6341 int err; 6342 6343 for (i = 0; i < obj->nr_programs; i++) { 6344 prog = &obj->programs[i]; 6345 err = bpf_object__sanitize_prog(obj, prog); 6346 if (err) 6347 return err; 6348 } 6349 6350 for (i = 0; i < obj->nr_programs; i++) { 6351 prog = &obj->programs[i]; 6352 if (prog_is_subprog(obj, prog)) 6353 continue; 6354 if (!prog->load) { 6355 pr_debug("prog '%s': skipped loading\n", prog->name); 6356 continue; 6357 } 6358 prog->log_level |= log_level; 6359 err = bpf_program__load(prog, obj->license, obj->kern_version); 6360 if (err) 6361 return err; 6362 } 6363 if (obj->gen_loader) 6364 bpf_object__free_relocs(obj); 6365 return 0; 6366 } 6367 6368 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 6369 6370 static struct bpf_object * 6371 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, 6372 const struct bpf_object_open_opts *opts) 6373 { 6374 const char *obj_name, *kconfig, *btf_tmp_path; 6375 struct bpf_program *prog; 6376 struct bpf_object *obj; 6377 char tmp_name[64]; 6378 int err; 6379 6380 if (elf_version(EV_CURRENT) == EV_NONE) { 6381 pr_warn("failed to init libelf for %s\n", 6382 path ? : "(mem buf)"); 6383 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 6384 } 6385 6386 if (!OPTS_VALID(opts, bpf_object_open_opts)) 6387 return ERR_PTR(-EINVAL); 6388 6389 obj_name = OPTS_GET(opts, object_name, NULL); 6390 if (obj_buf) { 6391 if (!obj_name) { 6392 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 6393 (unsigned long)obj_buf, 6394 (unsigned long)obj_buf_sz); 6395 obj_name = tmp_name; 6396 } 6397 path = obj_name; 6398 pr_debug("loading object '%s' from buffer\n", obj_name); 6399 } 6400 6401 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 6402 if (IS_ERR(obj)) 6403 return obj; 6404 6405 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 6406 if (btf_tmp_path) { 6407 if (strlen(btf_tmp_path) >= PATH_MAX) { 6408 err = -ENAMETOOLONG; 6409 goto out; 6410 } 6411 obj->btf_custom_path = strdup(btf_tmp_path); 6412 if (!obj->btf_custom_path) { 6413 err = -ENOMEM; 6414 goto out; 6415 } 6416 } 6417 6418 kconfig = OPTS_GET(opts, kconfig, NULL); 6419 if (kconfig) { 6420 obj->kconfig = strdup(kconfig); 6421 if (!obj->kconfig) { 6422 err = -ENOMEM; 6423 goto out; 6424 } 6425 } 6426 6427 err = bpf_object__elf_init(obj); 6428 err = err ? : bpf_object__check_endianness(obj); 6429 err = err ? : bpf_object__elf_collect(obj); 6430 err = err ? : bpf_object__collect_externs(obj); 6431 err = err ? : bpf_object__finalize_btf(obj); 6432 err = err ? : bpf_object__init_maps(obj, opts); 6433 err = err ? : bpf_object__collect_relos(obj); 6434 if (err) 6435 goto out; 6436 bpf_object__elf_finish(obj); 6437 6438 bpf_object__for_each_program(prog, obj) { 6439 prog->sec_def = find_sec_def(prog->sec_name); 6440 if (!prog->sec_def) { 6441 /* couldn't guess, but user might manually specify */ 6442 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 6443 prog->name, prog->sec_name); 6444 continue; 6445 } 6446 6447 if (prog->sec_def->is_sleepable) 6448 prog->prog_flags |= BPF_F_SLEEPABLE; 6449 bpf_program__set_type(prog, prog->sec_def->prog_type); 6450 bpf_program__set_expected_attach_type(prog, 6451 prog->sec_def->expected_attach_type); 6452 6453 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || 6454 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) 6455 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); 6456 } 6457 6458 return obj; 6459 out: 6460 bpf_object__close(obj); 6461 return ERR_PTR(err); 6462 } 6463 6464 static struct bpf_object * 6465 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) 6466 { 6467 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 6468 .relaxed_maps = flags & MAPS_RELAX_COMPAT, 6469 ); 6470 6471 /* param validation */ 6472 if (!attr->file) 6473 return NULL; 6474 6475 pr_debug("loading %s\n", attr->file); 6476 return __bpf_object__open(attr->file, NULL, 0, &opts); 6477 } 6478 6479 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 6480 { 6481 return libbpf_ptr(__bpf_object__open_xattr(attr, 0)); 6482 } 6483 6484 struct bpf_object *bpf_object__open(const char *path) 6485 { 6486 struct bpf_object_open_attr attr = { 6487 .file = path, 6488 .prog_type = BPF_PROG_TYPE_UNSPEC, 6489 }; 6490 6491 return libbpf_ptr(__bpf_object__open_xattr(&attr, 0)); 6492 } 6493 6494 struct bpf_object * 6495 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 6496 { 6497 if (!path) 6498 return libbpf_err_ptr(-EINVAL); 6499 6500 pr_debug("loading %s\n", path); 6501 6502 return libbpf_ptr(__bpf_object__open(path, NULL, 0, opts)); 6503 } 6504 6505 struct bpf_object * 6506 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 6507 const struct bpf_object_open_opts *opts) 6508 { 6509 if (!obj_buf || obj_buf_sz == 0) 6510 return libbpf_err_ptr(-EINVAL); 6511 6512 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, opts)); 6513 } 6514 6515 struct bpf_object * 6516 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, 6517 const char *name) 6518 { 6519 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 6520 .object_name = name, 6521 /* wrong default, but backwards-compatible */ 6522 .relaxed_maps = true, 6523 ); 6524 6525 /* returning NULL is wrong, but backwards-compatible */ 6526 if (!obj_buf || obj_buf_sz == 0) 6527 return errno = EINVAL, NULL; 6528 6529 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, &opts)); 6530 } 6531 6532 int bpf_object__unload(struct bpf_object *obj) 6533 { 6534 size_t i; 6535 6536 if (!obj) 6537 return libbpf_err(-EINVAL); 6538 6539 for (i = 0; i < obj->nr_maps; i++) { 6540 zclose(obj->maps[i].fd); 6541 if (obj->maps[i].st_ops) 6542 zfree(&obj->maps[i].st_ops->kern_vdata); 6543 } 6544 6545 for (i = 0; i < obj->nr_programs; i++) 6546 bpf_program__unload(&obj->programs[i]); 6547 6548 return 0; 6549 } 6550 6551 static int bpf_object__sanitize_maps(struct bpf_object *obj) 6552 { 6553 struct bpf_map *m; 6554 6555 bpf_object__for_each_map(m, obj) { 6556 if (!bpf_map__is_internal(m)) 6557 continue; 6558 if (!kernel_supports(obj, FEAT_GLOBAL_DATA)) { 6559 pr_warn("kernel doesn't support global data\n"); 6560 return -ENOTSUP; 6561 } 6562 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 6563 m->def.map_flags ^= BPF_F_MMAPABLE; 6564 } 6565 6566 return 0; 6567 } 6568 6569 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 6570 { 6571 char sym_type, sym_name[500]; 6572 unsigned long long sym_addr; 6573 const struct btf_type *t; 6574 struct extern_desc *ext; 6575 int ret, err = 0; 6576 FILE *f; 6577 6578 f = fopen("/proc/kallsyms", "r"); 6579 if (!f) { 6580 err = -errno; 6581 pr_warn("failed to open /proc/kallsyms: %d\n", err); 6582 return err; 6583 } 6584 6585 while (true) { 6586 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 6587 &sym_addr, &sym_type, sym_name); 6588 if (ret == EOF && feof(f)) 6589 break; 6590 if (ret != 3) { 6591 pr_warn("failed to read kallsyms entry: %d\n", ret); 6592 err = -EINVAL; 6593 goto out; 6594 } 6595 6596 ext = find_extern_by_name(obj, sym_name); 6597 if (!ext || ext->type != EXT_KSYM) 6598 continue; 6599 6600 t = btf__type_by_id(obj->btf, ext->btf_id); 6601 if (!btf_is_var(t)) 6602 continue; 6603 6604 if (ext->is_set && ext->ksym.addr != sym_addr) { 6605 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", 6606 sym_name, ext->ksym.addr, sym_addr); 6607 err = -EINVAL; 6608 goto out; 6609 } 6610 if (!ext->is_set) { 6611 ext->is_set = true; 6612 ext->ksym.addr = sym_addr; 6613 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); 6614 } 6615 } 6616 6617 out: 6618 fclose(f); 6619 return err; 6620 } 6621 6622 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 6623 __u16 kind, struct btf **res_btf, 6624 int *res_btf_fd) 6625 { 6626 int i, id, btf_fd, err; 6627 struct btf *btf; 6628 6629 btf = obj->btf_vmlinux; 6630 btf_fd = 0; 6631 id = btf__find_by_name_kind(btf, ksym_name, kind); 6632 6633 if (id == -ENOENT) { 6634 err = load_module_btfs(obj); 6635 if (err) 6636 return err; 6637 6638 for (i = 0; i < obj->btf_module_cnt; i++) { 6639 btf = obj->btf_modules[i].btf; 6640 /* we assume module BTF FD is always >0 */ 6641 btf_fd = obj->btf_modules[i].fd; 6642 id = btf__find_by_name_kind(btf, ksym_name, kind); 6643 if (id != -ENOENT) 6644 break; 6645 } 6646 } 6647 if (id <= 0) 6648 return -ESRCH; 6649 6650 *res_btf = btf; 6651 *res_btf_fd = btf_fd; 6652 return id; 6653 } 6654 6655 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 6656 struct extern_desc *ext) 6657 { 6658 const struct btf_type *targ_var, *targ_type; 6659 __u32 targ_type_id, local_type_id; 6660 const char *targ_var_name; 6661 int id, btf_fd = 0, err; 6662 struct btf *btf = NULL; 6663 6664 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd); 6665 if (id == -ESRCH && ext->is_weak) { 6666 return 0; 6667 } else if (id < 0) { 6668 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 6669 ext->name); 6670 return id; 6671 } 6672 6673 /* find local type_id */ 6674 local_type_id = ext->ksym.type_id; 6675 6676 /* find target type_id */ 6677 targ_var = btf__type_by_id(btf, id); 6678 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 6679 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 6680 6681 err = bpf_core_types_are_compat(obj->btf, local_type_id, 6682 btf, targ_type_id); 6683 if (err <= 0) { 6684 const struct btf_type *local_type; 6685 const char *targ_name, *local_name; 6686 6687 local_type = btf__type_by_id(obj->btf, local_type_id); 6688 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 6689 targ_name = btf__name_by_offset(btf, targ_type->name_off); 6690 6691 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 6692 ext->name, local_type_id, 6693 btf_kind_str(local_type), local_name, targ_type_id, 6694 btf_kind_str(targ_type), targ_name); 6695 return -EINVAL; 6696 } 6697 6698 ext->is_set = true; 6699 ext->ksym.kernel_btf_obj_fd = btf_fd; 6700 ext->ksym.kernel_btf_id = id; 6701 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 6702 ext->name, id, btf_kind_str(targ_var), targ_var_name); 6703 6704 return 0; 6705 } 6706 6707 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 6708 struct extern_desc *ext) 6709 { 6710 int local_func_proto_id, kfunc_proto_id, kfunc_id; 6711 const struct btf_type *kern_func; 6712 struct btf *kern_btf = NULL; 6713 int ret, kern_btf_fd = 0; 6714 6715 local_func_proto_id = ext->ksym.type_id; 6716 6717 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, 6718 &kern_btf, &kern_btf_fd); 6719 if (kfunc_id < 0) { 6720 pr_warn("extern (func ksym) '%s': not found in kernel BTF\n", 6721 ext->name); 6722 return kfunc_id; 6723 } 6724 6725 if (kern_btf != obj->btf_vmlinux) { 6726 pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n", 6727 ext->name); 6728 return -ENOTSUP; 6729 } 6730 6731 kern_func = btf__type_by_id(kern_btf, kfunc_id); 6732 kfunc_proto_id = kern_func->type; 6733 6734 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 6735 kern_btf, kfunc_proto_id); 6736 if (ret <= 0) { 6737 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n", 6738 ext->name, local_func_proto_id, kfunc_proto_id); 6739 return -EINVAL; 6740 } 6741 6742 ext->is_set = true; 6743 ext->ksym.kernel_btf_obj_fd = kern_btf_fd; 6744 ext->ksym.kernel_btf_id = kfunc_id; 6745 pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n", 6746 ext->name, kfunc_id); 6747 6748 return 0; 6749 } 6750 6751 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 6752 { 6753 const struct btf_type *t; 6754 struct extern_desc *ext; 6755 int i, err; 6756 6757 for (i = 0; i < obj->nr_extern; i++) { 6758 ext = &obj->externs[i]; 6759 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 6760 continue; 6761 6762 if (obj->gen_loader) { 6763 ext->is_set = true; 6764 ext->ksym.kernel_btf_obj_fd = 0; 6765 ext->ksym.kernel_btf_id = 0; 6766 continue; 6767 } 6768 t = btf__type_by_id(obj->btf, ext->btf_id); 6769 if (btf_is_var(t)) 6770 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 6771 else 6772 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 6773 if (err) 6774 return err; 6775 } 6776 return 0; 6777 } 6778 6779 static int bpf_object__resolve_externs(struct bpf_object *obj, 6780 const char *extra_kconfig) 6781 { 6782 bool need_config = false, need_kallsyms = false; 6783 bool need_vmlinux_btf = false; 6784 struct extern_desc *ext; 6785 void *kcfg_data = NULL; 6786 int err, i; 6787 6788 if (obj->nr_extern == 0) 6789 return 0; 6790 6791 if (obj->kconfig_map_idx >= 0) 6792 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 6793 6794 for (i = 0; i < obj->nr_extern; i++) { 6795 ext = &obj->externs[i]; 6796 6797 if (ext->type == EXT_KCFG && 6798 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 6799 void *ext_val = kcfg_data + ext->kcfg.data_off; 6800 __u32 kver = get_kernel_version(); 6801 6802 if (!kver) { 6803 pr_warn("failed to get kernel version\n"); 6804 return -EINVAL; 6805 } 6806 err = set_kcfg_value_num(ext, ext_val, kver); 6807 if (err) 6808 return err; 6809 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); 6810 } else if (ext->type == EXT_KCFG && 6811 strncmp(ext->name, "CONFIG_", 7) == 0) { 6812 need_config = true; 6813 } else if (ext->type == EXT_KSYM) { 6814 if (ext->ksym.type_id) 6815 need_vmlinux_btf = true; 6816 else 6817 need_kallsyms = true; 6818 } else { 6819 pr_warn("unrecognized extern '%s'\n", ext->name); 6820 return -EINVAL; 6821 } 6822 } 6823 if (need_config && extra_kconfig) { 6824 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 6825 if (err) 6826 return -EINVAL; 6827 need_config = false; 6828 for (i = 0; i < obj->nr_extern; i++) { 6829 ext = &obj->externs[i]; 6830 if (ext->type == EXT_KCFG && !ext->is_set) { 6831 need_config = true; 6832 break; 6833 } 6834 } 6835 } 6836 if (need_config) { 6837 err = bpf_object__read_kconfig_file(obj, kcfg_data); 6838 if (err) 6839 return -EINVAL; 6840 } 6841 if (need_kallsyms) { 6842 err = bpf_object__read_kallsyms_file(obj); 6843 if (err) 6844 return -EINVAL; 6845 } 6846 if (need_vmlinux_btf) { 6847 err = bpf_object__resolve_ksyms_btf_id(obj); 6848 if (err) 6849 return -EINVAL; 6850 } 6851 for (i = 0; i < obj->nr_extern; i++) { 6852 ext = &obj->externs[i]; 6853 6854 if (!ext->is_set && !ext->is_weak) { 6855 pr_warn("extern %s (strong) not resolved\n", ext->name); 6856 return -ESRCH; 6857 } else if (!ext->is_set) { 6858 pr_debug("extern %s (weak) not resolved, defaulting to zero\n", 6859 ext->name); 6860 } 6861 } 6862 6863 return 0; 6864 } 6865 6866 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 6867 { 6868 struct bpf_object *obj; 6869 int err, i; 6870 6871 if (!attr) 6872 return libbpf_err(-EINVAL); 6873 obj = attr->obj; 6874 if (!obj) 6875 return libbpf_err(-EINVAL); 6876 6877 if (obj->loaded) { 6878 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 6879 return libbpf_err(-EINVAL); 6880 } 6881 6882 if (obj->gen_loader) 6883 bpf_gen__init(obj->gen_loader, attr->log_level); 6884 6885 err = bpf_object__probe_loading(obj); 6886 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 6887 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 6888 err = err ? : bpf_object__sanitize_and_load_btf(obj); 6889 err = err ? : bpf_object__sanitize_maps(obj); 6890 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 6891 err = err ? : bpf_object__create_maps(obj); 6892 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : attr->target_btf_path); 6893 err = err ? : bpf_object__load_progs(obj, attr->log_level); 6894 6895 if (obj->gen_loader) { 6896 /* reset FDs */ 6897 if (obj->btf) 6898 btf__set_fd(obj->btf, -1); 6899 for (i = 0; i < obj->nr_maps; i++) 6900 obj->maps[i].fd = -1; 6901 if (!err) 6902 err = bpf_gen__finish(obj->gen_loader); 6903 } 6904 6905 /* clean up module BTFs */ 6906 for (i = 0; i < obj->btf_module_cnt; i++) { 6907 close(obj->btf_modules[i].fd); 6908 btf__free(obj->btf_modules[i].btf); 6909 free(obj->btf_modules[i].name); 6910 } 6911 free(obj->btf_modules); 6912 6913 /* clean up vmlinux BTF */ 6914 btf__free(obj->btf_vmlinux); 6915 obj->btf_vmlinux = NULL; 6916 6917 obj->loaded = true; /* doesn't matter if successfully or not */ 6918 6919 if (err) 6920 goto out; 6921 6922 return 0; 6923 out: 6924 /* unpin any maps that were auto-pinned during load */ 6925 for (i = 0; i < obj->nr_maps; i++) 6926 if (obj->maps[i].pinned && !obj->maps[i].reused) 6927 bpf_map__unpin(&obj->maps[i], NULL); 6928 6929 bpf_object__unload(obj); 6930 pr_warn("failed to load object '%s'\n", obj->path); 6931 return libbpf_err(err); 6932 } 6933 6934 int bpf_object__load(struct bpf_object *obj) 6935 { 6936 struct bpf_object_load_attr attr = { 6937 .obj = obj, 6938 }; 6939 6940 return bpf_object__load_xattr(&attr); 6941 } 6942 6943 static int make_parent_dir(const char *path) 6944 { 6945 char *cp, errmsg[STRERR_BUFSIZE]; 6946 char *dname, *dir; 6947 int err = 0; 6948 6949 dname = strdup(path); 6950 if (dname == NULL) 6951 return -ENOMEM; 6952 6953 dir = dirname(dname); 6954 if (mkdir(dir, 0700) && errno != EEXIST) 6955 err = -errno; 6956 6957 free(dname); 6958 if (err) { 6959 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 6960 pr_warn("failed to mkdir %s: %s\n", path, cp); 6961 } 6962 return err; 6963 } 6964 6965 static int check_path(const char *path) 6966 { 6967 char *cp, errmsg[STRERR_BUFSIZE]; 6968 struct statfs st_fs; 6969 char *dname, *dir; 6970 int err = 0; 6971 6972 if (path == NULL) 6973 return -EINVAL; 6974 6975 dname = strdup(path); 6976 if (dname == NULL) 6977 return -ENOMEM; 6978 6979 dir = dirname(dname); 6980 if (statfs(dir, &st_fs)) { 6981 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6982 pr_warn("failed to statfs %s: %s\n", dir, cp); 6983 err = -errno; 6984 } 6985 free(dname); 6986 6987 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 6988 pr_warn("specified path %s is not on BPF FS\n", path); 6989 err = -EINVAL; 6990 } 6991 6992 return err; 6993 } 6994 6995 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 6996 int instance) 6997 { 6998 char *cp, errmsg[STRERR_BUFSIZE]; 6999 int err; 7000 7001 err = make_parent_dir(path); 7002 if (err) 7003 return libbpf_err(err); 7004 7005 err = check_path(path); 7006 if (err) 7007 return libbpf_err(err); 7008 7009 if (prog == NULL) { 7010 pr_warn("invalid program pointer\n"); 7011 return libbpf_err(-EINVAL); 7012 } 7013 7014 if (instance < 0 || instance >= prog->instances.nr) { 7015 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7016 instance, prog->name, prog->instances.nr); 7017 return libbpf_err(-EINVAL); 7018 } 7019 7020 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 7021 err = -errno; 7022 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 7023 pr_warn("failed to pin program: %s\n", cp); 7024 return libbpf_err(err); 7025 } 7026 pr_debug("pinned program '%s'\n", path); 7027 7028 return 0; 7029 } 7030 7031 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 7032 int instance) 7033 { 7034 int err; 7035 7036 err = check_path(path); 7037 if (err) 7038 return libbpf_err(err); 7039 7040 if (prog == NULL) { 7041 pr_warn("invalid program pointer\n"); 7042 return libbpf_err(-EINVAL); 7043 } 7044 7045 if (instance < 0 || instance >= prog->instances.nr) { 7046 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7047 instance, prog->name, prog->instances.nr); 7048 return libbpf_err(-EINVAL); 7049 } 7050 7051 err = unlink(path); 7052 if (err != 0) 7053 return libbpf_err(-errno); 7054 7055 pr_debug("unpinned program '%s'\n", path); 7056 7057 return 0; 7058 } 7059 7060 int bpf_program__pin(struct bpf_program *prog, const char *path) 7061 { 7062 int i, err; 7063 7064 err = make_parent_dir(path); 7065 if (err) 7066 return libbpf_err(err); 7067 7068 err = check_path(path); 7069 if (err) 7070 return libbpf_err(err); 7071 7072 if (prog == NULL) { 7073 pr_warn("invalid program pointer\n"); 7074 return libbpf_err(-EINVAL); 7075 } 7076 7077 if (prog->instances.nr <= 0) { 7078 pr_warn("no instances of prog %s to pin\n", prog->name); 7079 return libbpf_err(-EINVAL); 7080 } 7081 7082 if (prog->instances.nr == 1) { 7083 /* don't create subdirs when pinning single instance */ 7084 return bpf_program__pin_instance(prog, path, 0); 7085 } 7086 7087 for (i = 0; i < prog->instances.nr; i++) { 7088 char buf[PATH_MAX]; 7089 int len; 7090 7091 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7092 if (len < 0) { 7093 err = -EINVAL; 7094 goto err_unpin; 7095 } else if (len >= PATH_MAX) { 7096 err = -ENAMETOOLONG; 7097 goto err_unpin; 7098 } 7099 7100 err = bpf_program__pin_instance(prog, buf, i); 7101 if (err) 7102 goto err_unpin; 7103 } 7104 7105 return 0; 7106 7107 err_unpin: 7108 for (i = i - 1; i >= 0; i--) { 7109 char buf[PATH_MAX]; 7110 int len; 7111 7112 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7113 if (len < 0) 7114 continue; 7115 else if (len >= PATH_MAX) 7116 continue; 7117 7118 bpf_program__unpin_instance(prog, buf, i); 7119 } 7120 7121 rmdir(path); 7122 7123 return libbpf_err(err); 7124 } 7125 7126 int bpf_program__unpin(struct bpf_program *prog, const char *path) 7127 { 7128 int i, err; 7129 7130 err = check_path(path); 7131 if (err) 7132 return libbpf_err(err); 7133 7134 if (prog == NULL) { 7135 pr_warn("invalid program pointer\n"); 7136 return libbpf_err(-EINVAL); 7137 } 7138 7139 if (prog->instances.nr <= 0) { 7140 pr_warn("no instances of prog %s to pin\n", prog->name); 7141 return libbpf_err(-EINVAL); 7142 } 7143 7144 if (prog->instances.nr == 1) { 7145 /* don't create subdirs when pinning single instance */ 7146 return bpf_program__unpin_instance(prog, path, 0); 7147 } 7148 7149 for (i = 0; i < prog->instances.nr; i++) { 7150 char buf[PATH_MAX]; 7151 int len; 7152 7153 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7154 if (len < 0) 7155 return libbpf_err(-EINVAL); 7156 else if (len >= PATH_MAX) 7157 return libbpf_err(-ENAMETOOLONG); 7158 7159 err = bpf_program__unpin_instance(prog, buf, i); 7160 if (err) 7161 return err; 7162 } 7163 7164 err = rmdir(path); 7165 if (err) 7166 return libbpf_err(-errno); 7167 7168 return 0; 7169 } 7170 7171 int bpf_map__pin(struct bpf_map *map, const char *path) 7172 { 7173 char *cp, errmsg[STRERR_BUFSIZE]; 7174 int err; 7175 7176 if (map == NULL) { 7177 pr_warn("invalid map pointer\n"); 7178 return libbpf_err(-EINVAL); 7179 } 7180 7181 if (map->pin_path) { 7182 if (path && strcmp(path, map->pin_path)) { 7183 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 7184 bpf_map__name(map), map->pin_path, path); 7185 return libbpf_err(-EINVAL); 7186 } else if (map->pinned) { 7187 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 7188 bpf_map__name(map), map->pin_path); 7189 return 0; 7190 } 7191 } else { 7192 if (!path) { 7193 pr_warn("missing a path to pin map '%s' at\n", 7194 bpf_map__name(map)); 7195 return libbpf_err(-EINVAL); 7196 } else if (map->pinned) { 7197 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 7198 return libbpf_err(-EEXIST); 7199 } 7200 7201 map->pin_path = strdup(path); 7202 if (!map->pin_path) { 7203 err = -errno; 7204 goto out_err; 7205 } 7206 } 7207 7208 err = make_parent_dir(map->pin_path); 7209 if (err) 7210 return libbpf_err(err); 7211 7212 err = check_path(map->pin_path); 7213 if (err) 7214 return libbpf_err(err); 7215 7216 if (bpf_obj_pin(map->fd, map->pin_path)) { 7217 err = -errno; 7218 goto out_err; 7219 } 7220 7221 map->pinned = true; 7222 pr_debug("pinned map '%s'\n", map->pin_path); 7223 7224 return 0; 7225 7226 out_err: 7227 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 7228 pr_warn("failed to pin map: %s\n", cp); 7229 return libbpf_err(err); 7230 } 7231 7232 int bpf_map__unpin(struct bpf_map *map, const char *path) 7233 { 7234 int err; 7235 7236 if (map == NULL) { 7237 pr_warn("invalid map pointer\n"); 7238 return libbpf_err(-EINVAL); 7239 } 7240 7241 if (map->pin_path) { 7242 if (path && strcmp(path, map->pin_path)) { 7243 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 7244 bpf_map__name(map), map->pin_path, path); 7245 return libbpf_err(-EINVAL); 7246 } 7247 path = map->pin_path; 7248 } else if (!path) { 7249 pr_warn("no path to unpin map '%s' from\n", 7250 bpf_map__name(map)); 7251 return libbpf_err(-EINVAL); 7252 } 7253 7254 err = check_path(path); 7255 if (err) 7256 return libbpf_err(err); 7257 7258 err = unlink(path); 7259 if (err != 0) 7260 return libbpf_err(-errno); 7261 7262 map->pinned = false; 7263 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 7264 7265 return 0; 7266 } 7267 7268 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 7269 { 7270 char *new = NULL; 7271 7272 if (path) { 7273 new = strdup(path); 7274 if (!new) 7275 return libbpf_err(-errno); 7276 } 7277 7278 free(map->pin_path); 7279 map->pin_path = new; 7280 return 0; 7281 } 7282 7283 const char *bpf_map__get_pin_path(const struct bpf_map *map) 7284 { 7285 return map->pin_path; 7286 } 7287 7288 const char *bpf_map__pin_path(const struct bpf_map *map) 7289 { 7290 return map->pin_path; 7291 } 7292 7293 bool bpf_map__is_pinned(const struct bpf_map *map) 7294 { 7295 return map->pinned; 7296 } 7297 7298 static void sanitize_pin_path(char *s) 7299 { 7300 /* bpffs disallows periods in path names */ 7301 while (*s) { 7302 if (*s == '.') 7303 *s = '_'; 7304 s++; 7305 } 7306 } 7307 7308 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 7309 { 7310 struct bpf_map *map; 7311 int err; 7312 7313 if (!obj) 7314 return libbpf_err(-ENOENT); 7315 7316 if (!obj->loaded) { 7317 pr_warn("object not yet loaded; load it first\n"); 7318 return libbpf_err(-ENOENT); 7319 } 7320 7321 bpf_object__for_each_map(map, obj) { 7322 char *pin_path = NULL; 7323 char buf[PATH_MAX]; 7324 7325 if (path) { 7326 int len; 7327 7328 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7329 bpf_map__name(map)); 7330 if (len < 0) { 7331 err = -EINVAL; 7332 goto err_unpin_maps; 7333 } else if (len >= PATH_MAX) { 7334 err = -ENAMETOOLONG; 7335 goto err_unpin_maps; 7336 } 7337 sanitize_pin_path(buf); 7338 pin_path = buf; 7339 } else if (!map->pin_path) { 7340 continue; 7341 } 7342 7343 err = bpf_map__pin(map, pin_path); 7344 if (err) 7345 goto err_unpin_maps; 7346 } 7347 7348 return 0; 7349 7350 err_unpin_maps: 7351 while ((map = bpf_map__prev(map, obj))) { 7352 if (!map->pin_path) 7353 continue; 7354 7355 bpf_map__unpin(map, NULL); 7356 } 7357 7358 return libbpf_err(err); 7359 } 7360 7361 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 7362 { 7363 struct bpf_map *map; 7364 int err; 7365 7366 if (!obj) 7367 return libbpf_err(-ENOENT); 7368 7369 bpf_object__for_each_map(map, obj) { 7370 char *pin_path = NULL; 7371 char buf[PATH_MAX]; 7372 7373 if (path) { 7374 int len; 7375 7376 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7377 bpf_map__name(map)); 7378 if (len < 0) 7379 return libbpf_err(-EINVAL); 7380 else if (len >= PATH_MAX) 7381 return libbpf_err(-ENAMETOOLONG); 7382 sanitize_pin_path(buf); 7383 pin_path = buf; 7384 } else if (!map->pin_path) { 7385 continue; 7386 } 7387 7388 err = bpf_map__unpin(map, pin_path); 7389 if (err) 7390 return libbpf_err(err); 7391 } 7392 7393 return 0; 7394 } 7395 7396 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 7397 { 7398 struct bpf_program *prog; 7399 int err; 7400 7401 if (!obj) 7402 return libbpf_err(-ENOENT); 7403 7404 if (!obj->loaded) { 7405 pr_warn("object not yet loaded; load it first\n"); 7406 return libbpf_err(-ENOENT); 7407 } 7408 7409 bpf_object__for_each_program(prog, obj) { 7410 char buf[PATH_MAX]; 7411 int len; 7412 7413 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7414 prog->pin_name); 7415 if (len < 0) { 7416 err = -EINVAL; 7417 goto err_unpin_programs; 7418 } else if (len >= PATH_MAX) { 7419 err = -ENAMETOOLONG; 7420 goto err_unpin_programs; 7421 } 7422 7423 err = bpf_program__pin(prog, buf); 7424 if (err) 7425 goto err_unpin_programs; 7426 } 7427 7428 return 0; 7429 7430 err_unpin_programs: 7431 while ((prog = bpf_program__prev(prog, obj))) { 7432 char buf[PATH_MAX]; 7433 int len; 7434 7435 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7436 prog->pin_name); 7437 if (len < 0) 7438 continue; 7439 else if (len >= PATH_MAX) 7440 continue; 7441 7442 bpf_program__unpin(prog, buf); 7443 } 7444 7445 return libbpf_err(err); 7446 } 7447 7448 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 7449 { 7450 struct bpf_program *prog; 7451 int err; 7452 7453 if (!obj) 7454 return libbpf_err(-ENOENT); 7455 7456 bpf_object__for_each_program(prog, obj) { 7457 char buf[PATH_MAX]; 7458 int len; 7459 7460 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7461 prog->pin_name); 7462 if (len < 0) 7463 return libbpf_err(-EINVAL); 7464 else if (len >= PATH_MAX) 7465 return libbpf_err(-ENAMETOOLONG); 7466 7467 err = bpf_program__unpin(prog, buf); 7468 if (err) 7469 return libbpf_err(err); 7470 } 7471 7472 return 0; 7473 } 7474 7475 int bpf_object__pin(struct bpf_object *obj, const char *path) 7476 { 7477 int err; 7478 7479 err = bpf_object__pin_maps(obj, path); 7480 if (err) 7481 return libbpf_err(err); 7482 7483 err = bpf_object__pin_programs(obj, path); 7484 if (err) { 7485 bpf_object__unpin_maps(obj, path); 7486 return libbpf_err(err); 7487 } 7488 7489 return 0; 7490 } 7491 7492 static void bpf_map__destroy(struct bpf_map *map) 7493 { 7494 if (map->clear_priv) 7495 map->clear_priv(map, map->priv); 7496 map->priv = NULL; 7497 map->clear_priv = NULL; 7498 7499 if (map->inner_map) { 7500 bpf_map__destroy(map->inner_map); 7501 zfree(&map->inner_map); 7502 } 7503 7504 zfree(&map->init_slots); 7505 map->init_slots_sz = 0; 7506 7507 if (map->mmaped) { 7508 munmap(map->mmaped, bpf_map_mmap_sz(map)); 7509 map->mmaped = NULL; 7510 } 7511 7512 if (map->st_ops) { 7513 zfree(&map->st_ops->data); 7514 zfree(&map->st_ops->progs); 7515 zfree(&map->st_ops->kern_func_off); 7516 zfree(&map->st_ops); 7517 } 7518 7519 zfree(&map->name); 7520 zfree(&map->pin_path); 7521 7522 if (map->fd >= 0) 7523 zclose(map->fd); 7524 } 7525 7526 void bpf_object__close(struct bpf_object *obj) 7527 { 7528 size_t i; 7529 7530 if (IS_ERR_OR_NULL(obj)) 7531 return; 7532 7533 if (obj->clear_priv) 7534 obj->clear_priv(obj, obj->priv); 7535 7536 bpf_gen__free(obj->gen_loader); 7537 bpf_object__elf_finish(obj); 7538 bpf_object__unload(obj); 7539 btf__free(obj->btf); 7540 btf_ext__free(obj->btf_ext); 7541 7542 for (i = 0; i < obj->nr_maps; i++) 7543 bpf_map__destroy(&obj->maps[i]); 7544 7545 zfree(&obj->btf_custom_path); 7546 zfree(&obj->kconfig); 7547 zfree(&obj->externs); 7548 obj->nr_extern = 0; 7549 7550 zfree(&obj->maps); 7551 obj->nr_maps = 0; 7552 7553 if (obj->programs && obj->nr_programs) { 7554 for (i = 0; i < obj->nr_programs; i++) 7555 bpf_program__exit(&obj->programs[i]); 7556 } 7557 zfree(&obj->programs); 7558 7559 list_del(&obj->list); 7560 free(obj); 7561 } 7562 7563 struct bpf_object * 7564 bpf_object__next(struct bpf_object *prev) 7565 { 7566 struct bpf_object *next; 7567 7568 if (!prev) 7569 next = list_first_entry(&bpf_objects_list, 7570 struct bpf_object, 7571 list); 7572 else 7573 next = list_next_entry(prev, list); 7574 7575 /* Empty list is noticed here so don't need checking on entry. */ 7576 if (&next->list == &bpf_objects_list) 7577 return NULL; 7578 7579 return next; 7580 } 7581 7582 const char *bpf_object__name(const struct bpf_object *obj) 7583 { 7584 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 7585 } 7586 7587 unsigned int bpf_object__kversion(const struct bpf_object *obj) 7588 { 7589 return obj ? obj->kern_version : 0; 7590 } 7591 7592 struct btf *bpf_object__btf(const struct bpf_object *obj) 7593 { 7594 return obj ? obj->btf : NULL; 7595 } 7596 7597 int bpf_object__btf_fd(const struct bpf_object *obj) 7598 { 7599 return obj->btf ? btf__fd(obj->btf) : -1; 7600 } 7601 7602 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 7603 { 7604 if (obj->loaded) 7605 return libbpf_err(-EINVAL); 7606 7607 obj->kern_version = kern_version; 7608 7609 return 0; 7610 } 7611 7612 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 7613 bpf_object_clear_priv_t clear_priv) 7614 { 7615 if (obj->priv && obj->clear_priv) 7616 obj->clear_priv(obj, obj->priv); 7617 7618 obj->priv = priv; 7619 obj->clear_priv = clear_priv; 7620 return 0; 7621 } 7622 7623 void *bpf_object__priv(const struct bpf_object *obj) 7624 { 7625 return obj ? obj->priv : libbpf_err_ptr(-EINVAL); 7626 } 7627 7628 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 7629 { 7630 struct bpf_gen *gen; 7631 7632 if (!opts) 7633 return -EFAULT; 7634 if (!OPTS_VALID(opts, gen_loader_opts)) 7635 return -EINVAL; 7636 gen = calloc(sizeof(*gen), 1); 7637 if (!gen) 7638 return -ENOMEM; 7639 gen->opts = opts; 7640 obj->gen_loader = gen; 7641 return 0; 7642 } 7643 7644 static struct bpf_program * 7645 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 7646 bool forward) 7647 { 7648 size_t nr_programs = obj->nr_programs; 7649 ssize_t idx; 7650 7651 if (!nr_programs) 7652 return NULL; 7653 7654 if (!p) 7655 /* Iter from the beginning */ 7656 return forward ? &obj->programs[0] : 7657 &obj->programs[nr_programs - 1]; 7658 7659 if (p->obj != obj) { 7660 pr_warn("error: program handler doesn't match object\n"); 7661 return errno = EINVAL, NULL; 7662 } 7663 7664 idx = (p - obj->programs) + (forward ? 1 : -1); 7665 if (idx >= obj->nr_programs || idx < 0) 7666 return NULL; 7667 return &obj->programs[idx]; 7668 } 7669 7670 struct bpf_program * 7671 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 7672 { 7673 struct bpf_program *prog = prev; 7674 7675 do { 7676 prog = __bpf_program__iter(prog, obj, true); 7677 } while (prog && prog_is_subprog(obj, prog)); 7678 7679 return prog; 7680 } 7681 7682 struct bpf_program * 7683 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 7684 { 7685 struct bpf_program *prog = next; 7686 7687 do { 7688 prog = __bpf_program__iter(prog, obj, false); 7689 } while (prog && prog_is_subprog(obj, prog)); 7690 7691 return prog; 7692 } 7693 7694 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 7695 bpf_program_clear_priv_t clear_priv) 7696 { 7697 if (prog->priv && prog->clear_priv) 7698 prog->clear_priv(prog, prog->priv); 7699 7700 prog->priv = priv; 7701 prog->clear_priv = clear_priv; 7702 return 0; 7703 } 7704 7705 void *bpf_program__priv(const struct bpf_program *prog) 7706 { 7707 return prog ? prog->priv : libbpf_err_ptr(-EINVAL); 7708 } 7709 7710 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 7711 { 7712 prog->prog_ifindex = ifindex; 7713 } 7714 7715 const char *bpf_program__name(const struct bpf_program *prog) 7716 { 7717 return prog->name; 7718 } 7719 7720 const char *bpf_program__section_name(const struct bpf_program *prog) 7721 { 7722 return prog->sec_name; 7723 } 7724 7725 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 7726 { 7727 const char *title; 7728 7729 title = prog->sec_name; 7730 if (needs_copy) { 7731 title = strdup(title); 7732 if (!title) { 7733 pr_warn("failed to strdup program title\n"); 7734 return libbpf_err_ptr(-ENOMEM); 7735 } 7736 } 7737 7738 return title; 7739 } 7740 7741 bool bpf_program__autoload(const struct bpf_program *prog) 7742 { 7743 return prog->load; 7744 } 7745 7746 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 7747 { 7748 if (prog->obj->loaded) 7749 return libbpf_err(-EINVAL); 7750 7751 prog->load = autoload; 7752 return 0; 7753 } 7754 7755 int bpf_program__fd(const struct bpf_program *prog) 7756 { 7757 return bpf_program__nth_fd(prog, 0); 7758 } 7759 7760 size_t bpf_program__size(const struct bpf_program *prog) 7761 { 7762 return prog->insns_cnt * BPF_INSN_SZ; 7763 } 7764 7765 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 7766 bpf_program_prep_t prep) 7767 { 7768 int *instances_fds; 7769 7770 if (nr_instances <= 0 || !prep) 7771 return libbpf_err(-EINVAL); 7772 7773 if (prog->instances.nr > 0 || prog->instances.fds) { 7774 pr_warn("Can't set pre-processor after loading\n"); 7775 return libbpf_err(-EINVAL); 7776 } 7777 7778 instances_fds = malloc(sizeof(int) * nr_instances); 7779 if (!instances_fds) { 7780 pr_warn("alloc memory failed for fds\n"); 7781 return libbpf_err(-ENOMEM); 7782 } 7783 7784 /* fill all fd with -1 */ 7785 memset(instances_fds, -1, sizeof(int) * nr_instances); 7786 7787 prog->instances.nr = nr_instances; 7788 prog->instances.fds = instances_fds; 7789 prog->preprocessor = prep; 7790 return 0; 7791 } 7792 7793 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 7794 { 7795 int fd; 7796 7797 if (!prog) 7798 return libbpf_err(-EINVAL); 7799 7800 if (n >= prog->instances.nr || n < 0) { 7801 pr_warn("Can't get the %dth fd from program %s: only %d instances\n", 7802 n, prog->name, prog->instances.nr); 7803 return libbpf_err(-EINVAL); 7804 } 7805 7806 fd = prog->instances.fds[n]; 7807 if (fd < 0) { 7808 pr_warn("%dth instance of program '%s' is invalid\n", 7809 n, prog->name); 7810 return libbpf_err(-ENOENT); 7811 } 7812 7813 return fd; 7814 } 7815 7816 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog) 7817 { 7818 return prog->type; 7819 } 7820 7821 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 7822 { 7823 prog->type = type; 7824 } 7825 7826 static bool bpf_program__is_type(const struct bpf_program *prog, 7827 enum bpf_prog_type type) 7828 { 7829 return prog ? (prog->type == type) : false; 7830 } 7831 7832 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 7833 int bpf_program__set_##NAME(struct bpf_program *prog) \ 7834 { \ 7835 if (!prog) \ 7836 return libbpf_err(-EINVAL); \ 7837 bpf_program__set_type(prog, TYPE); \ 7838 return 0; \ 7839 } \ 7840 \ 7841 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 7842 { \ 7843 return bpf_program__is_type(prog, TYPE); \ 7844 } \ 7845 7846 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 7847 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); 7848 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 7849 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 7850 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 7851 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 7852 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 7853 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 7854 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 7855 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); 7856 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); 7857 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); 7858 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP); 7859 7860 enum bpf_attach_type 7861 bpf_program__get_expected_attach_type(const struct bpf_program *prog) 7862 { 7863 return prog->expected_attach_type; 7864 } 7865 7866 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 7867 enum bpf_attach_type type) 7868 { 7869 prog->expected_attach_type = type; 7870 } 7871 7872 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \ 7873 attachable, attach_btf) \ 7874 { \ 7875 .sec = string, \ 7876 .len = sizeof(string) - 1, \ 7877 .prog_type = ptype, \ 7878 .expected_attach_type = eatype, \ 7879 .is_exp_attach_type_optional = eatype_optional, \ 7880 .is_attachable = attachable, \ 7881 .is_attach_btf = attach_btf, \ 7882 } 7883 7884 /* Programs that can NOT be attached. */ 7885 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) 7886 7887 /* Programs that can be attached. */ 7888 #define BPF_APROG_SEC(string, ptype, atype) \ 7889 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0) 7890 7891 /* Programs that must specify expected attach type at load time. */ 7892 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 7893 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0) 7894 7895 /* Programs that use BTF to identify attach point */ 7896 #define BPF_PROG_BTF(string, ptype, eatype) \ 7897 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1) 7898 7899 /* Programs that can be attached but attach type can't be identified by section 7900 * name. Kept for backward compatibility. 7901 */ 7902 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 7903 7904 #define SEC_DEF(sec_pfx, ptype, ...) { \ 7905 .sec = sec_pfx, \ 7906 .len = sizeof(sec_pfx) - 1, \ 7907 .prog_type = BPF_PROG_TYPE_##ptype, \ 7908 __VA_ARGS__ \ 7909 } 7910 7911 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 7912 struct bpf_program *prog); 7913 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 7914 struct bpf_program *prog); 7915 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 7916 struct bpf_program *prog); 7917 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 7918 struct bpf_program *prog); 7919 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 7920 struct bpf_program *prog); 7921 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 7922 struct bpf_program *prog); 7923 7924 static const struct bpf_sec_def section_defs[] = { 7925 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 7926 BPF_EAPROG_SEC("sk_reuseport/migrate", BPF_PROG_TYPE_SK_REUSEPORT, 7927 BPF_SK_REUSEPORT_SELECT_OR_MIGRATE), 7928 BPF_EAPROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT, 7929 BPF_SK_REUSEPORT_SELECT), 7930 SEC_DEF("kprobe/", KPROBE, 7931 .attach_fn = attach_kprobe), 7932 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), 7933 SEC_DEF("kretprobe/", KPROBE, 7934 .attach_fn = attach_kprobe), 7935 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), 7936 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 7937 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 7938 SEC_DEF("tracepoint/", TRACEPOINT, 7939 .attach_fn = attach_tp), 7940 SEC_DEF("tp/", TRACEPOINT, 7941 .attach_fn = attach_tp), 7942 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 7943 .attach_fn = attach_raw_tp), 7944 SEC_DEF("raw_tp/", RAW_TRACEPOINT, 7945 .attach_fn = attach_raw_tp), 7946 SEC_DEF("tp_btf/", TRACING, 7947 .expected_attach_type = BPF_TRACE_RAW_TP, 7948 .is_attach_btf = true, 7949 .attach_fn = attach_trace), 7950 SEC_DEF("fentry/", TRACING, 7951 .expected_attach_type = BPF_TRACE_FENTRY, 7952 .is_attach_btf = true, 7953 .attach_fn = attach_trace), 7954 SEC_DEF("fmod_ret/", TRACING, 7955 .expected_attach_type = BPF_MODIFY_RETURN, 7956 .is_attach_btf = true, 7957 .attach_fn = attach_trace), 7958 SEC_DEF("fexit/", TRACING, 7959 .expected_attach_type = BPF_TRACE_FEXIT, 7960 .is_attach_btf = true, 7961 .attach_fn = attach_trace), 7962 SEC_DEF("fentry.s/", TRACING, 7963 .expected_attach_type = BPF_TRACE_FENTRY, 7964 .is_attach_btf = true, 7965 .is_sleepable = true, 7966 .attach_fn = attach_trace), 7967 SEC_DEF("fmod_ret.s/", TRACING, 7968 .expected_attach_type = BPF_MODIFY_RETURN, 7969 .is_attach_btf = true, 7970 .is_sleepable = true, 7971 .attach_fn = attach_trace), 7972 SEC_DEF("fexit.s/", TRACING, 7973 .expected_attach_type = BPF_TRACE_FEXIT, 7974 .is_attach_btf = true, 7975 .is_sleepable = true, 7976 .attach_fn = attach_trace), 7977 SEC_DEF("freplace/", EXT, 7978 .is_attach_btf = true, 7979 .attach_fn = attach_trace), 7980 SEC_DEF("lsm/", LSM, 7981 .is_attach_btf = true, 7982 .expected_attach_type = BPF_LSM_MAC, 7983 .attach_fn = attach_lsm), 7984 SEC_DEF("lsm.s/", LSM, 7985 .is_attach_btf = true, 7986 .is_sleepable = true, 7987 .expected_attach_type = BPF_LSM_MAC, 7988 .attach_fn = attach_lsm), 7989 SEC_DEF("iter/", TRACING, 7990 .expected_attach_type = BPF_TRACE_ITER, 7991 .is_attach_btf = true, 7992 .attach_fn = attach_iter), 7993 SEC_DEF("syscall", SYSCALL, 7994 .is_sleepable = true), 7995 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP, 7996 BPF_XDP_DEVMAP), 7997 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP, 7998 BPF_XDP_CPUMAP), 7999 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP, 8000 BPF_XDP), 8001 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 8002 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 8003 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 8004 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 8005 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 8006 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 8007 BPF_CGROUP_INET_INGRESS), 8008 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 8009 BPF_CGROUP_INET_EGRESS), 8010 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 8011 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK, 8012 BPF_CGROUP_INET_SOCK_CREATE), 8013 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK, 8014 BPF_CGROUP_INET_SOCK_RELEASE), 8015 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 8016 BPF_CGROUP_INET_SOCK_CREATE), 8017 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 8018 BPF_CGROUP_INET4_POST_BIND), 8019 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 8020 BPF_CGROUP_INET6_POST_BIND), 8021 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 8022 BPF_CGROUP_DEVICE), 8023 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 8024 BPF_CGROUP_SOCK_OPS), 8025 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 8026 BPF_SK_SKB_STREAM_PARSER), 8027 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 8028 BPF_SK_SKB_STREAM_VERDICT), 8029 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 8030 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 8031 BPF_SK_MSG_VERDICT), 8032 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 8033 BPF_LIRC_MODE2), 8034 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 8035 BPF_FLOW_DISSECTOR), 8036 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8037 BPF_CGROUP_INET4_BIND), 8038 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8039 BPF_CGROUP_INET6_BIND), 8040 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8041 BPF_CGROUP_INET4_CONNECT), 8042 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8043 BPF_CGROUP_INET6_CONNECT), 8044 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8045 BPF_CGROUP_UDP4_SENDMSG), 8046 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8047 BPF_CGROUP_UDP6_SENDMSG), 8048 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8049 BPF_CGROUP_UDP4_RECVMSG), 8050 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8051 BPF_CGROUP_UDP6_RECVMSG), 8052 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8053 BPF_CGROUP_INET4_GETPEERNAME), 8054 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8055 BPF_CGROUP_INET6_GETPEERNAME), 8056 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8057 BPF_CGROUP_INET4_GETSOCKNAME), 8058 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8059 BPF_CGROUP_INET6_GETSOCKNAME), 8060 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 8061 BPF_CGROUP_SYSCTL), 8062 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8063 BPF_CGROUP_GETSOCKOPT), 8064 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8065 BPF_CGROUP_SETSOCKOPT), 8066 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), 8067 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP, 8068 BPF_SK_LOOKUP), 8069 }; 8070 8071 #undef BPF_PROG_SEC_IMPL 8072 #undef BPF_PROG_SEC 8073 #undef BPF_APROG_SEC 8074 #undef BPF_EAPROG_SEC 8075 #undef BPF_APROG_COMPAT 8076 #undef SEC_DEF 8077 8078 #define MAX_TYPE_NAME_SIZE 32 8079 8080 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 8081 { 8082 int i, n = ARRAY_SIZE(section_defs); 8083 8084 for (i = 0; i < n; i++) { 8085 if (strncmp(sec_name, 8086 section_defs[i].sec, section_defs[i].len)) 8087 continue; 8088 return §ion_defs[i]; 8089 } 8090 return NULL; 8091 } 8092 8093 static char *libbpf_get_type_names(bool attach_type) 8094 { 8095 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 8096 char *buf; 8097 8098 buf = malloc(len); 8099 if (!buf) 8100 return NULL; 8101 8102 buf[0] = '\0'; 8103 /* Forge string buf with all available names */ 8104 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8105 if (attach_type && !section_defs[i].is_attachable) 8106 continue; 8107 8108 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 8109 free(buf); 8110 return NULL; 8111 } 8112 strcat(buf, " "); 8113 strcat(buf, section_defs[i].sec); 8114 } 8115 8116 return buf; 8117 } 8118 8119 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 8120 enum bpf_attach_type *expected_attach_type) 8121 { 8122 const struct bpf_sec_def *sec_def; 8123 char *type_names; 8124 8125 if (!name) 8126 return libbpf_err(-EINVAL); 8127 8128 sec_def = find_sec_def(name); 8129 if (sec_def) { 8130 *prog_type = sec_def->prog_type; 8131 *expected_attach_type = sec_def->expected_attach_type; 8132 return 0; 8133 } 8134 8135 pr_debug("failed to guess program type from ELF section '%s'\n", name); 8136 type_names = libbpf_get_type_names(false); 8137 if (type_names != NULL) { 8138 pr_debug("supported section(type) names are:%s\n", type_names); 8139 free(type_names); 8140 } 8141 8142 return libbpf_err(-ESRCH); 8143 } 8144 8145 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 8146 size_t offset) 8147 { 8148 struct bpf_map *map; 8149 size_t i; 8150 8151 for (i = 0; i < obj->nr_maps; i++) { 8152 map = &obj->maps[i]; 8153 if (!bpf_map__is_struct_ops(map)) 8154 continue; 8155 if (map->sec_offset <= offset && 8156 offset - map->sec_offset < map->def.value_size) 8157 return map; 8158 } 8159 8160 return NULL; 8161 } 8162 8163 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 8164 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 8165 GElf_Shdr *shdr, Elf_Data *data) 8166 { 8167 const struct btf_member *member; 8168 struct bpf_struct_ops *st_ops; 8169 struct bpf_program *prog; 8170 unsigned int shdr_idx; 8171 const struct btf *btf; 8172 struct bpf_map *map; 8173 Elf_Data *symbols; 8174 unsigned int moff, insn_idx; 8175 const char *name; 8176 __u32 member_idx; 8177 GElf_Sym sym; 8178 GElf_Rel rel; 8179 int i, nrels; 8180 8181 symbols = obj->efile.symbols; 8182 btf = obj->btf; 8183 nrels = shdr->sh_size / shdr->sh_entsize; 8184 for (i = 0; i < nrels; i++) { 8185 if (!gelf_getrel(data, i, &rel)) { 8186 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 8187 return -LIBBPF_ERRNO__FORMAT; 8188 } 8189 8190 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 8191 pr_warn("struct_ops reloc: symbol %zx not found\n", 8192 (size_t)GELF_R_SYM(rel.r_info)); 8193 return -LIBBPF_ERRNO__FORMAT; 8194 } 8195 8196 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 8197 map = find_struct_ops_map_by_offset(obj, rel.r_offset); 8198 if (!map) { 8199 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", 8200 (size_t)rel.r_offset); 8201 return -EINVAL; 8202 } 8203 8204 moff = rel.r_offset - map->sec_offset; 8205 shdr_idx = sym.st_shndx; 8206 st_ops = map->st_ops; 8207 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 8208 map->name, 8209 (long long)(rel.r_info >> 32), 8210 (long long)sym.st_value, 8211 shdr_idx, (size_t)rel.r_offset, 8212 map->sec_offset, sym.st_name, name); 8213 8214 if (shdr_idx >= SHN_LORESERVE) { 8215 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", 8216 map->name, (size_t)rel.r_offset, shdr_idx); 8217 return -LIBBPF_ERRNO__RELOC; 8218 } 8219 if (sym.st_value % BPF_INSN_SZ) { 8220 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 8221 map->name, (unsigned long long)sym.st_value); 8222 return -LIBBPF_ERRNO__FORMAT; 8223 } 8224 insn_idx = sym.st_value / BPF_INSN_SZ; 8225 8226 member = find_member_by_offset(st_ops->type, moff * 8); 8227 if (!member) { 8228 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 8229 map->name, moff); 8230 return -EINVAL; 8231 } 8232 member_idx = member - btf_members(st_ops->type); 8233 name = btf__name_by_offset(btf, member->name_off); 8234 8235 if (!resolve_func_ptr(btf, member->type, NULL)) { 8236 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 8237 map->name, name); 8238 return -EINVAL; 8239 } 8240 8241 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 8242 if (!prog) { 8243 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 8244 map->name, shdr_idx, name); 8245 return -EINVAL; 8246 } 8247 8248 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 8249 const struct bpf_sec_def *sec_def; 8250 8251 sec_def = find_sec_def(prog->sec_name); 8252 if (sec_def && 8253 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) { 8254 /* for pr_warn */ 8255 prog->type = sec_def->prog_type; 8256 goto invalid_prog; 8257 } 8258 8259 prog->type = BPF_PROG_TYPE_STRUCT_OPS; 8260 prog->attach_btf_id = st_ops->type_id; 8261 prog->expected_attach_type = member_idx; 8262 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || 8263 prog->attach_btf_id != st_ops->type_id || 8264 prog->expected_attach_type != member_idx) { 8265 goto invalid_prog; 8266 } 8267 st_ops->progs[member_idx] = prog; 8268 } 8269 8270 return 0; 8271 8272 invalid_prog: 8273 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 8274 map->name, prog->name, prog->sec_name, prog->type, 8275 prog->attach_btf_id, prog->expected_attach_type, name); 8276 return -EINVAL; 8277 } 8278 8279 #define BTF_TRACE_PREFIX "btf_trace_" 8280 #define BTF_LSM_PREFIX "bpf_lsm_" 8281 #define BTF_ITER_PREFIX "bpf_iter_" 8282 #define BTF_MAX_NAME_SIZE 128 8283 8284 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 8285 const char **prefix, int *kind) 8286 { 8287 switch (attach_type) { 8288 case BPF_TRACE_RAW_TP: 8289 *prefix = BTF_TRACE_PREFIX; 8290 *kind = BTF_KIND_TYPEDEF; 8291 break; 8292 case BPF_LSM_MAC: 8293 *prefix = BTF_LSM_PREFIX; 8294 *kind = BTF_KIND_FUNC; 8295 break; 8296 case BPF_TRACE_ITER: 8297 *prefix = BTF_ITER_PREFIX; 8298 *kind = BTF_KIND_FUNC; 8299 break; 8300 default: 8301 *prefix = ""; 8302 *kind = BTF_KIND_FUNC; 8303 } 8304 } 8305 8306 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 8307 const char *name, __u32 kind) 8308 { 8309 char btf_type_name[BTF_MAX_NAME_SIZE]; 8310 int ret; 8311 8312 ret = snprintf(btf_type_name, sizeof(btf_type_name), 8313 "%s%s", prefix, name); 8314 /* snprintf returns the number of characters written excluding the 8315 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 8316 * indicates truncation. 8317 */ 8318 if (ret < 0 || ret >= sizeof(btf_type_name)) 8319 return -ENAMETOOLONG; 8320 return btf__find_by_name_kind(btf, btf_type_name, kind); 8321 } 8322 8323 static inline int find_attach_btf_id(struct btf *btf, const char *name, 8324 enum bpf_attach_type attach_type) 8325 { 8326 const char *prefix; 8327 int kind; 8328 8329 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 8330 return find_btf_by_prefix_kind(btf, prefix, name, kind); 8331 } 8332 8333 int libbpf_find_vmlinux_btf_id(const char *name, 8334 enum bpf_attach_type attach_type) 8335 { 8336 struct btf *btf; 8337 int err; 8338 8339 btf = btf__load_vmlinux_btf(); 8340 err = libbpf_get_error(btf); 8341 if (err) { 8342 pr_warn("vmlinux BTF is not found\n"); 8343 return libbpf_err(err); 8344 } 8345 8346 err = find_attach_btf_id(btf, name, attach_type); 8347 if (err <= 0) 8348 pr_warn("%s is not found in vmlinux BTF\n", name); 8349 8350 btf__free(btf); 8351 return libbpf_err(err); 8352 } 8353 8354 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 8355 { 8356 struct bpf_prog_info_linear *info_linear; 8357 struct bpf_prog_info *info; 8358 struct btf *btf; 8359 int err; 8360 8361 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); 8362 err = libbpf_get_error(info_linear); 8363 if (err) { 8364 pr_warn("failed get_prog_info_linear for FD %d\n", 8365 attach_prog_fd); 8366 return err; 8367 } 8368 8369 err = -EINVAL; 8370 info = &info_linear->info; 8371 if (!info->btf_id) { 8372 pr_warn("The target program doesn't have BTF\n"); 8373 goto out; 8374 } 8375 btf = btf__load_from_kernel_by_id(info->btf_id); 8376 if (libbpf_get_error(btf)) { 8377 pr_warn("Failed to get BTF of the program\n"); 8378 goto out; 8379 } 8380 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 8381 btf__free(btf); 8382 if (err <= 0) { 8383 pr_warn("%s is not found in prog's BTF\n", name); 8384 goto out; 8385 } 8386 out: 8387 free(info_linear); 8388 return err; 8389 } 8390 8391 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 8392 enum bpf_attach_type attach_type, 8393 int *btf_obj_fd, int *btf_type_id) 8394 { 8395 int ret, i; 8396 8397 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 8398 if (ret > 0) { 8399 *btf_obj_fd = 0; /* vmlinux BTF */ 8400 *btf_type_id = ret; 8401 return 0; 8402 } 8403 if (ret != -ENOENT) 8404 return ret; 8405 8406 ret = load_module_btfs(obj); 8407 if (ret) 8408 return ret; 8409 8410 for (i = 0; i < obj->btf_module_cnt; i++) { 8411 const struct module_btf *mod = &obj->btf_modules[i]; 8412 8413 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 8414 if (ret > 0) { 8415 *btf_obj_fd = mod->fd; 8416 *btf_type_id = ret; 8417 return 0; 8418 } 8419 if (ret == -ENOENT) 8420 continue; 8421 8422 return ret; 8423 } 8424 8425 return -ESRCH; 8426 } 8427 8428 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id) 8429 { 8430 enum bpf_attach_type attach_type = prog->expected_attach_type; 8431 __u32 attach_prog_fd = prog->attach_prog_fd; 8432 const char *name = prog->sec_name, *attach_name; 8433 const struct bpf_sec_def *sec = NULL; 8434 int i, err = 0; 8435 8436 if (!name) 8437 return -EINVAL; 8438 8439 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8440 if (!section_defs[i].is_attach_btf) 8441 continue; 8442 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 8443 continue; 8444 8445 sec = §ion_defs[i]; 8446 break; 8447 } 8448 8449 if (!sec) { 8450 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name); 8451 return -ESRCH; 8452 } 8453 attach_name = name + sec->len; 8454 8455 /* BPF program's BTF ID */ 8456 if (attach_prog_fd) { 8457 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 8458 if (err < 0) { 8459 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 8460 attach_prog_fd, attach_name, err); 8461 return err; 8462 } 8463 *btf_obj_fd = 0; 8464 *btf_type_id = err; 8465 return 0; 8466 } 8467 8468 /* kernel/module BTF ID */ 8469 if (prog->obj->gen_loader) { 8470 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 8471 *btf_obj_fd = 0; 8472 *btf_type_id = 1; 8473 } else { 8474 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); 8475 } 8476 if (err) { 8477 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err); 8478 return err; 8479 } 8480 return 0; 8481 } 8482 8483 int libbpf_attach_type_by_name(const char *name, 8484 enum bpf_attach_type *attach_type) 8485 { 8486 char *type_names; 8487 int i; 8488 8489 if (!name) 8490 return libbpf_err(-EINVAL); 8491 8492 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8493 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 8494 continue; 8495 if (!section_defs[i].is_attachable) 8496 return libbpf_err(-EINVAL); 8497 *attach_type = section_defs[i].expected_attach_type; 8498 return 0; 8499 } 8500 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 8501 type_names = libbpf_get_type_names(true); 8502 if (type_names != NULL) { 8503 pr_debug("attachable section(type) names are:%s\n", type_names); 8504 free(type_names); 8505 } 8506 8507 return libbpf_err(-EINVAL); 8508 } 8509 8510 int bpf_map__fd(const struct bpf_map *map) 8511 { 8512 return map ? map->fd : libbpf_err(-EINVAL); 8513 } 8514 8515 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 8516 { 8517 return map ? &map->def : libbpf_err_ptr(-EINVAL); 8518 } 8519 8520 const char *bpf_map__name(const struct bpf_map *map) 8521 { 8522 return map ? map->name : NULL; 8523 } 8524 8525 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 8526 { 8527 return map->def.type; 8528 } 8529 8530 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 8531 { 8532 if (map->fd >= 0) 8533 return libbpf_err(-EBUSY); 8534 map->def.type = type; 8535 return 0; 8536 } 8537 8538 __u32 bpf_map__map_flags(const struct bpf_map *map) 8539 { 8540 return map->def.map_flags; 8541 } 8542 8543 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 8544 { 8545 if (map->fd >= 0) 8546 return libbpf_err(-EBUSY); 8547 map->def.map_flags = flags; 8548 return 0; 8549 } 8550 8551 __u32 bpf_map__numa_node(const struct bpf_map *map) 8552 { 8553 return map->numa_node; 8554 } 8555 8556 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 8557 { 8558 if (map->fd >= 0) 8559 return libbpf_err(-EBUSY); 8560 map->numa_node = numa_node; 8561 return 0; 8562 } 8563 8564 __u32 bpf_map__key_size(const struct bpf_map *map) 8565 { 8566 return map->def.key_size; 8567 } 8568 8569 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 8570 { 8571 if (map->fd >= 0) 8572 return libbpf_err(-EBUSY); 8573 map->def.key_size = size; 8574 return 0; 8575 } 8576 8577 __u32 bpf_map__value_size(const struct bpf_map *map) 8578 { 8579 return map->def.value_size; 8580 } 8581 8582 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 8583 { 8584 if (map->fd >= 0) 8585 return libbpf_err(-EBUSY); 8586 map->def.value_size = size; 8587 return 0; 8588 } 8589 8590 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 8591 { 8592 return map ? map->btf_key_type_id : 0; 8593 } 8594 8595 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 8596 { 8597 return map ? map->btf_value_type_id : 0; 8598 } 8599 8600 int bpf_map__set_priv(struct bpf_map *map, void *priv, 8601 bpf_map_clear_priv_t clear_priv) 8602 { 8603 if (!map) 8604 return libbpf_err(-EINVAL); 8605 8606 if (map->priv) { 8607 if (map->clear_priv) 8608 map->clear_priv(map, map->priv); 8609 } 8610 8611 map->priv = priv; 8612 map->clear_priv = clear_priv; 8613 return 0; 8614 } 8615 8616 void *bpf_map__priv(const struct bpf_map *map) 8617 { 8618 return map ? map->priv : libbpf_err_ptr(-EINVAL); 8619 } 8620 8621 int bpf_map__set_initial_value(struct bpf_map *map, 8622 const void *data, size_t size) 8623 { 8624 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 8625 size != map->def.value_size || map->fd >= 0) 8626 return libbpf_err(-EINVAL); 8627 8628 memcpy(map->mmaped, data, size); 8629 return 0; 8630 } 8631 8632 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) 8633 { 8634 if (!map->mmaped) 8635 return NULL; 8636 *psize = map->def.value_size; 8637 return map->mmaped; 8638 } 8639 8640 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 8641 { 8642 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 8643 } 8644 8645 bool bpf_map__is_internal(const struct bpf_map *map) 8646 { 8647 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 8648 } 8649 8650 __u32 bpf_map__ifindex(const struct bpf_map *map) 8651 { 8652 return map->map_ifindex; 8653 } 8654 8655 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 8656 { 8657 if (map->fd >= 0) 8658 return libbpf_err(-EBUSY); 8659 map->map_ifindex = ifindex; 8660 return 0; 8661 } 8662 8663 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 8664 { 8665 if (!bpf_map_type__is_map_in_map(map->def.type)) { 8666 pr_warn("error: unsupported map type\n"); 8667 return libbpf_err(-EINVAL); 8668 } 8669 if (map->inner_map_fd != -1) { 8670 pr_warn("error: inner_map_fd already specified\n"); 8671 return libbpf_err(-EINVAL); 8672 } 8673 zfree(&map->inner_map); 8674 map->inner_map_fd = fd; 8675 return 0; 8676 } 8677 8678 static struct bpf_map * 8679 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 8680 { 8681 ssize_t idx; 8682 struct bpf_map *s, *e; 8683 8684 if (!obj || !obj->maps) 8685 return errno = EINVAL, NULL; 8686 8687 s = obj->maps; 8688 e = obj->maps + obj->nr_maps; 8689 8690 if ((m < s) || (m >= e)) { 8691 pr_warn("error in %s: map handler doesn't belong to object\n", 8692 __func__); 8693 return errno = EINVAL, NULL; 8694 } 8695 8696 idx = (m - obj->maps) + i; 8697 if (idx >= obj->nr_maps || idx < 0) 8698 return NULL; 8699 return &obj->maps[idx]; 8700 } 8701 8702 struct bpf_map * 8703 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 8704 { 8705 if (prev == NULL) 8706 return obj->maps; 8707 8708 return __bpf_map__iter(prev, obj, 1); 8709 } 8710 8711 struct bpf_map * 8712 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 8713 { 8714 if (next == NULL) { 8715 if (!obj->nr_maps) 8716 return NULL; 8717 return obj->maps + obj->nr_maps - 1; 8718 } 8719 8720 return __bpf_map__iter(next, obj, -1); 8721 } 8722 8723 struct bpf_map * 8724 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 8725 { 8726 struct bpf_map *pos; 8727 8728 bpf_object__for_each_map(pos, obj) { 8729 if (pos->name && !strcmp(pos->name, name)) 8730 return pos; 8731 } 8732 return errno = ENOENT, NULL; 8733 } 8734 8735 int 8736 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 8737 { 8738 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 8739 } 8740 8741 struct bpf_map * 8742 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 8743 { 8744 return libbpf_err_ptr(-ENOTSUP); 8745 } 8746 8747 long libbpf_get_error(const void *ptr) 8748 { 8749 if (!IS_ERR_OR_NULL(ptr)) 8750 return 0; 8751 8752 if (IS_ERR(ptr)) 8753 errno = -PTR_ERR(ptr); 8754 8755 /* If ptr == NULL, then errno should be already set by the failing 8756 * API, because libbpf never returns NULL on success and it now always 8757 * sets errno on error. So no extra errno handling for ptr == NULL 8758 * case. 8759 */ 8760 return -errno; 8761 } 8762 8763 int bpf_prog_load(const char *file, enum bpf_prog_type type, 8764 struct bpf_object **pobj, int *prog_fd) 8765 { 8766 struct bpf_prog_load_attr attr; 8767 8768 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 8769 attr.file = file; 8770 attr.prog_type = type; 8771 attr.expected_attach_type = 0; 8772 8773 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 8774 } 8775 8776 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 8777 struct bpf_object **pobj, int *prog_fd) 8778 { 8779 struct bpf_object_open_attr open_attr = {}; 8780 struct bpf_program *prog, *first_prog = NULL; 8781 struct bpf_object *obj; 8782 struct bpf_map *map; 8783 int err; 8784 8785 if (!attr) 8786 return libbpf_err(-EINVAL); 8787 if (!attr->file) 8788 return libbpf_err(-EINVAL); 8789 8790 open_attr.file = attr->file; 8791 open_attr.prog_type = attr->prog_type; 8792 8793 obj = bpf_object__open_xattr(&open_attr); 8794 err = libbpf_get_error(obj); 8795 if (err) 8796 return libbpf_err(-ENOENT); 8797 8798 bpf_object__for_each_program(prog, obj) { 8799 enum bpf_attach_type attach_type = attr->expected_attach_type; 8800 /* 8801 * to preserve backwards compatibility, bpf_prog_load treats 8802 * attr->prog_type, if specified, as an override to whatever 8803 * bpf_object__open guessed 8804 */ 8805 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { 8806 bpf_program__set_type(prog, attr->prog_type); 8807 bpf_program__set_expected_attach_type(prog, 8808 attach_type); 8809 } 8810 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { 8811 /* 8812 * we haven't guessed from section name and user 8813 * didn't provide a fallback type, too bad... 8814 */ 8815 bpf_object__close(obj); 8816 return libbpf_err(-EINVAL); 8817 } 8818 8819 prog->prog_ifindex = attr->ifindex; 8820 prog->log_level = attr->log_level; 8821 prog->prog_flags |= attr->prog_flags; 8822 if (!first_prog) 8823 first_prog = prog; 8824 } 8825 8826 bpf_object__for_each_map(map, obj) { 8827 if (!bpf_map__is_offload_neutral(map)) 8828 map->map_ifindex = attr->ifindex; 8829 } 8830 8831 if (!first_prog) { 8832 pr_warn("object file doesn't contain bpf program\n"); 8833 bpf_object__close(obj); 8834 return libbpf_err(-ENOENT); 8835 } 8836 8837 err = bpf_object__load(obj); 8838 if (err) { 8839 bpf_object__close(obj); 8840 return libbpf_err(err); 8841 } 8842 8843 *pobj = obj; 8844 *prog_fd = bpf_program__fd(first_prog); 8845 return 0; 8846 } 8847 8848 struct bpf_link { 8849 int (*detach)(struct bpf_link *link); 8850 void (*dealloc)(struct bpf_link *link); 8851 char *pin_path; /* NULL, if not pinned */ 8852 int fd; /* hook FD, -1 if not applicable */ 8853 bool disconnected; 8854 }; 8855 8856 /* Replace link's underlying BPF program with the new one */ 8857 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 8858 { 8859 int ret; 8860 8861 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 8862 return libbpf_err_errno(ret); 8863 } 8864 8865 /* Release "ownership" of underlying BPF resource (typically, BPF program 8866 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 8867 * link, when destructed through bpf_link__destroy() call won't attempt to 8868 * detach/unregisted that BPF resource. This is useful in situations where, 8869 * say, attached BPF program has to outlive userspace program that attached it 8870 * in the system. Depending on type of BPF program, though, there might be 8871 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 8872 * exit of userspace program doesn't trigger automatic detachment and clean up 8873 * inside the kernel. 8874 */ 8875 void bpf_link__disconnect(struct bpf_link *link) 8876 { 8877 link->disconnected = true; 8878 } 8879 8880 int bpf_link__destroy(struct bpf_link *link) 8881 { 8882 int err = 0; 8883 8884 if (IS_ERR_OR_NULL(link)) 8885 return 0; 8886 8887 if (!link->disconnected && link->detach) 8888 err = link->detach(link); 8889 if (link->pin_path) 8890 free(link->pin_path); 8891 if (link->dealloc) 8892 link->dealloc(link); 8893 else 8894 free(link); 8895 8896 return libbpf_err(err); 8897 } 8898 8899 int bpf_link__fd(const struct bpf_link *link) 8900 { 8901 return link->fd; 8902 } 8903 8904 const char *bpf_link__pin_path(const struct bpf_link *link) 8905 { 8906 return link->pin_path; 8907 } 8908 8909 static int bpf_link__detach_fd(struct bpf_link *link) 8910 { 8911 return libbpf_err_errno(close(link->fd)); 8912 } 8913 8914 struct bpf_link *bpf_link__open(const char *path) 8915 { 8916 struct bpf_link *link; 8917 int fd; 8918 8919 fd = bpf_obj_get(path); 8920 if (fd < 0) { 8921 fd = -errno; 8922 pr_warn("failed to open link at %s: %d\n", path, fd); 8923 return libbpf_err_ptr(fd); 8924 } 8925 8926 link = calloc(1, sizeof(*link)); 8927 if (!link) { 8928 close(fd); 8929 return libbpf_err_ptr(-ENOMEM); 8930 } 8931 link->detach = &bpf_link__detach_fd; 8932 link->fd = fd; 8933 8934 link->pin_path = strdup(path); 8935 if (!link->pin_path) { 8936 bpf_link__destroy(link); 8937 return libbpf_err_ptr(-ENOMEM); 8938 } 8939 8940 return link; 8941 } 8942 8943 int bpf_link__detach(struct bpf_link *link) 8944 { 8945 return bpf_link_detach(link->fd) ? -errno : 0; 8946 } 8947 8948 int bpf_link__pin(struct bpf_link *link, const char *path) 8949 { 8950 int err; 8951 8952 if (link->pin_path) 8953 return libbpf_err(-EBUSY); 8954 err = make_parent_dir(path); 8955 if (err) 8956 return libbpf_err(err); 8957 err = check_path(path); 8958 if (err) 8959 return libbpf_err(err); 8960 8961 link->pin_path = strdup(path); 8962 if (!link->pin_path) 8963 return libbpf_err(-ENOMEM); 8964 8965 if (bpf_obj_pin(link->fd, link->pin_path)) { 8966 err = -errno; 8967 zfree(&link->pin_path); 8968 return libbpf_err(err); 8969 } 8970 8971 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 8972 return 0; 8973 } 8974 8975 int bpf_link__unpin(struct bpf_link *link) 8976 { 8977 int err; 8978 8979 if (!link->pin_path) 8980 return libbpf_err(-EINVAL); 8981 8982 err = unlink(link->pin_path); 8983 if (err != 0) 8984 return -errno; 8985 8986 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 8987 zfree(&link->pin_path); 8988 return 0; 8989 } 8990 8991 struct bpf_link_perf { 8992 struct bpf_link link; 8993 int perf_event_fd; 8994 }; 8995 8996 static int bpf_link_perf_detach(struct bpf_link *link) 8997 { 8998 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 8999 int err = 0; 9000 9001 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 9002 err = -errno; 9003 9004 if (perf_link->perf_event_fd != link->fd) 9005 close(perf_link->perf_event_fd); 9006 close(link->fd); 9007 9008 return libbpf_err(err); 9009 } 9010 9011 static void bpf_link_perf_dealloc(struct bpf_link *link) 9012 { 9013 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 9014 9015 free(perf_link); 9016 } 9017 9018 struct bpf_link *bpf_program__attach_perf_event_opts(struct bpf_program *prog, int pfd, 9019 const struct bpf_perf_event_opts *opts) 9020 { 9021 char errmsg[STRERR_BUFSIZE]; 9022 struct bpf_link_perf *link; 9023 int prog_fd, link_fd = -1, err; 9024 9025 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 9026 return libbpf_err_ptr(-EINVAL); 9027 9028 if (pfd < 0) { 9029 pr_warn("prog '%s': invalid perf event FD %d\n", 9030 prog->name, pfd); 9031 return libbpf_err_ptr(-EINVAL); 9032 } 9033 prog_fd = bpf_program__fd(prog); 9034 if (prog_fd < 0) { 9035 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 9036 prog->name); 9037 return libbpf_err_ptr(-EINVAL); 9038 } 9039 9040 link = calloc(1, sizeof(*link)); 9041 if (!link) 9042 return libbpf_err_ptr(-ENOMEM); 9043 link->link.detach = &bpf_link_perf_detach; 9044 link->link.dealloc = &bpf_link_perf_dealloc; 9045 link->perf_event_fd = pfd; 9046 9047 if (kernel_supports(prog->obj, FEAT_PERF_LINK)) { 9048 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 9049 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 9050 9051 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 9052 if (link_fd < 0) { 9053 err = -errno; 9054 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", 9055 prog->name, pfd, 9056 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9057 goto err_out; 9058 } 9059 link->link.fd = link_fd; 9060 } else { 9061 if (OPTS_GET(opts, bpf_cookie, 0)) { 9062 pr_warn("prog '%s': user context value is not supported\n", prog->name); 9063 err = -EOPNOTSUPP; 9064 goto err_out; 9065 } 9066 9067 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 9068 err = -errno; 9069 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 9070 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9071 if (err == -EPROTO) 9072 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 9073 prog->name, pfd); 9074 goto err_out; 9075 } 9076 link->link.fd = pfd; 9077 } 9078 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 9079 err = -errno; 9080 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 9081 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9082 goto err_out; 9083 } 9084 9085 return &link->link; 9086 err_out: 9087 if (link_fd >= 0) 9088 close(link_fd); 9089 free(link); 9090 return libbpf_err_ptr(err); 9091 } 9092 9093 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, int pfd) 9094 { 9095 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 9096 } 9097 9098 /* 9099 * this function is expected to parse integer in the range of [0, 2^31-1] from 9100 * given file using scanf format string fmt. If actual parsed value is 9101 * negative, the result might be indistinguishable from error 9102 */ 9103 static int parse_uint_from_file(const char *file, const char *fmt) 9104 { 9105 char buf[STRERR_BUFSIZE]; 9106 int err, ret; 9107 FILE *f; 9108 9109 f = fopen(file, "r"); 9110 if (!f) { 9111 err = -errno; 9112 pr_debug("failed to open '%s': %s\n", file, 9113 libbpf_strerror_r(err, buf, sizeof(buf))); 9114 return err; 9115 } 9116 err = fscanf(f, fmt, &ret); 9117 if (err != 1) { 9118 err = err == EOF ? -EIO : -errno; 9119 pr_debug("failed to parse '%s': %s\n", file, 9120 libbpf_strerror_r(err, buf, sizeof(buf))); 9121 fclose(f); 9122 return err; 9123 } 9124 fclose(f); 9125 return ret; 9126 } 9127 9128 static int determine_kprobe_perf_type(void) 9129 { 9130 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 9131 9132 return parse_uint_from_file(file, "%d\n"); 9133 } 9134 9135 static int determine_uprobe_perf_type(void) 9136 { 9137 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 9138 9139 return parse_uint_from_file(file, "%d\n"); 9140 } 9141 9142 static int determine_kprobe_retprobe_bit(void) 9143 { 9144 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 9145 9146 return parse_uint_from_file(file, "config:%d\n"); 9147 } 9148 9149 static int determine_uprobe_retprobe_bit(void) 9150 { 9151 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 9152 9153 return parse_uint_from_file(file, "config:%d\n"); 9154 } 9155 9156 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 9157 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 9158 9159 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 9160 uint64_t offset, int pid, size_t ref_ctr_off) 9161 { 9162 struct perf_event_attr attr = {}; 9163 char errmsg[STRERR_BUFSIZE]; 9164 int type, pfd, err; 9165 9166 if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 9167 return -EINVAL; 9168 9169 type = uprobe ? determine_uprobe_perf_type() 9170 : determine_kprobe_perf_type(); 9171 if (type < 0) { 9172 pr_warn("failed to determine %s perf type: %s\n", 9173 uprobe ? "uprobe" : "kprobe", 9174 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 9175 return type; 9176 } 9177 if (retprobe) { 9178 int bit = uprobe ? determine_uprobe_retprobe_bit() 9179 : determine_kprobe_retprobe_bit(); 9180 9181 if (bit < 0) { 9182 pr_warn("failed to determine %s retprobe bit: %s\n", 9183 uprobe ? "uprobe" : "kprobe", 9184 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 9185 return bit; 9186 } 9187 attr.config |= 1 << bit; 9188 } 9189 attr.size = sizeof(attr); 9190 attr.type = type; 9191 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 9192 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 9193 attr.config2 = offset; /* kprobe_addr or probe_offset */ 9194 9195 /* pid filter is meaningful only for uprobes */ 9196 pfd = syscall(__NR_perf_event_open, &attr, 9197 pid < 0 ? -1 : pid /* pid */, 9198 pid == -1 ? 0 : -1 /* cpu */, 9199 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9200 if (pfd < 0) { 9201 err = -errno; 9202 pr_warn("%s perf_event_open() failed: %s\n", 9203 uprobe ? "uprobe" : "kprobe", 9204 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9205 return err; 9206 } 9207 return pfd; 9208 } 9209 9210 struct bpf_link * 9211 bpf_program__attach_kprobe_opts(struct bpf_program *prog, 9212 const char *func_name, 9213 const struct bpf_kprobe_opts *opts) 9214 { 9215 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9216 char errmsg[STRERR_BUFSIZE]; 9217 struct bpf_link *link; 9218 unsigned long offset; 9219 bool retprobe; 9220 int pfd, err; 9221 9222 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 9223 return libbpf_err_ptr(-EINVAL); 9224 9225 retprobe = OPTS_GET(opts, retprobe, false); 9226 offset = OPTS_GET(opts, offset, 0); 9227 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9228 9229 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 9230 offset, -1 /* pid */, 0 /* ref_ctr_off */); 9231 if (pfd < 0) { 9232 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n", 9233 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 9234 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9235 return libbpf_err_ptr(pfd); 9236 } 9237 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9238 err = libbpf_get_error(link); 9239 if (err) { 9240 close(pfd); 9241 pr_warn("prog '%s': failed to attach to %s '%s': %s\n", 9242 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 9243 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9244 return libbpf_err_ptr(err); 9245 } 9246 return link; 9247 } 9248 9249 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 9250 bool retprobe, 9251 const char *func_name) 9252 { 9253 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 9254 .retprobe = retprobe, 9255 ); 9256 9257 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 9258 } 9259 9260 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 9261 struct bpf_program *prog) 9262 { 9263 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 9264 unsigned long offset = 0; 9265 struct bpf_link *link; 9266 const char *func_name; 9267 char *func; 9268 int n, err; 9269 9270 func_name = prog->sec_name + sec->len; 9271 opts.retprobe = strcmp(sec->sec, "kretprobe/") == 0; 9272 9273 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 9274 if (n < 1) { 9275 err = -EINVAL; 9276 pr_warn("kprobe name is invalid: %s\n", func_name); 9277 return libbpf_err_ptr(err); 9278 } 9279 if (opts.retprobe && offset != 0) { 9280 free(func); 9281 err = -EINVAL; 9282 pr_warn("kretprobes do not support offset specification\n"); 9283 return libbpf_err_ptr(err); 9284 } 9285 9286 opts.offset = offset; 9287 link = bpf_program__attach_kprobe_opts(prog, func, &opts); 9288 free(func); 9289 return link; 9290 } 9291 9292 LIBBPF_API struct bpf_link * 9293 bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid, 9294 const char *binary_path, size_t func_offset, 9295 const struct bpf_uprobe_opts *opts) 9296 { 9297 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9298 char errmsg[STRERR_BUFSIZE]; 9299 struct bpf_link *link; 9300 size_t ref_ctr_off; 9301 int pfd, err; 9302 bool retprobe; 9303 9304 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 9305 return libbpf_err_ptr(-EINVAL); 9306 9307 retprobe = OPTS_GET(opts, retprobe, false); 9308 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 9309 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9310 9311 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 9312 func_offset, pid, ref_ctr_off); 9313 if (pfd < 0) { 9314 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 9315 prog->name, retprobe ? "uretprobe" : "uprobe", 9316 binary_path, func_offset, 9317 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9318 return libbpf_err_ptr(pfd); 9319 } 9320 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9321 err = libbpf_get_error(link); 9322 if (err) { 9323 close(pfd); 9324 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 9325 prog->name, retprobe ? "uretprobe" : "uprobe", 9326 binary_path, func_offset, 9327 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9328 return libbpf_err_ptr(err); 9329 } 9330 return link; 9331 } 9332 9333 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 9334 bool retprobe, pid_t pid, 9335 const char *binary_path, 9336 size_t func_offset) 9337 { 9338 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 9339 9340 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 9341 } 9342 9343 static int determine_tracepoint_id(const char *tp_category, 9344 const char *tp_name) 9345 { 9346 char file[PATH_MAX]; 9347 int ret; 9348 9349 ret = snprintf(file, sizeof(file), 9350 "/sys/kernel/debug/tracing/events/%s/%s/id", 9351 tp_category, tp_name); 9352 if (ret < 0) 9353 return -errno; 9354 if (ret >= sizeof(file)) { 9355 pr_debug("tracepoint %s/%s path is too long\n", 9356 tp_category, tp_name); 9357 return -E2BIG; 9358 } 9359 return parse_uint_from_file(file, "%d\n"); 9360 } 9361 9362 static int perf_event_open_tracepoint(const char *tp_category, 9363 const char *tp_name) 9364 { 9365 struct perf_event_attr attr = {}; 9366 char errmsg[STRERR_BUFSIZE]; 9367 int tp_id, pfd, err; 9368 9369 tp_id = determine_tracepoint_id(tp_category, tp_name); 9370 if (tp_id < 0) { 9371 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 9372 tp_category, tp_name, 9373 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 9374 return tp_id; 9375 } 9376 9377 attr.type = PERF_TYPE_TRACEPOINT; 9378 attr.size = sizeof(attr); 9379 attr.config = tp_id; 9380 9381 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 9382 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9383 if (pfd < 0) { 9384 err = -errno; 9385 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 9386 tp_category, tp_name, 9387 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9388 return err; 9389 } 9390 return pfd; 9391 } 9392 9393 struct bpf_link *bpf_program__attach_tracepoint_opts(struct bpf_program *prog, 9394 const char *tp_category, 9395 const char *tp_name, 9396 const struct bpf_tracepoint_opts *opts) 9397 { 9398 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9399 char errmsg[STRERR_BUFSIZE]; 9400 struct bpf_link *link; 9401 int pfd, err; 9402 9403 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 9404 return libbpf_err_ptr(-EINVAL); 9405 9406 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9407 9408 pfd = perf_event_open_tracepoint(tp_category, tp_name); 9409 if (pfd < 0) { 9410 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 9411 prog->name, tp_category, tp_name, 9412 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9413 return libbpf_err_ptr(pfd); 9414 } 9415 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9416 err = libbpf_get_error(link); 9417 if (err) { 9418 close(pfd); 9419 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 9420 prog->name, tp_category, tp_name, 9421 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9422 return libbpf_err_ptr(err); 9423 } 9424 return link; 9425 } 9426 9427 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 9428 const char *tp_category, 9429 const char *tp_name) 9430 { 9431 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 9432 } 9433 9434 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 9435 struct bpf_program *prog) 9436 { 9437 char *sec_name, *tp_cat, *tp_name; 9438 struct bpf_link *link; 9439 9440 sec_name = strdup(prog->sec_name); 9441 if (!sec_name) 9442 return libbpf_err_ptr(-ENOMEM); 9443 9444 /* extract "tp/<category>/<name>" */ 9445 tp_cat = sec_name + sec->len; 9446 tp_name = strchr(tp_cat, '/'); 9447 if (!tp_name) { 9448 free(sec_name); 9449 return libbpf_err_ptr(-EINVAL); 9450 } 9451 *tp_name = '\0'; 9452 tp_name++; 9453 9454 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 9455 free(sec_name); 9456 return link; 9457 } 9458 9459 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 9460 const char *tp_name) 9461 { 9462 char errmsg[STRERR_BUFSIZE]; 9463 struct bpf_link *link; 9464 int prog_fd, pfd; 9465 9466 prog_fd = bpf_program__fd(prog); 9467 if (prog_fd < 0) { 9468 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9469 return libbpf_err_ptr(-EINVAL); 9470 } 9471 9472 link = calloc(1, sizeof(*link)); 9473 if (!link) 9474 return libbpf_err_ptr(-ENOMEM); 9475 link->detach = &bpf_link__detach_fd; 9476 9477 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 9478 if (pfd < 0) { 9479 pfd = -errno; 9480 free(link); 9481 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 9482 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9483 return libbpf_err_ptr(pfd); 9484 } 9485 link->fd = pfd; 9486 return link; 9487 } 9488 9489 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 9490 struct bpf_program *prog) 9491 { 9492 const char *tp_name = prog->sec_name + sec->len; 9493 9494 return bpf_program__attach_raw_tracepoint(prog, tp_name); 9495 } 9496 9497 /* Common logic for all BPF program types that attach to a btf_id */ 9498 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog) 9499 { 9500 char errmsg[STRERR_BUFSIZE]; 9501 struct bpf_link *link; 9502 int prog_fd, pfd; 9503 9504 prog_fd = bpf_program__fd(prog); 9505 if (prog_fd < 0) { 9506 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9507 return libbpf_err_ptr(-EINVAL); 9508 } 9509 9510 link = calloc(1, sizeof(*link)); 9511 if (!link) 9512 return libbpf_err_ptr(-ENOMEM); 9513 link->detach = &bpf_link__detach_fd; 9514 9515 pfd = bpf_raw_tracepoint_open(NULL, prog_fd); 9516 if (pfd < 0) { 9517 pfd = -errno; 9518 free(link); 9519 pr_warn("prog '%s': failed to attach: %s\n", 9520 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9521 return libbpf_err_ptr(pfd); 9522 } 9523 link->fd = pfd; 9524 return (struct bpf_link *)link; 9525 } 9526 9527 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog) 9528 { 9529 return bpf_program__attach_btf_id(prog); 9530 } 9531 9532 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog) 9533 { 9534 return bpf_program__attach_btf_id(prog); 9535 } 9536 9537 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 9538 struct bpf_program *prog) 9539 { 9540 return bpf_program__attach_trace(prog); 9541 } 9542 9543 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 9544 struct bpf_program *prog) 9545 { 9546 return bpf_program__attach_lsm(prog); 9547 } 9548 9549 static struct bpf_link * 9550 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id, 9551 const char *target_name) 9552 { 9553 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, 9554 .target_btf_id = btf_id); 9555 enum bpf_attach_type attach_type; 9556 char errmsg[STRERR_BUFSIZE]; 9557 struct bpf_link *link; 9558 int prog_fd, link_fd; 9559 9560 prog_fd = bpf_program__fd(prog); 9561 if (prog_fd < 0) { 9562 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9563 return libbpf_err_ptr(-EINVAL); 9564 } 9565 9566 link = calloc(1, sizeof(*link)); 9567 if (!link) 9568 return libbpf_err_ptr(-ENOMEM); 9569 link->detach = &bpf_link__detach_fd; 9570 9571 attach_type = bpf_program__get_expected_attach_type(prog); 9572 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); 9573 if (link_fd < 0) { 9574 link_fd = -errno; 9575 free(link); 9576 pr_warn("prog '%s': failed to attach to %s: %s\n", 9577 prog->name, target_name, 9578 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 9579 return libbpf_err_ptr(link_fd); 9580 } 9581 link->fd = link_fd; 9582 return link; 9583 } 9584 9585 struct bpf_link * 9586 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd) 9587 { 9588 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup"); 9589 } 9590 9591 struct bpf_link * 9592 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd) 9593 { 9594 return bpf_program__attach_fd(prog, netns_fd, 0, "netns"); 9595 } 9596 9597 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex) 9598 { 9599 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 9600 return bpf_program__attach_fd(prog, ifindex, 0, "xdp"); 9601 } 9602 9603 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog, 9604 int target_fd, 9605 const char *attach_func_name) 9606 { 9607 int btf_id; 9608 9609 if (!!target_fd != !!attach_func_name) { 9610 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 9611 prog->name); 9612 return libbpf_err_ptr(-EINVAL); 9613 } 9614 9615 if (prog->type != BPF_PROG_TYPE_EXT) { 9616 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 9617 prog->name); 9618 return libbpf_err_ptr(-EINVAL); 9619 } 9620 9621 if (target_fd) { 9622 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 9623 if (btf_id < 0) 9624 return libbpf_err_ptr(btf_id); 9625 9626 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace"); 9627 } else { 9628 /* no target, so use raw_tracepoint_open for compatibility 9629 * with old kernels 9630 */ 9631 return bpf_program__attach_trace(prog); 9632 } 9633 } 9634 9635 struct bpf_link * 9636 bpf_program__attach_iter(struct bpf_program *prog, 9637 const struct bpf_iter_attach_opts *opts) 9638 { 9639 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 9640 char errmsg[STRERR_BUFSIZE]; 9641 struct bpf_link *link; 9642 int prog_fd, link_fd; 9643 __u32 target_fd = 0; 9644 9645 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 9646 return libbpf_err_ptr(-EINVAL); 9647 9648 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 9649 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 9650 9651 prog_fd = bpf_program__fd(prog); 9652 if (prog_fd < 0) { 9653 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9654 return libbpf_err_ptr(-EINVAL); 9655 } 9656 9657 link = calloc(1, sizeof(*link)); 9658 if (!link) 9659 return libbpf_err_ptr(-ENOMEM); 9660 link->detach = &bpf_link__detach_fd; 9661 9662 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 9663 &link_create_opts); 9664 if (link_fd < 0) { 9665 link_fd = -errno; 9666 free(link); 9667 pr_warn("prog '%s': failed to attach to iterator: %s\n", 9668 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 9669 return libbpf_err_ptr(link_fd); 9670 } 9671 link->fd = link_fd; 9672 return link; 9673 } 9674 9675 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 9676 struct bpf_program *prog) 9677 { 9678 return bpf_program__attach_iter(prog, NULL); 9679 } 9680 9681 struct bpf_link *bpf_program__attach(struct bpf_program *prog) 9682 { 9683 const struct bpf_sec_def *sec_def; 9684 9685 sec_def = find_sec_def(prog->sec_name); 9686 if (!sec_def || !sec_def->attach_fn) 9687 return libbpf_err_ptr(-ESRCH); 9688 9689 return sec_def->attach_fn(sec_def, prog); 9690 } 9691 9692 static int bpf_link__detach_struct_ops(struct bpf_link *link) 9693 { 9694 __u32 zero = 0; 9695 9696 if (bpf_map_delete_elem(link->fd, &zero)) 9697 return -errno; 9698 9699 return 0; 9700 } 9701 9702 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map) 9703 { 9704 struct bpf_struct_ops *st_ops; 9705 struct bpf_link *link; 9706 __u32 i, zero = 0; 9707 int err; 9708 9709 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 9710 return libbpf_err_ptr(-EINVAL); 9711 9712 link = calloc(1, sizeof(*link)); 9713 if (!link) 9714 return libbpf_err_ptr(-EINVAL); 9715 9716 st_ops = map->st_ops; 9717 for (i = 0; i < btf_vlen(st_ops->type); i++) { 9718 struct bpf_program *prog = st_ops->progs[i]; 9719 void *kern_data; 9720 int prog_fd; 9721 9722 if (!prog) 9723 continue; 9724 9725 prog_fd = bpf_program__fd(prog); 9726 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 9727 *(unsigned long *)kern_data = prog_fd; 9728 } 9729 9730 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); 9731 if (err) { 9732 err = -errno; 9733 free(link); 9734 return libbpf_err_ptr(err); 9735 } 9736 9737 link->detach = bpf_link__detach_struct_ops; 9738 link->fd = map->fd; 9739 9740 return link; 9741 } 9742 9743 enum bpf_perf_event_ret 9744 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 9745 void **copy_mem, size_t *copy_size, 9746 bpf_perf_event_print_t fn, void *private_data) 9747 { 9748 struct perf_event_mmap_page *header = mmap_mem; 9749 __u64 data_head = ring_buffer_read_head(header); 9750 __u64 data_tail = header->data_tail; 9751 void *base = ((__u8 *)header) + page_size; 9752 int ret = LIBBPF_PERF_EVENT_CONT; 9753 struct perf_event_header *ehdr; 9754 size_t ehdr_size; 9755 9756 while (data_head != data_tail) { 9757 ehdr = base + (data_tail & (mmap_size - 1)); 9758 ehdr_size = ehdr->size; 9759 9760 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 9761 void *copy_start = ehdr; 9762 size_t len_first = base + mmap_size - copy_start; 9763 size_t len_secnd = ehdr_size - len_first; 9764 9765 if (*copy_size < ehdr_size) { 9766 free(*copy_mem); 9767 *copy_mem = malloc(ehdr_size); 9768 if (!*copy_mem) { 9769 *copy_size = 0; 9770 ret = LIBBPF_PERF_EVENT_ERROR; 9771 break; 9772 } 9773 *copy_size = ehdr_size; 9774 } 9775 9776 memcpy(*copy_mem, copy_start, len_first); 9777 memcpy(*copy_mem + len_first, base, len_secnd); 9778 ehdr = *copy_mem; 9779 } 9780 9781 ret = fn(ehdr, private_data); 9782 data_tail += ehdr_size; 9783 if (ret != LIBBPF_PERF_EVENT_CONT) 9784 break; 9785 } 9786 9787 ring_buffer_write_tail(header, data_tail); 9788 return libbpf_err(ret); 9789 } 9790 9791 struct perf_buffer; 9792 9793 struct perf_buffer_params { 9794 struct perf_event_attr *attr; 9795 /* if event_cb is specified, it takes precendence */ 9796 perf_buffer_event_fn event_cb; 9797 /* sample_cb and lost_cb are higher-level common-case callbacks */ 9798 perf_buffer_sample_fn sample_cb; 9799 perf_buffer_lost_fn lost_cb; 9800 void *ctx; 9801 int cpu_cnt; 9802 int *cpus; 9803 int *map_keys; 9804 }; 9805 9806 struct perf_cpu_buf { 9807 struct perf_buffer *pb; 9808 void *base; /* mmap()'ed memory */ 9809 void *buf; /* for reconstructing segmented data */ 9810 size_t buf_size; 9811 int fd; 9812 int cpu; 9813 int map_key; 9814 }; 9815 9816 struct perf_buffer { 9817 perf_buffer_event_fn event_cb; 9818 perf_buffer_sample_fn sample_cb; 9819 perf_buffer_lost_fn lost_cb; 9820 void *ctx; /* passed into callbacks */ 9821 9822 size_t page_size; 9823 size_t mmap_size; 9824 struct perf_cpu_buf **cpu_bufs; 9825 struct epoll_event *events; 9826 int cpu_cnt; /* number of allocated CPU buffers */ 9827 int epoll_fd; /* perf event FD */ 9828 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 9829 }; 9830 9831 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 9832 struct perf_cpu_buf *cpu_buf) 9833 { 9834 if (!cpu_buf) 9835 return; 9836 if (cpu_buf->base && 9837 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 9838 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 9839 if (cpu_buf->fd >= 0) { 9840 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 9841 close(cpu_buf->fd); 9842 } 9843 free(cpu_buf->buf); 9844 free(cpu_buf); 9845 } 9846 9847 void perf_buffer__free(struct perf_buffer *pb) 9848 { 9849 int i; 9850 9851 if (IS_ERR_OR_NULL(pb)) 9852 return; 9853 if (pb->cpu_bufs) { 9854 for (i = 0; i < pb->cpu_cnt; i++) { 9855 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 9856 9857 if (!cpu_buf) 9858 continue; 9859 9860 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 9861 perf_buffer__free_cpu_buf(pb, cpu_buf); 9862 } 9863 free(pb->cpu_bufs); 9864 } 9865 if (pb->epoll_fd >= 0) 9866 close(pb->epoll_fd); 9867 free(pb->events); 9868 free(pb); 9869 } 9870 9871 static struct perf_cpu_buf * 9872 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 9873 int cpu, int map_key) 9874 { 9875 struct perf_cpu_buf *cpu_buf; 9876 char msg[STRERR_BUFSIZE]; 9877 int err; 9878 9879 cpu_buf = calloc(1, sizeof(*cpu_buf)); 9880 if (!cpu_buf) 9881 return ERR_PTR(-ENOMEM); 9882 9883 cpu_buf->pb = pb; 9884 cpu_buf->cpu = cpu; 9885 cpu_buf->map_key = map_key; 9886 9887 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 9888 -1, PERF_FLAG_FD_CLOEXEC); 9889 if (cpu_buf->fd < 0) { 9890 err = -errno; 9891 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 9892 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 9893 goto error; 9894 } 9895 9896 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 9897 PROT_READ | PROT_WRITE, MAP_SHARED, 9898 cpu_buf->fd, 0); 9899 if (cpu_buf->base == MAP_FAILED) { 9900 cpu_buf->base = NULL; 9901 err = -errno; 9902 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 9903 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 9904 goto error; 9905 } 9906 9907 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 9908 err = -errno; 9909 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 9910 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 9911 goto error; 9912 } 9913 9914 return cpu_buf; 9915 9916 error: 9917 perf_buffer__free_cpu_buf(pb, cpu_buf); 9918 return (struct perf_cpu_buf *)ERR_PTR(err); 9919 } 9920 9921 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 9922 struct perf_buffer_params *p); 9923 9924 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 9925 const struct perf_buffer_opts *opts) 9926 { 9927 struct perf_buffer_params p = {}; 9928 struct perf_event_attr attr = { 0, }; 9929 9930 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 9931 attr.type = PERF_TYPE_SOFTWARE; 9932 attr.sample_type = PERF_SAMPLE_RAW; 9933 attr.sample_period = 1; 9934 attr.wakeup_events = 1; 9935 9936 p.attr = &attr; 9937 p.sample_cb = opts ? opts->sample_cb : NULL; 9938 p.lost_cb = opts ? opts->lost_cb : NULL; 9939 p.ctx = opts ? opts->ctx : NULL; 9940 9941 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 9942 } 9943 9944 struct perf_buffer * 9945 perf_buffer__new_raw(int map_fd, size_t page_cnt, 9946 const struct perf_buffer_raw_opts *opts) 9947 { 9948 struct perf_buffer_params p = {}; 9949 9950 p.attr = opts->attr; 9951 p.event_cb = opts->event_cb; 9952 p.ctx = opts->ctx; 9953 p.cpu_cnt = opts->cpu_cnt; 9954 p.cpus = opts->cpus; 9955 p.map_keys = opts->map_keys; 9956 9957 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 9958 } 9959 9960 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 9961 struct perf_buffer_params *p) 9962 { 9963 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 9964 struct bpf_map_info map; 9965 char msg[STRERR_BUFSIZE]; 9966 struct perf_buffer *pb; 9967 bool *online = NULL; 9968 __u32 map_info_len; 9969 int err, i, j, n; 9970 9971 if (page_cnt & (page_cnt - 1)) { 9972 pr_warn("page count should be power of two, but is %zu\n", 9973 page_cnt); 9974 return ERR_PTR(-EINVAL); 9975 } 9976 9977 /* best-effort sanity checks */ 9978 memset(&map, 0, sizeof(map)); 9979 map_info_len = sizeof(map); 9980 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 9981 if (err) { 9982 err = -errno; 9983 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 9984 * -EBADFD, -EFAULT, or -E2BIG on real error 9985 */ 9986 if (err != -EINVAL) { 9987 pr_warn("failed to get map info for map FD %d: %s\n", 9988 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 9989 return ERR_PTR(err); 9990 } 9991 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 9992 map_fd); 9993 } else { 9994 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 9995 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 9996 map.name); 9997 return ERR_PTR(-EINVAL); 9998 } 9999 } 10000 10001 pb = calloc(1, sizeof(*pb)); 10002 if (!pb) 10003 return ERR_PTR(-ENOMEM); 10004 10005 pb->event_cb = p->event_cb; 10006 pb->sample_cb = p->sample_cb; 10007 pb->lost_cb = p->lost_cb; 10008 pb->ctx = p->ctx; 10009 10010 pb->page_size = getpagesize(); 10011 pb->mmap_size = pb->page_size * page_cnt; 10012 pb->map_fd = map_fd; 10013 10014 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 10015 if (pb->epoll_fd < 0) { 10016 err = -errno; 10017 pr_warn("failed to create epoll instance: %s\n", 10018 libbpf_strerror_r(err, msg, sizeof(msg))); 10019 goto error; 10020 } 10021 10022 if (p->cpu_cnt > 0) { 10023 pb->cpu_cnt = p->cpu_cnt; 10024 } else { 10025 pb->cpu_cnt = libbpf_num_possible_cpus(); 10026 if (pb->cpu_cnt < 0) { 10027 err = pb->cpu_cnt; 10028 goto error; 10029 } 10030 if (map.max_entries && map.max_entries < pb->cpu_cnt) 10031 pb->cpu_cnt = map.max_entries; 10032 } 10033 10034 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 10035 if (!pb->events) { 10036 err = -ENOMEM; 10037 pr_warn("failed to allocate events: out of memory\n"); 10038 goto error; 10039 } 10040 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 10041 if (!pb->cpu_bufs) { 10042 err = -ENOMEM; 10043 pr_warn("failed to allocate buffers: out of memory\n"); 10044 goto error; 10045 } 10046 10047 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 10048 if (err) { 10049 pr_warn("failed to get online CPU mask: %d\n", err); 10050 goto error; 10051 } 10052 10053 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 10054 struct perf_cpu_buf *cpu_buf; 10055 int cpu, map_key; 10056 10057 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 10058 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 10059 10060 /* in case user didn't explicitly requested particular CPUs to 10061 * be attached to, skip offline/not present CPUs 10062 */ 10063 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 10064 continue; 10065 10066 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 10067 if (IS_ERR(cpu_buf)) { 10068 err = PTR_ERR(cpu_buf); 10069 goto error; 10070 } 10071 10072 pb->cpu_bufs[j] = cpu_buf; 10073 10074 err = bpf_map_update_elem(pb->map_fd, &map_key, 10075 &cpu_buf->fd, 0); 10076 if (err) { 10077 err = -errno; 10078 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 10079 cpu, map_key, cpu_buf->fd, 10080 libbpf_strerror_r(err, msg, sizeof(msg))); 10081 goto error; 10082 } 10083 10084 pb->events[j].events = EPOLLIN; 10085 pb->events[j].data.ptr = cpu_buf; 10086 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 10087 &pb->events[j]) < 0) { 10088 err = -errno; 10089 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 10090 cpu, cpu_buf->fd, 10091 libbpf_strerror_r(err, msg, sizeof(msg))); 10092 goto error; 10093 } 10094 j++; 10095 } 10096 pb->cpu_cnt = j; 10097 free(online); 10098 10099 return pb; 10100 10101 error: 10102 free(online); 10103 if (pb) 10104 perf_buffer__free(pb); 10105 return ERR_PTR(err); 10106 } 10107 10108 struct perf_sample_raw { 10109 struct perf_event_header header; 10110 uint32_t size; 10111 char data[]; 10112 }; 10113 10114 struct perf_sample_lost { 10115 struct perf_event_header header; 10116 uint64_t id; 10117 uint64_t lost; 10118 uint64_t sample_id; 10119 }; 10120 10121 static enum bpf_perf_event_ret 10122 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 10123 { 10124 struct perf_cpu_buf *cpu_buf = ctx; 10125 struct perf_buffer *pb = cpu_buf->pb; 10126 void *data = e; 10127 10128 /* user wants full control over parsing perf event */ 10129 if (pb->event_cb) 10130 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 10131 10132 switch (e->type) { 10133 case PERF_RECORD_SAMPLE: { 10134 struct perf_sample_raw *s = data; 10135 10136 if (pb->sample_cb) 10137 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 10138 break; 10139 } 10140 case PERF_RECORD_LOST: { 10141 struct perf_sample_lost *s = data; 10142 10143 if (pb->lost_cb) 10144 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 10145 break; 10146 } 10147 default: 10148 pr_warn("unknown perf sample type %d\n", e->type); 10149 return LIBBPF_PERF_EVENT_ERROR; 10150 } 10151 return LIBBPF_PERF_EVENT_CONT; 10152 } 10153 10154 static int perf_buffer__process_records(struct perf_buffer *pb, 10155 struct perf_cpu_buf *cpu_buf) 10156 { 10157 enum bpf_perf_event_ret ret; 10158 10159 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 10160 pb->page_size, &cpu_buf->buf, 10161 &cpu_buf->buf_size, 10162 perf_buffer__process_record, cpu_buf); 10163 if (ret != LIBBPF_PERF_EVENT_CONT) 10164 return ret; 10165 return 0; 10166 } 10167 10168 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 10169 { 10170 return pb->epoll_fd; 10171 } 10172 10173 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 10174 { 10175 int i, cnt, err; 10176 10177 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 10178 if (cnt < 0) 10179 return -errno; 10180 10181 for (i = 0; i < cnt; i++) { 10182 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 10183 10184 err = perf_buffer__process_records(pb, cpu_buf); 10185 if (err) { 10186 pr_warn("error while processing records: %d\n", err); 10187 return libbpf_err(err); 10188 } 10189 } 10190 return cnt; 10191 } 10192 10193 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 10194 * manager. 10195 */ 10196 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 10197 { 10198 return pb->cpu_cnt; 10199 } 10200 10201 /* 10202 * Return perf_event FD of a ring buffer in *buf_idx* slot of 10203 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 10204 * select()/poll()/epoll() Linux syscalls. 10205 */ 10206 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 10207 { 10208 struct perf_cpu_buf *cpu_buf; 10209 10210 if (buf_idx >= pb->cpu_cnt) 10211 return libbpf_err(-EINVAL); 10212 10213 cpu_buf = pb->cpu_bufs[buf_idx]; 10214 if (!cpu_buf) 10215 return libbpf_err(-ENOENT); 10216 10217 return cpu_buf->fd; 10218 } 10219 10220 /* 10221 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 10222 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 10223 * consume, do nothing and return success. 10224 * Returns: 10225 * - 0 on success; 10226 * - <0 on failure. 10227 */ 10228 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 10229 { 10230 struct perf_cpu_buf *cpu_buf; 10231 10232 if (buf_idx >= pb->cpu_cnt) 10233 return libbpf_err(-EINVAL); 10234 10235 cpu_buf = pb->cpu_bufs[buf_idx]; 10236 if (!cpu_buf) 10237 return libbpf_err(-ENOENT); 10238 10239 return perf_buffer__process_records(pb, cpu_buf); 10240 } 10241 10242 int perf_buffer__consume(struct perf_buffer *pb) 10243 { 10244 int i, err; 10245 10246 for (i = 0; i < pb->cpu_cnt; i++) { 10247 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 10248 10249 if (!cpu_buf) 10250 continue; 10251 10252 err = perf_buffer__process_records(pb, cpu_buf); 10253 if (err) { 10254 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 10255 return libbpf_err(err); 10256 } 10257 } 10258 return 0; 10259 } 10260 10261 struct bpf_prog_info_array_desc { 10262 int array_offset; /* e.g. offset of jited_prog_insns */ 10263 int count_offset; /* e.g. offset of jited_prog_len */ 10264 int size_offset; /* > 0: offset of rec size, 10265 * < 0: fix size of -size_offset 10266 */ 10267 }; 10268 10269 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 10270 [BPF_PROG_INFO_JITED_INSNS] = { 10271 offsetof(struct bpf_prog_info, jited_prog_insns), 10272 offsetof(struct bpf_prog_info, jited_prog_len), 10273 -1, 10274 }, 10275 [BPF_PROG_INFO_XLATED_INSNS] = { 10276 offsetof(struct bpf_prog_info, xlated_prog_insns), 10277 offsetof(struct bpf_prog_info, xlated_prog_len), 10278 -1, 10279 }, 10280 [BPF_PROG_INFO_MAP_IDS] = { 10281 offsetof(struct bpf_prog_info, map_ids), 10282 offsetof(struct bpf_prog_info, nr_map_ids), 10283 -(int)sizeof(__u32), 10284 }, 10285 [BPF_PROG_INFO_JITED_KSYMS] = { 10286 offsetof(struct bpf_prog_info, jited_ksyms), 10287 offsetof(struct bpf_prog_info, nr_jited_ksyms), 10288 -(int)sizeof(__u64), 10289 }, 10290 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 10291 offsetof(struct bpf_prog_info, jited_func_lens), 10292 offsetof(struct bpf_prog_info, nr_jited_func_lens), 10293 -(int)sizeof(__u32), 10294 }, 10295 [BPF_PROG_INFO_FUNC_INFO] = { 10296 offsetof(struct bpf_prog_info, func_info), 10297 offsetof(struct bpf_prog_info, nr_func_info), 10298 offsetof(struct bpf_prog_info, func_info_rec_size), 10299 }, 10300 [BPF_PROG_INFO_LINE_INFO] = { 10301 offsetof(struct bpf_prog_info, line_info), 10302 offsetof(struct bpf_prog_info, nr_line_info), 10303 offsetof(struct bpf_prog_info, line_info_rec_size), 10304 }, 10305 [BPF_PROG_INFO_JITED_LINE_INFO] = { 10306 offsetof(struct bpf_prog_info, jited_line_info), 10307 offsetof(struct bpf_prog_info, nr_jited_line_info), 10308 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 10309 }, 10310 [BPF_PROG_INFO_PROG_TAGS] = { 10311 offsetof(struct bpf_prog_info, prog_tags), 10312 offsetof(struct bpf_prog_info, nr_prog_tags), 10313 -(int)sizeof(__u8) * BPF_TAG_SIZE, 10314 }, 10315 10316 }; 10317 10318 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 10319 int offset) 10320 { 10321 __u32 *array = (__u32 *)info; 10322 10323 if (offset >= 0) 10324 return array[offset / sizeof(__u32)]; 10325 return -(int)offset; 10326 } 10327 10328 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 10329 int offset) 10330 { 10331 __u64 *array = (__u64 *)info; 10332 10333 if (offset >= 0) 10334 return array[offset / sizeof(__u64)]; 10335 return -(int)offset; 10336 } 10337 10338 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 10339 __u32 val) 10340 { 10341 __u32 *array = (__u32 *)info; 10342 10343 if (offset >= 0) 10344 array[offset / sizeof(__u32)] = val; 10345 } 10346 10347 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 10348 __u64 val) 10349 { 10350 __u64 *array = (__u64 *)info; 10351 10352 if (offset >= 0) 10353 array[offset / sizeof(__u64)] = val; 10354 } 10355 10356 struct bpf_prog_info_linear * 10357 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 10358 { 10359 struct bpf_prog_info_linear *info_linear; 10360 struct bpf_prog_info info = {}; 10361 __u32 info_len = sizeof(info); 10362 __u32 data_len = 0; 10363 int i, err; 10364 void *ptr; 10365 10366 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 10367 return libbpf_err_ptr(-EINVAL); 10368 10369 /* step 1: get array dimensions */ 10370 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 10371 if (err) { 10372 pr_debug("can't get prog info: %s", strerror(errno)); 10373 return libbpf_err_ptr(-EFAULT); 10374 } 10375 10376 /* step 2: calculate total size of all arrays */ 10377 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10378 bool include_array = (arrays & (1UL << i)) > 0; 10379 struct bpf_prog_info_array_desc *desc; 10380 __u32 count, size; 10381 10382 desc = bpf_prog_info_array_desc + i; 10383 10384 /* kernel is too old to support this field */ 10385 if (info_len < desc->array_offset + sizeof(__u32) || 10386 info_len < desc->count_offset + sizeof(__u32) || 10387 (desc->size_offset > 0 && info_len < desc->size_offset)) 10388 include_array = false; 10389 10390 if (!include_array) { 10391 arrays &= ~(1UL << i); /* clear the bit */ 10392 continue; 10393 } 10394 10395 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10396 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10397 10398 data_len += count * size; 10399 } 10400 10401 /* step 3: allocate continuous memory */ 10402 data_len = roundup(data_len, sizeof(__u64)); 10403 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 10404 if (!info_linear) 10405 return libbpf_err_ptr(-ENOMEM); 10406 10407 /* step 4: fill data to info_linear->info */ 10408 info_linear->arrays = arrays; 10409 memset(&info_linear->info, 0, sizeof(info)); 10410 ptr = info_linear->data; 10411 10412 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10413 struct bpf_prog_info_array_desc *desc; 10414 __u32 count, size; 10415 10416 if ((arrays & (1UL << i)) == 0) 10417 continue; 10418 10419 desc = bpf_prog_info_array_desc + i; 10420 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10421 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10422 bpf_prog_info_set_offset_u32(&info_linear->info, 10423 desc->count_offset, count); 10424 bpf_prog_info_set_offset_u32(&info_linear->info, 10425 desc->size_offset, size); 10426 bpf_prog_info_set_offset_u64(&info_linear->info, 10427 desc->array_offset, 10428 ptr_to_u64(ptr)); 10429 ptr += count * size; 10430 } 10431 10432 /* step 5: call syscall again to get required arrays */ 10433 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 10434 if (err) { 10435 pr_debug("can't get prog info: %s", strerror(errno)); 10436 free(info_linear); 10437 return libbpf_err_ptr(-EFAULT); 10438 } 10439 10440 /* step 6: verify the data */ 10441 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10442 struct bpf_prog_info_array_desc *desc; 10443 __u32 v1, v2; 10444 10445 if ((arrays & (1UL << i)) == 0) 10446 continue; 10447 10448 desc = bpf_prog_info_array_desc + i; 10449 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10450 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 10451 desc->count_offset); 10452 if (v1 != v2) 10453 pr_warn("%s: mismatch in element count\n", __func__); 10454 10455 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10456 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 10457 desc->size_offset); 10458 if (v1 != v2) 10459 pr_warn("%s: mismatch in rec size\n", __func__); 10460 } 10461 10462 /* step 7: update info_len and data_len */ 10463 info_linear->info_len = sizeof(struct bpf_prog_info); 10464 info_linear->data_len = data_len; 10465 10466 return info_linear; 10467 } 10468 10469 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 10470 { 10471 int i; 10472 10473 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10474 struct bpf_prog_info_array_desc *desc; 10475 __u64 addr, offs; 10476 10477 if ((info_linear->arrays & (1UL << i)) == 0) 10478 continue; 10479 10480 desc = bpf_prog_info_array_desc + i; 10481 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 10482 desc->array_offset); 10483 offs = addr - ptr_to_u64(info_linear->data); 10484 bpf_prog_info_set_offset_u64(&info_linear->info, 10485 desc->array_offset, offs); 10486 } 10487 } 10488 10489 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 10490 { 10491 int i; 10492 10493 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10494 struct bpf_prog_info_array_desc *desc; 10495 __u64 addr, offs; 10496 10497 if ((info_linear->arrays & (1UL << i)) == 0) 10498 continue; 10499 10500 desc = bpf_prog_info_array_desc + i; 10501 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 10502 desc->array_offset); 10503 addr = offs + ptr_to_u64(info_linear->data); 10504 bpf_prog_info_set_offset_u64(&info_linear->info, 10505 desc->array_offset, addr); 10506 } 10507 } 10508 10509 int bpf_program__set_attach_target(struct bpf_program *prog, 10510 int attach_prog_fd, 10511 const char *attach_func_name) 10512 { 10513 int btf_obj_fd = 0, btf_id = 0, err; 10514 10515 if (!prog || attach_prog_fd < 0 || !attach_func_name) 10516 return libbpf_err(-EINVAL); 10517 10518 if (prog->obj->loaded) 10519 return libbpf_err(-EINVAL); 10520 10521 if (attach_prog_fd) { 10522 btf_id = libbpf_find_prog_btf_id(attach_func_name, 10523 attach_prog_fd); 10524 if (btf_id < 0) 10525 return libbpf_err(btf_id); 10526 } else { 10527 /* load btf_vmlinux, if not yet */ 10528 err = bpf_object__load_vmlinux_btf(prog->obj, true); 10529 if (err) 10530 return libbpf_err(err); 10531 err = find_kernel_btf_id(prog->obj, attach_func_name, 10532 prog->expected_attach_type, 10533 &btf_obj_fd, &btf_id); 10534 if (err) 10535 return libbpf_err(err); 10536 } 10537 10538 prog->attach_btf_id = btf_id; 10539 prog->attach_btf_obj_fd = btf_obj_fd; 10540 prog->attach_prog_fd = attach_prog_fd; 10541 return 0; 10542 } 10543 10544 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 10545 { 10546 int err = 0, n, len, start, end = -1; 10547 bool *tmp; 10548 10549 *mask = NULL; 10550 *mask_sz = 0; 10551 10552 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 10553 while (*s) { 10554 if (*s == ',' || *s == '\n') { 10555 s++; 10556 continue; 10557 } 10558 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 10559 if (n <= 0 || n > 2) { 10560 pr_warn("Failed to get CPU range %s: %d\n", s, n); 10561 err = -EINVAL; 10562 goto cleanup; 10563 } else if (n == 1) { 10564 end = start; 10565 } 10566 if (start < 0 || start > end) { 10567 pr_warn("Invalid CPU range [%d,%d] in %s\n", 10568 start, end, s); 10569 err = -EINVAL; 10570 goto cleanup; 10571 } 10572 tmp = realloc(*mask, end + 1); 10573 if (!tmp) { 10574 err = -ENOMEM; 10575 goto cleanup; 10576 } 10577 *mask = tmp; 10578 memset(tmp + *mask_sz, 0, start - *mask_sz); 10579 memset(tmp + start, 1, end - start + 1); 10580 *mask_sz = end + 1; 10581 s += len; 10582 } 10583 if (!*mask_sz) { 10584 pr_warn("Empty CPU range\n"); 10585 return -EINVAL; 10586 } 10587 return 0; 10588 cleanup: 10589 free(*mask); 10590 *mask = NULL; 10591 return err; 10592 } 10593 10594 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 10595 { 10596 int fd, err = 0, len; 10597 char buf[128]; 10598 10599 fd = open(fcpu, O_RDONLY); 10600 if (fd < 0) { 10601 err = -errno; 10602 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 10603 return err; 10604 } 10605 len = read(fd, buf, sizeof(buf)); 10606 close(fd); 10607 if (len <= 0) { 10608 err = len ? -errno : -EINVAL; 10609 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 10610 return err; 10611 } 10612 if (len >= sizeof(buf)) { 10613 pr_warn("CPU mask is too big in file %s\n", fcpu); 10614 return -E2BIG; 10615 } 10616 buf[len] = '\0'; 10617 10618 return parse_cpu_mask_str(buf, mask, mask_sz); 10619 } 10620 10621 int libbpf_num_possible_cpus(void) 10622 { 10623 static const char *fcpu = "/sys/devices/system/cpu/possible"; 10624 static int cpus; 10625 int err, n, i, tmp_cpus; 10626 bool *mask; 10627 10628 tmp_cpus = READ_ONCE(cpus); 10629 if (tmp_cpus > 0) 10630 return tmp_cpus; 10631 10632 err = parse_cpu_mask_file(fcpu, &mask, &n); 10633 if (err) 10634 return libbpf_err(err); 10635 10636 tmp_cpus = 0; 10637 for (i = 0; i < n; i++) { 10638 if (mask[i]) 10639 tmp_cpus++; 10640 } 10641 free(mask); 10642 10643 WRITE_ONCE(cpus, tmp_cpus); 10644 return tmp_cpus; 10645 } 10646 10647 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 10648 const struct bpf_object_open_opts *opts) 10649 { 10650 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 10651 .object_name = s->name, 10652 ); 10653 struct bpf_object *obj; 10654 int i, err; 10655 10656 /* Attempt to preserve opts->object_name, unless overriden by user 10657 * explicitly. Overwriting object name for skeletons is discouraged, 10658 * as it breaks global data maps, because they contain object name 10659 * prefix as their own map name prefix. When skeleton is generated, 10660 * bpftool is making an assumption that this name will stay the same. 10661 */ 10662 if (opts) { 10663 memcpy(&skel_opts, opts, sizeof(*opts)); 10664 if (!opts->object_name) 10665 skel_opts.object_name = s->name; 10666 } 10667 10668 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 10669 err = libbpf_get_error(obj); 10670 if (err) { 10671 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", 10672 s->name, err); 10673 return libbpf_err(err); 10674 } 10675 10676 *s->obj = obj; 10677 10678 for (i = 0; i < s->map_cnt; i++) { 10679 struct bpf_map **map = s->maps[i].map; 10680 const char *name = s->maps[i].name; 10681 void **mmaped = s->maps[i].mmaped; 10682 10683 *map = bpf_object__find_map_by_name(obj, name); 10684 if (!*map) { 10685 pr_warn("failed to find skeleton map '%s'\n", name); 10686 return libbpf_err(-ESRCH); 10687 } 10688 10689 /* externs shouldn't be pre-setup from user code */ 10690 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 10691 *mmaped = (*map)->mmaped; 10692 } 10693 10694 for (i = 0; i < s->prog_cnt; i++) { 10695 struct bpf_program **prog = s->progs[i].prog; 10696 const char *name = s->progs[i].name; 10697 10698 *prog = bpf_object__find_program_by_name(obj, name); 10699 if (!*prog) { 10700 pr_warn("failed to find skeleton program '%s'\n", name); 10701 return libbpf_err(-ESRCH); 10702 } 10703 } 10704 10705 return 0; 10706 } 10707 10708 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 10709 { 10710 int i, err; 10711 10712 err = bpf_object__load(*s->obj); 10713 if (err) { 10714 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 10715 return libbpf_err(err); 10716 } 10717 10718 for (i = 0; i < s->map_cnt; i++) { 10719 struct bpf_map *map = *s->maps[i].map; 10720 size_t mmap_sz = bpf_map_mmap_sz(map); 10721 int prot, map_fd = bpf_map__fd(map); 10722 void **mmaped = s->maps[i].mmaped; 10723 10724 if (!mmaped) 10725 continue; 10726 10727 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 10728 *mmaped = NULL; 10729 continue; 10730 } 10731 10732 if (map->def.map_flags & BPF_F_RDONLY_PROG) 10733 prot = PROT_READ; 10734 else 10735 prot = PROT_READ | PROT_WRITE; 10736 10737 /* Remap anonymous mmap()-ed "map initialization image" as 10738 * a BPF map-backed mmap()-ed memory, but preserving the same 10739 * memory address. This will cause kernel to change process' 10740 * page table to point to a different piece of kernel memory, 10741 * but from userspace point of view memory address (and its 10742 * contents, being identical at this point) will stay the 10743 * same. This mapping will be released by bpf_object__close() 10744 * as per normal clean up procedure, so we don't need to worry 10745 * about it from skeleton's clean up perspective. 10746 */ 10747 *mmaped = mmap(map->mmaped, mmap_sz, prot, 10748 MAP_SHARED | MAP_FIXED, map_fd, 0); 10749 if (*mmaped == MAP_FAILED) { 10750 err = -errno; 10751 *mmaped = NULL; 10752 pr_warn("failed to re-mmap() map '%s': %d\n", 10753 bpf_map__name(map), err); 10754 return libbpf_err(err); 10755 } 10756 } 10757 10758 return 0; 10759 } 10760 10761 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 10762 { 10763 int i, err; 10764 10765 for (i = 0; i < s->prog_cnt; i++) { 10766 struct bpf_program *prog = *s->progs[i].prog; 10767 struct bpf_link **link = s->progs[i].link; 10768 const struct bpf_sec_def *sec_def; 10769 10770 if (!prog->load) 10771 continue; 10772 10773 sec_def = find_sec_def(prog->sec_name); 10774 if (!sec_def || !sec_def->attach_fn) 10775 continue; 10776 10777 *link = sec_def->attach_fn(sec_def, prog); 10778 err = libbpf_get_error(*link); 10779 if (err) { 10780 pr_warn("failed to auto-attach program '%s': %d\n", 10781 bpf_program__name(prog), err); 10782 return libbpf_err(err); 10783 } 10784 } 10785 10786 return 0; 10787 } 10788 10789 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 10790 { 10791 int i; 10792 10793 for (i = 0; i < s->prog_cnt; i++) { 10794 struct bpf_link **link = s->progs[i].link; 10795 10796 bpf_link__destroy(*link); 10797 *link = NULL; 10798 } 10799 } 10800 10801 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 10802 { 10803 if (s->progs) 10804 bpf_object__detach_skeleton(s); 10805 if (s->obj) 10806 bpf_object__close(*s->obj); 10807 free(s->maps); 10808 free(s->progs); 10809 free(s); 10810 } 10811