1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/list.h> 35 #include <linux/limits.h> 36 #include <linux/perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <linux/version.h> 39 #include <sys/epoll.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <sys/stat.h> 43 #include <sys/types.h> 44 #include <sys/vfs.h> 45 #include <sys/utsname.h> 46 #include <sys/resource.h> 47 #include <libelf.h> 48 #include <gelf.h> 49 #include <zlib.h> 50 51 #include "libbpf.h" 52 #include "bpf.h" 53 #include "btf.h" 54 #include "str_error.h" 55 #include "libbpf_internal.h" 56 #include "hashmap.h" 57 #include "bpf_gen_internal.h" 58 59 #ifndef BPF_FS_MAGIC 60 #define BPF_FS_MAGIC 0xcafe4a11 61 #endif 62 63 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 64 65 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 66 * compilation if user enables corresponding warning. Disable it explicitly. 67 */ 68 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 69 70 #define __printf(a, b) __attribute__((format(printf, a, b))) 71 72 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 73 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 74 75 static int __base_pr(enum libbpf_print_level level, const char *format, 76 va_list args) 77 { 78 if (level == LIBBPF_DEBUG) 79 return 0; 80 81 return vfprintf(stderr, format, args); 82 } 83 84 static libbpf_print_fn_t __libbpf_pr = __base_pr; 85 86 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 87 { 88 libbpf_print_fn_t old_print_fn = __libbpf_pr; 89 90 __libbpf_pr = fn; 91 return old_print_fn; 92 } 93 94 __printf(2, 3) 95 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 96 { 97 va_list args; 98 99 if (!__libbpf_pr) 100 return; 101 102 va_start(args, format); 103 __libbpf_pr(level, format, args); 104 va_end(args); 105 } 106 107 static void pr_perm_msg(int err) 108 { 109 struct rlimit limit; 110 char buf[100]; 111 112 if (err != -EPERM || geteuid() != 0) 113 return; 114 115 err = getrlimit(RLIMIT_MEMLOCK, &limit); 116 if (err) 117 return; 118 119 if (limit.rlim_cur == RLIM_INFINITY) 120 return; 121 122 if (limit.rlim_cur < 1024) 123 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 124 else if (limit.rlim_cur < 1024*1024) 125 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 126 else 127 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 128 129 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 130 buf); 131 } 132 133 #define STRERR_BUFSIZE 128 134 135 /* Copied from tools/perf/util/util.h */ 136 #ifndef zfree 137 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 138 #endif 139 140 #ifndef zclose 141 # define zclose(fd) ({ \ 142 int ___err = 0; \ 143 if ((fd) >= 0) \ 144 ___err = close((fd)); \ 145 fd = -1; \ 146 ___err; }) 147 #endif 148 149 static inline __u64 ptr_to_u64(const void *ptr) 150 { 151 return (__u64) (unsigned long) ptr; 152 } 153 154 /* this goes away in libbpf 1.0 */ 155 enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE; 156 157 int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 158 { 159 /* __LIBBPF_STRICT_LAST is the last power-of-2 value used + 1, so to 160 * get all possible values we compensate last +1, and then (2*x - 1) 161 * to get the bit mask 162 */ 163 if (mode != LIBBPF_STRICT_ALL 164 && (mode & ~((__LIBBPF_STRICT_LAST - 1) * 2 - 1))) 165 return errno = EINVAL, -EINVAL; 166 167 libbpf_mode = mode; 168 return 0; 169 } 170 171 enum kern_feature_id { 172 /* v4.14: kernel support for program & map names. */ 173 FEAT_PROG_NAME, 174 /* v5.2: kernel support for global data sections. */ 175 FEAT_GLOBAL_DATA, 176 /* BTF support */ 177 FEAT_BTF, 178 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 179 FEAT_BTF_FUNC, 180 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 181 FEAT_BTF_DATASEC, 182 /* BTF_FUNC_GLOBAL is supported */ 183 FEAT_BTF_GLOBAL_FUNC, 184 /* BPF_F_MMAPABLE is supported for arrays */ 185 FEAT_ARRAY_MMAP, 186 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 187 FEAT_EXP_ATTACH_TYPE, 188 /* bpf_probe_read_{kernel,user}[_str] helpers */ 189 FEAT_PROBE_READ_KERN, 190 /* BPF_PROG_BIND_MAP is supported */ 191 FEAT_PROG_BIND_MAP, 192 /* Kernel support for module BTFs */ 193 FEAT_MODULE_BTF, 194 /* BTF_KIND_FLOAT support */ 195 FEAT_BTF_FLOAT, 196 /* BPF perf link support */ 197 FEAT_PERF_LINK, 198 __FEAT_CNT, 199 }; 200 201 static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); 202 203 enum reloc_type { 204 RELO_LD64, 205 RELO_CALL, 206 RELO_DATA, 207 RELO_EXTERN_VAR, 208 RELO_EXTERN_FUNC, 209 RELO_SUBPROG_ADDR, 210 }; 211 212 struct reloc_desc { 213 enum reloc_type type; 214 int insn_idx; 215 int map_idx; 216 int sym_off; 217 }; 218 219 struct bpf_sec_def; 220 221 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec, 222 struct bpf_program *prog); 223 224 struct bpf_sec_def { 225 const char *sec; 226 size_t len; 227 enum bpf_prog_type prog_type; 228 enum bpf_attach_type expected_attach_type; 229 bool is_exp_attach_type_optional; 230 bool is_attachable; 231 bool is_attach_btf; 232 bool is_sleepable; 233 attach_fn_t attach_fn; 234 }; 235 236 /* 237 * bpf_prog should be a better name but it has been used in 238 * linux/filter.h. 239 */ 240 struct bpf_program { 241 const struct bpf_sec_def *sec_def; 242 char *sec_name; 243 size_t sec_idx; 244 /* this program's instruction offset (in number of instructions) 245 * within its containing ELF section 246 */ 247 size_t sec_insn_off; 248 /* number of original instructions in ELF section belonging to this 249 * program, not taking into account subprogram instructions possible 250 * appended later during relocation 251 */ 252 size_t sec_insn_cnt; 253 /* Offset (in number of instructions) of the start of instruction 254 * belonging to this BPF program within its containing main BPF 255 * program. For the entry-point (main) BPF program, this is always 256 * zero. For a sub-program, this gets reset before each of main BPF 257 * programs are processed and relocated and is used to determined 258 * whether sub-program was already appended to the main program, and 259 * if yes, at which instruction offset. 260 */ 261 size_t sub_insn_off; 262 263 char *name; 264 /* sec_name with / replaced by _; makes recursive pinning 265 * in bpf_object__pin_programs easier 266 */ 267 char *pin_name; 268 269 /* instructions that belong to BPF program; insns[0] is located at 270 * sec_insn_off instruction within its ELF section in ELF file, so 271 * when mapping ELF file instruction index to the local instruction, 272 * one needs to subtract sec_insn_off; and vice versa. 273 */ 274 struct bpf_insn *insns; 275 /* actual number of instruction in this BPF program's image; for 276 * entry-point BPF programs this includes the size of main program 277 * itself plus all the used sub-programs, appended at the end 278 */ 279 size_t insns_cnt; 280 281 struct reloc_desc *reloc_desc; 282 int nr_reloc; 283 int log_level; 284 285 struct { 286 int nr; 287 int *fds; 288 } instances; 289 bpf_program_prep_t preprocessor; 290 291 struct bpf_object *obj; 292 void *priv; 293 bpf_program_clear_priv_t clear_priv; 294 295 bool load; 296 bool mark_btf_static; 297 enum bpf_prog_type type; 298 enum bpf_attach_type expected_attach_type; 299 int prog_ifindex; 300 __u32 attach_btf_obj_fd; 301 __u32 attach_btf_id; 302 __u32 attach_prog_fd; 303 void *func_info; 304 __u32 func_info_rec_size; 305 __u32 func_info_cnt; 306 307 void *line_info; 308 __u32 line_info_rec_size; 309 __u32 line_info_cnt; 310 __u32 prog_flags; 311 }; 312 313 struct bpf_struct_ops { 314 const char *tname; 315 const struct btf_type *type; 316 struct bpf_program **progs; 317 __u32 *kern_func_off; 318 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 319 void *data; 320 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 321 * btf_vmlinux's format. 322 * struct bpf_struct_ops_tcp_congestion_ops { 323 * [... some other kernel fields ...] 324 * struct tcp_congestion_ops data; 325 * } 326 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 327 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 328 * from "data". 329 */ 330 void *kern_vdata; 331 __u32 type_id; 332 }; 333 334 #define DATA_SEC ".data" 335 #define BSS_SEC ".bss" 336 #define RODATA_SEC ".rodata" 337 #define KCONFIG_SEC ".kconfig" 338 #define KSYMS_SEC ".ksyms" 339 #define STRUCT_OPS_SEC ".struct_ops" 340 341 enum libbpf_map_type { 342 LIBBPF_MAP_UNSPEC, 343 LIBBPF_MAP_DATA, 344 LIBBPF_MAP_BSS, 345 LIBBPF_MAP_RODATA, 346 LIBBPF_MAP_KCONFIG, 347 }; 348 349 static const char * const libbpf_type_to_btf_name[] = { 350 [LIBBPF_MAP_DATA] = DATA_SEC, 351 [LIBBPF_MAP_BSS] = BSS_SEC, 352 [LIBBPF_MAP_RODATA] = RODATA_SEC, 353 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, 354 }; 355 356 struct bpf_map { 357 char *name; 358 int fd; 359 int sec_idx; 360 size_t sec_offset; 361 int map_ifindex; 362 int inner_map_fd; 363 struct bpf_map_def def; 364 __u32 numa_node; 365 __u32 btf_var_idx; 366 __u32 btf_key_type_id; 367 __u32 btf_value_type_id; 368 __u32 btf_vmlinux_value_type_id; 369 void *priv; 370 bpf_map_clear_priv_t clear_priv; 371 enum libbpf_map_type libbpf_type; 372 void *mmaped; 373 struct bpf_struct_ops *st_ops; 374 struct bpf_map *inner_map; 375 void **init_slots; 376 int init_slots_sz; 377 char *pin_path; 378 bool pinned; 379 bool reused; 380 }; 381 382 enum extern_type { 383 EXT_UNKNOWN, 384 EXT_KCFG, 385 EXT_KSYM, 386 }; 387 388 enum kcfg_type { 389 KCFG_UNKNOWN, 390 KCFG_CHAR, 391 KCFG_BOOL, 392 KCFG_INT, 393 KCFG_TRISTATE, 394 KCFG_CHAR_ARR, 395 }; 396 397 struct extern_desc { 398 enum extern_type type; 399 int sym_idx; 400 int btf_id; 401 int sec_btf_id; 402 const char *name; 403 bool is_set; 404 bool is_weak; 405 union { 406 struct { 407 enum kcfg_type type; 408 int sz; 409 int align; 410 int data_off; 411 bool is_signed; 412 } kcfg; 413 struct { 414 unsigned long long addr; 415 416 /* target btf_id of the corresponding kernel var. */ 417 int kernel_btf_obj_fd; 418 int kernel_btf_id; 419 420 /* local btf_id of the ksym extern's type. */ 421 __u32 type_id; 422 } ksym; 423 }; 424 }; 425 426 static LIST_HEAD(bpf_objects_list); 427 428 struct module_btf { 429 struct btf *btf; 430 char *name; 431 __u32 id; 432 int fd; 433 }; 434 435 struct bpf_object { 436 char name[BPF_OBJ_NAME_LEN]; 437 char license[64]; 438 __u32 kern_version; 439 440 struct bpf_program *programs; 441 size_t nr_programs; 442 struct bpf_map *maps; 443 size_t nr_maps; 444 size_t maps_cap; 445 446 char *kconfig; 447 struct extern_desc *externs; 448 int nr_extern; 449 int kconfig_map_idx; 450 int rodata_map_idx; 451 452 bool loaded; 453 bool has_subcalls; 454 455 struct bpf_gen *gen_loader; 456 457 /* 458 * Information when doing elf related work. Only valid if fd 459 * is valid. 460 */ 461 struct { 462 int fd; 463 const void *obj_buf; 464 size_t obj_buf_sz; 465 Elf *elf; 466 GElf_Ehdr ehdr; 467 Elf_Data *symbols; 468 Elf_Data *data; 469 Elf_Data *rodata; 470 Elf_Data *bss; 471 Elf_Data *st_ops_data; 472 size_t shstrndx; /* section index for section name strings */ 473 size_t strtabidx; 474 struct { 475 GElf_Shdr shdr; 476 Elf_Data *data; 477 } *reloc_sects; 478 int nr_reloc_sects; 479 int maps_shndx; 480 int btf_maps_shndx; 481 __u32 btf_maps_sec_btf_id; 482 int text_shndx; 483 int symbols_shndx; 484 int data_shndx; 485 int rodata_shndx; 486 int bss_shndx; 487 int st_ops_shndx; 488 } efile; 489 /* 490 * All loaded bpf_object is linked in a list, which is 491 * hidden to caller. bpf_objects__<func> handlers deal with 492 * all objects. 493 */ 494 struct list_head list; 495 496 struct btf *btf; 497 struct btf_ext *btf_ext; 498 499 /* Parse and load BTF vmlinux if any of the programs in the object need 500 * it at load time. 501 */ 502 struct btf *btf_vmlinux; 503 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 504 * override for vmlinux BTF. 505 */ 506 char *btf_custom_path; 507 /* vmlinux BTF override for CO-RE relocations */ 508 struct btf *btf_vmlinux_override; 509 /* Lazily initialized kernel module BTFs */ 510 struct module_btf *btf_modules; 511 bool btf_modules_loaded; 512 size_t btf_module_cnt; 513 size_t btf_module_cap; 514 515 void *priv; 516 bpf_object_clear_priv_t clear_priv; 517 518 char path[]; 519 }; 520 #define obj_elf_valid(o) ((o)->efile.elf) 521 522 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 523 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 524 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 525 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 526 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr); 527 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 528 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 529 530 void bpf_program__unload(struct bpf_program *prog) 531 { 532 int i; 533 534 if (!prog) 535 return; 536 537 /* 538 * If the object is opened but the program was never loaded, 539 * it is possible that prog->instances.nr == -1. 540 */ 541 if (prog->instances.nr > 0) { 542 for (i = 0; i < prog->instances.nr; i++) 543 zclose(prog->instances.fds[i]); 544 } else if (prog->instances.nr != -1) { 545 pr_warn("Internal error: instances.nr is %d\n", 546 prog->instances.nr); 547 } 548 549 prog->instances.nr = -1; 550 zfree(&prog->instances.fds); 551 552 zfree(&prog->func_info); 553 zfree(&prog->line_info); 554 } 555 556 static void bpf_program__exit(struct bpf_program *prog) 557 { 558 if (!prog) 559 return; 560 561 if (prog->clear_priv) 562 prog->clear_priv(prog, prog->priv); 563 564 prog->priv = NULL; 565 prog->clear_priv = NULL; 566 567 bpf_program__unload(prog); 568 zfree(&prog->name); 569 zfree(&prog->sec_name); 570 zfree(&prog->pin_name); 571 zfree(&prog->insns); 572 zfree(&prog->reloc_desc); 573 574 prog->nr_reloc = 0; 575 prog->insns_cnt = 0; 576 prog->sec_idx = -1; 577 } 578 579 static char *__bpf_program__pin_name(struct bpf_program *prog) 580 { 581 char *name, *p; 582 583 name = p = strdup(prog->sec_name); 584 while ((p = strchr(p, '/'))) 585 *p = '_'; 586 587 return name; 588 } 589 590 static bool insn_is_subprog_call(const struct bpf_insn *insn) 591 { 592 return BPF_CLASS(insn->code) == BPF_JMP && 593 BPF_OP(insn->code) == BPF_CALL && 594 BPF_SRC(insn->code) == BPF_K && 595 insn->src_reg == BPF_PSEUDO_CALL && 596 insn->dst_reg == 0 && 597 insn->off == 0; 598 } 599 600 static bool is_call_insn(const struct bpf_insn *insn) 601 { 602 return insn->code == (BPF_JMP | BPF_CALL); 603 } 604 605 static bool insn_is_pseudo_func(struct bpf_insn *insn) 606 { 607 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 608 } 609 610 static int 611 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 612 const char *name, size_t sec_idx, const char *sec_name, 613 size_t sec_off, void *insn_data, size_t insn_data_sz) 614 { 615 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 616 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 617 sec_name, name, sec_off, insn_data_sz); 618 return -EINVAL; 619 } 620 621 memset(prog, 0, sizeof(*prog)); 622 prog->obj = obj; 623 624 prog->sec_idx = sec_idx; 625 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 626 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 627 /* insns_cnt can later be increased by appending used subprograms */ 628 prog->insns_cnt = prog->sec_insn_cnt; 629 630 prog->type = BPF_PROG_TYPE_UNSPEC; 631 prog->load = true; 632 633 prog->instances.fds = NULL; 634 prog->instances.nr = -1; 635 636 prog->sec_name = strdup(sec_name); 637 if (!prog->sec_name) 638 goto errout; 639 640 prog->name = strdup(name); 641 if (!prog->name) 642 goto errout; 643 644 prog->pin_name = __bpf_program__pin_name(prog); 645 if (!prog->pin_name) 646 goto errout; 647 648 prog->insns = malloc(insn_data_sz); 649 if (!prog->insns) 650 goto errout; 651 memcpy(prog->insns, insn_data, insn_data_sz); 652 653 return 0; 654 errout: 655 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 656 bpf_program__exit(prog); 657 return -ENOMEM; 658 } 659 660 static int 661 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 662 const char *sec_name, int sec_idx) 663 { 664 Elf_Data *symbols = obj->efile.symbols; 665 struct bpf_program *prog, *progs; 666 void *data = sec_data->d_buf; 667 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 668 int nr_progs, err, i; 669 const char *name; 670 GElf_Sym sym; 671 672 progs = obj->programs; 673 nr_progs = obj->nr_programs; 674 nr_syms = symbols->d_size / sizeof(GElf_Sym); 675 sec_off = 0; 676 677 for (i = 0; i < nr_syms; i++) { 678 if (!gelf_getsym(symbols, i, &sym)) 679 continue; 680 if (sym.st_shndx != sec_idx) 681 continue; 682 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC) 683 continue; 684 685 prog_sz = sym.st_size; 686 sec_off = sym.st_value; 687 688 name = elf_sym_str(obj, sym.st_name); 689 if (!name) { 690 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 691 sec_name, sec_off); 692 return -LIBBPF_ERRNO__FORMAT; 693 } 694 695 if (sec_off + prog_sz > sec_sz) { 696 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 697 sec_name, sec_off); 698 return -LIBBPF_ERRNO__FORMAT; 699 } 700 701 if (sec_idx != obj->efile.text_shndx && GELF_ST_BIND(sym.st_info) == STB_LOCAL) { 702 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 703 return -ENOTSUP; 704 } 705 706 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 707 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 708 709 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 710 if (!progs) { 711 /* 712 * In this case the original obj->programs 713 * is still valid, so don't need special treat for 714 * bpf_close_object(). 715 */ 716 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 717 sec_name, name); 718 return -ENOMEM; 719 } 720 obj->programs = progs; 721 722 prog = &progs[nr_progs]; 723 724 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 725 sec_off, data + sec_off, prog_sz); 726 if (err) 727 return err; 728 729 /* if function is a global/weak symbol, but has restricted 730 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 731 * as static to enable more permissive BPF verification mode 732 * with more outside context available to BPF verifier 733 */ 734 if (GELF_ST_BIND(sym.st_info) != STB_LOCAL 735 && (GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN 736 || GELF_ST_VISIBILITY(sym.st_other) == STV_INTERNAL)) 737 prog->mark_btf_static = true; 738 739 nr_progs++; 740 obj->nr_programs = nr_progs; 741 } 742 743 return 0; 744 } 745 746 static __u32 get_kernel_version(void) 747 { 748 __u32 major, minor, patch; 749 struct utsname info; 750 751 uname(&info); 752 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) 753 return 0; 754 return KERNEL_VERSION(major, minor, patch); 755 } 756 757 static const struct btf_member * 758 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 759 { 760 struct btf_member *m; 761 int i; 762 763 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 764 if (btf_member_bit_offset(t, i) == bit_offset) 765 return m; 766 } 767 768 return NULL; 769 } 770 771 static const struct btf_member * 772 find_member_by_name(const struct btf *btf, const struct btf_type *t, 773 const char *name) 774 { 775 struct btf_member *m; 776 int i; 777 778 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 779 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 780 return m; 781 } 782 783 return NULL; 784 } 785 786 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 787 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 788 const char *name, __u32 kind); 789 790 static int 791 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 792 const struct btf_type **type, __u32 *type_id, 793 const struct btf_type **vtype, __u32 *vtype_id, 794 const struct btf_member **data_member) 795 { 796 const struct btf_type *kern_type, *kern_vtype; 797 const struct btf_member *kern_data_member; 798 __s32 kern_vtype_id, kern_type_id; 799 __u32 i; 800 801 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 802 if (kern_type_id < 0) { 803 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 804 tname); 805 return kern_type_id; 806 } 807 kern_type = btf__type_by_id(btf, kern_type_id); 808 809 /* Find the corresponding "map_value" type that will be used 810 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 811 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 812 * btf_vmlinux. 813 */ 814 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 815 tname, BTF_KIND_STRUCT); 816 if (kern_vtype_id < 0) { 817 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 818 STRUCT_OPS_VALUE_PREFIX, tname); 819 return kern_vtype_id; 820 } 821 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 822 823 /* Find "struct tcp_congestion_ops" from 824 * struct bpf_struct_ops_tcp_congestion_ops { 825 * [ ... ] 826 * struct tcp_congestion_ops data; 827 * } 828 */ 829 kern_data_member = btf_members(kern_vtype); 830 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 831 if (kern_data_member->type == kern_type_id) 832 break; 833 } 834 if (i == btf_vlen(kern_vtype)) { 835 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 836 tname, STRUCT_OPS_VALUE_PREFIX, tname); 837 return -EINVAL; 838 } 839 840 *type = kern_type; 841 *type_id = kern_type_id; 842 *vtype = kern_vtype; 843 *vtype_id = kern_vtype_id; 844 *data_member = kern_data_member; 845 846 return 0; 847 } 848 849 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 850 { 851 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 852 } 853 854 /* Init the map's fields that depend on kern_btf */ 855 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 856 const struct btf *btf, 857 const struct btf *kern_btf) 858 { 859 const struct btf_member *member, *kern_member, *kern_data_member; 860 const struct btf_type *type, *kern_type, *kern_vtype; 861 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 862 struct bpf_struct_ops *st_ops; 863 void *data, *kern_data; 864 const char *tname; 865 int err; 866 867 st_ops = map->st_ops; 868 type = st_ops->type; 869 tname = st_ops->tname; 870 err = find_struct_ops_kern_types(kern_btf, tname, 871 &kern_type, &kern_type_id, 872 &kern_vtype, &kern_vtype_id, 873 &kern_data_member); 874 if (err) 875 return err; 876 877 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 878 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 879 880 map->def.value_size = kern_vtype->size; 881 map->btf_vmlinux_value_type_id = kern_vtype_id; 882 883 st_ops->kern_vdata = calloc(1, kern_vtype->size); 884 if (!st_ops->kern_vdata) 885 return -ENOMEM; 886 887 data = st_ops->data; 888 kern_data_off = kern_data_member->offset / 8; 889 kern_data = st_ops->kern_vdata + kern_data_off; 890 891 member = btf_members(type); 892 for (i = 0; i < btf_vlen(type); i++, member++) { 893 const struct btf_type *mtype, *kern_mtype; 894 __u32 mtype_id, kern_mtype_id; 895 void *mdata, *kern_mdata; 896 __s64 msize, kern_msize; 897 __u32 moff, kern_moff; 898 __u32 kern_member_idx; 899 const char *mname; 900 901 mname = btf__name_by_offset(btf, member->name_off); 902 kern_member = find_member_by_name(kern_btf, kern_type, mname); 903 if (!kern_member) { 904 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 905 map->name, mname); 906 return -ENOTSUP; 907 } 908 909 kern_member_idx = kern_member - btf_members(kern_type); 910 if (btf_member_bitfield_size(type, i) || 911 btf_member_bitfield_size(kern_type, kern_member_idx)) { 912 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 913 map->name, mname); 914 return -ENOTSUP; 915 } 916 917 moff = member->offset / 8; 918 kern_moff = kern_member->offset / 8; 919 920 mdata = data + moff; 921 kern_mdata = kern_data + kern_moff; 922 923 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 924 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 925 &kern_mtype_id); 926 if (BTF_INFO_KIND(mtype->info) != 927 BTF_INFO_KIND(kern_mtype->info)) { 928 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 929 map->name, mname, BTF_INFO_KIND(mtype->info), 930 BTF_INFO_KIND(kern_mtype->info)); 931 return -ENOTSUP; 932 } 933 934 if (btf_is_ptr(mtype)) { 935 struct bpf_program *prog; 936 937 prog = st_ops->progs[i]; 938 if (!prog) 939 continue; 940 941 kern_mtype = skip_mods_and_typedefs(kern_btf, 942 kern_mtype->type, 943 &kern_mtype_id); 944 945 /* mtype->type must be a func_proto which was 946 * guaranteed in bpf_object__collect_st_ops_relos(), 947 * so only check kern_mtype for func_proto here. 948 */ 949 if (!btf_is_func_proto(kern_mtype)) { 950 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 951 map->name, mname); 952 return -ENOTSUP; 953 } 954 955 prog->attach_btf_id = kern_type_id; 956 prog->expected_attach_type = kern_member_idx; 957 958 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 959 960 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 961 map->name, mname, prog->name, moff, 962 kern_moff); 963 964 continue; 965 } 966 967 msize = btf__resolve_size(btf, mtype_id); 968 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 969 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 970 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 971 map->name, mname, (ssize_t)msize, 972 (ssize_t)kern_msize); 973 return -ENOTSUP; 974 } 975 976 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 977 map->name, mname, (unsigned int)msize, 978 moff, kern_moff); 979 memcpy(kern_mdata, mdata, msize); 980 } 981 982 return 0; 983 } 984 985 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 986 { 987 struct bpf_map *map; 988 size_t i; 989 int err; 990 991 for (i = 0; i < obj->nr_maps; i++) { 992 map = &obj->maps[i]; 993 994 if (!bpf_map__is_struct_ops(map)) 995 continue; 996 997 err = bpf_map__init_kern_struct_ops(map, obj->btf, 998 obj->btf_vmlinux); 999 if (err) 1000 return err; 1001 } 1002 1003 return 0; 1004 } 1005 1006 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) 1007 { 1008 const struct btf_type *type, *datasec; 1009 const struct btf_var_secinfo *vsi; 1010 struct bpf_struct_ops *st_ops; 1011 const char *tname, *var_name; 1012 __s32 type_id, datasec_id; 1013 const struct btf *btf; 1014 struct bpf_map *map; 1015 __u32 i; 1016 1017 if (obj->efile.st_ops_shndx == -1) 1018 return 0; 1019 1020 btf = obj->btf; 1021 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, 1022 BTF_KIND_DATASEC); 1023 if (datasec_id < 0) { 1024 pr_warn("struct_ops init: DATASEC %s not found\n", 1025 STRUCT_OPS_SEC); 1026 return -EINVAL; 1027 } 1028 1029 datasec = btf__type_by_id(btf, datasec_id); 1030 vsi = btf_var_secinfos(datasec); 1031 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1032 type = btf__type_by_id(obj->btf, vsi->type); 1033 var_name = btf__name_by_offset(obj->btf, type->name_off); 1034 1035 type_id = btf__resolve_type(obj->btf, vsi->type); 1036 if (type_id < 0) { 1037 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1038 vsi->type, STRUCT_OPS_SEC); 1039 return -EINVAL; 1040 } 1041 1042 type = btf__type_by_id(obj->btf, type_id); 1043 tname = btf__name_by_offset(obj->btf, type->name_off); 1044 if (!tname[0]) { 1045 pr_warn("struct_ops init: anonymous type is not supported\n"); 1046 return -ENOTSUP; 1047 } 1048 if (!btf_is_struct(type)) { 1049 pr_warn("struct_ops init: %s is not a struct\n", tname); 1050 return -EINVAL; 1051 } 1052 1053 map = bpf_object__add_map(obj); 1054 if (IS_ERR(map)) 1055 return PTR_ERR(map); 1056 1057 map->sec_idx = obj->efile.st_ops_shndx; 1058 map->sec_offset = vsi->offset; 1059 map->name = strdup(var_name); 1060 if (!map->name) 1061 return -ENOMEM; 1062 1063 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1064 map->def.key_size = sizeof(int); 1065 map->def.value_size = type->size; 1066 map->def.max_entries = 1; 1067 1068 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1069 if (!map->st_ops) 1070 return -ENOMEM; 1071 st_ops = map->st_ops; 1072 st_ops->data = malloc(type->size); 1073 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1074 st_ops->kern_func_off = malloc(btf_vlen(type) * 1075 sizeof(*st_ops->kern_func_off)); 1076 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1077 return -ENOMEM; 1078 1079 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { 1080 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1081 var_name, STRUCT_OPS_SEC); 1082 return -EINVAL; 1083 } 1084 1085 memcpy(st_ops->data, 1086 obj->efile.st_ops_data->d_buf + vsi->offset, 1087 type->size); 1088 st_ops->tname = tname; 1089 st_ops->type = type; 1090 st_ops->type_id = type_id; 1091 1092 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1093 tname, type_id, var_name, vsi->offset); 1094 } 1095 1096 return 0; 1097 } 1098 1099 static struct bpf_object *bpf_object__new(const char *path, 1100 const void *obj_buf, 1101 size_t obj_buf_sz, 1102 const char *obj_name) 1103 { 1104 struct bpf_object *obj; 1105 char *end; 1106 1107 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1108 if (!obj) { 1109 pr_warn("alloc memory failed for %s\n", path); 1110 return ERR_PTR(-ENOMEM); 1111 } 1112 1113 strcpy(obj->path, path); 1114 if (obj_name) { 1115 strncpy(obj->name, obj_name, sizeof(obj->name) - 1); 1116 obj->name[sizeof(obj->name) - 1] = 0; 1117 } else { 1118 /* Using basename() GNU version which doesn't modify arg. */ 1119 strncpy(obj->name, basename((void *)path), 1120 sizeof(obj->name) - 1); 1121 end = strchr(obj->name, '.'); 1122 if (end) 1123 *end = 0; 1124 } 1125 1126 obj->efile.fd = -1; 1127 /* 1128 * Caller of this function should also call 1129 * bpf_object__elf_finish() after data collection to return 1130 * obj_buf to user. If not, we should duplicate the buffer to 1131 * avoid user freeing them before elf finish. 1132 */ 1133 obj->efile.obj_buf = obj_buf; 1134 obj->efile.obj_buf_sz = obj_buf_sz; 1135 obj->efile.maps_shndx = -1; 1136 obj->efile.btf_maps_shndx = -1; 1137 obj->efile.data_shndx = -1; 1138 obj->efile.rodata_shndx = -1; 1139 obj->efile.bss_shndx = -1; 1140 obj->efile.st_ops_shndx = -1; 1141 obj->kconfig_map_idx = -1; 1142 obj->rodata_map_idx = -1; 1143 1144 obj->kern_version = get_kernel_version(); 1145 obj->loaded = false; 1146 1147 INIT_LIST_HEAD(&obj->list); 1148 list_add(&obj->list, &bpf_objects_list); 1149 return obj; 1150 } 1151 1152 static void bpf_object__elf_finish(struct bpf_object *obj) 1153 { 1154 if (!obj_elf_valid(obj)) 1155 return; 1156 1157 if (obj->efile.elf) { 1158 elf_end(obj->efile.elf); 1159 obj->efile.elf = NULL; 1160 } 1161 obj->efile.symbols = NULL; 1162 obj->efile.data = NULL; 1163 obj->efile.rodata = NULL; 1164 obj->efile.bss = NULL; 1165 obj->efile.st_ops_data = NULL; 1166 1167 zfree(&obj->efile.reloc_sects); 1168 obj->efile.nr_reloc_sects = 0; 1169 zclose(obj->efile.fd); 1170 obj->efile.obj_buf = NULL; 1171 obj->efile.obj_buf_sz = 0; 1172 } 1173 1174 static int bpf_object__elf_init(struct bpf_object *obj) 1175 { 1176 int err = 0; 1177 GElf_Ehdr *ep; 1178 1179 if (obj_elf_valid(obj)) { 1180 pr_warn("elf: init internal error\n"); 1181 return -LIBBPF_ERRNO__LIBELF; 1182 } 1183 1184 if (obj->efile.obj_buf_sz > 0) { 1185 /* 1186 * obj_buf should have been validated by 1187 * bpf_object__open_buffer(). 1188 */ 1189 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, 1190 obj->efile.obj_buf_sz); 1191 } else { 1192 obj->efile.fd = open(obj->path, O_RDONLY); 1193 if (obj->efile.fd < 0) { 1194 char errmsg[STRERR_BUFSIZE], *cp; 1195 1196 err = -errno; 1197 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1198 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1199 return err; 1200 } 1201 1202 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1203 } 1204 1205 if (!obj->efile.elf) { 1206 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1207 err = -LIBBPF_ERRNO__LIBELF; 1208 goto errout; 1209 } 1210 1211 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 1212 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1213 err = -LIBBPF_ERRNO__FORMAT; 1214 goto errout; 1215 } 1216 ep = &obj->efile.ehdr; 1217 1218 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) { 1219 pr_warn("elf: failed to get section names section index for %s: %s\n", 1220 obj->path, elf_errmsg(-1)); 1221 err = -LIBBPF_ERRNO__FORMAT; 1222 goto errout; 1223 } 1224 1225 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1226 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) { 1227 pr_warn("elf: failed to get section names strings from %s: %s\n", 1228 obj->path, elf_errmsg(-1)); 1229 err = -LIBBPF_ERRNO__FORMAT; 1230 goto errout; 1231 } 1232 1233 /* Old LLVM set e_machine to EM_NONE */ 1234 if (ep->e_type != ET_REL || 1235 (ep->e_machine && ep->e_machine != EM_BPF)) { 1236 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1237 err = -LIBBPF_ERRNO__FORMAT; 1238 goto errout; 1239 } 1240 1241 return 0; 1242 errout: 1243 bpf_object__elf_finish(obj); 1244 return err; 1245 } 1246 1247 static int bpf_object__check_endianness(struct bpf_object *obj) 1248 { 1249 #if __BYTE_ORDER == __LITTLE_ENDIAN 1250 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 1251 return 0; 1252 #elif __BYTE_ORDER == __BIG_ENDIAN 1253 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 1254 return 0; 1255 #else 1256 # error "Unrecognized __BYTE_ORDER__" 1257 #endif 1258 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1259 return -LIBBPF_ERRNO__ENDIAN; 1260 } 1261 1262 static int 1263 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1264 { 1265 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 1266 pr_debug("license of %s is %s\n", obj->path, obj->license); 1267 return 0; 1268 } 1269 1270 static int 1271 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1272 { 1273 __u32 kver; 1274 1275 if (size != sizeof(kver)) { 1276 pr_warn("invalid kver section in %s\n", obj->path); 1277 return -LIBBPF_ERRNO__FORMAT; 1278 } 1279 memcpy(&kver, data, sizeof(kver)); 1280 obj->kern_version = kver; 1281 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1282 return 0; 1283 } 1284 1285 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1286 { 1287 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1288 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1289 return true; 1290 return false; 1291 } 1292 1293 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 1294 __u32 *size) 1295 { 1296 int ret = -ENOENT; 1297 1298 *size = 0; 1299 if (!name) { 1300 return -EINVAL; 1301 } else if (!strcmp(name, DATA_SEC)) { 1302 if (obj->efile.data) 1303 *size = obj->efile.data->d_size; 1304 } else if (!strcmp(name, BSS_SEC)) { 1305 if (obj->efile.bss) 1306 *size = obj->efile.bss->d_size; 1307 } else if (!strcmp(name, RODATA_SEC)) { 1308 if (obj->efile.rodata) 1309 *size = obj->efile.rodata->d_size; 1310 } else if (!strcmp(name, STRUCT_OPS_SEC)) { 1311 if (obj->efile.st_ops_data) 1312 *size = obj->efile.st_ops_data->d_size; 1313 } else { 1314 Elf_Scn *scn = elf_sec_by_name(obj, name); 1315 Elf_Data *data = elf_sec_data(obj, scn); 1316 1317 if (data) { 1318 ret = 0; /* found it */ 1319 *size = data->d_size; 1320 } 1321 } 1322 1323 return *size ? 0 : ret; 1324 } 1325 1326 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 1327 __u32 *off) 1328 { 1329 Elf_Data *symbols = obj->efile.symbols; 1330 const char *sname; 1331 size_t si; 1332 1333 if (!name || !off) 1334 return -EINVAL; 1335 1336 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 1337 GElf_Sym sym; 1338 1339 if (!gelf_getsym(symbols, si, &sym)) 1340 continue; 1341 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 1342 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 1343 continue; 1344 1345 sname = elf_sym_str(obj, sym.st_name); 1346 if (!sname) { 1347 pr_warn("failed to get sym name string for var %s\n", 1348 name); 1349 return -EIO; 1350 } 1351 if (strcmp(name, sname) == 0) { 1352 *off = sym.st_value; 1353 return 0; 1354 } 1355 } 1356 1357 return -ENOENT; 1358 } 1359 1360 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1361 { 1362 struct bpf_map *new_maps; 1363 size_t new_cap; 1364 int i; 1365 1366 if (obj->nr_maps < obj->maps_cap) 1367 return &obj->maps[obj->nr_maps++]; 1368 1369 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 1370 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps)); 1371 if (!new_maps) { 1372 pr_warn("alloc maps for object failed\n"); 1373 return ERR_PTR(-ENOMEM); 1374 } 1375 1376 obj->maps_cap = new_cap; 1377 obj->maps = new_maps; 1378 1379 /* zero out new maps */ 1380 memset(obj->maps + obj->nr_maps, 0, 1381 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 1382 /* 1383 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 1384 * when failure (zclose won't close negative fd)). 1385 */ 1386 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 1387 obj->maps[i].fd = -1; 1388 obj->maps[i].inner_map_fd = -1; 1389 } 1390 1391 return &obj->maps[obj->nr_maps++]; 1392 } 1393 1394 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1395 { 1396 long page_sz = sysconf(_SC_PAGE_SIZE); 1397 size_t map_sz; 1398 1399 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; 1400 map_sz = roundup(map_sz, page_sz); 1401 return map_sz; 1402 } 1403 1404 static char *internal_map_name(struct bpf_object *obj, 1405 enum libbpf_map_type type) 1406 { 1407 char map_name[BPF_OBJ_NAME_LEN], *p; 1408 const char *sfx = libbpf_type_to_btf_name[type]; 1409 int sfx_len = max((size_t)7, strlen(sfx)); 1410 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, 1411 strlen(obj->name)); 1412 1413 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1414 sfx_len, libbpf_type_to_btf_name[type]); 1415 1416 /* sanitise map name to characters allowed by kernel */ 1417 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1418 if (!isalnum(*p) && *p != '_' && *p != '.') 1419 *p = '_'; 1420 1421 return strdup(map_name); 1422 } 1423 1424 static int 1425 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1426 int sec_idx, void *data, size_t data_sz) 1427 { 1428 struct bpf_map_def *def; 1429 struct bpf_map *map; 1430 int err; 1431 1432 map = bpf_object__add_map(obj); 1433 if (IS_ERR(map)) 1434 return PTR_ERR(map); 1435 1436 map->libbpf_type = type; 1437 map->sec_idx = sec_idx; 1438 map->sec_offset = 0; 1439 map->name = internal_map_name(obj, type); 1440 if (!map->name) { 1441 pr_warn("failed to alloc map name\n"); 1442 return -ENOMEM; 1443 } 1444 1445 def = &map->def; 1446 def->type = BPF_MAP_TYPE_ARRAY; 1447 def->key_size = sizeof(int); 1448 def->value_size = data_sz; 1449 def->max_entries = 1; 1450 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1451 ? BPF_F_RDONLY_PROG : 0; 1452 def->map_flags |= BPF_F_MMAPABLE; 1453 1454 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1455 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1456 1457 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 1458 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1459 if (map->mmaped == MAP_FAILED) { 1460 err = -errno; 1461 map->mmaped = NULL; 1462 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1463 map->name, err); 1464 zfree(&map->name); 1465 return err; 1466 } 1467 1468 if (data) 1469 memcpy(map->mmaped, data, data_sz); 1470 1471 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1472 return 0; 1473 } 1474 1475 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1476 { 1477 int err; 1478 1479 /* 1480 * Populate obj->maps with libbpf internal maps. 1481 */ 1482 if (obj->efile.data_shndx >= 0) { 1483 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1484 obj->efile.data_shndx, 1485 obj->efile.data->d_buf, 1486 obj->efile.data->d_size); 1487 if (err) 1488 return err; 1489 } 1490 if (obj->efile.rodata_shndx >= 0) { 1491 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1492 obj->efile.rodata_shndx, 1493 obj->efile.rodata->d_buf, 1494 obj->efile.rodata->d_size); 1495 if (err) 1496 return err; 1497 1498 obj->rodata_map_idx = obj->nr_maps - 1; 1499 } 1500 if (obj->efile.bss_shndx >= 0) { 1501 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1502 obj->efile.bss_shndx, 1503 NULL, 1504 obj->efile.bss->d_size); 1505 if (err) 1506 return err; 1507 } 1508 return 0; 1509 } 1510 1511 1512 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1513 const void *name) 1514 { 1515 int i; 1516 1517 for (i = 0; i < obj->nr_extern; i++) { 1518 if (strcmp(obj->externs[i].name, name) == 0) 1519 return &obj->externs[i]; 1520 } 1521 return NULL; 1522 } 1523 1524 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1525 char value) 1526 { 1527 switch (ext->kcfg.type) { 1528 case KCFG_BOOL: 1529 if (value == 'm') { 1530 pr_warn("extern (kcfg) %s=%c should be tristate or char\n", 1531 ext->name, value); 1532 return -EINVAL; 1533 } 1534 *(bool *)ext_val = value == 'y' ? true : false; 1535 break; 1536 case KCFG_TRISTATE: 1537 if (value == 'y') 1538 *(enum libbpf_tristate *)ext_val = TRI_YES; 1539 else if (value == 'm') 1540 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1541 else /* value == 'n' */ 1542 *(enum libbpf_tristate *)ext_val = TRI_NO; 1543 break; 1544 case KCFG_CHAR: 1545 *(char *)ext_val = value; 1546 break; 1547 case KCFG_UNKNOWN: 1548 case KCFG_INT: 1549 case KCFG_CHAR_ARR: 1550 default: 1551 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", 1552 ext->name, value); 1553 return -EINVAL; 1554 } 1555 ext->is_set = true; 1556 return 0; 1557 } 1558 1559 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1560 const char *value) 1561 { 1562 size_t len; 1563 1564 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1565 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); 1566 return -EINVAL; 1567 } 1568 1569 len = strlen(value); 1570 if (value[len - 1] != '"') { 1571 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1572 ext->name, value); 1573 return -EINVAL; 1574 } 1575 1576 /* strip quotes */ 1577 len -= 2; 1578 if (len >= ext->kcfg.sz) { 1579 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", 1580 ext->name, value, len, ext->kcfg.sz - 1); 1581 len = ext->kcfg.sz - 1; 1582 } 1583 memcpy(ext_val, value + 1, len); 1584 ext_val[len] = '\0'; 1585 ext->is_set = true; 1586 return 0; 1587 } 1588 1589 static int parse_u64(const char *value, __u64 *res) 1590 { 1591 char *value_end; 1592 int err; 1593 1594 errno = 0; 1595 *res = strtoull(value, &value_end, 0); 1596 if (errno) { 1597 err = -errno; 1598 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1599 return err; 1600 } 1601 if (*value_end) { 1602 pr_warn("failed to parse '%s' as integer completely\n", value); 1603 return -EINVAL; 1604 } 1605 return 0; 1606 } 1607 1608 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1609 { 1610 int bit_sz = ext->kcfg.sz * 8; 1611 1612 if (ext->kcfg.sz == 8) 1613 return true; 1614 1615 /* Validate that value stored in u64 fits in integer of `ext->sz` 1616 * bytes size without any loss of information. If the target integer 1617 * is signed, we rely on the following limits of integer type of 1618 * Y bits and subsequent transformation: 1619 * 1620 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1621 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1622 * 0 <= X + 2^(Y-1) < 2^Y 1623 * 1624 * For unsigned target integer, check that all the (64 - Y) bits are 1625 * zero. 1626 */ 1627 if (ext->kcfg.is_signed) 1628 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1629 else 1630 return (v >> bit_sz) == 0; 1631 } 1632 1633 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1634 __u64 value) 1635 { 1636 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 1637 pr_warn("extern (kcfg) %s=%llu should be integer\n", 1638 ext->name, (unsigned long long)value); 1639 return -EINVAL; 1640 } 1641 if (!is_kcfg_value_in_range(ext, value)) { 1642 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", 1643 ext->name, (unsigned long long)value, ext->kcfg.sz); 1644 return -ERANGE; 1645 } 1646 switch (ext->kcfg.sz) { 1647 case 1: *(__u8 *)ext_val = value; break; 1648 case 2: *(__u16 *)ext_val = value; break; 1649 case 4: *(__u32 *)ext_val = value; break; 1650 case 8: *(__u64 *)ext_val = value; break; 1651 default: 1652 return -EINVAL; 1653 } 1654 ext->is_set = true; 1655 return 0; 1656 } 1657 1658 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1659 char *buf, void *data) 1660 { 1661 struct extern_desc *ext; 1662 char *sep, *value; 1663 int len, err = 0; 1664 void *ext_val; 1665 __u64 num; 1666 1667 if (strncmp(buf, "CONFIG_", 7)) 1668 return 0; 1669 1670 sep = strchr(buf, '='); 1671 if (!sep) { 1672 pr_warn("failed to parse '%s': no separator\n", buf); 1673 return -EINVAL; 1674 } 1675 1676 /* Trim ending '\n' */ 1677 len = strlen(buf); 1678 if (buf[len - 1] == '\n') 1679 buf[len - 1] = '\0'; 1680 /* Split on '=' and ensure that a value is present. */ 1681 *sep = '\0'; 1682 if (!sep[1]) { 1683 *sep = '='; 1684 pr_warn("failed to parse '%s': no value\n", buf); 1685 return -EINVAL; 1686 } 1687 1688 ext = find_extern_by_name(obj, buf); 1689 if (!ext || ext->is_set) 1690 return 0; 1691 1692 ext_val = data + ext->kcfg.data_off; 1693 value = sep + 1; 1694 1695 switch (*value) { 1696 case 'y': case 'n': case 'm': 1697 err = set_kcfg_value_tri(ext, ext_val, *value); 1698 break; 1699 case '"': 1700 err = set_kcfg_value_str(ext, ext_val, value); 1701 break; 1702 default: 1703 /* assume integer */ 1704 err = parse_u64(value, &num); 1705 if (err) { 1706 pr_warn("extern (kcfg) %s=%s should be integer\n", 1707 ext->name, value); 1708 return err; 1709 } 1710 err = set_kcfg_value_num(ext, ext_val, num); 1711 break; 1712 } 1713 if (err) 1714 return err; 1715 pr_debug("extern (kcfg) %s=%s\n", ext->name, value); 1716 return 0; 1717 } 1718 1719 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1720 { 1721 char buf[PATH_MAX]; 1722 struct utsname uts; 1723 int len, err = 0; 1724 gzFile file; 1725 1726 uname(&uts); 1727 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1728 if (len < 0) 1729 return -EINVAL; 1730 else if (len >= PATH_MAX) 1731 return -ENAMETOOLONG; 1732 1733 /* gzopen also accepts uncompressed files. */ 1734 file = gzopen(buf, "r"); 1735 if (!file) 1736 file = gzopen("/proc/config.gz", "r"); 1737 1738 if (!file) { 1739 pr_warn("failed to open system Kconfig\n"); 1740 return -ENOENT; 1741 } 1742 1743 while (gzgets(file, buf, sizeof(buf))) { 1744 err = bpf_object__process_kconfig_line(obj, buf, data); 1745 if (err) { 1746 pr_warn("error parsing system Kconfig line '%s': %d\n", 1747 buf, err); 1748 goto out; 1749 } 1750 } 1751 1752 out: 1753 gzclose(file); 1754 return err; 1755 } 1756 1757 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 1758 const char *config, void *data) 1759 { 1760 char buf[PATH_MAX]; 1761 int err = 0; 1762 FILE *file; 1763 1764 file = fmemopen((void *)config, strlen(config), "r"); 1765 if (!file) { 1766 err = -errno; 1767 pr_warn("failed to open in-memory Kconfig: %d\n", err); 1768 return err; 1769 } 1770 1771 while (fgets(buf, sizeof(buf), file)) { 1772 err = bpf_object__process_kconfig_line(obj, buf, data); 1773 if (err) { 1774 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 1775 buf, err); 1776 break; 1777 } 1778 } 1779 1780 fclose(file); 1781 return err; 1782 } 1783 1784 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 1785 { 1786 struct extern_desc *last_ext = NULL, *ext; 1787 size_t map_sz; 1788 int i, err; 1789 1790 for (i = 0; i < obj->nr_extern; i++) { 1791 ext = &obj->externs[i]; 1792 if (ext->type == EXT_KCFG) 1793 last_ext = ext; 1794 } 1795 1796 if (!last_ext) 1797 return 0; 1798 1799 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 1800 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 1801 obj->efile.symbols_shndx, 1802 NULL, map_sz); 1803 if (err) 1804 return err; 1805 1806 obj->kconfig_map_idx = obj->nr_maps - 1; 1807 1808 return 0; 1809 } 1810 1811 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 1812 { 1813 Elf_Data *symbols = obj->efile.symbols; 1814 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 1815 Elf_Data *data = NULL; 1816 Elf_Scn *scn; 1817 1818 if (obj->efile.maps_shndx < 0) 1819 return 0; 1820 1821 if (!symbols) 1822 return -EINVAL; 1823 1824 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx); 1825 data = elf_sec_data(obj, scn); 1826 if (!scn || !data) { 1827 pr_warn("elf: failed to get legacy map definitions for %s\n", 1828 obj->path); 1829 return -EINVAL; 1830 } 1831 1832 /* 1833 * Count number of maps. Each map has a name. 1834 * Array of maps is not supported: only the first element is 1835 * considered. 1836 * 1837 * TODO: Detect array of map and report error. 1838 */ 1839 nr_syms = symbols->d_size / sizeof(GElf_Sym); 1840 for (i = 0; i < nr_syms; i++) { 1841 GElf_Sym sym; 1842 1843 if (!gelf_getsym(symbols, i, &sym)) 1844 continue; 1845 if (sym.st_shndx != obj->efile.maps_shndx) 1846 continue; 1847 nr_maps++; 1848 } 1849 /* Assume equally sized map definitions */ 1850 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n", 1851 nr_maps, data->d_size, obj->path); 1852 1853 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { 1854 pr_warn("elf: unable to determine legacy map definition size in %s\n", 1855 obj->path); 1856 return -EINVAL; 1857 } 1858 map_def_sz = data->d_size / nr_maps; 1859 1860 /* Fill obj->maps using data in "maps" section. */ 1861 for (i = 0; i < nr_syms; i++) { 1862 GElf_Sym sym; 1863 const char *map_name; 1864 struct bpf_map_def *def; 1865 struct bpf_map *map; 1866 1867 if (!gelf_getsym(symbols, i, &sym)) 1868 continue; 1869 if (sym.st_shndx != obj->efile.maps_shndx) 1870 continue; 1871 1872 map = bpf_object__add_map(obj); 1873 if (IS_ERR(map)) 1874 return PTR_ERR(map); 1875 1876 map_name = elf_sym_str(obj, sym.st_name); 1877 if (!map_name) { 1878 pr_warn("failed to get map #%d name sym string for obj %s\n", 1879 i, obj->path); 1880 return -LIBBPF_ERRNO__FORMAT; 1881 } 1882 1883 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION 1884 || GELF_ST_BIND(sym.st_info) == STB_LOCAL) { 1885 pr_warn("map '%s' (legacy): static maps are not supported\n", map_name); 1886 return -ENOTSUP; 1887 } 1888 1889 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1890 map->sec_idx = sym.st_shndx; 1891 map->sec_offset = sym.st_value; 1892 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 1893 map_name, map->sec_idx, map->sec_offset); 1894 if (sym.st_value + map_def_sz > data->d_size) { 1895 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", 1896 obj->path, map_name); 1897 return -EINVAL; 1898 } 1899 1900 map->name = strdup(map_name); 1901 if (!map->name) { 1902 pr_warn("failed to alloc map name\n"); 1903 return -ENOMEM; 1904 } 1905 pr_debug("map %d is \"%s\"\n", i, map->name); 1906 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 1907 /* 1908 * If the definition of the map in the object file fits in 1909 * bpf_map_def, copy it. Any extra fields in our version 1910 * of bpf_map_def will default to zero as a result of the 1911 * calloc above. 1912 */ 1913 if (map_def_sz <= sizeof(struct bpf_map_def)) { 1914 memcpy(&map->def, def, map_def_sz); 1915 } else { 1916 /* 1917 * Here the map structure being read is bigger than what 1918 * we expect, truncate if the excess bits are all zero. 1919 * If they are not zero, reject this map as 1920 * incompatible. 1921 */ 1922 char *b; 1923 1924 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1925 b < ((char *)def) + map_def_sz; b++) { 1926 if (*b != 0) { 1927 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", 1928 obj->path, map_name); 1929 if (strict) 1930 return -EINVAL; 1931 } 1932 } 1933 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1934 } 1935 } 1936 return 0; 1937 } 1938 1939 const struct btf_type * 1940 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 1941 { 1942 const struct btf_type *t = btf__type_by_id(btf, id); 1943 1944 if (res_id) 1945 *res_id = id; 1946 1947 while (btf_is_mod(t) || btf_is_typedef(t)) { 1948 if (res_id) 1949 *res_id = t->type; 1950 t = btf__type_by_id(btf, t->type); 1951 } 1952 1953 return t; 1954 } 1955 1956 static const struct btf_type * 1957 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 1958 { 1959 const struct btf_type *t; 1960 1961 t = skip_mods_and_typedefs(btf, id, NULL); 1962 if (!btf_is_ptr(t)) 1963 return NULL; 1964 1965 t = skip_mods_and_typedefs(btf, t->type, res_id); 1966 1967 return btf_is_func_proto(t) ? t : NULL; 1968 } 1969 1970 static const char *__btf_kind_str(__u16 kind) 1971 { 1972 switch (kind) { 1973 case BTF_KIND_UNKN: return "void"; 1974 case BTF_KIND_INT: return "int"; 1975 case BTF_KIND_PTR: return "ptr"; 1976 case BTF_KIND_ARRAY: return "array"; 1977 case BTF_KIND_STRUCT: return "struct"; 1978 case BTF_KIND_UNION: return "union"; 1979 case BTF_KIND_ENUM: return "enum"; 1980 case BTF_KIND_FWD: return "fwd"; 1981 case BTF_KIND_TYPEDEF: return "typedef"; 1982 case BTF_KIND_VOLATILE: return "volatile"; 1983 case BTF_KIND_CONST: return "const"; 1984 case BTF_KIND_RESTRICT: return "restrict"; 1985 case BTF_KIND_FUNC: return "func"; 1986 case BTF_KIND_FUNC_PROTO: return "func_proto"; 1987 case BTF_KIND_VAR: return "var"; 1988 case BTF_KIND_DATASEC: return "datasec"; 1989 case BTF_KIND_FLOAT: return "float"; 1990 default: return "unknown"; 1991 } 1992 } 1993 1994 const char *btf_kind_str(const struct btf_type *t) 1995 { 1996 return __btf_kind_str(btf_kind(t)); 1997 } 1998 1999 /* 2000 * Fetch integer attribute of BTF map definition. Such attributes are 2001 * represented using a pointer to an array, in which dimensionality of array 2002 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2003 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2004 * type definition, while using only sizeof(void *) space in ELF data section. 2005 */ 2006 static bool get_map_field_int(const char *map_name, const struct btf *btf, 2007 const struct btf_member *m, __u32 *res) 2008 { 2009 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2010 const char *name = btf__name_by_offset(btf, m->name_off); 2011 const struct btf_array *arr_info; 2012 const struct btf_type *arr_t; 2013 2014 if (!btf_is_ptr(t)) { 2015 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2016 map_name, name, btf_kind_str(t)); 2017 return false; 2018 } 2019 2020 arr_t = btf__type_by_id(btf, t->type); 2021 if (!arr_t) { 2022 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2023 map_name, name, t->type); 2024 return false; 2025 } 2026 if (!btf_is_array(arr_t)) { 2027 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2028 map_name, name, btf_kind_str(arr_t)); 2029 return false; 2030 } 2031 arr_info = btf_array(arr_t); 2032 *res = arr_info->nelems; 2033 return true; 2034 } 2035 2036 static int build_map_pin_path(struct bpf_map *map, const char *path) 2037 { 2038 char buf[PATH_MAX]; 2039 int len; 2040 2041 if (!path) 2042 path = "/sys/fs/bpf"; 2043 2044 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); 2045 if (len < 0) 2046 return -EINVAL; 2047 else if (len >= PATH_MAX) 2048 return -ENAMETOOLONG; 2049 2050 return bpf_map__set_pin_path(map, buf); 2051 } 2052 2053 int parse_btf_map_def(const char *map_name, struct btf *btf, 2054 const struct btf_type *def_t, bool strict, 2055 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2056 { 2057 const struct btf_type *t; 2058 const struct btf_member *m; 2059 bool is_inner = inner_def == NULL; 2060 int vlen, i; 2061 2062 vlen = btf_vlen(def_t); 2063 m = btf_members(def_t); 2064 for (i = 0; i < vlen; i++, m++) { 2065 const char *name = btf__name_by_offset(btf, m->name_off); 2066 2067 if (!name) { 2068 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2069 return -EINVAL; 2070 } 2071 if (strcmp(name, "type") == 0) { 2072 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2073 return -EINVAL; 2074 map_def->parts |= MAP_DEF_MAP_TYPE; 2075 } else if (strcmp(name, "max_entries") == 0) { 2076 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2077 return -EINVAL; 2078 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2079 } else if (strcmp(name, "map_flags") == 0) { 2080 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2081 return -EINVAL; 2082 map_def->parts |= MAP_DEF_MAP_FLAGS; 2083 } else if (strcmp(name, "numa_node") == 0) { 2084 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2085 return -EINVAL; 2086 map_def->parts |= MAP_DEF_NUMA_NODE; 2087 } else if (strcmp(name, "key_size") == 0) { 2088 __u32 sz; 2089 2090 if (!get_map_field_int(map_name, btf, m, &sz)) 2091 return -EINVAL; 2092 if (map_def->key_size && map_def->key_size != sz) { 2093 pr_warn("map '%s': conflicting key size %u != %u.\n", 2094 map_name, map_def->key_size, sz); 2095 return -EINVAL; 2096 } 2097 map_def->key_size = sz; 2098 map_def->parts |= MAP_DEF_KEY_SIZE; 2099 } else if (strcmp(name, "key") == 0) { 2100 __s64 sz; 2101 2102 t = btf__type_by_id(btf, m->type); 2103 if (!t) { 2104 pr_warn("map '%s': key type [%d] not found.\n", 2105 map_name, m->type); 2106 return -EINVAL; 2107 } 2108 if (!btf_is_ptr(t)) { 2109 pr_warn("map '%s': key spec is not PTR: %s.\n", 2110 map_name, btf_kind_str(t)); 2111 return -EINVAL; 2112 } 2113 sz = btf__resolve_size(btf, t->type); 2114 if (sz < 0) { 2115 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2116 map_name, t->type, (ssize_t)sz); 2117 return sz; 2118 } 2119 if (map_def->key_size && map_def->key_size != sz) { 2120 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2121 map_name, map_def->key_size, (ssize_t)sz); 2122 return -EINVAL; 2123 } 2124 map_def->key_size = sz; 2125 map_def->key_type_id = t->type; 2126 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2127 } else if (strcmp(name, "value_size") == 0) { 2128 __u32 sz; 2129 2130 if (!get_map_field_int(map_name, btf, m, &sz)) 2131 return -EINVAL; 2132 if (map_def->value_size && map_def->value_size != sz) { 2133 pr_warn("map '%s': conflicting value size %u != %u.\n", 2134 map_name, map_def->value_size, sz); 2135 return -EINVAL; 2136 } 2137 map_def->value_size = sz; 2138 map_def->parts |= MAP_DEF_VALUE_SIZE; 2139 } else if (strcmp(name, "value") == 0) { 2140 __s64 sz; 2141 2142 t = btf__type_by_id(btf, m->type); 2143 if (!t) { 2144 pr_warn("map '%s': value type [%d] not found.\n", 2145 map_name, m->type); 2146 return -EINVAL; 2147 } 2148 if (!btf_is_ptr(t)) { 2149 pr_warn("map '%s': value spec is not PTR: %s.\n", 2150 map_name, btf_kind_str(t)); 2151 return -EINVAL; 2152 } 2153 sz = btf__resolve_size(btf, t->type); 2154 if (sz < 0) { 2155 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2156 map_name, t->type, (ssize_t)sz); 2157 return sz; 2158 } 2159 if (map_def->value_size && map_def->value_size != sz) { 2160 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2161 map_name, map_def->value_size, (ssize_t)sz); 2162 return -EINVAL; 2163 } 2164 map_def->value_size = sz; 2165 map_def->value_type_id = t->type; 2166 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2167 } 2168 else if (strcmp(name, "values") == 0) { 2169 char inner_map_name[128]; 2170 int err; 2171 2172 if (is_inner) { 2173 pr_warn("map '%s': multi-level inner maps not supported.\n", 2174 map_name); 2175 return -ENOTSUP; 2176 } 2177 if (i != vlen - 1) { 2178 pr_warn("map '%s': '%s' member should be last.\n", 2179 map_name, name); 2180 return -EINVAL; 2181 } 2182 if (!bpf_map_type__is_map_in_map(map_def->map_type)) { 2183 pr_warn("map '%s': should be map-in-map.\n", 2184 map_name); 2185 return -ENOTSUP; 2186 } 2187 if (map_def->value_size && map_def->value_size != 4) { 2188 pr_warn("map '%s': conflicting value size %u != 4.\n", 2189 map_name, map_def->value_size); 2190 return -EINVAL; 2191 } 2192 map_def->value_size = 4; 2193 t = btf__type_by_id(btf, m->type); 2194 if (!t) { 2195 pr_warn("map '%s': map-in-map inner type [%d] not found.\n", 2196 map_name, m->type); 2197 return -EINVAL; 2198 } 2199 if (!btf_is_array(t) || btf_array(t)->nelems) { 2200 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n", 2201 map_name); 2202 return -EINVAL; 2203 } 2204 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2205 if (!btf_is_ptr(t)) { 2206 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2207 map_name, btf_kind_str(t)); 2208 return -EINVAL; 2209 } 2210 t = skip_mods_and_typedefs(btf, t->type, NULL); 2211 if (!btf_is_struct(t)) { 2212 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2213 map_name, btf_kind_str(t)); 2214 return -EINVAL; 2215 } 2216 2217 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2218 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2219 if (err) 2220 return err; 2221 2222 map_def->parts |= MAP_DEF_INNER_MAP; 2223 } else if (strcmp(name, "pinning") == 0) { 2224 __u32 val; 2225 2226 if (is_inner) { 2227 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2228 return -EINVAL; 2229 } 2230 if (!get_map_field_int(map_name, btf, m, &val)) 2231 return -EINVAL; 2232 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2233 pr_warn("map '%s': invalid pinning value %u.\n", 2234 map_name, val); 2235 return -EINVAL; 2236 } 2237 map_def->pinning = val; 2238 map_def->parts |= MAP_DEF_PINNING; 2239 } else { 2240 if (strict) { 2241 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2242 return -ENOTSUP; 2243 } 2244 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2245 } 2246 } 2247 2248 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2249 pr_warn("map '%s': map type isn't specified.\n", map_name); 2250 return -EINVAL; 2251 } 2252 2253 return 0; 2254 } 2255 2256 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2257 { 2258 map->def.type = def->map_type; 2259 map->def.key_size = def->key_size; 2260 map->def.value_size = def->value_size; 2261 map->def.max_entries = def->max_entries; 2262 map->def.map_flags = def->map_flags; 2263 2264 map->numa_node = def->numa_node; 2265 map->btf_key_type_id = def->key_type_id; 2266 map->btf_value_type_id = def->value_type_id; 2267 2268 if (def->parts & MAP_DEF_MAP_TYPE) 2269 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2270 2271 if (def->parts & MAP_DEF_KEY_TYPE) 2272 pr_debug("map '%s': found key [%u], sz = %u.\n", 2273 map->name, def->key_type_id, def->key_size); 2274 else if (def->parts & MAP_DEF_KEY_SIZE) 2275 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2276 2277 if (def->parts & MAP_DEF_VALUE_TYPE) 2278 pr_debug("map '%s': found value [%u], sz = %u.\n", 2279 map->name, def->value_type_id, def->value_size); 2280 else if (def->parts & MAP_DEF_VALUE_SIZE) 2281 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2282 2283 if (def->parts & MAP_DEF_MAX_ENTRIES) 2284 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2285 if (def->parts & MAP_DEF_MAP_FLAGS) 2286 pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags); 2287 if (def->parts & MAP_DEF_PINNING) 2288 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2289 if (def->parts & MAP_DEF_NUMA_NODE) 2290 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2291 2292 if (def->parts & MAP_DEF_INNER_MAP) 2293 pr_debug("map '%s': found inner map definition.\n", map->name); 2294 } 2295 2296 static const char *btf_var_linkage_str(__u32 linkage) 2297 { 2298 switch (linkage) { 2299 case BTF_VAR_STATIC: return "static"; 2300 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2301 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2302 default: return "unknown"; 2303 } 2304 } 2305 2306 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2307 const struct btf_type *sec, 2308 int var_idx, int sec_idx, 2309 const Elf_Data *data, bool strict, 2310 const char *pin_root_path) 2311 { 2312 struct btf_map_def map_def = {}, inner_def = {}; 2313 const struct btf_type *var, *def; 2314 const struct btf_var_secinfo *vi; 2315 const struct btf_var *var_extra; 2316 const char *map_name; 2317 struct bpf_map *map; 2318 int err; 2319 2320 vi = btf_var_secinfos(sec) + var_idx; 2321 var = btf__type_by_id(obj->btf, vi->type); 2322 var_extra = btf_var(var); 2323 map_name = btf__name_by_offset(obj->btf, var->name_off); 2324 2325 if (map_name == NULL || map_name[0] == '\0') { 2326 pr_warn("map #%d: empty name.\n", var_idx); 2327 return -EINVAL; 2328 } 2329 if ((__u64)vi->offset + vi->size > data->d_size) { 2330 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2331 return -EINVAL; 2332 } 2333 if (!btf_is_var(var)) { 2334 pr_warn("map '%s': unexpected var kind %s.\n", 2335 map_name, btf_kind_str(var)); 2336 return -EINVAL; 2337 } 2338 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2339 pr_warn("map '%s': unsupported map linkage %s.\n", 2340 map_name, btf_var_linkage_str(var_extra->linkage)); 2341 return -EOPNOTSUPP; 2342 } 2343 2344 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2345 if (!btf_is_struct(def)) { 2346 pr_warn("map '%s': unexpected def kind %s.\n", 2347 map_name, btf_kind_str(var)); 2348 return -EINVAL; 2349 } 2350 if (def->size > vi->size) { 2351 pr_warn("map '%s': invalid def size.\n", map_name); 2352 return -EINVAL; 2353 } 2354 2355 map = bpf_object__add_map(obj); 2356 if (IS_ERR(map)) 2357 return PTR_ERR(map); 2358 map->name = strdup(map_name); 2359 if (!map->name) { 2360 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2361 return -ENOMEM; 2362 } 2363 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2364 map->def.type = BPF_MAP_TYPE_UNSPEC; 2365 map->sec_idx = sec_idx; 2366 map->sec_offset = vi->offset; 2367 map->btf_var_idx = var_idx; 2368 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2369 map_name, map->sec_idx, map->sec_offset); 2370 2371 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2372 if (err) 2373 return err; 2374 2375 fill_map_from_def(map, &map_def); 2376 2377 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2378 err = build_map_pin_path(map, pin_root_path); 2379 if (err) { 2380 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2381 return err; 2382 } 2383 } 2384 2385 if (map_def.parts & MAP_DEF_INNER_MAP) { 2386 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2387 if (!map->inner_map) 2388 return -ENOMEM; 2389 map->inner_map->fd = -1; 2390 map->inner_map->sec_idx = sec_idx; 2391 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2392 if (!map->inner_map->name) 2393 return -ENOMEM; 2394 sprintf(map->inner_map->name, "%s.inner", map_name); 2395 2396 fill_map_from_def(map->inner_map, &inner_def); 2397 } 2398 2399 return 0; 2400 } 2401 2402 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2403 const char *pin_root_path) 2404 { 2405 const struct btf_type *sec = NULL; 2406 int nr_types, i, vlen, err; 2407 const struct btf_type *t; 2408 const char *name; 2409 Elf_Data *data; 2410 Elf_Scn *scn; 2411 2412 if (obj->efile.btf_maps_shndx < 0) 2413 return 0; 2414 2415 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2416 data = elf_sec_data(obj, scn); 2417 if (!scn || !data) { 2418 pr_warn("elf: failed to get %s map definitions for %s\n", 2419 MAPS_ELF_SEC, obj->path); 2420 return -EINVAL; 2421 } 2422 2423 nr_types = btf__get_nr_types(obj->btf); 2424 for (i = 1; i <= nr_types; i++) { 2425 t = btf__type_by_id(obj->btf, i); 2426 if (!btf_is_datasec(t)) 2427 continue; 2428 name = btf__name_by_offset(obj->btf, t->name_off); 2429 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2430 sec = t; 2431 obj->efile.btf_maps_sec_btf_id = i; 2432 break; 2433 } 2434 } 2435 2436 if (!sec) { 2437 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2438 return -ENOENT; 2439 } 2440 2441 vlen = btf_vlen(sec); 2442 for (i = 0; i < vlen; i++) { 2443 err = bpf_object__init_user_btf_map(obj, sec, i, 2444 obj->efile.btf_maps_shndx, 2445 data, strict, 2446 pin_root_path); 2447 if (err) 2448 return err; 2449 } 2450 2451 return 0; 2452 } 2453 2454 static int bpf_object__init_maps(struct bpf_object *obj, 2455 const struct bpf_object_open_opts *opts) 2456 { 2457 const char *pin_root_path; 2458 bool strict; 2459 int err; 2460 2461 strict = !OPTS_GET(opts, relaxed_maps, false); 2462 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2463 2464 err = bpf_object__init_user_maps(obj, strict); 2465 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2466 err = err ?: bpf_object__init_global_data_maps(obj); 2467 err = err ?: bpf_object__init_kconfig_map(obj); 2468 err = err ?: bpf_object__init_struct_ops_maps(obj); 2469 2470 return err; 2471 } 2472 2473 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2474 { 2475 GElf_Shdr sh; 2476 2477 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh)) 2478 return false; 2479 2480 return sh.sh_flags & SHF_EXECINSTR; 2481 } 2482 2483 static bool btf_needs_sanitization(struct bpf_object *obj) 2484 { 2485 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2486 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2487 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2488 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2489 2490 return !has_func || !has_datasec || !has_func_global || !has_float; 2491 } 2492 2493 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2494 { 2495 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2496 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2497 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2498 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2499 struct btf_type *t; 2500 int i, j, vlen; 2501 2502 for (i = 1; i <= btf__get_nr_types(btf); i++) { 2503 t = (struct btf_type *)btf__type_by_id(btf, i); 2504 2505 if (!has_datasec && btf_is_var(t)) { 2506 /* replace VAR with INT */ 2507 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2508 /* 2509 * using size = 1 is the safest choice, 4 will be too 2510 * big and cause kernel BTF validation failure if 2511 * original variable took less than 4 bytes 2512 */ 2513 t->size = 1; 2514 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2515 } else if (!has_datasec && btf_is_datasec(t)) { 2516 /* replace DATASEC with STRUCT */ 2517 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2518 struct btf_member *m = btf_members(t); 2519 struct btf_type *vt; 2520 char *name; 2521 2522 name = (char *)btf__name_by_offset(btf, t->name_off); 2523 while (*name) { 2524 if (*name == '.') 2525 *name = '_'; 2526 name++; 2527 } 2528 2529 vlen = btf_vlen(t); 2530 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2531 for (j = 0; j < vlen; j++, v++, m++) { 2532 /* order of field assignments is important */ 2533 m->offset = v->offset * 8; 2534 m->type = v->type; 2535 /* preserve variable name as member name */ 2536 vt = (void *)btf__type_by_id(btf, v->type); 2537 m->name_off = vt->name_off; 2538 } 2539 } else if (!has_func && btf_is_func_proto(t)) { 2540 /* replace FUNC_PROTO with ENUM */ 2541 vlen = btf_vlen(t); 2542 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2543 t->size = sizeof(__u32); /* kernel enforced */ 2544 } else if (!has_func && btf_is_func(t)) { 2545 /* replace FUNC with TYPEDEF */ 2546 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2547 } else if (!has_func_global && btf_is_func(t)) { 2548 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2549 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2550 } else if (!has_float && btf_is_float(t)) { 2551 /* replace FLOAT with an equally-sized empty STRUCT; 2552 * since C compilers do not accept e.g. "float" as a 2553 * valid struct name, make it anonymous 2554 */ 2555 t->name_off = 0; 2556 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2557 } 2558 } 2559 } 2560 2561 static bool libbpf_needs_btf(const struct bpf_object *obj) 2562 { 2563 return obj->efile.btf_maps_shndx >= 0 || 2564 obj->efile.st_ops_shndx >= 0 || 2565 obj->nr_extern > 0; 2566 } 2567 2568 static bool kernel_needs_btf(const struct bpf_object *obj) 2569 { 2570 return obj->efile.st_ops_shndx >= 0; 2571 } 2572 2573 static int bpf_object__init_btf(struct bpf_object *obj, 2574 Elf_Data *btf_data, 2575 Elf_Data *btf_ext_data) 2576 { 2577 int err = -ENOENT; 2578 2579 if (btf_data) { 2580 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2581 err = libbpf_get_error(obj->btf); 2582 if (err) { 2583 obj->btf = NULL; 2584 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); 2585 goto out; 2586 } 2587 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2588 btf__set_pointer_size(obj->btf, 8); 2589 } 2590 if (btf_ext_data) { 2591 if (!obj->btf) { 2592 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2593 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2594 goto out; 2595 } 2596 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 2597 err = libbpf_get_error(obj->btf_ext); 2598 if (err) { 2599 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", 2600 BTF_EXT_ELF_SEC, err); 2601 obj->btf_ext = NULL; 2602 goto out; 2603 } 2604 } 2605 out: 2606 if (err && libbpf_needs_btf(obj)) { 2607 pr_warn("BTF is required, but is missing or corrupted.\n"); 2608 return err; 2609 } 2610 return 0; 2611 } 2612 2613 static int bpf_object__finalize_btf(struct bpf_object *obj) 2614 { 2615 int err; 2616 2617 if (!obj->btf) 2618 return 0; 2619 2620 err = btf__finalize_data(obj, obj->btf); 2621 if (err) { 2622 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); 2623 return err; 2624 } 2625 2626 return 0; 2627 } 2628 2629 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 2630 { 2631 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 2632 prog->type == BPF_PROG_TYPE_LSM) 2633 return true; 2634 2635 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 2636 * also need vmlinux BTF 2637 */ 2638 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 2639 return true; 2640 2641 return false; 2642 } 2643 2644 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 2645 { 2646 struct bpf_program *prog; 2647 int i; 2648 2649 /* CO-RE relocations need kernel BTF, only when btf_custom_path 2650 * is not specified 2651 */ 2652 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 2653 return true; 2654 2655 /* Support for typed ksyms needs kernel BTF */ 2656 for (i = 0; i < obj->nr_extern; i++) { 2657 const struct extern_desc *ext; 2658 2659 ext = &obj->externs[i]; 2660 if (ext->type == EXT_KSYM && ext->ksym.type_id) 2661 return true; 2662 } 2663 2664 bpf_object__for_each_program(prog, obj) { 2665 if (!prog->load) 2666 continue; 2667 if (prog_needs_vmlinux_btf(prog)) 2668 return true; 2669 } 2670 2671 return false; 2672 } 2673 2674 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 2675 { 2676 int err; 2677 2678 /* btf_vmlinux could be loaded earlier */ 2679 if (obj->btf_vmlinux || obj->gen_loader) 2680 return 0; 2681 2682 if (!force && !obj_needs_vmlinux_btf(obj)) 2683 return 0; 2684 2685 obj->btf_vmlinux = btf__load_vmlinux_btf(); 2686 err = libbpf_get_error(obj->btf_vmlinux); 2687 if (err) { 2688 pr_warn("Error loading vmlinux BTF: %d\n", err); 2689 obj->btf_vmlinux = NULL; 2690 return err; 2691 } 2692 return 0; 2693 } 2694 2695 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 2696 { 2697 struct btf *kern_btf = obj->btf; 2698 bool btf_mandatory, sanitize; 2699 int i, err = 0; 2700 2701 if (!obj->btf) 2702 return 0; 2703 2704 if (!kernel_supports(obj, FEAT_BTF)) { 2705 if (kernel_needs_btf(obj)) { 2706 err = -EOPNOTSUPP; 2707 goto report; 2708 } 2709 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 2710 return 0; 2711 } 2712 2713 /* Even though some subprogs are global/weak, user might prefer more 2714 * permissive BPF verification process that BPF verifier performs for 2715 * static functions, taking into account more context from the caller 2716 * functions. In such case, they need to mark such subprogs with 2717 * __attribute__((visibility("hidden"))) and libbpf will adjust 2718 * corresponding FUNC BTF type to be marked as static and trigger more 2719 * involved BPF verification process. 2720 */ 2721 for (i = 0; i < obj->nr_programs; i++) { 2722 struct bpf_program *prog = &obj->programs[i]; 2723 struct btf_type *t; 2724 const char *name; 2725 int j, n; 2726 2727 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 2728 continue; 2729 2730 n = btf__get_nr_types(obj->btf); 2731 for (j = 1; j <= n; j++) { 2732 t = btf_type_by_id(obj->btf, j); 2733 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 2734 continue; 2735 2736 name = btf__str_by_offset(obj->btf, t->name_off); 2737 if (strcmp(name, prog->name) != 0) 2738 continue; 2739 2740 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 2741 break; 2742 } 2743 } 2744 2745 sanitize = btf_needs_sanitization(obj); 2746 if (sanitize) { 2747 const void *raw_data; 2748 __u32 sz; 2749 2750 /* clone BTF to sanitize a copy and leave the original intact */ 2751 raw_data = btf__get_raw_data(obj->btf, &sz); 2752 kern_btf = btf__new(raw_data, sz); 2753 err = libbpf_get_error(kern_btf); 2754 if (err) 2755 return err; 2756 2757 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2758 btf__set_pointer_size(obj->btf, 8); 2759 bpf_object__sanitize_btf(obj, kern_btf); 2760 } 2761 2762 if (obj->gen_loader) { 2763 __u32 raw_size = 0; 2764 const void *raw_data = btf__get_raw_data(kern_btf, &raw_size); 2765 2766 if (!raw_data) 2767 return -ENOMEM; 2768 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 2769 /* Pretend to have valid FD to pass various fd >= 0 checks. 2770 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 2771 */ 2772 btf__set_fd(kern_btf, 0); 2773 } else { 2774 err = btf__load_into_kernel(kern_btf); 2775 } 2776 if (sanitize) { 2777 if (!err) { 2778 /* move fd to libbpf's BTF */ 2779 btf__set_fd(obj->btf, btf__fd(kern_btf)); 2780 btf__set_fd(kern_btf, -1); 2781 } 2782 btf__free(kern_btf); 2783 } 2784 report: 2785 if (err) { 2786 btf_mandatory = kernel_needs_btf(obj); 2787 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 2788 btf_mandatory ? "BTF is mandatory, can't proceed." 2789 : "BTF is optional, ignoring."); 2790 if (!btf_mandatory) 2791 err = 0; 2792 } 2793 return err; 2794 } 2795 2796 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 2797 { 2798 const char *name; 2799 2800 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 2801 if (!name) { 2802 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2803 off, obj->path, elf_errmsg(-1)); 2804 return NULL; 2805 } 2806 2807 return name; 2808 } 2809 2810 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 2811 { 2812 const char *name; 2813 2814 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 2815 if (!name) { 2816 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 2817 off, obj->path, elf_errmsg(-1)); 2818 return NULL; 2819 } 2820 2821 return name; 2822 } 2823 2824 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 2825 { 2826 Elf_Scn *scn; 2827 2828 scn = elf_getscn(obj->efile.elf, idx); 2829 if (!scn) { 2830 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 2831 idx, obj->path, elf_errmsg(-1)); 2832 return NULL; 2833 } 2834 return scn; 2835 } 2836 2837 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 2838 { 2839 Elf_Scn *scn = NULL; 2840 Elf *elf = obj->efile.elf; 2841 const char *sec_name; 2842 2843 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2844 sec_name = elf_sec_name(obj, scn); 2845 if (!sec_name) 2846 return NULL; 2847 2848 if (strcmp(sec_name, name) != 0) 2849 continue; 2850 2851 return scn; 2852 } 2853 return NULL; 2854 } 2855 2856 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr) 2857 { 2858 if (!scn) 2859 return -EINVAL; 2860 2861 if (gelf_getshdr(scn, hdr) != hdr) { 2862 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 2863 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2864 return -EINVAL; 2865 } 2866 2867 return 0; 2868 } 2869 2870 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 2871 { 2872 const char *name; 2873 GElf_Shdr sh; 2874 2875 if (!scn) 2876 return NULL; 2877 2878 if (elf_sec_hdr(obj, scn, &sh)) 2879 return NULL; 2880 2881 name = elf_sec_str(obj, sh.sh_name); 2882 if (!name) { 2883 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 2884 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 2885 return NULL; 2886 } 2887 2888 return name; 2889 } 2890 2891 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 2892 { 2893 Elf_Data *data; 2894 2895 if (!scn) 2896 return NULL; 2897 2898 data = elf_getdata(scn, 0); 2899 if (!data) { 2900 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 2901 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 2902 obj->path, elf_errmsg(-1)); 2903 return NULL; 2904 } 2905 2906 return data; 2907 } 2908 2909 static bool is_sec_name_dwarf(const char *name) 2910 { 2911 /* approximation, but the actual list is too long */ 2912 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 2913 } 2914 2915 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name) 2916 { 2917 /* no special handling of .strtab */ 2918 if (hdr->sh_type == SHT_STRTAB) 2919 return true; 2920 2921 /* ignore .llvm_addrsig section as well */ 2922 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 2923 return true; 2924 2925 /* no subprograms will lead to an empty .text section, ignore it */ 2926 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 2927 strcmp(name, ".text") == 0) 2928 return true; 2929 2930 /* DWARF sections */ 2931 if (is_sec_name_dwarf(name)) 2932 return true; 2933 2934 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 2935 name += sizeof(".rel") - 1; 2936 /* DWARF section relocations */ 2937 if (is_sec_name_dwarf(name)) 2938 return true; 2939 2940 /* .BTF and .BTF.ext don't need relocations */ 2941 if (strcmp(name, BTF_ELF_SEC) == 0 || 2942 strcmp(name, BTF_EXT_ELF_SEC) == 0) 2943 return true; 2944 } 2945 2946 return false; 2947 } 2948 2949 static int cmp_progs(const void *_a, const void *_b) 2950 { 2951 const struct bpf_program *a = _a; 2952 const struct bpf_program *b = _b; 2953 2954 if (a->sec_idx != b->sec_idx) 2955 return a->sec_idx < b->sec_idx ? -1 : 1; 2956 2957 /* sec_insn_off can't be the same within the section */ 2958 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 2959 } 2960 2961 static int bpf_object__elf_collect(struct bpf_object *obj) 2962 { 2963 Elf *elf = obj->efile.elf; 2964 Elf_Data *btf_ext_data = NULL; 2965 Elf_Data *btf_data = NULL; 2966 int idx = 0, err = 0; 2967 const char *name; 2968 Elf_Data *data; 2969 Elf_Scn *scn; 2970 GElf_Shdr sh; 2971 2972 /* a bunch of ELF parsing functionality depends on processing symbols, 2973 * so do the first pass and find the symbol table 2974 */ 2975 scn = NULL; 2976 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2977 if (elf_sec_hdr(obj, scn, &sh)) 2978 return -LIBBPF_ERRNO__FORMAT; 2979 2980 if (sh.sh_type == SHT_SYMTAB) { 2981 if (obj->efile.symbols) { 2982 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 2983 return -LIBBPF_ERRNO__FORMAT; 2984 } 2985 2986 data = elf_sec_data(obj, scn); 2987 if (!data) 2988 return -LIBBPF_ERRNO__FORMAT; 2989 2990 obj->efile.symbols = data; 2991 obj->efile.symbols_shndx = elf_ndxscn(scn); 2992 obj->efile.strtabidx = sh.sh_link; 2993 } 2994 } 2995 2996 scn = NULL; 2997 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2998 idx++; 2999 3000 if (elf_sec_hdr(obj, scn, &sh)) 3001 return -LIBBPF_ERRNO__FORMAT; 3002 3003 name = elf_sec_str(obj, sh.sh_name); 3004 if (!name) 3005 return -LIBBPF_ERRNO__FORMAT; 3006 3007 if (ignore_elf_section(&sh, name)) 3008 continue; 3009 3010 data = elf_sec_data(obj, scn); 3011 if (!data) 3012 return -LIBBPF_ERRNO__FORMAT; 3013 3014 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 3015 idx, name, (unsigned long)data->d_size, 3016 (int)sh.sh_link, (unsigned long)sh.sh_flags, 3017 (int)sh.sh_type); 3018 3019 if (strcmp(name, "license") == 0) { 3020 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 3021 if (err) 3022 return err; 3023 } else if (strcmp(name, "version") == 0) { 3024 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 3025 if (err) 3026 return err; 3027 } else if (strcmp(name, "maps") == 0) { 3028 obj->efile.maps_shndx = idx; 3029 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 3030 obj->efile.btf_maps_shndx = idx; 3031 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 3032 btf_data = data; 3033 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 3034 btf_ext_data = data; 3035 } else if (sh.sh_type == SHT_SYMTAB) { 3036 /* already processed during the first pass above */ 3037 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 3038 if (sh.sh_flags & SHF_EXECINSTR) { 3039 if (strcmp(name, ".text") == 0) 3040 obj->efile.text_shndx = idx; 3041 err = bpf_object__add_programs(obj, data, name, idx); 3042 if (err) 3043 return err; 3044 } else if (strcmp(name, DATA_SEC) == 0) { 3045 obj->efile.data = data; 3046 obj->efile.data_shndx = idx; 3047 } else if (strcmp(name, RODATA_SEC) == 0) { 3048 obj->efile.rodata = data; 3049 obj->efile.rodata_shndx = idx; 3050 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 3051 obj->efile.st_ops_data = data; 3052 obj->efile.st_ops_shndx = idx; 3053 } else { 3054 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3055 idx, name); 3056 } 3057 } else if (sh.sh_type == SHT_REL) { 3058 int nr_sects = obj->efile.nr_reloc_sects; 3059 void *sects = obj->efile.reloc_sects; 3060 int sec = sh.sh_info; /* points to other section */ 3061 3062 /* Only do relo for section with exec instructions */ 3063 if (!section_have_execinstr(obj, sec) && 3064 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3065 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3066 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3067 idx, name, sec, 3068 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>"); 3069 continue; 3070 } 3071 3072 sects = libbpf_reallocarray(sects, nr_sects + 1, 3073 sizeof(*obj->efile.reloc_sects)); 3074 if (!sects) 3075 return -ENOMEM; 3076 3077 obj->efile.reloc_sects = sects; 3078 obj->efile.nr_reloc_sects++; 3079 3080 obj->efile.reloc_sects[nr_sects].shdr = sh; 3081 obj->efile.reloc_sects[nr_sects].data = data; 3082 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) { 3083 obj->efile.bss = data; 3084 obj->efile.bss_shndx = idx; 3085 } else { 3086 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3087 (size_t)sh.sh_size); 3088 } 3089 } 3090 3091 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3092 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3093 return -LIBBPF_ERRNO__FORMAT; 3094 } 3095 3096 /* sort BPF programs by section name and in-section instruction offset 3097 * for faster search */ 3098 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3099 3100 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3101 } 3102 3103 static bool sym_is_extern(const GElf_Sym *sym) 3104 { 3105 int bind = GELF_ST_BIND(sym->st_info); 3106 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3107 return sym->st_shndx == SHN_UNDEF && 3108 (bind == STB_GLOBAL || bind == STB_WEAK) && 3109 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 3110 } 3111 3112 static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) 3113 { 3114 int bind = GELF_ST_BIND(sym->st_info); 3115 int type = GELF_ST_TYPE(sym->st_info); 3116 3117 /* in .text section */ 3118 if (sym->st_shndx != text_shndx) 3119 return false; 3120 3121 /* local function */ 3122 if (bind == STB_LOCAL && type == STT_SECTION) 3123 return true; 3124 3125 /* global function */ 3126 return bind == STB_GLOBAL && type == STT_FUNC; 3127 } 3128 3129 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3130 { 3131 const struct btf_type *t; 3132 const char *tname; 3133 int i, n; 3134 3135 if (!btf) 3136 return -ESRCH; 3137 3138 n = btf__get_nr_types(btf); 3139 for (i = 1; i <= n; i++) { 3140 t = btf__type_by_id(btf, i); 3141 3142 if (!btf_is_var(t) && !btf_is_func(t)) 3143 continue; 3144 3145 tname = btf__name_by_offset(btf, t->name_off); 3146 if (strcmp(tname, ext_name)) 3147 continue; 3148 3149 if (btf_is_var(t) && 3150 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3151 return -EINVAL; 3152 3153 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3154 return -EINVAL; 3155 3156 return i; 3157 } 3158 3159 return -ENOENT; 3160 } 3161 3162 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 3163 const struct btf_var_secinfo *vs; 3164 const struct btf_type *t; 3165 int i, j, n; 3166 3167 if (!btf) 3168 return -ESRCH; 3169 3170 n = btf__get_nr_types(btf); 3171 for (i = 1; i <= n; i++) { 3172 t = btf__type_by_id(btf, i); 3173 3174 if (!btf_is_datasec(t)) 3175 continue; 3176 3177 vs = btf_var_secinfos(t); 3178 for (j = 0; j < btf_vlen(t); j++, vs++) { 3179 if (vs->type == ext_btf_id) 3180 return i; 3181 } 3182 } 3183 3184 return -ENOENT; 3185 } 3186 3187 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 3188 bool *is_signed) 3189 { 3190 const struct btf_type *t; 3191 const char *name; 3192 3193 t = skip_mods_and_typedefs(btf, id, NULL); 3194 name = btf__name_by_offset(btf, t->name_off); 3195 3196 if (is_signed) 3197 *is_signed = false; 3198 switch (btf_kind(t)) { 3199 case BTF_KIND_INT: { 3200 int enc = btf_int_encoding(t); 3201 3202 if (enc & BTF_INT_BOOL) 3203 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 3204 if (is_signed) 3205 *is_signed = enc & BTF_INT_SIGNED; 3206 if (t->size == 1) 3207 return KCFG_CHAR; 3208 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 3209 return KCFG_UNKNOWN; 3210 return KCFG_INT; 3211 } 3212 case BTF_KIND_ENUM: 3213 if (t->size != 4) 3214 return KCFG_UNKNOWN; 3215 if (strcmp(name, "libbpf_tristate")) 3216 return KCFG_UNKNOWN; 3217 return KCFG_TRISTATE; 3218 case BTF_KIND_ARRAY: 3219 if (btf_array(t)->nelems == 0) 3220 return KCFG_UNKNOWN; 3221 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 3222 return KCFG_UNKNOWN; 3223 return KCFG_CHAR_ARR; 3224 default: 3225 return KCFG_UNKNOWN; 3226 } 3227 } 3228 3229 static int cmp_externs(const void *_a, const void *_b) 3230 { 3231 const struct extern_desc *a = _a; 3232 const struct extern_desc *b = _b; 3233 3234 if (a->type != b->type) 3235 return a->type < b->type ? -1 : 1; 3236 3237 if (a->type == EXT_KCFG) { 3238 /* descending order by alignment requirements */ 3239 if (a->kcfg.align != b->kcfg.align) 3240 return a->kcfg.align > b->kcfg.align ? -1 : 1; 3241 /* ascending order by size, within same alignment class */ 3242 if (a->kcfg.sz != b->kcfg.sz) 3243 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 3244 } 3245 3246 /* resolve ties by name */ 3247 return strcmp(a->name, b->name); 3248 } 3249 3250 static int find_int_btf_id(const struct btf *btf) 3251 { 3252 const struct btf_type *t; 3253 int i, n; 3254 3255 n = btf__get_nr_types(btf); 3256 for (i = 1; i <= n; i++) { 3257 t = btf__type_by_id(btf, i); 3258 3259 if (btf_is_int(t) && btf_int_bits(t) == 32) 3260 return i; 3261 } 3262 3263 return 0; 3264 } 3265 3266 static int add_dummy_ksym_var(struct btf *btf) 3267 { 3268 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 3269 const struct btf_var_secinfo *vs; 3270 const struct btf_type *sec; 3271 3272 if (!btf) 3273 return 0; 3274 3275 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 3276 BTF_KIND_DATASEC); 3277 if (sec_btf_id < 0) 3278 return 0; 3279 3280 sec = btf__type_by_id(btf, sec_btf_id); 3281 vs = btf_var_secinfos(sec); 3282 for (i = 0; i < btf_vlen(sec); i++, vs++) { 3283 const struct btf_type *vt; 3284 3285 vt = btf__type_by_id(btf, vs->type); 3286 if (btf_is_func(vt)) 3287 break; 3288 } 3289 3290 /* No func in ksyms sec. No need to add dummy var. */ 3291 if (i == btf_vlen(sec)) 3292 return 0; 3293 3294 int_btf_id = find_int_btf_id(btf); 3295 dummy_var_btf_id = btf__add_var(btf, 3296 "dummy_ksym", 3297 BTF_VAR_GLOBAL_ALLOCATED, 3298 int_btf_id); 3299 if (dummy_var_btf_id < 0) 3300 pr_warn("cannot create a dummy_ksym var\n"); 3301 3302 return dummy_var_btf_id; 3303 } 3304 3305 static int bpf_object__collect_externs(struct bpf_object *obj) 3306 { 3307 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 3308 const struct btf_type *t; 3309 struct extern_desc *ext; 3310 int i, n, off, dummy_var_btf_id; 3311 const char *ext_name, *sec_name; 3312 Elf_Scn *scn; 3313 GElf_Shdr sh; 3314 3315 if (!obj->efile.symbols) 3316 return 0; 3317 3318 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 3319 if (elf_sec_hdr(obj, scn, &sh)) 3320 return -LIBBPF_ERRNO__FORMAT; 3321 3322 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 3323 if (dummy_var_btf_id < 0) 3324 return dummy_var_btf_id; 3325 3326 n = sh.sh_size / sh.sh_entsize; 3327 pr_debug("looking for externs among %d symbols...\n", n); 3328 3329 for (i = 0; i < n; i++) { 3330 GElf_Sym sym; 3331 3332 if (!gelf_getsym(obj->efile.symbols, i, &sym)) 3333 return -LIBBPF_ERRNO__FORMAT; 3334 if (!sym_is_extern(&sym)) 3335 continue; 3336 ext_name = elf_sym_str(obj, sym.st_name); 3337 if (!ext_name || !ext_name[0]) 3338 continue; 3339 3340 ext = obj->externs; 3341 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 3342 if (!ext) 3343 return -ENOMEM; 3344 obj->externs = ext; 3345 ext = &ext[obj->nr_extern]; 3346 memset(ext, 0, sizeof(*ext)); 3347 obj->nr_extern++; 3348 3349 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 3350 if (ext->btf_id <= 0) { 3351 pr_warn("failed to find BTF for extern '%s': %d\n", 3352 ext_name, ext->btf_id); 3353 return ext->btf_id; 3354 } 3355 t = btf__type_by_id(obj->btf, ext->btf_id); 3356 ext->name = btf__name_by_offset(obj->btf, t->name_off); 3357 ext->sym_idx = i; 3358 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; 3359 3360 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 3361 if (ext->sec_btf_id <= 0) { 3362 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 3363 ext_name, ext->btf_id, ext->sec_btf_id); 3364 return ext->sec_btf_id; 3365 } 3366 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 3367 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 3368 3369 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 3370 if (btf_is_func(t)) { 3371 pr_warn("extern function %s is unsupported under %s section\n", 3372 ext->name, KCONFIG_SEC); 3373 return -ENOTSUP; 3374 } 3375 kcfg_sec = sec; 3376 ext->type = EXT_KCFG; 3377 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 3378 if (ext->kcfg.sz <= 0) { 3379 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 3380 ext_name, ext->kcfg.sz); 3381 return ext->kcfg.sz; 3382 } 3383 ext->kcfg.align = btf__align_of(obj->btf, t->type); 3384 if (ext->kcfg.align <= 0) { 3385 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 3386 ext_name, ext->kcfg.align); 3387 return -EINVAL; 3388 } 3389 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 3390 &ext->kcfg.is_signed); 3391 if (ext->kcfg.type == KCFG_UNKNOWN) { 3392 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); 3393 return -ENOTSUP; 3394 } 3395 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 3396 if (btf_is_func(t) && ext->is_weak) { 3397 pr_warn("extern weak function %s is unsupported\n", 3398 ext->name); 3399 return -ENOTSUP; 3400 } 3401 ksym_sec = sec; 3402 ext->type = EXT_KSYM; 3403 skip_mods_and_typedefs(obj->btf, t->type, 3404 &ext->ksym.type_id); 3405 } else { 3406 pr_warn("unrecognized extern section '%s'\n", sec_name); 3407 return -ENOTSUP; 3408 } 3409 } 3410 pr_debug("collected %d externs total\n", obj->nr_extern); 3411 3412 if (!obj->nr_extern) 3413 return 0; 3414 3415 /* sort externs by type, for kcfg ones also by (align, size, name) */ 3416 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 3417 3418 /* for .ksyms section, we need to turn all externs into allocated 3419 * variables in BTF to pass kernel verification; we do this by 3420 * pretending that each extern is a 8-byte variable 3421 */ 3422 if (ksym_sec) { 3423 /* find existing 4-byte integer type in BTF to use for fake 3424 * extern variables in DATASEC 3425 */ 3426 int int_btf_id = find_int_btf_id(obj->btf); 3427 /* For extern function, a dummy_var added earlier 3428 * will be used to replace the vs->type and 3429 * its name string will be used to refill 3430 * the missing param's name. 3431 */ 3432 const struct btf_type *dummy_var; 3433 3434 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 3435 for (i = 0; i < obj->nr_extern; i++) { 3436 ext = &obj->externs[i]; 3437 if (ext->type != EXT_KSYM) 3438 continue; 3439 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 3440 i, ext->sym_idx, ext->name); 3441 } 3442 3443 sec = ksym_sec; 3444 n = btf_vlen(sec); 3445 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 3446 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3447 struct btf_type *vt; 3448 3449 vt = (void *)btf__type_by_id(obj->btf, vs->type); 3450 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 3451 ext = find_extern_by_name(obj, ext_name); 3452 if (!ext) { 3453 pr_warn("failed to find extern definition for BTF %s '%s'\n", 3454 btf_kind_str(vt), ext_name); 3455 return -ESRCH; 3456 } 3457 if (btf_is_func(vt)) { 3458 const struct btf_type *func_proto; 3459 struct btf_param *param; 3460 int j; 3461 3462 func_proto = btf__type_by_id(obj->btf, 3463 vt->type); 3464 param = btf_params(func_proto); 3465 /* Reuse the dummy_var string if the 3466 * func proto does not have param name. 3467 */ 3468 for (j = 0; j < btf_vlen(func_proto); j++) 3469 if (param[j].type && !param[j].name_off) 3470 param[j].name_off = 3471 dummy_var->name_off; 3472 vs->type = dummy_var_btf_id; 3473 vt->info &= ~0xffff; 3474 vt->info |= BTF_FUNC_GLOBAL; 3475 } else { 3476 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3477 vt->type = int_btf_id; 3478 } 3479 vs->offset = off; 3480 vs->size = sizeof(int); 3481 } 3482 sec->size = off; 3483 } 3484 3485 if (kcfg_sec) { 3486 sec = kcfg_sec; 3487 /* for kcfg externs calculate their offsets within a .kconfig map */ 3488 off = 0; 3489 for (i = 0; i < obj->nr_extern; i++) { 3490 ext = &obj->externs[i]; 3491 if (ext->type != EXT_KCFG) 3492 continue; 3493 3494 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 3495 off = ext->kcfg.data_off + ext->kcfg.sz; 3496 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 3497 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 3498 } 3499 sec->size = off; 3500 n = btf_vlen(sec); 3501 for (i = 0; i < n; i++) { 3502 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3503 3504 t = btf__type_by_id(obj->btf, vs->type); 3505 ext_name = btf__name_by_offset(obj->btf, t->name_off); 3506 ext = find_extern_by_name(obj, ext_name); 3507 if (!ext) { 3508 pr_warn("failed to find extern definition for BTF var '%s'\n", 3509 ext_name); 3510 return -ESRCH; 3511 } 3512 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3513 vs->offset = ext->kcfg.data_off; 3514 } 3515 } 3516 return 0; 3517 } 3518 3519 struct bpf_program * 3520 bpf_object__find_program_by_title(const struct bpf_object *obj, 3521 const char *title) 3522 { 3523 struct bpf_program *pos; 3524 3525 bpf_object__for_each_program(pos, obj) { 3526 if (pos->sec_name && !strcmp(pos->sec_name, title)) 3527 return pos; 3528 } 3529 return errno = ENOENT, NULL; 3530 } 3531 3532 static bool prog_is_subprog(const struct bpf_object *obj, 3533 const struct bpf_program *prog) 3534 { 3535 /* For legacy reasons, libbpf supports an entry-point BPF programs 3536 * without SEC() attribute, i.e., those in the .text section. But if 3537 * there are 2 or more such programs in the .text section, they all 3538 * must be subprograms called from entry-point BPF programs in 3539 * designated SEC()'tions, otherwise there is no way to distinguish 3540 * which of those programs should be loaded vs which are a subprogram. 3541 * Similarly, if there is a function/program in .text and at least one 3542 * other BPF program with custom SEC() attribute, then we just assume 3543 * .text programs are subprograms (even if they are not called from 3544 * other programs), because libbpf never explicitly supported mixing 3545 * SEC()-designated BPF programs and .text entry-point BPF programs. 3546 */ 3547 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 3548 } 3549 3550 struct bpf_program * 3551 bpf_object__find_program_by_name(const struct bpf_object *obj, 3552 const char *name) 3553 { 3554 struct bpf_program *prog; 3555 3556 bpf_object__for_each_program(prog, obj) { 3557 if (prog_is_subprog(obj, prog)) 3558 continue; 3559 if (!strcmp(prog->name, name)) 3560 return prog; 3561 } 3562 return errno = ENOENT, NULL; 3563 } 3564 3565 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 3566 int shndx) 3567 { 3568 return shndx == obj->efile.data_shndx || 3569 shndx == obj->efile.bss_shndx || 3570 shndx == obj->efile.rodata_shndx; 3571 } 3572 3573 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 3574 int shndx) 3575 { 3576 return shndx == obj->efile.maps_shndx || 3577 shndx == obj->efile.btf_maps_shndx; 3578 } 3579 3580 static enum libbpf_map_type 3581 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 3582 { 3583 if (shndx == obj->efile.data_shndx) 3584 return LIBBPF_MAP_DATA; 3585 else if (shndx == obj->efile.bss_shndx) 3586 return LIBBPF_MAP_BSS; 3587 else if (shndx == obj->efile.rodata_shndx) 3588 return LIBBPF_MAP_RODATA; 3589 else if (shndx == obj->efile.symbols_shndx) 3590 return LIBBPF_MAP_KCONFIG; 3591 else 3592 return LIBBPF_MAP_UNSPEC; 3593 } 3594 3595 static int bpf_program__record_reloc(struct bpf_program *prog, 3596 struct reloc_desc *reloc_desc, 3597 __u32 insn_idx, const char *sym_name, 3598 const GElf_Sym *sym, const GElf_Rel *rel) 3599 { 3600 struct bpf_insn *insn = &prog->insns[insn_idx]; 3601 size_t map_idx, nr_maps = prog->obj->nr_maps; 3602 struct bpf_object *obj = prog->obj; 3603 __u32 shdr_idx = sym->st_shndx; 3604 enum libbpf_map_type type; 3605 const char *sym_sec_name; 3606 struct bpf_map *map; 3607 3608 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 3609 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 3610 prog->name, sym_name, insn_idx, insn->code); 3611 return -LIBBPF_ERRNO__RELOC; 3612 } 3613 3614 if (sym_is_extern(sym)) { 3615 int sym_idx = GELF_R_SYM(rel->r_info); 3616 int i, n = obj->nr_extern; 3617 struct extern_desc *ext; 3618 3619 for (i = 0; i < n; i++) { 3620 ext = &obj->externs[i]; 3621 if (ext->sym_idx == sym_idx) 3622 break; 3623 } 3624 if (i >= n) { 3625 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 3626 prog->name, sym_name, sym_idx); 3627 return -LIBBPF_ERRNO__RELOC; 3628 } 3629 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 3630 prog->name, i, ext->name, ext->sym_idx, insn_idx); 3631 if (insn->code == (BPF_JMP | BPF_CALL)) 3632 reloc_desc->type = RELO_EXTERN_FUNC; 3633 else 3634 reloc_desc->type = RELO_EXTERN_VAR; 3635 reloc_desc->insn_idx = insn_idx; 3636 reloc_desc->sym_off = i; /* sym_off stores extern index */ 3637 return 0; 3638 } 3639 3640 /* sub-program call relocation */ 3641 if (is_call_insn(insn)) { 3642 if (insn->src_reg != BPF_PSEUDO_CALL) { 3643 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 3644 return -LIBBPF_ERRNO__RELOC; 3645 } 3646 /* text_shndx can be 0, if no default "main" program exists */ 3647 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 3648 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3649 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 3650 prog->name, sym_name, sym_sec_name); 3651 return -LIBBPF_ERRNO__RELOC; 3652 } 3653 if (sym->st_value % BPF_INSN_SZ) { 3654 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 3655 prog->name, sym_name, (size_t)sym->st_value); 3656 return -LIBBPF_ERRNO__RELOC; 3657 } 3658 reloc_desc->type = RELO_CALL; 3659 reloc_desc->insn_idx = insn_idx; 3660 reloc_desc->sym_off = sym->st_value; 3661 return 0; 3662 } 3663 3664 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 3665 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 3666 prog->name, sym_name, shdr_idx); 3667 return -LIBBPF_ERRNO__RELOC; 3668 } 3669 3670 /* loading subprog addresses */ 3671 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 3672 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 3673 * local_func: sym->st_value = 0, insn->imm = offset in the section. 3674 */ 3675 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 3676 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 3677 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 3678 return -LIBBPF_ERRNO__RELOC; 3679 } 3680 3681 reloc_desc->type = RELO_SUBPROG_ADDR; 3682 reloc_desc->insn_idx = insn_idx; 3683 reloc_desc->sym_off = sym->st_value; 3684 return 0; 3685 } 3686 3687 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 3688 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 3689 3690 /* generic map reference relocation */ 3691 if (type == LIBBPF_MAP_UNSPEC) { 3692 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 3693 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 3694 prog->name, sym_name, sym_sec_name); 3695 return -LIBBPF_ERRNO__RELOC; 3696 } 3697 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3698 map = &obj->maps[map_idx]; 3699 if (map->libbpf_type != type || 3700 map->sec_idx != sym->st_shndx || 3701 map->sec_offset != sym->st_value) 3702 continue; 3703 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 3704 prog->name, map_idx, map->name, map->sec_idx, 3705 map->sec_offset, insn_idx); 3706 break; 3707 } 3708 if (map_idx >= nr_maps) { 3709 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 3710 prog->name, sym_sec_name, (size_t)sym->st_value); 3711 return -LIBBPF_ERRNO__RELOC; 3712 } 3713 reloc_desc->type = RELO_LD64; 3714 reloc_desc->insn_idx = insn_idx; 3715 reloc_desc->map_idx = map_idx; 3716 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 3717 return 0; 3718 } 3719 3720 /* global data map relocation */ 3721 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 3722 pr_warn("prog '%s': bad data relo against section '%s'\n", 3723 prog->name, sym_sec_name); 3724 return -LIBBPF_ERRNO__RELOC; 3725 } 3726 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3727 map = &obj->maps[map_idx]; 3728 if (map->libbpf_type != type) 3729 continue; 3730 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 3731 prog->name, map_idx, map->name, map->sec_idx, 3732 map->sec_offset, insn_idx); 3733 break; 3734 } 3735 if (map_idx >= nr_maps) { 3736 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 3737 prog->name, sym_sec_name); 3738 return -LIBBPF_ERRNO__RELOC; 3739 } 3740 3741 reloc_desc->type = RELO_DATA; 3742 reloc_desc->insn_idx = insn_idx; 3743 reloc_desc->map_idx = map_idx; 3744 reloc_desc->sym_off = sym->st_value; 3745 return 0; 3746 } 3747 3748 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 3749 { 3750 return insn_idx >= prog->sec_insn_off && 3751 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 3752 } 3753 3754 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 3755 size_t sec_idx, size_t insn_idx) 3756 { 3757 int l = 0, r = obj->nr_programs - 1, m; 3758 struct bpf_program *prog; 3759 3760 while (l < r) { 3761 m = l + (r - l + 1) / 2; 3762 prog = &obj->programs[m]; 3763 3764 if (prog->sec_idx < sec_idx || 3765 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 3766 l = m; 3767 else 3768 r = m - 1; 3769 } 3770 /* matching program could be at index l, but it still might be the 3771 * wrong one, so we need to double check conditions for the last time 3772 */ 3773 prog = &obj->programs[l]; 3774 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 3775 return prog; 3776 return NULL; 3777 } 3778 3779 static int 3780 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data) 3781 { 3782 Elf_Data *symbols = obj->efile.symbols; 3783 const char *relo_sec_name, *sec_name; 3784 size_t sec_idx = shdr->sh_info; 3785 struct bpf_program *prog; 3786 struct reloc_desc *relos; 3787 int err, i, nrels; 3788 const char *sym_name; 3789 __u32 insn_idx; 3790 Elf_Scn *scn; 3791 Elf_Data *scn_data; 3792 GElf_Sym sym; 3793 GElf_Rel rel; 3794 3795 scn = elf_sec_by_idx(obj, sec_idx); 3796 scn_data = elf_sec_data(obj, scn); 3797 3798 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 3799 sec_name = elf_sec_name(obj, scn); 3800 if (!relo_sec_name || !sec_name) 3801 return -EINVAL; 3802 3803 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 3804 relo_sec_name, sec_idx, sec_name); 3805 nrels = shdr->sh_size / shdr->sh_entsize; 3806 3807 for (i = 0; i < nrels; i++) { 3808 if (!gelf_getrel(data, i, &rel)) { 3809 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 3810 return -LIBBPF_ERRNO__FORMAT; 3811 } 3812 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 3813 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n", 3814 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3815 return -LIBBPF_ERRNO__FORMAT; 3816 } 3817 3818 if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) { 3819 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 3820 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i); 3821 return -LIBBPF_ERRNO__FORMAT; 3822 } 3823 3824 insn_idx = rel.r_offset / BPF_INSN_SZ; 3825 /* relocations against static functions are recorded as 3826 * relocations against the section that contains a function; 3827 * in such case, symbol will be STT_SECTION and sym.st_name 3828 * will point to empty string (0), so fetch section name 3829 * instead 3830 */ 3831 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0) 3832 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx)); 3833 else 3834 sym_name = elf_sym_str(obj, sym.st_name); 3835 sym_name = sym_name ?: "<?"; 3836 3837 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 3838 relo_sec_name, i, insn_idx, sym_name); 3839 3840 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 3841 if (!prog) { 3842 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 3843 relo_sec_name, i, sec_name, insn_idx); 3844 continue; 3845 } 3846 3847 relos = libbpf_reallocarray(prog->reloc_desc, 3848 prog->nr_reloc + 1, sizeof(*relos)); 3849 if (!relos) 3850 return -ENOMEM; 3851 prog->reloc_desc = relos; 3852 3853 /* adjust insn_idx to local BPF program frame of reference */ 3854 insn_idx -= prog->sec_insn_off; 3855 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 3856 insn_idx, sym_name, &sym, &rel); 3857 if (err) 3858 return err; 3859 3860 prog->nr_reloc++; 3861 } 3862 return 0; 3863 } 3864 3865 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 3866 { 3867 struct bpf_map_def *def = &map->def; 3868 __u32 key_type_id = 0, value_type_id = 0; 3869 int ret; 3870 3871 /* if it's BTF-defined map, we don't need to search for type IDs. 3872 * For struct_ops map, it does not need btf_key_type_id and 3873 * btf_value_type_id. 3874 */ 3875 if (map->sec_idx == obj->efile.btf_maps_shndx || 3876 bpf_map__is_struct_ops(map)) 3877 return 0; 3878 3879 if (!bpf_map__is_internal(map)) { 3880 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 3881 def->value_size, &key_type_id, 3882 &value_type_id); 3883 } else { 3884 /* 3885 * LLVM annotates global data differently in BTF, that is, 3886 * only as '.data', '.bss' or '.rodata'. 3887 */ 3888 ret = btf__find_by_name(obj->btf, 3889 libbpf_type_to_btf_name[map->libbpf_type]); 3890 } 3891 if (ret < 0) 3892 return ret; 3893 3894 map->btf_key_type_id = key_type_id; 3895 map->btf_value_type_id = bpf_map__is_internal(map) ? 3896 ret : value_type_id; 3897 return 0; 3898 } 3899 3900 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 3901 { 3902 char file[PATH_MAX], buff[4096]; 3903 FILE *fp; 3904 __u32 val; 3905 int err; 3906 3907 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 3908 memset(info, 0, sizeof(*info)); 3909 3910 fp = fopen(file, "r"); 3911 if (!fp) { 3912 err = -errno; 3913 pr_warn("failed to open %s: %d. No procfs support?\n", file, 3914 err); 3915 return err; 3916 } 3917 3918 while (fgets(buff, sizeof(buff), fp)) { 3919 if (sscanf(buff, "map_type:\t%u", &val) == 1) 3920 info->type = val; 3921 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 3922 info->key_size = val; 3923 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 3924 info->value_size = val; 3925 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 3926 info->max_entries = val; 3927 else if (sscanf(buff, "map_flags:\t%i", &val) == 1) 3928 info->map_flags = val; 3929 } 3930 3931 fclose(fp); 3932 3933 return 0; 3934 } 3935 3936 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 3937 { 3938 struct bpf_map_info info = {}; 3939 __u32 len = sizeof(info); 3940 int new_fd, err; 3941 char *new_name; 3942 3943 err = bpf_obj_get_info_by_fd(fd, &info, &len); 3944 if (err && errno == EINVAL) 3945 err = bpf_get_map_info_from_fdinfo(fd, &info); 3946 if (err) 3947 return libbpf_err(err); 3948 3949 new_name = strdup(info.name); 3950 if (!new_name) 3951 return libbpf_err(-errno); 3952 3953 new_fd = open("/", O_RDONLY | O_CLOEXEC); 3954 if (new_fd < 0) { 3955 err = -errno; 3956 goto err_free_new_name; 3957 } 3958 3959 new_fd = dup3(fd, new_fd, O_CLOEXEC); 3960 if (new_fd < 0) { 3961 err = -errno; 3962 goto err_close_new_fd; 3963 } 3964 3965 err = zclose(map->fd); 3966 if (err) { 3967 err = -errno; 3968 goto err_close_new_fd; 3969 } 3970 free(map->name); 3971 3972 map->fd = new_fd; 3973 map->name = new_name; 3974 map->def.type = info.type; 3975 map->def.key_size = info.key_size; 3976 map->def.value_size = info.value_size; 3977 map->def.max_entries = info.max_entries; 3978 map->def.map_flags = info.map_flags; 3979 map->btf_key_type_id = info.btf_key_type_id; 3980 map->btf_value_type_id = info.btf_value_type_id; 3981 map->reused = true; 3982 3983 return 0; 3984 3985 err_close_new_fd: 3986 close(new_fd); 3987 err_free_new_name: 3988 free(new_name); 3989 return libbpf_err(err); 3990 } 3991 3992 __u32 bpf_map__max_entries(const struct bpf_map *map) 3993 { 3994 return map->def.max_entries; 3995 } 3996 3997 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 3998 { 3999 if (!bpf_map_type__is_map_in_map(map->def.type)) 4000 return errno = EINVAL, NULL; 4001 4002 return map->inner_map; 4003 } 4004 4005 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 4006 { 4007 if (map->fd >= 0) 4008 return libbpf_err(-EBUSY); 4009 map->def.max_entries = max_entries; 4010 return 0; 4011 } 4012 4013 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 4014 { 4015 if (!map || !max_entries) 4016 return libbpf_err(-EINVAL); 4017 4018 return bpf_map__set_max_entries(map, max_entries); 4019 } 4020 4021 static int 4022 bpf_object__probe_loading(struct bpf_object *obj) 4023 { 4024 struct bpf_load_program_attr attr; 4025 char *cp, errmsg[STRERR_BUFSIZE]; 4026 struct bpf_insn insns[] = { 4027 BPF_MOV64_IMM(BPF_REG_0, 0), 4028 BPF_EXIT_INSN(), 4029 }; 4030 int ret; 4031 4032 if (obj->gen_loader) 4033 return 0; 4034 4035 /* make sure basic loading works */ 4036 4037 memset(&attr, 0, sizeof(attr)); 4038 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4039 attr.insns = insns; 4040 attr.insns_cnt = ARRAY_SIZE(insns); 4041 attr.license = "GPL"; 4042 4043 ret = bpf_load_program_xattr(&attr, NULL, 0); 4044 if (ret < 0) { 4045 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; 4046 ret = bpf_load_program_xattr(&attr, NULL, 0); 4047 } 4048 if (ret < 0) { 4049 ret = errno; 4050 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4051 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 4052 "program. Make sure your kernel supports BPF " 4053 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 4054 "set to big enough value.\n", __func__, cp, ret); 4055 return -ret; 4056 } 4057 close(ret); 4058 4059 return 0; 4060 } 4061 4062 static int probe_fd(int fd) 4063 { 4064 if (fd >= 0) 4065 close(fd); 4066 return fd >= 0; 4067 } 4068 4069 static int probe_kern_prog_name(void) 4070 { 4071 struct bpf_load_program_attr attr; 4072 struct bpf_insn insns[] = { 4073 BPF_MOV64_IMM(BPF_REG_0, 0), 4074 BPF_EXIT_INSN(), 4075 }; 4076 int ret; 4077 4078 /* make sure loading with name works */ 4079 4080 memset(&attr, 0, sizeof(attr)); 4081 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4082 attr.insns = insns; 4083 attr.insns_cnt = ARRAY_SIZE(insns); 4084 attr.license = "GPL"; 4085 attr.name = "test"; 4086 ret = bpf_load_program_xattr(&attr, NULL, 0); 4087 return probe_fd(ret); 4088 } 4089 4090 static int probe_kern_global_data(void) 4091 { 4092 struct bpf_load_program_attr prg_attr; 4093 struct bpf_create_map_attr map_attr; 4094 char *cp, errmsg[STRERR_BUFSIZE]; 4095 struct bpf_insn insns[] = { 4096 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 4097 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 4098 BPF_MOV64_IMM(BPF_REG_0, 0), 4099 BPF_EXIT_INSN(), 4100 }; 4101 int ret, map; 4102 4103 memset(&map_attr, 0, sizeof(map_attr)); 4104 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4105 map_attr.key_size = sizeof(int); 4106 map_attr.value_size = 32; 4107 map_attr.max_entries = 1; 4108 4109 map = bpf_create_map_xattr(&map_attr); 4110 if (map < 0) { 4111 ret = -errno; 4112 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4113 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4114 __func__, cp, -ret); 4115 return ret; 4116 } 4117 4118 insns[0].imm = map; 4119 4120 memset(&prg_attr, 0, sizeof(prg_attr)); 4121 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4122 prg_attr.insns = insns; 4123 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4124 prg_attr.license = "GPL"; 4125 4126 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 4127 close(map); 4128 return probe_fd(ret); 4129 } 4130 4131 static int probe_kern_btf(void) 4132 { 4133 static const char strs[] = "\0int"; 4134 __u32 types[] = { 4135 /* int */ 4136 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4137 }; 4138 4139 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4140 strs, sizeof(strs))); 4141 } 4142 4143 static int probe_kern_btf_func(void) 4144 { 4145 static const char strs[] = "\0int\0x\0a"; 4146 /* void x(int a) {} */ 4147 __u32 types[] = { 4148 /* int */ 4149 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4150 /* FUNC_PROTO */ /* [2] */ 4151 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4152 BTF_PARAM_ENC(7, 1), 4153 /* FUNC x */ /* [3] */ 4154 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 4155 }; 4156 4157 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4158 strs, sizeof(strs))); 4159 } 4160 4161 static int probe_kern_btf_func_global(void) 4162 { 4163 static const char strs[] = "\0int\0x\0a"; 4164 /* static void x(int a) {} */ 4165 __u32 types[] = { 4166 /* int */ 4167 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4168 /* FUNC_PROTO */ /* [2] */ 4169 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4170 BTF_PARAM_ENC(7, 1), 4171 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 4172 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 4173 }; 4174 4175 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4176 strs, sizeof(strs))); 4177 } 4178 4179 static int probe_kern_btf_datasec(void) 4180 { 4181 static const char strs[] = "\0x\0.data"; 4182 /* static int a; */ 4183 __u32 types[] = { 4184 /* int */ 4185 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4186 /* VAR x */ /* [2] */ 4187 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4188 BTF_VAR_STATIC, 4189 /* DATASEC val */ /* [3] */ 4190 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 4191 BTF_VAR_SECINFO_ENC(2, 0, 4), 4192 }; 4193 4194 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4195 strs, sizeof(strs))); 4196 } 4197 4198 static int probe_kern_btf_float(void) 4199 { 4200 static const char strs[] = "\0float"; 4201 __u32 types[] = { 4202 /* float */ 4203 BTF_TYPE_FLOAT_ENC(1, 4), 4204 }; 4205 4206 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4207 strs, sizeof(strs))); 4208 } 4209 4210 static int probe_kern_array_mmap(void) 4211 { 4212 struct bpf_create_map_attr attr = { 4213 .map_type = BPF_MAP_TYPE_ARRAY, 4214 .map_flags = BPF_F_MMAPABLE, 4215 .key_size = sizeof(int), 4216 .value_size = sizeof(int), 4217 .max_entries = 1, 4218 }; 4219 4220 return probe_fd(bpf_create_map_xattr(&attr)); 4221 } 4222 4223 static int probe_kern_exp_attach_type(void) 4224 { 4225 struct bpf_load_program_attr attr; 4226 struct bpf_insn insns[] = { 4227 BPF_MOV64_IMM(BPF_REG_0, 0), 4228 BPF_EXIT_INSN(), 4229 }; 4230 4231 memset(&attr, 0, sizeof(attr)); 4232 /* use any valid combination of program type and (optional) 4233 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 4234 * to see if kernel supports expected_attach_type field for 4235 * BPF_PROG_LOAD command 4236 */ 4237 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 4238 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; 4239 attr.insns = insns; 4240 attr.insns_cnt = ARRAY_SIZE(insns); 4241 attr.license = "GPL"; 4242 4243 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4244 } 4245 4246 static int probe_kern_probe_read_kernel(void) 4247 { 4248 struct bpf_load_program_attr attr; 4249 struct bpf_insn insns[] = { 4250 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 4251 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 4252 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 4253 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 4254 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 4255 BPF_EXIT_INSN(), 4256 }; 4257 4258 memset(&attr, 0, sizeof(attr)); 4259 attr.prog_type = BPF_PROG_TYPE_KPROBE; 4260 attr.insns = insns; 4261 attr.insns_cnt = ARRAY_SIZE(insns); 4262 attr.license = "GPL"; 4263 4264 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); 4265 } 4266 4267 static int probe_prog_bind_map(void) 4268 { 4269 struct bpf_load_program_attr prg_attr; 4270 struct bpf_create_map_attr map_attr; 4271 char *cp, errmsg[STRERR_BUFSIZE]; 4272 struct bpf_insn insns[] = { 4273 BPF_MOV64_IMM(BPF_REG_0, 0), 4274 BPF_EXIT_INSN(), 4275 }; 4276 int ret, map, prog; 4277 4278 memset(&map_attr, 0, sizeof(map_attr)); 4279 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 4280 map_attr.key_size = sizeof(int); 4281 map_attr.value_size = 32; 4282 map_attr.max_entries = 1; 4283 4284 map = bpf_create_map_xattr(&map_attr); 4285 if (map < 0) { 4286 ret = -errno; 4287 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4288 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4289 __func__, cp, -ret); 4290 return ret; 4291 } 4292 4293 memset(&prg_attr, 0, sizeof(prg_attr)); 4294 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4295 prg_attr.insns = insns; 4296 prg_attr.insns_cnt = ARRAY_SIZE(insns); 4297 prg_attr.license = "GPL"; 4298 4299 prog = bpf_load_program_xattr(&prg_attr, NULL, 0); 4300 if (prog < 0) { 4301 close(map); 4302 return 0; 4303 } 4304 4305 ret = bpf_prog_bind_map(prog, map, NULL); 4306 4307 close(map); 4308 close(prog); 4309 4310 return ret >= 0; 4311 } 4312 4313 static int probe_module_btf(void) 4314 { 4315 static const char strs[] = "\0int"; 4316 __u32 types[] = { 4317 /* int */ 4318 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4319 }; 4320 struct bpf_btf_info info; 4321 __u32 len = sizeof(info); 4322 char name[16]; 4323 int fd, err; 4324 4325 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); 4326 if (fd < 0) 4327 return 0; /* BTF not supported at all */ 4328 4329 memset(&info, 0, sizeof(info)); 4330 info.name = ptr_to_u64(name); 4331 info.name_len = sizeof(name); 4332 4333 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 4334 * kernel's module BTF support coincides with support for 4335 * name/name_len fields in struct bpf_btf_info. 4336 */ 4337 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4338 close(fd); 4339 return !err; 4340 } 4341 4342 static int probe_perf_link(void) 4343 { 4344 struct bpf_load_program_attr attr; 4345 struct bpf_insn insns[] = { 4346 BPF_MOV64_IMM(BPF_REG_0, 0), 4347 BPF_EXIT_INSN(), 4348 }; 4349 int prog_fd, link_fd, err; 4350 4351 memset(&attr, 0, sizeof(attr)); 4352 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; 4353 attr.insns = insns; 4354 attr.insns_cnt = ARRAY_SIZE(insns); 4355 attr.license = "GPL"; 4356 prog_fd = bpf_load_program_xattr(&attr, NULL, 0); 4357 if (prog_fd < 0) 4358 return -errno; 4359 4360 /* use invalid perf_event FD to get EBADF, if link is supported; 4361 * otherwise EINVAL should be returned 4362 */ 4363 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); 4364 err = -errno; /* close() can clobber errno */ 4365 4366 if (link_fd >= 0) 4367 close(link_fd); 4368 close(prog_fd); 4369 4370 return link_fd < 0 && err == -EBADF; 4371 } 4372 4373 enum kern_feature_result { 4374 FEAT_UNKNOWN = 0, 4375 FEAT_SUPPORTED = 1, 4376 FEAT_MISSING = 2, 4377 }; 4378 4379 typedef int (*feature_probe_fn)(void); 4380 4381 static struct kern_feature_desc { 4382 const char *desc; 4383 feature_probe_fn probe; 4384 enum kern_feature_result res; 4385 } feature_probes[__FEAT_CNT] = { 4386 [FEAT_PROG_NAME] = { 4387 "BPF program name", probe_kern_prog_name, 4388 }, 4389 [FEAT_GLOBAL_DATA] = { 4390 "global variables", probe_kern_global_data, 4391 }, 4392 [FEAT_BTF] = { 4393 "minimal BTF", probe_kern_btf, 4394 }, 4395 [FEAT_BTF_FUNC] = { 4396 "BTF functions", probe_kern_btf_func, 4397 }, 4398 [FEAT_BTF_GLOBAL_FUNC] = { 4399 "BTF global function", probe_kern_btf_func_global, 4400 }, 4401 [FEAT_BTF_DATASEC] = { 4402 "BTF data section and variable", probe_kern_btf_datasec, 4403 }, 4404 [FEAT_ARRAY_MMAP] = { 4405 "ARRAY map mmap()", probe_kern_array_mmap, 4406 }, 4407 [FEAT_EXP_ATTACH_TYPE] = { 4408 "BPF_PROG_LOAD expected_attach_type attribute", 4409 probe_kern_exp_attach_type, 4410 }, 4411 [FEAT_PROBE_READ_KERN] = { 4412 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 4413 }, 4414 [FEAT_PROG_BIND_MAP] = { 4415 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 4416 }, 4417 [FEAT_MODULE_BTF] = { 4418 "module BTF support", probe_module_btf, 4419 }, 4420 [FEAT_BTF_FLOAT] = { 4421 "BTF_KIND_FLOAT support", probe_kern_btf_float, 4422 }, 4423 [FEAT_PERF_LINK] = { 4424 "BPF perf link support", probe_perf_link, 4425 }, 4426 }; 4427 4428 static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 4429 { 4430 struct kern_feature_desc *feat = &feature_probes[feat_id]; 4431 int ret; 4432 4433 if (obj->gen_loader) 4434 /* To generate loader program assume the latest kernel 4435 * to avoid doing extra prog_load, map_create syscalls. 4436 */ 4437 return true; 4438 4439 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { 4440 ret = feat->probe(); 4441 if (ret > 0) { 4442 WRITE_ONCE(feat->res, FEAT_SUPPORTED); 4443 } else if (ret == 0) { 4444 WRITE_ONCE(feat->res, FEAT_MISSING); 4445 } else { 4446 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); 4447 WRITE_ONCE(feat->res, FEAT_MISSING); 4448 } 4449 } 4450 4451 return READ_ONCE(feat->res) == FEAT_SUPPORTED; 4452 } 4453 4454 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 4455 { 4456 struct bpf_map_info map_info = {}; 4457 char msg[STRERR_BUFSIZE]; 4458 __u32 map_info_len; 4459 int err; 4460 4461 map_info_len = sizeof(map_info); 4462 4463 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len); 4464 if (err && errno == EINVAL) 4465 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 4466 if (err) { 4467 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 4468 libbpf_strerror_r(errno, msg, sizeof(msg))); 4469 return false; 4470 } 4471 4472 return (map_info.type == map->def.type && 4473 map_info.key_size == map->def.key_size && 4474 map_info.value_size == map->def.value_size && 4475 map_info.max_entries == map->def.max_entries && 4476 map_info.map_flags == map->def.map_flags); 4477 } 4478 4479 static int 4480 bpf_object__reuse_map(struct bpf_map *map) 4481 { 4482 char *cp, errmsg[STRERR_BUFSIZE]; 4483 int err, pin_fd; 4484 4485 pin_fd = bpf_obj_get(map->pin_path); 4486 if (pin_fd < 0) { 4487 err = -errno; 4488 if (err == -ENOENT) { 4489 pr_debug("found no pinned map to reuse at '%s'\n", 4490 map->pin_path); 4491 return 0; 4492 } 4493 4494 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 4495 pr_warn("couldn't retrieve pinned map '%s': %s\n", 4496 map->pin_path, cp); 4497 return err; 4498 } 4499 4500 if (!map_is_reuse_compat(map, pin_fd)) { 4501 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 4502 map->pin_path); 4503 close(pin_fd); 4504 return -EINVAL; 4505 } 4506 4507 err = bpf_map__reuse_fd(map, pin_fd); 4508 if (err) { 4509 close(pin_fd); 4510 return err; 4511 } 4512 map->pinned = true; 4513 pr_debug("reused pinned map at '%s'\n", map->pin_path); 4514 4515 return 0; 4516 } 4517 4518 static int 4519 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 4520 { 4521 enum libbpf_map_type map_type = map->libbpf_type; 4522 char *cp, errmsg[STRERR_BUFSIZE]; 4523 int err, zero = 0; 4524 4525 if (obj->gen_loader) { 4526 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 4527 map->mmaped, map->def.value_size); 4528 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 4529 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 4530 return 0; 4531 } 4532 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 4533 if (err) { 4534 err = -errno; 4535 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4536 pr_warn("Error setting initial map(%s) contents: %s\n", 4537 map->name, cp); 4538 return err; 4539 } 4540 4541 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 4542 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 4543 err = bpf_map_freeze(map->fd); 4544 if (err) { 4545 err = -errno; 4546 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4547 pr_warn("Error freezing map(%s) as read-only: %s\n", 4548 map->name, cp); 4549 return err; 4550 } 4551 } 4552 return 0; 4553 } 4554 4555 static void bpf_map__destroy(struct bpf_map *map); 4556 4557 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 4558 { 4559 struct bpf_create_map_attr create_attr; 4560 struct bpf_map_def *def = &map->def; 4561 int err = 0; 4562 4563 memset(&create_attr, 0, sizeof(create_attr)); 4564 4565 if (kernel_supports(obj, FEAT_PROG_NAME)) 4566 create_attr.name = map->name; 4567 create_attr.map_ifindex = map->map_ifindex; 4568 create_attr.map_type = def->type; 4569 create_attr.map_flags = def->map_flags; 4570 create_attr.key_size = def->key_size; 4571 create_attr.value_size = def->value_size; 4572 create_attr.numa_node = map->numa_node; 4573 4574 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) { 4575 int nr_cpus; 4576 4577 nr_cpus = libbpf_num_possible_cpus(); 4578 if (nr_cpus < 0) { 4579 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 4580 map->name, nr_cpus); 4581 return nr_cpus; 4582 } 4583 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 4584 create_attr.max_entries = nr_cpus; 4585 } else { 4586 create_attr.max_entries = def->max_entries; 4587 } 4588 4589 if (bpf_map__is_struct_ops(map)) 4590 create_attr.btf_vmlinux_value_type_id = 4591 map->btf_vmlinux_value_type_id; 4592 4593 create_attr.btf_fd = 0; 4594 create_attr.btf_key_type_id = 0; 4595 create_attr.btf_value_type_id = 0; 4596 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) { 4597 create_attr.btf_fd = btf__fd(obj->btf); 4598 create_attr.btf_key_type_id = map->btf_key_type_id; 4599 create_attr.btf_value_type_id = map->btf_value_type_id; 4600 } 4601 4602 if (bpf_map_type__is_map_in_map(def->type)) { 4603 if (map->inner_map) { 4604 err = bpf_object__create_map(obj, map->inner_map, true); 4605 if (err) { 4606 pr_warn("map '%s': failed to create inner map: %d\n", 4607 map->name, err); 4608 return err; 4609 } 4610 map->inner_map_fd = bpf_map__fd(map->inner_map); 4611 } 4612 if (map->inner_map_fd >= 0) 4613 create_attr.inner_map_fd = map->inner_map_fd; 4614 } 4615 4616 if (obj->gen_loader) { 4617 bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps); 4618 /* Pretend to have valid FD to pass various fd >= 0 checks. 4619 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 4620 */ 4621 map->fd = 0; 4622 } else { 4623 map->fd = bpf_create_map_xattr(&create_attr); 4624 } 4625 if (map->fd < 0 && (create_attr.btf_key_type_id || 4626 create_attr.btf_value_type_id)) { 4627 char *cp, errmsg[STRERR_BUFSIZE]; 4628 4629 err = -errno; 4630 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4631 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 4632 map->name, cp, err); 4633 create_attr.btf_fd = 0; 4634 create_attr.btf_key_type_id = 0; 4635 create_attr.btf_value_type_id = 0; 4636 map->btf_key_type_id = 0; 4637 map->btf_value_type_id = 0; 4638 map->fd = bpf_create_map_xattr(&create_attr); 4639 } 4640 4641 err = map->fd < 0 ? -errno : 0; 4642 4643 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 4644 if (obj->gen_loader) 4645 map->inner_map->fd = -1; 4646 bpf_map__destroy(map->inner_map); 4647 zfree(&map->inner_map); 4648 } 4649 4650 return err; 4651 } 4652 4653 static int init_map_slots(struct bpf_object *obj, struct bpf_map *map) 4654 { 4655 const struct bpf_map *targ_map; 4656 unsigned int i; 4657 int fd, err = 0; 4658 4659 for (i = 0; i < map->init_slots_sz; i++) { 4660 if (!map->init_slots[i]) 4661 continue; 4662 4663 targ_map = map->init_slots[i]; 4664 fd = bpf_map__fd(targ_map); 4665 if (obj->gen_loader) { 4666 pr_warn("// TODO map_update_elem: idx %td key %d value==map_idx %td\n", 4667 map - obj->maps, i, targ_map - obj->maps); 4668 return -ENOTSUP; 4669 } else { 4670 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 4671 } 4672 if (err) { 4673 err = -errno; 4674 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 4675 map->name, i, targ_map->name, 4676 fd, err); 4677 return err; 4678 } 4679 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 4680 map->name, i, targ_map->name, fd); 4681 } 4682 4683 zfree(&map->init_slots); 4684 map->init_slots_sz = 0; 4685 4686 return 0; 4687 } 4688 4689 static int 4690 bpf_object__create_maps(struct bpf_object *obj) 4691 { 4692 struct bpf_map *map; 4693 char *cp, errmsg[STRERR_BUFSIZE]; 4694 unsigned int i, j; 4695 int err; 4696 bool retried; 4697 4698 for (i = 0; i < obj->nr_maps; i++) { 4699 map = &obj->maps[i]; 4700 4701 retried = false; 4702 retry: 4703 if (map->pin_path) { 4704 err = bpf_object__reuse_map(map); 4705 if (err) { 4706 pr_warn("map '%s': error reusing pinned map\n", 4707 map->name); 4708 goto err_out; 4709 } 4710 if (retried && map->fd < 0) { 4711 pr_warn("map '%s': cannot find pinned map\n", 4712 map->name); 4713 err = -ENOENT; 4714 goto err_out; 4715 } 4716 } 4717 4718 if (map->fd >= 0) { 4719 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 4720 map->name, map->fd); 4721 } else { 4722 err = bpf_object__create_map(obj, map, false); 4723 if (err) 4724 goto err_out; 4725 4726 pr_debug("map '%s': created successfully, fd=%d\n", 4727 map->name, map->fd); 4728 4729 if (bpf_map__is_internal(map)) { 4730 err = bpf_object__populate_internal_map(obj, map); 4731 if (err < 0) { 4732 zclose(map->fd); 4733 goto err_out; 4734 } 4735 } 4736 4737 if (map->init_slots_sz) { 4738 err = init_map_slots(obj, map); 4739 if (err < 0) { 4740 zclose(map->fd); 4741 goto err_out; 4742 } 4743 } 4744 } 4745 4746 if (map->pin_path && !map->pinned) { 4747 err = bpf_map__pin(map, NULL); 4748 if (err) { 4749 zclose(map->fd); 4750 if (!retried && err == -EEXIST) { 4751 retried = true; 4752 goto retry; 4753 } 4754 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 4755 map->name, map->pin_path, err); 4756 goto err_out; 4757 } 4758 } 4759 } 4760 4761 return 0; 4762 4763 err_out: 4764 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4765 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 4766 pr_perm_msg(err); 4767 for (j = 0; j < i; j++) 4768 zclose(obj->maps[j].fd); 4769 return err; 4770 } 4771 4772 static bool bpf_core_is_flavor_sep(const char *s) 4773 { 4774 /* check X___Y name pattern, where X and Y are not underscores */ 4775 return s[0] != '_' && /* X */ 4776 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 4777 s[4] != '_'; /* Y */ 4778 } 4779 4780 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 4781 * before last triple underscore. Struct name part after last triple 4782 * underscore is ignored by BPF CO-RE relocation during relocation matching. 4783 */ 4784 size_t bpf_core_essential_name_len(const char *name) 4785 { 4786 size_t n = strlen(name); 4787 int i; 4788 4789 for (i = n - 5; i >= 0; i--) { 4790 if (bpf_core_is_flavor_sep(name + i)) 4791 return i + 1; 4792 } 4793 return n; 4794 } 4795 4796 static void bpf_core_free_cands(struct bpf_core_cand_list *cands) 4797 { 4798 free(cands->cands); 4799 free(cands); 4800 } 4801 4802 static int bpf_core_add_cands(struct bpf_core_cand *local_cand, 4803 size_t local_essent_len, 4804 const struct btf *targ_btf, 4805 const char *targ_btf_name, 4806 int targ_start_id, 4807 struct bpf_core_cand_list *cands) 4808 { 4809 struct bpf_core_cand *new_cands, *cand; 4810 const struct btf_type *t; 4811 const char *targ_name; 4812 size_t targ_essent_len; 4813 int n, i; 4814 4815 n = btf__get_nr_types(targ_btf); 4816 for (i = targ_start_id; i <= n; i++) { 4817 t = btf__type_by_id(targ_btf, i); 4818 if (btf_kind(t) != btf_kind(local_cand->t)) 4819 continue; 4820 4821 targ_name = btf__name_by_offset(targ_btf, t->name_off); 4822 if (str_is_empty(targ_name)) 4823 continue; 4824 4825 targ_essent_len = bpf_core_essential_name_len(targ_name); 4826 if (targ_essent_len != local_essent_len) 4827 continue; 4828 4829 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0) 4830 continue; 4831 4832 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 4833 local_cand->id, btf_kind_str(local_cand->t), 4834 local_cand->name, i, btf_kind_str(t), targ_name, 4835 targ_btf_name); 4836 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 4837 sizeof(*cands->cands)); 4838 if (!new_cands) 4839 return -ENOMEM; 4840 4841 cand = &new_cands[cands->len]; 4842 cand->btf = targ_btf; 4843 cand->t = t; 4844 cand->name = targ_name; 4845 cand->id = i; 4846 4847 cands->cands = new_cands; 4848 cands->len++; 4849 } 4850 return 0; 4851 } 4852 4853 static int load_module_btfs(struct bpf_object *obj) 4854 { 4855 struct bpf_btf_info info; 4856 struct module_btf *mod_btf; 4857 struct btf *btf; 4858 char name[64]; 4859 __u32 id = 0, len; 4860 int err, fd; 4861 4862 if (obj->btf_modules_loaded) 4863 return 0; 4864 4865 if (obj->gen_loader) 4866 return 0; 4867 4868 /* don't do this again, even if we find no module BTFs */ 4869 obj->btf_modules_loaded = true; 4870 4871 /* kernel too old to support module BTFs */ 4872 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 4873 return 0; 4874 4875 while (true) { 4876 err = bpf_btf_get_next_id(id, &id); 4877 if (err && errno == ENOENT) 4878 return 0; 4879 if (err) { 4880 err = -errno; 4881 pr_warn("failed to iterate BTF objects: %d\n", err); 4882 return err; 4883 } 4884 4885 fd = bpf_btf_get_fd_by_id(id); 4886 if (fd < 0) { 4887 if (errno == ENOENT) 4888 continue; /* expected race: BTF was unloaded */ 4889 err = -errno; 4890 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 4891 return err; 4892 } 4893 4894 len = sizeof(info); 4895 memset(&info, 0, sizeof(info)); 4896 info.name = ptr_to_u64(name); 4897 info.name_len = sizeof(name); 4898 4899 err = bpf_obj_get_info_by_fd(fd, &info, &len); 4900 if (err) { 4901 err = -errno; 4902 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 4903 goto err_out; 4904 } 4905 4906 /* ignore non-module BTFs */ 4907 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 4908 close(fd); 4909 continue; 4910 } 4911 4912 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 4913 err = libbpf_get_error(btf); 4914 if (err) { 4915 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", 4916 name, id, err); 4917 goto err_out; 4918 } 4919 4920 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 4921 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 4922 if (err) 4923 goto err_out; 4924 4925 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 4926 4927 mod_btf->btf = btf; 4928 mod_btf->id = id; 4929 mod_btf->fd = fd; 4930 mod_btf->name = strdup(name); 4931 if (!mod_btf->name) { 4932 err = -ENOMEM; 4933 goto err_out; 4934 } 4935 continue; 4936 4937 err_out: 4938 close(fd); 4939 return err; 4940 } 4941 4942 return 0; 4943 } 4944 4945 static struct bpf_core_cand_list * 4946 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 4947 { 4948 struct bpf_core_cand local_cand = {}; 4949 struct bpf_core_cand_list *cands; 4950 const struct btf *main_btf; 4951 size_t local_essent_len; 4952 int err, i; 4953 4954 local_cand.btf = local_btf; 4955 local_cand.t = btf__type_by_id(local_btf, local_type_id); 4956 if (!local_cand.t) 4957 return ERR_PTR(-EINVAL); 4958 4959 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off); 4960 if (str_is_empty(local_cand.name)) 4961 return ERR_PTR(-EINVAL); 4962 local_essent_len = bpf_core_essential_name_len(local_cand.name); 4963 4964 cands = calloc(1, sizeof(*cands)); 4965 if (!cands) 4966 return ERR_PTR(-ENOMEM); 4967 4968 /* Attempt to find target candidates in vmlinux BTF first */ 4969 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 4970 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 4971 if (err) 4972 goto err_out; 4973 4974 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 4975 if (cands->len) 4976 return cands; 4977 4978 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 4979 if (obj->btf_vmlinux_override) 4980 return cands; 4981 4982 /* now look through module BTFs, trying to still find candidates */ 4983 err = load_module_btfs(obj); 4984 if (err) 4985 goto err_out; 4986 4987 for (i = 0; i < obj->btf_module_cnt; i++) { 4988 err = bpf_core_add_cands(&local_cand, local_essent_len, 4989 obj->btf_modules[i].btf, 4990 obj->btf_modules[i].name, 4991 btf__get_nr_types(obj->btf_vmlinux) + 1, 4992 cands); 4993 if (err) 4994 goto err_out; 4995 } 4996 4997 return cands; 4998 err_out: 4999 bpf_core_free_cands(cands); 5000 return ERR_PTR(err); 5001 } 5002 5003 /* Check local and target types for compatibility. This check is used for 5004 * type-based CO-RE relocations and follow slightly different rules than 5005 * field-based relocations. This function assumes that root types were already 5006 * checked for name match. Beyond that initial root-level name check, names 5007 * are completely ignored. Compatibility rules are as follows: 5008 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 5009 * kind should match for local and target types (i.e., STRUCT is not 5010 * compatible with UNION); 5011 * - for ENUMs, the size is ignored; 5012 * - for INT, size and signedness are ignored; 5013 * - for ARRAY, dimensionality is ignored, element types are checked for 5014 * compatibility recursively; 5015 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 5016 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 5017 * - FUNC_PROTOs are compatible if they have compatible signature: same 5018 * number of input args and compatible return and argument types. 5019 * These rules are not set in stone and probably will be adjusted as we get 5020 * more experience with using BPF CO-RE relocations. 5021 */ 5022 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 5023 const struct btf *targ_btf, __u32 targ_id) 5024 { 5025 const struct btf_type *local_type, *targ_type; 5026 int depth = 32; /* max recursion depth */ 5027 5028 /* caller made sure that names match (ignoring flavor suffix) */ 5029 local_type = btf__type_by_id(local_btf, local_id); 5030 targ_type = btf__type_by_id(targ_btf, targ_id); 5031 if (btf_kind(local_type) != btf_kind(targ_type)) 5032 return 0; 5033 5034 recur: 5035 depth--; 5036 if (depth < 0) 5037 return -EINVAL; 5038 5039 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 5040 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 5041 if (!local_type || !targ_type) 5042 return -EINVAL; 5043 5044 if (btf_kind(local_type) != btf_kind(targ_type)) 5045 return 0; 5046 5047 switch (btf_kind(local_type)) { 5048 case BTF_KIND_UNKN: 5049 case BTF_KIND_STRUCT: 5050 case BTF_KIND_UNION: 5051 case BTF_KIND_ENUM: 5052 case BTF_KIND_FWD: 5053 return 1; 5054 case BTF_KIND_INT: 5055 /* just reject deprecated bitfield-like integers; all other 5056 * integers are by default compatible between each other 5057 */ 5058 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; 5059 case BTF_KIND_PTR: 5060 local_id = local_type->type; 5061 targ_id = targ_type->type; 5062 goto recur; 5063 case BTF_KIND_ARRAY: 5064 local_id = btf_array(local_type)->type; 5065 targ_id = btf_array(targ_type)->type; 5066 goto recur; 5067 case BTF_KIND_FUNC_PROTO: { 5068 struct btf_param *local_p = btf_params(local_type); 5069 struct btf_param *targ_p = btf_params(targ_type); 5070 __u16 local_vlen = btf_vlen(local_type); 5071 __u16 targ_vlen = btf_vlen(targ_type); 5072 int i, err; 5073 5074 if (local_vlen != targ_vlen) 5075 return 0; 5076 5077 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { 5078 skip_mods_and_typedefs(local_btf, local_p->type, &local_id); 5079 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); 5080 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id); 5081 if (err <= 0) 5082 return err; 5083 } 5084 5085 /* tail recurse for return type check */ 5086 skip_mods_and_typedefs(local_btf, local_type->type, &local_id); 5087 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); 5088 goto recur; 5089 } 5090 default: 5091 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", 5092 btf_kind_str(local_type), local_id, targ_id); 5093 return 0; 5094 } 5095 } 5096 5097 static size_t bpf_core_hash_fn(const void *key, void *ctx) 5098 { 5099 return (size_t)key; 5100 } 5101 5102 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) 5103 { 5104 return k1 == k2; 5105 } 5106 5107 static void *u32_as_hash_key(__u32 x) 5108 { 5109 return (void *)(uintptr_t)x; 5110 } 5111 5112 static int bpf_core_apply_relo(struct bpf_program *prog, 5113 const struct bpf_core_relo *relo, 5114 int relo_idx, 5115 const struct btf *local_btf, 5116 struct hashmap *cand_cache) 5117 { 5118 const void *type_key = u32_as_hash_key(relo->type_id); 5119 struct bpf_core_cand_list *cands = NULL; 5120 const char *prog_name = prog->name; 5121 const struct btf_type *local_type; 5122 const char *local_name; 5123 __u32 local_id = relo->type_id; 5124 struct bpf_insn *insn; 5125 int insn_idx, err; 5126 5127 if (relo->insn_off % BPF_INSN_SZ) 5128 return -EINVAL; 5129 insn_idx = relo->insn_off / BPF_INSN_SZ; 5130 /* adjust insn_idx from section frame of reference to the local 5131 * program's frame of reference; (sub-)program code is not yet 5132 * relocated, so it's enough to just subtract in-section offset 5133 */ 5134 insn_idx = insn_idx - prog->sec_insn_off; 5135 if (insn_idx > prog->insns_cnt) 5136 return -EINVAL; 5137 insn = &prog->insns[insn_idx]; 5138 5139 local_type = btf__type_by_id(local_btf, local_id); 5140 if (!local_type) 5141 return -EINVAL; 5142 5143 local_name = btf__name_by_offset(local_btf, local_type->name_off); 5144 if (!local_name) 5145 return -EINVAL; 5146 5147 if (prog->obj->gen_loader) { 5148 pr_warn("// TODO core_relo: prog %td insn[%d] %s kind %d\n", 5149 prog - prog->obj->programs, relo->insn_off / 8, 5150 local_name, relo->kind); 5151 return -ENOTSUP; 5152 } 5153 5154 if (relo->kind != BPF_TYPE_ID_LOCAL && 5155 !hashmap__find(cand_cache, type_key, (void **)&cands)) { 5156 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 5157 if (IS_ERR(cands)) { 5158 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 5159 prog_name, relo_idx, local_id, btf_kind_str(local_type), 5160 local_name, PTR_ERR(cands)); 5161 return PTR_ERR(cands); 5162 } 5163 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL); 5164 if (err) { 5165 bpf_core_free_cands(cands); 5166 return err; 5167 } 5168 } 5169 5170 return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands); 5171 } 5172 5173 static int 5174 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 5175 { 5176 const struct btf_ext_info_sec *sec; 5177 const struct bpf_core_relo *rec; 5178 const struct btf_ext_info *seg; 5179 struct hashmap_entry *entry; 5180 struct hashmap *cand_cache = NULL; 5181 struct bpf_program *prog; 5182 const char *sec_name; 5183 int i, err = 0, insn_idx, sec_idx; 5184 5185 if (obj->btf_ext->core_relo_info.len == 0) 5186 return 0; 5187 5188 if (targ_btf_path) { 5189 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 5190 err = libbpf_get_error(obj->btf_vmlinux_override); 5191 if (err) { 5192 pr_warn("failed to parse target BTF: %d\n", err); 5193 return err; 5194 } 5195 } 5196 5197 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 5198 if (IS_ERR(cand_cache)) { 5199 err = PTR_ERR(cand_cache); 5200 goto out; 5201 } 5202 5203 seg = &obj->btf_ext->core_relo_info; 5204 for_each_btf_ext_sec(seg, sec) { 5205 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5206 if (str_is_empty(sec_name)) { 5207 err = -EINVAL; 5208 goto out; 5209 } 5210 /* bpf_object's ELF is gone by now so it's not easy to find 5211 * section index by section name, but we can find *any* 5212 * bpf_program within desired section name and use it's 5213 * prog->sec_idx to do a proper search by section index and 5214 * instruction offset 5215 */ 5216 prog = NULL; 5217 for (i = 0; i < obj->nr_programs; i++) { 5218 prog = &obj->programs[i]; 5219 if (strcmp(prog->sec_name, sec_name) == 0) 5220 break; 5221 } 5222 if (!prog) { 5223 pr_warn("sec '%s': failed to find a BPF program\n", sec_name); 5224 return -ENOENT; 5225 } 5226 sec_idx = prog->sec_idx; 5227 5228 pr_debug("sec '%s': found %d CO-RE relocations\n", 5229 sec_name, sec->num_info); 5230 5231 for_each_btf_ext_rec(seg, sec, i, rec) { 5232 insn_idx = rec->insn_off / BPF_INSN_SZ; 5233 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 5234 if (!prog) { 5235 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n", 5236 sec_name, insn_idx, i); 5237 err = -EINVAL; 5238 goto out; 5239 } 5240 /* no need to apply CO-RE relocation if the program is 5241 * not going to be loaded 5242 */ 5243 if (!prog->load) 5244 continue; 5245 5246 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache); 5247 if (err) { 5248 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 5249 prog->name, i, err); 5250 goto out; 5251 } 5252 } 5253 } 5254 5255 out: 5256 /* obj->btf_vmlinux and module BTFs are freed after object load */ 5257 btf__free(obj->btf_vmlinux_override); 5258 obj->btf_vmlinux_override = NULL; 5259 5260 if (!IS_ERR_OR_NULL(cand_cache)) { 5261 hashmap__for_each_entry(cand_cache, entry, i) { 5262 bpf_core_free_cands(entry->value); 5263 } 5264 hashmap__free(cand_cache); 5265 } 5266 return err; 5267 } 5268 5269 /* Relocate data references within program code: 5270 * - map references; 5271 * - global variable references; 5272 * - extern references. 5273 */ 5274 static int 5275 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 5276 { 5277 int i; 5278 5279 for (i = 0; i < prog->nr_reloc; i++) { 5280 struct reloc_desc *relo = &prog->reloc_desc[i]; 5281 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5282 struct extern_desc *ext; 5283 5284 switch (relo->type) { 5285 case RELO_LD64: 5286 if (obj->gen_loader) { 5287 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 5288 insn[0].imm = relo->map_idx; 5289 } else { 5290 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 5291 insn[0].imm = obj->maps[relo->map_idx].fd; 5292 } 5293 break; 5294 case RELO_DATA: 5295 insn[1].imm = insn[0].imm + relo->sym_off; 5296 if (obj->gen_loader) { 5297 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5298 insn[0].imm = relo->map_idx; 5299 } else { 5300 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5301 insn[0].imm = obj->maps[relo->map_idx].fd; 5302 } 5303 break; 5304 case RELO_EXTERN_VAR: 5305 ext = &obj->externs[relo->sym_off]; 5306 if (ext->type == EXT_KCFG) { 5307 if (obj->gen_loader) { 5308 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5309 insn[0].imm = obj->kconfig_map_idx; 5310 } else { 5311 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5312 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 5313 } 5314 insn[1].imm = ext->kcfg.data_off; 5315 } else /* EXT_KSYM */ { 5316 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 5317 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 5318 insn[0].imm = ext->ksym.kernel_btf_id; 5319 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 5320 } else { /* typeless ksyms or unresolved typed ksyms */ 5321 insn[0].imm = (__u32)ext->ksym.addr; 5322 insn[1].imm = ext->ksym.addr >> 32; 5323 } 5324 } 5325 break; 5326 case RELO_EXTERN_FUNC: 5327 ext = &obj->externs[relo->sym_off]; 5328 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 5329 insn[0].imm = ext->ksym.kernel_btf_id; 5330 break; 5331 case RELO_SUBPROG_ADDR: 5332 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 5333 pr_warn("prog '%s': relo #%d: bad insn\n", 5334 prog->name, i); 5335 return -EINVAL; 5336 } 5337 /* handled already */ 5338 break; 5339 case RELO_CALL: 5340 /* handled already */ 5341 break; 5342 default: 5343 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 5344 prog->name, i, relo->type); 5345 return -EINVAL; 5346 } 5347 } 5348 5349 return 0; 5350 } 5351 5352 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 5353 const struct bpf_program *prog, 5354 const struct btf_ext_info *ext_info, 5355 void **prog_info, __u32 *prog_rec_cnt, 5356 __u32 *prog_rec_sz) 5357 { 5358 void *copy_start = NULL, *copy_end = NULL; 5359 void *rec, *rec_end, *new_prog_info; 5360 const struct btf_ext_info_sec *sec; 5361 size_t old_sz, new_sz; 5362 const char *sec_name; 5363 int i, off_adj; 5364 5365 for_each_btf_ext_sec(ext_info, sec) { 5366 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5367 if (!sec_name) 5368 return -EINVAL; 5369 if (strcmp(sec_name, prog->sec_name) != 0) 5370 continue; 5371 5372 for_each_btf_ext_rec(ext_info, sec, i, rec) { 5373 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 5374 5375 if (insn_off < prog->sec_insn_off) 5376 continue; 5377 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 5378 break; 5379 5380 if (!copy_start) 5381 copy_start = rec; 5382 copy_end = rec + ext_info->rec_size; 5383 } 5384 5385 if (!copy_start) 5386 return -ENOENT; 5387 5388 /* append func/line info of a given (sub-)program to the main 5389 * program func/line info 5390 */ 5391 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 5392 new_sz = old_sz + (copy_end - copy_start); 5393 new_prog_info = realloc(*prog_info, new_sz); 5394 if (!new_prog_info) 5395 return -ENOMEM; 5396 *prog_info = new_prog_info; 5397 *prog_rec_cnt = new_sz / ext_info->rec_size; 5398 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 5399 5400 /* Kernel instruction offsets are in units of 8-byte 5401 * instructions, while .BTF.ext instruction offsets generated 5402 * by Clang are in units of bytes. So convert Clang offsets 5403 * into kernel offsets and adjust offset according to program 5404 * relocated position. 5405 */ 5406 off_adj = prog->sub_insn_off - prog->sec_insn_off; 5407 rec = new_prog_info + old_sz; 5408 rec_end = new_prog_info + new_sz; 5409 for (; rec < rec_end; rec += ext_info->rec_size) { 5410 __u32 *insn_off = rec; 5411 5412 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 5413 } 5414 *prog_rec_sz = ext_info->rec_size; 5415 return 0; 5416 } 5417 5418 return -ENOENT; 5419 } 5420 5421 static int 5422 reloc_prog_func_and_line_info(const struct bpf_object *obj, 5423 struct bpf_program *main_prog, 5424 const struct bpf_program *prog) 5425 { 5426 int err; 5427 5428 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 5429 * supprot func/line info 5430 */ 5431 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 5432 return 0; 5433 5434 /* only attempt func info relocation if main program's func_info 5435 * relocation was successful 5436 */ 5437 if (main_prog != prog && !main_prog->func_info) 5438 goto line_info; 5439 5440 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 5441 &main_prog->func_info, 5442 &main_prog->func_info_cnt, 5443 &main_prog->func_info_rec_size); 5444 if (err) { 5445 if (err != -ENOENT) { 5446 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 5447 prog->name, err); 5448 return err; 5449 } 5450 if (main_prog->func_info) { 5451 /* 5452 * Some info has already been found but has problem 5453 * in the last btf_ext reloc. Must have to error out. 5454 */ 5455 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 5456 return err; 5457 } 5458 /* Have problem loading the very first info. Ignore the rest. */ 5459 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 5460 prog->name); 5461 } 5462 5463 line_info: 5464 /* don't relocate line info if main program's relocation failed */ 5465 if (main_prog != prog && !main_prog->line_info) 5466 return 0; 5467 5468 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 5469 &main_prog->line_info, 5470 &main_prog->line_info_cnt, 5471 &main_prog->line_info_rec_size); 5472 if (err) { 5473 if (err != -ENOENT) { 5474 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 5475 prog->name, err); 5476 return err; 5477 } 5478 if (main_prog->line_info) { 5479 /* 5480 * Some info has already been found but has problem 5481 * in the last btf_ext reloc. Must have to error out. 5482 */ 5483 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 5484 return err; 5485 } 5486 /* Have problem loading the very first info. Ignore the rest. */ 5487 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 5488 prog->name); 5489 } 5490 return 0; 5491 } 5492 5493 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 5494 { 5495 size_t insn_idx = *(const size_t *)key; 5496 const struct reloc_desc *relo = elem; 5497 5498 if (insn_idx == relo->insn_idx) 5499 return 0; 5500 return insn_idx < relo->insn_idx ? -1 : 1; 5501 } 5502 5503 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 5504 { 5505 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 5506 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 5507 } 5508 5509 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 5510 { 5511 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 5512 struct reloc_desc *relos; 5513 int i; 5514 5515 if (main_prog == subprog) 5516 return 0; 5517 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 5518 if (!relos) 5519 return -ENOMEM; 5520 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 5521 sizeof(*relos) * subprog->nr_reloc); 5522 5523 for (i = main_prog->nr_reloc; i < new_cnt; i++) 5524 relos[i].insn_idx += subprog->sub_insn_off; 5525 /* After insn_idx adjustment the 'relos' array is still sorted 5526 * by insn_idx and doesn't break bsearch. 5527 */ 5528 main_prog->reloc_desc = relos; 5529 main_prog->nr_reloc = new_cnt; 5530 return 0; 5531 } 5532 5533 static int 5534 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 5535 struct bpf_program *prog) 5536 { 5537 size_t sub_insn_idx, insn_idx, new_cnt; 5538 struct bpf_program *subprog; 5539 struct bpf_insn *insns, *insn; 5540 struct reloc_desc *relo; 5541 int err; 5542 5543 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 5544 if (err) 5545 return err; 5546 5547 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 5548 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 5549 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 5550 continue; 5551 5552 relo = find_prog_insn_relo(prog, insn_idx); 5553 if (relo && relo->type == RELO_EXTERN_FUNC) 5554 /* kfunc relocations will be handled later 5555 * in bpf_object__relocate_data() 5556 */ 5557 continue; 5558 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 5559 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 5560 prog->name, insn_idx, relo->type); 5561 return -LIBBPF_ERRNO__RELOC; 5562 } 5563 if (relo) { 5564 /* sub-program instruction index is a combination of 5565 * an offset of a symbol pointed to by relocation and 5566 * call instruction's imm field; for global functions, 5567 * call always has imm = -1, but for static functions 5568 * relocation is against STT_SECTION and insn->imm 5569 * points to a start of a static function 5570 * 5571 * for subprog addr relocation, the relo->sym_off + insn->imm is 5572 * the byte offset in the corresponding section. 5573 */ 5574 if (relo->type == RELO_CALL) 5575 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 5576 else 5577 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 5578 } else if (insn_is_pseudo_func(insn)) { 5579 /* 5580 * RELO_SUBPROG_ADDR relo is always emitted even if both 5581 * functions are in the same section, so it shouldn't reach here. 5582 */ 5583 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 5584 prog->name, insn_idx); 5585 return -LIBBPF_ERRNO__RELOC; 5586 } else { 5587 /* if subprogram call is to a static function within 5588 * the same ELF section, there won't be any relocation 5589 * emitted, but it also means there is no additional 5590 * offset necessary, insns->imm is relative to 5591 * instruction's original position within the section 5592 */ 5593 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 5594 } 5595 5596 /* we enforce that sub-programs should be in .text section */ 5597 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 5598 if (!subprog) { 5599 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 5600 prog->name); 5601 return -LIBBPF_ERRNO__RELOC; 5602 } 5603 5604 /* if it's the first call instruction calling into this 5605 * subprogram (meaning this subprog hasn't been processed 5606 * yet) within the context of current main program: 5607 * - append it at the end of main program's instructions blog; 5608 * - process is recursively, while current program is put on hold; 5609 * - if that subprogram calls some other not yet processes 5610 * subprogram, same thing will happen recursively until 5611 * there are no more unprocesses subprograms left to append 5612 * and relocate. 5613 */ 5614 if (subprog->sub_insn_off == 0) { 5615 subprog->sub_insn_off = main_prog->insns_cnt; 5616 5617 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 5618 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 5619 if (!insns) { 5620 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 5621 return -ENOMEM; 5622 } 5623 main_prog->insns = insns; 5624 main_prog->insns_cnt = new_cnt; 5625 5626 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 5627 subprog->insns_cnt * sizeof(*insns)); 5628 5629 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 5630 main_prog->name, subprog->insns_cnt, subprog->name); 5631 5632 /* The subprog insns are now appended. Append its relos too. */ 5633 err = append_subprog_relos(main_prog, subprog); 5634 if (err) 5635 return err; 5636 err = bpf_object__reloc_code(obj, main_prog, subprog); 5637 if (err) 5638 return err; 5639 } 5640 5641 /* main_prog->insns memory could have been re-allocated, so 5642 * calculate pointer again 5643 */ 5644 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 5645 /* calculate correct instruction position within current main 5646 * prog; each main prog can have a different set of 5647 * subprograms appended (potentially in different order as 5648 * well), so position of any subprog can be different for 5649 * different main programs */ 5650 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 5651 5652 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 5653 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 5654 } 5655 5656 return 0; 5657 } 5658 5659 /* 5660 * Relocate sub-program calls. 5661 * 5662 * Algorithm operates as follows. Each entry-point BPF program (referred to as 5663 * main prog) is processed separately. For each subprog (non-entry functions, 5664 * that can be called from either entry progs or other subprogs) gets their 5665 * sub_insn_off reset to zero. This serves as indicator that this subprogram 5666 * hasn't been yet appended and relocated within current main prog. Once its 5667 * relocated, sub_insn_off will point at the position within current main prog 5668 * where given subprog was appended. This will further be used to relocate all 5669 * the call instructions jumping into this subprog. 5670 * 5671 * We start with main program and process all call instructions. If the call 5672 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 5673 * is zero), subprog instructions are appended at the end of main program's 5674 * instruction array. Then main program is "put on hold" while we recursively 5675 * process newly appended subprogram. If that subprogram calls into another 5676 * subprogram that hasn't been appended, new subprogram is appended again to 5677 * the *main* prog's instructions (subprog's instructions are always left 5678 * untouched, as they need to be in unmodified state for subsequent main progs 5679 * and subprog instructions are always sent only as part of a main prog) and 5680 * the process continues recursively. Once all the subprogs called from a main 5681 * prog or any of its subprogs are appended (and relocated), all their 5682 * positions within finalized instructions array are known, so it's easy to 5683 * rewrite call instructions with correct relative offsets, corresponding to 5684 * desired target subprog. 5685 * 5686 * Its important to realize that some subprogs might not be called from some 5687 * main prog and any of its called/used subprogs. Those will keep their 5688 * subprog->sub_insn_off as zero at all times and won't be appended to current 5689 * main prog and won't be relocated within the context of current main prog. 5690 * They might still be used from other main progs later. 5691 * 5692 * Visually this process can be shown as below. Suppose we have two main 5693 * programs mainA and mainB and BPF object contains three subprogs: subA, 5694 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 5695 * subC both call subB: 5696 * 5697 * +--------+ +-------+ 5698 * | v v | 5699 * +--+---+ +--+-+-+ +---+--+ 5700 * | subA | | subB | | subC | 5701 * +--+---+ +------+ +---+--+ 5702 * ^ ^ 5703 * | | 5704 * +---+-------+ +------+----+ 5705 * | mainA | | mainB | 5706 * +-----------+ +-----------+ 5707 * 5708 * We'll start relocating mainA, will find subA, append it and start 5709 * processing sub A recursively: 5710 * 5711 * +-----------+------+ 5712 * | mainA | subA | 5713 * +-----------+------+ 5714 * 5715 * At this point we notice that subB is used from subA, so we append it and 5716 * relocate (there are no further subcalls from subB): 5717 * 5718 * +-----------+------+------+ 5719 * | mainA | subA | subB | 5720 * +-----------+------+------+ 5721 * 5722 * At this point, we relocate subA calls, then go one level up and finish with 5723 * relocatin mainA calls. mainA is done. 5724 * 5725 * For mainB process is similar but results in different order. We start with 5726 * mainB and skip subA and subB, as mainB never calls them (at least 5727 * directly), but we see subC is needed, so we append and start processing it: 5728 * 5729 * +-----------+------+ 5730 * | mainB | subC | 5731 * +-----------+------+ 5732 * Now we see subC needs subB, so we go back to it, append and relocate it: 5733 * 5734 * +-----------+------+------+ 5735 * | mainB | subC | subB | 5736 * +-----------+------+------+ 5737 * 5738 * At this point we unwind recursion, relocate calls in subC, then in mainB. 5739 */ 5740 static int 5741 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 5742 { 5743 struct bpf_program *subprog; 5744 int i, err; 5745 5746 /* mark all subprogs as not relocated (yet) within the context of 5747 * current main program 5748 */ 5749 for (i = 0; i < obj->nr_programs; i++) { 5750 subprog = &obj->programs[i]; 5751 if (!prog_is_subprog(obj, subprog)) 5752 continue; 5753 5754 subprog->sub_insn_off = 0; 5755 } 5756 5757 err = bpf_object__reloc_code(obj, prog, prog); 5758 if (err) 5759 return err; 5760 5761 5762 return 0; 5763 } 5764 5765 static void 5766 bpf_object__free_relocs(struct bpf_object *obj) 5767 { 5768 struct bpf_program *prog; 5769 int i; 5770 5771 /* free up relocation descriptors */ 5772 for (i = 0; i < obj->nr_programs; i++) { 5773 prog = &obj->programs[i]; 5774 zfree(&prog->reloc_desc); 5775 prog->nr_reloc = 0; 5776 } 5777 } 5778 5779 static int 5780 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 5781 { 5782 struct bpf_program *prog; 5783 size_t i, j; 5784 int err; 5785 5786 if (obj->btf_ext) { 5787 err = bpf_object__relocate_core(obj, targ_btf_path); 5788 if (err) { 5789 pr_warn("failed to perform CO-RE relocations: %d\n", 5790 err); 5791 return err; 5792 } 5793 } 5794 5795 /* Before relocating calls pre-process relocations and mark 5796 * few ld_imm64 instructions that points to subprogs. 5797 * Otherwise bpf_object__reloc_code() later would have to consider 5798 * all ld_imm64 insns as relocation candidates. That would 5799 * reduce relocation speed, since amount of find_prog_insn_relo() 5800 * would increase and most of them will fail to find a relo. 5801 */ 5802 for (i = 0; i < obj->nr_programs; i++) { 5803 prog = &obj->programs[i]; 5804 for (j = 0; j < prog->nr_reloc; j++) { 5805 struct reloc_desc *relo = &prog->reloc_desc[j]; 5806 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5807 5808 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 5809 if (relo->type == RELO_SUBPROG_ADDR) 5810 insn[0].src_reg = BPF_PSEUDO_FUNC; 5811 } 5812 } 5813 5814 /* relocate subprogram calls and append used subprograms to main 5815 * programs; each copy of subprogram code needs to be relocated 5816 * differently for each main program, because its code location might 5817 * have changed. 5818 * Append subprog relos to main programs to allow data relos to be 5819 * processed after text is completely relocated. 5820 */ 5821 for (i = 0; i < obj->nr_programs; i++) { 5822 prog = &obj->programs[i]; 5823 /* sub-program's sub-calls are relocated within the context of 5824 * its main program only 5825 */ 5826 if (prog_is_subprog(obj, prog)) 5827 continue; 5828 5829 err = bpf_object__relocate_calls(obj, prog); 5830 if (err) { 5831 pr_warn("prog '%s': failed to relocate calls: %d\n", 5832 prog->name, err); 5833 return err; 5834 } 5835 } 5836 /* Process data relos for main programs */ 5837 for (i = 0; i < obj->nr_programs; i++) { 5838 prog = &obj->programs[i]; 5839 if (prog_is_subprog(obj, prog)) 5840 continue; 5841 err = bpf_object__relocate_data(obj, prog); 5842 if (err) { 5843 pr_warn("prog '%s': failed to relocate data references: %d\n", 5844 prog->name, err); 5845 return err; 5846 } 5847 } 5848 if (!obj->gen_loader) 5849 bpf_object__free_relocs(obj); 5850 return 0; 5851 } 5852 5853 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 5854 GElf_Shdr *shdr, Elf_Data *data); 5855 5856 static int bpf_object__collect_map_relos(struct bpf_object *obj, 5857 GElf_Shdr *shdr, Elf_Data *data) 5858 { 5859 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 5860 int i, j, nrels, new_sz; 5861 const struct btf_var_secinfo *vi = NULL; 5862 const struct btf_type *sec, *var, *def; 5863 struct bpf_map *map = NULL, *targ_map; 5864 const struct btf_member *member; 5865 const char *name, *mname; 5866 Elf_Data *symbols; 5867 unsigned int moff; 5868 GElf_Sym sym; 5869 GElf_Rel rel; 5870 void *tmp; 5871 5872 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 5873 return -EINVAL; 5874 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 5875 if (!sec) 5876 return -EINVAL; 5877 5878 symbols = obj->efile.symbols; 5879 nrels = shdr->sh_size / shdr->sh_entsize; 5880 for (i = 0; i < nrels; i++) { 5881 if (!gelf_getrel(data, i, &rel)) { 5882 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 5883 return -LIBBPF_ERRNO__FORMAT; 5884 } 5885 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 5886 pr_warn(".maps relo #%d: symbol %zx not found\n", 5887 i, (size_t)GELF_R_SYM(rel.r_info)); 5888 return -LIBBPF_ERRNO__FORMAT; 5889 } 5890 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 5891 if (sym.st_shndx != obj->efile.btf_maps_shndx) { 5892 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 5893 i, name); 5894 return -LIBBPF_ERRNO__RELOC; 5895 } 5896 5897 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n", 5898 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value, 5899 (size_t)rel.r_offset, sym.st_name, name); 5900 5901 for (j = 0; j < obj->nr_maps; j++) { 5902 map = &obj->maps[j]; 5903 if (map->sec_idx != obj->efile.btf_maps_shndx) 5904 continue; 5905 5906 vi = btf_var_secinfos(sec) + map->btf_var_idx; 5907 if (vi->offset <= rel.r_offset && 5908 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size) 5909 break; 5910 } 5911 if (j == obj->nr_maps) { 5912 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n", 5913 i, name, (size_t)rel.r_offset); 5914 return -EINVAL; 5915 } 5916 5917 if (!bpf_map_type__is_map_in_map(map->def.type)) 5918 return -EINVAL; 5919 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 5920 map->def.key_size != sizeof(int)) { 5921 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 5922 i, map->name, sizeof(int)); 5923 return -EINVAL; 5924 } 5925 5926 targ_map = bpf_object__find_map_by_name(obj, name); 5927 if (!targ_map) 5928 return -ESRCH; 5929 5930 var = btf__type_by_id(obj->btf, vi->type); 5931 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 5932 if (btf_vlen(def) == 0) 5933 return -EINVAL; 5934 member = btf_members(def) + btf_vlen(def) - 1; 5935 mname = btf__name_by_offset(obj->btf, member->name_off); 5936 if (strcmp(mname, "values")) 5937 return -EINVAL; 5938 5939 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 5940 if (rel.r_offset - vi->offset < moff) 5941 return -EINVAL; 5942 5943 moff = rel.r_offset - vi->offset - moff; 5944 /* here we use BPF pointer size, which is always 64 bit, as we 5945 * are parsing ELF that was built for BPF target 5946 */ 5947 if (moff % bpf_ptr_sz) 5948 return -EINVAL; 5949 moff /= bpf_ptr_sz; 5950 if (moff >= map->init_slots_sz) { 5951 new_sz = moff + 1; 5952 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 5953 if (!tmp) 5954 return -ENOMEM; 5955 map->init_slots = tmp; 5956 memset(map->init_slots + map->init_slots_sz, 0, 5957 (new_sz - map->init_slots_sz) * host_ptr_sz); 5958 map->init_slots_sz = new_sz; 5959 } 5960 map->init_slots[moff] = targ_map; 5961 5962 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n", 5963 i, map->name, moff, name); 5964 } 5965 5966 return 0; 5967 } 5968 5969 static int cmp_relocs(const void *_a, const void *_b) 5970 { 5971 const struct reloc_desc *a = _a; 5972 const struct reloc_desc *b = _b; 5973 5974 if (a->insn_idx != b->insn_idx) 5975 return a->insn_idx < b->insn_idx ? -1 : 1; 5976 5977 /* no two relocations should have the same insn_idx, but ... */ 5978 if (a->type != b->type) 5979 return a->type < b->type ? -1 : 1; 5980 5981 return 0; 5982 } 5983 5984 static int bpf_object__collect_relos(struct bpf_object *obj) 5985 { 5986 int i, err; 5987 5988 for (i = 0; i < obj->efile.nr_reloc_sects; i++) { 5989 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; 5990 Elf_Data *data = obj->efile.reloc_sects[i].data; 5991 int idx = shdr->sh_info; 5992 5993 if (shdr->sh_type != SHT_REL) { 5994 pr_warn("internal error at %d\n", __LINE__); 5995 return -LIBBPF_ERRNO__INTERNAL; 5996 } 5997 5998 if (idx == obj->efile.st_ops_shndx) 5999 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 6000 else if (idx == obj->efile.btf_maps_shndx) 6001 err = bpf_object__collect_map_relos(obj, shdr, data); 6002 else 6003 err = bpf_object__collect_prog_relos(obj, shdr, data); 6004 if (err) 6005 return err; 6006 } 6007 6008 for (i = 0; i < obj->nr_programs; i++) { 6009 struct bpf_program *p = &obj->programs[i]; 6010 6011 if (!p->nr_reloc) 6012 continue; 6013 6014 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 6015 } 6016 return 0; 6017 } 6018 6019 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 6020 { 6021 if (BPF_CLASS(insn->code) == BPF_JMP && 6022 BPF_OP(insn->code) == BPF_CALL && 6023 BPF_SRC(insn->code) == BPF_K && 6024 insn->src_reg == 0 && 6025 insn->dst_reg == 0) { 6026 *func_id = insn->imm; 6027 return true; 6028 } 6029 return false; 6030 } 6031 6032 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 6033 { 6034 struct bpf_insn *insn = prog->insns; 6035 enum bpf_func_id func_id; 6036 int i; 6037 6038 if (obj->gen_loader) 6039 return 0; 6040 6041 for (i = 0; i < prog->insns_cnt; i++, insn++) { 6042 if (!insn_is_helper_call(insn, &func_id)) 6043 continue; 6044 6045 /* on kernels that don't yet support 6046 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 6047 * to bpf_probe_read() which works well for old kernels 6048 */ 6049 switch (func_id) { 6050 case BPF_FUNC_probe_read_kernel: 6051 case BPF_FUNC_probe_read_user: 6052 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6053 insn->imm = BPF_FUNC_probe_read; 6054 break; 6055 case BPF_FUNC_probe_read_kernel_str: 6056 case BPF_FUNC_probe_read_user_str: 6057 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6058 insn->imm = BPF_FUNC_probe_read_str; 6059 break; 6060 default: 6061 break; 6062 } 6063 } 6064 return 0; 6065 } 6066 6067 static int 6068 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 6069 char *license, __u32 kern_version, int *pfd) 6070 { 6071 struct bpf_prog_load_params load_attr = {}; 6072 char *cp, errmsg[STRERR_BUFSIZE]; 6073 size_t log_buf_size = 0; 6074 char *log_buf = NULL; 6075 int btf_fd, ret; 6076 6077 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 6078 /* 6079 * The program type must be set. Most likely we couldn't find a proper 6080 * section definition at load time, and thus we didn't infer the type. 6081 */ 6082 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 6083 prog->name, prog->sec_name); 6084 return -EINVAL; 6085 } 6086 6087 if (!insns || !insns_cnt) 6088 return -EINVAL; 6089 6090 load_attr.prog_type = prog->type; 6091 /* old kernels might not support specifying expected_attach_type */ 6092 if (!kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE) && prog->sec_def && 6093 prog->sec_def->is_exp_attach_type_optional) 6094 load_attr.expected_attach_type = 0; 6095 else 6096 load_attr.expected_attach_type = prog->expected_attach_type; 6097 if (kernel_supports(prog->obj, FEAT_PROG_NAME)) 6098 load_attr.name = prog->name; 6099 load_attr.insns = insns; 6100 load_attr.insn_cnt = insns_cnt; 6101 load_attr.license = license; 6102 load_attr.attach_btf_id = prog->attach_btf_id; 6103 if (prog->attach_prog_fd) 6104 load_attr.attach_prog_fd = prog->attach_prog_fd; 6105 else 6106 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 6107 load_attr.attach_btf_id = prog->attach_btf_id; 6108 load_attr.kern_version = kern_version; 6109 load_attr.prog_ifindex = prog->prog_ifindex; 6110 6111 /* specify func_info/line_info only if kernel supports them */ 6112 btf_fd = bpf_object__btf_fd(prog->obj); 6113 if (btf_fd >= 0 && kernel_supports(prog->obj, FEAT_BTF_FUNC)) { 6114 load_attr.prog_btf_fd = btf_fd; 6115 load_attr.func_info = prog->func_info; 6116 load_attr.func_info_rec_size = prog->func_info_rec_size; 6117 load_attr.func_info_cnt = prog->func_info_cnt; 6118 load_attr.line_info = prog->line_info; 6119 load_attr.line_info_rec_size = prog->line_info_rec_size; 6120 load_attr.line_info_cnt = prog->line_info_cnt; 6121 } 6122 load_attr.log_level = prog->log_level; 6123 load_attr.prog_flags = prog->prog_flags; 6124 6125 if (prog->obj->gen_loader) { 6126 bpf_gen__prog_load(prog->obj->gen_loader, &load_attr, 6127 prog - prog->obj->programs); 6128 *pfd = -1; 6129 return 0; 6130 } 6131 retry_load: 6132 if (log_buf_size) { 6133 log_buf = malloc(log_buf_size); 6134 if (!log_buf) 6135 return -ENOMEM; 6136 6137 *log_buf = 0; 6138 } 6139 6140 load_attr.log_buf = log_buf; 6141 load_attr.log_buf_sz = log_buf_size; 6142 ret = libbpf__bpf_prog_load(&load_attr); 6143 6144 if (ret >= 0) { 6145 if (log_buf && load_attr.log_level) 6146 pr_debug("verifier log:\n%s", log_buf); 6147 6148 if (prog->obj->rodata_map_idx >= 0 && 6149 kernel_supports(prog->obj, FEAT_PROG_BIND_MAP)) { 6150 struct bpf_map *rodata_map = 6151 &prog->obj->maps[prog->obj->rodata_map_idx]; 6152 6153 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) { 6154 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6155 pr_warn("prog '%s': failed to bind .rodata map: %s\n", 6156 prog->name, cp); 6157 /* Don't fail hard if can't bind rodata. */ 6158 } 6159 } 6160 6161 *pfd = ret; 6162 ret = 0; 6163 goto out; 6164 } 6165 6166 if (!log_buf || errno == ENOSPC) { 6167 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, 6168 log_buf_size << 1); 6169 6170 free(log_buf); 6171 goto retry_load; 6172 } 6173 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD; 6174 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6175 pr_warn("load bpf program failed: %s\n", cp); 6176 pr_perm_msg(ret); 6177 6178 if (log_buf && log_buf[0] != '\0') { 6179 ret = -LIBBPF_ERRNO__VERIFY; 6180 pr_warn("-- BEGIN DUMP LOG ---\n"); 6181 pr_warn("\n%s\n", log_buf); 6182 pr_warn("-- END LOG --\n"); 6183 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) { 6184 pr_warn("Program too large (%zu insns), at most %d insns\n", 6185 load_attr.insn_cnt, BPF_MAXINSNS); 6186 ret = -LIBBPF_ERRNO__PROG2BIG; 6187 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 6188 /* Wrong program type? */ 6189 int fd; 6190 6191 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 6192 load_attr.expected_attach_type = 0; 6193 load_attr.log_buf = NULL; 6194 load_attr.log_buf_sz = 0; 6195 fd = libbpf__bpf_prog_load(&load_attr); 6196 if (fd >= 0) { 6197 close(fd); 6198 ret = -LIBBPF_ERRNO__PROGTYPE; 6199 goto out; 6200 } 6201 } 6202 6203 out: 6204 free(log_buf); 6205 return ret; 6206 } 6207 6208 static int bpf_program__record_externs(struct bpf_program *prog) 6209 { 6210 struct bpf_object *obj = prog->obj; 6211 int i; 6212 6213 for (i = 0; i < prog->nr_reloc; i++) { 6214 struct reloc_desc *relo = &prog->reloc_desc[i]; 6215 struct extern_desc *ext = &obj->externs[relo->sym_off]; 6216 6217 switch (relo->type) { 6218 case RELO_EXTERN_VAR: 6219 if (ext->type != EXT_KSYM) 6220 continue; 6221 if (!ext->ksym.type_id) { 6222 pr_warn("typeless ksym %s is not supported yet\n", 6223 ext->name); 6224 return -ENOTSUP; 6225 } 6226 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR, 6227 relo->insn_idx); 6228 break; 6229 case RELO_EXTERN_FUNC: 6230 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC, 6231 relo->insn_idx); 6232 break; 6233 default: 6234 continue; 6235 } 6236 } 6237 return 0; 6238 } 6239 6240 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id); 6241 6242 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) 6243 { 6244 int err = 0, fd, i; 6245 6246 if (prog->obj->loaded) { 6247 pr_warn("prog '%s': can't load after object was loaded\n", prog->name); 6248 return libbpf_err(-EINVAL); 6249 } 6250 6251 if ((prog->type == BPF_PROG_TYPE_TRACING || 6252 prog->type == BPF_PROG_TYPE_LSM || 6253 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { 6254 int btf_obj_fd = 0, btf_type_id = 0; 6255 6256 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id); 6257 if (err) 6258 return libbpf_err(err); 6259 6260 prog->attach_btf_obj_fd = btf_obj_fd; 6261 prog->attach_btf_id = btf_type_id; 6262 } 6263 6264 if (prog->instances.nr < 0 || !prog->instances.fds) { 6265 if (prog->preprocessor) { 6266 pr_warn("Internal error: can't load program '%s'\n", 6267 prog->name); 6268 return libbpf_err(-LIBBPF_ERRNO__INTERNAL); 6269 } 6270 6271 prog->instances.fds = malloc(sizeof(int)); 6272 if (!prog->instances.fds) { 6273 pr_warn("Not enough memory for BPF fds\n"); 6274 return libbpf_err(-ENOMEM); 6275 } 6276 prog->instances.nr = 1; 6277 prog->instances.fds[0] = -1; 6278 } 6279 6280 if (!prog->preprocessor) { 6281 if (prog->instances.nr != 1) { 6282 pr_warn("prog '%s': inconsistent nr(%d) != 1\n", 6283 prog->name, prog->instances.nr); 6284 } 6285 if (prog->obj->gen_loader) 6286 bpf_program__record_externs(prog); 6287 err = load_program(prog, prog->insns, prog->insns_cnt, 6288 license, kern_ver, &fd); 6289 if (!err) 6290 prog->instances.fds[0] = fd; 6291 goto out; 6292 } 6293 6294 for (i = 0; i < prog->instances.nr; i++) { 6295 struct bpf_prog_prep_result result; 6296 bpf_program_prep_t preprocessor = prog->preprocessor; 6297 6298 memset(&result, 0, sizeof(result)); 6299 err = preprocessor(prog, i, prog->insns, 6300 prog->insns_cnt, &result); 6301 if (err) { 6302 pr_warn("Preprocessing the %dth instance of program '%s' failed\n", 6303 i, prog->name); 6304 goto out; 6305 } 6306 6307 if (!result.new_insn_ptr || !result.new_insn_cnt) { 6308 pr_debug("Skip loading the %dth instance of program '%s'\n", 6309 i, prog->name); 6310 prog->instances.fds[i] = -1; 6311 if (result.pfd) 6312 *result.pfd = -1; 6313 continue; 6314 } 6315 6316 err = load_program(prog, result.new_insn_ptr, 6317 result.new_insn_cnt, license, kern_ver, &fd); 6318 if (err) { 6319 pr_warn("Loading the %dth instance of program '%s' failed\n", 6320 i, prog->name); 6321 goto out; 6322 } 6323 6324 if (result.pfd) 6325 *result.pfd = fd; 6326 prog->instances.fds[i] = fd; 6327 } 6328 out: 6329 if (err) 6330 pr_warn("failed to load program '%s'\n", prog->name); 6331 zfree(&prog->insns); 6332 prog->insns_cnt = 0; 6333 return libbpf_err(err); 6334 } 6335 6336 static int 6337 bpf_object__load_progs(struct bpf_object *obj, int log_level) 6338 { 6339 struct bpf_program *prog; 6340 size_t i; 6341 int err; 6342 6343 for (i = 0; i < obj->nr_programs; i++) { 6344 prog = &obj->programs[i]; 6345 err = bpf_object__sanitize_prog(obj, prog); 6346 if (err) 6347 return err; 6348 } 6349 6350 for (i = 0; i < obj->nr_programs; i++) { 6351 prog = &obj->programs[i]; 6352 if (prog_is_subprog(obj, prog)) 6353 continue; 6354 if (!prog->load) { 6355 pr_debug("prog '%s': skipped loading\n", prog->name); 6356 continue; 6357 } 6358 prog->log_level |= log_level; 6359 err = bpf_program__load(prog, obj->license, obj->kern_version); 6360 if (err) 6361 return err; 6362 } 6363 if (obj->gen_loader) 6364 bpf_object__free_relocs(obj); 6365 return 0; 6366 } 6367 6368 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 6369 6370 static struct bpf_object * 6371 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, 6372 const struct bpf_object_open_opts *opts) 6373 { 6374 const char *obj_name, *kconfig, *btf_tmp_path; 6375 struct bpf_program *prog; 6376 struct bpf_object *obj; 6377 char tmp_name[64]; 6378 int err; 6379 6380 if (elf_version(EV_CURRENT) == EV_NONE) { 6381 pr_warn("failed to init libelf for %s\n", 6382 path ? : "(mem buf)"); 6383 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 6384 } 6385 6386 if (!OPTS_VALID(opts, bpf_object_open_opts)) 6387 return ERR_PTR(-EINVAL); 6388 6389 obj_name = OPTS_GET(opts, object_name, NULL); 6390 if (obj_buf) { 6391 if (!obj_name) { 6392 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 6393 (unsigned long)obj_buf, 6394 (unsigned long)obj_buf_sz); 6395 obj_name = tmp_name; 6396 } 6397 path = obj_name; 6398 pr_debug("loading object '%s' from buffer\n", obj_name); 6399 } 6400 6401 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 6402 if (IS_ERR(obj)) 6403 return obj; 6404 6405 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 6406 if (btf_tmp_path) { 6407 if (strlen(btf_tmp_path) >= PATH_MAX) { 6408 err = -ENAMETOOLONG; 6409 goto out; 6410 } 6411 obj->btf_custom_path = strdup(btf_tmp_path); 6412 if (!obj->btf_custom_path) { 6413 err = -ENOMEM; 6414 goto out; 6415 } 6416 } 6417 6418 kconfig = OPTS_GET(opts, kconfig, NULL); 6419 if (kconfig) { 6420 obj->kconfig = strdup(kconfig); 6421 if (!obj->kconfig) { 6422 err = -ENOMEM; 6423 goto out; 6424 } 6425 } 6426 6427 err = bpf_object__elf_init(obj); 6428 err = err ? : bpf_object__check_endianness(obj); 6429 err = err ? : bpf_object__elf_collect(obj); 6430 err = err ? : bpf_object__collect_externs(obj); 6431 err = err ? : bpf_object__finalize_btf(obj); 6432 err = err ? : bpf_object__init_maps(obj, opts); 6433 err = err ? : bpf_object__collect_relos(obj); 6434 if (err) 6435 goto out; 6436 bpf_object__elf_finish(obj); 6437 6438 bpf_object__for_each_program(prog, obj) { 6439 prog->sec_def = find_sec_def(prog->sec_name); 6440 if (!prog->sec_def) { 6441 /* couldn't guess, but user might manually specify */ 6442 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 6443 prog->name, prog->sec_name); 6444 continue; 6445 } 6446 6447 if (prog->sec_def->is_sleepable) 6448 prog->prog_flags |= BPF_F_SLEEPABLE; 6449 bpf_program__set_type(prog, prog->sec_def->prog_type); 6450 bpf_program__set_expected_attach_type(prog, 6451 prog->sec_def->expected_attach_type); 6452 6453 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || 6454 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) 6455 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); 6456 } 6457 6458 return obj; 6459 out: 6460 bpf_object__close(obj); 6461 return ERR_PTR(err); 6462 } 6463 6464 static struct bpf_object * 6465 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) 6466 { 6467 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 6468 .relaxed_maps = flags & MAPS_RELAX_COMPAT, 6469 ); 6470 6471 /* param validation */ 6472 if (!attr->file) 6473 return NULL; 6474 6475 pr_debug("loading %s\n", attr->file); 6476 return __bpf_object__open(attr->file, NULL, 0, &opts); 6477 } 6478 6479 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 6480 { 6481 return libbpf_ptr(__bpf_object__open_xattr(attr, 0)); 6482 } 6483 6484 struct bpf_object *bpf_object__open(const char *path) 6485 { 6486 struct bpf_object_open_attr attr = { 6487 .file = path, 6488 .prog_type = BPF_PROG_TYPE_UNSPEC, 6489 }; 6490 6491 return libbpf_ptr(__bpf_object__open_xattr(&attr, 0)); 6492 } 6493 6494 struct bpf_object * 6495 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 6496 { 6497 if (!path) 6498 return libbpf_err_ptr(-EINVAL); 6499 6500 pr_debug("loading %s\n", path); 6501 6502 return libbpf_ptr(__bpf_object__open(path, NULL, 0, opts)); 6503 } 6504 6505 struct bpf_object * 6506 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 6507 const struct bpf_object_open_opts *opts) 6508 { 6509 if (!obj_buf || obj_buf_sz == 0) 6510 return libbpf_err_ptr(-EINVAL); 6511 6512 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, opts)); 6513 } 6514 6515 struct bpf_object * 6516 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, 6517 const char *name) 6518 { 6519 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 6520 .object_name = name, 6521 /* wrong default, but backwards-compatible */ 6522 .relaxed_maps = true, 6523 ); 6524 6525 /* returning NULL is wrong, but backwards-compatible */ 6526 if (!obj_buf || obj_buf_sz == 0) 6527 return errno = EINVAL, NULL; 6528 6529 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, &opts)); 6530 } 6531 6532 int bpf_object__unload(struct bpf_object *obj) 6533 { 6534 size_t i; 6535 6536 if (!obj) 6537 return libbpf_err(-EINVAL); 6538 6539 for (i = 0; i < obj->nr_maps; i++) { 6540 zclose(obj->maps[i].fd); 6541 if (obj->maps[i].st_ops) 6542 zfree(&obj->maps[i].st_ops->kern_vdata); 6543 } 6544 6545 for (i = 0; i < obj->nr_programs; i++) 6546 bpf_program__unload(&obj->programs[i]); 6547 6548 return 0; 6549 } 6550 6551 static int bpf_object__sanitize_maps(struct bpf_object *obj) 6552 { 6553 struct bpf_map *m; 6554 6555 bpf_object__for_each_map(m, obj) { 6556 if (!bpf_map__is_internal(m)) 6557 continue; 6558 if (!kernel_supports(obj, FEAT_GLOBAL_DATA)) { 6559 pr_warn("kernel doesn't support global data\n"); 6560 return -ENOTSUP; 6561 } 6562 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 6563 m->def.map_flags ^= BPF_F_MMAPABLE; 6564 } 6565 6566 return 0; 6567 } 6568 6569 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 6570 { 6571 char sym_type, sym_name[500]; 6572 unsigned long long sym_addr; 6573 const struct btf_type *t; 6574 struct extern_desc *ext; 6575 int ret, err = 0; 6576 FILE *f; 6577 6578 f = fopen("/proc/kallsyms", "r"); 6579 if (!f) { 6580 err = -errno; 6581 pr_warn("failed to open /proc/kallsyms: %d\n", err); 6582 return err; 6583 } 6584 6585 while (true) { 6586 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 6587 &sym_addr, &sym_type, sym_name); 6588 if (ret == EOF && feof(f)) 6589 break; 6590 if (ret != 3) { 6591 pr_warn("failed to read kallsyms entry: %d\n", ret); 6592 err = -EINVAL; 6593 goto out; 6594 } 6595 6596 ext = find_extern_by_name(obj, sym_name); 6597 if (!ext || ext->type != EXT_KSYM) 6598 continue; 6599 6600 t = btf__type_by_id(obj->btf, ext->btf_id); 6601 if (!btf_is_var(t)) 6602 continue; 6603 6604 if (ext->is_set && ext->ksym.addr != sym_addr) { 6605 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", 6606 sym_name, ext->ksym.addr, sym_addr); 6607 err = -EINVAL; 6608 goto out; 6609 } 6610 if (!ext->is_set) { 6611 ext->is_set = true; 6612 ext->ksym.addr = sym_addr; 6613 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); 6614 } 6615 } 6616 6617 out: 6618 fclose(f); 6619 return err; 6620 } 6621 6622 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 6623 __u16 kind, struct btf **res_btf, 6624 int *res_btf_fd) 6625 { 6626 int i, id, btf_fd, err; 6627 struct btf *btf; 6628 6629 btf = obj->btf_vmlinux; 6630 btf_fd = 0; 6631 id = btf__find_by_name_kind(btf, ksym_name, kind); 6632 6633 if (id == -ENOENT) { 6634 err = load_module_btfs(obj); 6635 if (err) 6636 return err; 6637 6638 for (i = 0; i < obj->btf_module_cnt; i++) { 6639 btf = obj->btf_modules[i].btf; 6640 /* we assume module BTF FD is always >0 */ 6641 btf_fd = obj->btf_modules[i].fd; 6642 id = btf__find_by_name_kind(btf, ksym_name, kind); 6643 if (id != -ENOENT) 6644 break; 6645 } 6646 } 6647 if (id <= 0) 6648 return -ESRCH; 6649 6650 *res_btf = btf; 6651 *res_btf_fd = btf_fd; 6652 return id; 6653 } 6654 6655 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 6656 struct extern_desc *ext) 6657 { 6658 const struct btf_type *targ_var, *targ_type; 6659 __u32 targ_type_id, local_type_id; 6660 const char *targ_var_name; 6661 int id, btf_fd = 0, err; 6662 struct btf *btf = NULL; 6663 6664 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd); 6665 if (id == -ESRCH && ext->is_weak) { 6666 return 0; 6667 } else if (id < 0) { 6668 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 6669 ext->name); 6670 return id; 6671 } 6672 6673 /* find local type_id */ 6674 local_type_id = ext->ksym.type_id; 6675 6676 /* find target type_id */ 6677 targ_var = btf__type_by_id(btf, id); 6678 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 6679 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 6680 6681 err = bpf_core_types_are_compat(obj->btf, local_type_id, 6682 btf, targ_type_id); 6683 if (err <= 0) { 6684 const struct btf_type *local_type; 6685 const char *targ_name, *local_name; 6686 6687 local_type = btf__type_by_id(obj->btf, local_type_id); 6688 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 6689 targ_name = btf__name_by_offset(btf, targ_type->name_off); 6690 6691 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 6692 ext->name, local_type_id, 6693 btf_kind_str(local_type), local_name, targ_type_id, 6694 btf_kind_str(targ_type), targ_name); 6695 return -EINVAL; 6696 } 6697 6698 ext->is_set = true; 6699 ext->ksym.kernel_btf_obj_fd = btf_fd; 6700 ext->ksym.kernel_btf_id = id; 6701 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 6702 ext->name, id, btf_kind_str(targ_var), targ_var_name); 6703 6704 return 0; 6705 } 6706 6707 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 6708 struct extern_desc *ext) 6709 { 6710 int local_func_proto_id, kfunc_proto_id, kfunc_id; 6711 const struct btf_type *kern_func; 6712 struct btf *kern_btf = NULL; 6713 int ret, kern_btf_fd = 0; 6714 6715 local_func_proto_id = ext->ksym.type_id; 6716 6717 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, 6718 &kern_btf, &kern_btf_fd); 6719 if (kfunc_id < 0) { 6720 pr_warn("extern (func ksym) '%s': not found in kernel BTF\n", 6721 ext->name); 6722 return kfunc_id; 6723 } 6724 6725 if (kern_btf != obj->btf_vmlinux) { 6726 pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n", 6727 ext->name); 6728 return -ENOTSUP; 6729 } 6730 6731 kern_func = btf__type_by_id(kern_btf, kfunc_id); 6732 kfunc_proto_id = kern_func->type; 6733 6734 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 6735 kern_btf, kfunc_proto_id); 6736 if (ret <= 0) { 6737 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n", 6738 ext->name, local_func_proto_id, kfunc_proto_id); 6739 return -EINVAL; 6740 } 6741 6742 ext->is_set = true; 6743 ext->ksym.kernel_btf_obj_fd = kern_btf_fd; 6744 ext->ksym.kernel_btf_id = kfunc_id; 6745 pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n", 6746 ext->name, kfunc_id); 6747 6748 return 0; 6749 } 6750 6751 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 6752 { 6753 const struct btf_type *t; 6754 struct extern_desc *ext; 6755 int i, err; 6756 6757 for (i = 0; i < obj->nr_extern; i++) { 6758 ext = &obj->externs[i]; 6759 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 6760 continue; 6761 6762 if (obj->gen_loader) { 6763 ext->is_set = true; 6764 ext->ksym.kernel_btf_obj_fd = 0; 6765 ext->ksym.kernel_btf_id = 0; 6766 continue; 6767 } 6768 t = btf__type_by_id(obj->btf, ext->btf_id); 6769 if (btf_is_var(t)) 6770 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 6771 else 6772 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 6773 if (err) 6774 return err; 6775 } 6776 return 0; 6777 } 6778 6779 static int bpf_object__resolve_externs(struct bpf_object *obj, 6780 const char *extra_kconfig) 6781 { 6782 bool need_config = false, need_kallsyms = false; 6783 bool need_vmlinux_btf = false; 6784 struct extern_desc *ext; 6785 void *kcfg_data = NULL; 6786 int err, i; 6787 6788 if (obj->nr_extern == 0) 6789 return 0; 6790 6791 if (obj->kconfig_map_idx >= 0) 6792 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 6793 6794 for (i = 0; i < obj->nr_extern; i++) { 6795 ext = &obj->externs[i]; 6796 6797 if (ext->type == EXT_KCFG && 6798 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 6799 void *ext_val = kcfg_data + ext->kcfg.data_off; 6800 __u32 kver = get_kernel_version(); 6801 6802 if (!kver) { 6803 pr_warn("failed to get kernel version\n"); 6804 return -EINVAL; 6805 } 6806 err = set_kcfg_value_num(ext, ext_val, kver); 6807 if (err) 6808 return err; 6809 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); 6810 } else if (ext->type == EXT_KCFG && 6811 strncmp(ext->name, "CONFIG_", 7) == 0) { 6812 need_config = true; 6813 } else if (ext->type == EXT_KSYM) { 6814 if (ext->ksym.type_id) 6815 need_vmlinux_btf = true; 6816 else 6817 need_kallsyms = true; 6818 } else { 6819 pr_warn("unrecognized extern '%s'\n", ext->name); 6820 return -EINVAL; 6821 } 6822 } 6823 if (need_config && extra_kconfig) { 6824 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 6825 if (err) 6826 return -EINVAL; 6827 need_config = false; 6828 for (i = 0; i < obj->nr_extern; i++) { 6829 ext = &obj->externs[i]; 6830 if (ext->type == EXT_KCFG && !ext->is_set) { 6831 need_config = true; 6832 break; 6833 } 6834 } 6835 } 6836 if (need_config) { 6837 err = bpf_object__read_kconfig_file(obj, kcfg_data); 6838 if (err) 6839 return -EINVAL; 6840 } 6841 if (need_kallsyms) { 6842 err = bpf_object__read_kallsyms_file(obj); 6843 if (err) 6844 return -EINVAL; 6845 } 6846 if (need_vmlinux_btf) { 6847 err = bpf_object__resolve_ksyms_btf_id(obj); 6848 if (err) 6849 return -EINVAL; 6850 } 6851 for (i = 0; i < obj->nr_extern; i++) { 6852 ext = &obj->externs[i]; 6853 6854 if (!ext->is_set && !ext->is_weak) { 6855 pr_warn("extern %s (strong) not resolved\n", ext->name); 6856 return -ESRCH; 6857 } else if (!ext->is_set) { 6858 pr_debug("extern %s (weak) not resolved, defaulting to zero\n", 6859 ext->name); 6860 } 6861 } 6862 6863 return 0; 6864 } 6865 6866 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 6867 { 6868 struct bpf_object *obj; 6869 int err, i; 6870 6871 if (!attr) 6872 return libbpf_err(-EINVAL); 6873 obj = attr->obj; 6874 if (!obj) 6875 return libbpf_err(-EINVAL); 6876 6877 if (obj->loaded) { 6878 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 6879 return libbpf_err(-EINVAL); 6880 } 6881 6882 if (obj->gen_loader) 6883 bpf_gen__init(obj->gen_loader, attr->log_level); 6884 6885 err = bpf_object__probe_loading(obj); 6886 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 6887 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 6888 err = err ? : bpf_object__sanitize_and_load_btf(obj); 6889 err = err ? : bpf_object__sanitize_maps(obj); 6890 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 6891 err = err ? : bpf_object__create_maps(obj); 6892 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : attr->target_btf_path); 6893 err = err ? : bpf_object__load_progs(obj, attr->log_level); 6894 6895 if (obj->gen_loader) { 6896 /* reset FDs */ 6897 btf__set_fd(obj->btf, -1); 6898 for (i = 0; i < obj->nr_maps; i++) 6899 obj->maps[i].fd = -1; 6900 if (!err) 6901 err = bpf_gen__finish(obj->gen_loader); 6902 } 6903 6904 /* clean up module BTFs */ 6905 for (i = 0; i < obj->btf_module_cnt; i++) { 6906 close(obj->btf_modules[i].fd); 6907 btf__free(obj->btf_modules[i].btf); 6908 free(obj->btf_modules[i].name); 6909 } 6910 free(obj->btf_modules); 6911 6912 /* clean up vmlinux BTF */ 6913 btf__free(obj->btf_vmlinux); 6914 obj->btf_vmlinux = NULL; 6915 6916 obj->loaded = true; /* doesn't matter if successfully or not */ 6917 6918 if (err) 6919 goto out; 6920 6921 return 0; 6922 out: 6923 /* unpin any maps that were auto-pinned during load */ 6924 for (i = 0; i < obj->nr_maps; i++) 6925 if (obj->maps[i].pinned && !obj->maps[i].reused) 6926 bpf_map__unpin(&obj->maps[i], NULL); 6927 6928 bpf_object__unload(obj); 6929 pr_warn("failed to load object '%s'\n", obj->path); 6930 return libbpf_err(err); 6931 } 6932 6933 int bpf_object__load(struct bpf_object *obj) 6934 { 6935 struct bpf_object_load_attr attr = { 6936 .obj = obj, 6937 }; 6938 6939 return bpf_object__load_xattr(&attr); 6940 } 6941 6942 static int make_parent_dir(const char *path) 6943 { 6944 char *cp, errmsg[STRERR_BUFSIZE]; 6945 char *dname, *dir; 6946 int err = 0; 6947 6948 dname = strdup(path); 6949 if (dname == NULL) 6950 return -ENOMEM; 6951 6952 dir = dirname(dname); 6953 if (mkdir(dir, 0700) && errno != EEXIST) 6954 err = -errno; 6955 6956 free(dname); 6957 if (err) { 6958 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 6959 pr_warn("failed to mkdir %s: %s\n", path, cp); 6960 } 6961 return err; 6962 } 6963 6964 static int check_path(const char *path) 6965 { 6966 char *cp, errmsg[STRERR_BUFSIZE]; 6967 struct statfs st_fs; 6968 char *dname, *dir; 6969 int err = 0; 6970 6971 if (path == NULL) 6972 return -EINVAL; 6973 6974 dname = strdup(path); 6975 if (dname == NULL) 6976 return -ENOMEM; 6977 6978 dir = dirname(dname); 6979 if (statfs(dir, &st_fs)) { 6980 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 6981 pr_warn("failed to statfs %s: %s\n", dir, cp); 6982 err = -errno; 6983 } 6984 free(dname); 6985 6986 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 6987 pr_warn("specified path %s is not on BPF FS\n", path); 6988 err = -EINVAL; 6989 } 6990 6991 return err; 6992 } 6993 6994 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 6995 int instance) 6996 { 6997 char *cp, errmsg[STRERR_BUFSIZE]; 6998 int err; 6999 7000 err = make_parent_dir(path); 7001 if (err) 7002 return libbpf_err(err); 7003 7004 err = check_path(path); 7005 if (err) 7006 return libbpf_err(err); 7007 7008 if (prog == NULL) { 7009 pr_warn("invalid program pointer\n"); 7010 return libbpf_err(-EINVAL); 7011 } 7012 7013 if (instance < 0 || instance >= prog->instances.nr) { 7014 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7015 instance, prog->name, prog->instances.nr); 7016 return libbpf_err(-EINVAL); 7017 } 7018 7019 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 7020 err = -errno; 7021 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 7022 pr_warn("failed to pin program: %s\n", cp); 7023 return libbpf_err(err); 7024 } 7025 pr_debug("pinned program '%s'\n", path); 7026 7027 return 0; 7028 } 7029 7030 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 7031 int instance) 7032 { 7033 int err; 7034 7035 err = check_path(path); 7036 if (err) 7037 return libbpf_err(err); 7038 7039 if (prog == NULL) { 7040 pr_warn("invalid program pointer\n"); 7041 return libbpf_err(-EINVAL); 7042 } 7043 7044 if (instance < 0 || instance >= prog->instances.nr) { 7045 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 7046 instance, prog->name, prog->instances.nr); 7047 return libbpf_err(-EINVAL); 7048 } 7049 7050 err = unlink(path); 7051 if (err != 0) 7052 return libbpf_err(-errno); 7053 7054 pr_debug("unpinned program '%s'\n", path); 7055 7056 return 0; 7057 } 7058 7059 int bpf_program__pin(struct bpf_program *prog, const char *path) 7060 { 7061 int i, err; 7062 7063 err = make_parent_dir(path); 7064 if (err) 7065 return libbpf_err(err); 7066 7067 err = check_path(path); 7068 if (err) 7069 return libbpf_err(err); 7070 7071 if (prog == NULL) { 7072 pr_warn("invalid program pointer\n"); 7073 return libbpf_err(-EINVAL); 7074 } 7075 7076 if (prog->instances.nr <= 0) { 7077 pr_warn("no instances of prog %s to pin\n", prog->name); 7078 return libbpf_err(-EINVAL); 7079 } 7080 7081 if (prog->instances.nr == 1) { 7082 /* don't create subdirs when pinning single instance */ 7083 return bpf_program__pin_instance(prog, path, 0); 7084 } 7085 7086 for (i = 0; i < prog->instances.nr; i++) { 7087 char buf[PATH_MAX]; 7088 int len; 7089 7090 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7091 if (len < 0) { 7092 err = -EINVAL; 7093 goto err_unpin; 7094 } else if (len >= PATH_MAX) { 7095 err = -ENAMETOOLONG; 7096 goto err_unpin; 7097 } 7098 7099 err = bpf_program__pin_instance(prog, buf, i); 7100 if (err) 7101 goto err_unpin; 7102 } 7103 7104 return 0; 7105 7106 err_unpin: 7107 for (i = i - 1; i >= 0; i--) { 7108 char buf[PATH_MAX]; 7109 int len; 7110 7111 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7112 if (len < 0) 7113 continue; 7114 else if (len >= PATH_MAX) 7115 continue; 7116 7117 bpf_program__unpin_instance(prog, buf, i); 7118 } 7119 7120 rmdir(path); 7121 7122 return libbpf_err(err); 7123 } 7124 7125 int bpf_program__unpin(struct bpf_program *prog, const char *path) 7126 { 7127 int i, err; 7128 7129 err = check_path(path); 7130 if (err) 7131 return libbpf_err(err); 7132 7133 if (prog == NULL) { 7134 pr_warn("invalid program pointer\n"); 7135 return libbpf_err(-EINVAL); 7136 } 7137 7138 if (prog->instances.nr <= 0) { 7139 pr_warn("no instances of prog %s to pin\n", prog->name); 7140 return libbpf_err(-EINVAL); 7141 } 7142 7143 if (prog->instances.nr == 1) { 7144 /* don't create subdirs when pinning single instance */ 7145 return bpf_program__unpin_instance(prog, path, 0); 7146 } 7147 7148 for (i = 0; i < prog->instances.nr; i++) { 7149 char buf[PATH_MAX]; 7150 int len; 7151 7152 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 7153 if (len < 0) 7154 return libbpf_err(-EINVAL); 7155 else if (len >= PATH_MAX) 7156 return libbpf_err(-ENAMETOOLONG); 7157 7158 err = bpf_program__unpin_instance(prog, buf, i); 7159 if (err) 7160 return err; 7161 } 7162 7163 err = rmdir(path); 7164 if (err) 7165 return libbpf_err(-errno); 7166 7167 return 0; 7168 } 7169 7170 int bpf_map__pin(struct bpf_map *map, const char *path) 7171 { 7172 char *cp, errmsg[STRERR_BUFSIZE]; 7173 int err; 7174 7175 if (map == NULL) { 7176 pr_warn("invalid map pointer\n"); 7177 return libbpf_err(-EINVAL); 7178 } 7179 7180 if (map->pin_path) { 7181 if (path && strcmp(path, map->pin_path)) { 7182 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 7183 bpf_map__name(map), map->pin_path, path); 7184 return libbpf_err(-EINVAL); 7185 } else if (map->pinned) { 7186 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 7187 bpf_map__name(map), map->pin_path); 7188 return 0; 7189 } 7190 } else { 7191 if (!path) { 7192 pr_warn("missing a path to pin map '%s' at\n", 7193 bpf_map__name(map)); 7194 return libbpf_err(-EINVAL); 7195 } else if (map->pinned) { 7196 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 7197 return libbpf_err(-EEXIST); 7198 } 7199 7200 map->pin_path = strdup(path); 7201 if (!map->pin_path) { 7202 err = -errno; 7203 goto out_err; 7204 } 7205 } 7206 7207 err = make_parent_dir(map->pin_path); 7208 if (err) 7209 return libbpf_err(err); 7210 7211 err = check_path(map->pin_path); 7212 if (err) 7213 return libbpf_err(err); 7214 7215 if (bpf_obj_pin(map->fd, map->pin_path)) { 7216 err = -errno; 7217 goto out_err; 7218 } 7219 7220 map->pinned = true; 7221 pr_debug("pinned map '%s'\n", map->pin_path); 7222 7223 return 0; 7224 7225 out_err: 7226 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 7227 pr_warn("failed to pin map: %s\n", cp); 7228 return libbpf_err(err); 7229 } 7230 7231 int bpf_map__unpin(struct bpf_map *map, const char *path) 7232 { 7233 int err; 7234 7235 if (map == NULL) { 7236 pr_warn("invalid map pointer\n"); 7237 return libbpf_err(-EINVAL); 7238 } 7239 7240 if (map->pin_path) { 7241 if (path && strcmp(path, map->pin_path)) { 7242 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 7243 bpf_map__name(map), map->pin_path, path); 7244 return libbpf_err(-EINVAL); 7245 } 7246 path = map->pin_path; 7247 } else if (!path) { 7248 pr_warn("no path to unpin map '%s' from\n", 7249 bpf_map__name(map)); 7250 return libbpf_err(-EINVAL); 7251 } 7252 7253 err = check_path(path); 7254 if (err) 7255 return libbpf_err(err); 7256 7257 err = unlink(path); 7258 if (err != 0) 7259 return libbpf_err(-errno); 7260 7261 map->pinned = false; 7262 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 7263 7264 return 0; 7265 } 7266 7267 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 7268 { 7269 char *new = NULL; 7270 7271 if (path) { 7272 new = strdup(path); 7273 if (!new) 7274 return libbpf_err(-errno); 7275 } 7276 7277 free(map->pin_path); 7278 map->pin_path = new; 7279 return 0; 7280 } 7281 7282 const char *bpf_map__get_pin_path(const struct bpf_map *map) 7283 { 7284 return map->pin_path; 7285 } 7286 7287 const char *bpf_map__pin_path(const struct bpf_map *map) 7288 { 7289 return map->pin_path; 7290 } 7291 7292 bool bpf_map__is_pinned(const struct bpf_map *map) 7293 { 7294 return map->pinned; 7295 } 7296 7297 static void sanitize_pin_path(char *s) 7298 { 7299 /* bpffs disallows periods in path names */ 7300 while (*s) { 7301 if (*s == '.') 7302 *s = '_'; 7303 s++; 7304 } 7305 } 7306 7307 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 7308 { 7309 struct bpf_map *map; 7310 int err; 7311 7312 if (!obj) 7313 return libbpf_err(-ENOENT); 7314 7315 if (!obj->loaded) { 7316 pr_warn("object not yet loaded; load it first\n"); 7317 return libbpf_err(-ENOENT); 7318 } 7319 7320 bpf_object__for_each_map(map, obj) { 7321 char *pin_path = NULL; 7322 char buf[PATH_MAX]; 7323 7324 if (path) { 7325 int len; 7326 7327 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7328 bpf_map__name(map)); 7329 if (len < 0) { 7330 err = -EINVAL; 7331 goto err_unpin_maps; 7332 } else if (len >= PATH_MAX) { 7333 err = -ENAMETOOLONG; 7334 goto err_unpin_maps; 7335 } 7336 sanitize_pin_path(buf); 7337 pin_path = buf; 7338 } else if (!map->pin_path) { 7339 continue; 7340 } 7341 7342 err = bpf_map__pin(map, pin_path); 7343 if (err) 7344 goto err_unpin_maps; 7345 } 7346 7347 return 0; 7348 7349 err_unpin_maps: 7350 while ((map = bpf_map__prev(map, obj))) { 7351 if (!map->pin_path) 7352 continue; 7353 7354 bpf_map__unpin(map, NULL); 7355 } 7356 7357 return libbpf_err(err); 7358 } 7359 7360 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 7361 { 7362 struct bpf_map *map; 7363 int err; 7364 7365 if (!obj) 7366 return libbpf_err(-ENOENT); 7367 7368 bpf_object__for_each_map(map, obj) { 7369 char *pin_path = NULL; 7370 char buf[PATH_MAX]; 7371 7372 if (path) { 7373 int len; 7374 7375 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7376 bpf_map__name(map)); 7377 if (len < 0) 7378 return libbpf_err(-EINVAL); 7379 else if (len >= PATH_MAX) 7380 return libbpf_err(-ENAMETOOLONG); 7381 sanitize_pin_path(buf); 7382 pin_path = buf; 7383 } else if (!map->pin_path) { 7384 continue; 7385 } 7386 7387 err = bpf_map__unpin(map, pin_path); 7388 if (err) 7389 return libbpf_err(err); 7390 } 7391 7392 return 0; 7393 } 7394 7395 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 7396 { 7397 struct bpf_program *prog; 7398 int err; 7399 7400 if (!obj) 7401 return libbpf_err(-ENOENT); 7402 7403 if (!obj->loaded) { 7404 pr_warn("object not yet loaded; load it first\n"); 7405 return libbpf_err(-ENOENT); 7406 } 7407 7408 bpf_object__for_each_program(prog, obj) { 7409 char buf[PATH_MAX]; 7410 int len; 7411 7412 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7413 prog->pin_name); 7414 if (len < 0) { 7415 err = -EINVAL; 7416 goto err_unpin_programs; 7417 } else if (len >= PATH_MAX) { 7418 err = -ENAMETOOLONG; 7419 goto err_unpin_programs; 7420 } 7421 7422 err = bpf_program__pin(prog, buf); 7423 if (err) 7424 goto err_unpin_programs; 7425 } 7426 7427 return 0; 7428 7429 err_unpin_programs: 7430 while ((prog = bpf_program__prev(prog, obj))) { 7431 char buf[PATH_MAX]; 7432 int len; 7433 7434 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7435 prog->pin_name); 7436 if (len < 0) 7437 continue; 7438 else if (len >= PATH_MAX) 7439 continue; 7440 7441 bpf_program__unpin(prog, buf); 7442 } 7443 7444 return libbpf_err(err); 7445 } 7446 7447 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 7448 { 7449 struct bpf_program *prog; 7450 int err; 7451 7452 if (!obj) 7453 return libbpf_err(-ENOENT); 7454 7455 bpf_object__for_each_program(prog, obj) { 7456 char buf[PATH_MAX]; 7457 int len; 7458 7459 len = snprintf(buf, PATH_MAX, "%s/%s", path, 7460 prog->pin_name); 7461 if (len < 0) 7462 return libbpf_err(-EINVAL); 7463 else if (len >= PATH_MAX) 7464 return libbpf_err(-ENAMETOOLONG); 7465 7466 err = bpf_program__unpin(prog, buf); 7467 if (err) 7468 return libbpf_err(err); 7469 } 7470 7471 return 0; 7472 } 7473 7474 int bpf_object__pin(struct bpf_object *obj, const char *path) 7475 { 7476 int err; 7477 7478 err = bpf_object__pin_maps(obj, path); 7479 if (err) 7480 return libbpf_err(err); 7481 7482 err = bpf_object__pin_programs(obj, path); 7483 if (err) { 7484 bpf_object__unpin_maps(obj, path); 7485 return libbpf_err(err); 7486 } 7487 7488 return 0; 7489 } 7490 7491 static void bpf_map__destroy(struct bpf_map *map) 7492 { 7493 if (map->clear_priv) 7494 map->clear_priv(map, map->priv); 7495 map->priv = NULL; 7496 map->clear_priv = NULL; 7497 7498 if (map->inner_map) { 7499 bpf_map__destroy(map->inner_map); 7500 zfree(&map->inner_map); 7501 } 7502 7503 zfree(&map->init_slots); 7504 map->init_slots_sz = 0; 7505 7506 if (map->mmaped) { 7507 munmap(map->mmaped, bpf_map_mmap_sz(map)); 7508 map->mmaped = NULL; 7509 } 7510 7511 if (map->st_ops) { 7512 zfree(&map->st_ops->data); 7513 zfree(&map->st_ops->progs); 7514 zfree(&map->st_ops->kern_func_off); 7515 zfree(&map->st_ops); 7516 } 7517 7518 zfree(&map->name); 7519 zfree(&map->pin_path); 7520 7521 if (map->fd >= 0) 7522 zclose(map->fd); 7523 } 7524 7525 void bpf_object__close(struct bpf_object *obj) 7526 { 7527 size_t i; 7528 7529 if (IS_ERR_OR_NULL(obj)) 7530 return; 7531 7532 if (obj->clear_priv) 7533 obj->clear_priv(obj, obj->priv); 7534 7535 bpf_gen__free(obj->gen_loader); 7536 bpf_object__elf_finish(obj); 7537 bpf_object__unload(obj); 7538 btf__free(obj->btf); 7539 btf_ext__free(obj->btf_ext); 7540 7541 for (i = 0; i < obj->nr_maps; i++) 7542 bpf_map__destroy(&obj->maps[i]); 7543 7544 zfree(&obj->btf_custom_path); 7545 zfree(&obj->kconfig); 7546 zfree(&obj->externs); 7547 obj->nr_extern = 0; 7548 7549 zfree(&obj->maps); 7550 obj->nr_maps = 0; 7551 7552 if (obj->programs && obj->nr_programs) { 7553 for (i = 0; i < obj->nr_programs; i++) 7554 bpf_program__exit(&obj->programs[i]); 7555 } 7556 zfree(&obj->programs); 7557 7558 list_del(&obj->list); 7559 free(obj); 7560 } 7561 7562 struct bpf_object * 7563 bpf_object__next(struct bpf_object *prev) 7564 { 7565 struct bpf_object *next; 7566 7567 if (!prev) 7568 next = list_first_entry(&bpf_objects_list, 7569 struct bpf_object, 7570 list); 7571 else 7572 next = list_next_entry(prev, list); 7573 7574 /* Empty list is noticed here so don't need checking on entry. */ 7575 if (&next->list == &bpf_objects_list) 7576 return NULL; 7577 7578 return next; 7579 } 7580 7581 const char *bpf_object__name(const struct bpf_object *obj) 7582 { 7583 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 7584 } 7585 7586 unsigned int bpf_object__kversion(const struct bpf_object *obj) 7587 { 7588 return obj ? obj->kern_version : 0; 7589 } 7590 7591 struct btf *bpf_object__btf(const struct bpf_object *obj) 7592 { 7593 return obj ? obj->btf : NULL; 7594 } 7595 7596 int bpf_object__btf_fd(const struct bpf_object *obj) 7597 { 7598 return obj->btf ? btf__fd(obj->btf) : -1; 7599 } 7600 7601 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 7602 { 7603 if (obj->loaded) 7604 return libbpf_err(-EINVAL); 7605 7606 obj->kern_version = kern_version; 7607 7608 return 0; 7609 } 7610 7611 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 7612 bpf_object_clear_priv_t clear_priv) 7613 { 7614 if (obj->priv && obj->clear_priv) 7615 obj->clear_priv(obj, obj->priv); 7616 7617 obj->priv = priv; 7618 obj->clear_priv = clear_priv; 7619 return 0; 7620 } 7621 7622 void *bpf_object__priv(const struct bpf_object *obj) 7623 { 7624 return obj ? obj->priv : libbpf_err_ptr(-EINVAL); 7625 } 7626 7627 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 7628 { 7629 struct bpf_gen *gen; 7630 7631 if (!opts) 7632 return -EFAULT; 7633 if (!OPTS_VALID(opts, gen_loader_opts)) 7634 return -EINVAL; 7635 gen = calloc(sizeof(*gen), 1); 7636 if (!gen) 7637 return -ENOMEM; 7638 gen->opts = opts; 7639 obj->gen_loader = gen; 7640 return 0; 7641 } 7642 7643 static struct bpf_program * 7644 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 7645 bool forward) 7646 { 7647 size_t nr_programs = obj->nr_programs; 7648 ssize_t idx; 7649 7650 if (!nr_programs) 7651 return NULL; 7652 7653 if (!p) 7654 /* Iter from the beginning */ 7655 return forward ? &obj->programs[0] : 7656 &obj->programs[nr_programs - 1]; 7657 7658 if (p->obj != obj) { 7659 pr_warn("error: program handler doesn't match object\n"); 7660 return errno = EINVAL, NULL; 7661 } 7662 7663 idx = (p - obj->programs) + (forward ? 1 : -1); 7664 if (idx >= obj->nr_programs || idx < 0) 7665 return NULL; 7666 return &obj->programs[idx]; 7667 } 7668 7669 struct bpf_program * 7670 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 7671 { 7672 struct bpf_program *prog = prev; 7673 7674 do { 7675 prog = __bpf_program__iter(prog, obj, true); 7676 } while (prog && prog_is_subprog(obj, prog)); 7677 7678 return prog; 7679 } 7680 7681 struct bpf_program * 7682 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 7683 { 7684 struct bpf_program *prog = next; 7685 7686 do { 7687 prog = __bpf_program__iter(prog, obj, false); 7688 } while (prog && prog_is_subprog(obj, prog)); 7689 7690 return prog; 7691 } 7692 7693 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 7694 bpf_program_clear_priv_t clear_priv) 7695 { 7696 if (prog->priv && prog->clear_priv) 7697 prog->clear_priv(prog, prog->priv); 7698 7699 prog->priv = priv; 7700 prog->clear_priv = clear_priv; 7701 return 0; 7702 } 7703 7704 void *bpf_program__priv(const struct bpf_program *prog) 7705 { 7706 return prog ? prog->priv : libbpf_err_ptr(-EINVAL); 7707 } 7708 7709 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 7710 { 7711 prog->prog_ifindex = ifindex; 7712 } 7713 7714 const char *bpf_program__name(const struct bpf_program *prog) 7715 { 7716 return prog->name; 7717 } 7718 7719 const char *bpf_program__section_name(const struct bpf_program *prog) 7720 { 7721 return prog->sec_name; 7722 } 7723 7724 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 7725 { 7726 const char *title; 7727 7728 title = prog->sec_name; 7729 if (needs_copy) { 7730 title = strdup(title); 7731 if (!title) { 7732 pr_warn("failed to strdup program title\n"); 7733 return libbpf_err_ptr(-ENOMEM); 7734 } 7735 } 7736 7737 return title; 7738 } 7739 7740 bool bpf_program__autoload(const struct bpf_program *prog) 7741 { 7742 return prog->load; 7743 } 7744 7745 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 7746 { 7747 if (prog->obj->loaded) 7748 return libbpf_err(-EINVAL); 7749 7750 prog->load = autoload; 7751 return 0; 7752 } 7753 7754 int bpf_program__fd(const struct bpf_program *prog) 7755 { 7756 return bpf_program__nth_fd(prog, 0); 7757 } 7758 7759 size_t bpf_program__size(const struct bpf_program *prog) 7760 { 7761 return prog->insns_cnt * BPF_INSN_SZ; 7762 } 7763 7764 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 7765 bpf_program_prep_t prep) 7766 { 7767 int *instances_fds; 7768 7769 if (nr_instances <= 0 || !prep) 7770 return libbpf_err(-EINVAL); 7771 7772 if (prog->instances.nr > 0 || prog->instances.fds) { 7773 pr_warn("Can't set pre-processor after loading\n"); 7774 return libbpf_err(-EINVAL); 7775 } 7776 7777 instances_fds = malloc(sizeof(int) * nr_instances); 7778 if (!instances_fds) { 7779 pr_warn("alloc memory failed for fds\n"); 7780 return libbpf_err(-ENOMEM); 7781 } 7782 7783 /* fill all fd with -1 */ 7784 memset(instances_fds, -1, sizeof(int) * nr_instances); 7785 7786 prog->instances.nr = nr_instances; 7787 prog->instances.fds = instances_fds; 7788 prog->preprocessor = prep; 7789 return 0; 7790 } 7791 7792 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 7793 { 7794 int fd; 7795 7796 if (!prog) 7797 return libbpf_err(-EINVAL); 7798 7799 if (n >= prog->instances.nr || n < 0) { 7800 pr_warn("Can't get the %dth fd from program %s: only %d instances\n", 7801 n, prog->name, prog->instances.nr); 7802 return libbpf_err(-EINVAL); 7803 } 7804 7805 fd = prog->instances.fds[n]; 7806 if (fd < 0) { 7807 pr_warn("%dth instance of program '%s' is invalid\n", 7808 n, prog->name); 7809 return libbpf_err(-ENOENT); 7810 } 7811 7812 return fd; 7813 } 7814 7815 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog) 7816 { 7817 return prog->type; 7818 } 7819 7820 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 7821 { 7822 prog->type = type; 7823 } 7824 7825 static bool bpf_program__is_type(const struct bpf_program *prog, 7826 enum bpf_prog_type type) 7827 { 7828 return prog ? (prog->type == type) : false; 7829 } 7830 7831 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 7832 int bpf_program__set_##NAME(struct bpf_program *prog) \ 7833 { \ 7834 if (!prog) \ 7835 return libbpf_err(-EINVAL); \ 7836 bpf_program__set_type(prog, TYPE); \ 7837 return 0; \ 7838 } \ 7839 \ 7840 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 7841 { \ 7842 return bpf_program__is_type(prog, TYPE); \ 7843 } \ 7844 7845 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 7846 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); 7847 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 7848 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 7849 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 7850 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 7851 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 7852 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 7853 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 7854 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); 7855 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); 7856 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); 7857 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP); 7858 7859 enum bpf_attach_type 7860 bpf_program__get_expected_attach_type(const struct bpf_program *prog) 7861 { 7862 return prog->expected_attach_type; 7863 } 7864 7865 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 7866 enum bpf_attach_type type) 7867 { 7868 prog->expected_attach_type = type; 7869 } 7870 7871 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \ 7872 attachable, attach_btf) \ 7873 { \ 7874 .sec = string, \ 7875 .len = sizeof(string) - 1, \ 7876 .prog_type = ptype, \ 7877 .expected_attach_type = eatype, \ 7878 .is_exp_attach_type_optional = eatype_optional, \ 7879 .is_attachable = attachable, \ 7880 .is_attach_btf = attach_btf, \ 7881 } 7882 7883 /* Programs that can NOT be attached. */ 7884 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) 7885 7886 /* Programs that can be attached. */ 7887 #define BPF_APROG_SEC(string, ptype, atype) \ 7888 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0) 7889 7890 /* Programs that must specify expected attach type at load time. */ 7891 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 7892 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0) 7893 7894 /* Programs that use BTF to identify attach point */ 7895 #define BPF_PROG_BTF(string, ptype, eatype) \ 7896 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1) 7897 7898 /* Programs that can be attached but attach type can't be identified by section 7899 * name. Kept for backward compatibility. 7900 */ 7901 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 7902 7903 #define SEC_DEF(sec_pfx, ptype, ...) { \ 7904 .sec = sec_pfx, \ 7905 .len = sizeof(sec_pfx) - 1, \ 7906 .prog_type = BPF_PROG_TYPE_##ptype, \ 7907 __VA_ARGS__ \ 7908 } 7909 7910 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 7911 struct bpf_program *prog); 7912 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 7913 struct bpf_program *prog); 7914 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 7915 struct bpf_program *prog); 7916 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 7917 struct bpf_program *prog); 7918 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 7919 struct bpf_program *prog); 7920 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 7921 struct bpf_program *prog); 7922 7923 static const struct bpf_sec_def section_defs[] = { 7924 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 7925 BPF_EAPROG_SEC("sk_reuseport/migrate", BPF_PROG_TYPE_SK_REUSEPORT, 7926 BPF_SK_REUSEPORT_SELECT_OR_MIGRATE), 7927 BPF_EAPROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT, 7928 BPF_SK_REUSEPORT_SELECT), 7929 SEC_DEF("kprobe/", KPROBE, 7930 .attach_fn = attach_kprobe), 7931 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), 7932 SEC_DEF("kretprobe/", KPROBE, 7933 .attach_fn = attach_kprobe), 7934 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), 7935 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 7936 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 7937 SEC_DEF("tracepoint/", TRACEPOINT, 7938 .attach_fn = attach_tp), 7939 SEC_DEF("tp/", TRACEPOINT, 7940 .attach_fn = attach_tp), 7941 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 7942 .attach_fn = attach_raw_tp), 7943 SEC_DEF("raw_tp/", RAW_TRACEPOINT, 7944 .attach_fn = attach_raw_tp), 7945 SEC_DEF("tp_btf/", TRACING, 7946 .expected_attach_type = BPF_TRACE_RAW_TP, 7947 .is_attach_btf = true, 7948 .attach_fn = attach_trace), 7949 SEC_DEF("fentry/", TRACING, 7950 .expected_attach_type = BPF_TRACE_FENTRY, 7951 .is_attach_btf = true, 7952 .attach_fn = attach_trace), 7953 SEC_DEF("fmod_ret/", TRACING, 7954 .expected_attach_type = BPF_MODIFY_RETURN, 7955 .is_attach_btf = true, 7956 .attach_fn = attach_trace), 7957 SEC_DEF("fexit/", TRACING, 7958 .expected_attach_type = BPF_TRACE_FEXIT, 7959 .is_attach_btf = true, 7960 .attach_fn = attach_trace), 7961 SEC_DEF("fentry.s/", TRACING, 7962 .expected_attach_type = BPF_TRACE_FENTRY, 7963 .is_attach_btf = true, 7964 .is_sleepable = true, 7965 .attach_fn = attach_trace), 7966 SEC_DEF("fmod_ret.s/", TRACING, 7967 .expected_attach_type = BPF_MODIFY_RETURN, 7968 .is_attach_btf = true, 7969 .is_sleepable = true, 7970 .attach_fn = attach_trace), 7971 SEC_DEF("fexit.s/", TRACING, 7972 .expected_attach_type = BPF_TRACE_FEXIT, 7973 .is_attach_btf = true, 7974 .is_sleepable = true, 7975 .attach_fn = attach_trace), 7976 SEC_DEF("freplace/", EXT, 7977 .is_attach_btf = true, 7978 .attach_fn = attach_trace), 7979 SEC_DEF("lsm/", LSM, 7980 .is_attach_btf = true, 7981 .expected_attach_type = BPF_LSM_MAC, 7982 .attach_fn = attach_lsm), 7983 SEC_DEF("lsm.s/", LSM, 7984 .is_attach_btf = true, 7985 .is_sleepable = true, 7986 .expected_attach_type = BPF_LSM_MAC, 7987 .attach_fn = attach_lsm), 7988 SEC_DEF("iter/", TRACING, 7989 .expected_attach_type = BPF_TRACE_ITER, 7990 .is_attach_btf = true, 7991 .attach_fn = attach_iter), 7992 SEC_DEF("syscall", SYSCALL, 7993 .is_sleepable = true), 7994 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP, 7995 BPF_XDP_DEVMAP), 7996 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP, 7997 BPF_XDP_CPUMAP), 7998 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP, 7999 BPF_XDP), 8000 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 8001 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 8002 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 8003 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 8004 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 8005 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 8006 BPF_CGROUP_INET_INGRESS), 8007 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 8008 BPF_CGROUP_INET_EGRESS), 8009 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 8010 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK, 8011 BPF_CGROUP_INET_SOCK_CREATE), 8012 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK, 8013 BPF_CGROUP_INET_SOCK_RELEASE), 8014 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 8015 BPF_CGROUP_INET_SOCK_CREATE), 8016 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 8017 BPF_CGROUP_INET4_POST_BIND), 8018 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 8019 BPF_CGROUP_INET6_POST_BIND), 8020 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 8021 BPF_CGROUP_DEVICE), 8022 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 8023 BPF_CGROUP_SOCK_OPS), 8024 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 8025 BPF_SK_SKB_STREAM_PARSER), 8026 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 8027 BPF_SK_SKB_STREAM_VERDICT), 8028 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 8029 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 8030 BPF_SK_MSG_VERDICT), 8031 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 8032 BPF_LIRC_MODE2), 8033 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 8034 BPF_FLOW_DISSECTOR), 8035 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8036 BPF_CGROUP_INET4_BIND), 8037 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8038 BPF_CGROUP_INET6_BIND), 8039 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8040 BPF_CGROUP_INET4_CONNECT), 8041 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8042 BPF_CGROUP_INET6_CONNECT), 8043 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8044 BPF_CGROUP_UDP4_SENDMSG), 8045 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8046 BPF_CGROUP_UDP6_SENDMSG), 8047 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8048 BPF_CGROUP_UDP4_RECVMSG), 8049 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8050 BPF_CGROUP_UDP6_RECVMSG), 8051 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8052 BPF_CGROUP_INET4_GETPEERNAME), 8053 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8054 BPF_CGROUP_INET6_GETPEERNAME), 8055 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8056 BPF_CGROUP_INET4_GETSOCKNAME), 8057 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 8058 BPF_CGROUP_INET6_GETSOCKNAME), 8059 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 8060 BPF_CGROUP_SYSCTL), 8061 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8062 BPF_CGROUP_GETSOCKOPT), 8063 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 8064 BPF_CGROUP_SETSOCKOPT), 8065 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), 8066 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP, 8067 BPF_SK_LOOKUP), 8068 }; 8069 8070 #undef BPF_PROG_SEC_IMPL 8071 #undef BPF_PROG_SEC 8072 #undef BPF_APROG_SEC 8073 #undef BPF_EAPROG_SEC 8074 #undef BPF_APROG_COMPAT 8075 #undef SEC_DEF 8076 8077 #define MAX_TYPE_NAME_SIZE 32 8078 8079 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 8080 { 8081 int i, n = ARRAY_SIZE(section_defs); 8082 8083 for (i = 0; i < n; i++) { 8084 if (strncmp(sec_name, 8085 section_defs[i].sec, section_defs[i].len)) 8086 continue; 8087 return §ion_defs[i]; 8088 } 8089 return NULL; 8090 } 8091 8092 static char *libbpf_get_type_names(bool attach_type) 8093 { 8094 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 8095 char *buf; 8096 8097 buf = malloc(len); 8098 if (!buf) 8099 return NULL; 8100 8101 buf[0] = '\0'; 8102 /* Forge string buf with all available names */ 8103 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8104 if (attach_type && !section_defs[i].is_attachable) 8105 continue; 8106 8107 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 8108 free(buf); 8109 return NULL; 8110 } 8111 strcat(buf, " "); 8112 strcat(buf, section_defs[i].sec); 8113 } 8114 8115 return buf; 8116 } 8117 8118 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 8119 enum bpf_attach_type *expected_attach_type) 8120 { 8121 const struct bpf_sec_def *sec_def; 8122 char *type_names; 8123 8124 if (!name) 8125 return libbpf_err(-EINVAL); 8126 8127 sec_def = find_sec_def(name); 8128 if (sec_def) { 8129 *prog_type = sec_def->prog_type; 8130 *expected_attach_type = sec_def->expected_attach_type; 8131 return 0; 8132 } 8133 8134 pr_debug("failed to guess program type from ELF section '%s'\n", name); 8135 type_names = libbpf_get_type_names(false); 8136 if (type_names != NULL) { 8137 pr_debug("supported section(type) names are:%s\n", type_names); 8138 free(type_names); 8139 } 8140 8141 return libbpf_err(-ESRCH); 8142 } 8143 8144 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 8145 size_t offset) 8146 { 8147 struct bpf_map *map; 8148 size_t i; 8149 8150 for (i = 0; i < obj->nr_maps; i++) { 8151 map = &obj->maps[i]; 8152 if (!bpf_map__is_struct_ops(map)) 8153 continue; 8154 if (map->sec_offset <= offset && 8155 offset - map->sec_offset < map->def.value_size) 8156 return map; 8157 } 8158 8159 return NULL; 8160 } 8161 8162 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 8163 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 8164 GElf_Shdr *shdr, Elf_Data *data) 8165 { 8166 const struct btf_member *member; 8167 struct bpf_struct_ops *st_ops; 8168 struct bpf_program *prog; 8169 unsigned int shdr_idx; 8170 const struct btf *btf; 8171 struct bpf_map *map; 8172 Elf_Data *symbols; 8173 unsigned int moff, insn_idx; 8174 const char *name; 8175 __u32 member_idx; 8176 GElf_Sym sym; 8177 GElf_Rel rel; 8178 int i, nrels; 8179 8180 symbols = obj->efile.symbols; 8181 btf = obj->btf; 8182 nrels = shdr->sh_size / shdr->sh_entsize; 8183 for (i = 0; i < nrels; i++) { 8184 if (!gelf_getrel(data, i, &rel)) { 8185 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 8186 return -LIBBPF_ERRNO__FORMAT; 8187 } 8188 8189 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 8190 pr_warn("struct_ops reloc: symbol %zx not found\n", 8191 (size_t)GELF_R_SYM(rel.r_info)); 8192 return -LIBBPF_ERRNO__FORMAT; 8193 } 8194 8195 name = elf_sym_str(obj, sym.st_name) ?: "<?>"; 8196 map = find_struct_ops_map_by_offset(obj, rel.r_offset); 8197 if (!map) { 8198 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", 8199 (size_t)rel.r_offset); 8200 return -EINVAL; 8201 } 8202 8203 moff = rel.r_offset - map->sec_offset; 8204 shdr_idx = sym.st_shndx; 8205 st_ops = map->st_ops; 8206 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 8207 map->name, 8208 (long long)(rel.r_info >> 32), 8209 (long long)sym.st_value, 8210 shdr_idx, (size_t)rel.r_offset, 8211 map->sec_offset, sym.st_name, name); 8212 8213 if (shdr_idx >= SHN_LORESERVE) { 8214 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", 8215 map->name, (size_t)rel.r_offset, shdr_idx); 8216 return -LIBBPF_ERRNO__RELOC; 8217 } 8218 if (sym.st_value % BPF_INSN_SZ) { 8219 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 8220 map->name, (unsigned long long)sym.st_value); 8221 return -LIBBPF_ERRNO__FORMAT; 8222 } 8223 insn_idx = sym.st_value / BPF_INSN_SZ; 8224 8225 member = find_member_by_offset(st_ops->type, moff * 8); 8226 if (!member) { 8227 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 8228 map->name, moff); 8229 return -EINVAL; 8230 } 8231 member_idx = member - btf_members(st_ops->type); 8232 name = btf__name_by_offset(btf, member->name_off); 8233 8234 if (!resolve_func_ptr(btf, member->type, NULL)) { 8235 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 8236 map->name, name); 8237 return -EINVAL; 8238 } 8239 8240 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 8241 if (!prog) { 8242 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 8243 map->name, shdr_idx, name); 8244 return -EINVAL; 8245 } 8246 8247 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 8248 const struct bpf_sec_def *sec_def; 8249 8250 sec_def = find_sec_def(prog->sec_name); 8251 if (sec_def && 8252 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) { 8253 /* for pr_warn */ 8254 prog->type = sec_def->prog_type; 8255 goto invalid_prog; 8256 } 8257 8258 prog->type = BPF_PROG_TYPE_STRUCT_OPS; 8259 prog->attach_btf_id = st_ops->type_id; 8260 prog->expected_attach_type = member_idx; 8261 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || 8262 prog->attach_btf_id != st_ops->type_id || 8263 prog->expected_attach_type != member_idx) { 8264 goto invalid_prog; 8265 } 8266 st_ops->progs[member_idx] = prog; 8267 } 8268 8269 return 0; 8270 8271 invalid_prog: 8272 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 8273 map->name, prog->name, prog->sec_name, prog->type, 8274 prog->attach_btf_id, prog->expected_attach_type, name); 8275 return -EINVAL; 8276 } 8277 8278 #define BTF_TRACE_PREFIX "btf_trace_" 8279 #define BTF_LSM_PREFIX "bpf_lsm_" 8280 #define BTF_ITER_PREFIX "bpf_iter_" 8281 #define BTF_MAX_NAME_SIZE 128 8282 8283 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 8284 const char **prefix, int *kind) 8285 { 8286 switch (attach_type) { 8287 case BPF_TRACE_RAW_TP: 8288 *prefix = BTF_TRACE_PREFIX; 8289 *kind = BTF_KIND_TYPEDEF; 8290 break; 8291 case BPF_LSM_MAC: 8292 *prefix = BTF_LSM_PREFIX; 8293 *kind = BTF_KIND_FUNC; 8294 break; 8295 case BPF_TRACE_ITER: 8296 *prefix = BTF_ITER_PREFIX; 8297 *kind = BTF_KIND_FUNC; 8298 break; 8299 default: 8300 *prefix = ""; 8301 *kind = BTF_KIND_FUNC; 8302 } 8303 } 8304 8305 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 8306 const char *name, __u32 kind) 8307 { 8308 char btf_type_name[BTF_MAX_NAME_SIZE]; 8309 int ret; 8310 8311 ret = snprintf(btf_type_name, sizeof(btf_type_name), 8312 "%s%s", prefix, name); 8313 /* snprintf returns the number of characters written excluding the 8314 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 8315 * indicates truncation. 8316 */ 8317 if (ret < 0 || ret >= sizeof(btf_type_name)) 8318 return -ENAMETOOLONG; 8319 return btf__find_by_name_kind(btf, btf_type_name, kind); 8320 } 8321 8322 static inline int find_attach_btf_id(struct btf *btf, const char *name, 8323 enum bpf_attach_type attach_type) 8324 { 8325 const char *prefix; 8326 int kind; 8327 8328 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 8329 return find_btf_by_prefix_kind(btf, prefix, name, kind); 8330 } 8331 8332 int libbpf_find_vmlinux_btf_id(const char *name, 8333 enum bpf_attach_type attach_type) 8334 { 8335 struct btf *btf; 8336 int err; 8337 8338 btf = btf__load_vmlinux_btf(); 8339 err = libbpf_get_error(btf); 8340 if (err) { 8341 pr_warn("vmlinux BTF is not found\n"); 8342 return libbpf_err(err); 8343 } 8344 8345 err = find_attach_btf_id(btf, name, attach_type); 8346 if (err <= 0) 8347 pr_warn("%s is not found in vmlinux BTF\n", name); 8348 8349 btf__free(btf); 8350 return libbpf_err(err); 8351 } 8352 8353 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 8354 { 8355 struct bpf_prog_info_linear *info_linear; 8356 struct bpf_prog_info *info; 8357 struct btf *btf; 8358 int err; 8359 8360 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); 8361 err = libbpf_get_error(info_linear); 8362 if (err) { 8363 pr_warn("failed get_prog_info_linear for FD %d\n", 8364 attach_prog_fd); 8365 return err; 8366 } 8367 8368 err = -EINVAL; 8369 info = &info_linear->info; 8370 if (!info->btf_id) { 8371 pr_warn("The target program doesn't have BTF\n"); 8372 goto out; 8373 } 8374 btf = btf__load_from_kernel_by_id(info->btf_id); 8375 if (libbpf_get_error(btf)) { 8376 pr_warn("Failed to get BTF of the program\n"); 8377 goto out; 8378 } 8379 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 8380 btf__free(btf); 8381 if (err <= 0) { 8382 pr_warn("%s is not found in prog's BTF\n", name); 8383 goto out; 8384 } 8385 out: 8386 free(info_linear); 8387 return err; 8388 } 8389 8390 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 8391 enum bpf_attach_type attach_type, 8392 int *btf_obj_fd, int *btf_type_id) 8393 { 8394 int ret, i; 8395 8396 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 8397 if (ret > 0) { 8398 *btf_obj_fd = 0; /* vmlinux BTF */ 8399 *btf_type_id = ret; 8400 return 0; 8401 } 8402 if (ret != -ENOENT) 8403 return ret; 8404 8405 ret = load_module_btfs(obj); 8406 if (ret) 8407 return ret; 8408 8409 for (i = 0; i < obj->btf_module_cnt; i++) { 8410 const struct module_btf *mod = &obj->btf_modules[i]; 8411 8412 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 8413 if (ret > 0) { 8414 *btf_obj_fd = mod->fd; 8415 *btf_type_id = ret; 8416 return 0; 8417 } 8418 if (ret == -ENOENT) 8419 continue; 8420 8421 return ret; 8422 } 8423 8424 return -ESRCH; 8425 } 8426 8427 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id) 8428 { 8429 enum bpf_attach_type attach_type = prog->expected_attach_type; 8430 __u32 attach_prog_fd = prog->attach_prog_fd; 8431 const char *name = prog->sec_name, *attach_name; 8432 const struct bpf_sec_def *sec = NULL; 8433 int i, err = 0; 8434 8435 if (!name) 8436 return -EINVAL; 8437 8438 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8439 if (!section_defs[i].is_attach_btf) 8440 continue; 8441 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 8442 continue; 8443 8444 sec = §ion_defs[i]; 8445 break; 8446 } 8447 8448 if (!sec) { 8449 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name); 8450 return -ESRCH; 8451 } 8452 attach_name = name + sec->len; 8453 8454 /* BPF program's BTF ID */ 8455 if (attach_prog_fd) { 8456 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 8457 if (err < 0) { 8458 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 8459 attach_prog_fd, attach_name, err); 8460 return err; 8461 } 8462 *btf_obj_fd = 0; 8463 *btf_type_id = err; 8464 return 0; 8465 } 8466 8467 /* kernel/module BTF ID */ 8468 if (prog->obj->gen_loader) { 8469 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 8470 *btf_obj_fd = 0; 8471 *btf_type_id = 1; 8472 } else { 8473 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); 8474 } 8475 if (err) { 8476 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err); 8477 return err; 8478 } 8479 return 0; 8480 } 8481 8482 int libbpf_attach_type_by_name(const char *name, 8483 enum bpf_attach_type *attach_type) 8484 { 8485 char *type_names; 8486 int i; 8487 8488 if (!name) 8489 return libbpf_err(-EINVAL); 8490 8491 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 8492 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 8493 continue; 8494 if (!section_defs[i].is_attachable) 8495 return libbpf_err(-EINVAL); 8496 *attach_type = section_defs[i].expected_attach_type; 8497 return 0; 8498 } 8499 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 8500 type_names = libbpf_get_type_names(true); 8501 if (type_names != NULL) { 8502 pr_debug("attachable section(type) names are:%s\n", type_names); 8503 free(type_names); 8504 } 8505 8506 return libbpf_err(-EINVAL); 8507 } 8508 8509 int bpf_map__fd(const struct bpf_map *map) 8510 { 8511 return map ? map->fd : libbpf_err(-EINVAL); 8512 } 8513 8514 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 8515 { 8516 return map ? &map->def : libbpf_err_ptr(-EINVAL); 8517 } 8518 8519 const char *bpf_map__name(const struct bpf_map *map) 8520 { 8521 return map ? map->name : NULL; 8522 } 8523 8524 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 8525 { 8526 return map->def.type; 8527 } 8528 8529 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 8530 { 8531 if (map->fd >= 0) 8532 return libbpf_err(-EBUSY); 8533 map->def.type = type; 8534 return 0; 8535 } 8536 8537 __u32 bpf_map__map_flags(const struct bpf_map *map) 8538 { 8539 return map->def.map_flags; 8540 } 8541 8542 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 8543 { 8544 if (map->fd >= 0) 8545 return libbpf_err(-EBUSY); 8546 map->def.map_flags = flags; 8547 return 0; 8548 } 8549 8550 __u32 bpf_map__numa_node(const struct bpf_map *map) 8551 { 8552 return map->numa_node; 8553 } 8554 8555 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 8556 { 8557 if (map->fd >= 0) 8558 return libbpf_err(-EBUSY); 8559 map->numa_node = numa_node; 8560 return 0; 8561 } 8562 8563 __u32 bpf_map__key_size(const struct bpf_map *map) 8564 { 8565 return map->def.key_size; 8566 } 8567 8568 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 8569 { 8570 if (map->fd >= 0) 8571 return libbpf_err(-EBUSY); 8572 map->def.key_size = size; 8573 return 0; 8574 } 8575 8576 __u32 bpf_map__value_size(const struct bpf_map *map) 8577 { 8578 return map->def.value_size; 8579 } 8580 8581 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 8582 { 8583 if (map->fd >= 0) 8584 return libbpf_err(-EBUSY); 8585 map->def.value_size = size; 8586 return 0; 8587 } 8588 8589 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 8590 { 8591 return map ? map->btf_key_type_id : 0; 8592 } 8593 8594 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 8595 { 8596 return map ? map->btf_value_type_id : 0; 8597 } 8598 8599 int bpf_map__set_priv(struct bpf_map *map, void *priv, 8600 bpf_map_clear_priv_t clear_priv) 8601 { 8602 if (!map) 8603 return libbpf_err(-EINVAL); 8604 8605 if (map->priv) { 8606 if (map->clear_priv) 8607 map->clear_priv(map, map->priv); 8608 } 8609 8610 map->priv = priv; 8611 map->clear_priv = clear_priv; 8612 return 0; 8613 } 8614 8615 void *bpf_map__priv(const struct bpf_map *map) 8616 { 8617 return map ? map->priv : libbpf_err_ptr(-EINVAL); 8618 } 8619 8620 int bpf_map__set_initial_value(struct bpf_map *map, 8621 const void *data, size_t size) 8622 { 8623 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 8624 size != map->def.value_size || map->fd >= 0) 8625 return libbpf_err(-EINVAL); 8626 8627 memcpy(map->mmaped, data, size); 8628 return 0; 8629 } 8630 8631 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) 8632 { 8633 if (!map->mmaped) 8634 return NULL; 8635 *psize = map->def.value_size; 8636 return map->mmaped; 8637 } 8638 8639 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 8640 { 8641 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 8642 } 8643 8644 bool bpf_map__is_internal(const struct bpf_map *map) 8645 { 8646 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 8647 } 8648 8649 __u32 bpf_map__ifindex(const struct bpf_map *map) 8650 { 8651 return map->map_ifindex; 8652 } 8653 8654 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 8655 { 8656 if (map->fd >= 0) 8657 return libbpf_err(-EBUSY); 8658 map->map_ifindex = ifindex; 8659 return 0; 8660 } 8661 8662 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 8663 { 8664 if (!bpf_map_type__is_map_in_map(map->def.type)) { 8665 pr_warn("error: unsupported map type\n"); 8666 return libbpf_err(-EINVAL); 8667 } 8668 if (map->inner_map_fd != -1) { 8669 pr_warn("error: inner_map_fd already specified\n"); 8670 return libbpf_err(-EINVAL); 8671 } 8672 zfree(&map->inner_map); 8673 map->inner_map_fd = fd; 8674 return 0; 8675 } 8676 8677 static struct bpf_map * 8678 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 8679 { 8680 ssize_t idx; 8681 struct bpf_map *s, *e; 8682 8683 if (!obj || !obj->maps) 8684 return errno = EINVAL, NULL; 8685 8686 s = obj->maps; 8687 e = obj->maps + obj->nr_maps; 8688 8689 if ((m < s) || (m >= e)) { 8690 pr_warn("error in %s: map handler doesn't belong to object\n", 8691 __func__); 8692 return errno = EINVAL, NULL; 8693 } 8694 8695 idx = (m - obj->maps) + i; 8696 if (idx >= obj->nr_maps || idx < 0) 8697 return NULL; 8698 return &obj->maps[idx]; 8699 } 8700 8701 struct bpf_map * 8702 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 8703 { 8704 if (prev == NULL) 8705 return obj->maps; 8706 8707 return __bpf_map__iter(prev, obj, 1); 8708 } 8709 8710 struct bpf_map * 8711 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 8712 { 8713 if (next == NULL) { 8714 if (!obj->nr_maps) 8715 return NULL; 8716 return obj->maps + obj->nr_maps - 1; 8717 } 8718 8719 return __bpf_map__iter(next, obj, -1); 8720 } 8721 8722 struct bpf_map * 8723 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 8724 { 8725 struct bpf_map *pos; 8726 8727 bpf_object__for_each_map(pos, obj) { 8728 if (pos->name && !strcmp(pos->name, name)) 8729 return pos; 8730 } 8731 return errno = ENOENT, NULL; 8732 } 8733 8734 int 8735 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 8736 { 8737 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 8738 } 8739 8740 struct bpf_map * 8741 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 8742 { 8743 return libbpf_err_ptr(-ENOTSUP); 8744 } 8745 8746 long libbpf_get_error(const void *ptr) 8747 { 8748 if (!IS_ERR_OR_NULL(ptr)) 8749 return 0; 8750 8751 if (IS_ERR(ptr)) 8752 errno = -PTR_ERR(ptr); 8753 8754 /* If ptr == NULL, then errno should be already set by the failing 8755 * API, because libbpf never returns NULL on success and it now always 8756 * sets errno on error. So no extra errno handling for ptr == NULL 8757 * case. 8758 */ 8759 return -errno; 8760 } 8761 8762 int bpf_prog_load(const char *file, enum bpf_prog_type type, 8763 struct bpf_object **pobj, int *prog_fd) 8764 { 8765 struct bpf_prog_load_attr attr; 8766 8767 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 8768 attr.file = file; 8769 attr.prog_type = type; 8770 attr.expected_attach_type = 0; 8771 8772 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 8773 } 8774 8775 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 8776 struct bpf_object **pobj, int *prog_fd) 8777 { 8778 struct bpf_object_open_attr open_attr = {}; 8779 struct bpf_program *prog, *first_prog = NULL; 8780 struct bpf_object *obj; 8781 struct bpf_map *map; 8782 int err; 8783 8784 if (!attr) 8785 return libbpf_err(-EINVAL); 8786 if (!attr->file) 8787 return libbpf_err(-EINVAL); 8788 8789 open_attr.file = attr->file; 8790 open_attr.prog_type = attr->prog_type; 8791 8792 obj = bpf_object__open_xattr(&open_attr); 8793 err = libbpf_get_error(obj); 8794 if (err) 8795 return libbpf_err(-ENOENT); 8796 8797 bpf_object__for_each_program(prog, obj) { 8798 enum bpf_attach_type attach_type = attr->expected_attach_type; 8799 /* 8800 * to preserve backwards compatibility, bpf_prog_load treats 8801 * attr->prog_type, if specified, as an override to whatever 8802 * bpf_object__open guessed 8803 */ 8804 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { 8805 bpf_program__set_type(prog, attr->prog_type); 8806 bpf_program__set_expected_attach_type(prog, 8807 attach_type); 8808 } 8809 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { 8810 /* 8811 * we haven't guessed from section name and user 8812 * didn't provide a fallback type, too bad... 8813 */ 8814 bpf_object__close(obj); 8815 return libbpf_err(-EINVAL); 8816 } 8817 8818 prog->prog_ifindex = attr->ifindex; 8819 prog->log_level = attr->log_level; 8820 prog->prog_flags |= attr->prog_flags; 8821 if (!first_prog) 8822 first_prog = prog; 8823 } 8824 8825 bpf_object__for_each_map(map, obj) { 8826 if (!bpf_map__is_offload_neutral(map)) 8827 map->map_ifindex = attr->ifindex; 8828 } 8829 8830 if (!first_prog) { 8831 pr_warn("object file doesn't contain bpf program\n"); 8832 bpf_object__close(obj); 8833 return libbpf_err(-ENOENT); 8834 } 8835 8836 err = bpf_object__load(obj); 8837 if (err) { 8838 bpf_object__close(obj); 8839 return libbpf_err(err); 8840 } 8841 8842 *pobj = obj; 8843 *prog_fd = bpf_program__fd(first_prog); 8844 return 0; 8845 } 8846 8847 struct bpf_link { 8848 int (*detach)(struct bpf_link *link); 8849 void (*dealloc)(struct bpf_link *link); 8850 char *pin_path; /* NULL, if not pinned */ 8851 int fd; /* hook FD, -1 if not applicable */ 8852 bool disconnected; 8853 }; 8854 8855 /* Replace link's underlying BPF program with the new one */ 8856 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 8857 { 8858 int ret; 8859 8860 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 8861 return libbpf_err_errno(ret); 8862 } 8863 8864 /* Release "ownership" of underlying BPF resource (typically, BPF program 8865 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 8866 * link, when destructed through bpf_link__destroy() call won't attempt to 8867 * detach/unregisted that BPF resource. This is useful in situations where, 8868 * say, attached BPF program has to outlive userspace program that attached it 8869 * in the system. Depending on type of BPF program, though, there might be 8870 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 8871 * exit of userspace program doesn't trigger automatic detachment and clean up 8872 * inside the kernel. 8873 */ 8874 void bpf_link__disconnect(struct bpf_link *link) 8875 { 8876 link->disconnected = true; 8877 } 8878 8879 int bpf_link__destroy(struct bpf_link *link) 8880 { 8881 int err = 0; 8882 8883 if (IS_ERR_OR_NULL(link)) 8884 return 0; 8885 8886 if (!link->disconnected && link->detach) 8887 err = link->detach(link); 8888 if (link->pin_path) 8889 free(link->pin_path); 8890 if (link->dealloc) 8891 link->dealloc(link); 8892 else 8893 free(link); 8894 8895 return libbpf_err(err); 8896 } 8897 8898 int bpf_link__fd(const struct bpf_link *link) 8899 { 8900 return link->fd; 8901 } 8902 8903 const char *bpf_link__pin_path(const struct bpf_link *link) 8904 { 8905 return link->pin_path; 8906 } 8907 8908 static int bpf_link__detach_fd(struct bpf_link *link) 8909 { 8910 return libbpf_err_errno(close(link->fd)); 8911 } 8912 8913 struct bpf_link *bpf_link__open(const char *path) 8914 { 8915 struct bpf_link *link; 8916 int fd; 8917 8918 fd = bpf_obj_get(path); 8919 if (fd < 0) { 8920 fd = -errno; 8921 pr_warn("failed to open link at %s: %d\n", path, fd); 8922 return libbpf_err_ptr(fd); 8923 } 8924 8925 link = calloc(1, sizeof(*link)); 8926 if (!link) { 8927 close(fd); 8928 return libbpf_err_ptr(-ENOMEM); 8929 } 8930 link->detach = &bpf_link__detach_fd; 8931 link->fd = fd; 8932 8933 link->pin_path = strdup(path); 8934 if (!link->pin_path) { 8935 bpf_link__destroy(link); 8936 return libbpf_err_ptr(-ENOMEM); 8937 } 8938 8939 return link; 8940 } 8941 8942 int bpf_link__detach(struct bpf_link *link) 8943 { 8944 return bpf_link_detach(link->fd) ? -errno : 0; 8945 } 8946 8947 int bpf_link__pin(struct bpf_link *link, const char *path) 8948 { 8949 int err; 8950 8951 if (link->pin_path) 8952 return libbpf_err(-EBUSY); 8953 err = make_parent_dir(path); 8954 if (err) 8955 return libbpf_err(err); 8956 err = check_path(path); 8957 if (err) 8958 return libbpf_err(err); 8959 8960 link->pin_path = strdup(path); 8961 if (!link->pin_path) 8962 return libbpf_err(-ENOMEM); 8963 8964 if (bpf_obj_pin(link->fd, link->pin_path)) { 8965 err = -errno; 8966 zfree(&link->pin_path); 8967 return libbpf_err(err); 8968 } 8969 8970 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 8971 return 0; 8972 } 8973 8974 int bpf_link__unpin(struct bpf_link *link) 8975 { 8976 int err; 8977 8978 if (!link->pin_path) 8979 return libbpf_err(-EINVAL); 8980 8981 err = unlink(link->pin_path); 8982 if (err != 0) 8983 return -errno; 8984 8985 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 8986 zfree(&link->pin_path); 8987 return 0; 8988 } 8989 8990 struct bpf_link_perf { 8991 struct bpf_link link; 8992 int perf_event_fd; 8993 }; 8994 8995 static int bpf_link_perf_detach(struct bpf_link *link) 8996 { 8997 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 8998 int err = 0; 8999 9000 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 9001 err = -errno; 9002 9003 if (perf_link->perf_event_fd != link->fd) 9004 close(perf_link->perf_event_fd); 9005 close(link->fd); 9006 9007 return libbpf_err(err); 9008 } 9009 9010 static void bpf_link_perf_dealloc(struct bpf_link *link) 9011 { 9012 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 9013 9014 free(perf_link); 9015 } 9016 9017 struct bpf_link *bpf_program__attach_perf_event_opts(struct bpf_program *prog, int pfd, 9018 const struct bpf_perf_event_opts *opts) 9019 { 9020 char errmsg[STRERR_BUFSIZE]; 9021 struct bpf_link_perf *link; 9022 int prog_fd, link_fd = -1, err; 9023 9024 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 9025 return libbpf_err_ptr(-EINVAL); 9026 9027 if (pfd < 0) { 9028 pr_warn("prog '%s': invalid perf event FD %d\n", 9029 prog->name, pfd); 9030 return libbpf_err_ptr(-EINVAL); 9031 } 9032 prog_fd = bpf_program__fd(prog); 9033 if (prog_fd < 0) { 9034 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 9035 prog->name); 9036 return libbpf_err_ptr(-EINVAL); 9037 } 9038 9039 link = calloc(1, sizeof(*link)); 9040 if (!link) 9041 return libbpf_err_ptr(-ENOMEM); 9042 link->link.detach = &bpf_link_perf_detach; 9043 link->link.dealloc = &bpf_link_perf_dealloc; 9044 link->perf_event_fd = pfd; 9045 9046 if (kernel_supports(prog->obj, FEAT_PERF_LINK)) { 9047 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 9048 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 9049 9050 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 9051 if (link_fd < 0) { 9052 err = -errno; 9053 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", 9054 prog->name, pfd, 9055 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9056 goto err_out; 9057 } 9058 link->link.fd = link_fd; 9059 } else { 9060 if (OPTS_GET(opts, bpf_cookie, 0)) { 9061 pr_warn("prog '%s': user context value is not supported\n", prog->name); 9062 err = -EOPNOTSUPP; 9063 goto err_out; 9064 } 9065 9066 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 9067 err = -errno; 9068 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 9069 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9070 if (err == -EPROTO) 9071 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 9072 prog->name, pfd); 9073 goto err_out; 9074 } 9075 link->link.fd = pfd; 9076 } 9077 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 9078 err = -errno; 9079 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 9080 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9081 goto err_out; 9082 } 9083 9084 return &link->link; 9085 err_out: 9086 if (link_fd >= 0) 9087 close(link_fd); 9088 free(link); 9089 return libbpf_err_ptr(err); 9090 } 9091 9092 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, int pfd) 9093 { 9094 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 9095 } 9096 9097 /* 9098 * this function is expected to parse integer in the range of [0, 2^31-1] from 9099 * given file using scanf format string fmt. If actual parsed value is 9100 * negative, the result might be indistinguishable from error 9101 */ 9102 static int parse_uint_from_file(const char *file, const char *fmt) 9103 { 9104 char buf[STRERR_BUFSIZE]; 9105 int err, ret; 9106 FILE *f; 9107 9108 f = fopen(file, "r"); 9109 if (!f) { 9110 err = -errno; 9111 pr_debug("failed to open '%s': %s\n", file, 9112 libbpf_strerror_r(err, buf, sizeof(buf))); 9113 return err; 9114 } 9115 err = fscanf(f, fmt, &ret); 9116 if (err != 1) { 9117 err = err == EOF ? -EIO : -errno; 9118 pr_debug("failed to parse '%s': %s\n", file, 9119 libbpf_strerror_r(err, buf, sizeof(buf))); 9120 fclose(f); 9121 return err; 9122 } 9123 fclose(f); 9124 return ret; 9125 } 9126 9127 static int determine_kprobe_perf_type(void) 9128 { 9129 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 9130 9131 return parse_uint_from_file(file, "%d\n"); 9132 } 9133 9134 static int determine_uprobe_perf_type(void) 9135 { 9136 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 9137 9138 return parse_uint_from_file(file, "%d\n"); 9139 } 9140 9141 static int determine_kprobe_retprobe_bit(void) 9142 { 9143 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 9144 9145 return parse_uint_from_file(file, "config:%d\n"); 9146 } 9147 9148 static int determine_uprobe_retprobe_bit(void) 9149 { 9150 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 9151 9152 return parse_uint_from_file(file, "config:%d\n"); 9153 } 9154 9155 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 9156 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 9157 9158 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 9159 uint64_t offset, int pid, size_t ref_ctr_off) 9160 { 9161 struct perf_event_attr attr = {}; 9162 char errmsg[STRERR_BUFSIZE]; 9163 int type, pfd, err; 9164 9165 if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 9166 return -EINVAL; 9167 9168 type = uprobe ? determine_uprobe_perf_type() 9169 : determine_kprobe_perf_type(); 9170 if (type < 0) { 9171 pr_warn("failed to determine %s perf type: %s\n", 9172 uprobe ? "uprobe" : "kprobe", 9173 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 9174 return type; 9175 } 9176 if (retprobe) { 9177 int bit = uprobe ? determine_uprobe_retprobe_bit() 9178 : determine_kprobe_retprobe_bit(); 9179 9180 if (bit < 0) { 9181 pr_warn("failed to determine %s retprobe bit: %s\n", 9182 uprobe ? "uprobe" : "kprobe", 9183 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 9184 return bit; 9185 } 9186 attr.config |= 1 << bit; 9187 } 9188 attr.size = sizeof(attr); 9189 attr.type = type; 9190 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 9191 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 9192 attr.config2 = offset; /* kprobe_addr or probe_offset */ 9193 9194 /* pid filter is meaningful only for uprobes */ 9195 pfd = syscall(__NR_perf_event_open, &attr, 9196 pid < 0 ? -1 : pid /* pid */, 9197 pid == -1 ? 0 : -1 /* cpu */, 9198 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9199 if (pfd < 0) { 9200 err = -errno; 9201 pr_warn("%s perf_event_open() failed: %s\n", 9202 uprobe ? "uprobe" : "kprobe", 9203 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9204 return err; 9205 } 9206 return pfd; 9207 } 9208 9209 struct bpf_link * 9210 bpf_program__attach_kprobe_opts(struct bpf_program *prog, 9211 const char *func_name, 9212 const struct bpf_kprobe_opts *opts) 9213 { 9214 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9215 char errmsg[STRERR_BUFSIZE]; 9216 struct bpf_link *link; 9217 unsigned long offset; 9218 bool retprobe; 9219 int pfd, err; 9220 9221 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 9222 return libbpf_err_ptr(-EINVAL); 9223 9224 retprobe = OPTS_GET(opts, retprobe, false); 9225 offset = OPTS_GET(opts, offset, 0); 9226 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9227 9228 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 9229 offset, -1 /* pid */, 0 /* ref_ctr_off */); 9230 if (pfd < 0) { 9231 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n", 9232 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 9233 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9234 return libbpf_err_ptr(pfd); 9235 } 9236 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9237 err = libbpf_get_error(link); 9238 if (err) { 9239 close(pfd); 9240 pr_warn("prog '%s': failed to attach to %s '%s': %s\n", 9241 prog->name, retprobe ? "kretprobe" : "kprobe", func_name, 9242 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9243 return libbpf_err_ptr(err); 9244 } 9245 return link; 9246 } 9247 9248 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 9249 bool retprobe, 9250 const char *func_name) 9251 { 9252 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 9253 .retprobe = retprobe, 9254 ); 9255 9256 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 9257 } 9258 9259 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 9260 struct bpf_program *prog) 9261 { 9262 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 9263 unsigned long offset = 0; 9264 struct bpf_link *link; 9265 const char *func_name; 9266 char *func; 9267 int n, err; 9268 9269 func_name = prog->sec_name + sec->len; 9270 opts.retprobe = strcmp(sec->sec, "kretprobe/") == 0; 9271 9272 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 9273 if (n < 1) { 9274 err = -EINVAL; 9275 pr_warn("kprobe name is invalid: %s\n", func_name); 9276 return libbpf_err_ptr(err); 9277 } 9278 if (opts.retprobe && offset != 0) { 9279 free(func); 9280 err = -EINVAL; 9281 pr_warn("kretprobes do not support offset specification\n"); 9282 return libbpf_err_ptr(err); 9283 } 9284 9285 opts.offset = offset; 9286 link = bpf_program__attach_kprobe_opts(prog, func, &opts); 9287 free(func); 9288 return link; 9289 } 9290 9291 LIBBPF_API struct bpf_link * 9292 bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid, 9293 const char *binary_path, size_t func_offset, 9294 const struct bpf_uprobe_opts *opts) 9295 { 9296 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9297 char errmsg[STRERR_BUFSIZE]; 9298 struct bpf_link *link; 9299 size_t ref_ctr_off; 9300 int pfd, err; 9301 bool retprobe; 9302 9303 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 9304 return libbpf_err_ptr(-EINVAL); 9305 9306 retprobe = OPTS_GET(opts, retprobe, false); 9307 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 9308 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9309 9310 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 9311 func_offset, pid, ref_ctr_off); 9312 if (pfd < 0) { 9313 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 9314 prog->name, retprobe ? "uretprobe" : "uprobe", 9315 binary_path, func_offset, 9316 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9317 return libbpf_err_ptr(pfd); 9318 } 9319 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9320 err = libbpf_get_error(link); 9321 if (err) { 9322 close(pfd); 9323 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 9324 prog->name, retprobe ? "uretprobe" : "uprobe", 9325 binary_path, func_offset, 9326 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9327 return libbpf_err_ptr(err); 9328 } 9329 return link; 9330 } 9331 9332 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 9333 bool retprobe, pid_t pid, 9334 const char *binary_path, 9335 size_t func_offset) 9336 { 9337 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 9338 9339 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 9340 } 9341 9342 static int determine_tracepoint_id(const char *tp_category, 9343 const char *tp_name) 9344 { 9345 char file[PATH_MAX]; 9346 int ret; 9347 9348 ret = snprintf(file, sizeof(file), 9349 "/sys/kernel/debug/tracing/events/%s/%s/id", 9350 tp_category, tp_name); 9351 if (ret < 0) 9352 return -errno; 9353 if (ret >= sizeof(file)) { 9354 pr_debug("tracepoint %s/%s path is too long\n", 9355 tp_category, tp_name); 9356 return -E2BIG; 9357 } 9358 return parse_uint_from_file(file, "%d\n"); 9359 } 9360 9361 static int perf_event_open_tracepoint(const char *tp_category, 9362 const char *tp_name) 9363 { 9364 struct perf_event_attr attr = {}; 9365 char errmsg[STRERR_BUFSIZE]; 9366 int tp_id, pfd, err; 9367 9368 tp_id = determine_tracepoint_id(tp_category, tp_name); 9369 if (tp_id < 0) { 9370 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 9371 tp_category, tp_name, 9372 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 9373 return tp_id; 9374 } 9375 9376 attr.type = PERF_TYPE_TRACEPOINT; 9377 attr.size = sizeof(attr); 9378 attr.config = tp_id; 9379 9380 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 9381 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 9382 if (pfd < 0) { 9383 err = -errno; 9384 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 9385 tp_category, tp_name, 9386 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9387 return err; 9388 } 9389 return pfd; 9390 } 9391 9392 struct bpf_link *bpf_program__attach_tracepoint_opts(struct bpf_program *prog, 9393 const char *tp_category, 9394 const char *tp_name, 9395 const struct bpf_tracepoint_opts *opts) 9396 { 9397 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 9398 char errmsg[STRERR_BUFSIZE]; 9399 struct bpf_link *link; 9400 int pfd, err; 9401 9402 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 9403 return libbpf_err_ptr(-EINVAL); 9404 9405 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 9406 9407 pfd = perf_event_open_tracepoint(tp_category, tp_name); 9408 if (pfd < 0) { 9409 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 9410 prog->name, tp_category, tp_name, 9411 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9412 return libbpf_err_ptr(pfd); 9413 } 9414 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 9415 err = libbpf_get_error(link); 9416 if (err) { 9417 close(pfd); 9418 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 9419 prog->name, tp_category, tp_name, 9420 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 9421 return libbpf_err_ptr(err); 9422 } 9423 return link; 9424 } 9425 9426 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 9427 const char *tp_category, 9428 const char *tp_name) 9429 { 9430 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 9431 } 9432 9433 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 9434 struct bpf_program *prog) 9435 { 9436 char *sec_name, *tp_cat, *tp_name; 9437 struct bpf_link *link; 9438 9439 sec_name = strdup(prog->sec_name); 9440 if (!sec_name) 9441 return libbpf_err_ptr(-ENOMEM); 9442 9443 /* extract "tp/<category>/<name>" */ 9444 tp_cat = sec_name + sec->len; 9445 tp_name = strchr(tp_cat, '/'); 9446 if (!tp_name) { 9447 free(sec_name); 9448 return libbpf_err_ptr(-EINVAL); 9449 } 9450 *tp_name = '\0'; 9451 tp_name++; 9452 9453 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 9454 free(sec_name); 9455 return link; 9456 } 9457 9458 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 9459 const char *tp_name) 9460 { 9461 char errmsg[STRERR_BUFSIZE]; 9462 struct bpf_link *link; 9463 int prog_fd, pfd; 9464 9465 prog_fd = bpf_program__fd(prog); 9466 if (prog_fd < 0) { 9467 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9468 return libbpf_err_ptr(-EINVAL); 9469 } 9470 9471 link = calloc(1, sizeof(*link)); 9472 if (!link) 9473 return libbpf_err_ptr(-ENOMEM); 9474 link->detach = &bpf_link__detach_fd; 9475 9476 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 9477 if (pfd < 0) { 9478 pfd = -errno; 9479 free(link); 9480 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 9481 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9482 return libbpf_err_ptr(pfd); 9483 } 9484 link->fd = pfd; 9485 return link; 9486 } 9487 9488 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 9489 struct bpf_program *prog) 9490 { 9491 const char *tp_name = prog->sec_name + sec->len; 9492 9493 return bpf_program__attach_raw_tracepoint(prog, tp_name); 9494 } 9495 9496 /* Common logic for all BPF program types that attach to a btf_id */ 9497 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog) 9498 { 9499 char errmsg[STRERR_BUFSIZE]; 9500 struct bpf_link *link; 9501 int prog_fd, pfd; 9502 9503 prog_fd = bpf_program__fd(prog); 9504 if (prog_fd < 0) { 9505 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9506 return libbpf_err_ptr(-EINVAL); 9507 } 9508 9509 link = calloc(1, sizeof(*link)); 9510 if (!link) 9511 return libbpf_err_ptr(-ENOMEM); 9512 link->detach = &bpf_link__detach_fd; 9513 9514 pfd = bpf_raw_tracepoint_open(NULL, prog_fd); 9515 if (pfd < 0) { 9516 pfd = -errno; 9517 free(link); 9518 pr_warn("prog '%s': failed to attach: %s\n", 9519 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 9520 return libbpf_err_ptr(pfd); 9521 } 9522 link->fd = pfd; 9523 return (struct bpf_link *)link; 9524 } 9525 9526 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog) 9527 { 9528 return bpf_program__attach_btf_id(prog); 9529 } 9530 9531 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog) 9532 { 9533 return bpf_program__attach_btf_id(prog); 9534 } 9535 9536 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 9537 struct bpf_program *prog) 9538 { 9539 return bpf_program__attach_trace(prog); 9540 } 9541 9542 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 9543 struct bpf_program *prog) 9544 { 9545 return bpf_program__attach_lsm(prog); 9546 } 9547 9548 static struct bpf_link * 9549 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id, 9550 const char *target_name) 9551 { 9552 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, 9553 .target_btf_id = btf_id); 9554 enum bpf_attach_type attach_type; 9555 char errmsg[STRERR_BUFSIZE]; 9556 struct bpf_link *link; 9557 int prog_fd, link_fd; 9558 9559 prog_fd = bpf_program__fd(prog); 9560 if (prog_fd < 0) { 9561 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9562 return libbpf_err_ptr(-EINVAL); 9563 } 9564 9565 link = calloc(1, sizeof(*link)); 9566 if (!link) 9567 return libbpf_err_ptr(-ENOMEM); 9568 link->detach = &bpf_link__detach_fd; 9569 9570 attach_type = bpf_program__get_expected_attach_type(prog); 9571 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); 9572 if (link_fd < 0) { 9573 link_fd = -errno; 9574 free(link); 9575 pr_warn("prog '%s': failed to attach to %s: %s\n", 9576 prog->name, target_name, 9577 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 9578 return libbpf_err_ptr(link_fd); 9579 } 9580 link->fd = link_fd; 9581 return link; 9582 } 9583 9584 struct bpf_link * 9585 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd) 9586 { 9587 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup"); 9588 } 9589 9590 struct bpf_link * 9591 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd) 9592 { 9593 return bpf_program__attach_fd(prog, netns_fd, 0, "netns"); 9594 } 9595 9596 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex) 9597 { 9598 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 9599 return bpf_program__attach_fd(prog, ifindex, 0, "xdp"); 9600 } 9601 9602 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog, 9603 int target_fd, 9604 const char *attach_func_name) 9605 { 9606 int btf_id; 9607 9608 if (!!target_fd != !!attach_func_name) { 9609 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 9610 prog->name); 9611 return libbpf_err_ptr(-EINVAL); 9612 } 9613 9614 if (prog->type != BPF_PROG_TYPE_EXT) { 9615 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 9616 prog->name); 9617 return libbpf_err_ptr(-EINVAL); 9618 } 9619 9620 if (target_fd) { 9621 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 9622 if (btf_id < 0) 9623 return libbpf_err_ptr(btf_id); 9624 9625 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace"); 9626 } else { 9627 /* no target, so use raw_tracepoint_open for compatibility 9628 * with old kernels 9629 */ 9630 return bpf_program__attach_trace(prog); 9631 } 9632 } 9633 9634 struct bpf_link * 9635 bpf_program__attach_iter(struct bpf_program *prog, 9636 const struct bpf_iter_attach_opts *opts) 9637 { 9638 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 9639 char errmsg[STRERR_BUFSIZE]; 9640 struct bpf_link *link; 9641 int prog_fd, link_fd; 9642 __u32 target_fd = 0; 9643 9644 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 9645 return libbpf_err_ptr(-EINVAL); 9646 9647 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 9648 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 9649 9650 prog_fd = bpf_program__fd(prog); 9651 if (prog_fd < 0) { 9652 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 9653 return libbpf_err_ptr(-EINVAL); 9654 } 9655 9656 link = calloc(1, sizeof(*link)); 9657 if (!link) 9658 return libbpf_err_ptr(-ENOMEM); 9659 link->detach = &bpf_link__detach_fd; 9660 9661 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 9662 &link_create_opts); 9663 if (link_fd < 0) { 9664 link_fd = -errno; 9665 free(link); 9666 pr_warn("prog '%s': failed to attach to iterator: %s\n", 9667 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 9668 return libbpf_err_ptr(link_fd); 9669 } 9670 link->fd = link_fd; 9671 return link; 9672 } 9673 9674 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 9675 struct bpf_program *prog) 9676 { 9677 return bpf_program__attach_iter(prog, NULL); 9678 } 9679 9680 struct bpf_link *bpf_program__attach(struct bpf_program *prog) 9681 { 9682 const struct bpf_sec_def *sec_def; 9683 9684 sec_def = find_sec_def(prog->sec_name); 9685 if (!sec_def || !sec_def->attach_fn) 9686 return libbpf_err_ptr(-ESRCH); 9687 9688 return sec_def->attach_fn(sec_def, prog); 9689 } 9690 9691 static int bpf_link__detach_struct_ops(struct bpf_link *link) 9692 { 9693 __u32 zero = 0; 9694 9695 if (bpf_map_delete_elem(link->fd, &zero)) 9696 return -errno; 9697 9698 return 0; 9699 } 9700 9701 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map) 9702 { 9703 struct bpf_struct_ops *st_ops; 9704 struct bpf_link *link; 9705 __u32 i, zero = 0; 9706 int err; 9707 9708 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 9709 return libbpf_err_ptr(-EINVAL); 9710 9711 link = calloc(1, sizeof(*link)); 9712 if (!link) 9713 return libbpf_err_ptr(-EINVAL); 9714 9715 st_ops = map->st_ops; 9716 for (i = 0; i < btf_vlen(st_ops->type); i++) { 9717 struct bpf_program *prog = st_ops->progs[i]; 9718 void *kern_data; 9719 int prog_fd; 9720 9721 if (!prog) 9722 continue; 9723 9724 prog_fd = bpf_program__fd(prog); 9725 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 9726 *(unsigned long *)kern_data = prog_fd; 9727 } 9728 9729 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); 9730 if (err) { 9731 err = -errno; 9732 free(link); 9733 return libbpf_err_ptr(err); 9734 } 9735 9736 link->detach = bpf_link__detach_struct_ops; 9737 link->fd = map->fd; 9738 9739 return link; 9740 } 9741 9742 enum bpf_perf_event_ret 9743 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 9744 void **copy_mem, size_t *copy_size, 9745 bpf_perf_event_print_t fn, void *private_data) 9746 { 9747 struct perf_event_mmap_page *header = mmap_mem; 9748 __u64 data_head = ring_buffer_read_head(header); 9749 __u64 data_tail = header->data_tail; 9750 void *base = ((__u8 *)header) + page_size; 9751 int ret = LIBBPF_PERF_EVENT_CONT; 9752 struct perf_event_header *ehdr; 9753 size_t ehdr_size; 9754 9755 while (data_head != data_tail) { 9756 ehdr = base + (data_tail & (mmap_size - 1)); 9757 ehdr_size = ehdr->size; 9758 9759 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 9760 void *copy_start = ehdr; 9761 size_t len_first = base + mmap_size - copy_start; 9762 size_t len_secnd = ehdr_size - len_first; 9763 9764 if (*copy_size < ehdr_size) { 9765 free(*copy_mem); 9766 *copy_mem = malloc(ehdr_size); 9767 if (!*copy_mem) { 9768 *copy_size = 0; 9769 ret = LIBBPF_PERF_EVENT_ERROR; 9770 break; 9771 } 9772 *copy_size = ehdr_size; 9773 } 9774 9775 memcpy(*copy_mem, copy_start, len_first); 9776 memcpy(*copy_mem + len_first, base, len_secnd); 9777 ehdr = *copy_mem; 9778 } 9779 9780 ret = fn(ehdr, private_data); 9781 data_tail += ehdr_size; 9782 if (ret != LIBBPF_PERF_EVENT_CONT) 9783 break; 9784 } 9785 9786 ring_buffer_write_tail(header, data_tail); 9787 return libbpf_err(ret); 9788 } 9789 9790 struct perf_buffer; 9791 9792 struct perf_buffer_params { 9793 struct perf_event_attr *attr; 9794 /* if event_cb is specified, it takes precendence */ 9795 perf_buffer_event_fn event_cb; 9796 /* sample_cb and lost_cb are higher-level common-case callbacks */ 9797 perf_buffer_sample_fn sample_cb; 9798 perf_buffer_lost_fn lost_cb; 9799 void *ctx; 9800 int cpu_cnt; 9801 int *cpus; 9802 int *map_keys; 9803 }; 9804 9805 struct perf_cpu_buf { 9806 struct perf_buffer *pb; 9807 void *base; /* mmap()'ed memory */ 9808 void *buf; /* for reconstructing segmented data */ 9809 size_t buf_size; 9810 int fd; 9811 int cpu; 9812 int map_key; 9813 }; 9814 9815 struct perf_buffer { 9816 perf_buffer_event_fn event_cb; 9817 perf_buffer_sample_fn sample_cb; 9818 perf_buffer_lost_fn lost_cb; 9819 void *ctx; /* passed into callbacks */ 9820 9821 size_t page_size; 9822 size_t mmap_size; 9823 struct perf_cpu_buf **cpu_bufs; 9824 struct epoll_event *events; 9825 int cpu_cnt; /* number of allocated CPU buffers */ 9826 int epoll_fd; /* perf event FD */ 9827 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 9828 }; 9829 9830 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 9831 struct perf_cpu_buf *cpu_buf) 9832 { 9833 if (!cpu_buf) 9834 return; 9835 if (cpu_buf->base && 9836 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 9837 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 9838 if (cpu_buf->fd >= 0) { 9839 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 9840 close(cpu_buf->fd); 9841 } 9842 free(cpu_buf->buf); 9843 free(cpu_buf); 9844 } 9845 9846 void perf_buffer__free(struct perf_buffer *pb) 9847 { 9848 int i; 9849 9850 if (IS_ERR_OR_NULL(pb)) 9851 return; 9852 if (pb->cpu_bufs) { 9853 for (i = 0; i < pb->cpu_cnt; i++) { 9854 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 9855 9856 if (!cpu_buf) 9857 continue; 9858 9859 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 9860 perf_buffer__free_cpu_buf(pb, cpu_buf); 9861 } 9862 free(pb->cpu_bufs); 9863 } 9864 if (pb->epoll_fd >= 0) 9865 close(pb->epoll_fd); 9866 free(pb->events); 9867 free(pb); 9868 } 9869 9870 static struct perf_cpu_buf * 9871 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 9872 int cpu, int map_key) 9873 { 9874 struct perf_cpu_buf *cpu_buf; 9875 char msg[STRERR_BUFSIZE]; 9876 int err; 9877 9878 cpu_buf = calloc(1, sizeof(*cpu_buf)); 9879 if (!cpu_buf) 9880 return ERR_PTR(-ENOMEM); 9881 9882 cpu_buf->pb = pb; 9883 cpu_buf->cpu = cpu; 9884 cpu_buf->map_key = map_key; 9885 9886 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 9887 -1, PERF_FLAG_FD_CLOEXEC); 9888 if (cpu_buf->fd < 0) { 9889 err = -errno; 9890 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 9891 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 9892 goto error; 9893 } 9894 9895 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 9896 PROT_READ | PROT_WRITE, MAP_SHARED, 9897 cpu_buf->fd, 0); 9898 if (cpu_buf->base == MAP_FAILED) { 9899 cpu_buf->base = NULL; 9900 err = -errno; 9901 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 9902 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 9903 goto error; 9904 } 9905 9906 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 9907 err = -errno; 9908 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 9909 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 9910 goto error; 9911 } 9912 9913 return cpu_buf; 9914 9915 error: 9916 perf_buffer__free_cpu_buf(pb, cpu_buf); 9917 return (struct perf_cpu_buf *)ERR_PTR(err); 9918 } 9919 9920 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 9921 struct perf_buffer_params *p); 9922 9923 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 9924 const struct perf_buffer_opts *opts) 9925 { 9926 struct perf_buffer_params p = {}; 9927 struct perf_event_attr attr = { 0, }; 9928 9929 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 9930 attr.type = PERF_TYPE_SOFTWARE; 9931 attr.sample_type = PERF_SAMPLE_RAW; 9932 attr.sample_period = 1; 9933 attr.wakeup_events = 1; 9934 9935 p.attr = &attr; 9936 p.sample_cb = opts ? opts->sample_cb : NULL; 9937 p.lost_cb = opts ? opts->lost_cb : NULL; 9938 p.ctx = opts ? opts->ctx : NULL; 9939 9940 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 9941 } 9942 9943 struct perf_buffer * 9944 perf_buffer__new_raw(int map_fd, size_t page_cnt, 9945 const struct perf_buffer_raw_opts *opts) 9946 { 9947 struct perf_buffer_params p = {}; 9948 9949 p.attr = opts->attr; 9950 p.event_cb = opts->event_cb; 9951 p.ctx = opts->ctx; 9952 p.cpu_cnt = opts->cpu_cnt; 9953 p.cpus = opts->cpus; 9954 p.map_keys = opts->map_keys; 9955 9956 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 9957 } 9958 9959 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 9960 struct perf_buffer_params *p) 9961 { 9962 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 9963 struct bpf_map_info map; 9964 char msg[STRERR_BUFSIZE]; 9965 struct perf_buffer *pb; 9966 bool *online = NULL; 9967 __u32 map_info_len; 9968 int err, i, j, n; 9969 9970 if (page_cnt & (page_cnt - 1)) { 9971 pr_warn("page count should be power of two, but is %zu\n", 9972 page_cnt); 9973 return ERR_PTR(-EINVAL); 9974 } 9975 9976 /* best-effort sanity checks */ 9977 memset(&map, 0, sizeof(map)); 9978 map_info_len = sizeof(map); 9979 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 9980 if (err) { 9981 err = -errno; 9982 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 9983 * -EBADFD, -EFAULT, or -E2BIG on real error 9984 */ 9985 if (err != -EINVAL) { 9986 pr_warn("failed to get map info for map FD %d: %s\n", 9987 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 9988 return ERR_PTR(err); 9989 } 9990 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 9991 map_fd); 9992 } else { 9993 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 9994 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 9995 map.name); 9996 return ERR_PTR(-EINVAL); 9997 } 9998 } 9999 10000 pb = calloc(1, sizeof(*pb)); 10001 if (!pb) 10002 return ERR_PTR(-ENOMEM); 10003 10004 pb->event_cb = p->event_cb; 10005 pb->sample_cb = p->sample_cb; 10006 pb->lost_cb = p->lost_cb; 10007 pb->ctx = p->ctx; 10008 10009 pb->page_size = getpagesize(); 10010 pb->mmap_size = pb->page_size * page_cnt; 10011 pb->map_fd = map_fd; 10012 10013 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 10014 if (pb->epoll_fd < 0) { 10015 err = -errno; 10016 pr_warn("failed to create epoll instance: %s\n", 10017 libbpf_strerror_r(err, msg, sizeof(msg))); 10018 goto error; 10019 } 10020 10021 if (p->cpu_cnt > 0) { 10022 pb->cpu_cnt = p->cpu_cnt; 10023 } else { 10024 pb->cpu_cnt = libbpf_num_possible_cpus(); 10025 if (pb->cpu_cnt < 0) { 10026 err = pb->cpu_cnt; 10027 goto error; 10028 } 10029 if (map.max_entries && map.max_entries < pb->cpu_cnt) 10030 pb->cpu_cnt = map.max_entries; 10031 } 10032 10033 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 10034 if (!pb->events) { 10035 err = -ENOMEM; 10036 pr_warn("failed to allocate events: out of memory\n"); 10037 goto error; 10038 } 10039 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 10040 if (!pb->cpu_bufs) { 10041 err = -ENOMEM; 10042 pr_warn("failed to allocate buffers: out of memory\n"); 10043 goto error; 10044 } 10045 10046 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 10047 if (err) { 10048 pr_warn("failed to get online CPU mask: %d\n", err); 10049 goto error; 10050 } 10051 10052 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 10053 struct perf_cpu_buf *cpu_buf; 10054 int cpu, map_key; 10055 10056 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 10057 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 10058 10059 /* in case user didn't explicitly requested particular CPUs to 10060 * be attached to, skip offline/not present CPUs 10061 */ 10062 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 10063 continue; 10064 10065 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 10066 if (IS_ERR(cpu_buf)) { 10067 err = PTR_ERR(cpu_buf); 10068 goto error; 10069 } 10070 10071 pb->cpu_bufs[j] = cpu_buf; 10072 10073 err = bpf_map_update_elem(pb->map_fd, &map_key, 10074 &cpu_buf->fd, 0); 10075 if (err) { 10076 err = -errno; 10077 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 10078 cpu, map_key, cpu_buf->fd, 10079 libbpf_strerror_r(err, msg, sizeof(msg))); 10080 goto error; 10081 } 10082 10083 pb->events[j].events = EPOLLIN; 10084 pb->events[j].data.ptr = cpu_buf; 10085 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 10086 &pb->events[j]) < 0) { 10087 err = -errno; 10088 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 10089 cpu, cpu_buf->fd, 10090 libbpf_strerror_r(err, msg, sizeof(msg))); 10091 goto error; 10092 } 10093 j++; 10094 } 10095 pb->cpu_cnt = j; 10096 free(online); 10097 10098 return pb; 10099 10100 error: 10101 free(online); 10102 if (pb) 10103 perf_buffer__free(pb); 10104 return ERR_PTR(err); 10105 } 10106 10107 struct perf_sample_raw { 10108 struct perf_event_header header; 10109 uint32_t size; 10110 char data[]; 10111 }; 10112 10113 struct perf_sample_lost { 10114 struct perf_event_header header; 10115 uint64_t id; 10116 uint64_t lost; 10117 uint64_t sample_id; 10118 }; 10119 10120 static enum bpf_perf_event_ret 10121 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 10122 { 10123 struct perf_cpu_buf *cpu_buf = ctx; 10124 struct perf_buffer *pb = cpu_buf->pb; 10125 void *data = e; 10126 10127 /* user wants full control over parsing perf event */ 10128 if (pb->event_cb) 10129 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 10130 10131 switch (e->type) { 10132 case PERF_RECORD_SAMPLE: { 10133 struct perf_sample_raw *s = data; 10134 10135 if (pb->sample_cb) 10136 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 10137 break; 10138 } 10139 case PERF_RECORD_LOST: { 10140 struct perf_sample_lost *s = data; 10141 10142 if (pb->lost_cb) 10143 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 10144 break; 10145 } 10146 default: 10147 pr_warn("unknown perf sample type %d\n", e->type); 10148 return LIBBPF_PERF_EVENT_ERROR; 10149 } 10150 return LIBBPF_PERF_EVENT_CONT; 10151 } 10152 10153 static int perf_buffer__process_records(struct perf_buffer *pb, 10154 struct perf_cpu_buf *cpu_buf) 10155 { 10156 enum bpf_perf_event_ret ret; 10157 10158 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 10159 pb->page_size, &cpu_buf->buf, 10160 &cpu_buf->buf_size, 10161 perf_buffer__process_record, cpu_buf); 10162 if (ret != LIBBPF_PERF_EVENT_CONT) 10163 return ret; 10164 return 0; 10165 } 10166 10167 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 10168 { 10169 return pb->epoll_fd; 10170 } 10171 10172 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 10173 { 10174 int i, cnt, err; 10175 10176 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 10177 if (cnt < 0) 10178 return -errno; 10179 10180 for (i = 0; i < cnt; i++) { 10181 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 10182 10183 err = perf_buffer__process_records(pb, cpu_buf); 10184 if (err) { 10185 pr_warn("error while processing records: %d\n", err); 10186 return libbpf_err(err); 10187 } 10188 } 10189 return cnt; 10190 } 10191 10192 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 10193 * manager. 10194 */ 10195 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 10196 { 10197 return pb->cpu_cnt; 10198 } 10199 10200 /* 10201 * Return perf_event FD of a ring buffer in *buf_idx* slot of 10202 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 10203 * select()/poll()/epoll() Linux syscalls. 10204 */ 10205 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 10206 { 10207 struct perf_cpu_buf *cpu_buf; 10208 10209 if (buf_idx >= pb->cpu_cnt) 10210 return libbpf_err(-EINVAL); 10211 10212 cpu_buf = pb->cpu_bufs[buf_idx]; 10213 if (!cpu_buf) 10214 return libbpf_err(-ENOENT); 10215 10216 return cpu_buf->fd; 10217 } 10218 10219 /* 10220 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 10221 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 10222 * consume, do nothing and return success. 10223 * Returns: 10224 * - 0 on success; 10225 * - <0 on failure. 10226 */ 10227 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 10228 { 10229 struct perf_cpu_buf *cpu_buf; 10230 10231 if (buf_idx >= pb->cpu_cnt) 10232 return libbpf_err(-EINVAL); 10233 10234 cpu_buf = pb->cpu_bufs[buf_idx]; 10235 if (!cpu_buf) 10236 return libbpf_err(-ENOENT); 10237 10238 return perf_buffer__process_records(pb, cpu_buf); 10239 } 10240 10241 int perf_buffer__consume(struct perf_buffer *pb) 10242 { 10243 int i, err; 10244 10245 for (i = 0; i < pb->cpu_cnt; i++) { 10246 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 10247 10248 if (!cpu_buf) 10249 continue; 10250 10251 err = perf_buffer__process_records(pb, cpu_buf); 10252 if (err) { 10253 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 10254 return libbpf_err(err); 10255 } 10256 } 10257 return 0; 10258 } 10259 10260 struct bpf_prog_info_array_desc { 10261 int array_offset; /* e.g. offset of jited_prog_insns */ 10262 int count_offset; /* e.g. offset of jited_prog_len */ 10263 int size_offset; /* > 0: offset of rec size, 10264 * < 0: fix size of -size_offset 10265 */ 10266 }; 10267 10268 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 10269 [BPF_PROG_INFO_JITED_INSNS] = { 10270 offsetof(struct bpf_prog_info, jited_prog_insns), 10271 offsetof(struct bpf_prog_info, jited_prog_len), 10272 -1, 10273 }, 10274 [BPF_PROG_INFO_XLATED_INSNS] = { 10275 offsetof(struct bpf_prog_info, xlated_prog_insns), 10276 offsetof(struct bpf_prog_info, xlated_prog_len), 10277 -1, 10278 }, 10279 [BPF_PROG_INFO_MAP_IDS] = { 10280 offsetof(struct bpf_prog_info, map_ids), 10281 offsetof(struct bpf_prog_info, nr_map_ids), 10282 -(int)sizeof(__u32), 10283 }, 10284 [BPF_PROG_INFO_JITED_KSYMS] = { 10285 offsetof(struct bpf_prog_info, jited_ksyms), 10286 offsetof(struct bpf_prog_info, nr_jited_ksyms), 10287 -(int)sizeof(__u64), 10288 }, 10289 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 10290 offsetof(struct bpf_prog_info, jited_func_lens), 10291 offsetof(struct bpf_prog_info, nr_jited_func_lens), 10292 -(int)sizeof(__u32), 10293 }, 10294 [BPF_PROG_INFO_FUNC_INFO] = { 10295 offsetof(struct bpf_prog_info, func_info), 10296 offsetof(struct bpf_prog_info, nr_func_info), 10297 offsetof(struct bpf_prog_info, func_info_rec_size), 10298 }, 10299 [BPF_PROG_INFO_LINE_INFO] = { 10300 offsetof(struct bpf_prog_info, line_info), 10301 offsetof(struct bpf_prog_info, nr_line_info), 10302 offsetof(struct bpf_prog_info, line_info_rec_size), 10303 }, 10304 [BPF_PROG_INFO_JITED_LINE_INFO] = { 10305 offsetof(struct bpf_prog_info, jited_line_info), 10306 offsetof(struct bpf_prog_info, nr_jited_line_info), 10307 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 10308 }, 10309 [BPF_PROG_INFO_PROG_TAGS] = { 10310 offsetof(struct bpf_prog_info, prog_tags), 10311 offsetof(struct bpf_prog_info, nr_prog_tags), 10312 -(int)sizeof(__u8) * BPF_TAG_SIZE, 10313 }, 10314 10315 }; 10316 10317 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 10318 int offset) 10319 { 10320 __u32 *array = (__u32 *)info; 10321 10322 if (offset >= 0) 10323 return array[offset / sizeof(__u32)]; 10324 return -(int)offset; 10325 } 10326 10327 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 10328 int offset) 10329 { 10330 __u64 *array = (__u64 *)info; 10331 10332 if (offset >= 0) 10333 return array[offset / sizeof(__u64)]; 10334 return -(int)offset; 10335 } 10336 10337 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 10338 __u32 val) 10339 { 10340 __u32 *array = (__u32 *)info; 10341 10342 if (offset >= 0) 10343 array[offset / sizeof(__u32)] = val; 10344 } 10345 10346 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 10347 __u64 val) 10348 { 10349 __u64 *array = (__u64 *)info; 10350 10351 if (offset >= 0) 10352 array[offset / sizeof(__u64)] = val; 10353 } 10354 10355 struct bpf_prog_info_linear * 10356 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 10357 { 10358 struct bpf_prog_info_linear *info_linear; 10359 struct bpf_prog_info info = {}; 10360 __u32 info_len = sizeof(info); 10361 __u32 data_len = 0; 10362 int i, err; 10363 void *ptr; 10364 10365 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 10366 return libbpf_err_ptr(-EINVAL); 10367 10368 /* step 1: get array dimensions */ 10369 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 10370 if (err) { 10371 pr_debug("can't get prog info: %s", strerror(errno)); 10372 return libbpf_err_ptr(-EFAULT); 10373 } 10374 10375 /* step 2: calculate total size of all arrays */ 10376 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10377 bool include_array = (arrays & (1UL << i)) > 0; 10378 struct bpf_prog_info_array_desc *desc; 10379 __u32 count, size; 10380 10381 desc = bpf_prog_info_array_desc + i; 10382 10383 /* kernel is too old to support this field */ 10384 if (info_len < desc->array_offset + sizeof(__u32) || 10385 info_len < desc->count_offset + sizeof(__u32) || 10386 (desc->size_offset > 0 && info_len < desc->size_offset)) 10387 include_array = false; 10388 10389 if (!include_array) { 10390 arrays &= ~(1UL << i); /* clear the bit */ 10391 continue; 10392 } 10393 10394 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10395 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10396 10397 data_len += count * size; 10398 } 10399 10400 /* step 3: allocate continuous memory */ 10401 data_len = roundup(data_len, sizeof(__u64)); 10402 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 10403 if (!info_linear) 10404 return libbpf_err_ptr(-ENOMEM); 10405 10406 /* step 4: fill data to info_linear->info */ 10407 info_linear->arrays = arrays; 10408 memset(&info_linear->info, 0, sizeof(info)); 10409 ptr = info_linear->data; 10410 10411 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10412 struct bpf_prog_info_array_desc *desc; 10413 __u32 count, size; 10414 10415 if ((arrays & (1UL << i)) == 0) 10416 continue; 10417 10418 desc = bpf_prog_info_array_desc + i; 10419 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10420 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10421 bpf_prog_info_set_offset_u32(&info_linear->info, 10422 desc->count_offset, count); 10423 bpf_prog_info_set_offset_u32(&info_linear->info, 10424 desc->size_offset, size); 10425 bpf_prog_info_set_offset_u64(&info_linear->info, 10426 desc->array_offset, 10427 ptr_to_u64(ptr)); 10428 ptr += count * size; 10429 } 10430 10431 /* step 5: call syscall again to get required arrays */ 10432 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 10433 if (err) { 10434 pr_debug("can't get prog info: %s", strerror(errno)); 10435 free(info_linear); 10436 return libbpf_err_ptr(-EFAULT); 10437 } 10438 10439 /* step 6: verify the data */ 10440 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10441 struct bpf_prog_info_array_desc *desc; 10442 __u32 v1, v2; 10443 10444 if ((arrays & (1UL << i)) == 0) 10445 continue; 10446 10447 desc = bpf_prog_info_array_desc + i; 10448 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 10449 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 10450 desc->count_offset); 10451 if (v1 != v2) 10452 pr_warn("%s: mismatch in element count\n", __func__); 10453 10454 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 10455 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 10456 desc->size_offset); 10457 if (v1 != v2) 10458 pr_warn("%s: mismatch in rec size\n", __func__); 10459 } 10460 10461 /* step 7: update info_len and data_len */ 10462 info_linear->info_len = sizeof(struct bpf_prog_info); 10463 info_linear->data_len = data_len; 10464 10465 return info_linear; 10466 } 10467 10468 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 10469 { 10470 int i; 10471 10472 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10473 struct bpf_prog_info_array_desc *desc; 10474 __u64 addr, offs; 10475 10476 if ((info_linear->arrays & (1UL << i)) == 0) 10477 continue; 10478 10479 desc = bpf_prog_info_array_desc + i; 10480 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 10481 desc->array_offset); 10482 offs = addr - ptr_to_u64(info_linear->data); 10483 bpf_prog_info_set_offset_u64(&info_linear->info, 10484 desc->array_offset, offs); 10485 } 10486 } 10487 10488 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 10489 { 10490 int i; 10491 10492 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 10493 struct bpf_prog_info_array_desc *desc; 10494 __u64 addr, offs; 10495 10496 if ((info_linear->arrays & (1UL << i)) == 0) 10497 continue; 10498 10499 desc = bpf_prog_info_array_desc + i; 10500 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 10501 desc->array_offset); 10502 addr = offs + ptr_to_u64(info_linear->data); 10503 bpf_prog_info_set_offset_u64(&info_linear->info, 10504 desc->array_offset, addr); 10505 } 10506 } 10507 10508 int bpf_program__set_attach_target(struct bpf_program *prog, 10509 int attach_prog_fd, 10510 const char *attach_func_name) 10511 { 10512 int btf_obj_fd = 0, btf_id = 0, err; 10513 10514 if (!prog || attach_prog_fd < 0 || !attach_func_name) 10515 return libbpf_err(-EINVAL); 10516 10517 if (prog->obj->loaded) 10518 return libbpf_err(-EINVAL); 10519 10520 if (attach_prog_fd) { 10521 btf_id = libbpf_find_prog_btf_id(attach_func_name, 10522 attach_prog_fd); 10523 if (btf_id < 0) 10524 return libbpf_err(btf_id); 10525 } else { 10526 /* load btf_vmlinux, if not yet */ 10527 err = bpf_object__load_vmlinux_btf(prog->obj, true); 10528 if (err) 10529 return libbpf_err(err); 10530 err = find_kernel_btf_id(prog->obj, attach_func_name, 10531 prog->expected_attach_type, 10532 &btf_obj_fd, &btf_id); 10533 if (err) 10534 return libbpf_err(err); 10535 } 10536 10537 prog->attach_btf_id = btf_id; 10538 prog->attach_btf_obj_fd = btf_obj_fd; 10539 prog->attach_prog_fd = attach_prog_fd; 10540 return 0; 10541 } 10542 10543 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 10544 { 10545 int err = 0, n, len, start, end = -1; 10546 bool *tmp; 10547 10548 *mask = NULL; 10549 *mask_sz = 0; 10550 10551 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 10552 while (*s) { 10553 if (*s == ',' || *s == '\n') { 10554 s++; 10555 continue; 10556 } 10557 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 10558 if (n <= 0 || n > 2) { 10559 pr_warn("Failed to get CPU range %s: %d\n", s, n); 10560 err = -EINVAL; 10561 goto cleanup; 10562 } else if (n == 1) { 10563 end = start; 10564 } 10565 if (start < 0 || start > end) { 10566 pr_warn("Invalid CPU range [%d,%d] in %s\n", 10567 start, end, s); 10568 err = -EINVAL; 10569 goto cleanup; 10570 } 10571 tmp = realloc(*mask, end + 1); 10572 if (!tmp) { 10573 err = -ENOMEM; 10574 goto cleanup; 10575 } 10576 *mask = tmp; 10577 memset(tmp + *mask_sz, 0, start - *mask_sz); 10578 memset(tmp + start, 1, end - start + 1); 10579 *mask_sz = end + 1; 10580 s += len; 10581 } 10582 if (!*mask_sz) { 10583 pr_warn("Empty CPU range\n"); 10584 return -EINVAL; 10585 } 10586 return 0; 10587 cleanup: 10588 free(*mask); 10589 *mask = NULL; 10590 return err; 10591 } 10592 10593 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 10594 { 10595 int fd, err = 0, len; 10596 char buf[128]; 10597 10598 fd = open(fcpu, O_RDONLY); 10599 if (fd < 0) { 10600 err = -errno; 10601 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 10602 return err; 10603 } 10604 len = read(fd, buf, sizeof(buf)); 10605 close(fd); 10606 if (len <= 0) { 10607 err = len ? -errno : -EINVAL; 10608 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 10609 return err; 10610 } 10611 if (len >= sizeof(buf)) { 10612 pr_warn("CPU mask is too big in file %s\n", fcpu); 10613 return -E2BIG; 10614 } 10615 buf[len] = '\0'; 10616 10617 return parse_cpu_mask_str(buf, mask, mask_sz); 10618 } 10619 10620 int libbpf_num_possible_cpus(void) 10621 { 10622 static const char *fcpu = "/sys/devices/system/cpu/possible"; 10623 static int cpus; 10624 int err, n, i, tmp_cpus; 10625 bool *mask; 10626 10627 tmp_cpus = READ_ONCE(cpus); 10628 if (tmp_cpus > 0) 10629 return tmp_cpus; 10630 10631 err = parse_cpu_mask_file(fcpu, &mask, &n); 10632 if (err) 10633 return libbpf_err(err); 10634 10635 tmp_cpus = 0; 10636 for (i = 0; i < n; i++) { 10637 if (mask[i]) 10638 tmp_cpus++; 10639 } 10640 free(mask); 10641 10642 WRITE_ONCE(cpus, tmp_cpus); 10643 return tmp_cpus; 10644 } 10645 10646 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 10647 const struct bpf_object_open_opts *opts) 10648 { 10649 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 10650 .object_name = s->name, 10651 ); 10652 struct bpf_object *obj; 10653 int i, err; 10654 10655 /* Attempt to preserve opts->object_name, unless overriden by user 10656 * explicitly. Overwriting object name for skeletons is discouraged, 10657 * as it breaks global data maps, because they contain object name 10658 * prefix as their own map name prefix. When skeleton is generated, 10659 * bpftool is making an assumption that this name will stay the same. 10660 */ 10661 if (opts) { 10662 memcpy(&skel_opts, opts, sizeof(*opts)); 10663 if (!opts->object_name) 10664 skel_opts.object_name = s->name; 10665 } 10666 10667 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 10668 err = libbpf_get_error(obj); 10669 if (err) { 10670 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", 10671 s->name, err); 10672 return libbpf_err(err); 10673 } 10674 10675 *s->obj = obj; 10676 10677 for (i = 0; i < s->map_cnt; i++) { 10678 struct bpf_map **map = s->maps[i].map; 10679 const char *name = s->maps[i].name; 10680 void **mmaped = s->maps[i].mmaped; 10681 10682 *map = bpf_object__find_map_by_name(obj, name); 10683 if (!*map) { 10684 pr_warn("failed to find skeleton map '%s'\n", name); 10685 return libbpf_err(-ESRCH); 10686 } 10687 10688 /* externs shouldn't be pre-setup from user code */ 10689 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 10690 *mmaped = (*map)->mmaped; 10691 } 10692 10693 for (i = 0; i < s->prog_cnt; i++) { 10694 struct bpf_program **prog = s->progs[i].prog; 10695 const char *name = s->progs[i].name; 10696 10697 *prog = bpf_object__find_program_by_name(obj, name); 10698 if (!*prog) { 10699 pr_warn("failed to find skeleton program '%s'\n", name); 10700 return libbpf_err(-ESRCH); 10701 } 10702 } 10703 10704 return 0; 10705 } 10706 10707 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 10708 { 10709 int i, err; 10710 10711 err = bpf_object__load(*s->obj); 10712 if (err) { 10713 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 10714 return libbpf_err(err); 10715 } 10716 10717 for (i = 0; i < s->map_cnt; i++) { 10718 struct bpf_map *map = *s->maps[i].map; 10719 size_t mmap_sz = bpf_map_mmap_sz(map); 10720 int prot, map_fd = bpf_map__fd(map); 10721 void **mmaped = s->maps[i].mmaped; 10722 10723 if (!mmaped) 10724 continue; 10725 10726 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 10727 *mmaped = NULL; 10728 continue; 10729 } 10730 10731 if (map->def.map_flags & BPF_F_RDONLY_PROG) 10732 prot = PROT_READ; 10733 else 10734 prot = PROT_READ | PROT_WRITE; 10735 10736 /* Remap anonymous mmap()-ed "map initialization image" as 10737 * a BPF map-backed mmap()-ed memory, but preserving the same 10738 * memory address. This will cause kernel to change process' 10739 * page table to point to a different piece of kernel memory, 10740 * but from userspace point of view memory address (and its 10741 * contents, being identical at this point) will stay the 10742 * same. This mapping will be released by bpf_object__close() 10743 * as per normal clean up procedure, so we don't need to worry 10744 * about it from skeleton's clean up perspective. 10745 */ 10746 *mmaped = mmap(map->mmaped, mmap_sz, prot, 10747 MAP_SHARED | MAP_FIXED, map_fd, 0); 10748 if (*mmaped == MAP_FAILED) { 10749 err = -errno; 10750 *mmaped = NULL; 10751 pr_warn("failed to re-mmap() map '%s': %d\n", 10752 bpf_map__name(map), err); 10753 return libbpf_err(err); 10754 } 10755 } 10756 10757 return 0; 10758 } 10759 10760 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 10761 { 10762 int i, err; 10763 10764 for (i = 0; i < s->prog_cnt; i++) { 10765 struct bpf_program *prog = *s->progs[i].prog; 10766 struct bpf_link **link = s->progs[i].link; 10767 const struct bpf_sec_def *sec_def; 10768 10769 if (!prog->load) 10770 continue; 10771 10772 sec_def = find_sec_def(prog->sec_name); 10773 if (!sec_def || !sec_def->attach_fn) 10774 continue; 10775 10776 *link = sec_def->attach_fn(sec_def, prog); 10777 err = libbpf_get_error(*link); 10778 if (err) { 10779 pr_warn("failed to auto-attach program '%s': %d\n", 10780 bpf_program__name(prog), err); 10781 return libbpf_err(err); 10782 } 10783 } 10784 10785 return 0; 10786 } 10787 10788 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 10789 { 10790 int i; 10791 10792 for (i = 0; i < s->prog_cnt; i++) { 10793 struct bpf_link **link = s->progs[i].link; 10794 10795 bpf_link__destroy(*link); 10796 *link = NULL; 10797 } 10798 } 10799 10800 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 10801 { 10802 if (s->progs) 10803 bpf_object__detach_skeleton(s); 10804 if (s->obj) 10805 bpf_object__close(*s->obj); 10806 free(s->maps); 10807 free(s->progs); 10808 free(s); 10809 } 10810