1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/list.h> 35 #include <linux/limits.h> 36 #include <linux/perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <linux/version.h> 39 #include <sys/epoll.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <sys/stat.h> 43 #include <sys/types.h> 44 #include <sys/vfs.h> 45 #include <sys/utsname.h> 46 #include <sys/resource.h> 47 #include <tools/libc_compat.h> 48 #include <libelf.h> 49 #include <gelf.h> 50 #include <zlib.h> 51 52 #include "libbpf.h" 53 #include "bpf.h" 54 #include "btf.h" 55 #include "str_error.h" 56 #include "libbpf_internal.h" 57 #include "hashmap.h" 58 59 /* make sure libbpf doesn't use kernel-only integer typedefs */ 60 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 61 62 #ifndef EM_BPF 63 #define EM_BPF 247 64 #endif 65 66 #ifndef BPF_FS_MAGIC 67 #define BPF_FS_MAGIC 0xcafe4a11 68 #endif 69 70 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 71 * compilation if user enables corresponding warning. Disable it explicitly. 72 */ 73 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 74 75 #define __printf(a, b) __attribute__((format(printf, a, b))) 76 77 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 78 static struct bpf_program *bpf_object__find_prog_by_idx(struct bpf_object *obj, 79 int idx); 80 static const struct btf_type * 81 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); 82 83 static int __base_pr(enum libbpf_print_level level, const char *format, 84 va_list args) 85 { 86 if (level == LIBBPF_DEBUG) 87 return 0; 88 89 return vfprintf(stderr, format, args); 90 } 91 92 static libbpf_print_fn_t __libbpf_pr = __base_pr; 93 94 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 95 { 96 libbpf_print_fn_t old_print_fn = __libbpf_pr; 97 98 __libbpf_pr = fn; 99 return old_print_fn; 100 } 101 102 __printf(2, 3) 103 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 104 { 105 va_list args; 106 107 if (!__libbpf_pr) 108 return; 109 110 va_start(args, format); 111 __libbpf_pr(level, format, args); 112 va_end(args); 113 } 114 115 static void pr_perm_msg(int err) 116 { 117 struct rlimit limit; 118 char buf[100]; 119 120 if (err != -EPERM || geteuid() != 0) 121 return; 122 123 err = getrlimit(RLIMIT_MEMLOCK, &limit); 124 if (err) 125 return; 126 127 if (limit.rlim_cur == RLIM_INFINITY) 128 return; 129 130 if (limit.rlim_cur < 1024) 131 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 132 else if (limit.rlim_cur < 1024*1024) 133 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 134 else 135 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 136 137 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 138 buf); 139 } 140 141 #define STRERR_BUFSIZE 128 142 143 /* Copied from tools/perf/util/util.h */ 144 #ifndef zfree 145 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 146 #endif 147 148 #ifndef zclose 149 # define zclose(fd) ({ \ 150 int ___err = 0; \ 151 if ((fd) >= 0) \ 152 ___err = close((fd)); \ 153 fd = -1; \ 154 ___err; }) 155 #endif 156 157 #ifdef HAVE_LIBELF_MMAP_SUPPORT 158 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 159 #else 160 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 161 #endif 162 163 static inline __u64 ptr_to_u64(const void *ptr) 164 { 165 return (__u64) (unsigned long) ptr; 166 } 167 168 struct bpf_capabilities { 169 /* v4.14: kernel support for program & map names. */ 170 __u32 name:1; 171 /* v5.2: kernel support for global data sections. */ 172 __u32 global_data:1; 173 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 174 __u32 btf_func:1; 175 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 176 __u32 btf_datasec:1; 177 /* BPF_F_MMAPABLE is supported for arrays */ 178 __u32 array_mmap:1; 179 /* BTF_FUNC_GLOBAL is supported */ 180 __u32 btf_func_global:1; 181 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 182 __u32 exp_attach_type:1; 183 }; 184 185 enum reloc_type { 186 RELO_LD64, 187 RELO_CALL, 188 RELO_DATA, 189 RELO_EXTERN, 190 }; 191 192 struct reloc_desc { 193 enum reloc_type type; 194 int insn_idx; 195 int map_idx; 196 int sym_off; 197 }; 198 199 struct bpf_sec_def; 200 201 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec, 202 struct bpf_program *prog); 203 204 struct bpf_sec_def { 205 const char *sec; 206 size_t len; 207 enum bpf_prog_type prog_type; 208 enum bpf_attach_type expected_attach_type; 209 bool is_exp_attach_type_optional; 210 bool is_attachable; 211 bool is_attach_btf; 212 attach_fn_t attach_fn; 213 }; 214 215 /* 216 * bpf_prog should be a better name but it has been used in 217 * linux/filter.h. 218 */ 219 struct bpf_program { 220 /* Index in elf obj file, for relocation use. */ 221 int idx; 222 char *name; 223 int prog_ifindex; 224 char *section_name; 225 const struct bpf_sec_def *sec_def; 226 /* section_name with / replaced by _; makes recursive pinning 227 * in bpf_object__pin_programs easier 228 */ 229 char *pin_name; 230 struct bpf_insn *insns; 231 size_t insns_cnt, main_prog_cnt; 232 enum bpf_prog_type type; 233 234 struct reloc_desc *reloc_desc; 235 int nr_reloc; 236 int log_level; 237 238 struct { 239 int nr; 240 int *fds; 241 } instances; 242 bpf_program_prep_t preprocessor; 243 244 struct bpf_object *obj; 245 void *priv; 246 bpf_program_clear_priv_t clear_priv; 247 248 enum bpf_attach_type expected_attach_type; 249 __u32 attach_btf_id; 250 __u32 attach_prog_fd; 251 void *func_info; 252 __u32 func_info_rec_size; 253 __u32 func_info_cnt; 254 255 struct bpf_capabilities *caps; 256 257 void *line_info; 258 __u32 line_info_rec_size; 259 __u32 line_info_cnt; 260 __u32 prog_flags; 261 }; 262 263 struct bpf_struct_ops { 264 const char *tname; 265 const struct btf_type *type; 266 struct bpf_program **progs; 267 __u32 *kern_func_off; 268 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 269 void *data; 270 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 271 * btf_vmlinux's format. 272 * struct bpf_struct_ops_tcp_congestion_ops { 273 * [... some other kernel fields ...] 274 * struct tcp_congestion_ops data; 275 * } 276 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 277 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 278 * from "data". 279 */ 280 void *kern_vdata; 281 __u32 type_id; 282 }; 283 284 #define DATA_SEC ".data" 285 #define BSS_SEC ".bss" 286 #define RODATA_SEC ".rodata" 287 #define KCONFIG_SEC ".kconfig" 288 #define STRUCT_OPS_SEC ".struct_ops" 289 290 enum libbpf_map_type { 291 LIBBPF_MAP_UNSPEC, 292 LIBBPF_MAP_DATA, 293 LIBBPF_MAP_BSS, 294 LIBBPF_MAP_RODATA, 295 LIBBPF_MAP_KCONFIG, 296 }; 297 298 static const char * const libbpf_type_to_btf_name[] = { 299 [LIBBPF_MAP_DATA] = DATA_SEC, 300 [LIBBPF_MAP_BSS] = BSS_SEC, 301 [LIBBPF_MAP_RODATA] = RODATA_SEC, 302 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, 303 }; 304 305 struct bpf_map { 306 char *name; 307 int fd; 308 int sec_idx; 309 size_t sec_offset; 310 int map_ifindex; 311 int inner_map_fd; 312 struct bpf_map_def def; 313 __u32 btf_var_idx; 314 __u32 btf_key_type_id; 315 __u32 btf_value_type_id; 316 __u32 btf_vmlinux_value_type_id; 317 void *priv; 318 bpf_map_clear_priv_t clear_priv; 319 enum libbpf_map_type libbpf_type; 320 void *mmaped; 321 struct bpf_struct_ops *st_ops; 322 struct bpf_map *inner_map; 323 void **init_slots; 324 int init_slots_sz; 325 char *pin_path; 326 bool pinned; 327 bool reused; 328 }; 329 330 enum extern_type { 331 EXT_UNKNOWN, 332 EXT_CHAR, 333 EXT_BOOL, 334 EXT_INT, 335 EXT_TRISTATE, 336 EXT_CHAR_ARR, 337 }; 338 339 struct extern_desc { 340 const char *name; 341 int sym_idx; 342 int btf_id; 343 enum extern_type type; 344 int sz; 345 int align; 346 int data_off; 347 bool is_signed; 348 bool is_weak; 349 bool is_set; 350 }; 351 352 static LIST_HEAD(bpf_objects_list); 353 354 struct bpf_object { 355 char name[BPF_OBJ_NAME_LEN]; 356 char license[64]; 357 __u32 kern_version; 358 359 struct bpf_program *programs; 360 size_t nr_programs; 361 struct bpf_map *maps; 362 size_t nr_maps; 363 size_t maps_cap; 364 365 char *kconfig; 366 struct extern_desc *externs; 367 int nr_extern; 368 int kconfig_map_idx; 369 370 bool loaded; 371 bool has_pseudo_calls; 372 373 /* 374 * Information when doing elf related work. Only valid if fd 375 * is valid. 376 */ 377 struct { 378 int fd; 379 const void *obj_buf; 380 size_t obj_buf_sz; 381 Elf *elf; 382 GElf_Ehdr ehdr; 383 Elf_Data *symbols; 384 Elf_Data *data; 385 Elf_Data *rodata; 386 Elf_Data *bss; 387 Elf_Data *st_ops_data; 388 size_t strtabidx; 389 struct { 390 GElf_Shdr shdr; 391 Elf_Data *data; 392 } *reloc_sects; 393 int nr_reloc_sects; 394 int maps_shndx; 395 int btf_maps_shndx; 396 __u32 btf_maps_sec_btf_id; 397 int text_shndx; 398 int symbols_shndx; 399 int data_shndx; 400 int rodata_shndx; 401 int bss_shndx; 402 int st_ops_shndx; 403 } efile; 404 /* 405 * All loaded bpf_object is linked in a list, which is 406 * hidden to caller. bpf_objects__<func> handlers deal with 407 * all objects. 408 */ 409 struct list_head list; 410 411 struct btf *btf; 412 /* Parse and load BTF vmlinux if any of the programs in the object need 413 * it at load time. 414 */ 415 struct btf *btf_vmlinux; 416 struct btf_ext *btf_ext; 417 418 void *priv; 419 bpf_object_clear_priv_t clear_priv; 420 421 struct bpf_capabilities caps; 422 423 char path[]; 424 }; 425 #define obj_elf_valid(o) ((o)->efile.elf) 426 427 void bpf_program__unload(struct bpf_program *prog) 428 { 429 int i; 430 431 if (!prog) 432 return; 433 434 /* 435 * If the object is opened but the program was never loaded, 436 * it is possible that prog->instances.nr == -1. 437 */ 438 if (prog->instances.nr > 0) { 439 for (i = 0; i < prog->instances.nr; i++) 440 zclose(prog->instances.fds[i]); 441 } else if (prog->instances.nr != -1) { 442 pr_warn("Internal error: instances.nr is %d\n", 443 prog->instances.nr); 444 } 445 446 prog->instances.nr = -1; 447 zfree(&prog->instances.fds); 448 449 zfree(&prog->func_info); 450 zfree(&prog->line_info); 451 } 452 453 static void bpf_program__exit(struct bpf_program *prog) 454 { 455 if (!prog) 456 return; 457 458 if (prog->clear_priv) 459 prog->clear_priv(prog, prog->priv); 460 461 prog->priv = NULL; 462 prog->clear_priv = NULL; 463 464 bpf_program__unload(prog); 465 zfree(&prog->name); 466 zfree(&prog->section_name); 467 zfree(&prog->pin_name); 468 zfree(&prog->insns); 469 zfree(&prog->reloc_desc); 470 471 prog->nr_reloc = 0; 472 prog->insns_cnt = 0; 473 prog->idx = -1; 474 } 475 476 static char *__bpf_program__pin_name(struct bpf_program *prog) 477 { 478 char *name, *p; 479 480 name = p = strdup(prog->section_name); 481 while ((p = strchr(p, '/'))) 482 *p = '_'; 483 484 return name; 485 } 486 487 static int 488 bpf_program__init(void *data, size_t size, char *section_name, int idx, 489 struct bpf_program *prog) 490 { 491 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 492 493 if (size == 0 || size % bpf_insn_sz) { 494 pr_warn("corrupted section '%s', size: %zu\n", 495 section_name, size); 496 return -EINVAL; 497 } 498 499 memset(prog, 0, sizeof(*prog)); 500 501 prog->section_name = strdup(section_name); 502 if (!prog->section_name) { 503 pr_warn("failed to alloc name for prog under section(%d) %s\n", 504 idx, section_name); 505 goto errout; 506 } 507 508 prog->pin_name = __bpf_program__pin_name(prog); 509 if (!prog->pin_name) { 510 pr_warn("failed to alloc pin name for prog under section(%d) %s\n", 511 idx, section_name); 512 goto errout; 513 } 514 515 prog->insns = malloc(size); 516 if (!prog->insns) { 517 pr_warn("failed to alloc insns for prog under section %s\n", 518 section_name); 519 goto errout; 520 } 521 prog->insns_cnt = size / bpf_insn_sz; 522 memcpy(prog->insns, data, size); 523 prog->idx = idx; 524 prog->instances.fds = NULL; 525 prog->instances.nr = -1; 526 prog->type = BPF_PROG_TYPE_UNSPEC; 527 528 return 0; 529 errout: 530 bpf_program__exit(prog); 531 return -ENOMEM; 532 } 533 534 static int 535 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 536 char *section_name, int idx) 537 { 538 struct bpf_program prog, *progs; 539 int nr_progs, err; 540 541 err = bpf_program__init(data, size, section_name, idx, &prog); 542 if (err) 543 return err; 544 545 prog.caps = &obj->caps; 546 progs = obj->programs; 547 nr_progs = obj->nr_programs; 548 549 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 550 if (!progs) { 551 /* 552 * In this case the original obj->programs 553 * is still valid, so don't need special treat for 554 * bpf_close_object(). 555 */ 556 pr_warn("failed to alloc a new program under section '%s'\n", 557 section_name); 558 bpf_program__exit(&prog); 559 return -ENOMEM; 560 } 561 562 pr_debug("found program %s\n", prog.section_name); 563 obj->programs = progs; 564 obj->nr_programs = nr_progs + 1; 565 prog.obj = obj; 566 progs[nr_progs] = prog; 567 return 0; 568 } 569 570 static int 571 bpf_object__init_prog_names(struct bpf_object *obj) 572 { 573 Elf_Data *symbols = obj->efile.symbols; 574 struct bpf_program *prog; 575 size_t pi, si; 576 577 for (pi = 0; pi < obj->nr_programs; pi++) { 578 const char *name = NULL; 579 580 prog = &obj->programs[pi]; 581 582 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 583 si++) { 584 GElf_Sym sym; 585 586 if (!gelf_getsym(symbols, si, &sym)) 587 continue; 588 if (sym.st_shndx != prog->idx) 589 continue; 590 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 591 continue; 592 593 name = elf_strptr(obj->efile.elf, 594 obj->efile.strtabidx, 595 sym.st_name); 596 if (!name) { 597 pr_warn("failed to get sym name string for prog %s\n", 598 prog->section_name); 599 return -LIBBPF_ERRNO__LIBELF; 600 } 601 } 602 603 if (!name && prog->idx == obj->efile.text_shndx) 604 name = ".text"; 605 606 if (!name) { 607 pr_warn("failed to find sym for prog %s\n", 608 prog->section_name); 609 return -EINVAL; 610 } 611 612 prog->name = strdup(name); 613 if (!prog->name) { 614 pr_warn("failed to allocate memory for prog sym %s\n", 615 name); 616 return -ENOMEM; 617 } 618 } 619 620 return 0; 621 } 622 623 static __u32 get_kernel_version(void) 624 { 625 __u32 major, minor, patch; 626 struct utsname info; 627 628 uname(&info); 629 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) 630 return 0; 631 return KERNEL_VERSION(major, minor, patch); 632 } 633 634 static const struct btf_member * 635 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 636 { 637 struct btf_member *m; 638 int i; 639 640 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 641 if (btf_member_bit_offset(t, i) == bit_offset) 642 return m; 643 } 644 645 return NULL; 646 } 647 648 static const struct btf_member * 649 find_member_by_name(const struct btf *btf, const struct btf_type *t, 650 const char *name) 651 { 652 struct btf_member *m; 653 int i; 654 655 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 656 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 657 return m; 658 } 659 660 return NULL; 661 } 662 663 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 664 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 665 const char *name, __u32 kind); 666 667 static int 668 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 669 const struct btf_type **type, __u32 *type_id, 670 const struct btf_type **vtype, __u32 *vtype_id, 671 const struct btf_member **data_member) 672 { 673 const struct btf_type *kern_type, *kern_vtype; 674 const struct btf_member *kern_data_member; 675 __s32 kern_vtype_id, kern_type_id; 676 __u32 i; 677 678 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 679 if (kern_type_id < 0) { 680 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 681 tname); 682 return kern_type_id; 683 } 684 kern_type = btf__type_by_id(btf, kern_type_id); 685 686 /* Find the corresponding "map_value" type that will be used 687 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 688 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 689 * btf_vmlinux. 690 */ 691 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 692 tname, BTF_KIND_STRUCT); 693 if (kern_vtype_id < 0) { 694 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 695 STRUCT_OPS_VALUE_PREFIX, tname); 696 return kern_vtype_id; 697 } 698 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 699 700 /* Find "struct tcp_congestion_ops" from 701 * struct bpf_struct_ops_tcp_congestion_ops { 702 * [ ... ] 703 * struct tcp_congestion_ops data; 704 * } 705 */ 706 kern_data_member = btf_members(kern_vtype); 707 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 708 if (kern_data_member->type == kern_type_id) 709 break; 710 } 711 if (i == btf_vlen(kern_vtype)) { 712 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 713 tname, STRUCT_OPS_VALUE_PREFIX, tname); 714 return -EINVAL; 715 } 716 717 *type = kern_type; 718 *type_id = kern_type_id; 719 *vtype = kern_vtype; 720 *vtype_id = kern_vtype_id; 721 *data_member = kern_data_member; 722 723 return 0; 724 } 725 726 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 727 { 728 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 729 } 730 731 /* Init the map's fields that depend on kern_btf */ 732 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 733 const struct btf *btf, 734 const struct btf *kern_btf) 735 { 736 const struct btf_member *member, *kern_member, *kern_data_member; 737 const struct btf_type *type, *kern_type, *kern_vtype; 738 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 739 struct bpf_struct_ops *st_ops; 740 void *data, *kern_data; 741 const char *tname; 742 int err; 743 744 st_ops = map->st_ops; 745 type = st_ops->type; 746 tname = st_ops->tname; 747 err = find_struct_ops_kern_types(kern_btf, tname, 748 &kern_type, &kern_type_id, 749 &kern_vtype, &kern_vtype_id, 750 &kern_data_member); 751 if (err) 752 return err; 753 754 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 755 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 756 757 map->def.value_size = kern_vtype->size; 758 map->btf_vmlinux_value_type_id = kern_vtype_id; 759 760 st_ops->kern_vdata = calloc(1, kern_vtype->size); 761 if (!st_ops->kern_vdata) 762 return -ENOMEM; 763 764 data = st_ops->data; 765 kern_data_off = kern_data_member->offset / 8; 766 kern_data = st_ops->kern_vdata + kern_data_off; 767 768 member = btf_members(type); 769 for (i = 0; i < btf_vlen(type); i++, member++) { 770 const struct btf_type *mtype, *kern_mtype; 771 __u32 mtype_id, kern_mtype_id; 772 void *mdata, *kern_mdata; 773 __s64 msize, kern_msize; 774 __u32 moff, kern_moff; 775 __u32 kern_member_idx; 776 const char *mname; 777 778 mname = btf__name_by_offset(btf, member->name_off); 779 kern_member = find_member_by_name(kern_btf, kern_type, mname); 780 if (!kern_member) { 781 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 782 map->name, mname); 783 return -ENOTSUP; 784 } 785 786 kern_member_idx = kern_member - btf_members(kern_type); 787 if (btf_member_bitfield_size(type, i) || 788 btf_member_bitfield_size(kern_type, kern_member_idx)) { 789 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 790 map->name, mname); 791 return -ENOTSUP; 792 } 793 794 moff = member->offset / 8; 795 kern_moff = kern_member->offset / 8; 796 797 mdata = data + moff; 798 kern_mdata = kern_data + kern_moff; 799 800 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 801 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 802 &kern_mtype_id); 803 if (BTF_INFO_KIND(mtype->info) != 804 BTF_INFO_KIND(kern_mtype->info)) { 805 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 806 map->name, mname, BTF_INFO_KIND(mtype->info), 807 BTF_INFO_KIND(kern_mtype->info)); 808 return -ENOTSUP; 809 } 810 811 if (btf_is_ptr(mtype)) { 812 struct bpf_program *prog; 813 814 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id); 815 kern_mtype = skip_mods_and_typedefs(kern_btf, 816 kern_mtype->type, 817 &kern_mtype_id); 818 if (!btf_is_func_proto(mtype) || 819 !btf_is_func_proto(kern_mtype)) { 820 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n", 821 map->name, mname); 822 return -ENOTSUP; 823 } 824 825 prog = st_ops->progs[i]; 826 if (!prog) { 827 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n", 828 map->name, mname); 829 continue; 830 } 831 832 prog->attach_btf_id = kern_type_id; 833 prog->expected_attach_type = kern_member_idx; 834 835 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 836 837 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 838 map->name, mname, prog->name, moff, 839 kern_moff); 840 841 continue; 842 } 843 844 msize = btf__resolve_size(btf, mtype_id); 845 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 846 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 847 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 848 map->name, mname, (ssize_t)msize, 849 (ssize_t)kern_msize); 850 return -ENOTSUP; 851 } 852 853 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 854 map->name, mname, (unsigned int)msize, 855 moff, kern_moff); 856 memcpy(kern_mdata, mdata, msize); 857 } 858 859 return 0; 860 } 861 862 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 863 { 864 struct bpf_map *map; 865 size_t i; 866 int err; 867 868 for (i = 0; i < obj->nr_maps; i++) { 869 map = &obj->maps[i]; 870 871 if (!bpf_map__is_struct_ops(map)) 872 continue; 873 874 err = bpf_map__init_kern_struct_ops(map, obj->btf, 875 obj->btf_vmlinux); 876 if (err) 877 return err; 878 } 879 880 return 0; 881 } 882 883 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) 884 { 885 const struct btf_type *type, *datasec; 886 const struct btf_var_secinfo *vsi; 887 struct bpf_struct_ops *st_ops; 888 const char *tname, *var_name; 889 __s32 type_id, datasec_id; 890 const struct btf *btf; 891 struct bpf_map *map; 892 __u32 i; 893 894 if (obj->efile.st_ops_shndx == -1) 895 return 0; 896 897 btf = obj->btf; 898 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, 899 BTF_KIND_DATASEC); 900 if (datasec_id < 0) { 901 pr_warn("struct_ops init: DATASEC %s not found\n", 902 STRUCT_OPS_SEC); 903 return -EINVAL; 904 } 905 906 datasec = btf__type_by_id(btf, datasec_id); 907 vsi = btf_var_secinfos(datasec); 908 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 909 type = btf__type_by_id(obj->btf, vsi->type); 910 var_name = btf__name_by_offset(obj->btf, type->name_off); 911 912 type_id = btf__resolve_type(obj->btf, vsi->type); 913 if (type_id < 0) { 914 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 915 vsi->type, STRUCT_OPS_SEC); 916 return -EINVAL; 917 } 918 919 type = btf__type_by_id(obj->btf, type_id); 920 tname = btf__name_by_offset(obj->btf, type->name_off); 921 if (!tname[0]) { 922 pr_warn("struct_ops init: anonymous type is not supported\n"); 923 return -ENOTSUP; 924 } 925 if (!btf_is_struct(type)) { 926 pr_warn("struct_ops init: %s is not a struct\n", tname); 927 return -EINVAL; 928 } 929 930 map = bpf_object__add_map(obj); 931 if (IS_ERR(map)) 932 return PTR_ERR(map); 933 934 map->sec_idx = obj->efile.st_ops_shndx; 935 map->sec_offset = vsi->offset; 936 map->name = strdup(var_name); 937 if (!map->name) 938 return -ENOMEM; 939 940 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 941 map->def.key_size = sizeof(int); 942 map->def.value_size = type->size; 943 map->def.max_entries = 1; 944 945 map->st_ops = calloc(1, sizeof(*map->st_ops)); 946 if (!map->st_ops) 947 return -ENOMEM; 948 st_ops = map->st_ops; 949 st_ops->data = malloc(type->size); 950 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 951 st_ops->kern_func_off = malloc(btf_vlen(type) * 952 sizeof(*st_ops->kern_func_off)); 953 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 954 return -ENOMEM; 955 956 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { 957 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 958 var_name, STRUCT_OPS_SEC); 959 return -EINVAL; 960 } 961 962 memcpy(st_ops->data, 963 obj->efile.st_ops_data->d_buf + vsi->offset, 964 type->size); 965 st_ops->tname = tname; 966 st_ops->type = type; 967 st_ops->type_id = type_id; 968 969 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 970 tname, type_id, var_name, vsi->offset); 971 } 972 973 return 0; 974 } 975 976 static struct bpf_object *bpf_object__new(const char *path, 977 const void *obj_buf, 978 size_t obj_buf_sz, 979 const char *obj_name) 980 { 981 struct bpf_object *obj; 982 char *end; 983 984 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 985 if (!obj) { 986 pr_warn("alloc memory failed for %s\n", path); 987 return ERR_PTR(-ENOMEM); 988 } 989 990 strcpy(obj->path, path); 991 if (obj_name) { 992 strncpy(obj->name, obj_name, sizeof(obj->name) - 1); 993 obj->name[sizeof(obj->name) - 1] = 0; 994 } else { 995 /* Using basename() GNU version which doesn't modify arg. */ 996 strncpy(obj->name, basename((void *)path), 997 sizeof(obj->name) - 1); 998 end = strchr(obj->name, '.'); 999 if (end) 1000 *end = 0; 1001 } 1002 1003 obj->efile.fd = -1; 1004 /* 1005 * Caller of this function should also call 1006 * bpf_object__elf_finish() after data collection to return 1007 * obj_buf to user. If not, we should duplicate the buffer to 1008 * avoid user freeing them before elf finish. 1009 */ 1010 obj->efile.obj_buf = obj_buf; 1011 obj->efile.obj_buf_sz = obj_buf_sz; 1012 obj->efile.maps_shndx = -1; 1013 obj->efile.btf_maps_shndx = -1; 1014 obj->efile.data_shndx = -1; 1015 obj->efile.rodata_shndx = -1; 1016 obj->efile.bss_shndx = -1; 1017 obj->efile.st_ops_shndx = -1; 1018 obj->kconfig_map_idx = -1; 1019 1020 obj->kern_version = get_kernel_version(); 1021 obj->loaded = false; 1022 1023 INIT_LIST_HEAD(&obj->list); 1024 list_add(&obj->list, &bpf_objects_list); 1025 return obj; 1026 } 1027 1028 static void bpf_object__elf_finish(struct bpf_object *obj) 1029 { 1030 if (!obj_elf_valid(obj)) 1031 return; 1032 1033 if (obj->efile.elf) { 1034 elf_end(obj->efile.elf); 1035 obj->efile.elf = NULL; 1036 } 1037 obj->efile.symbols = NULL; 1038 obj->efile.data = NULL; 1039 obj->efile.rodata = NULL; 1040 obj->efile.bss = NULL; 1041 obj->efile.st_ops_data = NULL; 1042 1043 zfree(&obj->efile.reloc_sects); 1044 obj->efile.nr_reloc_sects = 0; 1045 zclose(obj->efile.fd); 1046 obj->efile.obj_buf = NULL; 1047 obj->efile.obj_buf_sz = 0; 1048 } 1049 1050 static int bpf_object__elf_init(struct bpf_object *obj) 1051 { 1052 int err = 0; 1053 GElf_Ehdr *ep; 1054 1055 if (obj_elf_valid(obj)) { 1056 pr_warn("elf init: internal error\n"); 1057 return -LIBBPF_ERRNO__LIBELF; 1058 } 1059 1060 if (obj->efile.obj_buf_sz > 0) { 1061 /* 1062 * obj_buf should have been validated by 1063 * bpf_object__open_buffer(). 1064 */ 1065 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, 1066 obj->efile.obj_buf_sz); 1067 } else { 1068 obj->efile.fd = open(obj->path, O_RDONLY); 1069 if (obj->efile.fd < 0) { 1070 char errmsg[STRERR_BUFSIZE], *cp; 1071 1072 err = -errno; 1073 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1074 pr_warn("failed to open %s: %s\n", obj->path, cp); 1075 return err; 1076 } 1077 1078 obj->efile.elf = elf_begin(obj->efile.fd, 1079 LIBBPF_ELF_C_READ_MMAP, NULL); 1080 } 1081 1082 if (!obj->efile.elf) { 1083 pr_warn("failed to open %s as ELF file\n", obj->path); 1084 err = -LIBBPF_ERRNO__LIBELF; 1085 goto errout; 1086 } 1087 1088 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 1089 pr_warn("failed to get EHDR from %s\n", obj->path); 1090 err = -LIBBPF_ERRNO__FORMAT; 1091 goto errout; 1092 } 1093 ep = &obj->efile.ehdr; 1094 1095 /* Old LLVM set e_machine to EM_NONE */ 1096 if (ep->e_type != ET_REL || 1097 (ep->e_machine && ep->e_machine != EM_BPF)) { 1098 pr_warn("%s is not an eBPF object file\n", obj->path); 1099 err = -LIBBPF_ERRNO__FORMAT; 1100 goto errout; 1101 } 1102 1103 return 0; 1104 errout: 1105 bpf_object__elf_finish(obj); 1106 return err; 1107 } 1108 1109 static int bpf_object__check_endianness(struct bpf_object *obj) 1110 { 1111 #if __BYTE_ORDER == __LITTLE_ENDIAN 1112 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 1113 return 0; 1114 #elif __BYTE_ORDER == __BIG_ENDIAN 1115 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 1116 return 0; 1117 #else 1118 # error "Unrecognized __BYTE_ORDER__" 1119 #endif 1120 pr_warn("endianness mismatch.\n"); 1121 return -LIBBPF_ERRNO__ENDIAN; 1122 } 1123 1124 static int 1125 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1126 { 1127 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 1128 pr_debug("license of %s is %s\n", obj->path, obj->license); 1129 return 0; 1130 } 1131 1132 static int 1133 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1134 { 1135 __u32 kver; 1136 1137 if (size != sizeof(kver)) { 1138 pr_warn("invalid kver section in %s\n", obj->path); 1139 return -LIBBPF_ERRNO__FORMAT; 1140 } 1141 memcpy(&kver, data, sizeof(kver)); 1142 obj->kern_version = kver; 1143 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1144 return 0; 1145 } 1146 1147 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1148 { 1149 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1150 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1151 return true; 1152 return false; 1153 } 1154 1155 static int bpf_object_search_section_size(const struct bpf_object *obj, 1156 const char *name, size_t *d_size) 1157 { 1158 const GElf_Ehdr *ep = &obj->efile.ehdr; 1159 Elf *elf = obj->efile.elf; 1160 Elf_Scn *scn = NULL; 1161 int idx = 0; 1162 1163 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1164 const char *sec_name; 1165 Elf_Data *data; 1166 GElf_Shdr sh; 1167 1168 idx++; 1169 if (gelf_getshdr(scn, &sh) != &sh) { 1170 pr_warn("failed to get section(%d) header from %s\n", 1171 idx, obj->path); 1172 return -EIO; 1173 } 1174 1175 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1176 if (!sec_name) { 1177 pr_warn("failed to get section(%d) name from %s\n", 1178 idx, obj->path); 1179 return -EIO; 1180 } 1181 1182 if (strcmp(name, sec_name)) 1183 continue; 1184 1185 data = elf_getdata(scn, 0); 1186 if (!data) { 1187 pr_warn("failed to get section(%d) data from %s(%s)\n", 1188 idx, name, obj->path); 1189 return -EIO; 1190 } 1191 1192 *d_size = data->d_size; 1193 return 0; 1194 } 1195 1196 return -ENOENT; 1197 } 1198 1199 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 1200 __u32 *size) 1201 { 1202 int ret = -ENOENT; 1203 size_t d_size; 1204 1205 *size = 0; 1206 if (!name) { 1207 return -EINVAL; 1208 } else if (!strcmp(name, DATA_SEC)) { 1209 if (obj->efile.data) 1210 *size = obj->efile.data->d_size; 1211 } else if (!strcmp(name, BSS_SEC)) { 1212 if (obj->efile.bss) 1213 *size = obj->efile.bss->d_size; 1214 } else if (!strcmp(name, RODATA_SEC)) { 1215 if (obj->efile.rodata) 1216 *size = obj->efile.rodata->d_size; 1217 } else if (!strcmp(name, STRUCT_OPS_SEC)) { 1218 if (obj->efile.st_ops_data) 1219 *size = obj->efile.st_ops_data->d_size; 1220 } else { 1221 ret = bpf_object_search_section_size(obj, name, &d_size); 1222 if (!ret) 1223 *size = d_size; 1224 } 1225 1226 return *size ? 0 : ret; 1227 } 1228 1229 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 1230 __u32 *off) 1231 { 1232 Elf_Data *symbols = obj->efile.symbols; 1233 const char *sname; 1234 size_t si; 1235 1236 if (!name || !off) 1237 return -EINVAL; 1238 1239 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 1240 GElf_Sym sym; 1241 1242 if (!gelf_getsym(symbols, si, &sym)) 1243 continue; 1244 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 1245 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 1246 continue; 1247 1248 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1249 sym.st_name); 1250 if (!sname) { 1251 pr_warn("failed to get sym name string for var %s\n", 1252 name); 1253 return -EIO; 1254 } 1255 if (strcmp(name, sname) == 0) { 1256 *off = sym.st_value; 1257 return 0; 1258 } 1259 } 1260 1261 return -ENOENT; 1262 } 1263 1264 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1265 { 1266 struct bpf_map *new_maps; 1267 size_t new_cap; 1268 int i; 1269 1270 if (obj->nr_maps < obj->maps_cap) 1271 return &obj->maps[obj->nr_maps++]; 1272 1273 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 1274 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); 1275 if (!new_maps) { 1276 pr_warn("alloc maps for object failed\n"); 1277 return ERR_PTR(-ENOMEM); 1278 } 1279 1280 obj->maps_cap = new_cap; 1281 obj->maps = new_maps; 1282 1283 /* zero out new maps */ 1284 memset(obj->maps + obj->nr_maps, 0, 1285 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 1286 /* 1287 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 1288 * when failure (zclose won't close negative fd)). 1289 */ 1290 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 1291 obj->maps[i].fd = -1; 1292 obj->maps[i].inner_map_fd = -1; 1293 } 1294 1295 return &obj->maps[obj->nr_maps++]; 1296 } 1297 1298 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1299 { 1300 long page_sz = sysconf(_SC_PAGE_SIZE); 1301 size_t map_sz; 1302 1303 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; 1304 map_sz = roundup(map_sz, page_sz); 1305 return map_sz; 1306 } 1307 1308 static char *internal_map_name(struct bpf_object *obj, 1309 enum libbpf_map_type type) 1310 { 1311 char map_name[BPF_OBJ_NAME_LEN], *p; 1312 const char *sfx = libbpf_type_to_btf_name[type]; 1313 int sfx_len = max((size_t)7, strlen(sfx)); 1314 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, 1315 strlen(obj->name)); 1316 1317 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1318 sfx_len, libbpf_type_to_btf_name[type]); 1319 1320 /* sanitise map name to characters allowed by kernel */ 1321 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1322 if (!isalnum(*p) && *p != '_' && *p != '.') 1323 *p = '_'; 1324 1325 return strdup(map_name); 1326 } 1327 1328 static int 1329 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1330 int sec_idx, void *data, size_t data_sz) 1331 { 1332 struct bpf_map_def *def; 1333 struct bpf_map *map; 1334 int err; 1335 1336 map = bpf_object__add_map(obj); 1337 if (IS_ERR(map)) 1338 return PTR_ERR(map); 1339 1340 map->libbpf_type = type; 1341 map->sec_idx = sec_idx; 1342 map->sec_offset = 0; 1343 map->name = internal_map_name(obj, type); 1344 if (!map->name) { 1345 pr_warn("failed to alloc map name\n"); 1346 return -ENOMEM; 1347 } 1348 1349 def = &map->def; 1350 def->type = BPF_MAP_TYPE_ARRAY; 1351 def->key_size = sizeof(int); 1352 def->value_size = data_sz; 1353 def->max_entries = 1; 1354 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1355 ? BPF_F_RDONLY_PROG : 0; 1356 def->map_flags |= BPF_F_MMAPABLE; 1357 1358 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1359 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1360 1361 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 1362 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1363 if (map->mmaped == MAP_FAILED) { 1364 err = -errno; 1365 map->mmaped = NULL; 1366 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1367 map->name, err); 1368 zfree(&map->name); 1369 return err; 1370 } 1371 1372 if (data) 1373 memcpy(map->mmaped, data, data_sz); 1374 1375 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1376 return 0; 1377 } 1378 1379 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1380 { 1381 int err; 1382 1383 /* 1384 * Populate obj->maps with libbpf internal maps. 1385 */ 1386 if (obj->efile.data_shndx >= 0) { 1387 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1388 obj->efile.data_shndx, 1389 obj->efile.data->d_buf, 1390 obj->efile.data->d_size); 1391 if (err) 1392 return err; 1393 } 1394 if (obj->efile.rodata_shndx >= 0) { 1395 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1396 obj->efile.rodata_shndx, 1397 obj->efile.rodata->d_buf, 1398 obj->efile.rodata->d_size); 1399 if (err) 1400 return err; 1401 } 1402 if (obj->efile.bss_shndx >= 0) { 1403 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1404 obj->efile.bss_shndx, 1405 NULL, 1406 obj->efile.bss->d_size); 1407 if (err) 1408 return err; 1409 } 1410 return 0; 1411 } 1412 1413 1414 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1415 const void *name) 1416 { 1417 int i; 1418 1419 for (i = 0; i < obj->nr_extern; i++) { 1420 if (strcmp(obj->externs[i].name, name) == 0) 1421 return &obj->externs[i]; 1422 } 1423 return NULL; 1424 } 1425 1426 static int set_ext_value_tri(struct extern_desc *ext, void *ext_val, 1427 char value) 1428 { 1429 switch (ext->type) { 1430 case EXT_BOOL: 1431 if (value == 'm') { 1432 pr_warn("extern %s=%c should be tristate or char\n", 1433 ext->name, value); 1434 return -EINVAL; 1435 } 1436 *(bool *)ext_val = value == 'y' ? true : false; 1437 break; 1438 case EXT_TRISTATE: 1439 if (value == 'y') 1440 *(enum libbpf_tristate *)ext_val = TRI_YES; 1441 else if (value == 'm') 1442 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1443 else /* value == 'n' */ 1444 *(enum libbpf_tristate *)ext_val = TRI_NO; 1445 break; 1446 case EXT_CHAR: 1447 *(char *)ext_val = value; 1448 break; 1449 case EXT_UNKNOWN: 1450 case EXT_INT: 1451 case EXT_CHAR_ARR: 1452 default: 1453 pr_warn("extern %s=%c should be bool, tristate, or char\n", 1454 ext->name, value); 1455 return -EINVAL; 1456 } 1457 ext->is_set = true; 1458 return 0; 1459 } 1460 1461 static int set_ext_value_str(struct extern_desc *ext, char *ext_val, 1462 const char *value) 1463 { 1464 size_t len; 1465 1466 if (ext->type != EXT_CHAR_ARR) { 1467 pr_warn("extern %s=%s should char array\n", ext->name, value); 1468 return -EINVAL; 1469 } 1470 1471 len = strlen(value); 1472 if (value[len - 1] != '"') { 1473 pr_warn("extern '%s': invalid string config '%s'\n", 1474 ext->name, value); 1475 return -EINVAL; 1476 } 1477 1478 /* strip quotes */ 1479 len -= 2; 1480 if (len >= ext->sz) { 1481 pr_warn("extern '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", 1482 ext->name, value, len, ext->sz - 1); 1483 len = ext->sz - 1; 1484 } 1485 memcpy(ext_val, value + 1, len); 1486 ext_val[len] = '\0'; 1487 ext->is_set = true; 1488 return 0; 1489 } 1490 1491 static int parse_u64(const char *value, __u64 *res) 1492 { 1493 char *value_end; 1494 int err; 1495 1496 errno = 0; 1497 *res = strtoull(value, &value_end, 0); 1498 if (errno) { 1499 err = -errno; 1500 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1501 return err; 1502 } 1503 if (*value_end) { 1504 pr_warn("failed to parse '%s' as integer completely\n", value); 1505 return -EINVAL; 1506 } 1507 return 0; 1508 } 1509 1510 static bool is_ext_value_in_range(const struct extern_desc *ext, __u64 v) 1511 { 1512 int bit_sz = ext->sz * 8; 1513 1514 if (ext->sz == 8) 1515 return true; 1516 1517 /* Validate that value stored in u64 fits in integer of `ext->sz` 1518 * bytes size without any loss of information. If the target integer 1519 * is signed, we rely on the following limits of integer type of 1520 * Y bits and subsequent transformation: 1521 * 1522 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1523 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1524 * 0 <= X + 2^(Y-1) < 2^Y 1525 * 1526 * For unsigned target integer, check that all the (64 - Y) bits are 1527 * zero. 1528 */ 1529 if (ext->is_signed) 1530 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1531 else 1532 return (v >> bit_sz) == 0; 1533 } 1534 1535 static int set_ext_value_num(struct extern_desc *ext, void *ext_val, 1536 __u64 value) 1537 { 1538 if (ext->type != EXT_INT && ext->type != EXT_CHAR) { 1539 pr_warn("extern %s=%llu should be integer\n", 1540 ext->name, (unsigned long long)value); 1541 return -EINVAL; 1542 } 1543 if (!is_ext_value_in_range(ext, value)) { 1544 pr_warn("extern %s=%llu value doesn't fit in %d bytes\n", 1545 ext->name, (unsigned long long)value, ext->sz); 1546 return -ERANGE; 1547 } 1548 switch (ext->sz) { 1549 case 1: *(__u8 *)ext_val = value; break; 1550 case 2: *(__u16 *)ext_val = value; break; 1551 case 4: *(__u32 *)ext_val = value; break; 1552 case 8: *(__u64 *)ext_val = value; break; 1553 default: 1554 return -EINVAL; 1555 } 1556 ext->is_set = true; 1557 return 0; 1558 } 1559 1560 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1561 char *buf, void *data) 1562 { 1563 struct extern_desc *ext; 1564 char *sep, *value; 1565 int len, err = 0; 1566 void *ext_val; 1567 __u64 num; 1568 1569 if (strncmp(buf, "CONFIG_", 7)) 1570 return 0; 1571 1572 sep = strchr(buf, '='); 1573 if (!sep) { 1574 pr_warn("failed to parse '%s': no separator\n", buf); 1575 return -EINVAL; 1576 } 1577 1578 /* Trim ending '\n' */ 1579 len = strlen(buf); 1580 if (buf[len - 1] == '\n') 1581 buf[len - 1] = '\0'; 1582 /* Split on '=' and ensure that a value is present. */ 1583 *sep = '\0'; 1584 if (!sep[1]) { 1585 *sep = '='; 1586 pr_warn("failed to parse '%s': no value\n", buf); 1587 return -EINVAL; 1588 } 1589 1590 ext = find_extern_by_name(obj, buf); 1591 if (!ext || ext->is_set) 1592 return 0; 1593 1594 ext_val = data + ext->data_off; 1595 value = sep + 1; 1596 1597 switch (*value) { 1598 case 'y': case 'n': case 'm': 1599 err = set_ext_value_tri(ext, ext_val, *value); 1600 break; 1601 case '"': 1602 err = set_ext_value_str(ext, ext_val, value); 1603 break; 1604 default: 1605 /* assume integer */ 1606 err = parse_u64(value, &num); 1607 if (err) { 1608 pr_warn("extern %s=%s should be integer\n", 1609 ext->name, value); 1610 return err; 1611 } 1612 err = set_ext_value_num(ext, ext_val, num); 1613 break; 1614 } 1615 if (err) 1616 return err; 1617 pr_debug("extern %s=%s\n", ext->name, value); 1618 return 0; 1619 } 1620 1621 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1622 { 1623 char buf[PATH_MAX]; 1624 struct utsname uts; 1625 int len, err = 0; 1626 gzFile file; 1627 1628 uname(&uts); 1629 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1630 if (len < 0) 1631 return -EINVAL; 1632 else if (len >= PATH_MAX) 1633 return -ENAMETOOLONG; 1634 1635 /* gzopen also accepts uncompressed files. */ 1636 file = gzopen(buf, "r"); 1637 if (!file) 1638 file = gzopen("/proc/config.gz", "r"); 1639 1640 if (!file) { 1641 pr_warn("failed to open system Kconfig\n"); 1642 return -ENOENT; 1643 } 1644 1645 while (gzgets(file, buf, sizeof(buf))) { 1646 err = bpf_object__process_kconfig_line(obj, buf, data); 1647 if (err) { 1648 pr_warn("error parsing system Kconfig line '%s': %d\n", 1649 buf, err); 1650 goto out; 1651 } 1652 } 1653 1654 out: 1655 gzclose(file); 1656 return err; 1657 } 1658 1659 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 1660 const char *config, void *data) 1661 { 1662 char buf[PATH_MAX]; 1663 int err = 0; 1664 FILE *file; 1665 1666 file = fmemopen((void *)config, strlen(config), "r"); 1667 if (!file) { 1668 err = -errno; 1669 pr_warn("failed to open in-memory Kconfig: %d\n", err); 1670 return err; 1671 } 1672 1673 while (fgets(buf, sizeof(buf), file)) { 1674 err = bpf_object__process_kconfig_line(obj, buf, data); 1675 if (err) { 1676 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 1677 buf, err); 1678 break; 1679 } 1680 } 1681 1682 fclose(file); 1683 return err; 1684 } 1685 1686 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 1687 { 1688 struct extern_desc *last_ext; 1689 size_t map_sz; 1690 int err; 1691 1692 if (obj->nr_extern == 0) 1693 return 0; 1694 1695 last_ext = &obj->externs[obj->nr_extern - 1]; 1696 map_sz = last_ext->data_off + last_ext->sz; 1697 1698 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 1699 obj->efile.symbols_shndx, 1700 NULL, map_sz); 1701 if (err) 1702 return err; 1703 1704 obj->kconfig_map_idx = obj->nr_maps - 1; 1705 1706 return 0; 1707 } 1708 1709 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 1710 { 1711 Elf_Data *symbols = obj->efile.symbols; 1712 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 1713 Elf_Data *data = NULL; 1714 Elf_Scn *scn; 1715 1716 if (obj->efile.maps_shndx < 0) 1717 return 0; 1718 1719 if (!symbols) 1720 return -EINVAL; 1721 1722 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 1723 if (scn) 1724 data = elf_getdata(scn, NULL); 1725 if (!scn || !data) { 1726 pr_warn("failed to get Elf_Data from map section %d\n", 1727 obj->efile.maps_shndx); 1728 return -EINVAL; 1729 } 1730 1731 /* 1732 * Count number of maps. Each map has a name. 1733 * Array of maps is not supported: only the first element is 1734 * considered. 1735 * 1736 * TODO: Detect array of map and report error. 1737 */ 1738 nr_syms = symbols->d_size / sizeof(GElf_Sym); 1739 for (i = 0; i < nr_syms; i++) { 1740 GElf_Sym sym; 1741 1742 if (!gelf_getsym(symbols, i, &sym)) 1743 continue; 1744 if (sym.st_shndx != obj->efile.maps_shndx) 1745 continue; 1746 nr_maps++; 1747 } 1748 /* Assume equally sized map definitions */ 1749 pr_debug("maps in %s: %d maps in %zd bytes\n", 1750 obj->path, nr_maps, data->d_size); 1751 1752 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { 1753 pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n", 1754 obj->path, nr_maps, data->d_size); 1755 return -EINVAL; 1756 } 1757 map_def_sz = data->d_size / nr_maps; 1758 1759 /* Fill obj->maps using data in "maps" section. */ 1760 for (i = 0; i < nr_syms; i++) { 1761 GElf_Sym sym; 1762 const char *map_name; 1763 struct bpf_map_def *def; 1764 struct bpf_map *map; 1765 1766 if (!gelf_getsym(symbols, i, &sym)) 1767 continue; 1768 if (sym.st_shndx != obj->efile.maps_shndx) 1769 continue; 1770 1771 map = bpf_object__add_map(obj); 1772 if (IS_ERR(map)) 1773 return PTR_ERR(map); 1774 1775 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1776 sym.st_name); 1777 if (!map_name) { 1778 pr_warn("failed to get map #%d name sym string for obj %s\n", 1779 i, obj->path); 1780 return -LIBBPF_ERRNO__FORMAT; 1781 } 1782 1783 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1784 map->sec_idx = sym.st_shndx; 1785 map->sec_offset = sym.st_value; 1786 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 1787 map_name, map->sec_idx, map->sec_offset); 1788 if (sym.st_value + map_def_sz > data->d_size) { 1789 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", 1790 obj->path, map_name); 1791 return -EINVAL; 1792 } 1793 1794 map->name = strdup(map_name); 1795 if (!map->name) { 1796 pr_warn("failed to alloc map name\n"); 1797 return -ENOMEM; 1798 } 1799 pr_debug("map %d is \"%s\"\n", i, map->name); 1800 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 1801 /* 1802 * If the definition of the map in the object file fits in 1803 * bpf_map_def, copy it. Any extra fields in our version 1804 * of bpf_map_def will default to zero as a result of the 1805 * calloc above. 1806 */ 1807 if (map_def_sz <= sizeof(struct bpf_map_def)) { 1808 memcpy(&map->def, def, map_def_sz); 1809 } else { 1810 /* 1811 * Here the map structure being read is bigger than what 1812 * we expect, truncate if the excess bits are all zero. 1813 * If they are not zero, reject this map as 1814 * incompatible. 1815 */ 1816 char *b; 1817 1818 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1819 b < ((char *)def) + map_def_sz; b++) { 1820 if (*b != 0) { 1821 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", 1822 obj->path, map_name); 1823 if (strict) 1824 return -EINVAL; 1825 } 1826 } 1827 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1828 } 1829 } 1830 return 0; 1831 } 1832 1833 static const struct btf_type * 1834 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 1835 { 1836 const struct btf_type *t = btf__type_by_id(btf, id); 1837 1838 if (res_id) 1839 *res_id = id; 1840 1841 while (btf_is_mod(t) || btf_is_typedef(t)) { 1842 if (res_id) 1843 *res_id = t->type; 1844 t = btf__type_by_id(btf, t->type); 1845 } 1846 1847 return t; 1848 } 1849 1850 static const struct btf_type * 1851 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 1852 { 1853 const struct btf_type *t; 1854 1855 t = skip_mods_and_typedefs(btf, id, NULL); 1856 if (!btf_is_ptr(t)) 1857 return NULL; 1858 1859 t = skip_mods_and_typedefs(btf, t->type, res_id); 1860 1861 return btf_is_func_proto(t) ? t : NULL; 1862 } 1863 1864 /* 1865 * Fetch integer attribute of BTF map definition. Such attributes are 1866 * represented using a pointer to an array, in which dimensionality of array 1867 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 1868 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 1869 * type definition, while using only sizeof(void *) space in ELF data section. 1870 */ 1871 static bool get_map_field_int(const char *map_name, const struct btf *btf, 1872 const struct btf_member *m, __u32 *res) 1873 { 1874 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 1875 const char *name = btf__name_by_offset(btf, m->name_off); 1876 const struct btf_array *arr_info; 1877 const struct btf_type *arr_t; 1878 1879 if (!btf_is_ptr(t)) { 1880 pr_warn("map '%s': attr '%s': expected PTR, got %u.\n", 1881 map_name, name, btf_kind(t)); 1882 return false; 1883 } 1884 1885 arr_t = btf__type_by_id(btf, t->type); 1886 if (!arr_t) { 1887 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 1888 map_name, name, t->type); 1889 return false; 1890 } 1891 if (!btf_is_array(arr_t)) { 1892 pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n", 1893 map_name, name, btf_kind(arr_t)); 1894 return false; 1895 } 1896 arr_info = btf_array(arr_t); 1897 *res = arr_info->nelems; 1898 return true; 1899 } 1900 1901 static int build_map_pin_path(struct bpf_map *map, const char *path) 1902 { 1903 char buf[PATH_MAX]; 1904 int err, len; 1905 1906 if (!path) 1907 path = "/sys/fs/bpf"; 1908 1909 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); 1910 if (len < 0) 1911 return -EINVAL; 1912 else if (len >= PATH_MAX) 1913 return -ENAMETOOLONG; 1914 1915 err = bpf_map__set_pin_path(map, buf); 1916 if (err) 1917 return err; 1918 1919 return 0; 1920 } 1921 1922 1923 static int parse_btf_map_def(struct bpf_object *obj, 1924 struct bpf_map *map, 1925 const struct btf_type *def, 1926 bool strict, bool is_inner, 1927 const char *pin_root_path) 1928 { 1929 const struct btf_type *t; 1930 const struct btf_member *m; 1931 int vlen, i; 1932 1933 vlen = btf_vlen(def); 1934 m = btf_members(def); 1935 for (i = 0; i < vlen; i++, m++) { 1936 const char *name = btf__name_by_offset(obj->btf, m->name_off); 1937 1938 if (!name) { 1939 pr_warn("map '%s': invalid field #%d.\n", map->name, i); 1940 return -EINVAL; 1941 } 1942 if (strcmp(name, "type") == 0) { 1943 if (!get_map_field_int(map->name, obj->btf, m, 1944 &map->def.type)) 1945 return -EINVAL; 1946 pr_debug("map '%s': found type = %u.\n", 1947 map->name, map->def.type); 1948 } else if (strcmp(name, "max_entries") == 0) { 1949 if (!get_map_field_int(map->name, obj->btf, m, 1950 &map->def.max_entries)) 1951 return -EINVAL; 1952 pr_debug("map '%s': found max_entries = %u.\n", 1953 map->name, map->def.max_entries); 1954 } else if (strcmp(name, "map_flags") == 0) { 1955 if (!get_map_field_int(map->name, obj->btf, m, 1956 &map->def.map_flags)) 1957 return -EINVAL; 1958 pr_debug("map '%s': found map_flags = %u.\n", 1959 map->name, map->def.map_flags); 1960 } else if (strcmp(name, "key_size") == 0) { 1961 __u32 sz; 1962 1963 if (!get_map_field_int(map->name, obj->btf, m, &sz)) 1964 return -EINVAL; 1965 pr_debug("map '%s': found key_size = %u.\n", 1966 map->name, sz); 1967 if (map->def.key_size && map->def.key_size != sz) { 1968 pr_warn("map '%s': conflicting key size %u != %u.\n", 1969 map->name, map->def.key_size, sz); 1970 return -EINVAL; 1971 } 1972 map->def.key_size = sz; 1973 } else if (strcmp(name, "key") == 0) { 1974 __s64 sz; 1975 1976 t = btf__type_by_id(obj->btf, m->type); 1977 if (!t) { 1978 pr_warn("map '%s': key type [%d] not found.\n", 1979 map->name, m->type); 1980 return -EINVAL; 1981 } 1982 if (!btf_is_ptr(t)) { 1983 pr_warn("map '%s': key spec is not PTR: %u.\n", 1984 map->name, btf_kind(t)); 1985 return -EINVAL; 1986 } 1987 sz = btf__resolve_size(obj->btf, t->type); 1988 if (sz < 0) { 1989 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 1990 map->name, t->type, (ssize_t)sz); 1991 return sz; 1992 } 1993 pr_debug("map '%s': found key [%u], sz = %zd.\n", 1994 map->name, t->type, (ssize_t)sz); 1995 if (map->def.key_size && map->def.key_size != sz) { 1996 pr_warn("map '%s': conflicting key size %u != %zd.\n", 1997 map->name, map->def.key_size, (ssize_t)sz); 1998 return -EINVAL; 1999 } 2000 map->def.key_size = sz; 2001 map->btf_key_type_id = t->type; 2002 } else if (strcmp(name, "value_size") == 0) { 2003 __u32 sz; 2004 2005 if (!get_map_field_int(map->name, obj->btf, m, &sz)) 2006 return -EINVAL; 2007 pr_debug("map '%s': found value_size = %u.\n", 2008 map->name, sz); 2009 if (map->def.value_size && map->def.value_size != sz) { 2010 pr_warn("map '%s': conflicting value size %u != %u.\n", 2011 map->name, map->def.value_size, sz); 2012 return -EINVAL; 2013 } 2014 map->def.value_size = sz; 2015 } else if (strcmp(name, "value") == 0) { 2016 __s64 sz; 2017 2018 t = btf__type_by_id(obj->btf, m->type); 2019 if (!t) { 2020 pr_warn("map '%s': value type [%d] not found.\n", 2021 map->name, m->type); 2022 return -EINVAL; 2023 } 2024 if (!btf_is_ptr(t)) { 2025 pr_warn("map '%s': value spec is not PTR: %u.\n", 2026 map->name, btf_kind(t)); 2027 return -EINVAL; 2028 } 2029 sz = btf__resolve_size(obj->btf, t->type); 2030 if (sz < 0) { 2031 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2032 map->name, t->type, (ssize_t)sz); 2033 return sz; 2034 } 2035 pr_debug("map '%s': found value [%u], sz = %zd.\n", 2036 map->name, t->type, (ssize_t)sz); 2037 if (map->def.value_size && map->def.value_size != sz) { 2038 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2039 map->name, map->def.value_size, (ssize_t)sz); 2040 return -EINVAL; 2041 } 2042 map->def.value_size = sz; 2043 map->btf_value_type_id = t->type; 2044 } 2045 else if (strcmp(name, "values") == 0) { 2046 int err; 2047 2048 if (is_inner) { 2049 pr_warn("map '%s': multi-level inner maps not supported.\n", 2050 map->name); 2051 return -ENOTSUP; 2052 } 2053 if (i != vlen - 1) { 2054 pr_warn("map '%s': '%s' member should be last.\n", 2055 map->name, name); 2056 return -EINVAL; 2057 } 2058 if (!bpf_map_type__is_map_in_map(map->def.type)) { 2059 pr_warn("map '%s': should be map-in-map.\n", 2060 map->name); 2061 return -ENOTSUP; 2062 } 2063 if (map->def.value_size && map->def.value_size != 4) { 2064 pr_warn("map '%s': conflicting value size %u != 4.\n", 2065 map->name, map->def.value_size); 2066 return -EINVAL; 2067 } 2068 map->def.value_size = 4; 2069 t = btf__type_by_id(obj->btf, m->type); 2070 if (!t) { 2071 pr_warn("map '%s': map-in-map inner type [%d] not found.\n", 2072 map->name, m->type); 2073 return -EINVAL; 2074 } 2075 if (!btf_is_array(t) || btf_array(t)->nelems) { 2076 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n", 2077 map->name); 2078 return -EINVAL; 2079 } 2080 t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type, 2081 NULL); 2082 if (!btf_is_ptr(t)) { 2083 pr_warn("map '%s': map-in-map inner def is of unexpected kind %u.\n", 2084 map->name, btf_kind(t)); 2085 return -EINVAL; 2086 } 2087 t = skip_mods_and_typedefs(obj->btf, t->type, NULL); 2088 if (!btf_is_struct(t)) { 2089 pr_warn("map '%s': map-in-map inner def is of unexpected kind %u.\n", 2090 map->name, btf_kind(t)); 2091 return -EINVAL; 2092 } 2093 2094 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2095 if (!map->inner_map) 2096 return -ENOMEM; 2097 map->inner_map->sec_idx = obj->efile.btf_maps_shndx; 2098 map->inner_map->name = malloc(strlen(map->name) + 2099 sizeof(".inner") + 1); 2100 if (!map->inner_map->name) 2101 return -ENOMEM; 2102 sprintf(map->inner_map->name, "%s.inner", map->name); 2103 2104 err = parse_btf_map_def(obj, map->inner_map, t, strict, 2105 true /* is_inner */, NULL); 2106 if (err) 2107 return err; 2108 } else if (strcmp(name, "pinning") == 0) { 2109 __u32 val; 2110 int err; 2111 2112 if (is_inner) { 2113 pr_debug("map '%s': inner def can't be pinned.\n", 2114 map->name); 2115 return -EINVAL; 2116 } 2117 if (!get_map_field_int(map->name, obj->btf, m, &val)) 2118 return -EINVAL; 2119 pr_debug("map '%s': found pinning = %u.\n", 2120 map->name, val); 2121 2122 if (val != LIBBPF_PIN_NONE && 2123 val != LIBBPF_PIN_BY_NAME) { 2124 pr_warn("map '%s': invalid pinning value %u.\n", 2125 map->name, val); 2126 return -EINVAL; 2127 } 2128 if (val == LIBBPF_PIN_BY_NAME) { 2129 err = build_map_pin_path(map, pin_root_path); 2130 if (err) { 2131 pr_warn("map '%s': couldn't build pin path.\n", 2132 map->name); 2133 return err; 2134 } 2135 } 2136 } else { 2137 if (strict) { 2138 pr_warn("map '%s': unknown field '%s'.\n", 2139 map->name, name); 2140 return -ENOTSUP; 2141 } 2142 pr_debug("map '%s': ignoring unknown field '%s'.\n", 2143 map->name, name); 2144 } 2145 } 2146 2147 if (map->def.type == BPF_MAP_TYPE_UNSPEC) { 2148 pr_warn("map '%s': map type isn't specified.\n", map->name); 2149 return -EINVAL; 2150 } 2151 2152 return 0; 2153 } 2154 2155 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2156 const struct btf_type *sec, 2157 int var_idx, int sec_idx, 2158 const Elf_Data *data, bool strict, 2159 const char *pin_root_path) 2160 { 2161 const struct btf_type *var, *def; 2162 const struct btf_var_secinfo *vi; 2163 const struct btf_var *var_extra; 2164 const char *map_name; 2165 struct bpf_map *map; 2166 2167 vi = btf_var_secinfos(sec) + var_idx; 2168 var = btf__type_by_id(obj->btf, vi->type); 2169 var_extra = btf_var(var); 2170 map_name = btf__name_by_offset(obj->btf, var->name_off); 2171 2172 if (map_name == NULL || map_name[0] == '\0') { 2173 pr_warn("map #%d: empty name.\n", var_idx); 2174 return -EINVAL; 2175 } 2176 if ((__u64)vi->offset + vi->size > data->d_size) { 2177 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2178 return -EINVAL; 2179 } 2180 if (!btf_is_var(var)) { 2181 pr_warn("map '%s': unexpected var kind %u.\n", 2182 map_name, btf_kind(var)); 2183 return -EINVAL; 2184 } 2185 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && 2186 var_extra->linkage != BTF_VAR_STATIC) { 2187 pr_warn("map '%s': unsupported var linkage %u.\n", 2188 map_name, var_extra->linkage); 2189 return -EOPNOTSUPP; 2190 } 2191 2192 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2193 if (!btf_is_struct(def)) { 2194 pr_warn("map '%s': unexpected def kind %u.\n", 2195 map_name, btf_kind(var)); 2196 return -EINVAL; 2197 } 2198 if (def->size > vi->size) { 2199 pr_warn("map '%s': invalid def size.\n", map_name); 2200 return -EINVAL; 2201 } 2202 2203 map = bpf_object__add_map(obj); 2204 if (IS_ERR(map)) 2205 return PTR_ERR(map); 2206 map->name = strdup(map_name); 2207 if (!map->name) { 2208 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2209 return -ENOMEM; 2210 } 2211 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2212 map->def.type = BPF_MAP_TYPE_UNSPEC; 2213 map->sec_idx = sec_idx; 2214 map->sec_offset = vi->offset; 2215 map->btf_var_idx = var_idx; 2216 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2217 map_name, map->sec_idx, map->sec_offset); 2218 2219 return parse_btf_map_def(obj, map, def, strict, false, pin_root_path); 2220 } 2221 2222 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2223 const char *pin_root_path) 2224 { 2225 const struct btf_type *sec = NULL; 2226 int nr_types, i, vlen, err; 2227 const struct btf_type *t; 2228 const char *name; 2229 Elf_Data *data; 2230 Elf_Scn *scn; 2231 2232 if (obj->efile.btf_maps_shndx < 0) 2233 return 0; 2234 2235 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); 2236 if (scn) 2237 data = elf_getdata(scn, NULL); 2238 if (!scn || !data) { 2239 pr_warn("failed to get Elf_Data from map section %d (%s)\n", 2240 obj->efile.maps_shndx, MAPS_ELF_SEC); 2241 return -EINVAL; 2242 } 2243 2244 nr_types = btf__get_nr_types(obj->btf); 2245 for (i = 1; i <= nr_types; i++) { 2246 t = btf__type_by_id(obj->btf, i); 2247 if (!btf_is_datasec(t)) 2248 continue; 2249 name = btf__name_by_offset(obj->btf, t->name_off); 2250 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2251 sec = t; 2252 obj->efile.btf_maps_sec_btf_id = i; 2253 break; 2254 } 2255 } 2256 2257 if (!sec) { 2258 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2259 return -ENOENT; 2260 } 2261 2262 vlen = btf_vlen(sec); 2263 for (i = 0; i < vlen; i++) { 2264 err = bpf_object__init_user_btf_map(obj, sec, i, 2265 obj->efile.btf_maps_shndx, 2266 data, strict, 2267 pin_root_path); 2268 if (err) 2269 return err; 2270 } 2271 2272 return 0; 2273 } 2274 2275 static int bpf_object__init_maps(struct bpf_object *obj, 2276 const struct bpf_object_open_opts *opts) 2277 { 2278 const char *pin_root_path; 2279 bool strict; 2280 int err; 2281 2282 strict = !OPTS_GET(opts, relaxed_maps, false); 2283 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2284 2285 err = bpf_object__init_user_maps(obj, strict); 2286 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2287 err = err ?: bpf_object__init_global_data_maps(obj); 2288 err = err ?: bpf_object__init_kconfig_map(obj); 2289 err = err ?: bpf_object__init_struct_ops_maps(obj); 2290 if (err) 2291 return err; 2292 2293 return 0; 2294 } 2295 2296 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2297 { 2298 Elf_Scn *scn; 2299 GElf_Shdr sh; 2300 2301 scn = elf_getscn(obj->efile.elf, idx); 2302 if (!scn) 2303 return false; 2304 2305 if (gelf_getshdr(scn, &sh) != &sh) 2306 return false; 2307 2308 if (sh.sh_flags & SHF_EXECINSTR) 2309 return true; 2310 2311 return false; 2312 } 2313 2314 static void bpf_object__sanitize_btf(struct bpf_object *obj) 2315 { 2316 bool has_func_global = obj->caps.btf_func_global; 2317 bool has_datasec = obj->caps.btf_datasec; 2318 bool has_func = obj->caps.btf_func; 2319 struct btf *btf = obj->btf; 2320 struct btf_type *t; 2321 int i, j, vlen; 2322 2323 if (!obj->btf || (has_func && has_datasec && has_func_global)) 2324 return; 2325 2326 for (i = 1; i <= btf__get_nr_types(btf); i++) { 2327 t = (struct btf_type *)btf__type_by_id(btf, i); 2328 2329 if (!has_datasec && btf_is_var(t)) { 2330 /* replace VAR with INT */ 2331 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2332 /* 2333 * using size = 1 is the safest choice, 4 will be too 2334 * big and cause kernel BTF validation failure if 2335 * original variable took less than 4 bytes 2336 */ 2337 t->size = 1; 2338 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2339 } else if (!has_datasec && btf_is_datasec(t)) { 2340 /* replace DATASEC with STRUCT */ 2341 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2342 struct btf_member *m = btf_members(t); 2343 struct btf_type *vt; 2344 char *name; 2345 2346 name = (char *)btf__name_by_offset(btf, t->name_off); 2347 while (*name) { 2348 if (*name == '.') 2349 *name = '_'; 2350 name++; 2351 } 2352 2353 vlen = btf_vlen(t); 2354 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2355 for (j = 0; j < vlen; j++, v++, m++) { 2356 /* order of field assignments is important */ 2357 m->offset = v->offset * 8; 2358 m->type = v->type; 2359 /* preserve variable name as member name */ 2360 vt = (void *)btf__type_by_id(btf, v->type); 2361 m->name_off = vt->name_off; 2362 } 2363 } else if (!has_func && btf_is_func_proto(t)) { 2364 /* replace FUNC_PROTO with ENUM */ 2365 vlen = btf_vlen(t); 2366 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2367 t->size = sizeof(__u32); /* kernel enforced */ 2368 } else if (!has_func && btf_is_func(t)) { 2369 /* replace FUNC with TYPEDEF */ 2370 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2371 } else if (!has_func_global && btf_is_func(t)) { 2372 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2373 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2374 } 2375 } 2376 } 2377 2378 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 2379 { 2380 if (!obj->btf_ext) 2381 return; 2382 2383 if (!obj->caps.btf_func) { 2384 btf_ext__free(obj->btf_ext); 2385 obj->btf_ext = NULL; 2386 } 2387 } 2388 2389 static bool libbpf_needs_btf(const struct bpf_object *obj) 2390 { 2391 return obj->efile.btf_maps_shndx >= 0 || 2392 obj->efile.st_ops_shndx >= 0 || 2393 obj->nr_extern > 0; 2394 } 2395 2396 static bool kernel_needs_btf(const struct bpf_object *obj) 2397 { 2398 return obj->efile.st_ops_shndx >= 0; 2399 } 2400 2401 static int bpf_object__init_btf(struct bpf_object *obj, 2402 Elf_Data *btf_data, 2403 Elf_Data *btf_ext_data) 2404 { 2405 int err = -ENOENT; 2406 2407 if (btf_data) { 2408 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2409 if (IS_ERR(obj->btf)) { 2410 err = PTR_ERR(obj->btf); 2411 obj->btf = NULL; 2412 pr_warn("Error loading ELF section %s: %d.\n", 2413 BTF_ELF_SEC, err); 2414 goto out; 2415 } 2416 err = 0; 2417 } 2418 if (btf_ext_data) { 2419 if (!obj->btf) { 2420 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2421 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2422 goto out; 2423 } 2424 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 2425 btf_ext_data->d_size); 2426 if (IS_ERR(obj->btf_ext)) { 2427 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n", 2428 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); 2429 obj->btf_ext = NULL; 2430 goto out; 2431 } 2432 } 2433 out: 2434 if (err && libbpf_needs_btf(obj)) { 2435 pr_warn("BTF is required, but is missing or corrupted.\n"); 2436 return err; 2437 } 2438 return 0; 2439 } 2440 2441 static int bpf_object__finalize_btf(struct bpf_object *obj) 2442 { 2443 int err; 2444 2445 if (!obj->btf) 2446 return 0; 2447 2448 err = btf__finalize_data(obj, obj->btf); 2449 if (!err) 2450 return 0; 2451 2452 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); 2453 btf__free(obj->btf); 2454 obj->btf = NULL; 2455 btf_ext__free(obj->btf_ext); 2456 obj->btf_ext = NULL; 2457 2458 if (libbpf_needs_btf(obj)) { 2459 pr_warn("BTF is required, but is missing or corrupted.\n"); 2460 return -ENOENT; 2461 } 2462 return 0; 2463 } 2464 2465 static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog) 2466 { 2467 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 2468 prog->type == BPF_PROG_TYPE_LSM) 2469 return true; 2470 2471 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 2472 * also need vmlinux BTF 2473 */ 2474 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 2475 return true; 2476 2477 return false; 2478 } 2479 2480 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj) 2481 { 2482 struct bpf_program *prog; 2483 int err; 2484 2485 bpf_object__for_each_program(prog, obj) { 2486 if (libbpf_prog_needs_vmlinux_btf(prog)) { 2487 obj->btf_vmlinux = libbpf_find_kernel_btf(); 2488 if (IS_ERR(obj->btf_vmlinux)) { 2489 err = PTR_ERR(obj->btf_vmlinux); 2490 pr_warn("Error loading vmlinux BTF: %d\n", err); 2491 obj->btf_vmlinux = NULL; 2492 return err; 2493 } 2494 return 0; 2495 } 2496 } 2497 2498 return 0; 2499 } 2500 2501 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 2502 { 2503 int err = 0; 2504 2505 if (!obj->btf) 2506 return 0; 2507 2508 bpf_object__sanitize_btf(obj); 2509 bpf_object__sanitize_btf_ext(obj); 2510 2511 err = btf__load(obj->btf); 2512 if (err) { 2513 pr_warn("Error loading %s into kernel: %d.\n", 2514 BTF_ELF_SEC, err); 2515 btf__free(obj->btf); 2516 obj->btf = NULL; 2517 /* btf_ext can't exist without btf, so free it as well */ 2518 if (obj->btf_ext) { 2519 btf_ext__free(obj->btf_ext); 2520 obj->btf_ext = NULL; 2521 } 2522 2523 if (kernel_needs_btf(obj)) 2524 return err; 2525 } 2526 return 0; 2527 } 2528 2529 static int bpf_object__elf_collect(struct bpf_object *obj) 2530 { 2531 Elf *elf = obj->efile.elf; 2532 GElf_Ehdr *ep = &obj->efile.ehdr; 2533 Elf_Data *btf_ext_data = NULL; 2534 Elf_Data *btf_data = NULL; 2535 Elf_Scn *scn = NULL; 2536 int idx = 0, err = 0; 2537 2538 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 2539 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 2540 pr_warn("failed to get e_shstrndx from %s\n", obj->path); 2541 return -LIBBPF_ERRNO__FORMAT; 2542 } 2543 2544 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2545 char *name; 2546 GElf_Shdr sh; 2547 Elf_Data *data; 2548 2549 idx++; 2550 if (gelf_getshdr(scn, &sh) != &sh) { 2551 pr_warn("failed to get section(%d) header from %s\n", 2552 idx, obj->path); 2553 return -LIBBPF_ERRNO__FORMAT; 2554 } 2555 2556 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 2557 if (!name) { 2558 pr_warn("failed to get section(%d) name from %s\n", 2559 idx, obj->path); 2560 return -LIBBPF_ERRNO__FORMAT; 2561 } 2562 2563 data = elf_getdata(scn, 0); 2564 if (!data) { 2565 pr_warn("failed to get section(%d) data from %s(%s)\n", 2566 idx, name, obj->path); 2567 return -LIBBPF_ERRNO__FORMAT; 2568 } 2569 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 2570 idx, name, (unsigned long)data->d_size, 2571 (int)sh.sh_link, (unsigned long)sh.sh_flags, 2572 (int)sh.sh_type); 2573 2574 if (strcmp(name, "license") == 0) { 2575 err = bpf_object__init_license(obj, 2576 data->d_buf, 2577 data->d_size); 2578 if (err) 2579 return err; 2580 } else if (strcmp(name, "version") == 0) { 2581 err = bpf_object__init_kversion(obj, 2582 data->d_buf, 2583 data->d_size); 2584 if (err) 2585 return err; 2586 } else if (strcmp(name, "maps") == 0) { 2587 obj->efile.maps_shndx = idx; 2588 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 2589 obj->efile.btf_maps_shndx = idx; 2590 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 2591 btf_data = data; 2592 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 2593 btf_ext_data = data; 2594 } else if (sh.sh_type == SHT_SYMTAB) { 2595 if (obj->efile.symbols) { 2596 pr_warn("bpf: multiple SYMTAB in %s\n", 2597 obj->path); 2598 return -LIBBPF_ERRNO__FORMAT; 2599 } 2600 obj->efile.symbols = data; 2601 obj->efile.symbols_shndx = idx; 2602 obj->efile.strtabidx = sh.sh_link; 2603 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 2604 if (sh.sh_flags & SHF_EXECINSTR) { 2605 if (strcmp(name, ".text") == 0) 2606 obj->efile.text_shndx = idx; 2607 err = bpf_object__add_program(obj, data->d_buf, 2608 data->d_size, 2609 name, idx); 2610 if (err) { 2611 char errmsg[STRERR_BUFSIZE]; 2612 char *cp; 2613 2614 cp = libbpf_strerror_r(-err, errmsg, 2615 sizeof(errmsg)); 2616 pr_warn("failed to alloc program %s (%s): %s", 2617 name, obj->path, cp); 2618 return err; 2619 } 2620 } else if (strcmp(name, DATA_SEC) == 0) { 2621 obj->efile.data = data; 2622 obj->efile.data_shndx = idx; 2623 } else if (strcmp(name, RODATA_SEC) == 0) { 2624 obj->efile.rodata = data; 2625 obj->efile.rodata_shndx = idx; 2626 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 2627 obj->efile.st_ops_data = data; 2628 obj->efile.st_ops_shndx = idx; 2629 } else { 2630 pr_debug("skip section(%d) %s\n", idx, name); 2631 } 2632 } else if (sh.sh_type == SHT_REL) { 2633 int nr_sects = obj->efile.nr_reloc_sects; 2634 void *sects = obj->efile.reloc_sects; 2635 int sec = sh.sh_info; /* points to other section */ 2636 2637 /* Only do relo for section with exec instructions */ 2638 if (!section_have_execinstr(obj, sec) && 2639 strcmp(name, ".rel" STRUCT_OPS_SEC) && 2640 strcmp(name, ".rel" MAPS_ELF_SEC)) { 2641 pr_debug("skip relo %s(%d) for section(%d)\n", 2642 name, idx, sec); 2643 continue; 2644 } 2645 2646 sects = reallocarray(sects, nr_sects + 1, 2647 sizeof(*obj->efile.reloc_sects)); 2648 if (!sects) { 2649 pr_warn("reloc_sects realloc failed\n"); 2650 return -ENOMEM; 2651 } 2652 2653 obj->efile.reloc_sects = sects; 2654 obj->efile.nr_reloc_sects++; 2655 2656 obj->efile.reloc_sects[nr_sects].shdr = sh; 2657 obj->efile.reloc_sects[nr_sects].data = data; 2658 } else if (sh.sh_type == SHT_NOBITS && 2659 strcmp(name, BSS_SEC) == 0) { 2660 obj->efile.bss = data; 2661 obj->efile.bss_shndx = idx; 2662 } else { 2663 pr_debug("skip section(%d) %s\n", idx, name); 2664 } 2665 } 2666 2667 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 2668 pr_warn("Corrupted ELF file: index of strtab invalid\n"); 2669 return -LIBBPF_ERRNO__FORMAT; 2670 } 2671 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 2672 } 2673 2674 static bool sym_is_extern(const GElf_Sym *sym) 2675 { 2676 int bind = GELF_ST_BIND(sym->st_info); 2677 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 2678 return sym->st_shndx == SHN_UNDEF && 2679 (bind == STB_GLOBAL || bind == STB_WEAK) && 2680 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 2681 } 2682 2683 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 2684 { 2685 const struct btf_type *t; 2686 const char *var_name; 2687 int i, n; 2688 2689 if (!btf) 2690 return -ESRCH; 2691 2692 n = btf__get_nr_types(btf); 2693 for (i = 1; i <= n; i++) { 2694 t = btf__type_by_id(btf, i); 2695 2696 if (!btf_is_var(t)) 2697 continue; 2698 2699 var_name = btf__name_by_offset(btf, t->name_off); 2700 if (strcmp(var_name, ext_name)) 2701 continue; 2702 2703 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 2704 return -EINVAL; 2705 2706 return i; 2707 } 2708 2709 return -ENOENT; 2710 } 2711 2712 static enum extern_type find_extern_type(const struct btf *btf, int id, 2713 bool *is_signed) 2714 { 2715 const struct btf_type *t; 2716 const char *name; 2717 2718 t = skip_mods_and_typedefs(btf, id, NULL); 2719 name = btf__name_by_offset(btf, t->name_off); 2720 2721 if (is_signed) 2722 *is_signed = false; 2723 switch (btf_kind(t)) { 2724 case BTF_KIND_INT: { 2725 int enc = btf_int_encoding(t); 2726 2727 if (enc & BTF_INT_BOOL) 2728 return t->size == 1 ? EXT_BOOL : EXT_UNKNOWN; 2729 if (is_signed) 2730 *is_signed = enc & BTF_INT_SIGNED; 2731 if (t->size == 1) 2732 return EXT_CHAR; 2733 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 2734 return EXT_UNKNOWN; 2735 return EXT_INT; 2736 } 2737 case BTF_KIND_ENUM: 2738 if (t->size != 4) 2739 return EXT_UNKNOWN; 2740 if (strcmp(name, "libbpf_tristate")) 2741 return EXT_UNKNOWN; 2742 return EXT_TRISTATE; 2743 case BTF_KIND_ARRAY: 2744 if (btf_array(t)->nelems == 0) 2745 return EXT_UNKNOWN; 2746 if (find_extern_type(btf, btf_array(t)->type, NULL) != EXT_CHAR) 2747 return EXT_UNKNOWN; 2748 return EXT_CHAR_ARR; 2749 default: 2750 return EXT_UNKNOWN; 2751 } 2752 } 2753 2754 static int cmp_externs(const void *_a, const void *_b) 2755 { 2756 const struct extern_desc *a = _a; 2757 const struct extern_desc *b = _b; 2758 2759 /* descending order by alignment requirements */ 2760 if (a->align != b->align) 2761 return a->align > b->align ? -1 : 1; 2762 /* ascending order by size, within same alignment class */ 2763 if (a->sz != b->sz) 2764 return a->sz < b->sz ? -1 : 1; 2765 /* resolve ties by name */ 2766 return strcmp(a->name, b->name); 2767 } 2768 2769 static int bpf_object__collect_externs(struct bpf_object *obj) 2770 { 2771 const struct btf_type *t; 2772 struct extern_desc *ext; 2773 int i, n, off, btf_id; 2774 struct btf_type *sec; 2775 const char *ext_name; 2776 Elf_Scn *scn; 2777 GElf_Shdr sh; 2778 2779 if (!obj->efile.symbols) 2780 return 0; 2781 2782 scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx); 2783 if (!scn) 2784 return -LIBBPF_ERRNO__FORMAT; 2785 if (gelf_getshdr(scn, &sh) != &sh) 2786 return -LIBBPF_ERRNO__FORMAT; 2787 n = sh.sh_size / sh.sh_entsize; 2788 2789 pr_debug("looking for externs among %d symbols...\n", n); 2790 for (i = 0; i < n; i++) { 2791 GElf_Sym sym; 2792 2793 if (!gelf_getsym(obj->efile.symbols, i, &sym)) 2794 return -LIBBPF_ERRNO__FORMAT; 2795 if (!sym_is_extern(&sym)) 2796 continue; 2797 ext_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 2798 sym.st_name); 2799 if (!ext_name || !ext_name[0]) 2800 continue; 2801 2802 ext = obj->externs; 2803 ext = reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 2804 if (!ext) 2805 return -ENOMEM; 2806 obj->externs = ext; 2807 ext = &ext[obj->nr_extern]; 2808 memset(ext, 0, sizeof(*ext)); 2809 obj->nr_extern++; 2810 2811 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 2812 if (ext->btf_id <= 0) { 2813 pr_warn("failed to find BTF for extern '%s': %d\n", 2814 ext_name, ext->btf_id); 2815 return ext->btf_id; 2816 } 2817 t = btf__type_by_id(obj->btf, ext->btf_id); 2818 ext->name = btf__name_by_offset(obj->btf, t->name_off); 2819 ext->sym_idx = i; 2820 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; 2821 ext->sz = btf__resolve_size(obj->btf, t->type); 2822 if (ext->sz <= 0) { 2823 pr_warn("failed to resolve size of extern '%s': %d\n", 2824 ext_name, ext->sz); 2825 return ext->sz; 2826 } 2827 ext->align = btf__align_of(obj->btf, t->type); 2828 if (ext->align <= 0) { 2829 pr_warn("failed to determine alignment of extern '%s': %d\n", 2830 ext_name, ext->align); 2831 return -EINVAL; 2832 } 2833 ext->type = find_extern_type(obj->btf, t->type, 2834 &ext->is_signed); 2835 if (ext->type == EXT_UNKNOWN) { 2836 pr_warn("extern '%s' type is unsupported\n", ext_name); 2837 return -ENOTSUP; 2838 } 2839 } 2840 pr_debug("collected %d externs total\n", obj->nr_extern); 2841 2842 if (!obj->nr_extern) 2843 return 0; 2844 2845 /* sort externs by (alignment, size, name) and calculate their offsets 2846 * within a map */ 2847 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 2848 off = 0; 2849 for (i = 0; i < obj->nr_extern; i++) { 2850 ext = &obj->externs[i]; 2851 ext->data_off = roundup(off, ext->align); 2852 off = ext->data_off + ext->sz; 2853 pr_debug("extern #%d: symbol %d, off %u, name %s\n", 2854 i, ext->sym_idx, ext->data_off, ext->name); 2855 } 2856 2857 btf_id = btf__find_by_name(obj->btf, KCONFIG_SEC); 2858 if (btf_id <= 0) { 2859 pr_warn("no BTF info found for '%s' datasec\n", KCONFIG_SEC); 2860 return -ESRCH; 2861 } 2862 2863 sec = (struct btf_type *)btf__type_by_id(obj->btf, btf_id); 2864 sec->size = off; 2865 n = btf_vlen(sec); 2866 for (i = 0; i < n; i++) { 2867 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 2868 2869 t = btf__type_by_id(obj->btf, vs->type); 2870 ext_name = btf__name_by_offset(obj->btf, t->name_off); 2871 ext = find_extern_by_name(obj, ext_name); 2872 if (!ext) { 2873 pr_warn("failed to find extern definition for BTF var '%s'\n", 2874 ext_name); 2875 return -ESRCH; 2876 } 2877 vs->offset = ext->data_off; 2878 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 2879 } 2880 2881 return 0; 2882 } 2883 2884 static struct bpf_program * 2885 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 2886 { 2887 struct bpf_program *prog; 2888 size_t i; 2889 2890 for (i = 0; i < obj->nr_programs; i++) { 2891 prog = &obj->programs[i]; 2892 if (prog->idx == idx) 2893 return prog; 2894 } 2895 return NULL; 2896 } 2897 2898 struct bpf_program * 2899 bpf_object__find_program_by_title(const struct bpf_object *obj, 2900 const char *title) 2901 { 2902 struct bpf_program *pos; 2903 2904 bpf_object__for_each_program(pos, obj) { 2905 if (pos->section_name && !strcmp(pos->section_name, title)) 2906 return pos; 2907 } 2908 return NULL; 2909 } 2910 2911 struct bpf_program * 2912 bpf_object__find_program_by_name(const struct bpf_object *obj, 2913 const char *name) 2914 { 2915 struct bpf_program *prog; 2916 2917 bpf_object__for_each_program(prog, obj) { 2918 if (!strcmp(prog->name, name)) 2919 return prog; 2920 } 2921 return NULL; 2922 } 2923 2924 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 2925 int shndx) 2926 { 2927 return shndx == obj->efile.data_shndx || 2928 shndx == obj->efile.bss_shndx || 2929 shndx == obj->efile.rodata_shndx; 2930 } 2931 2932 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 2933 int shndx) 2934 { 2935 return shndx == obj->efile.maps_shndx || 2936 shndx == obj->efile.btf_maps_shndx; 2937 } 2938 2939 static enum libbpf_map_type 2940 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 2941 { 2942 if (shndx == obj->efile.data_shndx) 2943 return LIBBPF_MAP_DATA; 2944 else if (shndx == obj->efile.bss_shndx) 2945 return LIBBPF_MAP_BSS; 2946 else if (shndx == obj->efile.rodata_shndx) 2947 return LIBBPF_MAP_RODATA; 2948 else if (shndx == obj->efile.symbols_shndx) 2949 return LIBBPF_MAP_KCONFIG; 2950 else 2951 return LIBBPF_MAP_UNSPEC; 2952 } 2953 2954 static int bpf_program__record_reloc(struct bpf_program *prog, 2955 struct reloc_desc *reloc_desc, 2956 __u32 insn_idx, const char *name, 2957 const GElf_Sym *sym, const GElf_Rel *rel) 2958 { 2959 struct bpf_insn *insn = &prog->insns[insn_idx]; 2960 size_t map_idx, nr_maps = prog->obj->nr_maps; 2961 struct bpf_object *obj = prog->obj; 2962 __u32 shdr_idx = sym->st_shndx; 2963 enum libbpf_map_type type; 2964 struct bpf_map *map; 2965 2966 /* sub-program call relocation */ 2967 if (insn->code == (BPF_JMP | BPF_CALL)) { 2968 if (insn->src_reg != BPF_PSEUDO_CALL) { 2969 pr_warn("incorrect bpf_call opcode\n"); 2970 return -LIBBPF_ERRNO__RELOC; 2971 } 2972 /* text_shndx can be 0, if no default "main" program exists */ 2973 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 2974 pr_warn("bad call relo against section %u\n", shdr_idx); 2975 return -LIBBPF_ERRNO__RELOC; 2976 } 2977 if (sym->st_value % 8) { 2978 pr_warn("bad call relo offset: %zu\n", 2979 (size_t)sym->st_value); 2980 return -LIBBPF_ERRNO__RELOC; 2981 } 2982 reloc_desc->type = RELO_CALL; 2983 reloc_desc->insn_idx = insn_idx; 2984 reloc_desc->sym_off = sym->st_value; 2985 obj->has_pseudo_calls = true; 2986 return 0; 2987 } 2988 2989 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) { 2990 pr_warn("invalid relo for insns[%d].code 0x%x\n", 2991 insn_idx, insn->code); 2992 return -LIBBPF_ERRNO__RELOC; 2993 } 2994 2995 if (sym_is_extern(sym)) { 2996 int sym_idx = GELF_R_SYM(rel->r_info); 2997 int i, n = obj->nr_extern; 2998 struct extern_desc *ext; 2999 3000 for (i = 0; i < n; i++) { 3001 ext = &obj->externs[i]; 3002 if (ext->sym_idx == sym_idx) 3003 break; 3004 } 3005 if (i >= n) { 3006 pr_warn("extern relo failed to find extern for sym %d\n", 3007 sym_idx); 3008 return -LIBBPF_ERRNO__RELOC; 3009 } 3010 pr_debug("found extern #%d '%s' (sym %d, off %u) for insn %u\n", 3011 i, ext->name, ext->sym_idx, ext->data_off, insn_idx); 3012 reloc_desc->type = RELO_EXTERN; 3013 reloc_desc->insn_idx = insn_idx; 3014 reloc_desc->sym_off = ext->data_off; 3015 return 0; 3016 } 3017 3018 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 3019 pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n", 3020 name, shdr_idx); 3021 return -LIBBPF_ERRNO__RELOC; 3022 } 3023 3024 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 3025 3026 /* generic map reference relocation */ 3027 if (type == LIBBPF_MAP_UNSPEC) { 3028 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 3029 pr_warn("bad map relo against section %u\n", 3030 shdr_idx); 3031 return -LIBBPF_ERRNO__RELOC; 3032 } 3033 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3034 map = &obj->maps[map_idx]; 3035 if (map->libbpf_type != type || 3036 map->sec_idx != sym->st_shndx || 3037 map->sec_offset != sym->st_value) 3038 continue; 3039 pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n", 3040 map_idx, map->name, map->sec_idx, 3041 map->sec_offset, insn_idx); 3042 break; 3043 } 3044 if (map_idx >= nr_maps) { 3045 pr_warn("map relo failed to find map for sec %u, off %zu\n", 3046 shdr_idx, (size_t)sym->st_value); 3047 return -LIBBPF_ERRNO__RELOC; 3048 } 3049 reloc_desc->type = RELO_LD64; 3050 reloc_desc->insn_idx = insn_idx; 3051 reloc_desc->map_idx = map_idx; 3052 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 3053 return 0; 3054 } 3055 3056 /* global data map relocation */ 3057 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 3058 pr_warn("bad data relo against section %u\n", shdr_idx); 3059 return -LIBBPF_ERRNO__RELOC; 3060 } 3061 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3062 map = &obj->maps[map_idx]; 3063 if (map->libbpf_type != type) 3064 continue; 3065 pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n", 3066 map_idx, map->name, map->sec_idx, map->sec_offset, 3067 insn_idx); 3068 break; 3069 } 3070 if (map_idx >= nr_maps) { 3071 pr_warn("data relo failed to find map for sec %u\n", 3072 shdr_idx); 3073 return -LIBBPF_ERRNO__RELOC; 3074 } 3075 3076 reloc_desc->type = RELO_DATA; 3077 reloc_desc->insn_idx = insn_idx; 3078 reloc_desc->map_idx = map_idx; 3079 reloc_desc->sym_off = sym->st_value; 3080 return 0; 3081 } 3082 3083 static int 3084 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 3085 Elf_Data *data, struct bpf_object *obj) 3086 { 3087 Elf_Data *symbols = obj->efile.symbols; 3088 int err, i, nrels; 3089 3090 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 3091 nrels = shdr->sh_size / shdr->sh_entsize; 3092 3093 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 3094 if (!prog->reloc_desc) { 3095 pr_warn("failed to alloc memory in relocation\n"); 3096 return -ENOMEM; 3097 } 3098 prog->nr_reloc = nrels; 3099 3100 for (i = 0; i < nrels; i++) { 3101 const char *name; 3102 __u32 insn_idx; 3103 GElf_Sym sym; 3104 GElf_Rel rel; 3105 3106 if (!gelf_getrel(data, i, &rel)) { 3107 pr_warn("relocation: failed to get %d reloc\n", i); 3108 return -LIBBPF_ERRNO__FORMAT; 3109 } 3110 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 3111 pr_warn("relocation: symbol %"PRIx64" not found\n", 3112 GELF_R_SYM(rel.r_info)); 3113 return -LIBBPF_ERRNO__FORMAT; 3114 } 3115 if (rel.r_offset % sizeof(struct bpf_insn)) 3116 return -LIBBPF_ERRNO__FORMAT; 3117 3118 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 3119 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 3120 sym.st_name) ? : "<?>"; 3121 3122 pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n", 3123 (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info), 3124 (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info), 3125 GELF_ST_BIND(sym.st_info), sym.st_name, name, 3126 insn_idx); 3127 3128 err = bpf_program__record_reloc(prog, &prog->reloc_desc[i], 3129 insn_idx, name, &sym, &rel); 3130 if (err) 3131 return err; 3132 } 3133 return 0; 3134 } 3135 3136 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 3137 { 3138 struct bpf_map_def *def = &map->def; 3139 __u32 key_type_id = 0, value_type_id = 0; 3140 int ret; 3141 3142 /* if it's BTF-defined map, we don't need to search for type IDs. 3143 * For struct_ops map, it does not need btf_key_type_id and 3144 * btf_value_type_id. 3145 */ 3146 if (map->sec_idx == obj->efile.btf_maps_shndx || 3147 bpf_map__is_struct_ops(map)) 3148 return 0; 3149 3150 if (!bpf_map__is_internal(map)) { 3151 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 3152 def->value_size, &key_type_id, 3153 &value_type_id); 3154 } else { 3155 /* 3156 * LLVM annotates global data differently in BTF, that is, 3157 * only as '.data', '.bss' or '.rodata'. 3158 */ 3159 ret = btf__find_by_name(obj->btf, 3160 libbpf_type_to_btf_name[map->libbpf_type]); 3161 } 3162 if (ret < 0) 3163 return ret; 3164 3165 map->btf_key_type_id = key_type_id; 3166 map->btf_value_type_id = bpf_map__is_internal(map) ? 3167 ret : value_type_id; 3168 return 0; 3169 } 3170 3171 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 3172 { 3173 struct bpf_map_info info = {}; 3174 __u32 len = sizeof(info); 3175 int new_fd, err; 3176 char *new_name; 3177 3178 err = bpf_obj_get_info_by_fd(fd, &info, &len); 3179 if (err) 3180 return err; 3181 3182 new_name = strdup(info.name); 3183 if (!new_name) 3184 return -errno; 3185 3186 new_fd = open("/", O_RDONLY | O_CLOEXEC); 3187 if (new_fd < 0) { 3188 err = -errno; 3189 goto err_free_new_name; 3190 } 3191 3192 new_fd = dup3(fd, new_fd, O_CLOEXEC); 3193 if (new_fd < 0) { 3194 err = -errno; 3195 goto err_close_new_fd; 3196 } 3197 3198 err = zclose(map->fd); 3199 if (err) { 3200 err = -errno; 3201 goto err_close_new_fd; 3202 } 3203 free(map->name); 3204 3205 map->fd = new_fd; 3206 map->name = new_name; 3207 map->def.type = info.type; 3208 map->def.key_size = info.key_size; 3209 map->def.value_size = info.value_size; 3210 map->def.max_entries = info.max_entries; 3211 map->def.map_flags = info.map_flags; 3212 map->btf_key_type_id = info.btf_key_type_id; 3213 map->btf_value_type_id = info.btf_value_type_id; 3214 map->reused = true; 3215 3216 return 0; 3217 3218 err_close_new_fd: 3219 close(new_fd); 3220 err_free_new_name: 3221 free(new_name); 3222 return err; 3223 } 3224 3225 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 3226 { 3227 if (!map || !max_entries) 3228 return -EINVAL; 3229 3230 /* If map already created, its attributes can't be changed. */ 3231 if (map->fd >= 0) 3232 return -EBUSY; 3233 3234 map->def.max_entries = max_entries; 3235 3236 return 0; 3237 } 3238 3239 static int 3240 bpf_object__probe_loading(struct bpf_object *obj) 3241 { 3242 struct bpf_load_program_attr attr; 3243 char *cp, errmsg[STRERR_BUFSIZE]; 3244 struct bpf_insn insns[] = { 3245 BPF_MOV64_IMM(BPF_REG_0, 0), 3246 BPF_EXIT_INSN(), 3247 }; 3248 int ret; 3249 3250 /* make sure basic loading works */ 3251 3252 memset(&attr, 0, sizeof(attr)); 3253 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3254 attr.insns = insns; 3255 attr.insns_cnt = ARRAY_SIZE(insns); 3256 attr.license = "GPL"; 3257 3258 ret = bpf_load_program_xattr(&attr, NULL, 0); 3259 if (ret < 0) { 3260 ret = errno; 3261 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 3262 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 3263 "program. Make sure your kernel supports BPF " 3264 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 3265 "set to big enough value.\n", __func__, cp, ret); 3266 return -ret; 3267 } 3268 close(ret); 3269 3270 return 0; 3271 } 3272 3273 static int 3274 bpf_object__probe_name(struct bpf_object *obj) 3275 { 3276 struct bpf_load_program_attr attr; 3277 struct bpf_insn insns[] = { 3278 BPF_MOV64_IMM(BPF_REG_0, 0), 3279 BPF_EXIT_INSN(), 3280 }; 3281 int ret; 3282 3283 /* make sure loading with name works */ 3284 3285 memset(&attr, 0, sizeof(attr)); 3286 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3287 attr.insns = insns; 3288 attr.insns_cnt = ARRAY_SIZE(insns); 3289 attr.license = "GPL"; 3290 attr.name = "test"; 3291 ret = bpf_load_program_xattr(&attr, NULL, 0); 3292 if (ret >= 0) { 3293 obj->caps.name = 1; 3294 close(ret); 3295 } 3296 3297 return 0; 3298 } 3299 3300 static int 3301 bpf_object__probe_global_data(struct bpf_object *obj) 3302 { 3303 struct bpf_load_program_attr prg_attr; 3304 struct bpf_create_map_attr map_attr; 3305 char *cp, errmsg[STRERR_BUFSIZE]; 3306 struct bpf_insn insns[] = { 3307 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 3308 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 3309 BPF_MOV64_IMM(BPF_REG_0, 0), 3310 BPF_EXIT_INSN(), 3311 }; 3312 int ret, map; 3313 3314 memset(&map_attr, 0, sizeof(map_attr)); 3315 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 3316 map_attr.key_size = sizeof(int); 3317 map_attr.value_size = 32; 3318 map_attr.max_entries = 1; 3319 3320 map = bpf_create_map_xattr(&map_attr); 3321 if (map < 0) { 3322 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 3323 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 3324 __func__, cp, errno); 3325 return -errno; 3326 } 3327 3328 insns[0].imm = map; 3329 3330 memset(&prg_attr, 0, sizeof(prg_attr)); 3331 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3332 prg_attr.insns = insns; 3333 prg_attr.insns_cnt = ARRAY_SIZE(insns); 3334 prg_attr.license = "GPL"; 3335 3336 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 3337 if (ret >= 0) { 3338 obj->caps.global_data = 1; 3339 close(ret); 3340 } 3341 3342 close(map); 3343 return 0; 3344 } 3345 3346 static int bpf_object__probe_btf_func(struct bpf_object *obj) 3347 { 3348 static const char strs[] = "\0int\0x\0a"; 3349 /* void x(int a) {} */ 3350 __u32 types[] = { 3351 /* int */ 3352 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 3353 /* FUNC_PROTO */ /* [2] */ 3354 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 3355 BTF_PARAM_ENC(7, 1), 3356 /* FUNC x */ /* [3] */ 3357 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 3358 }; 3359 int btf_fd; 3360 3361 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 3362 strs, sizeof(strs)); 3363 if (btf_fd >= 0) { 3364 obj->caps.btf_func = 1; 3365 close(btf_fd); 3366 return 1; 3367 } 3368 3369 return 0; 3370 } 3371 3372 static int bpf_object__probe_btf_func_global(struct bpf_object *obj) 3373 { 3374 static const char strs[] = "\0int\0x\0a"; 3375 /* static void x(int a) {} */ 3376 __u32 types[] = { 3377 /* int */ 3378 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 3379 /* FUNC_PROTO */ /* [2] */ 3380 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 3381 BTF_PARAM_ENC(7, 1), 3382 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 3383 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 3384 }; 3385 int btf_fd; 3386 3387 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 3388 strs, sizeof(strs)); 3389 if (btf_fd >= 0) { 3390 obj->caps.btf_func_global = 1; 3391 close(btf_fd); 3392 return 1; 3393 } 3394 3395 return 0; 3396 } 3397 3398 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 3399 { 3400 static const char strs[] = "\0x\0.data"; 3401 /* static int a; */ 3402 __u32 types[] = { 3403 /* int */ 3404 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 3405 /* VAR x */ /* [2] */ 3406 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 3407 BTF_VAR_STATIC, 3408 /* DATASEC val */ /* [3] */ 3409 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 3410 BTF_VAR_SECINFO_ENC(2, 0, 4), 3411 }; 3412 int btf_fd; 3413 3414 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 3415 strs, sizeof(strs)); 3416 if (btf_fd >= 0) { 3417 obj->caps.btf_datasec = 1; 3418 close(btf_fd); 3419 return 1; 3420 } 3421 3422 return 0; 3423 } 3424 3425 static int bpf_object__probe_array_mmap(struct bpf_object *obj) 3426 { 3427 struct bpf_create_map_attr attr = { 3428 .map_type = BPF_MAP_TYPE_ARRAY, 3429 .map_flags = BPF_F_MMAPABLE, 3430 .key_size = sizeof(int), 3431 .value_size = sizeof(int), 3432 .max_entries = 1, 3433 }; 3434 int fd; 3435 3436 fd = bpf_create_map_xattr(&attr); 3437 if (fd >= 0) { 3438 obj->caps.array_mmap = 1; 3439 close(fd); 3440 return 1; 3441 } 3442 3443 return 0; 3444 } 3445 3446 static int 3447 bpf_object__probe_exp_attach_type(struct bpf_object *obj) 3448 { 3449 struct bpf_load_program_attr attr; 3450 struct bpf_insn insns[] = { 3451 BPF_MOV64_IMM(BPF_REG_0, 0), 3452 BPF_EXIT_INSN(), 3453 }; 3454 int fd; 3455 3456 memset(&attr, 0, sizeof(attr)); 3457 /* use any valid combination of program type and (optional) 3458 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 3459 * to see if kernel supports expected_attach_type field for 3460 * BPF_PROG_LOAD command 3461 */ 3462 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 3463 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; 3464 attr.insns = insns; 3465 attr.insns_cnt = ARRAY_SIZE(insns); 3466 attr.license = "GPL"; 3467 3468 fd = bpf_load_program_xattr(&attr, NULL, 0); 3469 if (fd >= 0) { 3470 obj->caps.exp_attach_type = 1; 3471 close(fd); 3472 return 1; 3473 } 3474 return 0; 3475 } 3476 3477 static int 3478 bpf_object__probe_caps(struct bpf_object *obj) 3479 { 3480 int (*probe_fn[])(struct bpf_object *obj) = { 3481 bpf_object__probe_name, 3482 bpf_object__probe_global_data, 3483 bpf_object__probe_btf_func, 3484 bpf_object__probe_btf_func_global, 3485 bpf_object__probe_btf_datasec, 3486 bpf_object__probe_array_mmap, 3487 bpf_object__probe_exp_attach_type, 3488 }; 3489 int i, ret; 3490 3491 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 3492 ret = probe_fn[i](obj); 3493 if (ret < 0) 3494 pr_debug("Probe #%d failed with %d.\n", i, ret); 3495 } 3496 3497 return 0; 3498 } 3499 3500 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 3501 { 3502 struct bpf_map_info map_info = {}; 3503 char msg[STRERR_BUFSIZE]; 3504 __u32 map_info_len; 3505 3506 map_info_len = sizeof(map_info); 3507 3508 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) { 3509 pr_warn("failed to get map info for map FD %d: %s\n", 3510 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg))); 3511 return false; 3512 } 3513 3514 return (map_info.type == map->def.type && 3515 map_info.key_size == map->def.key_size && 3516 map_info.value_size == map->def.value_size && 3517 map_info.max_entries == map->def.max_entries && 3518 map_info.map_flags == map->def.map_flags); 3519 } 3520 3521 static int 3522 bpf_object__reuse_map(struct bpf_map *map) 3523 { 3524 char *cp, errmsg[STRERR_BUFSIZE]; 3525 int err, pin_fd; 3526 3527 pin_fd = bpf_obj_get(map->pin_path); 3528 if (pin_fd < 0) { 3529 err = -errno; 3530 if (err == -ENOENT) { 3531 pr_debug("found no pinned map to reuse at '%s'\n", 3532 map->pin_path); 3533 return 0; 3534 } 3535 3536 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 3537 pr_warn("couldn't retrieve pinned map '%s': %s\n", 3538 map->pin_path, cp); 3539 return err; 3540 } 3541 3542 if (!map_is_reuse_compat(map, pin_fd)) { 3543 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 3544 map->pin_path); 3545 close(pin_fd); 3546 return -EINVAL; 3547 } 3548 3549 err = bpf_map__reuse_fd(map, pin_fd); 3550 if (err) { 3551 close(pin_fd); 3552 return err; 3553 } 3554 map->pinned = true; 3555 pr_debug("reused pinned map at '%s'\n", map->pin_path); 3556 3557 return 0; 3558 } 3559 3560 static int 3561 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 3562 { 3563 enum libbpf_map_type map_type = map->libbpf_type; 3564 char *cp, errmsg[STRERR_BUFSIZE]; 3565 int err, zero = 0; 3566 3567 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 3568 if (err) { 3569 err = -errno; 3570 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3571 pr_warn("Error setting initial map(%s) contents: %s\n", 3572 map->name, cp); 3573 return err; 3574 } 3575 3576 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 3577 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 3578 err = bpf_map_freeze(map->fd); 3579 if (err) { 3580 err = -errno; 3581 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3582 pr_warn("Error freezing map(%s) as read-only: %s\n", 3583 map->name, cp); 3584 return err; 3585 } 3586 } 3587 return 0; 3588 } 3589 3590 static void bpf_map__destroy(struct bpf_map *map); 3591 3592 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map) 3593 { 3594 struct bpf_create_map_attr create_attr; 3595 struct bpf_map_def *def = &map->def; 3596 3597 memset(&create_attr, 0, sizeof(create_attr)); 3598 3599 if (obj->caps.name) 3600 create_attr.name = map->name; 3601 create_attr.map_ifindex = map->map_ifindex; 3602 create_attr.map_type = def->type; 3603 create_attr.map_flags = def->map_flags; 3604 create_attr.key_size = def->key_size; 3605 create_attr.value_size = def->value_size; 3606 3607 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) { 3608 int nr_cpus; 3609 3610 nr_cpus = libbpf_num_possible_cpus(); 3611 if (nr_cpus < 0) { 3612 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 3613 map->name, nr_cpus); 3614 return nr_cpus; 3615 } 3616 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 3617 create_attr.max_entries = nr_cpus; 3618 } else { 3619 create_attr.max_entries = def->max_entries; 3620 } 3621 3622 if (bpf_map__is_struct_ops(map)) 3623 create_attr.btf_vmlinux_value_type_id = 3624 map->btf_vmlinux_value_type_id; 3625 3626 create_attr.btf_fd = 0; 3627 create_attr.btf_key_type_id = 0; 3628 create_attr.btf_value_type_id = 0; 3629 if (obj->btf && !bpf_map_find_btf_info(obj, map)) { 3630 create_attr.btf_fd = btf__fd(obj->btf); 3631 create_attr.btf_key_type_id = map->btf_key_type_id; 3632 create_attr.btf_value_type_id = map->btf_value_type_id; 3633 } 3634 3635 if (bpf_map_type__is_map_in_map(def->type)) { 3636 if (map->inner_map) { 3637 int err; 3638 3639 err = bpf_object__create_map(obj, map->inner_map); 3640 if (err) { 3641 pr_warn("map '%s': failed to create inner map: %d\n", 3642 map->name, err); 3643 return err; 3644 } 3645 map->inner_map_fd = bpf_map__fd(map->inner_map); 3646 } 3647 if (map->inner_map_fd >= 0) 3648 create_attr.inner_map_fd = map->inner_map_fd; 3649 } 3650 3651 map->fd = bpf_create_map_xattr(&create_attr); 3652 if (map->fd < 0 && (create_attr.btf_key_type_id || 3653 create_attr.btf_value_type_id)) { 3654 char *cp, errmsg[STRERR_BUFSIZE]; 3655 int err = -errno; 3656 3657 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3658 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 3659 map->name, cp, err); 3660 create_attr.btf_fd = 0; 3661 create_attr.btf_key_type_id = 0; 3662 create_attr.btf_value_type_id = 0; 3663 map->btf_key_type_id = 0; 3664 map->btf_value_type_id = 0; 3665 map->fd = bpf_create_map_xattr(&create_attr); 3666 } 3667 3668 if (map->fd < 0) 3669 return -errno; 3670 3671 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 3672 bpf_map__destroy(map->inner_map); 3673 zfree(&map->inner_map); 3674 } 3675 3676 return 0; 3677 } 3678 3679 static int 3680 bpf_object__create_maps(struct bpf_object *obj) 3681 { 3682 struct bpf_map *map; 3683 char *cp, errmsg[STRERR_BUFSIZE]; 3684 unsigned int i, j; 3685 int err; 3686 3687 for (i = 0; i < obj->nr_maps; i++) { 3688 map = &obj->maps[i]; 3689 3690 if (map->pin_path) { 3691 err = bpf_object__reuse_map(map); 3692 if (err) { 3693 pr_warn("map '%s': error reusing pinned map\n", 3694 map->name); 3695 goto err_out; 3696 } 3697 } 3698 3699 if (map->fd >= 0) { 3700 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 3701 map->name, map->fd); 3702 continue; 3703 } 3704 3705 err = bpf_object__create_map(obj, map); 3706 if (err) 3707 goto err_out; 3708 3709 pr_debug("map '%s': created successfully, fd=%d\n", map->name, 3710 map->fd); 3711 3712 if (bpf_map__is_internal(map)) { 3713 err = bpf_object__populate_internal_map(obj, map); 3714 if (err < 0) { 3715 zclose(map->fd); 3716 goto err_out; 3717 } 3718 } 3719 3720 if (map->init_slots_sz) { 3721 for (j = 0; j < map->init_slots_sz; j++) { 3722 const struct bpf_map *targ_map; 3723 int fd; 3724 3725 if (!map->init_slots[j]) 3726 continue; 3727 3728 targ_map = map->init_slots[j]; 3729 fd = bpf_map__fd(targ_map); 3730 err = bpf_map_update_elem(map->fd, &j, &fd, 0); 3731 if (err) { 3732 err = -errno; 3733 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 3734 map->name, j, targ_map->name, 3735 fd, err); 3736 goto err_out; 3737 } 3738 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 3739 map->name, j, targ_map->name, fd); 3740 } 3741 zfree(&map->init_slots); 3742 map->init_slots_sz = 0; 3743 } 3744 3745 if (map->pin_path && !map->pinned) { 3746 err = bpf_map__pin(map, NULL); 3747 if (err) { 3748 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 3749 map->name, map->pin_path, err); 3750 zclose(map->fd); 3751 goto err_out; 3752 } 3753 } 3754 } 3755 3756 return 0; 3757 3758 err_out: 3759 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3760 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 3761 pr_perm_msg(err); 3762 for (j = 0; j < i; j++) 3763 zclose(obj->maps[j].fd); 3764 return err; 3765 } 3766 3767 static int 3768 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 3769 void *btf_prog_info, const char *info_name) 3770 { 3771 if (err != -ENOENT) { 3772 pr_warn("Error in loading %s for sec %s.\n", 3773 info_name, prog->section_name); 3774 return err; 3775 } 3776 3777 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 3778 3779 if (btf_prog_info) { 3780 /* 3781 * Some info has already been found but has problem 3782 * in the last btf_ext reloc. Must have to error out. 3783 */ 3784 pr_warn("Error in relocating %s for sec %s.\n", 3785 info_name, prog->section_name); 3786 return err; 3787 } 3788 3789 /* Have problem loading the very first info. Ignore the rest. */ 3790 pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n", 3791 info_name, prog->section_name, info_name); 3792 return 0; 3793 } 3794 3795 static int 3796 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 3797 const char *section_name, __u32 insn_offset) 3798 { 3799 int err; 3800 3801 if (!insn_offset || prog->func_info) { 3802 /* 3803 * !insn_offset => main program 3804 * 3805 * For sub prog, the main program's func_info has to 3806 * be loaded first (i.e. prog->func_info != NULL) 3807 */ 3808 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 3809 section_name, insn_offset, 3810 &prog->func_info, 3811 &prog->func_info_cnt); 3812 if (err) 3813 return check_btf_ext_reloc_err(prog, err, 3814 prog->func_info, 3815 "bpf_func_info"); 3816 3817 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 3818 } 3819 3820 if (!insn_offset || prog->line_info) { 3821 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 3822 section_name, insn_offset, 3823 &prog->line_info, 3824 &prog->line_info_cnt); 3825 if (err) 3826 return check_btf_ext_reloc_err(prog, err, 3827 prog->line_info, 3828 "bpf_line_info"); 3829 3830 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 3831 } 3832 3833 return 0; 3834 } 3835 3836 #define BPF_CORE_SPEC_MAX_LEN 64 3837 3838 /* represents BPF CO-RE field or array element accessor */ 3839 struct bpf_core_accessor { 3840 __u32 type_id; /* struct/union type or array element type */ 3841 __u32 idx; /* field index or array index */ 3842 const char *name; /* field name or NULL for array accessor */ 3843 }; 3844 3845 struct bpf_core_spec { 3846 const struct btf *btf; 3847 /* high-level spec: named fields and array indices only */ 3848 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; 3849 /* high-level spec length */ 3850 int len; 3851 /* raw, low-level spec: 1-to-1 with accessor spec string */ 3852 int raw_spec[BPF_CORE_SPEC_MAX_LEN]; 3853 /* raw spec length */ 3854 int raw_len; 3855 /* field bit offset represented by spec */ 3856 __u32 bit_offset; 3857 }; 3858 3859 static bool str_is_empty(const char *s) 3860 { 3861 return !s || !s[0]; 3862 } 3863 3864 static bool is_flex_arr(const struct btf *btf, 3865 const struct bpf_core_accessor *acc, 3866 const struct btf_array *arr) 3867 { 3868 const struct btf_type *t; 3869 3870 /* not a flexible array, if not inside a struct or has non-zero size */ 3871 if (!acc->name || arr->nelems > 0) 3872 return false; 3873 3874 /* has to be the last member of enclosing struct */ 3875 t = btf__type_by_id(btf, acc->type_id); 3876 return acc->idx == btf_vlen(t) - 1; 3877 } 3878 3879 /* 3880 * Turn bpf_field_reloc into a low- and high-level spec representation, 3881 * validating correctness along the way, as well as calculating resulting 3882 * field bit offset, specified by accessor string. Low-level spec captures 3883 * every single level of nestedness, including traversing anonymous 3884 * struct/union members. High-level one only captures semantically meaningful 3885 * "turning points": named fields and array indicies. 3886 * E.g., for this case: 3887 * 3888 * struct sample { 3889 * int __unimportant; 3890 * struct { 3891 * int __1; 3892 * int __2; 3893 * int a[7]; 3894 * }; 3895 * }; 3896 * 3897 * struct sample *s = ...; 3898 * 3899 * int x = &s->a[3]; // access string = '0:1:2:3' 3900 * 3901 * Low-level spec has 1:1 mapping with each element of access string (it's 3902 * just a parsed access string representation): [0, 1, 2, 3]. 3903 * 3904 * High-level spec will capture only 3 points: 3905 * - intial zero-index access by pointer (&s->... is the same as &s[0]...); 3906 * - field 'a' access (corresponds to '2' in low-level spec); 3907 * - array element #3 access (corresponds to '3' in low-level spec). 3908 * 3909 */ 3910 static int bpf_core_spec_parse(const struct btf *btf, 3911 __u32 type_id, 3912 const char *spec_str, 3913 struct bpf_core_spec *spec) 3914 { 3915 int access_idx, parsed_len, i; 3916 struct bpf_core_accessor *acc; 3917 const struct btf_type *t; 3918 const char *name; 3919 __u32 id; 3920 __s64 sz; 3921 3922 if (str_is_empty(spec_str) || *spec_str == ':') 3923 return -EINVAL; 3924 3925 memset(spec, 0, sizeof(*spec)); 3926 spec->btf = btf; 3927 3928 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ 3929 while (*spec_str) { 3930 if (*spec_str == ':') 3931 ++spec_str; 3932 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) 3933 return -EINVAL; 3934 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 3935 return -E2BIG; 3936 spec_str += parsed_len; 3937 spec->raw_spec[spec->raw_len++] = access_idx; 3938 } 3939 3940 if (spec->raw_len == 0) 3941 return -EINVAL; 3942 3943 /* first spec value is always reloc type array index */ 3944 t = skip_mods_and_typedefs(btf, type_id, &id); 3945 if (!t) 3946 return -EINVAL; 3947 3948 access_idx = spec->raw_spec[0]; 3949 spec->spec[0].type_id = id; 3950 spec->spec[0].idx = access_idx; 3951 spec->len++; 3952 3953 sz = btf__resolve_size(btf, id); 3954 if (sz < 0) 3955 return sz; 3956 spec->bit_offset = access_idx * sz * 8; 3957 3958 for (i = 1; i < spec->raw_len; i++) { 3959 t = skip_mods_and_typedefs(btf, id, &id); 3960 if (!t) 3961 return -EINVAL; 3962 3963 access_idx = spec->raw_spec[i]; 3964 acc = &spec->spec[spec->len]; 3965 3966 if (btf_is_composite(t)) { 3967 const struct btf_member *m; 3968 __u32 bit_offset; 3969 3970 if (access_idx >= btf_vlen(t)) 3971 return -EINVAL; 3972 3973 bit_offset = btf_member_bit_offset(t, access_idx); 3974 spec->bit_offset += bit_offset; 3975 3976 m = btf_members(t) + access_idx; 3977 if (m->name_off) { 3978 name = btf__name_by_offset(btf, m->name_off); 3979 if (str_is_empty(name)) 3980 return -EINVAL; 3981 3982 acc->type_id = id; 3983 acc->idx = access_idx; 3984 acc->name = name; 3985 spec->len++; 3986 } 3987 3988 id = m->type; 3989 } else if (btf_is_array(t)) { 3990 const struct btf_array *a = btf_array(t); 3991 bool flex; 3992 3993 t = skip_mods_and_typedefs(btf, a->type, &id); 3994 if (!t) 3995 return -EINVAL; 3996 3997 flex = is_flex_arr(btf, acc - 1, a); 3998 if (!flex && access_idx >= a->nelems) 3999 return -EINVAL; 4000 4001 spec->spec[spec->len].type_id = id; 4002 spec->spec[spec->len].idx = access_idx; 4003 spec->len++; 4004 4005 sz = btf__resolve_size(btf, id); 4006 if (sz < 0) 4007 return sz; 4008 spec->bit_offset += access_idx * sz * 8; 4009 } else { 4010 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", 4011 type_id, spec_str, i, id, btf_kind(t)); 4012 return -EINVAL; 4013 } 4014 } 4015 4016 return 0; 4017 } 4018 4019 static bool bpf_core_is_flavor_sep(const char *s) 4020 { 4021 /* check X___Y name pattern, where X and Y are not underscores */ 4022 return s[0] != '_' && /* X */ 4023 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 4024 s[4] != '_'; /* Y */ 4025 } 4026 4027 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 4028 * before last triple underscore. Struct name part after last triple 4029 * underscore is ignored by BPF CO-RE relocation during relocation matching. 4030 */ 4031 static size_t bpf_core_essential_name_len(const char *name) 4032 { 4033 size_t n = strlen(name); 4034 int i; 4035 4036 for (i = n - 5; i >= 0; i--) { 4037 if (bpf_core_is_flavor_sep(name + i)) 4038 return i + 1; 4039 } 4040 return n; 4041 } 4042 4043 /* dynamically sized list of type IDs */ 4044 struct ids_vec { 4045 __u32 *data; 4046 int len; 4047 }; 4048 4049 static void bpf_core_free_cands(struct ids_vec *cand_ids) 4050 { 4051 free(cand_ids->data); 4052 free(cand_ids); 4053 } 4054 4055 static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf, 4056 __u32 local_type_id, 4057 const struct btf *targ_btf) 4058 { 4059 size_t local_essent_len, targ_essent_len; 4060 const char *local_name, *targ_name; 4061 const struct btf_type *t; 4062 struct ids_vec *cand_ids; 4063 __u32 *new_ids; 4064 int i, err, n; 4065 4066 t = btf__type_by_id(local_btf, local_type_id); 4067 if (!t) 4068 return ERR_PTR(-EINVAL); 4069 4070 local_name = btf__name_by_offset(local_btf, t->name_off); 4071 if (str_is_empty(local_name)) 4072 return ERR_PTR(-EINVAL); 4073 local_essent_len = bpf_core_essential_name_len(local_name); 4074 4075 cand_ids = calloc(1, sizeof(*cand_ids)); 4076 if (!cand_ids) 4077 return ERR_PTR(-ENOMEM); 4078 4079 n = btf__get_nr_types(targ_btf); 4080 for (i = 1; i <= n; i++) { 4081 t = btf__type_by_id(targ_btf, i); 4082 targ_name = btf__name_by_offset(targ_btf, t->name_off); 4083 if (str_is_empty(targ_name)) 4084 continue; 4085 4086 t = skip_mods_and_typedefs(targ_btf, i, NULL); 4087 if (!btf_is_composite(t) && !btf_is_array(t)) 4088 continue; 4089 4090 targ_essent_len = bpf_core_essential_name_len(targ_name); 4091 if (targ_essent_len != local_essent_len) 4092 continue; 4093 4094 if (strncmp(local_name, targ_name, local_essent_len) == 0) { 4095 pr_debug("[%d] %s: found candidate [%d] %s\n", 4096 local_type_id, local_name, i, targ_name); 4097 new_ids = reallocarray(cand_ids->data, 4098 cand_ids->len + 1, 4099 sizeof(*cand_ids->data)); 4100 if (!new_ids) { 4101 err = -ENOMEM; 4102 goto err_out; 4103 } 4104 cand_ids->data = new_ids; 4105 cand_ids->data[cand_ids->len++] = i; 4106 } 4107 } 4108 return cand_ids; 4109 err_out: 4110 bpf_core_free_cands(cand_ids); 4111 return ERR_PTR(err); 4112 } 4113 4114 /* Check two types for compatibility, skipping const/volatile/restrict and 4115 * typedefs, to ensure we are relocating compatible entities: 4116 * - any two STRUCTs/UNIONs are compatible and can be mixed; 4117 * - any two FWDs are compatible, if their names match (modulo flavor suffix); 4118 * - any two PTRs are always compatible; 4119 * - for ENUMs, names should be the same (ignoring flavor suffix) or at 4120 * least one of enums should be anonymous; 4121 * - for ENUMs, check sizes, names are ignored; 4122 * - for INT, size and signedness are ignored; 4123 * - for ARRAY, dimensionality is ignored, element types are checked for 4124 * compatibility recursively; 4125 * - everything else shouldn't be ever a target of relocation. 4126 * These rules are not set in stone and probably will be adjusted as we get 4127 * more experience with using BPF CO-RE relocations. 4128 */ 4129 static int bpf_core_fields_are_compat(const struct btf *local_btf, 4130 __u32 local_id, 4131 const struct btf *targ_btf, 4132 __u32 targ_id) 4133 { 4134 const struct btf_type *local_type, *targ_type; 4135 4136 recur: 4137 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 4138 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 4139 if (!local_type || !targ_type) 4140 return -EINVAL; 4141 4142 if (btf_is_composite(local_type) && btf_is_composite(targ_type)) 4143 return 1; 4144 if (btf_kind(local_type) != btf_kind(targ_type)) 4145 return 0; 4146 4147 switch (btf_kind(local_type)) { 4148 case BTF_KIND_PTR: 4149 return 1; 4150 case BTF_KIND_FWD: 4151 case BTF_KIND_ENUM: { 4152 const char *local_name, *targ_name; 4153 size_t local_len, targ_len; 4154 4155 local_name = btf__name_by_offset(local_btf, 4156 local_type->name_off); 4157 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); 4158 local_len = bpf_core_essential_name_len(local_name); 4159 targ_len = bpf_core_essential_name_len(targ_name); 4160 /* one of them is anonymous or both w/ same flavor-less names */ 4161 return local_len == 0 || targ_len == 0 || 4162 (local_len == targ_len && 4163 strncmp(local_name, targ_name, local_len) == 0); 4164 } 4165 case BTF_KIND_INT: 4166 /* just reject deprecated bitfield-like integers; all other 4167 * integers are by default compatible between each other 4168 */ 4169 return btf_int_offset(local_type) == 0 && 4170 btf_int_offset(targ_type) == 0; 4171 case BTF_KIND_ARRAY: 4172 local_id = btf_array(local_type)->type; 4173 targ_id = btf_array(targ_type)->type; 4174 goto recur; 4175 default: 4176 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", 4177 btf_kind(local_type), local_id, targ_id); 4178 return 0; 4179 } 4180 } 4181 4182 /* 4183 * Given single high-level named field accessor in local type, find 4184 * corresponding high-level accessor for a target type. Along the way, 4185 * maintain low-level spec for target as well. Also keep updating target 4186 * bit offset. 4187 * 4188 * Searching is performed through recursive exhaustive enumeration of all 4189 * fields of a struct/union. If there are any anonymous (embedded) 4190 * structs/unions, they are recursively searched as well. If field with 4191 * desired name is found, check compatibility between local and target types, 4192 * before returning result. 4193 * 4194 * 1 is returned, if field is found. 4195 * 0 is returned if no compatible field is found. 4196 * <0 is returned on error. 4197 */ 4198 static int bpf_core_match_member(const struct btf *local_btf, 4199 const struct bpf_core_accessor *local_acc, 4200 const struct btf *targ_btf, 4201 __u32 targ_id, 4202 struct bpf_core_spec *spec, 4203 __u32 *next_targ_id) 4204 { 4205 const struct btf_type *local_type, *targ_type; 4206 const struct btf_member *local_member, *m; 4207 const char *local_name, *targ_name; 4208 __u32 local_id; 4209 int i, n, found; 4210 4211 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 4212 if (!targ_type) 4213 return -EINVAL; 4214 if (!btf_is_composite(targ_type)) 4215 return 0; 4216 4217 local_id = local_acc->type_id; 4218 local_type = btf__type_by_id(local_btf, local_id); 4219 local_member = btf_members(local_type) + local_acc->idx; 4220 local_name = btf__name_by_offset(local_btf, local_member->name_off); 4221 4222 n = btf_vlen(targ_type); 4223 m = btf_members(targ_type); 4224 for (i = 0; i < n; i++, m++) { 4225 __u32 bit_offset; 4226 4227 bit_offset = btf_member_bit_offset(targ_type, i); 4228 4229 /* too deep struct/union/array nesting */ 4230 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 4231 return -E2BIG; 4232 4233 /* speculate this member will be the good one */ 4234 spec->bit_offset += bit_offset; 4235 spec->raw_spec[spec->raw_len++] = i; 4236 4237 targ_name = btf__name_by_offset(targ_btf, m->name_off); 4238 if (str_is_empty(targ_name)) { 4239 /* embedded struct/union, we need to go deeper */ 4240 found = bpf_core_match_member(local_btf, local_acc, 4241 targ_btf, m->type, 4242 spec, next_targ_id); 4243 if (found) /* either found or error */ 4244 return found; 4245 } else if (strcmp(local_name, targ_name) == 0) { 4246 /* matching named field */ 4247 struct bpf_core_accessor *targ_acc; 4248 4249 targ_acc = &spec->spec[spec->len++]; 4250 targ_acc->type_id = targ_id; 4251 targ_acc->idx = i; 4252 targ_acc->name = targ_name; 4253 4254 *next_targ_id = m->type; 4255 found = bpf_core_fields_are_compat(local_btf, 4256 local_member->type, 4257 targ_btf, m->type); 4258 if (!found) 4259 spec->len--; /* pop accessor */ 4260 return found; 4261 } 4262 /* member turned out not to be what we looked for */ 4263 spec->bit_offset -= bit_offset; 4264 spec->raw_len--; 4265 } 4266 4267 return 0; 4268 } 4269 4270 /* 4271 * Try to match local spec to a target type and, if successful, produce full 4272 * target spec (high-level, low-level + bit offset). 4273 */ 4274 static int bpf_core_spec_match(struct bpf_core_spec *local_spec, 4275 const struct btf *targ_btf, __u32 targ_id, 4276 struct bpf_core_spec *targ_spec) 4277 { 4278 const struct btf_type *targ_type; 4279 const struct bpf_core_accessor *local_acc; 4280 struct bpf_core_accessor *targ_acc; 4281 int i, sz, matched; 4282 4283 memset(targ_spec, 0, sizeof(*targ_spec)); 4284 targ_spec->btf = targ_btf; 4285 4286 local_acc = &local_spec->spec[0]; 4287 targ_acc = &targ_spec->spec[0]; 4288 4289 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { 4290 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, 4291 &targ_id); 4292 if (!targ_type) 4293 return -EINVAL; 4294 4295 if (local_acc->name) { 4296 matched = bpf_core_match_member(local_spec->btf, 4297 local_acc, 4298 targ_btf, targ_id, 4299 targ_spec, &targ_id); 4300 if (matched <= 0) 4301 return matched; 4302 } else { 4303 /* for i=0, targ_id is already treated as array element 4304 * type (because it's the original struct), for others 4305 * we should find array element type first 4306 */ 4307 if (i > 0) { 4308 const struct btf_array *a; 4309 bool flex; 4310 4311 if (!btf_is_array(targ_type)) 4312 return 0; 4313 4314 a = btf_array(targ_type); 4315 flex = is_flex_arr(targ_btf, targ_acc - 1, a); 4316 if (!flex && local_acc->idx >= a->nelems) 4317 return 0; 4318 if (!skip_mods_and_typedefs(targ_btf, a->type, 4319 &targ_id)) 4320 return -EINVAL; 4321 } 4322 4323 /* too deep struct/union/array nesting */ 4324 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 4325 return -E2BIG; 4326 4327 targ_acc->type_id = targ_id; 4328 targ_acc->idx = local_acc->idx; 4329 targ_acc->name = NULL; 4330 targ_spec->len++; 4331 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; 4332 targ_spec->raw_len++; 4333 4334 sz = btf__resolve_size(targ_btf, targ_id); 4335 if (sz < 0) 4336 return sz; 4337 targ_spec->bit_offset += local_acc->idx * sz * 8; 4338 } 4339 } 4340 4341 return 1; 4342 } 4343 4344 static int bpf_core_calc_field_relo(const struct bpf_program *prog, 4345 const struct bpf_field_reloc *relo, 4346 const struct bpf_core_spec *spec, 4347 __u32 *val, bool *validate) 4348 { 4349 const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1]; 4350 const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id); 4351 __u32 byte_off, byte_sz, bit_off, bit_sz; 4352 const struct btf_member *m; 4353 const struct btf_type *mt; 4354 bool bitfield; 4355 __s64 sz; 4356 4357 /* a[n] accessor needs special handling */ 4358 if (!acc->name) { 4359 if (relo->kind == BPF_FIELD_BYTE_OFFSET) { 4360 *val = spec->bit_offset / 8; 4361 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) { 4362 sz = btf__resolve_size(spec->btf, acc->type_id); 4363 if (sz < 0) 4364 return -EINVAL; 4365 *val = sz; 4366 } else { 4367 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", 4368 bpf_program__title(prog, false), 4369 relo->kind, relo->insn_off / 8); 4370 return -EINVAL; 4371 } 4372 if (validate) 4373 *validate = true; 4374 return 0; 4375 } 4376 4377 m = btf_members(t) + acc->idx; 4378 mt = skip_mods_and_typedefs(spec->btf, m->type, NULL); 4379 bit_off = spec->bit_offset; 4380 bit_sz = btf_member_bitfield_size(t, acc->idx); 4381 4382 bitfield = bit_sz > 0; 4383 if (bitfield) { 4384 byte_sz = mt->size; 4385 byte_off = bit_off / 8 / byte_sz * byte_sz; 4386 /* figure out smallest int size necessary for bitfield load */ 4387 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { 4388 if (byte_sz >= 8) { 4389 /* bitfield can't be read with 64-bit read */ 4390 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", 4391 bpf_program__title(prog, false), 4392 relo->kind, relo->insn_off / 8); 4393 return -E2BIG; 4394 } 4395 byte_sz *= 2; 4396 byte_off = bit_off / 8 / byte_sz * byte_sz; 4397 } 4398 } else { 4399 sz = btf__resolve_size(spec->btf, m->type); 4400 if (sz < 0) 4401 return -EINVAL; 4402 byte_sz = sz; 4403 byte_off = spec->bit_offset / 8; 4404 bit_sz = byte_sz * 8; 4405 } 4406 4407 /* for bitfields, all the relocatable aspects are ambiguous and we 4408 * might disagree with compiler, so turn off validation of expected 4409 * value, except for signedness 4410 */ 4411 if (validate) 4412 *validate = !bitfield; 4413 4414 switch (relo->kind) { 4415 case BPF_FIELD_BYTE_OFFSET: 4416 *val = byte_off; 4417 break; 4418 case BPF_FIELD_BYTE_SIZE: 4419 *val = byte_sz; 4420 break; 4421 case BPF_FIELD_SIGNED: 4422 /* enums will be assumed unsigned */ 4423 *val = btf_is_enum(mt) || 4424 (btf_int_encoding(mt) & BTF_INT_SIGNED); 4425 if (validate) 4426 *validate = true; /* signedness is never ambiguous */ 4427 break; 4428 case BPF_FIELD_LSHIFT_U64: 4429 #if __BYTE_ORDER == __LITTLE_ENDIAN 4430 *val = 64 - (bit_off + bit_sz - byte_off * 8); 4431 #else 4432 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); 4433 #endif 4434 break; 4435 case BPF_FIELD_RSHIFT_U64: 4436 *val = 64 - bit_sz; 4437 if (validate) 4438 *validate = true; /* right shift is never ambiguous */ 4439 break; 4440 case BPF_FIELD_EXISTS: 4441 default: 4442 pr_warn("prog '%s': unknown relo %d at insn #%d\n", 4443 bpf_program__title(prog, false), 4444 relo->kind, relo->insn_off / 8); 4445 return -EINVAL; 4446 } 4447 4448 return 0; 4449 } 4450 4451 /* 4452 * Patch relocatable BPF instruction. 4453 * 4454 * Patched value is determined by relocation kind and target specification. 4455 * For field existence relocation target spec will be NULL if field is not 4456 * found. 4457 * Expected insn->imm value is determined using relocation kind and local 4458 * spec, and is checked before patching instruction. If actual insn->imm value 4459 * is wrong, bail out with error. 4460 * 4461 * Currently three kinds of BPF instructions are supported: 4462 * 1. rX = <imm> (assignment with immediate operand); 4463 * 2. rX += <imm> (arithmetic operations with immediate operand); 4464 */ 4465 static int bpf_core_reloc_insn(struct bpf_program *prog, 4466 const struct bpf_field_reloc *relo, 4467 int relo_idx, 4468 const struct bpf_core_spec *local_spec, 4469 const struct bpf_core_spec *targ_spec) 4470 { 4471 __u32 orig_val, new_val; 4472 struct bpf_insn *insn; 4473 bool validate = true; 4474 int insn_idx, err; 4475 __u8 class; 4476 4477 if (relo->insn_off % sizeof(struct bpf_insn)) 4478 return -EINVAL; 4479 insn_idx = relo->insn_off / sizeof(struct bpf_insn); 4480 insn = &prog->insns[insn_idx]; 4481 class = BPF_CLASS(insn->code); 4482 4483 if (relo->kind == BPF_FIELD_EXISTS) { 4484 orig_val = 1; /* can't generate EXISTS relo w/o local field */ 4485 new_val = targ_spec ? 1 : 0; 4486 } else if (!targ_spec) { 4487 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", 4488 bpf_program__title(prog, false), relo_idx, insn_idx); 4489 insn->code = BPF_JMP | BPF_CALL; 4490 insn->dst_reg = 0; 4491 insn->src_reg = 0; 4492 insn->off = 0; 4493 /* if this instruction is reachable (not a dead code), 4494 * verifier will complain with the following message: 4495 * invalid func unknown#195896080 4496 */ 4497 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ 4498 return 0; 4499 } else { 4500 err = bpf_core_calc_field_relo(prog, relo, local_spec, 4501 &orig_val, &validate); 4502 if (err) 4503 return err; 4504 err = bpf_core_calc_field_relo(prog, relo, targ_spec, 4505 &new_val, NULL); 4506 if (err) 4507 return err; 4508 } 4509 4510 switch (class) { 4511 case BPF_ALU: 4512 case BPF_ALU64: 4513 if (BPF_SRC(insn->code) != BPF_K) 4514 return -EINVAL; 4515 if (validate && insn->imm != orig_val) { 4516 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n", 4517 bpf_program__title(prog, false), relo_idx, 4518 insn_idx, insn->imm, orig_val, new_val); 4519 return -EINVAL; 4520 } 4521 orig_val = insn->imm; 4522 insn->imm = new_val; 4523 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n", 4524 bpf_program__title(prog, false), relo_idx, insn_idx, 4525 orig_val, new_val); 4526 break; 4527 case BPF_LDX: 4528 case BPF_ST: 4529 case BPF_STX: 4530 if (validate && insn->off != orig_val) { 4531 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n", 4532 bpf_program__title(prog, false), relo_idx, 4533 insn_idx, insn->off, orig_val, new_val); 4534 return -EINVAL; 4535 } 4536 if (new_val > SHRT_MAX) { 4537 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n", 4538 bpf_program__title(prog, false), relo_idx, 4539 insn_idx, new_val); 4540 return -ERANGE; 4541 } 4542 orig_val = insn->off; 4543 insn->off = new_val; 4544 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n", 4545 bpf_program__title(prog, false), relo_idx, insn_idx, 4546 orig_val, new_val); 4547 break; 4548 default: 4549 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", 4550 bpf_program__title(prog, false), relo_idx, 4551 insn_idx, insn->code, insn->src_reg, insn->dst_reg, 4552 insn->off, insn->imm); 4553 return -EINVAL; 4554 } 4555 4556 return 0; 4557 } 4558 4559 /* Output spec definition in the format: 4560 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, 4561 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b 4562 */ 4563 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) 4564 { 4565 const struct btf_type *t; 4566 const char *s; 4567 __u32 type_id; 4568 int i; 4569 4570 type_id = spec->spec[0].type_id; 4571 t = btf__type_by_id(spec->btf, type_id); 4572 s = btf__name_by_offset(spec->btf, t->name_off); 4573 libbpf_print(level, "[%u] %s + ", type_id, s); 4574 4575 for (i = 0; i < spec->raw_len; i++) 4576 libbpf_print(level, "%d%s", spec->raw_spec[i], 4577 i == spec->raw_len - 1 ? " => " : ":"); 4578 4579 libbpf_print(level, "%u.%u @ &x", 4580 spec->bit_offset / 8, spec->bit_offset % 8); 4581 4582 for (i = 0; i < spec->len; i++) { 4583 if (spec->spec[i].name) 4584 libbpf_print(level, ".%s", spec->spec[i].name); 4585 else 4586 libbpf_print(level, "[%u]", spec->spec[i].idx); 4587 } 4588 4589 } 4590 4591 static size_t bpf_core_hash_fn(const void *key, void *ctx) 4592 { 4593 return (size_t)key; 4594 } 4595 4596 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) 4597 { 4598 return k1 == k2; 4599 } 4600 4601 static void *u32_as_hash_key(__u32 x) 4602 { 4603 return (void *)(uintptr_t)x; 4604 } 4605 4606 /* 4607 * CO-RE relocate single instruction. 4608 * 4609 * The outline and important points of the algorithm: 4610 * 1. For given local type, find corresponding candidate target types. 4611 * Candidate type is a type with the same "essential" name, ignoring 4612 * everything after last triple underscore (___). E.g., `sample`, 4613 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates 4614 * for each other. Names with triple underscore are referred to as 4615 * "flavors" and are useful, among other things, to allow to 4616 * specify/support incompatible variations of the same kernel struct, which 4617 * might differ between different kernel versions and/or build 4618 * configurations. 4619 * 4620 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C 4621 * converter, when deduplicated BTF of a kernel still contains more than 4622 * one different types with the same name. In that case, ___2, ___3, etc 4623 * are appended starting from second name conflict. But start flavors are 4624 * also useful to be defined "locally", in BPF program, to extract same 4625 * data from incompatible changes between different kernel 4626 * versions/configurations. For instance, to handle field renames between 4627 * kernel versions, one can use two flavors of the struct name with the 4628 * same common name and use conditional relocations to extract that field, 4629 * depending on target kernel version. 4630 * 2. For each candidate type, try to match local specification to this 4631 * candidate target type. Matching involves finding corresponding 4632 * high-level spec accessors, meaning that all named fields should match, 4633 * as well as all array accesses should be within the actual bounds. Also, 4634 * types should be compatible (see bpf_core_fields_are_compat for details). 4635 * 3. It is supported and expected that there might be multiple flavors 4636 * matching the spec. As long as all the specs resolve to the same set of 4637 * offsets across all candidates, there is no error. If there is any 4638 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate 4639 * imprefection of BTF deduplication, which can cause slight duplication of 4640 * the same BTF type, if some directly or indirectly referenced (by 4641 * pointer) type gets resolved to different actual types in different 4642 * object files. If such situation occurs, deduplicated BTF will end up 4643 * with two (or more) structurally identical types, which differ only in 4644 * types they refer to through pointer. This should be OK in most cases and 4645 * is not an error. 4646 * 4. Candidate types search is performed by linearly scanning through all 4647 * types in target BTF. It is anticipated that this is overall more 4648 * efficient memory-wise and not significantly worse (if not better) 4649 * CPU-wise compared to prebuilding a map from all local type names to 4650 * a list of candidate type names. It's also sped up by caching resolved 4651 * list of matching candidates per each local "root" type ID, that has at 4652 * least one bpf_field_reloc associated with it. This list is shared 4653 * between multiple relocations for the same type ID and is updated as some 4654 * of the candidates are pruned due to structural incompatibility. 4655 */ 4656 static int bpf_core_reloc_field(struct bpf_program *prog, 4657 const struct bpf_field_reloc *relo, 4658 int relo_idx, 4659 const struct btf *local_btf, 4660 const struct btf *targ_btf, 4661 struct hashmap *cand_cache) 4662 { 4663 const char *prog_name = bpf_program__title(prog, false); 4664 struct bpf_core_spec local_spec, cand_spec, targ_spec; 4665 const void *type_key = u32_as_hash_key(relo->type_id); 4666 const struct btf_type *local_type, *cand_type; 4667 const char *local_name, *cand_name; 4668 struct ids_vec *cand_ids; 4669 __u32 local_id, cand_id; 4670 const char *spec_str; 4671 int i, j, err; 4672 4673 local_id = relo->type_id; 4674 local_type = btf__type_by_id(local_btf, local_id); 4675 if (!local_type) 4676 return -EINVAL; 4677 4678 local_name = btf__name_by_offset(local_btf, local_type->name_off); 4679 if (str_is_empty(local_name)) 4680 return -EINVAL; 4681 4682 spec_str = btf__name_by_offset(local_btf, relo->access_str_off); 4683 if (str_is_empty(spec_str)) 4684 return -EINVAL; 4685 4686 err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec); 4687 if (err) { 4688 pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", 4689 prog_name, relo_idx, local_id, local_name, spec_str, 4690 err); 4691 return -EINVAL; 4692 } 4693 4694 pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx, 4695 relo->kind); 4696 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); 4697 libbpf_print(LIBBPF_DEBUG, "\n"); 4698 4699 if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) { 4700 cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf); 4701 if (IS_ERR(cand_ids)) { 4702 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", 4703 prog_name, relo_idx, local_id, local_name, 4704 PTR_ERR(cand_ids)); 4705 return PTR_ERR(cand_ids); 4706 } 4707 err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL); 4708 if (err) { 4709 bpf_core_free_cands(cand_ids); 4710 return err; 4711 } 4712 } 4713 4714 for (i = 0, j = 0; i < cand_ids->len; i++) { 4715 cand_id = cand_ids->data[i]; 4716 cand_type = btf__type_by_id(targ_btf, cand_id); 4717 cand_name = btf__name_by_offset(targ_btf, cand_type->name_off); 4718 4719 err = bpf_core_spec_match(&local_spec, targ_btf, 4720 cand_id, &cand_spec); 4721 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ", 4722 prog_name, relo_idx, i, cand_name); 4723 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); 4724 libbpf_print(LIBBPF_DEBUG, ": %d\n", err); 4725 if (err < 0) { 4726 pr_warn("prog '%s': relo #%d: matching error: %d\n", 4727 prog_name, relo_idx, err); 4728 return err; 4729 } 4730 if (err == 0) 4731 continue; 4732 4733 if (j == 0) { 4734 targ_spec = cand_spec; 4735 } else if (cand_spec.bit_offset != targ_spec.bit_offset) { 4736 /* if there are many candidates, they should all 4737 * resolve to the same bit offset 4738 */ 4739 pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n", 4740 prog_name, relo_idx, cand_spec.bit_offset, 4741 targ_spec.bit_offset); 4742 return -EINVAL; 4743 } 4744 4745 cand_ids->data[j++] = cand_spec.spec[0].type_id; 4746 } 4747 4748 /* 4749 * For BPF_FIELD_EXISTS relo or when used BPF program has field 4750 * existence checks or kernel version/config checks, it's expected 4751 * that we might not find any candidates. In this case, if field 4752 * wasn't found in any candidate, the list of candidates shouldn't 4753 * change at all, we'll just handle relocating appropriately, 4754 * depending on relo's kind. 4755 */ 4756 if (j > 0) 4757 cand_ids->len = j; 4758 4759 /* 4760 * If no candidates were found, it might be both a programmer error, 4761 * as well as expected case, depending whether instruction w/ 4762 * relocation is guarded in some way that makes it unreachable (dead 4763 * code) if relocation can't be resolved. This is handled in 4764 * bpf_core_reloc_insn() uniformly by replacing that instruction with 4765 * BPF helper call insn (using invalid helper ID). If that instruction 4766 * is indeed unreachable, then it will be ignored and eliminated by 4767 * verifier. If it was an error, then verifier will complain and point 4768 * to a specific instruction number in its log. 4769 */ 4770 if (j == 0) 4771 pr_debug("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", 4772 prog_name, relo_idx, local_id, local_name, spec_str); 4773 4774 /* bpf_core_reloc_insn should know how to handle missing targ_spec */ 4775 err = bpf_core_reloc_insn(prog, relo, relo_idx, &local_spec, 4776 j ? &targ_spec : NULL); 4777 if (err) { 4778 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", 4779 prog_name, relo_idx, relo->insn_off, err); 4780 return -EINVAL; 4781 } 4782 4783 return 0; 4784 } 4785 4786 static int 4787 bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) 4788 { 4789 const struct btf_ext_info_sec *sec; 4790 const struct bpf_field_reloc *rec; 4791 const struct btf_ext_info *seg; 4792 struct hashmap_entry *entry; 4793 struct hashmap *cand_cache = NULL; 4794 struct bpf_program *prog; 4795 struct btf *targ_btf; 4796 const char *sec_name; 4797 int i, err = 0; 4798 4799 if (targ_btf_path) 4800 targ_btf = btf__parse_elf(targ_btf_path, NULL); 4801 else 4802 targ_btf = libbpf_find_kernel_btf(); 4803 if (IS_ERR(targ_btf)) { 4804 pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf)); 4805 return PTR_ERR(targ_btf); 4806 } 4807 4808 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 4809 if (IS_ERR(cand_cache)) { 4810 err = PTR_ERR(cand_cache); 4811 goto out; 4812 } 4813 4814 seg = &obj->btf_ext->field_reloc_info; 4815 for_each_btf_ext_sec(seg, sec) { 4816 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 4817 if (str_is_empty(sec_name)) { 4818 err = -EINVAL; 4819 goto out; 4820 } 4821 prog = bpf_object__find_program_by_title(obj, sec_name); 4822 if (!prog) { 4823 pr_warn("failed to find program '%s' for CO-RE offset relocation\n", 4824 sec_name); 4825 err = -EINVAL; 4826 goto out; 4827 } 4828 4829 pr_debug("prog '%s': performing %d CO-RE offset relocs\n", 4830 sec_name, sec->num_info); 4831 4832 for_each_btf_ext_rec(seg, sec, i, rec) { 4833 err = bpf_core_reloc_field(prog, rec, i, obj->btf, 4834 targ_btf, cand_cache); 4835 if (err) { 4836 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 4837 sec_name, i, err); 4838 goto out; 4839 } 4840 } 4841 } 4842 4843 out: 4844 btf__free(targ_btf); 4845 if (!IS_ERR_OR_NULL(cand_cache)) { 4846 hashmap__for_each_entry(cand_cache, entry, i) { 4847 bpf_core_free_cands(entry->value); 4848 } 4849 hashmap__free(cand_cache); 4850 } 4851 return err; 4852 } 4853 4854 static int 4855 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 4856 { 4857 int err = 0; 4858 4859 if (obj->btf_ext->field_reloc_info.len) 4860 err = bpf_core_reloc_fields(obj, targ_btf_path); 4861 4862 return err; 4863 } 4864 4865 static int 4866 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 4867 struct reloc_desc *relo) 4868 { 4869 struct bpf_insn *insn, *new_insn; 4870 struct bpf_program *text; 4871 size_t new_cnt; 4872 int err; 4873 4874 if (prog->idx != obj->efile.text_shndx && prog->main_prog_cnt == 0) { 4875 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 4876 if (!text) { 4877 pr_warn("no .text section found yet relo into text exist\n"); 4878 return -LIBBPF_ERRNO__RELOC; 4879 } 4880 new_cnt = prog->insns_cnt + text->insns_cnt; 4881 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 4882 if (!new_insn) { 4883 pr_warn("oom in prog realloc\n"); 4884 return -ENOMEM; 4885 } 4886 prog->insns = new_insn; 4887 4888 if (obj->btf_ext) { 4889 err = bpf_program_reloc_btf_ext(prog, obj, 4890 text->section_name, 4891 prog->insns_cnt); 4892 if (err) 4893 return err; 4894 } 4895 4896 memcpy(new_insn + prog->insns_cnt, text->insns, 4897 text->insns_cnt * sizeof(*insn)); 4898 prog->main_prog_cnt = prog->insns_cnt; 4899 prog->insns_cnt = new_cnt; 4900 pr_debug("added %zd insn from %s to prog %s\n", 4901 text->insns_cnt, text->section_name, 4902 prog->section_name); 4903 } 4904 4905 insn = &prog->insns[relo->insn_idx]; 4906 insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx; 4907 return 0; 4908 } 4909 4910 static int 4911 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 4912 { 4913 int i, err; 4914 4915 if (!prog) 4916 return 0; 4917 4918 if (obj->btf_ext) { 4919 err = bpf_program_reloc_btf_ext(prog, obj, 4920 prog->section_name, 0); 4921 if (err) 4922 return err; 4923 } 4924 4925 if (!prog->reloc_desc) 4926 return 0; 4927 4928 for (i = 0; i < prog->nr_reloc; i++) { 4929 struct reloc_desc *relo = &prog->reloc_desc[i]; 4930 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 4931 4932 if (relo->insn_idx + 1 >= (int)prog->insns_cnt) { 4933 pr_warn("relocation out of range: '%s'\n", 4934 prog->section_name); 4935 return -LIBBPF_ERRNO__RELOC; 4936 } 4937 4938 switch (relo->type) { 4939 case RELO_LD64: 4940 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 4941 insn[0].imm = obj->maps[relo->map_idx].fd; 4942 break; 4943 case RELO_DATA: 4944 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 4945 insn[1].imm = insn[0].imm + relo->sym_off; 4946 insn[0].imm = obj->maps[relo->map_idx].fd; 4947 break; 4948 case RELO_EXTERN: 4949 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 4950 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 4951 insn[1].imm = relo->sym_off; 4952 break; 4953 case RELO_CALL: 4954 err = bpf_program__reloc_text(prog, obj, relo); 4955 if (err) 4956 return err; 4957 break; 4958 default: 4959 pr_warn("relo #%d: bad relo type %d\n", i, relo->type); 4960 return -EINVAL; 4961 } 4962 } 4963 4964 zfree(&prog->reloc_desc); 4965 prog->nr_reloc = 0; 4966 return 0; 4967 } 4968 4969 static int 4970 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 4971 { 4972 struct bpf_program *prog; 4973 size_t i; 4974 int err; 4975 4976 if (obj->btf_ext) { 4977 err = bpf_object__relocate_core(obj, targ_btf_path); 4978 if (err) { 4979 pr_warn("failed to perform CO-RE relocations: %d\n", 4980 err); 4981 return err; 4982 } 4983 } 4984 /* ensure .text is relocated first, as it's going to be copied as-is 4985 * later for sub-program calls 4986 */ 4987 for (i = 0; i < obj->nr_programs; i++) { 4988 prog = &obj->programs[i]; 4989 if (prog->idx != obj->efile.text_shndx) 4990 continue; 4991 4992 err = bpf_program__relocate(prog, obj); 4993 if (err) { 4994 pr_warn("failed to relocate '%s'\n", prog->section_name); 4995 return err; 4996 } 4997 break; 4998 } 4999 /* now relocate everything but .text, which by now is relocated 5000 * properly, so we can copy raw sub-program instructions as is safely 5001 */ 5002 for (i = 0; i < obj->nr_programs; i++) { 5003 prog = &obj->programs[i]; 5004 if (prog->idx == obj->efile.text_shndx) 5005 continue; 5006 5007 err = bpf_program__relocate(prog, obj); 5008 if (err) { 5009 pr_warn("failed to relocate '%s'\n", prog->section_name); 5010 return err; 5011 } 5012 } 5013 return 0; 5014 } 5015 5016 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 5017 GElf_Shdr *shdr, Elf_Data *data); 5018 5019 static int bpf_object__collect_map_relos(struct bpf_object *obj, 5020 GElf_Shdr *shdr, Elf_Data *data) 5021 { 5022 int i, j, nrels, new_sz, ptr_sz = sizeof(void *); 5023 const struct btf_var_secinfo *vi = NULL; 5024 const struct btf_type *sec, *var, *def; 5025 const struct btf_member *member; 5026 struct bpf_map *map, *targ_map; 5027 const char *name, *mname; 5028 Elf_Data *symbols; 5029 unsigned int moff; 5030 GElf_Sym sym; 5031 GElf_Rel rel; 5032 void *tmp; 5033 5034 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 5035 return -EINVAL; 5036 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 5037 if (!sec) 5038 return -EINVAL; 5039 5040 symbols = obj->efile.symbols; 5041 nrels = shdr->sh_size / shdr->sh_entsize; 5042 for (i = 0; i < nrels; i++) { 5043 if (!gelf_getrel(data, i, &rel)) { 5044 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 5045 return -LIBBPF_ERRNO__FORMAT; 5046 } 5047 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 5048 pr_warn(".maps relo #%d: symbol %zx not found\n", 5049 i, (size_t)GELF_R_SYM(rel.r_info)); 5050 return -LIBBPF_ERRNO__FORMAT; 5051 } 5052 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 5053 sym.st_name) ? : "<?>"; 5054 if (sym.st_shndx != obj->efile.btf_maps_shndx) { 5055 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 5056 i, name); 5057 return -LIBBPF_ERRNO__RELOC; 5058 } 5059 5060 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n", 5061 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value, 5062 (size_t)rel.r_offset, sym.st_name, name); 5063 5064 for (j = 0; j < obj->nr_maps; j++) { 5065 map = &obj->maps[j]; 5066 if (map->sec_idx != obj->efile.btf_maps_shndx) 5067 continue; 5068 5069 vi = btf_var_secinfos(sec) + map->btf_var_idx; 5070 if (vi->offset <= rel.r_offset && 5071 rel.r_offset + sizeof(void *) <= vi->offset + vi->size) 5072 break; 5073 } 5074 if (j == obj->nr_maps) { 5075 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n", 5076 i, name, (size_t)rel.r_offset); 5077 return -EINVAL; 5078 } 5079 5080 if (!bpf_map_type__is_map_in_map(map->def.type)) 5081 return -EINVAL; 5082 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 5083 map->def.key_size != sizeof(int)) { 5084 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 5085 i, map->name, sizeof(int)); 5086 return -EINVAL; 5087 } 5088 5089 targ_map = bpf_object__find_map_by_name(obj, name); 5090 if (!targ_map) 5091 return -ESRCH; 5092 5093 var = btf__type_by_id(obj->btf, vi->type); 5094 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 5095 if (btf_vlen(def) == 0) 5096 return -EINVAL; 5097 member = btf_members(def) + btf_vlen(def) - 1; 5098 mname = btf__name_by_offset(obj->btf, member->name_off); 5099 if (strcmp(mname, "values")) 5100 return -EINVAL; 5101 5102 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 5103 if (rel.r_offset - vi->offset < moff) 5104 return -EINVAL; 5105 5106 moff = rel.r_offset - vi->offset - moff; 5107 if (moff % ptr_sz) 5108 return -EINVAL; 5109 moff /= ptr_sz; 5110 if (moff >= map->init_slots_sz) { 5111 new_sz = moff + 1; 5112 tmp = realloc(map->init_slots, new_sz * ptr_sz); 5113 if (!tmp) 5114 return -ENOMEM; 5115 map->init_slots = tmp; 5116 memset(map->init_slots + map->init_slots_sz, 0, 5117 (new_sz - map->init_slots_sz) * ptr_sz); 5118 map->init_slots_sz = new_sz; 5119 } 5120 map->init_slots[moff] = targ_map; 5121 5122 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n", 5123 i, map->name, moff, name); 5124 } 5125 5126 return 0; 5127 } 5128 5129 static int bpf_object__collect_reloc(struct bpf_object *obj) 5130 { 5131 int i, err; 5132 5133 if (!obj_elf_valid(obj)) { 5134 pr_warn("Internal error: elf object is closed\n"); 5135 return -LIBBPF_ERRNO__INTERNAL; 5136 } 5137 5138 for (i = 0; i < obj->efile.nr_reloc_sects; i++) { 5139 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; 5140 Elf_Data *data = obj->efile.reloc_sects[i].data; 5141 int idx = shdr->sh_info; 5142 struct bpf_program *prog; 5143 5144 if (shdr->sh_type != SHT_REL) { 5145 pr_warn("internal error at %d\n", __LINE__); 5146 return -LIBBPF_ERRNO__INTERNAL; 5147 } 5148 5149 if (idx == obj->efile.st_ops_shndx) { 5150 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 5151 } else if (idx == obj->efile.btf_maps_shndx) { 5152 err = bpf_object__collect_map_relos(obj, shdr, data); 5153 } else { 5154 prog = bpf_object__find_prog_by_idx(obj, idx); 5155 if (!prog) { 5156 pr_warn("relocation failed: no prog in section(%d)\n", idx); 5157 return -LIBBPF_ERRNO__RELOC; 5158 } 5159 err = bpf_program__collect_reloc(prog, shdr, data, obj); 5160 } 5161 if (err) 5162 return err; 5163 } 5164 return 0; 5165 } 5166 5167 static int 5168 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 5169 char *license, __u32 kern_version, int *pfd) 5170 { 5171 struct bpf_load_program_attr load_attr; 5172 char *cp, errmsg[STRERR_BUFSIZE]; 5173 size_t log_buf_size = 0; 5174 char *log_buf = NULL; 5175 int btf_fd, ret; 5176 5177 if (!insns || !insns_cnt) 5178 return -EINVAL; 5179 5180 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 5181 load_attr.prog_type = prog->type; 5182 /* old kernels might not support specifying expected_attach_type */ 5183 if (!prog->caps->exp_attach_type && prog->sec_def && 5184 prog->sec_def->is_exp_attach_type_optional) 5185 load_attr.expected_attach_type = 0; 5186 else 5187 load_attr.expected_attach_type = prog->expected_attach_type; 5188 if (prog->caps->name) 5189 load_attr.name = prog->name; 5190 load_attr.insns = insns; 5191 load_attr.insns_cnt = insns_cnt; 5192 load_attr.license = license; 5193 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 5194 prog->type == BPF_PROG_TYPE_LSM) { 5195 load_attr.attach_btf_id = prog->attach_btf_id; 5196 } else if (prog->type == BPF_PROG_TYPE_TRACING || 5197 prog->type == BPF_PROG_TYPE_EXT) { 5198 load_attr.attach_prog_fd = prog->attach_prog_fd; 5199 load_attr.attach_btf_id = prog->attach_btf_id; 5200 } else { 5201 load_attr.kern_version = kern_version; 5202 load_attr.prog_ifindex = prog->prog_ifindex; 5203 } 5204 /* if .BTF.ext was loaded, kernel supports associated BTF for prog */ 5205 if (prog->obj->btf_ext) 5206 btf_fd = bpf_object__btf_fd(prog->obj); 5207 else 5208 btf_fd = -1; 5209 load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0; 5210 load_attr.func_info = prog->func_info; 5211 load_attr.func_info_rec_size = prog->func_info_rec_size; 5212 load_attr.func_info_cnt = prog->func_info_cnt; 5213 load_attr.line_info = prog->line_info; 5214 load_attr.line_info_rec_size = prog->line_info_rec_size; 5215 load_attr.line_info_cnt = prog->line_info_cnt; 5216 load_attr.log_level = prog->log_level; 5217 load_attr.prog_flags = prog->prog_flags; 5218 5219 retry_load: 5220 if (log_buf_size) { 5221 log_buf = malloc(log_buf_size); 5222 if (!log_buf) 5223 return -ENOMEM; 5224 5225 *log_buf = 0; 5226 } 5227 5228 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 5229 5230 if (ret >= 0) { 5231 if (log_buf && load_attr.log_level) 5232 pr_debug("verifier log:\n%s", log_buf); 5233 *pfd = ret; 5234 ret = 0; 5235 goto out; 5236 } 5237 5238 if (!log_buf || errno == ENOSPC) { 5239 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, 5240 log_buf_size << 1); 5241 5242 free(log_buf); 5243 goto retry_load; 5244 } 5245 ret = -errno; 5246 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 5247 pr_warn("load bpf program failed: %s\n", cp); 5248 pr_perm_msg(ret); 5249 5250 if (log_buf && log_buf[0] != '\0') { 5251 ret = -LIBBPF_ERRNO__VERIFY; 5252 pr_warn("-- BEGIN DUMP LOG ---\n"); 5253 pr_warn("\n%s\n", log_buf); 5254 pr_warn("-- END LOG --\n"); 5255 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 5256 pr_warn("Program too large (%zu insns), at most %d insns\n", 5257 load_attr.insns_cnt, BPF_MAXINSNS); 5258 ret = -LIBBPF_ERRNO__PROG2BIG; 5259 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 5260 /* Wrong program type? */ 5261 int fd; 5262 5263 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 5264 load_attr.expected_attach_type = 0; 5265 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 5266 if (fd >= 0) { 5267 close(fd); 5268 ret = -LIBBPF_ERRNO__PROGTYPE; 5269 goto out; 5270 } 5271 } 5272 5273 out: 5274 free(log_buf); 5275 return ret; 5276 } 5277 5278 static int libbpf_find_attach_btf_id(struct bpf_program *prog); 5279 5280 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) 5281 { 5282 int err = 0, fd, i, btf_id; 5283 5284 if ((prog->type == BPF_PROG_TYPE_TRACING || 5285 prog->type == BPF_PROG_TYPE_LSM || 5286 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { 5287 btf_id = libbpf_find_attach_btf_id(prog); 5288 if (btf_id <= 0) 5289 return btf_id; 5290 prog->attach_btf_id = btf_id; 5291 } 5292 5293 if (prog->instances.nr < 0 || !prog->instances.fds) { 5294 if (prog->preprocessor) { 5295 pr_warn("Internal error: can't load program '%s'\n", 5296 prog->section_name); 5297 return -LIBBPF_ERRNO__INTERNAL; 5298 } 5299 5300 prog->instances.fds = malloc(sizeof(int)); 5301 if (!prog->instances.fds) { 5302 pr_warn("Not enough memory for BPF fds\n"); 5303 return -ENOMEM; 5304 } 5305 prog->instances.nr = 1; 5306 prog->instances.fds[0] = -1; 5307 } 5308 5309 if (!prog->preprocessor) { 5310 if (prog->instances.nr != 1) { 5311 pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n", 5312 prog->section_name, prog->instances.nr); 5313 } 5314 err = load_program(prog, prog->insns, prog->insns_cnt, 5315 license, kern_ver, &fd); 5316 if (!err) 5317 prog->instances.fds[0] = fd; 5318 goto out; 5319 } 5320 5321 for (i = 0; i < prog->instances.nr; i++) { 5322 struct bpf_prog_prep_result result; 5323 bpf_program_prep_t preprocessor = prog->preprocessor; 5324 5325 memset(&result, 0, sizeof(result)); 5326 err = preprocessor(prog, i, prog->insns, 5327 prog->insns_cnt, &result); 5328 if (err) { 5329 pr_warn("Preprocessing the %dth instance of program '%s' failed\n", 5330 i, prog->section_name); 5331 goto out; 5332 } 5333 5334 if (!result.new_insn_ptr || !result.new_insn_cnt) { 5335 pr_debug("Skip loading the %dth instance of program '%s'\n", 5336 i, prog->section_name); 5337 prog->instances.fds[i] = -1; 5338 if (result.pfd) 5339 *result.pfd = -1; 5340 continue; 5341 } 5342 5343 err = load_program(prog, result.new_insn_ptr, 5344 result.new_insn_cnt, license, kern_ver, &fd); 5345 if (err) { 5346 pr_warn("Loading the %dth instance of program '%s' failed\n", 5347 i, prog->section_name); 5348 goto out; 5349 } 5350 5351 if (result.pfd) 5352 *result.pfd = fd; 5353 prog->instances.fds[i] = fd; 5354 } 5355 out: 5356 if (err) 5357 pr_warn("failed to load program '%s'\n", prog->section_name); 5358 zfree(&prog->insns); 5359 prog->insns_cnt = 0; 5360 return err; 5361 } 5362 5363 static bool bpf_program__is_function_storage(const struct bpf_program *prog, 5364 const struct bpf_object *obj) 5365 { 5366 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 5367 } 5368 5369 static int 5370 bpf_object__load_progs(struct bpf_object *obj, int log_level) 5371 { 5372 size_t i; 5373 int err; 5374 5375 for (i = 0; i < obj->nr_programs; i++) { 5376 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 5377 continue; 5378 obj->programs[i].log_level |= log_level; 5379 err = bpf_program__load(&obj->programs[i], 5380 obj->license, 5381 obj->kern_version); 5382 if (err) 5383 return err; 5384 } 5385 return 0; 5386 } 5387 5388 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 5389 5390 static struct bpf_object * 5391 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, 5392 const struct bpf_object_open_opts *opts) 5393 { 5394 const char *obj_name, *kconfig; 5395 struct bpf_program *prog; 5396 struct bpf_object *obj; 5397 char tmp_name[64]; 5398 int err; 5399 5400 if (elf_version(EV_CURRENT) == EV_NONE) { 5401 pr_warn("failed to init libelf for %s\n", 5402 path ? : "(mem buf)"); 5403 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 5404 } 5405 5406 if (!OPTS_VALID(opts, bpf_object_open_opts)) 5407 return ERR_PTR(-EINVAL); 5408 5409 obj_name = OPTS_GET(opts, object_name, NULL); 5410 if (obj_buf) { 5411 if (!obj_name) { 5412 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 5413 (unsigned long)obj_buf, 5414 (unsigned long)obj_buf_sz); 5415 obj_name = tmp_name; 5416 } 5417 path = obj_name; 5418 pr_debug("loading object '%s' from buffer\n", obj_name); 5419 } 5420 5421 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 5422 if (IS_ERR(obj)) 5423 return obj; 5424 5425 kconfig = OPTS_GET(opts, kconfig, NULL); 5426 if (kconfig) { 5427 obj->kconfig = strdup(kconfig); 5428 if (!obj->kconfig) 5429 return ERR_PTR(-ENOMEM); 5430 } 5431 5432 err = bpf_object__elf_init(obj); 5433 err = err ? : bpf_object__check_endianness(obj); 5434 err = err ? : bpf_object__elf_collect(obj); 5435 err = err ? : bpf_object__collect_externs(obj); 5436 err = err ? : bpf_object__finalize_btf(obj); 5437 err = err ? : bpf_object__init_maps(obj, opts); 5438 err = err ? : bpf_object__init_prog_names(obj); 5439 err = err ? : bpf_object__collect_reloc(obj); 5440 if (err) 5441 goto out; 5442 bpf_object__elf_finish(obj); 5443 5444 bpf_object__for_each_program(prog, obj) { 5445 prog->sec_def = find_sec_def(prog->section_name); 5446 if (!prog->sec_def) 5447 /* couldn't guess, but user might manually specify */ 5448 continue; 5449 5450 bpf_program__set_type(prog, prog->sec_def->prog_type); 5451 bpf_program__set_expected_attach_type(prog, 5452 prog->sec_def->expected_attach_type); 5453 5454 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || 5455 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) 5456 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); 5457 } 5458 5459 return obj; 5460 out: 5461 bpf_object__close(obj); 5462 return ERR_PTR(err); 5463 } 5464 5465 static struct bpf_object * 5466 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) 5467 { 5468 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 5469 .relaxed_maps = flags & MAPS_RELAX_COMPAT, 5470 ); 5471 5472 /* param validation */ 5473 if (!attr->file) 5474 return NULL; 5475 5476 pr_debug("loading %s\n", attr->file); 5477 return __bpf_object__open(attr->file, NULL, 0, &opts); 5478 } 5479 5480 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 5481 { 5482 return __bpf_object__open_xattr(attr, 0); 5483 } 5484 5485 struct bpf_object *bpf_object__open(const char *path) 5486 { 5487 struct bpf_object_open_attr attr = { 5488 .file = path, 5489 .prog_type = BPF_PROG_TYPE_UNSPEC, 5490 }; 5491 5492 return bpf_object__open_xattr(&attr); 5493 } 5494 5495 struct bpf_object * 5496 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 5497 { 5498 if (!path) 5499 return ERR_PTR(-EINVAL); 5500 5501 pr_debug("loading %s\n", path); 5502 5503 return __bpf_object__open(path, NULL, 0, opts); 5504 } 5505 5506 struct bpf_object * 5507 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 5508 const struct bpf_object_open_opts *opts) 5509 { 5510 if (!obj_buf || obj_buf_sz == 0) 5511 return ERR_PTR(-EINVAL); 5512 5513 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts); 5514 } 5515 5516 struct bpf_object * 5517 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, 5518 const char *name) 5519 { 5520 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 5521 .object_name = name, 5522 /* wrong default, but backwards-compatible */ 5523 .relaxed_maps = true, 5524 ); 5525 5526 /* returning NULL is wrong, but backwards-compatible */ 5527 if (!obj_buf || obj_buf_sz == 0) 5528 return NULL; 5529 5530 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts); 5531 } 5532 5533 int bpf_object__unload(struct bpf_object *obj) 5534 { 5535 size_t i; 5536 5537 if (!obj) 5538 return -EINVAL; 5539 5540 for (i = 0; i < obj->nr_maps; i++) { 5541 zclose(obj->maps[i].fd); 5542 if (obj->maps[i].st_ops) 5543 zfree(&obj->maps[i].st_ops->kern_vdata); 5544 } 5545 5546 for (i = 0; i < obj->nr_programs; i++) 5547 bpf_program__unload(&obj->programs[i]); 5548 5549 return 0; 5550 } 5551 5552 static int bpf_object__sanitize_maps(struct bpf_object *obj) 5553 { 5554 struct bpf_map *m; 5555 5556 bpf_object__for_each_map(m, obj) { 5557 if (!bpf_map__is_internal(m)) 5558 continue; 5559 if (!obj->caps.global_data) { 5560 pr_warn("kernel doesn't support global data\n"); 5561 return -ENOTSUP; 5562 } 5563 if (!obj->caps.array_mmap) 5564 m->def.map_flags ^= BPF_F_MMAPABLE; 5565 } 5566 5567 return 0; 5568 } 5569 5570 static int bpf_object__resolve_externs(struct bpf_object *obj, 5571 const char *extra_kconfig) 5572 { 5573 bool need_config = false; 5574 struct extern_desc *ext; 5575 int err, i; 5576 void *data; 5577 5578 if (obj->nr_extern == 0) 5579 return 0; 5580 5581 data = obj->maps[obj->kconfig_map_idx].mmaped; 5582 5583 for (i = 0; i < obj->nr_extern; i++) { 5584 ext = &obj->externs[i]; 5585 5586 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 5587 void *ext_val = data + ext->data_off; 5588 __u32 kver = get_kernel_version(); 5589 5590 if (!kver) { 5591 pr_warn("failed to get kernel version\n"); 5592 return -EINVAL; 5593 } 5594 err = set_ext_value_num(ext, ext_val, kver); 5595 if (err) 5596 return err; 5597 pr_debug("extern %s=0x%x\n", ext->name, kver); 5598 } else if (strncmp(ext->name, "CONFIG_", 7) == 0) { 5599 need_config = true; 5600 } else { 5601 pr_warn("unrecognized extern '%s'\n", ext->name); 5602 return -EINVAL; 5603 } 5604 } 5605 if (need_config && extra_kconfig) { 5606 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, data); 5607 if (err) 5608 return -EINVAL; 5609 need_config = false; 5610 for (i = 0; i < obj->nr_extern; i++) { 5611 ext = &obj->externs[i]; 5612 if (!ext->is_set) { 5613 need_config = true; 5614 break; 5615 } 5616 } 5617 } 5618 if (need_config) { 5619 err = bpf_object__read_kconfig_file(obj, data); 5620 if (err) 5621 return -EINVAL; 5622 } 5623 for (i = 0; i < obj->nr_extern; i++) { 5624 ext = &obj->externs[i]; 5625 5626 if (!ext->is_set && !ext->is_weak) { 5627 pr_warn("extern %s (strong) not resolved\n", ext->name); 5628 return -ESRCH; 5629 } else if (!ext->is_set) { 5630 pr_debug("extern %s (weak) not resolved, defaulting to zero\n", 5631 ext->name); 5632 } 5633 } 5634 5635 return 0; 5636 } 5637 5638 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 5639 { 5640 struct bpf_object *obj; 5641 int err, i; 5642 5643 if (!attr) 5644 return -EINVAL; 5645 obj = attr->obj; 5646 if (!obj) 5647 return -EINVAL; 5648 5649 if (obj->loaded) { 5650 pr_warn("object should not be loaded twice\n"); 5651 return -EINVAL; 5652 } 5653 5654 obj->loaded = true; 5655 5656 err = bpf_object__probe_loading(obj); 5657 err = err ? : bpf_object__probe_caps(obj); 5658 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 5659 err = err ? : bpf_object__sanitize_and_load_btf(obj); 5660 err = err ? : bpf_object__sanitize_maps(obj); 5661 err = err ? : bpf_object__load_vmlinux_btf(obj); 5662 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 5663 err = err ? : bpf_object__create_maps(obj); 5664 err = err ? : bpf_object__relocate(obj, attr->target_btf_path); 5665 err = err ? : bpf_object__load_progs(obj, attr->log_level); 5666 5667 btf__free(obj->btf_vmlinux); 5668 obj->btf_vmlinux = NULL; 5669 5670 if (err) 5671 goto out; 5672 5673 return 0; 5674 out: 5675 /* unpin any maps that were auto-pinned during load */ 5676 for (i = 0; i < obj->nr_maps; i++) 5677 if (obj->maps[i].pinned && !obj->maps[i].reused) 5678 bpf_map__unpin(&obj->maps[i], NULL); 5679 5680 bpf_object__unload(obj); 5681 pr_warn("failed to load object '%s'\n", obj->path); 5682 return err; 5683 } 5684 5685 int bpf_object__load(struct bpf_object *obj) 5686 { 5687 struct bpf_object_load_attr attr = { 5688 .obj = obj, 5689 }; 5690 5691 return bpf_object__load_xattr(&attr); 5692 } 5693 5694 static int make_parent_dir(const char *path) 5695 { 5696 char *cp, errmsg[STRERR_BUFSIZE]; 5697 char *dname, *dir; 5698 int err = 0; 5699 5700 dname = strdup(path); 5701 if (dname == NULL) 5702 return -ENOMEM; 5703 5704 dir = dirname(dname); 5705 if (mkdir(dir, 0700) && errno != EEXIST) 5706 err = -errno; 5707 5708 free(dname); 5709 if (err) { 5710 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 5711 pr_warn("failed to mkdir %s: %s\n", path, cp); 5712 } 5713 return err; 5714 } 5715 5716 static int check_path(const char *path) 5717 { 5718 char *cp, errmsg[STRERR_BUFSIZE]; 5719 struct statfs st_fs; 5720 char *dname, *dir; 5721 int err = 0; 5722 5723 if (path == NULL) 5724 return -EINVAL; 5725 5726 dname = strdup(path); 5727 if (dname == NULL) 5728 return -ENOMEM; 5729 5730 dir = dirname(dname); 5731 if (statfs(dir, &st_fs)) { 5732 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 5733 pr_warn("failed to statfs %s: %s\n", dir, cp); 5734 err = -errno; 5735 } 5736 free(dname); 5737 5738 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 5739 pr_warn("specified path %s is not on BPF FS\n", path); 5740 err = -EINVAL; 5741 } 5742 5743 return err; 5744 } 5745 5746 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 5747 int instance) 5748 { 5749 char *cp, errmsg[STRERR_BUFSIZE]; 5750 int err; 5751 5752 err = make_parent_dir(path); 5753 if (err) 5754 return err; 5755 5756 err = check_path(path); 5757 if (err) 5758 return err; 5759 5760 if (prog == NULL) { 5761 pr_warn("invalid program pointer\n"); 5762 return -EINVAL; 5763 } 5764 5765 if (instance < 0 || instance >= prog->instances.nr) { 5766 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 5767 instance, prog->section_name, prog->instances.nr); 5768 return -EINVAL; 5769 } 5770 5771 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 5772 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 5773 pr_warn("failed to pin program: %s\n", cp); 5774 return -errno; 5775 } 5776 pr_debug("pinned program '%s'\n", path); 5777 5778 return 0; 5779 } 5780 5781 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 5782 int instance) 5783 { 5784 int err; 5785 5786 err = check_path(path); 5787 if (err) 5788 return err; 5789 5790 if (prog == NULL) { 5791 pr_warn("invalid program pointer\n"); 5792 return -EINVAL; 5793 } 5794 5795 if (instance < 0 || instance >= prog->instances.nr) { 5796 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 5797 instance, prog->section_name, prog->instances.nr); 5798 return -EINVAL; 5799 } 5800 5801 err = unlink(path); 5802 if (err != 0) 5803 return -errno; 5804 pr_debug("unpinned program '%s'\n", path); 5805 5806 return 0; 5807 } 5808 5809 int bpf_program__pin(struct bpf_program *prog, const char *path) 5810 { 5811 int i, err; 5812 5813 err = make_parent_dir(path); 5814 if (err) 5815 return err; 5816 5817 err = check_path(path); 5818 if (err) 5819 return err; 5820 5821 if (prog == NULL) { 5822 pr_warn("invalid program pointer\n"); 5823 return -EINVAL; 5824 } 5825 5826 if (prog->instances.nr <= 0) { 5827 pr_warn("no instances of prog %s to pin\n", 5828 prog->section_name); 5829 return -EINVAL; 5830 } 5831 5832 if (prog->instances.nr == 1) { 5833 /* don't create subdirs when pinning single instance */ 5834 return bpf_program__pin_instance(prog, path, 0); 5835 } 5836 5837 for (i = 0; i < prog->instances.nr; i++) { 5838 char buf[PATH_MAX]; 5839 int len; 5840 5841 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 5842 if (len < 0) { 5843 err = -EINVAL; 5844 goto err_unpin; 5845 } else if (len >= PATH_MAX) { 5846 err = -ENAMETOOLONG; 5847 goto err_unpin; 5848 } 5849 5850 err = bpf_program__pin_instance(prog, buf, i); 5851 if (err) 5852 goto err_unpin; 5853 } 5854 5855 return 0; 5856 5857 err_unpin: 5858 for (i = i - 1; i >= 0; i--) { 5859 char buf[PATH_MAX]; 5860 int len; 5861 5862 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 5863 if (len < 0) 5864 continue; 5865 else if (len >= PATH_MAX) 5866 continue; 5867 5868 bpf_program__unpin_instance(prog, buf, i); 5869 } 5870 5871 rmdir(path); 5872 5873 return err; 5874 } 5875 5876 int bpf_program__unpin(struct bpf_program *prog, const char *path) 5877 { 5878 int i, err; 5879 5880 err = check_path(path); 5881 if (err) 5882 return err; 5883 5884 if (prog == NULL) { 5885 pr_warn("invalid program pointer\n"); 5886 return -EINVAL; 5887 } 5888 5889 if (prog->instances.nr <= 0) { 5890 pr_warn("no instances of prog %s to pin\n", 5891 prog->section_name); 5892 return -EINVAL; 5893 } 5894 5895 if (prog->instances.nr == 1) { 5896 /* don't create subdirs when pinning single instance */ 5897 return bpf_program__unpin_instance(prog, path, 0); 5898 } 5899 5900 for (i = 0; i < prog->instances.nr; i++) { 5901 char buf[PATH_MAX]; 5902 int len; 5903 5904 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 5905 if (len < 0) 5906 return -EINVAL; 5907 else if (len >= PATH_MAX) 5908 return -ENAMETOOLONG; 5909 5910 err = bpf_program__unpin_instance(prog, buf, i); 5911 if (err) 5912 return err; 5913 } 5914 5915 err = rmdir(path); 5916 if (err) 5917 return -errno; 5918 5919 return 0; 5920 } 5921 5922 int bpf_map__pin(struct bpf_map *map, const char *path) 5923 { 5924 char *cp, errmsg[STRERR_BUFSIZE]; 5925 int err; 5926 5927 if (map == NULL) { 5928 pr_warn("invalid map pointer\n"); 5929 return -EINVAL; 5930 } 5931 5932 if (map->pin_path) { 5933 if (path && strcmp(path, map->pin_path)) { 5934 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 5935 bpf_map__name(map), map->pin_path, path); 5936 return -EINVAL; 5937 } else if (map->pinned) { 5938 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 5939 bpf_map__name(map), map->pin_path); 5940 return 0; 5941 } 5942 } else { 5943 if (!path) { 5944 pr_warn("missing a path to pin map '%s' at\n", 5945 bpf_map__name(map)); 5946 return -EINVAL; 5947 } else if (map->pinned) { 5948 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 5949 return -EEXIST; 5950 } 5951 5952 map->pin_path = strdup(path); 5953 if (!map->pin_path) { 5954 err = -errno; 5955 goto out_err; 5956 } 5957 } 5958 5959 err = make_parent_dir(map->pin_path); 5960 if (err) 5961 return err; 5962 5963 err = check_path(map->pin_path); 5964 if (err) 5965 return err; 5966 5967 if (bpf_obj_pin(map->fd, map->pin_path)) { 5968 err = -errno; 5969 goto out_err; 5970 } 5971 5972 map->pinned = true; 5973 pr_debug("pinned map '%s'\n", map->pin_path); 5974 5975 return 0; 5976 5977 out_err: 5978 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 5979 pr_warn("failed to pin map: %s\n", cp); 5980 return err; 5981 } 5982 5983 int bpf_map__unpin(struct bpf_map *map, const char *path) 5984 { 5985 int err; 5986 5987 if (map == NULL) { 5988 pr_warn("invalid map pointer\n"); 5989 return -EINVAL; 5990 } 5991 5992 if (map->pin_path) { 5993 if (path && strcmp(path, map->pin_path)) { 5994 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 5995 bpf_map__name(map), map->pin_path, path); 5996 return -EINVAL; 5997 } 5998 path = map->pin_path; 5999 } else if (!path) { 6000 pr_warn("no path to unpin map '%s' from\n", 6001 bpf_map__name(map)); 6002 return -EINVAL; 6003 } 6004 6005 err = check_path(path); 6006 if (err) 6007 return err; 6008 6009 err = unlink(path); 6010 if (err != 0) 6011 return -errno; 6012 6013 map->pinned = false; 6014 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 6015 6016 return 0; 6017 } 6018 6019 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 6020 { 6021 char *new = NULL; 6022 6023 if (path) { 6024 new = strdup(path); 6025 if (!new) 6026 return -errno; 6027 } 6028 6029 free(map->pin_path); 6030 map->pin_path = new; 6031 return 0; 6032 } 6033 6034 const char *bpf_map__get_pin_path(const struct bpf_map *map) 6035 { 6036 return map->pin_path; 6037 } 6038 6039 bool bpf_map__is_pinned(const struct bpf_map *map) 6040 { 6041 return map->pinned; 6042 } 6043 6044 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 6045 { 6046 struct bpf_map *map; 6047 int err; 6048 6049 if (!obj) 6050 return -ENOENT; 6051 6052 if (!obj->loaded) { 6053 pr_warn("object not yet loaded; load it first\n"); 6054 return -ENOENT; 6055 } 6056 6057 bpf_object__for_each_map(map, obj) { 6058 char *pin_path = NULL; 6059 char buf[PATH_MAX]; 6060 6061 if (path) { 6062 int len; 6063 6064 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6065 bpf_map__name(map)); 6066 if (len < 0) { 6067 err = -EINVAL; 6068 goto err_unpin_maps; 6069 } else if (len >= PATH_MAX) { 6070 err = -ENAMETOOLONG; 6071 goto err_unpin_maps; 6072 } 6073 pin_path = buf; 6074 } else if (!map->pin_path) { 6075 continue; 6076 } 6077 6078 err = bpf_map__pin(map, pin_path); 6079 if (err) 6080 goto err_unpin_maps; 6081 } 6082 6083 return 0; 6084 6085 err_unpin_maps: 6086 while ((map = bpf_map__prev(map, obj))) { 6087 if (!map->pin_path) 6088 continue; 6089 6090 bpf_map__unpin(map, NULL); 6091 } 6092 6093 return err; 6094 } 6095 6096 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 6097 { 6098 struct bpf_map *map; 6099 int err; 6100 6101 if (!obj) 6102 return -ENOENT; 6103 6104 bpf_object__for_each_map(map, obj) { 6105 char *pin_path = NULL; 6106 char buf[PATH_MAX]; 6107 6108 if (path) { 6109 int len; 6110 6111 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6112 bpf_map__name(map)); 6113 if (len < 0) 6114 return -EINVAL; 6115 else if (len >= PATH_MAX) 6116 return -ENAMETOOLONG; 6117 pin_path = buf; 6118 } else if (!map->pin_path) { 6119 continue; 6120 } 6121 6122 err = bpf_map__unpin(map, pin_path); 6123 if (err) 6124 return err; 6125 } 6126 6127 return 0; 6128 } 6129 6130 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 6131 { 6132 struct bpf_program *prog; 6133 int err; 6134 6135 if (!obj) 6136 return -ENOENT; 6137 6138 if (!obj->loaded) { 6139 pr_warn("object not yet loaded; load it first\n"); 6140 return -ENOENT; 6141 } 6142 6143 bpf_object__for_each_program(prog, obj) { 6144 char buf[PATH_MAX]; 6145 int len; 6146 6147 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6148 prog->pin_name); 6149 if (len < 0) { 6150 err = -EINVAL; 6151 goto err_unpin_programs; 6152 } else if (len >= PATH_MAX) { 6153 err = -ENAMETOOLONG; 6154 goto err_unpin_programs; 6155 } 6156 6157 err = bpf_program__pin(prog, buf); 6158 if (err) 6159 goto err_unpin_programs; 6160 } 6161 6162 return 0; 6163 6164 err_unpin_programs: 6165 while ((prog = bpf_program__prev(prog, obj))) { 6166 char buf[PATH_MAX]; 6167 int len; 6168 6169 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6170 prog->pin_name); 6171 if (len < 0) 6172 continue; 6173 else if (len >= PATH_MAX) 6174 continue; 6175 6176 bpf_program__unpin(prog, buf); 6177 } 6178 6179 return err; 6180 } 6181 6182 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 6183 { 6184 struct bpf_program *prog; 6185 int err; 6186 6187 if (!obj) 6188 return -ENOENT; 6189 6190 bpf_object__for_each_program(prog, obj) { 6191 char buf[PATH_MAX]; 6192 int len; 6193 6194 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6195 prog->pin_name); 6196 if (len < 0) 6197 return -EINVAL; 6198 else if (len >= PATH_MAX) 6199 return -ENAMETOOLONG; 6200 6201 err = bpf_program__unpin(prog, buf); 6202 if (err) 6203 return err; 6204 } 6205 6206 return 0; 6207 } 6208 6209 int bpf_object__pin(struct bpf_object *obj, const char *path) 6210 { 6211 int err; 6212 6213 err = bpf_object__pin_maps(obj, path); 6214 if (err) 6215 return err; 6216 6217 err = bpf_object__pin_programs(obj, path); 6218 if (err) { 6219 bpf_object__unpin_maps(obj, path); 6220 return err; 6221 } 6222 6223 return 0; 6224 } 6225 6226 static void bpf_map__destroy(struct bpf_map *map) 6227 { 6228 if (map->clear_priv) 6229 map->clear_priv(map, map->priv); 6230 map->priv = NULL; 6231 map->clear_priv = NULL; 6232 6233 if (map->inner_map) { 6234 bpf_map__destroy(map->inner_map); 6235 zfree(&map->inner_map); 6236 } 6237 6238 zfree(&map->init_slots); 6239 map->init_slots_sz = 0; 6240 6241 if (map->mmaped) { 6242 munmap(map->mmaped, bpf_map_mmap_sz(map)); 6243 map->mmaped = NULL; 6244 } 6245 6246 if (map->st_ops) { 6247 zfree(&map->st_ops->data); 6248 zfree(&map->st_ops->progs); 6249 zfree(&map->st_ops->kern_func_off); 6250 zfree(&map->st_ops); 6251 } 6252 6253 zfree(&map->name); 6254 zfree(&map->pin_path); 6255 6256 if (map->fd >= 0) 6257 zclose(map->fd); 6258 } 6259 6260 void bpf_object__close(struct bpf_object *obj) 6261 { 6262 size_t i; 6263 6264 if (!obj) 6265 return; 6266 6267 if (obj->clear_priv) 6268 obj->clear_priv(obj, obj->priv); 6269 6270 bpf_object__elf_finish(obj); 6271 bpf_object__unload(obj); 6272 btf__free(obj->btf); 6273 btf_ext__free(obj->btf_ext); 6274 6275 for (i = 0; i < obj->nr_maps; i++) 6276 bpf_map__destroy(&obj->maps[i]); 6277 6278 zfree(&obj->kconfig); 6279 zfree(&obj->externs); 6280 obj->nr_extern = 0; 6281 6282 zfree(&obj->maps); 6283 obj->nr_maps = 0; 6284 6285 if (obj->programs && obj->nr_programs) { 6286 for (i = 0; i < obj->nr_programs; i++) 6287 bpf_program__exit(&obj->programs[i]); 6288 } 6289 zfree(&obj->programs); 6290 6291 list_del(&obj->list); 6292 free(obj); 6293 } 6294 6295 struct bpf_object * 6296 bpf_object__next(struct bpf_object *prev) 6297 { 6298 struct bpf_object *next; 6299 6300 if (!prev) 6301 next = list_first_entry(&bpf_objects_list, 6302 struct bpf_object, 6303 list); 6304 else 6305 next = list_next_entry(prev, list); 6306 6307 /* Empty list is noticed here so don't need checking on entry. */ 6308 if (&next->list == &bpf_objects_list) 6309 return NULL; 6310 6311 return next; 6312 } 6313 6314 const char *bpf_object__name(const struct bpf_object *obj) 6315 { 6316 return obj ? obj->name : ERR_PTR(-EINVAL); 6317 } 6318 6319 unsigned int bpf_object__kversion(const struct bpf_object *obj) 6320 { 6321 return obj ? obj->kern_version : 0; 6322 } 6323 6324 struct btf *bpf_object__btf(const struct bpf_object *obj) 6325 { 6326 return obj ? obj->btf : NULL; 6327 } 6328 6329 int bpf_object__btf_fd(const struct bpf_object *obj) 6330 { 6331 return obj->btf ? btf__fd(obj->btf) : -1; 6332 } 6333 6334 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 6335 bpf_object_clear_priv_t clear_priv) 6336 { 6337 if (obj->priv && obj->clear_priv) 6338 obj->clear_priv(obj, obj->priv); 6339 6340 obj->priv = priv; 6341 obj->clear_priv = clear_priv; 6342 return 0; 6343 } 6344 6345 void *bpf_object__priv(const struct bpf_object *obj) 6346 { 6347 return obj ? obj->priv : ERR_PTR(-EINVAL); 6348 } 6349 6350 static struct bpf_program * 6351 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 6352 bool forward) 6353 { 6354 size_t nr_programs = obj->nr_programs; 6355 ssize_t idx; 6356 6357 if (!nr_programs) 6358 return NULL; 6359 6360 if (!p) 6361 /* Iter from the beginning */ 6362 return forward ? &obj->programs[0] : 6363 &obj->programs[nr_programs - 1]; 6364 6365 if (p->obj != obj) { 6366 pr_warn("error: program handler doesn't match object\n"); 6367 return NULL; 6368 } 6369 6370 idx = (p - obj->programs) + (forward ? 1 : -1); 6371 if (idx >= obj->nr_programs || idx < 0) 6372 return NULL; 6373 return &obj->programs[idx]; 6374 } 6375 6376 struct bpf_program * 6377 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 6378 { 6379 struct bpf_program *prog = prev; 6380 6381 do { 6382 prog = __bpf_program__iter(prog, obj, true); 6383 } while (prog && bpf_program__is_function_storage(prog, obj)); 6384 6385 return prog; 6386 } 6387 6388 struct bpf_program * 6389 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 6390 { 6391 struct bpf_program *prog = next; 6392 6393 do { 6394 prog = __bpf_program__iter(prog, obj, false); 6395 } while (prog && bpf_program__is_function_storage(prog, obj)); 6396 6397 return prog; 6398 } 6399 6400 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 6401 bpf_program_clear_priv_t clear_priv) 6402 { 6403 if (prog->priv && prog->clear_priv) 6404 prog->clear_priv(prog, prog->priv); 6405 6406 prog->priv = priv; 6407 prog->clear_priv = clear_priv; 6408 return 0; 6409 } 6410 6411 void *bpf_program__priv(const struct bpf_program *prog) 6412 { 6413 return prog ? prog->priv : ERR_PTR(-EINVAL); 6414 } 6415 6416 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 6417 { 6418 prog->prog_ifindex = ifindex; 6419 } 6420 6421 const char *bpf_program__name(const struct bpf_program *prog) 6422 { 6423 return prog->name; 6424 } 6425 6426 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 6427 { 6428 const char *title; 6429 6430 title = prog->section_name; 6431 if (needs_copy) { 6432 title = strdup(title); 6433 if (!title) { 6434 pr_warn("failed to strdup program title\n"); 6435 return ERR_PTR(-ENOMEM); 6436 } 6437 } 6438 6439 return title; 6440 } 6441 6442 int bpf_program__fd(const struct bpf_program *prog) 6443 { 6444 return bpf_program__nth_fd(prog, 0); 6445 } 6446 6447 size_t bpf_program__size(const struct bpf_program *prog) 6448 { 6449 return prog->insns_cnt * sizeof(struct bpf_insn); 6450 } 6451 6452 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 6453 bpf_program_prep_t prep) 6454 { 6455 int *instances_fds; 6456 6457 if (nr_instances <= 0 || !prep) 6458 return -EINVAL; 6459 6460 if (prog->instances.nr > 0 || prog->instances.fds) { 6461 pr_warn("Can't set pre-processor after loading\n"); 6462 return -EINVAL; 6463 } 6464 6465 instances_fds = malloc(sizeof(int) * nr_instances); 6466 if (!instances_fds) { 6467 pr_warn("alloc memory failed for fds\n"); 6468 return -ENOMEM; 6469 } 6470 6471 /* fill all fd with -1 */ 6472 memset(instances_fds, -1, sizeof(int) * nr_instances); 6473 6474 prog->instances.nr = nr_instances; 6475 prog->instances.fds = instances_fds; 6476 prog->preprocessor = prep; 6477 return 0; 6478 } 6479 6480 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 6481 { 6482 int fd; 6483 6484 if (!prog) 6485 return -EINVAL; 6486 6487 if (n >= prog->instances.nr || n < 0) { 6488 pr_warn("Can't get the %dth fd from program %s: only %d instances\n", 6489 n, prog->section_name, prog->instances.nr); 6490 return -EINVAL; 6491 } 6492 6493 fd = prog->instances.fds[n]; 6494 if (fd < 0) { 6495 pr_warn("%dth instance of program '%s' is invalid\n", 6496 n, prog->section_name); 6497 return -ENOENT; 6498 } 6499 6500 return fd; 6501 } 6502 6503 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog) 6504 { 6505 return prog->type; 6506 } 6507 6508 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 6509 { 6510 prog->type = type; 6511 } 6512 6513 static bool bpf_program__is_type(const struct bpf_program *prog, 6514 enum bpf_prog_type type) 6515 { 6516 return prog ? (prog->type == type) : false; 6517 } 6518 6519 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 6520 int bpf_program__set_##NAME(struct bpf_program *prog) \ 6521 { \ 6522 if (!prog) \ 6523 return -EINVAL; \ 6524 bpf_program__set_type(prog, TYPE); \ 6525 return 0; \ 6526 } \ 6527 \ 6528 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 6529 { \ 6530 return bpf_program__is_type(prog, TYPE); \ 6531 } \ 6532 6533 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 6534 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); 6535 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 6536 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 6537 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 6538 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 6539 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 6540 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 6541 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 6542 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); 6543 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); 6544 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); 6545 6546 enum bpf_attach_type 6547 bpf_program__get_expected_attach_type(struct bpf_program *prog) 6548 { 6549 return prog->expected_attach_type; 6550 } 6551 6552 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 6553 enum bpf_attach_type type) 6554 { 6555 prog->expected_attach_type = type; 6556 } 6557 6558 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \ 6559 attachable, attach_btf) \ 6560 { \ 6561 .sec = string, \ 6562 .len = sizeof(string) - 1, \ 6563 .prog_type = ptype, \ 6564 .expected_attach_type = eatype, \ 6565 .is_exp_attach_type_optional = eatype_optional, \ 6566 .is_attachable = attachable, \ 6567 .is_attach_btf = attach_btf, \ 6568 } 6569 6570 /* Programs that can NOT be attached. */ 6571 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) 6572 6573 /* Programs that can be attached. */ 6574 #define BPF_APROG_SEC(string, ptype, atype) \ 6575 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0) 6576 6577 /* Programs that must specify expected attach type at load time. */ 6578 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 6579 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0) 6580 6581 /* Programs that use BTF to identify attach point */ 6582 #define BPF_PROG_BTF(string, ptype, eatype) \ 6583 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1) 6584 6585 /* Programs that can be attached but attach type can't be identified by section 6586 * name. Kept for backward compatibility. 6587 */ 6588 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 6589 6590 #define SEC_DEF(sec_pfx, ptype, ...) { \ 6591 .sec = sec_pfx, \ 6592 .len = sizeof(sec_pfx) - 1, \ 6593 .prog_type = BPF_PROG_TYPE_##ptype, \ 6594 __VA_ARGS__ \ 6595 } 6596 6597 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 6598 struct bpf_program *prog); 6599 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 6600 struct bpf_program *prog); 6601 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 6602 struct bpf_program *prog); 6603 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 6604 struct bpf_program *prog); 6605 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 6606 struct bpf_program *prog); 6607 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 6608 struct bpf_program *prog); 6609 6610 static const struct bpf_sec_def section_defs[] = { 6611 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 6612 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT), 6613 SEC_DEF("kprobe/", KPROBE, 6614 .attach_fn = attach_kprobe), 6615 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), 6616 SEC_DEF("kretprobe/", KPROBE, 6617 .attach_fn = attach_kprobe), 6618 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), 6619 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 6620 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 6621 SEC_DEF("tracepoint/", TRACEPOINT, 6622 .attach_fn = attach_tp), 6623 SEC_DEF("tp/", TRACEPOINT, 6624 .attach_fn = attach_tp), 6625 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 6626 .attach_fn = attach_raw_tp), 6627 SEC_DEF("raw_tp/", RAW_TRACEPOINT, 6628 .attach_fn = attach_raw_tp), 6629 SEC_DEF("tp_btf/", TRACING, 6630 .expected_attach_type = BPF_TRACE_RAW_TP, 6631 .is_attach_btf = true, 6632 .attach_fn = attach_trace), 6633 SEC_DEF("fentry/", TRACING, 6634 .expected_attach_type = BPF_TRACE_FENTRY, 6635 .is_attach_btf = true, 6636 .attach_fn = attach_trace), 6637 SEC_DEF("fmod_ret/", TRACING, 6638 .expected_attach_type = BPF_MODIFY_RETURN, 6639 .is_attach_btf = true, 6640 .attach_fn = attach_trace), 6641 SEC_DEF("fexit/", TRACING, 6642 .expected_attach_type = BPF_TRACE_FEXIT, 6643 .is_attach_btf = true, 6644 .attach_fn = attach_trace), 6645 SEC_DEF("freplace/", EXT, 6646 .is_attach_btf = true, 6647 .attach_fn = attach_trace), 6648 SEC_DEF("lsm/", LSM, 6649 .is_attach_btf = true, 6650 .expected_attach_type = BPF_LSM_MAC, 6651 .attach_fn = attach_lsm), 6652 SEC_DEF("iter/", TRACING, 6653 .expected_attach_type = BPF_TRACE_ITER, 6654 .is_attach_btf = true, 6655 .attach_fn = attach_iter), 6656 BPF_EAPROG_SEC("xdp_devmap", BPF_PROG_TYPE_XDP, 6657 BPF_XDP_DEVMAP), 6658 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 6659 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 6660 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 6661 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 6662 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 6663 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 6664 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 6665 BPF_CGROUP_INET_INGRESS), 6666 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 6667 BPF_CGROUP_INET_EGRESS), 6668 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 6669 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 6670 BPF_CGROUP_INET_SOCK_CREATE), 6671 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 6672 BPF_CGROUP_INET4_POST_BIND), 6673 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 6674 BPF_CGROUP_INET6_POST_BIND), 6675 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 6676 BPF_CGROUP_DEVICE), 6677 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 6678 BPF_CGROUP_SOCK_OPS), 6679 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 6680 BPF_SK_SKB_STREAM_PARSER), 6681 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 6682 BPF_SK_SKB_STREAM_VERDICT), 6683 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 6684 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 6685 BPF_SK_MSG_VERDICT), 6686 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 6687 BPF_LIRC_MODE2), 6688 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 6689 BPF_FLOW_DISSECTOR), 6690 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6691 BPF_CGROUP_INET4_BIND), 6692 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6693 BPF_CGROUP_INET6_BIND), 6694 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6695 BPF_CGROUP_INET4_CONNECT), 6696 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6697 BPF_CGROUP_INET6_CONNECT), 6698 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6699 BPF_CGROUP_UDP4_SENDMSG), 6700 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6701 BPF_CGROUP_UDP6_SENDMSG), 6702 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6703 BPF_CGROUP_UDP4_RECVMSG), 6704 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6705 BPF_CGROUP_UDP6_RECVMSG), 6706 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6707 BPF_CGROUP_INET4_GETPEERNAME), 6708 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6709 BPF_CGROUP_INET6_GETPEERNAME), 6710 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6711 BPF_CGROUP_INET4_GETSOCKNAME), 6712 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6713 BPF_CGROUP_INET6_GETSOCKNAME), 6714 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 6715 BPF_CGROUP_SYSCTL), 6716 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 6717 BPF_CGROUP_GETSOCKOPT), 6718 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 6719 BPF_CGROUP_SETSOCKOPT), 6720 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), 6721 }; 6722 6723 #undef BPF_PROG_SEC_IMPL 6724 #undef BPF_PROG_SEC 6725 #undef BPF_APROG_SEC 6726 #undef BPF_EAPROG_SEC 6727 #undef BPF_APROG_COMPAT 6728 #undef SEC_DEF 6729 6730 #define MAX_TYPE_NAME_SIZE 32 6731 6732 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 6733 { 6734 int i, n = ARRAY_SIZE(section_defs); 6735 6736 for (i = 0; i < n; i++) { 6737 if (strncmp(sec_name, 6738 section_defs[i].sec, section_defs[i].len)) 6739 continue; 6740 return §ion_defs[i]; 6741 } 6742 return NULL; 6743 } 6744 6745 static char *libbpf_get_type_names(bool attach_type) 6746 { 6747 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 6748 char *buf; 6749 6750 buf = malloc(len); 6751 if (!buf) 6752 return NULL; 6753 6754 buf[0] = '\0'; 6755 /* Forge string buf with all available names */ 6756 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 6757 if (attach_type && !section_defs[i].is_attachable) 6758 continue; 6759 6760 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 6761 free(buf); 6762 return NULL; 6763 } 6764 strcat(buf, " "); 6765 strcat(buf, section_defs[i].sec); 6766 } 6767 6768 return buf; 6769 } 6770 6771 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 6772 enum bpf_attach_type *expected_attach_type) 6773 { 6774 const struct bpf_sec_def *sec_def; 6775 char *type_names; 6776 6777 if (!name) 6778 return -EINVAL; 6779 6780 sec_def = find_sec_def(name); 6781 if (sec_def) { 6782 *prog_type = sec_def->prog_type; 6783 *expected_attach_type = sec_def->expected_attach_type; 6784 return 0; 6785 } 6786 6787 pr_debug("failed to guess program type from ELF section '%s'\n", name); 6788 type_names = libbpf_get_type_names(false); 6789 if (type_names != NULL) { 6790 pr_debug("supported section(type) names are:%s\n", type_names); 6791 free(type_names); 6792 } 6793 6794 return -ESRCH; 6795 } 6796 6797 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 6798 size_t offset) 6799 { 6800 struct bpf_map *map; 6801 size_t i; 6802 6803 for (i = 0; i < obj->nr_maps; i++) { 6804 map = &obj->maps[i]; 6805 if (!bpf_map__is_struct_ops(map)) 6806 continue; 6807 if (map->sec_offset <= offset && 6808 offset - map->sec_offset < map->def.value_size) 6809 return map; 6810 } 6811 6812 return NULL; 6813 } 6814 6815 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 6816 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 6817 GElf_Shdr *shdr, Elf_Data *data) 6818 { 6819 const struct btf_member *member; 6820 struct bpf_struct_ops *st_ops; 6821 struct bpf_program *prog; 6822 unsigned int shdr_idx; 6823 const struct btf *btf; 6824 struct bpf_map *map; 6825 Elf_Data *symbols; 6826 unsigned int moff; 6827 const char *name; 6828 __u32 member_idx; 6829 GElf_Sym sym; 6830 GElf_Rel rel; 6831 int i, nrels; 6832 6833 symbols = obj->efile.symbols; 6834 btf = obj->btf; 6835 nrels = shdr->sh_size / shdr->sh_entsize; 6836 for (i = 0; i < nrels; i++) { 6837 if (!gelf_getrel(data, i, &rel)) { 6838 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 6839 return -LIBBPF_ERRNO__FORMAT; 6840 } 6841 6842 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 6843 pr_warn("struct_ops reloc: symbol %zx not found\n", 6844 (size_t)GELF_R_SYM(rel.r_info)); 6845 return -LIBBPF_ERRNO__FORMAT; 6846 } 6847 6848 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 6849 sym.st_name) ? : "<?>"; 6850 map = find_struct_ops_map_by_offset(obj, rel.r_offset); 6851 if (!map) { 6852 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", 6853 (size_t)rel.r_offset); 6854 return -EINVAL; 6855 } 6856 6857 moff = rel.r_offset - map->sec_offset; 6858 shdr_idx = sym.st_shndx; 6859 st_ops = map->st_ops; 6860 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 6861 map->name, 6862 (long long)(rel.r_info >> 32), 6863 (long long)sym.st_value, 6864 shdr_idx, (size_t)rel.r_offset, 6865 map->sec_offset, sym.st_name, name); 6866 6867 if (shdr_idx >= SHN_LORESERVE) { 6868 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", 6869 map->name, (size_t)rel.r_offset, shdr_idx); 6870 return -LIBBPF_ERRNO__RELOC; 6871 } 6872 6873 member = find_member_by_offset(st_ops->type, moff * 8); 6874 if (!member) { 6875 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 6876 map->name, moff); 6877 return -EINVAL; 6878 } 6879 member_idx = member - btf_members(st_ops->type); 6880 name = btf__name_by_offset(btf, member->name_off); 6881 6882 if (!resolve_func_ptr(btf, member->type, NULL)) { 6883 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 6884 map->name, name); 6885 return -EINVAL; 6886 } 6887 6888 prog = bpf_object__find_prog_by_idx(obj, shdr_idx); 6889 if (!prog) { 6890 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 6891 map->name, shdr_idx, name); 6892 return -EINVAL; 6893 } 6894 6895 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 6896 const struct bpf_sec_def *sec_def; 6897 6898 sec_def = find_sec_def(prog->section_name); 6899 if (sec_def && 6900 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) { 6901 /* for pr_warn */ 6902 prog->type = sec_def->prog_type; 6903 goto invalid_prog; 6904 } 6905 6906 prog->type = BPF_PROG_TYPE_STRUCT_OPS; 6907 prog->attach_btf_id = st_ops->type_id; 6908 prog->expected_attach_type = member_idx; 6909 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || 6910 prog->attach_btf_id != st_ops->type_id || 6911 prog->expected_attach_type != member_idx) { 6912 goto invalid_prog; 6913 } 6914 st_ops->progs[member_idx] = prog; 6915 } 6916 6917 return 0; 6918 6919 invalid_prog: 6920 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 6921 map->name, prog->name, prog->section_name, prog->type, 6922 prog->attach_btf_id, prog->expected_attach_type, name); 6923 return -EINVAL; 6924 } 6925 6926 #define BTF_TRACE_PREFIX "btf_trace_" 6927 #define BTF_LSM_PREFIX "bpf_lsm_" 6928 #define BTF_ITER_PREFIX "bpf_iter_" 6929 #define BTF_MAX_NAME_SIZE 128 6930 6931 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 6932 const char *name, __u32 kind) 6933 { 6934 char btf_type_name[BTF_MAX_NAME_SIZE]; 6935 int ret; 6936 6937 ret = snprintf(btf_type_name, sizeof(btf_type_name), 6938 "%s%s", prefix, name); 6939 /* snprintf returns the number of characters written excluding the 6940 * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 6941 * indicates truncation. 6942 */ 6943 if (ret < 0 || ret >= sizeof(btf_type_name)) 6944 return -ENAMETOOLONG; 6945 return btf__find_by_name_kind(btf, btf_type_name, kind); 6946 } 6947 6948 static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name, 6949 enum bpf_attach_type attach_type) 6950 { 6951 int err; 6952 6953 if (attach_type == BPF_TRACE_RAW_TP) 6954 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name, 6955 BTF_KIND_TYPEDEF); 6956 else if (attach_type == BPF_LSM_MAC) 6957 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name, 6958 BTF_KIND_FUNC); 6959 else if (attach_type == BPF_TRACE_ITER) 6960 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name, 6961 BTF_KIND_FUNC); 6962 else 6963 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 6964 6965 if (err <= 0) 6966 pr_warn("%s is not found in vmlinux BTF\n", name); 6967 6968 return err; 6969 } 6970 6971 int libbpf_find_vmlinux_btf_id(const char *name, 6972 enum bpf_attach_type attach_type) 6973 { 6974 struct btf *btf; 6975 int err; 6976 6977 btf = libbpf_find_kernel_btf(); 6978 if (IS_ERR(btf)) { 6979 pr_warn("vmlinux BTF is not found\n"); 6980 return -EINVAL; 6981 } 6982 6983 err = __find_vmlinux_btf_id(btf, name, attach_type); 6984 btf__free(btf); 6985 return err; 6986 } 6987 6988 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 6989 { 6990 struct bpf_prog_info_linear *info_linear; 6991 struct bpf_prog_info *info; 6992 struct btf *btf = NULL; 6993 int err = -EINVAL; 6994 6995 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); 6996 if (IS_ERR_OR_NULL(info_linear)) { 6997 pr_warn("failed get_prog_info_linear for FD %d\n", 6998 attach_prog_fd); 6999 return -EINVAL; 7000 } 7001 info = &info_linear->info; 7002 if (!info->btf_id) { 7003 pr_warn("The target program doesn't have BTF\n"); 7004 goto out; 7005 } 7006 if (btf__get_from_id(info->btf_id, &btf)) { 7007 pr_warn("Failed to get BTF of the program\n"); 7008 goto out; 7009 } 7010 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 7011 btf__free(btf); 7012 if (err <= 0) { 7013 pr_warn("%s is not found in prog's BTF\n", name); 7014 goto out; 7015 } 7016 out: 7017 free(info_linear); 7018 return err; 7019 } 7020 7021 static int libbpf_find_attach_btf_id(struct bpf_program *prog) 7022 { 7023 enum bpf_attach_type attach_type = prog->expected_attach_type; 7024 __u32 attach_prog_fd = prog->attach_prog_fd; 7025 const char *name = prog->section_name; 7026 int i, err; 7027 7028 if (!name) 7029 return -EINVAL; 7030 7031 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 7032 if (!section_defs[i].is_attach_btf) 7033 continue; 7034 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 7035 continue; 7036 if (attach_prog_fd) 7037 err = libbpf_find_prog_btf_id(name + section_defs[i].len, 7038 attach_prog_fd); 7039 else 7040 err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux, 7041 name + section_defs[i].len, 7042 attach_type); 7043 return err; 7044 } 7045 pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name); 7046 return -ESRCH; 7047 } 7048 7049 int libbpf_attach_type_by_name(const char *name, 7050 enum bpf_attach_type *attach_type) 7051 { 7052 char *type_names; 7053 int i; 7054 7055 if (!name) 7056 return -EINVAL; 7057 7058 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 7059 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 7060 continue; 7061 if (!section_defs[i].is_attachable) 7062 return -EINVAL; 7063 *attach_type = section_defs[i].expected_attach_type; 7064 return 0; 7065 } 7066 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 7067 type_names = libbpf_get_type_names(true); 7068 if (type_names != NULL) { 7069 pr_debug("attachable section(type) names are:%s\n", type_names); 7070 free(type_names); 7071 } 7072 7073 return -EINVAL; 7074 } 7075 7076 int bpf_map__fd(const struct bpf_map *map) 7077 { 7078 return map ? map->fd : -EINVAL; 7079 } 7080 7081 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 7082 { 7083 return map ? &map->def : ERR_PTR(-EINVAL); 7084 } 7085 7086 const char *bpf_map__name(const struct bpf_map *map) 7087 { 7088 return map ? map->name : NULL; 7089 } 7090 7091 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 7092 { 7093 return map ? map->btf_key_type_id : 0; 7094 } 7095 7096 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 7097 { 7098 return map ? map->btf_value_type_id : 0; 7099 } 7100 7101 int bpf_map__set_priv(struct bpf_map *map, void *priv, 7102 bpf_map_clear_priv_t clear_priv) 7103 { 7104 if (!map) 7105 return -EINVAL; 7106 7107 if (map->priv) { 7108 if (map->clear_priv) 7109 map->clear_priv(map, map->priv); 7110 } 7111 7112 map->priv = priv; 7113 map->clear_priv = clear_priv; 7114 return 0; 7115 } 7116 7117 void *bpf_map__priv(const struct bpf_map *map) 7118 { 7119 return map ? map->priv : ERR_PTR(-EINVAL); 7120 } 7121 7122 int bpf_map__set_initial_value(struct bpf_map *map, 7123 const void *data, size_t size) 7124 { 7125 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 7126 size != map->def.value_size || map->fd >= 0) 7127 return -EINVAL; 7128 7129 memcpy(map->mmaped, data, size); 7130 return 0; 7131 } 7132 7133 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 7134 { 7135 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 7136 } 7137 7138 bool bpf_map__is_internal(const struct bpf_map *map) 7139 { 7140 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 7141 } 7142 7143 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 7144 { 7145 map->map_ifindex = ifindex; 7146 } 7147 7148 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 7149 { 7150 if (!bpf_map_type__is_map_in_map(map->def.type)) { 7151 pr_warn("error: unsupported map type\n"); 7152 return -EINVAL; 7153 } 7154 if (map->inner_map_fd != -1) { 7155 pr_warn("error: inner_map_fd already specified\n"); 7156 return -EINVAL; 7157 } 7158 map->inner_map_fd = fd; 7159 return 0; 7160 } 7161 7162 static struct bpf_map * 7163 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 7164 { 7165 ssize_t idx; 7166 struct bpf_map *s, *e; 7167 7168 if (!obj || !obj->maps) 7169 return NULL; 7170 7171 s = obj->maps; 7172 e = obj->maps + obj->nr_maps; 7173 7174 if ((m < s) || (m >= e)) { 7175 pr_warn("error in %s: map handler doesn't belong to object\n", 7176 __func__); 7177 return NULL; 7178 } 7179 7180 idx = (m - obj->maps) + i; 7181 if (idx >= obj->nr_maps || idx < 0) 7182 return NULL; 7183 return &obj->maps[idx]; 7184 } 7185 7186 struct bpf_map * 7187 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 7188 { 7189 if (prev == NULL) 7190 return obj->maps; 7191 7192 return __bpf_map__iter(prev, obj, 1); 7193 } 7194 7195 struct bpf_map * 7196 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 7197 { 7198 if (next == NULL) { 7199 if (!obj->nr_maps) 7200 return NULL; 7201 return obj->maps + obj->nr_maps - 1; 7202 } 7203 7204 return __bpf_map__iter(next, obj, -1); 7205 } 7206 7207 struct bpf_map * 7208 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 7209 { 7210 struct bpf_map *pos; 7211 7212 bpf_object__for_each_map(pos, obj) { 7213 if (pos->name && !strcmp(pos->name, name)) 7214 return pos; 7215 } 7216 return NULL; 7217 } 7218 7219 int 7220 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 7221 { 7222 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 7223 } 7224 7225 struct bpf_map * 7226 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 7227 { 7228 return ERR_PTR(-ENOTSUP); 7229 } 7230 7231 long libbpf_get_error(const void *ptr) 7232 { 7233 return PTR_ERR_OR_ZERO(ptr); 7234 } 7235 7236 int bpf_prog_load(const char *file, enum bpf_prog_type type, 7237 struct bpf_object **pobj, int *prog_fd) 7238 { 7239 struct bpf_prog_load_attr attr; 7240 7241 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 7242 attr.file = file; 7243 attr.prog_type = type; 7244 attr.expected_attach_type = 0; 7245 7246 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 7247 } 7248 7249 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 7250 struct bpf_object **pobj, int *prog_fd) 7251 { 7252 struct bpf_object_open_attr open_attr = {}; 7253 struct bpf_program *prog, *first_prog = NULL; 7254 struct bpf_object *obj; 7255 struct bpf_map *map; 7256 int err; 7257 7258 if (!attr) 7259 return -EINVAL; 7260 if (!attr->file) 7261 return -EINVAL; 7262 7263 open_attr.file = attr->file; 7264 open_attr.prog_type = attr->prog_type; 7265 7266 obj = bpf_object__open_xattr(&open_attr); 7267 if (IS_ERR_OR_NULL(obj)) 7268 return -ENOENT; 7269 7270 bpf_object__for_each_program(prog, obj) { 7271 enum bpf_attach_type attach_type = attr->expected_attach_type; 7272 /* 7273 * to preserve backwards compatibility, bpf_prog_load treats 7274 * attr->prog_type, if specified, as an override to whatever 7275 * bpf_object__open guessed 7276 */ 7277 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { 7278 bpf_program__set_type(prog, attr->prog_type); 7279 bpf_program__set_expected_attach_type(prog, 7280 attach_type); 7281 } 7282 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { 7283 /* 7284 * we haven't guessed from section name and user 7285 * didn't provide a fallback type, too bad... 7286 */ 7287 bpf_object__close(obj); 7288 return -EINVAL; 7289 } 7290 7291 prog->prog_ifindex = attr->ifindex; 7292 prog->log_level = attr->log_level; 7293 prog->prog_flags = attr->prog_flags; 7294 if (!first_prog) 7295 first_prog = prog; 7296 } 7297 7298 bpf_object__for_each_map(map, obj) { 7299 if (!bpf_map__is_offload_neutral(map)) 7300 map->map_ifindex = attr->ifindex; 7301 } 7302 7303 if (!first_prog) { 7304 pr_warn("object file doesn't contain bpf program\n"); 7305 bpf_object__close(obj); 7306 return -ENOENT; 7307 } 7308 7309 err = bpf_object__load(obj); 7310 if (err) { 7311 bpf_object__close(obj); 7312 return err; 7313 } 7314 7315 *pobj = obj; 7316 *prog_fd = bpf_program__fd(first_prog); 7317 return 0; 7318 } 7319 7320 struct bpf_link { 7321 int (*detach)(struct bpf_link *link); 7322 int (*destroy)(struct bpf_link *link); 7323 char *pin_path; /* NULL, if not pinned */ 7324 int fd; /* hook FD, -1 if not applicable */ 7325 bool disconnected; 7326 }; 7327 7328 /* Replace link's underlying BPF program with the new one */ 7329 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 7330 { 7331 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 7332 } 7333 7334 /* Release "ownership" of underlying BPF resource (typically, BPF program 7335 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 7336 * link, when destructed through bpf_link__destroy() call won't attempt to 7337 * detach/unregisted that BPF resource. This is useful in situations where, 7338 * say, attached BPF program has to outlive userspace program that attached it 7339 * in the system. Depending on type of BPF program, though, there might be 7340 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 7341 * exit of userspace program doesn't trigger automatic detachment and clean up 7342 * inside the kernel. 7343 */ 7344 void bpf_link__disconnect(struct bpf_link *link) 7345 { 7346 link->disconnected = true; 7347 } 7348 7349 int bpf_link__destroy(struct bpf_link *link) 7350 { 7351 int err = 0; 7352 7353 if (!link) 7354 return 0; 7355 7356 if (!link->disconnected && link->detach) 7357 err = link->detach(link); 7358 if (link->destroy) 7359 link->destroy(link); 7360 if (link->pin_path) 7361 free(link->pin_path); 7362 free(link); 7363 7364 return err; 7365 } 7366 7367 int bpf_link__fd(const struct bpf_link *link) 7368 { 7369 return link->fd; 7370 } 7371 7372 const char *bpf_link__pin_path(const struct bpf_link *link) 7373 { 7374 return link->pin_path; 7375 } 7376 7377 static int bpf_link__detach_fd(struct bpf_link *link) 7378 { 7379 return close(link->fd); 7380 } 7381 7382 struct bpf_link *bpf_link__open(const char *path) 7383 { 7384 struct bpf_link *link; 7385 int fd; 7386 7387 fd = bpf_obj_get(path); 7388 if (fd < 0) { 7389 fd = -errno; 7390 pr_warn("failed to open link at %s: %d\n", path, fd); 7391 return ERR_PTR(fd); 7392 } 7393 7394 link = calloc(1, sizeof(*link)); 7395 if (!link) { 7396 close(fd); 7397 return ERR_PTR(-ENOMEM); 7398 } 7399 link->detach = &bpf_link__detach_fd; 7400 link->fd = fd; 7401 7402 link->pin_path = strdup(path); 7403 if (!link->pin_path) { 7404 bpf_link__destroy(link); 7405 return ERR_PTR(-ENOMEM); 7406 } 7407 7408 return link; 7409 } 7410 7411 int bpf_link__pin(struct bpf_link *link, const char *path) 7412 { 7413 int err; 7414 7415 if (link->pin_path) 7416 return -EBUSY; 7417 err = make_parent_dir(path); 7418 if (err) 7419 return err; 7420 err = check_path(path); 7421 if (err) 7422 return err; 7423 7424 link->pin_path = strdup(path); 7425 if (!link->pin_path) 7426 return -ENOMEM; 7427 7428 if (bpf_obj_pin(link->fd, link->pin_path)) { 7429 err = -errno; 7430 zfree(&link->pin_path); 7431 return err; 7432 } 7433 7434 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 7435 return 0; 7436 } 7437 7438 int bpf_link__unpin(struct bpf_link *link) 7439 { 7440 int err; 7441 7442 if (!link->pin_path) 7443 return -EINVAL; 7444 7445 err = unlink(link->pin_path); 7446 if (err != 0) 7447 return -errno; 7448 7449 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 7450 zfree(&link->pin_path); 7451 return 0; 7452 } 7453 7454 static int bpf_link__detach_perf_event(struct bpf_link *link) 7455 { 7456 int err; 7457 7458 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0); 7459 if (err) 7460 err = -errno; 7461 7462 close(link->fd); 7463 return err; 7464 } 7465 7466 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, 7467 int pfd) 7468 { 7469 char errmsg[STRERR_BUFSIZE]; 7470 struct bpf_link *link; 7471 int prog_fd, err; 7472 7473 if (pfd < 0) { 7474 pr_warn("program '%s': invalid perf event FD %d\n", 7475 bpf_program__title(prog, false), pfd); 7476 return ERR_PTR(-EINVAL); 7477 } 7478 prog_fd = bpf_program__fd(prog); 7479 if (prog_fd < 0) { 7480 pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n", 7481 bpf_program__title(prog, false)); 7482 return ERR_PTR(-EINVAL); 7483 } 7484 7485 link = calloc(1, sizeof(*link)); 7486 if (!link) 7487 return ERR_PTR(-ENOMEM); 7488 link->detach = &bpf_link__detach_perf_event; 7489 link->fd = pfd; 7490 7491 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 7492 err = -errno; 7493 free(link); 7494 pr_warn("program '%s': failed to attach to pfd %d: %s\n", 7495 bpf_program__title(prog, false), pfd, 7496 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7497 return ERR_PTR(err); 7498 } 7499 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 7500 err = -errno; 7501 free(link); 7502 pr_warn("program '%s': failed to enable pfd %d: %s\n", 7503 bpf_program__title(prog, false), pfd, 7504 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7505 return ERR_PTR(err); 7506 } 7507 return link; 7508 } 7509 7510 /* 7511 * this function is expected to parse integer in the range of [0, 2^31-1] from 7512 * given file using scanf format string fmt. If actual parsed value is 7513 * negative, the result might be indistinguishable from error 7514 */ 7515 static int parse_uint_from_file(const char *file, const char *fmt) 7516 { 7517 char buf[STRERR_BUFSIZE]; 7518 int err, ret; 7519 FILE *f; 7520 7521 f = fopen(file, "r"); 7522 if (!f) { 7523 err = -errno; 7524 pr_debug("failed to open '%s': %s\n", file, 7525 libbpf_strerror_r(err, buf, sizeof(buf))); 7526 return err; 7527 } 7528 err = fscanf(f, fmt, &ret); 7529 if (err != 1) { 7530 err = err == EOF ? -EIO : -errno; 7531 pr_debug("failed to parse '%s': %s\n", file, 7532 libbpf_strerror_r(err, buf, sizeof(buf))); 7533 fclose(f); 7534 return err; 7535 } 7536 fclose(f); 7537 return ret; 7538 } 7539 7540 static int determine_kprobe_perf_type(void) 7541 { 7542 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 7543 7544 return parse_uint_from_file(file, "%d\n"); 7545 } 7546 7547 static int determine_uprobe_perf_type(void) 7548 { 7549 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 7550 7551 return parse_uint_from_file(file, "%d\n"); 7552 } 7553 7554 static int determine_kprobe_retprobe_bit(void) 7555 { 7556 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 7557 7558 return parse_uint_from_file(file, "config:%d\n"); 7559 } 7560 7561 static int determine_uprobe_retprobe_bit(void) 7562 { 7563 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 7564 7565 return parse_uint_from_file(file, "config:%d\n"); 7566 } 7567 7568 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 7569 uint64_t offset, int pid) 7570 { 7571 struct perf_event_attr attr = {}; 7572 char errmsg[STRERR_BUFSIZE]; 7573 int type, pfd, err; 7574 7575 type = uprobe ? determine_uprobe_perf_type() 7576 : determine_kprobe_perf_type(); 7577 if (type < 0) { 7578 pr_warn("failed to determine %s perf type: %s\n", 7579 uprobe ? "uprobe" : "kprobe", 7580 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 7581 return type; 7582 } 7583 if (retprobe) { 7584 int bit = uprobe ? determine_uprobe_retprobe_bit() 7585 : determine_kprobe_retprobe_bit(); 7586 7587 if (bit < 0) { 7588 pr_warn("failed to determine %s retprobe bit: %s\n", 7589 uprobe ? "uprobe" : "kprobe", 7590 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 7591 return bit; 7592 } 7593 attr.config |= 1 << bit; 7594 } 7595 attr.size = sizeof(attr); 7596 attr.type = type; 7597 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 7598 attr.config2 = offset; /* kprobe_addr or probe_offset */ 7599 7600 /* pid filter is meaningful only for uprobes */ 7601 pfd = syscall(__NR_perf_event_open, &attr, 7602 pid < 0 ? -1 : pid /* pid */, 7603 pid == -1 ? 0 : -1 /* cpu */, 7604 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 7605 if (pfd < 0) { 7606 err = -errno; 7607 pr_warn("%s perf_event_open() failed: %s\n", 7608 uprobe ? "uprobe" : "kprobe", 7609 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7610 return err; 7611 } 7612 return pfd; 7613 } 7614 7615 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 7616 bool retprobe, 7617 const char *func_name) 7618 { 7619 char errmsg[STRERR_BUFSIZE]; 7620 struct bpf_link *link; 7621 int pfd, err; 7622 7623 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 7624 0 /* offset */, -1 /* pid */); 7625 if (pfd < 0) { 7626 pr_warn("program '%s': failed to create %s '%s' perf event: %s\n", 7627 bpf_program__title(prog, false), 7628 retprobe ? "kretprobe" : "kprobe", func_name, 7629 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7630 return ERR_PTR(pfd); 7631 } 7632 link = bpf_program__attach_perf_event(prog, pfd); 7633 if (IS_ERR(link)) { 7634 close(pfd); 7635 err = PTR_ERR(link); 7636 pr_warn("program '%s': failed to attach to %s '%s': %s\n", 7637 bpf_program__title(prog, false), 7638 retprobe ? "kretprobe" : "kprobe", func_name, 7639 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7640 return link; 7641 } 7642 return link; 7643 } 7644 7645 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 7646 struct bpf_program *prog) 7647 { 7648 const char *func_name; 7649 bool retprobe; 7650 7651 func_name = bpf_program__title(prog, false) + sec->len; 7652 retprobe = strcmp(sec->sec, "kretprobe/") == 0; 7653 7654 return bpf_program__attach_kprobe(prog, retprobe, func_name); 7655 } 7656 7657 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 7658 bool retprobe, pid_t pid, 7659 const char *binary_path, 7660 size_t func_offset) 7661 { 7662 char errmsg[STRERR_BUFSIZE]; 7663 struct bpf_link *link; 7664 int pfd, err; 7665 7666 pfd = perf_event_open_probe(true /* uprobe */, retprobe, 7667 binary_path, func_offset, pid); 7668 if (pfd < 0) { 7669 pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 7670 bpf_program__title(prog, false), 7671 retprobe ? "uretprobe" : "uprobe", 7672 binary_path, func_offset, 7673 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7674 return ERR_PTR(pfd); 7675 } 7676 link = bpf_program__attach_perf_event(prog, pfd); 7677 if (IS_ERR(link)) { 7678 close(pfd); 7679 err = PTR_ERR(link); 7680 pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n", 7681 bpf_program__title(prog, false), 7682 retprobe ? "uretprobe" : "uprobe", 7683 binary_path, func_offset, 7684 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7685 return link; 7686 } 7687 return link; 7688 } 7689 7690 static int determine_tracepoint_id(const char *tp_category, 7691 const char *tp_name) 7692 { 7693 char file[PATH_MAX]; 7694 int ret; 7695 7696 ret = snprintf(file, sizeof(file), 7697 "/sys/kernel/debug/tracing/events/%s/%s/id", 7698 tp_category, tp_name); 7699 if (ret < 0) 7700 return -errno; 7701 if (ret >= sizeof(file)) { 7702 pr_debug("tracepoint %s/%s path is too long\n", 7703 tp_category, tp_name); 7704 return -E2BIG; 7705 } 7706 return parse_uint_from_file(file, "%d\n"); 7707 } 7708 7709 static int perf_event_open_tracepoint(const char *tp_category, 7710 const char *tp_name) 7711 { 7712 struct perf_event_attr attr = {}; 7713 char errmsg[STRERR_BUFSIZE]; 7714 int tp_id, pfd, err; 7715 7716 tp_id = determine_tracepoint_id(tp_category, tp_name); 7717 if (tp_id < 0) { 7718 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 7719 tp_category, tp_name, 7720 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 7721 return tp_id; 7722 } 7723 7724 attr.type = PERF_TYPE_TRACEPOINT; 7725 attr.size = sizeof(attr); 7726 attr.config = tp_id; 7727 7728 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 7729 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 7730 if (pfd < 0) { 7731 err = -errno; 7732 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 7733 tp_category, tp_name, 7734 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7735 return err; 7736 } 7737 return pfd; 7738 } 7739 7740 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 7741 const char *tp_category, 7742 const char *tp_name) 7743 { 7744 char errmsg[STRERR_BUFSIZE]; 7745 struct bpf_link *link; 7746 int pfd, err; 7747 7748 pfd = perf_event_open_tracepoint(tp_category, tp_name); 7749 if (pfd < 0) { 7750 pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 7751 bpf_program__title(prog, false), 7752 tp_category, tp_name, 7753 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7754 return ERR_PTR(pfd); 7755 } 7756 link = bpf_program__attach_perf_event(prog, pfd); 7757 if (IS_ERR(link)) { 7758 close(pfd); 7759 err = PTR_ERR(link); 7760 pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n", 7761 bpf_program__title(prog, false), 7762 tp_category, tp_name, 7763 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7764 return link; 7765 } 7766 return link; 7767 } 7768 7769 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 7770 struct bpf_program *prog) 7771 { 7772 char *sec_name, *tp_cat, *tp_name; 7773 struct bpf_link *link; 7774 7775 sec_name = strdup(bpf_program__title(prog, false)); 7776 if (!sec_name) 7777 return ERR_PTR(-ENOMEM); 7778 7779 /* extract "tp/<category>/<name>" */ 7780 tp_cat = sec_name + sec->len; 7781 tp_name = strchr(tp_cat, '/'); 7782 if (!tp_name) { 7783 link = ERR_PTR(-EINVAL); 7784 goto out; 7785 } 7786 *tp_name = '\0'; 7787 tp_name++; 7788 7789 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 7790 out: 7791 free(sec_name); 7792 return link; 7793 } 7794 7795 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 7796 const char *tp_name) 7797 { 7798 char errmsg[STRERR_BUFSIZE]; 7799 struct bpf_link *link; 7800 int prog_fd, pfd; 7801 7802 prog_fd = bpf_program__fd(prog); 7803 if (prog_fd < 0) { 7804 pr_warn("program '%s': can't attach before loaded\n", 7805 bpf_program__title(prog, false)); 7806 return ERR_PTR(-EINVAL); 7807 } 7808 7809 link = calloc(1, sizeof(*link)); 7810 if (!link) 7811 return ERR_PTR(-ENOMEM); 7812 link->detach = &bpf_link__detach_fd; 7813 7814 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 7815 if (pfd < 0) { 7816 pfd = -errno; 7817 free(link); 7818 pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n", 7819 bpf_program__title(prog, false), tp_name, 7820 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7821 return ERR_PTR(pfd); 7822 } 7823 link->fd = pfd; 7824 return link; 7825 } 7826 7827 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 7828 struct bpf_program *prog) 7829 { 7830 const char *tp_name = bpf_program__title(prog, false) + sec->len; 7831 7832 return bpf_program__attach_raw_tracepoint(prog, tp_name); 7833 } 7834 7835 /* Common logic for all BPF program types that attach to a btf_id */ 7836 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog) 7837 { 7838 char errmsg[STRERR_BUFSIZE]; 7839 struct bpf_link *link; 7840 int prog_fd, pfd; 7841 7842 prog_fd = bpf_program__fd(prog); 7843 if (prog_fd < 0) { 7844 pr_warn("program '%s': can't attach before loaded\n", 7845 bpf_program__title(prog, false)); 7846 return ERR_PTR(-EINVAL); 7847 } 7848 7849 link = calloc(1, sizeof(*link)); 7850 if (!link) 7851 return ERR_PTR(-ENOMEM); 7852 link->detach = &bpf_link__detach_fd; 7853 7854 pfd = bpf_raw_tracepoint_open(NULL, prog_fd); 7855 if (pfd < 0) { 7856 pfd = -errno; 7857 free(link); 7858 pr_warn("program '%s': failed to attach: %s\n", 7859 bpf_program__title(prog, false), 7860 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7861 return ERR_PTR(pfd); 7862 } 7863 link->fd = pfd; 7864 return (struct bpf_link *)link; 7865 } 7866 7867 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog) 7868 { 7869 return bpf_program__attach_btf_id(prog); 7870 } 7871 7872 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog) 7873 { 7874 return bpf_program__attach_btf_id(prog); 7875 } 7876 7877 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 7878 struct bpf_program *prog) 7879 { 7880 return bpf_program__attach_trace(prog); 7881 } 7882 7883 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 7884 struct bpf_program *prog) 7885 { 7886 return bpf_program__attach_lsm(prog); 7887 } 7888 7889 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 7890 struct bpf_program *prog) 7891 { 7892 return bpf_program__attach_iter(prog, NULL); 7893 } 7894 7895 static struct bpf_link * 7896 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, 7897 const char *target_name) 7898 { 7899 enum bpf_attach_type attach_type; 7900 char errmsg[STRERR_BUFSIZE]; 7901 struct bpf_link *link; 7902 int prog_fd, link_fd; 7903 7904 prog_fd = bpf_program__fd(prog); 7905 if (prog_fd < 0) { 7906 pr_warn("program '%s': can't attach before loaded\n", 7907 bpf_program__title(prog, false)); 7908 return ERR_PTR(-EINVAL); 7909 } 7910 7911 link = calloc(1, sizeof(*link)); 7912 if (!link) 7913 return ERR_PTR(-ENOMEM); 7914 link->detach = &bpf_link__detach_fd; 7915 7916 attach_type = bpf_program__get_expected_attach_type(prog); 7917 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, NULL); 7918 if (link_fd < 0) { 7919 link_fd = -errno; 7920 free(link); 7921 pr_warn("program '%s': failed to attach to %s: %s\n", 7922 bpf_program__title(prog, false), target_name, 7923 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 7924 return ERR_PTR(link_fd); 7925 } 7926 link->fd = link_fd; 7927 return link; 7928 } 7929 7930 struct bpf_link * 7931 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd) 7932 { 7933 return bpf_program__attach_fd(prog, cgroup_fd, "cgroup"); 7934 } 7935 7936 struct bpf_link * 7937 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd) 7938 { 7939 return bpf_program__attach_fd(prog, netns_fd, "netns"); 7940 } 7941 7942 struct bpf_link * 7943 bpf_program__attach_iter(struct bpf_program *prog, 7944 const struct bpf_iter_attach_opts *opts) 7945 { 7946 char errmsg[STRERR_BUFSIZE]; 7947 struct bpf_link *link; 7948 int prog_fd, link_fd; 7949 7950 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 7951 return ERR_PTR(-EINVAL); 7952 7953 prog_fd = bpf_program__fd(prog); 7954 if (prog_fd < 0) { 7955 pr_warn("program '%s': can't attach before loaded\n", 7956 bpf_program__title(prog, false)); 7957 return ERR_PTR(-EINVAL); 7958 } 7959 7960 link = calloc(1, sizeof(*link)); 7961 if (!link) 7962 return ERR_PTR(-ENOMEM); 7963 link->detach = &bpf_link__detach_fd; 7964 7965 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_ITER, NULL); 7966 if (link_fd < 0) { 7967 link_fd = -errno; 7968 free(link); 7969 pr_warn("program '%s': failed to attach to iterator: %s\n", 7970 bpf_program__title(prog, false), 7971 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 7972 return ERR_PTR(link_fd); 7973 } 7974 link->fd = link_fd; 7975 return link; 7976 } 7977 7978 struct bpf_link *bpf_program__attach(struct bpf_program *prog) 7979 { 7980 const struct bpf_sec_def *sec_def; 7981 7982 sec_def = find_sec_def(bpf_program__title(prog, false)); 7983 if (!sec_def || !sec_def->attach_fn) 7984 return ERR_PTR(-ESRCH); 7985 7986 return sec_def->attach_fn(sec_def, prog); 7987 } 7988 7989 static int bpf_link__detach_struct_ops(struct bpf_link *link) 7990 { 7991 __u32 zero = 0; 7992 7993 if (bpf_map_delete_elem(link->fd, &zero)) 7994 return -errno; 7995 7996 return 0; 7997 } 7998 7999 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map) 8000 { 8001 struct bpf_struct_ops *st_ops; 8002 struct bpf_link *link; 8003 __u32 i, zero = 0; 8004 int err; 8005 8006 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 8007 return ERR_PTR(-EINVAL); 8008 8009 link = calloc(1, sizeof(*link)); 8010 if (!link) 8011 return ERR_PTR(-EINVAL); 8012 8013 st_ops = map->st_ops; 8014 for (i = 0; i < btf_vlen(st_ops->type); i++) { 8015 struct bpf_program *prog = st_ops->progs[i]; 8016 void *kern_data; 8017 int prog_fd; 8018 8019 if (!prog) 8020 continue; 8021 8022 prog_fd = bpf_program__fd(prog); 8023 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 8024 *(unsigned long *)kern_data = prog_fd; 8025 } 8026 8027 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); 8028 if (err) { 8029 err = -errno; 8030 free(link); 8031 return ERR_PTR(err); 8032 } 8033 8034 link->detach = bpf_link__detach_struct_ops; 8035 link->fd = map->fd; 8036 8037 return link; 8038 } 8039 8040 enum bpf_perf_event_ret 8041 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 8042 void **copy_mem, size_t *copy_size, 8043 bpf_perf_event_print_t fn, void *private_data) 8044 { 8045 struct perf_event_mmap_page *header = mmap_mem; 8046 __u64 data_head = ring_buffer_read_head(header); 8047 __u64 data_tail = header->data_tail; 8048 void *base = ((__u8 *)header) + page_size; 8049 int ret = LIBBPF_PERF_EVENT_CONT; 8050 struct perf_event_header *ehdr; 8051 size_t ehdr_size; 8052 8053 while (data_head != data_tail) { 8054 ehdr = base + (data_tail & (mmap_size - 1)); 8055 ehdr_size = ehdr->size; 8056 8057 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 8058 void *copy_start = ehdr; 8059 size_t len_first = base + mmap_size - copy_start; 8060 size_t len_secnd = ehdr_size - len_first; 8061 8062 if (*copy_size < ehdr_size) { 8063 free(*copy_mem); 8064 *copy_mem = malloc(ehdr_size); 8065 if (!*copy_mem) { 8066 *copy_size = 0; 8067 ret = LIBBPF_PERF_EVENT_ERROR; 8068 break; 8069 } 8070 *copy_size = ehdr_size; 8071 } 8072 8073 memcpy(*copy_mem, copy_start, len_first); 8074 memcpy(*copy_mem + len_first, base, len_secnd); 8075 ehdr = *copy_mem; 8076 } 8077 8078 ret = fn(ehdr, private_data); 8079 data_tail += ehdr_size; 8080 if (ret != LIBBPF_PERF_EVENT_CONT) 8081 break; 8082 } 8083 8084 ring_buffer_write_tail(header, data_tail); 8085 return ret; 8086 } 8087 8088 struct perf_buffer; 8089 8090 struct perf_buffer_params { 8091 struct perf_event_attr *attr; 8092 /* if event_cb is specified, it takes precendence */ 8093 perf_buffer_event_fn event_cb; 8094 /* sample_cb and lost_cb are higher-level common-case callbacks */ 8095 perf_buffer_sample_fn sample_cb; 8096 perf_buffer_lost_fn lost_cb; 8097 void *ctx; 8098 int cpu_cnt; 8099 int *cpus; 8100 int *map_keys; 8101 }; 8102 8103 struct perf_cpu_buf { 8104 struct perf_buffer *pb; 8105 void *base; /* mmap()'ed memory */ 8106 void *buf; /* for reconstructing segmented data */ 8107 size_t buf_size; 8108 int fd; 8109 int cpu; 8110 int map_key; 8111 }; 8112 8113 struct perf_buffer { 8114 perf_buffer_event_fn event_cb; 8115 perf_buffer_sample_fn sample_cb; 8116 perf_buffer_lost_fn lost_cb; 8117 void *ctx; /* passed into callbacks */ 8118 8119 size_t page_size; 8120 size_t mmap_size; 8121 struct perf_cpu_buf **cpu_bufs; 8122 struct epoll_event *events; 8123 int cpu_cnt; /* number of allocated CPU buffers */ 8124 int epoll_fd; /* perf event FD */ 8125 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 8126 }; 8127 8128 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 8129 struct perf_cpu_buf *cpu_buf) 8130 { 8131 if (!cpu_buf) 8132 return; 8133 if (cpu_buf->base && 8134 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 8135 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 8136 if (cpu_buf->fd >= 0) { 8137 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 8138 close(cpu_buf->fd); 8139 } 8140 free(cpu_buf->buf); 8141 free(cpu_buf); 8142 } 8143 8144 void perf_buffer__free(struct perf_buffer *pb) 8145 { 8146 int i; 8147 8148 if (!pb) 8149 return; 8150 if (pb->cpu_bufs) { 8151 for (i = 0; i < pb->cpu_cnt; i++) { 8152 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 8153 8154 if (!cpu_buf) 8155 continue; 8156 8157 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 8158 perf_buffer__free_cpu_buf(pb, cpu_buf); 8159 } 8160 free(pb->cpu_bufs); 8161 } 8162 if (pb->epoll_fd >= 0) 8163 close(pb->epoll_fd); 8164 free(pb->events); 8165 free(pb); 8166 } 8167 8168 static struct perf_cpu_buf * 8169 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 8170 int cpu, int map_key) 8171 { 8172 struct perf_cpu_buf *cpu_buf; 8173 char msg[STRERR_BUFSIZE]; 8174 int err; 8175 8176 cpu_buf = calloc(1, sizeof(*cpu_buf)); 8177 if (!cpu_buf) 8178 return ERR_PTR(-ENOMEM); 8179 8180 cpu_buf->pb = pb; 8181 cpu_buf->cpu = cpu; 8182 cpu_buf->map_key = map_key; 8183 8184 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 8185 -1, PERF_FLAG_FD_CLOEXEC); 8186 if (cpu_buf->fd < 0) { 8187 err = -errno; 8188 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 8189 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 8190 goto error; 8191 } 8192 8193 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 8194 PROT_READ | PROT_WRITE, MAP_SHARED, 8195 cpu_buf->fd, 0); 8196 if (cpu_buf->base == MAP_FAILED) { 8197 cpu_buf->base = NULL; 8198 err = -errno; 8199 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 8200 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 8201 goto error; 8202 } 8203 8204 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 8205 err = -errno; 8206 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 8207 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 8208 goto error; 8209 } 8210 8211 return cpu_buf; 8212 8213 error: 8214 perf_buffer__free_cpu_buf(pb, cpu_buf); 8215 return (struct perf_cpu_buf *)ERR_PTR(err); 8216 } 8217 8218 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 8219 struct perf_buffer_params *p); 8220 8221 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 8222 const struct perf_buffer_opts *opts) 8223 { 8224 struct perf_buffer_params p = {}; 8225 struct perf_event_attr attr = { 0, }; 8226 8227 attr.config = PERF_COUNT_SW_BPF_OUTPUT, 8228 attr.type = PERF_TYPE_SOFTWARE; 8229 attr.sample_type = PERF_SAMPLE_RAW; 8230 attr.sample_period = 1; 8231 attr.wakeup_events = 1; 8232 8233 p.attr = &attr; 8234 p.sample_cb = opts ? opts->sample_cb : NULL; 8235 p.lost_cb = opts ? opts->lost_cb : NULL; 8236 p.ctx = opts ? opts->ctx : NULL; 8237 8238 return __perf_buffer__new(map_fd, page_cnt, &p); 8239 } 8240 8241 struct perf_buffer * 8242 perf_buffer__new_raw(int map_fd, size_t page_cnt, 8243 const struct perf_buffer_raw_opts *opts) 8244 { 8245 struct perf_buffer_params p = {}; 8246 8247 p.attr = opts->attr; 8248 p.event_cb = opts->event_cb; 8249 p.ctx = opts->ctx; 8250 p.cpu_cnt = opts->cpu_cnt; 8251 p.cpus = opts->cpus; 8252 p.map_keys = opts->map_keys; 8253 8254 return __perf_buffer__new(map_fd, page_cnt, &p); 8255 } 8256 8257 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 8258 struct perf_buffer_params *p) 8259 { 8260 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 8261 struct bpf_map_info map = {}; 8262 char msg[STRERR_BUFSIZE]; 8263 struct perf_buffer *pb; 8264 bool *online = NULL; 8265 __u32 map_info_len; 8266 int err, i, j, n; 8267 8268 if (page_cnt & (page_cnt - 1)) { 8269 pr_warn("page count should be power of two, but is %zu\n", 8270 page_cnt); 8271 return ERR_PTR(-EINVAL); 8272 } 8273 8274 map_info_len = sizeof(map); 8275 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 8276 if (err) { 8277 err = -errno; 8278 pr_warn("failed to get map info for map FD %d: %s\n", 8279 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 8280 return ERR_PTR(err); 8281 } 8282 8283 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 8284 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 8285 map.name); 8286 return ERR_PTR(-EINVAL); 8287 } 8288 8289 pb = calloc(1, sizeof(*pb)); 8290 if (!pb) 8291 return ERR_PTR(-ENOMEM); 8292 8293 pb->event_cb = p->event_cb; 8294 pb->sample_cb = p->sample_cb; 8295 pb->lost_cb = p->lost_cb; 8296 pb->ctx = p->ctx; 8297 8298 pb->page_size = getpagesize(); 8299 pb->mmap_size = pb->page_size * page_cnt; 8300 pb->map_fd = map_fd; 8301 8302 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 8303 if (pb->epoll_fd < 0) { 8304 err = -errno; 8305 pr_warn("failed to create epoll instance: %s\n", 8306 libbpf_strerror_r(err, msg, sizeof(msg))); 8307 goto error; 8308 } 8309 8310 if (p->cpu_cnt > 0) { 8311 pb->cpu_cnt = p->cpu_cnt; 8312 } else { 8313 pb->cpu_cnt = libbpf_num_possible_cpus(); 8314 if (pb->cpu_cnt < 0) { 8315 err = pb->cpu_cnt; 8316 goto error; 8317 } 8318 if (map.max_entries < pb->cpu_cnt) 8319 pb->cpu_cnt = map.max_entries; 8320 } 8321 8322 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 8323 if (!pb->events) { 8324 err = -ENOMEM; 8325 pr_warn("failed to allocate events: out of memory\n"); 8326 goto error; 8327 } 8328 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 8329 if (!pb->cpu_bufs) { 8330 err = -ENOMEM; 8331 pr_warn("failed to allocate buffers: out of memory\n"); 8332 goto error; 8333 } 8334 8335 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 8336 if (err) { 8337 pr_warn("failed to get online CPU mask: %d\n", err); 8338 goto error; 8339 } 8340 8341 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 8342 struct perf_cpu_buf *cpu_buf; 8343 int cpu, map_key; 8344 8345 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 8346 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 8347 8348 /* in case user didn't explicitly requested particular CPUs to 8349 * be attached to, skip offline/not present CPUs 8350 */ 8351 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 8352 continue; 8353 8354 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 8355 if (IS_ERR(cpu_buf)) { 8356 err = PTR_ERR(cpu_buf); 8357 goto error; 8358 } 8359 8360 pb->cpu_bufs[j] = cpu_buf; 8361 8362 err = bpf_map_update_elem(pb->map_fd, &map_key, 8363 &cpu_buf->fd, 0); 8364 if (err) { 8365 err = -errno; 8366 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 8367 cpu, map_key, cpu_buf->fd, 8368 libbpf_strerror_r(err, msg, sizeof(msg))); 8369 goto error; 8370 } 8371 8372 pb->events[j].events = EPOLLIN; 8373 pb->events[j].data.ptr = cpu_buf; 8374 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 8375 &pb->events[j]) < 0) { 8376 err = -errno; 8377 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 8378 cpu, cpu_buf->fd, 8379 libbpf_strerror_r(err, msg, sizeof(msg))); 8380 goto error; 8381 } 8382 j++; 8383 } 8384 pb->cpu_cnt = j; 8385 free(online); 8386 8387 return pb; 8388 8389 error: 8390 free(online); 8391 if (pb) 8392 perf_buffer__free(pb); 8393 return ERR_PTR(err); 8394 } 8395 8396 struct perf_sample_raw { 8397 struct perf_event_header header; 8398 uint32_t size; 8399 char data[]; 8400 }; 8401 8402 struct perf_sample_lost { 8403 struct perf_event_header header; 8404 uint64_t id; 8405 uint64_t lost; 8406 uint64_t sample_id; 8407 }; 8408 8409 static enum bpf_perf_event_ret 8410 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 8411 { 8412 struct perf_cpu_buf *cpu_buf = ctx; 8413 struct perf_buffer *pb = cpu_buf->pb; 8414 void *data = e; 8415 8416 /* user wants full control over parsing perf event */ 8417 if (pb->event_cb) 8418 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 8419 8420 switch (e->type) { 8421 case PERF_RECORD_SAMPLE: { 8422 struct perf_sample_raw *s = data; 8423 8424 if (pb->sample_cb) 8425 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 8426 break; 8427 } 8428 case PERF_RECORD_LOST: { 8429 struct perf_sample_lost *s = data; 8430 8431 if (pb->lost_cb) 8432 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 8433 break; 8434 } 8435 default: 8436 pr_warn("unknown perf sample type %d\n", e->type); 8437 return LIBBPF_PERF_EVENT_ERROR; 8438 } 8439 return LIBBPF_PERF_EVENT_CONT; 8440 } 8441 8442 static int perf_buffer__process_records(struct perf_buffer *pb, 8443 struct perf_cpu_buf *cpu_buf) 8444 { 8445 enum bpf_perf_event_ret ret; 8446 8447 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 8448 pb->page_size, &cpu_buf->buf, 8449 &cpu_buf->buf_size, 8450 perf_buffer__process_record, cpu_buf); 8451 if (ret != LIBBPF_PERF_EVENT_CONT) 8452 return ret; 8453 return 0; 8454 } 8455 8456 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 8457 { 8458 int i, cnt, err; 8459 8460 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 8461 for (i = 0; i < cnt; i++) { 8462 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 8463 8464 err = perf_buffer__process_records(pb, cpu_buf); 8465 if (err) { 8466 pr_warn("error while processing records: %d\n", err); 8467 return err; 8468 } 8469 } 8470 return cnt < 0 ? -errno : cnt; 8471 } 8472 8473 int perf_buffer__consume(struct perf_buffer *pb) 8474 { 8475 int i, err; 8476 8477 for (i = 0; i < pb->cpu_cnt; i++) { 8478 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 8479 8480 if (!cpu_buf) 8481 continue; 8482 8483 err = perf_buffer__process_records(pb, cpu_buf); 8484 if (err) { 8485 pr_warn("error while processing records: %d\n", err); 8486 return err; 8487 } 8488 } 8489 return 0; 8490 } 8491 8492 struct bpf_prog_info_array_desc { 8493 int array_offset; /* e.g. offset of jited_prog_insns */ 8494 int count_offset; /* e.g. offset of jited_prog_len */ 8495 int size_offset; /* > 0: offset of rec size, 8496 * < 0: fix size of -size_offset 8497 */ 8498 }; 8499 8500 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 8501 [BPF_PROG_INFO_JITED_INSNS] = { 8502 offsetof(struct bpf_prog_info, jited_prog_insns), 8503 offsetof(struct bpf_prog_info, jited_prog_len), 8504 -1, 8505 }, 8506 [BPF_PROG_INFO_XLATED_INSNS] = { 8507 offsetof(struct bpf_prog_info, xlated_prog_insns), 8508 offsetof(struct bpf_prog_info, xlated_prog_len), 8509 -1, 8510 }, 8511 [BPF_PROG_INFO_MAP_IDS] = { 8512 offsetof(struct bpf_prog_info, map_ids), 8513 offsetof(struct bpf_prog_info, nr_map_ids), 8514 -(int)sizeof(__u32), 8515 }, 8516 [BPF_PROG_INFO_JITED_KSYMS] = { 8517 offsetof(struct bpf_prog_info, jited_ksyms), 8518 offsetof(struct bpf_prog_info, nr_jited_ksyms), 8519 -(int)sizeof(__u64), 8520 }, 8521 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 8522 offsetof(struct bpf_prog_info, jited_func_lens), 8523 offsetof(struct bpf_prog_info, nr_jited_func_lens), 8524 -(int)sizeof(__u32), 8525 }, 8526 [BPF_PROG_INFO_FUNC_INFO] = { 8527 offsetof(struct bpf_prog_info, func_info), 8528 offsetof(struct bpf_prog_info, nr_func_info), 8529 offsetof(struct bpf_prog_info, func_info_rec_size), 8530 }, 8531 [BPF_PROG_INFO_LINE_INFO] = { 8532 offsetof(struct bpf_prog_info, line_info), 8533 offsetof(struct bpf_prog_info, nr_line_info), 8534 offsetof(struct bpf_prog_info, line_info_rec_size), 8535 }, 8536 [BPF_PROG_INFO_JITED_LINE_INFO] = { 8537 offsetof(struct bpf_prog_info, jited_line_info), 8538 offsetof(struct bpf_prog_info, nr_jited_line_info), 8539 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 8540 }, 8541 [BPF_PROG_INFO_PROG_TAGS] = { 8542 offsetof(struct bpf_prog_info, prog_tags), 8543 offsetof(struct bpf_prog_info, nr_prog_tags), 8544 -(int)sizeof(__u8) * BPF_TAG_SIZE, 8545 }, 8546 8547 }; 8548 8549 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 8550 int offset) 8551 { 8552 __u32 *array = (__u32 *)info; 8553 8554 if (offset >= 0) 8555 return array[offset / sizeof(__u32)]; 8556 return -(int)offset; 8557 } 8558 8559 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 8560 int offset) 8561 { 8562 __u64 *array = (__u64 *)info; 8563 8564 if (offset >= 0) 8565 return array[offset / sizeof(__u64)]; 8566 return -(int)offset; 8567 } 8568 8569 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 8570 __u32 val) 8571 { 8572 __u32 *array = (__u32 *)info; 8573 8574 if (offset >= 0) 8575 array[offset / sizeof(__u32)] = val; 8576 } 8577 8578 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 8579 __u64 val) 8580 { 8581 __u64 *array = (__u64 *)info; 8582 8583 if (offset >= 0) 8584 array[offset / sizeof(__u64)] = val; 8585 } 8586 8587 struct bpf_prog_info_linear * 8588 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 8589 { 8590 struct bpf_prog_info_linear *info_linear; 8591 struct bpf_prog_info info = {}; 8592 __u32 info_len = sizeof(info); 8593 __u32 data_len = 0; 8594 int i, err; 8595 void *ptr; 8596 8597 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 8598 return ERR_PTR(-EINVAL); 8599 8600 /* step 1: get array dimensions */ 8601 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 8602 if (err) { 8603 pr_debug("can't get prog info: %s", strerror(errno)); 8604 return ERR_PTR(-EFAULT); 8605 } 8606 8607 /* step 2: calculate total size of all arrays */ 8608 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8609 bool include_array = (arrays & (1UL << i)) > 0; 8610 struct bpf_prog_info_array_desc *desc; 8611 __u32 count, size; 8612 8613 desc = bpf_prog_info_array_desc + i; 8614 8615 /* kernel is too old to support this field */ 8616 if (info_len < desc->array_offset + sizeof(__u32) || 8617 info_len < desc->count_offset + sizeof(__u32) || 8618 (desc->size_offset > 0 && info_len < desc->size_offset)) 8619 include_array = false; 8620 8621 if (!include_array) { 8622 arrays &= ~(1UL << i); /* clear the bit */ 8623 continue; 8624 } 8625 8626 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 8627 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 8628 8629 data_len += count * size; 8630 } 8631 8632 /* step 3: allocate continuous memory */ 8633 data_len = roundup(data_len, sizeof(__u64)); 8634 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 8635 if (!info_linear) 8636 return ERR_PTR(-ENOMEM); 8637 8638 /* step 4: fill data to info_linear->info */ 8639 info_linear->arrays = arrays; 8640 memset(&info_linear->info, 0, sizeof(info)); 8641 ptr = info_linear->data; 8642 8643 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8644 struct bpf_prog_info_array_desc *desc; 8645 __u32 count, size; 8646 8647 if ((arrays & (1UL << i)) == 0) 8648 continue; 8649 8650 desc = bpf_prog_info_array_desc + i; 8651 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 8652 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 8653 bpf_prog_info_set_offset_u32(&info_linear->info, 8654 desc->count_offset, count); 8655 bpf_prog_info_set_offset_u32(&info_linear->info, 8656 desc->size_offset, size); 8657 bpf_prog_info_set_offset_u64(&info_linear->info, 8658 desc->array_offset, 8659 ptr_to_u64(ptr)); 8660 ptr += count * size; 8661 } 8662 8663 /* step 5: call syscall again to get required arrays */ 8664 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 8665 if (err) { 8666 pr_debug("can't get prog info: %s", strerror(errno)); 8667 free(info_linear); 8668 return ERR_PTR(-EFAULT); 8669 } 8670 8671 /* step 6: verify the data */ 8672 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8673 struct bpf_prog_info_array_desc *desc; 8674 __u32 v1, v2; 8675 8676 if ((arrays & (1UL << i)) == 0) 8677 continue; 8678 8679 desc = bpf_prog_info_array_desc + i; 8680 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 8681 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 8682 desc->count_offset); 8683 if (v1 != v2) 8684 pr_warn("%s: mismatch in element count\n", __func__); 8685 8686 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 8687 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 8688 desc->size_offset); 8689 if (v1 != v2) 8690 pr_warn("%s: mismatch in rec size\n", __func__); 8691 } 8692 8693 /* step 7: update info_len and data_len */ 8694 info_linear->info_len = sizeof(struct bpf_prog_info); 8695 info_linear->data_len = data_len; 8696 8697 return info_linear; 8698 } 8699 8700 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 8701 { 8702 int i; 8703 8704 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8705 struct bpf_prog_info_array_desc *desc; 8706 __u64 addr, offs; 8707 8708 if ((info_linear->arrays & (1UL << i)) == 0) 8709 continue; 8710 8711 desc = bpf_prog_info_array_desc + i; 8712 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 8713 desc->array_offset); 8714 offs = addr - ptr_to_u64(info_linear->data); 8715 bpf_prog_info_set_offset_u64(&info_linear->info, 8716 desc->array_offset, offs); 8717 } 8718 } 8719 8720 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 8721 { 8722 int i; 8723 8724 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8725 struct bpf_prog_info_array_desc *desc; 8726 __u64 addr, offs; 8727 8728 if ((info_linear->arrays & (1UL << i)) == 0) 8729 continue; 8730 8731 desc = bpf_prog_info_array_desc + i; 8732 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 8733 desc->array_offset); 8734 addr = offs + ptr_to_u64(info_linear->data); 8735 bpf_prog_info_set_offset_u64(&info_linear->info, 8736 desc->array_offset, addr); 8737 } 8738 } 8739 8740 int bpf_program__set_attach_target(struct bpf_program *prog, 8741 int attach_prog_fd, 8742 const char *attach_func_name) 8743 { 8744 int btf_id; 8745 8746 if (!prog || attach_prog_fd < 0 || !attach_func_name) 8747 return -EINVAL; 8748 8749 if (attach_prog_fd) 8750 btf_id = libbpf_find_prog_btf_id(attach_func_name, 8751 attach_prog_fd); 8752 else 8753 btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux, 8754 attach_func_name, 8755 prog->expected_attach_type); 8756 8757 if (btf_id < 0) 8758 return btf_id; 8759 8760 prog->attach_btf_id = btf_id; 8761 prog->attach_prog_fd = attach_prog_fd; 8762 return 0; 8763 } 8764 8765 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 8766 { 8767 int err = 0, n, len, start, end = -1; 8768 bool *tmp; 8769 8770 *mask = NULL; 8771 *mask_sz = 0; 8772 8773 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 8774 while (*s) { 8775 if (*s == ',' || *s == '\n') { 8776 s++; 8777 continue; 8778 } 8779 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 8780 if (n <= 0 || n > 2) { 8781 pr_warn("Failed to get CPU range %s: %d\n", s, n); 8782 err = -EINVAL; 8783 goto cleanup; 8784 } else if (n == 1) { 8785 end = start; 8786 } 8787 if (start < 0 || start > end) { 8788 pr_warn("Invalid CPU range [%d,%d] in %s\n", 8789 start, end, s); 8790 err = -EINVAL; 8791 goto cleanup; 8792 } 8793 tmp = realloc(*mask, end + 1); 8794 if (!tmp) { 8795 err = -ENOMEM; 8796 goto cleanup; 8797 } 8798 *mask = tmp; 8799 memset(tmp + *mask_sz, 0, start - *mask_sz); 8800 memset(tmp + start, 1, end - start + 1); 8801 *mask_sz = end + 1; 8802 s += len; 8803 } 8804 if (!*mask_sz) { 8805 pr_warn("Empty CPU range\n"); 8806 return -EINVAL; 8807 } 8808 return 0; 8809 cleanup: 8810 free(*mask); 8811 *mask = NULL; 8812 return err; 8813 } 8814 8815 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 8816 { 8817 int fd, err = 0, len; 8818 char buf[128]; 8819 8820 fd = open(fcpu, O_RDONLY); 8821 if (fd < 0) { 8822 err = -errno; 8823 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 8824 return err; 8825 } 8826 len = read(fd, buf, sizeof(buf)); 8827 close(fd); 8828 if (len <= 0) { 8829 err = len ? -errno : -EINVAL; 8830 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 8831 return err; 8832 } 8833 if (len >= sizeof(buf)) { 8834 pr_warn("CPU mask is too big in file %s\n", fcpu); 8835 return -E2BIG; 8836 } 8837 buf[len] = '\0'; 8838 8839 return parse_cpu_mask_str(buf, mask, mask_sz); 8840 } 8841 8842 int libbpf_num_possible_cpus(void) 8843 { 8844 static const char *fcpu = "/sys/devices/system/cpu/possible"; 8845 static int cpus; 8846 int err, n, i, tmp_cpus; 8847 bool *mask; 8848 8849 tmp_cpus = READ_ONCE(cpus); 8850 if (tmp_cpus > 0) 8851 return tmp_cpus; 8852 8853 err = parse_cpu_mask_file(fcpu, &mask, &n); 8854 if (err) 8855 return err; 8856 8857 tmp_cpus = 0; 8858 for (i = 0; i < n; i++) { 8859 if (mask[i]) 8860 tmp_cpus++; 8861 } 8862 free(mask); 8863 8864 WRITE_ONCE(cpus, tmp_cpus); 8865 return tmp_cpus; 8866 } 8867 8868 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 8869 const struct bpf_object_open_opts *opts) 8870 { 8871 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 8872 .object_name = s->name, 8873 ); 8874 struct bpf_object *obj; 8875 int i; 8876 8877 /* Attempt to preserve opts->object_name, unless overriden by user 8878 * explicitly. Overwriting object name for skeletons is discouraged, 8879 * as it breaks global data maps, because they contain object name 8880 * prefix as their own map name prefix. When skeleton is generated, 8881 * bpftool is making an assumption that this name will stay the same. 8882 */ 8883 if (opts) { 8884 memcpy(&skel_opts, opts, sizeof(*opts)); 8885 if (!opts->object_name) 8886 skel_opts.object_name = s->name; 8887 } 8888 8889 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 8890 if (IS_ERR(obj)) { 8891 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n", 8892 s->name, PTR_ERR(obj)); 8893 return PTR_ERR(obj); 8894 } 8895 8896 *s->obj = obj; 8897 8898 for (i = 0; i < s->map_cnt; i++) { 8899 struct bpf_map **map = s->maps[i].map; 8900 const char *name = s->maps[i].name; 8901 void **mmaped = s->maps[i].mmaped; 8902 8903 *map = bpf_object__find_map_by_name(obj, name); 8904 if (!*map) { 8905 pr_warn("failed to find skeleton map '%s'\n", name); 8906 return -ESRCH; 8907 } 8908 8909 /* externs shouldn't be pre-setup from user code */ 8910 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 8911 *mmaped = (*map)->mmaped; 8912 } 8913 8914 for (i = 0; i < s->prog_cnt; i++) { 8915 struct bpf_program **prog = s->progs[i].prog; 8916 const char *name = s->progs[i].name; 8917 8918 *prog = bpf_object__find_program_by_name(obj, name); 8919 if (!*prog) { 8920 pr_warn("failed to find skeleton program '%s'\n", name); 8921 return -ESRCH; 8922 } 8923 } 8924 8925 return 0; 8926 } 8927 8928 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 8929 { 8930 int i, err; 8931 8932 err = bpf_object__load(*s->obj); 8933 if (err) { 8934 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 8935 return err; 8936 } 8937 8938 for (i = 0; i < s->map_cnt; i++) { 8939 struct bpf_map *map = *s->maps[i].map; 8940 size_t mmap_sz = bpf_map_mmap_sz(map); 8941 int prot, map_fd = bpf_map__fd(map); 8942 void **mmaped = s->maps[i].mmaped; 8943 8944 if (!mmaped) 8945 continue; 8946 8947 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 8948 *mmaped = NULL; 8949 continue; 8950 } 8951 8952 if (map->def.map_flags & BPF_F_RDONLY_PROG) 8953 prot = PROT_READ; 8954 else 8955 prot = PROT_READ | PROT_WRITE; 8956 8957 /* Remap anonymous mmap()-ed "map initialization image" as 8958 * a BPF map-backed mmap()-ed memory, but preserving the same 8959 * memory address. This will cause kernel to change process' 8960 * page table to point to a different piece of kernel memory, 8961 * but from userspace point of view memory address (and its 8962 * contents, being identical at this point) will stay the 8963 * same. This mapping will be released by bpf_object__close() 8964 * as per normal clean up procedure, so we don't need to worry 8965 * about it from skeleton's clean up perspective. 8966 */ 8967 *mmaped = mmap(map->mmaped, mmap_sz, prot, 8968 MAP_SHARED | MAP_FIXED, map_fd, 0); 8969 if (*mmaped == MAP_FAILED) { 8970 err = -errno; 8971 *mmaped = NULL; 8972 pr_warn("failed to re-mmap() map '%s': %d\n", 8973 bpf_map__name(map), err); 8974 return err; 8975 } 8976 } 8977 8978 return 0; 8979 } 8980 8981 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 8982 { 8983 int i; 8984 8985 for (i = 0; i < s->prog_cnt; i++) { 8986 struct bpf_program *prog = *s->progs[i].prog; 8987 struct bpf_link **link = s->progs[i].link; 8988 const struct bpf_sec_def *sec_def; 8989 const char *sec_name = bpf_program__title(prog, false); 8990 8991 sec_def = find_sec_def(sec_name); 8992 if (!sec_def || !sec_def->attach_fn) 8993 continue; 8994 8995 *link = sec_def->attach_fn(sec_def, prog); 8996 if (IS_ERR(*link)) { 8997 pr_warn("failed to auto-attach program '%s': %ld\n", 8998 bpf_program__name(prog), PTR_ERR(*link)); 8999 return PTR_ERR(*link); 9000 } 9001 } 9002 9003 return 0; 9004 } 9005 9006 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 9007 { 9008 int i; 9009 9010 for (i = 0; i < s->prog_cnt; i++) { 9011 struct bpf_link **link = s->progs[i].link; 9012 9013 if (!IS_ERR_OR_NULL(*link)) 9014 bpf_link__destroy(*link); 9015 *link = NULL; 9016 } 9017 } 9018 9019 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 9020 { 9021 if (s->progs) 9022 bpf_object__detach_skeleton(s); 9023 if (s->obj) 9024 bpf_object__close(*s->obj); 9025 free(s->maps); 9026 free(s->progs); 9027 free(s); 9028 } 9029