1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/epoll.h> 36 #include <sys/ioctl.h> 37 #include <sys/mman.h> 38 #include <sys/stat.h> 39 #include <sys/types.h> 40 #include <sys/vfs.h> 41 #include <tools/libc_compat.h> 42 #include <libelf.h> 43 #include <gelf.h> 44 45 #include "libbpf.h" 46 #include "bpf.h" 47 #include "btf.h" 48 #include "str_error.h" 49 #include "libbpf_internal.h" 50 51 #ifndef EM_BPF 52 #define EM_BPF 247 53 #endif 54 55 #ifndef BPF_FS_MAGIC 56 #define BPF_FS_MAGIC 0xcafe4a11 57 #endif 58 59 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 60 * compilation if user enables corresponding warning. Disable it explicitly. 61 */ 62 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 63 64 #define __printf(a, b) __attribute__((format(printf, a, b))) 65 66 static int __base_pr(enum libbpf_print_level level, const char *format, 67 va_list args) 68 { 69 if (level == LIBBPF_DEBUG) 70 return 0; 71 72 return vfprintf(stderr, format, args); 73 } 74 75 static libbpf_print_fn_t __libbpf_pr = __base_pr; 76 77 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 78 { 79 libbpf_print_fn_t old_print_fn = __libbpf_pr; 80 81 __libbpf_pr = fn; 82 return old_print_fn; 83 } 84 85 __printf(2, 3) 86 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 87 { 88 va_list args; 89 90 if (!__libbpf_pr) 91 return; 92 93 va_start(args, format); 94 __libbpf_pr(level, format, args); 95 va_end(args); 96 } 97 98 #define STRERR_BUFSIZE 128 99 100 #define CHECK_ERR(action, err, out) do { \ 101 err = action; \ 102 if (err) \ 103 goto out; \ 104 } while(0) 105 106 107 /* Copied from tools/perf/util/util.h */ 108 #ifndef zfree 109 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 110 #endif 111 112 #ifndef zclose 113 # define zclose(fd) ({ \ 114 int ___err = 0; \ 115 if ((fd) >= 0) \ 116 ___err = close((fd)); \ 117 fd = -1; \ 118 ___err; }) 119 #endif 120 121 #ifdef HAVE_LIBELF_MMAP_SUPPORT 122 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 123 #else 124 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 125 #endif 126 127 static inline __u64 ptr_to_u64(const void *ptr) 128 { 129 return (__u64) (unsigned long) ptr; 130 } 131 132 struct bpf_capabilities { 133 /* v4.14: kernel support for program & map names. */ 134 __u32 name:1; 135 /* v5.2: kernel support for global data sections. */ 136 __u32 global_data:1; 137 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 138 __u32 btf_func:1; 139 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 140 __u32 btf_datasec:1; 141 }; 142 143 /* 144 * bpf_prog should be a better name but it has been used in 145 * linux/filter.h. 146 */ 147 struct bpf_program { 148 /* Index in elf obj file, for relocation use. */ 149 int idx; 150 char *name; 151 int prog_ifindex; 152 char *section_name; 153 /* section_name with / replaced by _; makes recursive pinning 154 * in bpf_object__pin_programs easier 155 */ 156 char *pin_name; 157 struct bpf_insn *insns; 158 size_t insns_cnt, main_prog_cnt; 159 enum bpf_prog_type type; 160 161 struct reloc_desc { 162 enum { 163 RELO_LD64, 164 RELO_CALL, 165 RELO_DATA, 166 } type; 167 int insn_idx; 168 union { 169 int map_idx; 170 int text_off; 171 }; 172 } *reloc_desc; 173 int nr_reloc; 174 int log_level; 175 176 struct { 177 int nr; 178 int *fds; 179 } instances; 180 bpf_program_prep_t preprocessor; 181 182 struct bpf_object *obj; 183 void *priv; 184 bpf_program_clear_priv_t clear_priv; 185 186 enum bpf_attach_type expected_attach_type; 187 int btf_fd; 188 void *func_info; 189 __u32 func_info_rec_size; 190 __u32 func_info_cnt; 191 192 struct bpf_capabilities *caps; 193 194 void *line_info; 195 __u32 line_info_rec_size; 196 __u32 line_info_cnt; 197 __u32 prog_flags; 198 }; 199 200 enum libbpf_map_type { 201 LIBBPF_MAP_UNSPEC, 202 LIBBPF_MAP_DATA, 203 LIBBPF_MAP_BSS, 204 LIBBPF_MAP_RODATA, 205 }; 206 207 static const char * const libbpf_type_to_btf_name[] = { 208 [LIBBPF_MAP_DATA] = ".data", 209 [LIBBPF_MAP_BSS] = ".bss", 210 [LIBBPF_MAP_RODATA] = ".rodata", 211 }; 212 213 struct bpf_map { 214 int fd; 215 char *name; 216 int sec_idx; 217 size_t sec_offset; 218 int map_ifindex; 219 int inner_map_fd; 220 struct bpf_map_def def; 221 __u32 btf_key_type_id; 222 __u32 btf_value_type_id; 223 void *priv; 224 bpf_map_clear_priv_t clear_priv; 225 enum libbpf_map_type libbpf_type; 226 }; 227 228 struct bpf_secdata { 229 void *rodata; 230 void *data; 231 }; 232 233 static LIST_HEAD(bpf_objects_list); 234 235 struct bpf_object { 236 char name[BPF_OBJ_NAME_LEN]; 237 char license[64]; 238 __u32 kern_version; 239 240 struct bpf_program *programs; 241 size_t nr_programs; 242 struct bpf_map *maps; 243 size_t nr_maps; 244 size_t maps_cap; 245 struct bpf_secdata sections; 246 247 bool loaded; 248 bool has_pseudo_calls; 249 250 /* 251 * Information when doing elf related work. Only valid if fd 252 * is valid. 253 */ 254 struct { 255 int fd; 256 void *obj_buf; 257 size_t obj_buf_sz; 258 Elf *elf; 259 GElf_Ehdr ehdr; 260 Elf_Data *symbols; 261 Elf_Data *data; 262 Elf_Data *rodata; 263 Elf_Data *bss; 264 size_t strtabidx; 265 struct { 266 GElf_Shdr shdr; 267 Elf_Data *data; 268 } *reloc; 269 int nr_reloc; 270 int maps_shndx; 271 int btf_maps_shndx; 272 int text_shndx; 273 int data_shndx; 274 int rodata_shndx; 275 int bss_shndx; 276 } efile; 277 /* 278 * All loaded bpf_object is linked in a list, which is 279 * hidden to caller. bpf_objects__<func> handlers deal with 280 * all objects. 281 */ 282 struct list_head list; 283 284 struct btf *btf; 285 struct btf_ext *btf_ext; 286 287 void *priv; 288 bpf_object_clear_priv_t clear_priv; 289 290 struct bpf_capabilities caps; 291 292 char path[]; 293 }; 294 #define obj_elf_valid(o) ((o)->efile.elf) 295 296 void bpf_program__unload(struct bpf_program *prog) 297 { 298 int i; 299 300 if (!prog) 301 return; 302 303 /* 304 * If the object is opened but the program was never loaded, 305 * it is possible that prog->instances.nr == -1. 306 */ 307 if (prog->instances.nr > 0) { 308 for (i = 0; i < prog->instances.nr; i++) 309 zclose(prog->instances.fds[i]); 310 } else if (prog->instances.nr != -1) { 311 pr_warning("Internal error: instances.nr is %d\n", 312 prog->instances.nr); 313 } 314 315 prog->instances.nr = -1; 316 zfree(&prog->instances.fds); 317 318 zclose(prog->btf_fd); 319 zfree(&prog->func_info); 320 zfree(&prog->line_info); 321 } 322 323 static void bpf_program__exit(struct bpf_program *prog) 324 { 325 if (!prog) 326 return; 327 328 if (prog->clear_priv) 329 prog->clear_priv(prog, prog->priv); 330 331 prog->priv = NULL; 332 prog->clear_priv = NULL; 333 334 bpf_program__unload(prog); 335 zfree(&prog->name); 336 zfree(&prog->section_name); 337 zfree(&prog->pin_name); 338 zfree(&prog->insns); 339 zfree(&prog->reloc_desc); 340 341 prog->nr_reloc = 0; 342 prog->insns_cnt = 0; 343 prog->idx = -1; 344 } 345 346 static char *__bpf_program__pin_name(struct bpf_program *prog) 347 { 348 char *name, *p; 349 350 name = p = strdup(prog->section_name); 351 while ((p = strchr(p, '/'))) 352 *p = '_'; 353 354 return name; 355 } 356 357 static int 358 bpf_program__init(void *data, size_t size, char *section_name, int idx, 359 struct bpf_program *prog) 360 { 361 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 362 363 if (size == 0 || size % bpf_insn_sz) { 364 pr_warning("corrupted section '%s', size: %zu\n", 365 section_name, size); 366 return -EINVAL; 367 } 368 369 memset(prog, 0, sizeof(*prog)); 370 371 prog->section_name = strdup(section_name); 372 if (!prog->section_name) { 373 pr_warning("failed to alloc name for prog under section(%d) %s\n", 374 idx, section_name); 375 goto errout; 376 } 377 378 prog->pin_name = __bpf_program__pin_name(prog); 379 if (!prog->pin_name) { 380 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 381 idx, section_name); 382 goto errout; 383 } 384 385 prog->insns = malloc(size); 386 if (!prog->insns) { 387 pr_warning("failed to alloc insns for prog under section %s\n", 388 section_name); 389 goto errout; 390 } 391 prog->insns_cnt = size / bpf_insn_sz; 392 memcpy(prog->insns, data, size); 393 prog->idx = idx; 394 prog->instances.fds = NULL; 395 prog->instances.nr = -1; 396 prog->type = BPF_PROG_TYPE_UNSPEC; 397 prog->btf_fd = -1; 398 399 return 0; 400 errout: 401 bpf_program__exit(prog); 402 return -ENOMEM; 403 } 404 405 static int 406 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 407 char *section_name, int idx) 408 { 409 struct bpf_program prog, *progs; 410 int nr_progs, err; 411 412 err = bpf_program__init(data, size, section_name, idx, &prog); 413 if (err) 414 return err; 415 416 prog.caps = &obj->caps; 417 progs = obj->programs; 418 nr_progs = obj->nr_programs; 419 420 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 421 if (!progs) { 422 /* 423 * In this case the original obj->programs 424 * is still valid, so don't need special treat for 425 * bpf_close_object(). 426 */ 427 pr_warning("failed to alloc a new program under section '%s'\n", 428 section_name); 429 bpf_program__exit(&prog); 430 return -ENOMEM; 431 } 432 433 pr_debug("found program %s\n", prog.section_name); 434 obj->programs = progs; 435 obj->nr_programs = nr_progs + 1; 436 prog.obj = obj; 437 progs[nr_progs] = prog; 438 return 0; 439 } 440 441 static int 442 bpf_object__init_prog_names(struct bpf_object *obj) 443 { 444 Elf_Data *symbols = obj->efile.symbols; 445 struct bpf_program *prog; 446 size_t pi, si; 447 448 for (pi = 0; pi < obj->nr_programs; pi++) { 449 const char *name = NULL; 450 451 prog = &obj->programs[pi]; 452 453 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 454 si++) { 455 GElf_Sym sym; 456 457 if (!gelf_getsym(symbols, si, &sym)) 458 continue; 459 if (sym.st_shndx != prog->idx) 460 continue; 461 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 462 continue; 463 464 name = elf_strptr(obj->efile.elf, 465 obj->efile.strtabidx, 466 sym.st_name); 467 if (!name) { 468 pr_warning("failed to get sym name string for prog %s\n", 469 prog->section_name); 470 return -LIBBPF_ERRNO__LIBELF; 471 } 472 } 473 474 if (!name && prog->idx == obj->efile.text_shndx) 475 name = ".text"; 476 477 if (!name) { 478 pr_warning("failed to find sym for prog %s\n", 479 prog->section_name); 480 return -EINVAL; 481 } 482 483 prog->name = strdup(name); 484 if (!prog->name) { 485 pr_warning("failed to allocate memory for prog sym %s\n", 486 name); 487 return -ENOMEM; 488 } 489 } 490 491 return 0; 492 } 493 494 static struct bpf_object *bpf_object__new(const char *path, 495 void *obj_buf, 496 size_t obj_buf_sz) 497 { 498 struct bpf_object *obj; 499 char *end; 500 501 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 502 if (!obj) { 503 pr_warning("alloc memory failed for %s\n", path); 504 return ERR_PTR(-ENOMEM); 505 } 506 507 strcpy(obj->path, path); 508 /* Using basename() GNU version which doesn't modify arg. */ 509 strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); 510 end = strchr(obj->name, '.'); 511 if (end) 512 *end = 0; 513 514 obj->efile.fd = -1; 515 /* 516 * Caller of this function should also call 517 * bpf_object__elf_finish() after data collection to return 518 * obj_buf to user. If not, we should duplicate the buffer to 519 * avoid user freeing them before elf finish. 520 */ 521 obj->efile.obj_buf = obj_buf; 522 obj->efile.obj_buf_sz = obj_buf_sz; 523 obj->efile.maps_shndx = -1; 524 obj->efile.btf_maps_shndx = -1; 525 obj->efile.data_shndx = -1; 526 obj->efile.rodata_shndx = -1; 527 obj->efile.bss_shndx = -1; 528 529 obj->loaded = false; 530 531 INIT_LIST_HEAD(&obj->list); 532 list_add(&obj->list, &bpf_objects_list); 533 return obj; 534 } 535 536 static void bpf_object__elf_finish(struct bpf_object *obj) 537 { 538 if (!obj_elf_valid(obj)) 539 return; 540 541 if (obj->efile.elf) { 542 elf_end(obj->efile.elf); 543 obj->efile.elf = NULL; 544 } 545 obj->efile.symbols = NULL; 546 obj->efile.data = NULL; 547 obj->efile.rodata = NULL; 548 obj->efile.bss = NULL; 549 550 zfree(&obj->efile.reloc); 551 obj->efile.nr_reloc = 0; 552 zclose(obj->efile.fd); 553 obj->efile.obj_buf = NULL; 554 obj->efile.obj_buf_sz = 0; 555 } 556 557 static int bpf_object__elf_init(struct bpf_object *obj) 558 { 559 int err = 0; 560 GElf_Ehdr *ep; 561 562 if (obj_elf_valid(obj)) { 563 pr_warning("elf init: internal error\n"); 564 return -LIBBPF_ERRNO__LIBELF; 565 } 566 567 if (obj->efile.obj_buf_sz > 0) { 568 /* 569 * obj_buf should have been validated by 570 * bpf_object__open_buffer(). 571 */ 572 obj->efile.elf = elf_memory(obj->efile.obj_buf, 573 obj->efile.obj_buf_sz); 574 } else { 575 obj->efile.fd = open(obj->path, O_RDONLY); 576 if (obj->efile.fd < 0) { 577 char errmsg[STRERR_BUFSIZE], *cp; 578 579 err = -errno; 580 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 581 pr_warning("failed to open %s: %s\n", obj->path, cp); 582 return err; 583 } 584 585 obj->efile.elf = elf_begin(obj->efile.fd, 586 LIBBPF_ELF_C_READ_MMAP, NULL); 587 } 588 589 if (!obj->efile.elf) { 590 pr_warning("failed to open %s as ELF file\n", obj->path); 591 err = -LIBBPF_ERRNO__LIBELF; 592 goto errout; 593 } 594 595 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 596 pr_warning("failed to get EHDR from %s\n", obj->path); 597 err = -LIBBPF_ERRNO__FORMAT; 598 goto errout; 599 } 600 ep = &obj->efile.ehdr; 601 602 /* Old LLVM set e_machine to EM_NONE */ 603 if (ep->e_type != ET_REL || 604 (ep->e_machine && ep->e_machine != EM_BPF)) { 605 pr_warning("%s is not an eBPF object file\n", obj->path); 606 err = -LIBBPF_ERRNO__FORMAT; 607 goto errout; 608 } 609 610 return 0; 611 errout: 612 bpf_object__elf_finish(obj); 613 return err; 614 } 615 616 static int bpf_object__check_endianness(struct bpf_object *obj) 617 { 618 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 619 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 620 return 0; 621 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 622 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 623 return 0; 624 #else 625 # error "Unrecognized __BYTE_ORDER__" 626 #endif 627 pr_warning("endianness mismatch.\n"); 628 return -LIBBPF_ERRNO__ENDIAN; 629 } 630 631 static int 632 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 633 { 634 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 635 pr_debug("license of %s is %s\n", obj->path, obj->license); 636 return 0; 637 } 638 639 static int 640 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 641 { 642 __u32 kver; 643 644 if (size != sizeof(kver)) { 645 pr_warning("invalid kver section in %s\n", obj->path); 646 return -LIBBPF_ERRNO__FORMAT; 647 } 648 memcpy(&kver, data, sizeof(kver)); 649 obj->kern_version = kver; 650 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 651 return 0; 652 } 653 654 static int compare_bpf_map(const void *_a, const void *_b) 655 { 656 const struct bpf_map *a = _a; 657 const struct bpf_map *b = _b; 658 659 if (a->sec_idx != b->sec_idx) 660 return a->sec_idx - b->sec_idx; 661 return a->sec_offset - b->sec_offset; 662 } 663 664 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 665 { 666 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 667 type == BPF_MAP_TYPE_HASH_OF_MAPS) 668 return true; 669 return false; 670 } 671 672 static int bpf_object_search_section_size(const struct bpf_object *obj, 673 const char *name, size_t *d_size) 674 { 675 const GElf_Ehdr *ep = &obj->efile.ehdr; 676 Elf *elf = obj->efile.elf; 677 Elf_Scn *scn = NULL; 678 int idx = 0; 679 680 while ((scn = elf_nextscn(elf, scn)) != NULL) { 681 const char *sec_name; 682 Elf_Data *data; 683 GElf_Shdr sh; 684 685 idx++; 686 if (gelf_getshdr(scn, &sh) != &sh) { 687 pr_warning("failed to get section(%d) header from %s\n", 688 idx, obj->path); 689 return -EIO; 690 } 691 692 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 693 if (!sec_name) { 694 pr_warning("failed to get section(%d) name from %s\n", 695 idx, obj->path); 696 return -EIO; 697 } 698 699 if (strcmp(name, sec_name)) 700 continue; 701 702 data = elf_getdata(scn, 0); 703 if (!data) { 704 pr_warning("failed to get section(%d) data from %s(%s)\n", 705 idx, name, obj->path); 706 return -EIO; 707 } 708 709 *d_size = data->d_size; 710 return 0; 711 } 712 713 return -ENOENT; 714 } 715 716 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 717 __u32 *size) 718 { 719 int ret = -ENOENT; 720 size_t d_size; 721 722 *size = 0; 723 if (!name) { 724 return -EINVAL; 725 } else if (!strcmp(name, ".data")) { 726 if (obj->efile.data) 727 *size = obj->efile.data->d_size; 728 } else if (!strcmp(name, ".bss")) { 729 if (obj->efile.bss) 730 *size = obj->efile.bss->d_size; 731 } else if (!strcmp(name, ".rodata")) { 732 if (obj->efile.rodata) 733 *size = obj->efile.rodata->d_size; 734 } else { 735 ret = bpf_object_search_section_size(obj, name, &d_size); 736 if (!ret) 737 *size = d_size; 738 } 739 740 return *size ? 0 : ret; 741 } 742 743 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 744 __u32 *off) 745 { 746 Elf_Data *symbols = obj->efile.symbols; 747 const char *sname; 748 size_t si; 749 750 if (!name || !off) 751 return -EINVAL; 752 753 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 754 GElf_Sym sym; 755 756 if (!gelf_getsym(symbols, si, &sym)) 757 continue; 758 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 759 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 760 continue; 761 762 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 763 sym.st_name); 764 if (!sname) { 765 pr_warning("failed to get sym name string for var %s\n", 766 name); 767 return -EIO; 768 } 769 if (strcmp(name, sname) == 0) { 770 *off = sym.st_value; 771 return 0; 772 } 773 } 774 775 return -ENOENT; 776 } 777 778 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 779 { 780 struct bpf_map *new_maps; 781 size_t new_cap; 782 int i; 783 784 if (obj->nr_maps < obj->maps_cap) 785 return &obj->maps[obj->nr_maps++]; 786 787 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 788 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); 789 if (!new_maps) { 790 pr_warning("alloc maps for object failed\n"); 791 return ERR_PTR(-ENOMEM); 792 } 793 794 obj->maps_cap = new_cap; 795 obj->maps = new_maps; 796 797 /* zero out new maps */ 798 memset(obj->maps + obj->nr_maps, 0, 799 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 800 /* 801 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 802 * when failure (zclose won't close negative fd)). 803 */ 804 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 805 obj->maps[i].fd = -1; 806 obj->maps[i].inner_map_fd = -1; 807 } 808 809 return &obj->maps[obj->nr_maps++]; 810 } 811 812 static int 813 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 814 int sec_idx, Elf_Data *data, void **data_buff) 815 { 816 char map_name[BPF_OBJ_NAME_LEN]; 817 struct bpf_map_def *def; 818 struct bpf_map *map; 819 820 map = bpf_object__add_map(obj); 821 if (IS_ERR(map)) 822 return PTR_ERR(map); 823 824 map->libbpf_type = type; 825 map->sec_idx = sec_idx; 826 map->sec_offset = 0; 827 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 828 libbpf_type_to_btf_name[type]); 829 map->name = strdup(map_name); 830 if (!map->name) { 831 pr_warning("failed to alloc map name\n"); 832 return -ENOMEM; 833 } 834 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", 835 map_name, map->sec_idx, map->sec_offset); 836 837 def = &map->def; 838 def->type = BPF_MAP_TYPE_ARRAY; 839 def->key_size = sizeof(int); 840 def->value_size = data->d_size; 841 def->max_entries = 1; 842 def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; 843 if (data_buff) { 844 *data_buff = malloc(data->d_size); 845 if (!*data_buff) { 846 zfree(&map->name); 847 pr_warning("failed to alloc map content buffer\n"); 848 return -ENOMEM; 849 } 850 memcpy(*data_buff, data->d_buf, data->d_size); 851 } 852 853 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 854 return 0; 855 } 856 857 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 858 { 859 int err; 860 861 if (!obj->caps.global_data) 862 return 0; 863 /* 864 * Populate obj->maps with libbpf internal maps. 865 */ 866 if (obj->efile.data_shndx >= 0) { 867 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 868 obj->efile.data_shndx, 869 obj->efile.data, 870 &obj->sections.data); 871 if (err) 872 return err; 873 } 874 if (obj->efile.rodata_shndx >= 0) { 875 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 876 obj->efile.rodata_shndx, 877 obj->efile.rodata, 878 &obj->sections.rodata); 879 if (err) 880 return err; 881 } 882 if (obj->efile.bss_shndx >= 0) { 883 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 884 obj->efile.bss_shndx, 885 obj->efile.bss, NULL); 886 if (err) 887 return err; 888 } 889 return 0; 890 } 891 892 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 893 { 894 Elf_Data *symbols = obj->efile.symbols; 895 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 896 Elf_Data *data = NULL; 897 Elf_Scn *scn; 898 899 if (obj->efile.maps_shndx < 0) 900 return 0; 901 902 if (!symbols) 903 return -EINVAL; 904 905 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 906 if (scn) 907 data = elf_getdata(scn, NULL); 908 if (!scn || !data) { 909 pr_warning("failed to get Elf_Data from map section %d\n", 910 obj->efile.maps_shndx); 911 return -EINVAL; 912 } 913 914 /* 915 * Count number of maps. Each map has a name. 916 * Array of maps is not supported: only the first element is 917 * considered. 918 * 919 * TODO: Detect array of map and report error. 920 */ 921 nr_syms = symbols->d_size / sizeof(GElf_Sym); 922 for (i = 0; i < nr_syms; i++) { 923 GElf_Sym sym; 924 925 if (!gelf_getsym(symbols, i, &sym)) 926 continue; 927 if (sym.st_shndx != obj->efile.maps_shndx) 928 continue; 929 nr_maps++; 930 } 931 /* Assume equally sized map definitions */ 932 pr_debug("maps in %s: %d maps in %zd bytes\n", 933 obj->path, nr_maps, data->d_size); 934 935 map_def_sz = data->d_size / nr_maps; 936 if (!data->d_size || (data->d_size % nr_maps) != 0) { 937 pr_warning("unable to determine map definition size " 938 "section %s, %d maps in %zd bytes\n", 939 obj->path, nr_maps, data->d_size); 940 return -EINVAL; 941 } 942 943 /* Fill obj->maps using data in "maps" section. */ 944 for (i = 0; i < nr_syms; i++) { 945 GElf_Sym sym; 946 const char *map_name; 947 struct bpf_map_def *def; 948 struct bpf_map *map; 949 950 if (!gelf_getsym(symbols, i, &sym)) 951 continue; 952 if (sym.st_shndx != obj->efile.maps_shndx) 953 continue; 954 955 map = bpf_object__add_map(obj); 956 if (IS_ERR(map)) 957 return PTR_ERR(map); 958 959 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 960 sym.st_name); 961 if (!map_name) { 962 pr_warning("failed to get map #%d name sym string for obj %s\n", 963 i, obj->path); 964 return -LIBBPF_ERRNO__FORMAT; 965 } 966 967 map->libbpf_type = LIBBPF_MAP_UNSPEC; 968 map->sec_idx = sym.st_shndx; 969 map->sec_offset = sym.st_value; 970 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 971 map_name, map->sec_idx, map->sec_offset); 972 if (sym.st_value + map_def_sz > data->d_size) { 973 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 974 obj->path, map_name); 975 return -EINVAL; 976 } 977 978 map->name = strdup(map_name); 979 if (!map->name) { 980 pr_warning("failed to alloc map name\n"); 981 return -ENOMEM; 982 } 983 pr_debug("map %d is \"%s\"\n", i, map->name); 984 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 985 /* 986 * If the definition of the map in the object file fits in 987 * bpf_map_def, copy it. Any extra fields in our version 988 * of bpf_map_def will default to zero as a result of the 989 * calloc above. 990 */ 991 if (map_def_sz <= sizeof(struct bpf_map_def)) { 992 memcpy(&map->def, def, map_def_sz); 993 } else { 994 /* 995 * Here the map structure being read is bigger than what 996 * we expect, truncate if the excess bits are all zero. 997 * If they are not zero, reject this map as 998 * incompatible. 999 */ 1000 char *b; 1001 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1002 b < ((char *)def) + map_def_sz; b++) { 1003 if (*b != 0) { 1004 pr_warning("maps section in %s: \"%s\" " 1005 "has unrecognized, non-zero " 1006 "options\n", 1007 obj->path, map_name); 1008 if (strict) 1009 return -EINVAL; 1010 } 1011 } 1012 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1013 } 1014 } 1015 return 0; 1016 } 1017 1018 static const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, 1019 __u32 id) 1020 { 1021 const struct btf_type *t = btf__type_by_id(btf, id); 1022 1023 while (true) { 1024 switch (BTF_INFO_KIND(t->info)) { 1025 case BTF_KIND_VOLATILE: 1026 case BTF_KIND_CONST: 1027 case BTF_KIND_RESTRICT: 1028 case BTF_KIND_TYPEDEF: 1029 t = btf__type_by_id(btf, t->type); 1030 break; 1031 default: 1032 return t; 1033 } 1034 } 1035 } 1036 1037 /* 1038 * Fetch integer attribute of BTF map definition. Such attributes are 1039 * represented using a pointer to an array, in which dimensionality of array 1040 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 1041 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 1042 * type definition, while using only sizeof(void *) space in ELF data section. 1043 */ 1044 static bool get_map_field_int(const char *map_name, const struct btf *btf, 1045 const struct btf_type *def, 1046 const struct btf_member *m, __u32 *res) { 1047 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type); 1048 const char *name = btf__name_by_offset(btf, m->name_off); 1049 const struct btf_array *arr_info; 1050 const struct btf_type *arr_t; 1051 1052 if (!btf_is_ptr(t)) { 1053 pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", 1054 map_name, name, btf_kind(t)); 1055 return false; 1056 } 1057 1058 arr_t = btf__type_by_id(btf, t->type); 1059 if (!arr_t) { 1060 pr_warning("map '%s': attr '%s': type [%u] not found.\n", 1061 map_name, name, t->type); 1062 return false; 1063 } 1064 if (!btf_is_array(arr_t)) { 1065 pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", 1066 map_name, name, btf_kind(arr_t)); 1067 return false; 1068 } 1069 arr_info = btf_array(arr_t); 1070 *res = arr_info->nelems; 1071 return true; 1072 } 1073 1074 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 1075 const struct btf_type *sec, 1076 int var_idx, int sec_idx, 1077 const Elf_Data *data, bool strict) 1078 { 1079 const struct btf_type *var, *def, *t; 1080 const struct btf_var_secinfo *vi; 1081 const struct btf_var *var_extra; 1082 const struct btf_member *m; 1083 const char *map_name; 1084 struct bpf_map *map; 1085 int vlen, i; 1086 1087 vi = btf_var_secinfos(sec) + var_idx; 1088 var = btf__type_by_id(obj->btf, vi->type); 1089 var_extra = btf_var(var); 1090 map_name = btf__name_by_offset(obj->btf, var->name_off); 1091 vlen = btf_vlen(var); 1092 1093 if (map_name == NULL || map_name[0] == '\0') { 1094 pr_warning("map #%d: empty name.\n", var_idx); 1095 return -EINVAL; 1096 } 1097 if ((__u64)vi->offset + vi->size > data->d_size) { 1098 pr_warning("map '%s' BTF data is corrupted.\n", map_name); 1099 return -EINVAL; 1100 } 1101 if (!btf_is_var(var)) { 1102 pr_warning("map '%s': unexpected var kind %u.\n", 1103 map_name, btf_kind(var)); 1104 return -EINVAL; 1105 } 1106 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && 1107 var_extra->linkage != BTF_VAR_STATIC) { 1108 pr_warning("map '%s': unsupported var linkage %u.\n", 1109 map_name, var_extra->linkage); 1110 return -EOPNOTSUPP; 1111 } 1112 1113 def = skip_mods_and_typedefs(obj->btf, var->type); 1114 if (!btf_is_struct(def)) { 1115 pr_warning("map '%s': unexpected def kind %u.\n", 1116 map_name, btf_kind(var)); 1117 return -EINVAL; 1118 } 1119 if (def->size > vi->size) { 1120 pr_warning("map '%s': invalid def size.\n", map_name); 1121 return -EINVAL; 1122 } 1123 1124 map = bpf_object__add_map(obj); 1125 if (IS_ERR(map)) 1126 return PTR_ERR(map); 1127 map->name = strdup(map_name); 1128 if (!map->name) { 1129 pr_warning("map '%s': failed to alloc map name.\n", map_name); 1130 return -ENOMEM; 1131 } 1132 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1133 map->def.type = BPF_MAP_TYPE_UNSPEC; 1134 map->sec_idx = sec_idx; 1135 map->sec_offset = vi->offset; 1136 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 1137 map_name, map->sec_idx, map->sec_offset); 1138 1139 vlen = btf_vlen(def); 1140 m = btf_members(def); 1141 for (i = 0; i < vlen; i++, m++) { 1142 const char *name = btf__name_by_offset(obj->btf, m->name_off); 1143 1144 if (!name) { 1145 pr_warning("map '%s': invalid field #%d.\n", 1146 map_name, i); 1147 return -EINVAL; 1148 } 1149 if (strcmp(name, "type") == 0) { 1150 if (!get_map_field_int(map_name, obj->btf, def, m, 1151 &map->def.type)) 1152 return -EINVAL; 1153 pr_debug("map '%s': found type = %u.\n", 1154 map_name, map->def.type); 1155 } else if (strcmp(name, "max_entries") == 0) { 1156 if (!get_map_field_int(map_name, obj->btf, def, m, 1157 &map->def.max_entries)) 1158 return -EINVAL; 1159 pr_debug("map '%s': found max_entries = %u.\n", 1160 map_name, map->def.max_entries); 1161 } else if (strcmp(name, "map_flags") == 0) { 1162 if (!get_map_field_int(map_name, obj->btf, def, m, 1163 &map->def.map_flags)) 1164 return -EINVAL; 1165 pr_debug("map '%s': found map_flags = %u.\n", 1166 map_name, map->def.map_flags); 1167 } else if (strcmp(name, "key_size") == 0) { 1168 __u32 sz; 1169 1170 if (!get_map_field_int(map_name, obj->btf, def, m, 1171 &sz)) 1172 return -EINVAL; 1173 pr_debug("map '%s': found key_size = %u.\n", 1174 map_name, sz); 1175 if (map->def.key_size && map->def.key_size != sz) { 1176 pr_warning("map '%s': conflicting key size %u != %u.\n", 1177 map_name, map->def.key_size, sz); 1178 return -EINVAL; 1179 } 1180 map->def.key_size = sz; 1181 } else if (strcmp(name, "key") == 0) { 1182 __s64 sz; 1183 1184 t = btf__type_by_id(obj->btf, m->type); 1185 if (!t) { 1186 pr_warning("map '%s': key type [%d] not found.\n", 1187 map_name, m->type); 1188 return -EINVAL; 1189 } 1190 if (!btf_is_ptr(t)) { 1191 pr_warning("map '%s': key spec is not PTR: %u.\n", 1192 map_name, btf_kind(t)); 1193 return -EINVAL; 1194 } 1195 sz = btf__resolve_size(obj->btf, t->type); 1196 if (sz < 0) { 1197 pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", 1198 map_name, t->type, sz); 1199 return sz; 1200 } 1201 pr_debug("map '%s': found key [%u], sz = %lld.\n", 1202 map_name, t->type, sz); 1203 if (map->def.key_size && map->def.key_size != sz) { 1204 pr_warning("map '%s': conflicting key size %u != %lld.\n", 1205 map_name, map->def.key_size, sz); 1206 return -EINVAL; 1207 } 1208 map->def.key_size = sz; 1209 map->btf_key_type_id = t->type; 1210 } else if (strcmp(name, "value_size") == 0) { 1211 __u32 sz; 1212 1213 if (!get_map_field_int(map_name, obj->btf, def, m, 1214 &sz)) 1215 return -EINVAL; 1216 pr_debug("map '%s': found value_size = %u.\n", 1217 map_name, sz); 1218 if (map->def.value_size && map->def.value_size != sz) { 1219 pr_warning("map '%s': conflicting value size %u != %u.\n", 1220 map_name, map->def.value_size, sz); 1221 return -EINVAL; 1222 } 1223 map->def.value_size = sz; 1224 } else if (strcmp(name, "value") == 0) { 1225 __s64 sz; 1226 1227 t = btf__type_by_id(obj->btf, m->type); 1228 if (!t) { 1229 pr_warning("map '%s': value type [%d] not found.\n", 1230 map_name, m->type); 1231 return -EINVAL; 1232 } 1233 if (!btf_is_ptr(t)) { 1234 pr_warning("map '%s': value spec is not PTR: %u.\n", 1235 map_name, btf_kind(t)); 1236 return -EINVAL; 1237 } 1238 sz = btf__resolve_size(obj->btf, t->type); 1239 if (sz < 0) { 1240 pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", 1241 map_name, t->type, sz); 1242 return sz; 1243 } 1244 pr_debug("map '%s': found value [%u], sz = %lld.\n", 1245 map_name, t->type, sz); 1246 if (map->def.value_size && map->def.value_size != sz) { 1247 pr_warning("map '%s': conflicting value size %u != %lld.\n", 1248 map_name, map->def.value_size, sz); 1249 return -EINVAL; 1250 } 1251 map->def.value_size = sz; 1252 map->btf_value_type_id = t->type; 1253 } else { 1254 if (strict) { 1255 pr_warning("map '%s': unknown field '%s'.\n", 1256 map_name, name); 1257 return -ENOTSUP; 1258 } 1259 pr_debug("map '%s': ignoring unknown field '%s'.\n", 1260 map_name, name); 1261 } 1262 } 1263 1264 if (map->def.type == BPF_MAP_TYPE_UNSPEC) { 1265 pr_warning("map '%s': map type isn't specified.\n", map_name); 1266 return -EINVAL; 1267 } 1268 1269 return 0; 1270 } 1271 1272 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) 1273 { 1274 const struct btf_type *sec = NULL; 1275 int nr_types, i, vlen, err; 1276 const struct btf_type *t; 1277 const char *name; 1278 Elf_Data *data; 1279 Elf_Scn *scn; 1280 1281 if (obj->efile.btf_maps_shndx < 0) 1282 return 0; 1283 1284 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); 1285 if (scn) 1286 data = elf_getdata(scn, NULL); 1287 if (!scn || !data) { 1288 pr_warning("failed to get Elf_Data from map section %d (%s)\n", 1289 obj->efile.maps_shndx, MAPS_ELF_SEC); 1290 return -EINVAL; 1291 } 1292 1293 nr_types = btf__get_nr_types(obj->btf); 1294 for (i = 1; i <= nr_types; i++) { 1295 t = btf__type_by_id(obj->btf, i); 1296 if (!btf_is_datasec(t)) 1297 continue; 1298 name = btf__name_by_offset(obj->btf, t->name_off); 1299 if (strcmp(name, MAPS_ELF_SEC) == 0) { 1300 sec = t; 1301 break; 1302 } 1303 } 1304 1305 if (!sec) { 1306 pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 1307 return -ENOENT; 1308 } 1309 1310 vlen = btf_vlen(sec); 1311 for (i = 0; i < vlen; i++) { 1312 err = bpf_object__init_user_btf_map(obj, sec, i, 1313 obj->efile.btf_maps_shndx, 1314 data, strict); 1315 if (err) 1316 return err; 1317 } 1318 1319 return 0; 1320 } 1321 1322 static int bpf_object__init_maps(struct bpf_object *obj, int flags) 1323 { 1324 bool strict = !(flags & MAPS_RELAX_COMPAT); 1325 int err; 1326 1327 err = bpf_object__init_user_maps(obj, strict); 1328 if (err) 1329 return err; 1330 1331 err = bpf_object__init_user_btf_maps(obj, strict); 1332 if (err) 1333 return err; 1334 1335 err = bpf_object__init_global_data_maps(obj); 1336 if (err) 1337 return err; 1338 1339 if (obj->nr_maps) { 1340 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1341 compare_bpf_map); 1342 } 1343 return 0; 1344 } 1345 1346 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1347 { 1348 Elf_Scn *scn; 1349 GElf_Shdr sh; 1350 1351 scn = elf_getscn(obj->efile.elf, idx); 1352 if (!scn) 1353 return false; 1354 1355 if (gelf_getshdr(scn, &sh) != &sh) 1356 return false; 1357 1358 if (sh.sh_flags & SHF_EXECINSTR) 1359 return true; 1360 1361 return false; 1362 } 1363 1364 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1365 { 1366 bool has_datasec = obj->caps.btf_datasec; 1367 bool has_func = obj->caps.btf_func; 1368 struct btf *btf = obj->btf; 1369 struct btf_type *t; 1370 int i, j, vlen; 1371 1372 if (!obj->btf || (has_func && has_datasec)) 1373 return; 1374 1375 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1376 t = (struct btf_type *)btf__type_by_id(btf, i); 1377 1378 if (!has_datasec && btf_is_var(t)) { 1379 /* replace VAR with INT */ 1380 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1381 t->size = sizeof(int); 1382 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 32); 1383 } else if (!has_datasec && btf_is_datasec(t)) { 1384 /* replace DATASEC with STRUCT */ 1385 const struct btf_var_secinfo *v = btf_var_secinfos(t); 1386 struct btf_member *m = btf_members(t); 1387 struct btf_type *vt; 1388 char *name; 1389 1390 name = (char *)btf__name_by_offset(btf, t->name_off); 1391 while (*name) { 1392 if (*name == '.') 1393 *name = '_'; 1394 name++; 1395 } 1396 1397 vlen = btf_vlen(t); 1398 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1399 for (j = 0; j < vlen; j++, v++, m++) { 1400 /* order of field assignments is important */ 1401 m->offset = v->offset * 8; 1402 m->type = v->type; 1403 /* preserve variable name as member name */ 1404 vt = (void *)btf__type_by_id(btf, v->type); 1405 m->name_off = vt->name_off; 1406 } 1407 } else if (!has_func && btf_is_func_proto(t)) { 1408 /* replace FUNC_PROTO with ENUM */ 1409 vlen = btf_vlen(t); 1410 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1411 t->size = sizeof(__u32); /* kernel enforced */ 1412 } else if (!has_func && btf_is_func(t)) { 1413 /* replace FUNC with TYPEDEF */ 1414 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1415 } 1416 } 1417 } 1418 1419 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1420 { 1421 if (!obj->btf_ext) 1422 return; 1423 1424 if (!obj->caps.btf_func) { 1425 btf_ext__free(obj->btf_ext); 1426 obj->btf_ext = NULL; 1427 } 1428 } 1429 1430 static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) 1431 { 1432 return obj->efile.btf_maps_shndx >= 0; 1433 } 1434 1435 static int bpf_object__init_btf(struct bpf_object *obj, 1436 Elf_Data *btf_data, 1437 Elf_Data *btf_ext_data) 1438 { 1439 bool btf_required = bpf_object__is_btf_mandatory(obj); 1440 int err = 0; 1441 1442 if (btf_data) { 1443 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1444 if (IS_ERR(obj->btf)) { 1445 pr_warning("Error loading ELF section %s: %d.\n", 1446 BTF_ELF_SEC, err); 1447 goto out; 1448 } 1449 err = btf__finalize_data(obj, obj->btf); 1450 if (err) { 1451 pr_warning("Error finalizing %s: %d.\n", 1452 BTF_ELF_SEC, err); 1453 goto out; 1454 } 1455 } 1456 if (btf_ext_data) { 1457 if (!obj->btf) { 1458 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1459 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1460 goto out; 1461 } 1462 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1463 btf_ext_data->d_size); 1464 if (IS_ERR(obj->btf_ext)) { 1465 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1466 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); 1467 obj->btf_ext = NULL; 1468 goto out; 1469 } 1470 } 1471 out: 1472 if (err || IS_ERR(obj->btf)) { 1473 if (btf_required) 1474 err = err ? : PTR_ERR(obj->btf); 1475 else 1476 err = 0; 1477 if (!IS_ERR_OR_NULL(obj->btf)) 1478 btf__free(obj->btf); 1479 obj->btf = NULL; 1480 } 1481 if (btf_required && !obj->btf) { 1482 pr_warning("BTF is required, but is missing or corrupted.\n"); 1483 return err == 0 ? -ENOENT : err; 1484 } 1485 return 0; 1486 } 1487 1488 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 1489 { 1490 int err = 0; 1491 1492 if (!obj->btf) 1493 return 0; 1494 1495 bpf_object__sanitize_btf(obj); 1496 bpf_object__sanitize_btf_ext(obj); 1497 1498 err = btf__load(obj->btf); 1499 if (err) { 1500 pr_warning("Error loading %s into kernel: %d.\n", 1501 BTF_ELF_SEC, err); 1502 btf__free(obj->btf); 1503 obj->btf = NULL; 1504 if (bpf_object__is_btf_mandatory(obj)) 1505 return err; 1506 } 1507 return 0; 1508 } 1509 1510 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1511 { 1512 Elf *elf = obj->efile.elf; 1513 GElf_Ehdr *ep = &obj->efile.ehdr; 1514 Elf_Data *btf_ext_data = NULL; 1515 Elf_Data *btf_data = NULL; 1516 Elf_Scn *scn = NULL; 1517 int idx = 0, err = 0; 1518 1519 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1520 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1521 pr_warning("failed to get e_shstrndx from %s\n", obj->path); 1522 return -LIBBPF_ERRNO__FORMAT; 1523 } 1524 1525 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1526 char *name; 1527 GElf_Shdr sh; 1528 Elf_Data *data; 1529 1530 idx++; 1531 if (gelf_getshdr(scn, &sh) != &sh) { 1532 pr_warning("failed to get section(%d) header from %s\n", 1533 idx, obj->path); 1534 return -LIBBPF_ERRNO__FORMAT; 1535 } 1536 1537 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1538 if (!name) { 1539 pr_warning("failed to get section(%d) name from %s\n", 1540 idx, obj->path); 1541 return -LIBBPF_ERRNO__FORMAT; 1542 } 1543 1544 data = elf_getdata(scn, 0); 1545 if (!data) { 1546 pr_warning("failed to get section(%d) data from %s(%s)\n", 1547 idx, name, obj->path); 1548 return -LIBBPF_ERRNO__FORMAT; 1549 } 1550 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1551 idx, name, (unsigned long)data->d_size, 1552 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1553 (int)sh.sh_type); 1554 1555 if (strcmp(name, "license") == 0) { 1556 err = bpf_object__init_license(obj, 1557 data->d_buf, 1558 data->d_size); 1559 if (err) 1560 return err; 1561 } else if (strcmp(name, "version") == 0) { 1562 err = bpf_object__init_kversion(obj, 1563 data->d_buf, 1564 data->d_size); 1565 if (err) 1566 return err; 1567 } else if (strcmp(name, "maps") == 0) { 1568 obj->efile.maps_shndx = idx; 1569 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 1570 obj->efile.btf_maps_shndx = idx; 1571 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1572 btf_data = data; 1573 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1574 btf_ext_data = data; 1575 } else if (sh.sh_type == SHT_SYMTAB) { 1576 if (obj->efile.symbols) { 1577 pr_warning("bpf: multiple SYMTAB in %s\n", 1578 obj->path); 1579 return -LIBBPF_ERRNO__FORMAT; 1580 } 1581 obj->efile.symbols = data; 1582 obj->efile.strtabidx = sh.sh_link; 1583 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1584 if (sh.sh_flags & SHF_EXECINSTR) { 1585 if (strcmp(name, ".text") == 0) 1586 obj->efile.text_shndx = idx; 1587 err = bpf_object__add_program(obj, data->d_buf, 1588 data->d_size, name, idx); 1589 if (err) { 1590 char errmsg[STRERR_BUFSIZE]; 1591 char *cp = libbpf_strerror_r(-err, errmsg, 1592 sizeof(errmsg)); 1593 1594 pr_warning("failed to alloc program %s (%s): %s", 1595 name, obj->path, cp); 1596 return err; 1597 } 1598 } else if (strcmp(name, ".data") == 0) { 1599 obj->efile.data = data; 1600 obj->efile.data_shndx = idx; 1601 } else if (strcmp(name, ".rodata") == 0) { 1602 obj->efile.rodata = data; 1603 obj->efile.rodata_shndx = idx; 1604 } else { 1605 pr_debug("skip section(%d) %s\n", idx, name); 1606 } 1607 } else if (sh.sh_type == SHT_REL) { 1608 int nr_reloc = obj->efile.nr_reloc; 1609 void *reloc = obj->efile.reloc; 1610 int sec = sh.sh_info; /* points to other section */ 1611 1612 /* Only do relo for section with exec instructions */ 1613 if (!section_have_execinstr(obj, sec)) { 1614 pr_debug("skip relo %s(%d) for section(%d)\n", 1615 name, idx, sec); 1616 continue; 1617 } 1618 1619 reloc = reallocarray(reloc, nr_reloc + 1, 1620 sizeof(*obj->efile.reloc)); 1621 if (!reloc) { 1622 pr_warning("realloc failed\n"); 1623 return -ENOMEM; 1624 } 1625 1626 obj->efile.reloc = reloc; 1627 obj->efile.nr_reloc++; 1628 1629 obj->efile.reloc[nr_reloc].shdr = sh; 1630 obj->efile.reloc[nr_reloc].data = data; 1631 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1632 obj->efile.bss = data; 1633 obj->efile.bss_shndx = idx; 1634 } else { 1635 pr_debug("skip section(%d) %s\n", idx, name); 1636 } 1637 } 1638 1639 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1640 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1641 return -LIBBPF_ERRNO__FORMAT; 1642 } 1643 err = bpf_object__init_btf(obj, btf_data, btf_ext_data); 1644 if (!err) 1645 err = bpf_object__init_maps(obj, flags); 1646 if (!err) 1647 err = bpf_object__sanitize_and_load_btf(obj); 1648 if (!err) 1649 err = bpf_object__init_prog_names(obj); 1650 return err; 1651 } 1652 1653 static struct bpf_program * 1654 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1655 { 1656 struct bpf_program *prog; 1657 size_t i; 1658 1659 for (i = 0; i < obj->nr_programs; i++) { 1660 prog = &obj->programs[i]; 1661 if (prog->idx == idx) 1662 return prog; 1663 } 1664 return NULL; 1665 } 1666 1667 struct bpf_program * 1668 bpf_object__find_program_by_title(const struct bpf_object *obj, 1669 const char *title) 1670 { 1671 struct bpf_program *pos; 1672 1673 bpf_object__for_each_program(pos, obj) { 1674 if (pos->section_name && !strcmp(pos->section_name, title)) 1675 return pos; 1676 } 1677 return NULL; 1678 } 1679 1680 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1681 int shndx) 1682 { 1683 return shndx == obj->efile.data_shndx || 1684 shndx == obj->efile.bss_shndx || 1685 shndx == obj->efile.rodata_shndx; 1686 } 1687 1688 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1689 int shndx) 1690 { 1691 return shndx == obj->efile.maps_shndx || 1692 shndx == obj->efile.btf_maps_shndx; 1693 } 1694 1695 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1696 int shndx) 1697 { 1698 return shndx == obj->efile.text_shndx || 1699 bpf_object__shndx_is_maps(obj, shndx) || 1700 bpf_object__shndx_is_data(obj, shndx); 1701 } 1702 1703 static enum libbpf_map_type 1704 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1705 { 1706 if (shndx == obj->efile.data_shndx) 1707 return LIBBPF_MAP_DATA; 1708 else if (shndx == obj->efile.bss_shndx) 1709 return LIBBPF_MAP_BSS; 1710 else if (shndx == obj->efile.rodata_shndx) 1711 return LIBBPF_MAP_RODATA; 1712 else 1713 return LIBBPF_MAP_UNSPEC; 1714 } 1715 1716 static int 1717 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1718 Elf_Data *data, struct bpf_object *obj) 1719 { 1720 Elf_Data *symbols = obj->efile.symbols; 1721 struct bpf_map *maps = obj->maps; 1722 size_t nr_maps = obj->nr_maps; 1723 int i, nrels; 1724 1725 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 1726 nrels = shdr->sh_size / shdr->sh_entsize; 1727 1728 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1729 if (!prog->reloc_desc) { 1730 pr_warning("failed to alloc memory in relocation\n"); 1731 return -ENOMEM; 1732 } 1733 prog->nr_reloc = nrels; 1734 1735 for (i = 0; i < nrels; i++) { 1736 struct bpf_insn *insns = prog->insns; 1737 enum libbpf_map_type type; 1738 unsigned int insn_idx; 1739 unsigned int shdr_idx; 1740 const char *name; 1741 size_t map_idx; 1742 GElf_Sym sym; 1743 GElf_Rel rel; 1744 1745 if (!gelf_getrel(data, i, &rel)) { 1746 pr_warning("relocation: failed to get %d reloc\n", i); 1747 return -LIBBPF_ERRNO__FORMAT; 1748 } 1749 1750 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 1751 pr_warning("relocation: symbol %"PRIx64" not found\n", 1752 GELF_R_SYM(rel.r_info)); 1753 return -LIBBPF_ERRNO__FORMAT; 1754 } 1755 1756 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1757 sym.st_name) ? : "<?>"; 1758 1759 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1760 (long long) (rel.r_info >> 32), 1761 (long long) sym.st_value, sym.st_name, name); 1762 1763 shdr_idx = sym.st_shndx; 1764 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1765 pr_debug("relocation: insn_idx=%u, shdr_idx=%u\n", 1766 insn_idx, shdr_idx); 1767 1768 if (shdr_idx >= SHN_LORESERVE) { 1769 pr_warning("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n", 1770 name, shdr_idx, insn_idx, 1771 insns[insn_idx].code); 1772 return -LIBBPF_ERRNO__RELOC; 1773 } 1774 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1775 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1776 prog->section_name, shdr_idx); 1777 return -LIBBPF_ERRNO__RELOC; 1778 } 1779 1780 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1781 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1782 pr_warning("incorrect bpf_call opcode\n"); 1783 return -LIBBPF_ERRNO__RELOC; 1784 } 1785 prog->reloc_desc[i].type = RELO_CALL; 1786 prog->reloc_desc[i].insn_idx = insn_idx; 1787 prog->reloc_desc[i].text_off = sym.st_value; 1788 obj->has_pseudo_calls = true; 1789 continue; 1790 } 1791 1792 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1793 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1794 insn_idx, insns[insn_idx].code); 1795 return -LIBBPF_ERRNO__RELOC; 1796 } 1797 1798 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1799 bpf_object__shndx_is_data(obj, shdr_idx)) { 1800 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1801 if (type != LIBBPF_MAP_UNSPEC) { 1802 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1803 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1804 name, insn_idx, insns[insn_idx].code); 1805 return -LIBBPF_ERRNO__RELOC; 1806 } 1807 if (!obj->caps.global_data) { 1808 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1809 name, insn_idx); 1810 return -LIBBPF_ERRNO__RELOC; 1811 } 1812 } 1813 1814 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1815 if (maps[map_idx].libbpf_type != type) 1816 continue; 1817 if (type != LIBBPF_MAP_UNSPEC || 1818 (maps[map_idx].sec_idx == sym.st_shndx && 1819 maps[map_idx].sec_offset == sym.st_value)) { 1820 pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", 1821 map_idx, maps[map_idx].name, 1822 maps[map_idx].sec_idx, 1823 maps[map_idx].sec_offset, 1824 insn_idx); 1825 break; 1826 } 1827 } 1828 1829 if (map_idx >= nr_maps) { 1830 pr_warning("bpf relocation: map_idx %d larger than %d\n", 1831 (int)map_idx, (int)nr_maps - 1); 1832 return -LIBBPF_ERRNO__RELOC; 1833 } 1834 1835 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1836 RELO_DATA : RELO_LD64; 1837 prog->reloc_desc[i].insn_idx = insn_idx; 1838 prog->reloc_desc[i].map_idx = map_idx; 1839 } 1840 } 1841 return 0; 1842 } 1843 1844 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 1845 { 1846 struct bpf_map_def *def = &map->def; 1847 __u32 key_type_id = 0, value_type_id = 0; 1848 int ret; 1849 1850 /* if it's BTF-defined map, we don't need to search for type IDs */ 1851 if (map->sec_idx == obj->efile.btf_maps_shndx) 1852 return 0; 1853 1854 if (!bpf_map__is_internal(map)) { 1855 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 1856 def->value_size, &key_type_id, 1857 &value_type_id); 1858 } else { 1859 /* 1860 * LLVM annotates global data differently in BTF, that is, 1861 * only as '.data', '.bss' or '.rodata'. 1862 */ 1863 ret = btf__find_by_name(obj->btf, 1864 libbpf_type_to_btf_name[map->libbpf_type]); 1865 } 1866 if (ret < 0) 1867 return ret; 1868 1869 map->btf_key_type_id = key_type_id; 1870 map->btf_value_type_id = bpf_map__is_internal(map) ? 1871 ret : value_type_id; 1872 return 0; 1873 } 1874 1875 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1876 { 1877 struct bpf_map_info info = {}; 1878 __u32 len = sizeof(info); 1879 int new_fd, err; 1880 char *new_name; 1881 1882 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1883 if (err) 1884 return err; 1885 1886 new_name = strdup(info.name); 1887 if (!new_name) 1888 return -errno; 1889 1890 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1891 if (new_fd < 0) 1892 goto err_free_new_name; 1893 1894 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1895 if (new_fd < 0) 1896 goto err_close_new_fd; 1897 1898 err = zclose(map->fd); 1899 if (err) 1900 goto err_close_new_fd; 1901 free(map->name); 1902 1903 map->fd = new_fd; 1904 map->name = new_name; 1905 map->def.type = info.type; 1906 map->def.key_size = info.key_size; 1907 map->def.value_size = info.value_size; 1908 map->def.max_entries = info.max_entries; 1909 map->def.map_flags = info.map_flags; 1910 map->btf_key_type_id = info.btf_key_type_id; 1911 map->btf_value_type_id = info.btf_value_type_id; 1912 1913 return 0; 1914 1915 err_close_new_fd: 1916 close(new_fd); 1917 err_free_new_name: 1918 free(new_name); 1919 return -errno; 1920 } 1921 1922 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1923 { 1924 if (!map || !max_entries) 1925 return -EINVAL; 1926 1927 /* If map already created, its attributes can't be changed. */ 1928 if (map->fd >= 0) 1929 return -EBUSY; 1930 1931 map->def.max_entries = max_entries; 1932 1933 return 0; 1934 } 1935 1936 static int 1937 bpf_object__probe_name(struct bpf_object *obj) 1938 { 1939 struct bpf_load_program_attr attr; 1940 char *cp, errmsg[STRERR_BUFSIZE]; 1941 struct bpf_insn insns[] = { 1942 BPF_MOV64_IMM(BPF_REG_0, 0), 1943 BPF_EXIT_INSN(), 1944 }; 1945 int ret; 1946 1947 /* make sure basic loading works */ 1948 1949 memset(&attr, 0, sizeof(attr)); 1950 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1951 attr.insns = insns; 1952 attr.insns_cnt = ARRAY_SIZE(insns); 1953 attr.license = "GPL"; 1954 1955 ret = bpf_load_program_xattr(&attr, NULL, 0); 1956 if (ret < 0) { 1957 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1958 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1959 __func__, cp, errno); 1960 return -errno; 1961 } 1962 close(ret); 1963 1964 /* now try the same program, but with the name */ 1965 1966 attr.name = "test"; 1967 ret = bpf_load_program_xattr(&attr, NULL, 0); 1968 if (ret >= 0) { 1969 obj->caps.name = 1; 1970 close(ret); 1971 } 1972 1973 return 0; 1974 } 1975 1976 static int 1977 bpf_object__probe_global_data(struct bpf_object *obj) 1978 { 1979 struct bpf_load_program_attr prg_attr; 1980 struct bpf_create_map_attr map_attr; 1981 char *cp, errmsg[STRERR_BUFSIZE]; 1982 struct bpf_insn insns[] = { 1983 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1984 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1985 BPF_MOV64_IMM(BPF_REG_0, 0), 1986 BPF_EXIT_INSN(), 1987 }; 1988 int ret, map; 1989 1990 memset(&map_attr, 0, sizeof(map_attr)); 1991 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1992 map_attr.key_size = sizeof(int); 1993 map_attr.value_size = 32; 1994 map_attr.max_entries = 1; 1995 1996 map = bpf_create_map_xattr(&map_attr); 1997 if (map < 0) { 1998 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1999 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 2000 __func__, cp, errno); 2001 return -errno; 2002 } 2003 2004 insns[0].imm = map; 2005 2006 memset(&prg_attr, 0, sizeof(prg_attr)); 2007 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 2008 prg_attr.insns = insns; 2009 prg_attr.insns_cnt = ARRAY_SIZE(insns); 2010 prg_attr.license = "GPL"; 2011 2012 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 2013 if (ret >= 0) { 2014 obj->caps.global_data = 1; 2015 close(ret); 2016 } 2017 2018 close(map); 2019 return 0; 2020 } 2021 2022 static int bpf_object__probe_btf_func(struct bpf_object *obj) 2023 { 2024 const char strs[] = "\0int\0x\0a"; 2025 /* void x(int a) {} */ 2026 __u32 types[] = { 2027 /* int */ 2028 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 2029 /* FUNC_PROTO */ /* [2] */ 2030 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 2031 BTF_PARAM_ENC(7, 1), 2032 /* FUNC x */ /* [3] */ 2033 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 2034 }; 2035 int btf_fd; 2036 2037 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 2038 strs, sizeof(strs)); 2039 if (btf_fd >= 0) { 2040 obj->caps.btf_func = 1; 2041 close(btf_fd); 2042 return 1; 2043 } 2044 2045 return 0; 2046 } 2047 2048 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 2049 { 2050 const char strs[] = "\0x\0.data"; 2051 /* static int a; */ 2052 __u32 types[] = { 2053 /* int */ 2054 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 2055 /* VAR x */ /* [2] */ 2056 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 2057 BTF_VAR_STATIC, 2058 /* DATASEC val */ /* [3] */ 2059 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 2060 BTF_VAR_SECINFO_ENC(2, 0, 4), 2061 }; 2062 int btf_fd; 2063 2064 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 2065 strs, sizeof(strs)); 2066 if (btf_fd >= 0) { 2067 obj->caps.btf_datasec = 1; 2068 close(btf_fd); 2069 return 1; 2070 } 2071 2072 return 0; 2073 } 2074 2075 static int 2076 bpf_object__probe_caps(struct bpf_object *obj) 2077 { 2078 int (*probe_fn[])(struct bpf_object *obj) = { 2079 bpf_object__probe_name, 2080 bpf_object__probe_global_data, 2081 bpf_object__probe_btf_func, 2082 bpf_object__probe_btf_datasec, 2083 }; 2084 int i, ret; 2085 2086 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 2087 ret = probe_fn[i](obj); 2088 if (ret < 0) 2089 pr_debug("Probe #%d failed with %d.\n", i, ret); 2090 } 2091 2092 return 0; 2093 } 2094 2095 static int 2096 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 2097 { 2098 char *cp, errmsg[STRERR_BUFSIZE]; 2099 int err, zero = 0; 2100 __u8 *data; 2101 2102 /* Nothing to do here since kernel already zero-initializes .bss map. */ 2103 if (map->libbpf_type == LIBBPF_MAP_BSS) 2104 return 0; 2105 2106 data = map->libbpf_type == LIBBPF_MAP_DATA ? 2107 obj->sections.data : obj->sections.rodata; 2108 2109 err = bpf_map_update_elem(map->fd, &zero, data, 0); 2110 /* Freeze .rodata map as read-only from syscall side. */ 2111 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 2112 err = bpf_map_freeze(map->fd); 2113 if (err) { 2114 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2115 pr_warning("Error freezing map(%s) as read-only: %s\n", 2116 map->name, cp); 2117 err = 0; 2118 } 2119 } 2120 return err; 2121 } 2122 2123 static int 2124 bpf_object__create_maps(struct bpf_object *obj) 2125 { 2126 struct bpf_create_map_attr create_attr = {}; 2127 int nr_cpus = 0; 2128 unsigned int i; 2129 int err; 2130 2131 for (i = 0; i < obj->nr_maps; i++) { 2132 struct bpf_map *map = &obj->maps[i]; 2133 struct bpf_map_def *def = &map->def; 2134 char *cp, errmsg[STRERR_BUFSIZE]; 2135 int *pfd = &map->fd; 2136 2137 if (map->fd >= 0) { 2138 pr_debug("skip map create (preset) %s: fd=%d\n", 2139 map->name, map->fd); 2140 continue; 2141 } 2142 2143 if (obj->caps.name) 2144 create_attr.name = map->name; 2145 create_attr.map_ifindex = map->map_ifindex; 2146 create_attr.map_type = def->type; 2147 create_attr.map_flags = def->map_flags; 2148 create_attr.key_size = def->key_size; 2149 create_attr.value_size = def->value_size; 2150 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && 2151 !def->max_entries) { 2152 if (!nr_cpus) 2153 nr_cpus = libbpf_num_possible_cpus(); 2154 if (nr_cpus < 0) { 2155 pr_warning("failed to determine number of system CPUs: %d\n", 2156 nr_cpus); 2157 err = nr_cpus; 2158 goto err_out; 2159 } 2160 pr_debug("map '%s': setting size to %d\n", 2161 map->name, nr_cpus); 2162 create_attr.max_entries = nr_cpus; 2163 } else { 2164 create_attr.max_entries = def->max_entries; 2165 } 2166 create_attr.btf_fd = 0; 2167 create_attr.btf_key_type_id = 0; 2168 create_attr.btf_value_type_id = 0; 2169 if (bpf_map_type__is_map_in_map(def->type) && 2170 map->inner_map_fd >= 0) 2171 create_attr.inner_map_fd = map->inner_map_fd; 2172 2173 if (obj->btf && !bpf_map_find_btf_info(obj, map)) { 2174 create_attr.btf_fd = btf__fd(obj->btf); 2175 create_attr.btf_key_type_id = map->btf_key_type_id; 2176 create_attr.btf_value_type_id = map->btf_value_type_id; 2177 } 2178 2179 *pfd = bpf_create_map_xattr(&create_attr); 2180 if (*pfd < 0 && (create_attr.btf_key_type_id || 2181 create_attr.btf_value_type_id)) { 2182 err = -errno; 2183 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 2184 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 2185 map->name, cp, err); 2186 create_attr.btf_fd = 0; 2187 create_attr.btf_key_type_id = 0; 2188 create_attr.btf_value_type_id = 0; 2189 map->btf_key_type_id = 0; 2190 map->btf_value_type_id = 0; 2191 *pfd = bpf_create_map_xattr(&create_attr); 2192 } 2193 2194 if (*pfd < 0) { 2195 size_t j; 2196 2197 err = -errno; 2198 err_out: 2199 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 2200 pr_warning("failed to create map (name: '%s'): %s(%d)\n", 2201 map->name, cp, err); 2202 for (j = 0; j < i; j++) 2203 zclose(obj->maps[j].fd); 2204 return err; 2205 } 2206 2207 if (bpf_map__is_internal(map)) { 2208 err = bpf_object__populate_internal_map(obj, map); 2209 if (err < 0) { 2210 zclose(*pfd); 2211 goto err_out; 2212 } 2213 } 2214 2215 pr_debug("created map %s: fd=%d\n", map->name, *pfd); 2216 } 2217 2218 return 0; 2219 } 2220 2221 static int 2222 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 2223 void *btf_prog_info, const char *info_name) 2224 { 2225 if (err != -ENOENT) { 2226 pr_warning("Error in loading %s for sec %s.\n", 2227 info_name, prog->section_name); 2228 return err; 2229 } 2230 2231 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 2232 2233 if (btf_prog_info) { 2234 /* 2235 * Some info has already been found but has problem 2236 * in the last btf_ext reloc. Must have to error out. 2237 */ 2238 pr_warning("Error in relocating %s for sec %s.\n", 2239 info_name, prog->section_name); 2240 return err; 2241 } 2242 2243 /* Have problem loading the very first info. Ignore the rest. */ 2244 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 2245 info_name, prog->section_name, info_name); 2246 return 0; 2247 } 2248 2249 static int 2250 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 2251 const char *section_name, __u32 insn_offset) 2252 { 2253 int err; 2254 2255 if (!insn_offset || prog->func_info) { 2256 /* 2257 * !insn_offset => main program 2258 * 2259 * For sub prog, the main program's func_info has to 2260 * be loaded first (i.e. prog->func_info != NULL) 2261 */ 2262 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 2263 section_name, insn_offset, 2264 &prog->func_info, 2265 &prog->func_info_cnt); 2266 if (err) 2267 return check_btf_ext_reloc_err(prog, err, 2268 prog->func_info, 2269 "bpf_func_info"); 2270 2271 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 2272 } 2273 2274 if (!insn_offset || prog->line_info) { 2275 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 2276 section_name, insn_offset, 2277 &prog->line_info, 2278 &prog->line_info_cnt); 2279 if (err) 2280 return check_btf_ext_reloc_err(prog, err, 2281 prog->line_info, 2282 "bpf_line_info"); 2283 2284 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 2285 } 2286 2287 if (!insn_offset) 2288 prog->btf_fd = btf__fd(obj->btf); 2289 2290 return 0; 2291 } 2292 2293 static int 2294 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 2295 struct reloc_desc *relo) 2296 { 2297 struct bpf_insn *insn, *new_insn; 2298 struct bpf_program *text; 2299 size_t new_cnt; 2300 int err; 2301 2302 if (relo->type != RELO_CALL) 2303 return -LIBBPF_ERRNO__RELOC; 2304 2305 if (prog->idx == obj->efile.text_shndx) { 2306 pr_warning("relo in .text insn %d into off %d\n", 2307 relo->insn_idx, relo->text_off); 2308 return -LIBBPF_ERRNO__RELOC; 2309 } 2310 2311 if (prog->main_prog_cnt == 0) { 2312 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 2313 if (!text) { 2314 pr_warning("no .text section found yet relo into text exist\n"); 2315 return -LIBBPF_ERRNO__RELOC; 2316 } 2317 new_cnt = prog->insns_cnt + text->insns_cnt; 2318 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 2319 if (!new_insn) { 2320 pr_warning("oom in prog realloc\n"); 2321 return -ENOMEM; 2322 } 2323 2324 if (obj->btf_ext) { 2325 err = bpf_program_reloc_btf_ext(prog, obj, 2326 text->section_name, 2327 prog->insns_cnt); 2328 if (err) 2329 return err; 2330 } 2331 2332 memcpy(new_insn + prog->insns_cnt, text->insns, 2333 text->insns_cnt * sizeof(*insn)); 2334 prog->insns = new_insn; 2335 prog->main_prog_cnt = prog->insns_cnt; 2336 prog->insns_cnt = new_cnt; 2337 pr_debug("added %zd insn from %s to prog %s\n", 2338 text->insns_cnt, text->section_name, 2339 prog->section_name); 2340 } 2341 insn = &prog->insns[relo->insn_idx]; 2342 insn->imm += prog->main_prog_cnt - relo->insn_idx; 2343 return 0; 2344 } 2345 2346 static int 2347 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 2348 { 2349 int i, err; 2350 2351 if (!prog) 2352 return 0; 2353 2354 if (obj->btf_ext) { 2355 err = bpf_program_reloc_btf_ext(prog, obj, 2356 prog->section_name, 0); 2357 if (err) 2358 return err; 2359 } 2360 2361 if (!prog->reloc_desc) 2362 return 0; 2363 2364 for (i = 0; i < prog->nr_reloc; i++) { 2365 if (prog->reloc_desc[i].type == RELO_LD64 || 2366 prog->reloc_desc[i].type == RELO_DATA) { 2367 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 2368 struct bpf_insn *insns = prog->insns; 2369 int insn_idx, map_idx; 2370 2371 insn_idx = prog->reloc_desc[i].insn_idx; 2372 map_idx = prog->reloc_desc[i].map_idx; 2373 2374 if (insn_idx + 1 >= (int)prog->insns_cnt) { 2375 pr_warning("relocation out of range: '%s'\n", 2376 prog->section_name); 2377 return -LIBBPF_ERRNO__RELOC; 2378 } 2379 2380 if (!relo_data) { 2381 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 2382 } else { 2383 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 2384 insns[insn_idx + 1].imm = insns[insn_idx].imm; 2385 } 2386 insns[insn_idx].imm = obj->maps[map_idx].fd; 2387 } else if (prog->reloc_desc[i].type == RELO_CALL) { 2388 err = bpf_program__reloc_text(prog, obj, 2389 &prog->reloc_desc[i]); 2390 if (err) 2391 return err; 2392 } 2393 } 2394 2395 zfree(&prog->reloc_desc); 2396 prog->nr_reloc = 0; 2397 return 0; 2398 } 2399 2400 2401 static int 2402 bpf_object__relocate(struct bpf_object *obj) 2403 { 2404 struct bpf_program *prog; 2405 size_t i; 2406 int err; 2407 2408 for (i = 0; i < obj->nr_programs; i++) { 2409 prog = &obj->programs[i]; 2410 2411 err = bpf_program__relocate(prog, obj); 2412 if (err) { 2413 pr_warning("failed to relocate '%s'\n", 2414 prog->section_name); 2415 return err; 2416 } 2417 } 2418 return 0; 2419 } 2420 2421 static int bpf_object__collect_reloc(struct bpf_object *obj) 2422 { 2423 int i, err; 2424 2425 if (!obj_elf_valid(obj)) { 2426 pr_warning("Internal error: elf object is closed\n"); 2427 return -LIBBPF_ERRNO__INTERNAL; 2428 } 2429 2430 for (i = 0; i < obj->efile.nr_reloc; i++) { 2431 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2432 Elf_Data *data = obj->efile.reloc[i].data; 2433 int idx = shdr->sh_info; 2434 struct bpf_program *prog; 2435 2436 if (shdr->sh_type != SHT_REL) { 2437 pr_warning("internal error at %d\n", __LINE__); 2438 return -LIBBPF_ERRNO__INTERNAL; 2439 } 2440 2441 prog = bpf_object__find_prog_by_idx(obj, idx); 2442 if (!prog) { 2443 pr_warning("relocation failed: no section(%d)\n", idx); 2444 return -LIBBPF_ERRNO__RELOC; 2445 } 2446 2447 err = bpf_program__collect_reloc(prog, shdr, data, obj); 2448 if (err) 2449 return err; 2450 } 2451 return 0; 2452 } 2453 2454 static int 2455 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2456 char *license, __u32 kern_version, int *pfd) 2457 { 2458 struct bpf_load_program_attr load_attr; 2459 char *cp, errmsg[STRERR_BUFSIZE]; 2460 int log_buf_size = BPF_LOG_BUF_SIZE; 2461 char *log_buf; 2462 int ret; 2463 2464 if (!insns || !insns_cnt) 2465 return -EINVAL; 2466 2467 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2468 load_attr.prog_type = prog->type; 2469 load_attr.expected_attach_type = prog->expected_attach_type; 2470 if (prog->caps->name) 2471 load_attr.name = prog->name; 2472 load_attr.insns = insns; 2473 load_attr.insns_cnt = insns_cnt; 2474 load_attr.license = license; 2475 load_attr.kern_version = kern_version; 2476 load_attr.prog_ifindex = prog->prog_ifindex; 2477 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 2478 load_attr.func_info = prog->func_info; 2479 load_attr.func_info_rec_size = prog->func_info_rec_size; 2480 load_attr.func_info_cnt = prog->func_info_cnt; 2481 load_attr.line_info = prog->line_info; 2482 load_attr.line_info_rec_size = prog->line_info_rec_size; 2483 load_attr.line_info_cnt = prog->line_info_cnt; 2484 load_attr.log_level = prog->log_level; 2485 load_attr.prog_flags = prog->prog_flags; 2486 2487 retry_load: 2488 log_buf = malloc(log_buf_size); 2489 if (!log_buf) 2490 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2491 2492 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2493 2494 if (ret >= 0) { 2495 if (load_attr.log_level) 2496 pr_debug("verifier log:\n%s", log_buf); 2497 *pfd = ret; 2498 ret = 0; 2499 goto out; 2500 } 2501 2502 if (errno == ENOSPC) { 2503 log_buf_size <<= 1; 2504 free(log_buf); 2505 goto retry_load; 2506 } 2507 ret = -LIBBPF_ERRNO__LOAD; 2508 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2509 pr_warning("load bpf program failed: %s\n", cp); 2510 2511 if (log_buf && log_buf[0] != '\0') { 2512 ret = -LIBBPF_ERRNO__VERIFY; 2513 pr_warning("-- BEGIN DUMP LOG ---\n"); 2514 pr_warning("\n%s\n", log_buf); 2515 pr_warning("-- END LOG --\n"); 2516 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2517 pr_warning("Program too large (%zu insns), at most %d insns\n", 2518 load_attr.insns_cnt, BPF_MAXINSNS); 2519 ret = -LIBBPF_ERRNO__PROG2BIG; 2520 } else { 2521 /* Wrong program type? */ 2522 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2523 int fd; 2524 2525 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2526 load_attr.expected_attach_type = 0; 2527 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2528 if (fd >= 0) { 2529 close(fd); 2530 ret = -LIBBPF_ERRNO__PROGTYPE; 2531 goto out; 2532 } 2533 } 2534 2535 if (log_buf) 2536 ret = -LIBBPF_ERRNO__KVER; 2537 } 2538 2539 out: 2540 free(log_buf); 2541 return ret; 2542 } 2543 2544 int 2545 bpf_program__load(struct bpf_program *prog, 2546 char *license, __u32 kern_version) 2547 { 2548 int err = 0, fd, i; 2549 2550 if (prog->instances.nr < 0 || !prog->instances.fds) { 2551 if (prog->preprocessor) { 2552 pr_warning("Internal error: can't load program '%s'\n", 2553 prog->section_name); 2554 return -LIBBPF_ERRNO__INTERNAL; 2555 } 2556 2557 prog->instances.fds = malloc(sizeof(int)); 2558 if (!prog->instances.fds) { 2559 pr_warning("Not enough memory for BPF fds\n"); 2560 return -ENOMEM; 2561 } 2562 prog->instances.nr = 1; 2563 prog->instances.fds[0] = -1; 2564 } 2565 2566 if (!prog->preprocessor) { 2567 if (prog->instances.nr != 1) { 2568 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2569 prog->section_name, prog->instances.nr); 2570 } 2571 err = load_program(prog, prog->insns, prog->insns_cnt, 2572 license, kern_version, &fd); 2573 if (!err) 2574 prog->instances.fds[0] = fd; 2575 goto out; 2576 } 2577 2578 for (i = 0; i < prog->instances.nr; i++) { 2579 struct bpf_prog_prep_result result; 2580 bpf_program_prep_t preprocessor = prog->preprocessor; 2581 2582 memset(&result, 0, sizeof(result)); 2583 err = preprocessor(prog, i, prog->insns, 2584 prog->insns_cnt, &result); 2585 if (err) { 2586 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2587 i, prog->section_name); 2588 goto out; 2589 } 2590 2591 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2592 pr_debug("Skip loading the %dth instance of program '%s'\n", 2593 i, prog->section_name); 2594 prog->instances.fds[i] = -1; 2595 if (result.pfd) 2596 *result.pfd = -1; 2597 continue; 2598 } 2599 2600 err = load_program(prog, result.new_insn_ptr, 2601 result.new_insn_cnt, 2602 license, kern_version, &fd); 2603 2604 if (err) { 2605 pr_warning("Loading the %dth instance of program '%s' failed\n", 2606 i, prog->section_name); 2607 goto out; 2608 } 2609 2610 if (result.pfd) 2611 *result.pfd = fd; 2612 prog->instances.fds[i] = fd; 2613 } 2614 out: 2615 if (err) 2616 pr_warning("failed to load program '%s'\n", 2617 prog->section_name); 2618 zfree(&prog->insns); 2619 prog->insns_cnt = 0; 2620 return err; 2621 } 2622 2623 static bool bpf_program__is_function_storage(const struct bpf_program *prog, 2624 const struct bpf_object *obj) 2625 { 2626 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2627 } 2628 2629 static int 2630 bpf_object__load_progs(struct bpf_object *obj, int log_level) 2631 { 2632 size_t i; 2633 int err; 2634 2635 for (i = 0; i < obj->nr_programs; i++) { 2636 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2637 continue; 2638 obj->programs[i].log_level |= log_level; 2639 err = bpf_program__load(&obj->programs[i], 2640 obj->license, 2641 obj->kern_version); 2642 if (err) 2643 return err; 2644 } 2645 return 0; 2646 } 2647 2648 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2649 { 2650 switch (type) { 2651 case BPF_PROG_TYPE_SOCKET_FILTER: 2652 case BPF_PROG_TYPE_SCHED_CLS: 2653 case BPF_PROG_TYPE_SCHED_ACT: 2654 case BPF_PROG_TYPE_XDP: 2655 case BPF_PROG_TYPE_CGROUP_SKB: 2656 case BPF_PROG_TYPE_CGROUP_SOCK: 2657 case BPF_PROG_TYPE_LWT_IN: 2658 case BPF_PROG_TYPE_LWT_OUT: 2659 case BPF_PROG_TYPE_LWT_XMIT: 2660 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2661 case BPF_PROG_TYPE_SOCK_OPS: 2662 case BPF_PROG_TYPE_SK_SKB: 2663 case BPF_PROG_TYPE_CGROUP_DEVICE: 2664 case BPF_PROG_TYPE_SK_MSG: 2665 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2666 case BPF_PROG_TYPE_LIRC_MODE2: 2667 case BPF_PROG_TYPE_SK_REUSEPORT: 2668 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2669 case BPF_PROG_TYPE_UNSPEC: 2670 case BPF_PROG_TYPE_TRACEPOINT: 2671 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2672 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2673 case BPF_PROG_TYPE_PERF_EVENT: 2674 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2675 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2676 return false; 2677 case BPF_PROG_TYPE_KPROBE: 2678 default: 2679 return true; 2680 } 2681 } 2682 2683 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2684 { 2685 if (needs_kver && obj->kern_version == 0) { 2686 pr_warning("%s doesn't provide kernel version\n", 2687 obj->path); 2688 return -LIBBPF_ERRNO__KVERSION; 2689 } 2690 return 0; 2691 } 2692 2693 static struct bpf_object * 2694 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2695 bool needs_kver, int flags) 2696 { 2697 struct bpf_object *obj; 2698 int err; 2699 2700 if (elf_version(EV_CURRENT) == EV_NONE) { 2701 pr_warning("failed to init libelf for %s\n", path); 2702 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2703 } 2704 2705 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2706 if (IS_ERR(obj)) 2707 return obj; 2708 2709 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2710 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2711 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2712 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2713 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2714 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2715 2716 bpf_object__elf_finish(obj); 2717 return obj; 2718 out: 2719 bpf_object__close(obj); 2720 return ERR_PTR(err); 2721 } 2722 2723 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2724 int flags) 2725 { 2726 /* param validation */ 2727 if (!attr->file) 2728 return NULL; 2729 2730 pr_debug("loading %s\n", attr->file); 2731 2732 return __bpf_object__open(attr->file, NULL, 0, 2733 bpf_prog_type__needs_kver(attr->prog_type), 2734 flags); 2735 } 2736 2737 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2738 { 2739 return __bpf_object__open_xattr(attr, 0); 2740 } 2741 2742 struct bpf_object *bpf_object__open(const char *path) 2743 { 2744 struct bpf_object_open_attr attr = { 2745 .file = path, 2746 .prog_type = BPF_PROG_TYPE_UNSPEC, 2747 }; 2748 2749 return bpf_object__open_xattr(&attr); 2750 } 2751 2752 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2753 size_t obj_buf_sz, 2754 const char *name) 2755 { 2756 char tmp_name[64]; 2757 2758 /* param validation */ 2759 if (!obj_buf || obj_buf_sz <= 0) 2760 return NULL; 2761 2762 if (!name) { 2763 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2764 (unsigned long)obj_buf, 2765 (unsigned long)obj_buf_sz); 2766 name = tmp_name; 2767 } 2768 pr_debug("loading object '%s' from buffer\n", name); 2769 2770 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2771 } 2772 2773 int bpf_object__unload(struct bpf_object *obj) 2774 { 2775 size_t i; 2776 2777 if (!obj) 2778 return -EINVAL; 2779 2780 for (i = 0; i < obj->nr_maps; i++) 2781 zclose(obj->maps[i].fd); 2782 2783 for (i = 0; i < obj->nr_programs; i++) 2784 bpf_program__unload(&obj->programs[i]); 2785 2786 return 0; 2787 } 2788 2789 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 2790 { 2791 struct bpf_object *obj; 2792 int err; 2793 2794 if (!attr) 2795 return -EINVAL; 2796 obj = attr->obj; 2797 if (!obj) 2798 return -EINVAL; 2799 2800 if (obj->loaded) { 2801 pr_warning("object should not be loaded twice\n"); 2802 return -EINVAL; 2803 } 2804 2805 obj->loaded = true; 2806 2807 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2808 CHECK_ERR(bpf_object__relocate(obj), err, out); 2809 CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); 2810 2811 return 0; 2812 out: 2813 bpf_object__unload(obj); 2814 pr_warning("failed to load object '%s'\n", obj->path); 2815 return err; 2816 } 2817 2818 int bpf_object__load(struct bpf_object *obj) 2819 { 2820 struct bpf_object_load_attr attr = { 2821 .obj = obj, 2822 }; 2823 2824 return bpf_object__load_xattr(&attr); 2825 } 2826 2827 static int check_path(const char *path) 2828 { 2829 char *cp, errmsg[STRERR_BUFSIZE]; 2830 struct statfs st_fs; 2831 char *dname, *dir; 2832 int err = 0; 2833 2834 if (path == NULL) 2835 return -EINVAL; 2836 2837 dname = strdup(path); 2838 if (dname == NULL) 2839 return -ENOMEM; 2840 2841 dir = dirname(dname); 2842 if (statfs(dir, &st_fs)) { 2843 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2844 pr_warning("failed to statfs %s: %s\n", dir, cp); 2845 err = -errno; 2846 } 2847 free(dname); 2848 2849 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2850 pr_warning("specified path %s is not on BPF FS\n", path); 2851 err = -EINVAL; 2852 } 2853 2854 return err; 2855 } 2856 2857 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2858 int instance) 2859 { 2860 char *cp, errmsg[STRERR_BUFSIZE]; 2861 int err; 2862 2863 err = check_path(path); 2864 if (err) 2865 return err; 2866 2867 if (prog == NULL) { 2868 pr_warning("invalid program pointer\n"); 2869 return -EINVAL; 2870 } 2871 2872 if (instance < 0 || instance >= prog->instances.nr) { 2873 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2874 instance, prog->section_name, prog->instances.nr); 2875 return -EINVAL; 2876 } 2877 2878 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2879 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2880 pr_warning("failed to pin program: %s\n", cp); 2881 return -errno; 2882 } 2883 pr_debug("pinned program '%s'\n", path); 2884 2885 return 0; 2886 } 2887 2888 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2889 int instance) 2890 { 2891 int err; 2892 2893 err = check_path(path); 2894 if (err) 2895 return err; 2896 2897 if (prog == NULL) { 2898 pr_warning("invalid program pointer\n"); 2899 return -EINVAL; 2900 } 2901 2902 if (instance < 0 || instance >= prog->instances.nr) { 2903 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2904 instance, prog->section_name, prog->instances.nr); 2905 return -EINVAL; 2906 } 2907 2908 err = unlink(path); 2909 if (err != 0) 2910 return -errno; 2911 pr_debug("unpinned program '%s'\n", path); 2912 2913 return 0; 2914 } 2915 2916 static int make_dir(const char *path) 2917 { 2918 char *cp, errmsg[STRERR_BUFSIZE]; 2919 int err = 0; 2920 2921 if (mkdir(path, 0700) && errno != EEXIST) 2922 err = -errno; 2923 2924 if (err) { 2925 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2926 pr_warning("failed to mkdir %s: %s\n", path, cp); 2927 } 2928 return err; 2929 } 2930 2931 int bpf_program__pin(struct bpf_program *prog, const char *path) 2932 { 2933 int i, err; 2934 2935 err = check_path(path); 2936 if (err) 2937 return err; 2938 2939 if (prog == NULL) { 2940 pr_warning("invalid program pointer\n"); 2941 return -EINVAL; 2942 } 2943 2944 if (prog->instances.nr <= 0) { 2945 pr_warning("no instances of prog %s to pin\n", 2946 prog->section_name); 2947 return -EINVAL; 2948 } 2949 2950 if (prog->instances.nr == 1) { 2951 /* don't create subdirs when pinning single instance */ 2952 return bpf_program__pin_instance(prog, path, 0); 2953 } 2954 2955 err = make_dir(path); 2956 if (err) 2957 return err; 2958 2959 for (i = 0; i < prog->instances.nr; i++) { 2960 char buf[PATH_MAX]; 2961 int len; 2962 2963 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2964 if (len < 0) { 2965 err = -EINVAL; 2966 goto err_unpin; 2967 } else if (len >= PATH_MAX) { 2968 err = -ENAMETOOLONG; 2969 goto err_unpin; 2970 } 2971 2972 err = bpf_program__pin_instance(prog, buf, i); 2973 if (err) 2974 goto err_unpin; 2975 } 2976 2977 return 0; 2978 2979 err_unpin: 2980 for (i = i - 1; i >= 0; i--) { 2981 char buf[PATH_MAX]; 2982 int len; 2983 2984 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2985 if (len < 0) 2986 continue; 2987 else if (len >= PATH_MAX) 2988 continue; 2989 2990 bpf_program__unpin_instance(prog, buf, i); 2991 } 2992 2993 rmdir(path); 2994 2995 return err; 2996 } 2997 2998 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2999 { 3000 int i, err; 3001 3002 err = check_path(path); 3003 if (err) 3004 return err; 3005 3006 if (prog == NULL) { 3007 pr_warning("invalid program pointer\n"); 3008 return -EINVAL; 3009 } 3010 3011 if (prog->instances.nr <= 0) { 3012 pr_warning("no instances of prog %s to pin\n", 3013 prog->section_name); 3014 return -EINVAL; 3015 } 3016 3017 if (prog->instances.nr == 1) { 3018 /* don't create subdirs when pinning single instance */ 3019 return bpf_program__unpin_instance(prog, path, 0); 3020 } 3021 3022 for (i = 0; i < prog->instances.nr; i++) { 3023 char buf[PATH_MAX]; 3024 int len; 3025 3026 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 3027 if (len < 0) 3028 return -EINVAL; 3029 else if (len >= PATH_MAX) 3030 return -ENAMETOOLONG; 3031 3032 err = bpf_program__unpin_instance(prog, buf, i); 3033 if (err) 3034 return err; 3035 } 3036 3037 err = rmdir(path); 3038 if (err) 3039 return -errno; 3040 3041 return 0; 3042 } 3043 3044 int bpf_map__pin(struct bpf_map *map, const char *path) 3045 { 3046 char *cp, errmsg[STRERR_BUFSIZE]; 3047 int err; 3048 3049 err = check_path(path); 3050 if (err) 3051 return err; 3052 3053 if (map == NULL) { 3054 pr_warning("invalid map pointer\n"); 3055 return -EINVAL; 3056 } 3057 3058 if (bpf_obj_pin(map->fd, path)) { 3059 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 3060 pr_warning("failed to pin map: %s\n", cp); 3061 return -errno; 3062 } 3063 3064 pr_debug("pinned map '%s'\n", path); 3065 3066 return 0; 3067 } 3068 3069 int bpf_map__unpin(struct bpf_map *map, const char *path) 3070 { 3071 int err; 3072 3073 err = check_path(path); 3074 if (err) 3075 return err; 3076 3077 if (map == NULL) { 3078 pr_warning("invalid map pointer\n"); 3079 return -EINVAL; 3080 } 3081 3082 err = unlink(path); 3083 if (err != 0) 3084 return -errno; 3085 pr_debug("unpinned map '%s'\n", path); 3086 3087 return 0; 3088 } 3089 3090 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 3091 { 3092 struct bpf_map *map; 3093 int err; 3094 3095 if (!obj) 3096 return -ENOENT; 3097 3098 if (!obj->loaded) { 3099 pr_warning("object not yet loaded; load it first\n"); 3100 return -ENOENT; 3101 } 3102 3103 err = make_dir(path); 3104 if (err) 3105 return err; 3106 3107 bpf_object__for_each_map(map, obj) { 3108 char buf[PATH_MAX]; 3109 int len; 3110 3111 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3112 bpf_map__name(map)); 3113 if (len < 0) { 3114 err = -EINVAL; 3115 goto err_unpin_maps; 3116 } else if (len >= PATH_MAX) { 3117 err = -ENAMETOOLONG; 3118 goto err_unpin_maps; 3119 } 3120 3121 err = bpf_map__pin(map, buf); 3122 if (err) 3123 goto err_unpin_maps; 3124 } 3125 3126 return 0; 3127 3128 err_unpin_maps: 3129 while ((map = bpf_map__prev(map, obj))) { 3130 char buf[PATH_MAX]; 3131 int len; 3132 3133 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3134 bpf_map__name(map)); 3135 if (len < 0) 3136 continue; 3137 else if (len >= PATH_MAX) 3138 continue; 3139 3140 bpf_map__unpin(map, buf); 3141 } 3142 3143 return err; 3144 } 3145 3146 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 3147 { 3148 struct bpf_map *map; 3149 int err; 3150 3151 if (!obj) 3152 return -ENOENT; 3153 3154 bpf_object__for_each_map(map, obj) { 3155 char buf[PATH_MAX]; 3156 int len; 3157 3158 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3159 bpf_map__name(map)); 3160 if (len < 0) 3161 return -EINVAL; 3162 else if (len >= PATH_MAX) 3163 return -ENAMETOOLONG; 3164 3165 err = bpf_map__unpin(map, buf); 3166 if (err) 3167 return err; 3168 } 3169 3170 return 0; 3171 } 3172 3173 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 3174 { 3175 struct bpf_program *prog; 3176 int err; 3177 3178 if (!obj) 3179 return -ENOENT; 3180 3181 if (!obj->loaded) { 3182 pr_warning("object not yet loaded; load it first\n"); 3183 return -ENOENT; 3184 } 3185 3186 err = make_dir(path); 3187 if (err) 3188 return err; 3189 3190 bpf_object__for_each_program(prog, obj) { 3191 char buf[PATH_MAX]; 3192 int len; 3193 3194 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3195 prog->pin_name); 3196 if (len < 0) { 3197 err = -EINVAL; 3198 goto err_unpin_programs; 3199 } else if (len >= PATH_MAX) { 3200 err = -ENAMETOOLONG; 3201 goto err_unpin_programs; 3202 } 3203 3204 err = bpf_program__pin(prog, buf); 3205 if (err) 3206 goto err_unpin_programs; 3207 } 3208 3209 return 0; 3210 3211 err_unpin_programs: 3212 while ((prog = bpf_program__prev(prog, obj))) { 3213 char buf[PATH_MAX]; 3214 int len; 3215 3216 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3217 prog->pin_name); 3218 if (len < 0) 3219 continue; 3220 else if (len >= PATH_MAX) 3221 continue; 3222 3223 bpf_program__unpin(prog, buf); 3224 } 3225 3226 return err; 3227 } 3228 3229 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 3230 { 3231 struct bpf_program *prog; 3232 int err; 3233 3234 if (!obj) 3235 return -ENOENT; 3236 3237 bpf_object__for_each_program(prog, obj) { 3238 char buf[PATH_MAX]; 3239 int len; 3240 3241 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3242 prog->pin_name); 3243 if (len < 0) 3244 return -EINVAL; 3245 else if (len >= PATH_MAX) 3246 return -ENAMETOOLONG; 3247 3248 err = bpf_program__unpin(prog, buf); 3249 if (err) 3250 return err; 3251 } 3252 3253 return 0; 3254 } 3255 3256 int bpf_object__pin(struct bpf_object *obj, const char *path) 3257 { 3258 int err; 3259 3260 err = bpf_object__pin_maps(obj, path); 3261 if (err) 3262 return err; 3263 3264 err = bpf_object__pin_programs(obj, path); 3265 if (err) { 3266 bpf_object__unpin_maps(obj, path); 3267 return err; 3268 } 3269 3270 return 0; 3271 } 3272 3273 void bpf_object__close(struct bpf_object *obj) 3274 { 3275 size_t i; 3276 3277 if (!obj) 3278 return; 3279 3280 if (obj->clear_priv) 3281 obj->clear_priv(obj, obj->priv); 3282 3283 bpf_object__elf_finish(obj); 3284 bpf_object__unload(obj); 3285 btf__free(obj->btf); 3286 btf_ext__free(obj->btf_ext); 3287 3288 for (i = 0; i < obj->nr_maps; i++) { 3289 zfree(&obj->maps[i].name); 3290 if (obj->maps[i].clear_priv) 3291 obj->maps[i].clear_priv(&obj->maps[i], 3292 obj->maps[i].priv); 3293 obj->maps[i].priv = NULL; 3294 obj->maps[i].clear_priv = NULL; 3295 } 3296 3297 zfree(&obj->sections.rodata); 3298 zfree(&obj->sections.data); 3299 zfree(&obj->maps); 3300 obj->nr_maps = 0; 3301 3302 if (obj->programs && obj->nr_programs) { 3303 for (i = 0; i < obj->nr_programs; i++) 3304 bpf_program__exit(&obj->programs[i]); 3305 } 3306 zfree(&obj->programs); 3307 3308 list_del(&obj->list); 3309 free(obj); 3310 } 3311 3312 struct bpf_object * 3313 bpf_object__next(struct bpf_object *prev) 3314 { 3315 struct bpf_object *next; 3316 3317 if (!prev) 3318 next = list_first_entry(&bpf_objects_list, 3319 struct bpf_object, 3320 list); 3321 else 3322 next = list_next_entry(prev, list); 3323 3324 /* Empty list is noticed here so don't need checking on entry. */ 3325 if (&next->list == &bpf_objects_list) 3326 return NULL; 3327 3328 return next; 3329 } 3330 3331 const char *bpf_object__name(const struct bpf_object *obj) 3332 { 3333 return obj ? obj->path : ERR_PTR(-EINVAL); 3334 } 3335 3336 unsigned int bpf_object__kversion(const struct bpf_object *obj) 3337 { 3338 return obj ? obj->kern_version : 0; 3339 } 3340 3341 struct btf *bpf_object__btf(const struct bpf_object *obj) 3342 { 3343 return obj ? obj->btf : NULL; 3344 } 3345 3346 int bpf_object__btf_fd(const struct bpf_object *obj) 3347 { 3348 return obj->btf ? btf__fd(obj->btf) : -1; 3349 } 3350 3351 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 3352 bpf_object_clear_priv_t clear_priv) 3353 { 3354 if (obj->priv && obj->clear_priv) 3355 obj->clear_priv(obj, obj->priv); 3356 3357 obj->priv = priv; 3358 obj->clear_priv = clear_priv; 3359 return 0; 3360 } 3361 3362 void *bpf_object__priv(const struct bpf_object *obj) 3363 { 3364 return obj ? obj->priv : ERR_PTR(-EINVAL); 3365 } 3366 3367 static struct bpf_program * 3368 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 3369 bool forward) 3370 { 3371 size_t nr_programs = obj->nr_programs; 3372 ssize_t idx; 3373 3374 if (!nr_programs) 3375 return NULL; 3376 3377 if (!p) 3378 /* Iter from the beginning */ 3379 return forward ? &obj->programs[0] : 3380 &obj->programs[nr_programs - 1]; 3381 3382 if (p->obj != obj) { 3383 pr_warning("error: program handler doesn't match object\n"); 3384 return NULL; 3385 } 3386 3387 idx = (p - obj->programs) + (forward ? 1 : -1); 3388 if (idx >= obj->nr_programs || idx < 0) 3389 return NULL; 3390 return &obj->programs[idx]; 3391 } 3392 3393 struct bpf_program * 3394 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 3395 { 3396 struct bpf_program *prog = prev; 3397 3398 do { 3399 prog = __bpf_program__iter(prog, obj, true); 3400 } while (prog && bpf_program__is_function_storage(prog, obj)); 3401 3402 return prog; 3403 } 3404 3405 struct bpf_program * 3406 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 3407 { 3408 struct bpf_program *prog = next; 3409 3410 do { 3411 prog = __bpf_program__iter(prog, obj, false); 3412 } while (prog && bpf_program__is_function_storage(prog, obj)); 3413 3414 return prog; 3415 } 3416 3417 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 3418 bpf_program_clear_priv_t clear_priv) 3419 { 3420 if (prog->priv && prog->clear_priv) 3421 prog->clear_priv(prog, prog->priv); 3422 3423 prog->priv = priv; 3424 prog->clear_priv = clear_priv; 3425 return 0; 3426 } 3427 3428 void *bpf_program__priv(const struct bpf_program *prog) 3429 { 3430 return prog ? prog->priv : ERR_PTR(-EINVAL); 3431 } 3432 3433 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3434 { 3435 prog->prog_ifindex = ifindex; 3436 } 3437 3438 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 3439 { 3440 const char *title; 3441 3442 title = prog->section_name; 3443 if (needs_copy) { 3444 title = strdup(title); 3445 if (!title) { 3446 pr_warning("failed to strdup program title\n"); 3447 return ERR_PTR(-ENOMEM); 3448 } 3449 } 3450 3451 return title; 3452 } 3453 3454 int bpf_program__fd(const struct bpf_program *prog) 3455 { 3456 return bpf_program__nth_fd(prog, 0); 3457 } 3458 3459 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3460 bpf_program_prep_t prep) 3461 { 3462 int *instances_fds; 3463 3464 if (nr_instances <= 0 || !prep) 3465 return -EINVAL; 3466 3467 if (prog->instances.nr > 0 || prog->instances.fds) { 3468 pr_warning("Can't set pre-processor after loading\n"); 3469 return -EINVAL; 3470 } 3471 3472 instances_fds = malloc(sizeof(int) * nr_instances); 3473 if (!instances_fds) { 3474 pr_warning("alloc memory failed for fds\n"); 3475 return -ENOMEM; 3476 } 3477 3478 /* fill all fd with -1 */ 3479 memset(instances_fds, -1, sizeof(int) * nr_instances); 3480 3481 prog->instances.nr = nr_instances; 3482 prog->instances.fds = instances_fds; 3483 prog->preprocessor = prep; 3484 return 0; 3485 } 3486 3487 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 3488 { 3489 int fd; 3490 3491 if (!prog) 3492 return -EINVAL; 3493 3494 if (n >= prog->instances.nr || n < 0) { 3495 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3496 n, prog->section_name, prog->instances.nr); 3497 return -EINVAL; 3498 } 3499 3500 fd = prog->instances.fds[n]; 3501 if (fd < 0) { 3502 pr_warning("%dth instance of program '%s' is invalid\n", 3503 n, prog->section_name); 3504 return -ENOENT; 3505 } 3506 3507 return fd; 3508 } 3509 3510 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3511 { 3512 prog->type = type; 3513 } 3514 3515 static bool bpf_program__is_type(const struct bpf_program *prog, 3516 enum bpf_prog_type type) 3517 { 3518 return prog ? (prog->type == type) : false; 3519 } 3520 3521 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3522 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3523 { \ 3524 if (!prog) \ 3525 return -EINVAL; \ 3526 bpf_program__set_type(prog, TYPE); \ 3527 return 0; \ 3528 } \ 3529 \ 3530 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 3531 { \ 3532 return bpf_program__is_type(prog, TYPE); \ 3533 } \ 3534 3535 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3536 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3537 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3538 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3539 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3540 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3541 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3542 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3543 3544 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3545 enum bpf_attach_type type) 3546 { 3547 prog->expected_attach_type = type; 3548 } 3549 3550 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3551 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3552 3553 /* Programs that can NOT be attached. */ 3554 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3555 3556 /* Programs that can be attached. */ 3557 #define BPF_APROG_SEC(string, ptype, atype) \ 3558 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3559 3560 /* Programs that must specify expected attach type at load time. */ 3561 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3562 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3563 3564 /* Programs that can be attached but attach type can't be identified by section 3565 * name. Kept for backward compatibility. 3566 */ 3567 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3568 3569 static const struct { 3570 const char *sec; 3571 size_t len; 3572 enum bpf_prog_type prog_type; 3573 enum bpf_attach_type expected_attach_type; 3574 int is_attachable; 3575 enum bpf_attach_type attach_type; 3576 } section_names[] = { 3577 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3578 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3579 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3580 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3581 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3582 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3583 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3584 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3585 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3586 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3587 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3588 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3589 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3590 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3591 BPF_CGROUP_INET_INGRESS), 3592 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3593 BPF_CGROUP_INET_EGRESS), 3594 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3595 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3596 BPF_CGROUP_INET_SOCK_CREATE), 3597 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3598 BPF_CGROUP_INET4_POST_BIND), 3599 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3600 BPF_CGROUP_INET6_POST_BIND), 3601 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3602 BPF_CGROUP_DEVICE), 3603 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3604 BPF_CGROUP_SOCK_OPS), 3605 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3606 BPF_SK_SKB_STREAM_PARSER), 3607 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3608 BPF_SK_SKB_STREAM_VERDICT), 3609 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3610 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3611 BPF_SK_MSG_VERDICT), 3612 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3613 BPF_LIRC_MODE2), 3614 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3615 BPF_FLOW_DISSECTOR), 3616 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3617 BPF_CGROUP_INET4_BIND), 3618 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3619 BPF_CGROUP_INET6_BIND), 3620 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3621 BPF_CGROUP_INET4_CONNECT), 3622 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3623 BPF_CGROUP_INET6_CONNECT), 3624 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3625 BPF_CGROUP_UDP4_SENDMSG), 3626 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3627 BPF_CGROUP_UDP6_SENDMSG), 3628 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3629 BPF_CGROUP_UDP4_RECVMSG), 3630 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3631 BPF_CGROUP_UDP6_RECVMSG), 3632 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3633 BPF_CGROUP_SYSCTL), 3634 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 3635 BPF_CGROUP_GETSOCKOPT), 3636 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 3637 BPF_CGROUP_SETSOCKOPT), 3638 }; 3639 3640 #undef BPF_PROG_SEC_IMPL 3641 #undef BPF_PROG_SEC 3642 #undef BPF_APROG_SEC 3643 #undef BPF_EAPROG_SEC 3644 #undef BPF_APROG_COMPAT 3645 3646 #define MAX_TYPE_NAME_SIZE 32 3647 3648 static char *libbpf_get_type_names(bool attach_type) 3649 { 3650 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3651 char *buf; 3652 3653 buf = malloc(len); 3654 if (!buf) 3655 return NULL; 3656 3657 buf[0] = '\0'; 3658 /* Forge string buf with all available names */ 3659 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3660 if (attach_type && !section_names[i].is_attachable) 3661 continue; 3662 3663 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3664 free(buf); 3665 return NULL; 3666 } 3667 strcat(buf, " "); 3668 strcat(buf, section_names[i].sec); 3669 } 3670 3671 return buf; 3672 } 3673 3674 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3675 enum bpf_attach_type *expected_attach_type) 3676 { 3677 char *type_names; 3678 int i; 3679 3680 if (!name) 3681 return -EINVAL; 3682 3683 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3684 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3685 continue; 3686 *prog_type = section_names[i].prog_type; 3687 *expected_attach_type = section_names[i].expected_attach_type; 3688 return 0; 3689 } 3690 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3691 type_names = libbpf_get_type_names(false); 3692 if (type_names != NULL) { 3693 pr_info("supported section(type) names are:%s\n", type_names); 3694 free(type_names); 3695 } 3696 3697 return -EINVAL; 3698 } 3699 3700 int libbpf_attach_type_by_name(const char *name, 3701 enum bpf_attach_type *attach_type) 3702 { 3703 char *type_names; 3704 int i; 3705 3706 if (!name) 3707 return -EINVAL; 3708 3709 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3710 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3711 continue; 3712 if (!section_names[i].is_attachable) 3713 return -EINVAL; 3714 *attach_type = section_names[i].attach_type; 3715 return 0; 3716 } 3717 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3718 type_names = libbpf_get_type_names(true); 3719 if (type_names != NULL) { 3720 pr_info("attachable section(type) names are:%s\n", type_names); 3721 free(type_names); 3722 } 3723 3724 return -EINVAL; 3725 } 3726 3727 static int 3728 bpf_program__identify_section(struct bpf_program *prog, 3729 enum bpf_prog_type *prog_type, 3730 enum bpf_attach_type *expected_attach_type) 3731 { 3732 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3733 expected_attach_type); 3734 } 3735 3736 int bpf_map__fd(const struct bpf_map *map) 3737 { 3738 return map ? map->fd : -EINVAL; 3739 } 3740 3741 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 3742 { 3743 return map ? &map->def : ERR_PTR(-EINVAL); 3744 } 3745 3746 const char *bpf_map__name(const struct bpf_map *map) 3747 { 3748 return map ? map->name : NULL; 3749 } 3750 3751 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3752 { 3753 return map ? map->btf_key_type_id : 0; 3754 } 3755 3756 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3757 { 3758 return map ? map->btf_value_type_id : 0; 3759 } 3760 3761 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3762 bpf_map_clear_priv_t clear_priv) 3763 { 3764 if (!map) 3765 return -EINVAL; 3766 3767 if (map->priv) { 3768 if (map->clear_priv) 3769 map->clear_priv(map, map->priv); 3770 } 3771 3772 map->priv = priv; 3773 map->clear_priv = clear_priv; 3774 return 0; 3775 } 3776 3777 void *bpf_map__priv(const struct bpf_map *map) 3778 { 3779 return map ? map->priv : ERR_PTR(-EINVAL); 3780 } 3781 3782 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 3783 { 3784 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3785 } 3786 3787 bool bpf_map__is_internal(const struct bpf_map *map) 3788 { 3789 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3790 } 3791 3792 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3793 { 3794 map->map_ifindex = ifindex; 3795 } 3796 3797 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3798 { 3799 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3800 pr_warning("error: unsupported map type\n"); 3801 return -EINVAL; 3802 } 3803 if (map->inner_map_fd != -1) { 3804 pr_warning("error: inner_map_fd already specified\n"); 3805 return -EINVAL; 3806 } 3807 map->inner_map_fd = fd; 3808 return 0; 3809 } 3810 3811 static struct bpf_map * 3812 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 3813 { 3814 ssize_t idx; 3815 struct bpf_map *s, *e; 3816 3817 if (!obj || !obj->maps) 3818 return NULL; 3819 3820 s = obj->maps; 3821 e = obj->maps + obj->nr_maps; 3822 3823 if ((m < s) || (m >= e)) { 3824 pr_warning("error in %s: map handler doesn't belong to object\n", 3825 __func__); 3826 return NULL; 3827 } 3828 3829 idx = (m - obj->maps) + i; 3830 if (idx >= obj->nr_maps || idx < 0) 3831 return NULL; 3832 return &obj->maps[idx]; 3833 } 3834 3835 struct bpf_map * 3836 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 3837 { 3838 if (prev == NULL) 3839 return obj->maps; 3840 3841 return __bpf_map__iter(prev, obj, 1); 3842 } 3843 3844 struct bpf_map * 3845 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 3846 { 3847 if (next == NULL) { 3848 if (!obj->nr_maps) 3849 return NULL; 3850 return obj->maps + obj->nr_maps - 1; 3851 } 3852 3853 return __bpf_map__iter(next, obj, -1); 3854 } 3855 3856 struct bpf_map * 3857 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 3858 { 3859 struct bpf_map *pos; 3860 3861 bpf_object__for_each_map(pos, obj) { 3862 if (pos->name && !strcmp(pos->name, name)) 3863 return pos; 3864 } 3865 return NULL; 3866 } 3867 3868 int 3869 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 3870 { 3871 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3872 } 3873 3874 struct bpf_map * 3875 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3876 { 3877 return ERR_PTR(-ENOTSUP); 3878 } 3879 3880 long libbpf_get_error(const void *ptr) 3881 { 3882 return PTR_ERR_OR_ZERO(ptr); 3883 } 3884 3885 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3886 struct bpf_object **pobj, int *prog_fd) 3887 { 3888 struct bpf_prog_load_attr attr; 3889 3890 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3891 attr.file = file; 3892 attr.prog_type = type; 3893 attr.expected_attach_type = 0; 3894 3895 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3896 } 3897 3898 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3899 struct bpf_object **pobj, int *prog_fd) 3900 { 3901 struct bpf_object_open_attr open_attr = {}; 3902 struct bpf_program *prog, *first_prog = NULL; 3903 enum bpf_attach_type expected_attach_type; 3904 enum bpf_prog_type prog_type; 3905 struct bpf_object *obj; 3906 struct bpf_map *map; 3907 int err; 3908 3909 if (!attr) 3910 return -EINVAL; 3911 if (!attr->file) 3912 return -EINVAL; 3913 3914 open_attr.file = attr->file; 3915 open_attr.prog_type = attr->prog_type; 3916 3917 obj = bpf_object__open_xattr(&open_attr); 3918 if (IS_ERR_OR_NULL(obj)) 3919 return -ENOENT; 3920 3921 bpf_object__for_each_program(prog, obj) { 3922 /* 3923 * If type is not specified, try to guess it based on 3924 * section name. 3925 */ 3926 prog_type = attr->prog_type; 3927 prog->prog_ifindex = attr->ifindex; 3928 expected_attach_type = attr->expected_attach_type; 3929 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3930 err = bpf_program__identify_section(prog, &prog_type, 3931 &expected_attach_type); 3932 if (err < 0) { 3933 bpf_object__close(obj); 3934 return -EINVAL; 3935 } 3936 } 3937 3938 bpf_program__set_type(prog, prog_type); 3939 bpf_program__set_expected_attach_type(prog, 3940 expected_attach_type); 3941 3942 prog->log_level = attr->log_level; 3943 prog->prog_flags = attr->prog_flags; 3944 if (!first_prog) 3945 first_prog = prog; 3946 } 3947 3948 bpf_object__for_each_map(map, obj) { 3949 if (!bpf_map__is_offload_neutral(map)) 3950 map->map_ifindex = attr->ifindex; 3951 } 3952 3953 if (!first_prog) { 3954 pr_warning("object file doesn't contain bpf program\n"); 3955 bpf_object__close(obj); 3956 return -ENOENT; 3957 } 3958 3959 err = bpf_object__load(obj); 3960 if (err) { 3961 bpf_object__close(obj); 3962 return -EINVAL; 3963 } 3964 3965 *pobj = obj; 3966 *prog_fd = bpf_program__fd(first_prog); 3967 return 0; 3968 } 3969 3970 struct bpf_link { 3971 int (*destroy)(struct bpf_link *link); 3972 }; 3973 3974 int bpf_link__destroy(struct bpf_link *link) 3975 { 3976 int err; 3977 3978 if (!link) 3979 return 0; 3980 3981 err = link->destroy(link); 3982 free(link); 3983 3984 return err; 3985 } 3986 3987 struct bpf_link_fd { 3988 struct bpf_link link; /* has to be at the top of struct */ 3989 int fd; /* hook FD */ 3990 }; 3991 3992 static int bpf_link__destroy_perf_event(struct bpf_link *link) 3993 { 3994 struct bpf_link_fd *l = (void *)link; 3995 int err; 3996 3997 err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); 3998 if (err) 3999 err = -errno; 4000 4001 close(l->fd); 4002 return err; 4003 } 4004 4005 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, 4006 int pfd) 4007 { 4008 char errmsg[STRERR_BUFSIZE]; 4009 struct bpf_link_fd *link; 4010 int prog_fd, err; 4011 4012 if (pfd < 0) { 4013 pr_warning("program '%s': invalid perf event FD %d\n", 4014 bpf_program__title(prog, false), pfd); 4015 return ERR_PTR(-EINVAL); 4016 } 4017 prog_fd = bpf_program__fd(prog); 4018 if (prog_fd < 0) { 4019 pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", 4020 bpf_program__title(prog, false)); 4021 return ERR_PTR(-EINVAL); 4022 } 4023 4024 link = malloc(sizeof(*link)); 4025 if (!link) 4026 return ERR_PTR(-ENOMEM); 4027 link->link.destroy = &bpf_link__destroy_perf_event; 4028 link->fd = pfd; 4029 4030 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 4031 err = -errno; 4032 free(link); 4033 pr_warning("program '%s': failed to attach to pfd %d: %s\n", 4034 bpf_program__title(prog, false), pfd, 4035 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4036 return ERR_PTR(err); 4037 } 4038 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 4039 err = -errno; 4040 free(link); 4041 pr_warning("program '%s': failed to enable pfd %d: %s\n", 4042 bpf_program__title(prog, false), pfd, 4043 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4044 return ERR_PTR(err); 4045 } 4046 return (struct bpf_link *)link; 4047 } 4048 4049 /* 4050 * this function is expected to parse integer in the range of [0, 2^31-1] from 4051 * given file using scanf format string fmt. If actual parsed value is 4052 * negative, the result might be indistinguishable from error 4053 */ 4054 static int parse_uint_from_file(const char *file, const char *fmt) 4055 { 4056 char buf[STRERR_BUFSIZE]; 4057 int err, ret; 4058 FILE *f; 4059 4060 f = fopen(file, "r"); 4061 if (!f) { 4062 err = -errno; 4063 pr_debug("failed to open '%s': %s\n", file, 4064 libbpf_strerror_r(err, buf, sizeof(buf))); 4065 return err; 4066 } 4067 err = fscanf(f, fmt, &ret); 4068 if (err != 1) { 4069 err = err == EOF ? -EIO : -errno; 4070 pr_debug("failed to parse '%s': %s\n", file, 4071 libbpf_strerror_r(err, buf, sizeof(buf))); 4072 fclose(f); 4073 return err; 4074 } 4075 fclose(f); 4076 return ret; 4077 } 4078 4079 static int determine_kprobe_perf_type(void) 4080 { 4081 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 4082 4083 return parse_uint_from_file(file, "%d\n"); 4084 } 4085 4086 static int determine_uprobe_perf_type(void) 4087 { 4088 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 4089 4090 return parse_uint_from_file(file, "%d\n"); 4091 } 4092 4093 static int determine_kprobe_retprobe_bit(void) 4094 { 4095 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 4096 4097 return parse_uint_from_file(file, "config:%d\n"); 4098 } 4099 4100 static int determine_uprobe_retprobe_bit(void) 4101 { 4102 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 4103 4104 return parse_uint_from_file(file, "config:%d\n"); 4105 } 4106 4107 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 4108 uint64_t offset, int pid) 4109 { 4110 struct perf_event_attr attr = {}; 4111 char errmsg[STRERR_BUFSIZE]; 4112 int type, pfd, err; 4113 4114 type = uprobe ? determine_uprobe_perf_type() 4115 : determine_kprobe_perf_type(); 4116 if (type < 0) { 4117 pr_warning("failed to determine %s perf type: %s\n", 4118 uprobe ? "uprobe" : "kprobe", 4119 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 4120 return type; 4121 } 4122 if (retprobe) { 4123 int bit = uprobe ? determine_uprobe_retprobe_bit() 4124 : determine_kprobe_retprobe_bit(); 4125 4126 if (bit < 0) { 4127 pr_warning("failed to determine %s retprobe bit: %s\n", 4128 uprobe ? "uprobe" : "kprobe", 4129 libbpf_strerror_r(bit, errmsg, 4130 sizeof(errmsg))); 4131 return bit; 4132 } 4133 attr.config |= 1 << bit; 4134 } 4135 attr.size = sizeof(attr); 4136 attr.type = type; 4137 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 4138 attr.config2 = offset; /* kprobe_addr or probe_offset */ 4139 4140 /* pid filter is meaningful only for uprobes */ 4141 pfd = syscall(__NR_perf_event_open, &attr, 4142 pid < 0 ? -1 : pid /* pid */, 4143 pid == -1 ? 0 : -1 /* cpu */, 4144 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 4145 if (pfd < 0) { 4146 err = -errno; 4147 pr_warning("%s perf_event_open() failed: %s\n", 4148 uprobe ? "uprobe" : "kprobe", 4149 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4150 return err; 4151 } 4152 return pfd; 4153 } 4154 4155 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 4156 bool retprobe, 4157 const char *func_name) 4158 { 4159 char errmsg[STRERR_BUFSIZE]; 4160 struct bpf_link *link; 4161 int pfd, err; 4162 4163 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 4164 0 /* offset */, -1 /* pid */); 4165 if (pfd < 0) { 4166 pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", 4167 bpf_program__title(prog, false), 4168 retprobe ? "kretprobe" : "kprobe", func_name, 4169 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4170 return ERR_PTR(pfd); 4171 } 4172 link = bpf_program__attach_perf_event(prog, pfd); 4173 if (IS_ERR(link)) { 4174 close(pfd); 4175 err = PTR_ERR(link); 4176 pr_warning("program '%s': failed to attach to %s '%s': %s\n", 4177 bpf_program__title(prog, false), 4178 retprobe ? "kretprobe" : "kprobe", func_name, 4179 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4180 return link; 4181 } 4182 return link; 4183 } 4184 4185 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 4186 bool retprobe, pid_t pid, 4187 const char *binary_path, 4188 size_t func_offset) 4189 { 4190 char errmsg[STRERR_BUFSIZE]; 4191 struct bpf_link *link; 4192 int pfd, err; 4193 4194 pfd = perf_event_open_probe(true /* uprobe */, retprobe, 4195 binary_path, func_offset, pid); 4196 if (pfd < 0) { 4197 pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 4198 bpf_program__title(prog, false), 4199 retprobe ? "uretprobe" : "uprobe", 4200 binary_path, func_offset, 4201 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4202 return ERR_PTR(pfd); 4203 } 4204 link = bpf_program__attach_perf_event(prog, pfd); 4205 if (IS_ERR(link)) { 4206 close(pfd); 4207 err = PTR_ERR(link); 4208 pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", 4209 bpf_program__title(prog, false), 4210 retprobe ? "uretprobe" : "uprobe", 4211 binary_path, func_offset, 4212 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4213 return link; 4214 } 4215 return link; 4216 } 4217 4218 static int determine_tracepoint_id(const char *tp_category, 4219 const char *tp_name) 4220 { 4221 char file[PATH_MAX]; 4222 int ret; 4223 4224 ret = snprintf(file, sizeof(file), 4225 "/sys/kernel/debug/tracing/events/%s/%s/id", 4226 tp_category, tp_name); 4227 if (ret < 0) 4228 return -errno; 4229 if (ret >= sizeof(file)) { 4230 pr_debug("tracepoint %s/%s path is too long\n", 4231 tp_category, tp_name); 4232 return -E2BIG; 4233 } 4234 return parse_uint_from_file(file, "%d\n"); 4235 } 4236 4237 static int perf_event_open_tracepoint(const char *tp_category, 4238 const char *tp_name) 4239 { 4240 struct perf_event_attr attr = {}; 4241 char errmsg[STRERR_BUFSIZE]; 4242 int tp_id, pfd, err; 4243 4244 tp_id = determine_tracepoint_id(tp_category, tp_name); 4245 if (tp_id < 0) { 4246 pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 4247 tp_category, tp_name, 4248 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 4249 return tp_id; 4250 } 4251 4252 attr.type = PERF_TYPE_TRACEPOINT; 4253 attr.size = sizeof(attr); 4254 attr.config = tp_id; 4255 4256 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 4257 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 4258 if (pfd < 0) { 4259 err = -errno; 4260 pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", 4261 tp_category, tp_name, 4262 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4263 return err; 4264 } 4265 return pfd; 4266 } 4267 4268 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 4269 const char *tp_category, 4270 const char *tp_name) 4271 { 4272 char errmsg[STRERR_BUFSIZE]; 4273 struct bpf_link *link; 4274 int pfd, err; 4275 4276 pfd = perf_event_open_tracepoint(tp_category, tp_name); 4277 if (pfd < 0) { 4278 pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 4279 bpf_program__title(prog, false), 4280 tp_category, tp_name, 4281 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4282 return ERR_PTR(pfd); 4283 } 4284 link = bpf_program__attach_perf_event(prog, pfd); 4285 if (IS_ERR(link)) { 4286 close(pfd); 4287 err = PTR_ERR(link); 4288 pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", 4289 bpf_program__title(prog, false), 4290 tp_category, tp_name, 4291 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4292 return link; 4293 } 4294 return link; 4295 } 4296 4297 static int bpf_link__destroy_fd(struct bpf_link *link) 4298 { 4299 struct bpf_link_fd *l = (void *)link; 4300 4301 return close(l->fd); 4302 } 4303 4304 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 4305 const char *tp_name) 4306 { 4307 char errmsg[STRERR_BUFSIZE]; 4308 struct bpf_link_fd *link; 4309 int prog_fd, pfd; 4310 4311 prog_fd = bpf_program__fd(prog); 4312 if (prog_fd < 0) { 4313 pr_warning("program '%s': can't attach before loaded\n", 4314 bpf_program__title(prog, false)); 4315 return ERR_PTR(-EINVAL); 4316 } 4317 4318 link = malloc(sizeof(*link)); 4319 if (!link) 4320 return ERR_PTR(-ENOMEM); 4321 link->link.destroy = &bpf_link__destroy_fd; 4322 4323 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 4324 if (pfd < 0) { 4325 pfd = -errno; 4326 free(link); 4327 pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", 4328 bpf_program__title(prog, false), tp_name, 4329 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4330 return ERR_PTR(pfd); 4331 } 4332 link->fd = pfd; 4333 return (struct bpf_link *)link; 4334 } 4335 4336 enum bpf_perf_event_ret 4337 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 4338 void **copy_mem, size_t *copy_size, 4339 bpf_perf_event_print_t fn, void *private_data) 4340 { 4341 struct perf_event_mmap_page *header = mmap_mem; 4342 __u64 data_head = ring_buffer_read_head(header); 4343 __u64 data_tail = header->data_tail; 4344 void *base = ((__u8 *)header) + page_size; 4345 int ret = LIBBPF_PERF_EVENT_CONT; 4346 struct perf_event_header *ehdr; 4347 size_t ehdr_size; 4348 4349 while (data_head != data_tail) { 4350 ehdr = base + (data_tail & (mmap_size - 1)); 4351 ehdr_size = ehdr->size; 4352 4353 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 4354 void *copy_start = ehdr; 4355 size_t len_first = base + mmap_size - copy_start; 4356 size_t len_secnd = ehdr_size - len_first; 4357 4358 if (*copy_size < ehdr_size) { 4359 free(*copy_mem); 4360 *copy_mem = malloc(ehdr_size); 4361 if (!*copy_mem) { 4362 *copy_size = 0; 4363 ret = LIBBPF_PERF_EVENT_ERROR; 4364 break; 4365 } 4366 *copy_size = ehdr_size; 4367 } 4368 4369 memcpy(*copy_mem, copy_start, len_first); 4370 memcpy(*copy_mem + len_first, base, len_secnd); 4371 ehdr = *copy_mem; 4372 } 4373 4374 ret = fn(ehdr, private_data); 4375 data_tail += ehdr_size; 4376 if (ret != LIBBPF_PERF_EVENT_CONT) 4377 break; 4378 } 4379 4380 ring_buffer_write_tail(header, data_tail); 4381 return ret; 4382 } 4383 4384 struct perf_buffer; 4385 4386 struct perf_buffer_params { 4387 struct perf_event_attr *attr; 4388 /* if event_cb is specified, it takes precendence */ 4389 perf_buffer_event_fn event_cb; 4390 /* sample_cb and lost_cb are higher-level common-case callbacks */ 4391 perf_buffer_sample_fn sample_cb; 4392 perf_buffer_lost_fn lost_cb; 4393 void *ctx; 4394 int cpu_cnt; 4395 int *cpus; 4396 int *map_keys; 4397 }; 4398 4399 struct perf_cpu_buf { 4400 struct perf_buffer *pb; 4401 void *base; /* mmap()'ed memory */ 4402 void *buf; /* for reconstructing segmented data */ 4403 size_t buf_size; 4404 int fd; 4405 int cpu; 4406 int map_key; 4407 }; 4408 4409 struct perf_buffer { 4410 perf_buffer_event_fn event_cb; 4411 perf_buffer_sample_fn sample_cb; 4412 perf_buffer_lost_fn lost_cb; 4413 void *ctx; /* passed into callbacks */ 4414 4415 size_t page_size; 4416 size_t mmap_size; 4417 struct perf_cpu_buf **cpu_bufs; 4418 struct epoll_event *events; 4419 int cpu_cnt; 4420 int epoll_fd; /* perf event FD */ 4421 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 4422 }; 4423 4424 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 4425 struct perf_cpu_buf *cpu_buf) 4426 { 4427 if (!cpu_buf) 4428 return; 4429 if (cpu_buf->base && 4430 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 4431 pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 4432 if (cpu_buf->fd >= 0) { 4433 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 4434 close(cpu_buf->fd); 4435 } 4436 free(cpu_buf->buf); 4437 free(cpu_buf); 4438 } 4439 4440 void perf_buffer__free(struct perf_buffer *pb) 4441 { 4442 int i; 4443 4444 if (!pb) 4445 return; 4446 if (pb->cpu_bufs) { 4447 for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { 4448 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 4449 4450 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 4451 perf_buffer__free_cpu_buf(pb, cpu_buf); 4452 } 4453 free(pb->cpu_bufs); 4454 } 4455 if (pb->epoll_fd >= 0) 4456 close(pb->epoll_fd); 4457 free(pb->events); 4458 free(pb); 4459 } 4460 4461 static struct perf_cpu_buf * 4462 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 4463 int cpu, int map_key) 4464 { 4465 struct perf_cpu_buf *cpu_buf; 4466 char msg[STRERR_BUFSIZE]; 4467 int err; 4468 4469 cpu_buf = calloc(1, sizeof(*cpu_buf)); 4470 if (!cpu_buf) 4471 return ERR_PTR(-ENOMEM); 4472 4473 cpu_buf->pb = pb; 4474 cpu_buf->cpu = cpu; 4475 cpu_buf->map_key = map_key; 4476 4477 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 4478 -1, PERF_FLAG_FD_CLOEXEC); 4479 if (cpu_buf->fd < 0) { 4480 err = -errno; 4481 pr_warning("failed to open perf buffer event on cpu #%d: %s\n", 4482 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 4483 goto error; 4484 } 4485 4486 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 4487 PROT_READ | PROT_WRITE, MAP_SHARED, 4488 cpu_buf->fd, 0); 4489 if (cpu_buf->base == MAP_FAILED) { 4490 cpu_buf->base = NULL; 4491 err = -errno; 4492 pr_warning("failed to mmap perf buffer on cpu #%d: %s\n", 4493 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 4494 goto error; 4495 } 4496 4497 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 4498 err = -errno; 4499 pr_warning("failed to enable perf buffer event on cpu #%d: %s\n", 4500 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 4501 goto error; 4502 } 4503 4504 return cpu_buf; 4505 4506 error: 4507 perf_buffer__free_cpu_buf(pb, cpu_buf); 4508 return (struct perf_cpu_buf *)ERR_PTR(err); 4509 } 4510 4511 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 4512 struct perf_buffer_params *p); 4513 4514 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 4515 const struct perf_buffer_opts *opts) 4516 { 4517 struct perf_buffer_params p = {}; 4518 struct perf_event_attr attr = { 4519 .config = PERF_COUNT_SW_BPF_OUTPUT, 4520 .type = PERF_TYPE_SOFTWARE, 4521 .sample_type = PERF_SAMPLE_RAW, 4522 .sample_period = 1, 4523 .wakeup_events = 1, 4524 }; 4525 4526 p.attr = &attr; 4527 p.sample_cb = opts ? opts->sample_cb : NULL; 4528 p.lost_cb = opts ? opts->lost_cb : NULL; 4529 p.ctx = opts ? opts->ctx : NULL; 4530 4531 return __perf_buffer__new(map_fd, page_cnt, &p); 4532 } 4533 4534 struct perf_buffer * 4535 perf_buffer__new_raw(int map_fd, size_t page_cnt, 4536 const struct perf_buffer_raw_opts *opts) 4537 { 4538 struct perf_buffer_params p = {}; 4539 4540 p.attr = opts->attr; 4541 p.event_cb = opts->event_cb; 4542 p.ctx = opts->ctx; 4543 p.cpu_cnt = opts->cpu_cnt; 4544 p.cpus = opts->cpus; 4545 p.map_keys = opts->map_keys; 4546 4547 return __perf_buffer__new(map_fd, page_cnt, &p); 4548 } 4549 4550 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 4551 struct perf_buffer_params *p) 4552 { 4553 struct bpf_map_info map = {}; 4554 char msg[STRERR_BUFSIZE]; 4555 struct perf_buffer *pb; 4556 __u32 map_info_len; 4557 int err, i; 4558 4559 if (page_cnt & (page_cnt - 1)) { 4560 pr_warning("page count should be power of two, but is %zu\n", 4561 page_cnt); 4562 return ERR_PTR(-EINVAL); 4563 } 4564 4565 map_info_len = sizeof(map); 4566 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 4567 if (err) { 4568 err = -errno; 4569 pr_warning("failed to get map info for map FD %d: %s\n", 4570 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 4571 return ERR_PTR(err); 4572 } 4573 4574 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 4575 pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 4576 map.name); 4577 return ERR_PTR(-EINVAL); 4578 } 4579 4580 pb = calloc(1, sizeof(*pb)); 4581 if (!pb) 4582 return ERR_PTR(-ENOMEM); 4583 4584 pb->event_cb = p->event_cb; 4585 pb->sample_cb = p->sample_cb; 4586 pb->lost_cb = p->lost_cb; 4587 pb->ctx = p->ctx; 4588 4589 pb->page_size = getpagesize(); 4590 pb->mmap_size = pb->page_size * page_cnt; 4591 pb->map_fd = map_fd; 4592 4593 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 4594 if (pb->epoll_fd < 0) { 4595 err = -errno; 4596 pr_warning("failed to create epoll instance: %s\n", 4597 libbpf_strerror_r(err, msg, sizeof(msg))); 4598 goto error; 4599 } 4600 4601 if (p->cpu_cnt > 0) { 4602 pb->cpu_cnt = p->cpu_cnt; 4603 } else { 4604 pb->cpu_cnt = libbpf_num_possible_cpus(); 4605 if (pb->cpu_cnt < 0) { 4606 err = pb->cpu_cnt; 4607 goto error; 4608 } 4609 if (map.max_entries < pb->cpu_cnt) 4610 pb->cpu_cnt = map.max_entries; 4611 } 4612 4613 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 4614 if (!pb->events) { 4615 err = -ENOMEM; 4616 pr_warning("failed to allocate events: out of memory\n"); 4617 goto error; 4618 } 4619 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 4620 if (!pb->cpu_bufs) { 4621 err = -ENOMEM; 4622 pr_warning("failed to allocate buffers: out of memory\n"); 4623 goto error; 4624 } 4625 4626 for (i = 0; i < pb->cpu_cnt; i++) { 4627 struct perf_cpu_buf *cpu_buf; 4628 int cpu, map_key; 4629 4630 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 4631 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 4632 4633 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 4634 if (IS_ERR(cpu_buf)) { 4635 err = PTR_ERR(cpu_buf); 4636 goto error; 4637 } 4638 4639 pb->cpu_bufs[i] = cpu_buf; 4640 4641 err = bpf_map_update_elem(pb->map_fd, &map_key, 4642 &cpu_buf->fd, 0); 4643 if (err) { 4644 err = -errno; 4645 pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 4646 cpu, map_key, cpu_buf->fd, 4647 libbpf_strerror_r(err, msg, sizeof(msg))); 4648 goto error; 4649 } 4650 4651 pb->events[i].events = EPOLLIN; 4652 pb->events[i].data.ptr = cpu_buf; 4653 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 4654 &pb->events[i]) < 0) { 4655 err = -errno; 4656 pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 4657 cpu, cpu_buf->fd, 4658 libbpf_strerror_r(err, msg, sizeof(msg))); 4659 goto error; 4660 } 4661 } 4662 4663 return pb; 4664 4665 error: 4666 if (pb) 4667 perf_buffer__free(pb); 4668 return ERR_PTR(err); 4669 } 4670 4671 struct perf_sample_raw { 4672 struct perf_event_header header; 4673 uint32_t size; 4674 char data[0]; 4675 }; 4676 4677 struct perf_sample_lost { 4678 struct perf_event_header header; 4679 uint64_t id; 4680 uint64_t lost; 4681 uint64_t sample_id; 4682 }; 4683 4684 static enum bpf_perf_event_ret 4685 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 4686 { 4687 struct perf_cpu_buf *cpu_buf = ctx; 4688 struct perf_buffer *pb = cpu_buf->pb; 4689 void *data = e; 4690 4691 /* user wants full control over parsing perf event */ 4692 if (pb->event_cb) 4693 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 4694 4695 switch (e->type) { 4696 case PERF_RECORD_SAMPLE: { 4697 struct perf_sample_raw *s = data; 4698 4699 if (pb->sample_cb) 4700 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 4701 break; 4702 } 4703 case PERF_RECORD_LOST: { 4704 struct perf_sample_lost *s = data; 4705 4706 if (pb->lost_cb) 4707 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 4708 break; 4709 } 4710 default: 4711 pr_warning("unknown perf sample type %d\n", e->type); 4712 return LIBBPF_PERF_EVENT_ERROR; 4713 } 4714 return LIBBPF_PERF_EVENT_CONT; 4715 } 4716 4717 static int perf_buffer__process_records(struct perf_buffer *pb, 4718 struct perf_cpu_buf *cpu_buf) 4719 { 4720 enum bpf_perf_event_ret ret; 4721 4722 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 4723 pb->page_size, &cpu_buf->buf, 4724 &cpu_buf->buf_size, 4725 perf_buffer__process_record, cpu_buf); 4726 if (ret != LIBBPF_PERF_EVENT_CONT) 4727 return ret; 4728 return 0; 4729 } 4730 4731 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 4732 { 4733 int i, cnt, err; 4734 4735 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 4736 for (i = 0; i < cnt; i++) { 4737 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 4738 4739 err = perf_buffer__process_records(pb, cpu_buf); 4740 if (err) { 4741 pr_warning("error while processing records: %d\n", err); 4742 return err; 4743 } 4744 } 4745 return cnt < 0 ? -errno : cnt; 4746 } 4747 4748 struct bpf_prog_info_array_desc { 4749 int array_offset; /* e.g. offset of jited_prog_insns */ 4750 int count_offset; /* e.g. offset of jited_prog_len */ 4751 int size_offset; /* > 0: offset of rec size, 4752 * < 0: fix size of -size_offset 4753 */ 4754 }; 4755 4756 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 4757 [BPF_PROG_INFO_JITED_INSNS] = { 4758 offsetof(struct bpf_prog_info, jited_prog_insns), 4759 offsetof(struct bpf_prog_info, jited_prog_len), 4760 -1, 4761 }, 4762 [BPF_PROG_INFO_XLATED_INSNS] = { 4763 offsetof(struct bpf_prog_info, xlated_prog_insns), 4764 offsetof(struct bpf_prog_info, xlated_prog_len), 4765 -1, 4766 }, 4767 [BPF_PROG_INFO_MAP_IDS] = { 4768 offsetof(struct bpf_prog_info, map_ids), 4769 offsetof(struct bpf_prog_info, nr_map_ids), 4770 -(int)sizeof(__u32), 4771 }, 4772 [BPF_PROG_INFO_JITED_KSYMS] = { 4773 offsetof(struct bpf_prog_info, jited_ksyms), 4774 offsetof(struct bpf_prog_info, nr_jited_ksyms), 4775 -(int)sizeof(__u64), 4776 }, 4777 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 4778 offsetof(struct bpf_prog_info, jited_func_lens), 4779 offsetof(struct bpf_prog_info, nr_jited_func_lens), 4780 -(int)sizeof(__u32), 4781 }, 4782 [BPF_PROG_INFO_FUNC_INFO] = { 4783 offsetof(struct bpf_prog_info, func_info), 4784 offsetof(struct bpf_prog_info, nr_func_info), 4785 offsetof(struct bpf_prog_info, func_info_rec_size), 4786 }, 4787 [BPF_PROG_INFO_LINE_INFO] = { 4788 offsetof(struct bpf_prog_info, line_info), 4789 offsetof(struct bpf_prog_info, nr_line_info), 4790 offsetof(struct bpf_prog_info, line_info_rec_size), 4791 }, 4792 [BPF_PROG_INFO_JITED_LINE_INFO] = { 4793 offsetof(struct bpf_prog_info, jited_line_info), 4794 offsetof(struct bpf_prog_info, nr_jited_line_info), 4795 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 4796 }, 4797 [BPF_PROG_INFO_PROG_TAGS] = { 4798 offsetof(struct bpf_prog_info, prog_tags), 4799 offsetof(struct bpf_prog_info, nr_prog_tags), 4800 -(int)sizeof(__u8) * BPF_TAG_SIZE, 4801 }, 4802 4803 }; 4804 4805 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 4806 { 4807 __u32 *array = (__u32 *)info; 4808 4809 if (offset >= 0) 4810 return array[offset / sizeof(__u32)]; 4811 return -(int)offset; 4812 } 4813 4814 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 4815 { 4816 __u64 *array = (__u64 *)info; 4817 4818 if (offset >= 0) 4819 return array[offset / sizeof(__u64)]; 4820 return -(int)offset; 4821 } 4822 4823 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 4824 __u32 val) 4825 { 4826 __u32 *array = (__u32 *)info; 4827 4828 if (offset >= 0) 4829 array[offset / sizeof(__u32)] = val; 4830 } 4831 4832 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 4833 __u64 val) 4834 { 4835 __u64 *array = (__u64 *)info; 4836 4837 if (offset >= 0) 4838 array[offset / sizeof(__u64)] = val; 4839 } 4840 4841 struct bpf_prog_info_linear * 4842 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 4843 { 4844 struct bpf_prog_info_linear *info_linear; 4845 struct bpf_prog_info info = {}; 4846 __u32 info_len = sizeof(info); 4847 __u32 data_len = 0; 4848 int i, err; 4849 void *ptr; 4850 4851 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 4852 return ERR_PTR(-EINVAL); 4853 4854 /* step 1: get array dimensions */ 4855 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 4856 if (err) { 4857 pr_debug("can't get prog info: %s", strerror(errno)); 4858 return ERR_PTR(-EFAULT); 4859 } 4860 4861 /* step 2: calculate total size of all arrays */ 4862 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4863 bool include_array = (arrays & (1UL << i)) > 0; 4864 struct bpf_prog_info_array_desc *desc; 4865 __u32 count, size; 4866 4867 desc = bpf_prog_info_array_desc + i; 4868 4869 /* kernel is too old to support this field */ 4870 if (info_len < desc->array_offset + sizeof(__u32) || 4871 info_len < desc->count_offset + sizeof(__u32) || 4872 (desc->size_offset > 0 && info_len < desc->size_offset)) 4873 include_array = false; 4874 4875 if (!include_array) { 4876 arrays &= ~(1UL << i); /* clear the bit */ 4877 continue; 4878 } 4879 4880 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4881 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4882 4883 data_len += count * size; 4884 } 4885 4886 /* step 3: allocate continuous memory */ 4887 data_len = roundup(data_len, sizeof(__u64)); 4888 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 4889 if (!info_linear) 4890 return ERR_PTR(-ENOMEM); 4891 4892 /* step 4: fill data to info_linear->info */ 4893 info_linear->arrays = arrays; 4894 memset(&info_linear->info, 0, sizeof(info)); 4895 ptr = info_linear->data; 4896 4897 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4898 struct bpf_prog_info_array_desc *desc; 4899 __u32 count, size; 4900 4901 if ((arrays & (1UL << i)) == 0) 4902 continue; 4903 4904 desc = bpf_prog_info_array_desc + i; 4905 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4906 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4907 bpf_prog_info_set_offset_u32(&info_linear->info, 4908 desc->count_offset, count); 4909 bpf_prog_info_set_offset_u32(&info_linear->info, 4910 desc->size_offset, size); 4911 bpf_prog_info_set_offset_u64(&info_linear->info, 4912 desc->array_offset, 4913 ptr_to_u64(ptr)); 4914 ptr += count * size; 4915 } 4916 4917 /* step 5: call syscall again to get required arrays */ 4918 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 4919 if (err) { 4920 pr_debug("can't get prog info: %s", strerror(errno)); 4921 free(info_linear); 4922 return ERR_PTR(-EFAULT); 4923 } 4924 4925 /* step 6: verify the data */ 4926 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4927 struct bpf_prog_info_array_desc *desc; 4928 __u32 v1, v2; 4929 4930 if ((arrays & (1UL << i)) == 0) 4931 continue; 4932 4933 desc = bpf_prog_info_array_desc + i; 4934 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4935 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 4936 desc->count_offset); 4937 if (v1 != v2) 4938 pr_warning("%s: mismatch in element count\n", __func__); 4939 4940 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4941 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 4942 desc->size_offset); 4943 if (v1 != v2) 4944 pr_warning("%s: mismatch in rec size\n", __func__); 4945 } 4946 4947 /* step 7: update info_len and data_len */ 4948 info_linear->info_len = sizeof(struct bpf_prog_info); 4949 info_linear->data_len = data_len; 4950 4951 return info_linear; 4952 } 4953 4954 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 4955 { 4956 int i; 4957 4958 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4959 struct bpf_prog_info_array_desc *desc; 4960 __u64 addr, offs; 4961 4962 if ((info_linear->arrays & (1UL << i)) == 0) 4963 continue; 4964 4965 desc = bpf_prog_info_array_desc + i; 4966 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 4967 desc->array_offset); 4968 offs = addr - ptr_to_u64(info_linear->data); 4969 bpf_prog_info_set_offset_u64(&info_linear->info, 4970 desc->array_offset, offs); 4971 } 4972 } 4973 4974 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 4975 { 4976 int i; 4977 4978 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4979 struct bpf_prog_info_array_desc *desc; 4980 __u64 addr, offs; 4981 4982 if ((info_linear->arrays & (1UL << i)) == 0) 4983 continue; 4984 4985 desc = bpf_prog_info_array_desc + i; 4986 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 4987 desc->array_offset); 4988 addr = offs + ptr_to_u64(info_linear->data); 4989 bpf_prog_info_set_offset_u64(&info_linear->info, 4990 desc->array_offset, addr); 4991 } 4992 } 4993 4994 int libbpf_num_possible_cpus(void) 4995 { 4996 static const char *fcpu = "/sys/devices/system/cpu/possible"; 4997 int len = 0, n = 0, il = 0, ir = 0; 4998 unsigned int start = 0, end = 0; 4999 static int cpus; 5000 char buf[128]; 5001 int error = 0; 5002 int fd = -1; 5003 5004 if (cpus > 0) 5005 return cpus; 5006 5007 fd = open(fcpu, O_RDONLY); 5008 if (fd < 0) { 5009 error = errno; 5010 pr_warning("Failed to open file %s: %s\n", 5011 fcpu, strerror(error)); 5012 return -error; 5013 } 5014 len = read(fd, buf, sizeof(buf)); 5015 close(fd); 5016 if (len <= 0) { 5017 error = len ? errno : EINVAL; 5018 pr_warning("Failed to read # of possible cpus from %s: %s\n", 5019 fcpu, strerror(error)); 5020 return -error; 5021 } 5022 if (len == sizeof(buf)) { 5023 pr_warning("File %s size overflow\n", fcpu); 5024 return -EOVERFLOW; 5025 } 5026 buf[len] = '\0'; 5027 5028 for (ir = 0, cpus = 0; ir <= len; ir++) { 5029 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 5030 if (buf[ir] == ',' || buf[ir] == '\0') { 5031 buf[ir] = '\0'; 5032 n = sscanf(&buf[il], "%u-%u", &start, &end); 5033 if (n <= 0) { 5034 pr_warning("Failed to get # CPUs from %s\n", 5035 &buf[il]); 5036 return -EINVAL; 5037 } else if (n == 1) { 5038 end = start; 5039 } 5040 cpus += end - start + 1; 5041 il = ir + 1; 5042 } 5043 } 5044 if (cpus <= 0) { 5045 pr_warning("Invalid #CPUs %d from %s\n", cpus, fcpu); 5046 return -EINVAL; 5047 } 5048 return cpus; 5049 } 5050