1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <endian.h> 24 #include <fcntl.h> 25 #include <errno.h> 26 #include <asm/unistd.h> 27 #include <linux/err.h> 28 #include <linux/kernel.h> 29 #include <linux/bpf.h> 30 #include <linux/btf.h> 31 #include <linux/filter.h> 32 #include <linux/list.h> 33 #include <linux/limits.h> 34 #include <linux/perf_event.h> 35 #include <linux/ring_buffer.h> 36 #include <sys/epoll.h> 37 #include <sys/ioctl.h> 38 #include <sys/mman.h> 39 #include <sys/stat.h> 40 #include <sys/types.h> 41 #include <sys/vfs.h> 42 #include <tools/libc_compat.h> 43 #include <libelf.h> 44 #include <gelf.h> 45 46 #include "libbpf.h" 47 #include "bpf.h" 48 #include "btf.h" 49 #include "str_error.h" 50 #include "libbpf_internal.h" 51 52 #ifndef EM_BPF 53 #define EM_BPF 247 54 #endif 55 56 #ifndef BPF_FS_MAGIC 57 #define BPF_FS_MAGIC 0xcafe4a11 58 #endif 59 60 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 61 * compilation if user enables corresponding warning. Disable it explicitly. 62 */ 63 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 64 65 #define __printf(a, b) __attribute__((format(printf, a, b))) 66 67 static int __base_pr(enum libbpf_print_level level, const char *format, 68 va_list args) 69 { 70 if (level == LIBBPF_DEBUG) 71 return 0; 72 73 return vfprintf(stderr, format, args); 74 } 75 76 static libbpf_print_fn_t __libbpf_pr = __base_pr; 77 78 void libbpf_set_print(libbpf_print_fn_t fn) 79 { 80 __libbpf_pr = fn; 81 } 82 83 __printf(2, 3) 84 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 85 { 86 va_list args; 87 88 if (!__libbpf_pr) 89 return; 90 91 va_start(args, format); 92 __libbpf_pr(level, format, args); 93 va_end(args); 94 } 95 96 #define STRERR_BUFSIZE 128 97 98 #define CHECK_ERR(action, err, out) do { \ 99 err = action; \ 100 if (err) \ 101 goto out; \ 102 } while(0) 103 104 105 /* Copied from tools/perf/util/util.h */ 106 #ifndef zfree 107 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 108 #endif 109 110 #ifndef zclose 111 # define zclose(fd) ({ \ 112 int ___err = 0; \ 113 if ((fd) >= 0) \ 114 ___err = close((fd)); \ 115 fd = -1; \ 116 ___err; }) 117 #endif 118 119 #ifdef HAVE_LIBELF_MMAP_SUPPORT 120 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 121 #else 122 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 123 #endif 124 125 static inline __u64 ptr_to_u64(const void *ptr) 126 { 127 return (__u64) (unsigned long) ptr; 128 } 129 130 struct bpf_capabilities { 131 /* v4.14: kernel support for program & map names. */ 132 __u32 name:1; 133 /* v5.2: kernel support for global data sections. */ 134 __u32 global_data:1; 135 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 136 __u32 btf_func:1; 137 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 138 __u32 btf_datasec:1; 139 }; 140 141 /* 142 * bpf_prog should be a better name but it has been used in 143 * linux/filter.h. 144 */ 145 struct bpf_program { 146 /* Index in elf obj file, for relocation use. */ 147 int idx; 148 char *name; 149 int prog_ifindex; 150 char *section_name; 151 /* section_name with / replaced by _; makes recursive pinning 152 * in bpf_object__pin_programs easier 153 */ 154 char *pin_name; 155 struct bpf_insn *insns; 156 size_t insns_cnt, main_prog_cnt; 157 enum bpf_prog_type type; 158 159 struct reloc_desc { 160 enum { 161 RELO_LD64, 162 RELO_CALL, 163 RELO_DATA, 164 } type; 165 int insn_idx; 166 union { 167 int map_idx; 168 int text_off; 169 }; 170 } *reloc_desc; 171 int nr_reloc; 172 int log_level; 173 174 struct { 175 int nr; 176 int *fds; 177 } instances; 178 bpf_program_prep_t preprocessor; 179 180 struct bpf_object *obj; 181 void *priv; 182 bpf_program_clear_priv_t clear_priv; 183 184 enum bpf_attach_type expected_attach_type; 185 void *func_info; 186 __u32 func_info_rec_size; 187 __u32 func_info_cnt; 188 189 struct bpf_capabilities *caps; 190 191 void *line_info; 192 __u32 line_info_rec_size; 193 __u32 line_info_cnt; 194 __u32 prog_flags; 195 }; 196 197 enum libbpf_map_type { 198 LIBBPF_MAP_UNSPEC, 199 LIBBPF_MAP_DATA, 200 LIBBPF_MAP_BSS, 201 LIBBPF_MAP_RODATA, 202 }; 203 204 static const char * const libbpf_type_to_btf_name[] = { 205 [LIBBPF_MAP_DATA] = ".data", 206 [LIBBPF_MAP_BSS] = ".bss", 207 [LIBBPF_MAP_RODATA] = ".rodata", 208 }; 209 210 struct bpf_map { 211 int fd; 212 char *name; 213 int sec_idx; 214 size_t sec_offset; 215 int map_ifindex; 216 int inner_map_fd; 217 struct bpf_map_def def; 218 __u32 btf_key_type_id; 219 __u32 btf_value_type_id; 220 void *priv; 221 bpf_map_clear_priv_t clear_priv; 222 enum libbpf_map_type libbpf_type; 223 }; 224 225 struct bpf_secdata { 226 void *rodata; 227 void *data; 228 }; 229 230 static LIST_HEAD(bpf_objects_list); 231 232 struct bpf_object { 233 char name[BPF_OBJ_NAME_LEN]; 234 char license[64]; 235 __u32 kern_version; 236 237 struct bpf_program *programs; 238 size_t nr_programs; 239 struct bpf_map *maps; 240 size_t nr_maps; 241 size_t maps_cap; 242 struct bpf_secdata sections; 243 244 bool loaded; 245 bool has_pseudo_calls; 246 247 /* 248 * Information when doing elf related work. Only valid if fd 249 * is valid. 250 */ 251 struct { 252 int fd; 253 void *obj_buf; 254 size_t obj_buf_sz; 255 Elf *elf; 256 GElf_Ehdr ehdr; 257 Elf_Data *symbols; 258 Elf_Data *data; 259 Elf_Data *rodata; 260 Elf_Data *bss; 261 size_t strtabidx; 262 struct { 263 GElf_Shdr shdr; 264 Elf_Data *data; 265 } *reloc; 266 int nr_reloc; 267 int maps_shndx; 268 int btf_maps_shndx; 269 int text_shndx; 270 int data_shndx; 271 int rodata_shndx; 272 int bss_shndx; 273 } efile; 274 /* 275 * All loaded bpf_object is linked in a list, which is 276 * hidden to caller. bpf_objects__<func> handlers deal with 277 * all objects. 278 */ 279 struct list_head list; 280 281 struct btf *btf; 282 struct btf_ext *btf_ext; 283 284 void *priv; 285 bpf_object_clear_priv_t clear_priv; 286 287 struct bpf_capabilities caps; 288 289 char path[]; 290 }; 291 #define obj_elf_valid(o) ((o)->efile.elf) 292 293 void bpf_program__unload(struct bpf_program *prog) 294 { 295 int i; 296 297 if (!prog) 298 return; 299 300 /* 301 * If the object is opened but the program was never loaded, 302 * it is possible that prog->instances.nr == -1. 303 */ 304 if (prog->instances.nr > 0) { 305 for (i = 0; i < prog->instances.nr; i++) 306 zclose(prog->instances.fds[i]); 307 } else if (prog->instances.nr != -1) { 308 pr_warning("Internal error: instances.nr is %d\n", 309 prog->instances.nr); 310 } 311 312 prog->instances.nr = -1; 313 zfree(&prog->instances.fds); 314 315 zfree(&prog->func_info); 316 zfree(&prog->line_info); 317 } 318 319 static void bpf_program__exit(struct bpf_program *prog) 320 { 321 if (!prog) 322 return; 323 324 if (prog->clear_priv) 325 prog->clear_priv(prog, prog->priv); 326 327 prog->priv = NULL; 328 prog->clear_priv = NULL; 329 330 bpf_program__unload(prog); 331 zfree(&prog->name); 332 zfree(&prog->section_name); 333 zfree(&prog->pin_name); 334 zfree(&prog->insns); 335 zfree(&prog->reloc_desc); 336 337 prog->nr_reloc = 0; 338 prog->insns_cnt = 0; 339 prog->idx = -1; 340 } 341 342 static char *__bpf_program__pin_name(struct bpf_program *prog) 343 { 344 char *name, *p; 345 346 name = p = strdup(prog->section_name); 347 while ((p = strchr(p, '/'))) 348 *p = '_'; 349 350 return name; 351 } 352 353 static int 354 bpf_program__init(void *data, size_t size, char *section_name, int idx, 355 struct bpf_program *prog) 356 { 357 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 358 359 if (size == 0 || size % bpf_insn_sz) { 360 pr_warning("corrupted section '%s', size: %zu\n", 361 section_name, size); 362 return -EINVAL; 363 } 364 365 memset(prog, 0, sizeof(*prog)); 366 367 prog->section_name = strdup(section_name); 368 if (!prog->section_name) { 369 pr_warning("failed to alloc name for prog under section(%d) %s\n", 370 idx, section_name); 371 goto errout; 372 } 373 374 prog->pin_name = __bpf_program__pin_name(prog); 375 if (!prog->pin_name) { 376 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 377 idx, section_name); 378 goto errout; 379 } 380 381 prog->insns = malloc(size); 382 if (!prog->insns) { 383 pr_warning("failed to alloc insns for prog under section %s\n", 384 section_name); 385 goto errout; 386 } 387 prog->insns_cnt = size / bpf_insn_sz; 388 memcpy(prog->insns, data, size); 389 prog->idx = idx; 390 prog->instances.fds = NULL; 391 prog->instances.nr = -1; 392 prog->type = BPF_PROG_TYPE_UNSPEC; 393 394 return 0; 395 errout: 396 bpf_program__exit(prog); 397 return -ENOMEM; 398 } 399 400 static int 401 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 402 char *section_name, int idx) 403 { 404 struct bpf_program prog, *progs; 405 int nr_progs, err; 406 407 err = bpf_program__init(data, size, section_name, idx, &prog); 408 if (err) 409 return err; 410 411 prog.caps = &obj->caps; 412 progs = obj->programs; 413 nr_progs = obj->nr_programs; 414 415 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 416 if (!progs) { 417 /* 418 * In this case the original obj->programs 419 * is still valid, so don't need special treat for 420 * bpf_close_object(). 421 */ 422 pr_warning("failed to alloc a new program under section '%s'\n", 423 section_name); 424 bpf_program__exit(&prog); 425 return -ENOMEM; 426 } 427 428 pr_debug("found program %s\n", prog.section_name); 429 obj->programs = progs; 430 obj->nr_programs = nr_progs + 1; 431 prog.obj = obj; 432 progs[nr_progs] = prog; 433 return 0; 434 } 435 436 static int 437 bpf_object__init_prog_names(struct bpf_object *obj) 438 { 439 Elf_Data *symbols = obj->efile.symbols; 440 struct bpf_program *prog; 441 size_t pi, si; 442 443 for (pi = 0; pi < obj->nr_programs; pi++) { 444 const char *name = NULL; 445 446 prog = &obj->programs[pi]; 447 448 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 449 si++) { 450 GElf_Sym sym; 451 452 if (!gelf_getsym(symbols, si, &sym)) 453 continue; 454 if (sym.st_shndx != prog->idx) 455 continue; 456 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 457 continue; 458 459 name = elf_strptr(obj->efile.elf, 460 obj->efile.strtabidx, 461 sym.st_name); 462 if (!name) { 463 pr_warning("failed to get sym name string for prog %s\n", 464 prog->section_name); 465 return -LIBBPF_ERRNO__LIBELF; 466 } 467 } 468 469 if (!name && prog->idx == obj->efile.text_shndx) 470 name = ".text"; 471 472 if (!name) { 473 pr_warning("failed to find sym for prog %s\n", 474 prog->section_name); 475 return -EINVAL; 476 } 477 478 prog->name = strdup(name); 479 if (!prog->name) { 480 pr_warning("failed to allocate memory for prog sym %s\n", 481 name); 482 return -ENOMEM; 483 } 484 } 485 486 return 0; 487 } 488 489 static struct bpf_object *bpf_object__new(const char *path, 490 void *obj_buf, 491 size_t obj_buf_sz) 492 { 493 struct bpf_object *obj; 494 char *end; 495 496 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 497 if (!obj) { 498 pr_warning("alloc memory failed for %s\n", path); 499 return ERR_PTR(-ENOMEM); 500 } 501 502 strcpy(obj->path, path); 503 /* Using basename() GNU version which doesn't modify arg. */ 504 strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); 505 end = strchr(obj->name, '.'); 506 if (end) 507 *end = 0; 508 509 obj->efile.fd = -1; 510 /* 511 * Caller of this function should also call 512 * bpf_object__elf_finish() after data collection to return 513 * obj_buf to user. If not, we should duplicate the buffer to 514 * avoid user freeing them before elf finish. 515 */ 516 obj->efile.obj_buf = obj_buf; 517 obj->efile.obj_buf_sz = obj_buf_sz; 518 obj->efile.maps_shndx = -1; 519 obj->efile.btf_maps_shndx = -1; 520 obj->efile.data_shndx = -1; 521 obj->efile.rodata_shndx = -1; 522 obj->efile.bss_shndx = -1; 523 524 obj->loaded = false; 525 526 INIT_LIST_HEAD(&obj->list); 527 list_add(&obj->list, &bpf_objects_list); 528 return obj; 529 } 530 531 static void bpf_object__elf_finish(struct bpf_object *obj) 532 { 533 if (!obj_elf_valid(obj)) 534 return; 535 536 if (obj->efile.elf) { 537 elf_end(obj->efile.elf); 538 obj->efile.elf = NULL; 539 } 540 obj->efile.symbols = NULL; 541 obj->efile.data = NULL; 542 obj->efile.rodata = NULL; 543 obj->efile.bss = NULL; 544 545 zfree(&obj->efile.reloc); 546 obj->efile.nr_reloc = 0; 547 zclose(obj->efile.fd); 548 obj->efile.obj_buf = NULL; 549 obj->efile.obj_buf_sz = 0; 550 } 551 552 static int bpf_object__elf_init(struct bpf_object *obj) 553 { 554 int err = 0; 555 GElf_Ehdr *ep; 556 557 if (obj_elf_valid(obj)) { 558 pr_warning("elf init: internal error\n"); 559 return -LIBBPF_ERRNO__LIBELF; 560 } 561 562 if (obj->efile.obj_buf_sz > 0) { 563 /* 564 * obj_buf should have been validated by 565 * bpf_object__open_buffer(). 566 */ 567 obj->efile.elf = elf_memory(obj->efile.obj_buf, 568 obj->efile.obj_buf_sz); 569 } else { 570 obj->efile.fd = open(obj->path, O_RDONLY); 571 if (obj->efile.fd < 0) { 572 char errmsg[STRERR_BUFSIZE], *cp; 573 574 err = -errno; 575 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 576 pr_warning("failed to open %s: %s\n", obj->path, cp); 577 return err; 578 } 579 580 obj->efile.elf = elf_begin(obj->efile.fd, 581 LIBBPF_ELF_C_READ_MMAP, NULL); 582 } 583 584 if (!obj->efile.elf) { 585 pr_warning("failed to open %s as ELF file\n", obj->path); 586 err = -LIBBPF_ERRNO__LIBELF; 587 goto errout; 588 } 589 590 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 591 pr_warning("failed to get EHDR from %s\n", obj->path); 592 err = -LIBBPF_ERRNO__FORMAT; 593 goto errout; 594 } 595 ep = &obj->efile.ehdr; 596 597 /* Old LLVM set e_machine to EM_NONE */ 598 if (ep->e_type != ET_REL || 599 (ep->e_machine && ep->e_machine != EM_BPF)) { 600 pr_warning("%s is not an eBPF object file\n", obj->path); 601 err = -LIBBPF_ERRNO__FORMAT; 602 goto errout; 603 } 604 605 return 0; 606 errout: 607 bpf_object__elf_finish(obj); 608 return err; 609 } 610 611 static int bpf_object__check_endianness(struct bpf_object *obj) 612 { 613 #if __BYTE_ORDER == __LITTLE_ENDIAN 614 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 615 return 0; 616 #elif __BYTE_ORDER == __BIG_ENDIAN 617 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 618 return 0; 619 #else 620 # error "Unrecognized __BYTE_ORDER__" 621 #endif 622 pr_warning("endianness mismatch.\n"); 623 return -LIBBPF_ERRNO__ENDIAN; 624 } 625 626 static int 627 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 628 { 629 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 630 pr_debug("license of %s is %s\n", obj->path, obj->license); 631 return 0; 632 } 633 634 static int 635 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 636 { 637 __u32 kver; 638 639 if (size != sizeof(kver)) { 640 pr_warning("invalid kver section in %s\n", obj->path); 641 return -LIBBPF_ERRNO__FORMAT; 642 } 643 memcpy(&kver, data, sizeof(kver)); 644 obj->kern_version = kver; 645 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 646 return 0; 647 } 648 649 static int compare_bpf_map(const void *_a, const void *_b) 650 { 651 const struct bpf_map *a = _a; 652 const struct bpf_map *b = _b; 653 654 if (a->sec_idx != b->sec_idx) 655 return a->sec_idx - b->sec_idx; 656 return a->sec_offset - b->sec_offset; 657 } 658 659 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 660 { 661 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 662 type == BPF_MAP_TYPE_HASH_OF_MAPS) 663 return true; 664 return false; 665 } 666 667 static int bpf_object_search_section_size(const struct bpf_object *obj, 668 const char *name, size_t *d_size) 669 { 670 const GElf_Ehdr *ep = &obj->efile.ehdr; 671 Elf *elf = obj->efile.elf; 672 Elf_Scn *scn = NULL; 673 int idx = 0; 674 675 while ((scn = elf_nextscn(elf, scn)) != NULL) { 676 const char *sec_name; 677 Elf_Data *data; 678 GElf_Shdr sh; 679 680 idx++; 681 if (gelf_getshdr(scn, &sh) != &sh) { 682 pr_warning("failed to get section(%d) header from %s\n", 683 idx, obj->path); 684 return -EIO; 685 } 686 687 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 688 if (!sec_name) { 689 pr_warning("failed to get section(%d) name from %s\n", 690 idx, obj->path); 691 return -EIO; 692 } 693 694 if (strcmp(name, sec_name)) 695 continue; 696 697 data = elf_getdata(scn, 0); 698 if (!data) { 699 pr_warning("failed to get section(%d) data from %s(%s)\n", 700 idx, name, obj->path); 701 return -EIO; 702 } 703 704 *d_size = data->d_size; 705 return 0; 706 } 707 708 return -ENOENT; 709 } 710 711 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 712 __u32 *size) 713 { 714 int ret = -ENOENT; 715 size_t d_size; 716 717 *size = 0; 718 if (!name) { 719 return -EINVAL; 720 } else if (!strcmp(name, ".data")) { 721 if (obj->efile.data) 722 *size = obj->efile.data->d_size; 723 } else if (!strcmp(name, ".bss")) { 724 if (obj->efile.bss) 725 *size = obj->efile.bss->d_size; 726 } else if (!strcmp(name, ".rodata")) { 727 if (obj->efile.rodata) 728 *size = obj->efile.rodata->d_size; 729 } else { 730 ret = bpf_object_search_section_size(obj, name, &d_size); 731 if (!ret) 732 *size = d_size; 733 } 734 735 return *size ? 0 : ret; 736 } 737 738 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 739 __u32 *off) 740 { 741 Elf_Data *symbols = obj->efile.symbols; 742 const char *sname; 743 size_t si; 744 745 if (!name || !off) 746 return -EINVAL; 747 748 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 749 GElf_Sym sym; 750 751 if (!gelf_getsym(symbols, si, &sym)) 752 continue; 753 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 754 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 755 continue; 756 757 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 758 sym.st_name); 759 if (!sname) { 760 pr_warning("failed to get sym name string for var %s\n", 761 name); 762 return -EIO; 763 } 764 if (strcmp(name, sname) == 0) { 765 *off = sym.st_value; 766 return 0; 767 } 768 } 769 770 return -ENOENT; 771 } 772 773 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 774 { 775 struct bpf_map *new_maps; 776 size_t new_cap; 777 int i; 778 779 if (obj->nr_maps < obj->maps_cap) 780 return &obj->maps[obj->nr_maps++]; 781 782 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 783 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); 784 if (!new_maps) { 785 pr_warning("alloc maps for object failed\n"); 786 return ERR_PTR(-ENOMEM); 787 } 788 789 obj->maps_cap = new_cap; 790 obj->maps = new_maps; 791 792 /* zero out new maps */ 793 memset(obj->maps + obj->nr_maps, 0, 794 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 795 /* 796 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 797 * when failure (zclose won't close negative fd)). 798 */ 799 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 800 obj->maps[i].fd = -1; 801 obj->maps[i].inner_map_fd = -1; 802 } 803 804 return &obj->maps[obj->nr_maps++]; 805 } 806 807 static int 808 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 809 int sec_idx, Elf_Data *data, void **data_buff) 810 { 811 char map_name[BPF_OBJ_NAME_LEN]; 812 struct bpf_map_def *def; 813 struct bpf_map *map; 814 815 map = bpf_object__add_map(obj); 816 if (IS_ERR(map)) 817 return PTR_ERR(map); 818 819 map->libbpf_type = type; 820 map->sec_idx = sec_idx; 821 map->sec_offset = 0; 822 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 823 libbpf_type_to_btf_name[type]); 824 map->name = strdup(map_name); 825 if (!map->name) { 826 pr_warning("failed to alloc map name\n"); 827 return -ENOMEM; 828 } 829 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", 830 map_name, map->sec_idx, map->sec_offset); 831 832 def = &map->def; 833 def->type = BPF_MAP_TYPE_ARRAY; 834 def->key_size = sizeof(int); 835 def->value_size = data->d_size; 836 def->max_entries = 1; 837 def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; 838 if (data_buff) { 839 *data_buff = malloc(data->d_size); 840 if (!*data_buff) { 841 zfree(&map->name); 842 pr_warning("failed to alloc map content buffer\n"); 843 return -ENOMEM; 844 } 845 memcpy(*data_buff, data->d_buf, data->d_size); 846 } 847 848 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 849 return 0; 850 } 851 852 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 853 { 854 int err; 855 856 if (!obj->caps.global_data) 857 return 0; 858 /* 859 * Populate obj->maps with libbpf internal maps. 860 */ 861 if (obj->efile.data_shndx >= 0) { 862 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 863 obj->efile.data_shndx, 864 obj->efile.data, 865 &obj->sections.data); 866 if (err) 867 return err; 868 } 869 if (obj->efile.rodata_shndx >= 0) { 870 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 871 obj->efile.rodata_shndx, 872 obj->efile.rodata, 873 &obj->sections.rodata); 874 if (err) 875 return err; 876 } 877 if (obj->efile.bss_shndx >= 0) { 878 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 879 obj->efile.bss_shndx, 880 obj->efile.bss, NULL); 881 if (err) 882 return err; 883 } 884 return 0; 885 } 886 887 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 888 { 889 Elf_Data *symbols = obj->efile.symbols; 890 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 891 Elf_Data *data = NULL; 892 Elf_Scn *scn; 893 894 if (obj->efile.maps_shndx < 0) 895 return 0; 896 897 if (!symbols) 898 return -EINVAL; 899 900 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 901 if (scn) 902 data = elf_getdata(scn, NULL); 903 if (!scn || !data) { 904 pr_warning("failed to get Elf_Data from map section %d\n", 905 obj->efile.maps_shndx); 906 return -EINVAL; 907 } 908 909 /* 910 * Count number of maps. Each map has a name. 911 * Array of maps is not supported: only the first element is 912 * considered. 913 * 914 * TODO: Detect array of map and report error. 915 */ 916 nr_syms = symbols->d_size / sizeof(GElf_Sym); 917 for (i = 0; i < nr_syms; i++) { 918 GElf_Sym sym; 919 920 if (!gelf_getsym(symbols, i, &sym)) 921 continue; 922 if (sym.st_shndx != obj->efile.maps_shndx) 923 continue; 924 nr_maps++; 925 } 926 /* Assume equally sized map definitions */ 927 pr_debug("maps in %s: %d maps in %zd bytes\n", 928 obj->path, nr_maps, data->d_size); 929 930 map_def_sz = data->d_size / nr_maps; 931 if (!data->d_size || (data->d_size % nr_maps) != 0) { 932 pr_warning("unable to determine map definition size " 933 "section %s, %d maps in %zd bytes\n", 934 obj->path, nr_maps, data->d_size); 935 return -EINVAL; 936 } 937 938 /* Fill obj->maps using data in "maps" section. */ 939 for (i = 0; i < nr_syms; i++) { 940 GElf_Sym sym; 941 const char *map_name; 942 struct bpf_map_def *def; 943 struct bpf_map *map; 944 945 if (!gelf_getsym(symbols, i, &sym)) 946 continue; 947 if (sym.st_shndx != obj->efile.maps_shndx) 948 continue; 949 950 map = bpf_object__add_map(obj); 951 if (IS_ERR(map)) 952 return PTR_ERR(map); 953 954 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 955 sym.st_name); 956 if (!map_name) { 957 pr_warning("failed to get map #%d name sym string for obj %s\n", 958 i, obj->path); 959 return -LIBBPF_ERRNO__FORMAT; 960 } 961 962 map->libbpf_type = LIBBPF_MAP_UNSPEC; 963 map->sec_idx = sym.st_shndx; 964 map->sec_offset = sym.st_value; 965 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 966 map_name, map->sec_idx, map->sec_offset); 967 if (sym.st_value + map_def_sz > data->d_size) { 968 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 969 obj->path, map_name); 970 return -EINVAL; 971 } 972 973 map->name = strdup(map_name); 974 if (!map->name) { 975 pr_warning("failed to alloc map name\n"); 976 return -ENOMEM; 977 } 978 pr_debug("map %d is \"%s\"\n", i, map->name); 979 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 980 /* 981 * If the definition of the map in the object file fits in 982 * bpf_map_def, copy it. Any extra fields in our version 983 * of bpf_map_def will default to zero as a result of the 984 * calloc above. 985 */ 986 if (map_def_sz <= sizeof(struct bpf_map_def)) { 987 memcpy(&map->def, def, map_def_sz); 988 } else { 989 /* 990 * Here the map structure being read is bigger than what 991 * we expect, truncate if the excess bits are all zero. 992 * If they are not zero, reject this map as 993 * incompatible. 994 */ 995 char *b; 996 for (b = ((char *)def) + sizeof(struct bpf_map_def); 997 b < ((char *)def) + map_def_sz; b++) { 998 if (*b != 0) { 999 pr_warning("maps section in %s: \"%s\" " 1000 "has unrecognized, non-zero " 1001 "options\n", 1002 obj->path, map_name); 1003 if (strict) 1004 return -EINVAL; 1005 } 1006 } 1007 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1008 } 1009 } 1010 return 0; 1011 } 1012 1013 static const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, 1014 __u32 id) 1015 { 1016 const struct btf_type *t = btf__type_by_id(btf, id); 1017 1018 while (true) { 1019 switch (BTF_INFO_KIND(t->info)) { 1020 case BTF_KIND_VOLATILE: 1021 case BTF_KIND_CONST: 1022 case BTF_KIND_RESTRICT: 1023 case BTF_KIND_TYPEDEF: 1024 t = btf__type_by_id(btf, t->type); 1025 break; 1026 default: 1027 return t; 1028 } 1029 } 1030 } 1031 1032 /* 1033 * Fetch integer attribute of BTF map definition. Such attributes are 1034 * represented using a pointer to an array, in which dimensionality of array 1035 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 1036 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 1037 * type definition, while using only sizeof(void *) space in ELF data section. 1038 */ 1039 static bool get_map_field_int(const char *map_name, const struct btf *btf, 1040 const struct btf_type *def, 1041 const struct btf_member *m, __u32 *res) { 1042 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type); 1043 const char *name = btf__name_by_offset(btf, m->name_off); 1044 const struct btf_array *arr_info; 1045 const struct btf_type *arr_t; 1046 1047 if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { 1048 pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", 1049 map_name, name, BTF_INFO_KIND(t->info)); 1050 return false; 1051 } 1052 1053 arr_t = btf__type_by_id(btf, t->type); 1054 if (!arr_t) { 1055 pr_warning("map '%s': attr '%s': type [%u] not found.\n", 1056 map_name, name, t->type); 1057 return false; 1058 } 1059 if (BTF_INFO_KIND(arr_t->info) != BTF_KIND_ARRAY) { 1060 pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", 1061 map_name, name, BTF_INFO_KIND(arr_t->info)); 1062 return false; 1063 } 1064 arr_info = (const void *)(arr_t + 1); 1065 *res = arr_info->nelems; 1066 return true; 1067 } 1068 1069 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 1070 const struct btf_type *sec, 1071 int var_idx, int sec_idx, 1072 const Elf_Data *data, bool strict) 1073 { 1074 const struct btf_type *var, *def, *t; 1075 const struct btf_var_secinfo *vi; 1076 const struct btf_var *var_extra; 1077 const struct btf_member *m; 1078 const char *map_name; 1079 struct bpf_map *map; 1080 int vlen, i; 1081 1082 vi = (const struct btf_var_secinfo *)(const void *)(sec + 1) + var_idx; 1083 var = btf__type_by_id(obj->btf, vi->type); 1084 var_extra = (const void *)(var + 1); 1085 map_name = btf__name_by_offset(obj->btf, var->name_off); 1086 vlen = BTF_INFO_VLEN(var->info); 1087 1088 if (map_name == NULL || map_name[0] == '\0') { 1089 pr_warning("map #%d: empty name.\n", var_idx); 1090 return -EINVAL; 1091 } 1092 if ((__u64)vi->offset + vi->size > data->d_size) { 1093 pr_warning("map '%s' BTF data is corrupted.\n", map_name); 1094 return -EINVAL; 1095 } 1096 if (BTF_INFO_KIND(var->info) != BTF_KIND_VAR) { 1097 pr_warning("map '%s': unexpected var kind %u.\n", 1098 map_name, BTF_INFO_KIND(var->info)); 1099 return -EINVAL; 1100 } 1101 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && 1102 var_extra->linkage != BTF_VAR_STATIC) { 1103 pr_warning("map '%s': unsupported var linkage %u.\n", 1104 map_name, var_extra->linkage); 1105 return -EOPNOTSUPP; 1106 } 1107 1108 def = skip_mods_and_typedefs(obj->btf, var->type); 1109 if (BTF_INFO_KIND(def->info) != BTF_KIND_STRUCT) { 1110 pr_warning("map '%s': unexpected def kind %u.\n", 1111 map_name, BTF_INFO_KIND(var->info)); 1112 return -EINVAL; 1113 } 1114 if (def->size > vi->size) { 1115 pr_warning("map '%s': invalid def size.\n", map_name); 1116 return -EINVAL; 1117 } 1118 1119 map = bpf_object__add_map(obj); 1120 if (IS_ERR(map)) 1121 return PTR_ERR(map); 1122 map->name = strdup(map_name); 1123 if (!map->name) { 1124 pr_warning("map '%s': failed to alloc map name.\n", map_name); 1125 return -ENOMEM; 1126 } 1127 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1128 map->def.type = BPF_MAP_TYPE_UNSPEC; 1129 map->sec_idx = sec_idx; 1130 map->sec_offset = vi->offset; 1131 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 1132 map_name, map->sec_idx, map->sec_offset); 1133 1134 vlen = BTF_INFO_VLEN(def->info); 1135 m = (const void *)(def + 1); 1136 for (i = 0; i < vlen; i++, m++) { 1137 const char *name = btf__name_by_offset(obj->btf, m->name_off); 1138 1139 if (!name) { 1140 pr_warning("map '%s': invalid field #%d.\n", 1141 map_name, i); 1142 return -EINVAL; 1143 } 1144 if (strcmp(name, "type") == 0) { 1145 if (!get_map_field_int(map_name, obj->btf, def, m, 1146 &map->def.type)) 1147 return -EINVAL; 1148 pr_debug("map '%s': found type = %u.\n", 1149 map_name, map->def.type); 1150 } else if (strcmp(name, "max_entries") == 0) { 1151 if (!get_map_field_int(map_name, obj->btf, def, m, 1152 &map->def.max_entries)) 1153 return -EINVAL; 1154 pr_debug("map '%s': found max_entries = %u.\n", 1155 map_name, map->def.max_entries); 1156 } else if (strcmp(name, "map_flags") == 0) { 1157 if (!get_map_field_int(map_name, obj->btf, def, m, 1158 &map->def.map_flags)) 1159 return -EINVAL; 1160 pr_debug("map '%s': found map_flags = %u.\n", 1161 map_name, map->def.map_flags); 1162 } else if (strcmp(name, "key_size") == 0) { 1163 __u32 sz; 1164 1165 if (!get_map_field_int(map_name, obj->btf, def, m, 1166 &sz)) 1167 return -EINVAL; 1168 pr_debug("map '%s': found key_size = %u.\n", 1169 map_name, sz); 1170 if (map->def.key_size && map->def.key_size != sz) { 1171 pr_warning("map '%s': conflicting key size %u != %u.\n", 1172 map_name, map->def.key_size, sz); 1173 return -EINVAL; 1174 } 1175 map->def.key_size = sz; 1176 } else if (strcmp(name, "key") == 0) { 1177 __s64 sz; 1178 1179 t = btf__type_by_id(obj->btf, m->type); 1180 if (!t) { 1181 pr_warning("map '%s': key type [%d] not found.\n", 1182 map_name, m->type); 1183 return -EINVAL; 1184 } 1185 if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { 1186 pr_warning("map '%s': key spec is not PTR: %u.\n", 1187 map_name, BTF_INFO_KIND(t->info)); 1188 return -EINVAL; 1189 } 1190 sz = btf__resolve_size(obj->btf, t->type); 1191 if (sz < 0) { 1192 pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", 1193 map_name, t->type, sz); 1194 return sz; 1195 } 1196 pr_debug("map '%s': found key [%u], sz = %lld.\n", 1197 map_name, t->type, sz); 1198 if (map->def.key_size && map->def.key_size != sz) { 1199 pr_warning("map '%s': conflicting key size %u != %lld.\n", 1200 map_name, map->def.key_size, sz); 1201 return -EINVAL; 1202 } 1203 map->def.key_size = sz; 1204 map->btf_key_type_id = t->type; 1205 } else if (strcmp(name, "value_size") == 0) { 1206 __u32 sz; 1207 1208 if (!get_map_field_int(map_name, obj->btf, def, m, 1209 &sz)) 1210 return -EINVAL; 1211 pr_debug("map '%s': found value_size = %u.\n", 1212 map_name, sz); 1213 if (map->def.value_size && map->def.value_size != sz) { 1214 pr_warning("map '%s': conflicting value size %u != %u.\n", 1215 map_name, map->def.value_size, sz); 1216 return -EINVAL; 1217 } 1218 map->def.value_size = sz; 1219 } else if (strcmp(name, "value") == 0) { 1220 __s64 sz; 1221 1222 t = btf__type_by_id(obj->btf, m->type); 1223 if (!t) { 1224 pr_warning("map '%s': value type [%d] not found.\n", 1225 map_name, m->type); 1226 return -EINVAL; 1227 } 1228 if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { 1229 pr_warning("map '%s': value spec is not PTR: %u.\n", 1230 map_name, BTF_INFO_KIND(t->info)); 1231 return -EINVAL; 1232 } 1233 sz = btf__resolve_size(obj->btf, t->type); 1234 if (sz < 0) { 1235 pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", 1236 map_name, t->type, sz); 1237 return sz; 1238 } 1239 pr_debug("map '%s': found value [%u], sz = %lld.\n", 1240 map_name, t->type, sz); 1241 if (map->def.value_size && map->def.value_size != sz) { 1242 pr_warning("map '%s': conflicting value size %u != %lld.\n", 1243 map_name, map->def.value_size, sz); 1244 return -EINVAL; 1245 } 1246 map->def.value_size = sz; 1247 map->btf_value_type_id = t->type; 1248 } else { 1249 if (strict) { 1250 pr_warning("map '%s': unknown field '%s'.\n", 1251 map_name, name); 1252 return -ENOTSUP; 1253 } 1254 pr_debug("map '%s': ignoring unknown field '%s'.\n", 1255 map_name, name); 1256 } 1257 } 1258 1259 if (map->def.type == BPF_MAP_TYPE_UNSPEC) { 1260 pr_warning("map '%s': map type isn't specified.\n", map_name); 1261 return -EINVAL; 1262 } 1263 1264 return 0; 1265 } 1266 1267 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) 1268 { 1269 const struct btf_type *sec = NULL; 1270 int nr_types, i, vlen, err; 1271 const struct btf_type *t; 1272 const char *name; 1273 Elf_Data *data; 1274 Elf_Scn *scn; 1275 1276 if (obj->efile.btf_maps_shndx < 0) 1277 return 0; 1278 1279 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); 1280 if (scn) 1281 data = elf_getdata(scn, NULL); 1282 if (!scn || !data) { 1283 pr_warning("failed to get Elf_Data from map section %d (%s)\n", 1284 obj->efile.maps_shndx, MAPS_ELF_SEC); 1285 return -EINVAL; 1286 } 1287 1288 nr_types = btf__get_nr_types(obj->btf); 1289 for (i = 1; i <= nr_types; i++) { 1290 t = btf__type_by_id(obj->btf, i); 1291 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 1292 continue; 1293 name = btf__name_by_offset(obj->btf, t->name_off); 1294 if (strcmp(name, MAPS_ELF_SEC) == 0) { 1295 sec = t; 1296 break; 1297 } 1298 } 1299 1300 if (!sec) { 1301 pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 1302 return -ENOENT; 1303 } 1304 1305 vlen = BTF_INFO_VLEN(sec->info); 1306 for (i = 0; i < vlen; i++) { 1307 err = bpf_object__init_user_btf_map(obj, sec, i, 1308 obj->efile.btf_maps_shndx, 1309 data, strict); 1310 if (err) 1311 return err; 1312 } 1313 1314 return 0; 1315 } 1316 1317 static int bpf_object__init_maps(struct bpf_object *obj, int flags) 1318 { 1319 bool strict = !(flags & MAPS_RELAX_COMPAT); 1320 int err; 1321 1322 err = bpf_object__init_user_maps(obj, strict); 1323 if (err) 1324 return err; 1325 1326 err = bpf_object__init_user_btf_maps(obj, strict); 1327 if (err) 1328 return err; 1329 1330 err = bpf_object__init_global_data_maps(obj); 1331 if (err) 1332 return err; 1333 1334 if (obj->nr_maps) { 1335 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1336 compare_bpf_map); 1337 } 1338 return 0; 1339 } 1340 1341 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1342 { 1343 Elf_Scn *scn; 1344 GElf_Shdr sh; 1345 1346 scn = elf_getscn(obj->efile.elf, idx); 1347 if (!scn) 1348 return false; 1349 1350 if (gelf_getshdr(scn, &sh) != &sh) 1351 return false; 1352 1353 if (sh.sh_flags & SHF_EXECINSTR) 1354 return true; 1355 1356 return false; 1357 } 1358 1359 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1360 { 1361 bool has_datasec = obj->caps.btf_datasec; 1362 bool has_func = obj->caps.btf_func; 1363 struct btf *btf = obj->btf; 1364 struct btf_type *t; 1365 int i, j, vlen; 1366 __u16 kind; 1367 1368 if (!obj->btf || (has_func && has_datasec)) 1369 return; 1370 1371 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1372 t = (struct btf_type *)btf__type_by_id(btf, i); 1373 kind = BTF_INFO_KIND(t->info); 1374 1375 if (!has_datasec && kind == BTF_KIND_VAR) { 1376 /* replace VAR with INT */ 1377 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1378 /* 1379 * using size = 1 is the safest choice, 4 will be too 1380 * big and cause kernel BTF validation failure if 1381 * original variable took less than 4 bytes 1382 */ 1383 t->size = 1; 1384 *(int *)(t+1) = BTF_INT_ENC(0, 0, 8); 1385 } else if (!has_datasec && kind == BTF_KIND_DATASEC) { 1386 /* replace DATASEC with STRUCT */ 1387 struct btf_var_secinfo *v = (void *)(t + 1); 1388 struct btf_member *m = (void *)(t + 1); 1389 struct btf_type *vt; 1390 char *name; 1391 1392 name = (char *)btf__name_by_offset(btf, t->name_off); 1393 while (*name) { 1394 if (*name == '.') 1395 *name = '_'; 1396 name++; 1397 } 1398 1399 vlen = BTF_INFO_VLEN(t->info); 1400 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1401 for (j = 0; j < vlen; j++, v++, m++) { 1402 /* order of field assignments is important */ 1403 m->offset = v->offset * 8; 1404 m->type = v->type; 1405 /* preserve variable name as member name */ 1406 vt = (void *)btf__type_by_id(btf, v->type); 1407 m->name_off = vt->name_off; 1408 } 1409 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) { 1410 /* replace FUNC_PROTO with ENUM */ 1411 vlen = BTF_INFO_VLEN(t->info); 1412 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1413 t->size = sizeof(__u32); /* kernel enforced */ 1414 } else if (!has_func && kind == BTF_KIND_FUNC) { 1415 /* replace FUNC with TYPEDEF */ 1416 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1417 } 1418 } 1419 } 1420 1421 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1422 { 1423 if (!obj->btf_ext) 1424 return; 1425 1426 if (!obj->caps.btf_func) { 1427 btf_ext__free(obj->btf_ext); 1428 obj->btf_ext = NULL; 1429 } 1430 } 1431 1432 static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) 1433 { 1434 return obj->efile.btf_maps_shndx >= 0; 1435 } 1436 1437 static int bpf_object__init_btf(struct bpf_object *obj, 1438 Elf_Data *btf_data, 1439 Elf_Data *btf_ext_data) 1440 { 1441 bool btf_required = bpf_object__is_btf_mandatory(obj); 1442 int err = 0; 1443 1444 if (btf_data) { 1445 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1446 if (IS_ERR(obj->btf)) { 1447 pr_warning("Error loading ELF section %s: %d.\n", 1448 BTF_ELF_SEC, err); 1449 goto out; 1450 } 1451 err = btf__finalize_data(obj, obj->btf); 1452 if (err) { 1453 pr_warning("Error finalizing %s: %d.\n", 1454 BTF_ELF_SEC, err); 1455 goto out; 1456 } 1457 } 1458 if (btf_ext_data) { 1459 if (!obj->btf) { 1460 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1461 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1462 goto out; 1463 } 1464 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1465 btf_ext_data->d_size); 1466 if (IS_ERR(obj->btf_ext)) { 1467 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1468 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); 1469 obj->btf_ext = NULL; 1470 goto out; 1471 } 1472 } 1473 out: 1474 if (err || IS_ERR(obj->btf)) { 1475 if (btf_required) 1476 err = err ? : PTR_ERR(obj->btf); 1477 else 1478 err = 0; 1479 if (!IS_ERR_OR_NULL(obj->btf)) 1480 btf__free(obj->btf); 1481 obj->btf = NULL; 1482 } 1483 if (btf_required && !obj->btf) { 1484 pr_warning("BTF is required, but is missing or corrupted.\n"); 1485 return err == 0 ? -ENOENT : err; 1486 } 1487 return 0; 1488 } 1489 1490 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 1491 { 1492 int err = 0; 1493 1494 if (!obj->btf) 1495 return 0; 1496 1497 bpf_object__sanitize_btf(obj); 1498 bpf_object__sanitize_btf_ext(obj); 1499 1500 err = btf__load(obj->btf); 1501 if (err) { 1502 pr_warning("Error loading %s into kernel: %d.\n", 1503 BTF_ELF_SEC, err); 1504 btf__free(obj->btf); 1505 obj->btf = NULL; 1506 /* btf_ext can't exist without btf, so free it as well */ 1507 if (obj->btf_ext) { 1508 btf_ext__free(obj->btf_ext); 1509 obj->btf_ext = NULL; 1510 } 1511 1512 if (bpf_object__is_btf_mandatory(obj)) 1513 return err; 1514 } 1515 return 0; 1516 } 1517 1518 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1519 { 1520 Elf *elf = obj->efile.elf; 1521 GElf_Ehdr *ep = &obj->efile.ehdr; 1522 Elf_Data *btf_ext_data = NULL; 1523 Elf_Data *btf_data = NULL; 1524 Elf_Scn *scn = NULL; 1525 int idx = 0, err = 0; 1526 1527 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1528 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1529 pr_warning("failed to get e_shstrndx from %s\n", obj->path); 1530 return -LIBBPF_ERRNO__FORMAT; 1531 } 1532 1533 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1534 char *name; 1535 GElf_Shdr sh; 1536 Elf_Data *data; 1537 1538 idx++; 1539 if (gelf_getshdr(scn, &sh) != &sh) { 1540 pr_warning("failed to get section(%d) header from %s\n", 1541 idx, obj->path); 1542 return -LIBBPF_ERRNO__FORMAT; 1543 } 1544 1545 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1546 if (!name) { 1547 pr_warning("failed to get section(%d) name from %s\n", 1548 idx, obj->path); 1549 return -LIBBPF_ERRNO__FORMAT; 1550 } 1551 1552 data = elf_getdata(scn, 0); 1553 if (!data) { 1554 pr_warning("failed to get section(%d) data from %s(%s)\n", 1555 idx, name, obj->path); 1556 return -LIBBPF_ERRNO__FORMAT; 1557 } 1558 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1559 idx, name, (unsigned long)data->d_size, 1560 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1561 (int)sh.sh_type); 1562 1563 if (strcmp(name, "license") == 0) { 1564 err = bpf_object__init_license(obj, 1565 data->d_buf, 1566 data->d_size); 1567 if (err) 1568 return err; 1569 } else if (strcmp(name, "version") == 0) { 1570 err = bpf_object__init_kversion(obj, 1571 data->d_buf, 1572 data->d_size); 1573 if (err) 1574 return err; 1575 } else if (strcmp(name, "maps") == 0) { 1576 obj->efile.maps_shndx = idx; 1577 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 1578 obj->efile.btf_maps_shndx = idx; 1579 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1580 btf_data = data; 1581 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1582 btf_ext_data = data; 1583 } else if (sh.sh_type == SHT_SYMTAB) { 1584 if (obj->efile.symbols) { 1585 pr_warning("bpf: multiple SYMTAB in %s\n", 1586 obj->path); 1587 return -LIBBPF_ERRNO__FORMAT; 1588 } 1589 obj->efile.symbols = data; 1590 obj->efile.strtabidx = sh.sh_link; 1591 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1592 if (sh.sh_flags & SHF_EXECINSTR) { 1593 if (strcmp(name, ".text") == 0) 1594 obj->efile.text_shndx = idx; 1595 err = bpf_object__add_program(obj, data->d_buf, 1596 data->d_size, name, idx); 1597 if (err) { 1598 char errmsg[STRERR_BUFSIZE]; 1599 char *cp = libbpf_strerror_r(-err, errmsg, 1600 sizeof(errmsg)); 1601 1602 pr_warning("failed to alloc program %s (%s): %s", 1603 name, obj->path, cp); 1604 return err; 1605 } 1606 } else if (strcmp(name, ".data") == 0) { 1607 obj->efile.data = data; 1608 obj->efile.data_shndx = idx; 1609 } else if (strcmp(name, ".rodata") == 0) { 1610 obj->efile.rodata = data; 1611 obj->efile.rodata_shndx = idx; 1612 } else { 1613 pr_debug("skip section(%d) %s\n", idx, name); 1614 } 1615 } else if (sh.sh_type == SHT_REL) { 1616 int nr_reloc = obj->efile.nr_reloc; 1617 void *reloc = obj->efile.reloc; 1618 int sec = sh.sh_info; /* points to other section */ 1619 1620 /* Only do relo for section with exec instructions */ 1621 if (!section_have_execinstr(obj, sec)) { 1622 pr_debug("skip relo %s(%d) for section(%d)\n", 1623 name, idx, sec); 1624 continue; 1625 } 1626 1627 reloc = reallocarray(reloc, nr_reloc + 1, 1628 sizeof(*obj->efile.reloc)); 1629 if (!reloc) { 1630 pr_warning("realloc failed\n"); 1631 return -ENOMEM; 1632 } 1633 1634 obj->efile.reloc = reloc; 1635 obj->efile.nr_reloc++; 1636 1637 obj->efile.reloc[nr_reloc].shdr = sh; 1638 obj->efile.reloc[nr_reloc].data = data; 1639 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1640 obj->efile.bss = data; 1641 obj->efile.bss_shndx = idx; 1642 } else { 1643 pr_debug("skip section(%d) %s\n", idx, name); 1644 } 1645 } 1646 1647 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1648 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1649 return -LIBBPF_ERRNO__FORMAT; 1650 } 1651 err = bpf_object__init_btf(obj, btf_data, btf_ext_data); 1652 if (!err) 1653 err = bpf_object__init_maps(obj, flags); 1654 if (!err) 1655 err = bpf_object__sanitize_and_load_btf(obj); 1656 if (!err) 1657 err = bpf_object__init_prog_names(obj); 1658 return err; 1659 } 1660 1661 static struct bpf_program * 1662 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1663 { 1664 struct bpf_program *prog; 1665 size_t i; 1666 1667 for (i = 0; i < obj->nr_programs; i++) { 1668 prog = &obj->programs[i]; 1669 if (prog->idx == idx) 1670 return prog; 1671 } 1672 return NULL; 1673 } 1674 1675 struct bpf_program * 1676 bpf_object__find_program_by_title(const struct bpf_object *obj, 1677 const char *title) 1678 { 1679 struct bpf_program *pos; 1680 1681 bpf_object__for_each_program(pos, obj) { 1682 if (pos->section_name && !strcmp(pos->section_name, title)) 1683 return pos; 1684 } 1685 return NULL; 1686 } 1687 1688 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1689 int shndx) 1690 { 1691 return shndx == obj->efile.data_shndx || 1692 shndx == obj->efile.bss_shndx || 1693 shndx == obj->efile.rodata_shndx; 1694 } 1695 1696 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1697 int shndx) 1698 { 1699 return shndx == obj->efile.maps_shndx || 1700 shndx == obj->efile.btf_maps_shndx; 1701 } 1702 1703 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1704 int shndx) 1705 { 1706 return shndx == obj->efile.text_shndx || 1707 bpf_object__shndx_is_maps(obj, shndx) || 1708 bpf_object__shndx_is_data(obj, shndx); 1709 } 1710 1711 static enum libbpf_map_type 1712 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1713 { 1714 if (shndx == obj->efile.data_shndx) 1715 return LIBBPF_MAP_DATA; 1716 else if (shndx == obj->efile.bss_shndx) 1717 return LIBBPF_MAP_BSS; 1718 else if (shndx == obj->efile.rodata_shndx) 1719 return LIBBPF_MAP_RODATA; 1720 else 1721 return LIBBPF_MAP_UNSPEC; 1722 } 1723 1724 static int 1725 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1726 Elf_Data *data, struct bpf_object *obj) 1727 { 1728 Elf_Data *symbols = obj->efile.symbols; 1729 struct bpf_map *maps = obj->maps; 1730 size_t nr_maps = obj->nr_maps; 1731 int i, nrels; 1732 1733 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 1734 nrels = shdr->sh_size / shdr->sh_entsize; 1735 1736 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1737 if (!prog->reloc_desc) { 1738 pr_warning("failed to alloc memory in relocation\n"); 1739 return -ENOMEM; 1740 } 1741 prog->nr_reloc = nrels; 1742 1743 for (i = 0; i < nrels; i++) { 1744 struct bpf_insn *insns = prog->insns; 1745 enum libbpf_map_type type; 1746 unsigned int insn_idx; 1747 unsigned int shdr_idx; 1748 const char *name; 1749 size_t map_idx; 1750 GElf_Sym sym; 1751 GElf_Rel rel; 1752 1753 if (!gelf_getrel(data, i, &rel)) { 1754 pr_warning("relocation: failed to get %d reloc\n", i); 1755 return -LIBBPF_ERRNO__FORMAT; 1756 } 1757 1758 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 1759 pr_warning("relocation: symbol %"PRIx64" not found\n", 1760 GELF_R_SYM(rel.r_info)); 1761 return -LIBBPF_ERRNO__FORMAT; 1762 } 1763 1764 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1765 sym.st_name) ? : "<?>"; 1766 1767 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1768 (long long) (rel.r_info >> 32), 1769 (long long) sym.st_value, sym.st_name, name); 1770 1771 shdr_idx = sym.st_shndx; 1772 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1773 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1774 prog->section_name, shdr_idx); 1775 return -LIBBPF_ERRNO__RELOC; 1776 } 1777 1778 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1779 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1780 1781 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1782 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1783 pr_warning("incorrect bpf_call opcode\n"); 1784 return -LIBBPF_ERRNO__RELOC; 1785 } 1786 prog->reloc_desc[i].type = RELO_CALL; 1787 prog->reloc_desc[i].insn_idx = insn_idx; 1788 prog->reloc_desc[i].text_off = sym.st_value; 1789 obj->has_pseudo_calls = true; 1790 continue; 1791 } 1792 1793 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1794 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1795 insn_idx, insns[insn_idx].code); 1796 return -LIBBPF_ERRNO__RELOC; 1797 } 1798 1799 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1800 bpf_object__shndx_is_data(obj, shdr_idx)) { 1801 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1802 if (type != LIBBPF_MAP_UNSPEC) { 1803 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1804 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1805 name, insn_idx, insns[insn_idx].code); 1806 return -LIBBPF_ERRNO__RELOC; 1807 } 1808 if (!obj->caps.global_data) { 1809 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1810 name, insn_idx); 1811 return -LIBBPF_ERRNO__RELOC; 1812 } 1813 } 1814 1815 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1816 if (maps[map_idx].libbpf_type != type) 1817 continue; 1818 if (type != LIBBPF_MAP_UNSPEC || 1819 (maps[map_idx].sec_idx == sym.st_shndx && 1820 maps[map_idx].sec_offset == sym.st_value)) { 1821 pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", 1822 map_idx, maps[map_idx].name, 1823 maps[map_idx].sec_idx, 1824 maps[map_idx].sec_offset, 1825 insn_idx); 1826 break; 1827 } 1828 } 1829 1830 if (map_idx >= nr_maps) { 1831 pr_warning("bpf relocation: map_idx %d larger than %d\n", 1832 (int)map_idx, (int)nr_maps - 1); 1833 return -LIBBPF_ERRNO__RELOC; 1834 } 1835 1836 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1837 RELO_DATA : RELO_LD64; 1838 prog->reloc_desc[i].insn_idx = insn_idx; 1839 prog->reloc_desc[i].map_idx = map_idx; 1840 } 1841 } 1842 return 0; 1843 } 1844 1845 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 1846 { 1847 struct bpf_map_def *def = &map->def; 1848 __u32 key_type_id = 0, value_type_id = 0; 1849 int ret; 1850 1851 /* if it's BTF-defined map, we don't need to search for type IDs */ 1852 if (map->sec_idx == obj->efile.btf_maps_shndx) 1853 return 0; 1854 1855 if (!bpf_map__is_internal(map)) { 1856 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 1857 def->value_size, &key_type_id, 1858 &value_type_id); 1859 } else { 1860 /* 1861 * LLVM annotates global data differently in BTF, that is, 1862 * only as '.data', '.bss' or '.rodata'. 1863 */ 1864 ret = btf__find_by_name(obj->btf, 1865 libbpf_type_to_btf_name[map->libbpf_type]); 1866 } 1867 if (ret < 0) 1868 return ret; 1869 1870 map->btf_key_type_id = key_type_id; 1871 map->btf_value_type_id = bpf_map__is_internal(map) ? 1872 ret : value_type_id; 1873 return 0; 1874 } 1875 1876 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1877 { 1878 struct bpf_map_info info = {}; 1879 __u32 len = sizeof(info); 1880 int new_fd, err; 1881 char *new_name; 1882 1883 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1884 if (err) 1885 return err; 1886 1887 new_name = strdup(info.name); 1888 if (!new_name) 1889 return -errno; 1890 1891 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1892 if (new_fd < 0) 1893 goto err_free_new_name; 1894 1895 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1896 if (new_fd < 0) 1897 goto err_close_new_fd; 1898 1899 err = zclose(map->fd); 1900 if (err) 1901 goto err_close_new_fd; 1902 free(map->name); 1903 1904 map->fd = new_fd; 1905 map->name = new_name; 1906 map->def.type = info.type; 1907 map->def.key_size = info.key_size; 1908 map->def.value_size = info.value_size; 1909 map->def.max_entries = info.max_entries; 1910 map->def.map_flags = info.map_flags; 1911 map->btf_key_type_id = info.btf_key_type_id; 1912 map->btf_value_type_id = info.btf_value_type_id; 1913 1914 return 0; 1915 1916 err_close_new_fd: 1917 close(new_fd); 1918 err_free_new_name: 1919 free(new_name); 1920 return -errno; 1921 } 1922 1923 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1924 { 1925 if (!map || !max_entries) 1926 return -EINVAL; 1927 1928 /* If map already created, its attributes can't be changed. */ 1929 if (map->fd >= 0) 1930 return -EBUSY; 1931 1932 map->def.max_entries = max_entries; 1933 1934 return 0; 1935 } 1936 1937 static int 1938 bpf_object__probe_name(struct bpf_object *obj) 1939 { 1940 struct bpf_load_program_attr attr; 1941 char *cp, errmsg[STRERR_BUFSIZE]; 1942 struct bpf_insn insns[] = { 1943 BPF_MOV64_IMM(BPF_REG_0, 0), 1944 BPF_EXIT_INSN(), 1945 }; 1946 int ret; 1947 1948 /* make sure basic loading works */ 1949 1950 memset(&attr, 0, sizeof(attr)); 1951 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1952 attr.insns = insns; 1953 attr.insns_cnt = ARRAY_SIZE(insns); 1954 attr.license = "GPL"; 1955 1956 ret = bpf_load_program_xattr(&attr, NULL, 0); 1957 if (ret < 0) { 1958 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1959 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1960 __func__, cp, errno); 1961 return -errno; 1962 } 1963 close(ret); 1964 1965 /* now try the same program, but with the name */ 1966 1967 attr.name = "test"; 1968 ret = bpf_load_program_xattr(&attr, NULL, 0); 1969 if (ret >= 0) { 1970 obj->caps.name = 1; 1971 close(ret); 1972 } 1973 1974 return 0; 1975 } 1976 1977 static int 1978 bpf_object__probe_global_data(struct bpf_object *obj) 1979 { 1980 struct bpf_load_program_attr prg_attr; 1981 struct bpf_create_map_attr map_attr; 1982 char *cp, errmsg[STRERR_BUFSIZE]; 1983 struct bpf_insn insns[] = { 1984 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1985 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1986 BPF_MOV64_IMM(BPF_REG_0, 0), 1987 BPF_EXIT_INSN(), 1988 }; 1989 int ret, map; 1990 1991 memset(&map_attr, 0, sizeof(map_attr)); 1992 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1993 map_attr.key_size = sizeof(int); 1994 map_attr.value_size = 32; 1995 map_attr.max_entries = 1; 1996 1997 map = bpf_create_map_xattr(&map_attr); 1998 if (map < 0) { 1999 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2000 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 2001 __func__, cp, errno); 2002 return -errno; 2003 } 2004 2005 insns[0].imm = map; 2006 2007 memset(&prg_attr, 0, sizeof(prg_attr)); 2008 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 2009 prg_attr.insns = insns; 2010 prg_attr.insns_cnt = ARRAY_SIZE(insns); 2011 prg_attr.license = "GPL"; 2012 2013 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 2014 if (ret >= 0) { 2015 obj->caps.global_data = 1; 2016 close(ret); 2017 } 2018 2019 close(map); 2020 return 0; 2021 } 2022 2023 static int bpf_object__probe_btf_func(struct bpf_object *obj) 2024 { 2025 const char strs[] = "\0int\0x\0a"; 2026 /* void x(int a) {} */ 2027 __u32 types[] = { 2028 /* int */ 2029 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 2030 /* FUNC_PROTO */ /* [2] */ 2031 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 2032 BTF_PARAM_ENC(7, 1), 2033 /* FUNC x */ /* [3] */ 2034 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 2035 }; 2036 int btf_fd; 2037 2038 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 2039 strs, sizeof(strs)); 2040 if (btf_fd >= 0) { 2041 obj->caps.btf_func = 1; 2042 close(btf_fd); 2043 return 1; 2044 } 2045 2046 return 0; 2047 } 2048 2049 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 2050 { 2051 const char strs[] = "\0x\0.data"; 2052 /* static int a; */ 2053 __u32 types[] = { 2054 /* int */ 2055 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 2056 /* VAR x */ /* [2] */ 2057 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 2058 BTF_VAR_STATIC, 2059 /* DATASEC val */ /* [3] */ 2060 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 2061 BTF_VAR_SECINFO_ENC(2, 0, 4), 2062 }; 2063 int btf_fd; 2064 2065 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 2066 strs, sizeof(strs)); 2067 if (btf_fd >= 0) { 2068 obj->caps.btf_datasec = 1; 2069 close(btf_fd); 2070 return 1; 2071 } 2072 2073 return 0; 2074 } 2075 2076 static int 2077 bpf_object__probe_caps(struct bpf_object *obj) 2078 { 2079 int (*probe_fn[])(struct bpf_object *obj) = { 2080 bpf_object__probe_name, 2081 bpf_object__probe_global_data, 2082 bpf_object__probe_btf_func, 2083 bpf_object__probe_btf_datasec, 2084 }; 2085 int i, ret; 2086 2087 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 2088 ret = probe_fn[i](obj); 2089 if (ret < 0) 2090 pr_debug("Probe #%d failed with %d.\n", i, ret); 2091 } 2092 2093 return 0; 2094 } 2095 2096 static int 2097 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 2098 { 2099 char *cp, errmsg[STRERR_BUFSIZE]; 2100 int err, zero = 0; 2101 __u8 *data; 2102 2103 /* Nothing to do here since kernel already zero-initializes .bss map. */ 2104 if (map->libbpf_type == LIBBPF_MAP_BSS) 2105 return 0; 2106 2107 data = map->libbpf_type == LIBBPF_MAP_DATA ? 2108 obj->sections.data : obj->sections.rodata; 2109 2110 err = bpf_map_update_elem(map->fd, &zero, data, 0); 2111 /* Freeze .rodata map as read-only from syscall side. */ 2112 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 2113 err = bpf_map_freeze(map->fd); 2114 if (err) { 2115 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2116 pr_warning("Error freezing map(%s) as read-only: %s\n", 2117 map->name, cp); 2118 err = 0; 2119 } 2120 } 2121 return err; 2122 } 2123 2124 static int 2125 bpf_object__create_maps(struct bpf_object *obj) 2126 { 2127 struct bpf_create_map_attr create_attr = {}; 2128 int nr_cpus = 0; 2129 unsigned int i; 2130 int err; 2131 2132 for (i = 0; i < obj->nr_maps; i++) { 2133 struct bpf_map *map = &obj->maps[i]; 2134 struct bpf_map_def *def = &map->def; 2135 char *cp, errmsg[STRERR_BUFSIZE]; 2136 int *pfd = &map->fd; 2137 2138 if (map->fd >= 0) { 2139 pr_debug("skip map create (preset) %s: fd=%d\n", 2140 map->name, map->fd); 2141 continue; 2142 } 2143 2144 if (obj->caps.name) 2145 create_attr.name = map->name; 2146 create_attr.map_ifindex = map->map_ifindex; 2147 create_attr.map_type = def->type; 2148 create_attr.map_flags = def->map_flags; 2149 create_attr.key_size = def->key_size; 2150 create_attr.value_size = def->value_size; 2151 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && 2152 !def->max_entries) { 2153 if (!nr_cpus) 2154 nr_cpus = libbpf_num_possible_cpus(); 2155 if (nr_cpus < 0) { 2156 pr_warning("failed to determine number of system CPUs: %d\n", 2157 nr_cpus); 2158 err = nr_cpus; 2159 goto err_out; 2160 } 2161 pr_debug("map '%s': setting size to %d\n", 2162 map->name, nr_cpus); 2163 create_attr.max_entries = nr_cpus; 2164 } else { 2165 create_attr.max_entries = def->max_entries; 2166 } 2167 create_attr.btf_fd = 0; 2168 create_attr.btf_key_type_id = 0; 2169 create_attr.btf_value_type_id = 0; 2170 if (bpf_map_type__is_map_in_map(def->type) && 2171 map->inner_map_fd >= 0) 2172 create_attr.inner_map_fd = map->inner_map_fd; 2173 2174 if (obj->btf && !bpf_map_find_btf_info(obj, map)) { 2175 create_attr.btf_fd = btf__fd(obj->btf); 2176 create_attr.btf_key_type_id = map->btf_key_type_id; 2177 create_attr.btf_value_type_id = map->btf_value_type_id; 2178 } 2179 2180 *pfd = bpf_create_map_xattr(&create_attr); 2181 if (*pfd < 0 && (create_attr.btf_key_type_id || 2182 create_attr.btf_value_type_id)) { 2183 err = -errno; 2184 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 2185 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 2186 map->name, cp, err); 2187 create_attr.btf_fd = 0; 2188 create_attr.btf_key_type_id = 0; 2189 create_attr.btf_value_type_id = 0; 2190 map->btf_key_type_id = 0; 2191 map->btf_value_type_id = 0; 2192 *pfd = bpf_create_map_xattr(&create_attr); 2193 } 2194 2195 if (*pfd < 0) { 2196 size_t j; 2197 2198 err = -errno; 2199 err_out: 2200 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 2201 pr_warning("failed to create map (name: '%s'): %s(%d)\n", 2202 map->name, cp, err); 2203 for (j = 0; j < i; j++) 2204 zclose(obj->maps[j].fd); 2205 return err; 2206 } 2207 2208 if (bpf_map__is_internal(map)) { 2209 err = bpf_object__populate_internal_map(obj, map); 2210 if (err < 0) { 2211 zclose(*pfd); 2212 goto err_out; 2213 } 2214 } 2215 2216 pr_debug("created map %s: fd=%d\n", map->name, *pfd); 2217 } 2218 2219 return 0; 2220 } 2221 2222 static int 2223 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 2224 void *btf_prog_info, const char *info_name) 2225 { 2226 if (err != -ENOENT) { 2227 pr_warning("Error in loading %s for sec %s.\n", 2228 info_name, prog->section_name); 2229 return err; 2230 } 2231 2232 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 2233 2234 if (btf_prog_info) { 2235 /* 2236 * Some info has already been found but has problem 2237 * in the last btf_ext reloc. Must have to error out. 2238 */ 2239 pr_warning("Error in relocating %s for sec %s.\n", 2240 info_name, prog->section_name); 2241 return err; 2242 } 2243 2244 /* Have problem loading the very first info. Ignore the rest. */ 2245 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 2246 info_name, prog->section_name, info_name); 2247 return 0; 2248 } 2249 2250 static int 2251 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 2252 const char *section_name, __u32 insn_offset) 2253 { 2254 int err; 2255 2256 if (!insn_offset || prog->func_info) { 2257 /* 2258 * !insn_offset => main program 2259 * 2260 * For sub prog, the main program's func_info has to 2261 * be loaded first (i.e. prog->func_info != NULL) 2262 */ 2263 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 2264 section_name, insn_offset, 2265 &prog->func_info, 2266 &prog->func_info_cnt); 2267 if (err) 2268 return check_btf_ext_reloc_err(prog, err, 2269 prog->func_info, 2270 "bpf_func_info"); 2271 2272 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 2273 } 2274 2275 if (!insn_offset || prog->line_info) { 2276 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 2277 section_name, insn_offset, 2278 &prog->line_info, 2279 &prog->line_info_cnt); 2280 if (err) 2281 return check_btf_ext_reloc_err(prog, err, 2282 prog->line_info, 2283 "bpf_line_info"); 2284 2285 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 2286 } 2287 2288 return 0; 2289 } 2290 2291 static int 2292 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 2293 struct reloc_desc *relo) 2294 { 2295 struct bpf_insn *insn, *new_insn; 2296 struct bpf_program *text; 2297 size_t new_cnt; 2298 int err; 2299 2300 if (relo->type != RELO_CALL) 2301 return -LIBBPF_ERRNO__RELOC; 2302 2303 if (prog->idx == obj->efile.text_shndx) { 2304 pr_warning("relo in .text insn %d into off %d\n", 2305 relo->insn_idx, relo->text_off); 2306 return -LIBBPF_ERRNO__RELOC; 2307 } 2308 2309 if (prog->main_prog_cnt == 0) { 2310 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 2311 if (!text) { 2312 pr_warning("no .text section found yet relo into text exist\n"); 2313 return -LIBBPF_ERRNO__RELOC; 2314 } 2315 new_cnt = prog->insns_cnt + text->insns_cnt; 2316 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 2317 if (!new_insn) { 2318 pr_warning("oom in prog realloc\n"); 2319 return -ENOMEM; 2320 } 2321 2322 if (obj->btf_ext) { 2323 err = bpf_program_reloc_btf_ext(prog, obj, 2324 text->section_name, 2325 prog->insns_cnt); 2326 if (err) 2327 return err; 2328 } 2329 2330 memcpy(new_insn + prog->insns_cnt, text->insns, 2331 text->insns_cnt * sizeof(*insn)); 2332 prog->insns = new_insn; 2333 prog->main_prog_cnt = prog->insns_cnt; 2334 prog->insns_cnt = new_cnt; 2335 pr_debug("added %zd insn from %s to prog %s\n", 2336 text->insns_cnt, text->section_name, 2337 prog->section_name); 2338 } 2339 insn = &prog->insns[relo->insn_idx]; 2340 insn->imm += prog->main_prog_cnt - relo->insn_idx; 2341 return 0; 2342 } 2343 2344 static int 2345 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 2346 { 2347 int i, err; 2348 2349 if (!prog) 2350 return 0; 2351 2352 if (obj->btf_ext) { 2353 err = bpf_program_reloc_btf_ext(prog, obj, 2354 prog->section_name, 0); 2355 if (err) 2356 return err; 2357 } 2358 2359 if (!prog->reloc_desc) 2360 return 0; 2361 2362 for (i = 0; i < prog->nr_reloc; i++) { 2363 if (prog->reloc_desc[i].type == RELO_LD64 || 2364 prog->reloc_desc[i].type == RELO_DATA) { 2365 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 2366 struct bpf_insn *insns = prog->insns; 2367 int insn_idx, map_idx; 2368 2369 insn_idx = prog->reloc_desc[i].insn_idx; 2370 map_idx = prog->reloc_desc[i].map_idx; 2371 2372 if (insn_idx + 1 >= (int)prog->insns_cnt) { 2373 pr_warning("relocation out of range: '%s'\n", 2374 prog->section_name); 2375 return -LIBBPF_ERRNO__RELOC; 2376 } 2377 2378 if (!relo_data) { 2379 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 2380 } else { 2381 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 2382 insns[insn_idx + 1].imm = insns[insn_idx].imm; 2383 } 2384 insns[insn_idx].imm = obj->maps[map_idx].fd; 2385 } else if (prog->reloc_desc[i].type == RELO_CALL) { 2386 err = bpf_program__reloc_text(prog, obj, 2387 &prog->reloc_desc[i]); 2388 if (err) 2389 return err; 2390 } 2391 } 2392 2393 zfree(&prog->reloc_desc); 2394 prog->nr_reloc = 0; 2395 return 0; 2396 } 2397 2398 2399 static int 2400 bpf_object__relocate(struct bpf_object *obj) 2401 { 2402 struct bpf_program *prog; 2403 size_t i; 2404 int err; 2405 2406 for (i = 0; i < obj->nr_programs; i++) { 2407 prog = &obj->programs[i]; 2408 2409 err = bpf_program__relocate(prog, obj); 2410 if (err) { 2411 pr_warning("failed to relocate '%s'\n", 2412 prog->section_name); 2413 return err; 2414 } 2415 } 2416 return 0; 2417 } 2418 2419 static int bpf_object__collect_reloc(struct bpf_object *obj) 2420 { 2421 int i, err; 2422 2423 if (!obj_elf_valid(obj)) { 2424 pr_warning("Internal error: elf object is closed\n"); 2425 return -LIBBPF_ERRNO__INTERNAL; 2426 } 2427 2428 for (i = 0; i < obj->efile.nr_reloc; i++) { 2429 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2430 Elf_Data *data = obj->efile.reloc[i].data; 2431 int idx = shdr->sh_info; 2432 struct bpf_program *prog; 2433 2434 if (shdr->sh_type != SHT_REL) { 2435 pr_warning("internal error at %d\n", __LINE__); 2436 return -LIBBPF_ERRNO__INTERNAL; 2437 } 2438 2439 prog = bpf_object__find_prog_by_idx(obj, idx); 2440 if (!prog) { 2441 pr_warning("relocation failed: no section(%d)\n", idx); 2442 return -LIBBPF_ERRNO__RELOC; 2443 } 2444 2445 err = bpf_program__collect_reloc(prog, shdr, data, obj); 2446 if (err) 2447 return err; 2448 } 2449 return 0; 2450 } 2451 2452 static int 2453 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2454 char *license, __u32 kern_version, int *pfd) 2455 { 2456 struct bpf_load_program_attr load_attr; 2457 char *cp, errmsg[STRERR_BUFSIZE]; 2458 int log_buf_size = BPF_LOG_BUF_SIZE; 2459 char *log_buf; 2460 int btf_fd, ret; 2461 2462 if (!insns || !insns_cnt) 2463 return -EINVAL; 2464 2465 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2466 load_attr.prog_type = prog->type; 2467 load_attr.expected_attach_type = prog->expected_attach_type; 2468 if (prog->caps->name) 2469 load_attr.name = prog->name; 2470 load_attr.insns = insns; 2471 load_attr.insns_cnt = insns_cnt; 2472 load_attr.license = license; 2473 load_attr.kern_version = kern_version; 2474 load_attr.prog_ifindex = prog->prog_ifindex; 2475 /* if .BTF.ext was loaded, kernel supports associated BTF for prog */ 2476 if (prog->obj->btf_ext) 2477 btf_fd = bpf_object__btf_fd(prog->obj); 2478 else 2479 btf_fd = -1; 2480 load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0; 2481 load_attr.func_info = prog->func_info; 2482 load_attr.func_info_rec_size = prog->func_info_rec_size; 2483 load_attr.func_info_cnt = prog->func_info_cnt; 2484 load_attr.line_info = prog->line_info; 2485 load_attr.line_info_rec_size = prog->line_info_rec_size; 2486 load_attr.line_info_cnt = prog->line_info_cnt; 2487 load_attr.log_level = prog->log_level; 2488 load_attr.prog_flags = prog->prog_flags; 2489 2490 retry_load: 2491 log_buf = malloc(log_buf_size); 2492 if (!log_buf) 2493 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2494 2495 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2496 2497 if (ret >= 0) { 2498 if (load_attr.log_level) 2499 pr_debug("verifier log:\n%s", log_buf); 2500 *pfd = ret; 2501 ret = 0; 2502 goto out; 2503 } 2504 2505 if (errno == ENOSPC) { 2506 log_buf_size <<= 1; 2507 free(log_buf); 2508 goto retry_load; 2509 } 2510 ret = -LIBBPF_ERRNO__LOAD; 2511 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2512 pr_warning("load bpf program failed: %s\n", cp); 2513 2514 if (log_buf && log_buf[0] != '\0') { 2515 ret = -LIBBPF_ERRNO__VERIFY; 2516 pr_warning("-- BEGIN DUMP LOG ---\n"); 2517 pr_warning("\n%s\n", log_buf); 2518 pr_warning("-- END LOG --\n"); 2519 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2520 pr_warning("Program too large (%zu insns), at most %d insns\n", 2521 load_attr.insns_cnt, BPF_MAXINSNS); 2522 ret = -LIBBPF_ERRNO__PROG2BIG; 2523 } else { 2524 /* Wrong program type? */ 2525 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2526 int fd; 2527 2528 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2529 load_attr.expected_attach_type = 0; 2530 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2531 if (fd >= 0) { 2532 close(fd); 2533 ret = -LIBBPF_ERRNO__PROGTYPE; 2534 goto out; 2535 } 2536 } 2537 2538 if (log_buf) 2539 ret = -LIBBPF_ERRNO__KVER; 2540 } 2541 2542 out: 2543 free(log_buf); 2544 return ret; 2545 } 2546 2547 int 2548 bpf_program__load(struct bpf_program *prog, 2549 char *license, __u32 kern_version) 2550 { 2551 int err = 0, fd, i; 2552 2553 if (prog->instances.nr < 0 || !prog->instances.fds) { 2554 if (prog->preprocessor) { 2555 pr_warning("Internal error: can't load program '%s'\n", 2556 prog->section_name); 2557 return -LIBBPF_ERRNO__INTERNAL; 2558 } 2559 2560 prog->instances.fds = malloc(sizeof(int)); 2561 if (!prog->instances.fds) { 2562 pr_warning("Not enough memory for BPF fds\n"); 2563 return -ENOMEM; 2564 } 2565 prog->instances.nr = 1; 2566 prog->instances.fds[0] = -1; 2567 } 2568 2569 if (!prog->preprocessor) { 2570 if (prog->instances.nr != 1) { 2571 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2572 prog->section_name, prog->instances.nr); 2573 } 2574 err = load_program(prog, prog->insns, prog->insns_cnt, 2575 license, kern_version, &fd); 2576 if (!err) 2577 prog->instances.fds[0] = fd; 2578 goto out; 2579 } 2580 2581 for (i = 0; i < prog->instances.nr; i++) { 2582 struct bpf_prog_prep_result result; 2583 bpf_program_prep_t preprocessor = prog->preprocessor; 2584 2585 memset(&result, 0, sizeof(result)); 2586 err = preprocessor(prog, i, prog->insns, 2587 prog->insns_cnt, &result); 2588 if (err) { 2589 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2590 i, prog->section_name); 2591 goto out; 2592 } 2593 2594 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2595 pr_debug("Skip loading the %dth instance of program '%s'\n", 2596 i, prog->section_name); 2597 prog->instances.fds[i] = -1; 2598 if (result.pfd) 2599 *result.pfd = -1; 2600 continue; 2601 } 2602 2603 err = load_program(prog, result.new_insn_ptr, 2604 result.new_insn_cnt, 2605 license, kern_version, &fd); 2606 2607 if (err) { 2608 pr_warning("Loading the %dth instance of program '%s' failed\n", 2609 i, prog->section_name); 2610 goto out; 2611 } 2612 2613 if (result.pfd) 2614 *result.pfd = fd; 2615 prog->instances.fds[i] = fd; 2616 } 2617 out: 2618 if (err) 2619 pr_warning("failed to load program '%s'\n", 2620 prog->section_name); 2621 zfree(&prog->insns); 2622 prog->insns_cnt = 0; 2623 return err; 2624 } 2625 2626 static bool bpf_program__is_function_storage(const struct bpf_program *prog, 2627 const struct bpf_object *obj) 2628 { 2629 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2630 } 2631 2632 static int 2633 bpf_object__load_progs(struct bpf_object *obj, int log_level) 2634 { 2635 size_t i; 2636 int err; 2637 2638 for (i = 0; i < obj->nr_programs; i++) { 2639 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2640 continue; 2641 obj->programs[i].log_level |= log_level; 2642 err = bpf_program__load(&obj->programs[i], 2643 obj->license, 2644 obj->kern_version); 2645 if (err) 2646 return err; 2647 } 2648 return 0; 2649 } 2650 2651 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2652 { 2653 switch (type) { 2654 case BPF_PROG_TYPE_SOCKET_FILTER: 2655 case BPF_PROG_TYPE_SCHED_CLS: 2656 case BPF_PROG_TYPE_SCHED_ACT: 2657 case BPF_PROG_TYPE_XDP: 2658 case BPF_PROG_TYPE_CGROUP_SKB: 2659 case BPF_PROG_TYPE_CGROUP_SOCK: 2660 case BPF_PROG_TYPE_LWT_IN: 2661 case BPF_PROG_TYPE_LWT_OUT: 2662 case BPF_PROG_TYPE_LWT_XMIT: 2663 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2664 case BPF_PROG_TYPE_SOCK_OPS: 2665 case BPF_PROG_TYPE_SK_SKB: 2666 case BPF_PROG_TYPE_CGROUP_DEVICE: 2667 case BPF_PROG_TYPE_SK_MSG: 2668 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2669 case BPF_PROG_TYPE_LIRC_MODE2: 2670 case BPF_PROG_TYPE_SK_REUSEPORT: 2671 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2672 case BPF_PROG_TYPE_UNSPEC: 2673 case BPF_PROG_TYPE_TRACEPOINT: 2674 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2675 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2676 case BPF_PROG_TYPE_PERF_EVENT: 2677 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2678 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2679 return false; 2680 case BPF_PROG_TYPE_KPROBE: 2681 default: 2682 return true; 2683 } 2684 } 2685 2686 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2687 { 2688 if (needs_kver && obj->kern_version == 0) { 2689 pr_warning("%s doesn't provide kernel version\n", 2690 obj->path); 2691 return -LIBBPF_ERRNO__KVERSION; 2692 } 2693 return 0; 2694 } 2695 2696 static struct bpf_object * 2697 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2698 bool needs_kver, int flags) 2699 { 2700 struct bpf_object *obj; 2701 int err; 2702 2703 if (elf_version(EV_CURRENT) == EV_NONE) { 2704 pr_warning("failed to init libelf for %s\n", path); 2705 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2706 } 2707 2708 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2709 if (IS_ERR(obj)) 2710 return obj; 2711 2712 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2713 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2714 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2715 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2716 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2717 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2718 2719 bpf_object__elf_finish(obj); 2720 return obj; 2721 out: 2722 bpf_object__close(obj); 2723 return ERR_PTR(err); 2724 } 2725 2726 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2727 int flags) 2728 { 2729 /* param validation */ 2730 if (!attr->file) 2731 return NULL; 2732 2733 pr_debug("loading %s\n", attr->file); 2734 2735 return __bpf_object__open(attr->file, NULL, 0, 2736 bpf_prog_type__needs_kver(attr->prog_type), 2737 flags); 2738 } 2739 2740 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2741 { 2742 return __bpf_object__open_xattr(attr, 0); 2743 } 2744 2745 struct bpf_object *bpf_object__open(const char *path) 2746 { 2747 struct bpf_object_open_attr attr = { 2748 .file = path, 2749 .prog_type = BPF_PROG_TYPE_UNSPEC, 2750 }; 2751 2752 return bpf_object__open_xattr(&attr); 2753 } 2754 2755 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2756 size_t obj_buf_sz, 2757 const char *name) 2758 { 2759 char tmp_name[64]; 2760 2761 /* param validation */ 2762 if (!obj_buf || obj_buf_sz <= 0) 2763 return NULL; 2764 2765 if (!name) { 2766 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2767 (unsigned long)obj_buf, 2768 (unsigned long)obj_buf_sz); 2769 name = tmp_name; 2770 } 2771 pr_debug("loading object '%s' from buffer\n", name); 2772 2773 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2774 } 2775 2776 int bpf_object__unload(struct bpf_object *obj) 2777 { 2778 size_t i; 2779 2780 if (!obj) 2781 return -EINVAL; 2782 2783 for (i = 0; i < obj->nr_maps; i++) 2784 zclose(obj->maps[i].fd); 2785 2786 for (i = 0; i < obj->nr_programs; i++) 2787 bpf_program__unload(&obj->programs[i]); 2788 2789 return 0; 2790 } 2791 2792 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 2793 { 2794 struct bpf_object *obj; 2795 int err; 2796 2797 if (!attr) 2798 return -EINVAL; 2799 obj = attr->obj; 2800 if (!obj) 2801 return -EINVAL; 2802 2803 if (obj->loaded) { 2804 pr_warning("object should not be loaded twice\n"); 2805 return -EINVAL; 2806 } 2807 2808 obj->loaded = true; 2809 2810 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2811 CHECK_ERR(bpf_object__relocate(obj), err, out); 2812 CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); 2813 2814 return 0; 2815 out: 2816 bpf_object__unload(obj); 2817 pr_warning("failed to load object '%s'\n", obj->path); 2818 return err; 2819 } 2820 2821 int bpf_object__load(struct bpf_object *obj) 2822 { 2823 struct bpf_object_load_attr attr = { 2824 .obj = obj, 2825 }; 2826 2827 return bpf_object__load_xattr(&attr); 2828 } 2829 2830 static int check_path(const char *path) 2831 { 2832 char *cp, errmsg[STRERR_BUFSIZE]; 2833 struct statfs st_fs; 2834 char *dname, *dir; 2835 int err = 0; 2836 2837 if (path == NULL) 2838 return -EINVAL; 2839 2840 dname = strdup(path); 2841 if (dname == NULL) 2842 return -ENOMEM; 2843 2844 dir = dirname(dname); 2845 if (statfs(dir, &st_fs)) { 2846 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2847 pr_warning("failed to statfs %s: %s\n", dir, cp); 2848 err = -errno; 2849 } 2850 free(dname); 2851 2852 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2853 pr_warning("specified path %s is not on BPF FS\n", path); 2854 err = -EINVAL; 2855 } 2856 2857 return err; 2858 } 2859 2860 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2861 int instance) 2862 { 2863 char *cp, errmsg[STRERR_BUFSIZE]; 2864 int err; 2865 2866 err = check_path(path); 2867 if (err) 2868 return err; 2869 2870 if (prog == NULL) { 2871 pr_warning("invalid program pointer\n"); 2872 return -EINVAL; 2873 } 2874 2875 if (instance < 0 || instance >= prog->instances.nr) { 2876 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2877 instance, prog->section_name, prog->instances.nr); 2878 return -EINVAL; 2879 } 2880 2881 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2882 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2883 pr_warning("failed to pin program: %s\n", cp); 2884 return -errno; 2885 } 2886 pr_debug("pinned program '%s'\n", path); 2887 2888 return 0; 2889 } 2890 2891 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2892 int instance) 2893 { 2894 int err; 2895 2896 err = check_path(path); 2897 if (err) 2898 return err; 2899 2900 if (prog == NULL) { 2901 pr_warning("invalid program pointer\n"); 2902 return -EINVAL; 2903 } 2904 2905 if (instance < 0 || instance >= prog->instances.nr) { 2906 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2907 instance, prog->section_name, prog->instances.nr); 2908 return -EINVAL; 2909 } 2910 2911 err = unlink(path); 2912 if (err != 0) 2913 return -errno; 2914 pr_debug("unpinned program '%s'\n", path); 2915 2916 return 0; 2917 } 2918 2919 static int make_dir(const char *path) 2920 { 2921 char *cp, errmsg[STRERR_BUFSIZE]; 2922 int err = 0; 2923 2924 if (mkdir(path, 0700) && errno != EEXIST) 2925 err = -errno; 2926 2927 if (err) { 2928 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2929 pr_warning("failed to mkdir %s: %s\n", path, cp); 2930 } 2931 return err; 2932 } 2933 2934 int bpf_program__pin(struct bpf_program *prog, const char *path) 2935 { 2936 int i, err; 2937 2938 err = check_path(path); 2939 if (err) 2940 return err; 2941 2942 if (prog == NULL) { 2943 pr_warning("invalid program pointer\n"); 2944 return -EINVAL; 2945 } 2946 2947 if (prog->instances.nr <= 0) { 2948 pr_warning("no instances of prog %s to pin\n", 2949 prog->section_name); 2950 return -EINVAL; 2951 } 2952 2953 if (prog->instances.nr == 1) { 2954 /* don't create subdirs when pinning single instance */ 2955 return bpf_program__pin_instance(prog, path, 0); 2956 } 2957 2958 err = make_dir(path); 2959 if (err) 2960 return err; 2961 2962 for (i = 0; i < prog->instances.nr; i++) { 2963 char buf[PATH_MAX]; 2964 int len; 2965 2966 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2967 if (len < 0) { 2968 err = -EINVAL; 2969 goto err_unpin; 2970 } else if (len >= PATH_MAX) { 2971 err = -ENAMETOOLONG; 2972 goto err_unpin; 2973 } 2974 2975 err = bpf_program__pin_instance(prog, buf, i); 2976 if (err) 2977 goto err_unpin; 2978 } 2979 2980 return 0; 2981 2982 err_unpin: 2983 for (i = i - 1; i >= 0; i--) { 2984 char buf[PATH_MAX]; 2985 int len; 2986 2987 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2988 if (len < 0) 2989 continue; 2990 else if (len >= PATH_MAX) 2991 continue; 2992 2993 bpf_program__unpin_instance(prog, buf, i); 2994 } 2995 2996 rmdir(path); 2997 2998 return err; 2999 } 3000 3001 int bpf_program__unpin(struct bpf_program *prog, const char *path) 3002 { 3003 int i, err; 3004 3005 err = check_path(path); 3006 if (err) 3007 return err; 3008 3009 if (prog == NULL) { 3010 pr_warning("invalid program pointer\n"); 3011 return -EINVAL; 3012 } 3013 3014 if (prog->instances.nr <= 0) { 3015 pr_warning("no instances of prog %s to pin\n", 3016 prog->section_name); 3017 return -EINVAL; 3018 } 3019 3020 if (prog->instances.nr == 1) { 3021 /* don't create subdirs when pinning single instance */ 3022 return bpf_program__unpin_instance(prog, path, 0); 3023 } 3024 3025 for (i = 0; i < prog->instances.nr; i++) { 3026 char buf[PATH_MAX]; 3027 int len; 3028 3029 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 3030 if (len < 0) 3031 return -EINVAL; 3032 else if (len >= PATH_MAX) 3033 return -ENAMETOOLONG; 3034 3035 err = bpf_program__unpin_instance(prog, buf, i); 3036 if (err) 3037 return err; 3038 } 3039 3040 err = rmdir(path); 3041 if (err) 3042 return -errno; 3043 3044 return 0; 3045 } 3046 3047 int bpf_map__pin(struct bpf_map *map, const char *path) 3048 { 3049 char *cp, errmsg[STRERR_BUFSIZE]; 3050 int err; 3051 3052 err = check_path(path); 3053 if (err) 3054 return err; 3055 3056 if (map == NULL) { 3057 pr_warning("invalid map pointer\n"); 3058 return -EINVAL; 3059 } 3060 3061 if (bpf_obj_pin(map->fd, path)) { 3062 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 3063 pr_warning("failed to pin map: %s\n", cp); 3064 return -errno; 3065 } 3066 3067 pr_debug("pinned map '%s'\n", path); 3068 3069 return 0; 3070 } 3071 3072 int bpf_map__unpin(struct bpf_map *map, const char *path) 3073 { 3074 int err; 3075 3076 err = check_path(path); 3077 if (err) 3078 return err; 3079 3080 if (map == NULL) { 3081 pr_warning("invalid map pointer\n"); 3082 return -EINVAL; 3083 } 3084 3085 err = unlink(path); 3086 if (err != 0) 3087 return -errno; 3088 pr_debug("unpinned map '%s'\n", path); 3089 3090 return 0; 3091 } 3092 3093 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 3094 { 3095 struct bpf_map *map; 3096 int err; 3097 3098 if (!obj) 3099 return -ENOENT; 3100 3101 if (!obj->loaded) { 3102 pr_warning("object not yet loaded; load it first\n"); 3103 return -ENOENT; 3104 } 3105 3106 err = make_dir(path); 3107 if (err) 3108 return err; 3109 3110 bpf_object__for_each_map(map, obj) { 3111 char buf[PATH_MAX]; 3112 int len; 3113 3114 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3115 bpf_map__name(map)); 3116 if (len < 0) { 3117 err = -EINVAL; 3118 goto err_unpin_maps; 3119 } else if (len >= PATH_MAX) { 3120 err = -ENAMETOOLONG; 3121 goto err_unpin_maps; 3122 } 3123 3124 err = bpf_map__pin(map, buf); 3125 if (err) 3126 goto err_unpin_maps; 3127 } 3128 3129 return 0; 3130 3131 err_unpin_maps: 3132 while ((map = bpf_map__prev(map, obj))) { 3133 char buf[PATH_MAX]; 3134 int len; 3135 3136 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3137 bpf_map__name(map)); 3138 if (len < 0) 3139 continue; 3140 else if (len >= PATH_MAX) 3141 continue; 3142 3143 bpf_map__unpin(map, buf); 3144 } 3145 3146 return err; 3147 } 3148 3149 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 3150 { 3151 struct bpf_map *map; 3152 int err; 3153 3154 if (!obj) 3155 return -ENOENT; 3156 3157 bpf_object__for_each_map(map, obj) { 3158 char buf[PATH_MAX]; 3159 int len; 3160 3161 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3162 bpf_map__name(map)); 3163 if (len < 0) 3164 return -EINVAL; 3165 else if (len >= PATH_MAX) 3166 return -ENAMETOOLONG; 3167 3168 err = bpf_map__unpin(map, buf); 3169 if (err) 3170 return err; 3171 } 3172 3173 return 0; 3174 } 3175 3176 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 3177 { 3178 struct bpf_program *prog; 3179 int err; 3180 3181 if (!obj) 3182 return -ENOENT; 3183 3184 if (!obj->loaded) { 3185 pr_warning("object not yet loaded; load it first\n"); 3186 return -ENOENT; 3187 } 3188 3189 err = make_dir(path); 3190 if (err) 3191 return err; 3192 3193 bpf_object__for_each_program(prog, obj) { 3194 char buf[PATH_MAX]; 3195 int len; 3196 3197 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3198 prog->pin_name); 3199 if (len < 0) { 3200 err = -EINVAL; 3201 goto err_unpin_programs; 3202 } else if (len >= PATH_MAX) { 3203 err = -ENAMETOOLONG; 3204 goto err_unpin_programs; 3205 } 3206 3207 err = bpf_program__pin(prog, buf); 3208 if (err) 3209 goto err_unpin_programs; 3210 } 3211 3212 return 0; 3213 3214 err_unpin_programs: 3215 while ((prog = bpf_program__prev(prog, obj))) { 3216 char buf[PATH_MAX]; 3217 int len; 3218 3219 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3220 prog->pin_name); 3221 if (len < 0) 3222 continue; 3223 else if (len >= PATH_MAX) 3224 continue; 3225 3226 bpf_program__unpin(prog, buf); 3227 } 3228 3229 return err; 3230 } 3231 3232 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 3233 { 3234 struct bpf_program *prog; 3235 int err; 3236 3237 if (!obj) 3238 return -ENOENT; 3239 3240 bpf_object__for_each_program(prog, obj) { 3241 char buf[PATH_MAX]; 3242 int len; 3243 3244 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3245 prog->pin_name); 3246 if (len < 0) 3247 return -EINVAL; 3248 else if (len >= PATH_MAX) 3249 return -ENAMETOOLONG; 3250 3251 err = bpf_program__unpin(prog, buf); 3252 if (err) 3253 return err; 3254 } 3255 3256 return 0; 3257 } 3258 3259 int bpf_object__pin(struct bpf_object *obj, const char *path) 3260 { 3261 int err; 3262 3263 err = bpf_object__pin_maps(obj, path); 3264 if (err) 3265 return err; 3266 3267 err = bpf_object__pin_programs(obj, path); 3268 if (err) { 3269 bpf_object__unpin_maps(obj, path); 3270 return err; 3271 } 3272 3273 return 0; 3274 } 3275 3276 void bpf_object__close(struct bpf_object *obj) 3277 { 3278 size_t i; 3279 3280 if (!obj) 3281 return; 3282 3283 if (obj->clear_priv) 3284 obj->clear_priv(obj, obj->priv); 3285 3286 bpf_object__elf_finish(obj); 3287 bpf_object__unload(obj); 3288 btf__free(obj->btf); 3289 btf_ext__free(obj->btf_ext); 3290 3291 for (i = 0; i < obj->nr_maps; i++) { 3292 zfree(&obj->maps[i].name); 3293 if (obj->maps[i].clear_priv) 3294 obj->maps[i].clear_priv(&obj->maps[i], 3295 obj->maps[i].priv); 3296 obj->maps[i].priv = NULL; 3297 obj->maps[i].clear_priv = NULL; 3298 } 3299 3300 zfree(&obj->sections.rodata); 3301 zfree(&obj->sections.data); 3302 zfree(&obj->maps); 3303 obj->nr_maps = 0; 3304 3305 if (obj->programs && obj->nr_programs) { 3306 for (i = 0; i < obj->nr_programs; i++) 3307 bpf_program__exit(&obj->programs[i]); 3308 } 3309 zfree(&obj->programs); 3310 3311 list_del(&obj->list); 3312 free(obj); 3313 } 3314 3315 struct bpf_object * 3316 bpf_object__next(struct bpf_object *prev) 3317 { 3318 struct bpf_object *next; 3319 3320 if (!prev) 3321 next = list_first_entry(&bpf_objects_list, 3322 struct bpf_object, 3323 list); 3324 else 3325 next = list_next_entry(prev, list); 3326 3327 /* Empty list is noticed here so don't need checking on entry. */ 3328 if (&next->list == &bpf_objects_list) 3329 return NULL; 3330 3331 return next; 3332 } 3333 3334 const char *bpf_object__name(const struct bpf_object *obj) 3335 { 3336 return obj ? obj->path : ERR_PTR(-EINVAL); 3337 } 3338 3339 unsigned int bpf_object__kversion(const struct bpf_object *obj) 3340 { 3341 return obj ? obj->kern_version : 0; 3342 } 3343 3344 struct btf *bpf_object__btf(const struct bpf_object *obj) 3345 { 3346 return obj ? obj->btf : NULL; 3347 } 3348 3349 int bpf_object__btf_fd(const struct bpf_object *obj) 3350 { 3351 return obj->btf ? btf__fd(obj->btf) : -1; 3352 } 3353 3354 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 3355 bpf_object_clear_priv_t clear_priv) 3356 { 3357 if (obj->priv && obj->clear_priv) 3358 obj->clear_priv(obj, obj->priv); 3359 3360 obj->priv = priv; 3361 obj->clear_priv = clear_priv; 3362 return 0; 3363 } 3364 3365 void *bpf_object__priv(const struct bpf_object *obj) 3366 { 3367 return obj ? obj->priv : ERR_PTR(-EINVAL); 3368 } 3369 3370 static struct bpf_program * 3371 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 3372 bool forward) 3373 { 3374 size_t nr_programs = obj->nr_programs; 3375 ssize_t idx; 3376 3377 if (!nr_programs) 3378 return NULL; 3379 3380 if (!p) 3381 /* Iter from the beginning */ 3382 return forward ? &obj->programs[0] : 3383 &obj->programs[nr_programs - 1]; 3384 3385 if (p->obj != obj) { 3386 pr_warning("error: program handler doesn't match object\n"); 3387 return NULL; 3388 } 3389 3390 idx = (p - obj->programs) + (forward ? 1 : -1); 3391 if (idx >= obj->nr_programs || idx < 0) 3392 return NULL; 3393 return &obj->programs[idx]; 3394 } 3395 3396 struct bpf_program * 3397 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 3398 { 3399 struct bpf_program *prog = prev; 3400 3401 do { 3402 prog = __bpf_program__iter(prog, obj, true); 3403 } while (prog && bpf_program__is_function_storage(prog, obj)); 3404 3405 return prog; 3406 } 3407 3408 struct bpf_program * 3409 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 3410 { 3411 struct bpf_program *prog = next; 3412 3413 do { 3414 prog = __bpf_program__iter(prog, obj, false); 3415 } while (prog && bpf_program__is_function_storage(prog, obj)); 3416 3417 return prog; 3418 } 3419 3420 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 3421 bpf_program_clear_priv_t clear_priv) 3422 { 3423 if (prog->priv && prog->clear_priv) 3424 prog->clear_priv(prog, prog->priv); 3425 3426 prog->priv = priv; 3427 prog->clear_priv = clear_priv; 3428 return 0; 3429 } 3430 3431 void *bpf_program__priv(const struct bpf_program *prog) 3432 { 3433 return prog ? prog->priv : ERR_PTR(-EINVAL); 3434 } 3435 3436 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3437 { 3438 prog->prog_ifindex = ifindex; 3439 } 3440 3441 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 3442 { 3443 const char *title; 3444 3445 title = prog->section_name; 3446 if (needs_copy) { 3447 title = strdup(title); 3448 if (!title) { 3449 pr_warning("failed to strdup program title\n"); 3450 return ERR_PTR(-ENOMEM); 3451 } 3452 } 3453 3454 return title; 3455 } 3456 3457 int bpf_program__fd(const struct bpf_program *prog) 3458 { 3459 return bpf_program__nth_fd(prog, 0); 3460 } 3461 3462 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3463 bpf_program_prep_t prep) 3464 { 3465 int *instances_fds; 3466 3467 if (nr_instances <= 0 || !prep) 3468 return -EINVAL; 3469 3470 if (prog->instances.nr > 0 || prog->instances.fds) { 3471 pr_warning("Can't set pre-processor after loading\n"); 3472 return -EINVAL; 3473 } 3474 3475 instances_fds = malloc(sizeof(int) * nr_instances); 3476 if (!instances_fds) { 3477 pr_warning("alloc memory failed for fds\n"); 3478 return -ENOMEM; 3479 } 3480 3481 /* fill all fd with -1 */ 3482 memset(instances_fds, -1, sizeof(int) * nr_instances); 3483 3484 prog->instances.nr = nr_instances; 3485 prog->instances.fds = instances_fds; 3486 prog->preprocessor = prep; 3487 return 0; 3488 } 3489 3490 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 3491 { 3492 int fd; 3493 3494 if (!prog) 3495 return -EINVAL; 3496 3497 if (n >= prog->instances.nr || n < 0) { 3498 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3499 n, prog->section_name, prog->instances.nr); 3500 return -EINVAL; 3501 } 3502 3503 fd = prog->instances.fds[n]; 3504 if (fd < 0) { 3505 pr_warning("%dth instance of program '%s' is invalid\n", 3506 n, prog->section_name); 3507 return -ENOENT; 3508 } 3509 3510 return fd; 3511 } 3512 3513 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3514 { 3515 prog->type = type; 3516 } 3517 3518 static bool bpf_program__is_type(const struct bpf_program *prog, 3519 enum bpf_prog_type type) 3520 { 3521 return prog ? (prog->type == type) : false; 3522 } 3523 3524 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3525 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3526 { \ 3527 if (!prog) \ 3528 return -EINVAL; \ 3529 bpf_program__set_type(prog, TYPE); \ 3530 return 0; \ 3531 } \ 3532 \ 3533 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 3534 { \ 3535 return bpf_program__is_type(prog, TYPE); \ 3536 } \ 3537 3538 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3539 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3540 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3541 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3542 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3543 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3544 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3545 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3546 3547 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3548 enum bpf_attach_type type) 3549 { 3550 prog->expected_attach_type = type; 3551 } 3552 3553 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3554 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3555 3556 /* Programs that can NOT be attached. */ 3557 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3558 3559 /* Programs that can be attached. */ 3560 #define BPF_APROG_SEC(string, ptype, atype) \ 3561 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3562 3563 /* Programs that must specify expected attach type at load time. */ 3564 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3565 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3566 3567 /* Programs that can be attached but attach type can't be identified by section 3568 * name. Kept for backward compatibility. 3569 */ 3570 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3571 3572 static const struct { 3573 const char *sec; 3574 size_t len; 3575 enum bpf_prog_type prog_type; 3576 enum bpf_attach_type expected_attach_type; 3577 int is_attachable; 3578 enum bpf_attach_type attach_type; 3579 } section_names[] = { 3580 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3581 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3582 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3583 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3584 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3585 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3586 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3587 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3588 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3589 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3590 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3591 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3592 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3593 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3594 BPF_CGROUP_INET_INGRESS), 3595 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3596 BPF_CGROUP_INET_EGRESS), 3597 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3598 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3599 BPF_CGROUP_INET_SOCK_CREATE), 3600 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3601 BPF_CGROUP_INET4_POST_BIND), 3602 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3603 BPF_CGROUP_INET6_POST_BIND), 3604 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3605 BPF_CGROUP_DEVICE), 3606 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3607 BPF_CGROUP_SOCK_OPS), 3608 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3609 BPF_SK_SKB_STREAM_PARSER), 3610 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3611 BPF_SK_SKB_STREAM_VERDICT), 3612 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3613 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3614 BPF_SK_MSG_VERDICT), 3615 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3616 BPF_LIRC_MODE2), 3617 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3618 BPF_FLOW_DISSECTOR), 3619 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3620 BPF_CGROUP_INET4_BIND), 3621 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3622 BPF_CGROUP_INET6_BIND), 3623 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3624 BPF_CGROUP_INET4_CONNECT), 3625 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3626 BPF_CGROUP_INET6_CONNECT), 3627 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3628 BPF_CGROUP_UDP4_SENDMSG), 3629 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3630 BPF_CGROUP_UDP6_SENDMSG), 3631 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3632 BPF_CGROUP_UDP4_RECVMSG), 3633 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3634 BPF_CGROUP_UDP6_RECVMSG), 3635 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3636 BPF_CGROUP_SYSCTL), 3637 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 3638 BPF_CGROUP_GETSOCKOPT), 3639 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 3640 BPF_CGROUP_SETSOCKOPT), 3641 }; 3642 3643 #undef BPF_PROG_SEC_IMPL 3644 #undef BPF_PROG_SEC 3645 #undef BPF_APROG_SEC 3646 #undef BPF_EAPROG_SEC 3647 #undef BPF_APROG_COMPAT 3648 3649 #define MAX_TYPE_NAME_SIZE 32 3650 3651 static char *libbpf_get_type_names(bool attach_type) 3652 { 3653 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3654 char *buf; 3655 3656 buf = malloc(len); 3657 if (!buf) 3658 return NULL; 3659 3660 buf[0] = '\0'; 3661 /* Forge string buf with all available names */ 3662 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3663 if (attach_type && !section_names[i].is_attachable) 3664 continue; 3665 3666 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3667 free(buf); 3668 return NULL; 3669 } 3670 strcat(buf, " "); 3671 strcat(buf, section_names[i].sec); 3672 } 3673 3674 return buf; 3675 } 3676 3677 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3678 enum bpf_attach_type *expected_attach_type) 3679 { 3680 char *type_names; 3681 int i; 3682 3683 if (!name) 3684 return -EINVAL; 3685 3686 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3687 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3688 continue; 3689 *prog_type = section_names[i].prog_type; 3690 *expected_attach_type = section_names[i].expected_attach_type; 3691 return 0; 3692 } 3693 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3694 type_names = libbpf_get_type_names(false); 3695 if (type_names != NULL) { 3696 pr_info("supported section(type) names are:%s\n", type_names); 3697 free(type_names); 3698 } 3699 3700 return -EINVAL; 3701 } 3702 3703 int libbpf_attach_type_by_name(const char *name, 3704 enum bpf_attach_type *attach_type) 3705 { 3706 char *type_names; 3707 int i; 3708 3709 if (!name) 3710 return -EINVAL; 3711 3712 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3713 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3714 continue; 3715 if (!section_names[i].is_attachable) 3716 return -EINVAL; 3717 *attach_type = section_names[i].attach_type; 3718 return 0; 3719 } 3720 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3721 type_names = libbpf_get_type_names(true); 3722 if (type_names != NULL) { 3723 pr_info("attachable section(type) names are:%s\n", type_names); 3724 free(type_names); 3725 } 3726 3727 return -EINVAL; 3728 } 3729 3730 static int 3731 bpf_program__identify_section(struct bpf_program *prog, 3732 enum bpf_prog_type *prog_type, 3733 enum bpf_attach_type *expected_attach_type) 3734 { 3735 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3736 expected_attach_type); 3737 } 3738 3739 int bpf_map__fd(const struct bpf_map *map) 3740 { 3741 return map ? map->fd : -EINVAL; 3742 } 3743 3744 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 3745 { 3746 return map ? &map->def : ERR_PTR(-EINVAL); 3747 } 3748 3749 const char *bpf_map__name(const struct bpf_map *map) 3750 { 3751 return map ? map->name : NULL; 3752 } 3753 3754 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3755 { 3756 return map ? map->btf_key_type_id : 0; 3757 } 3758 3759 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3760 { 3761 return map ? map->btf_value_type_id : 0; 3762 } 3763 3764 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3765 bpf_map_clear_priv_t clear_priv) 3766 { 3767 if (!map) 3768 return -EINVAL; 3769 3770 if (map->priv) { 3771 if (map->clear_priv) 3772 map->clear_priv(map, map->priv); 3773 } 3774 3775 map->priv = priv; 3776 map->clear_priv = clear_priv; 3777 return 0; 3778 } 3779 3780 void *bpf_map__priv(const struct bpf_map *map) 3781 { 3782 return map ? map->priv : ERR_PTR(-EINVAL); 3783 } 3784 3785 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 3786 { 3787 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3788 } 3789 3790 bool bpf_map__is_internal(const struct bpf_map *map) 3791 { 3792 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3793 } 3794 3795 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3796 { 3797 map->map_ifindex = ifindex; 3798 } 3799 3800 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3801 { 3802 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3803 pr_warning("error: unsupported map type\n"); 3804 return -EINVAL; 3805 } 3806 if (map->inner_map_fd != -1) { 3807 pr_warning("error: inner_map_fd already specified\n"); 3808 return -EINVAL; 3809 } 3810 map->inner_map_fd = fd; 3811 return 0; 3812 } 3813 3814 static struct bpf_map * 3815 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 3816 { 3817 ssize_t idx; 3818 struct bpf_map *s, *e; 3819 3820 if (!obj || !obj->maps) 3821 return NULL; 3822 3823 s = obj->maps; 3824 e = obj->maps + obj->nr_maps; 3825 3826 if ((m < s) || (m >= e)) { 3827 pr_warning("error in %s: map handler doesn't belong to object\n", 3828 __func__); 3829 return NULL; 3830 } 3831 3832 idx = (m - obj->maps) + i; 3833 if (idx >= obj->nr_maps || idx < 0) 3834 return NULL; 3835 return &obj->maps[idx]; 3836 } 3837 3838 struct bpf_map * 3839 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 3840 { 3841 if (prev == NULL) 3842 return obj->maps; 3843 3844 return __bpf_map__iter(prev, obj, 1); 3845 } 3846 3847 struct bpf_map * 3848 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 3849 { 3850 if (next == NULL) { 3851 if (!obj->nr_maps) 3852 return NULL; 3853 return obj->maps + obj->nr_maps - 1; 3854 } 3855 3856 return __bpf_map__iter(next, obj, -1); 3857 } 3858 3859 struct bpf_map * 3860 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 3861 { 3862 struct bpf_map *pos; 3863 3864 bpf_object__for_each_map(pos, obj) { 3865 if (pos->name && !strcmp(pos->name, name)) 3866 return pos; 3867 } 3868 return NULL; 3869 } 3870 3871 int 3872 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 3873 { 3874 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3875 } 3876 3877 struct bpf_map * 3878 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3879 { 3880 return ERR_PTR(-ENOTSUP); 3881 } 3882 3883 long libbpf_get_error(const void *ptr) 3884 { 3885 return PTR_ERR_OR_ZERO(ptr); 3886 } 3887 3888 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3889 struct bpf_object **pobj, int *prog_fd) 3890 { 3891 struct bpf_prog_load_attr attr; 3892 3893 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3894 attr.file = file; 3895 attr.prog_type = type; 3896 attr.expected_attach_type = 0; 3897 3898 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3899 } 3900 3901 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3902 struct bpf_object **pobj, int *prog_fd) 3903 { 3904 struct bpf_object_open_attr open_attr = {}; 3905 struct bpf_program *prog, *first_prog = NULL; 3906 enum bpf_attach_type expected_attach_type; 3907 enum bpf_prog_type prog_type; 3908 struct bpf_object *obj; 3909 struct bpf_map *map; 3910 int err; 3911 3912 if (!attr) 3913 return -EINVAL; 3914 if (!attr->file) 3915 return -EINVAL; 3916 3917 open_attr.file = attr->file; 3918 open_attr.prog_type = attr->prog_type; 3919 3920 obj = bpf_object__open_xattr(&open_attr); 3921 if (IS_ERR_OR_NULL(obj)) 3922 return -ENOENT; 3923 3924 bpf_object__for_each_program(prog, obj) { 3925 /* 3926 * If type is not specified, try to guess it based on 3927 * section name. 3928 */ 3929 prog_type = attr->prog_type; 3930 prog->prog_ifindex = attr->ifindex; 3931 expected_attach_type = attr->expected_attach_type; 3932 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3933 err = bpf_program__identify_section(prog, &prog_type, 3934 &expected_attach_type); 3935 if (err < 0) { 3936 bpf_object__close(obj); 3937 return -EINVAL; 3938 } 3939 } 3940 3941 bpf_program__set_type(prog, prog_type); 3942 bpf_program__set_expected_attach_type(prog, 3943 expected_attach_type); 3944 3945 prog->log_level = attr->log_level; 3946 prog->prog_flags = attr->prog_flags; 3947 if (!first_prog) 3948 first_prog = prog; 3949 } 3950 3951 bpf_object__for_each_map(map, obj) { 3952 if (!bpf_map__is_offload_neutral(map)) 3953 map->map_ifindex = attr->ifindex; 3954 } 3955 3956 if (!first_prog) { 3957 pr_warning("object file doesn't contain bpf program\n"); 3958 bpf_object__close(obj); 3959 return -ENOENT; 3960 } 3961 3962 err = bpf_object__load(obj); 3963 if (err) { 3964 bpf_object__close(obj); 3965 return -EINVAL; 3966 } 3967 3968 *pobj = obj; 3969 *prog_fd = bpf_program__fd(first_prog); 3970 return 0; 3971 } 3972 3973 struct bpf_link { 3974 int (*destroy)(struct bpf_link *link); 3975 }; 3976 3977 int bpf_link__destroy(struct bpf_link *link) 3978 { 3979 int err; 3980 3981 if (!link) 3982 return 0; 3983 3984 err = link->destroy(link); 3985 free(link); 3986 3987 return err; 3988 } 3989 3990 struct bpf_link_fd { 3991 struct bpf_link link; /* has to be at the top of struct */ 3992 int fd; /* hook FD */ 3993 }; 3994 3995 static int bpf_link__destroy_perf_event(struct bpf_link *link) 3996 { 3997 struct bpf_link_fd *l = (void *)link; 3998 int err; 3999 4000 err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); 4001 if (err) 4002 err = -errno; 4003 4004 close(l->fd); 4005 return err; 4006 } 4007 4008 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, 4009 int pfd) 4010 { 4011 char errmsg[STRERR_BUFSIZE]; 4012 struct bpf_link_fd *link; 4013 int prog_fd, err; 4014 4015 if (pfd < 0) { 4016 pr_warning("program '%s': invalid perf event FD %d\n", 4017 bpf_program__title(prog, false), pfd); 4018 return ERR_PTR(-EINVAL); 4019 } 4020 prog_fd = bpf_program__fd(prog); 4021 if (prog_fd < 0) { 4022 pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", 4023 bpf_program__title(prog, false)); 4024 return ERR_PTR(-EINVAL); 4025 } 4026 4027 link = malloc(sizeof(*link)); 4028 if (!link) 4029 return ERR_PTR(-ENOMEM); 4030 link->link.destroy = &bpf_link__destroy_perf_event; 4031 link->fd = pfd; 4032 4033 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 4034 err = -errno; 4035 free(link); 4036 pr_warning("program '%s': failed to attach to pfd %d: %s\n", 4037 bpf_program__title(prog, false), pfd, 4038 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4039 return ERR_PTR(err); 4040 } 4041 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 4042 err = -errno; 4043 free(link); 4044 pr_warning("program '%s': failed to enable pfd %d: %s\n", 4045 bpf_program__title(prog, false), pfd, 4046 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4047 return ERR_PTR(err); 4048 } 4049 return (struct bpf_link *)link; 4050 } 4051 4052 /* 4053 * this function is expected to parse integer in the range of [0, 2^31-1] from 4054 * given file using scanf format string fmt. If actual parsed value is 4055 * negative, the result might be indistinguishable from error 4056 */ 4057 static int parse_uint_from_file(const char *file, const char *fmt) 4058 { 4059 char buf[STRERR_BUFSIZE]; 4060 int err, ret; 4061 FILE *f; 4062 4063 f = fopen(file, "r"); 4064 if (!f) { 4065 err = -errno; 4066 pr_debug("failed to open '%s': %s\n", file, 4067 libbpf_strerror_r(err, buf, sizeof(buf))); 4068 return err; 4069 } 4070 err = fscanf(f, fmt, &ret); 4071 if (err != 1) { 4072 err = err == EOF ? -EIO : -errno; 4073 pr_debug("failed to parse '%s': %s\n", file, 4074 libbpf_strerror_r(err, buf, sizeof(buf))); 4075 fclose(f); 4076 return err; 4077 } 4078 fclose(f); 4079 return ret; 4080 } 4081 4082 static int determine_kprobe_perf_type(void) 4083 { 4084 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 4085 4086 return parse_uint_from_file(file, "%d\n"); 4087 } 4088 4089 static int determine_uprobe_perf_type(void) 4090 { 4091 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 4092 4093 return parse_uint_from_file(file, "%d\n"); 4094 } 4095 4096 static int determine_kprobe_retprobe_bit(void) 4097 { 4098 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 4099 4100 return parse_uint_from_file(file, "config:%d\n"); 4101 } 4102 4103 static int determine_uprobe_retprobe_bit(void) 4104 { 4105 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 4106 4107 return parse_uint_from_file(file, "config:%d\n"); 4108 } 4109 4110 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 4111 uint64_t offset, int pid) 4112 { 4113 struct perf_event_attr attr = {}; 4114 char errmsg[STRERR_BUFSIZE]; 4115 int type, pfd, err; 4116 4117 type = uprobe ? determine_uprobe_perf_type() 4118 : determine_kprobe_perf_type(); 4119 if (type < 0) { 4120 pr_warning("failed to determine %s perf type: %s\n", 4121 uprobe ? "uprobe" : "kprobe", 4122 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 4123 return type; 4124 } 4125 if (retprobe) { 4126 int bit = uprobe ? determine_uprobe_retprobe_bit() 4127 : determine_kprobe_retprobe_bit(); 4128 4129 if (bit < 0) { 4130 pr_warning("failed to determine %s retprobe bit: %s\n", 4131 uprobe ? "uprobe" : "kprobe", 4132 libbpf_strerror_r(bit, errmsg, 4133 sizeof(errmsg))); 4134 return bit; 4135 } 4136 attr.config |= 1 << bit; 4137 } 4138 attr.size = sizeof(attr); 4139 attr.type = type; 4140 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 4141 attr.config2 = offset; /* kprobe_addr or probe_offset */ 4142 4143 /* pid filter is meaningful only for uprobes */ 4144 pfd = syscall(__NR_perf_event_open, &attr, 4145 pid < 0 ? -1 : pid /* pid */, 4146 pid == -1 ? 0 : -1 /* cpu */, 4147 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 4148 if (pfd < 0) { 4149 err = -errno; 4150 pr_warning("%s perf_event_open() failed: %s\n", 4151 uprobe ? "uprobe" : "kprobe", 4152 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4153 return err; 4154 } 4155 return pfd; 4156 } 4157 4158 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 4159 bool retprobe, 4160 const char *func_name) 4161 { 4162 char errmsg[STRERR_BUFSIZE]; 4163 struct bpf_link *link; 4164 int pfd, err; 4165 4166 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 4167 0 /* offset */, -1 /* pid */); 4168 if (pfd < 0) { 4169 pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", 4170 bpf_program__title(prog, false), 4171 retprobe ? "kretprobe" : "kprobe", func_name, 4172 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4173 return ERR_PTR(pfd); 4174 } 4175 link = bpf_program__attach_perf_event(prog, pfd); 4176 if (IS_ERR(link)) { 4177 close(pfd); 4178 err = PTR_ERR(link); 4179 pr_warning("program '%s': failed to attach to %s '%s': %s\n", 4180 bpf_program__title(prog, false), 4181 retprobe ? "kretprobe" : "kprobe", func_name, 4182 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4183 return link; 4184 } 4185 return link; 4186 } 4187 4188 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 4189 bool retprobe, pid_t pid, 4190 const char *binary_path, 4191 size_t func_offset) 4192 { 4193 char errmsg[STRERR_BUFSIZE]; 4194 struct bpf_link *link; 4195 int pfd, err; 4196 4197 pfd = perf_event_open_probe(true /* uprobe */, retprobe, 4198 binary_path, func_offset, pid); 4199 if (pfd < 0) { 4200 pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 4201 bpf_program__title(prog, false), 4202 retprobe ? "uretprobe" : "uprobe", 4203 binary_path, func_offset, 4204 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4205 return ERR_PTR(pfd); 4206 } 4207 link = bpf_program__attach_perf_event(prog, pfd); 4208 if (IS_ERR(link)) { 4209 close(pfd); 4210 err = PTR_ERR(link); 4211 pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", 4212 bpf_program__title(prog, false), 4213 retprobe ? "uretprobe" : "uprobe", 4214 binary_path, func_offset, 4215 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4216 return link; 4217 } 4218 return link; 4219 } 4220 4221 static int determine_tracepoint_id(const char *tp_category, 4222 const char *tp_name) 4223 { 4224 char file[PATH_MAX]; 4225 int ret; 4226 4227 ret = snprintf(file, sizeof(file), 4228 "/sys/kernel/debug/tracing/events/%s/%s/id", 4229 tp_category, tp_name); 4230 if (ret < 0) 4231 return -errno; 4232 if (ret >= sizeof(file)) { 4233 pr_debug("tracepoint %s/%s path is too long\n", 4234 tp_category, tp_name); 4235 return -E2BIG; 4236 } 4237 return parse_uint_from_file(file, "%d\n"); 4238 } 4239 4240 static int perf_event_open_tracepoint(const char *tp_category, 4241 const char *tp_name) 4242 { 4243 struct perf_event_attr attr = {}; 4244 char errmsg[STRERR_BUFSIZE]; 4245 int tp_id, pfd, err; 4246 4247 tp_id = determine_tracepoint_id(tp_category, tp_name); 4248 if (tp_id < 0) { 4249 pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 4250 tp_category, tp_name, 4251 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 4252 return tp_id; 4253 } 4254 4255 attr.type = PERF_TYPE_TRACEPOINT; 4256 attr.size = sizeof(attr); 4257 attr.config = tp_id; 4258 4259 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 4260 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 4261 if (pfd < 0) { 4262 err = -errno; 4263 pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", 4264 tp_category, tp_name, 4265 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4266 return err; 4267 } 4268 return pfd; 4269 } 4270 4271 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 4272 const char *tp_category, 4273 const char *tp_name) 4274 { 4275 char errmsg[STRERR_BUFSIZE]; 4276 struct bpf_link *link; 4277 int pfd, err; 4278 4279 pfd = perf_event_open_tracepoint(tp_category, tp_name); 4280 if (pfd < 0) { 4281 pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 4282 bpf_program__title(prog, false), 4283 tp_category, tp_name, 4284 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4285 return ERR_PTR(pfd); 4286 } 4287 link = bpf_program__attach_perf_event(prog, pfd); 4288 if (IS_ERR(link)) { 4289 close(pfd); 4290 err = PTR_ERR(link); 4291 pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", 4292 bpf_program__title(prog, false), 4293 tp_category, tp_name, 4294 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4295 return link; 4296 } 4297 return link; 4298 } 4299 4300 static int bpf_link__destroy_fd(struct bpf_link *link) 4301 { 4302 struct bpf_link_fd *l = (void *)link; 4303 4304 return close(l->fd); 4305 } 4306 4307 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 4308 const char *tp_name) 4309 { 4310 char errmsg[STRERR_BUFSIZE]; 4311 struct bpf_link_fd *link; 4312 int prog_fd, pfd; 4313 4314 prog_fd = bpf_program__fd(prog); 4315 if (prog_fd < 0) { 4316 pr_warning("program '%s': can't attach before loaded\n", 4317 bpf_program__title(prog, false)); 4318 return ERR_PTR(-EINVAL); 4319 } 4320 4321 link = malloc(sizeof(*link)); 4322 if (!link) 4323 return ERR_PTR(-ENOMEM); 4324 link->link.destroy = &bpf_link__destroy_fd; 4325 4326 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 4327 if (pfd < 0) { 4328 pfd = -errno; 4329 free(link); 4330 pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", 4331 bpf_program__title(prog, false), tp_name, 4332 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4333 return ERR_PTR(pfd); 4334 } 4335 link->fd = pfd; 4336 return (struct bpf_link *)link; 4337 } 4338 4339 enum bpf_perf_event_ret 4340 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 4341 void **copy_mem, size_t *copy_size, 4342 bpf_perf_event_print_t fn, void *private_data) 4343 { 4344 struct perf_event_mmap_page *header = mmap_mem; 4345 __u64 data_head = ring_buffer_read_head(header); 4346 __u64 data_tail = header->data_tail; 4347 void *base = ((__u8 *)header) + page_size; 4348 int ret = LIBBPF_PERF_EVENT_CONT; 4349 struct perf_event_header *ehdr; 4350 size_t ehdr_size; 4351 4352 while (data_head != data_tail) { 4353 ehdr = base + (data_tail & (mmap_size - 1)); 4354 ehdr_size = ehdr->size; 4355 4356 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 4357 void *copy_start = ehdr; 4358 size_t len_first = base + mmap_size - copy_start; 4359 size_t len_secnd = ehdr_size - len_first; 4360 4361 if (*copy_size < ehdr_size) { 4362 free(*copy_mem); 4363 *copy_mem = malloc(ehdr_size); 4364 if (!*copy_mem) { 4365 *copy_size = 0; 4366 ret = LIBBPF_PERF_EVENT_ERROR; 4367 break; 4368 } 4369 *copy_size = ehdr_size; 4370 } 4371 4372 memcpy(*copy_mem, copy_start, len_first); 4373 memcpy(*copy_mem + len_first, base, len_secnd); 4374 ehdr = *copy_mem; 4375 } 4376 4377 ret = fn(ehdr, private_data); 4378 data_tail += ehdr_size; 4379 if (ret != LIBBPF_PERF_EVENT_CONT) 4380 break; 4381 } 4382 4383 ring_buffer_write_tail(header, data_tail); 4384 return ret; 4385 } 4386 4387 struct perf_buffer; 4388 4389 struct perf_buffer_params { 4390 struct perf_event_attr *attr; 4391 /* if event_cb is specified, it takes precendence */ 4392 perf_buffer_event_fn event_cb; 4393 /* sample_cb and lost_cb are higher-level common-case callbacks */ 4394 perf_buffer_sample_fn sample_cb; 4395 perf_buffer_lost_fn lost_cb; 4396 void *ctx; 4397 int cpu_cnt; 4398 int *cpus; 4399 int *map_keys; 4400 }; 4401 4402 struct perf_cpu_buf { 4403 struct perf_buffer *pb; 4404 void *base; /* mmap()'ed memory */ 4405 void *buf; /* for reconstructing segmented data */ 4406 size_t buf_size; 4407 int fd; 4408 int cpu; 4409 int map_key; 4410 }; 4411 4412 struct perf_buffer { 4413 perf_buffer_event_fn event_cb; 4414 perf_buffer_sample_fn sample_cb; 4415 perf_buffer_lost_fn lost_cb; 4416 void *ctx; /* passed into callbacks */ 4417 4418 size_t page_size; 4419 size_t mmap_size; 4420 struct perf_cpu_buf **cpu_bufs; 4421 struct epoll_event *events; 4422 int cpu_cnt; 4423 int epoll_fd; /* perf event FD */ 4424 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 4425 }; 4426 4427 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 4428 struct perf_cpu_buf *cpu_buf) 4429 { 4430 if (!cpu_buf) 4431 return; 4432 if (cpu_buf->base && 4433 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 4434 pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 4435 if (cpu_buf->fd >= 0) { 4436 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 4437 close(cpu_buf->fd); 4438 } 4439 free(cpu_buf->buf); 4440 free(cpu_buf); 4441 } 4442 4443 void perf_buffer__free(struct perf_buffer *pb) 4444 { 4445 int i; 4446 4447 if (!pb) 4448 return; 4449 if (pb->cpu_bufs) { 4450 for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { 4451 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 4452 4453 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 4454 perf_buffer__free_cpu_buf(pb, cpu_buf); 4455 } 4456 free(pb->cpu_bufs); 4457 } 4458 if (pb->epoll_fd >= 0) 4459 close(pb->epoll_fd); 4460 free(pb->events); 4461 free(pb); 4462 } 4463 4464 static struct perf_cpu_buf * 4465 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 4466 int cpu, int map_key) 4467 { 4468 struct perf_cpu_buf *cpu_buf; 4469 char msg[STRERR_BUFSIZE]; 4470 int err; 4471 4472 cpu_buf = calloc(1, sizeof(*cpu_buf)); 4473 if (!cpu_buf) 4474 return ERR_PTR(-ENOMEM); 4475 4476 cpu_buf->pb = pb; 4477 cpu_buf->cpu = cpu; 4478 cpu_buf->map_key = map_key; 4479 4480 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 4481 -1, PERF_FLAG_FD_CLOEXEC); 4482 if (cpu_buf->fd < 0) { 4483 err = -errno; 4484 pr_warning("failed to open perf buffer event on cpu #%d: %s\n", 4485 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 4486 goto error; 4487 } 4488 4489 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 4490 PROT_READ | PROT_WRITE, MAP_SHARED, 4491 cpu_buf->fd, 0); 4492 if (cpu_buf->base == MAP_FAILED) { 4493 cpu_buf->base = NULL; 4494 err = -errno; 4495 pr_warning("failed to mmap perf buffer on cpu #%d: %s\n", 4496 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 4497 goto error; 4498 } 4499 4500 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 4501 err = -errno; 4502 pr_warning("failed to enable perf buffer event on cpu #%d: %s\n", 4503 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 4504 goto error; 4505 } 4506 4507 return cpu_buf; 4508 4509 error: 4510 perf_buffer__free_cpu_buf(pb, cpu_buf); 4511 return (struct perf_cpu_buf *)ERR_PTR(err); 4512 } 4513 4514 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 4515 struct perf_buffer_params *p); 4516 4517 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 4518 const struct perf_buffer_opts *opts) 4519 { 4520 struct perf_buffer_params p = {}; 4521 struct perf_event_attr attr = { 0, }; 4522 4523 attr.config = PERF_COUNT_SW_BPF_OUTPUT, 4524 attr.type = PERF_TYPE_SOFTWARE; 4525 attr.sample_type = PERF_SAMPLE_RAW; 4526 attr.sample_period = 1; 4527 attr.wakeup_events = 1; 4528 4529 p.attr = &attr; 4530 p.sample_cb = opts ? opts->sample_cb : NULL; 4531 p.lost_cb = opts ? opts->lost_cb : NULL; 4532 p.ctx = opts ? opts->ctx : NULL; 4533 4534 return __perf_buffer__new(map_fd, page_cnt, &p); 4535 } 4536 4537 struct perf_buffer * 4538 perf_buffer__new_raw(int map_fd, size_t page_cnt, 4539 const struct perf_buffer_raw_opts *opts) 4540 { 4541 struct perf_buffer_params p = {}; 4542 4543 p.attr = opts->attr; 4544 p.event_cb = opts->event_cb; 4545 p.ctx = opts->ctx; 4546 p.cpu_cnt = opts->cpu_cnt; 4547 p.cpus = opts->cpus; 4548 p.map_keys = opts->map_keys; 4549 4550 return __perf_buffer__new(map_fd, page_cnt, &p); 4551 } 4552 4553 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 4554 struct perf_buffer_params *p) 4555 { 4556 struct bpf_map_info map = {}; 4557 char msg[STRERR_BUFSIZE]; 4558 struct perf_buffer *pb; 4559 __u32 map_info_len; 4560 int err, i; 4561 4562 if (page_cnt & (page_cnt - 1)) { 4563 pr_warning("page count should be power of two, but is %zu\n", 4564 page_cnt); 4565 return ERR_PTR(-EINVAL); 4566 } 4567 4568 map_info_len = sizeof(map); 4569 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 4570 if (err) { 4571 err = -errno; 4572 pr_warning("failed to get map info for map FD %d: %s\n", 4573 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 4574 return ERR_PTR(err); 4575 } 4576 4577 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 4578 pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 4579 map.name); 4580 return ERR_PTR(-EINVAL); 4581 } 4582 4583 pb = calloc(1, sizeof(*pb)); 4584 if (!pb) 4585 return ERR_PTR(-ENOMEM); 4586 4587 pb->event_cb = p->event_cb; 4588 pb->sample_cb = p->sample_cb; 4589 pb->lost_cb = p->lost_cb; 4590 pb->ctx = p->ctx; 4591 4592 pb->page_size = getpagesize(); 4593 pb->mmap_size = pb->page_size * page_cnt; 4594 pb->map_fd = map_fd; 4595 4596 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 4597 if (pb->epoll_fd < 0) { 4598 err = -errno; 4599 pr_warning("failed to create epoll instance: %s\n", 4600 libbpf_strerror_r(err, msg, sizeof(msg))); 4601 goto error; 4602 } 4603 4604 if (p->cpu_cnt > 0) { 4605 pb->cpu_cnt = p->cpu_cnt; 4606 } else { 4607 pb->cpu_cnt = libbpf_num_possible_cpus(); 4608 if (pb->cpu_cnt < 0) { 4609 err = pb->cpu_cnt; 4610 goto error; 4611 } 4612 if (map.max_entries < pb->cpu_cnt) 4613 pb->cpu_cnt = map.max_entries; 4614 } 4615 4616 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 4617 if (!pb->events) { 4618 err = -ENOMEM; 4619 pr_warning("failed to allocate events: out of memory\n"); 4620 goto error; 4621 } 4622 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 4623 if (!pb->cpu_bufs) { 4624 err = -ENOMEM; 4625 pr_warning("failed to allocate buffers: out of memory\n"); 4626 goto error; 4627 } 4628 4629 for (i = 0; i < pb->cpu_cnt; i++) { 4630 struct perf_cpu_buf *cpu_buf; 4631 int cpu, map_key; 4632 4633 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 4634 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 4635 4636 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 4637 if (IS_ERR(cpu_buf)) { 4638 err = PTR_ERR(cpu_buf); 4639 goto error; 4640 } 4641 4642 pb->cpu_bufs[i] = cpu_buf; 4643 4644 err = bpf_map_update_elem(pb->map_fd, &map_key, 4645 &cpu_buf->fd, 0); 4646 if (err) { 4647 err = -errno; 4648 pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 4649 cpu, map_key, cpu_buf->fd, 4650 libbpf_strerror_r(err, msg, sizeof(msg))); 4651 goto error; 4652 } 4653 4654 pb->events[i].events = EPOLLIN; 4655 pb->events[i].data.ptr = cpu_buf; 4656 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 4657 &pb->events[i]) < 0) { 4658 err = -errno; 4659 pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 4660 cpu, cpu_buf->fd, 4661 libbpf_strerror_r(err, msg, sizeof(msg))); 4662 goto error; 4663 } 4664 } 4665 4666 return pb; 4667 4668 error: 4669 if (pb) 4670 perf_buffer__free(pb); 4671 return ERR_PTR(err); 4672 } 4673 4674 struct perf_sample_raw { 4675 struct perf_event_header header; 4676 uint32_t size; 4677 char data[0]; 4678 }; 4679 4680 struct perf_sample_lost { 4681 struct perf_event_header header; 4682 uint64_t id; 4683 uint64_t lost; 4684 uint64_t sample_id; 4685 }; 4686 4687 static enum bpf_perf_event_ret 4688 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 4689 { 4690 struct perf_cpu_buf *cpu_buf = ctx; 4691 struct perf_buffer *pb = cpu_buf->pb; 4692 void *data = e; 4693 4694 /* user wants full control over parsing perf event */ 4695 if (pb->event_cb) 4696 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 4697 4698 switch (e->type) { 4699 case PERF_RECORD_SAMPLE: { 4700 struct perf_sample_raw *s = data; 4701 4702 if (pb->sample_cb) 4703 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 4704 break; 4705 } 4706 case PERF_RECORD_LOST: { 4707 struct perf_sample_lost *s = data; 4708 4709 if (pb->lost_cb) 4710 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 4711 break; 4712 } 4713 default: 4714 pr_warning("unknown perf sample type %d\n", e->type); 4715 return LIBBPF_PERF_EVENT_ERROR; 4716 } 4717 return LIBBPF_PERF_EVENT_CONT; 4718 } 4719 4720 static int perf_buffer__process_records(struct perf_buffer *pb, 4721 struct perf_cpu_buf *cpu_buf) 4722 { 4723 enum bpf_perf_event_ret ret; 4724 4725 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 4726 pb->page_size, &cpu_buf->buf, 4727 &cpu_buf->buf_size, 4728 perf_buffer__process_record, cpu_buf); 4729 if (ret != LIBBPF_PERF_EVENT_CONT) 4730 return ret; 4731 return 0; 4732 } 4733 4734 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 4735 { 4736 int i, cnt, err; 4737 4738 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 4739 for (i = 0; i < cnt; i++) { 4740 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 4741 4742 err = perf_buffer__process_records(pb, cpu_buf); 4743 if (err) { 4744 pr_warning("error while processing records: %d\n", err); 4745 return err; 4746 } 4747 } 4748 return cnt < 0 ? -errno : cnt; 4749 } 4750 4751 struct bpf_prog_info_array_desc { 4752 int array_offset; /* e.g. offset of jited_prog_insns */ 4753 int count_offset; /* e.g. offset of jited_prog_len */ 4754 int size_offset; /* > 0: offset of rec size, 4755 * < 0: fix size of -size_offset 4756 */ 4757 }; 4758 4759 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 4760 [BPF_PROG_INFO_JITED_INSNS] = { 4761 offsetof(struct bpf_prog_info, jited_prog_insns), 4762 offsetof(struct bpf_prog_info, jited_prog_len), 4763 -1, 4764 }, 4765 [BPF_PROG_INFO_XLATED_INSNS] = { 4766 offsetof(struct bpf_prog_info, xlated_prog_insns), 4767 offsetof(struct bpf_prog_info, xlated_prog_len), 4768 -1, 4769 }, 4770 [BPF_PROG_INFO_MAP_IDS] = { 4771 offsetof(struct bpf_prog_info, map_ids), 4772 offsetof(struct bpf_prog_info, nr_map_ids), 4773 -(int)sizeof(__u32), 4774 }, 4775 [BPF_PROG_INFO_JITED_KSYMS] = { 4776 offsetof(struct bpf_prog_info, jited_ksyms), 4777 offsetof(struct bpf_prog_info, nr_jited_ksyms), 4778 -(int)sizeof(__u64), 4779 }, 4780 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 4781 offsetof(struct bpf_prog_info, jited_func_lens), 4782 offsetof(struct bpf_prog_info, nr_jited_func_lens), 4783 -(int)sizeof(__u32), 4784 }, 4785 [BPF_PROG_INFO_FUNC_INFO] = { 4786 offsetof(struct bpf_prog_info, func_info), 4787 offsetof(struct bpf_prog_info, nr_func_info), 4788 offsetof(struct bpf_prog_info, func_info_rec_size), 4789 }, 4790 [BPF_PROG_INFO_LINE_INFO] = { 4791 offsetof(struct bpf_prog_info, line_info), 4792 offsetof(struct bpf_prog_info, nr_line_info), 4793 offsetof(struct bpf_prog_info, line_info_rec_size), 4794 }, 4795 [BPF_PROG_INFO_JITED_LINE_INFO] = { 4796 offsetof(struct bpf_prog_info, jited_line_info), 4797 offsetof(struct bpf_prog_info, nr_jited_line_info), 4798 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 4799 }, 4800 [BPF_PROG_INFO_PROG_TAGS] = { 4801 offsetof(struct bpf_prog_info, prog_tags), 4802 offsetof(struct bpf_prog_info, nr_prog_tags), 4803 -(int)sizeof(__u8) * BPF_TAG_SIZE, 4804 }, 4805 4806 }; 4807 4808 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 4809 { 4810 __u32 *array = (__u32 *)info; 4811 4812 if (offset >= 0) 4813 return array[offset / sizeof(__u32)]; 4814 return -(int)offset; 4815 } 4816 4817 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 4818 { 4819 __u64 *array = (__u64 *)info; 4820 4821 if (offset >= 0) 4822 return array[offset / sizeof(__u64)]; 4823 return -(int)offset; 4824 } 4825 4826 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 4827 __u32 val) 4828 { 4829 __u32 *array = (__u32 *)info; 4830 4831 if (offset >= 0) 4832 array[offset / sizeof(__u32)] = val; 4833 } 4834 4835 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 4836 __u64 val) 4837 { 4838 __u64 *array = (__u64 *)info; 4839 4840 if (offset >= 0) 4841 array[offset / sizeof(__u64)] = val; 4842 } 4843 4844 struct bpf_prog_info_linear * 4845 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 4846 { 4847 struct bpf_prog_info_linear *info_linear; 4848 struct bpf_prog_info info = {}; 4849 __u32 info_len = sizeof(info); 4850 __u32 data_len = 0; 4851 int i, err; 4852 void *ptr; 4853 4854 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 4855 return ERR_PTR(-EINVAL); 4856 4857 /* step 1: get array dimensions */ 4858 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 4859 if (err) { 4860 pr_debug("can't get prog info: %s", strerror(errno)); 4861 return ERR_PTR(-EFAULT); 4862 } 4863 4864 /* step 2: calculate total size of all arrays */ 4865 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4866 bool include_array = (arrays & (1UL << i)) > 0; 4867 struct bpf_prog_info_array_desc *desc; 4868 __u32 count, size; 4869 4870 desc = bpf_prog_info_array_desc + i; 4871 4872 /* kernel is too old to support this field */ 4873 if (info_len < desc->array_offset + sizeof(__u32) || 4874 info_len < desc->count_offset + sizeof(__u32) || 4875 (desc->size_offset > 0 && info_len < desc->size_offset)) 4876 include_array = false; 4877 4878 if (!include_array) { 4879 arrays &= ~(1UL << i); /* clear the bit */ 4880 continue; 4881 } 4882 4883 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4884 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4885 4886 data_len += count * size; 4887 } 4888 4889 /* step 3: allocate continuous memory */ 4890 data_len = roundup(data_len, sizeof(__u64)); 4891 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 4892 if (!info_linear) 4893 return ERR_PTR(-ENOMEM); 4894 4895 /* step 4: fill data to info_linear->info */ 4896 info_linear->arrays = arrays; 4897 memset(&info_linear->info, 0, sizeof(info)); 4898 ptr = info_linear->data; 4899 4900 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4901 struct bpf_prog_info_array_desc *desc; 4902 __u32 count, size; 4903 4904 if ((arrays & (1UL << i)) == 0) 4905 continue; 4906 4907 desc = bpf_prog_info_array_desc + i; 4908 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4909 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4910 bpf_prog_info_set_offset_u32(&info_linear->info, 4911 desc->count_offset, count); 4912 bpf_prog_info_set_offset_u32(&info_linear->info, 4913 desc->size_offset, size); 4914 bpf_prog_info_set_offset_u64(&info_linear->info, 4915 desc->array_offset, 4916 ptr_to_u64(ptr)); 4917 ptr += count * size; 4918 } 4919 4920 /* step 5: call syscall again to get required arrays */ 4921 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 4922 if (err) { 4923 pr_debug("can't get prog info: %s", strerror(errno)); 4924 free(info_linear); 4925 return ERR_PTR(-EFAULT); 4926 } 4927 4928 /* step 6: verify the data */ 4929 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4930 struct bpf_prog_info_array_desc *desc; 4931 __u32 v1, v2; 4932 4933 if ((arrays & (1UL << i)) == 0) 4934 continue; 4935 4936 desc = bpf_prog_info_array_desc + i; 4937 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4938 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 4939 desc->count_offset); 4940 if (v1 != v2) 4941 pr_warning("%s: mismatch in element count\n", __func__); 4942 4943 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4944 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 4945 desc->size_offset); 4946 if (v1 != v2) 4947 pr_warning("%s: mismatch in rec size\n", __func__); 4948 } 4949 4950 /* step 7: update info_len and data_len */ 4951 info_linear->info_len = sizeof(struct bpf_prog_info); 4952 info_linear->data_len = data_len; 4953 4954 return info_linear; 4955 } 4956 4957 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 4958 { 4959 int i; 4960 4961 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4962 struct bpf_prog_info_array_desc *desc; 4963 __u64 addr, offs; 4964 4965 if ((info_linear->arrays & (1UL << i)) == 0) 4966 continue; 4967 4968 desc = bpf_prog_info_array_desc + i; 4969 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 4970 desc->array_offset); 4971 offs = addr - ptr_to_u64(info_linear->data); 4972 bpf_prog_info_set_offset_u64(&info_linear->info, 4973 desc->array_offset, offs); 4974 } 4975 } 4976 4977 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 4978 { 4979 int i; 4980 4981 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4982 struct bpf_prog_info_array_desc *desc; 4983 __u64 addr, offs; 4984 4985 if ((info_linear->arrays & (1UL << i)) == 0) 4986 continue; 4987 4988 desc = bpf_prog_info_array_desc + i; 4989 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 4990 desc->array_offset); 4991 addr = offs + ptr_to_u64(info_linear->data); 4992 bpf_prog_info_set_offset_u64(&info_linear->info, 4993 desc->array_offset, addr); 4994 } 4995 } 4996 4997 int libbpf_num_possible_cpus(void) 4998 { 4999 static const char *fcpu = "/sys/devices/system/cpu/possible"; 5000 int len = 0, n = 0, il = 0, ir = 0; 5001 unsigned int start = 0, end = 0; 5002 int tmp_cpus = 0; 5003 static int cpus; 5004 char buf[128]; 5005 int error = 0; 5006 int fd = -1; 5007 5008 tmp_cpus = READ_ONCE(cpus); 5009 if (tmp_cpus > 0) 5010 return tmp_cpus; 5011 5012 fd = open(fcpu, O_RDONLY); 5013 if (fd < 0) { 5014 error = errno; 5015 pr_warning("Failed to open file %s: %s\n", 5016 fcpu, strerror(error)); 5017 return -error; 5018 } 5019 len = read(fd, buf, sizeof(buf)); 5020 close(fd); 5021 if (len <= 0) { 5022 error = len ? errno : EINVAL; 5023 pr_warning("Failed to read # of possible cpus from %s: %s\n", 5024 fcpu, strerror(error)); 5025 return -error; 5026 } 5027 if (len == sizeof(buf)) { 5028 pr_warning("File %s size overflow\n", fcpu); 5029 return -EOVERFLOW; 5030 } 5031 buf[len] = '\0'; 5032 5033 for (ir = 0, tmp_cpus = 0; ir <= len; ir++) { 5034 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 5035 if (buf[ir] == ',' || buf[ir] == '\0') { 5036 buf[ir] = '\0'; 5037 n = sscanf(&buf[il], "%u-%u", &start, &end); 5038 if (n <= 0) { 5039 pr_warning("Failed to get # CPUs from %s\n", 5040 &buf[il]); 5041 return -EINVAL; 5042 } else if (n == 1) { 5043 end = start; 5044 } 5045 tmp_cpus += end - start + 1; 5046 il = ir + 1; 5047 } 5048 } 5049 if (tmp_cpus <= 0) { 5050 pr_warning("Invalid #CPUs %d from %s\n", tmp_cpus, fcpu); 5051 return -EINVAL; 5052 } 5053 5054 WRITE_ONCE(cpus, tmp_cpus); 5055 return tmp_cpus; 5056 } 5057