1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/stat.h> 36 #include <sys/types.h> 37 #include <sys/vfs.h> 38 #include <tools/libc_compat.h> 39 #include <libelf.h> 40 #include <gelf.h> 41 42 #include "libbpf.h" 43 #include "bpf.h" 44 #include "btf.h" 45 #include "str_error.h" 46 #include "libbpf_util.h" 47 #include "libbpf_internal.h" 48 49 #ifndef EM_BPF 50 #define EM_BPF 247 51 #endif 52 53 #ifndef BPF_FS_MAGIC 54 #define BPF_FS_MAGIC 0xcafe4a11 55 #endif 56 57 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 58 * compilation if user enables corresponding warning. Disable it explicitly. 59 */ 60 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 61 62 #define __printf(a, b) __attribute__((format(printf, a, b))) 63 64 static int __base_pr(enum libbpf_print_level level, const char *format, 65 va_list args) 66 { 67 if (level == LIBBPF_DEBUG) 68 return 0; 69 70 return vfprintf(stderr, format, args); 71 } 72 73 static libbpf_print_fn_t __libbpf_pr = __base_pr; 74 75 void libbpf_set_print(libbpf_print_fn_t fn) 76 { 77 __libbpf_pr = fn; 78 } 79 80 __printf(2, 3) 81 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 82 { 83 va_list args; 84 85 if (!__libbpf_pr) 86 return; 87 88 va_start(args, format); 89 __libbpf_pr(level, format, args); 90 va_end(args); 91 } 92 93 #define STRERR_BUFSIZE 128 94 95 #define CHECK_ERR(action, err, out) do { \ 96 err = action; \ 97 if (err) \ 98 goto out; \ 99 } while(0) 100 101 102 /* Copied from tools/perf/util/util.h */ 103 #ifndef zfree 104 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 105 #endif 106 107 #ifndef zclose 108 # define zclose(fd) ({ \ 109 int ___err = 0; \ 110 if ((fd) >= 0) \ 111 ___err = close((fd)); \ 112 fd = -1; \ 113 ___err; }) 114 #endif 115 116 #ifdef HAVE_LIBELF_MMAP_SUPPORT 117 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 118 #else 119 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 120 #endif 121 122 static inline __u64 ptr_to_u64(const void *ptr) 123 { 124 return (__u64) (unsigned long) ptr; 125 } 126 127 struct bpf_capabilities { 128 /* v4.14: kernel support for program & map names. */ 129 __u32 name:1; 130 /* v5.2: kernel support for global data sections. */ 131 __u32 global_data:1; 132 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 133 __u32 btf_func:1; 134 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 135 __u32 btf_datasec:1; 136 }; 137 138 /* 139 * bpf_prog should be a better name but it has been used in 140 * linux/filter.h. 141 */ 142 struct bpf_program { 143 /* Index in elf obj file, for relocation use. */ 144 int idx; 145 char *name; 146 int prog_ifindex; 147 char *section_name; 148 /* section_name with / replaced by _; makes recursive pinning 149 * in bpf_object__pin_programs easier 150 */ 151 char *pin_name; 152 struct bpf_insn *insns; 153 size_t insns_cnt, main_prog_cnt; 154 enum bpf_prog_type type; 155 156 struct reloc_desc { 157 enum { 158 RELO_LD64, 159 RELO_CALL, 160 RELO_DATA, 161 } type; 162 int insn_idx; 163 union { 164 int map_idx; 165 int text_off; 166 }; 167 } *reloc_desc; 168 int nr_reloc; 169 int log_level; 170 171 struct { 172 int nr; 173 int *fds; 174 } instances; 175 bpf_program_prep_t preprocessor; 176 177 struct bpf_object *obj; 178 void *priv; 179 bpf_program_clear_priv_t clear_priv; 180 181 enum bpf_attach_type expected_attach_type; 182 int btf_fd; 183 void *func_info; 184 __u32 func_info_rec_size; 185 __u32 func_info_cnt; 186 187 struct bpf_capabilities *caps; 188 189 void *line_info; 190 __u32 line_info_rec_size; 191 __u32 line_info_cnt; 192 }; 193 194 enum libbpf_map_type { 195 LIBBPF_MAP_UNSPEC, 196 LIBBPF_MAP_DATA, 197 LIBBPF_MAP_BSS, 198 LIBBPF_MAP_RODATA, 199 }; 200 201 static const char * const libbpf_type_to_btf_name[] = { 202 [LIBBPF_MAP_DATA] = ".data", 203 [LIBBPF_MAP_BSS] = ".bss", 204 [LIBBPF_MAP_RODATA] = ".rodata", 205 }; 206 207 struct bpf_map { 208 int fd; 209 char *name; 210 size_t offset; 211 int map_ifindex; 212 int inner_map_fd; 213 struct bpf_map_def def; 214 __u32 btf_key_type_id; 215 __u32 btf_value_type_id; 216 void *priv; 217 bpf_map_clear_priv_t clear_priv; 218 enum libbpf_map_type libbpf_type; 219 }; 220 221 struct bpf_secdata { 222 void *rodata; 223 void *data; 224 }; 225 226 static LIST_HEAD(bpf_objects_list); 227 228 struct bpf_object { 229 char name[BPF_OBJ_NAME_LEN]; 230 char license[64]; 231 __u32 kern_version; 232 233 struct bpf_program *programs; 234 size_t nr_programs; 235 struct bpf_map *maps; 236 size_t nr_maps; 237 struct bpf_secdata sections; 238 239 bool loaded; 240 bool has_pseudo_calls; 241 242 /* 243 * Information when doing elf related work. Only valid if fd 244 * is valid. 245 */ 246 struct { 247 int fd; 248 void *obj_buf; 249 size_t obj_buf_sz; 250 Elf *elf; 251 GElf_Ehdr ehdr; 252 Elf_Data *symbols; 253 Elf_Data *data; 254 Elf_Data *rodata; 255 Elf_Data *bss; 256 size_t strtabidx; 257 struct { 258 GElf_Shdr shdr; 259 Elf_Data *data; 260 } *reloc; 261 int nr_reloc; 262 int maps_shndx; 263 int text_shndx; 264 int data_shndx; 265 int rodata_shndx; 266 int bss_shndx; 267 } efile; 268 /* 269 * All loaded bpf_object is linked in a list, which is 270 * hidden to caller. bpf_objects__<func> handlers deal with 271 * all objects. 272 */ 273 struct list_head list; 274 275 struct btf *btf; 276 struct btf_ext *btf_ext; 277 278 void *priv; 279 bpf_object_clear_priv_t clear_priv; 280 281 struct bpf_capabilities caps; 282 283 char path[]; 284 }; 285 #define obj_elf_valid(o) ((o)->efile.elf) 286 287 void bpf_program__unload(struct bpf_program *prog) 288 { 289 int i; 290 291 if (!prog) 292 return; 293 294 /* 295 * If the object is opened but the program was never loaded, 296 * it is possible that prog->instances.nr == -1. 297 */ 298 if (prog->instances.nr > 0) { 299 for (i = 0; i < prog->instances.nr; i++) 300 zclose(prog->instances.fds[i]); 301 } else if (prog->instances.nr != -1) { 302 pr_warning("Internal error: instances.nr is %d\n", 303 prog->instances.nr); 304 } 305 306 prog->instances.nr = -1; 307 zfree(&prog->instances.fds); 308 309 zclose(prog->btf_fd); 310 zfree(&prog->func_info); 311 zfree(&prog->line_info); 312 } 313 314 static void bpf_program__exit(struct bpf_program *prog) 315 { 316 if (!prog) 317 return; 318 319 if (prog->clear_priv) 320 prog->clear_priv(prog, prog->priv); 321 322 prog->priv = NULL; 323 prog->clear_priv = NULL; 324 325 bpf_program__unload(prog); 326 zfree(&prog->name); 327 zfree(&prog->section_name); 328 zfree(&prog->pin_name); 329 zfree(&prog->insns); 330 zfree(&prog->reloc_desc); 331 332 prog->nr_reloc = 0; 333 prog->insns_cnt = 0; 334 prog->idx = -1; 335 } 336 337 static char *__bpf_program__pin_name(struct bpf_program *prog) 338 { 339 char *name, *p; 340 341 name = p = strdup(prog->section_name); 342 while ((p = strchr(p, '/'))) 343 *p = '_'; 344 345 return name; 346 } 347 348 static int 349 bpf_program__init(void *data, size_t size, char *section_name, int idx, 350 struct bpf_program *prog) 351 { 352 if (size < sizeof(struct bpf_insn)) { 353 pr_warning("corrupted section '%s'\n", section_name); 354 return -EINVAL; 355 } 356 357 memset(prog, 0, sizeof(*prog)); 358 359 prog->section_name = strdup(section_name); 360 if (!prog->section_name) { 361 pr_warning("failed to alloc name for prog under section(%d) %s\n", 362 idx, section_name); 363 goto errout; 364 } 365 366 prog->pin_name = __bpf_program__pin_name(prog); 367 if (!prog->pin_name) { 368 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 369 idx, section_name); 370 goto errout; 371 } 372 373 prog->insns = malloc(size); 374 if (!prog->insns) { 375 pr_warning("failed to alloc insns for prog under section %s\n", 376 section_name); 377 goto errout; 378 } 379 prog->insns_cnt = size / sizeof(struct bpf_insn); 380 memcpy(prog->insns, data, 381 prog->insns_cnt * sizeof(struct bpf_insn)); 382 prog->idx = idx; 383 prog->instances.fds = NULL; 384 prog->instances.nr = -1; 385 prog->type = BPF_PROG_TYPE_UNSPEC; 386 prog->btf_fd = -1; 387 388 return 0; 389 errout: 390 bpf_program__exit(prog); 391 return -ENOMEM; 392 } 393 394 static int 395 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 396 char *section_name, int idx) 397 { 398 struct bpf_program prog, *progs; 399 int nr_progs, err; 400 401 err = bpf_program__init(data, size, section_name, idx, &prog); 402 if (err) 403 return err; 404 405 prog.caps = &obj->caps; 406 progs = obj->programs; 407 nr_progs = obj->nr_programs; 408 409 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 410 if (!progs) { 411 /* 412 * In this case the original obj->programs 413 * is still valid, so don't need special treat for 414 * bpf_close_object(). 415 */ 416 pr_warning("failed to alloc a new program under section '%s'\n", 417 section_name); 418 bpf_program__exit(&prog); 419 return -ENOMEM; 420 } 421 422 pr_debug("found program %s\n", prog.section_name); 423 obj->programs = progs; 424 obj->nr_programs = nr_progs + 1; 425 prog.obj = obj; 426 progs[nr_progs] = prog; 427 return 0; 428 } 429 430 static int 431 bpf_object__init_prog_names(struct bpf_object *obj) 432 { 433 Elf_Data *symbols = obj->efile.symbols; 434 struct bpf_program *prog; 435 size_t pi, si; 436 437 for (pi = 0; pi < obj->nr_programs; pi++) { 438 const char *name = NULL; 439 440 prog = &obj->programs[pi]; 441 442 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 443 si++) { 444 GElf_Sym sym; 445 446 if (!gelf_getsym(symbols, si, &sym)) 447 continue; 448 if (sym.st_shndx != prog->idx) 449 continue; 450 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 451 continue; 452 453 name = elf_strptr(obj->efile.elf, 454 obj->efile.strtabidx, 455 sym.st_name); 456 if (!name) { 457 pr_warning("failed to get sym name string for prog %s\n", 458 prog->section_name); 459 return -LIBBPF_ERRNO__LIBELF; 460 } 461 } 462 463 if (!name && prog->idx == obj->efile.text_shndx) 464 name = ".text"; 465 466 if (!name) { 467 pr_warning("failed to find sym for prog %s\n", 468 prog->section_name); 469 return -EINVAL; 470 } 471 472 prog->name = strdup(name); 473 if (!prog->name) { 474 pr_warning("failed to allocate memory for prog sym %s\n", 475 name); 476 return -ENOMEM; 477 } 478 } 479 480 return 0; 481 } 482 483 static struct bpf_object *bpf_object__new(const char *path, 484 void *obj_buf, 485 size_t obj_buf_sz) 486 { 487 struct bpf_object *obj; 488 char *end; 489 490 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 491 if (!obj) { 492 pr_warning("alloc memory failed for %s\n", path); 493 return ERR_PTR(-ENOMEM); 494 } 495 496 strcpy(obj->path, path); 497 /* Using basename() GNU version which doesn't modify arg. */ 498 strncpy(obj->name, basename((void *)path), 499 sizeof(obj->name) - 1); 500 end = strchr(obj->name, '.'); 501 if (end) 502 *end = 0; 503 504 obj->efile.fd = -1; 505 /* 506 * Caller of this function should also calls 507 * bpf_object__elf_finish() after data collection to return 508 * obj_buf to user. If not, we should duplicate the buffer to 509 * avoid user freeing them before elf finish. 510 */ 511 obj->efile.obj_buf = obj_buf; 512 obj->efile.obj_buf_sz = obj_buf_sz; 513 obj->efile.maps_shndx = -1; 514 obj->efile.data_shndx = -1; 515 obj->efile.rodata_shndx = -1; 516 obj->efile.bss_shndx = -1; 517 518 obj->loaded = false; 519 520 INIT_LIST_HEAD(&obj->list); 521 list_add(&obj->list, &bpf_objects_list); 522 return obj; 523 } 524 525 static void bpf_object__elf_finish(struct bpf_object *obj) 526 { 527 if (!obj_elf_valid(obj)) 528 return; 529 530 if (obj->efile.elf) { 531 elf_end(obj->efile.elf); 532 obj->efile.elf = NULL; 533 } 534 obj->efile.symbols = NULL; 535 obj->efile.data = NULL; 536 obj->efile.rodata = NULL; 537 obj->efile.bss = NULL; 538 539 zfree(&obj->efile.reloc); 540 obj->efile.nr_reloc = 0; 541 zclose(obj->efile.fd); 542 obj->efile.obj_buf = NULL; 543 obj->efile.obj_buf_sz = 0; 544 } 545 546 static int bpf_object__elf_init(struct bpf_object *obj) 547 { 548 int err = 0; 549 GElf_Ehdr *ep; 550 551 if (obj_elf_valid(obj)) { 552 pr_warning("elf init: internal error\n"); 553 return -LIBBPF_ERRNO__LIBELF; 554 } 555 556 if (obj->efile.obj_buf_sz > 0) { 557 /* 558 * obj_buf should have been validated by 559 * bpf_object__open_buffer(). 560 */ 561 obj->efile.elf = elf_memory(obj->efile.obj_buf, 562 obj->efile.obj_buf_sz); 563 } else { 564 obj->efile.fd = open(obj->path, O_RDONLY); 565 if (obj->efile.fd < 0) { 566 char errmsg[STRERR_BUFSIZE]; 567 char *cp = libbpf_strerror_r(errno, errmsg, 568 sizeof(errmsg)); 569 570 pr_warning("failed to open %s: %s\n", obj->path, cp); 571 return -errno; 572 } 573 574 obj->efile.elf = elf_begin(obj->efile.fd, 575 LIBBPF_ELF_C_READ_MMAP, 576 NULL); 577 } 578 579 if (!obj->efile.elf) { 580 pr_warning("failed to open %s as ELF file\n", 581 obj->path); 582 err = -LIBBPF_ERRNO__LIBELF; 583 goto errout; 584 } 585 586 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 587 pr_warning("failed to get EHDR from %s\n", 588 obj->path); 589 err = -LIBBPF_ERRNO__FORMAT; 590 goto errout; 591 } 592 ep = &obj->efile.ehdr; 593 594 /* Old LLVM set e_machine to EM_NONE */ 595 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 596 pr_warning("%s is not an eBPF object file\n", 597 obj->path); 598 err = -LIBBPF_ERRNO__FORMAT; 599 goto errout; 600 } 601 602 return 0; 603 errout: 604 bpf_object__elf_finish(obj); 605 return err; 606 } 607 608 static int 609 bpf_object__check_endianness(struct bpf_object *obj) 610 { 611 static unsigned int const endian = 1; 612 613 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 614 case ELFDATA2LSB: 615 /* We are big endian, BPF obj is little endian. */ 616 if (*(unsigned char const *)&endian != 1) 617 goto mismatch; 618 break; 619 620 case ELFDATA2MSB: 621 /* We are little endian, BPF obj is big endian. */ 622 if (*(unsigned char const *)&endian != 0) 623 goto mismatch; 624 break; 625 default: 626 return -LIBBPF_ERRNO__ENDIAN; 627 } 628 629 return 0; 630 631 mismatch: 632 pr_warning("Error: endianness mismatch.\n"); 633 return -LIBBPF_ERRNO__ENDIAN; 634 } 635 636 static int 637 bpf_object__init_license(struct bpf_object *obj, 638 void *data, size_t size) 639 { 640 memcpy(obj->license, data, 641 min(size, sizeof(obj->license) - 1)); 642 pr_debug("license of %s is %s\n", obj->path, obj->license); 643 return 0; 644 } 645 646 static int 647 bpf_object__init_kversion(struct bpf_object *obj, 648 void *data, size_t size) 649 { 650 __u32 kver; 651 652 if (size != sizeof(kver)) { 653 pr_warning("invalid kver section in %s\n", obj->path); 654 return -LIBBPF_ERRNO__FORMAT; 655 } 656 memcpy(&kver, data, sizeof(kver)); 657 obj->kern_version = kver; 658 pr_debug("kernel version of %s is %x\n", obj->path, 659 obj->kern_version); 660 return 0; 661 } 662 663 static int compare_bpf_map(const void *_a, const void *_b) 664 { 665 const struct bpf_map *a = _a; 666 const struct bpf_map *b = _b; 667 668 return a->offset - b->offset; 669 } 670 671 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 672 { 673 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 674 type == BPF_MAP_TYPE_HASH_OF_MAPS) 675 return true; 676 return false; 677 } 678 679 static int bpf_object_search_section_size(const struct bpf_object *obj, 680 const char *name, size_t *d_size) 681 { 682 const GElf_Ehdr *ep = &obj->efile.ehdr; 683 Elf *elf = obj->efile.elf; 684 Elf_Scn *scn = NULL; 685 int idx = 0; 686 687 while ((scn = elf_nextscn(elf, scn)) != NULL) { 688 const char *sec_name; 689 Elf_Data *data; 690 GElf_Shdr sh; 691 692 idx++; 693 if (gelf_getshdr(scn, &sh) != &sh) { 694 pr_warning("failed to get section(%d) header from %s\n", 695 idx, obj->path); 696 return -EIO; 697 } 698 699 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 700 if (!sec_name) { 701 pr_warning("failed to get section(%d) name from %s\n", 702 idx, obj->path); 703 return -EIO; 704 } 705 706 if (strcmp(name, sec_name)) 707 continue; 708 709 data = elf_getdata(scn, 0); 710 if (!data) { 711 pr_warning("failed to get section(%d) data from %s(%s)\n", 712 idx, name, obj->path); 713 return -EIO; 714 } 715 716 *d_size = data->d_size; 717 return 0; 718 } 719 720 return -ENOENT; 721 } 722 723 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 724 __u32 *size) 725 { 726 int ret = -ENOENT; 727 size_t d_size; 728 729 *size = 0; 730 if (!name) { 731 return -EINVAL; 732 } else if (!strcmp(name, ".data")) { 733 if (obj->efile.data) 734 *size = obj->efile.data->d_size; 735 } else if (!strcmp(name, ".bss")) { 736 if (obj->efile.bss) 737 *size = obj->efile.bss->d_size; 738 } else if (!strcmp(name, ".rodata")) { 739 if (obj->efile.rodata) 740 *size = obj->efile.rodata->d_size; 741 } else { 742 ret = bpf_object_search_section_size(obj, name, &d_size); 743 if (!ret) 744 *size = d_size; 745 } 746 747 return *size ? 0 : ret; 748 } 749 750 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 751 __u32 *off) 752 { 753 Elf_Data *symbols = obj->efile.symbols; 754 const char *sname; 755 size_t si; 756 757 if (!name || !off) 758 return -EINVAL; 759 760 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 761 GElf_Sym sym; 762 763 if (!gelf_getsym(symbols, si, &sym)) 764 continue; 765 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 766 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 767 continue; 768 769 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 770 sym.st_name); 771 if (!sname) { 772 pr_warning("failed to get sym name string for var %s\n", 773 name); 774 return -EIO; 775 } 776 if (strcmp(name, sname) == 0) { 777 *off = sym.st_value; 778 return 0; 779 } 780 } 781 782 return -ENOENT; 783 } 784 785 static bool bpf_object__has_maps(const struct bpf_object *obj) 786 { 787 return obj->efile.maps_shndx >= 0 || 788 obj->efile.data_shndx >= 0 || 789 obj->efile.rodata_shndx >= 0 || 790 obj->efile.bss_shndx >= 0; 791 } 792 793 static int 794 bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, 795 enum libbpf_map_type type, Elf_Data *data, 796 void **data_buff) 797 { 798 struct bpf_map_def *def = &map->def; 799 char map_name[BPF_OBJ_NAME_LEN]; 800 801 map->libbpf_type = type; 802 map->offset = ~(typeof(map->offset))0; 803 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 804 libbpf_type_to_btf_name[type]); 805 map->name = strdup(map_name); 806 if (!map->name) { 807 pr_warning("failed to alloc map name\n"); 808 return -ENOMEM; 809 } 810 811 def->type = BPF_MAP_TYPE_ARRAY; 812 def->key_size = sizeof(int); 813 def->value_size = data->d_size; 814 def->max_entries = 1; 815 def->map_flags = type == LIBBPF_MAP_RODATA ? 816 BPF_F_RDONLY_PROG : 0; 817 if (data_buff) { 818 *data_buff = malloc(data->d_size); 819 if (!*data_buff) { 820 zfree(&map->name); 821 pr_warning("failed to alloc map content buffer\n"); 822 return -ENOMEM; 823 } 824 memcpy(*data_buff, data->d_buf, data->d_size); 825 } 826 827 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 828 return 0; 829 } 830 831 static int 832 bpf_object__init_maps(struct bpf_object *obj, int flags) 833 { 834 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; 835 bool strict = !(flags & MAPS_RELAX_COMPAT); 836 Elf_Data *symbols = obj->efile.symbols; 837 Elf_Data *data = NULL; 838 int ret = 0; 839 840 if (!symbols) 841 return -EINVAL; 842 nr_syms = symbols->d_size / sizeof(GElf_Sym); 843 844 if (obj->efile.maps_shndx >= 0) { 845 Elf_Scn *scn = elf_getscn(obj->efile.elf, 846 obj->efile.maps_shndx); 847 848 if (scn) 849 data = elf_getdata(scn, NULL); 850 if (!scn || !data) { 851 pr_warning("failed to get Elf_Data from map section %d\n", 852 obj->efile.maps_shndx); 853 return -EINVAL; 854 } 855 } 856 857 /* 858 * Count number of maps. Each map has a name. 859 * Array of maps is not supported: only the first element is 860 * considered. 861 * 862 * TODO: Detect array of map and report error. 863 */ 864 if (obj->caps.global_data) { 865 if (obj->efile.data_shndx >= 0) 866 nr_maps_glob++; 867 if (obj->efile.rodata_shndx >= 0) 868 nr_maps_glob++; 869 if (obj->efile.bss_shndx >= 0) 870 nr_maps_glob++; 871 } 872 873 for (i = 0; data && i < nr_syms; i++) { 874 GElf_Sym sym; 875 876 if (!gelf_getsym(symbols, i, &sym)) 877 continue; 878 if (sym.st_shndx != obj->efile.maps_shndx) 879 continue; 880 nr_maps++; 881 } 882 883 if (!nr_maps && !nr_maps_glob) 884 return 0; 885 886 /* Assume equally sized map definitions */ 887 if (data) { 888 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 889 nr_maps, data->d_size); 890 891 map_def_sz = data->d_size / nr_maps; 892 if (!data->d_size || (data->d_size % nr_maps) != 0) { 893 pr_warning("unable to determine map definition size " 894 "section %s, %d maps in %zd bytes\n", 895 obj->path, nr_maps, data->d_size); 896 return -EINVAL; 897 } 898 } 899 900 nr_maps += nr_maps_glob; 901 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 902 if (!obj->maps) { 903 pr_warning("alloc maps for object failed\n"); 904 return -ENOMEM; 905 } 906 obj->nr_maps = nr_maps; 907 908 for (i = 0; i < nr_maps; i++) { 909 /* 910 * fill all fd with -1 so won't close incorrect 911 * fd (fd=0 is stdin) when failure (zclose won't close 912 * negative fd)). 913 */ 914 obj->maps[i].fd = -1; 915 obj->maps[i].inner_map_fd = -1; 916 } 917 918 /* 919 * Fill obj->maps using data in "maps" section. 920 */ 921 for (i = 0, map_idx = 0; data && i < nr_syms; i++) { 922 GElf_Sym sym; 923 const char *map_name; 924 struct bpf_map_def *def; 925 926 if (!gelf_getsym(symbols, i, &sym)) 927 continue; 928 if (sym.st_shndx != obj->efile.maps_shndx) 929 continue; 930 931 map_name = elf_strptr(obj->efile.elf, 932 obj->efile.strtabidx, 933 sym.st_name); 934 935 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; 936 obj->maps[map_idx].offset = sym.st_value; 937 if (sym.st_value + map_def_sz > data->d_size) { 938 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 939 obj->path, map_name); 940 return -EINVAL; 941 } 942 943 obj->maps[map_idx].name = strdup(map_name); 944 if (!obj->maps[map_idx].name) { 945 pr_warning("failed to alloc map name\n"); 946 return -ENOMEM; 947 } 948 pr_debug("map %d is \"%s\"\n", map_idx, 949 obj->maps[map_idx].name); 950 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 951 /* 952 * If the definition of the map in the object file fits in 953 * bpf_map_def, copy it. Any extra fields in our version 954 * of bpf_map_def will default to zero as a result of the 955 * calloc above. 956 */ 957 if (map_def_sz <= sizeof(struct bpf_map_def)) { 958 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 959 } else { 960 /* 961 * Here the map structure being read is bigger than what 962 * we expect, truncate if the excess bits are all zero. 963 * If they are not zero, reject this map as 964 * incompatible. 965 */ 966 char *b; 967 for (b = ((char *)def) + sizeof(struct bpf_map_def); 968 b < ((char *)def) + map_def_sz; b++) { 969 if (*b != 0) { 970 pr_warning("maps section in %s: \"%s\" " 971 "has unrecognized, non-zero " 972 "options\n", 973 obj->path, map_name); 974 if (strict) 975 return -EINVAL; 976 } 977 } 978 memcpy(&obj->maps[map_idx].def, def, 979 sizeof(struct bpf_map_def)); 980 } 981 map_idx++; 982 } 983 984 if (!obj->caps.global_data) 985 goto finalize; 986 987 /* 988 * Populate rest of obj->maps with libbpf internal maps. 989 */ 990 if (obj->efile.data_shndx >= 0) 991 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 992 LIBBPF_MAP_DATA, 993 obj->efile.data, 994 &obj->sections.data); 995 if (!ret && obj->efile.rodata_shndx >= 0) 996 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 997 LIBBPF_MAP_RODATA, 998 obj->efile.rodata, 999 &obj->sections.rodata); 1000 if (!ret && obj->efile.bss_shndx >= 0) 1001 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 1002 LIBBPF_MAP_BSS, 1003 obj->efile.bss, NULL); 1004 finalize: 1005 if (!ret) 1006 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1007 compare_bpf_map); 1008 return ret; 1009 } 1010 1011 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1012 { 1013 Elf_Scn *scn; 1014 GElf_Shdr sh; 1015 1016 scn = elf_getscn(obj->efile.elf, idx); 1017 if (!scn) 1018 return false; 1019 1020 if (gelf_getshdr(scn, &sh) != &sh) 1021 return false; 1022 1023 if (sh.sh_flags & SHF_EXECINSTR) 1024 return true; 1025 1026 return false; 1027 } 1028 1029 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1030 { 1031 bool has_datasec = obj->caps.btf_datasec; 1032 bool has_func = obj->caps.btf_func; 1033 struct btf *btf = obj->btf; 1034 struct btf_type *t; 1035 int i, j, vlen; 1036 __u16 kind; 1037 1038 if (!obj->btf || (has_func && has_datasec)) 1039 return; 1040 1041 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1042 t = (struct btf_type *)btf__type_by_id(btf, i); 1043 kind = BTF_INFO_KIND(t->info); 1044 1045 if (!has_datasec && kind == BTF_KIND_VAR) { 1046 /* replace VAR with INT */ 1047 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1048 t->size = sizeof(int); 1049 *(int *)(t+1) = BTF_INT_ENC(0, 0, 32); 1050 } else if (!has_datasec && kind == BTF_KIND_DATASEC) { 1051 /* replace DATASEC with STRUCT */ 1052 struct btf_var_secinfo *v = (void *)(t + 1); 1053 struct btf_member *m = (void *)(t + 1); 1054 struct btf_type *vt; 1055 char *name; 1056 1057 name = (char *)btf__name_by_offset(btf, t->name_off); 1058 while (*name) { 1059 if (*name == '.') 1060 *name = '_'; 1061 name++; 1062 } 1063 1064 vlen = BTF_INFO_VLEN(t->info); 1065 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1066 for (j = 0; j < vlen; j++, v++, m++) { 1067 /* order of field assignments is important */ 1068 m->offset = v->offset * 8; 1069 m->type = v->type; 1070 /* preserve variable name as member name */ 1071 vt = (void *)btf__type_by_id(btf, v->type); 1072 m->name_off = vt->name_off; 1073 } 1074 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) { 1075 /* replace FUNC_PROTO with ENUM */ 1076 vlen = BTF_INFO_VLEN(t->info); 1077 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1078 t->size = sizeof(__u32); /* kernel enforced */ 1079 } else if (!has_func && kind == BTF_KIND_FUNC) { 1080 /* replace FUNC with TYPEDEF */ 1081 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1082 } 1083 } 1084 } 1085 1086 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1087 { 1088 if (!obj->btf_ext) 1089 return; 1090 1091 if (!obj->caps.btf_func) { 1092 btf_ext__free(obj->btf_ext); 1093 obj->btf_ext = NULL; 1094 } 1095 } 1096 1097 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1098 { 1099 Elf *elf = obj->efile.elf; 1100 GElf_Ehdr *ep = &obj->efile.ehdr; 1101 Elf_Data *btf_ext_data = NULL; 1102 Elf_Data *btf_data = NULL; 1103 Elf_Scn *scn = NULL; 1104 int idx = 0, err = 0; 1105 1106 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1107 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1108 pr_warning("failed to get e_shstrndx from %s\n", 1109 obj->path); 1110 return -LIBBPF_ERRNO__FORMAT; 1111 } 1112 1113 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1114 char *name; 1115 GElf_Shdr sh; 1116 Elf_Data *data; 1117 1118 idx++; 1119 if (gelf_getshdr(scn, &sh) != &sh) { 1120 pr_warning("failed to get section(%d) header from %s\n", 1121 idx, obj->path); 1122 err = -LIBBPF_ERRNO__FORMAT; 1123 goto out; 1124 } 1125 1126 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1127 if (!name) { 1128 pr_warning("failed to get section(%d) name from %s\n", 1129 idx, obj->path); 1130 err = -LIBBPF_ERRNO__FORMAT; 1131 goto out; 1132 } 1133 1134 data = elf_getdata(scn, 0); 1135 if (!data) { 1136 pr_warning("failed to get section(%d) data from %s(%s)\n", 1137 idx, name, obj->path); 1138 err = -LIBBPF_ERRNO__FORMAT; 1139 goto out; 1140 } 1141 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1142 idx, name, (unsigned long)data->d_size, 1143 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1144 (int)sh.sh_type); 1145 1146 if (strcmp(name, "license") == 0) { 1147 err = bpf_object__init_license(obj, 1148 data->d_buf, 1149 data->d_size); 1150 } else if (strcmp(name, "version") == 0) { 1151 err = bpf_object__init_kversion(obj, 1152 data->d_buf, 1153 data->d_size); 1154 } else if (strcmp(name, "maps") == 0) { 1155 obj->efile.maps_shndx = idx; 1156 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1157 btf_data = data; 1158 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1159 btf_ext_data = data; 1160 } else if (sh.sh_type == SHT_SYMTAB) { 1161 if (obj->efile.symbols) { 1162 pr_warning("bpf: multiple SYMTAB in %s\n", 1163 obj->path); 1164 err = -LIBBPF_ERRNO__FORMAT; 1165 } else { 1166 obj->efile.symbols = data; 1167 obj->efile.strtabidx = sh.sh_link; 1168 } 1169 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1170 if (sh.sh_flags & SHF_EXECINSTR) { 1171 if (strcmp(name, ".text") == 0) 1172 obj->efile.text_shndx = idx; 1173 err = bpf_object__add_program(obj, data->d_buf, 1174 data->d_size, name, idx); 1175 if (err) { 1176 char errmsg[STRERR_BUFSIZE]; 1177 char *cp = libbpf_strerror_r(-err, errmsg, 1178 sizeof(errmsg)); 1179 1180 pr_warning("failed to alloc program %s (%s): %s", 1181 name, obj->path, cp); 1182 } 1183 } else if (strcmp(name, ".data") == 0) { 1184 obj->efile.data = data; 1185 obj->efile.data_shndx = idx; 1186 } else if (strcmp(name, ".rodata") == 0) { 1187 obj->efile.rodata = data; 1188 obj->efile.rodata_shndx = idx; 1189 } else { 1190 pr_debug("skip section(%d) %s\n", idx, name); 1191 } 1192 } else if (sh.sh_type == SHT_REL) { 1193 void *reloc = obj->efile.reloc; 1194 int nr_reloc = obj->efile.nr_reloc + 1; 1195 int sec = sh.sh_info; /* points to other section */ 1196 1197 /* Only do relo for section with exec instructions */ 1198 if (!section_have_execinstr(obj, sec)) { 1199 pr_debug("skip relo %s(%d) for section(%d)\n", 1200 name, idx, sec); 1201 continue; 1202 } 1203 1204 reloc = reallocarray(reloc, nr_reloc, 1205 sizeof(*obj->efile.reloc)); 1206 if (!reloc) { 1207 pr_warning("realloc failed\n"); 1208 err = -ENOMEM; 1209 } else { 1210 int n = nr_reloc - 1; 1211 1212 obj->efile.reloc = reloc; 1213 obj->efile.nr_reloc = nr_reloc; 1214 1215 obj->efile.reloc[n].shdr = sh; 1216 obj->efile.reloc[n].data = data; 1217 } 1218 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1219 obj->efile.bss = data; 1220 obj->efile.bss_shndx = idx; 1221 } else { 1222 pr_debug("skip section(%d) %s\n", idx, name); 1223 } 1224 if (err) 1225 goto out; 1226 } 1227 1228 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1229 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1230 return LIBBPF_ERRNO__FORMAT; 1231 } 1232 if (btf_data) { 1233 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1234 if (IS_ERR(obj->btf)) { 1235 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1236 BTF_ELF_SEC, PTR_ERR(obj->btf)); 1237 obj->btf = NULL; 1238 } else { 1239 err = btf__finalize_data(obj, obj->btf); 1240 if (!err) { 1241 bpf_object__sanitize_btf(obj); 1242 err = btf__load(obj->btf); 1243 } 1244 if (err) { 1245 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", 1246 BTF_ELF_SEC, err); 1247 btf__free(obj->btf); 1248 obj->btf = NULL; 1249 err = 0; 1250 } 1251 } 1252 } 1253 if (btf_ext_data) { 1254 if (!obj->btf) { 1255 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1256 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1257 } else { 1258 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1259 btf_ext_data->d_size); 1260 if (IS_ERR(obj->btf_ext)) { 1261 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1262 BTF_EXT_ELF_SEC, 1263 PTR_ERR(obj->btf_ext)); 1264 obj->btf_ext = NULL; 1265 } else { 1266 bpf_object__sanitize_btf_ext(obj); 1267 } 1268 } 1269 } 1270 if (bpf_object__has_maps(obj)) { 1271 err = bpf_object__init_maps(obj, flags); 1272 if (err) 1273 goto out; 1274 } 1275 err = bpf_object__init_prog_names(obj); 1276 out: 1277 return err; 1278 } 1279 1280 static struct bpf_program * 1281 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1282 { 1283 struct bpf_program *prog; 1284 size_t i; 1285 1286 for (i = 0; i < obj->nr_programs; i++) { 1287 prog = &obj->programs[i]; 1288 if (prog->idx == idx) 1289 return prog; 1290 } 1291 return NULL; 1292 } 1293 1294 struct bpf_program * 1295 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 1296 { 1297 struct bpf_program *pos; 1298 1299 bpf_object__for_each_program(pos, obj) { 1300 if (pos->section_name && !strcmp(pos->section_name, title)) 1301 return pos; 1302 } 1303 return NULL; 1304 } 1305 1306 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1307 int shndx) 1308 { 1309 return shndx == obj->efile.data_shndx || 1310 shndx == obj->efile.bss_shndx || 1311 shndx == obj->efile.rodata_shndx; 1312 } 1313 1314 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1315 int shndx) 1316 { 1317 return shndx == obj->efile.maps_shndx; 1318 } 1319 1320 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1321 int shndx) 1322 { 1323 return shndx == obj->efile.text_shndx || 1324 bpf_object__shndx_is_maps(obj, shndx) || 1325 bpf_object__shndx_is_data(obj, shndx); 1326 } 1327 1328 static enum libbpf_map_type 1329 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1330 { 1331 if (shndx == obj->efile.data_shndx) 1332 return LIBBPF_MAP_DATA; 1333 else if (shndx == obj->efile.bss_shndx) 1334 return LIBBPF_MAP_BSS; 1335 else if (shndx == obj->efile.rodata_shndx) 1336 return LIBBPF_MAP_RODATA; 1337 else 1338 return LIBBPF_MAP_UNSPEC; 1339 } 1340 1341 static int 1342 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1343 Elf_Data *data, struct bpf_object *obj) 1344 { 1345 Elf_Data *symbols = obj->efile.symbols; 1346 struct bpf_map *maps = obj->maps; 1347 size_t nr_maps = obj->nr_maps; 1348 int i, nrels; 1349 1350 pr_debug("collecting relocating info for: '%s'\n", 1351 prog->section_name); 1352 nrels = shdr->sh_size / shdr->sh_entsize; 1353 1354 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1355 if (!prog->reloc_desc) { 1356 pr_warning("failed to alloc memory in relocation\n"); 1357 return -ENOMEM; 1358 } 1359 prog->nr_reloc = nrels; 1360 1361 for (i = 0; i < nrels; i++) { 1362 GElf_Sym sym; 1363 GElf_Rel rel; 1364 unsigned int insn_idx; 1365 unsigned int shdr_idx; 1366 struct bpf_insn *insns = prog->insns; 1367 enum libbpf_map_type type; 1368 const char *name; 1369 size_t map_idx; 1370 1371 if (!gelf_getrel(data, i, &rel)) { 1372 pr_warning("relocation: failed to get %d reloc\n", i); 1373 return -LIBBPF_ERRNO__FORMAT; 1374 } 1375 1376 if (!gelf_getsym(symbols, 1377 GELF_R_SYM(rel.r_info), 1378 &sym)) { 1379 pr_warning("relocation: symbol %"PRIx64" not found\n", 1380 GELF_R_SYM(rel.r_info)); 1381 return -LIBBPF_ERRNO__FORMAT; 1382 } 1383 1384 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1385 sym.st_name) ? : "<?>"; 1386 1387 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1388 (long long) (rel.r_info >> 32), 1389 (long long) sym.st_value, sym.st_name, name); 1390 1391 shdr_idx = sym.st_shndx; 1392 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1393 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1394 prog->section_name, shdr_idx); 1395 return -LIBBPF_ERRNO__RELOC; 1396 } 1397 1398 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1399 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1400 1401 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1402 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1403 pr_warning("incorrect bpf_call opcode\n"); 1404 return -LIBBPF_ERRNO__RELOC; 1405 } 1406 prog->reloc_desc[i].type = RELO_CALL; 1407 prog->reloc_desc[i].insn_idx = insn_idx; 1408 prog->reloc_desc[i].text_off = sym.st_value; 1409 obj->has_pseudo_calls = true; 1410 continue; 1411 } 1412 1413 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1414 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1415 insn_idx, insns[insn_idx].code); 1416 return -LIBBPF_ERRNO__RELOC; 1417 } 1418 1419 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1420 bpf_object__shndx_is_data(obj, shdr_idx)) { 1421 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1422 if (type != LIBBPF_MAP_UNSPEC) { 1423 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1424 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1425 name, insn_idx, insns[insn_idx].code); 1426 return -LIBBPF_ERRNO__RELOC; 1427 } 1428 if (!obj->caps.global_data) { 1429 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1430 name, insn_idx); 1431 return -LIBBPF_ERRNO__RELOC; 1432 } 1433 } 1434 1435 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1436 if (maps[map_idx].libbpf_type != type) 1437 continue; 1438 if (type != LIBBPF_MAP_UNSPEC || 1439 (type == LIBBPF_MAP_UNSPEC && 1440 maps[map_idx].offset == sym.st_value)) { 1441 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1442 map_idx, maps[map_idx].name, insn_idx); 1443 break; 1444 } 1445 } 1446 1447 if (map_idx >= nr_maps) { 1448 pr_warning("bpf relocation: map_idx %d large than %d\n", 1449 (int)map_idx, (int)nr_maps - 1); 1450 return -LIBBPF_ERRNO__RELOC; 1451 } 1452 1453 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1454 RELO_DATA : RELO_LD64; 1455 prog->reloc_desc[i].insn_idx = insn_idx; 1456 prog->reloc_desc[i].map_idx = map_idx; 1457 } 1458 } 1459 return 0; 1460 } 1461 1462 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1463 { 1464 struct bpf_map_def *def = &map->def; 1465 __u32 key_type_id = 0, value_type_id = 0; 1466 int ret; 1467 1468 if (!bpf_map__is_internal(map)) { 1469 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1470 def->value_size, &key_type_id, 1471 &value_type_id); 1472 } else { 1473 /* 1474 * LLVM annotates global data differently in BTF, that is, 1475 * only as '.data', '.bss' or '.rodata'. 1476 */ 1477 ret = btf__find_by_name(btf, 1478 libbpf_type_to_btf_name[map->libbpf_type]); 1479 } 1480 if (ret < 0) 1481 return ret; 1482 1483 map->btf_key_type_id = key_type_id; 1484 map->btf_value_type_id = bpf_map__is_internal(map) ? 1485 ret : value_type_id; 1486 return 0; 1487 } 1488 1489 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1490 { 1491 struct bpf_map_info info = {}; 1492 __u32 len = sizeof(info); 1493 int new_fd, err; 1494 char *new_name; 1495 1496 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1497 if (err) 1498 return err; 1499 1500 new_name = strdup(info.name); 1501 if (!new_name) 1502 return -errno; 1503 1504 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1505 if (new_fd < 0) 1506 goto err_free_new_name; 1507 1508 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1509 if (new_fd < 0) 1510 goto err_close_new_fd; 1511 1512 err = zclose(map->fd); 1513 if (err) 1514 goto err_close_new_fd; 1515 free(map->name); 1516 1517 map->fd = new_fd; 1518 map->name = new_name; 1519 map->def.type = info.type; 1520 map->def.key_size = info.key_size; 1521 map->def.value_size = info.value_size; 1522 map->def.max_entries = info.max_entries; 1523 map->def.map_flags = info.map_flags; 1524 map->btf_key_type_id = info.btf_key_type_id; 1525 map->btf_value_type_id = info.btf_value_type_id; 1526 1527 return 0; 1528 1529 err_close_new_fd: 1530 close(new_fd); 1531 err_free_new_name: 1532 free(new_name); 1533 return -errno; 1534 } 1535 1536 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1537 { 1538 if (!map || !max_entries) 1539 return -EINVAL; 1540 1541 /* If map already created, its attributes can't be changed. */ 1542 if (map->fd >= 0) 1543 return -EBUSY; 1544 1545 map->def.max_entries = max_entries; 1546 1547 return 0; 1548 } 1549 1550 static int 1551 bpf_object__probe_name(struct bpf_object *obj) 1552 { 1553 struct bpf_load_program_attr attr; 1554 char *cp, errmsg[STRERR_BUFSIZE]; 1555 struct bpf_insn insns[] = { 1556 BPF_MOV64_IMM(BPF_REG_0, 0), 1557 BPF_EXIT_INSN(), 1558 }; 1559 int ret; 1560 1561 /* make sure basic loading works */ 1562 1563 memset(&attr, 0, sizeof(attr)); 1564 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1565 attr.insns = insns; 1566 attr.insns_cnt = ARRAY_SIZE(insns); 1567 attr.license = "GPL"; 1568 1569 ret = bpf_load_program_xattr(&attr, NULL, 0); 1570 if (ret < 0) { 1571 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1572 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1573 __func__, cp, errno); 1574 return -errno; 1575 } 1576 close(ret); 1577 1578 /* now try the same program, but with the name */ 1579 1580 attr.name = "test"; 1581 ret = bpf_load_program_xattr(&attr, NULL, 0); 1582 if (ret >= 0) { 1583 obj->caps.name = 1; 1584 close(ret); 1585 } 1586 1587 return 0; 1588 } 1589 1590 static int 1591 bpf_object__probe_global_data(struct bpf_object *obj) 1592 { 1593 struct bpf_load_program_attr prg_attr; 1594 struct bpf_create_map_attr map_attr; 1595 char *cp, errmsg[STRERR_BUFSIZE]; 1596 struct bpf_insn insns[] = { 1597 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1598 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1599 BPF_MOV64_IMM(BPF_REG_0, 0), 1600 BPF_EXIT_INSN(), 1601 }; 1602 int ret, map; 1603 1604 memset(&map_attr, 0, sizeof(map_attr)); 1605 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1606 map_attr.key_size = sizeof(int); 1607 map_attr.value_size = 32; 1608 map_attr.max_entries = 1; 1609 1610 map = bpf_create_map_xattr(&map_attr); 1611 if (map < 0) { 1612 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1613 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1614 __func__, cp, errno); 1615 return -errno; 1616 } 1617 1618 insns[0].imm = map; 1619 1620 memset(&prg_attr, 0, sizeof(prg_attr)); 1621 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1622 prg_attr.insns = insns; 1623 prg_attr.insns_cnt = ARRAY_SIZE(insns); 1624 prg_attr.license = "GPL"; 1625 1626 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 1627 if (ret >= 0) { 1628 obj->caps.global_data = 1; 1629 close(ret); 1630 } 1631 1632 close(map); 1633 return 0; 1634 } 1635 1636 static int bpf_object__probe_btf_func(struct bpf_object *obj) 1637 { 1638 const char strs[] = "\0int\0x\0a"; 1639 /* void x(int a) {} */ 1640 __u32 types[] = { 1641 /* int */ 1642 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1643 /* FUNC_PROTO */ /* [2] */ 1644 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 1645 BTF_PARAM_ENC(7, 1), 1646 /* FUNC x */ /* [3] */ 1647 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 1648 }; 1649 int res; 1650 1651 res = libbpf__probe_raw_btf((char *)types, sizeof(types), 1652 strs, sizeof(strs)); 1653 if (res < 0) 1654 return res; 1655 if (res > 0) 1656 obj->caps.btf_func = 1; 1657 return 0; 1658 } 1659 1660 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 1661 { 1662 const char strs[] = "\0x\0.data"; 1663 /* static int a; */ 1664 __u32 types[] = { 1665 /* int */ 1666 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1667 /* VAR x */ /* [2] */ 1668 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 1669 BTF_VAR_STATIC, 1670 /* DATASEC val */ /* [3] */ 1671 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 1672 BTF_VAR_SECINFO_ENC(2, 0, 4), 1673 }; 1674 int res; 1675 1676 res = libbpf__probe_raw_btf((char *)types, sizeof(types), 1677 strs, sizeof(strs)); 1678 if (res < 0) 1679 return res; 1680 if (res > 0) 1681 obj->caps.btf_datasec = 1; 1682 return 0; 1683 } 1684 1685 static int 1686 bpf_object__probe_caps(struct bpf_object *obj) 1687 { 1688 int (*probe_fn[])(struct bpf_object *obj) = { 1689 bpf_object__probe_name, 1690 bpf_object__probe_global_data, 1691 bpf_object__probe_btf_func, 1692 bpf_object__probe_btf_datasec, 1693 }; 1694 int i, ret; 1695 1696 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 1697 ret = probe_fn[i](obj); 1698 if (ret < 0) 1699 return ret; 1700 } 1701 1702 return 0; 1703 } 1704 1705 static int 1706 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 1707 { 1708 char *cp, errmsg[STRERR_BUFSIZE]; 1709 int err, zero = 0; 1710 __u8 *data; 1711 1712 /* Nothing to do here since kernel already zero-initializes .bss map. */ 1713 if (map->libbpf_type == LIBBPF_MAP_BSS) 1714 return 0; 1715 1716 data = map->libbpf_type == LIBBPF_MAP_DATA ? 1717 obj->sections.data : obj->sections.rodata; 1718 1719 err = bpf_map_update_elem(map->fd, &zero, data, 0); 1720 /* Freeze .rodata map as read-only from syscall side. */ 1721 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 1722 err = bpf_map_freeze(map->fd); 1723 if (err) { 1724 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1725 pr_warning("Error freezing map(%s) as read-only: %s\n", 1726 map->name, cp); 1727 err = 0; 1728 } 1729 } 1730 return err; 1731 } 1732 1733 static int 1734 bpf_object__create_maps(struct bpf_object *obj) 1735 { 1736 struct bpf_create_map_attr create_attr = {}; 1737 unsigned int i; 1738 int err; 1739 1740 for (i = 0; i < obj->nr_maps; i++) { 1741 struct bpf_map *map = &obj->maps[i]; 1742 struct bpf_map_def *def = &map->def; 1743 char *cp, errmsg[STRERR_BUFSIZE]; 1744 int *pfd = &map->fd; 1745 1746 if (map->fd >= 0) { 1747 pr_debug("skip map create (preset) %s: fd=%d\n", 1748 map->name, map->fd); 1749 continue; 1750 } 1751 1752 if (obj->caps.name) 1753 create_attr.name = map->name; 1754 create_attr.map_ifindex = map->map_ifindex; 1755 create_attr.map_type = def->type; 1756 create_attr.map_flags = def->map_flags; 1757 create_attr.key_size = def->key_size; 1758 create_attr.value_size = def->value_size; 1759 create_attr.max_entries = def->max_entries; 1760 create_attr.btf_fd = 0; 1761 create_attr.btf_key_type_id = 0; 1762 create_attr.btf_value_type_id = 0; 1763 if (bpf_map_type__is_map_in_map(def->type) && 1764 map->inner_map_fd >= 0) 1765 create_attr.inner_map_fd = map->inner_map_fd; 1766 1767 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1768 create_attr.btf_fd = btf__fd(obj->btf); 1769 create_attr.btf_key_type_id = map->btf_key_type_id; 1770 create_attr.btf_value_type_id = map->btf_value_type_id; 1771 } 1772 1773 *pfd = bpf_create_map_xattr(&create_attr); 1774 if (*pfd < 0 && create_attr.btf_key_type_id) { 1775 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1776 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1777 map->name, cp, errno); 1778 create_attr.btf_fd = 0; 1779 create_attr.btf_key_type_id = 0; 1780 create_attr.btf_value_type_id = 0; 1781 map->btf_key_type_id = 0; 1782 map->btf_value_type_id = 0; 1783 *pfd = bpf_create_map_xattr(&create_attr); 1784 } 1785 1786 if (*pfd < 0) { 1787 size_t j; 1788 1789 err = *pfd; 1790 err_out: 1791 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1792 pr_warning("failed to create map (name: '%s'): %s\n", 1793 map->name, cp); 1794 for (j = 0; j < i; j++) 1795 zclose(obj->maps[j].fd); 1796 return err; 1797 } 1798 1799 if (bpf_map__is_internal(map)) { 1800 err = bpf_object__populate_internal_map(obj, map); 1801 if (err < 0) { 1802 zclose(*pfd); 1803 goto err_out; 1804 } 1805 } 1806 1807 pr_debug("create map %s: fd=%d\n", map->name, *pfd); 1808 } 1809 1810 return 0; 1811 } 1812 1813 static int 1814 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1815 void *btf_prog_info, const char *info_name) 1816 { 1817 if (err != -ENOENT) { 1818 pr_warning("Error in loading %s for sec %s.\n", 1819 info_name, prog->section_name); 1820 return err; 1821 } 1822 1823 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1824 1825 if (btf_prog_info) { 1826 /* 1827 * Some info has already been found but has problem 1828 * in the last btf_ext reloc. Must have to error 1829 * out. 1830 */ 1831 pr_warning("Error in relocating %s for sec %s.\n", 1832 info_name, prog->section_name); 1833 return err; 1834 } 1835 1836 /* 1837 * Have problem loading the very first info. Ignore 1838 * the rest. 1839 */ 1840 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1841 info_name, prog->section_name, info_name); 1842 return 0; 1843 } 1844 1845 static int 1846 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1847 const char *section_name, __u32 insn_offset) 1848 { 1849 int err; 1850 1851 if (!insn_offset || prog->func_info) { 1852 /* 1853 * !insn_offset => main program 1854 * 1855 * For sub prog, the main program's func_info has to 1856 * be loaded first (i.e. prog->func_info != NULL) 1857 */ 1858 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1859 section_name, insn_offset, 1860 &prog->func_info, 1861 &prog->func_info_cnt); 1862 if (err) 1863 return check_btf_ext_reloc_err(prog, err, 1864 prog->func_info, 1865 "bpf_func_info"); 1866 1867 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1868 } 1869 1870 if (!insn_offset || prog->line_info) { 1871 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1872 section_name, insn_offset, 1873 &prog->line_info, 1874 &prog->line_info_cnt); 1875 if (err) 1876 return check_btf_ext_reloc_err(prog, err, 1877 prog->line_info, 1878 "bpf_line_info"); 1879 1880 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1881 } 1882 1883 if (!insn_offset) 1884 prog->btf_fd = btf__fd(obj->btf); 1885 1886 return 0; 1887 } 1888 1889 static int 1890 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1891 struct reloc_desc *relo) 1892 { 1893 struct bpf_insn *insn, *new_insn; 1894 struct bpf_program *text; 1895 size_t new_cnt; 1896 int err; 1897 1898 if (relo->type != RELO_CALL) 1899 return -LIBBPF_ERRNO__RELOC; 1900 1901 if (prog->idx == obj->efile.text_shndx) { 1902 pr_warning("relo in .text insn %d into off %d\n", 1903 relo->insn_idx, relo->text_off); 1904 return -LIBBPF_ERRNO__RELOC; 1905 } 1906 1907 if (prog->main_prog_cnt == 0) { 1908 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1909 if (!text) { 1910 pr_warning("no .text section found yet relo into text exist\n"); 1911 return -LIBBPF_ERRNO__RELOC; 1912 } 1913 new_cnt = prog->insns_cnt + text->insns_cnt; 1914 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1915 if (!new_insn) { 1916 pr_warning("oom in prog realloc\n"); 1917 return -ENOMEM; 1918 } 1919 1920 if (obj->btf_ext) { 1921 err = bpf_program_reloc_btf_ext(prog, obj, 1922 text->section_name, 1923 prog->insns_cnt); 1924 if (err) 1925 return err; 1926 } 1927 1928 memcpy(new_insn + prog->insns_cnt, text->insns, 1929 text->insns_cnt * sizeof(*insn)); 1930 prog->insns = new_insn; 1931 prog->main_prog_cnt = prog->insns_cnt; 1932 prog->insns_cnt = new_cnt; 1933 pr_debug("added %zd insn from %s to prog %s\n", 1934 text->insns_cnt, text->section_name, 1935 prog->section_name); 1936 } 1937 insn = &prog->insns[relo->insn_idx]; 1938 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1939 return 0; 1940 } 1941 1942 static int 1943 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1944 { 1945 int i, err; 1946 1947 if (!prog) 1948 return 0; 1949 1950 if (obj->btf_ext) { 1951 err = bpf_program_reloc_btf_ext(prog, obj, 1952 prog->section_name, 0); 1953 if (err) 1954 return err; 1955 } 1956 1957 if (!prog->reloc_desc) 1958 return 0; 1959 1960 for (i = 0; i < prog->nr_reloc; i++) { 1961 if (prog->reloc_desc[i].type == RELO_LD64 || 1962 prog->reloc_desc[i].type == RELO_DATA) { 1963 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 1964 struct bpf_insn *insns = prog->insns; 1965 int insn_idx, map_idx; 1966 1967 insn_idx = prog->reloc_desc[i].insn_idx; 1968 map_idx = prog->reloc_desc[i].map_idx; 1969 1970 if (insn_idx + 1 >= (int)prog->insns_cnt) { 1971 pr_warning("relocation out of range: '%s'\n", 1972 prog->section_name); 1973 return -LIBBPF_ERRNO__RELOC; 1974 } 1975 1976 if (!relo_data) { 1977 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1978 } else { 1979 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 1980 insns[insn_idx + 1].imm = insns[insn_idx].imm; 1981 } 1982 insns[insn_idx].imm = obj->maps[map_idx].fd; 1983 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1984 err = bpf_program__reloc_text(prog, obj, 1985 &prog->reloc_desc[i]); 1986 if (err) 1987 return err; 1988 } 1989 } 1990 1991 zfree(&prog->reloc_desc); 1992 prog->nr_reloc = 0; 1993 return 0; 1994 } 1995 1996 1997 static int 1998 bpf_object__relocate(struct bpf_object *obj) 1999 { 2000 struct bpf_program *prog; 2001 size_t i; 2002 int err; 2003 2004 for (i = 0; i < obj->nr_programs; i++) { 2005 prog = &obj->programs[i]; 2006 2007 err = bpf_program__relocate(prog, obj); 2008 if (err) { 2009 pr_warning("failed to relocate '%s'\n", 2010 prog->section_name); 2011 return err; 2012 } 2013 } 2014 return 0; 2015 } 2016 2017 static int bpf_object__collect_reloc(struct bpf_object *obj) 2018 { 2019 int i, err; 2020 2021 if (!obj_elf_valid(obj)) { 2022 pr_warning("Internal error: elf object is closed\n"); 2023 return -LIBBPF_ERRNO__INTERNAL; 2024 } 2025 2026 for (i = 0; i < obj->efile.nr_reloc; i++) { 2027 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2028 Elf_Data *data = obj->efile.reloc[i].data; 2029 int idx = shdr->sh_info; 2030 struct bpf_program *prog; 2031 2032 if (shdr->sh_type != SHT_REL) { 2033 pr_warning("internal error at %d\n", __LINE__); 2034 return -LIBBPF_ERRNO__INTERNAL; 2035 } 2036 2037 prog = bpf_object__find_prog_by_idx(obj, idx); 2038 if (!prog) { 2039 pr_warning("relocation failed: no section(%d)\n", idx); 2040 return -LIBBPF_ERRNO__RELOC; 2041 } 2042 2043 err = bpf_program__collect_reloc(prog, 2044 shdr, data, 2045 obj); 2046 if (err) 2047 return err; 2048 } 2049 return 0; 2050 } 2051 2052 static int 2053 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2054 char *license, __u32 kern_version, int *pfd) 2055 { 2056 struct bpf_load_program_attr load_attr; 2057 char *cp, errmsg[STRERR_BUFSIZE]; 2058 int log_buf_size = BPF_LOG_BUF_SIZE; 2059 char *log_buf; 2060 int ret; 2061 2062 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2063 load_attr.prog_type = prog->type; 2064 load_attr.expected_attach_type = prog->expected_attach_type; 2065 if (prog->caps->name) 2066 load_attr.name = prog->name; 2067 load_attr.insns = insns; 2068 load_attr.insns_cnt = insns_cnt; 2069 load_attr.license = license; 2070 load_attr.kern_version = kern_version; 2071 load_attr.prog_ifindex = prog->prog_ifindex; 2072 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 2073 load_attr.func_info = prog->func_info; 2074 load_attr.func_info_rec_size = prog->func_info_rec_size; 2075 load_attr.func_info_cnt = prog->func_info_cnt; 2076 load_attr.line_info = prog->line_info; 2077 load_attr.line_info_rec_size = prog->line_info_rec_size; 2078 load_attr.line_info_cnt = prog->line_info_cnt; 2079 load_attr.log_level = prog->log_level; 2080 if (!load_attr.insns || !load_attr.insns_cnt) 2081 return -EINVAL; 2082 2083 retry_load: 2084 log_buf = malloc(log_buf_size); 2085 if (!log_buf) 2086 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2087 2088 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2089 2090 if (ret >= 0) { 2091 if (load_attr.log_level) 2092 pr_debug("verifier log:\n%s", log_buf); 2093 *pfd = ret; 2094 ret = 0; 2095 goto out; 2096 } 2097 2098 if (errno == ENOSPC) { 2099 log_buf_size <<= 1; 2100 free(log_buf); 2101 goto retry_load; 2102 } 2103 ret = -LIBBPF_ERRNO__LOAD; 2104 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2105 pr_warning("load bpf program failed: %s\n", cp); 2106 2107 if (log_buf && log_buf[0] != '\0') { 2108 ret = -LIBBPF_ERRNO__VERIFY; 2109 pr_warning("-- BEGIN DUMP LOG ---\n"); 2110 pr_warning("\n%s\n", log_buf); 2111 pr_warning("-- END LOG --\n"); 2112 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2113 pr_warning("Program too large (%zu insns), at most %d insns\n", 2114 load_attr.insns_cnt, BPF_MAXINSNS); 2115 ret = -LIBBPF_ERRNO__PROG2BIG; 2116 } else { 2117 /* Wrong program type? */ 2118 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2119 int fd; 2120 2121 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2122 load_attr.expected_attach_type = 0; 2123 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2124 if (fd >= 0) { 2125 close(fd); 2126 ret = -LIBBPF_ERRNO__PROGTYPE; 2127 goto out; 2128 } 2129 } 2130 2131 if (log_buf) 2132 ret = -LIBBPF_ERRNO__KVER; 2133 } 2134 2135 out: 2136 free(log_buf); 2137 return ret; 2138 } 2139 2140 int 2141 bpf_program__load(struct bpf_program *prog, 2142 char *license, __u32 kern_version) 2143 { 2144 int err = 0, fd, i; 2145 2146 if (prog->instances.nr < 0 || !prog->instances.fds) { 2147 if (prog->preprocessor) { 2148 pr_warning("Internal error: can't load program '%s'\n", 2149 prog->section_name); 2150 return -LIBBPF_ERRNO__INTERNAL; 2151 } 2152 2153 prog->instances.fds = malloc(sizeof(int)); 2154 if (!prog->instances.fds) { 2155 pr_warning("Not enough memory for BPF fds\n"); 2156 return -ENOMEM; 2157 } 2158 prog->instances.nr = 1; 2159 prog->instances.fds[0] = -1; 2160 } 2161 2162 if (!prog->preprocessor) { 2163 if (prog->instances.nr != 1) { 2164 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2165 prog->section_name, prog->instances.nr); 2166 } 2167 err = load_program(prog, prog->insns, prog->insns_cnt, 2168 license, kern_version, &fd); 2169 if (!err) 2170 prog->instances.fds[0] = fd; 2171 goto out; 2172 } 2173 2174 for (i = 0; i < prog->instances.nr; i++) { 2175 struct bpf_prog_prep_result result; 2176 bpf_program_prep_t preprocessor = prog->preprocessor; 2177 2178 memset(&result, 0, sizeof(result)); 2179 err = preprocessor(prog, i, prog->insns, 2180 prog->insns_cnt, &result); 2181 if (err) { 2182 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2183 i, prog->section_name); 2184 goto out; 2185 } 2186 2187 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2188 pr_debug("Skip loading the %dth instance of program '%s'\n", 2189 i, prog->section_name); 2190 prog->instances.fds[i] = -1; 2191 if (result.pfd) 2192 *result.pfd = -1; 2193 continue; 2194 } 2195 2196 err = load_program(prog, result.new_insn_ptr, 2197 result.new_insn_cnt, 2198 license, kern_version, &fd); 2199 2200 if (err) { 2201 pr_warning("Loading the %dth instance of program '%s' failed\n", 2202 i, prog->section_name); 2203 goto out; 2204 } 2205 2206 if (result.pfd) 2207 *result.pfd = fd; 2208 prog->instances.fds[i] = fd; 2209 } 2210 out: 2211 if (err) 2212 pr_warning("failed to load program '%s'\n", 2213 prog->section_name); 2214 zfree(&prog->insns); 2215 prog->insns_cnt = 0; 2216 return err; 2217 } 2218 2219 static bool bpf_program__is_function_storage(struct bpf_program *prog, 2220 struct bpf_object *obj) 2221 { 2222 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2223 } 2224 2225 static int 2226 bpf_object__load_progs(struct bpf_object *obj) 2227 { 2228 size_t i; 2229 int err; 2230 2231 for (i = 0; i < obj->nr_programs; i++) { 2232 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2233 continue; 2234 err = bpf_program__load(&obj->programs[i], 2235 obj->license, 2236 obj->kern_version); 2237 if (err) 2238 return err; 2239 } 2240 return 0; 2241 } 2242 2243 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2244 { 2245 switch (type) { 2246 case BPF_PROG_TYPE_SOCKET_FILTER: 2247 case BPF_PROG_TYPE_SCHED_CLS: 2248 case BPF_PROG_TYPE_SCHED_ACT: 2249 case BPF_PROG_TYPE_XDP: 2250 case BPF_PROG_TYPE_CGROUP_SKB: 2251 case BPF_PROG_TYPE_CGROUP_SOCK: 2252 case BPF_PROG_TYPE_LWT_IN: 2253 case BPF_PROG_TYPE_LWT_OUT: 2254 case BPF_PROG_TYPE_LWT_XMIT: 2255 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2256 case BPF_PROG_TYPE_SOCK_OPS: 2257 case BPF_PROG_TYPE_SK_SKB: 2258 case BPF_PROG_TYPE_CGROUP_DEVICE: 2259 case BPF_PROG_TYPE_SK_MSG: 2260 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2261 case BPF_PROG_TYPE_LIRC_MODE2: 2262 case BPF_PROG_TYPE_SK_REUSEPORT: 2263 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2264 case BPF_PROG_TYPE_UNSPEC: 2265 case BPF_PROG_TYPE_TRACEPOINT: 2266 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2267 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2268 case BPF_PROG_TYPE_PERF_EVENT: 2269 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2270 return false; 2271 case BPF_PROG_TYPE_KPROBE: 2272 default: 2273 return true; 2274 } 2275 } 2276 2277 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2278 { 2279 if (needs_kver && obj->kern_version == 0) { 2280 pr_warning("%s doesn't provide kernel version\n", 2281 obj->path); 2282 return -LIBBPF_ERRNO__KVERSION; 2283 } 2284 return 0; 2285 } 2286 2287 static struct bpf_object * 2288 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2289 bool needs_kver, int flags) 2290 { 2291 struct bpf_object *obj; 2292 int err; 2293 2294 if (elf_version(EV_CURRENT) == EV_NONE) { 2295 pr_warning("failed to init libelf for %s\n", path); 2296 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2297 } 2298 2299 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2300 if (IS_ERR(obj)) 2301 return obj; 2302 2303 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2304 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2305 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2306 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2307 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2308 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2309 2310 bpf_object__elf_finish(obj); 2311 return obj; 2312 out: 2313 bpf_object__close(obj); 2314 return ERR_PTR(err); 2315 } 2316 2317 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2318 int flags) 2319 { 2320 /* param validation */ 2321 if (!attr->file) 2322 return NULL; 2323 2324 pr_debug("loading %s\n", attr->file); 2325 2326 return __bpf_object__open(attr->file, NULL, 0, 2327 bpf_prog_type__needs_kver(attr->prog_type), 2328 flags); 2329 } 2330 2331 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2332 { 2333 return __bpf_object__open_xattr(attr, 0); 2334 } 2335 2336 struct bpf_object *bpf_object__open(const char *path) 2337 { 2338 struct bpf_object_open_attr attr = { 2339 .file = path, 2340 .prog_type = BPF_PROG_TYPE_UNSPEC, 2341 }; 2342 2343 return bpf_object__open_xattr(&attr); 2344 } 2345 2346 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2347 size_t obj_buf_sz, 2348 const char *name) 2349 { 2350 char tmp_name[64]; 2351 2352 /* param validation */ 2353 if (!obj_buf || obj_buf_sz <= 0) 2354 return NULL; 2355 2356 if (!name) { 2357 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2358 (unsigned long)obj_buf, 2359 (unsigned long)obj_buf_sz); 2360 tmp_name[sizeof(tmp_name) - 1] = '\0'; 2361 name = tmp_name; 2362 } 2363 pr_debug("loading object '%s' from buffer\n", 2364 name); 2365 2366 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2367 } 2368 2369 int bpf_object__unload(struct bpf_object *obj) 2370 { 2371 size_t i; 2372 2373 if (!obj) 2374 return -EINVAL; 2375 2376 for (i = 0; i < obj->nr_maps; i++) 2377 zclose(obj->maps[i].fd); 2378 2379 for (i = 0; i < obj->nr_programs; i++) 2380 bpf_program__unload(&obj->programs[i]); 2381 2382 return 0; 2383 } 2384 2385 int bpf_object__load(struct bpf_object *obj) 2386 { 2387 int err; 2388 2389 if (!obj) 2390 return -EINVAL; 2391 2392 if (obj->loaded) { 2393 pr_warning("object should not be loaded twice\n"); 2394 return -EINVAL; 2395 } 2396 2397 obj->loaded = true; 2398 2399 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2400 CHECK_ERR(bpf_object__relocate(obj), err, out); 2401 CHECK_ERR(bpf_object__load_progs(obj), err, out); 2402 2403 return 0; 2404 out: 2405 bpf_object__unload(obj); 2406 pr_warning("failed to load object '%s'\n", obj->path); 2407 return err; 2408 } 2409 2410 static int check_path(const char *path) 2411 { 2412 char *cp, errmsg[STRERR_BUFSIZE]; 2413 struct statfs st_fs; 2414 char *dname, *dir; 2415 int err = 0; 2416 2417 if (path == NULL) 2418 return -EINVAL; 2419 2420 dname = strdup(path); 2421 if (dname == NULL) 2422 return -ENOMEM; 2423 2424 dir = dirname(dname); 2425 if (statfs(dir, &st_fs)) { 2426 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2427 pr_warning("failed to statfs %s: %s\n", dir, cp); 2428 err = -errno; 2429 } 2430 free(dname); 2431 2432 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2433 pr_warning("specified path %s is not on BPF FS\n", path); 2434 err = -EINVAL; 2435 } 2436 2437 return err; 2438 } 2439 2440 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2441 int instance) 2442 { 2443 char *cp, errmsg[STRERR_BUFSIZE]; 2444 int err; 2445 2446 err = check_path(path); 2447 if (err) 2448 return err; 2449 2450 if (prog == NULL) { 2451 pr_warning("invalid program pointer\n"); 2452 return -EINVAL; 2453 } 2454 2455 if (instance < 0 || instance >= prog->instances.nr) { 2456 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2457 instance, prog->section_name, prog->instances.nr); 2458 return -EINVAL; 2459 } 2460 2461 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2462 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2463 pr_warning("failed to pin program: %s\n", cp); 2464 return -errno; 2465 } 2466 pr_debug("pinned program '%s'\n", path); 2467 2468 return 0; 2469 } 2470 2471 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2472 int instance) 2473 { 2474 int err; 2475 2476 err = check_path(path); 2477 if (err) 2478 return err; 2479 2480 if (prog == NULL) { 2481 pr_warning("invalid program pointer\n"); 2482 return -EINVAL; 2483 } 2484 2485 if (instance < 0 || instance >= prog->instances.nr) { 2486 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2487 instance, prog->section_name, prog->instances.nr); 2488 return -EINVAL; 2489 } 2490 2491 err = unlink(path); 2492 if (err != 0) 2493 return -errno; 2494 pr_debug("unpinned program '%s'\n", path); 2495 2496 return 0; 2497 } 2498 2499 static int make_dir(const char *path) 2500 { 2501 char *cp, errmsg[STRERR_BUFSIZE]; 2502 int err = 0; 2503 2504 if (mkdir(path, 0700) && errno != EEXIST) 2505 err = -errno; 2506 2507 if (err) { 2508 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2509 pr_warning("failed to mkdir %s: %s\n", path, cp); 2510 } 2511 return err; 2512 } 2513 2514 int bpf_program__pin(struct bpf_program *prog, const char *path) 2515 { 2516 int i, err; 2517 2518 err = check_path(path); 2519 if (err) 2520 return err; 2521 2522 if (prog == NULL) { 2523 pr_warning("invalid program pointer\n"); 2524 return -EINVAL; 2525 } 2526 2527 if (prog->instances.nr <= 0) { 2528 pr_warning("no instances of prog %s to pin\n", 2529 prog->section_name); 2530 return -EINVAL; 2531 } 2532 2533 if (prog->instances.nr == 1) { 2534 /* don't create subdirs when pinning single instance */ 2535 return bpf_program__pin_instance(prog, path, 0); 2536 } 2537 2538 err = make_dir(path); 2539 if (err) 2540 return err; 2541 2542 for (i = 0; i < prog->instances.nr; i++) { 2543 char buf[PATH_MAX]; 2544 int len; 2545 2546 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2547 if (len < 0) { 2548 err = -EINVAL; 2549 goto err_unpin; 2550 } else if (len >= PATH_MAX) { 2551 err = -ENAMETOOLONG; 2552 goto err_unpin; 2553 } 2554 2555 err = bpf_program__pin_instance(prog, buf, i); 2556 if (err) 2557 goto err_unpin; 2558 } 2559 2560 return 0; 2561 2562 err_unpin: 2563 for (i = i - 1; i >= 0; i--) { 2564 char buf[PATH_MAX]; 2565 int len; 2566 2567 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2568 if (len < 0) 2569 continue; 2570 else if (len >= PATH_MAX) 2571 continue; 2572 2573 bpf_program__unpin_instance(prog, buf, i); 2574 } 2575 2576 rmdir(path); 2577 2578 return err; 2579 } 2580 2581 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2582 { 2583 int i, err; 2584 2585 err = check_path(path); 2586 if (err) 2587 return err; 2588 2589 if (prog == NULL) { 2590 pr_warning("invalid program pointer\n"); 2591 return -EINVAL; 2592 } 2593 2594 if (prog->instances.nr <= 0) { 2595 pr_warning("no instances of prog %s to pin\n", 2596 prog->section_name); 2597 return -EINVAL; 2598 } 2599 2600 if (prog->instances.nr == 1) { 2601 /* don't create subdirs when pinning single instance */ 2602 return bpf_program__unpin_instance(prog, path, 0); 2603 } 2604 2605 for (i = 0; i < prog->instances.nr; i++) { 2606 char buf[PATH_MAX]; 2607 int len; 2608 2609 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2610 if (len < 0) 2611 return -EINVAL; 2612 else if (len >= PATH_MAX) 2613 return -ENAMETOOLONG; 2614 2615 err = bpf_program__unpin_instance(prog, buf, i); 2616 if (err) 2617 return err; 2618 } 2619 2620 err = rmdir(path); 2621 if (err) 2622 return -errno; 2623 2624 return 0; 2625 } 2626 2627 int bpf_map__pin(struct bpf_map *map, const char *path) 2628 { 2629 char *cp, errmsg[STRERR_BUFSIZE]; 2630 int err; 2631 2632 err = check_path(path); 2633 if (err) 2634 return err; 2635 2636 if (map == NULL) { 2637 pr_warning("invalid map pointer\n"); 2638 return -EINVAL; 2639 } 2640 2641 if (bpf_obj_pin(map->fd, path)) { 2642 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2643 pr_warning("failed to pin map: %s\n", cp); 2644 return -errno; 2645 } 2646 2647 pr_debug("pinned map '%s'\n", path); 2648 2649 return 0; 2650 } 2651 2652 int bpf_map__unpin(struct bpf_map *map, const char *path) 2653 { 2654 int err; 2655 2656 err = check_path(path); 2657 if (err) 2658 return err; 2659 2660 if (map == NULL) { 2661 pr_warning("invalid map pointer\n"); 2662 return -EINVAL; 2663 } 2664 2665 err = unlink(path); 2666 if (err != 0) 2667 return -errno; 2668 pr_debug("unpinned map '%s'\n", path); 2669 2670 return 0; 2671 } 2672 2673 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2674 { 2675 struct bpf_map *map; 2676 int err; 2677 2678 if (!obj) 2679 return -ENOENT; 2680 2681 if (!obj->loaded) { 2682 pr_warning("object not yet loaded; load it first\n"); 2683 return -ENOENT; 2684 } 2685 2686 err = make_dir(path); 2687 if (err) 2688 return err; 2689 2690 bpf_object__for_each_map(map, obj) { 2691 char buf[PATH_MAX]; 2692 int len; 2693 2694 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2695 bpf_map__name(map)); 2696 if (len < 0) { 2697 err = -EINVAL; 2698 goto err_unpin_maps; 2699 } else if (len >= PATH_MAX) { 2700 err = -ENAMETOOLONG; 2701 goto err_unpin_maps; 2702 } 2703 2704 err = bpf_map__pin(map, buf); 2705 if (err) 2706 goto err_unpin_maps; 2707 } 2708 2709 return 0; 2710 2711 err_unpin_maps: 2712 while ((map = bpf_map__prev(map, obj))) { 2713 char buf[PATH_MAX]; 2714 int len; 2715 2716 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2717 bpf_map__name(map)); 2718 if (len < 0) 2719 continue; 2720 else if (len >= PATH_MAX) 2721 continue; 2722 2723 bpf_map__unpin(map, buf); 2724 } 2725 2726 return err; 2727 } 2728 2729 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2730 { 2731 struct bpf_map *map; 2732 int err; 2733 2734 if (!obj) 2735 return -ENOENT; 2736 2737 bpf_object__for_each_map(map, obj) { 2738 char buf[PATH_MAX]; 2739 int len; 2740 2741 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2742 bpf_map__name(map)); 2743 if (len < 0) 2744 return -EINVAL; 2745 else if (len >= PATH_MAX) 2746 return -ENAMETOOLONG; 2747 2748 err = bpf_map__unpin(map, buf); 2749 if (err) 2750 return err; 2751 } 2752 2753 return 0; 2754 } 2755 2756 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2757 { 2758 struct bpf_program *prog; 2759 int err; 2760 2761 if (!obj) 2762 return -ENOENT; 2763 2764 if (!obj->loaded) { 2765 pr_warning("object not yet loaded; load it first\n"); 2766 return -ENOENT; 2767 } 2768 2769 err = make_dir(path); 2770 if (err) 2771 return err; 2772 2773 bpf_object__for_each_program(prog, obj) { 2774 char buf[PATH_MAX]; 2775 int len; 2776 2777 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2778 prog->pin_name); 2779 if (len < 0) { 2780 err = -EINVAL; 2781 goto err_unpin_programs; 2782 } else if (len >= PATH_MAX) { 2783 err = -ENAMETOOLONG; 2784 goto err_unpin_programs; 2785 } 2786 2787 err = bpf_program__pin(prog, buf); 2788 if (err) 2789 goto err_unpin_programs; 2790 } 2791 2792 return 0; 2793 2794 err_unpin_programs: 2795 while ((prog = bpf_program__prev(prog, obj))) { 2796 char buf[PATH_MAX]; 2797 int len; 2798 2799 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2800 prog->pin_name); 2801 if (len < 0) 2802 continue; 2803 else if (len >= PATH_MAX) 2804 continue; 2805 2806 bpf_program__unpin(prog, buf); 2807 } 2808 2809 return err; 2810 } 2811 2812 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2813 { 2814 struct bpf_program *prog; 2815 int err; 2816 2817 if (!obj) 2818 return -ENOENT; 2819 2820 bpf_object__for_each_program(prog, obj) { 2821 char buf[PATH_MAX]; 2822 int len; 2823 2824 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2825 prog->pin_name); 2826 if (len < 0) 2827 return -EINVAL; 2828 else if (len >= PATH_MAX) 2829 return -ENAMETOOLONG; 2830 2831 err = bpf_program__unpin(prog, buf); 2832 if (err) 2833 return err; 2834 } 2835 2836 return 0; 2837 } 2838 2839 int bpf_object__pin(struct bpf_object *obj, const char *path) 2840 { 2841 int err; 2842 2843 err = bpf_object__pin_maps(obj, path); 2844 if (err) 2845 return err; 2846 2847 err = bpf_object__pin_programs(obj, path); 2848 if (err) { 2849 bpf_object__unpin_maps(obj, path); 2850 return err; 2851 } 2852 2853 return 0; 2854 } 2855 2856 void bpf_object__close(struct bpf_object *obj) 2857 { 2858 size_t i; 2859 2860 if (!obj) 2861 return; 2862 2863 if (obj->clear_priv) 2864 obj->clear_priv(obj, obj->priv); 2865 2866 bpf_object__elf_finish(obj); 2867 bpf_object__unload(obj); 2868 btf__free(obj->btf); 2869 btf_ext__free(obj->btf_ext); 2870 2871 for (i = 0; i < obj->nr_maps; i++) { 2872 zfree(&obj->maps[i].name); 2873 if (obj->maps[i].clear_priv) 2874 obj->maps[i].clear_priv(&obj->maps[i], 2875 obj->maps[i].priv); 2876 obj->maps[i].priv = NULL; 2877 obj->maps[i].clear_priv = NULL; 2878 } 2879 2880 zfree(&obj->sections.rodata); 2881 zfree(&obj->sections.data); 2882 zfree(&obj->maps); 2883 obj->nr_maps = 0; 2884 2885 if (obj->programs && obj->nr_programs) { 2886 for (i = 0; i < obj->nr_programs; i++) 2887 bpf_program__exit(&obj->programs[i]); 2888 } 2889 zfree(&obj->programs); 2890 2891 list_del(&obj->list); 2892 free(obj); 2893 } 2894 2895 struct bpf_object * 2896 bpf_object__next(struct bpf_object *prev) 2897 { 2898 struct bpf_object *next; 2899 2900 if (!prev) 2901 next = list_first_entry(&bpf_objects_list, 2902 struct bpf_object, 2903 list); 2904 else 2905 next = list_next_entry(prev, list); 2906 2907 /* Empty list is noticed here so don't need checking on entry. */ 2908 if (&next->list == &bpf_objects_list) 2909 return NULL; 2910 2911 return next; 2912 } 2913 2914 const char *bpf_object__name(struct bpf_object *obj) 2915 { 2916 return obj ? obj->path : ERR_PTR(-EINVAL); 2917 } 2918 2919 unsigned int bpf_object__kversion(struct bpf_object *obj) 2920 { 2921 return obj ? obj->kern_version : 0; 2922 } 2923 2924 struct btf *bpf_object__btf(struct bpf_object *obj) 2925 { 2926 return obj ? obj->btf : NULL; 2927 } 2928 2929 int bpf_object__btf_fd(const struct bpf_object *obj) 2930 { 2931 return obj->btf ? btf__fd(obj->btf) : -1; 2932 } 2933 2934 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2935 bpf_object_clear_priv_t clear_priv) 2936 { 2937 if (obj->priv && obj->clear_priv) 2938 obj->clear_priv(obj, obj->priv); 2939 2940 obj->priv = priv; 2941 obj->clear_priv = clear_priv; 2942 return 0; 2943 } 2944 2945 void *bpf_object__priv(struct bpf_object *obj) 2946 { 2947 return obj ? obj->priv : ERR_PTR(-EINVAL); 2948 } 2949 2950 static struct bpf_program * 2951 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2952 { 2953 size_t nr_programs = obj->nr_programs; 2954 ssize_t idx; 2955 2956 if (!nr_programs) 2957 return NULL; 2958 2959 if (!p) 2960 /* Iter from the beginning */ 2961 return forward ? &obj->programs[0] : 2962 &obj->programs[nr_programs - 1]; 2963 2964 if (p->obj != obj) { 2965 pr_warning("error: program handler doesn't match object\n"); 2966 return NULL; 2967 } 2968 2969 idx = (p - obj->programs) + (forward ? 1 : -1); 2970 if (idx >= obj->nr_programs || idx < 0) 2971 return NULL; 2972 return &obj->programs[idx]; 2973 } 2974 2975 struct bpf_program * 2976 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2977 { 2978 struct bpf_program *prog = prev; 2979 2980 do { 2981 prog = __bpf_program__iter(prog, obj, true); 2982 } while (prog && bpf_program__is_function_storage(prog, obj)); 2983 2984 return prog; 2985 } 2986 2987 struct bpf_program * 2988 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2989 { 2990 struct bpf_program *prog = next; 2991 2992 do { 2993 prog = __bpf_program__iter(prog, obj, false); 2994 } while (prog && bpf_program__is_function_storage(prog, obj)); 2995 2996 return prog; 2997 } 2998 2999 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 3000 bpf_program_clear_priv_t clear_priv) 3001 { 3002 if (prog->priv && prog->clear_priv) 3003 prog->clear_priv(prog, prog->priv); 3004 3005 prog->priv = priv; 3006 prog->clear_priv = clear_priv; 3007 return 0; 3008 } 3009 3010 void *bpf_program__priv(struct bpf_program *prog) 3011 { 3012 return prog ? prog->priv : ERR_PTR(-EINVAL); 3013 } 3014 3015 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3016 { 3017 prog->prog_ifindex = ifindex; 3018 } 3019 3020 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 3021 { 3022 const char *title; 3023 3024 title = prog->section_name; 3025 if (needs_copy) { 3026 title = strdup(title); 3027 if (!title) { 3028 pr_warning("failed to strdup program title\n"); 3029 return ERR_PTR(-ENOMEM); 3030 } 3031 } 3032 3033 return title; 3034 } 3035 3036 int bpf_program__fd(struct bpf_program *prog) 3037 { 3038 return bpf_program__nth_fd(prog, 0); 3039 } 3040 3041 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3042 bpf_program_prep_t prep) 3043 { 3044 int *instances_fds; 3045 3046 if (nr_instances <= 0 || !prep) 3047 return -EINVAL; 3048 3049 if (prog->instances.nr > 0 || prog->instances.fds) { 3050 pr_warning("Can't set pre-processor after loading\n"); 3051 return -EINVAL; 3052 } 3053 3054 instances_fds = malloc(sizeof(int) * nr_instances); 3055 if (!instances_fds) { 3056 pr_warning("alloc memory failed for fds\n"); 3057 return -ENOMEM; 3058 } 3059 3060 /* fill all fd with -1 */ 3061 memset(instances_fds, -1, sizeof(int) * nr_instances); 3062 3063 prog->instances.nr = nr_instances; 3064 prog->instances.fds = instances_fds; 3065 prog->preprocessor = prep; 3066 return 0; 3067 } 3068 3069 int bpf_program__nth_fd(struct bpf_program *prog, int n) 3070 { 3071 int fd; 3072 3073 if (!prog) 3074 return -EINVAL; 3075 3076 if (n >= prog->instances.nr || n < 0) { 3077 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3078 n, prog->section_name, prog->instances.nr); 3079 return -EINVAL; 3080 } 3081 3082 fd = prog->instances.fds[n]; 3083 if (fd < 0) { 3084 pr_warning("%dth instance of program '%s' is invalid\n", 3085 n, prog->section_name); 3086 return -ENOENT; 3087 } 3088 3089 return fd; 3090 } 3091 3092 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3093 { 3094 prog->type = type; 3095 } 3096 3097 static bool bpf_program__is_type(struct bpf_program *prog, 3098 enum bpf_prog_type type) 3099 { 3100 return prog ? (prog->type == type) : false; 3101 } 3102 3103 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3104 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3105 { \ 3106 if (!prog) \ 3107 return -EINVAL; \ 3108 bpf_program__set_type(prog, TYPE); \ 3109 return 0; \ 3110 } \ 3111 \ 3112 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 3113 { \ 3114 return bpf_program__is_type(prog, TYPE); \ 3115 } \ 3116 3117 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3118 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3119 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3120 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3121 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3122 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3123 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3124 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3125 3126 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3127 enum bpf_attach_type type) 3128 { 3129 prog->expected_attach_type = type; 3130 } 3131 3132 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3133 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3134 3135 /* Programs that can NOT be attached. */ 3136 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3137 3138 /* Programs that can be attached. */ 3139 #define BPF_APROG_SEC(string, ptype, atype) \ 3140 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3141 3142 /* Programs that must specify expected attach type at load time. */ 3143 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3144 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3145 3146 /* Programs that can be attached but attach type can't be identified by section 3147 * name. Kept for backward compatibility. 3148 */ 3149 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3150 3151 static const struct { 3152 const char *sec; 3153 size_t len; 3154 enum bpf_prog_type prog_type; 3155 enum bpf_attach_type expected_attach_type; 3156 int is_attachable; 3157 enum bpf_attach_type attach_type; 3158 } section_names[] = { 3159 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3160 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3161 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3162 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3163 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3164 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3165 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3166 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3167 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3168 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3169 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3170 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3171 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3172 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3173 BPF_CGROUP_INET_INGRESS), 3174 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3175 BPF_CGROUP_INET_EGRESS), 3176 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3177 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3178 BPF_CGROUP_INET_SOCK_CREATE), 3179 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3180 BPF_CGROUP_INET4_POST_BIND), 3181 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3182 BPF_CGROUP_INET6_POST_BIND), 3183 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3184 BPF_CGROUP_DEVICE), 3185 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3186 BPF_CGROUP_SOCK_OPS), 3187 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3188 BPF_SK_SKB_STREAM_PARSER), 3189 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3190 BPF_SK_SKB_STREAM_VERDICT), 3191 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3192 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3193 BPF_SK_MSG_VERDICT), 3194 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3195 BPF_LIRC_MODE2), 3196 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3197 BPF_FLOW_DISSECTOR), 3198 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3199 BPF_CGROUP_INET4_BIND), 3200 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3201 BPF_CGROUP_INET6_BIND), 3202 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3203 BPF_CGROUP_INET4_CONNECT), 3204 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3205 BPF_CGROUP_INET6_CONNECT), 3206 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3207 BPF_CGROUP_UDP4_SENDMSG), 3208 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3209 BPF_CGROUP_UDP6_SENDMSG), 3210 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3211 BPF_CGROUP_SYSCTL), 3212 }; 3213 3214 #undef BPF_PROG_SEC_IMPL 3215 #undef BPF_PROG_SEC 3216 #undef BPF_APROG_SEC 3217 #undef BPF_EAPROG_SEC 3218 #undef BPF_APROG_COMPAT 3219 3220 #define MAX_TYPE_NAME_SIZE 32 3221 3222 static char *libbpf_get_type_names(bool attach_type) 3223 { 3224 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3225 char *buf; 3226 3227 buf = malloc(len); 3228 if (!buf) 3229 return NULL; 3230 3231 buf[0] = '\0'; 3232 /* Forge string buf with all available names */ 3233 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3234 if (attach_type && !section_names[i].is_attachable) 3235 continue; 3236 3237 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3238 free(buf); 3239 return NULL; 3240 } 3241 strcat(buf, " "); 3242 strcat(buf, section_names[i].sec); 3243 } 3244 3245 return buf; 3246 } 3247 3248 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3249 enum bpf_attach_type *expected_attach_type) 3250 { 3251 char *type_names; 3252 int i; 3253 3254 if (!name) 3255 return -EINVAL; 3256 3257 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3258 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3259 continue; 3260 *prog_type = section_names[i].prog_type; 3261 *expected_attach_type = section_names[i].expected_attach_type; 3262 return 0; 3263 } 3264 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3265 type_names = libbpf_get_type_names(false); 3266 if (type_names != NULL) { 3267 pr_info("supported section(type) names are:%s\n", type_names); 3268 free(type_names); 3269 } 3270 3271 return -EINVAL; 3272 } 3273 3274 int libbpf_attach_type_by_name(const char *name, 3275 enum bpf_attach_type *attach_type) 3276 { 3277 char *type_names; 3278 int i; 3279 3280 if (!name) 3281 return -EINVAL; 3282 3283 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3284 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3285 continue; 3286 if (!section_names[i].is_attachable) 3287 return -EINVAL; 3288 *attach_type = section_names[i].attach_type; 3289 return 0; 3290 } 3291 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3292 type_names = libbpf_get_type_names(true); 3293 if (type_names != NULL) { 3294 pr_info("attachable section(type) names are:%s\n", type_names); 3295 free(type_names); 3296 } 3297 3298 return -EINVAL; 3299 } 3300 3301 static int 3302 bpf_program__identify_section(struct bpf_program *prog, 3303 enum bpf_prog_type *prog_type, 3304 enum bpf_attach_type *expected_attach_type) 3305 { 3306 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3307 expected_attach_type); 3308 } 3309 3310 int bpf_map__fd(struct bpf_map *map) 3311 { 3312 return map ? map->fd : -EINVAL; 3313 } 3314 3315 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 3316 { 3317 return map ? &map->def : ERR_PTR(-EINVAL); 3318 } 3319 3320 const char *bpf_map__name(struct bpf_map *map) 3321 { 3322 return map ? map->name : NULL; 3323 } 3324 3325 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3326 { 3327 return map ? map->btf_key_type_id : 0; 3328 } 3329 3330 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3331 { 3332 return map ? map->btf_value_type_id : 0; 3333 } 3334 3335 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3336 bpf_map_clear_priv_t clear_priv) 3337 { 3338 if (!map) 3339 return -EINVAL; 3340 3341 if (map->priv) { 3342 if (map->clear_priv) 3343 map->clear_priv(map, map->priv); 3344 } 3345 3346 map->priv = priv; 3347 map->clear_priv = clear_priv; 3348 return 0; 3349 } 3350 3351 void *bpf_map__priv(struct bpf_map *map) 3352 { 3353 return map ? map->priv : ERR_PTR(-EINVAL); 3354 } 3355 3356 bool bpf_map__is_offload_neutral(struct bpf_map *map) 3357 { 3358 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3359 } 3360 3361 bool bpf_map__is_internal(struct bpf_map *map) 3362 { 3363 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3364 } 3365 3366 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3367 { 3368 map->map_ifindex = ifindex; 3369 } 3370 3371 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3372 { 3373 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3374 pr_warning("error: unsupported map type\n"); 3375 return -EINVAL; 3376 } 3377 if (map->inner_map_fd != -1) { 3378 pr_warning("error: inner_map_fd already specified\n"); 3379 return -EINVAL; 3380 } 3381 map->inner_map_fd = fd; 3382 return 0; 3383 } 3384 3385 static struct bpf_map * 3386 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 3387 { 3388 ssize_t idx; 3389 struct bpf_map *s, *e; 3390 3391 if (!obj || !obj->maps) 3392 return NULL; 3393 3394 s = obj->maps; 3395 e = obj->maps + obj->nr_maps; 3396 3397 if ((m < s) || (m >= e)) { 3398 pr_warning("error in %s: map handler doesn't belong to object\n", 3399 __func__); 3400 return NULL; 3401 } 3402 3403 idx = (m - obj->maps) + i; 3404 if (idx >= obj->nr_maps || idx < 0) 3405 return NULL; 3406 return &obj->maps[idx]; 3407 } 3408 3409 struct bpf_map * 3410 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 3411 { 3412 if (prev == NULL) 3413 return obj->maps; 3414 3415 return __bpf_map__iter(prev, obj, 1); 3416 } 3417 3418 struct bpf_map * 3419 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 3420 { 3421 if (next == NULL) { 3422 if (!obj->nr_maps) 3423 return NULL; 3424 return obj->maps + obj->nr_maps - 1; 3425 } 3426 3427 return __bpf_map__iter(next, obj, -1); 3428 } 3429 3430 struct bpf_map * 3431 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 3432 { 3433 struct bpf_map *pos; 3434 3435 bpf_object__for_each_map(pos, obj) { 3436 if (pos->name && !strcmp(pos->name, name)) 3437 return pos; 3438 } 3439 return NULL; 3440 } 3441 3442 int 3443 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 3444 { 3445 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3446 } 3447 3448 struct bpf_map * 3449 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3450 { 3451 int i; 3452 3453 for (i = 0; i < obj->nr_maps; i++) { 3454 if (obj->maps[i].offset == offset) 3455 return &obj->maps[i]; 3456 } 3457 return ERR_PTR(-ENOENT); 3458 } 3459 3460 long libbpf_get_error(const void *ptr) 3461 { 3462 if (IS_ERR(ptr)) 3463 return PTR_ERR(ptr); 3464 return 0; 3465 } 3466 3467 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3468 struct bpf_object **pobj, int *prog_fd) 3469 { 3470 struct bpf_prog_load_attr attr; 3471 3472 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3473 attr.file = file; 3474 attr.prog_type = type; 3475 attr.expected_attach_type = 0; 3476 3477 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3478 } 3479 3480 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3481 struct bpf_object **pobj, int *prog_fd) 3482 { 3483 struct bpf_object_open_attr open_attr = { 3484 .file = attr->file, 3485 .prog_type = attr->prog_type, 3486 }; 3487 struct bpf_program *prog, *first_prog = NULL; 3488 enum bpf_attach_type expected_attach_type; 3489 enum bpf_prog_type prog_type; 3490 struct bpf_object *obj; 3491 struct bpf_map *map; 3492 int err; 3493 3494 if (!attr) 3495 return -EINVAL; 3496 if (!attr->file) 3497 return -EINVAL; 3498 3499 obj = bpf_object__open_xattr(&open_attr); 3500 if (IS_ERR_OR_NULL(obj)) 3501 return -ENOENT; 3502 3503 bpf_object__for_each_program(prog, obj) { 3504 /* 3505 * If type is not specified, try to guess it based on 3506 * section name. 3507 */ 3508 prog_type = attr->prog_type; 3509 prog->prog_ifindex = attr->ifindex; 3510 expected_attach_type = attr->expected_attach_type; 3511 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3512 err = bpf_program__identify_section(prog, &prog_type, 3513 &expected_attach_type); 3514 if (err < 0) { 3515 bpf_object__close(obj); 3516 return -EINVAL; 3517 } 3518 } 3519 3520 bpf_program__set_type(prog, prog_type); 3521 bpf_program__set_expected_attach_type(prog, 3522 expected_attach_type); 3523 3524 prog->log_level = attr->log_level; 3525 if (!first_prog) 3526 first_prog = prog; 3527 } 3528 3529 bpf_object__for_each_map(map, obj) { 3530 if (!bpf_map__is_offload_neutral(map)) 3531 map->map_ifindex = attr->ifindex; 3532 } 3533 3534 if (!first_prog) { 3535 pr_warning("object file doesn't contain bpf program\n"); 3536 bpf_object__close(obj); 3537 return -ENOENT; 3538 } 3539 3540 err = bpf_object__load(obj); 3541 if (err) { 3542 bpf_object__close(obj); 3543 return -EINVAL; 3544 } 3545 3546 *pobj = obj; 3547 *prog_fd = bpf_program__fd(first_prog); 3548 return 0; 3549 } 3550 3551 enum bpf_perf_event_ret 3552 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 3553 void **copy_mem, size_t *copy_size, 3554 bpf_perf_event_print_t fn, void *private_data) 3555 { 3556 struct perf_event_mmap_page *header = mmap_mem; 3557 __u64 data_head = ring_buffer_read_head(header); 3558 __u64 data_tail = header->data_tail; 3559 void *base = ((__u8 *)header) + page_size; 3560 int ret = LIBBPF_PERF_EVENT_CONT; 3561 struct perf_event_header *ehdr; 3562 size_t ehdr_size; 3563 3564 while (data_head != data_tail) { 3565 ehdr = base + (data_tail & (mmap_size - 1)); 3566 ehdr_size = ehdr->size; 3567 3568 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 3569 void *copy_start = ehdr; 3570 size_t len_first = base + mmap_size - copy_start; 3571 size_t len_secnd = ehdr_size - len_first; 3572 3573 if (*copy_size < ehdr_size) { 3574 free(*copy_mem); 3575 *copy_mem = malloc(ehdr_size); 3576 if (!*copy_mem) { 3577 *copy_size = 0; 3578 ret = LIBBPF_PERF_EVENT_ERROR; 3579 break; 3580 } 3581 *copy_size = ehdr_size; 3582 } 3583 3584 memcpy(*copy_mem, copy_start, len_first); 3585 memcpy(*copy_mem + len_first, base, len_secnd); 3586 ehdr = *copy_mem; 3587 } 3588 3589 ret = fn(ehdr, private_data); 3590 data_tail += ehdr_size; 3591 if (ret != LIBBPF_PERF_EVENT_CONT) 3592 break; 3593 } 3594 3595 ring_buffer_write_tail(header, data_tail); 3596 return ret; 3597 } 3598 3599 struct bpf_prog_info_array_desc { 3600 int array_offset; /* e.g. offset of jited_prog_insns */ 3601 int count_offset; /* e.g. offset of jited_prog_len */ 3602 int size_offset; /* > 0: offset of rec size, 3603 * < 0: fix size of -size_offset 3604 */ 3605 }; 3606 3607 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 3608 [BPF_PROG_INFO_JITED_INSNS] = { 3609 offsetof(struct bpf_prog_info, jited_prog_insns), 3610 offsetof(struct bpf_prog_info, jited_prog_len), 3611 -1, 3612 }, 3613 [BPF_PROG_INFO_XLATED_INSNS] = { 3614 offsetof(struct bpf_prog_info, xlated_prog_insns), 3615 offsetof(struct bpf_prog_info, xlated_prog_len), 3616 -1, 3617 }, 3618 [BPF_PROG_INFO_MAP_IDS] = { 3619 offsetof(struct bpf_prog_info, map_ids), 3620 offsetof(struct bpf_prog_info, nr_map_ids), 3621 -(int)sizeof(__u32), 3622 }, 3623 [BPF_PROG_INFO_JITED_KSYMS] = { 3624 offsetof(struct bpf_prog_info, jited_ksyms), 3625 offsetof(struct bpf_prog_info, nr_jited_ksyms), 3626 -(int)sizeof(__u64), 3627 }, 3628 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 3629 offsetof(struct bpf_prog_info, jited_func_lens), 3630 offsetof(struct bpf_prog_info, nr_jited_func_lens), 3631 -(int)sizeof(__u32), 3632 }, 3633 [BPF_PROG_INFO_FUNC_INFO] = { 3634 offsetof(struct bpf_prog_info, func_info), 3635 offsetof(struct bpf_prog_info, nr_func_info), 3636 offsetof(struct bpf_prog_info, func_info_rec_size), 3637 }, 3638 [BPF_PROG_INFO_LINE_INFO] = { 3639 offsetof(struct bpf_prog_info, line_info), 3640 offsetof(struct bpf_prog_info, nr_line_info), 3641 offsetof(struct bpf_prog_info, line_info_rec_size), 3642 }, 3643 [BPF_PROG_INFO_JITED_LINE_INFO] = { 3644 offsetof(struct bpf_prog_info, jited_line_info), 3645 offsetof(struct bpf_prog_info, nr_jited_line_info), 3646 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 3647 }, 3648 [BPF_PROG_INFO_PROG_TAGS] = { 3649 offsetof(struct bpf_prog_info, prog_tags), 3650 offsetof(struct bpf_prog_info, nr_prog_tags), 3651 -(int)sizeof(__u8) * BPF_TAG_SIZE, 3652 }, 3653 3654 }; 3655 3656 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 3657 { 3658 __u32 *array = (__u32 *)info; 3659 3660 if (offset >= 0) 3661 return array[offset / sizeof(__u32)]; 3662 return -(int)offset; 3663 } 3664 3665 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 3666 { 3667 __u64 *array = (__u64 *)info; 3668 3669 if (offset >= 0) 3670 return array[offset / sizeof(__u64)]; 3671 return -(int)offset; 3672 } 3673 3674 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 3675 __u32 val) 3676 { 3677 __u32 *array = (__u32 *)info; 3678 3679 if (offset >= 0) 3680 array[offset / sizeof(__u32)] = val; 3681 } 3682 3683 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 3684 __u64 val) 3685 { 3686 __u64 *array = (__u64 *)info; 3687 3688 if (offset >= 0) 3689 array[offset / sizeof(__u64)] = val; 3690 } 3691 3692 struct bpf_prog_info_linear * 3693 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 3694 { 3695 struct bpf_prog_info_linear *info_linear; 3696 struct bpf_prog_info info = {}; 3697 __u32 info_len = sizeof(info); 3698 __u32 data_len = 0; 3699 int i, err; 3700 void *ptr; 3701 3702 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 3703 return ERR_PTR(-EINVAL); 3704 3705 /* step 1: get array dimensions */ 3706 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 3707 if (err) { 3708 pr_debug("can't get prog info: %s", strerror(errno)); 3709 return ERR_PTR(-EFAULT); 3710 } 3711 3712 /* step 2: calculate total size of all arrays */ 3713 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3714 bool include_array = (arrays & (1UL << i)) > 0; 3715 struct bpf_prog_info_array_desc *desc; 3716 __u32 count, size; 3717 3718 desc = bpf_prog_info_array_desc + i; 3719 3720 /* kernel is too old to support this field */ 3721 if (info_len < desc->array_offset + sizeof(__u32) || 3722 info_len < desc->count_offset + sizeof(__u32) || 3723 (desc->size_offset > 0 && info_len < desc->size_offset)) 3724 include_array = false; 3725 3726 if (!include_array) { 3727 arrays &= ~(1UL << i); /* clear the bit */ 3728 continue; 3729 } 3730 3731 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3732 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3733 3734 data_len += count * size; 3735 } 3736 3737 /* step 3: allocate continuous memory */ 3738 data_len = roundup(data_len, sizeof(__u64)); 3739 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 3740 if (!info_linear) 3741 return ERR_PTR(-ENOMEM); 3742 3743 /* step 4: fill data to info_linear->info */ 3744 info_linear->arrays = arrays; 3745 memset(&info_linear->info, 0, sizeof(info)); 3746 ptr = info_linear->data; 3747 3748 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3749 struct bpf_prog_info_array_desc *desc; 3750 __u32 count, size; 3751 3752 if ((arrays & (1UL << i)) == 0) 3753 continue; 3754 3755 desc = bpf_prog_info_array_desc + i; 3756 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3757 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3758 bpf_prog_info_set_offset_u32(&info_linear->info, 3759 desc->count_offset, count); 3760 bpf_prog_info_set_offset_u32(&info_linear->info, 3761 desc->size_offset, size); 3762 bpf_prog_info_set_offset_u64(&info_linear->info, 3763 desc->array_offset, 3764 ptr_to_u64(ptr)); 3765 ptr += count * size; 3766 } 3767 3768 /* step 5: call syscall again to get required arrays */ 3769 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 3770 if (err) { 3771 pr_debug("can't get prog info: %s", strerror(errno)); 3772 free(info_linear); 3773 return ERR_PTR(-EFAULT); 3774 } 3775 3776 /* step 6: verify the data */ 3777 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3778 struct bpf_prog_info_array_desc *desc; 3779 __u32 v1, v2; 3780 3781 if ((arrays & (1UL << i)) == 0) 3782 continue; 3783 3784 desc = bpf_prog_info_array_desc + i; 3785 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3786 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3787 desc->count_offset); 3788 if (v1 != v2) 3789 pr_warning("%s: mismatch in element count\n", __func__); 3790 3791 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3792 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3793 desc->size_offset); 3794 if (v1 != v2) 3795 pr_warning("%s: mismatch in rec size\n", __func__); 3796 } 3797 3798 /* step 7: update info_len and data_len */ 3799 info_linear->info_len = sizeof(struct bpf_prog_info); 3800 info_linear->data_len = data_len; 3801 3802 return info_linear; 3803 } 3804 3805 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 3806 { 3807 int i; 3808 3809 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3810 struct bpf_prog_info_array_desc *desc; 3811 __u64 addr, offs; 3812 3813 if ((info_linear->arrays & (1UL << i)) == 0) 3814 continue; 3815 3816 desc = bpf_prog_info_array_desc + i; 3817 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 3818 desc->array_offset); 3819 offs = addr - ptr_to_u64(info_linear->data); 3820 bpf_prog_info_set_offset_u64(&info_linear->info, 3821 desc->array_offset, offs); 3822 } 3823 } 3824 3825 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 3826 { 3827 int i; 3828 3829 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3830 struct bpf_prog_info_array_desc *desc; 3831 __u64 addr, offs; 3832 3833 if ((info_linear->arrays & (1UL << i)) == 0) 3834 continue; 3835 3836 desc = bpf_prog_info_array_desc + i; 3837 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 3838 desc->array_offset); 3839 addr = offs + ptr_to_u64(info_linear->data); 3840 bpf_prog_info_set_offset_u64(&info_linear->info, 3841 desc->array_offset, addr); 3842 } 3843 } 3844