1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/stat.h> 36 #include <sys/types.h> 37 #include <sys/vfs.h> 38 #include <tools/libc_compat.h> 39 #include <libelf.h> 40 #include <gelf.h> 41 42 #include "libbpf.h" 43 #include "bpf.h" 44 #include "btf.h" 45 #include "str_error.h" 46 #include "libbpf_internal.h" 47 48 #ifndef EM_BPF 49 #define EM_BPF 247 50 #endif 51 52 #ifndef BPF_FS_MAGIC 53 #define BPF_FS_MAGIC 0xcafe4a11 54 #endif 55 56 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 57 * compilation if user enables corresponding warning. Disable it explicitly. 58 */ 59 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 60 61 #define __printf(a, b) __attribute__((format(printf, a, b))) 62 63 static int __base_pr(enum libbpf_print_level level, const char *format, 64 va_list args) 65 { 66 if (level == LIBBPF_DEBUG) 67 return 0; 68 69 return vfprintf(stderr, format, args); 70 } 71 72 static libbpf_print_fn_t __libbpf_pr = __base_pr; 73 74 void libbpf_set_print(libbpf_print_fn_t fn) 75 { 76 __libbpf_pr = fn; 77 } 78 79 __printf(2, 3) 80 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 81 { 82 va_list args; 83 84 if (!__libbpf_pr) 85 return; 86 87 va_start(args, format); 88 __libbpf_pr(level, format, args); 89 va_end(args); 90 } 91 92 #define STRERR_BUFSIZE 128 93 94 #define CHECK_ERR(action, err, out) do { \ 95 err = action; \ 96 if (err) \ 97 goto out; \ 98 } while(0) 99 100 101 /* Copied from tools/perf/util/util.h */ 102 #ifndef zfree 103 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 104 #endif 105 106 #ifndef zclose 107 # define zclose(fd) ({ \ 108 int ___err = 0; \ 109 if ((fd) >= 0) \ 110 ___err = close((fd)); \ 111 fd = -1; \ 112 ___err; }) 113 #endif 114 115 #ifdef HAVE_LIBELF_MMAP_SUPPORT 116 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 117 #else 118 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 119 #endif 120 121 static inline __u64 ptr_to_u64(const void *ptr) 122 { 123 return (__u64) (unsigned long) ptr; 124 } 125 126 struct bpf_capabilities { 127 /* v4.14: kernel support for program & map names. */ 128 __u32 name:1; 129 /* v5.2: kernel support for global data sections. */ 130 __u32 global_data:1; 131 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 132 __u32 btf_func:1; 133 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 134 __u32 btf_datasec:1; 135 }; 136 137 /* 138 * bpf_prog should be a better name but it has been used in 139 * linux/filter.h. 140 */ 141 struct bpf_program { 142 /* Index in elf obj file, for relocation use. */ 143 int idx; 144 char *name; 145 int prog_ifindex; 146 char *section_name; 147 /* section_name with / replaced by _; makes recursive pinning 148 * in bpf_object__pin_programs easier 149 */ 150 char *pin_name; 151 struct bpf_insn *insns; 152 size_t insns_cnt, main_prog_cnt; 153 enum bpf_prog_type type; 154 155 struct reloc_desc { 156 enum { 157 RELO_LD64, 158 RELO_CALL, 159 RELO_DATA, 160 } type; 161 int insn_idx; 162 union { 163 int map_idx; 164 int text_off; 165 }; 166 } *reloc_desc; 167 int nr_reloc; 168 int log_level; 169 170 struct { 171 int nr; 172 int *fds; 173 } instances; 174 bpf_program_prep_t preprocessor; 175 176 struct bpf_object *obj; 177 void *priv; 178 bpf_program_clear_priv_t clear_priv; 179 180 enum bpf_attach_type expected_attach_type; 181 int btf_fd; 182 void *func_info; 183 __u32 func_info_rec_size; 184 __u32 func_info_cnt; 185 186 struct bpf_capabilities *caps; 187 188 void *line_info; 189 __u32 line_info_rec_size; 190 __u32 line_info_cnt; 191 }; 192 193 enum libbpf_map_type { 194 LIBBPF_MAP_UNSPEC, 195 LIBBPF_MAP_DATA, 196 LIBBPF_MAP_BSS, 197 LIBBPF_MAP_RODATA, 198 }; 199 200 static const char * const libbpf_type_to_btf_name[] = { 201 [LIBBPF_MAP_DATA] = ".data", 202 [LIBBPF_MAP_BSS] = ".bss", 203 [LIBBPF_MAP_RODATA] = ".rodata", 204 }; 205 206 struct bpf_map { 207 int fd; 208 char *name; 209 size_t offset; 210 int map_ifindex; 211 int inner_map_fd; 212 struct bpf_map_def def; 213 __u32 btf_key_type_id; 214 __u32 btf_value_type_id; 215 void *priv; 216 bpf_map_clear_priv_t clear_priv; 217 enum libbpf_map_type libbpf_type; 218 }; 219 220 struct bpf_secdata { 221 void *rodata; 222 void *data; 223 }; 224 225 static LIST_HEAD(bpf_objects_list); 226 227 struct bpf_object { 228 char name[BPF_OBJ_NAME_LEN]; 229 char license[64]; 230 __u32 kern_version; 231 232 struct bpf_program *programs; 233 size_t nr_programs; 234 struct bpf_map *maps; 235 size_t nr_maps; 236 struct bpf_secdata sections; 237 238 bool loaded; 239 bool has_pseudo_calls; 240 241 /* 242 * Information when doing elf related work. Only valid if fd 243 * is valid. 244 */ 245 struct { 246 int fd; 247 void *obj_buf; 248 size_t obj_buf_sz; 249 Elf *elf; 250 GElf_Ehdr ehdr; 251 Elf_Data *symbols; 252 Elf_Data *data; 253 Elf_Data *rodata; 254 Elf_Data *bss; 255 size_t strtabidx; 256 struct { 257 GElf_Shdr shdr; 258 Elf_Data *data; 259 } *reloc; 260 int nr_reloc; 261 int maps_shndx; 262 int text_shndx; 263 int data_shndx; 264 int rodata_shndx; 265 int bss_shndx; 266 } efile; 267 /* 268 * All loaded bpf_object is linked in a list, which is 269 * hidden to caller. bpf_objects__<func> handlers deal with 270 * all objects. 271 */ 272 struct list_head list; 273 274 struct btf *btf; 275 struct btf_ext *btf_ext; 276 277 void *priv; 278 bpf_object_clear_priv_t clear_priv; 279 280 struct bpf_capabilities caps; 281 282 char path[]; 283 }; 284 #define obj_elf_valid(o) ((o)->efile.elf) 285 286 void bpf_program__unload(struct bpf_program *prog) 287 { 288 int i; 289 290 if (!prog) 291 return; 292 293 /* 294 * If the object is opened but the program was never loaded, 295 * it is possible that prog->instances.nr == -1. 296 */ 297 if (prog->instances.nr > 0) { 298 for (i = 0; i < prog->instances.nr; i++) 299 zclose(prog->instances.fds[i]); 300 } else if (prog->instances.nr != -1) { 301 pr_warning("Internal error: instances.nr is %d\n", 302 prog->instances.nr); 303 } 304 305 prog->instances.nr = -1; 306 zfree(&prog->instances.fds); 307 308 zclose(prog->btf_fd); 309 zfree(&prog->func_info); 310 zfree(&prog->line_info); 311 } 312 313 static void bpf_program__exit(struct bpf_program *prog) 314 { 315 if (!prog) 316 return; 317 318 if (prog->clear_priv) 319 prog->clear_priv(prog, prog->priv); 320 321 prog->priv = NULL; 322 prog->clear_priv = NULL; 323 324 bpf_program__unload(prog); 325 zfree(&prog->name); 326 zfree(&prog->section_name); 327 zfree(&prog->pin_name); 328 zfree(&prog->insns); 329 zfree(&prog->reloc_desc); 330 331 prog->nr_reloc = 0; 332 prog->insns_cnt = 0; 333 prog->idx = -1; 334 } 335 336 static char *__bpf_program__pin_name(struct bpf_program *prog) 337 { 338 char *name, *p; 339 340 name = p = strdup(prog->section_name); 341 while ((p = strchr(p, '/'))) 342 *p = '_'; 343 344 return name; 345 } 346 347 static int 348 bpf_program__init(void *data, size_t size, char *section_name, int idx, 349 struct bpf_program *prog) 350 { 351 if (size < sizeof(struct bpf_insn)) { 352 pr_warning("corrupted section '%s'\n", section_name); 353 return -EINVAL; 354 } 355 356 memset(prog, 0, sizeof(*prog)); 357 358 prog->section_name = strdup(section_name); 359 if (!prog->section_name) { 360 pr_warning("failed to alloc name for prog under section(%d) %s\n", 361 idx, section_name); 362 goto errout; 363 } 364 365 prog->pin_name = __bpf_program__pin_name(prog); 366 if (!prog->pin_name) { 367 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 368 idx, section_name); 369 goto errout; 370 } 371 372 prog->insns = malloc(size); 373 if (!prog->insns) { 374 pr_warning("failed to alloc insns for prog under section %s\n", 375 section_name); 376 goto errout; 377 } 378 prog->insns_cnt = size / sizeof(struct bpf_insn); 379 memcpy(prog->insns, data, 380 prog->insns_cnt * sizeof(struct bpf_insn)); 381 prog->idx = idx; 382 prog->instances.fds = NULL; 383 prog->instances.nr = -1; 384 prog->type = BPF_PROG_TYPE_UNSPEC; 385 prog->btf_fd = -1; 386 387 return 0; 388 errout: 389 bpf_program__exit(prog); 390 return -ENOMEM; 391 } 392 393 static int 394 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 395 char *section_name, int idx) 396 { 397 struct bpf_program prog, *progs; 398 int nr_progs, err; 399 400 err = bpf_program__init(data, size, section_name, idx, &prog); 401 if (err) 402 return err; 403 404 prog.caps = &obj->caps; 405 progs = obj->programs; 406 nr_progs = obj->nr_programs; 407 408 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 409 if (!progs) { 410 /* 411 * In this case the original obj->programs 412 * is still valid, so don't need special treat for 413 * bpf_close_object(). 414 */ 415 pr_warning("failed to alloc a new program under section '%s'\n", 416 section_name); 417 bpf_program__exit(&prog); 418 return -ENOMEM; 419 } 420 421 pr_debug("found program %s\n", prog.section_name); 422 obj->programs = progs; 423 obj->nr_programs = nr_progs + 1; 424 prog.obj = obj; 425 progs[nr_progs] = prog; 426 return 0; 427 } 428 429 static int 430 bpf_object__init_prog_names(struct bpf_object *obj) 431 { 432 Elf_Data *symbols = obj->efile.symbols; 433 struct bpf_program *prog; 434 size_t pi, si; 435 436 for (pi = 0; pi < obj->nr_programs; pi++) { 437 const char *name = NULL; 438 439 prog = &obj->programs[pi]; 440 441 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 442 si++) { 443 GElf_Sym sym; 444 445 if (!gelf_getsym(symbols, si, &sym)) 446 continue; 447 if (sym.st_shndx != prog->idx) 448 continue; 449 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 450 continue; 451 452 name = elf_strptr(obj->efile.elf, 453 obj->efile.strtabidx, 454 sym.st_name); 455 if (!name) { 456 pr_warning("failed to get sym name string for prog %s\n", 457 prog->section_name); 458 return -LIBBPF_ERRNO__LIBELF; 459 } 460 } 461 462 if (!name && prog->idx == obj->efile.text_shndx) 463 name = ".text"; 464 465 if (!name) { 466 pr_warning("failed to find sym for prog %s\n", 467 prog->section_name); 468 return -EINVAL; 469 } 470 471 prog->name = strdup(name); 472 if (!prog->name) { 473 pr_warning("failed to allocate memory for prog sym %s\n", 474 name); 475 return -ENOMEM; 476 } 477 } 478 479 return 0; 480 } 481 482 static struct bpf_object *bpf_object__new(const char *path, 483 void *obj_buf, 484 size_t obj_buf_sz) 485 { 486 struct bpf_object *obj; 487 char *end; 488 489 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 490 if (!obj) { 491 pr_warning("alloc memory failed for %s\n", path); 492 return ERR_PTR(-ENOMEM); 493 } 494 495 strcpy(obj->path, path); 496 /* Using basename() GNU version which doesn't modify arg. */ 497 strncpy(obj->name, basename((void *)path), 498 sizeof(obj->name) - 1); 499 end = strchr(obj->name, '.'); 500 if (end) 501 *end = 0; 502 503 obj->efile.fd = -1; 504 /* 505 * Caller of this function should also calls 506 * bpf_object__elf_finish() after data collection to return 507 * obj_buf to user. If not, we should duplicate the buffer to 508 * avoid user freeing them before elf finish. 509 */ 510 obj->efile.obj_buf = obj_buf; 511 obj->efile.obj_buf_sz = obj_buf_sz; 512 obj->efile.maps_shndx = -1; 513 obj->efile.data_shndx = -1; 514 obj->efile.rodata_shndx = -1; 515 obj->efile.bss_shndx = -1; 516 517 obj->loaded = false; 518 519 INIT_LIST_HEAD(&obj->list); 520 list_add(&obj->list, &bpf_objects_list); 521 return obj; 522 } 523 524 static void bpf_object__elf_finish(struct bpf_object *obj) 525 { 526 if (!obj_elf_valid(obj)) 527 return; 528 529 if (obj->efile.elf) { 530 elf_end(obj->efile.elf); 531 obj->efile.elf = NULL; 532 } 533 obj->efile.symbols = NULL; 534 obj->efile.data = NULL; 535 obj->efile.rodata = NULL; 536 obj->efile.bss = NULL; 537 538 zfree(&obj->efile.reloc); 539 obj->efile.nr_reloc = 0; 540 zclose(obj->efile.fd); 541 obj->efile.obj_buf = NULL; 542 obj->efile.obj_buf_sz = 0; 543 } 544 545 static int bpf_object__elf_init(struct bpf_object *obj) 546 { 547 int err = 0; 548 GElf_Ehdr *ep; 549 550 if (obj_elf_valid(obj)) { 551 pr_warning("elf init: internal error\n"); 552 return -LIBBPF_ERRNO__LIBELF; 553 } 554 555 if (obj->efile.obj_buf_sz > 0) { 556 /* 557 * obj_buf should have been validated by 558 * bpf_object__open_buffer(). 559 */ 560 obj->efile.elf = elf_memory(obj->efile.obj_buf, 561 obj->efile.obj_buf_sz); 562 } else { 563 obj->efile.fd = open(obj->path, O_RDONLY); 564 if (obj->efile.fd < 0) { 565 char errmsg[STRERR_BUFSIZE]; 566 char *cp = libbpf_strerror_r(errno, errmsg, 567 sizeof(errmsg)); 568 569 pr_warning("failed to open %s: %s\n", obj->path, cp); 570 return -errno; 571 } 572 573 obj->efile.elf = elf_begin(obj->efile.fd, 574 LIBBPF_ELF_C_READ_MMAP, 575 NULL); 576 } 577 578 if (!obj->efile.elf) { 579 pr_warning("failed to open %s as ELF file\n", 580 obj->path); 581 err = -LIBBPF_ERRNO__LIBELF; 582 goto errout; 583 } 584 585 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 586 pr_warning("failed to get EHDR from %s\n", 587 obj->path); 588 err = -LIBBPF_ERRNO__FORMAT; 589 goto errout; 590 } 591 ep = &obj->efile.ehdr; 592 593 /* Old LLVM set e_machine to EM_NONE */ 594 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 595 pr_warning("%s is not an eBPF object file\n", 596 obj->path); 597 err = -LIBBPF_ERRNO__FORMAT; 598 goto errout; 599 } 600 601 return 0; 602 errout: 603 bpf_object__elf_finish(obj); 604 return err; 605 } 606 607 static int 608 bpf_object__check_endianness(struct bpf_object *obj) 609 { 610 static unsigned int const endian = 1; 611 612 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 613 case ELFDATA2LSB: 614 /* We are big endian, BPF obj is little endian. */ 615 if (*(unsigned char const *)&endian != 1) 616 goto mismatch; 617 break; 618 619 case ELFDATA2MSB: 620 /* We are little endian, BPF obj is big endian. */ 621 if (*(unsigned char const *)&endian != 0) 622 goto mismatch; 623 break; 624 default: 625 return -LIBBPF_ERRNO__ENDIAN; 626 } 627 628 return 0; 629 630 mismatch: 631 pr_warning("Error: endianness mismatch.\n"); 632 return -LIBBPF_ERRNO__ENDIAN; 633 } 634 635 static int 636 bpf_object__init_license(struct bpf_object *obj, 637 void *data, size_t size) 638 { 639 memcpy(obj->license, data, 640 min(size, sizeof(obj->license) - 1)); 641 pr_debug("license of %s is %s\n", obj->path, obj->license); 642 return 0; 643 } 644 645 static int 646 bpf_object__init_kversion(struct bpf_object *obj, 647 void *data, size_t size) 648 { 649 __u32 kver; 650 651 if (size != sizeof(kver)) { 652 pr_warning("invalid kver section in %s\n", obj->path); 653 return -LIBBPF_ERRNO__FORMAT; 654 } 655 memcpy(&kver, data, sizeof(kver)); 656 obj->kern_version = kver; 657 pr_debug("kernel version of %s is %x\n", obj->path, 658 obj->kern_version); 659 return 0; 660 } 661 662 static int compare_bpf_map(const void *_a, const void *_b) 663 { 664 const struct bpf_map *a = _a; 665 const struct bpf_map *b = _b; 666 667 return a->offset - b->offset; 668 } 669 670 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 671 { 672 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 673 type == BPF_MAP_TYPE_HASH_OF_MAPS) 674 return true; 675 return false; 676 } 677 678 static int bpf_object_search_section_size(const struct bpf_object *obj, 679 const char *name, size_t *d_size) 680 { 681 const GElf_Ehdr *ep = &obj->efile.ehdr; 682 Elf *elf = obj->efile.elf; 683 Elf_Scn *scn = NULL; 684 int idx = 0; 685 686 while ((scn = elf_nextscn(elf, scn)) != NULL) { 687 const char *sec_name; 688 Elf_Data *data; 689 GElf_Shdr sh; 690 691 idx++; 692 if (gelf_getshdr(scn, &sh) != &sh) { 693 pr_warning("failed to get section(%d) header from %s\n", 694 idx, obj->path); 695 return -EIO; 696 } 697 698 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 699 if (!sec_name) { 700 pr_warning("failed to get section(%d) name from %s\n", 701 idx, obj->path); 702 return -EIO; 703 } 704 705 if (strcmp(name, sec_name)) 706 continue; 707 708 data = elf_getdata(scn, 0); 709 if (!data) { 710 pr_warning("failed to get section(%d) data from %s(%s)\n", 711 idx, name, obj->path); 712 return -EIO; 713 } 714 715 *d_size = data->d_size; 716 return 0; 717 } 718 719 return -ENOENT; 720 } 721 722 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 723 __u32 *size) 724 { 725 int ret = -ENOENT; 726 size_t d_size; 727 728 *size = 0; 729 if (!name) { 730 return -EINVAL; 731 } else if (!strcmp(name, ".data")) { 732 if (obj->efile.data) 733 *size = obj->efile.data->d_size; 734 } else if (!strcmp(name, ".bss")) { 735 if (obj->efile.bss) 736 *size = obj->efile.bss->d_size; 737 } else if (!strcmp(name, ".rodata")) { 738 if (obj->efile.rodata) 739 *size = obj->efile.rodata->d_size; 740 } else { 741 ret = bpf_object_search_section_size(obj, name, &d_size); 742 if (!ret) 743 *size = d_size; 744 } 745 746 return *size ? 0 : ret; 747 } 748 749 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 750 __u32 *off) 751 { 752 Elf_Data *symbols = obj->efile.symbols; 753 const char *sname; 754 size_t si; 755 756 if (!name || !off) 757 return -EINVAL; 758 759 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 760 GElf_Sym sym; 761 762 if (!gelf_getsym(symbols, si, &sym)) 763 continue; 764 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 765 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 766 continue; 767 768 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 769 sym.st_name); 770 if (!sname) { 771 pr_warning("failed to get sym name string for var %s\n", 772 name); 773 return -EIO; 774 } 775 if (strcmp(name, sname) == 0) { 776 *off = sym.st_value; 777 return 0; 778 } 779 } 780 781 return -ENOENT; 782 } 783 784 static bool bpf_object__has_maps(const struct bpf_object *obj) 785 { 786 return obj->efile.maps_shndx >= 0 || 787 obj->efile.data_shndx >= 0 || 788 obj->efile.rodata_shndx >= 0 || 789 obj->efile.bss_shndx >= 0; 790 } 791 792 static int 793 bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, 794 enum libbpf_map_type type, Elf_Data *data, 795 void **data_buff) 796 { 797 struct bpf_map_def *def = &map->def; 798 char map_name[BPF_OBJ_NAME_LEN]; 799 800 map->libbpf_type = type; 801 map->offset = ~(typeof(map->offset))0; 802 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 803 libbpf_type_to_btf_name[type]); 804 map->name = strdup(map_name); 805 if (!map->name) { 806 pr_warning("failed to alloc map name\n"); 807 return -ENOMEM; 808 } 809 810 def->type = BPF_MAP_TYPE_ARRAY; 811 def->key_size = sizeof(int); 812 def->value_size = data->d_size; 813 def->max_entries = 1; 814 def->map_flags = type == LIBBPF_MAP_RODATA ? 815 BPF_F_RDONLY_PROG : 0; 816 if (data_buff) { 817 *data_buff = malloc(data->d_size); 818 if (!*data_buff) { 819 zfree(&map->name); 820 pr_warning("failed to alloc map content buffer\n"); 821 return -ENOMEM; 822 } 823 memcpy(*data_buff, data->d_buf, data->d_size); 824 } 825 826 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 827 return 0; 828 } 829 830 static int 831 bpf_object__init_maps(struct bpf_object *obj, int flags) 832 { 833 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; 834 bool strict = !(flags & MAPS_RELAX_COMPAT); 835 Elf_Data *symbols = obj->efile.symbols; 836 Elf_Data *data = NULL; 837 int ret = 0; 838 839 if (!symbols) 840 return -EINVAL; 841 nr_syms = symbols->d_size / sizeof(GElf_Sym); 842 843 if (obj->efile.maps_shndx >= 0) { 844 Elf_Scn *scn = elf_getscn(obj->efile.elf, 845 obj->efile.maps_shndx); 846 847 if (scn) 848 data = elf_getdata(scn, NULL); 849 if (!scn || !data) { 850 pr_warning("failed to get Elf_Data from map section %d\n", 851 obj->efile.maps_shndx); 852 return -EINVAL; 853 } 854 } 855 856 /* 857 * Count number of maps. Each map has a name. 858 * Array of maps is not supported: only the first element is 859 * considered. 860 * 861 * TODO: Detect array of map and report error. 862 */ 863 if (obj->caps.global_data) { 864 if (obj->efile.data_shndx >= 0) 865 nr_maps_glob++; 866 if (obj->efile.rodata_shndx >= 0) 867 nr_maps_glob++; 868 if (obj->efile.bss_shndx >= 0) 869 nr_maps_glob++; 870 } 871 872 for (i = 0; data && i < nr_syms; i++) { 873 GElf_Sym sym; 874 875 if (!gelf_getsym(symbols, i, &sym)) 876 continue; 877 if (sym.st_shndx != obj->efile.maps_shndx) 878 continue; 879 nr_maps++; 880 } 881 882 if (!nr_maps && !nr_maps_glob) 883 return 0; 884 885 /* Assume equally sized map definitions */ 886 if (data) { 887 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 888 nr_maps, data->d_size); 889 890 map_def_sz = data->d_size / nr_maps; 891 if (!data->d_size || (data->d_size % nr_maps) != 0) { 892 pr_warning("unable to determine map definition size " 893 "section %s, %d maps in %zd bytes\n", 894 obj->path, nr_maps, data->d_size); 895 return -EINVAL; 896 } 897 } 898 899 nr_maps += nr_maps_glob; 900 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 901 if (!obj->maps) { 902 pr_warning("alloc maps for object failed\n"); 903 return -ENOMEM; 904 } 905 obj->nr_maps = nr_maps; 906 907 for (i = 0; i < nr_maps; i++) { 908 /* 909 * fill all fd with -1 so won't close incorrect 910 * fd (fd=0 is stdin) when failure (zclose won't close 911 * negative fd)). 912 */ 913 obj->maps[i].fd = -1; 914 obj->maps[i].inner_map_fd = -1; 915 } 916 917 /* 918 * Fill obj->maps using data in "maps" section. 919 */ 920 for (i = 0, map_idx = 0; data && i < nr_syms; i++) { 921 GElf_Sym sym; 922 const char *map_name; 923 struct bpf_map_def *def; 924 925 if (!gelf_getsym(symbols, i, &sym)) 926 continue; 927 if (sym.st_shndx != obj->efile.maps_shndx) 928 continue; 929 930 map_name = elf_strptr(obj->efile.elf, 931 obj->efile.strtabidx, 932 sym.st_name); 933 934 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; 935 obj->maps[map_idx].offset = sym.st_value; 936 if (sym.st_value + map_def_sz > data->d_size) { 937 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 938 obj->path, map_name); 939 return -EINVAL; 940 } 941 942 obj->maps[map_idx].name = strdup(map_name); 943 if (!obj->maps[map_idx].name) { 944 pr_warning("failed to alloc map name\n"); 945 return -ENOMEM; 946 } 947 pr_debug("map %d is \"%s\"\n", map_idx, 948 obj->maps[map_idx].name); 949 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 950 /* 951 * If the definition of the map in the object file fits in 952 * bpf_map_def, copy it. Any extra fields in our version 953 * of bpf_map_def will default to zero as a result of the 954 * calloc above. 955 */ 956 if (map_def_sz <= sizeof(struct bpf_map_def)) { 957 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 958 } else { 959 /* 960 * Here the map structure being read is bigger than what 961 * we expect, truncate if the excess bits are all zero. 962 * If they are not zero, reject this map as 963 * incompatible. 964 */ 965 char *b; 966 for (b = ((char *)def) + sizeof(struct bpf_map_def); 967 b < ((char *)def) + map_def_sz; b++) { 968 if (*b != 0) { 969 pr_warning("maps section in %s: \"%s\" " 970 "has unrecognized, non-zero " 971 "options\n", 972 obj->path, map_name); 973 if (strict) 974 return -EINVAL; 975 } 976 } 977 memcpy(&obj->maps[map_idx].def, def, 978 sizeof(struct bpf_map_def)); 979 } 980 map_idx++; 981 } 982 983 if (!obj->caps.global_data) 984 goto finalize; 985 986 /* 987 * Populate rest of obj->maps with libbpf internal maps. 988 */ 989 if (obj->efile.data_shndx >= 0) 990 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 991 LIBBPF_MAP_DATA, 992 obj->efile.data, 993 &obj->sections.data); 994 if (!ret && obj->efile.rodata_shndx >= 0) 995 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 996 LIBBPF_MAP_RODATA, 997 obj->efile.rodata, 998 &obj->sections.rodata); 999 if (!ret && obj->efile.bss_shndx >= 0) 1000 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 1001 LIBBPF_MAP_BSS, 1002 obj->efile.bss, NULL); 1003 finalize: 1004 if (!ret) 1005 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1006 compare_bpf_map); 1007 return ret; 1008 } 1009 1010 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1011 { 1012 Elf_Scn *scn; 1013 GElf_Shdr sh; 1014 1015 scn = elf_getscn(obj->efile.elf, idx); 1016 if (!scn) 1017 return false; 1018 1019 if (gelf_getshdr(scn, &sh) != &sh) 1020 return false; 1021 1022 if (sh.sh_flags & SHF_EXECINSTR) 1023 return true; 1024 1025 return false; 1026 } 1027 1028 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1029 { 1030 bool has_datasec = obj->caps.btf_datasec; 1031 bool has_func = obj->caps.btf_func; 1032 struct btf *btf = obj->btf; 1033 struct btf_type *t; 1034 int i, j, vlen; 1035 __u16 kind; 1036 1037 if (!obj->btf || (has_func && has_datasec)) 1038 return; 1039 1040 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1041 t = (struct btf_type *)btf__type_by_id(btf, i); 1042 kind = BTF_INFO_KIND(t->info); 1043 1044 if (!has_datasec && kind == BTF_KIND_VAR) { 1045 /* replace VAR with INT */ 1046 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1047 t->size = sizeof(int); 1048 *(int *)(t+1) = BTF_INT_ENC(0, 0, 32); 1049 } else if (!has_datasec && kind == BTF_KIND_DATASEC) { 1050 /* replace DATASEC with STRUCT */ 1051 struct btf_var_secinfo *v = (void *)(t + 1); 1052 struct btf_member *m = (void *)(t + 1); 1053 struct btf_type *vt; 1054 char *name; 1055 1056 name = (char *)btf__name_by_offset(btf, t->name_off); 1057 while (*name) { 1058 if (*name == '.') 1059 *name = '_'; 1060 name++; 1061 } 1062 1063 vlen = BTF_INFO_VLEN(t->info); 1064 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1065 for (j = 0; j < vlen; j++, v++, m++) { 1066 /* order of field assignments is important */ 1067 m->offset = v->offset * 8; 1068 m->type = v->type; 1069 /* preserve variable name as member name */ 1070 vt = (void *)btf__type_by_id(btf, v->type); 1071 m->name_off = vt->name_off; 1072 } 1073 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) { 1074 /* replace FUNC_PROTO with ENUM */ 1075 vlen = BTF_INFO_VLEN(t->info); 1076 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1077 t->size = sizeof(__u32); /* kernel enforced */ 1078 } else if (!has_func && kind == BTF_KIND_FUNC) { 1079 /* replace FUNC with TYPEDEF */ 1080 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1081 } 1082 } 1083 } 1084 1085 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1086 { 1087 if (!obj->btf_ext) 1088 return; 1089 1090 if (!obj->caps.btf_func) { 1091 btf_ext__free(obj->btf_ext); 1092 obj->btf_ext = NULL; 1093 } 1094 } 1095 1096 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1097 { 1098 Elf *elf = obj->efile.elf; 1099 GElf_Ehdr *ep = &obj->efile.ehdr; 1100 Elf_Data *btf_ext_data = NULL; 1101 Elf_Data *btf_data = NULL; 1102 Elf_Scn *scn = NULL; 1103 int idx = 0, err = 0; 1104 1105 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1106 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1107 pr_warning("failed to get e_shstrndx from %s\n", 1108 obj->path); 1109 return -LIBBPF_ERRNO__FORMAT; 1110 } 1111 1112 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1113 char *name; 1114 GElf_Shdr sh; 1115 Elf_Data *data; 1116 1117 idx++; 1118 if (gelf_getshdr(scn, &sh) != &sh) { 1119 pr_warning("failed to get section(%d) header from %s\n", 1120 idx, obj->path); 1121 err = -LIBBPF_ERRNO__FORMAT; 1122 goto out; 1123 } 1124 1125 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1126 if (!name) { 1127 pr_warning("failed to get section(%d) name from %s\n", 1128 idx, obj->path); 1129 err = -LIBBPF_ERRNO__FORMAT; 1130 goto out; 1131 } 1132 1133 data = elf_getdata(scn, 0); 1134 if (!data) { 1135 pr_warning("failed to get section(%d) data from %s(%s)\n", 1136 idx, name, obj->path); 1137 err = -LIBBPF_ERRNO__FORMAT; 1138 goto out; 1139 } 1140 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1141 idx, name, (unsigned long)data->d_size, 1142 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1143 (int)sh.sh_type); 1144 1145 if (strcmp(name, "license") == 0) { 1146 err = bpf_object__init_license(obj, 1147 data->d_buf, 1148 data->d_size); 1149 } else if (strcmp(name, "version") == 0) { 1150 err = bpf_object__init_kversion(obj, 1151 data->d_buf, 1152 data->d_size); 1153 } else if (strcmp(name, "maps") == 0) { 1154 obj->efile.maps_shndx = idx; 1155 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1156 btf_data = data; 1157 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1158 btf_ext_data = data; 1159 } else if (sh.sh_type == SHT_SYMTAB) { 1160 if (obj->efile.symbols) { 1161 pr_warning("bpf: multiple SYMTAB in %s\n", 1162 obj->path); 1163 err = -LIBBPF_ERRNO__FORMAT; 1164 } else { 1165 obj->efile.symbols = data; 1166 obj->efile.strtabidx = sh.sh_link; 1167 } 1168 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1169 if (sh.sh_flags & SHF_EXECINSTR) { 1170 if (strcmp(name, ".text") == 0) 1171 obj->efile.text_shndx = idx; 1172 err = bpf_object__add_program(obj, data->d_buf, 1173 data->d_size, name, idx); 1174 if (err) { 1175 char errmsg[STRERR_BUFSIZE]; 1176 char *cp = libbpf_strerror_r(-err, errmsg, 1177 sizeof(errmsg)); 1178 1179 pr_warning("failed to alloc program %s (%s): %s", 1180 name, obj->path, cp); 1181 } 1182 } else if (strcmp(name, ".data") == 0) { 1183 obj->efile.data = data; 1184 obj->efile.data_shndx = idx; 1185 } else if (strcmp(name, ".rodata") == 0) { 1186 obj->efile.rodata = data; 1187 obj->efile.rodata_shndx = idx; 1188 } else { 1189 pr_debug("skip section(%d) %s\n", idx, name); 1190 } 1191 } else if (sh.sh_type == SHT_REL) { 1192 void *reloc = obj->efile.reloc; 1193 int nr_reloc = obj->efile.nr_reloc + 1; 1194 int sec = sh.sh_info; /* points to other section */ 1195 1196 /* Only do relo for section with exec instructions */ 1197 if (!section_have_execinstr(obj, sec)) { 1198 pr_debug("skip relo %s(%d) for section(%d)\n", 1199 name, idx, sec); 1200 continue; 1201 } 1202 1203 reloc = reallocarray(reloc, nr_reloc, 1204 sizeof(*obj->efile.reloc)); 1205 if (!reloc) { 1206 pr_warning("realloc failed\n"); 1207 err = -ENOMEM; 1208 } else { 1209 int n = nr_reloc - 1; 1210 1211 obj->efile.reloc = reloc; 1212 obj->efile.nr_reloc = nr_reloc; 1213 1214 obj->efile.reloc[n].shdr = sh; 1215 obj->efile.reloc[n].data = data; 1216 } 1217 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1218 obj->efile.bss = data; 1219 obj->efile.bss_shndx = idx; 1220 } else { 1221 pr_debug("skip section(%d) %s\n", idx, name); 1222 } 1223 if (err) 1224 goto out; 1225 } 1226 1227 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1228 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1229 return LIBBPF_ERRNO__FORMAT; 1230 } 1231 if (btf_data) { 1232 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1233 if (IS_ERR(obj->btf)) { 1234 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1235 BTF_ELF_SEC, PTR_ERR(obj->btf)); 1236 obj->btf = NULL; 1237 } else { 1238 err = btf__finalize_data(obj, obj->btf); 1239 if (!err) { 1240 bpf_object__sanitize_btf(obj); 1241 err = btf__load(obj->btf); 1242 } 1243 if (err) { 1244 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", 1245 BTF_ELF_SEC, err); 1246 btf__free(obj->btf); 1247 obj->btf = NULL; 1248 err = 0; 1249 } 1250 } 1251 } 1252 if (btf_ext_data) { 1253 if (!obj->btf) { 1254 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1255 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1256 } else { 1257 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1258 btf_ext_data->d_size); 1259 if (IS_ERR(obj->btf_ext)) { 1260 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1261 BTF_EXT_ELF_SEC, 1262 PTR_ERR(obj->btf_ext)); 1263 obj->btf_ext = NULL; 1264 } else { 1265 bpf_object__sanitize_btf_ext(obj); 1266 } 1267 } 1268 } 1269 if (bpf_object__has_maps(obj)) { 1270 err = bpf_object__init_maps(obj, flags); 1271 if (err) 1272 goto out; 1273 } 1274 err = bpf_object__init_prog_names(obj); 1275 out: 1276 return err; 1277 } 1278 1279 static struct bpf_program * 1280 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1281 { 1282 struct bpf_program *prog; 1283 size_t i; 1284 1285 for (i = 0; i < obj->nr_programs; i++) { 1286 prog = &obj->programs[i]; 1287 if (prog->idx == idx) 1288 return prog; 1289 } 1290 return NULL; 1291 } 1292 1293 struct bpf_program * 1294 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 1295 { 1296 struct bpf_program *pos; 1297 1298 bpf_object__for_each_program(pos, obj) { 1299 if (pos->section_name && !strcmp(pos->section_name, title)) 1300 return pos; 1301 } 1302 return NULL; 1303 } 1304 1305 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1306 int shndx) 1307 { 1308 return shndx == obj->efile.data_shndx || 1309 shndx == obj->efile.bss_shndx || 1310 shndx == obj->efile.rodata_shndx; 1311 } 1312 1313 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1314 int shndx) 1315 { 1316 return shndx == obj->efile.maps_shndx; 1317 } 1318 1319 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1320 int shndx) 1321 { 1322 return shndx == obj->efile.text_shndx || 1323 bpf_object__shndx_is_maps(obj, shndx) || 1324 bpf_object__shndx_is_data(obj, shndx); 1325 } 1326 1327 static enum libbpf_map_type 1328 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1329 { 1330 if (shndx == obj->efile.data_shndx) 1331 return LIBBPF_MAP_DATA; 1332 else if (shndx == obj->efile.bss_shndx) 1333 return LIBBPF_MAP_BSS; 1334 else if (shndx == obj->efile.rodata_shndx) 1335 return LIBBPF_MAP_RODATA; 1336 else 1337 return LIBBPF_MAP_UNSPEC; 1338 } 1339 1340 static int 1341 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1342 Elf_Data *data, struct bpf_object *obj) 1343 { 1344 Elf_Data *symbols = obj->efile.symbols; 1345 struct bpf_map *maps = obj->maps; 1346 size_t nr_maps = obj->nr_maps; 1347 int i, nrels; 1348 1349 pr_debug("collecting relocating info for: '%s'\n", 1350 prog->section_name); 1351 nrels = shdr->sh_size / shdr->sh_entsize; 1352 1353 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1354 if (!prog->reloc_desc) { 1355 pr_warning("failed to alloc memory in relocation\n"); 1356 return -ENOMEM; 1357 } 1358 prog->nr_reloc = nrels; 1359 1360 for (i = 0; i < nrels; i++) { 1361 GElf_Sym sym; 1362 GElf_Rel rel; 1363 unsigned int insn_idx; 1364 unsigned int shdr_idx; 1365 struct bpf_insn *insns = prog->insns; 1366 enum libbpf_map_type type; 1367 const char *name; 1368 size_t map_idx; 1369 1370 if (!gelf_getrel(data, i, &rel)) { 1371 pr_warning("relocation: failed to get %d reloc\n", i); 1372 return -LIBBPF_ERRNO__FORMAT; 1373 } 1374 1375 if (!gelf_getsym(symbols, 1376 GELF_R_SYM(rel.r_info), 1377 &sym)) { 1378 pr_warning("relocation: symbol %"PRIx64" not found\n", 1379 GELF_R_SYM(rel.r_info)); 1380 return -LIBBPF_ERRNO__FORMAT; 1381 } 1382 1383 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1384 sym.st_name) ? : "<?>"; 1385 1386 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1387 (long long) (rel.r_info >> 32), 1388 (long long) sym.st_value, sym.st_name, name); 1389 1390 shdr_idx = sym.st_shndx; 1391 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1392 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1393 prog->section_name, shdr_idx); 1394 return -LIBBPF_ERRNO__RELOC; 1395 } 1396 1397 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1398 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1399 1400 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1401 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1402 pr_warning("incorrect bpf_call opcode\n"); 1403 return -LIBBPF_ERRNO__RELOC; 1404 } 1405 prog->reloc_desc[i].type = RELO_CALL; 1406 prog->reloc_desc[i].insn_idx = insn_idx; 1407 prog->reloc_desc[i].text_off = sym.st_value; 1408 obj->has_pseudo_calls = true; 1409 continue; 1410 } 1411 1412 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1413 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1414 insn_idx, insns[insn_idx].code); 1415 return -LIBBPF_ERRNO__RELOC; 1416 } 1417 1418 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1419 bpf_object__shndx_is_data(obj, shdr_idx)) { 1420 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1421 if (type != LIBBPF_MAP_UNSPEC) { 1422 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1423 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1424 name, insn_idx, insns[insn_idx].code); 1425 return -LIBBPF_ERRNO__RELOC; 1426 } 1427 if (!obj->caps.global_data) { 1428 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1429 name, insn_idx); 1430 return -LIBBPF_ERRNO__RELOC; 1431 } 1432 } 1433 1434 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1435 if (maps[map_idx].libbpf_type != type) 1436 continue; 1437 if (type != LIBBPF_MAP_UNSPEC || 1438 (type == LIBBPF_MAP_UNSPEC && 1439 maps[map_idx].offset == sym.st_value)) { 1440 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1441 map_idx, maps[map_idx].name, insn_idx); 1442 break; 1443 } 1444 } 1445 1446 if (map_idx >= nr_maps) { 1447 pr_warning("bpf relocation: map_idx %d large than %d\n", 1448 (int)map_idx, (int)nr_maps - 1); 1449 return -LIBBPF_ERRNO__RELOC; 1450 } 1451 1452 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1453 RELO_DATA : RELO_LD64; 1454 prog->reloc_desc[i].insn_idx = insn_idx; 1455 prog->reloc_desc[i].map_idx = map_idx; 1456 } 1457 } 1458 return 0; 1459 } 1460 1461 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1462 { 1463 struct bpf_map_def *def = &map->def; 1464 __u32 key_type_id = 0, value_type_id = 0; 1465 int ret; 1466 1467 if (!bpf_map__is_internal(map)) { 1468 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1469 def->value_size, &key_type_id, 1470 &value_type_id); 1471 } else { 1472 /* 1473 * LLVM annotates global data differently in BTF, that is, 1474 * only as '.data', '.bss' or '.rodata'. 1475 */ 1476 ret = btf__find_by_name(btf, 1477 libbpf_type_to_btf_name[map->libbpf_type]); 1478 } 1479 if (ret < 0) 1480 return ret; 1481 1482 map->btf_key_type_id = key_type_id; 1483 map->btf_value_type_id = bpf_map__is_internal(map) ? 1484 ret : value_type_id; 1485 return 0; 1486 } 1487 1488 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1489 { 1490 struct bpf_map_info info = {}; 1491 __u32 len = sizeof(info); 1492 int new_fd, err; 1493 char *new_name; 1494 1495 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1496 if (err) 1497 return err; 1498 1499 new_name = strdup(info.name); 1500 if (!new_name) 1501 return -errno; 1502 1503 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1504 if (new_fd < 0) 1505 goto err_free_new_name; 1506 1507 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1508 if (new_fd < 0) 1509 goto err_close_new_fd; 1510 1511 err = zclose(map->fd); 1512 if (err) 1513 goto err_close_new_fd; 1514 free(map->name); 1515 1516 map->fd = new_fd; 1517 map->name = new_name; 1518 map->def.type = info.type; 1519 map->def.key_size = info.key_size; 1520 map->def.value_size = info.value_size; 1521 map->def.max_entries = info.max_entries; 1522 map->def.map_flags = info.map_flags; 1523 map->btf_key_type_id = info.btf_key_type_id; 1524 map->btf_value_type_id = info.btf_value_type_id; 1525 1526 return 0; 1527 1528 err_close_new_fd: 1529 close(new_fd); 1530 err_free_new_name: 1531 free(new_name); 1532 return -errno; 1533 } 1534 1535 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1536 { 1537 if (!map || !max_entries) 1538 return -EINVAL; 1539 1540 /* If map already created, its attributes can't be changed. */ 1541 if (map->fd >= 0) 1542 return -EBUSY; 1543 1544 map->def.max_entries = max_entries; 1545 1546 return 0; 1547 } 1548 1549 static int 1550 bpf_object__probe_name(struct bpf_object *obj) 1551 { 1552 struct bpf_load_program_attr attr; 1553 char *cp, errmsg[STRERR_BUFSIZE]; 1554 struct bpf_insn insns[] = { 1555 BPF_MOV64_IMM(BPF_REG_0, 0), 1556 BPF_EXIT_INSN(), 1557 }; 1558 int ret; 1559 1560 /* make sure basic loading works */ 1561 1562 memset(&attr, 0, sizeof(attr)); 1563 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1564 attr.insns = insns; 1565 attr.insns_cnt = ARRAY_SIZE(insns); 1566 attr.license = "GPL"; 1567 1568 ret = bpf_load_program_xattr(&attr, NULL, 0); 1569 if (ret < 0) { 1570 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1571 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1572 __func__, cp, errno); 1573 return -errno; 1574 } 1575 close(ret); 1576 1577 /* now try the same program, but with the name */ 1578 1579 attr.name = "test"; 1580 ret = bpf_load_program_xattr(&attr, NULL, 0); 1581 if (ret >= 0) { 1582 obj->caps.name = 1; 1583 close(ret); 1584 } 1585 1586 return 0; 1587 } 1588 1589 static int 1590 bpf_object__probe_global_data(struct bpf_object *obj) 1591 { 1592 struct bpf_load_program_attr prg_attr; 1593 struct bpf_create_map_attr map_attr; 1594 char *cp, errmsg[STRERR_BUFSIZE]; 1595 struct bpf_insn insns[] = { 1596 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1597 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1598 BPF_MOV64_IMM(BPF_REG_0, 0), 1599 BPF_EXIT_INSN(), 1600 }; 1601 int ret, map; 1602 1603 memset(&map_attr, 0, sizeof(map_attr)); 1604 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1605 map_attr.key_size = sizeof(int); 1606 map_attr.value_size = 32; 1607 map_attr.max_entries = 1; 1608 1609 map = bpf_create_map_xattr(&map_attr); 1610 if (map < 0) { 1611 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1612 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1613 __func__, cp, errno); 1614 return -errno; 1615 } 1616 1617 insns[0].imm = map; 1618 1619 memset(&prg_attr, 0, sizeof(prg_attr)); 1620 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1621 prg_attr.insns = insns; 1622 prg_attr.insns_cnt = ARRAY_SIZE(insns); 1623 prg_attr.license = "GPL"; 1624 1625 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 1626 if (ret >= 0) { 1627 obj->caps.global_data = 1; 1628 close(ret); 1629 } 1630 1631 close(map); 1632 return 0; 1633 } 1634 1635 static int bpf_object__probe_btf_func(struct bpf_object *obj) 1636 { 1637 const char strs[] = "\0int\0x\0a"; 1638 /* void x(int a) {} */ 1639 __u32 types[] = { 1640 /* int */ 1641 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1642 /* FUNC_PROTO */ /* [2] */ 1643 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 1644 BTF_PARAM_ENC(7, 1), 1645 /* FUNC x */ /* [3] */ 1646 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 1647 }; 1648 int res; 1649 1650 res = libbpf__probe_raw_btf((char *)types, sizeof(types), 1651 strs, sizeof(strs)); 1652 if (res < 0) 1653 return res; 1654 if (res > 0) 1655 obj->caps.btf_func = 1; 1656 return 0; 1657 } 1658 1659 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 1660 { 1661 const char strs[] = "\0x\0.data"; 1662 /* static int a; */ 1663 __u32 types[] = { 1664 /* int */ 1665 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1666 /* VAR x */ /* [2] */ 1667 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 1668 BTF_VAR_STATIC, 1669 /* DATASEC val */ /* [3] */ 1670 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 1671 BTF_VAR_SECINFO_ENC(2, 0, 4), 1672 }; 1673 int res; 1674 1675 res = libbpf__probe_raw_btf((char *)types, sizeof(types), 1676 strs, sizeof(strs)); 1677 if (res < 0) 1678 return res; 1679 if (res > 0) 1680 obj->caps.btf_datasec = 1; 1681 return 0; 1682 } 1683 1684 static int 1685 bpf_object__probe_caps(struct bpf_object *obj) 1686 { 1687 int (*probe_fn[])(struct bpf_object *obj) = { 1688 bpf_object__probe_name, 1689 bpf_object__probe_global_data, 1690 bpf_object__probe_btf_func, 1691 bpf_object__probe_btf_datasec, 1692 }; 1693 int i, ret; 1694 1695 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 1696 ret = probe_fn[i](obj); 1697 if (ret < 0) 1698 pr_debug("Probe #%d failed with %d.\n", i, ret); 1699 } 1700 1701 return 0; 1702 } 1703 1704 static int 1705 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 1706 { 1707 char *cp, errmsg[STRERR_BUFSIZE]; 1708 int err, zero = 0; 1709 __u8 *data; 1710 1711 /* Nothing to do here since kernel already zero-initializes .bss map. */ 1712 if (map->libbpf_type == LIBBPF_MAP_BSS) 1713 return 0; 1714 1715 data = map->libbpf_type == LIBBPF_MAP_DATA ? 1716 obj->sections.data : obj->sections.rodata; 1717 1718 err = bpf_map_update_elem(map->fd, &zero, data, 0); 1719 /* Freeze .rodata map as read-only from syscall side. */ 1720 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 1721 err = bpf_map_freeze(map->fd); 1722 if (err) { 1723 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1724 pr_warning("Error freezing map(%s) as read-only: %s\n", 1725 map->name, cp); 1726 err = 0; 1727 } 1728 } 1729 return err; 1730 } 1731 1732 static int 1733 bpf_object__create_maps(struct bpf_object *obj) 1734 { 1735 struct bpf_create_map_attr create_attr = {}; 1736 unsigned int i; 1737 int err; 1738 1739 for (i = 0; i < obj->nr_maps; i++) { 1740 struct bpf_map *map = &obj->maps[i]; 1741 struct bpf_map_def *def = &map->def; 1742 char *cp, errmsg[STRERR_BUFSIZE]; 1743 int *pfd = &map->fd; 1744 1745 if (map->fd >= 0) { 1746 pr_debug("skip map create (preset) %s: fd=%d\n", 1747 map->name, map->fd); 1748 continue; 1749 } 1750 1751 if (obj->caps.name) 1752 create_attr.name = map->name; 1753 create_attr.map_ifindex = map->map_ifindex; 1754 create_attr.map_type = def->type; 1755 create_attr.map_flags = def->map_flags; 1756 create_attr.key_size = def->key_size; 1757 create_attr.value_size = def->value_size; 1758 create_attr.max_entries = def->max_entries; 1759 create_attr.btf_fd = 0; 1760 create_attr.btf_key_type_id = 0; 1761 create_attr.btf_value_type_id = 0; 1762 if (bpf_map_type__is_map_in_map(def->type) && 1763 map->inner_map_fd >= 0) 1764 create_attr.inner_map_fd = map->inner_map_fd; 1765 1766 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1767 create_attr.btf_fd = btf__fd(obj->btf); 1768 create_attr.btf_key_type_id = map->btf_key_type_id; 1769 create_attr.btf_value_type_id = map->btf_value_type_id; 1770 } 1771 1772 *pfd = bpf_create_map_xattr(&create_attr); 1773 if (*pfd < 0 && create_attr.btf_key_type_id) { 1774 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1775 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1776 map->name, cp, errno); 1777 create_attr.btf_fd = 0; 1778 create_attr.btf_key_type_id = 0; 1779 create_attr.btf_value_type_id = 0; 1780 map->btf_key_type_id = 0; 1781 map->btf_value_type_id = 0; 1782 *pfd = bpf_create_map_xattr(&create_attr); 1783 } 1784 1785 if (*pfd < 0) { 1786 size_t j; 1787 1788 err = *pfd; 1789 err_out: 1790 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1791 pr_warning("failed to create map (name: '%s'): %s\n", 1792 map->name, cp); 1793 for (j = 0; j < i; j++) 1794 zclose(obj->maps[j].fd); 1795 return err; 1796 } 1797 1798 if (bpf_map__is_internal(map)) { 1799 err = bpf_object__populate_internal_map(obj, map); 1800 if (err < 0) { 1801 zclose(*pfd); 1802 goto err_out; 1803 } 1804 } 1805 1806 pr_debug("create map %s: fd=%d\n", map->name, *pfd); 1807 } 1808 1809 return 0; 1810 } 1811 1812 static int 1813 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1814 void *btf_prog_info, const char *info_name) 1815 { 1816 if (err != -ENOENT) { 1817 pr_warning("Error in loading %s for sec %s.\n", 1818 info_name, prog->section_name); 1819 return err; 1820 } 1821 1822 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1823 1824 if (btf_prog_info) { 1825 /* 1826 * Some info has already been found but has problem 1827 * in the last btf_ext reloc. Must have to error 1828 * out. 1829 */ 1830 pr_warning("Error in relocating %s for sec %s.\n", 1831 info_name, prog->section_name); 1832 return err; 1833 } 1834 1835 /* 1836 * Have problem loading the very first info. Ignore 1837 * the rest. 1838 */ 1839 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1840 info_name, prog->section_name, info_name); 1841 return 0; 1842 } 1843 1844 static int 1845 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1846 const char *section_name, __u32 insn_offset) 1847 { 1848 int err; 1849 1850 if (!insn_offset || prog->func_info) { 1851 /* 1852 * !insn_offset => main program 1853 * 1854 * For sub prog, the main program's func_info has to 1855 * be loaded first (i.e. prog->func_info != NULL) 1856 */ 1857 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1858 section_name, insn_offset, 1859 &prog->func_info, 1860 &prog->func_info_cnt); 1861 if (err) 1862 return check_btf_ext_reloc_err(prog, err, 1863 prog->func_info, 1864 "bpf_func_info"); 1865 1866 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1867 } 1868 1869 if (!insn_offset || prog->line_info) { 1870 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1871 section_name, insn_offset, 1872 &prog->line_info, 1873 &prog->line_info_cnt); 1874 if (err) 1875 return check_btf_ext_reloc_err(prog, err, 1876 prog->line_info, 1877 "bpf_line_info"); 1878 1879 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1880 } 1881 1882 if (!insn_offset) 1883 prog->btf_fd = btf__fd(obj->btf); 1884 1885 return 0; 1886 } 1887 1888 static int 1889 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1890 struct reloc_desc *relo) 1891 { 1892 struct bpf_insn *insn, *new_insn; 1893 struct bpf_program *text; 1894 size_t new_cnt; 1895 int err; 1896 1897 if (relo->type != RELO_CALL) 1898 return -LIBBPF_ERRNO__RELOC; 1899 1900 if (prog->idx == obj->efile.text_shndx) { 1901 pr_warning("relo in .text insn %d into off %d\n", 1902 relo->insn_idx, relo->text_off); 1903 return -LIBBPF_ERRNO__RELOC; 1904 } 1905 1906 if (prog->main_prog_cnt == 0) { 1907 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1908 if (!text) { 1909 pr_warning("no .text section found yet relo into text exist\n"); 1910 return -LIBBPF_ERRNO__RELOC; 1911 } 1912 new_cnt = prog->insns_cnt + text->insns_cnt; 1913 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1914 if (!new_insn) { 1915 pr_warning("oom in prog realloc\n"); 1916 return -ENOMEM; 1917 } 1918 1919 if (obj->btf_ext) { 1920 err = bpf_program_reloc_btf_ext(prog, obj, 1921 text->section_name, 1922 prog->insns_cnt); 1923 if (err) 1924 return err; 1925 } 1926 1927 memcpy(new_insn + prog->insns_cnt, text->insns, 1928 text->insns_cnt * sizeof(*insn)); 1929 prog->insns = new_insn; 1930 prog->main_prog_cnt = prog->insns_cnt; 1931 prog->insns_cnt = new_cnt; 1932 pr_debug("added %zd insn from %s to prog %s\n", 1933 text->insns_cnt, text->section_name, 1934 prog->section_name); 1935 } 1936 insn = &prog->insns[relo->insn_idx]; 1937 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1938 return 0; 1939 } 1940 1941 static int 1942 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1943 { 1944 int i, err; 1945 1946 if (!prog) 1947 return 0; 1948 1949 if (obj->btf_ext) { 1950 err = bpf_program_reloc_btf_ext(prog, obj, 1951 prog->section_name, 0); 1952 if (err) 1953 return err; 1954 } 1955 1956 if (!prog->reloc_desc) 1957 return 0; 1958 1959 for (i = 0; i < prog->nr_reloc; i++) { 1960 if (prog->reloc_desc[i].type == RELO_LD64 || 1961 prog->reloc_desc[i].type == RELO_DATA) { 1962 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 1963 struct bpf_insn *insns = prog->insns; 1964 int insn_idx, map_idx; 1965 1966 insn_idx = prog->reloc_desc[i].insn_idx; 1967 map_idx = prog->reloc_desc[i].map_idx; 1968 1969 if (insn_idx + 1 >= (int)prog->insns_cnt) { 1970 pr_warning("relocation out of range: '%s'\n", 1971 prog->section_name); 1972 return -LIBBPF_ERRNO__RELOC; 1973 } 1974 1975 if (!relo_data) { 1976 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1977 } else { 1978 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 1979 insns[insn_idx + 1].imm = insns[insn_idx].imm; 1980 } 1981 insns[insn_idx].imm = obj->maps[map_idx].fd; 1982 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1983 err = bpf_program__reloc_text(prog, obj, 1984 &prog->reloc_desc[i]); 1985 if (err) 1986 return err; 1987 } 1988 } 1989 1990 zfree(&prog->reloc_desc); 1991 prog->nr_reloc = 0; 1992 return 0; 1993 } 1994 1995 1996 static int 1997 bpf_object__relocate(struct bpf_object *obj) 1998 { 1999 struct bpf_program *prog; 2000 size_t i; 2001 int err; 2002 2003 for (i = 0; i < obj->nr_programs; i++) { 2004 prog = &obj->programs[i]; 2005 2006 err = bpf_program__relocate(prog, obj); 2007 if (err) { 2008 pr_warning("failed to relocate '%s'\n", 2009 prog->section_name); 2010 return err; 2011 } 2012 } 2013 return 0; 2014 } 2015 2016 static int bpf_object__collect_reloc(struct bpf_object *obj) 2017 { 2018 int i, err; 2019 2020 if (!obj_elf_valid(obj)) { 2021 pr_warning("Internal error: elf object is closed\n"); 2022 return -LIBBPF_ERRNO__INTERNAL; 2023 } 2024 2025 for (i = 0; i < obj->efile.nr_reloc; i++) { 2026 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2027 Elf_Data *data = obj->efile.reloc[i].data; 2028 int idx = shdr->sh_info; 2029 struct bpf_program *prog; 2030 2031 if (shdr->sh_type != SHT_REL) { 2032 pr_warning("internal error at %d\n", __LINE__); 2033 return -LIBBPF_ERRNO__INTERNAL; 2034 } 2035 2036 prog = bpf_object__find_prog_by_idx(obj, idx); 2037 if (!prog) { 2038 pr_warning("relocation failed: no section(%d)\n", idx); 2039 return -LIBBPF_ERRNO__RELOC; 2040 } 2041 2042 err = bpf_program__collect_reloc(prog, 2043 shdr, data, 2044 obj); 2045 if (err) 2046 return err; 2047 } 2048 return 0; 2049 } 2050 2051 static int 2052 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2053 char *license, __u32 kern_version, int *pfd) 2054 { 2055 struct bpf_load_program_attr load_attr; 2056 char *cp, errmsg[STRERR_BUFSIZE]; 2057 int log_buf_size = BPF_LOG_BUF_SIZE; 2058 char *log_buf; 2059 int ret; 2060 2061 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2062 load_attr.prog_type = prog->type; 2063 load_attr.expected_attach_type = prog->expected_attach_type; 2064 if (prog->caps->name) 2065 load_attr.name = prog->name; 2066 load_attr.insns = insns; 2067 load_attr.insns_cnt = insns_cnt; 2068 load_attr.license = license; 2069 load_attr.kern_version = kern_version; 2070 load_attr.prog_ifindex = prog->prog_ifindex; 2071 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 2072 load_attr.func_info = prog->func_info; 2073 load_attr.func_info_rec_size = prog->func_info_rec_size; 2074 load_attr.func_info_cnt = prog->func_info_cnt; 2075 load_attr.line_info = prog->line_info; 2076 load_attr.line_info_rec_size = prog->line_info_rec_size; 2077 load_attr.line_info_cnt = prog->line_info_cnt; 2078 load_attr.log_level = prog->log_level; 2079 if (!load_attr.insns || !load_attr.insns_cnt) 2080 return -EINVAL; 2081 2082 retry_load: 2083 log_buf = malloc(log_buf_size); 2084 if (!log_buf) 2085 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2086 2087 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2088 2089 if (ret >= 0) { 2090 if (load_attr.log_level) 2091 pr_debug("verifier log:\n%s", log_buf); 2092 *pfd = ret; 2093 ret = 0; 2094 goto out; 2095 } 2096 2097 if (errno == ENOSPC) { 2098 log_buf_size <<= 1; 2099 free(log_buf); 2100 goto retry_load; 2101 } 2102 ret = -LIBBPF_ERRNO__LOAD; 2103 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2104 pr_warning("load bpf program failed: %s\n", cp); 2105 2106 if (log_buf && log_buf[0] != '\0') { 2107 ret = -LIBBPF_ERRNO__VERIFY; 2108 pr_warning("-- BEGIN DUMP LOG ---\n"); 2109 pr_warning("\n%s\n", log_buf); 2110 pr_warning("-- END LOG --\n"); 2111 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2112 pr_warning("Program too large (%zu insns), at most %d insns\n", 2113 load_attr.insns_cnt, BPF_MAXINSNS); 2114 ret = -LIBBPF_ERRNO__PROG2BIG; 2115 } else { 2116 /* Wrong program type? */ 2117 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2118 int fd; 2119 2120 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2121 load_attr.expected_attach_type = 0; 2122 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2123 if (fd >= 0) { 2124 close(fd); 2125 ret = -LIBBPF_ERRNO__PROGTYPE; 2126 goto out; 2127 } 2128 } 2129 2130 if (log_buf) 2131 ret = -LIBBPF_ERRNO__KVER; 2132 } 2133 2134 out: 2135 free(log_buf); 2136 return ret; 2137 } 2138 2139 int 2140 bpf_program__load(struct bpf_program *prog, 2141 char *license, __u32 kern_version) 2142 { 2143 int err = 0, fd, i; 2144 2145 if (prog->instances.nr < 0 || !prog->instances.fds) { 2146 if (prog->preprocessor) { 2147 pr_warning("Internal error: can't load program '%s'\n", 2148 prog->section_name); 2149 return -LIBBPF_ERRNO__INTERNAL; 2150 } 2151 2152 prog->instances.fds = malloc(sizeof(int)); 2153 if (!prog->instances.fds) { 2154 pr_warning("Not enough memory for BPF fds\n"); 2155 return -ENOMEM; 2156 } 2157 prog->instances.nr = 1; 2158 prog->instances.fds[0] = -1; 2159 } 2160 2161 if (!prog->preprocessor) { 2162 if (prog->instances.nr != 1) { 2163 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2164 prog->section_name, prog->instances.nr); 2165 } 2166 err = load_program(prog, prog->insns, prog->insns_cnt, 2167 license, kern_version, &fd); 2168 if (!err) 2169 prog->instances.fds[0] = fd; 2170 goto out; 2171 } 2172 2173 for (i = 0; i < prog->instances.nr; i++) { 2174 struct bpf_prog_prep_result result; 2175 bpf_program_prep_t preprocessor = prog->preprocessor; 2176 2177 memset(&result, 0, sizeof(result)); 2178 err = preprocessor(prog, i, prog->insns, 2179 prog->insns_cnt, &result); 2180 if (err) { 2181 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2182 i, prog->section_name); 2183 goto out; 2184 } 2185 2186 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2187 pr_debug("Skip loading the %dth instance of program '%s'\n", 2188 i, prog->section_name); 2189 prog->instances.fds[i] = -1; 2190 if (result.pfd) 2191 *result.pfd = -1; 2192 continue; 2193 } 2194 2195 err = load_program(prog, result.new_insn_ptr, 2196 result.new_insn_cnt, 2197 license, kern_version, &fd); 2198 2199 if (err) { 2200 pr_warning("Loading the %dth instance of program '%s' failed\n", 2201 i, prog->section_name); 2202 goto out; 2203 } 2204 2205 if (result.pfd) 2206 *result.pfd = fd; 2207 prog->instances.fds[i] = fd; 2208 } 2209 out: 2210 if (err) 2211 pr_warning("failed to load program '%s'\n", 2212 prog->section_name); 2213 zfree(&prog->insns); 2214 prog->insns_cnt = 0; 2215 return err; 2216 } 2217 2218 static bool bpf_program__is_function_storage(struct bpf_program *prog, 2219 struct bpf_object *obj) 2220 { 2221 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2222 } 2223 2224 static int 2225 bpf_object__load_progs(struct bpf_object *obj) 2226 { 2227 size_t i; 2228 int err; 2229 2230 for (i = 0; i < obj->nr_programs; i++) { 2231 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2232 continue; 2233 err = bpf_program__load(&obj->programs[i], 2234 obj->license, 2235 obj->kern_version); 2236 if (err) 2237 return err; 2238 } 2239 return 0; 2240 } 2241 2242 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2243 { 2244 switch (type) { 2245 case BPF_PROG_TYPE_SOCKET_FILTER: 2246 case BPF_PROG_TYPE_SCHED_CLS: 2247 case BPF_PROG_TYPE_SCHED_ACT: 2248 case BPF_PROG_TYPE_XDP: 2249 case BPF_PROG_TYPE_CGROUP_SKB: 2250 case BPF_PROG_TYPE_CGROUP_SOCK: 2251 case BPF_PROG_TYPE_LWT_IN: 2252 case BPF_PROG_TYPE_LWT_OUT: 2253 case BPF_PROG_TYPE_LWT_XMIT: 2254 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2255 case BPF_PROG_TYPE_SOCK_OPS: 2256 case BPF_PROG_TYPE_SK_SKB: 2257 case BPF_PROG_TYPE_CGROUP_DEVICE: 2258 case BPF_PROG_TYPE_SK_MSG: 2259 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2260 case BPF_PROG_TYPE_LIRC_MODE2: 2261 case BPF_PROG_TYPE_SK_REUSEPORT: 2262 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2263 case BPF_PROG_TYPE_UNSPEC: 2264 case BPF_PROG_TYPE_TRACEPOINT: 2265 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2266 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2267 case BPF_PROG_TYPE_PERF_EVENT: 2268 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2269 return false; 2270 case BPF_PROG_TYPE_KPROBE: 2271 default: 2272 return true; 2273 } 2274 } 2275 2276 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2277 { 2278 if (needs_kver && obj->kern_version == 0) { 2279 pr_warning("%s doesn't provide kernel version\n", 2280 obj->path); 2281 return -LIBBPF_ERRNO__KVERSION; 2282 } 2283 return 0; 2284 } 2285 2286 static struct bpf_object * 2287 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2288 bool needs_kver, int flags) 2289 { 2290 struct bpf_object *obj; 2291 int err; 2292 2293 if (elf_version(EV_CURRENT) == EV_NONE) { 2294 pr_warning("failed to init libelf for %s\n", path); 2295 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2296 } 2297 2298 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2299 if (IS_ERR(obj)) 2300 return obj; 2301 2302 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2303 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2304 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2305 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2306 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2307 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2308 2309 bpf_object__elf_finish(obj); 2310 return obj; 2311 out: 2312 bpf_object__close(obj); 2313 return ERR_PTR(err); 2314 } 2315 2316 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2317 int flags) 2318 { 2319 /* param validation */ 2320 if (!attr->file) 2321 return NULL; 2322 2323 pr_debug("loading %s\n", attr->file); 2324 2325 return __bpf_object__open(attr->file, NULL, 0, 2326 bpf_prog_type__needs_kver(attr->prog_type), 2327 flags); 2328 } 2329 2330 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2331 { 2332 return __bpf_object__open_xattr(attr, 0); 2333 } 2334 2335 struct bpf_object *bpf_object__open(const char *path) 2336 { 2337 struct bpf_object_open_attr attr = { 2338 .file = path, 2339 .prog_type = BPF_PROG_TYPE_UNSPEC, 2340 }; 2341 2342 return bpf_object__open_xattr(&attr); 2343 } 2344 2345 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2346 size_t obj_buf_sz, 2347 const char *name) 2348 { 2349 char tmp_name[64]; 2350 2351 /* param validation */ 2352 if (!obj_buf || obj_buf_sz <= 0) 2353 return NULL; 2354 2355 if (!name) { 2356 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2357 (unsigned long)obj_buf, 2358 (unsigned long)obj_buf_sz); 2359 tmp_name[sizeof(tmp_name) - 1] = '\0'; 2360 name = tmp_name; 2361 } 2362 pr_debug("loading object '%s' from buffer\n", 2363 name); 2364 2365 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2366 } 2367 2368 int bpf_object__unload(struct bpf_object *obj) 2369 { 2370 size_t i; 2371 2372 if (!obj) 2373 return -EINVAL; 2374 2375 for (i = 0; i < obj->nr_maps; i++) 2376 zclose(obj->maps[i].fd); 2377 2378 for (i = 0; i < obj->nr_programs; i++) 2379 bpf_program__unload(&obj->programs[i]); 2380 2381 return 0; 2382 } 2383 2384 int bpf_object__load(struct bpf_object *obj) 2385 { 2386 int err; 2387 2388 if (!obj) 2389 return -EINVAL; 2390 2391 if (obj->loaded) { 2392 pr_warning("object should not be loaded twice\n"); 2393 return -EINVAL; 2394 } 2395 2396 obj->loaded = true; 2397 2398 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2399 CHECK_ERR(bpf_object__relocate(obj), err, out); 2400 CHECK_ERR(bpf_object__load_progs(obj), err, out); 2401 2402 return 0; 2403 out: 2404 bpf_object__unload(obj); 2405 pr_warning("failed to load object '%s'\n", obj->path); 2406 return err; 2407 } 2408 2409 static int check_path(const char *path) 2410 { 2411 char *cp, errmsg[STRERR_BUFSIZE]; 2412 struct statfs st_fs; 2413 char *dname, *dir; 2414 int err = 0; 2415 2416 if (path == NULL) 2417 return -EINVAL; 2418 2419 dname = strdup(path); 2420 if (dname == NULL) 2421 return -ENOMEM; 2422 2423 dir = dirname(dname); 2424 if (statfs(dir, &st_fs)) { 2425 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2426 pr_warning("failed to statfs %s: %s\n", dir, cp); 2427 err = -errno; 2428 } 2429 free(dname); 2430 2431 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2432 pr_warning("specified path %s is not on BPF FS\n", path); 2433 err = -EINVAL; 2434 } 2435 2436 return err; 2437 } 2438 2439 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2440 int instance) 2441 { 2442 char *cp, errmsg[STRERR_BUFSIZE]; 2443 int err; 2444 2445 err = check_path(path); 2446 if (err) 2447 return err; 2448 2449 if (prog == NULL) { 2450 pr_warning("invalid program pointer\n"); 2451 return -EINVAL; 2452 } 2453 2454 if (instance < 0 || instance >= prog->instances.nr) { 2455 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2456 instance, prog->section_name, prog->instances.nr); 2457 return -EINVAL; 2458 } 2459 2460 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2461 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2462 pr_warning("failed to pin program: %s\n", cp); 2463 return -errno; 2464 } 2465 pr_debug("pinned program '%s'\n", path); 2466 2467 return 0; 2468 } 2469 2470 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2471 int instance) 2472 { 2473 int err; 2474 2475 err = check_path(path); 2476 if (err) 2477 return err; 2478 2479 if (prog == NULL) { 2480 pr_warning("invalid program pointer\n"); 2481 return -EINVAL; 2482 } 2483 2484 if (instance < 0 || instance >= prog->instances.nr) { 2485 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2486 instance, prog->section_name, prog->instances.nr); 2487 return -EINVAL; 2488 } 2489 2490 err = unlink(path); 2491 if (err != 0) 2492 return -errno; 2493 pr_debug("unpinned program '%s'\n", path); 2494 2495 return 0; 2496 } 2497 2498 static int make_dir(const char *path) 2499 { 2500 char *cp, errmsg[STRERR_BUFSIZE]; 2501 int err = 0; 2502 2503 if (mkdir(path, 0700) && errno != EEXIST) 2504 err = -errno; 2505 2506 if (err) { 2507 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2508 pr_warning("failed to mkdir %s: %s\n", path, cp); 2509 } 2510 return err; 2511 } 2512 2513 int bpf_program__pin(struct bpf_program *prog, const char *path) 2514 { 2515 int i, err; 2516 2517 err = check_path(path); 2518 if (err) 2519 return err; 2520 2521 if (prog == NULL) { 2522 pr_warning("invalid program pointer\n"); 2523 return -EINVAL; 2524 } 2525 2526 if (prog->instances.nr <= 0) { 2527 pr_warning("no instances of prog %s to pin\n", 2528 prog->section_name); 2529 return -EINVAL; 2530 } 2531 2532 if (prog->instances.nr == 1) { 2533 /* don't create subdirs when pinning single instance */ 2534 return bpf_program__pin_instance(prog, path, 0); 2535 } 2536 2537 err = make_dir(path); 2538 if (err) 2539 return err; 2540 2541 for (i = 0; i < prog->instances.nr; i++) { 2542 char buf[PATH_MAX]; 2543 int len; 2544 2545 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2546 if (len < 0) { 2547 err = -EINVAL; 2548 goto err_unpin; 2549 } else if (len >= PATH_MAX) { 2550 err = -ENAMETOOLONG; 2551 goto err_unpin; 2552 } 2553 2554 err = bpf_program__pin_instance(prog, buf, i); 2555 if (err) 2556 goto err_unpin; 2557 } 2558 2559 return 0; 2560 2561 err_unpin: 2562 for (i = i - 1; i >= 0; i--) { 2563 char buf[PATH_MAX]; 2564 int len; 2565 2566 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2567 if (len < 0) 2568 continue; 2569 else if (len >= PATH_MAX) 2570 continue; 2571 2572 bpf_program__unpin_instance(prog, buf, i); 2573 } 2574 2575 rmdir(path); 2576 2577 return err; 2578 } 2579 2580 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2581 { 2582 int i, err; 2583 2584 err = check_path(path); 2585 if (err) 2586 return err; 2587 2588 if (prog == NULL) { 2589 pr_warning("invalid program pointer\n"); 2590 return -EINVAL; 2591 } 2592 2593 if (prog->instances.nr <= 0) { 2594 pr_warning("no instances of prog %s to pin\n", 2595 prog->section_name); 2596 return -EINVAL; 2597 } 2598 2599 if (prog->instances.nr == 1) { 2600 /* don't create subdirs when pinning single instance */ 2601 return bpf_program__unpin_instance(prog, path, 0); 2602 } 2603 2604 for (i = 0; i < prog->instances.nr; i++) { 2605 char buf[PATH_MAX]; 2606 int len; 2607 2608 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2609 if (len < 0) 2610 return -EINVAL; 2611 else if (len >= PATH_MAX) 2612 return -ENAMETOOLONG; 2613 2614 err = bpf_program__unpin_instance(prog, buf, i); 2615 if (err) 2616 return err; 2617 } 2618 2619 err = rmdir(path); 2620 if (err) 2621 return -errno; 2622 2623 return 0; 2624 } 2625 2626 int bpf_map__pin(struct bpf_map *map, const char *path) 2627 { 2628 char *cp, errmsg[STRERR_BUFSIZE]; 2629 int err; 2630 2631 err = check_path(path); 2632 if (err) 2633 return err; 2634 2635 if (map == NULL) { 2636 pr_warning("invalid map pointer\n"); 2637 return -EINVAL; 2638 } 2639 2640 if (bpf_obj_pin(map->fd, path)) { 2641 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2642 pr_warning("failed to pin map: %s\n", cp); 2643 return -errno; 2644 } 2645 2646 pr_debug("pinned map '%s'\n", path); 2647 2648 return 0; 2649 } 2650 2651 int bpf_map__unpin(struct bpf_map *map, const char *path) 2652 { 2653 int err; 2654 2655 err = check_path(path); 2656 if (err) 2657 return err; 2658 2659 if (map == NULL) { 2660 pr_warning("invalid map pointer\n"); 2661 return -EINVAL; 2662 } 2663 2664 err = unlink(path); 2665 if (err != 0) 2666 return -errno; 2667 pr_debug("unpinned map '%s'\n", path); 2668 2669 return 0; 2670 } 2671 2672 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2673 { 2674 struct bpf_map *map; 2675 int err; 2676 2677 if (!obj) 2678 return -ENOENT; 2679 2680 if (!obj->loaded) { 2681 pr_warning("object not yet loaded; load it first\n"); 2682 return -ENOENT; 2683 } 2684 2685 err = make_dir(path); 2686 if (err) 2687 return err; 2688 2689 bpf_object__for_each_map(map, obj) { 2690 char buf[PATH_MAX]; 2691 int len; 2692 2693 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2694 bpf_map__name(map)); 2695 if (len < 0) { 2696 err = -EINVAL; 2697 goto err_unpin_maps; 2698 } else if (len >= PATH_MAX) { 2699 err = -ENAMETOOLONG; 2700 goto err_unpin_maps; 2701 } 2702 2703 err = bpf_map__pin(map, buf); 2704 if (err) 2705 goto err_unpin_maps; 2706 } 2707 2708 return 0; 2709 2710 err_unpin_maps: 2711 while ((map = bpf_map__prev(map, obj))) { 2712 char buf[PATH_MAX]; 2713 int len; 2714 2715 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2716 bpf_map__name(map)); 2717 if (len < 0) 2718 continue; 2719 else if (len >= PATH_MAX) 2720 continue; 2721 2722 bpf_map__unpin(map, buf); 2723 } 2724 2725 return err; 2726 } 2727 2728 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2729 { 2730 struct bpf_map *map; 2731 int err; 2732 2733 if (!obj) 2734 return -ENOENT; 2735 2736 bpf_object__for_each_map(map, obj) { 2737 char buf[PATH_MAX]; 2738 int len; 2739 2740 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2741 bpf_map__name(map)); 2742 if (len < 0) 2743 return -EINVAL; 2744 else if (len >= PATH_MAX) 2745 return -ENAMETOOLONG; 2746 2747 err = bpf_map__unpin(map, buf); 2748 if (err) 2749 return err; 2750 } 2751 2752 return 0; 2753 } 2754 2755 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2756 { 2757 struct bpf_program *prog; 2758 int err; 2759 2760 if (!obj) 2761 return -ENOENT; 2762 2763 if (!obj->loaded) { 2764 pr_warning("object not yet loaded; load it first\n"); 2765 return -ENOENT; 2766 } 2767 2768 err = make_dir(path); 2769 if (err) 2770 return err; 2771 2772 bpf_object__for_each_program(prog, obj) { 2773 char buf[PATH_MAX]; 2774 int len; 2775 2776 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2777 prog->pin_name); 2778 if (len < 0) { 2779 err = -EINVAL; 2780 goto err_unpin_programs; 2781 } else if (len >= PATH_MAX) { 2782 err = -ENAMETOOLONG; 2783 goto err_unpin_programs; 2784 } 2785 2786 err = bpf_program__pin(prog, buf); 2787 if (err) 2788 goto err_unpin_programs; 2789 } 2790 2791 return 0; 2792 2793 err_unpin_programs: 2794 while ((prog = bpf_program__prev(prog, obj))) { 2795 char buf[PATH_MAX]; 2796 int len; 2797 2798 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2799 prog->pin_name); 2800 if (len < 0) 2801 continue; 2802 else if (len >= PATH_MAX) 2803 continue; 2804 2805 bpf_program__unpin(prog, buf); 2806 } 2807 2808 return err; 2809 } 2810 2811 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2812 { 2813 struct bpf_program *prog; 2814 int err; 2815 2816 if (!obj) 2817 return -ENOENT; 2818 2819 bpf_object__for_each_program(prog, obj) { 2820 char buf[PATH_MAX]; 2821 int len; 2822 2823 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2824 prog->pin_name); 2825 if (len < 0) 2826 return -EINVAL; 2827 else if (len >= PATH_MAX) 2828 return -ENAMETOOLONG; 2829 2830 err = bpf_program__unpin(prog, buf); 2831 if (err) 2832 return err; 2833 } 2834 2835 return 0; 2836 } 2837 2838 int bpf_object__pin(struct bpf_object *obj, const char *path) 2839 { 2840 int err; 2841 2842 err = bpf_object__pin_maps(obj, path); 2843 if (err) 2844 return err; 2845 2846 err = bpf_object__pin_programs(obj, path); 2847 if (err) { 2848 bpf_object__unpin_maps(obj, path); 2849 return err; 2850 } 2851 2852 return 0; 2853 } 2854 2855 void bpf_object__close(struct bpf_object *obj) 2856 { 2857 size_t i; 2858 2859 if (!obj) 2860 return; 2861 2862 if (obj->clear_priv) 2863 obj->clear_priv(obj, obj->priv); 2864 2865 bpf_object__elf_finish(obj); 2866 bpf_object__unload(obj); 2867 btf__free(obj->btf); 2868 btf_ext__free(obj->btf_ext); 2869 2870 for (i = 0; i < obj->nr_maps; i++) { 2871 zfree(&obj->maps[i].name); 2872 if (obj->maps[i].clear_priv) 2873 obj->maps[i].clear_priv(&obj->maps[i], 2874 obj->maps[i].priv); 2875 obj->maps[i].priv = NULL; 2876 obj->maps[i].clear_priv = NULL; 2877 } 2878 2879 zfree(&obj->sections.rodata); 2880 zfree(&obj->sections.data); 2881 zfree(&obj->maps); 2882 obj->nr_maps = 0; 2883 2884 if (obj->programs && obj->nr_programs) { 2885 for (i = 0; i < obj->nr_programs; i++) 2886 bpf_program__exit(&obj->programs[i]); 2887 } 2888 zfree(&obj->programs); 2889 2890 list_del(&obj->list); 2891 free(obj); 2892 } 2893 2894 struct bpf_object * 2895 bpf_object__next(struct bpf_object *prev) 2896 { 2897 struct bpf_object *next; 2898 2899 if (!prev) 2900 next = list_first_entry(&bpf_objects_list, 2901 struct bpf_object, 2902 list); 2903 else 2904 next = list_next_entry(prev, list); 2905 2906 /* Empty list is noticed here so don't need checking on entry. */ 2907 if (&next->list == &bpf_objects_list) 2908 return NULL; 2909 2910 return next; 2911 } 2912 2913 const char *bpf_object__name(struct bpf_object *obj) 2914 { 2915 return obj ? obj->path : ERR_PTR(-EINVAL); 2916 } 2917 2918 unsigned int bpf_object__kversion(struct bpf_object *obj) 2919 { 2920 return obj ? obj->kern_version : 0; 2921 } 2922 2923 struct btf *bpf_object__btf(struct bpf_object *obj) 2924 { 2925 return obj ? obj->btf : NULL; 2926 } 2927 2928 int bpf_object__btf_fd(const struct bpf_object *obj) 2929 { 2930 return obj->btf ? btf__fd(obj->btf) : -1; 2931 } 2932 2933 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2934 bpf_object_clear_priv_t clear_priv) 2935 { 2936 if (obj->priv && obj->clear_priv) 2937 obj->clear_priv(obj, obj->priv); 2938 2939 obj->priv = priv; 2940 obj->clear_priv = clear_priv; 2941 return 0; 2942 } 2943 2944 void *bpf_object__priv(struct bpf_object *obj) 2945 { 2946 return obj ? obj->priv : ERR_PTR(-EINVAL); 2947 } 2948 2949 static struct bpf_program * 2950 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2951 { 2952 size_t nr_programs = obj->nr_programs; 2953 ssize_t idx; 2954 2955 if (!nr_programs) 2956 return NULL; 2957 2958 if (!p) 2959 /* Iter from the beginning */ 2960 return forward ? &obj->programs[0] : 2961 &obj->programs[nr_programs - 1]; 2962 2963 if (p->obj != obj) { 2964 pr_warning("error: program handler doesn't match object\n"); 2965 return NULL; 2966 } 2967 2968 idx = (p - obj->programs) + (forward ? 1 : -1); 2969 if (idx >= obj->nr_programs || idx < 0) 2970 return NULL; 2971 return &obj->programs[idx]; 2972 } 2973 2974 struct bpf_program * 2975 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2976 { 2977 struct bpf_program *prog = prev; 2978 2979 do { 2980 prog = __bpf_program__iter(prog, obj, true); 2981 } while (prog && bpf_program__is_function_storage(prog, obj)); 2982 2983 return prog; 2984 } 2985 2986 struct bpf_program * 2987 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2988 { 2989 struct bpf_program *prog = next; 2990 2991 do { 2992 prog = __bpf_program__iter(prog, obj, false); 2993 } while (prog && bpf_program__is_function_storage(prog, obj)); 2994 2995 return prog; 2996 } 2997 2998 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 2999 bpf_program_clear_priv_t clear_priv) 3000 { 3001 if (prog->priv && prog->clear_priv) 3002 prog->clear_priv(prog, prog->priv); 3003 3004 prog->priv = priv; 3005 prog->clear_priv = clear_priv; 3006 return 0; 3007 } 3008 3009 void *bpf_program__priv(struct bpf_program *prog) 3010 { 3011 return prog ? prog->priv : ERR_PTR(-EINVAL); 3012 } 3013 3014 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3015 { 3016 prog->prog_ifindex = ifindex; 3017 } 3018 3019 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 3020 { 3021 const char *title; 3022 3023 title = prog->section_name; 3024 if (needs_copy) { 3025 title = strdup(title); 3026 if (!title) { 3027 pr_warning("failed to strdup program title\n"); 3028 return ERR_PTR(-ENOMEM); 3029 } 3030 } 3031 3032 return title; 3033 } 3034 3035 int bpf_program__fd(struct bpf_program *prog) 3036 { 3037 return bpf_program__nth_fd(prog, 0); 3038 } 3039 3040 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3041 bpf_program_prep_t prep) 3042 { 3043 int *instances_fds; 3044 3045 if (nr_instances <= 0 || !prep) 3046 return -EINVAL; 3047 3048 if (prog->instances.nr > 0 || prog->instances.fds) { 3049 pr_warning("Can't set pre-processor after loading\n"); 3050 return -EINVAL; 3051 } 3052 3053 instances_fds = malloc(sizeof(int) * nr_instances); 3054 if (!instances_fds) { 3055 pr_warning("alloc memory failed for fds\n"); 3056 return -ENOMEM; 3057 } 3058 3059 /* fill all fd with -1 */ 3060 memset(instances_fds, -1, sizeof(int) * nr_instances); 3061 3062 prog->instances.nr = nr_instances; 3063 prog->instances.fds = instances_fds; 3064 prog->preprocessor = prep; 3065 return 0; 3066 } 3067 3068 int bpf_program__nth_fd(struct bpf_program *prog, int n) 3069 { 3070 int fd; 3071 3072 if (!prog) 3073 return -EINVAL; 3074 3075 if (n >= prog->instances.nr || n < 0) { 3076 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3077 n, prog->section_name, prog->instances.nr); 3078 return -EINVAL; 3079 } 3080 3081 fd = prog->instances.fds[n]; 3082 if (fd < 0) { 3083 pr_warning("%dth instance of program '%s' is invalid\n", 3084 n, prog->section_name); 3085 return -ENOENT; 3086 } 3087 3088 return fd; 3089 } 3090 3091 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3092 { 3093 prog->type = type; 3094 } 3095 3096 static bool bpf_program__is_type(struct bpf_program *prog, 3097 enum bpf_prog_type type) 3098 { 3099 return prog ? (prog->type == type) : false; 3100 } 3101 3102 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3103 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3104 { \ 3105 if (!prog) \ 3106 return -EINVAL; \ 3107 bpf_program__set_type(prog, TYPE); \ 3108 return 0; \ 3109 } \ 3110 \ 3111 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 3112 { \ 3113 return bpf_program__is_type(prog, TYPE); \ 3114 } \ 3115 3116 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3117 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3118 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3119 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3120 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3121 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3122 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3123 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3124 3125 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3126 enum bpf_attach_type type) 3127 { 3128 prog->expected_attach_type = type; 3129 } 3130 3131 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3132 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3133 3134 /* Programs that can NOT be attached. */ 3135 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3136 3137 /* Programs that can be attached. */ 3138 #define BPF_APROG_SEC(string, ptype, atype) \ 3139 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3140 3141 /* Programs that must specify expected attach type at load time. */ 3142 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3143 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3144 3145 /* Programs that can be attached but attach type can't be identified by section 3146 * name. Kept for backward compatibility. 3147 */ 3148 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3149 3150 static const struct { 3151 const char *sec; 3152 size_t len; 3153 enum bpf_prog_type prog_type; 3154 enum bpf_attach_type expected_attach_type; 3155 int is_attachable; 3156 enum bpf_attach_type attach_type; 3157 } section_names[] = { 3158 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3159 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3160 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3161 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3162 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3163 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3164 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3165 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3166 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3167 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3168 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3169 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3170 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3171 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3172 BPF_CGROUP_INET_INGRESS), 3173 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3174 BPF_CGROUP_INET_EGRESS), 3175 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3176 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3177 BPF_CGROUP_INET_SOCK_CREATE), 3178 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3179 BPF_CGROUP_INET4_POST_BIND), 3180 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3181 BPF_CGROUP_INET6_POST_BIND), 3182 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3183 BPF_CGROUP_DEVICE), 3184 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3185 BPF_CGROUP_SOCK_OPS), 3186 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3187 BPF_SK_SKB_STREAM_PARSER), 3188 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3189 BPF_SK_SKB_STREAM_VERDICT), 3190 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3191 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3192 BPF_SK_MSG_VERDICT), 3193 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3194 BPF_LIRC_MODE2), 3195 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3196 BPF_FLOW_DISSECTOR), 3197 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3198 BPF_CGROUP_INET4_BIND), 3199 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3200 BPF_CGROUP_INET6_BIND), 3201 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3202 BPF_CGROUP_INET4_CONNECT), 3203 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3204 BPF_CGROUP_INET6_CONNECT), 3205 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3206 BPF_CGROUP_UDP4_SENDMSG), 3207 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3208 BPF_CGROUP_UDP6_SENDMSG), 3209 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3210 BPF_CGROUP_SYSCTL), 3211 }; 3212 3213 #undef BPF_PROG_SEC_IMPL 3214 #undef BPF_PROG_SEC 3215 #undef BPF_APROG_SEC 3216 #undef BPF_EAPROG_SEC 3217 #undef BPF_APROG_COMPAT 3218 3219 #define MAX_TYPE_NAME_SIZE 32 3220 3221 static char *libbpf_get_type_names(bool attach_type) 3222 { 3223 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3224 char *buf; 3225 3226 buf = malloc(len); 3227 if (!buf) 3228 return NULL; 3229 3230 buf[0] = '\0'; 3231 /* Forge string buf with all available names */ 3232 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3233 if (attach_type && !section_names[i].is_attachable) 3234 continue; 3235 3236 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3237 free(buf); 3238 return NULL; 3239 } 3240 strcat(buf, " "); 3241 strcat(buf, section_names[i].sec); 3242 } 3243 3244 return buf; 3245 } 3246 3247 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3248 enum bpf_attach_type *expected_attach_type) 3249 { 3250 char *type_names; 3251 int i; 3252 3253 if (!name) 3254 return -EINVAL; 3255 3256 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3257 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3258 continue; 3259 *prog_type = section_names[i].prog_type; 3260 *expected_attach_type = section_names[i].expected_attach_type; 3261 return 0; 3262 } 3263 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3264 type_names = libbpf_get_type_names(false); 3265 if (type_names != NULL) { 3266 pr_info("supported section(type) names are:%s\n", type_names); 3267 free(type_names); 3268 } 3269 3270 return -EINVAL; 3271 } 3272 3273 int libbpf_attach_type_by_name(const char *name, 3274 enum bpf_attach_type *attach_type) 3275 { 3276 char *type_names; 3277 int i; 3278 3279 if (!name) 3280 return -EINVAL; 3281 3282 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3283 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3284 continue; 3285 if (!section_names[i].is_attachable) 3286 return -EINVAL; 3287 *attach_type = section_names[i].attach_type; 3288 return 0; 3289 } 3290 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3291 type_names = libbpf_get_type_names(true); 3292 if (type_names != NULL) { 3293 pr_info("attachable section(type) names are:%s\n", type_names); 3294 free(type_names); 3295 } 3296 3297 return -EINVAL; 3298 } 3299 3300 static int 3301 bpf_program__identify_section(struct bpf_program *prog, 3302 enum bpf_prog_type *prog_type, 3303 enum bpf_attach_type *expected_attach_type) 3304 { 3305 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3306 expected_attach_type); 3307 } 3308 3309 int bpf_map__fd(struct bpf_map *map) 3310 { 3311 return map ? map->fd : -EINVAL; 3312 } 3313 3314 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 3315 { 3316 return map ? &map->def : ERR_PTR(-EINVAL); 3317 } 3318 3319 const char *bpf_map__name(struct bpf_map *map) 3320 { 3321 return map ? map->name : NULL; 3322 } 3323 3324 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3325 { 3326 return map ? map->btf_key_type_id : 0; 3327 } 3328 3329 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3330 { 3331 return map ? map->btf_value_type_id : 0; 3332 } 3333 3334 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3335 bpf_map_clear_priv_t clear_priv) 3336 { 3337 if (!map) 3338 return -EINVAL; 3339 3340 if (map->priv) { 3341 if (map->clear_priv) 3342 map->clear_priv(map, map->priv); 3343 } 3344 3345 map->priv = priv; 3346 map->clear_priv = clear_priv; 3347 return 0; 3348 } 3349 3350 void *bpf_map__priv(struct bpf_map *map) 3351 { 3352 return map ? map->priv : ERR_PTR(-EINVAL); 3353 } 3354 3355 bool bpf_map__is_offload_neutral(struct bpf_map *map) 3356 { 3357 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3358 } 3359 3360 bool bpf_map__is_internal(struct bpf_map *map) 3361 { 3362 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3363 } 3364 3365 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3366 { 3367 map->map_ifindex = ifindex; 3368 } 3369 3370 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3371 { 3372 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3373 pr_warning("error: unsupported map type\n"); 3374 return -EINVAL; 3375 } 3376 if (map->inner_map_fd != -1) { 3377 pr_warning("error: inner_map_fd already specified\n"); 3378 return -EINVAL; 3379 } 3380 map->inner_map_fd = fd; 3381 return 0; 3382 } 3383 3384 static struct bpf_map * 3385 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 3386 { 3387 ssize_t idx; 3388 struct bpf_map *s, *e; 3389 3390 if (!obj || !obj->maps) 3391 return NULL; 3392 3393 s = obj->maps; 3394 e = obj->maps + obj->nr_maps; 3395 3396 if ((m < s) || (m >= e)) { 3397 pr_warning("error in %s: map handler doesn't belong to object\n", 3398 __func__); 3399 return NULL; 3400 } 3401 3402 idx = (m - obj->maps) + i; 3403 if (idx >= obj->nr_maps || idx < 0) 3404 return NULL; 3405 return &obj->maps[idx]; 3406 } 3407 3408 struct bpf_map * 3409 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 3410 { 3411 if (prev == NULL) 3412 return obj->maps; 3413 3414 return __bpf_map__iter(prev, obj, 1); 3415 } 3416 3417 struct bpf_map * 3418 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 3419 { 3420 if (next == NULL) { 3421 if (!obj->nr_maps) 3422 return NULL; 3423 return obj->maps + obj->nr_maps - 1; 3424 } 3425 3426 return __bpf_map__iter(next, obj, -1); 3427 } 3428 3429 struct bpf_map * 3430 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 3431 { 3432 struct bpf_map *pos; 3433 3434 bpf_object__for_each_map(pos, obj) { 3435 if (pos->name && !strcmp(pos->name, name)) 3436 return pos; 3437 } 3438 return NULL; 3439 } 3440 3441 int 3442 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 3443 { 3444 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3445 } 3446 3447 struct bpf_map * 3448 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3449 { 3450 int i; 3451 3452 for (i = 0; i < obj->nr_maps; i++) { 3453 if (obj->maps[i].offset == offset) 3454 return &obj->maps[i]; 3455 } 3456 return ERR_PTR(-ENOENT); 3457 } 3458 3459 long libbpf_get_error(const void *ptr) 3460 { 3461 if (IS_ERR(ptr)) 3462 return PTR_ERR(ptr); 3463 return 0; 3464 } 3465 3466 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3467 struct bpf_object **pobj, int *prog_fd) 3468 { 3469 struct bpf_prog_load_attr attr; 3470 3471 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3472 attr.file = file; 3473 attr.prog_type = type; 3474 attr.expected_attach_type = 0; 3475 3476 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3477 } 3478 3479 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3480 struct bpf_object **pobj, int *prog_fd) 3481 { 3482 struct bpf_object_open_attr open_attr = { 3483 .file = attr->file, 3484 .prog_type = attr->prog_type, 3485 }; 3486 struct bpf_program *prog, *first_prog = NULL; 3487 enum bpf_attach_type expected_attach_type; 3488 enum bpf_prog_type prog_type; 3489 struct bpf_object *obj; 3490 struct bpf_map *map; 3491 int err; 3492 3493 if (!attr) 3494 return -EINVAL; 3495 if (!attr->file) 3496 return -EINVAL; 3497 3498 obj = bpf_object__open_xattr(&open_attr); 3499 if (IS_ERR_OR_NULL(obj)) 3500 return -ENOENT; 3501 3502 bpf_object__for_each_program(prog, obj) { 3503 /* 3504 * If type is not specified, try to guess it based on 3505 * section name. 3506 */ 3507 prog_type = attr->prog_type; 3508 prog->prog_ifindex = attr->ifindex; 3509 expected_attach_type = attr->expected_attach_type; 3510 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3511 err = bpf_program__identify_section(prog, &prog_type, 3512 &expected_attach_type); 3513 if (err < 0) { 3514 bpf_object__close(obj); 3515 return -EINVAL; 3516 } 3517 } 3518 3519 bpf_program__set_type(prog, prog_type); 3520 bpf_program__set_expected_attach_type(prog, 3521 expected_attach_type); 3522 3523 prog->log_level = attr->log_level; 3524 if (!first_prog) 3525 first_prog = prog; 3526 } 3527 3528 bpf_object__for_each_map(map, obj) { 3529 if (!bpf_map__is_offload_neutral(map)) 3530 map->map_ifindex = attr->ifindex; 3531 } 3532 3533 if (!first_prog) { 3534 pr_warning("object file doesn't contain bpf program\n"); 3535 bpf_object__close(obj); 3536 return -ENOENT; 3537 } 3538 3539 err = bpf_object__load(obj); 3540 if (err) { 3541 bpf_object__close(obj); 3542 return -EINVAL; 3543 } 3544 3545 *pobj = obj; 3546 *prog_fd = bpf_program__fd(first_prog); 3547 return 0; 3548 } 3549 3550 enum bpf_perf_event_ret 3551 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 3552 void **copy_mem, size_t *copy_size, 3553 bpf_perf_event_print_t fn, void *private_data) 3554 { 3555 struct perf_event_mmap_page *header = mmap_mem; 3556 __u64 data_head = ring_buffer_read_head(header); 3557 __u64 data_tail = header->data_tail; 3558 void *base = ((__u8 *)header) + page_size; 3559 int ret = LIBBPF_PERF_EVENT_CONT; 3560 struct perf_event_header *ehdr; 3561 size_t ehdr_size; 3562 3563 while (data_head != data_tail) { 3564 ehdr = base + (data_tail & (mmap_size - 1)); 3565 ehdr_size = ehdr->size; 3566 3567 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 3568 void *copy_start = ehdr; 3569 size_t len_first = base + mmap_size - copy_start; 3570 size_t len_secnd = ehdr_size - len_first; 3571 3572 if (*copy_size < ehdr_size) { 3573 free(*copy_mem); 3574 *copy_mem = malloc(ehdr_size); 3575 if (!*copy_mem) { 3576 *copy_size = 0; 3577 ret = LIBBPF_PERF_EVENT_ERROR; 3578 break; 3579 } 3580 *copy_size = ehdr_size; 3581 } 3582 3583 memcpy(*copy_mem, copy_start, len_first); 3584 memcpy(*copy_mem + len_first, base, len_secnd); 3585 ehdr = *copy_mem; 3586 } 3587 3588 ret = fn(ehdr, private_data); 3589 data_tail += ehdr_size; 3590 if (ret != LIBBPF_PERF_EVENT_CONT) 3591 break; 3592 } 3593 3594 ring_buffer_write_tail(header, data_tail); 3595 return ret; 3596 } 3597 3598 struct bpf_prog_info_array_desc { 3599 int array_offset; /* e.g. offset of jited_prog_insns */ 3600 int count_offset; /* e.g. offset of jited_prog_len */ 3601 int size_offset; /* > 0: offset of rec size, 3602 * < 0: fix size of -size_offset 3603 */ 3604 }; 3605 3606 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 3607 [BPF_PROG_INFO_JITED_INSNS] = { 3608 offsetof(struct bpf_prog_info, jited_prog_insns), 3609 offsetof(struct bpf_prog_info, jited_prog_len), 3610 -1, 3611 }, 3612 [BPF_PROG_INFO_XLATED_INSNS] = { 3613 offsetof(struct bpf_prog_info, xlated_prog_insns), 3614 offsetof(struct bpf_prog_info, xlated_prog_len), 3615 -1, 3616 }, 3617 [BPF_PROG_INFO_MAP_IDS] = { 3618 offsetof(struct bpf_prog_info, map_ids), 3619 offsetof(struct bpf_prog_info, nr_map_ids), 3620 -(int)sizeof(__u32), 3621 }, 3622 [BPF_PROG_INFO_JITED_KSYMS] = { 3623 offsetof(struct bpf_prog_info, jited_ksyms), 3624 offsetof(struct bpf_prog_info, nr_jited_ksyms), 3625 -(int)sizeof(__u64), 3626 }, 3627 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 3628 offsetof(struct bpf_prog_info, jited_func_lens), 3629 offsetof(struct bpf_prog_info, nr_jited_func_lens), 3630 -(int)sizeof(__u32), 3631 }, 3632 [BPF_PROG_INFO_FUNC_INFO] = { 3633 offsetof(struct bpf_prog_info, func_info), 3634 offsetof(struct bpf_prog_info, nr_func_info), 3635 offsetof(struct bpf_prog_info, func_info_rec_size), 3636 }, 3637 [BPF_PROG_INFO_LINE_INFO] = { 3638 offsetof(struct bpf_prog_info, line_info), 3639 offsetof(struct bpf_prog_info, nr_line_info), 3640 offsetof(struct bpf_prog_info, line_info_rec_size), 3641 }, 3642 [BPF_PROG_INFO_JITED_LINE_INFO] = { 3643 offsetof(struct bpf_prog_info, jited_line_info), 3644 offsetof(struct bpf_prog_info, nr_jited_line_info), 3645 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 3646 }, 3647 [BPF_PROG_INFO_PROG_TAGS] = { 3648 offsetof(struct bpf_prog_info, prog_tags), 3649 offsetof(struct bpf_prog_info, nr_prog_tags), 3650 -(int)sizeof(__u8) * BPF_TAG_SIZE, 3651 }, 3652 3653 }; 3654 3655 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 3656 { 3657 __u32 *array = (__u32 *)info; 3658 3659 if (offset >= 0) 3660 return array[offset / sizeof(__u32)]; 3661 return -(int)offset; 3662 } 3663 3664 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 3665 { 3666 __u64 *array = (__u64 *)info; 3667 3668 if (offset >= 0) 3669 return array[offset / sizeof(__u64)]; 3670 return -(int)offset; 3671 } 3672 3673 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 3674 __u32 val) 3675 { 3676 __u32 *array = (__u32 *)info; 3677 3678 if (offset >= 0) 3679 array[offset / sizeof(__u32)] = val; 3680 } 3681 3682 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 3683 __u64 val) 3684 { 3685 __u64 *array = (__u64 *)info; 3686 3687 if (offset >= 0) 3688 array[offset / sizeof(__u64)] = val; 3689 } 3690 3691 struct bpf_prog_info_linear * 3692 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 3693 { 3694 struct bpf_prog_info_linear *info_linear; 3695 struct bpf_prog_info info = {}; 3696 __u32 info_len = sizeof(info); 3697 __u32 data_len = 0; 3698 int i, err; 3699 void *ptr; 3700 3701 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 3702 return ERR_PTR(-EINVAL); 3703 3704 /* step 1: get array dimensions */ 3705 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 3706 if (err) { 3707 pr_debug("can't get prog info: %s", strerror(errno)); 3708 return ERR_PTR(-EFAULT); 3709 } 3710 3711 /* step 2: calculate total size of all arrays */ 3712 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3713 bool include_array = (arrays & (1UL << i)) > 0; 3714 struct bpf_prog_info_array_desc *desc; 3715 __u32 count, size; 3716 3717 desc = bpf_prog_info_array_desc + i; 3718 3719 /* kernel is too old to support this field */ 3720 if (info_len < desc->array_offset + sizeof(__u32) || 3721 info_len < desc->count_offset + sizeof(__u32) || 3722 (desc->size_offset > 0 && info_len < desc->size_offset)) 3723 include_array = false; 3724 3725 if (!include_array) { 3726 arrays &= ~(1UL << i); /* clear the bit */ 3727 continue; 3728 } 3729 3730 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3731 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3732 3733 data_len += count * size; 3734 } 3735 3736 /* step 3: allocate continuous memory */ 3737 data_len = roundup(data_len, sizeof(__u64)); 3738 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 3739 if (!info_linear) 3740 return ERR_PTR(-ENOMEM); 3741 3742 /* step 4: fill data to info_linear->info */ 3743 info_linear->arrays = arrays; 3744 memset(&info_linear->info, 0, sizeof(info)); 3745 ptr = info_linear->data; 3746 3747 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3748 struct bpf_prog_info_array_desc *desc; 3749 __u32 count, size; 3750 3751 if ((arrays & (1UL << i)) == 0) 3752 continue; 3753 3754 desc = bpf_prog_info_array_desc + i; 3755 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3756 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3757 bpf_prog_info_set_offset_u32(&info_linear->info, 3758 desc->count_offset, count); 3759 bpf_prog_info_set_offset_u32(&info_linear->info, 3760 desc->size_offset, size); 3761 bpf_prog_info_set_offset_u64(&info_linear->info, 3762 desc->array_offset, 3763 ptr_to_u64(ptr)); 3764 ptr += count * size; 3765 } 3766 3767 /* step 5: call syscall again to get required arrays */ 3768 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 3769 if (err) { 3770 pr_debug("can't get prog info: %s", strerror(errno)); 3771 free(info_linear); 3772 return ERR_PTR(-EFAULT); 3773 } 3774 3775 /* step 6: verify the data */ 3776 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3777 struct bpf_prog_info_array_desc *desc; 3778 __u32 v1, v2; 3779 3780 if ((arrays & (1UL << i)) == 0) 3781 continue; 3782 3783 desc = bpf_prog_info_array_desc + i; 3784 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3785 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3786 desc->count_offset); 3787 if (v1 != v2) 3788 pr_warning("%s: mismatch in element count\n", __func__); 3789 3790 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3791 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3792 desc->size_offset); 3793 if (v1 != v2) 3794 pr_warning("%s: mismatch in rec size\n", __func__); 3795 } 3796 3797 /* step 7: update info_len and data_len */ 3798 info_linear->info_len = sizeof(struct bpf_prog_info); 3799 info_linear->data_len = data_len; 3800 3801 return info_linear; 3802 } 3803 3804 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 3805 { 3806 int i; 3807 3808 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3809 struct bpf_prog_info_array_desc *desc; 3810 __u64 addr, offs; 3811 3812 if ((info_linear->arrays & (1UL << i)) == 0) 3813 continue; 3814 3815 desc = bpf_prog_info_array_desc + i; 3816 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 3817 desc->array_offset); 3818 offs = addr - ptr_to_u64(info_linear->data); 3819 bpf_prog_info_set_offset_u64(&info_linear->info, 3820 desc->array_offset, offs); 3821 } 3822 } 3823 3824 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 3825 { 3826 int i; 3827 3828 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3829 struct bpf_prog_info_array_desc *desc; 3830 __u64 addr, offs; 3831 3832 if ((info_linear->arrays & (1UL << i)) == 0) 3833 continue; 3834 3835 desc = bpf_prog_info_array_desc + i; 3836 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 3837 desc->array_offset); 3838 addr = offs + ptr_to_u64(info_linear->data); 3839 bpf_prog_info_set_offset_u64(&info_linear->info, 3840 desc->array_offset, addr); 3841 } 3842 } 3843