1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 */ 11 12 #ifndef _GNU_SOURCE 13 #define _GNU_SOURCE 14 #endif 15 #include <stdlib.h> 16 #include <stdio.h> 17 #include <stdarg.h> 18 #include <libgen.h> 19 #include <inttypes.h> 20 #include <string.h> 21 #include <unistd.h> 22 #include <fcntl.h> 23 #include <errno.h> 24 #include <asm/unistd.h> 25 #include <linux/err.h> 26 #include <linux/kernel.h> 27 #include <linux/bpf.h> 28 #include <linux/btf.h> 29 #include <linux/filter.h> 30 #include <linux/list.h> 31 #include <linux/limits.h> 32 #include <linux/perf_event.h> 33 #include <linux/ring_buffer.h> 34 #include <sys/stat.h> 35 #include <sys/types.h> 36 #include <sys/vfs.h> 37 #include <tools/libc_compat.h> 38 #include <libelf.h> 39 #include <gelf.h> 40 41 #include "libbpf.h" 42 #include "bpf.h" 43 #include "btf.h" 44 #include "str_error.h" 45 #include "libbpf_util.h" 46 47 #ifndef EM_BPF 48 #define EM_BPF 247 49 #endif 50 51 #ifndef BPF_FS_MAGIC 52 #define BPF_FS_MAGIC 0xcafe4a11 53 #endif 54 55 #define __printf(a, b) __attribute__((format(printf, a, b))) 56 57 static int __base_pr(enum libbpf_print_level level, const char *format, 58 va_list args) 59 { 60 if (level == LIBBPF_DEBUG) 61 return 0; 62 63 return vfprintf(stderr, format, args); 64 } 65 66 static libbpf_print_fn_t __libbpf_pr = __base_pr; 67 68 void libbpf_set_print(libbpf_print_fn_t fn) 69 { 70 __libbpf_pr = fn; 71 } 72 73 __printf(2, 3) 74 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 75 { 76 va_list args; 77 78 if (!__libbpf_pr) 79 return; 80 81 va_start(args, format); 82 __libbpf_pr(level, format, args); 83 va_end(args); 84 } 85 86 #define STRERR_BUFSIZE 128 87 88 #define CHECK_ERR(action, err, out) do { \ 89 err = action; \ 90 if (err) \ 91 goto out; \ 92 } while(0) 93 94 95 /* Copied from tools/perf/util/util.h */ 96 #ifndef zfree 97 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 98 #endif 99 100 #ifndef zclose 101 # define zclose(fd) ({ \ 102 int ___err = 0; \ 103 if ((fd) >= 0) \ 104 ___err = close((fd)); \ 105 fd = -1; \ 106 ___err; }) 107 #endif 108 109 #ifdef HAVE_LIBELF_MMAP_SUPPORT 110 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 111 #else 112 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 113 #endif 114 115 struct bpf_capabilities { 116 /* v4.14: kernel support for program & map names. */ 117 __u32 name:1; 118 }; 119 120 /* 121 * bpf_prog should be a better name but it has been used in 122 * linux/filter.h. 123 */ 124 struct bpf_program { 125 /* Index in elf obj file, for relocation use. */ 126 int idx; 127 char *name; 128 int prog_ifindex; 129 char *section_name; 130 /* section_name with / replaced by _; makes recursive pinning 131 * in bpf_object__pin_programs easier 132 */ 133 char *pin_name; 134 struct bpf_insn *insns; 135 size_t insns_cnt, main_prog_cnt; 136 enum bpf_prog_type type; 137 138 struct reloc_desc { 139 enum { 140 RELO_LD64, 141 RELO_CALL, 142 } type; 143 int insn_idx; 144 union { 145 int map_idx; 146 int text_off; 147 }; 148 } *reloc_desc; 149 int nr_reloc; 150 151 struct { 152 int nr; 153 int *fds; 154 } instances; 155 bpf_program_prep_t preprocessor; 156 157 struct bpf_object *obj; 158 void *priv; 159 bpf_program_clear_priv_t clear_priv; 160 161 enum bpf_attach_type expected_attach_type; 162 int btf_fd; 163 void *func_info; 164 __u32 func_info_rec_size; 165 __u32 func_info_cnt; 166 167 struct bpf_capabilities *caps; 168 169 void *line_info; 170 __u32 line_info_rec_size; 171 __u32 line_info_cnt; 172 }; 173 174 struct bpf_map { 175 int fd; 176 char *name; 177 size_t offset; 178 int map_ifindex; 179 int inner_map_fd; 180 struct bpf_map_def def; 181 __u32 btf_key_type_id; 182 __u32 btf_value_type_id; 183 void *priv; 184 bpf_map_clear_priv_t clear_priv; 185 }; 186 187 static LIST_HEAD(bpf_objects_list); 188 189 struct bpf_object { 190 char license[64]; 191 __u32 kern_version; 192 193 struct bpf_program *programs; 194 size_t nr_programs; 195 struct bpf_map *maps; 196 size_t nr_maps; 197 198 bool loaded; 199 bool has_pseudo_calls; 200 201 /* 202 * Information when doing elf related work. Only valid if fd 203 * is valid. 204 */ 205 struct { 206 int fd; 207 void *obj_buf; 208 size_t obj_buf_sz; 209 Elf *elf; 210 GElf_Ehdr ehdr; 211 Elf_Data *symbols; 212 size_t strtabidx; 213 struct { 214 GElf_Shdr shdr; 215 Elf_Data *data; 216 } *reloc; 217 int nr_reloc; 218 int maps_shndx; 219 int text_shndx; 220 } efile; 221 /* 222 * All loaded bpf_object is linked in a list, which is 223 * hidden to caller. bpf_objects__<func> handlers deal with 224 * all objects. 225 */ 226 struct list_head list; 227 228 struct btf *btf; 229 struct btf_ext *btf_ext; 230 231 void *priv; 232 bpf_object_clear_priv_t clear_priv; 233 234 struct bpf_capabilities caps; 235 236 char path[]; 237 }; 238 #define obj_elf_valid(o) ((o)->efile.elf) 239 240 void bpf_program__unload(struct bpf_program *prog) 241 { 242 int i; 243 244 if (!prog) 245 return; 246 247 /* 248 * If the object is opened but the program was never loaded, 249 * it is possible that prog->instances.nr == -1. 250 */ 251 if (prog->instances.nr > 0) { 252 for (i = 0; i < prog->instances.nr; i++) 253 zclose(prog->instances.fds[i]); 254 } else if (prog->instances.nr != -1) { 255 pr_warning("Internal error: instances.nr is %d\n", 256 prog->instances.nr); 257 } 258 259 prog->instances.nr = -1; 260 zfree(&prog->instances.fds); 261 262 zclose(prog->btf_fd); 263 zfree(&prog->func_info); 264 zfree(&prog->line_info); 265 } 266 267 static void bpf_program__exit(struct bpf_program *prog) 268 { 269 if (!prog) 270 return; 271 272 if (prog->clear_priv) 273 prog->clear_priv(prog, prog->priv); 274 275 prog->priv = NULL; 276 prog->clear_priv = NULL; 277 278 bpf_program__unload(prog); 279 zfree(&prog->name); 280 zfree(&prog->section_name); 281 zfree(&prog->pin_name); 282 zfree(&prog->insns); 283 zfree(&prog->reloc_desc); 284 285 prog->nr_reloc = 0; 286 prog->insns_cnt = 0; 287 prog->idx = -1; 288 } 289 290 static char *__bpf_program__pin_name(struct bpf_program *prog) 291 { 292 char *name, *p; 293 294 name = p = strdup(prog->section_name); 295 while ((p = strchr(p, '/'))) 296 *p = '_'; 297 298 return name; 299 } 300 301 static int 302 bpf_program__init(void *data, size_t size, char *section_name, int idx, 303 struct bpf_program *prog) 304 { 305 if (size < sizeof(struct bpf_insn)) { 306 pr_warning("corrupted section '%s'\n", section_name); 307 return -EINVAL; 308 } 309 310 memset(prog, 0, sizeof(*prog)); 311 312 prog->section_name = strdup(section_name); 313 if (!prog->section_name) { 314 pr_warning("failed to alloc name for prog under section(%d) %s\n", 315 idx, section_name); 316 goto errout; 317 } 318 319 prog->pin_name = __bpf_program__pin_name(prog); 320 if (!prog->pin_name) { 321 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 322 idx, section_name); 323 goto errout; 324 } 325 326 prog->insns = malloc(size); 327 if (!prog->insns) { 328 pr_warning("failed to alloc insns for prog under section %s\n", 329 section_name); 330 goto errout; 331 } 332 prog->insns_cnt = size / sizeof(struct bpf_insn); 333 memcpy(prog->insns, data, 334 prog->insns_cnt * sizeof(struct bpf_insn)); 335 prog->idx = idx; 336 prog->instances.fds = NULL; 337 prog->instances.nr = -1; 338 prog->type = BPF_PROG_TYPE_UNSPEC; 339 prog->btf_fd = -1; 340 341 return 0; 342 errout: 343 bpf_program__exit(prog); 344 return -ENOMEM; 345 } 346 347 static int 348 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 349 char *section_name, int idx) 350 { 351 struct bpf_program prog, *progs; 352 int nr_progs, err; 353 354 err = bpf_program__init(data, size, section_name, idx, &prog); 355 if (err) 356 return err; 357 358 prog.caps = &obj->caps; 359 progs = obj->programs; 360 nr_progs = obj->nr_programs; 361 362 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 363 if (!progs) { 364 /* 365 * In this case the original obj->programs 366 * is still valid, so don't need special treat for 367 * bpf_close_object(). 368 */ 369 pr_warning("failed to alloc a new program under section '%s'\n", 370 section_name); 371 bpf_program__exit(&prog); 372 return -ENOMEM; 373 } 374 375 pr_debug("found program %s\n", prog.section_name); 376 obj->programs = progs; 377 obj->nr_programs = nr_progs + 1; 378 prog.obj = obj; 379 progs[nr_progs] = prog; 380 return 0; 381 } 382 383 static int 384 bpf_object__init_prog_names(struct bpf_object *obj) 385 { 386 Elf_Data *symbols = obj->efile.symbols; 387 struct bpf_program *prog; 388 size_t pi, si; 389 390 for (pi = 0; pi < obj->nr_programs; pi++) { 391 const char *name = NULL; 392 393 prog = &obj->programs[pi]; 394 395 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 396 si++) { 397 GElf_Sym sym; 398 399 if (!gelf_getsym(symbols, si, &sym)) 400 continue; 401 if (sym.st_shndx != prog->idx) 402 continue; 403 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 404 continue; 405 406 name = elf_strptr(obj->efile.elf, 407 obj->efile.strtabidx, 408 sym.st_name); 409 if (!name) { 410 pr_warning("failed to get sym name string for prog %s\n", 411 prog->section_name); 412 return -LIBBPF_ERRNO__LIBELF; 413 } 414 } 415 416 if (!name && prog->idx == obj->efile.text_shndx) 417 name = ".text"; 418 419 if (!name) { 420 pr_warning("failed to find sym for prog %s\n", 421 prog->section_name); 422 return -EINVAL; 423 } 424 425 prog->name = strdup(name); 426 if (!prog->name) { 427 pr_warning("failed to allocate memory for prog sym %s\n", 428 name); 429 return -ENOMEM; 430 } 431 } 432 433 return 0; 434 } 435 436 static struct bpf_object *bpf_object__new(const char *path, 437 void *obj_buf, 438 size_t obj_buf_sz) 439 { 440 struct bpf_object *obj; 441 442 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 443 if (!obj) { 444 pr_warning("alloc memory failed for %s\n", path); 445 return ERR_PTR(-ENOMEM); 446 } 447 448 strcpy(obj->path, path); 449 obj->efile.fd = -1; 450 451 /* 452 * Caller of this function should also calls 453 * bpf_object__elf_finish() after data collection to return 454 * obj_buf to user. If not, we should duplicate the buffer to 455 * avoid user freeing them before elf finish. 456 */ 457 obj->efile.obj_buf = obj_buf; 458 obj->efile.obj_buf_sz = obj_buf_sz; 459 obj->efile.maps_shndx = -1; 460 461 obj->loaded = false; 462 463 INIT_LIST_HEAD(&obj->list); 464 list_add(&obj->list, &bpf_objects_list); 465 return obj; 466 } 467 468 static void bpf_object__elf_finish(struct bpf_object *obj) 469 { 470 if (!obj_elf_valid(obj)) 471 return; 472 473 if (obj->efile.elf) { 474 elf_end(obj->efile.elf); 475 obj->efile.elf = NULL; 476 } 477 obj->efile.symbols = NULL; 478 479 zfree(&obj->efile.reloc); 480 obj->efile.nr_reloc = 0; 481 zclose(obj->efile.fd); 482 obj->efile.obj_buf = NULL; 483 obj->efile.obj_buf_sz = 0; 484 } 485 486 static int bpf_object__elf_init(struct bpf_object *obj) 487 { 488 int err = 0; 489 GElf_Ehdr *ep; 490 491 if (obj_elf_valid(obj)) { 492 pr_warning("elf init: internal error\n"); 493 return -LIBBPF_ERRNO__LIBELF; 494 } 495 496 if (obj->efile.obj_buf_sz > 0) { 497 /* 498 * obj_buf should have been validated by 499 * bpf_object__open_buffer(). 500 */ 501 obj->efile.elf = elf_memory(obj->efile.obj_buf, 502 obj->efile.obj_buf_sz); 503 } else { 504 obj->efile.fd = open(obj->path, O_RDONLY); 505 if (obj->efile.fd < 0) { 506 char errmsg[STRERR_BUFSIZE]; 507 char *cp = libbpf_strerror_r(errno, errmsg, 508 sizeof(errmsg)); 509 510 pr_warning("failed to open %s: %s\n", obj->path, cp); 511 return -errno; 512 } 513 514 obj->efile.elf = elf_begin(obj->efile.fd, 515 LIBBPF_ELF_C_READ_MMAP, 516 NULL); 517 } 518 519 if (!obj->efile.elf) { 520 pr_warning("failed to open %s as ELF file\n", 521 obj->path); 522 err = -LIBBPF_ERRNO__LIBELF; 523 goto errout; 524 } 525 526 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 527 pr_warning("failed to get EHDR from %s\n", 528 obj->path); 529 err = -LIBBPF_ERRNO__FORMAT; 530 goto errout; 531 } 532 ep = &obj->efile.ehdr; 533 534 /* Old LLVM set e_machine to EM_NONE */ 535 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 536 pr_warning("%s is not an eBPF object file\n", 537 obj->path); 538 err = -LIBBPF_ERRNO__FORMAT; 539 goto errout; 540 } 541 542 return 0; 543 errout: 544 bpf_object__elf_finish(obj); 545 return err; 546 } 547 548 static int 549 bpf_object__check_endianness(struct bpf_object *obj) 550 { 551 static unsigned int const endian = 1; 552 553 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 554 case ELFDATA2LSB: 555 /* We are big endian, BPF obj is little endian. */ 556 if (*(unsigned char const *)&endian != 1) 557 goto mismatch; 558 break; 559 560 case ELFDATA2MSB: 561 /* We are little endian, BPF obj is big endian. */ 562 if (*(unsigned char const *)&endian != 0) 563 goto mismatch; 564 break; 565 default: 566 return -LIBBPF_ERRNO__ENDIAN; 567 } 568 569 return 0; 570 571 mismatch: 572 pr_warning("Error: endianness mismatch.\n"); 573 return -LIBBPF_ERRNO__ENDIAN; 574 } 575 576 static int 577 bpf_object__init_license(struct bpf_object *obj, 578 void *data, size_t size) 579 { 580 memcpy(obj->license, data, 581 min(size, sizeof(obj->license) - 1)); 582 pr_debug("license of %s is %s\n", obj->path, obj->license); 583 return 0; 584 } 585 586 static int 587 bpf_object__init_kversion(struct bpf_object *obj, 588 void *data, size_t size) 589 { 590 __u32 kver; 591 592 if (size != sizeof(kver)) { 593 pr_warning("invalid kver section in %s\n", obj->path); 594 return -LIBBPF_ERRNO__FORMAT; 595 } 596 memcpy(&kver, data, sizeof(kver)); 597 obj->kern_version = kver; 598 pr_debug("kernel version of %s is %x\n", obj->path, 599 obj->kern_version); 600 return 0; 601 } 602 603 static int compare_bpf_map(const void *_a, const void *_b) 604 { 605 const struct bpf_map *a = _a; 606 const struct bpf_map *b = _b; 607 608 return a->offset - b->offset; 609 } 610 611 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 612 { 613 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 614 type == BPF_MAP_TYPE_HASH_OF_MAPS) 615 return true; 616 return false; 617 } 618 619 static int 620 bpf_object__init_maps(struct bpf_object *obj, int flags) 621 { 622 bool strict = !(flags & MAPS_RELAX_COMPAT); 623 int i, map_idx, map_def_sz, nr_maps = 0; 624 Elf_Scn *scn; 625 Elf_Data *data; 626 Elf_Data *symbols = obj->efile.symbols; 627 628 if (obj->efile.maps_shndx < 0) 629 return -EINVAL; 630 if (!symbols) 631 return -EINVAL; 632 633 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 634 if (scn) 635 data = elf_getdata(scn, NULL); 636 if (!scn || !data) { 637 pr_warning("failed to get Elf_Data from map section %d\n", 638 obj->efile.maps_shndx); 639 return -EINVAL; 640 } 641 642 /* 643 * Count number of maps. Each map has a name. 644 * Array of maps is not supported: only the first element is 645 * considered. 646 * 647 * TODO: Detect array of map and report error. 648 */ 649 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 650 GElf_Sym sym; 651 652 if (!gelf_getsym(symbols, i, &sym)) 653 continue; 654 if (sym.st_shndx != obj->efile.maps_shndx) 655 continue; 656 nr_maps++; 657 } 658 659 /* Alloc obj->maps and fill nr_maps. */ 660 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 661 nr_maps, data->d_size); 662 663 if (!nr_maps) 664 return 0; 665 666 /* Assume equally sized map definitions */ 667 map_def_sz = data->d_size / nr_maps; 668 if (!data->d_size || (data->d_size % nr_maps) != 0) { 669 pr_warning("unable to determine map definition size " 670 "section %s, %d maps in %zd bytes\n", 671 obj->path, nr_maps, data->d_size); 672 return -EINVAL; 673 } 674 675 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 676 if (!obj->maps) { 677 pr_warning("alloc maps for object failed\n"); 678 return -ENOMEM; 679 } 680 obj->nr_maps = nr_maps; 681 682 for (i = 0; i < nr_maps; i++) { 683 /* 684 * fill all fd with -1 so won't close incorrect 685 * fd (fd=0 is stdin) when failure (zclose won't close 686 * negative fd)). 687 */ 688 obj->maps[i].fd = -1; 689 obj->maps[i].inner_map_fd = -1; 690 } 691 692 /* 693 * Fill obj->maps using data in "maps" section. 694 */ 695 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 696 GElf_Sym sym; 697 const char *map_name; 698 struct bpf_map_def *def; 699 700 if (!gelf_getsym(symbols, i, &sym)) 701 continue; 702 if (sym.st_shndx != obj->efile.maps_shndx) 703 continue; 704 705 map_name = elf_strptr(obj->efile.elf, 706 obj->efile.strtabidx, 707 sym.st_name); 708 obj->maps[map_idx].offset = sym.st_value; 709 if (sym.st_value + map_def_sz > data->d_size) { 710 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 711 obj->path, map_name); 712 return -EINVAL; 713 } 714 715 obj->maps[map_idx].name = strdup(map_name); 716 if (!obj->maps[map_idx].name) { 717 pr_warning("failed to alloc map name\n"); 718 return -ENOMEM; 719 } 720 pr_debug("map %d is \"%s\"\n", map_idx, 721 obj->maps[map_idx].name); 722 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 723 /* 724 * If the definition of the map in the object file fits in 725 * bpf_map_def, copy it. Any extra fields in our version 726 * of bpf_map_def will default to zero as a result of the 727 * calloc above. 728 */ 729 if (map_def_sz <= sizeof(struct bpf_map_def)) { 730 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 731 } else { 732 /* 733 * Here the map structure being read is bigger than what 734 * we expect, truncate if the excess bits are all zero. 735 * If they are not zero, reject this map as 736 * incompatible. 737 */ 738 char *b; 739 for (b = ((char *)def) + sizeof(struct bpf_map_def); 740 b < ((char *)def) + map_def_sz; b++) { 741 if (*b != 0) { 742 pr_warning("maps section in %s: \"%s\" " 743 "has unrecognized, non-zero " 744 "options\n", 745 obj->path, map_name); 746 if (strict) 747 return -EINVAL; 748 } 749 } 750 memcpy(&obj->maps[map_idx].def, def, 751 sizeof(struct bpf_map_def)); 752 } 753 map_idx++; 754 } 755 756 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); 757 return 0; 758 } 759 760 static bool section_have_execinstr(struct bpf_object *obj, int idx) 761 { 762 Elf_Scn *scn; 763 GElf_Shdr sh; 764 765 scn = elf_getscn(obj->efile.elf, idx); 766 if (!scn) 767 return false; 768 769 if (gelf_getshdr(scn, &sh) != &sh) 770 return false; 771 772 if (sh.sh_flags & SHF_EXECINSTR) 773 return true; 774 775 return false; 776 } 777 778 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 779 { 780 Elf *elf = obj->efile.elf; 781 GElf_Ehdr *ep = &obj->efile.ehdr; 782 Elf_Data *btf_ext_data = NULL; 783 Elf_Scn *scn = NULL; 784 int idx = 0, err = 0; 785 786 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 787 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 788 pr_warning("failed to get e_shstrndx from %s\n", 789 obj->path); 790 return -LIBBPF_ERRNO__FORMAT; 791 } 792 793 while ((scn = elf_nextscn(elf, scn)) != NULL) { 794 char *name; 795 GElf_Shdr sh; 796 Elf_Data *data; 797 798 idx++; 799 if (gelf_getshdr(scn, &sh) != &sh) { 800 pr_warning("failed to get section(%d) header from %s\n", 801 idx, obj->path); 802 err = -LIBBPF_ERRNO__FORMAT; 803 goto out; 804 } 805 806 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 807 if (!name) { 808 pr_warning("failed to get section(%d) name from %s\n", 809 idx, obj->path); 810 err = -LIBBPF_ERRNO__FORMAT; 811 goto out; 812 } 813 814 data = elf_getdata(scn, 0); 815 if (!data) { 816 pr_warning("failed to get section(%d) data from %s(%s)\n", 817 idx, name, obj->path); 818 err = -LIBBPF_ERRNO__FORMAT; 819 goto out; 820 } 821 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 822 idx, name, (unsigned long)data->d_size, 823 (int)sh.sh_link, (unsigned long)sh.sh_flags, 824 (int)sh.sh_type); 825 826 if (strcmp(name, "license") == 0) 827 err = bpf_object__init_license(obj, 828 data->d_buf, 829 data->d_size); 830 else if (strcmp(name, "version") == 0) 831 err = bpf_object__init_kversion(obj, 832 data->d_buf, 833 data->d_size); 834 else if (strcmp(name, "maps") == 0) 835 obj->efile.maps_shndx = idx; 836 else if (strcmp(name, BTF_ELF_SEC) == 0) { 837 obj->btf = btf__new(data->d_buf, data->d_size); 838 if (IS_ERR(obj->btf) || btf__load(obj->btf)) { 839 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 840 BTF_ELF_SEC, PTR_ERR(obj->btf)); 841 if (!IS_ERR(obj->btf)) 842 btf__free(obj->btf); 843 obj->btf = NULL; 844 } 845 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 846 btf_ext_data = data; 847 } else if (sh.sh_type == SHT_SYMTAB) { 848 if (obj->efile.symbols) { 849 pr_warning("bpf: multiple SYMTAB in %s\n", 850 obj->path); 851 err = -LIBBPF_ERRNO__FORMAT; 852 } else { 853 obj->efile.symbols = data; 854 obj->efile.strtabidx = sh.sh_link; 855 } 856 } else if ((sh.sh_type == SHT_PROGBITS) && 857 (sh.sh_flags & SHF_EXECINSTR) && 858 (data->d_size > 0)) { 859 if (strcmp(name, ".text") == 0) 860 obj->efile.text_shndx = idx; 861 err = bpf_object__add_program(obj, data->d_buf, 862 data->d_size, name, idx); 863 if (err) { 864 char errmsg[STRERR_BUFSIZE]; 865 char *cp = libbpf_strerror_r(-err, errmsg, 866 sizeof(errmsg)); 867 868 pr_warning("failed to alloc program %s (%s): %s", 869 name, obj->path, cp); 870 } 871 } else if (sh.sh_type == SHT_REL) { 872 void *reloc = obj->efile.reloc; 873 int nr_reloc = obj->efile.nr_reloc + 1; 874 int sec = sh.sh_info; /* points to other section */ 875 876 /* Only do relo for section with exec instructions */ 877 if (!section_have_execinstr(obj, sec)) { 878 pr_debug("skip relo %s(%d) for section(%d)\n", 879 name, idx, sec); 880 continue; 881 } 882 883 reloc = reallocarray(reloc, nr_reloc, 884 sizeof(*obj->efile.reloc)); 885 if (!reloc) { 886 pr_warning("realloc failed\n"); 887 err = -ENOMEM; 888 } else { 889 int n = nr_reloc - 1; 890 891 obj->efile.reloc = reloc; 892 obj->efile.nr_reloc = nr_reloc; 893 894 obj->efile.reloc[n].shdr = sh; 895 obj->efile.reloc[n].data = data; 896 } 897 } else { 898 pr_debug("skip section(%d) %s\n", idx, name); 899 } 900 if (err) 901 goto out; 902 } 903 904 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 905 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 906 return LIBBPF_ERRNO__FORMAT; 907 } 908 if (btf_ext_data) { 909 if (!obj->btf) { 910 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 911 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 912 } else { 913 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 914 btf_ext_data->d_size); 915 if (IS_ERR(obj->btf_ext)) { 916 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 917 BTF_EXT_ELF_SEC, 918 PTR_ERR(obj->btf_ext)); 919 obj->btf_ext = NULL; 920 } 921 } 922 } 923 if (obj->efile.maps_shndx >= 0) { 924 err = bpf_object__init_maps(obj, flags); 925 if (err) 926 goto out; 927 } 928 err = bpf_object__init_prog_names(obj); 929 out: 930 return err; 931 } 932 933 static struct bpf_program * 934 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 935 { 936 struct bpf_program *prog; 937 size_t i; 938 939 for (i = 0; i < obj->nr_programs; i++) { 940 prog = &obj->programs[i]; 941 if (prog->idx == idx) 942 return prog; 943 } 944 return NULL; 945 } 946 947 struct bpf_program * 948 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 949 { 950 struct bpf_program *pos; 951 952 bpf_object__for_each_program(pos, obj) { 953 if (pos->section_name && !strcmp(pos->section_name, title)) 954 return pos; 955 } 956 return NULL; 957 } 958 959 static int 960 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 961 Elf_Data *data, struct bpf_object *obj) 962 { 963 Elf_Data *symbols = obj->efile.symbols; 964 int text_shndx = obj->efile.text_shndx; 965 int maps_shndx = obj->efile.maps_shndx; 966 struct bpf_map *maps = obj->maps; 967 size_t nr_maps = obj->nr_maps; 968 int i, nrels; 969 970 pr_debug("collecting relocating info for: '%s'\n", 971 prog->section_name); 972 nrels = shdr->sh_size / shdr->sh_entsize; 973 974 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 975 if (!prog->reloc_desc) { 976 pr_warning("failed to alloc memory in relocation\n"); 977 return -ENOMEM; 978 } 979 prog->nr_reloc = nrels; 980 981 for (i = 0; i < nrels; i++) { 982 GElf_Sym sym; 983 GElf_Rel rel; 984 unsigned int insn_idx; 985 struct bpf_insn *insns = prog->insns; 986 size_t map_idx; 987 988 if (!gelf_getrel(data, i, &rel)) { 989 pr_warning("relocation: failed to get %d reloc\n", i); 990 return -LIBBPF_ERRNO__FORMAT; 991 } 992 993 if (!gelf_getsym(symbols, 994 GELF_R_SYM(rel.r_info), 995 &sym)) { 996 pr_warning("relocation: symbol %"PRIx64" not found\n", 997 GELF_R_SYM(rel.r_info)); 998 return -LIBBPF_ERRNO__FORMAT; 999 } 1000 pr_debug("relo for %lld value %lld name %d\n", 1001 (long long) (rel.r_info >> 32), 1002 (long long) sym.st_value, sym.st_name); 1003 1004 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { 1005 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 1006 prog->section_name, sym.st_shndx); 1007 return -LIBBPF_ERRNO__RELOC; 1008 } 1009 1010 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1011 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1012 1013 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1014 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1015 pr_warning("incorrect bpf_call opcode\n"); 1016 return -LIBBPF_ERRNO__RELOC; 1017 } 1018 prog->reloc_desc[i].type = RELO_CALL; 1019 prog->reloc_desc[i].insn_idx = insn_idx; 1020 prog->reloc_desc[i].text_off = sym.st_value; 1021 obj->has_pseudo_calls = true; 1022 continue; 1023 } 1024 1025 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1026 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1027 insn_idx, insns[insn_idx].code); 1028 return -LIBBPF_ERRNO__RELOC; 1029 } 1030 1031 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ 1032 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1033 if (maps[map_idx].offset == sym.st_value) { 1034 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1035 map_idx, maps[map_idx].name, insn_idx); 1036 break; 1037 } 1038 } 1039 1040 if (map_idx >= nr_maps) { 1041 pr_warning("bpf relocation: map_idx %d large than %d\n", 1042 (int)map_idx, (int)nr_maps - 1); 1043 return -LIBBPF_ERRNO__RELOC; 1044 } 1045 1046 prog->reloc_desc[i].type = RELO_LD64; 1047 prog->reloc_desc[i].insn_idx = insn_idx; 1048 prog->reloc_desc[i].map_idx = map_idx; 1049 } 1050 return 0; 1051 } 1052 1053 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1054 { 1055 struct bpf_map_def *def = &map->def; 1056 __u32 key_type_id, value_type_id; 1057 int ret; 1058 1059 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1060 def->value_size, &key_type_id, 1061 &value_type_id); 1062 if (ret) 1063 return ret; 1064 1065 map->btf_key_type_id = key_type_id; 1066 map->btf_value_type_id = value_type_id; 1067 1068 return 0; 1069 } 1070 1071 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1072 { 1073 struct bpf_map_info info = {}; 1074 __u32 len = sizeof(info); 1075 int new_fd, err; 1076 char *new_name; 1077 1078 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1079 if (err) 1080 return err; 1081 1082 new_name = strdup(info.name); 1083 if (!new_name) 1084 return -errno; 1085 1086 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1087 if (new_fd < 0) 1088 goto err_free_new_name; 1089 1090 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1091 if (new_fd < 0) 1092 goto err_close_new_fd; 1093 1094 err = zclose(map->fd); 1095 if (err) 1096 goto err_close_new_fd; 1097 free(map->name); 1098 1099 map->fd = new_fd; 1100 map->name = new_name; 1101 map->def.type = info.type; 1102 map->def.key_size = info.key_size; 1103 map->def.value_size = info.value_size; 1104 map->def.max_entries = info.max_entries; 1105 map->def.map_flags = info.map_flags; 1106 map->btf_key_type_id = info.btf_key_type_id; 1107 map->btf_value_type_id = info.btf_value_type_id; 1108 1109 return 0; 1110 1111 err_close_new_fd: 1112 close(new_fd); 1113 err_free_new_name: 1114 free(new_name); 1115 return -errno; 1116 } 1117 1118 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1119 { 1120 if (!map || !max_entries) 1121 return -EINVAL; 1122 1123 /* If map already created, its attributes can't be changed. */ 1124 if (map->fd >= 0) 1125 return -EBUSY; 1126 1127 map->def.max_entries = max_entries; 1128 1129 return 0; 1130 } 1131 1132 static int 1133 bpf_object__probe_name(struct bpf_object *obj) 1134 { 1135 struct bpf_load_program_attr attr; 1136 char *cp, errmsg[STRERR_BUFSIZE]; 1137 struct bpf_insn insns[] = { 1138 BPF_MOV64_IMM(BPF_REG_0, 0), 1139 BPF_EXIT_INSN(), 1140 }; 1141 int ret; 1142 1143 /* make sure basic loading works */ 1144 1145 memset(&attr, 0, sizeof(attr)); 1146 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1147 attr.insns = insns; 1148 attr.insns_cnt = ARRAY_SIZE(insns); 1149 attr.license = "GPL"; 1150 1151 ret = bpf_load_program_xattr(&attr, NULL, 0); 1152 if (ret < 0) { 1153 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1154 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1155 __func__, cp, errno); 1156 return -errno; 1157 } 1158 close(ret); 1159 1160 /* now try the same program, but with the name */ 1161 1162 attr.name = "test"; 1163 ret = bpf_load_program_xattr(&attr, NULL, 0); 1164 if (ret >= 0) { 1165 obj->caps.name = 1; 1166 close(ret); 1167 } 1168 1169 return 0; 1170 } 1171 1172 static int 1173 bpf_object__probe_caps(struct bpf_object *obj) 1174 { 1175 return bpf_object__probe_name(obj); 1176 } 1177 1178 static int 1179 bpf_object__create_maps(struct bpf_object *obj) 1180 { 1181 struct bpf_create_map_attr create_attr = {}; 1182 unsigned int i; 1183 int err; 1184 1185 for (i = 0; i < obj->nr_maps; i++) { 1186 struct bpf_map *map = &obj->maps[i]; 1187 struct bpf_map_def *def = &map->def; 1188 char *cp, errmsg[STRERR_BUFSIZE]; 1189 int *pfd = &map->fd; 1190 1191 if (map->fd >= 0) { 1192 pr_debug("skip map create (preset) %s: fd=%d\n", 1193 map->name, map->fd); 1194 continue; 1195 } 1196 1197 if (obj->caps.name) 1198 create_attr.name = map->name; 1199 create_attr.map_ifindex = map->map_ifindex; 1200 create_attr.map_type = def->type; 1201 create_attr.map_flags = def->map_flags; 1202 create_attr.key_size = def->key_size; 1203 create_attr.value_size = def->value_size; 1204 create_attr.max_entries = def->max_entries; 1205 create_attr.btf_fd = 0; 1206 create_attr.btf_key_type_id = 0; 1207 create_attr.btf_value_type_id = 0; 1208 if (bpf_map_type__is_map_in_map(def->type) && 1209 map->inner_map_fd >= 0) 1210 create_attr.inner_map_fd = map->inner_map_fd; 1211 1212 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1213 create_attr.btf_fd = btf__fd(obj->btf); 1214 create_attr.btf_key_type_id = map->btf_key_type_id; 1215 create_attr.btf_value_type_id = map->btf_value_type_id; 1216 } 1217 1218 *pfd = bpf_create_map_xattr(&create_attr); 1219 if (*pfd < 0 && create_attr.btf_key_type_id) { 1220 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1221 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1222 map->name, cp, errno); 1223 create_attr.btf_fd = 0; 1224 create_attr.btf_key_type_id = 0; 1225 create_attr.btf_value_type_id = 0; 1226 map->btf_key_type_id = 0; 1227 map->btf_value_type_id = 0; 1228 *pfd = bpf_create_map_xattr(&create_attr); 1229 } 1230 1231 if (*pfd < 0) { 1232 size_t j; 1233 1234 err = *pfd; 1235 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1236 pr_warning("failed to create map (name: '%s'): %s\n", 1237 map->name, cp); 1238 for (j = 0; j < i; j++) 1239 zclose(obj->maps[j].fd); 1240 return err; 1241 } 1242 pr_debug("create map %s: fd=%d\n", map->name, *pfd); 1243 } 1244 1245 return 0; 1246 } 1247 1248 static int 1249 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1250 void *btf_prog_info, const char *info_name) 1251 { 1252 if (err != -ENOENT) { 1253 pr_warning("Error in loading %s for sec %s.\n", 1254 info_name, prog->section_name); 1255 return err; 1256 } 1257 1258 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1259 1260 if (btf_prog_info) { 1261 /* 1262 * Some info has already been found but has problem 1263 * in the last btf_ext reloc. Must have to error 1264 * out. 1265 */ 1266 pr_warning("Error in relocating %s for sec %s.\n", 1267 info_name, prog->section_name); 1268 return err; 1269 } 1270 1271 /* 1272 * Have problem loading the very first info. Ignore 1273 * the rest. 1274 */ 1275 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1276 info_name, prog->section_name, info_name); 1277 return 0; 1278 } 1279 1280 static int 1281 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1282 const char *section_name, __u32 insn_offset) 1283 { 1284 int err; 1285 1286 if (!insn_offset || prog->func_info) { 1287 /* 1288 * !insn_offset => main program 1289 * 1290 * For sub prog, the main program's func_info has to 1291 * be loaded first (i.e. prog->func_info != NULL) 1292 */ 1293 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1294 section_name, insn_offset, 1295 &prog->func_info, 1296 &prog->func_info_cnt); 1297 if (err) 1298 return check_btf_ext_reloc_err(prog, err, 1299 prog->func_info, 1300 "bpf_func_info"); 1301 1302 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1303 } 1304 1305 if (!insn_offset || prog->line_info) { 1306 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1307 section_name, insn_offset, 1308 &prog->line_info, 1309 &prog->line_info_cnt); 1310 if (err) 1311 return check_btf_ext_reloc_err(prog, err, 1312 prog->line_info, 1313 "bpf_line_info"); 1314 1315 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1316 } 1317 1318 if (!insn_offset) 1319 prog->btf_fd = btf__fd(obj->btf); 1320 1321 return 0; 1322 } 1323 1324 static int 1325 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1326 struct reloc_desc *relo) 1327 { 1328 struct bpf_insn *insn, *new_insn; 1329 struct bpf_program *text; 1330 size_t new_cnt; 1331 int err; 1332 1333 if (relo->type != RELO_CALL) 1334 return -LIBBPF_ERRNO__RELOC; 1335 1336 if (prog->idx == obj->efile.text_shndx) { 1337 pr_warning("relo in .text insn %d into off %d\n", 1338 relo->insn_idx, relo->text_off); 1339 return -LIBBPF_ERRNO__RELOC; 1340 } 1341 1342 if (prog->main_prog_cnt == 0) { 1343 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1344 if (!text) { 1345 pr_warning("no .text section found yet relo into text exist\n"); 1346 return -LIBBPF_ERRNO__RELOC; 1347 } 1348 new_cnt = prog->insns_cnt + text->insns_cnt; 1349 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1350 if (!new_insn) { 1351 pr_warning("oom in prog realloc\n"); 1352 return -ENOMEM; 1353 } 1354 1355 if (obj->btf_ext) { 1356 err = bpf_program_reloc_btf_ext(prog, obj, 1357 text->section_name, 1358 prog->insns_cnt); 1359 if (err) 1360 return err; 1361 } 1362 1363 memcpy(new_insn + prog->insns_cnt, text->insns, 1364 text->insns_cnt * sizeof(*insn)); 1365 prog->insns = new_insn; 1366 prog->main_prog_cnt = prog->insns_cnt; 1367 prog->insns_cnt = new_cnt; 1368 pr_debug("added %zd insn from %s to prog %s\n", 1369 text->insns_cnt, text->section_name, 1370 prog->section_name); 1371 } 1372 insn = &prog->insns[relo->insn_idx]; 1373 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1374 return 0; 1375 } 1376 1377 static int 1378 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1379 { 1380 int i, err; 1381 1382 if (!prog) 1383 return 0; 1384 1385 if (obj->btf_ext) { 1386 err = bpf_program_reloc_btf_ext(prog, obj, 1387 prog->section_name, 0); 1388 if (err) 1389 return err; 1390 } 1391 1392 if (!prog->reloc_desc) 1393 return 0; 1394 1395 for (i = 0; i < prog->nr_reloc; i++) { 1396 if (prog->reloc_desc[i].type == RELO_LD64) { 1397 struct bpf_insn *insns = prog->insns; 1398 int insn_idx, map_idx; 1399 1400 insn_idx = prog->reloc_desc[i].insn_idx; 1401 map_idx = prog->reloc_desc[i].map_idx; 1402 1403 if (insn_idx >= (int)prog->insns_cnt) { 1404 pr_warning("relocation out of range: '%s'\n", 1405 prog->section_name); 1406 return -LIBBPF_ERRNO__RELOC; 1407 } 1408 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1409 insns[insn_idx].imm = obj->maps[map_idx].fd; 1410 } else { 1411 err = bpf_program__reloc_text(prog, obj, 1412 &prog->reloc_desc[i]); 1413 if (err) 1414 return err; 1415 } 1416 } 1417 1418 zfree(&prog->reloc_desc); 1419 prog->nr_reloc = 0; 1420 return 0; 1421 } 1422 1423 1424 static int 1425 bpf_object__relocate(struct bpf_object *obj) 1426 { 1427 struct bpf_program *prog; 1428 size_t i; 1429 int err; 1430 1431 for (i = 0; i < obj->nr_programs; i++) { 1432 prog = &obj->programs[i]; 1433 1434 err = bpf_program__relocate(prog, obj); 1435 if (err) { 1436 pr_warning("failed to relocate '%s'\n", 1437 prog->section_name); 1438 return err; 1439 } 1440 } 1441 return 0; 1442 } 1443 1444 static int bpf_object__collect_reloc(struct bpf_object *obj) 1445 { 1446 int i, err; 1447 1448 if (!obj_elf_valid(obj)) { 1449 pr_warning("Internal error: elf object is closed\n"); 1450 return -LIBBPF_ERRNO__INTERNAL; 1451 } 1452 1453 for (i = 0; i < obj->efile.nr_reloc; i++) { 1454 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1455 Elf_Data *data = obj->efile.reloc[i].data; 1456 int idx = shdr->sh_info; 1457 struct bpf_program *prog; 1458 1459 if (shdr->sh_type != SHT_REL) { 1460 pr_warning("internal error at %d\n", __LINE__); 1461 return -LIBBPF_ERRNO__INTERNAL; 1462 } 1463 1464 prog = bpf_object__find_prog_by_idx(obj, idx); 1465 if (!prog) { 1466 pr_warning("relocation failed: no section(%d)\n", idx); 1467 return -LIBBPF_ERRNO__RELOC; 1468 } 1469 1470 err = bpf_program__collect_reloc(prog, 1471 shdr, data, 1472 obj); 1473 if (err) 1474 return err; 1475 } 1476 return 0; 1477 } 1478 1479 static int 1480 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 1481 char *license, __u32 kern_version, int *pfd) 1482 { 1483 struct bpf_load_program_attr load_attr; 1484 char *cp, errmsg[STRERR_BUFSIZE]; 1485 char *log_buf; 1486 int ret; 1487 1488 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 1489 load_attr.prog_type = prog->type; 1490 load_attr.expected_attach_type = prog->expected_attach_type; 1491 if (prog->caps->name) 1492 load_attr.name = prog->name; 1493 load_attr.insns = insns; 1494 load_attr.insns_cnt = insns_cnt; 1495 load_attr.license = license; 1496 load_attr.kern_version = kern_version; 1497 load_attr.prog_ifindex = prog->prog_ifindex; 1498 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 1499 load_attr.func_info = prog->func_info; 1500 load_attr.func_info_rec_size = prog->func_info_rec_size; 1501 load_attr.func_info_cnt = prog->func_info_cnt; 1502 load_attr.line_info = prog->line_info; 1503 load_attr.line_info_rec_size = prog->line_info_rec_size; 1504 load_attr.line_info_cnt = prog->line_info_cnt; 1505 if (!load_attr.insns || !load_attr.insns_cnt) 1506 return -EINVAL; 1507 1508 log_buf = malloc(BPF_LOG_BUF_SIZE); 1509 if (!log_buf) 1510 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1511 1512 ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE); 1513 1514 if (ret >= 0) { 1515 *pfd = ret; 1516 ret = 0; 1517 goto out; 1518 } 1519 1520 ret = -LIBBPF_ERRNO__LOAD; 1521 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1522 pr_warning("load bpf program failed: %s\n", cp); 1523 1524 if (log_buf && log_buf[0] != '\0') { 1525 ret = -LIBBPF_ERRNO__VERIFY; 1526 pr_warning("-- BEGIN DUMP LOG ---\n"); 1527 pr_warning("\n%s\n", log_buf); 1528 pr_warning("-- END LOG --\n"); 1529 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 1530 pr_warning("Program too large (%zu insns), at most %d insns\n", 1531 load_attr.insns_cnt, BPF_MAXINSNS); 1532 ret = -LIBBPF_ERRNO__PROG2BIG; 1533 } else { 1534 /* Wrong program type? */ 1535 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 1536 int fd; 1537 1538 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 1539 load_attr.expected_attach_type = 0; 1540 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 1541 if (fd >= 0) { 1542 close(fd); 1543 ret = -LIBBPF_ERRNO__PROGTYPE; 1544 goto out; 1545 } 1546 } 1547 1548 if (log_buf) 1549 ret = -LIBBPF_ERRNO__KVER; 1550 } 1551 1552 out: 1553 free(log_buf); 1554 return ret; 1555 } 1556 1557 int 1558 bpf_program__load(struct bpf_program *prog, 1559 char *license, __u32 kern_version) 1560 { 1561 int err = 0, fd, i; 1562 1563 if (prog->instances.nr < 0 || !prog->instances.fds) { 1564 if (prog->preprocessor) { 1565 pr_warning("Internal error: can't load program '%s'\n", 1566 prog->section_name); 1567 return -LIBBPF_ERRNO__INTERNAL; 1568 } 1569 1570 prog->instances.fds = malloc(sizeof(int)); 1571 if (!prog->instances.fds) { 1572 pr_warning("Not enough memory for BPF fds\n"); 1573 return -ENOMEM; 1574 } 1575 prog->instances.nr = 1; 1576 prog->instances.fds[0] = -1; 1577 } 1578 1579 if (!prog->preprocessor) { 1580 if (prog->instances.nr != 1) { 1581 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 1582 prog->section_name, prog->instances.nr); 1583 } 1584 err = load_program(prog, prog->insns, prog->insns_cnt, 1585 license, kern_version, &fd); 1586 if (!err) 1587 prog->instances.fds[0] = fd; 1588 goto out; 1589 } 1590 1591 for (i = 0; i < prog->instances.nr; i++) { 1592 struct bpf_prog_prep_result result; 1593 bpf_program_prep_t preprocessor = prog->preprocessor; 1594 1595 memset(&result, 0, sizeof(result)); 1596 err = preprocessor(prog, i, prog->insns, 1597 prog->insns_cnt, &result); 1598 if (err) { 1599 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 1600 i, prog->section_name); 1601 goto out; 1602 } 1603 1604 if (!result.new_insn_ptr || !result.new_insn_cnt) { 1605 pr_debug("Skip loading the %dth instance of program '%s'\n", 1606 i, prog->section_name); 1607 prog->instances.fds[i] = -1; 1608 if (result.pfd) 1609 *result.pfd = -1; 1610 continue; 1611 } 1612 1613 err = load_program(prog, result.new_insn_ptr, 1614 result.new_insn_cnt, 1615 license, kern_version, &fd); 1616 1617 if (err) { 1618 pr_warning("Loading the %dth instance of program '%s' failed\n", 1619 i, prog->section_name); 1620 goto out; 1621 } 1622 1623 if (result.pfd) 1624 *result.pfd = fd; 1625 prog->instances.fds[i] = fd; 1626 } 1627 out: 1628 if (err) 1629 pr_warning("failed to load program '%s'\n", 1630 prog->section_name); 1631 zfree(&prog->insns); 1632 prog->insns_cnt = 0; 1633 return err; 1634 } 1635 1636 static bool bpf_program__is_function_storage(struct bpf_program *prog, 1637 struct bpf_object *obj) 1638 { 1639 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 1640 } 1641 1642 static int 1643 bpf_object__load_progs(struct bpf_object *obj) 1644 { 1645 size_t i; 1646 int err; 1647 1648 for (i = 0; i < obj->nr_programs; i++) { 1649 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 1650 continue; 1651 err = bpf_program__load(&obj->programs[i], 1652 obj->license, 1653 obj->kern_version); 1654 if (err) 1655 return err; 1656 } 1657 return 0; 1658 } 1659 1660 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 1661 { 1662 switch (type) { 1663 case BPF_PROG_TYPE_SOCKET_FILTER: 1664 case BPF_PROG_TYPE_SCHED_CLS: 1665 case BPF_PROG_TYPE_SCHED_ACT: 1666 case BPF_PROG_TYPE_XDP: 1667 case BPF_PROG_TYPE_CGROUP_SKB: 1668 case BPF_PROG_TYPE_CGROUP_SOCK: 1669 case BPF_PROG_TYPE_LWT_IN: 1670 case BPF_PROG_TYPE_LWT_OUT: 1671 case BPF_PROG_TYPE_LWT_XMIT: 1672 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 1673 case BPF_PROG_TYPE_SOCK_OPS: 1674 case BPF_PROG_TYPE_SK_SKB: 1675 case BPF_PROG_TYPE_CGROUP_DEVICE: 1676 case BPF_PROG_TYPE_SK_MSG: 1677 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 1678 case BPF_PROG_TYPE_LIRC_MODE2: 1679 case BPF_PROG_TYPE_SK_REUSEPORT: 1680 case BPF_PROG_TYPE_FLOW_DISSECTOR: 1681 case BPF_PROG_TYPE_UNSPEC: 1682 case BPF_PROG_TYPE_TRACEPOINT: 1683 case BPF_PROG_TYPE_RAW_TRACEPOINT: 1684 case BPF_PROG_TYPE_PERF_EVENT: 1685 return false; 1686 case BPF_PROG_TYPE_KPROBE: 1687 default: 1688 return true; 1689 } 1690 } 1691 1692 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 1693 { 1694 if (needs_kver && obj->kern_version == 0) { 1695 pr_warning("%s doesn't provide kernel version\n", 1696 obj->path); 1697 return -LIBBPF_ERRNO__KVERSION; 1698 } 1699 return 0; 1700 } 1701 1702 static struct bpf_object * 1703 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 1704 bool needs_kver, int flags) 1705 { 1706 struct bpf_object *obj; 1707 int err; 1708 1709 if (elf_version(EV_CURRENT) == EV_NONE) { 1710 pr_warning("failed to init libelf for %s\n", path); 1711 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1712 } 1713 1714 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1715 if (IS_ERR(obj)) 1716 return obj; 1717 1718 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1719 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1720 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 1721 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1722 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 1723 1724 bpf_object__elf_finish(obj); 1725 return obj; 1726 out: 1727 bpf_object__close(obj); 1728 return ERR_PTR(err); 1729 } 1730 1731 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 1732 int flags) 1733 { 1734 /* param validation */ 1735 if (!attr->file) 1736 return NULL; 1737 1738 pr_debug("loading %s\n", attr->file); 1739 1740 return __bpf_object__open(attr->file, NULL, 0, 1741 bpf_prog_type__needs_kver(attr->prog_type), 1742 flags); 1743 } 1744 1745 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 1746 { 1747 return __bpf_object__open_xattr(attr, 0); 1748 } 1749 1750 struct bpf_object *bpf_object__open(const char *path) 1751 { 1752 struct bpf_object_open_attr attr = { 1753 .file = path, 1754 .prog_type = BPF_PROG_TYPE_UNSPEC, 1755 }; 1756 1757 return bpf_object__open_xattr(&attr); 1758 } 1759 1760 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1761 size_t obj_buf_sz, 1762 const char *name) 1763 { 1764 char tmp_name[64]; 1765 1766 /* param validation */ 1767 if (!obj_buf || obj_buf_sz <= 0) 1768 return NULL; 1769 1770 if (!name) { 1771 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1772 (unsigned long)obj_buf, 1773 (unsigned long)obj_buf_sz); 1774 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1775 name = tmp_name; 1776 } 1777 pr_debug("loading object '%s' from buffer\n", 1778 name); 1779 1780 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 1781 } 1782 1783 int bpf_object__unload(struct bpf_object *obj) 1784 { 1785 size_t i; 1786 1787 if (!obj) 1788 return -EINVAL; 1789 1790 for (i = 0; i < obj->nr_maps; i++) 1791 zclose(obj->maps[i].fd); 1792 1793 for (i = 0; i < obj->nr_programs; i++) 1794 bpf_program__unload(&obj->programs[i]); 1795 1796 return 0; 1797 } 1798 1799 int bpf_object__load(struct bpf_object *obj) 1800 { 1801 int err; 1802 1803 if (!obj) 1804 return -EINVAL; 1805 1806 if (obj->loaded) { 1807 pr_warning("object should not be loaded twice\n"); 1808 return -EINVAL; 1809 } 1810 1811 obj->loaded = true; 1812 1813 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 1814 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1815 CHECK_ERR(bpf_object__relocate(obj), err, out); 1816 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1817 1818 return 0; 1819 out: 1820 bpf_object__unload(obj); 1821 pr_warning("failed to load object '%s'\n", obj->path); 1822 return err; 1823 } 1824 1825 static int check_path(const char *path) 1826 { 1827 char *cp, errmsg[STRERR_BUFSIZE]; 1828 struct statfs st_fs; 1829 char *dname, *dir; 1830 int err = 0; 1831 1832 if (path == NULL) 1833 return -EINVAL; 1834 1835 dname = strdup(path); 1836 if (dname == NULL) 1837 return -ENOMEM; 1838 1839 dir = dirname(dname); 1840 if (statfs(dir, &st_fs)) { 1841 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1842 pr_warning("failed to statfs %s: %s\n", dir, cp); 1843 err = -errno; 1844 } 1845 free(dname); 1846 1847 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 1848 pr_warning("specified path %s is not on BPF FS\n", path); 1849 err = -EINVAL; 1850 } 1851 1852 return err; 1853 } 1854 1855 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 1856 int instance) 1857 { 1858 char *cp, errmsg[STRERR_BUFSIZE]; 1859 int err; 1860 1861 err = check_path(path); 1862 if (err) 1863 return err; 1864 1865 if (prog == NULL) { 1866 pr_warning("invalid program pointer\n"); 1867 return -EINVAL; 1868 } 1869 1870 if (instance < 0 || instance >= prog->instances.nr) { 1871 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 1872 instance, prog->section_name, prog->instances.nr); 1873 return -EINVAL; 1874 } 1875 1876 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 1877 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1878 pr_warning("failed to pin program: %s\n", cp); 1879 return -errno; 1880 } 1881 pr_debug("pinned program '%s'\n", path); 1882 1883 return 0; 1884 } 1885 1886 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 1887 int instance) 1888 { 1889 int err; 1890 1891 err = check_path(path); 1892 if (err) 1893 return err; 1894 1895 if (prog == NULL) { 1896 pr_warning("invalid program pointer\n"); 1897 return -EINVAL; 1898 } 1899 1900 if (instance < 0 || instance >= prog->instances.nr) { 1901 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 1902 instance, prog->section_name, prog->instances.nr); 1903 return -EINVAL; 1904 } 1905 1906 err = unlink(path); 1907 if (err != 0) 1908 return -errno; 1909 pr_debug("unpinned program '%s'\n", path); 1910 1911 return 0; 1912 } 1913 1914 static int make_dir(const char *path) 1915 { 1916 char *cp, errmsg[STRERR_BUFSIZE]; 1917 int err = 0; 1918 1919 if (mkdir(path, 0700) && errno != EEXIST) 1920 err = -errno; 1921 1922 if (err) { 1923 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 1924 pr_warning("failed to mkdir %s: %s\n", path, cp); 1925 } 1926 return err; 1927 } 1928 1929 int bpf_program__pin(struct bpf_program *prog, const char *path) 1930 { 1931 int i, err; 1932 1933 err = check_path(path); 1934 if (err) 1935 return err; 1936 1937 if (prog == NULL) { 1938 pr_warning("invalid program pointer\n"); 1939 return -EINVAL; 1940 } 1941 1942 if (prog->instances.nr <= 0) { 1943 pr_warning("no instances of prog %s to pin\n", 1944 prog->section_name); 1945 return -EINVAL; 1946 } 1947 1948 if (prog->instances.nr == 1) { 1949 /* don't create subdirs when pinning single instance */ 1950 return bpf_program__pin_instance(prog, path, 0); 1951 } 1952 1953 err = make_dir(path); 1954 if (err) 1955 return err; 1956 1957 for (i = 0; i < prog->instances.nr; i++) { 1958 char buf[PATH_MAX]; 1959 int len; 1960 1961 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 1962 if (len < 0) { 1963 err = -EINVAL; 1964 goto err_unpin; 1965 } else if (len >= PATH_MAX) { 1966 err = -ENAMETOOLONG; 1967 goto err_unpin; 1968 } 1969 1970 err = bpf_program__pin_instance(prog, buf, i); 1971 if (err) 1972 goto err_unpin; 1973 } 1974 1975 return 0; 1976 1977 err_unpin: 1978 for (i = i - 1; i >= 0; i--) { 1979 char buf[PATH_MAX]; 1980 int len; 1981 1982 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 1983 if (len < 0) 1984 continue; 1985 else if (len >= PATH_MAX) 1986 continue; 1987 1988 bpf_program__unpin_instance(prog, buf, i); 1989 } 1990 1991 rmdir(path); 1992 1993 return err; 1994 } 1995 1996 int bpf_program__unpin(struct bpf_program *prog, const char *path) 1997 { 1998 int i, err; 1999 2000 err = check_path(path); 2001 if (err) 2002 return err; 2003 2004 if (prog == NULL) { 2005 pr_warning("invalid program pointer\n"); 2006 return -EINVAL; 2007 } 2008 2009 if (prog->instances.nr <= 0) { 2010 pr_warning("no instances of prog %s to pin\n", 2011 prog->section_name); 2012 return -EINVAL; 2013 } 2014 2015 if (prog->instances.nr == 1) { 2016 /* don't create subdirs when pinning single instance */ 2017 return bpf_program__unpin_instance(prog, path, 0); 2018 } 2019 2020 for (i = 0; i < prog->instances.nr; i++) { 2021 char buf[PATH_MAX]; 2022 int len; 2023 2024 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2025 if (len < 0) 2026 return -EINVAL; 2027 else if (len >= PATH_MAX) 2028 return -ENAMETOOLONG; 2029 2030 err = bpf_program__unpin_instance(prog, buf, i); 2031 if (err) 2032 return err; 2033 } 2034 2035 err = rmdir(path); 2036 if (err) 2037 return -errno; 2038 2039 return 0; 2040 } 2041 2042 int bpf_map__pin(struct bpf_map *map, const char *path) 2043 { 2044 char *cp, errmsg[STRERR_BUFSIZE]; 2045 int err; 2046 2047 err = check_path(path); 2048 if (err) 2049 return err; 2050 2051 if (map == NULL) { 2052 pr_warning("invalid map pointer\n"); 2053 return -EINVAL; 2054 } 2055 2056 if (bpf_obj_pin(map->fd, path)) { 2057 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2058 pr_warning("failed to pin map: %s\n", cp); 2059 return -errno; 2060 } 2061 2062 pr_debug("pinned map '%s'\n", path); 2063 2064 return 0; 2065 } 2066 2067 int bpf_map__unpin(struct bpf_map *map, const char *path) 2068 { 2069 int err; 2070 2071 err = check_path(path); 2072 if (err) 2073 return err; 2074 2075 if (map == NULL) { 2076 pr_warning("invalid map pointer\n"); 2077 return -EINVAL; 2078 } 2079 2080 err = unlink(path); 2081 if (err != 0) 2082 return -errno; 2083 pr_debug("unpinned map '%s'\n", path); 2084 2085 return 0; 2086 } 2087 2088 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2089 { 2090 struct bpf_map *map; 2091 int err; 2092 2093 if (!obj) 2094 return -ENOENT; 2095 2096 if (!obj->loaded) { 2097 pr_warning("object not yet loaded; load it first\n"); 2098 return -ENOENT; 2099 } 2100 2101 err = make_dir(path); 2102 if (err) 2103 return err; 2104 2105 bpf_object__for_each_map(map, obj) { 2106 char buf[PATH_MAX]; 2107 int len; 2108 2109 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2110 bpf_map__name(map)); 2111 if (len < 0) { 2112 err = -EINVAL; 2113 goto err_unpin_maps; 2114 } else if (len >= PATH_MAX) { 2115 err = -ENAMETOOLONG; 2116 goto err_unpin_maps; 2117 } 2118 2119 err = bpf_map__pin(map, buf); 2120 if (err) 2121 goto err_unpin_maps; 2122 } 2123 2124 return 0; 2125 2126 err_unpin_maps: 2127 while ((map = bpf_map__prev(map, obj))) { 2128 char buf[PATH_MAX]; 2129 int len; 2130 2131 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2132 bpf_map__name(map)); 2133 if (len < 0) 2134 continue; 2135 else if (len >= PATH_MAX) 2136 continue; 2137 2138 bpf_map__unpin(map, buf); 2139 } 2140 2141 return err; 2142 } 2143 2144 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2145 { 2146 struct bpf_map *map; 2147 int err; 2148 2149 if (!obj) 2150 return -ENOENT; 2151 2152 bpf_object__for_each_map(map, obj) { 2153 char buf[PATH_MAX]; 2154 int len; 2155 2156 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2157 bpf_map__name(map)); 2158 if (len < 0) 2159 return -EINVAL; 2160 else if (len >= PATH_MAX) 2161 return -ENAMETOOLONG; 2162 2163 err = bpf_map__unpin(map, buf); 2164 if (err) 2165 return err; 2166 } 2167 2168 return 0; 2169 } 2170 2171 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2172 { 2173 struct bpf_program *prog; 2174 int err; 2175 2176 if (!obj) 2177 return -ENOENT; 2178 2179 if (!obj->loaded) { 2180 pr_warning("object not yet loaded; load it first\n"); 2181 return -ENOENT; 2182 } 2183 2184 err = make_dir(path); 2185 if (err) 2186 return err; 2187 2188 bpf_object__for_each_program(prog, obj) { 2189 char buf[PATH_MAX]; 2190 int len; 2191 2192 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2193 prog->pin_name); 2194 if (len < 0) { 2195 err = -EINVAL; 2196 goto err_unpin_programs; 2197 } else if (len >= PATH_MAX) { 2198 err = -ENAMETOOLONG; 2199 goto err_unpin_programs; 2200 } 2201 2202 err = bpf_program__pin(prog, buf); 2203 if (err) 2204 goto err_unpin_programs; 2205 } 2206 2207 return 0; 2208 2209 err_unpin_programs: 2210 while ((prog = bpf_program__prev(prog, obj))) { 2211 char buf[PATH_MAX]; 2212 int len; 2213 2214 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2215 prog->pin_name); 2216 if (len < 0) 2217 continue; 2218 else if (len >= PATH_MAX) 2219 continue; 2220 2221 bpf_program__unpin(prog, buf); 2222 } 2223 2224 return err; 2225 } 2226 2227 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2228 { 2229 struct bpf_program *prog; 2230 int err; 2231 2232 if (!obj) 2233 return -ENOENT; 2234 2235 bpf_object__for_each_program(prog, obj) { 2236 char buf[PATH_MAX]; 2237 int len; 2238 2239 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2240 prog->pin_name); 2241 if (len < 0) 2242 return -EINVAL; 2243 else if (len >= PATH_MAX) 2244 return -ENAMETOOLONG; 2245 2246 err = bpf_program__unpin(prog, buf); 2247 if (err) 2248 return err; 2249 } 2250 2251 return 0; 2252 } 2253 2254 int bpf_object__pin(struct bpf_object *obj, const char *path) 2255 { 2256 int err; 2257 2258 err = bpf_object__pin_maps(obj, path); 2259 if (err) 2260 return err; 2261 2262 err = bpf_object__pin_programs(obj, path); 2263 if (err) { 2264 bpf_object__unpin_maps(obj, path); 2265 return err; 2266 } 2267 2268 return 0; 2269 } 2270 2271 void bpf_object__close(struct bpf_object *obj) 2272 { 2273 size_t i; 2274 2275 if (!obj) 2276 return; 2277 2278 if (obj->clear_priv) 2279 obj->clear_priv(obj, obj->priv); 2280 2281 bpf_object__elf_finish(obj); 2282 bpf_object__unload(obj); 2283 btf__free(obj->btf); 2284 btf_ext__free(obj->btf_ext); 2285 2286 for (i = 0; i < obj->nr_maps; i++) { 2287 zfree(&obj->maps[i].name); 2288 if (obj->maps[i].clear_priv) 2289 obj->maps[i].clear_priv(&obj->maps[i], 2290 obj->maps[i].priv); 2291 obj->maps[i].priv = NULL; 2292 obj->maps[i].clear_priv = NULL; 2293 } 2294 zfree(&obj->maps); 2295 obj->nr_maps = 0; 2296 2297 if (obj->programs && obj->nr_programs) { 2298 for (i = 0; i < obj->nr_programs; i++) 2299 bpf_program__exit(&obj->programs[i]); 2300 } 2301 zfree(&obj->programs); 2302 2303 list_del(&obj->list); 2304 free(obj); 2305 } 2306 2307 struct bpf_object * 2308 bpf_object__next(struct bpf_object *prev) 2309 { 2310 struct bpf_object *next; 2311 2312 if (!prev) 2313 next = list_first_entry(&bpf_objects_list, 2314 struct bpf_object, 2315 list); 2316 else 2317 next = list_next_entry(prev, list); 2318 2319 /* Empty list is noticed here so don't need checking on entry. */ 2320 if (&next->list == &bpf_objects_list) 2321 return NULL; 2322 2323 return next; 2324 } 2325 2326 const char *bpf_object__name(struct bpf_object *obj) 2327 { 2328 return obj ? obj->path : ERR_PTR(-EINVAL); 2329 } 2330 2331 unsigned int bpf_object__kversion(struct bpf_object *obj) 2332 { 2333 return obj ? obj->kern_version : 0; 2334 } 2335 2336 struct btf *bpf_object__btf(struct bpf_object *obj) 2337 { 2338 return obj ? obj->btf : NULL; 2339 } 2340 2341 int bpf_object__btf_fd(const struct bpf_object *obj) 2342 { 2343 return obj->btf ? btf__fd(obj->btf) : -1; 2344 } 2345 2346 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2347 bpf_object_clear_priv_t clear_priv) 2348 { 2349 if (obj->priv && obj->clear_priv) 2350 obj->clear_priv(obj, obj->priv); 2351 2352 obj->priv = priv; 2353 obj->clear_priv = clear_priv; 2354 return 0; 2355 } 2356 2357 void *bpf_object__priv(struct bpf_object *obj) 2358 { 2359 return obj ? obj->priv : ERR_PTR(-EINVAL); 2360 } 2361 2362 static struct bpf_program * 2363 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2364 { 2365 size_t nr_programs = obj->nr_programs; 2366 ssize_t idx; 2367 2368 if (!nr_programs) 2369 return NULL; 2370 2371 if (!p) 2372 /* Iter from the beginning */ 2373 return forward ? &obj->programs[0] : 2374 &obj->programs[nr_programs - 1]; 2375 2376 if (p->obj != obj) { 2377 pr_warning("error: program handler doesn't match object\n"); 2378 return NULL; 2379 } 2380 2381 idx = (p - obj->programs) + (forward ? 1 : -1); 2382 if (idx >= obj->nr_programs || idx < 0) 2383 return NULL; 2384 return &obj->programs[idx]; 2385 } 2386 2387 struct bpf_program * 2388 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2389 { 2390 struct bpf_program *prog = prev; 2391 2392 do { 2393 prog = __bpf_program__iter(prog, obj, true); 2394 } while (prog && bpf_program__is_function_storage(prog, obj)); 2395 2396 return prog; 2397 } 2398 2399 struct bpf_program * 2400 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2401 { 2402 struct bpf_program *prog = next; 2403 2404 do { 2405 prog = __bpf_program__iter(prog, obj, false); 2406 } while (prog && bpf_program__is_function_storage(prog, obj)); 2407 2408 return prog; 2409 } 2410 2411 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 2412 bpf_program_clear_priv_t clear_priv) 2413 { 2414 if (prog->priv && prog->clear_priv) 2415 prog->clear_priv(prog, prog->priv); 2416 2417 prog->priv = priv; 2418 prog->clear_priv = clear_priv; 2419 return 0; 2420 } 2421 2422 void *bpf_program__priv(struct bpf_program *prog) 2423 { 2424 return prog ? prog->priv : ERR_PTR(-EINVAL); 2425 } 2426 2427 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 2428 { 2429 prog->prog_ifindex = ifindex; 2430 } 2431 2432 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 2433 { 2434 const char *title; 2435 2436 title = prog->section_name; 2437 if (needs_copy) { 2438 title = strdup(title); 2439 if (!title) { 2440 pr_warning("failed to strdup program title\n"); 2441 return ERR_PTR(-ENOMEM); 2442 } 2443 } 2444 2445 return title; 2446 } 2447 2448 int bpf_program__fd(struct bpf_program *prog) 2449 { 2450 return bpf_program__nth_fd(prog, 0); 2451 } 2452 2453 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 2454 bpf_program_prep_t prep) 2455 { 2456 int *instances_fds; 2457 2458 if (nr_instances <= 0 || !prep) 2459 return -EINVAL; 2460 2461 if (prog->instances.nr > 0 || prog->instances.fds) { 2462 pr_warning("Can't set pre-processor after loading\n"); 2463 return -EINVAL; 2464 } 2465 2466 instances_fds = malloc(sizeof(int) * nr_instances); 2467 if (!instances_fds) { 2468 pr_warning("alloc memory failed for fds\n"); 2469 return -ENOMEM; 2470 } 2471 2472 /* fill all fd with -1 */ 2473 memset(instances_fds, -1, sizeof(int) * nr_instances); 2474 2475 prog->instances.nr = nr_instances; 2476 prog->instances.fds = instances_fds; 2477 prog->preprocessor = prep; 2478 return 0; 2479 } 2480 2481 int bpf_program__nth_fd(struct bpf_program *prog, int n) 2482 { 2483 int fd; 2484 2485 if (!prog) 2486 return -EINVAL; 2487 2488 if (n >= prog->instances.nr || n < 0) { 2489 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 2490 n, prog->section_name, prog->instances.nr); 2491 return -EINVAL; 2492 } 2493 2494 fd = prog->instances.fds[n]; 2495 if (fd < 0) { 2496 pr_warning("%dth instance of program '%s' is invalid\n", 2497 n, prog->section_name); 2498 return -ENOENT; 2499 } 2500 2501 return fd; 2502 } 2503 2504 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 2505 { 2506 prog->type = type; 2507 } 2508 2509 static bool bpf_program__is_type(struct bpf_program *prog, 2510 enum bpf_prog_type type) 2511 { 2512 return prog ? (prog->type == type) : false; 2513 } 2514 2515 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 2516 int bpf_program__set_##NAME(struct bpf_program *prog) \ 2517 { \ 2518 if (!prog) \ 2519 return -EINVAL; \ 2520 bpf_program__set_type(prog, TYPE); \ 2521 return 0; \ 2522 } \ 2523 \ 2524 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 2525 { \ 2526 return bpf_program__is_type(prog, TYPE); \ 2527 } \ 2528 2529 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 2530 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 2531 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 2532 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 2533 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 2534 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 2535 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 2536 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 2537 2538 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 2539 enum bpf_attach_type type) 2540 { 2541 prog->expected_attach_type = type; 2542 } 2543 2544 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 2545 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 2546 2547 /* Programs that can NOT be attached. */ 2548 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 2549 2550 /* Programs that can be attached. */ 2551 #define BPF_APROG_SEC(string, ptype, atype) \ 2552 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 2553 2554 /* Programs that must specify expected attach type at load time. */ 2555 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 2556 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 2557 2558 /* Programs that can be attached but attach type can't be identified by section 2559 * name. Kept for backward compatibility. 2560 */ 2561 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 2562 2563 static const struct { 2564 const char *sec; 2565 size_t len; 2566 enum bpf_prog_type prog_type; 2567 enum bpf_attach_type expected_attach_type; 2568 int is_attachable; 2569 enum bpf_attach_type attach_type; 2570 } section_names[] = { 2571 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 2572 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 2573 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 2574 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 2575 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 2576 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 2577 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 2578 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 2579 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 2580 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 2581 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 2582 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 2583 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 2584 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 2585 BPF_CGROUP_INET_INGRESS), 2586 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 2587 BPF_CGROUP_INET_EGRESS), 2588 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 2589 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 2590 BPF_CGROUP_INET_SOCK_CREATE), 2591 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 2592 BPF_CGROUP_INET4_POST_BIND), 2593 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 2594 BPF_CGROUP_INET6_POST_BIND), 2595 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 2596 BPF_CGROUP_DEVICE), 2597 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 2598 BPF_CGROUP_SOCK_OPS), 2599 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 2600 BPF_SK_SKB_STREAM_PARSER), 2601 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 2602 BPF_SK_SKB_STREAM_VERDICT), 2603 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 2604 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 2605 BPF_SK_MSG_VERDICT), 2606 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 2607 BPF_LIRC_MODE2), 2608 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 2609 BPF_FLOW_DISSECTOR), 2610 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 2611 BPF_CGROUP_INET4_BIND), 2612 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 2613 BPF_CGROUP_INET6_BIND), 2614 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 2615 BPF_CGROUP_INET4_CONNECT), 2616 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 2617 BPF_CGROUP_INET6_CONNECT), 2618 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 2619 BPF_CGROUP_UDP4_SENDMSG), 2620 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 2621 BPF_CGROUP_UDP6_SENDMSG), 2622 }; 2623 2624 #undef BPF_PROG_SEC_IMPL 2625 #undef BPF_PROG_SEC 2626 #undef BPF_APROG_SEC 2627 #undef BPF_EAPROG_SEC 2628 #undef BPF_APROG_COMPAT 2629 2630 #define MAX_TYPE_NAME_SIZE 32 2631 2632 static char *libbpf_get_type_names(bool attach_type) 2633 { 2634 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 2635 char *buf; 2636 2637 buf = malloc(len); 2638 if (!buf) 2639 return NULL; 2640 2641 buf[0] = '\0'; 2642 /* Forge string buf with all available names */ 2643 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 2644 if (attach_type && !section_names[i].is_attachable) 2645 continue; 2646 2647 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 2648 free(buf); 2649 return NULL; 2650 } 2651 strcat(buf, " "); 2652 strcat(buf, section_names[i].sec); 2653 } 2654 2655 return buf; 2656 } 2657 2658 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 2659 enum bpf_attach_type *expected_attach_type) 2660 { 2661 char *type_names; 2662 int i; 2663 2664 if (!name) 2665 return -EINVAL; 2666 2667 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 2668 if (strncmp(name, section_names[i].sec, section_names[i].len)) 2669 continue; 2670 *prog_type = section_names[i].prog_type; 2671 *expected_attach_type = section_names[i].expected_attach_type; 2672 return 0; 2673 } 2674 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 2675 type_names = libbpf_get_type_names(false); 2676 if (type_names != NULL) { 2677 pr_info("supported section(type) names are:%s\n", type_names); 2678 free(type_names); 2679 } 2680 2681 return -EINVAL; 2682 } 2683 2684 int libbpf_attach_type_by_name(const char *name, 2685 enum bpf_attach_type *attach_type) 2686 { 2687 char *type_names; 2688 int i; 2689 2690 if (!name) 2691 return -EINVAL; 2692 2693 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 2694 if (strncmp(name, section_names[i].sec, section_names[i].len)) 2695 continue; 2696 if (!section_names[i].is_attachable) 2697 return -EINVAL; 2698 *attach_type = section_names[i].attach_type; 2699 return 0; 2700 } 2701 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 2702 type_names = libbpf_get_type_names(true); 2703 if (type_names != NULL) { 2704 pr_info("attachable section(type) names are:%s\n", type_names); 2705 free(type_names); 2706 } 2707 2708 return -EINVAL; 2709 } 2710 2711 static int 2712 bpf_program__identify_section(struct bpf_program *prog, 2713 enum bpf_prog_type *prog_type, 2714 enum bpf_attach_type *expected_attach_type) 2715 { 2716 return libbpf_prog_type_by_name(prog->section_name, prog_type, 2717 expected_attach_type); 2718 } 2719 2720 int bpf_map__fd(struct bpf_map *map) 2721 { 2722 return map ? map->fd : -EINVAL; 2723 } 2724 2725 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 2726 { 2727 return map ? &map->def : ERR_PTR(-EINVAL); 2728 } 2729 2730 const char *bpf_map__name(struct bpf_map *map) 2731 { 2732 return map ? map->name : NULL; 2733 } 2734 2735 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 2736 { 2737 return map ? map->btf_key_type_id : 0; 2738 } 2739 2740 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 2741 { 2742 return map ? map->btf_value_type_id : 0; 2743 } 2744 2745 int bpf_map__set_priv(struct bpf_map *map, void *priv, 2746 bpf_map_clear_priv_t clear_priv) 2747 { 2748 if (!map) 2749 return -EINVAL; 2750 2751 if (map->priv) { 2752 if (map->clear_priv) 2753 map->clear_priv(map, map->priv); 2754 } 2755 2756 map->priv = priv; 2757 map->clear_priv = clear_priv; 2758 return 0; 2759 } 2760 2761 void *bpf_map__priv(struct bpf_map *map) 2762 { 2763 return map ? map->priv : ERR_PTR(-EINVAL); 2764 } 2765 2766 bool bpf_map__is_offload_neutral(struct bpf_map *map) 2767 { 2768 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 2769 } 2770 2771 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 2772 { 2773 map->map_ifindex = ifindex; 2774 } 2775 2776 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 2777 { 2778 if (!bpf_map_type__is_map_in_map(map->def.type)) { 2779 pr_warning("error: unsupported map type\n"); 2780 return -EINVAL; 2781 } 2782 if (map->inner_map_fd != -1) { 2783 pr_warning("error: inner_map_fd already specified\n"); 2784 return -EINVAL; 2785 } 2786 map->inner_map_fd = fd; 2787 return 0; 2788 } 2789 2790 static struct bpf_map * 2791 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 2792 { 2793 ssize_t idx; 2794 struct bpf_map *s, *e; 2795 2796 if (!obj || !obj->maps) 2797 return NULL; 2798 2799 s = obj->maps; 2800 e = obj->maps + obj->nr_maps; 2801 2802 if ((m < s) || (m >= e)) { 2803 pr_warning("error in %s: map handler doesn't belong to object\n", 2804 __func__); 2805 return NULL; 2806 } 2807 2808 idx = (m - obj->maps) + i; 2809 if (idx >= obj->nr_maps || idx < 0) 2810 return NULL; 2811 return &obj->maps[idx]; 2812 } 2813 2814 struct bpf_map * 2815 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 2816 { 2817 if (prev == NULL) 2818 return obj->maps; 2819 2820 return __bpf_map__iter(prev, obj, 1); 2821 } 2822 2823 struct bpf_map * 2824 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 2825 { 2826 if (next == NULL) { 2827 if (!obj->nr_maps) 2828 return NULL; 2829 return obj->maps + obj->nr_maps - 1; 2830 } 2831 2832 return __bpf_map__iter(next, obj, -1); 2833 } 2834 2835 struct bpf_map * 2836 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 2837 { 2838 struct bpf_map *pos; 2839 2840 bpf_object__for_each_map(pos, obj) { 2841 if (pos->name && !strcmp(pos->name, name)) 2842 return pos; 2843 } 2844 return NULL; 2845 } 2846 2847 int 2848 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 2849 { 2850 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 2851 } 2852 2853 struct bpf_map * 2854 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 2855 { 2856 int i; 2857 2858 for (i = 0; i < obj->nr_maps; i++) { 2859 if (obj->maps[i].offset == offset) 2860 return &obj->maps[i]; 2861 } 2862 return ERR_PTR(-ENOENT); 2863 } 2864 2865 long libbpf_get_error(const void *ptr) 2866 { 2867 if (IS_ERR(ptr)) 2868 return PTR_ERR(ptr); 2869 return 0; 2870 } 2871 2872 int bpf_prog_load(const char *file, enum bpf_prog_type type, 2873 struct bpf_object **pobj, int *prog_fd) 2874 { 2875 struct bpf_prog_load_attr attr; 2876 2877 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 2878 attr.file = file; 2879 attr.prog_type = type; 2880 attr.expected_attach_type = 0; 2881 2882 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 2883 } 2884 2885 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 2886 struct bpf_object **pobj, int *prog_fd) 2887 { 2888 struct bpf_object_open_attr open_attr = { 2889 .file = attr->file, 2890 .prog_type = attr->prog_type, 2891 }; 2892 struct bpf_program *prog, *first_prog = NULL; 2893 enum bpf_attach_type expected_attach_type; 2894 enum bpf_prog_type prog_type; 2895 struct bpf_object *obj; 2896 struct bpf_map *map; 2897 int err; 2898 2899 if (!attr) 2900 return -EINVAL; 2901 if (!attr->file) 2902 return -EINVAL; 2903 2904 obj = bpf_object__open_xattr(&open_attr); 2905 if (IS_ERR_OR_NULL(obj)) 2906 return -ENOENT; 2907 2908 bpf_object__for_each_program(prog, obj) { 2909 /* 2910 * If type is not specified, try to guess it based on 2911 * section name. 2912 */ 2913 prog_type = attr->prog_type; 2914 prog->prog_ifindex = attr->ifindex; 2915 expected_attach_type = attr->expected_attach_type; 2916 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 2917 err = bpf_program__identify_section(prog, &prog_type, 2918 &expected_attach_type); 2919 if (err < 0) { 2920 bpf_object__close(obj); 2921 return -EINVAL; 2922 } 2923 } 2924 2925 bpf_program__set_type(prog, prog_type); 2926 bpf_program__set_expected_attach_type(prog, 2927 expected_attach_type); 2928 2929 if (!first_prog) 2930 first_prog = prog; 2931 } 2932 2933 bpf_object__for_each_map(map, obj) { 2934 if (!bpf_map__is_offload_neutral(map)) 2935 map->map_ifindex = attr->ifindex; 2936 } 2937 2938 if (!first_prog) { 2939 pr_warning("object file doesn't contain bpf program\n"); 2940 bpf_object__close(obj); 2941 return -ENOENT; 2942 } 2943 2944 err = bpf_object__load(obj); 2945 if (err) { 2946 bpf_object__close(obj); 2947 return -EINVAL; 2948 } 2949 2950 *pobj = obj; 2951 *prog_fd = bpf_program__fd(first_prog); 2952 return 0; 2953 } 2954 2955 enum bpf_perf_event_ret 2956 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 2957 void **copy_mem, size_t *copy_size, 2958 bpf_perf_event_print_t fn, void *private_data) 2959 { 2960 struct perf_event_mmap_page *header = mmap_mem; 2961 __u64 data_head = ring_buffer_read_head(header); 2962 __u64 data_tail = header->data_tail; 2963 void *base = ((__u8 *)header) + page_size; 2964 int ret = LIBBPF_PERF_EVENT_CONT; 2965 struct perf_event_header *ehdr; 2966 size_t ehdr_size; 2967 2968 while (data_head != data_tail) { 2969 ehdr = base + (data_tail & (mmap_size - 1)); 2970 ehdr_size = ehdr->size; 2971 2972 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 2973 void *copy_start = ehdr; 2974 size_t len_first = base + mmap_size - copy_start; 2975 size_t len_secnd = ehdr_size - len_first; 2976 2977 if (*copy_size < ehdr_size) { 2978 free(*copy_mem); 2979 *copy_mem = malloc(ehdr_size); 2980 if (!*copy_mem) { 2981 *copy_size = 0; 2982 ret = LIBBPF_PERF_EVENT_ERROR; 2983 break; 2984 } 2985 *copy_size = ehdr_size; 2986 } 2987 2988 memcpy(*copy_mem, copy_start, len_first); 2989 memcpy(*copy_mem + len_first, base, len_secnd); 2990 ehdr = *copy_mem; 2991 } 2992 2993 ret = fn(ehdr, private_data); 2994 data_tail += ehdr_size; 2995 if (ret != LIBBPF_PERF_EVENT_CONT) 2996 break; 2997 } 2998 2999 ring_buffer_write_tail(header, data_tail); 3000 return ret; 3001 } 3002