1 // SPDX-License-Identifier: LGPL-2.1 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU Lesser General Public 13 * License as published by the Free Software Foundation; 14 * version 2.1 of the License (not later!) 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU Lesser General Public License for more details. 20 * 21 * You should have received a copy of the GNU Lesser General Public 22 * License along with this program; if not, see <http://www.gnu.org/licenses> 23 */ 24 25 #include <stdlib.h> 26 #include <stdio.h> 27 #include <stdarg.h> 28 #include <libgen.h> 29 #include <inttypes.h> 30 #include <string.h> 31 #include <unistd.h> 32 #include <fcntl.h> 33 #include <errno.h> 34 #include <asm/unistd.h> 35 #include <linux/err.h> 36 #include <linux/kernel.h> 37 #include <linux/bpf.h> 38 #include <linux/list.h> 39 #include <linux/limits.h> 40 #include <sys/stat.h> 41 #include <sys/types.h> 42 #include <sys/vfs.h> 43 #include <libelf.h> 44 #include <gelf.h> 45 46 #include "libbpf.h" 47 #include "bpf.h" 48 49 #ifndef EM_BPF 50 #define EM_BPF 247 51 #endif 52 53 #ifndef BPF_FS_MAGIC 54 #define BPF_FS_MAGIC 0xcafe4a11 55 #endif 56 57 #define __printf(a, b) __attribute__((format(printf, a, b))) 58 59 __printf(1, 2) 60 static int __base_pr(const char *format, ...) 61 { 62 va_list args; 63 int err; 64 65 va_start(args, format); 66 err = vfprintf(stderr, format, args); 67 va_end(args); 68 return err; 69 } 70 71 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 72 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 73 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 74 75 #define __pr(func, fmt, ...) \ 76 do { \ 77 if ((func)) \ 78 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 79 } while (0) 80 81 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 82 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 83 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 84 85 void libbpf_set_print(libbpf_print_fn_t warn, 86 libbpf_print_fn_t info, 87 libbpf_print_fn_t debug) 88 { 89 __pr_warning = warn; 90 __pr_info = info; 91 __pr_debug = debug; 92 } 93 94 #define STRERR_BUFSIZE 128 95 96 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 97 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 98 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 99 100 static const char *libbpf_strerror_table[NR_ERRNO] = { 101 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 102 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 103 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 104 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", 105 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 106 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 107 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 108 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 109 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 110 [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", 111 [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message", 112 [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence", 113 }; 114 115 int libbpf_strerror(int err, char *buf, size_t size) 116 { 117 if (!buf || !size) 118 return -1; 119 120 err = err > 0 ? err : -err; 121 122 if (err < __LIBBPF_ERRNO__START) { 123 int ret; 124 125 ret = strerror_r(err, buf, size); 126 buf[size - 1] = '\0'; 127 return ret; 128 } 129 130 if (err < __LIBBPF_ERRNO__END) { 131 const char *msg; 132 133 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 134 snprintf(buf, size, "%s", msg); 135 buf[size - 1] = '\0'; 136 return 0; 137 } 138 139 snprintf(buf, size, "Unknown libbpf error %d", err); 140 buf[size - 1] = '\0'; 141 return -1; 142 } 143 144 #define CHECK_ERR(action, err, out) do { \ 145 err = action; \ 146 if (err) \ 147 goto out; \ 148 } while(0) 149 150 151 /* Copied from tools/perf/util/util.h */ 152 #ifndef zfree 153 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 154 #endif 155 156 #ifndef zclose 157 # define zclose(fd) ({ \ 158 int ___err = 0; \ 159 if ((fd) >= 0) \ 160 ___err = close((fd)); \ 161 fd = -1; \ 162 ___err; }) 163 #endif 164 165 #ifdef HAVE_LIBELF_MMAP_SUPPORT 166 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 167 #else 168 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 169 #endif 170 171 /* 172 * bpf_prog should be a better name but it has been used in 173 * linux/filter.h. 174 */ 175 struct bpf_program { 176 /* Index in elf obj file, for relocation use. */ 177 int idx; 178 char *name; 179 char *section_name; 180 struct bpf_insn *insns; 181 size_t insns_cnt, main_prog_cnt; 182 enum bpf_prog_type type; 183 184 struct reloc_desc { 185 enum { 186 RELO_LD64, 187 RELO_CALL, 188 } type; 189 int insn_idx; 190 union { 191 int map_idx; 192 int text_off; 193 }; 194 } *reloc_desc; 195 int nr_reloc; 196 197 struct { 198 int nr; 199 int *fds; 200 } instances; 201 bpf_program_prep_t preprocessor; 202 203 struct bpf_object *obj; 204 void *priv; 205 bpf_program_clear_priv_t clear_priv; 206 }; 207 208 struct bpf_map { 209 int fd; 210 char *name; 211 size_t offset; 212 struct bpf_map_def def; 213 void *priv; 214 bpf_map_clear_priv_t clear_priv; 215 }; 216 217 static LIST_HEAD(bpf_objects_list); 218 219 struct bpf_object { 220 char license[64]; 221 u32 kern_version; 222 223 struct bpf_program *programs; 224 size_t nr_programs; 225 struct bpf_map *maps; 226 size_t nr_maps; 227 228 bool loaded; 229 230 /* 231 * Information when doing elf related work. Only valid if fd 232 * is valid. 233 */ 234 struct { 235 int fd; 236 void *obj_buf; 237 size_t obj_buf_sz; 238 Elf *elf; 239 GElf_Ehdr ehdr; 240 Elf_Data *symbols; 241 size_t strtabidx; 242 struct { 243 GElf_Shdr shdr; 244 Elf_Data *data; 245 } *reloc; 246 int nr_reloc; 247 int maps_shndx; 248 int text_shndx; 249 } efile; 250 /* 251 * All loaded bpf_object is linked in a list, which is 252 * hidden to caller. bpf_objects__<func> handlers deal with 253 * all objects. 254 */ 255 struct list_head list; 256 257 void *priv; 258 bpf_object_clear_priv_t clear_priv; 259 260 char path[]; 261 }; 262 #define obj_elf_valid(o) ((o)->efile.elf) 263 264 static void bpf_program__unload(struct bpf_program *prog) 265 { 266 int i; 267 268 if (!prog) 269 return; 270 271 /* 272 * If the object is opened but the program was never loaded, 273 * it is possible that prog->instances.nr == -1. 274 */ 275 if (prog->instances.nr > 0) { 276 for (i = 0; i < prog->instances.nr; i++) 277 zclose(prog->instances.fds[i]); 278 } else if (prog->instances.nr != -1) { 279 pr_warning("Internal error: instances.nr is %d\n", 280 prog->instances.nr); 281 } 282 283 prog->instances.nr = -1; 284 zfree(&prog->instances.fds); 285 } 286 287 static void bpf_program__exit(struct bpf_program *prog) 288 { 289 if (!prog) 290 return; 291 292 if (prog->clear_priv) 293 prog->clear_priv(prog, prog->priv); 294 295 prog->priv = NULL; 296 prog->clear_priv = NULL; 297 298 bpf_program__unload(prog); 299 zfree(&prog->name); 300 zfree(&prog->section_name); 301 zfree(&prog->insns); 302 zfree(&prog->reloc_desc); 303 304 prog->nr_reloc = 0; 305 prog->insns_cnt = 0; 306 prog->idx = -1; 307 } 308 309 static int 310 bpf_program__init(void *data, size_t size, char *section_name, int idx, 311 struct bpf_program *prog) 312 { 313 if (size < sizeof(struct bpf_insn)) { 314 pr_warning("corrupted section '%s'\n", section_name); 315 return -EINVAL; 316 } 317 318 bzero(prog, sizeof(*prog)); 319 320 prog->section_name = strdup(section_name); 321 if (!prog->section_name) { 322 pr_warning("failed to alloc name for prog under section(%d) %s\n", 323 idx, section_name); 324 goto errout; 325 } 326 327 prog->insns = malloc(size); 328 if (!prog->insns) { 329 pr_warning("failed to alloc insns for prog under section %s\n", 330 section_name); 331 goto errout; 332 } 333 prog->insns_cnt = size / sizeof(struct bpf_insn); 334 memcpy(prog->insns, data, 335 prog->insns_cnt * sizeof(struct bpf_insn)); 336 prog->idx = idx; 337 prog->instances.fds = NULL; 338 prog->instances.nr = -1; 339 prog->type = BPF_PROG_TYPE_KPROBE; 340 341 return 0; 342 errout: 343 bpf_program__exit(prog); 344 return -ENOMEM; 345 } 346 347 static int 348 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 349 char *section_name, int idx) 350 { 351 struct bpf_program prog, *progs; 352 int nr_progs, err; 353 354 err = bpf_program__init(data, size, section_name, idx, &prog); 355 if (err) 356 return err; 357 358 progs = obj->programs; 359 nr_progs = obj->nr_programs; 360 361 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 362 if (!progs) { 363 /* 364 * In this case the original obj->programs 365 * is still valid, so don't need special treat for 366 * bpf_close_object(). 367 */ 368 pr_warning("failed to alloc a new program under section '%s'\n", 369 section_name); 370 bpf_program__exit(&prog); 371 return -ENOMEM; 372 } 373 374 pr_debug("found program %s\n", prog.section_name); 375 obj->programs = progs; 376 obj->nr_programs = nr_progs + 1; 377 prog.obj = obj; 378 progs[nr_progs] = prog; 379 return 0; 380 } 381 382 static int 383 bpf_object__init_prog_names(struct bpf_object *obj) 384 { 385 Elf_Data *symbols = obj->efile.symbols; 386 struct bpf_program *prog; 387 size_t pi, si; 388 389 for (pi = 0; pi < obj->nr_programs; pi++) { 390 const char *name = NULL; 391 392 prog = &obj->programs[pi]; 393 if (prog->idx == obj->efile.text_shndx) { 394 name = ".text"; 395 goto skip_search; 396 } 397 398 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 399 si++) { 400 GElf_Sym sym; 401 402 if (!gelf_getsym(symbols, si, &sym)) 403 continue; 404 if (sym.st_shndx != prog->idx) 405 continue; 406 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 407 continue; 408 409 name = elf_strptr(obj->efile.elf, 410 obj->efile.strtabidx, 411 sym.st_name); 412 if (!name) { 413 pr_warning("failed to get sym name string for prog %s\n", 414 prog->section_name); 415 return -LIBBPF_ERRNO__LIBELF; 416 } 417 } 418 419 if (!name) { 420 pr_warning("failed to find sym for prog %s\n", 421 prog->section_name); 422 return -EINVAL; 423 } 424 skip_search: 425 prog->name = strdup(name); 426 if (!prog->name) { 427 pr_warning("failed to allocate memory for prog sym %s\n", 428 name); 429 return -ENOMEM; 430 } 431 } 432 433 return 0; 434 } 435 436 static struct bpf_object *bpf_object__new(const char *path, 437 void *obj_buf, 438 size_t obj_buf_sz) 439 { 440 struct bpf_object *obj; 441 442 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 443 if (!obj) { 444 pr_warning("alloc memory failed for %s\n", path); 445 return ERR_PTR(-ENOMEM); 446 } 447 448 strcpy(obj->path, path); 449 obj->efile.fd = -1; 450 451 /* 452 * Caller of this function should also calls 453 * bpf_object__elf_finish() after data collection to return 454 * obj_buf to user. If not, we should duplicate the buffer to 455 * avoid user freeing them before elf finish. 456 */ 457 obj->efile.obj_buf = obj_buf; 458 obj->efile.obj_buf_sz = obj_buf_sz; 459 obj->efile.maps_shndx = -1; 460 461 obj->loaded = false; 462 463 INIT_LIST_HEAD(&obj->list); 464 list_add(&obj->list, &bpf_objects_list); 465 return obj; 466 } 467 468 static void bpf_object__elf_finish(struct bpf_object *obj) 469 { 470 if (!obj_elf_valid(obj)) 471 return; 472 473 if (obj->efile.elf) { 474 elf_end(obj->efile.elf); 475 obj->efile.elf = NULL; 476 } 477 obj->efile.symbols = NULL; 478 479 zfree(&obj->efile.reloc); 480 obj->efile.nr_reloc = 0; 481 zclose(obj->efile.fd); 482 obj->efile.obj_buf = NULL; 483 obj->efile.obj_buf_sz = 0; 484 } 485 486 static int bpf_object__elf_init(struct bpf_object *obj) 487 { 488 int err = 0; 489 GElf_Ehdr *ep; 490 491 if (obj_elf_valid(obj)) { 492 pr_warning("elf init: internal error\n"); 493 return -LIBBPF_ERRNO__LIBELF; 494 } 495 496 if (obj->efile.obj_buf_sz > 0) { 497 /* 498 * obj_buf should have been validated by 499 * bpf_object__open_buffer(). 500 */ 501 obj->efile.elf = elf_memory(obj->efile.obj_buf, 502 obj->efile.obj_buf_sz); 503 } else { 504 obj->efile.fd = open(obj->path, O_RDONLY); 505 if (obj->efile.fd < 0) { 506 pr_warning("failed to open %s: %s\n", obj->path, 507 strerror(errno)); 508 return -errno; 509 } 510 511 obj->efile.elf = elf_begin(obj->efile.fd, 512 LIBBPF_ELF_C_READ_MMAP, 513 NULL); 514 } 515 516 if (!obj->efile.elf) { 517 pr_warning("failed to open %s as ELF file\n", 518 obj->path); 519 err = -LIBBPF_ERRNO__LIBELF; 520 goto errout; 521 } 522 523 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 524 pr_warning("failed to get EHDR from %s\n", 525 obj->path); 526 err = -LIBBPF_ERRNO__FORMAT; 527 goto errout; 528 } 529 ep = &obj->efile.ehdr; 530 531 /* Old LLVM set e_machine to EM_NONE */ 532 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 533 pr_warning("%s is not an eBPF object file\n", 534 obj->path); 535 err = -LIBBPF_ERRNO__FORMAT; 536 goto errout; 537 } 538 539 return 0; 540 errout: 541 bpf_object__elf_finish(obj); 542 return err; 543 } 544 545 static int 546 bpf_object__check_endianness(struct bpf_object *obj) 547 { 548 static unsigned int const endian = 1; 549 550 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 551 case ELFDATA2LSB: 552 /* We are big endian, BPF obj is little endian. */ 553 if (*(unsigned char const *)&endian != 1) 554 goto mismatch; 555 break; 556 557 case ELFDATA2MSB: 558 /* We are little endian, BPF obj is big endian. */ 559 if (*(unsigned char const *)&endian != 0) 560 goto mismatch; 561 break; 562 default: 563 return -LIBBPF_ERRNO__ENDIAN; 564 } 565 566 return 0; 567 568 mismatch: 569 pr_warning("Error: endianness mismatch.\n"); 570 return -LIBBPF_ERRNO__ENDIAN; 571 } 572 573 static int 574 bpf_object__init_license(struct bpf_object *obj, 575 void *data, size_t size) 576 { 577 memcpy(obj->license, data, 578 min(size, sizeof(obj->license) - 1)); 579 pr_debug("license of %s is %s\n", obj->path, obj->license); 580 return 0; 581 } 582 583 static int 584 bpf_object__init_kversion(struct bpf_object *obj, 585 void *data, size_t size) 586 { 587 u32 kver; 588 589 if (size != sizeof(kver)) { 590 pr_warning("invalid kver section in %s\n", obj->path); 591 return -LIBBPF_ERRNO__FORMAT; 592 } 593 memcpy(&kver, data, sizeof(kver)); 594 obj->kern_version = kver; 595 pr_debug("kernel version of %s is %x\n", obj->path, 596 obj->kern_version); 597 return 0; 598 } 599 600 static int compare_bpf_map(const void *_a, const void *_b) 601 { 602 const struct bpf_map *a = _a; 603 const struct bpf_map *b = _b; 604 605 return a->offset - b->offset; 606 } 607 608 static int 609 bpf_object__init_maps(struct bpf_object *obj) 610 { 611 int i, map_idx, map_def_sz, nr_maps = 0; 612 Elf_Scn *scn; 613 Elf_Data *data; 614 Elf_Data *symbols = obj->efile.symbols; 615 616 if (obj->efile.maps_shndx < 0) 617 return -EINVAL; 618 if (!symbols) 619 return -EINVAL; 620 621 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 622 if (scn) 623 data = elf_getdata(scn, NULL); 624 if (!scn || !data) { 625 pr_warning("failed to get Elf_Data from map section %d\n", 626 obj->efile.maps_shndx); 627 return -EINVAL; 628 } 629 630 /* 631 * Count number of maps. Each map has a name. 632 * Array of maps is not supported: only the first element is 633 * considered. 634 * 635 * TODO: Detect array of map and report error. 636 */ 637 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 638 GElf_Sym sym; 639 640 if (!gelf_getsym(symbols, i, &sym)) 641 continue; 642 if (sym.st_shndx != obj->efile.maps_shndx) 643 continue; 644 nr_maps++; 645 } 646 647 /* Alloc obj->maps and fill nr_maps. */ 648 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 649 nr_maps, data->d_size); 650 651 if (!nr_maps) 652 return 0; 653 654 /* Assume equally sized map definitions */ 655 map_def_sz = data->d_size / nr_maps; 656 if (!data->d_size || (data->d_size % nr_maps) != 0) { 657 pr_warning("unable to determine map definition size " 658 "section %s, %d maps in %zd bytes\n", 659 obj->path, nr_maps, data->d_size); 660 return -EINVAL; 661 } 662 663 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 664 if (!obj->maps) { 665 pr_warning("alloc maps for object failed\n"); 666 return -ENOMEM; 667 } 668 obj->nr_maps = nr_maps; 669 670 /* 671 * fill all fd with -1 so won't close incorrect 672 * fd (fd=0 is stdin) when failure (zclose won't close 673 * negative fd)). 674 */ 675 for (i = 0; i < nr_maps; i++) 676 obj->maps[i].fd = -1; 677 678 /* 679 * Fill obj->maps using data in "maps" section. 680 */ 681 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 682 GElf_Sym sym; 683 const char *map_name; 684 struct bpf_map_def *def; 685 686 if (!gelf_getsym(symbols, i, &sym)) 687 continue; 688 if (sym.st_shndx != obj->efile.maps_shndx) 689 continue; 690 691 map_name = elf_strptr(obj->efile.elf, 692 obj->efile.strtabidx, 693 sym.st_name); 694 obj->maps[map_idx].offset = sym.st_value; 695 if (sym.st_value + map_def_sz > data->d_size) { 696 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 697 obj->path, map_name); 698 return -EINVAL; 699 } 700 701 obj->maps[map_idx].name = strdup(map_name); 702 if (!obj->maps[map_idx].name) { 703 pr_warning("failed to alloc map name\n"); 704 return -ENOMEM; 705 } 706 pr_debug("map %d is \"%s\"\n", map_idx, 707 obj->maps[map_idx].name); 708 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 709 /* 710 * If the definition of the map in the object file fits in 711 * bpf_map_def, copy it. Any extra fields in our version 712 * of bpf_map_def will default to zero as a result of the 713 * calloc above. 714 */ 715 if (map_def_sz <= sizeof(struct bpf_map_def)) { 716 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 717 } else { 718 /* 719 * Here the map structure being read is bigger than what 720 * we expect, truncate if the excess bits are all zero. 721 * If they are not zero, reject this map as 722 * incompatible. 723 */ 724 char *b; 725 for (b = ((char *)def) + sizeof(struct bpf_map_def); 726 b < ((char *)def) + map_def_sz; b++) { 727 if (*b != 0) { 728 pr_warning("maps section in %s: \"%s\" " 729 "has unrecognized, non-zero " 730 "options\n", 731 obj->path, map_name); 732 return -EINVAL; 733 } 734 } 735 memcpy(&obj->maps[map_idx].def, def, 736 sizeof(struct bpf_map_def)); 737 } 738 map_idx++; 739 } 740 741 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); 742 return 0; 743 } 744 745 static bool section_have_execinstr(struct bpf_object *obj, int idx) 746 { 747 Elf_Scn *scn; 748 GElf_Shdr sh; 749 750 scn = elf_getscn(obj->efile.elf, idx); 751 if (!scn) 752 return false; 753 754 if (gelf_getshdr(scn, &sh) != &sh) 755 return false; 756 757 if (sh.sh_flags & SHF_EXECINSTR) 758 return true; 759 760 return false; 761 } 762 763 static int bpf_object__elf_collect(struct bpf_object *obj) 764 { 765 Elf *elf = obj->efile.elf; 766 GElf_Ehdr *ep = &obj->efile.ehdr; 767 Elf_Scn *scn = NULL; 768 int idx = 0, err = 0; 769 770 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 771 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 772 pr_warning("failed to get e_shstrndx from %s\n", 773 obj->path); 774 return -LIBBPF_ERRNO__FORMAT; 775 } 776 777 while ((scn = elf_nextscn(elf, scn)) != NULL) { 778 char *name; 779 GElf_Shdr sh; 780 Elf_Data *data; 781 782 idx++; 783 if (gelf_getshdr(scn, &sh) != &sh) { 784 pr_warning("failed to get section(%d) header from %s\n", 785 idx, obj->path); 786 err = -LIBBPF_ERRNO__FORMAT; 787 goto out; 788 } 789 790 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 791 if (!name) { 792 pr_warning("failed to get section(%d) name from %s\n", 793 idx, obj->path); 794 err = -LIBBPF_ERRNO__FORMAT; 795 goto out; 796 } 797 798 data = elf_getdata(scn, 0); 799 if (!data) { 800 pr_warning("failed to get section(%d) data from %s(%s)\n", 801 idx, name, obj->path); 802 err = -LIBBPF_ERRNO__FORMAT; 803 goto out; 804 } 805 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 806 idx, name, (unsigned long)data->d_size, 807 (int)sh.sh_link, (unsigned long)sh.sh_flags, 808 (int)sh.sh_type); 809 810 if (strcmp(name, "license") == 0) 811 err = bpf_object__init_license(obj, 812 data->d_buf, 813 data->d_size); 814 else if (strcmp(name, "version") == 0) 815 err = bpf_object__init_kversion(obj, 816 data->d_buf, 817 data->d_size); 818 else if (strcmp(name, "maps") == 0) 819 obj->efile.maps_shndx = idx; 820 else if (sh.sh_type == SHT_SYMTAB) { 821 if (obj->efile.symbols) { 822 pr_warning("bpf: multiple SYMTAB in %s\n", 823 obj->path); 824 err = -LIBBPF_ERRNO__FORMAT; 825 } else { 826 obj->efile.symbols = data; 827 obj->efile.strtabidx = sh.sh_link; 828 } 829 } else if ((sh.sh_type == SHT_PROGBITS) && 830 (sh.sh_flags & SHF_EXECINSTR) && 831 (data->d_size > 0)) { 832 if (strcmp(name, ".text") == 0) 833 obj->efile.text_shndx = idx; 834 err = bpf_object__add_program(obj, data->d_buf, 835 data->d_size, name, idx); 836 if (err) { 837 char errmsg[STRERR_BUFSIZE]; 838 839 strerror_r(-err, errmsg, sizeof(errmsg)); 840 pr_warning("failed to alloc program %s (%s): %s", 841 name, obj->path, errmsg); 842 } 843 } else if (sh.sh_type == SHT_REL) { 844 void *reloc = obj->efile.reloc; 845 int nr_reloc = obj->efile.nr_reloc + 1; 846 int sec = sh.sh_info; /* points to other section */ 847 848 /* Only do relo for section with exec instructions */ 849 if (!section_have_execinstr(obj, sec)) { 850 pr_debug("skip relo %s(%d) for section(%d)\n", 851 name, idx, sec); 852 continue; 853 } 854 855 reloc = realloc(reloc, 856 sizeof(*obj->efile.reloc) * nr_reloc); 857 if (!reloc) { 858 pr_warning("realloc failed\n"); 859 err = -ENOMEM; 860 } else { 861 int n = nr_reloc - 1; 862 863 obj->efile.reloc = reloc; 864 obj->efile.nr_reloc = nr_reloc; 865 866 obj->efile.reloc[n].shdr = sh; 867 obj->efile.reloc[n].data = data; 868 } 869 } else { 870 pr_debug("skip section(%d) %s\n", idx, name); 871 } 872 if (err) 873 goto out; 874 } 875 876 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 877 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 878 return LIBBPF_ERRNO__FORMAT; 879 } 880 if (obj->efile.maps_shndx >= 0) { 881 err = bpf_object__init_maps(obj); 882 if (err) 883 goto out; 884 } 885 err = bpf_object__init_prog_names(obj); 886 out: 887 return err; 888 } 889 890 static struct bpf_program * 891 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 892 { 893 struct bpf_program *prog; 894 size_t i; 895 896 for (i = 0; i < obj->nr_programs; i++) { 897 prog = &obj->programs[i]; 898 if (prog->idx == idx) 899 return prog; 900 } 901 return NULL; 902 } 903 904 static int 905 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 906 Elf_Data *data, struct bpf_object *obj) 907 { 908 Elf_Data *symbols = obj->efile.symbols; 909 int text_shndx = obj->efile.text_shndx; 910 int maps_shndx = obj->efile.maps_shndx; 911 struct bpf_map *maps = obj->maps; 912 size_t nr_maps = obj->nr_maps; 913 int i, nrels; 914 915 pr_debug("collecting relocating info for: '%s'\n", 916 prog->section_name); 917 nrels = shdr->sh_size / shdr->sh_entsize; 918 919 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 920 if (!prog->reloc_desc) { 921 pr_warning("failed to alloc memory in relocation\n"); 922 return -ENOMEM; 923 } 924 prog->nr_reloc = nrels; 925 926 for (i = 0; i < nrels; i++) { 927 GElf_Sym sym; 928 GElf_Rel rel; 929 unsigned int insn_idx; 930 struct bpf_insn *insns = prog->insns; 931 size_t map_idx; 932 933 if (!gelf_getrel(data, i, &rel)) { 934 pr_warning("relocation: failed to get %d reloc\n", i); 935 return -LIBBPF_ERRNO__FORMAT; 936 } 937 938 if (!gelf_getsym(symbols, 939 GELF_R_SYM(rel.r_info), 940 &sym)) { 941 pr_warning("relocation: symbol %"PRIx64" not found\n", 942 GELF_R_SYM(rel.r_info)); 943 return -LIBBPF_ERRNO__FORMAT; 944 } 945 pr_debug("relo for %lld value %lld name %d\n", 946 (long long) (rel.r_info >> 32), 947 (long long) sym.st_value, sym.st_name); 948 949 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { 950 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 951 prog->section_name, sym.st_shndx); 952 return -LIBBPF_ERRNO__RELOC; 953 } 954 955 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 956 pr_debug("relocation: insn_idx=%u\n", insn_idx); 957 958 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 959 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 960 pr_warning("incorrect bpf_call opcode\n"); 961 return -LIBBPF_ERRNO__RELOC; 962 } 963 prog->reloc_desc[i].type = RELO_CALL; 964 prog->reloc_desc[i].insn_idx = insn_idx; 965 prog->reloc_desc[i].text_off = sym.st_value; 966 continue; 967 } 968 969 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 970 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 971 insn_idx, insns[insn_idx].code); 972 return -LIBBPF_ERRNO__RELOC; 973 } 974 975 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ 976 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 977 if (maps[map_idx].offset == sym.st_value) { 978 pr_debug("relocation: find map %zd (%s) for insn %u\n", 979 map_idx, maps[map_idx].name, insn_idx); 980 break; 981 } 982 } 983 984 if (map_idx >= nr_maps) { 985 pr_warning("bpf relocation: map_idx %d large than %d\n", 986 (int)map_idx, (int)nr_maps - 1); 987 return -LIBBPF_ERRNO__RELOC; 988 } 989 990 prog->reloc_desc[i].type = RELO_LD64; 991 prog->reloc_desc[i].insn_idx = insn_idx; 992 prog->reloc_desc[i].map_idx = map_idx; 993 } 994 return 0; 995 } 996 997 static int 998 bpf_object__create_maps(struct bpf_object *obj) 999 { 1000 unsigned int i; 1001 1002 for (i = 0; i < obj->nr_maps; i++) { 1003 struct bpf_map_def *def = &obj->maps[i].def; 1004 int *pfd = &obj->maps[i].fd; 1005 1006 *pfd = bpf_create_map_name(def->type, 1007 obj->maps[i].name, 1008 def->key_size, 1009 def->value_size, 1010 def->max_entries, 1011 def->map_flags); 1012 if (*pfd < 0) { 1013 size_t j; 1014 int err = *pfd; 1015 1016 pr_warning("failed to create map (name: '%s'): %s\n", 1017 obj->maps[i].name, 1018 strerror(errno)); 1019 for (j = 0; j < i; j++) 1020 zclose(obj->maps[j].fd); 1021 return err; 1022 } 1023 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); 1024 } 1025 1026 return 0; 1027 } 1028 1029 static int 1030 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1031 struct reloc_desc *relo) 1032 { 1033 struct bpf_insn *insn, *new_insn; 1034 struct bpf_program *text; 1035 size_t new_cnt; 1036 1037 if (relo->type != RELO_CALL) 1038 return -LIBBPF_ERRNO__RELOC; 1039 1040 if (prog->idx == obj->efile.text_shndx) { 1041 pr_warning("relo in .text insn %d into off %d\n", 1042 relo->insn_idx, relo->text_off); 1043 return -LIBBPF_ERRNO__RELOC; 1044 } 1045 1046 if (prog->main_prog_cnt == 0) { 1047 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1048 if (!text) { 1049 pr_warning("no .text section found yet relo into text exist\n"); 1050 return -LIBBPF_ERRNO__RELOC; 1051 } 1052 new_cnt = prog->insns_cnt + text->insns_cnt; 1053 new_insn = realloc(prog->insns, new_cnt * sizeof(*insn)); 1054 if (!new_insn) { 1055 pr_warning("oom in prog realloc\n"); 1056 return -ENOMEM; 1057 } 1058 memcpy(new_insn + prog->insns_cnt, text->insns, 1059 text->insns_cnt * sizeof(*insn)); 1060 prog->insns = new_insn; 1061 prog->main_prog_cnt = prog->insns_cnt; 1062 prog->insns_cnt = new_cnt; 1063 pr_debug("added %zd insn from %s to prog %s\n", 1064 text->insns_cnt, text->section_name, 1065 prog->section_name); 1066 } 1067 insn = &prog->insns[relo->insn_idx]; 1068 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1069 return 0; 1070 } 1071 1072 static int 1073 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1074 { 1075 int i, err; 1076 1077 if (!prog || !prog->reloc_desc) 1078 return 0; 1079 1080 for (i = 0; i < prog->nr_reloc; i++) { 1081 if (prog->reloc_desc[i].type == RELO_LD64) { 1082 struct bpf_insn *insns = prog->insns; 1083 int insn_idx, map_idx; 1084 1085 insn_idx = prog->reloc_desc[i].insn_idx; 1086 map_idx = prog->reloc_desc[i].map_idx; 1087 1088 if (insn_idx >= (int)prog->insns_cnt) { 1089 pr_warning("relocation out of range: '%s'\n", 1090 prog->section_name); 1091 return -LIBBPF_ERRNO__RELOC; 1092 } 1093 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1094 insns[insn_idx].imm = obj->maps[map_idx].fd; 1095 } else { 1096 err = bpf_program__reloc_text(prog, obj, 1097 &prog->reloc_desc[i]); 1098 if (err) 1099 return err; 1100 } 1101 } 1102 1103 zfree(&prog->reloc_desc); 1104 prog->nr_reloc = 0; 1105 return 0; 1106 } 1107 1108 1109 static int 1110 bpf_object__relocate(struct bpf_object *obj) 1111 { 1112 struct bpf_program *prog; 1113 size_t i; 1114 int err; 1115 1116 for (i = 0; i < obj->nr_programs; i++) { 1117 prog = &obj->programs[i]; 1118 1119 err = bpf_program__relocate(prog, obj); 1120 if (err) { 1121 pr_warning("failed to relocate '%s'\n", 1122 prog->section_name); 1123 return err; 1124 } 1125 } 1126 return 0; 1127 } 1128 1129 static int bpf_object__collect_reloc(struct bpf_object *obj) 1130 { 1131 int i, err; 1132 1133 if (!obj_elf_valid(obj)) { 1134 pr_warning("Internal error: elf object is closed\n"); 1135 return -LIBBPF_ERRNO__INTERNAL; 1136 } 1137 1138 for (i = 0; i < obj->efile.nr_reloc; i++) { 1139 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1140 Elf_Data *data = obj->efile.reloc[i].data; 1141 int idx = shdr->sh_info; 1142 struct bpf_program *prog; 1143 1144 if (shdr->sh_type != SHT_REL) { 1145 pr_warning("internal error at %d\n", __LINE__); 1146 return -LIBBPF_ERRNO__INTERNAL; 1147 } 1148 1149 prog = bpf_object__find_prog_by_idx(obj, idx); 1150 if (!prog) { 1151 pr_warning("relocation failed: no section(%d)\n", idx); 1152 return -LIBBPF_ERRNO__RELOC; 1153 } 1154 1155 err = bpf_program__collect_reloc(prog, 1156 shdr, data, 1157 obj); 1158 if (err) 1159 return err; 1160 } 1161 return 0; 1162 } 1163 1164 static int 1165 load_program(enum bpf_prog_type type, const char *name, struct bpf_insn *insns, 1166 int insns_cnt, char *license, u32 kern_version, int *pfd) 1167 { 1168 int ret; 1169 char *log_buf; 1170 1171 if (!insns || !insns_cnt) 1172 return -EINVAL; 1173 1174 log_buf = malloc(BPF_LOG_BUF_SIZE); 1175 if (!log_buf) 1176 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1177 1178 ret = bpf_load_program_name(type, name, insns, insns_cnt, license, 1179 kern_version, log_buf, BPF_LOG_BUF_SIZE); 1180 1181 if (ret >= 0) { 1182 *pfd = ret; 1183 ret = 0; 1184 goto out; 1185 } 1186 1187 ret = -LIBBPF_ERRNO__LOAD; 1188 pr_warning("load bpf program failed: %s\n", strerror(errno)); 1189 1190 if (log_buf && log_buf[0] != '\0') { 1191 ret = -LIBBPF_ERRNO__VERIFY; 1192 pr_warning("-- BEGIN DUMP LOG ---\n"); 1193 pr_warning("\n%s\n", log_buf); 1194 pr_warning("-- END LOG --\n"); 1195 } else if (insns_cnt >= BPF_MAXINSNS) { 1196 pr_warning("Program too large (%d insns), at most %d insns\n", 1197 insns_cnt, BPF_MAXINSNS); 1198 ret = -LIBBPF_ERRNO__PROG2BIG; 1199 } else { 1200 /* Wrong program type? */ 1201 if (type != BPF_PROG_TYPE_KPROBE) { 1202 int fd; 1203 1204 fd = bpf_load_program_name(BPF_PROG_TYPE_KPROBE, name, 1205 insns, insns_cnt, license, 1206 kern_version, NULL, 0); 1207 if (fd >= 0) { 1208 close(fd); 1209 ret = -LIBBPF_ERRNO__PROGTYPE; 1210 goto out; 1211 } 1212 } 1213 1214 if (log_buf) 1215 ret = -LIBBPF_ERRNO__KVER; 1216 } 1217 1218 out: 1219 free(log_buf); 1220 return ret; 1221 } 1222 1223 static int 1224 bpf_program__load(struct bpf_program *prog, 1225 char *license, u32 kern_version) 1226 { 1227 int err = 0, fd, i; 1228 1229 if (prog->instances.nr < 0 || !prog->instances.fds) { 1230 if (prog->preprocessor) { 1231 pr_warning("Internal error: can't load program '%s'\n", 1232 prog->section_name); 1233 return -LIBBPF_ERRNO__INTERNAL; 1234 } 1235 1236 prog->instances.fds = malloc(sizeof(int)); 1237 if (!prog->instances.fds) { 1238 pr_warning("Not enough memory for BPF fds\n"); 1239 return -ENOMEM; 1240 } 1241 prog->instances.nr = 1; 1242 prog->instances.fds[0] = -1; 1243 } 1244 1245 if (!prog->preprocessor) { 1246 if (prog->instances.nr != 1) { 1247 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 1248 prog->section_name, prog->instances.nr); 1249 } 1250 err = load_program(prog->type, prog->name, prog->insns, 1251 prog->insns_cnt, license, kern_version, &fd); 1252 if (!err) 1253 prog->instances.fds[0] = fd; 1254 goto out; 1255 } 1256 1257 for (i = 0; i < prog->instances.nr; i++) { 1258 struct bpf_prog_prep_result result; 1259 bpf_program_prep_t preprocessor = prog->preprocessor; 1260 1261 bzero(&result, sizeof(result)); 1262 err = preprocessor(prog, i, prog->insns, 1263 prog->insns_cnt, &result); 1264 if (err) { 1265 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 1266 i, prog->section_name); 1267 goto out; 1268 } 1269 1270 if (!result.new_insn_ptr || !result.new_insn_cnt) { 1271 pr_debug("Skip loading the %dth instance of program '%s'\n", 1272 i, prog->section_name); 1273 prog->instances.fds[i] = -1; 1274 if (result.pfd) 1275 *result.pfd = -1; 1276 continue; 1277 } 1278 1279 err = load_program(prog->type, prog->name, 1280 result.new_insn_ptr, 1281 result.new_insn_cnt, 1282 license, kern_version, &fd); 1283 1284 if (err) { 1285 pr_warning("Loading the %dth instance of program '%s' failed\n", 1286 i, prog->section_name); 1287 goto out; 1288 } 1289 1290 if (result.pfd) 1291 *result.pfd = fd; 1292 prog->instances.fds[i] = fd; 1293 } 1294 out: 1295 if (err) 1296 pr_warning("failed to load program '%s'\n", 1297 prog->section_name); 1298 zfree(&prog->insns); 1299 prog->insns_cnt = 0; 1300 return err; 1301 } 1302 1303 static int 1304 bpf_object__load_progs(struct bpf_object *obj) 1305 { 1306 size_t i; 1307 int err; 1308 1309 for (i = 0; i < obj->nr_programs; i++) { 1310 if (obj->programs[i].idx == obj->efile.text_shndx) 1311 continue; 1312 err = bpf_program__load(&obj->programs[i], 1313 obj->license, 1314 obj->kern_version); 1315 if (err) 1316 return err; 1317 } 1318 return 0; 1319 } 1320 1321 static int bpf_object__validate(struct bpf_object *obj) 1322 { 1323 if (obj->kern_version == 0) { 1324 pr_warning("%s doesn't provide kernel version\n", 1325 obj->path); 1326 return -LIBBPF_ERRNO__KVERSION; 1327 } 1328 return 0; 1329 } 1330 1331 static struct bpf_object * 1332 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 1333 { 1334 struct bpf_object *obj; 1335 int err; 1336 1337 if (elf_version(EV_CURRENT) == EV_NONE) { 1338 pr_warning("failed to init libelf for %s\n", path); 1339 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1340 } 1341 1342 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1343 if (IS_ERR(obj)) 1344 return obj; 1345 1346 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1347 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1348 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 1349 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1350 CHECK_ERR(bpf_object__validate(obj), err, out); 1351 1352 bpf_object__elf_finish(obj); 1353 return obj; 1354 out: 1355 bpf_object__close(obj); 1356 return ERR_PTR(err); 1357 } 1358 1359 struct bpf_object *bpf_object__open(const char *path) 1360 { 1361 /* param validation */ 1362 if (!path) 1363 return NULL; 1364 1365 pr_debug("loading %s\n", path); 1366 1367 return __bpf_object__open(path, NULL, 0); 1368 } 1369 1370 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1371 size_t obj_buf_sz, 1372 const char *name) 1373 { 1374 char tmp_name[64]; 1375 1376 /* param validation */ 1377 if (!obj_buf || obj_buf_sz <= 0) 1378 return NULL; 1379 1380 if (!name) { 1381 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1382 (unsigned long)obj_buf, 1383 (unsigned long)obj_buf_sz); 1384 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1385 name = tmp_name; 1386 } 1387 pr_debug("loading object '%s' from buffer\n", 1388 name); 1389 1390 return __bpf_object__open(name, obj_buf, obj_buf_sz); 1391 } 1392 1393 int bpf_object__unload(struct bpf_object *obj) 1394 { 1395 size_t i; 1396 1397 if (!obj) 1398 return -EINVAL; 1399 1400 for (i = 0; i < obj->nr_maps; i++) 1401 zclose(obj->maps[i].fd); 1402 1403 for (i = 0; i < obj->nr_programs; i++) 1404 bpf_program__unload(&obj->programs[i]); 1405 1406 return 0; 1407 } 1408 1409 int bpf_object__load(struct bpf_object *obj) 1410 { 1411 int err; 1412 1413 if (!obj) 1414 return -EINVAL; 1415 1416 if (obj->loaded) { 1417 pr_warning("object should not be loaded twice\n"); 1418 return -EINVAL; 1419 } 1420 1421 obj->loaded = true; 1422 1423 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1424 CHECK_ERR(bpf_object__relocate(obj), err, out); 1425 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1426 1427 return 0; 1428 out: 1429 bpf_object__unload(obj); 1430 pr_warning("failed to load object '%s'\n", obj->path); 1431 return err; 1432 } 1433 1434 static int check_path(const char *path) 1435 { 1436 struct statfs st_fs; 1437 char *dname, *dir; 1438 int err = 0; 1439 1440 if (path == NULL) 1441 return -EINVAL; 1442 1443 dname = strdup(path); 1444 if (dname == NULL) 1445 return -ENOMEM; 1446 1447 dir = dirname(dname); 1448 if (statfs(dir, &st_fs)) { 1449 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno)); 1450 err = -errno; 1451 } 1452 free(dname); 1453 1454 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 1455 pr_warning("specified path %s is not on BPF FS\n", path); 1456 err = -EINVAL; 1457 } 1458 1459 return err; 1460 } 1461 1462 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 1463 int instance) 1464 { 1465 int err; 1466 1467 err = check_path(path); 1468 if (err) 1469 return err; 1470 1471 if (prog == NULL) { 1472 pr_warning("invalid program pointer\n"); 1473 return -EINVAL; 1474 } 1475 1476 if (instance < 0 || instance >= prog->instances.nr) { 1477 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 1478 instance, prog->section_name, prog->instances.nr); 1479 return -EINVAL; 1480 } 1481 1482 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 1483 pr_warning("failed to pin program: %s\n", strerror(errno)); 1484 return -errno; 1485 } 1486 pr_debug("pinned program '%s'\n", path); 1487 1488 return 0; 1489 } 1490 1491 static int make_dir(const char *path) 1492 { 1493 int err = 0; 1494 1495 if (mkdir(path, 0700) && errno != EEXIST) 1496 err = -errno; 1497 1498 if (err) 1499 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err)); 1500 return err; 1501 } 1502 1503 int bpf_program__pin(struct bpf_program *prog, const char *path) 1504 { 1505 int i, err; 1506 1507 err = check_path(path); 1508 if (err) 1509 return err; 1510 1511 if (prog == NULL) { 1512 pr_warning("invalid program pointer\n"); 1513 return -EINVAL; 1514 } 1515 1516 if (prog->instances.nr <= 0) { 1517 pr_warning("no instances of prog %s to pin\n", 1518 prog->section_name); 1519 return -EINVAL; 1520 } 1521 1522 err = make_dir(path); 1523 if (err) 1524 return err; 1525 1526 for (i = 0; i < prog->instances.nr; i++) { 1527 char buf[PATH_MAX]; 1528 int len; 1529 1530 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 1531 if (len < 0) 1532 return -EINVAL; 1533 else if (len >= PATH_MAX) 1534 return -ENAMETOOLONG; 1535 1536 err = bpf_program__pin_instance(prog, buf, i); 1537 if (err) 1538 return err; 1539 } 1540 1541 return 0; 1542 } 1543 1544 int bpf_map__pin(struct bpf_map *map, const char *path) 1545 { 1546 int err; 1547 1548 err = check_path(path); 1549 if (err) 1550 return err; 1551 1552 if (map == NULL) { 1553 pr_warning("invalid map pointer\n"); 1554 return -EINVAL; 1555 } 1556 1557 if (bpf_obj_pin(map->fd, path)) { 1558 pr_warning("failed to pin map: %s\n", strerror(errno)); 1559 return -errno; 1560 } 1561 1562 pr_debug("pinned map '%s'\n", path); 1563 return 0; 1564 } 1565 1566 int bpf_object__pin(struct bpf_object *obj, const char *path) 1567 { 1568 struct bpf_program *prog; 1569 struct bpf_map *map; 1570 int err; 1571 1572 if (!obj) 1573 return -ENOENT; 1574 1575 if (!obj->loaded) { 1576 pr_warning("object not yet loaded; load it first\n"); 1577 return -ENOENT; 1578 } 1579 1580 err = make_dir(path); 1581 if (err) 1582 return err; 1583 1584 bpf_map__for_each(map, obj) { 1585 char buf[PATH_MAX]; 1586 int len; 1587 1588 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1589 bpf_map__name(map)); 1590 if (len < 0) 1591 return -EINVAL; 1592 else if (len >= PATH_MAX) 1593 return -ENAMETOOLONG; 1594 1595 err = bpf_map__pin(map, buf); 1596 if (err) 1597 return err; 1598 } 1599 1600 bpf_object__for_each_program(prog, obj) { 1601 char buf[PATH_MAX]; 1602 int len; 1603 1604 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1605 prog->section_name); 1606 if (len < 0) 1607 return -EINVAL; 1608 else if (len >= PATH_MAX) 1609 return -ENAMETOOLONG; 1610 1611 err = bpf_program__pin(prog, buf); 1612 if (err) 1613 return err; 1614 } 1615 1616 return 0; 1617 } 1618 1619 void bpf_object__close(struct bpf_object *obj) 1620 { 1621 size_t i; 1622 1623 if (!obj) 1624 return; 1625 1626 if (obj->clear_priv) 1627 obj->clear_priv(obj, obj->priv); 1628 1629 bpf_object__elf_finish(obj); 1630 bpf_object__unload(obj); 1631 1632 for (i = 0; i < obj->nr_maps; i++) { 1633 zfree(&obj->maps[i].name); 1634 if (obj->maps[i].clear_priv) 1635 obj->maps[i].clear_priv(&obj->maps[i], 1636 obj->maps[i].priv); 1637 obj->maps[i].priv = NULL; 1638 obj->maps[i].clear_priv = NULL; 1639 } 1640 zfree(&obj->maps); 1641 obj->nr_maps = 0; 1642 1643 if (obj->programs && obj->nr_programs) { 1644 for (i = 0; i < obj->nr_programs; i++) 1645 bpf_program__exit(&obj->programs[i]); 1646 } 1647 zfree(&obj->programs); 1648 1649 list_del(&obj->list); 1650 free(obj); 1651 } 1652 1653 struct bpf_object * 1654 bpf_object__next(struct bpf_object *prev) 1655 { 1656 struct bpf_object *next; 1657 1658 if (!prev) 1659 next = list_first_entry(&bpf_objects_list, 1660 struct bpf_object, 1661 list); 1662 else 1663 next = list_next_entry(prev, list); 1664 1665 /* Empty list is noticed here so don't need checking on entry. */ 1666 if (&next->list == &bpf_objects_list) 1667 return NULL; 1668 1669 return next; 1670 } 1671 1672 const char *bpf_object__name(struct bpf_object *obj) 1673 { 1674 return obj ? obj->path : ERR_PTR(-EINVAL); 1675 } 1676 1677 unsigned int bpf_object__kversion(struct bpf_object *obj) 1678 { 1679 return obj ? obj->kern_version : 0; 1680 } 1681 1682 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 1683 bpf_object_clear_priv_t clear_priv) 1684 { 1685 if (obj->priv && obj->clear_priv) 1686 obj->clear_priv(obj, obj->priv); 1687 1688 obj->priv = priv; 1689 obj->clear_priv = clear_priv; 1690 return 0; 1691 } 1692 1693 void *bpf_object__priv(struct bpf_object *obj) 1694 { 1695 return obj ? obj->priv : ERR_PTR(-EINVAL); 1696 } 1697 1698 struct bpf_program * 1699 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1700 { 1701 size_t idx; 1702 1703 if (!obj->programs) 1704 return NULL; 1705 /* First handler */ 1706 if (prev == NULL) 1707 return &obj->programs[0]; 1708 1709 if (prev->obj != obj) { 1710 pr_warning("error: program handler doesn't match object\n"); 1711 return NULL; 1712 } 1713 1714 idx = (prev - obj->programs) + 1; 1715 if (idx >= obj->nr_programs) 1716 return NULL; 1717 return &obj->programs[idx]; 1718 } 1719 1720 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 1721 bpf_program_clear_priv_t clear_priv) 1722 { 1723 if (prog->priv && prog->clear_priv) 1724 prog->clear_priv(prog, prog->priv); 1725 1726 prog->priv = priv; 1727 prog->clear_priv = clear_priv; 1728 return 0; 1729 } 1730 1731 void *bpf_program__priv(struct bpf_program *prog) 1732 { 1733 return prog ? prog->priv : ERR_PTR(-EINVAL); 1734 } 1735 1736 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1737 { 1738 const char *title; 1739 1740 title = prog->section_name; 1741 if (needs_copy) { 1742 title = strdup(title); 1743 if (!title) { 1744 pr_warning("failed to strdup program title\n"); 1745 return ERR_PTR(-ENOMEM); 1746 } 1747 } 1748 1749 return title; 1750 } 1751 1752 int bpf_program__fd(struct bpf_program *prog) 1753 { 1754 return bpf_program__nth_fd(prog, 0); 1755 } 1756 1757 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 1758 bpf_program_prep_t prep) 1759 { 1760 int *instances_fds; 1761 1762 if (nr_instances <= 0 || !prep) 1763 return -EINVAL; 1764 1765 if (prog->instances.nr > 0 || prog->instances.fds) { 1766 pr_warning("Can't set pre-processor after loading\n"); 1767 return -EINVAL; 1768 } 1769 1770 instances_fds = malloc(sizeof(int) * nr_instances); 1771 if (!instances_fds) { 1772 pr_warning("alloc memory failed for fds\n"); 1773 return -ENOMEM; 1774 } 1775 1776 /* fill all fd with -1 */ 1777 memset(instances_fds, -1, sizeof(int) * nr_instances); 1778 1779 prog->instances.nr = nr_instances; 1780 prog->instances.fds = instances_fds; 1781 prog->preprocessor = prep; 1782 return 0; 1783 } 1784 1785 int bpf_program__nth_fd(struct bpf_program *prog, int n) 1786 { 1787 int fd; 1788 1789 if (n >= prog->instances.nr || n < 0) { 1790 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 1791 n, prog->section_name, prog->instances.nr); 1792 return -EINVAL; 1793 } 1794 1795 fd = prog->instances.fds[n]; 1796 if (fd < 0) { 1797 pr_warning("%dth instance of program '%s' is invalid\n", 1798 n, prog->section_name); 1799 return -ENOENT; 1800 } 1801 1802 return fd; 1803 } 1804 1805 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 1806 { 1807 prog->type = type; 1808 } 1809 1810 static bool bpf_program__is_type(struct bpf_program *prog, 1811 enum bpf_prog_type type) 1812 { 1813 return prog ? (prog->type == type) : false; 1814 } 1815 1816 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 1817 int bpf_program__set_##NAME(struct bpf_program *prog) \ 1818 { \ 1819 if (!prog) \ 1820 return -EINVAL; \ 1821 bpf_program__set_type(prog, TYPE); \ 1822 return 0; \ 1823 } \ 1824 \ 1825 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 1826 { \ 1827 return bpf_program__is_type(prog, TYPE); \ 1828 } \ 1829 1830 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 1831 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 1832 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 1833 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 1834 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 1835 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 1836 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 1837 1838 #define BPF_PROG_SEC(string, type) { string, sizeof(string) - 1, type } 1839 static const struct { 1840 const char *sec; 1841 size_t len; 1842 enum bpf_prog_type prog_type; 1843 } section_names[] = { 1844 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 1845 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 1846 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 1847 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 1848 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 1849 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 1850 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 1851 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 1852 BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 1853 BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK), 1854 BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE), 1855 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 1856 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 1857 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 1858 BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS), 1859 BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB), 1860 }; 1861 #undef BPF_PROG_SEC 1862 1863 static enum bpf_prog_type bpf_program__guess_type(struct bpf_program *prog) 1864 { 1865 int i; 1866 1867 if (!prog->section_name) 1868 goto err; 1869 1870 for (i = 0; i < ARRAY_SIZE(section_names); i++) 1871 if (strncmp(prog->section_name, section_names[i].sec, 1872 section_names[i].len) == 0) 1873 return section_names[i].prog_type; 1874 1875 err: 1876 pr_warning("failed to guess program type based on section name %s\n", 1877 prog->section_name); 1878 1879 return BPF_PROG_TYPE_UNSPEC; 1880 } 1881 1882 int bpf_map__fd(struct bpf_map *map) 1883 { 1884 return map ? map->fd : -EINVAL; 1885 } 1886 1887 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 1888 { 1889 return map ? &map->def : ERR_PTR(-EINVAL); 1890 } 1891 1892 const char *bpf_map__name(struct bpf_map *map) 1893 { 1894 return map ? map->name : NULL; 1895 } 1896 1897 int bpf_map__set_priv(struct bpf_map *map, void *priv, 1898 bpf_map_clear_priv_t clear_priv) 1899 { 1900 if (!map) 1901 return -EINVAL; 1902 1903 if (map->priv) { 1904 if (map->clear_priv) 1905 map->clear_priv(map, map->priv); 1906 } 1907 1908 map->priv = priv; 1909 map->clear_priv = clear_priv; 1910 return 0; 1911 } 1912 1913 void *bpf_map__priv(struct bpf_map *map) 1914 { 1915 return map ? map->priv : ERR_PTR(-EINVAL); 1916 } 1917 1918 struct bpf_map * 1919 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 1920 { 1921 size_t idx; 1922 struct bpf_map *s, *e; 1923 1924 if (!obj || !obj->maps) 1925 return NULL; 1926 1927 s = obj->maps; 1928 e = obj->maps + obj->nr_maps; 1929 1930 if (prev == NULL) 1931 return s; 1932 1933 if ((prev < s) || (prev >= e)) { 1934 pr_warning("error in %s: map handler doesn't belong to object\n", 1935 __func__); 1936 return NULL; 1937 } 1938 1939 idx = (prev - obj->maps) + 1; 1940 if (idx >= obj->nr_maps) 1941 return NULL; 1942 return &obj->maps[idx]; 1943 } 1944 1945 struct bpf_map * 1946 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 1947 { 1948 struct bpf_map *pos; 1949 1950 bpf_map__for_each(pos, obj) { 1951 if (pos->name && !strcmp(pos->name, name)) 1952 return pos; 1953 } 1954 return NULL; 1955 } 1956 1957 struct bpf_map * 1958 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 1959 { 1960 int i; 1961 1962 for (i = 0; i < obj->nr_maps; i++) { 1963 if (obj->maps[i].offset == offset) 1964 return &obj->maps[i]; 1965 } 1966 return ERR_PTR(-ENOENT); 1967 } 1968 1969 long libbpf_get_error(const void *ptr) 1970 { 1971 if (IS_ERR(ptr)) 1972 return PTR_ERR(ptr); 1973 return 0; 1974 } 1975 1976 int bpf_prog_load(const char *file, enum bpf_prog_type type, 1977 struct bpf_object **pobj, int *prog_fd) 1978 { 1979 struct bpf_program *prog, *first_prog = NULL; 1980 struct bpf_object *obj; 1981 int err; 1982 1983 obj = bpf_object__open(file); 1984 if (IS_ERR(obj)) 1985 return -ENOENT; 1986 1987 bpf_object__for_each_program(prog, obj) { 1988 /* 1989 * If type is not specified, try to guess it based on 1990 * section name. 1991 */ 1992 if (type == BPF_PROG_TYPE_UNSPEC) { 1993 type = bpf_program__guess_type(prog); 1994 if (type == BPF_PROG_TYPE_UNSPEC) { 1995 bpf_object__close(obj); 1996 return -EINVAL; 1997 } 1998 } 1999 2000 bpf_program__set_type(prog, type); 2001 if (prog->idx != obj->efile.text_shndx && !first_prog) 2002 first_prog = prog; 2003 } 2004 2005 if (!first_prog) { 2006 pr_warning("object file doesn't contain bpf program\n"); 2007 bpf_object__close(obj); 2008 return -ENOENT; 2009 } 2010 2011 err = bpf_object__load(obj); 2012 if (err) { 2013 bpf_object__close(obj); 2014 return -EINVAL; 2015 } 2016 2017 *pobj = obj; 2018 *prog_fd = bpf_program__fd(first_prog); 2019 return 0; 2020 } 2021