1 /* 2 * Common eBPF ELF object loading operations. 3 * 4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 6 * Copyright (C) 2015 Huawei Inc. 7 * Copyright (C) 2017 Nicira, Inc. 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU Lesser General Public 11 * License as published by the Free Software Foundation; 12 * version 2.1 of the License (not later!) 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public 20 * License along with this program; if not, see <http://www.gnu.org/licenses> 21 */ 22 23 #include <stdlib.h> 24 #include <stdio.h> 25 #include <stdarg.h> 26 #include <libgen.h> 27 #include <inttypes.h> 28 #include <string.h> 29 #include <unistd.h> 30 #include <fcntl.h> 31 #include <errno.h> 32 #include <asm/unistd.h> 33 #include <linux/err.h> 34 #include <linux/kernel.h> 35 #include <linux/bpf.h> 36 #include <linux/list.h> 37 #include <linux/limits.h> 38 #include <sys/stat.h> 39 #include <sys/types.h> 40 #include <sys/vfs.h> 41 #include <libelf.h> 42 #include <gelf.h> 43 44 #include "libbpf.h" 45 #include "bpf.h" 46 47 #ifndef EM_BPF 48 #define EM_BPF 247 49 #endif 50 51 #ifndef BPF_FS_MAGIC 52 #define BPF_FS_MAGIC 0xcafe4a11 53 #endif 54 55 #define __printf(a, b) __attribute__((format(printf, a, b))) 56 57 __printf(1, 2) 58 static int __base_pr(const char *format, ...) 59 { 60 va_list args; 61 int err; 62 63 va_start(args, format); 64 err = vfprintf(stderr, format, args); 65 va_end(args); 66 return err; 67 } 68 69 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 70 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 71 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 72 73 #define __pr(func, fmt, ...) \ 74 do { \ 75 if ((func)) \ 76 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 77 } while (0) 78 79 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 80 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 81 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 82 83 void libbpf_set_print(libbpf_print_fn_t warn, 84 libbpf_print_fn_t info, 85 libbpf_print_fn_t debug) 86 { 87 __pr_warning = warn; 88 __pr_info = info; 89 __pr_debug = debug; 90 } 91 92 #define STRERR_BUFSIZE 128 93 94 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 95 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 96 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 97 98 static const char *libbpf_strerror_table[NR_ERRNO] = { 99 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 100 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 101 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 102 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", 103 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 104 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 105 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 106 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 107 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 108 [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", 109 }; 110 111 int libbpf_strerror(int err, char *buf, size_t size) 112 { 113 if (!buf || !size) 114 return -1; 115 116 err = err > 0 ? err : -err; 117 118 if (err < __LIBBPF_ERRNO__START) { 119 int ret; 120 121 ret = strerror_r(err, buf, size); 122 buf[size - 1] = '\0'; 123 return ret; 124 } 125 126 if (err < __LIBBPF_ERRNO__END) { 127 const char *msg; 128 129 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 130 snprintf(buf, size, "%s", msg); 131 buf[size - 1] = '\0'; 132 return 0; 133 } 134 135 snprintf(buf, size, "Unknown libbpf error %d", err); 136 buf[size - 1] = '\0'; 137 return -1; 138 } 139 140 #define CHECK_ERR(action, err, out) do { \ 141 err = action; \ 142 if (err) \ 143 goto out; \ 144 } while(0) 145 146 147 /* Copied from tools/perf/util/util.h */ 148 #ifndef zfree 149 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 150 #endif 151 152 #ifndef zclose 153 # define zclose(fd) ({ \ 154 int ___err = 0; \ 155 if ((fd) >= 0) \ 156 ___err = close((fd)); \ 157 fd = -1; \ 158 ___err; }) 159 #endif 160 161 #ifdef HAVE_LIBELF_MMAP_SUPPORT 162 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 163 #else 164 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 165 #endif 166 167 /* 168 * bpf_prog should be a better name but it has been used in 169 * linux/filter.h. 170 */ 171 struct bpf_program { 172 /* Index in elf obj file, for relocation use. */ 173 int idx; 174 char *name; 175 char *section_name; 176 struct bpf_insn *insns; 177 size_t insns_cnt, main_prog_cnt; 178 enum bpf_prog_type type; 179 180 struct reloc_desc { 181 enum { 182 RELO_LD64, 183 RELO_CALL, 184 } type; 185 int insn_idx; 186 union { 187 int map_idx; 188 int text_off; 189 }; 190 } *reloc_desc; 191 int nr_reloc; 192 193 struct { 194 int nr; 195 int *fds; 196 } instances; 197 bpf_program_prep_t preprocessor; 198 199 struct bpf_object *obj; 200 void *priv; 201 bpf_program_clear_priv_t clear_priv; 202 }; 203 204 struct bpf_map { 205 int fd; 206 char *name; 207 size_t offset; 208 struct bpf_map_def def; 209 void *priv; 210 bpf_map_clear_priv_t clear_priv; 211 }; 212 213 static LIST_HEAD(bpf_objects_list); 214 215 struct bpf_object { 216 char license[64]; 217 u32 kern_version; 218 219 struct bpf_program *programs; 220 size_t nr_programs; 221 struct bpf_map *maps; 222 size_t nr_maps; 223 224 bool loaded; 225 226 /* 227 * Information when doing elf related work. Only valid if fd 228 * is valid. 229 */ 230 struct { 231 int fd; 232 void *obj_buf; 233 size_t obj_buf_sz; 234 Elf *elf; 235 GElf_Ehdr ehdr; 236 Elf_Data *symbols; 237 size_t strtabidx; 238 struct { 239 GElf_Shdr shdr; 240 Elf_Data *data; 241 } *reloc; 242 int nr_reloc; 243 int maps_shndx; 244 int text_shndx; 245 } efile; 246 /* 247 * All loaded bpf_object is linked in a list, which is 248 * hidden to caller. bpf_objects__<func> handlers deal with 249 * all objects. 250 */ 251 struct list_head list; 252 253 void *priv; 254 bpf_object_clear_priv_t clear_priv; 255 256 char path[]; 257 }; 258 #define obj_elf_valid(o) ((o)->efile.elf) 259 260 static void bpf_program__unload(struct bpf_program *prog) 261 { 262 int i; 263 264 if (!prog) 265 return; 266 267 /* 268 * If the object is opened but the program was never loaded, 269 * it is possible that prog->instances.nr == -1. 270 */ 271 if (prog->instances.nr > 0) { 272 for (i = 0; i < prog->instances.nr; i++) 273 zclose(prog->instances.fds[i]); 274 } else if (prog->instances.nr != -1) { 275 pr_warning("Internal error: instances.nr is %d\n", 276 prog->instances.nr); 277 } 278 279 prog->instances.nr = -1; 280 zfree(&prog->instances.fds); 281 } 282 283 static void bpf_program__exit(struct bpf_program *prog) 284 { 285 if (!prog) 286 return; 287 288 if (prog->clear_priv) 289 prog->clear_priv(prog, prog->priv); 290 291 prog->priv = NULL; 292 prog->clear_priv = NULL; 293 294 bpf_program__unload(prog); 295 zfree(&prog->name); 296 zfree(&prog->section_name); 297 zfree(&prog->insns); 298 zfree(&prog->reloc_desc); 299 300 prog->nr_reloc = 0; 301 prog->insns_cnt = 0; 302 prog->idx = -1; 303 } 304 305 static int 306 bpf_program__init(void *data, size_t size, char *section_name, int idx, 307 struct bpf_program *prog) 308 { 309 if (size < sizeof(struct bpf_insn)) { 310 pr_warning("corrupted section '%s'\n", section_name); 311 return -EINVAL; 312 } 313 314 bzero(prog, sizeof(*prog)); 315 316 prog->section_name = strdup(section_name); 317 if (!prog->section_name) { 318 pr_warning("failed to alloc name for prog under section %s\n", 319 section_name); 320 goto errout; 321 } 322 323 prog->insns = malloc(size); 324 if (!prog->insns) { 325 pr_warning("failed to alloc insns for prog under section %s\n", 326 section_name); 327 goto errout; 328 } 329 prog->insns_cnt = size / sizeof(struct bpf_insn); 330 memcpy(prog->insns, data, 331 prog->insns_cnt * sizeof(struct bpf_insn)); 332 prog->idx = idx; 333 prog->instances.fds = NULL; 334 prog->instances.nr = -1; 335 prog->type = BPF_PROG_TYPE_KPROBE; 336 337 return 0; 338 errout: 339 bpf_program__exit(prog); 340 return -ENOMEM; 341 } 342 343 static int 344 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 345 char *section_name, int idx) 346 { 347 struct bpf_program prog, *progs; 348 int nr_progs, err; 349 350 err = bpf_program__init(data, size, section_name, idx, &prog); 351 if (err) 352 return err; 353 354 progs = obj->programs; 355 nr_progs = obj->nr_programs; 356 357 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 358 if (!progs) { 359 /* 360 * In this case the original obj->programs 361 * is still valid, so don't need special treat for 362 * bpf_close_object(). 363 */ 364 pr_warning("failed to alloc a new program under section '%s'\n", 365 section_name); 366 bpf_program__exit(&prog); 367 return -ENOMEM; 368 } 369 370 pr_debug("found program %s\n", prog.section_name); 371 obj->programs = progs; 372 obj->nr_programs = nr_progs + 1; 373 prog.obj = obj; 374 progs[nr_progs] = prog; 375 return 0; 376 } 377 378 static int 379 bpf_object__init_prog_names(struct bpf_object *obj) 380 { 381 Elf_Data *symbols = obj->efile.symbols; 382 struct bpf_program *prog; 383 size_t pi, si; 384 385 for (pi = 0; pi < obj->nr_programs; pi++) { 386 const char *name = NULL; 387 388 prog = &obj->programs[pi]; 389 if (prog->idx == obj->efile.text_shndx) { 390 name = ".text"; 391 goto skip_search; 392 } 393 394 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 395 si++) { 396 GElf_Sym sym; 397 398 if (!gelf_getsym(symbols, si, &sym)) 399 continue; 400 if (sym.st_shndx != prog->idx) 401 continue; 402 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 403 continue; 404 405 name = elf_strptr(obj->efile.elf, 406 obj->efile.strtabidx, 407 sym.st_name); 408 if (!name) { 409 pr_warning("failed to get sym name string for prog %s\n", 410 prog->section_name); 411 return -LIBBPF_ERRNO__LIBELF; 412 } 413 } 414 415 if (!name) { 416 pr_warning("failed to find sym for prog %s\n", 417 prog->section_name); 418 return -EINVAL; 419 } 420 skip_search: 421 prog->name = strdup(name); 422 if (!prog->name) { 423 pr_warning("failed to allocate memory for prog sym %s\n", 424 name); 425 return -ENOMEM; 426 } 427 } 428 429 return 0; 430 } 431 432 static struct bpf_object *bpf_object__new(const char *path, 433 void *obj_buf, 434 size_t obj_buf_sz) 435 { 436 struct bpf_object *obj; 437 438 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 439 if (!obj) { 440 pr_warning("alloc memory failed for %s\n", path); 441 return ERR_PTR(-ENOMEM); 442 } 443 444 strcpy(obj->path, path); 445 obj->efile.fd = -1; 446 447 /* 448 * Caller of this function should also calls 449 * bpf_object__elf_finish() after data collection to return 450 * obj_buf to user. If not, we should duplicate the buffer to 451 * avoid user freeing them before elf finish. 452 */ 453 obj->efile.obj_buf = obj_buf; 454 obj->efile.obj_buf_sz = obj_buf_sz; 455 obj->efile.maps_shndx = -1; 456 457 obj->loaded = false; 458 459 INIT_LIST_HEAD(&obj->list); 460 list_add(&obj->list, &bpf_objects_list); 461 return obj; 462 } 463 464 static void bpf_object__elf_finish(struct bpf_object *obj) 465 { 466 if (!obj_elf_valid(obj)) 467 return; 468 469 if (obj->efile.elf) { 470 elf_end(obj->efile.elf); 471 obj->efile.elf = NULL; 472 } 473 obj->efile.symbols = NULL; 474 475 zfree(&obj->efile.reloc); 476 obj->efile.nr_reloc = 0; 477 zclose(obj->efile.fd); 478 obj->efile.obj_buf = NULL; 479 obj->efile.obj_buf_sz = 0; 480 } 481 482 static int bpf_object__elf_init(struct bpf_object *obj) 483 { 484 int err = 0; 485 GElf_Ehdr *ep; 486 487 if (obj_elf_valid(obj)) { 488 pr_warning("elf init: internal error\n"); 489 return -LIBBPF_ERRNO__LIBELF; 490 } 491 492 if (obj->efile.obj_buf_sz > 0) { 493 /* 494 * obj_buf should have been validated by 495 * bpf_object__open_buffer(). 496 */ 497 obj->efile.elf = elf_memory(obj->efile.obj_buf, 498 obj->efile.obj_buf_sz); 499 } else { 500 obj->efile.fd = open(obj->path, O_RDONLY); 501 if (obj->efile.fd < 0) { 502 pr_warning("failed to open %s: %s\n", obj->path, 503 strerror(errno)); 504 return -errno; 505 } 506 507 obj->efile.elf = elf_begin(obj->efile.fd, 508 LIBBPF_ELF_C_READ_MMAP, 509 NULL); 510 } 511 512 if (!obj->efile.elf) { 513 pr_warning("failed to open %s as ELF file\n", 514 obj->path); 515 err = -LIBBPF_ERRNO__LIBELF; 516 goto errout; 517 } 518 519 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 520 pr_warning("failed to get EHDR from %s\n", 521 obj->path); 522 err = -LIBBPF_ERRNO__FORMAT; 523 goto errout; 524 } 525 ep = &obj->efile.ehdr; 526 527 /* Old LLVM set e_machine to EM_NONE */ 528 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 529 pr_warning("%s is not an eBPF object file\n", 530 obj->path); 531 err = -LIBBPF_ERRNO__FORMAT; 532 goto errout; 533 } 534 535 return 0; 536 errout: 537 bpf_object__elf_finish(obj); 538 return err; 539 } 540 541 static int 542 bpf_object__check_endianness(struct bpf_object *obj) 543 { 544 static unsigned int const endian = 1; 545 546 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 547 case ELFDATA2LSB: 548 /* We are big endian, BPF obj is little endian. */ 549 if (*(unsigned char const *)&endian != 1) 550 goto mismatch; 551 break; 552 553 case ELFDATA2MSB: 554 /* We are little endian, BPF obj is big endian. */ 555 if (*(unsigned char const *)&endian != 0) 556 goto mismatch; 557 break; 558 default: 559 return -LIBBPF_ERRNO__ENDIAN; 560 } 561 562 return 0; 563 564 mismatch: 565 pr_warning("Error: endianness mismatch.\n"); 566 return -LIBBPF_ERRNO__ENDIAN; 567 } 568 569 static int 570 bpf_object__init_license(struct bpf_object *obj, 571 void *data, size_t size) 572 { 573 memcpy(obj->license, data, 574 min(size, sizeof(obj->license) - 1)); 575 pr_debug("license of %s is %s\n", obj->path, obj->license); 576 return 0; 577 } 578 579 static int 580 bpf_object__init_kversion(struct bpf_object *obj, 581 void *data, size_t size) 582 { 583 u32 kver; 584 585 if (size != sizeof(kver)) { 586 pr_warning("invalid kver section in %s\n", obj->path); 587 return -LIBBPF_ERRNO__FORMAT; 588 } 589 memcpy(&kver, data, sizeof(kver)); 590 obj->kern_version = kver; 591 pr_debug("kernel version of %s is %x\n", obj->path, 592 obj->kern_version); 593 return 0; 594 } 595 596 static int compare_bpf_map(const void *_a, const void *_b) 597 { 598 const struct bpf_map *a = _a; 599 const struct bpf_map *b = _b; 600 601 return a->offset - b->offset; 602 } 603 604 static int 605 bpf_object__init_maps(struct bpf_object *obj) 606 { 607 int i, map_idx, map_def_sz, nr_maps = 0; 608 Elf_Scn *scn; 609 Elf_Data *data; 610 Elf_Data *symbols = obj->efile.symbols; 611 612 if (obj->efile.maps_shndx < 0) 613 return -EINVAL; 614 if (!symbols) 615 return -EINVAL; 616 617 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 618 if (scn) 619 data = elf_getdata(scn, NULL); 620 if (!scn || !data) { 621 pr_warning("failed to get Elf_Data from map section %d\n", 622 obj->efile.maps_shndx); 623 return -EINVAL; 624 } 625 626 /* 627 * Count number of maps. Each map has a name. 628 * Array of maps is not supported: only the first element is 629 * considered. 630 * 631 * TODO: Detect array of map and report error. 632 */ 633 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 634 GElf_Sym sym; 635 636 if (!gelf_getsym(symbols, i, &sym)) 637 continue; 638 if (sym.st_shndx != obj->efile.maps_shndx) 639 continue; 640 nr_maps++; 641 } 642 643 /* Alloc obj->maps and fill nr_maps. */ 644 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 645 nr_maps, data->d_size); 646 647 if (!nr_maps) 648 return 0; 649 650 /* Assume equally sized map definitions */ 651 map_def_sz = data->d_size / nr_maps; 652 if (!data->d_size || (data->d_size % nr_maps) != 0) { 653 pr_warning("unable to determine map definition size " 654 "section %s, %d maps in %zd bytes\n", 655 obj->path, nr_maps, data->d_size); 656 return -EINVAL; 657 } 658 659 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 660 if (!obj->maps) { 661 pr_warning("alloc maps for object failed\n"); 662 return -ENOMEM; 663 } 664 obj->nr_maps = nr_maps; 665 666 /* 667 * fill all fd with -1 so won't close incorrect 668 * fd (fd=0 is stdin) when failure (zclose won't close 669 * negative fd)). 670 */ 671 for (i = 0; i < nr_maps; i++) 672 obj->maps[i].fd = -1; 673 674 /* 675 * Fill obj->maps using data in "maps" section. 676 */ 677 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 678 GElf_Sym sym; 679 const char *map_name; 680 struct bpf_map_def *def; 681 682 if (!gelf_getsym(symbols, i, &sym)) 683 continue; 684 if (sym.st_shndx != obj->efile.maps_shndx) 685 continue; 686 687 map_name = elf_strptr(obj->efile.elf, 688 obj->efile.strtabidx, 689 sym.st_name); 690 obj->maps[map_idx].offset = sym.st_value; 691 if (sym.st_value + map_def_sz > data->d_size) { 692 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 693 obj->path, map_name); 694 return -EINVAL; 695 } 696 697 obj->maps[map_idx].name = strdup(map_name); 698 if (!obj->maps[map_idx].name) { 699 pr_warning("failed to alloc map name\n"); 700 return -ENOMEM; 701 } 702 pr_debug("map %d is \"%s\"\n", map_idx, 703 obj->maps[map_idx].name); 704 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 705 /* 706 * If the definition of the map in the object file fits in 707 * bpf_map_def, copy it. Any extra fields in our version 708 * of bpf_map_def will default to zero as a result of the 709 * calloc above. 710 */ 711 if (map_def_sz <= sizeof(struct bpf_map_def)) { 712 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 713 } else { 714 /* 715 * Here the map structure being read is bigger than what 716 * we expect, truncate if the excess bits are all zero. 717 * If they are not zero, reject this map as 718 * incompatible. 719 */ 720 char *b; 721 for (b = ((char *)def) + sizeof(struct bpf_map_def); 722 b < ((char *)def) + map_def_sz; b++) { 723 if (*b != 0) { 724 pr_warning("maps section in %s: \"%s\" " 725 "has unrecognized, non-zero " 726 "options\n", 727 obj->path, map_name); 728 return -EINVAL; 729 } 730 } 731 memcpy(&obj->maps[map_idx].def, def, 732 sizeof(struct bpf_map_def)); 733 } 734 map_idx++; 735 } 736 737 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); 738 return 0; 739 } 740 741 static int bpf_object__elf_collect(struct bpf_object *obj) 742 { 743 Elf *elf = obj->efile.elf; 744 GElf_Ehdr *ep = &obj->efile.ehdr; 745 Elf_Scn *scn = NULL; 746 int idx = 0, err = 0; 747 748 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 749 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 750 pr_warning("failed to get e_shstrndx from %s\n", 751 obj->path); 752 return -LIBBPF_ERRNO__FORMAT; 753 } 754 755 while ((scn = elf_nextscn(elf, scn)) != NULL) { 756 char *name; 757 GElf_Shdr sh; 758 Elf_Data *data; 759 760 idx++; 761 if (gelf_getshdr(scn, &sh) != &sh) { 762 pr_warning("failed to get section header from %s\n", 763 obj->path); 764 err = -LIBBPF_ERRNO__FORMAT; 765 goto out; 766 } 767 768 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 769 if (!name) { 770 pr_warning("failed to get section name from %s\n", 771 obj->path); 772 err = -LIBBPF_ERRNO__FORMAT; 773 goto out; 774 } 775 776 data = elf_getdata(scn, 0); 777 if (!data) { 778 pr_warning("failed to get section data from %s(%s)\n", 779 name, obj->path); 780 err = -LIBBPF_ERRNO__FORMAT; 781 goto out; 782 } 783 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", 784 name, (unsigned long)data->d_size, 785 (int)sh.sh_link, (unsigned long)sh.sh_flags, 786 (int)sh.sh_type); 787 788 if (strcmp(name, "license") == 0) 789 err = bpf_object__init_license(obj, 790 data->d_buf, 791 data->d_size); 792 else if (strcmp(name, "version") == 0) 793 err = bpf_object__init_kversion(obj, 794 data->d_buf, 795 data->d_size); 796 else if (strcmp(name, "maps") == 0) 797 obj->efile.maps_shndx = idx; 798 else if (sh.sh_type == SHT_SYMTAB) { 799 if (obj->efile.symbols) { 800 pr_warning("bpf: multiple SYMTAB in %s\n", 801 obj->path); 802 err = -LIBBPF_ERRNO__FORMAT; 803 } else { 804 obj->efile.symbols = data; 805 obj->efile.strtabidx = sh.sh_link; 806 } 807 } else if ((sh.sh_type == SHT_PROGBITS) && 808 (sh.sh_flags & SHF_EXECINSTR) && 809 (data->d_size > 0)) { 810 if (strcmp(name, ".text") == 0) 811 obj->efile.text_shndx = idx; 812 err = bpf_object__add_program(obj, data->d_buf, 813 data->d_size, name, idx); 814 if (err) { 815 char errmsg[STRERR_BUFSIZE]; 816 817 strerror_r(-err, errmsg, sizeof(errmsg)); 818 pr_warning("failed to alloc program %s (%s): %s", 819 name, obj->path, errmsg); 820 } 821 } else if (sh.sh_type == SHT_REL) { 822 void *reloc = obj->efile.reloc; 823 int nr_reloc = obj->efile.nr_reloc + 1; 824 825 reloc = realloc(reloc, 826 sizeof(*obj->efile.reloc) * nr_reloc); 827 if (!reloc) { 828 pr_warning("realloc failed\n"); 829 err = -ENOMEM; 830 } else { 831 int n = nr_reloc - 1; 832 833 obj->efile.reloc = reloc; 834 obj->efile.nr_reloc = nr_reloc; 835 836 obj->efile.reloc[n].shdr = sh; 837 obj->efile.reloc[n].data = data; 838 } 839 } 840 if (err) 841 goto out; 842 } 843 844 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 845 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 846 return LIBBPF_ERRNO__FORMAT; 847 } 848 if (obj->efile.maps_shndx >= 0) { 849 err = bpf_object__init_maps(obj); 850 if (err) 851 goto out; 852 } 853 err = bpf_object__init_prog_names(obj); 854 out: 855 return err; 856 } 857 858 static struct bpf_program * 859 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 860 { 861 struct bpf_program *prog; 862 size_t i; 863 864 for (i = 0; i < obj->nr_programs; i++) { 865 prog = &obj->programs[i]; 866 if (prog->idx == idx) 867 return prog; 868 } 869 return NULL; 870 } 871 872 static int 873 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 874 Elf_Data *data, struct bpf_object *obj) 875 { 876 Elf_Data *symbols = obj->efile.symbols; 877 int text_shndx = obj->efile.text_shndx; 878 int maps_shndx = obj->efile.maps_shndx; 879 struct bpf_map *maps = obj->maps; 880 size_t nr_maps = obj->nr_maps; 881 int i, nrels; 882 883 pr_debug("collecting relocating info for: '%s'\n", 884 prog->section_name); 885 nrels = shdr->sh_size / shdr->sh_entsize; 886 887 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 888 if (!prog->reloc_desc) { 889 pr_warning("failed to alloc memory in relocation\n"); 890 return -ENOMEM; 891 } 892 prog->nr_reloc = nrels; 893 894 for (i = 0; i < nrels; i++) { 895 GElf_Sym sym; 896 GElf_Rel rel; 897 unsigned int insn_idx; 898 struct bpf_insn *insns = prog->insns; 899 size_t map_idx; 900 901 if (!gelf_getrel(data, i, &rel)) { 902 pr_warning("relocation: failed to get %d reloc\n", i); 903 return -LIBBPF_ERRNO__FORMAT; 904 } 905 906 if (!gelf_getsym(symbols, 907 GELF_R_SYM(rel.r_info), 908 &sym)) { 909 pr_warning("relocation: symbol %"PRIx64" not found\n", 910 GELF_R_SYM(rel.r_info)); 911 return -LIBBPF_ERRNO__FORMAT; 912 } 913 pr_debug("relo for %lld value %lld name %d\n", 914 (long long) (rel.r_info >> 32), 915 (long long) sym.st_value, sym.st_name); 916 917 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { 918 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 919 prog->section_name, sym.st_shndx); 920 return -LIBBPF_ERRNO__RELOC; 921 } 922 923 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 924 pr_debug("relocation: insn_idx=%u\n", insn_idx); 925 926 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 927 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 928 pr_warning("incorrect bpf_call opcode\n"); 929 return -LIBBPF_ERRNO__RELOC; 930 } 931 prog->reloc_desc[i].type = RELO_CALL; 932 prog->reloc_desc[i].insn_idx = insn_idx; 933 prog->reloc_desc[i].text_off = sym.st_value; 934 continue; 935 } 936 937 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 938 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 939 insn_idx, insns[insn_idx].code); 940 return -LIBBPF_ERRNO__RELOC; 941 } 942 943 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ 944 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 945 if (maps[map_idx].offset == sym.st_value) { 946 pr_debug("relocation: find map %zd (%s) for insn %u\n", 947 map_idx, maps[map_idx].name, insn_idx); 948 break; 949 } 950 } 951 952 if (map_idx >= nr_maps) { 953 pr_warning("bpf relocation: map_idx %d large than %d\n", 954 (int)map_idx, (int)nr_maps - 1); 955 return -LIBBPF_ERRNO__RELOC; 956 } 957 958 prog->reloc_desc[i].type = RELO_LD64; 959 prog->reloc_desc[i].insn_idx = insn_idx; 960 prog->reloc_desc[i].map_idx = map_idx; 961 } 962 return 0; 963 } 964 965 static int 966 bpf_object__create_maps(struct bpf_object *obj) 967 { 968 unsigned int i; 969 970 for (i = 0; i < obj->nr_maps; i++) { 971 struct bpf_map_def *def = &obj->maps[i].def; 972 int *pfd = &obj->maps[i].fd; 973 974 *pfd = bpf_create_map_name(def->type, 975 obj->maps[i].name, 976 def->key_size, 977 def->value_size, 978 def->max_entries, 979 def->map_flags); 980 if (*pfd < 0) { 981 size_t j; 982 int err = *pfd; 983 984 pr_warning("failed to create map (name: '%s'): %s\n", 985 obj->maps[i].name, 986 strerror(errno)); 987 for (j = 0; j < i; j++) 988 zclose(obj->maps[j].fd); 989 return err; 990 } 991 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); 992 } 993 994 return 0; 995 } 996 997 static int 998 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 999 struct reloc_desc *relo) 1000 { 1001 struct bpf_insn *insn, *new_insn; 1002 struct bpf_program *text; 1003 size_t new_cnt; 1004 1005 if (relo->type != RELO_CALL) 1006 return -LIBBPF_ERRNO__RELOC; 1007 1008 if (prog->idx == obj->efile.text_shndx) { 1009 pr_warning("relo in .text insn %d into off %d\n", 1010 relo->insn_idx, relo->text_off); 1011 return -LIBBPF_ERRNO__RELOC; 1012 } 1013 1014 if (prog->main_prog_cnt == 0) { 1015 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1016 if (!text) { 1017 pr_warning("no .text section found yet relo into text exist\n"); 1018 return -LIBBPF_ERRNO__RELOC; 1019 } 1020 new_cnt = prog->insns_cnt + text->insns_cnt; 1021 new_insn = realloc(prog->insns, new_cnt * sizeof(*insn)); 1022 if (!new_insn) { 1023 pr_warning("oom in prog realloc\n"); 1024 return -ENOMEM; 1025 } 1026 memcpy(new_insn + prog->insns_cnt, text->insns, 1027 text->insns_cnt * sizeof(*insn)); 1028 prog->insns = new_insn; 1029 prog->main_prog_cnt = prog->insns_cnt; 1030 prog->insns_cnt = new_cnt; 1031 } 1032 insn = &prog->insns[relo->insn_idx]; 1033 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1034 pr_debug("added %zd insn from %s to prog %s\n", 1035 text->insns_cnt, text->section_name, prog->section_name); 1036 return 0; 1037 } 1038 1039 static int 1040 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1041 { 1042 int i, err; 1043 1044 if (!prog || !prog->reloc_desc) 1045 return 0; 1046 1047 for (i = 0; i < prog->nr_reloc; i++) { 1048 if (prog->reloc_desc[i].type == RELO_LD64) { 1049 struct bpf_insn *insns = prog->insns; 1050 int insn_idx, map_idx; 1051 1052 insn_idx = prog->reloc_desc[i].insn_idx; 1053 map_idx = prog->reloc_desc[i].map_idx; 1054 1055 if (insn_idx >= (int)prog->insns_cnt) { 1056 pr_warning("relocation out of range: '%s'\n", 1057 prog->section_name); 1058 return -LIBBPF_ERRNO__RELOC; 1059 } 1060 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1061 insns[insn_idx].imm = obj->maps[map_idx].fd; 1062 } else { 1063 err = bpf_program__reloc_text(prog, obj, 1064 &prog->reloc_desc[i]); 1065 if (err) 1066 return err; 1067 } 1068 } 1069 1070 zfree(&prog->reloc_desc); 1071 prog->nr_reloc = 0; 1072 return 0; 1073 } 1074 1075 1076 static int 1077 bpf_object__relocate(struct bpf_object *obj) 1078 { 1079 struct bpf_program *prog; 1080 size_t i; 1081 int err; 1082 1083 for (i = 0; i < obj->nr_programs; i++) { 1084 prog = &obj->programs[i]; 1085 1086 err = bpf_program__relocate(prog, obj); 1087 if (err) { 1088 pr_warning("failed to relocate '%s'\n", 1089 prog->section_name); 1090 return err; 1091 } 1092 } 1093 return 0; 1094 } 1095 1096 static int bpf_object__collect_reloc(struct bpf_object *obj) 1097 { 1098 int i, err; 1099 1100 if (!obj_elf_valid(obj)) { 1101 pr_warning("Internal error: elf object is closed\n"); 1102 return -LIBBPF_ERRNO__INTERNAL; 1103 } 1104 1105 for (i = 0; i < obj->efile.nr_reloc; i++) { 1106 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1107 Elf_Data *data = obj->efile.reloc[i].data; 1108 int idx = shdr->sh_info; 1109 struct bpf_program *prog; 1110 1111 if (shdr->sh_type != SHT_REL) { 1112 pr_warning("internal error at %d\n", __LINE__); 1113 return -LIBBPF_ERRNO__INTERNAL; 1114 } 1115 1116 prog = bpf_object__find_prog_by_idx(obj, idx); 1117 if (!prog) { 1118 pr_warning("relocation failed: no %d section\n", 1119 idx); 1120 return -LIBBPF_ERRNO__RELOC; 1121 } 1122 1123 err = bpf_program__collect_reloc(prog, 1124 shdr, data, 1125 obj); 1126 if (err) 1127 return err; 1128 } 1129 return 0; 1130 } 1131 1132 static int 1133 load_program(enum bpf_prog_type type, const char *name, struct bpf_insn *insns, 1134 int insns_cnt, char *license, u32 kern_version, int *pfd) 1135 { 1136 int ret; 1137 char *log_buf; 1138 1139 if (!insns || !insns_cnt) 1140 return -EINVAL; 1141 1142 log_buf = malloc(BPF_LOG_BUF_SIZE); 1143 if (!log_buf) 1144 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1145 1146 ret = bpf_load_program_name(type, name, insns, insns_cnt, license, 1147 kern_version, log_buf, BPF_LOG_BUF_SIZE); 1148 1149 if (ret >= 0) { 1150 *pfd = ret; 1151 ret = 0; 1152 goto out; 1153 } 1154 1155 ret = -LIBBPF_ERRNO__LOAD; 1156 pr_warning("load bpf program failed: %s\n", strerror(errno)); 1157 1158 if (log_buf && log_buf[0] != '\0') { 1159 ret = -LIBBPF_ERRNO__VERIFY; 1160 pr_warning("-- BEGIN DUMP LOG ---\n"); 1161 pr_warning("\n%s\n", log_buf); 1162 pr_warning("-- END LOG --\n"); 1163 } else if (insns_cnt >= BPF_MAXINSNS) { 1164 pr_warning("Program too large (%d insns), at most %d insns\n", 1165 insns_cnt, BPF_MAXINSNS); 1166 ret = -LIBBPF_ERRNO__PROG2BIG; 1167 } else { 1168 /* Wrong program type? */ 1169 if (type != BPF_PROG_TYPE_KPROBE) { 1170 int fd; 1171 1172 fd = bpf_load_program_name(BPF_PROG_TYPE_KPROBE, name, 1173 insns, insns_cnt, license, 1174 kern_version, NULL, 0); 1175 if (fd >= 0) { 1176 close(fd); 1177 ret = -LIBBPF_ERRNO__PROGTYPE; 1178 goto out; 1179 } 1180 } 1181 1182 if (log_buf) 1183 ret = -LIBBPF_ERRNO__KVER; 1184 } 1185 1186 out: 1187 free(log_buf); 1188 return ret; 1189 } 1190 1191 static int 1192 bpf_program__load(struct bpf_program *prog, 1193 char *license, u32 kern_version) 1194 { 1195 int err = 0, fd, i; 1196 1197 if (prog->instances.nr < 0 || !prog->instances.fds) { 1198 if (prog->preprocessor) { 1199 pr_warning("Internal error: can't load program '%s'\n", 1200 prog->section_name); 1201 return -LIBBPF_ERRNO__INTERNAL; 1202 } 1203 1204 prog->instances.fds = malloc(sizeof(int)); 1205 if (!prog->instances.fds) { 1206 pr_warning("Not enough memory for BPF fds\n"); 1207 return -ENOMEM; 1208 } 1209 prog->instances.nr = 1; 1210 prog->instances.fds[0] = -1; 1211 } 1212 1213 if (!prog->preprocessor) { 1214 if (prog->instances.nr != 1) { 1215 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 1216 prog->section_name, prog->instances.nr); 1217 } 1218 err = load_program(prog->type, prog->name, prog->insns, 1219 prog->insns_cnt, license, kern_version, &fd); 1220 if (!err) 1221 prog->instances.fds[0] = fd; 1222 goto out; 1223 } 1224 1225 for (i = 0; i < prog->instances.nr; i++) { 1226 struct bpf_prog_prep_result result; 1227 bpf_program_prep_t preprocessor = prog->preprocessor; 1228 1229 bzero(&result, sizeof(result)); 1230 err = preprocessor(prog, i, prog->insns, 1231 prog->insns_cnt, &result); 1232 if (err) { 1233 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 1234 i, prog->section_name); 1235 goto out; 1236 } 1237 1238 if (!result.new_insn_ptr || !result.new_insn_cnt) { 1239 pr_debug("Skip loading the %dth instance of program '%s'\n", 1240 i, prog->section_name); 1241 prog->instances.fds[i] = -1; 1242 if (result.pfd) 1243 *result.pfd = -1; 1244 continue; 1245 } 1246 1247 err = load_program(prog->type, prog->name, 1248 result.new_insn_ptr, 1249 result.new_insn_cnt, 1250 license, kern_version, &fd); 1251 1252 if (err) { 1253 pr_warning("Loading the %dth instance of program '%s' failed\n", 1254 i, prog->section_name); 1255 goto out; 1256 } 1257 1258 if (result.pfd) 1259 *result.pfd = fd; 1260 prog->instances.fds[i] = fd; 1261 } 1262 out: 1263 if (err) 1264 pr_warning("failed to load program '%s'\n", 1265 prog->section_name); 1266 zfree(&prog->insns); 1267 prog->insns_cnt = 0; 1268 return err; 1269 } 1270 1271 static int 1272 bpf_object__load_progs(struct bpf_object *obj) 1273 { 1274 size_t i; 1275 int err; 1276 1277 for (i = 0; i < obj->nr_programs; i++) { 1278 if (obj->programs[i].idx == obj->efile.text_shndx) 1279 continue; 1280 err = bpf_program__load(&obj->programs[i], 1281 obj->license, 1282 obj->kern_version); 1283 if (err) 1284 return err; 1285 } 1286 return 0; 1287 } 1288 1289 static int bpf_object__validate(struct bpf_object *obj) 1290 { 1291 if (obj->kern_version == 0) { 1292 pr_warning("%s doesn't provide kernel version\n", 1293 obj->path); 1294 return -LIBBPF_ERRNO__KVERSION; 1295 } 1296 return 0; 1297 } 1298 1299 static struct bpf_object * 1300 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 1301 { 1302 struct bpf_object *obj; 1303 int err; 1304 1305 if (elf_version(EV_CURRENT) == EV_NONE) { 1306 pr_warning("failed to init libelf for %s\n", path); 1307 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1308 } 1309 1310 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1311 if (IS_ERR(obj)) 1312 return obj; 1313 1314 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1315 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1316 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 1317 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1318 CHECK_ERR(bpf_object__validate(obj), err, out); 1319 1320 bpf_object__elf_finish(obj); 1321 return obj; 1322 out: 1323 bpf_object__close(obj); 1324 return ERR_PTR(err); 1325 } 1326 1327 struct bpf_object *bpf_object__open(const char *path) 1328 { 1329 /* param validation */ 1330 if (!path) 1331 return NULL; 1332 1333 pr_debug("loading %s\n", path); 1334 1335 return __bpf_object__open(path, NULL, 0); 1336 } 1337 1338 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1339 size_t obj_buf_sz, 1340 const char *name) 1341 { 1342 char tmp_name[64]; 1343 1344 /* param validation */ 1345 if (!obj_buf || obj_buf_sz <= 0) 1346 return NULL; 1347 1348 if (!name) { 1349 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1350 (unsigned long)obj_buf, 1351 (unsigned long)obj_buf_sz); 1352 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1353 name = tmp_name; 1354 } 1355 pr_debug("loading object '%s' from buffer\n", 1356 name); 1357 1358 return __bpf_object__open(name, obj_buf, obj_buf_sz); 1359 } 1360 1361 int bpf_object__unload(struct bpf_object *obj) 1362 { 1363 size_t i; 1364 1365 if (!obj) 1366 return -EINVAL; 1367 1368 for (i = 0; i < obj->nr_maps; i++) 1369 zclose(obj->maps[i].fd); 1370 1371 for (i = 0; i < obj->nr_programs; i++) 1372 bpf_program__unload(&obj->programs[i]); 1373 1374 return 0; 1375 } 1376 1377 int bpf_object__load(struct bpf_object *obj) 1378 { 1379 int err; 1380 1381 if (!obj) 1382 return -EINVAL; 1383 1384 if (obj->loaded) { 1385 pr_warning("object should not be loaded twice\n"); 1386 return -EINVAL; 1387 } 1388 1389 obj->loaded = true; 1390 1391 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1392 CHECK_ERR(bpf_object__relocate(obj), err, out); 1393 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1394 1395 return 0; 1396 out: 1397 bpf_object__unload(obj); 1398 pr_warning("failed to load object '%s'\n", obj->path); 1399 return err; 1400 } 1401 1402 static int check_path(const char *path) 1403 { 1404 struct statfs st_fs; 1405 char *dname, *dir; 1406 int err = 0; 1407 1408 if (path == NULL) 1409 return -EINVAL; 1410 1411 dname = strdup(path); 1412 if (dname == NULL) 1413 return -ENOMEM; 1414 1415 dir = dirname(dname); 1416 if (statfs(dir, &st_fs)) { 1417 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno)); 1418 err = -errno; 1419 } 1420 free(dname); 1421 1422 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 1423 pr_warning("specified path %s is not on BPF FS\n", path); 1424 err = -EINVAL; 1425 } 1426 1427 return err; 1428 } 1429 1430 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 1431 int instance) 1432 { 1433 int err; 1434 1435 err = check_path(path); 1436 if (err) 1437 return err; 1438 1439 if (prog == NULL) { 1440 pr_warning("invalid program pointer\n"); 1441 return -EINVAL; 1442 } 1443 1444 if (instance < 0 || instance >= prog->instances.nr) { 1445 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 1446 instance, prog->section_name, prog->instances.nr); 1447 return -EINVAL; 1448 } 1449 1450 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 1451 pr_warning("failed to pin program: %s\n", strerror(errno)); 1452 return -errno; 1453 } 1454 pr_debug("pinned program '%s'\n", path); 1455 1456 return 0; 1457 } 1458 1459 static int make_dir(const char *path) 1460 { 1461 int err = 0; 1462 1463 if (mkdir(path, 0700) && errno != EEXIST) 1464 err = -errno; 1465 1466 if (err) 1467 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err)); 1468 return err; 1469 } 1470 1471 int bpf_program__pin(struct bpf_program *prog, const char *path) 1472 { 1473 int i, err; 1474 1475 err = check_path(path); 1476 if (err) 1477 return err; 1478 1479 if (prog == NULL) { 1480 pr_warning("invalid program pointer\n"); 1481 return -EINVAL; 1482 } 1483 1484 if (prog->instances.nr <= 0) { 1485 pr_warning("no instances of prog %s to pin\n", 1486 prog->section_name); 1487 return -EINVAL; 1488 } 1489 1490 err = make_dir(path); 1491 if (err) 1492 return err; 1493 1494 for (i = 0; i < prog->instances.nr; i++) { 1495 char buf[PATH_MAX]; 1496 int len; 1497 1498 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 1499 if (len < 0) 1500 return -EINVAL; 1501 else if (len >= PATH_MAX) 1502 return -ENAMETOOLONG; 1503 1504 err = bpf_program__pin_instance(prog, buf, i); 1505 if (err) 1506 return err; 1507 } 1508 1509 return 0; 1510 } 1511 1512 int bpf_map__pin(struct bpf_map *map, const char *path) 1513 { 1514 int err; 1515 1516 err = check_path(path); 1517 if (err) 1518 return err; 1519 1520 if (map == NULL) { 1521 pr_warning("invalid map pointer\n"); 1522 return -EINVAL; 1523 } 1524 1525 if (bpf_obj_pin(map->fd, path)) { 1526 pr_warning("failed to pin map: %s\n", strerror(errno)); 1527 return -errno; 1528 } 1529 1530 pr_debug("pinned map '%s'\n", path); 1531 return 0; 1532 } 1533 1534 int bpf_object__pin(struct bpf_object *obj, const char *path) 1535 { 1536 struct bpf_program *prog; 1537 struct bpf_map *map; 1538 int err; 1539 1540 if (!obj) 1541 return -ENOENT; 1542 1543 if (!obj->loaded) { 1544 pr_warning("object not yet loaded; load it first\n"); 1545 return -ENOENT; 1546 } 1547 1548 err = make_dir(path); 1549 if (err) 1550 return err; 1551 1552 bpf_map__for_each(map, obj) { 1553 char buf[PATH_MAX]; 1554 int len; 1555 1556 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1557 bpf_map__name(map)); 1558 if (len < 0) 1559 return -EINVAL; 1560 else if (len >= PATH_MAX) 1561 return -ENAMETOOLONG; 1562 1563 err = bpf_map__pin(map, buf); 1564 if (err) 1565 return err; 1566 } 1567 1568 bpf_object__for_each_program(prog, obj) { 1569 char buf[PATH_MAX]; 1570 int len; 1571 1572 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1573 prog->section_name); 1574 if (len < 0) 1575 return -EINVAL; 1576 else if (len >= PATH_MAX) 1577 return -ENAMETOOLONG; 1578 1579 err = bpf_program__pin(prog, buf); 1580 if (err) 1581 return err; 1582 } 1583 1584 return 0; 1585 } 1586 1587 void bpf_object__close(struct bpf_object *obj) 1588 { 1589 size_t i; 1590 1591 if (!obj) 1592 return; 1593 1594 if (obj->clear_priv) 1595 obj->clear_priv(obj, obj->priv); 1596 1597 bpf_object__elf_finish(obj); 1598 bpf_object__unload(obj); 1599 1600 for (i = 0; i < obj->nr_maps; i++) { 1601 zfree(&obj->maps[i].name); 1602 if (obj->maps[i].clear_priv) 1603 obj->maps[i].clear_priv(&obj->maps[i], 1604 obj->maps[i].priv); 1605 obj->maps[i].priv = NULL; 1606 obj->maps[i].clear_priv = NULL; 1607 } 1608 zfree(&obj->maps); 1609 obj->nr_maps = 0; 1610 1611 if (obj->programs && obj->nr_programs) { 1612 for (i = 0; i < obj->nr_programs; i++) 1613 bpf_program__exit(&obj->programs[i]); 1614 } 1615 zfree(&obj->programs); 1616 1617 list_del(&obj->list); 1618 free(obj); 1619 } 1620 1621 struct bpf_object * 1622 bpf_object__next(struct bpf_object *prev) 1623 { 1624 struct bpf_object *next; 1625 1626 if (!prev) 1627 next = list_first_entry(&bpf_objects_list, 1628 struct bpf_object, 1629 list); 1630 else 1631 next = list_next_entry(prev, list); 1632 1633 /* Empty list is noticed here so don't need checking on entry. */ 1634 if (&next->list == &bpf_objects_list) 1635 return NULL; 1636 1637 return next; 1638 } 1639 1640 const char *bpf_object__name(struct bpf_object *obj) 1641 { 1642 return obj ? obj->path : ERR_PTR(-EINVAL); 1643 } 1644 1645 unsigned int bpf_object__kversion(struct bpf_object *obj) 1646 { 1647 return obj ? obj->kern_version : 0; 1648 } 1649 1650 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 1651 bpf_object_clear_priv_t clear_priv) 1652 { 1653 if (obj->priv && obj->clear_priv) 1654 obj->clear_priv(obj, obj->priv); 1655 1656 obj->priv = priv; 1657 obj->clear_priv = clear_priv; 1658 return 0; 1659 } 1660 1661 void *bpf_object__priv(struct bpf_object *obj) 1662 { 1663 return obj ? obj->priv : ERR_PTR(-EINVAL); 1664 } 1665 1666 struct bpf_program * 1667 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1668 { 1669 size_t idx; 1670 1671 if (!obj->programs) 1672 return NULL; 1673 /* First handler */ 1674 if (prev == NULL) 1675 return &obj->programs[0]; 1676 1677 if (prev->obj != obj) { 1678 pr_warning("error: program handler doesn't match object\n"); 1679 return NULL; 1680 } 1681 1682 idx = (prev - obj->programs) + 1; 1683 if (idx >= obj->nr_programs) 1684 return NULL; 1685 return &obj->programs[idx]; 1686 } 1687 1688 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 1689 bpf_program_clear_priv_t clear_priv) 1690 { 1691 if (prog->priv && prog->clear_priv) 1692 prog->clear_priv(prog, prog->priv); 1693 1694 prog->priv = priv; 1695 prog->clear_priv = clear_priv; 1696 return 0; 1697 } 1698 1699 void *bpf_program__priv(struct bpf_program *prog) 1700 { 1701 return prog ? prog->priv : ERR_PTR(-EINVAL); 1702 } 1703 1704 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1705 { 1706 const char *title; 1707 1708 title = prog->section_name; 1709 if (needs_copy) { 1710 title = strdup(title); 1711 if (!title) { 1712 pr_warning("failed to strdup program title\n"); 1713 return ERR_PTR(-ENOMEM); 1714 } 1715 } 1716 1717 return title; 1718 } 1719 1720 int bpf_program__fd(struct bpf_program *prog) 1721 { 1722 return bpf_program__nth_fd(prog, 0); 1723 } 1724 1725 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 1726 bpf_program_prep_t prep) 1727 { 1728 int *instances_fds; 1729 1730 if (nr_instances <= 0 || !prep) 1731 return -EINVAL; 1732 1733 if (prog->instances.nr > 0 || prog->instances.fds) { 1734 pr_warning("Can't set pre-processor after loading\n"); 1735 return -EINVAL; 1736 } 1737 1738 instances_fds = malloc(sizeof(int) * nr_instances); 1739 if (!instances_fds) { 1740 pr_warning("alloc memory failed for fds\n"); 1741 return -ENOMEM; 1742 } 1743 1744 /* fill all fd with -1 */ 1745 memset(instances_fds, -1, sizeof(int) * nr_instances); 1746 1747 prog->instances.nr = nr_instances; 1748 prog->instances.fds = instances_fds; 1749 prog->preprocessor = prep; 1750 return 0; 1751 } 1752 1753 int bpf_program__nth_fd(struct bpf_program *prog, int n) 1754 { 1755 int fd; 1756 1757 if (n >= prog->instances.nr || n < 0) { 1758 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 1759 n, prog->section_name, prog->instances.nr); 1760 return -EINVAL; 1761 } 1762 1763 fd = prog->instances.fds[n]; 1764 if (fd < 0) { 1765 pr_warning("%dth instance of program '%s' is invalid\n", 1766 n, prog->section_name); 1767 return -ENOENT; 1768 } 1769 1770 return fd; 1771 } 1772 1773 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 1774 { 1775 prog->type = type; 1776 } 1777 1778 static bool bpf_program__is_type(struct bpf_program *prog, 1779 enum bpf_prog_type type) 1780 { 1781 return prog ? (prog->type == type) : false; 1782 } 1783 1784 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 1785 int bpf_program__set_##NAME(struct bpf_program *prog) \ 1786 { \ 1787 if (!prog) \ 1788 return -EINVAL; \ 1789 bpf_program__set_type(prog, TYPE); \ 1790 return 0; \ 1791 } \ 1792 \ 1793 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 1794 { \ 1795 return bpf_program__is_type(prog, TYPE); \ 1796 } \ 1797 1798 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 1799 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 1800 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 1801 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 1802 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 1803 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 1804 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 1805 1806 #define BPF_PROG_SEC(string, type) { string, sizeof(string), type } 1807 static const struct { 1808 const char *sec; 1809 size_t len; 1810 enum bpf_prog_type prog_type; 1811 } section_names[] = { 1812 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 1813 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 1814 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 1815 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 1816 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 1817 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 1818 BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 1819 BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK), 1820 BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE), 1821 BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS), 1822 BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB), 1823 }; 1824 #undef BPF_PROG_SEC 1825 1826 static enum bpf_prog_type bpf_program__guess_type(struct bpf_program *prog) 1827 { 1828 int i; 1829 1830 if (!prog->section_name) 1831 goto err; 1832 1833 for (i = 0; i < ARRAY_SIZE(section_names); i++) 1834 if (strncmp(prog->section_name, section_names[i].sec, 1835 section_names[i].len) == 0) 1836 return section_names[i].prog_type; 1837 1838 err: 1839 pr_warning("failed to guess program type based on section name %s\n", 1840 prog->section_name); 1841 1842 return BPF_PROG_TYPE_UNSPEC; 1843 } 1844 1845 int bpf_map__fd(struct bpf_map *map) 1846 { 1847 return map ? map->fd : -EINVAL; 1848 } 1849 1850 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 1851 { 1852 return map ? &map->def : ERR_PTR(-EINVAL); 1853 } 1854 1855 const char *bpf_map__name(struct bpf_map *map) 1856 { 1857 return map ? map->name : NULL; 1858 } 1859 1860 int bpf_map__set_priv(struct bpf_map *map, void *priv, 1861 bpf_map_clear_priv_t clear_priv) 1862 { 1863 if (!map) 1864 return -EINVAL; 1865 1866 if (map->priv) { 1867 if (map->clear_priv) 1868 map->clear_priv(map, map->priv); 1869 } 1870 1871 map->priv = priv; 1872 map->clear_priv = clear_priv; 1873 return 0; 1874 } 1875 1876 void *bpf_map__priv(struct bpf_map *map) 1877 { 1878 return map ? map->priv : ERR_PTR(-EINVAL); 1879 } 1880 1881 struct bpf_map * 1882 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 1883 { 1884 size_t idx; 1885 struct bpf_map *s, *e; 1886 1887 if (!obj || !obj->maps) 1888 return NULL; 1889 1890 s = obj->maps; 1891 e = obj->maps + obj->nr_maps; 1892 1893 if (prev == NULL) 1894 return s; 1895 1896 if ((prev < s) || (prev >= e)) { 1897 pr_warning("error in %s: map handler doesn't belong to object\n", 1898 __func__); 1899 return NULL; 1900 } 1901 1902 idx = (prev - obj->maps) + 1; 1903 if (idx >= obj->nr_maps) 1904 return NULL; 1905 return &obj->maps[idx]; 1906 } 1907 1908 struct bpf_map * 1909 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 1910 { 1911 struct bpf_map *pos; 1912 1913 bpf_map__for_each(pos, obj) { 1914 if (pos->name && !strcmp(pos->name, name)) 1915 return pos; 1916 } 1917 return NULL; 1918 } 1919 1920 struct bpf_map * 1921 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 1922 { 1923 int i; 1924 1925 for (i = 0; i < obj->nr_maps; i++) { 1926 if (obj->maps[i].offset == offset) 1927 return &obj->maps[i]; 1928 } 1929 return ERR_PTR(-ENOENT); 1930 } 1931 1932 long libbpf_get_error(const void *ptr) 1933 { 1934 if (IS_ERR(ptr)) 1935 return PTR_ERR(ptr); 1936 return 0; 1937 } 1938 1939 int bpf_prog_load(const char *file, enum bpf_prog_type type, 1940 struct bpf_object **pobj, int *prog_fd) 1941 { 1942 struct bpf_program *prog, *first_prog = NULL; 1943 struct bpf_object *obj; 1944 int err; 1945 1946 obj = bpf_object__open(file); 1947 if (IS_ERR(obj)) 1948 return -ENOENT; 1949 1950 bpf_object__for_each_program(prog, obj) { 1951 /* 1952 * If type is not specified, try to guess it based on 1953 * section name. 1954 */ 1955 if (type == BPF_PROG_TYPE_UNSPEC) { 1956 type = bpf_program__guess_type(prog); 1957 if (type == BPF_PROG_TYPE_UNSPEC) { 1958 bpf_object__close(obj); 1959 return -EINVAL; 1960 } 1961 } 1962 1963 bpf_program__set_type(prog, type); 1964 if (prog->idx != obj->efile.text_shndx && !first_prog) 1965 first_prog = prog; 1966 } 1967 1968 if (!first_prog) { 1969 pr_warning("object file doesn't contain bpf program\n"); 1970 bpf_object__close(obj); 1971 return -ENOENT; 1972 } 1973 1974 err = bpf_object__load(obj); 1975 if (err) { 1976 bpf_object__close(obj); 1977 return -EINVAL; 1978 } 1979 1980 *pobj = obj; 1981 *prog_fd = bpf_program__fd(first_prog); 1982 return 0; 1983 } 1984