1 /* 2 * Common eBPF ELF object loading operations. 3 * 4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 6 * Copyright (C) 2015 Huawei Inc. 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; 11 * version 2.1 of the License (not later!) 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this program; if not, see <http://www.gnu.org/licenses> 20 */ 21 22 #include <stdlib.h> 23 #include <stdio.h> 24 #include <stdarg.h> 25 #include <inttypes.h> 26 #include <string.h> 27 #include <unistd.h> 28 #include <fcntl.h> 29 #include <errno.h> 30 #include <asm/unistd.h> 31 #include <linux/kernel.h> 32 #include <linux/bpf.h> 33 #include <linux/list.h> 34 #include <libelf.h> 35 #include <gelf.h> 36 37 #include "libbpf.h" 38 #include "bpf.h" 39 40 #ifndef EM_BPF 41 #define EM_BPF 247 42 #endif 43 44 #define __printf(a, b) __attribute__((format(printf, a, b))) 45 46 __printf(1, 2) 47 static int __base_pr(const char *format, ...) 48 { 49 va_list args; 50 int err; 51 52 va_start(args, format); 53 err = vfprintf(stderr, format, args); 54 va_end(args); 55 return err; 56 } 57 58 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 59 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 60 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 61 62 #define __pr(func, fmt, ...) \ 63 do { \ 64 if ((func)) \ 65 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 66 } while (0) 67 68 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 69 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 70 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 71 72 void libbpf_set_print(libbpf_print_fn_t warn, 73 libbpf_print_fn_t info, 74 libbpf_print_fn_t debug) 75 { 76 __pr_warning = warn; 77 __pr_info = info; 78 __pr_debug = debug; 79 } 80 81 #define STRERR_BUFSIZE 128 82 83 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 84 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 85 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 86 87 static const char *libbpf_strerror_table[NR_ERRNO] = { 88 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 89 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 90 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 91 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", 92 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 93 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 94 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 95 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 96 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 97 [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", 98 }; 99 100 int libbpf_strerror(int err, char *buf, size_t size) 101 { 102 if (!buf || !size) 103 return -1; 104 105 err = err > 0 ? err : -err; 106 107 if (err < __LIBBPF_ERRNO__START) { 108 int ret; 109 110 ret = strerror_r(err, buf, size); 111 buf[size - 1] = '\0'; 112 return ret; 113 } 114 115 if (err < __LIBBPF_ERRNO__END) { 116 const char *msg; 117 118 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 119 snprintf(buf, size, "%s", msg); 120 buf[size - 1] = '\0'; 121 return 0; 122 } 123 124 snprintf(buf, size, "Unknown libbpf error %d", err); 125 buf[size - 1] = '\0'; 126 return -1; 127 } 128 129 #define CHECK_ERR(action, err, out) do { \ 130 err = action; \ 131 if (err) \ 132 goto out; \ 133 } while(0) 134 135 136 /* Copied from tools/perf/util/util.h */ 137 #ifndef zfree 138 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 139 #endif 140 141 #ifndef zclose 142 # define zclose(fd) ({ \ 143 int ___err = 0; \ 144 if ((fd) >= 0) \ 145 ___err = close((fd)); \ 146 fd = -1; \ 147 ___err; }) 148 #endif 149 150 #ifdef HAVE_LIBELF_MMAP_SUPPORT 151 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 152 #else 153 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 154 #endif 155 156 /* 157 * bpf_prog should be a better name but it has been used in 158 * linux/filter.h. 159 */ 160 struct bpf_program { 161 /* Index in elf obj file, for relocation use. */ 162 int idx; 163 char *section_name; 164 struct bpf_insn *insns; 165 size_t insns_cnt; 166 enum bpf_prog_type type; 167 168 struct { 169 int insn_idx; 170 int map_idx; 171 } *reloc_desc; 172 int nr_reloc; 173 174 struct { 175 int nr; 176 int *fds; 177 } instances; 178 bpf_program_prep_t preprocessor; 179 180 struct bpf_object *obj; 181 void *priv; 182 bpf_program_clear_priv_t clear_priv; 183 }; 184 185 struct bpf_map { 186 int fd; 187 char *name; 188 struct bpf_map_def def; 189 void *priv; 190 bpf_map_clear_priv_t clear_priv; 191 }; 192 193 static LIST_HEAD(bpf_objects_list); 194 195 struct bpf_object { 196 char license[64]; 197 u32 kern_version; 198 199 struct bpf_program *programs; 200 size_t nr_programs; 201 struct bpf_map *maps; 202 size_t nr_maps; 203 204 bool loaded; 205 206 /* 207 * Information when doing elf related work. Only valid if fd 208 * is valid. 209 */ 210 struct { 211 int fd; 212 void *obj_buf; 213 size_t obj_buf_sz; 214 Elf *elf; 215 GElf_Ehdr ehdr; 216 Elf_Data *symbols; 217 size_t strtabidx; 218 struct { 219 GElf_Shdr shdr; 220 Elf_Data *data; 221 } *reloc; 222 int nr_reloc; 223 int maps_shndx; 224 } efile; 225 /* 226 * All loaded bpf_object is linked in a list, which is 227 * hidden to caller. bpf_objects__<func> handlers deal with 228 * all objects. 229 */ 230 struct list_head list; 231 char path[]; 232 }; 233 #define obj_elf_valid(o) ((o)->efile.elf) 234 235 static void bpf_program__unload(struct bpf_program *prog) 236 { 237 int i; 238 239 if (!prog) 240 return; 241 242 /* 243 * If the object is opened but the program was never loaded, 244 * it is possible that prog->instances.nr == -1. 245 */ 246 if (prog->instances.nr > 0) { 247 for (i = 0; i < prog->instances.nr; i++) 248 zclose(prog->instances.fds[i]); 249 } else if (prog->instances.nr != -1) { 250 pr_warning("Internal error: instances.nr is %d\n", 251 prog->instances.nr); 252 } 253 254 prog->instances.nr = -1; 255 zfree(&prog->instances.fds); 256 } 257 258 static void bpf_program__exit(struct bpf_program *prog) 259 { 260 if (!prog) 261 return; 262 263 if (prog->clear_priv) 264 prog->clear_priv(prog, prog->priv); 265 266 prog->priv = NULL; 267 prog->clear_priv = NULL; 268 269 bpf_program__unload(prog); 270 zfree(&prog->section_name); 271 zfree(&prog->insns); 272 zfree(&prog->reloc_desc); 273 274 prog->nr_reloc = 0; 275 prog->insns_cnt = 0; 276 prog->idx = -1; 277 } 278 279 static int 280 bpf_program__init(void *data, size_t size, char *name, int idx, 281 struct bpf_program *prog) 282 { 283 if (size < sizeof(struct bpf_insn)) { 284 pr_warning("corrupted section '%s'\n", name); 285 return -EINVAL; 286 } 287 288 bzero(prog, sizeof(*prog)); 289 290 prog->section_name = strdup(name); 291 if (!prog->section_name) { 292 pr_warning("failed to alloc name for prog %s\n", 293 name); 294 goto errout; 295 } 296 297 prog->insns = malloc(size); 298 if (!prog->insns) { 299 pr_warning("failed to alloc insns for %s\n", name); 300 goto errout; 301 } 302 prog->insns_cnt = size / sizeof(struct bpf_insn); 303 memcpy(prog->insns, data, 304 prog->insns_cnt * sizeof(struct bpf_insn)); 305 prog->idx = idx; 306 prog->instances.fds = NULL; 307 prog->instances.nr = -1; 308 prog->type = BPF_PROG_TYPE_KPROBE; 309 310 return 0; 311 errout: 312 bpf_program__exit(prog); 313 return -ENOMEM; 314 } 315 316 static int 317 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 318 char *name, int idx) 319 { 320 struct bpf_program prog, *progs; 321 int nr_progs, err; 322 323 err = bpf_program__init(data, size, name, idx, &prog); 324 if (err) 325 return err; 326 327 progs = obj->programs; 328 nr_progs = obj->nr_programs; 329 330 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 331 if (!progs) { 332 /* 333 * In this case the original obj->programs 334 * is still valid, so don't need special treat for 335 * bpf_close_object(). 336 */ 337 pr_warning("failed to alloc a new program '%s'\n", 338 name); 339 bpf_program__exit(&prog); 340 return -ENOMEM; 341 } 342 343 pr_debug("found program %s\n", prog.section_name); 344 obj->programs = progs; 345 obj->nr_programs = nr_progs + 1; 346 prog.obj = obj; 347 progs[nr_progs] = prog; 348 return 0; 349 } 350 351 static struct bpf_object *bpf_object__new(const char *path, 352 void *obj_buf, 353 size_t obj_buf_sz) 354 { 355 struct bpf_object *obj; 356 357 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 358 if (!obj) { 359 pr_warning("alloc memory failed for %s\n", path); 360 return ERR_PTR(-ENOMEM); 361 } 362 363 strcpy(obj->path, path); 364 obj->efile.fd = -1; 365 366 /* 367 * Caller of this function should also calls 368 * bpf_object__elf_finish() after data collection to return 369 * obj_buf to user. If not, we should duplicate the buffer to 370 * avoid user freeing them before elf finish. 371 */ 372 obj->efile.obj_buf = obj_buf; 373 obj->efile.obj_buf_sz = obj_buf_sz; 374 obj->efile.maps_shndx = -1; 375 376 obj->loaded = false; 377 378 INIT_LIST_HEAD(&obj->list); 379 list_add(&obj->list, &bpf_objects_list); 380 return obj; 381 } 382 383 static void bpf_object__elf_finish(struct bpf_object *obj) 384 { 385 if (!obj_elf_valid(obj)) 386 return; 387 388 if (obj->efile.elf) { 389 elf_end(obj->efile.elf); 390 obj->efile.elf = NULL; 391 } 392 obj->efile.symbols = NULL; 393 394 zfree(&obj->efile.reloc); 395 obj->efile.nr_reloc = 0; 396 zclose(obj->efile.fd); 397 obj->efile.obj_buf = NULL; 398 obj->efile.obj_buf_sz = 0; 399 } 400 401 static int bpf_object__elf_init(struct bpf_object *obj) 402 { 403 int err = 0; 404 GElf_Ehdr *ep; 405 406 if (obj_elf_valid(obj)) { 407 pr_warning("elf init: internal error\n"); 408 return -LIBBPF_ERRNO__LIBELF; 409 } 410 411 if (obj->efile.obj_buf_sz > 0) { 412 /* 413 * obj_buf should have been validated by 414 * bpf_object__open_buffer(). 415 */ 416 obj->efile.elf = elf_memory(obj->efile.obj_buf, 417 obj->efile.obj_buf_sz); 418 } else { 419 obj->efile.fd = open(obj->path, O_RDONLY); 420 if (obj->efile.fd < 0) { 421 pr_warning("failed to open %s: %s\n", obj->path, 422 strerror(errno)); 423 return -errno; 424 } 425 426 obj->efile.elf = elf_begin(obj->efile.fd, 427 LIBBPF_ELF_C_READ_MMAP, 428 NULL); 429 } 430 431 if (!obj->efile.elf) { 432 pr_warning("failed to open %s as ELF file\n", 433 obj->path); 434 err = -LIBBPF_ERRNO__LIBELF; 435 goto errout; 436 } 437 438 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 439 pr_warning("failed to get EHDR from %s\n", 440 obj->path); 441 err = -LIBBPF_ERRNO__FORMAT; 442 goto errout; 443 } 444 ep = &obj->efile.ehdr; 445 446 /* Old LLVM set e_machine to EM_NONE */ 447 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 448 pr_warning("%s is not an eBPF object file\n", 449 obj->path); 450 err = -LIBBPF_ERRNO__FORMAT; 451 goto errout; 452 } 453 454 return 0; 455 errout: 456 bpf_object__elf_finish(obj); 457 return err; 458 } 459 460 static int 461 bpf_object__check_endianness(struct bpf_object *obj) 462 { 463 static unsigned int const endian = 1; 464 465 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 466 case ELFDATA2LSB: 467 /* We are big endian, BPF obj is little endian. */ 468 if (*(unsigned char const *)&endian != 1) 469 goto mismatch; 470 break; 471 472 case ELFDATA2MSB: 473 /* We are little endian, BPF obj is big endian. */ 474 if (*(unsigned char const *)&endian != 0) 475 goto mismatch; 476 break; 477 default: 478 return -LIBBPF_ERRNO__ENDIAN; 479 } 480 481 return 0; 482 483 mismatch: 484 pr_warning("Error: endianness mismatch.\n"); 485 return -LIBBPF_ERRNO__ENDIAN; 486 } 487 488 static int 489 bpf_object__init_license(struct bpf_object *obj, 490 void *data, size_t size) 491 { 492 memcpy(obj->license, data, 493 min(size, sizeof(obj->license) - 1)); 494 pr_debug("license of %s is %s\n", obj->path, obj->license); 495 return 0; 496 } 497 498 static int 499 bpf_object__init_kversion(struct bpf_object *obj, 500 void *data, size_t size) 501 { 502 u32 kver; 503 504 if (size != sizeof(kver)) { 505 pr_warning("invalid kver section in %s\n", obj->path); 506 return -LIBBPF_ERRNO__FORMAT; 507 } 508 memcpy(&kver, data, sizeof(kver)); 509 obj->kern_version = kver; 510 pr_debug("kernel version of %s is %x\n", obj->path, 511 obj->kern_version); 512 return 0; 513 } 514 515 static int 516 bpf_object__init_maps(struct bpf_object *obj, void *data, 517 size_t size) 518 { 519 size_t nr_maps; 520 int i; 521 522 nr_maps = size / sizeof(struct bpf_map_def); 523 if (!data || !nr_maps) { 524 pr_debug("%s doesn't need map definition\n", 525 obj->path); 526 return 0; 527 } 528 529 pr_debug("maps in %s: %zd bytes\n", obj->path, size); 530 531 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 532 if (!obj->maps) { 533 pr_warning("alloc maps for object failed\n"); 534 return -ENOMEM; 535 } 536 obj->nr_maps = nr_maps; 537 538 for (i = 0; i < nr_maps; i++) { 539 struct bpf_map_def *def = &obj->maps[i].def; 540 541 /* 542 * fill all fd with -1 so won't close incorrect 543 * fd (fd=0 is stdin) when failure (zclose won't close 544 * negative fd)). 545 */ 546 obj->maps[i].fd = -1; 547 548 /* Save map definition into obj->maps */ 549 *def = ((struct bpf_map_def *)data)[i]; 550 } 551 return 0; 552 } 553 554 static int 555 bpf_object__init_maps_name(struct bpf_object *obj) 556 { 557 int i; 558 Elf_Data *symbols = obj->efile.symbols; 559 560 if (!symbols || obj->efile.maps_shndx < 0) 561 return -EINVAL; 562 563 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 564 GElf_Sym sym; 565 size_t map_idx; 566 const char *map_name; 567 568 if (!gelf_getsym(symbols, i, &sym)) 569 continue; 570 if (sym.st_shndx != obj->efile.maps_shndx) 571 continue; 572 573 map_name = elf_strptr(obj->efile.elf, 574 obj->efile.strtabidx, 575 sym.st_name); 576 map_idx = sym.st_value / sizeof(struct bpf_map_def); 577 if (map_idx >= obj->nr_maps) { 578 pr_warning("index of map \"%s\" is buggy: %zu > %zu\n", 579 map_name, map_idx, obj->nr_maps); 580 continue; 581 } 582 obj->maps[map_idx].name = strdup(map_name); 583 if (!obj->maps[map_idx].name) { 584 pr_warning("failed to alloc map name\n"); 585 return -ENOMEM; 586 } 587 pr_debug("map %zu is \"%s\"\n", map_idx, 588 obj->maps[map_idx].name); 589 } 590 return 0; 591 } 592 593 static int bpf_object__elf_collect(struct bpf_object *obj) 594 { 595 Elf *elf = obj->efile.elf; 596 GElf_Ehdr *ep = &obj->efile.ehdr; 597 Elf_Scn *scn = NULL; 598 int idx = 0, err = 0; 599 600 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 601 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 602 pr_warning("failed to get e_shstrndx from %s\n", 603 obj->path); 604 return -LIBBPF_ERRNO__FORMAT; 605 } 606 607 while ((scn = elf_nextscn(elf, scn)) != NULL) { 608 char *name; 609 GElf_Shdr sh; 610 Elf_Data *data; 611 612 idx++; 613 if (gelf_getshdr(scn, &sh) != &sh) { 614 pr_warning("failed to get section header from %s\n", 615 obj->path); 616 err = -LIBBPF_ERRNO__FORMAT; 617 goto out; 618 } 619 620 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 621 if (!name) { 622 pr_warning("failed to get section name from %s\n", 623 obj->path); 624 err = -LIBBPF_ERRNO__FORMAT; 625 goto out; 626 } 627 628 data = elf_getdata(scn, 0); 629 if (!data) { 630 pr_warning("failed to get section data from %s(%s)\n", 631 name, obj->path); 632 err = -LIBBPF_ERRNO__FORMAT; 633 goto out; 634 } 635 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", 636 name, (unsigned long)data->d_size, 637 (int)sh.sh_link, (unsigned long)sh.sh_flags, 638 (int)sh.sh_type); 639 640 if (strcmp(name, "license") == 0) 641 err = bpf_object__init_license(obj, 642 data->d_buf, 643 data->d_size); 644 else if (strcmp(name, "version") == 0) 645 err = bpf_object__init_kversion(obj, 646 data->d_buf, 647 data->d_size); 648 else if (strcmp(name, "maps") == 0) { 649 err = bpf_object__init_maps(obj, data->d_buf, 650 data->d_size); 651 obj->efile.maps_shndx = idx; 652 } else if (sh.sh_type == SHT_SYMTAB) { 653 if (obj->efile.symbols) { 654 pr_warning("bpf: multiple SYMTAB in %s\n", 655 obj->path); 656 err = -LIBBPF_ERRNO__FORMAT; 657 } else { 658 obj->efile.symbols = data; 659 obj->efile.strtabidx = sh.sh_link; 660 } 661 } else if ((sh.sh_type == SHT_PROGBITS) && 662 (sh.sh_flags & SHF_EXECINSTR) && 663 (data->d_size > 0)) { 664 err = bpf_object__add_program(obj, data->d_buf, 665 data->d_size, name, idx); 666 if (err) { 667 char errmsg[STRERR_BUFSIZE]; 668 669 strerror_r(-err, errmsg, sizeof(errmsg)); 670 pr_warning("failed to alloc program %s (%s): %s", 671 name, obj->path, errmsg); 672 } 673 } else if (sh.sh_type == SHT_REL) { 674 void *reloc = obj->efile.reloc; 675 int nr_reloc = obj->efile.nr_reloc + 1; 676 677 reloc = realloc(reloc, 678 sizeof(*obj->efile.reloc) * nr_reloc); 679 if (!reloc) { 680 pr_warning("realloc failed\n"); 681 err = -ENOMEM; 682 } else { 683 int n = nr_reloc - 1; 684 685 obj->efile.reloc = reloc; 686 obj->efile.nr_reloc = nr_reloc; 687 688 obj->efile.reloc[n].shdr = sh; 689 obj->efile.reloc[n].data = data; 690 } 691 } 692 if (err) 693 goto out; 694 } 695 696 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 697 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 698 return LIBBPF_ERRNO__FORMAT; 699 } 700 if (obj->efile.maps_shndx >= 0) 701 err = bpf_object__init_maps_name(obj); 702 out: 703 return err; 704 } 705 706 static struct bpf_program * 707 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 708 { 709 struct bpf_program *prog; 710 size_t i; 711 712 for (i = 0; i < obj->nr_programs; i++) { 713 prog = &obj->programs[i]; 714 if (prog->idx == idx) 715 return prog; 716 } 717 return NULL; 718 } 719 720 static int 721 bpf_program__collect_reloc(struct bpf_program *prog, 722 size_t nr_maps, GElf_Shdr *shdr, 723 Elf_Data *data, Elf_Data *symbols, 724 int maps_shndx) 725 { 726 int i, nrels; 727 728 pr_debug("collecting relocating info for: '%s'\n", 729 prog->section_name); 730 nrels = shdr->sh_size / shdr->sh_entsize; 731 732 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 733 if (!prog->reloc_desc) { 734 pr_warning("failed to alloc memory in relocation\n"); 735 return -ENOMEM; 736 } 737 prog->nr_reloc = nrels; 738 739 for (i = 0; i < nrels; i++) { 740 GElf_Sym sym; 741 GElf_Rel rel; 742 unsigned int insn_idx; 743 struct bpf_insn *insns = prog->insns; 744 size_t map_idx; 745 746 if (!gelf_getrel(data, i, &rel)) { 747 pr_warning("relocation: failed to get %d reloc\n", i); 748 return -LIBBPF_ERRNO__FORMAT; 749 } 750 751 if (!gelf_getsym(symbols, 752 GELF_R_SYM(rel.r_info), 753 &sym)) { 754 pr_warning("relocation: symbol %"PRIx64" not found\n", 755 GELF_R_SYM(rel.r_info)); 756 return -LIBBPF_ERRNO__FORMAT; 757 } 758 759 if (sym.st_shndx != maps_shndx) { 760 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 761 prog->section_name, sym.st_shndx); 762 return -LIBBPF_ERRNO__RELOC; 763 } 764 765 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 766 pr_debug("relocation: insn_idx=%u\n", insn_idx); 767 768 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 769 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 770 insn_idx, insns[insn_idx].code); 771 return -LIBBPF_ERRNO__RELOC; 772 } 773 774 map_idx = sym.st_value / sizeof(struct bpf_map_def); 775 if (map_idx >= nr_maps) { 776 pr_warning("bpf relocation: map_idx %d large than %d\n", 777 (int)map_idx, (int)nr_maps - 1); 778 return -LIBBPF_ERRNO__RELOC; 779 } 780 781 prog->reloc_desc[i].insn_idx = insn_idx; 782 prog->reloc_desc[i].map_idx = map_idx; 783 } 784 return 0; 785 } 786 787 static int 788 bpf_object__create_maps(struct bpf_object *obj) 789 { 790 unsigned int i; 791 792 for (i = 0; i < obj->nr_maps; i++) { 793 struct bpf_map_def *def = &obj->maps[i].def; 794 int *pfd = &obj->maps[i].fd; 795 796 *pfd = bpf_create_map(def->type, 797 def->key_size, 798 def->value_size, 799 def->max_entries); 800 if (*pfd < 0) { 801 size_t j; 802 int err = *pfd; 803 804 pr_warning("failed to create map: %s\n", 805 strerror(errno)); 806 for (j = 0; j < i; j++) 807 zclose(obj->maps[j].fd); 808 return err; 809 } 810 pr_debug("create map: fd=%d\n", *pfd); 811 } 812 813 return 0; 814 } 815 816 static int 817 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 818 { 819 int i; 820 821 if (!prog || !prog->reloc_desc) 822 return 0; 823 824 for (i = 0; i < prog->nr_reloc; i++) { 825 int insn_idx, map_idx; 826 struct bpf_insn *insns = prog->insns; 827 828 insn_idx = prog->reloc_desc[i].insn_idx; 829 map_idx = prog->reloc_desc[i].map_idx; 830 831 if (insn_idx >= (int)prog->insns_cnt) { 832 pr_warning("relocation out of range: '%s'\n", 833 prog->section_name); 834 return -LIBBPF_ERRNO__RELOC; 835 } 836 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 837 insns[insn_idx].imm = obj->maps[map_idx].fd; 838 } 839 840 zfree(&prog->reloc_desc); 841 prog->nr_reloc = 0; 842 return 0; 843 } 844 845 846 static int 847 bpf_object__relocate(struct bpf_object *obj) 848 { 849 struct bpf_program *prog; 850 size_t i; 851 int err; 852 853 for (i = 0; i < obj->nr_programs; i++) { 854 prog = &obj->programs[i]; 855 856 err = bpf_program__relocate(prog, obj); 857 if (err) { 858 pr_warning("failed to relocate '%s'\n", 859 prog->section_name); 860 return err; 861 } 862 } 863 return 0; 864 } 865 866 static int bpf_object__collect_reloc(struct bpf_object *obj) 867 { 868 int i, err; 869 870 if (!obj_elf_valid(obj)) { 871 pr_warning("Internal error: elf object is closed\n"); 872 return -LIBBPF_ERRNO__INTERNAL; 873 } 874 875 for (i = 0; i < obj->efile.nr_reloc; i++) { 876 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 877 Elf_Data *data = obj->efile.reloc[i].data; 878 int idx = shdr->sh_info; 879 struct bpf_program *prog; 880 size_t nr_maps = obj->nr_maps; 881 882 if (shdr->sh_type != SHT_REL) { 883 pr_warning("internal error at %d\n", __LINE__); 884 return -LIBBPF_ERRNO__INTERNAL; 885 } 886 887 prog = bpf_object__find_prog_by_idx(obj, idx); 888 if (!prog) { 889 pr_warning("relocation failed: no %d section\n", 890 idx); 891 return -LIBBPF_ERRNO__RELOC; 892 } 893 894 err = bpf_program__collect_reloc(prog, nr_maps, 895 shdr, data, 896 obj->efile.symbols, 897 obj->efile.maps_shndx); 898 if (err) 899 return err; 900 } 901 return 0; 902 } 903 904 static int 905 load_program(enum bpf_prog_type type, struct bpf_insn *insns, 906 int insns_cnt, char *license, u32 kern_version, int *pfd) 907 { 908 int ret; 909 char *log_buf; 910 911 if (!insns || !insns_cnt) 912 return -EINVAL; 913 914 log_buf = malloc(BPF_LOG_BUF_SIZE); 915 if (!log_buf) 916 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 917 918 ret = bpf_load_program(type, insns, insns_cnt, license, 919 kern_version, log_buf, BPF_LOG_BUF_SIZE); 920 921 if (ret >= 0) { 922 *pfd = ret; 923 ret = 0; 924 goto out; 925 } 926 927 ret = -LIBBPF_ERRNO__LOAD; 928 pr_warning("load bpf program failed: %s\n", strerror(errno)); 929 930 if (log_buf && log_buf[0] != '\0') { 931 ret = -LIBBPF_ERRNO__VERIFY; 932 pr_warning("-- BEGIN DUMP LOG ---\n"); 933 pr_warning("\n%s\n", log_buf); 934 pr_warning("-- END LOG --\n"); 935 } else if (insns_cnt >= BPF_MAXINSNS) { 936 pr_warning("Program too large (%d insns), at most %d insns\n", 937 insns_cnt, BPF_MAXINSNS); 938 ret = -LIBBPF_ERRNO__PROG2BIG; 939 } else { 940 /* Wrong program type? */ 941 if (type != BPF_PROG_TYPE_KPROBE) { 942 int fd; 943 944 fd = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, 945 insns_cnt, license, kern_version, 946 NULL, 0); 947 if (fd >= 0) { 948 close(fd); 949 ret = -LIBBPF_ERRNO__PROGTYPE; 950 goto out; 951 } 952 } 953 954 if (log_buf) 955 ret = -LIBBPF_ERRNO__KVER; 956 } 957 958 out: 959 free(log_buf); 960 return ret; 961 } 962 963 static int 964 bpf_program__load(struct bpf_program *prog, 965 char *license, u32 kern_version) 966 { 967 int err = 0, fd, i; 968 969 if (prog->instances.nr < 0 || !prog->instances.fds) { 970 if (prog->preprocessor) { 971 pr_warning("Internal error: can't load program '%s'\n", 972 prog->section_name); 973 return -LIBBPF_ERRNO__INTERNAL; 974 } 975 976 prog->instances.fds = malloc(sizeof(int)); 977 if (!prog->instances.fds) { 978 pr_warning("Not enough memory for BPF fds\n"); 979 return -ENOMEM; 980 } 981 prog->instances.nr = 1; 982 prog->instances.fds[0] = -1; 983 } 984 985 if (!prog->preprocessor) { 986 if (prog->instances.nr != 1) { 987 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 988 prog->section_name, prog->instances.nr); 989 } 990 err = load_program(prog->type, prog->insns, prog->insns_cnt, 991 license, kern_version, &fd); 992 if (!err) 993 prog->instances.fds[0] = fd; 994 goto out; 995 } 996 997 for (i = 0; i < prog->instances.nr; i++) { 998 struct bpf_prog_prep_result result; 999 bpf_program_prep_t preprocessor = prog->preprocessor; 1000 1001 bzero(&result, sizeof(result)); 1002 err = preprocessor(prog, i, prog->insns, 1003 prog->insns_cnt, &result); 1004 if (err) { 1005 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 1006 i, prog->section_name); 1007 goto out; 1008 } 1009 1010 if (!result.new_insn_ptr || !result.new_insn_cnt) { 1011 pr_debug("Skip loading the %dth instance of program '%s'\n", 1012 i, prog->section_name); 1013 prog->instances.fds[i] = -1; 1014 if (result.pfd) 1015 *result.pfd = -1; 1016 continue; 1017 } 1018 1019 err = load_program(prog->type, result.new_insn_ptr, 1020 result.new_insn_cnt, 1021 license, kern_version, &fd); 1022 1023 if (err) { 1024 pr_warning("Loading the %dth instance of program '%s' failed\n", 1025 i, prog->section_name); 1026 goto out; 1027 } 1028 1029 if (result.pfd) 1030 *result.pfd = fd; 1031 prog->instances.fds[i] = fd; 1032 } 1033 out: 1034 if (err) 1035 pr_warning("failed to load program '%s'\n", 1036 prog->section_name); 1037 zfree(&prog->insns); 1038 prog->insns_cnt = 0; 1039 return err; 1040 } 1041 1042 static int 1043 bpf_object__load_progs(struct bpf_object *obj) 1044 { 1045 size_t i; 1046 int err; 1047 1048 for (i = 0; i < obj->nr_programs; i++) { 1049 err = bpf_program__load(&obj->programs[i], 1050 obj->license, 1051 obj->kern_version); 1052 if (err) 1053 return err; 1054 } 1055 return 0; 1056 } 1057 1058 static int bpf_object__validate(struct bpf_object *obj) 1059 { 1060 if (obj->kern_version == 0) { 1061 pr_warning("%s doesn't provide kernel version\n", 1062 obj->path); 1063 return -LIBBPF_ERRNO__KVERSION; 1064 } 1065 return 0; 1066 } 1067 1068 static struct bpf_object * 1069 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 1070 { 1071 struct bpf_object *obj; 1072 int err; 1073 1074 if (elf_version(EV_CURRENT) == EV_NONE) { 1075 pr_warning("failed to init libelf for %s\n", path); 1076 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1077 } 1078 1079 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1080 if (IS_ERR(obj)) 1081 return obj; 1082 1083 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1084 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1085 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 1086 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1087 CHECK_ERR(bpf_object__validate(obj), err, out); 1088 1089 bpf_object__elf_finish(obj); 1090 return obj; 1091 out: 1092 bpf_object__close(obj); 1093 return ERR_PTR(err); 1094 } 1095 1096 struct bpf_object *bpf_object__open(const char *path) 1097 { 1098 /* param validation */ 1099 if (!path) 1100 return NULL; 1101 1102 pr_debug("loading %s\n", path); 1103 1104 return __bpf_object__open(path, NULL, 0); 1105 } 1106 1107 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1108 size_t obj_buf_sz, 1109 const char *name) 1110 { 1111 char tmp_name[64]; 1112 1113 /* param validation */ 1114 if (!obj_buf || obj_buf_sz <= 0) 1115 return NULL; 1116 1117 if (!name) { 1118 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1119 (unsigned long)obj_buf, 1120 (unsigned long)obj_buf_sz); 1121 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1122 name = tmp_name; 1123 } 1124 pr_debug("loading object '%s' from buffer\n", 1125 name); 1126 1127 return __bpf_object__open(name, obj_buf, obj_buf_sz); 1128 } 1129 1130 int bpf_object__unload(struct bpf_object *obj) 1131 { 1132 size_t i; 1133 1134 if (!obj) 1135 return -EINVAL; 1136 1137 for (i = 0; i < obj->nr_maps; i++) 1138 zclose(obj->maps[i].fd); 1139 1140 for (i = 0; i < obj->nr_programs; i++) 1141 bpf_program__unload(&obj->programs[i]); 1142 1143 return 0; 1144 } 1145 1146 int bpf_object__load(struct bpf_object *obj) 1147 { 1148 int err; 1149 1150 if (!obj) 1151 return -EINVAL; 1152 1153 if (obj->loaded) { 1154 pr_warning("object should not be loaded twice\n"); 1155 return -EINVAL; 1156 } 1157 1158 obj->loaded = true; 1159 1160 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1161 CHECK_ERR(bpf_object__relocate(obj), err, out); 1162 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1163 1164 return 0; 1165 out: 1166 bpf_object__unload(obj); 1167 pr_warning("failed to load object '%s'\n", obj->path); 1168 return err; 1169 } 1170 1171 void bpf_object__close(struct bpf_object *obj) 1172 { 1173 size_t i; 1174 1175 if (!obj) 1176 return; 1177 1178 bpf_object__elf_finish(obj); 1179 bpf_object__unload(obj); 1180 1181 for (i = 0; i < obj->nr_maps; i++) { 1182 zfree(&obj->maps[i].name); 1183 if (obj->maps[i].clear_priv) 1184 obj->maps[i].clear_priv(&obj->maps[i], 1185 obj->maps[i].priv); 1186 obj->maps[i].priv = NULL; 1187 obj->maps[i].clear_priv = NULL; 1188 } 1189 zfree(&obj->maps); 1190 obj->nr_maps = 0; 1191 1192 if (obj->programs && obj->nr_programs) { 1193 for (i = 0; i < obj->nr_programs; i++) 1194 bpf_program__exit(&obj->programs[i]); 1195 } 1196 zfree(&obj->programs); 1197 1198 list_del(&obj->list); 1199 free(obj); 1200 } 1201 1202 struct bpf_object * 1203 bpf_object__next(struct bpf_object *prev) 1204 { 1205 struct bpf_object *next; 1206 1207 if (!prev) 1208 next = list_first_entry(&bpf_objects_list, 1209 struct bpf_object, 1210 list); 1211 else 1212 next = list_next_entry(prev, list); 1213 1214 /* Empty list is noticed here so don't need checking on entry. */ 1215 if (&next->list == &bpf_objects_list) 1216 return NULL; 1217 1218 return next; 1219 } 1220 1221 const char *bpf_object__name(struct bpf_object *obj) 1222 { 1223 return obj ? obj->path : ERR_PTR(-EINVAL); 1224 } 1225 1226 unsigned int bpf_object__kversion(struct bpf_object *obj) 1227 { 1228 return obj ? obj->kern_version : 0; 1229 } 1230 1231 struct bpf_program * 1232 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1233 { 1234 size_t idx; 1235 1236 if (!obj->programs) 1237 return NULL; 1238 /* First handler */ 1239 if (prev == NULL) 1240 return &obj->programs[0]; 1241 1242 if (prev->obj != obj) { 1243 pr_warning("error: program handler doesn't match object\n"); 1244 return NULL; 1245 } 1246 1247 idx = (prev - obj->programs) + 1; 1248 if (idx >= obj->nr_programs) 1249 return NULL; 1250 return &obj->programs[idx]; 1251 } 1252 1253 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 1254 bpf_program_clear_priv_t clear_priv) 1255 { 1256 if (prog->priv && prog->clear_priv) 1257 prog->clear_priv(prog, prog->priv); 1258 1259 prog->priv = priv; 1260 prog->clear_priv = clear_priv; 1261 return 0; 1262 } 1263 1264 void *bpf_program__priv(struct bpf_program *prog) 1265 { 1266 return prog ? prog->priv : ERR_PTR(-EINVAL); 1267 } 1268 1269 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1270 { 1271 const char *title; 1272 1273 title = prog->section_name; 1274 if (needs_copy) { 1275 title = strdup(title); 1276 if (!title) { 1277 pr_warning("failed to strdup program title\n"); 1278 return ERR_PTR(-ENOMEM); 1279 } 1280 } 1281 1282 return title; 1283 } 1284 1285 int bpf_program__fd(struct bpf_program *prog) 1286 { 1287 return bpf_program__nth_fd(prog, 0); 1288 } 1289 1290 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 1291 bpf_program_prep_t prep) 1292 { 1293 int *instances_fds; 1294 1295 if (nr_instances <= 0 || !prep) 1296 return -EINVAL; 1297 1298 if (prog->instances.nr > 0 || prog->instances.fds) { 1299 pr_warning("Can't set pre-processor after loading\n"); 1300 return -EINVAL; 1301 } 1302 1303 instances_fds = malloc(sizeof(int) * nr_instances); 1304 if (!instances_fds) { 1305 pr_warning("alloc memory failed for fds\n"); 1306 return -ENOMEM; 1307 } 1308 1309 /* fill all fd with -1 */ 1310 memset(instances_fds, -1, sizeof(int) * nr_instances); 1311 1312 prog->instances.nr = nr_instances; 1313 prog->instances.fds = instances_fds; 1314 prog->preprocessor = prep; 1315 return 0; 1316 } 1317 1318 int bpf_program__nth_fd(struct bpf_program *prog, int n) 1319 { 1320 int fd; 1321 1322 if (n >= prog->instances.nr || n < 0) { 1323 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 1324 n, prog->section_name, prog->instances.nr); 1325 return -EINVAL; 1326 } 1327 1328 fd = prog->instances.fds[n]; 1329 if (fd < 0) { 1330 pr_warning("%dth instance of program '%s' is invalid\n", 1331 n, prog->section_name); 1332 return -ENOENT; 1333 } 1334 1335 return fd; 1336 } 1337 1338 static void bpf_program__set_type(struct bpf_program *prog, 1339 enum bpf_prog_type type) 1340 { 1341 prog->type = type; 1342 } 1343 1344 int bpf_program__set_tracepoint(struct bpf_program *prog) 1345 { 1346 if (!prog) 1347 return -EINVAL; 1348 bpf_program__set_type(prog, BPF_PROG_TYPE_TRACEPOINT); 1349 return 0; 1350 } 1351 1352 int bpf_program__set_kprobe(struct bpf_program *prog) 1353 { 1354 if (!prog) 1355 return -EINVAL; 1356 bpf_program__set_type(prog, BPF_PROG_TYPE_KPROBE); 1357 return 0; 1358 } 1359 1360 static bool bpf_program__is_type(struct bpf_program *prog, 1361 enum bpf_prog_type type) 1362 { 1363 return prog ? (prog->type == type) : false; 1364 } 1365 1366 bool bpf_program__is_tracepoint(struct bpf_program *prog) 1367 { 1368 return bpf_program__is_type(prog, BPF_PROG_TYPE_TRACEPOINT); 1369 } 1370 1371 bool bpf_program__is_kprobe(struct bpf_program *prog) 1372 { 1373 return bpf_program__is_type(prog, BPF_PROG_TYPE_KPROBE); 1374 } 1375 1376 int bpf_map__fd(struct bpf_map *map) 1377 { 1378 return map ? map->fd : -EINVAL; 1379 } 1380 1381 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 1382 { 1383 return map ? &map->def : ERR_PTR(-EINVAL); 1384 } 1385 1386 const char *bpf_map__name(struct bpf_map *map) 1387 { 1388 return map ? map->name : NULL; 1389 } 1390 1391 int bpf_map__set_priv(struct bpf_map *map, void *priv, 1392 bpf_map_clear_priv_t clear_priv) 1393 { 1394 if (!map) 1395 return -EINVAL; 1396 1397 if (map->priv) { 1398 if (map->clear_priv) 1399 map->clear_priv(map, map->priv); 1400 } 1401 1402 map->priv = priv; 1403 map->clear_priv = clear_priv; 1404 return 0; 1405 } 1406 1407 void *bpf_map__priv(struct bpf_map *map) 1408 { 1409 return map ? map->priv : ERR_PTR(-EINVAL); 1410 } 1411 1412 struct bpf_map * 1413 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 1414 { 1415 size_t idx; 1416 struct bpf_map *s, *e; 1417 1418 if (!obj || !obj->maps) 1419 return NULL; 1420 1421 s = obj->maps; 1422 e = obj->maps + obj->nr_maps; 1423 1424 if (prev == NULL) 1425 return s; 1426 1427 if ((prev < s) || (prev >= e)) { 1428 pr_warning("error in %s: map handler doesn't belong to object\n", 1429 __func__); 1430 return NULL; 1431 } 1432 1433 idx = (prev - obj->maps) + 1; 1434 if (idx >= obj->nr_maps) 1435 return NULL; 1436 return &obj->maps[idx]; 1437 } 1438 1439 struct bpf_map * 1440 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 1441 { 1442 struct bpf_map *pos; 1443 1444 bpf_map__for_each(pos, obj) { 1445 if (pos->name && !strcmp(pos->name, name)) 1446 return pos; 1447 } 1448 return NULL; 1449 } 1450