1 #include <sys/sysmacros.h> 2 #include <sys/types.h> 3 #include <errno.h> 4 #include <stdio.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <fcntl.h> 8 #include <unistd.h> 9 #include <inttypes.h> 10 #include <byteswap.h> 11 #include <sys/stat.h> 12 #include <sys/mman.h> 13 #include <linux/stringify.h> 14 15 #include "util.h" 16 #include "event.h" 17 #include "debug.h" 18 #include "evlist.h" 19 #include "symbol.h" 20 #include <elf.h> 21 22 #include "tsc.h" 23 #include "session.h" 24 #include "jit.h" 25 #include "jitdump.h" 26 #include "genelf.h" 27 #include "../builtin.h" 28 29 #include "sane_ctype.h" 30 31 struct jit_buf_desc { 32 struct perf_data_file *output; 33 struct perf_session *session; 34 struct machine *machine; 35 union jr_entry *entry; 36 void *buf; 37 uint64_t sample_type; 38 size_t bufsize; 39 FILE *in; 40 bool needs_bswap; /* handles cross-endianess */ 41 bool use_arch_timestamp; 42 void *debug_data; 43 void *unwinding_data; 44 uint64_t unwinding_size; 45 uint64_t unwinding_mapped_size; 46 uint64_t eh_frame_hdr_size; 47 size_t nr_debug_entries; 48 uint32_t code_load_count; 49 u64 bytes_written; 50 struct rb_root code_root; 51 char dir[PATH_MAX]; 52 }; 53 54 struct debug_line_info { 55 unsigned long vma; 56 unsigned int lineno; 57 /* The filename format is unspecified, absolute path, relative etc. */ 58 char const filename[0]; 59 }; 60 61 struct jit_tool { 62 struct perf_tool tool; 63 struct perf_data_file output; 64 struct perf_data_file input; 65 u64 bytes_written; 66 }; 67 68 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 69 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 70 71 static int 72 jit_emit_elf(char *filename, 73 const char *sym, 74 uint64_t code_addr, 75 const void *code, 76 int csize, 77 void *debug, 78 int nr_debug_entries, 79 void *unwinding, 80 uint32_t unwinding_header_size, 81 uint32_t unwinding_size) 82 { 83 int ret, fd; 84 85 if (verbose > 0) 86 fprintf(stderr, "write ELF image %s\n", filename); 87 88 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 89 if (fd == -1) { 90 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 91 return -1; 92 } 93 94 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries, 95 unwinding, unwinding_header_size, unwinding_size); 96 97 close(fd); 98 99 if (ret) 100 unlink(filename); 101 102 return ret; 103 } 104 105 static void 106 jit_close(struct jit_buf_desc *jd) 107 { 108 if (!(jd && jd->in)) 109 return; 110 funlockfile(jd->in); 111 fclose(jd->in); 112 jd->in = NULL; 113 } 114 115 static int 116 jit_validate_events(struct perf_session *session) 117 { 118 struct perf_evsel *evsel; 119 120 /* 121 * check that all events use CLOCK_MONOTONIC 122 */ 123 evlist__for_each_entry(session->evlist, evsel) { 124 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC) 125 return -1; 126 } 127 return 0; 128 } 129 130 static int 131 jit_open(struct jit_buf_desc *jd, const char *name) 132 { 133 struct jitheader header; 134 struct jr_prefix *prefix; 135 ssize_t bs, bsz = 0; 136 void *n, *buf = NULL; 137 int ret, retval = -1; 138 139 jd->in = fopen(name, "r"); 140 if (!jd->in) 141 return -1; 142 143 bsz = hmax(sizeof(header), sizeof(*prefix)); 144 145 buf = malloc(bsz); 146 if (!buf) 147 goto error; 148 149 /* 150 * protect from writer modifying the file while we are reading it 151 */ 152 flockfile(jd->in); 153 154 ret = fread(buf, sizeof(header), 1, jd->in); 155 if (ret != 1) 156 goto error; 157 158 memcpy(&header, buf, sizeof(header)); 159 160 if (header.magic != JITHEADER_MAGIC) { 161 if (header.magic != JITHEADER_MAGIC_SW) 162 goto error; 163 jd->needs_bswap = true; 164 } 165 166 if (jd->needs_bswap) { 167 header.version = bswap_32(header.version); 168 header.total_size = bswap_32(header.total_size); 169 header.pid = bswap_32(header.pid); 170 header.elf_mach = bswap_32(header.elf_mach); 171 header.timestamp = bswap_64(header.timestamp); 172 header.flags = bswap_64(header.flags); 173 } 174 175 jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP; 176 177 if (verbose > 2) 178 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n", 179 header.version, 180 header.total_size, 181 (unsigned long long)header.timestamp, 182 header.pid, 183 header.elf_mach, 184 jd->use_arch_timestamp); 185 186 if (header.version > JITHEADER_VERSION) { 187 pr_err("wrong jitdump version %u, expected " __stringify(JITHEADER_VERSION), 188 header.version); 189 goto error; 190 } 191 192 if (header.flags & JITDUMP_FLAGS_RESERVED) { 193 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 194 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 195 goto error; 196 } 197 198 if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) { 199 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n"); 200 goto error; 201 } 202 203 /* 204 * validate event is using the correct clockid 205 */ 206 if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) { 207 pr_err("error, jitted code must be sampled with perf record -k 1\n"); 208 goto error; 209 } 210 211 bs = header.total_size - sizeof(header); 212 213 if (bs > bsz) { 214 n = realloc(buf, bs); 215 if (!n) 216 goto error; 217 bsz = bs; 218 buf = n; 219 /* read extra we do not know about */ 220 ret = fread(buf, bs - bsz, 1, jd->in); 221 if (ret != 1) 222 goto error; 223 } 224 /* 225 * keep dirname for generating files and mmap records 226 */ 227 strcpy(jd->dir, name); 228 dirname(jd->dir); 229 230 return 0; 231 error: 232 funlockfile(jd->in); 233 fclose(jd->in); 234 return retval; 235 } 236 237 static union jr_entry * 238 jit_get_next_entry(struct jit_buf_desc *jd) 239 { 240 struct jr_prefix *prefix; 241 union jr_entry *jr; 242 void *addr; 243 size_t bs, size; 244 int id, ret; 245 246 if (!(jd && jd->in)) 247 return NULL; 248 249 if (jd->buf == NULL) { 250 size_t sz = getpagesize(); 251 if (sz < sizeof(*prefix)) 252 sz = sizeof(*prefix); 253 254 jd->buf = malloc(sz); 255 if (jd->buf == NULL) 256 return NULL; 257 258 jd->bufsize = sz; 259 } 260 261 prefix = jd->buf; 262 263 /* 264 * file is still locked at this point 265 */ 266 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 267 if (ret != 1) 268 return NULL; 269 270 if (jd->needs_bswap) { 271 prefix->id = bswap_32(prefix->id); 272 prefix->total_size = bswap_32(prefix->total_size); 273 prefix->timestamp = bswap_64(prefix->timestamp); 274 } 275 id = prefix->id; 276 size = prefix->total_size; 277 278 bs = (size_t)size; 279 if (bs < sizeof(*prefix)) 280 return NULL; 281 282 if (id >= JIT_CODE_MAX) { 283 pr_warning("next_entry: unknown record type %d, skipping\n", id); 284 } 285 if (bs > jd->bufsize) { 286 void *n; 287 n = realloc(jd->buf, bs); 288 if (!n) 289 return NULL; 290 jd->buf = n; 291 jd->bufsize = bs; 292 } 293 294 addr = ((void *)jd->buf) + sizeof(*prefix); 295 296 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 297 if (ret != 1) 298 return NULL; 299 300 jr = (union jr_entry *)jd->buf; 301 302 switch(id) { 303 case JIT_CODE_DEBUG_INFO: 304 if (jd->needs_bswap) { 305 uint64_t n; 306 jr->info.code_addr = bswap_64(jr->info.code_addr); 307 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 308 for (n = 0 ; n < jr->info.nr_entry; n++) { 309 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 310 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 311 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 312 } 313 } 314 break; 315 case JIT_CODE_UNWINDING_INFO: 316 if (jd->needs_bswap) { 317 jr->unwinding.unwinding_size = bswap_64(jr->unwinding.unwinding_size); 318 jr->unwinding.eh_frame_hdr_size = bswap_64(jr->unwinding.eh_frame_hdr_size); 319 jr->unwinding.mapped_size = bswap_64(jr->unwinding.mapped_size); 320 } 321 break; 322 case JIT_CODE_CLOSE: 323 break; 324 case JIT_CODE_LOAD: 325 if (jd->needs_bswap) { 326 jr->load.pid = bswap_32(jr->load.pid); 327 jr->load.tid = bswap_32(jr->load.tid); 328 jr->load.vma = bswap_64(jr->load.vma); 329 jr->load.code_addr = bswap_64(jr->load.code_addr); 330 jr->load.code_size = bswap_64(jr->load.code_size); 331 jr->load.code_index= bswap_64(jr->load.code_index); 332 } 333 jd->code_load_count++; 334 break; 335 case JIT_CODE_MOVE: 336 if (jd->needs_bswap) { 337 jr->move.pid = bswap_32(jr->move.pid); 338 jr->move.tid = bswap_32(jr->move.tid); 339 jr->move.vma = bswap_64(jr->move.vma); 340 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 341 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 342 jr->move.code_size = bswap_64(jr->move.code_size); 343 jr->move.code_index = bswap_64(jr->move.code_index); 344 } 345 break; 346 case JIT_CODE_MAX: 347 default: 348 /* skip unknown record (we have read them) */ 349 break; 350 } 351 return jr; 352 } 353 354 static int 355 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 356 { 357 ssize_t size; 358 359 size = perf_data_file__write(jd->output, event, event->header.size); 360 if (size < 0) 361 return -1; 362 363 jd->bytes_written += size; 364 return 0; 365 } 366 367 static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) 368 { 369 struct perf_tsc_conversion tc; 370 371 if (!jd->use_arch_timestamp) 372 return timestamp; 373 374 tc.time_shift = jd->session->time_conv.time_shift; 375 tc.time_mult = jd->session->time_conv.time_mult; 376 tc.time_zero = jd->session->time_conv.time_zero; 377 378 if (!tc.time_mult) 379 return 0; 380 381 return tsc_to_perf_time(timestamp, &tc); 382 } 383 384 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 385 { 386 struct perf_sample sample; 387 union perf_event *event; 388 struct perf_tool *tool = jd->session->tool; 389 uint64_t code, addr; 390 uintptr_t uaddr; 391 char *filename; 392 struct stat st; 393 size_t size; 394 u16 idr_size; 395 const char *sym; 396 uint32_t count; 397 int ret, csize, usize; 398 pid_t pid, tid; 399 struct { 400 u32 pid, tid; 401 u64 time; 402 } *id; 403 404 pid = jr->load.pid; 405 tid = jr->load.tid; 406 csize = jr->load.code_size; 407 usize = jd->unwinding_mapped_size; 408 addr = jr->load.code_addr; 409 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 410 code = (unsigned long)jr + jr->load.p.total_size - csize; 411 count = jr->load.code_index; 412 idr_size = jd->machine->id_hdr_size; 413 414 event = calloc(1, sizeof(*event) + idr_size); 415 if (!event) 416 return -1; 417 418 filename = event->mmap2.filename; 419 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 420 jd->dir, 421 pid, 422 count); 423 424 size++; /* for \0 */ 425 426 size = PERF_ALIGN(size, sizeof(u64)); 427 uaddr = (uintptr_t)code; 428 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries, 429 jd->unwinding_data, jd->eh_frame_hdr_size, jd->unwinding_size); 430 431 if (jd->debug_data && jd->nr_debug_entries) { 432 free(jd->debug_data); 433 jd->debug_data = NULL; 434 jd->nr_debug_entries = 0; 435 } 436 437 if (jd->unwinding_data && jd->eh_frame_hdr_size) { 438 free(jd->unwinding_data); 439 jd->unwinding_data = NULL; 440 jd->eh_frame_hdr_size = 0; 441 jd->unwinding_mapped_size = 0; 442 jd->unwinding_size = 0; 443 } 444 445 if (ret) { 446 free(event); 447 return -1; 448 } 449 if (stat(filename, &st)) 450 memset(&st, 0, sizeof(st)); 451 452 event->mmap2.header.type = PERF_RECORD_MMAP2; 453 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 454 event->mmap2.header.size = (sizeof(event->mmap2) - 455 (sizeof(event->mmap2.filename) - size) + idr_size); 456 457 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 458 event->mmap2.start = addr; 459 event->mmap2.len = usize ? ALIGN_8(csize) + usize : csize; 460 event->mmap2.pid = pid; 461 event->mmap2.tid = tid; 462 event->mmap2.ino = st.st_ino; 463 event->mmap2.maj = major(st.st_dev); 464 event->mmap2.min = minor(st.st_dev); 465 event->mmap2.prot = st.st_mode; 466 event->mmap2.flags = MAP_SHARED; 467 event->mmap2.ino_generation = 1; 468 469 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 470 if (jd->sample_type & PERF_SAMPLE_TID) { 471 id->pid = pid; 472 id->tid = tid; 473 } 474 if (jd->sample_type & PERF_SAMPLE_TIME) 475 id->time = convert_timestamp(jd, jr->load.p.timestamp); 476 477 /* 478 * create pseudo sample to induce dso hit increment 479 * use first address as sample address 480 */ 481 memset(&sample, 0, sizeof(sample)); 482 sample.cpumode = PERF_RECORD_MISC_USER; 483 sample.pid = pid; 484 sample.tid = tid; 485 sample.time = id->time; 486 sample.ip = addr; 487 488 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 489 if (ret) 490 return ret; 491 492 ret = jit_inject_event(jd, event); 493 /* 494 * mark dso as use to generate buildid in the header 495 */ 496 if (!ret) 497 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 498 499 return ret; 500 } 501 502 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 503 { 504 struct perf_sample sample; 505 union perf_event *event; 506 struct perf_tool *tool = jd->session->tool; 507 char *filename; 508 size_t size; 509 struct stat st; 510 int usize; 511 u16 idr_size; 512 int ret; 513 pid_t pid, tid; 514 struct { 515 u32 pid, tid; 516 u64 time; 517 } *id; 518 519 pid = jr->move.pid; 520 tid = jr->move.tid; 521 usize = jd->unwinding_mapped_size; 522 idr_size = jd->machine->id_hdr_size; 523 524 /* 525 * +16 to account for sample_id_all (hack) 526 */ 527 event = calloc(1, sizeof(*event) + 16); 528 if (!event) 529 return -1; 530 531 filename = event->mmap2.filename; 532 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 533 jd->dir, 534 pid, 535 jr->move.code_index); 536 537 size++; /* for \0 */ 538 539 if (stat(filename, &st)) 540 memset(&st, 0, sizeof(st)); 541 542 size = PERF_ALIGN(size, sizeof(u64)); 543 544 event->mmap2.header.type = PERF_RECORD_MMAP2; 545 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 546 event->mmap2.header.size = (sizeof(event->mmap2) - 547 (sizeof(event->mmap2.filename) - size) + idr_size); 548 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 549 event->mmap2.start = jr->move.new_code_addr; 550 event->mmap2.len = usize ? ALIGN_8(jr->move.code_size) + usize 551 : jr->move.code_size; 552 event->mmap2.pid = pid; 553 event->mmap2.tid = tid; 554 event->mmap2.ino = st.st_ino; 555 event->mmap2.maj = major(st.st_dev); 556 event->mmap2.min = minor(st.st_dev); 557 event->mmap2.prot = st.st_mode; 558 event->mmap2.flags = MAP_SHARED; 559 event->mmap2.ino_generation = 1; 560 561 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 562 if (jd->sample_type & PERF_SAMPLE_TID) { 563 id->pid = pid; 564 id->tid = tid; 565 } 566 if (jd->sample_type & PERF_SAMPLE_TIME) 567 id->time = convert_timestamp(jd, jr->load.p.timestamp); 568 569 /* 570 * create pseudo sample to induce dso hit increment 571 * use first address as sample address 572 */ 573 memset(&sample, 0, sizeof(sample)); 574 sample.cpumode = PERF_RECORD_MISC_USER; 575 sample.pid = pid; 576 sample.tid = tid; 577 sample.time = id->time; 578 sample.ip = jr->move.new_code_addr; 579 580 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 581 if (ret) 582 return ret; 583 584 ret = jit_inject_event(jd, event); 585 if (!ret) 586 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 587 588 return ret; 589 } 590 591 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 592 { 593 void *data; 594 size_t sz; 595 596 if (!(jd && jr)) 597 return -1; 598 599 sz = jr->prefix.total_size - sizeof(jr->info); 600 data = malloc(sz); 601 if (!data) 602 return -1; 603 604 memcpy(data, &jr->info.entries, sz); 605 606 jd->debug_data = data; 607 608 /* 609 * we must use nr_entry instead of size here because 610 * we cannot distinguish actual entry from padding otherwise 611 */ 612 jd->nr_debug_entries = jr->info.nr_entry; 613 614 return 0; 615 } 616 617 static int 618 jit_repipe_unwinding_info(struct jit_buf_desc *jd, union jr_entry *jr) 619 { 620 void *unwinding_data; 621 uint32_t unwinding_data_size; 622 623 if (!(jd && jr)) 624 return -1; 625 626 unwinding_data_size = jr->prefix.total_size - sizeof(jr->unwinding); 627 unwinding_data = malloc(unwinding_data_size); 628 if (!unwinding_data) 629 return -1; 630 631 memcpy(unwinding_data, &jr->unwinding.unwinding_data, 632 unwinding_data_size); 633 634 jd->eh_frame_hdr_size = jr->unwinding.eh_frame_hdr_size; 635 jd->unwinding_size = jr->unwinding.unwinding_size; 636 jd->unwinding_mapped_size = jr->unwinding.mapped_size; 637 jd->unwinding_data = unwinding_data; 638 639 return 0; 640 } 641 642 static int 643 jit_process_dump(struct jit_buf_desc *jd) 644 { 645 union jr_entry *jr; 646 int ret = 0; 647 648 while ((jr = jit_get_next_entry(jd))) { 649 switch(jr->prefix.id) { 650 case JIT_CODE_LOAD: 651 ret = jit_repipe_code_load(jd, jr); 652 break; 653 case JIT_CODE_MOVE: 654 ret = jit_repipe_code_move(jd, jr); 655 break; 656 case JIT_CODE_DEBUG_INFO: 657 ret = jit_repipe_debug_info(jd, jr); 658 break; 659 case JIT_CODE_UNWINDING_INFO: 660 ret = jit_repipe_unwinding_info(jd, jr); 661 break; 662 default: 663 ret = 0; 664 continue; 665 } 666 } 667 return ret; 668 } 669 670 static int 671 jit_inject(struct jit_buf_desc *jd, char *path) 672 { 673 int ret; 674 675 if (verbose > 0) 676 fprintf(stderr, "injecting: %s\n", path); 677 678 ret = jit_open(jd, path); 679 if (ret) 680 return -1; 681 682 ret = jit_process_dump(jd); 683 684 jit_close(jd); 685 686 if (verbose > 0) 687 fprintf(stderr, "injected: %s (%d)\n", path, ret); 688 689 return 0; 690 } 691 692 /* 693 * File must be with pattern .../jit-XXXX.dump 694 * where XXXX is the PID of the process which did the mmap() 695 * as captured in the RECORD_MMAP record 696 */ 697 static int 698 jit_detect(char *mmap_name, pid_t pid) 699 { 700 char *p; 701 char *end = NULL; 702 pid_t pid2; 703 704 if (verbose > 2) 705 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 706 /* 707 * get file name 708 */ 709 p = strrchr(mmap_name, '/'); 710 if (!p) 711 return -1; 712 713 /* 714 * match prefix 715 */ 716 if (strncmp(p, "/jit-", 5)) 717 return -1; 718 719 /* 720 * skip prefix 721 */ 722 p += 5; 723 724 /* 725 * must be followed by a pid 726 */ 727 if (!isdigit(*p)) 728 return -1; 729 730 pid2 = (int)strtol(p, &end, 10); 731 if (!end) 732 return -1; 733 734 /* 735 * pid does not match mmap pid 736 * pid==0 in system-wide mode (synthesized) 737 */ 738 if (pid && pid2 != pid) 739 return -1; 740 /* 741 * validate suffix 742 */ 743 if (strcmp(end, ".dump")) 744 return -1; 745 746 if (verbose > 0) 747 fprintf(stderr, "jit marker found: %s\n", mmap_name); 748 749 return 0; 750 } 751 752 int 753 jit_process(struct perf_session *session, 754 struct perf_data_file *output, 755 struct machine *machine, 756 char *filename, 757 pid_t pid, 758 u64 *nbytes) 759 { 760 struct perf_evsel *first; 761 struct jit_buf_desc jd; 762 int ret; 763 764 /* 765 * first, detect marker mmap (i.e., the jitdump mmap) 766 */ 767 if (jit_detect(filename, pid)) 768 return 0; 769 770 memset(&jd, 0, sizeof(jd)); 771 772 jd.session = session; 773 jd.output = output; 774 jd.machine = machine; 775 776 /* 777 * track sample_type to compute id_all layout 778 * perf sets the same sample type to all events as of now 779 */ 780 first = perf_evlist__first(session->evlist); 781 jd.sample_type = first->attr.sample_type; 782 783 *nbytes = 0; 784 785 ret = jit_inject(&jd, filename); 786 if (!ret) { 787 *nbytes = jd.bytes_written; 788 ret = 1; 789 } 790 791 return ret; 792 } 793