1 #include <sys/sysmacros.h> 2 #include <sys/types.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <inttypes.h> 9 #include <byteswap.h> 10 #include <sys/stat.h> 11 #include <sys/mman.h> 12 13 #include "util.h" 14 #include "event.h" 15 #include "debug.h" 16 #include "evlist.h" 17 #include "symbol.h" 18 #include "strlist.h" 19 #include <elf.h> 20 21 #include "tsc.h" 22 #include "session.h" 23 #include "jit.h" 24 #include "jitdump.h" 25 #include "genelf.h" 26 #include "../builtin.h" 27 28 struct jit_buf_desc { 29 struct perf_data_file *output; 30 struct perf_session *session; 31 struct machine *machine; 32 union jr_entry *entry; 33 void *buf; 34 uint64_t sample_type; 35 size_t bufsize; 36 FILE *in; 37 bool needs_bswap; /* handles cross-endianess */ 38 bool use_arch_timestamp; 39 void *debug_data; 40 void *unwinding_data; 41 uint64_t unwinding_size; 42 uint64_t unwinding_mapped_size; 43 uint64_t eh_frame_hdr_size; 44 size_t nr_debug_entries; 45 uint32_t code_load_count; 46 u64 bytes_written; 47 struct rb_root code_root; 48 char dir[PATH_MAX]; 49 }; 50 51 struct debug_line_info { 52 unsigned long vma; 53 unsigned int lineno; 54 /* The filename format is unspecified, absolute path, relative etc. */ 55 char const filename[0]; 56 }; 57 58 struct jit_tool { 59 struct perf_tool tool; 60 struct perf_data_file output; 61 struct perf_data_file input; 62 u64 bytes_written; 63 }; 64 65 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 66 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 67 68 static int 69 jit_emit_elf(char *filename, 70 const char *sym, 71 uint64_t code_addr, 72 const void *code, 73 int csize, 74 void *debug, 75 int nr_debug_entries, 76 void *unwinding, 77 uint32_t unwinding_header_size, 78 uint32_t unwinding_size) 79 { 80 int ret, fd; 81 82 if (verbose > 0) 83 fprintf(stderr, "write ELF image %s\n", filename); 84 85 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 86 if (fd == -1) { 87 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 88 return -1; 89 } 90 91 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries, 92 unwinding, unwinding_header_size, unwinding_size); 93 94 close(fd); 95 96 if (ret) 97 unlink(filename); 98 99 return ret; 100 } 101 102 static void 103 jit_close(struct jit_buf_desc *jd) 104 { 105 if (!(jd && jd->in)) 106 return; 107 funlockfile(jd->in); 108 fclose(jd->in); 109 jd->in = NULL; 110 } 111 112 static int 113 jit_validate_events(struct perf_session *session) 114 { 115 struct perf_evsel *evsel; 116 117 /* 118 * check that all events use CLOCK_MONOTONIC 119 */ 120 evlist__for_each_entry(session->evlist, evsel) { 121 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC) 122 return -1; 123 } 124 return 0; 125 } 126 127 static int 128 jit_open(struct jit_buf_desc *jd, const char *name) 129 { 130 struct jitheader header; 131 struct jr_prefix *prefix; 132 ssize_t bs, bsz = 0; 133 void *n, *buf = NULL; 134 int ret, retval = -1; 135 136 jd->in = fopen(name, "r"); 137 if (!jd->in) 138 return -1; 139 140 bsz = hmax(sizeof(header), sizeof(*prefix)); 141 142 buf = malloc(bsz); 143 if (!buf) 144 goto error; 145 146 /* 147 * protect from writer modifying the file while we are reading it 148 */ 149 flockfile(jd->in); 150 151 ret = fread(buf, sizeof(header), 1, jd->in); 152 if (ret != 1) 153 goto error; 154 155 memcpy(&header, buf, sizeof(header)); 156 157 if (header.magic != JITHEADER_MAGIC) { 158 if (header.magic != JITHEADER_MAGIC_SW) 159 goto error; 160 jd->needs_bswap = true; 161 } 162 163 if (jd->needs_bswap) { 164 header.version = bswap_32(header.version); 165 header.total_size = bswap_32(header.total_size); 166 header.pid = bswap_32(header.pid); 167 header.elf_mach = bswap_32(header.elf_mach); 168 header.timestamp = bswap_64(header.timestamp); 169 header.flags = bswap_64(header.flags); 170 } 171 172 jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP; 173 174 if (verbose > 2) 175 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n", 176 header.version, 177 header.total_size, 178 (unsigned long long)header.timestamp, 179 header.pid, 180 header.elf_mach, 181 jd->use_arch_timestamp); 182 183 if (header.version > JITHEADER_VERSION) { 184 pr_err("wrong jitdump version %u, expected " STR(JITHEADER_VERSION), 185 header.version); 186 goto error; 187 } 188 189 if (header.flags & JITDUMP_FLAGS_RESERVED) { 190 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 191 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 192 goto error; 193 } 194 195 if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) { 196 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n"); 197 goto error; 198 } 199 200 /* 201 * validate event is using the correct clockid 202 */ 203 if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) { 204 pr_err("error, jitted code must be sampled with perf record -k 1\n"); 205 goto error; 206 } 207 208 bs = header.total_size - sizeof(header); 209 210 if (bs > bsz) { 211 n = realloc(buf, bs); 212 if (!n) 213 goto error; 214 bsz = bs; 215 buf = n; 216 /* read extra we do not know about */ 217 ret = fread(buf, bs - bsz, 1, jd->in); 218 if (ret != 1) 219 goto error; 220 } 221 /* 222 * keep dirname for generating files and mmap records 223 */ 224 strcpy(jd->dir, name); 225 dirname(jd->dir); 226 227 return 0; 228 error: 229 funlockfile(jd->in); 230 fclose(jd->in); 231 return retval; 232 } 233 234 static union jr_entry * 235 jit_get_next_entry(struct jit_buf_desc *jd) 236 { 237 struct jr_prefix *prefix; 238 union jr_entry *jr; 239 void *addr; 240 size_t bs, size; 241 int id, ret; 242 243 if (!(jd && jd->in)) 244 return NULL; 245 246 if (jd->buf == NULL) { 247 size_t sz = getpagesize(); 248 if (sz < sizeof(*prefix)) 249 sz = sizeof(*prefix); 250 251 jd->buf = malloc(sz); 252 if (jd->buf == NULL) 253 return NULL; 254 255 jd->bufsize = sz; 256 } 257 258 prefix = jd->buf; 259 260 /* 261 * file is still locked at this point 262 */ 263 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 264 if (ret != 1) 265 return NULL; 266 267 if (jd->needs_bswap) { 268 prefix->id = bswap_32(prefix->id); 269 prefix->total_size = bswap_32(prefix->total_size); 270 prefix->timestamp = bswap_64(prefix->timestamp); 271 } 272 id = prefix->id; 273 size = prefix->total_size; 274 275 bs = (size_t)size; 276 if (bs < sizeof(*prefix)) 277 return NULL; 278 279 if (id >= JIT_CODE_MAX) { 280 pr_warning("next_entry: unknown record type %d, skipping\n", id); 281 } 282 if (bs > jd->bufsize) { 283 void *n; 284 n = realloc(jd->buf, bs); 285 if (!n) 286 return NULL; 287 jd->buf = n; 288 jd->bufsize = bs; 289 } 290 291 addr = ((void *)jd->buf) + sizeof(*prefix); 292 293 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 294 if (ret != 1) 295 return NULL; 296 297 jr = (union jr_entry *)jd->buf; 298 299 switch(id) { 300 case JIT_CODE_DEBUG_INFO: 301 if (jd->needs_bswap) { 302 uint64_t n; 303 jr->info.code_addr = bswap_64(jr->info.code_addr); 304 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 305 for (n = 0 ; n < jr->info.nr_entry; n++) { 306 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 307 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 308 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 309 } 310 } 311 break; 312 case JIT_CODE_UNWINDING_INFO: 313 if (jd->needs_bswap) { 314 jr->unwinding.unwinding_size = bswap_64(jr->unwinding.unwinding_size); 315 jr->unwinding.eh_frame_hdr_size = bswap_64(jr->unwinding.eh_frame_hdr_size); 316 jr->unwinding.mapped_size = bswap_64(jr->unwinding.mapped_size); 317 } 318 break; 319 case JIT_CODE_CLOSE: 320 break; 321 case JIT_CODE_LOAD: 322 if (jd->needs_bswap) { 323 jr->load.pid = bswap_32(jr->load.pid); 324 jr->load.tid = bswap_32(jr->load.tid); 325 jr->load.vma = bswap_64(jr->load.vma); 326 jr->load.code_addr = bswap_64(jr->load.code_addr); 327 jr->load.code_size = bswap_64(jr->load.code_size); 328 jr->load.code_index= bswap_64(jr->load.code_index); 329 } 330 jd->code_load_count++; 331 break; 332 case JIT_CODE_MOVE: 333 if (jd->needs_bswap) { 334 jr->move.pid = bswap_32(jr->move.pid); 335 jr->move.tid = bswap_32(jr->move.tid); 336 jr->move.vma = bswap_64(jr->move.vma); 337 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 338 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 339 jr->move.code_size = bswap_64(jr->move.code_size); 340 jr->move.code_index = bswap_64(jr->move.code_index); 341 } 342 break; 343 case JIT_CODE_MAX: 344 default: 345 /* skip unknown record (we have read them) */ 346 break; 347 } 348 return jr; 349 } 350 351 static int 352 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 353 { 354 ssize_t size; 355 356 size = perf_data_file__write(jd->output, event, event->header.size); 357 if (size < 0) 358 return -1; 359 360 jd->bytes_written += size; 361 return 0; 362 } 363 364 static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) 365 { 366 struct perf_tsc_conversion tc; 367 368 if (!jd->use_arch_timestamp) 369 return timestamp; 370 371 tc.time_shift = jd->session->time_conv.time_shift; 372 tc.time_mult = jd->session->time_conv.time_mult; 373 tc.time_zero = jd->session->time_conv.time_zero; 374 375 if (!tc.time_mult) 376 return 0; 377 378 return tsc_to_perf_time(timestamp, &tc); 379 } 380 381 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 382 { 383 struct perf_sample sample; 384 union perf_event *event; 385 struct perf_tool *tool = jd->session->tool; 386 uint64_t code, addr; 387 uintptr_t uaddr; 388 char *filename; 389 struct stat st; 390 size_t size; 391 u16 idr_size; 392 const char *sym; 393 uint32_t count; 394 int ret, csize, usize; 395 pid_t pid, tid; 396 struct { 397 u32 pid, tid; 398 u64 time; 399 } *id; 400 401 pid = jr->load.pid; 402 tid = jr->load.tid; 403 csize = jr->load.code_size; 404 usize = jd->unwinding_mapped_size; 405 addr = jr->load.code_addr; 406 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 407 code = (unsigned long)jr + jr->load.p.total_size - csize; 408 count = jr->load.code_index; 409 idr_size = jd->machine->id_hdr_size; 410 411 event = calloc(1, sizeof(*event) + idr_size); 412 if (!event) 413 return -1; 414 415 filename = event->mmap2.filename; 416 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 417 jd->dir, 418 pid, 419 count); 420 421 size++; /* for \0 */ 422 423 size = PERF_ALIGN(size, sizeof(u64)); 424 uaddr = (uintptr_t)code; 425 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries, 426 jd->unwinding_data, jd->eh_frame_hdr_size, jd->unwinding_size); 427 428 if (jd->debug_data && jd->nr_debug_entries) { 429 free(jd->debug_data); 430 jd->debug_data = NULL; 431 jd->nr_debug_entries = 0; 432 } 433 434 if (jd->unwinding_data && jd->eh_frame_hdr_size) { 435 free(jd->unwinding_data); 436 jd->unwinding_data = NULL; 437 jd->eh_frame_hdr_size = 0; 438 jd->unwinding_mapped_size = 0; 439 jd->unwinding_size = 0; 440 } 441 442 if (ret) { 443 free(event); 444 return -1; 445 } 446 if (stat(filename, &st)) 447 memset(&st, 0, sizeof(st)); 448 449 event->mmap2.header.type = PERF_RECORD_MMAP2; 450 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 451 event->mmap2.header.size = (sizeof(event->mmap2) - 452 (sizeof(event->mmap2.filename) - size) + idr_size); 453 454 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 455 event->mmap2.start = addr; 456 event->mmap2.len = usize ? ALIGN_8(csize) + usize : csize; 457 event->mmap2.pid = pid; 458 event->mmap2.tid = tid; 459 event->mmap2.ino = st.st_ino; 460 event->mmap2.maj = major(st.st_dev); 461 event->mmap2.min = minor(st.st_dev); 462 event->mmap2.prot = st.st_mode; 463 event->mmap2.flags = MAP_SHARED; 464 event->mmap2.ino_generation = 1; 465 466 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 467 if (jd->sample_type & PERF_SAMPLE_TID) { 468 id->pid = pid; 469 id->tid = tid; 470 } 471 if (jd->sample_type & PERF_SAMPLE_TIME) 472 id->time = convert_timestamp(jd, jr->load.p.timestamp); 473 474 /* 475 * create pseudo sample to induce dso hit increment 476 * use first address as sample address 477 */ 478 memset(&sample, 0, sizeof(sample)); 479 sample.cpumode = PERF_RECORD_MISC_USER; 480 sample.pid = pid; 481 sample.tid = tid; 482 sample.time = id->time; 483 sample.ip = addr; 484 485 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 486 if (ret) 487 return ret; 488 489 ret = jit_inject_event(jd, event); 490 /* 491 * mark dso as use to generate buildid in the header 492 */ 493 if (!ret) 494 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 495 496 return ret; 497 } 498 499 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 500 { 501 struct perf_sample sample; 502 union perf_event *event; 503 struct perf_tool *tool = jd->session->tool; 504 char *filename; 505 size_t size; 506 struct stat st; 507 int usize; 508 u16 idr_size; 509 int ret; 510 pid_t pid, tid; 511 struct { 512 u32 pid, tid; 513 u64 time; 514 } *id; 515 516 pid = jr->move.pid; 517 tid = jr->move.tid; 518 usize = jd->unwinding_mapped_size; 519 idr_size = jd->machine->id_hdr_size; 520 521 /* 522 * +16 to account for sample_id_all (hack) 523 */ 524 event = calloc(1, sizeof(*event) + 16); 525 if (!event) 526 return -1; 527 528 filename = event->mmap2.filename; 529 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 530 jd->dir, 531 pid, 532 jr->move.code_index); 533 534 size++; /* for \0 */ 535 536 if (stat(filename, &st)) 537 memset(&st, 0, sizeof(st)); 538 539 size = PERF_ALIGN(size, sizeof(u64)); 540 541 event->mmap2.header.type = PERF_RECORD_MMAP2; 542 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 543 event->mmap2.header.size = (sizeof(event->mmap2) - 544 (sizeof(event->mmap2.filename) - size) + idr_size); 545 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 546 event->mmap2.start = jr->move.new_code_addr; 547 event->mmap2.len = usize ? ALIGN_8(jr->move.code_size) + usize 548 : jr->move.code_size; 549 event->mmap2.pid = pid; 550 event->mmap2.tid = tid; 551 event->mmap2.ino = st.st_ino; 552 event->mmap2.maj = major(st.st_dev); 553 event->mmap2.min = minor(st.st_dev); 554 event->mmap2.prot = st.st_mode; 555 event->mmap2.flags = MAP_SHARED; 556 event->mmap2.ino_generation = 1; 557 558 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 559 if (jd->sample_type & PERF_SAMPLE_TID) { 560 id->pid = pid; 561 id->tid = tid; 562 } 563 if (jd->sample_type & PERF_SAMPLE_TIME) 564 id->time = convert_timestamp(jd, jr->load.p.timestamp); 565 566 /* 567 * create pseudo sample to induce dso hit increment 568 * use first address as sample address 569 */ 570 memset(&sample, 0, sizeof(sample)); 571 sample.cpumode = PERF_RECORD_MISC_USER; 572 sample.pid = pid; 573 sample.tid = tid; 574 sample.time = id->time; 575 sample.ip = jr->move.new_code_addr; 576 577 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 578 if (ret) 579 return ret; 580 581 ret = jit_inject_event(jd, event); 582 if (!ret) 583 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 584 585 return ret; 586 } 587 588 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 589 { 590 void *data; 591 size_t sz; 592 593 if (!(jd && jr)) 594 return -1; 595 596 sz = jr->prefix.total_size - sizeof(jr->info); 597 data = malloc(sz); 598 if (!data) 599 return -1; 600 601 memcpy(data, &jr->info.entries, sz); 602 603 jd->debug_data = data; 604 605 /* 606 * we must use nr_entry instead of size here because 607 * we cannot distinguish actual entry from padding otherwise 608 */ 609 jd->nr_debug_entries = jr->info.nr_entry; 610 611 return 0; 612 } 613 614 static int 615 jit_repipe_unwinding_info(struct jit_buf_desc *jd, union jr_entry *jr) 616 { 617 void *unwinding_data; 618 uint32_t unwinding_data_size; 619 620 if (!(jd && jr)) 621 return -1; 622 623 unwinding_data_size = jr->prefix.total_size - sizeof(jr->unwinding); 624 unwinding_data = malloc(unwinding_data_size); 625 if (!unwinding_data) 626 return -1; 627 628 memcpy(unwinding_data, &jr->unwinding.unwinding_data, 629 unwinding_data_size); 630 631 jd->eh_frame_hdr_size = jr->unwinding.eh_frame_hdr_size; 632 jd->unwinding_size = jr->unwinding.unwinding_size; 633 jd->unwinding_mapped_size = jr->unwinding.mapped_size; 634 jd->unwinding_data = unwinding_data; 635 636 return 0; 637 } 638 639 static int 640 jit_process_dump(struct jit_buf_desc *jd) 641 { 642 union jr_entry *jr; 643 int ret = 0; 644 645 while ((jr = jit_get_next_entry(jd))) { 646 switch(jr->prefix.id) { 647 case JIT_CODE_LOAD: 648 ret = jit_repipe_code_load(jd, jr); 649 break; 650 case JIT_CODE_MOVE: 651 ret = jit_repipe_code_move(jd, jr); 652 break; 653 case JIT_CODE_DEBUG_INFO: 654 ret = jit_repipe_debug_info(jd, jr); 655 break; 656 case JIT_CODE_UNWINDING_INFO: 657 ret = jit_repipe_unwinding_info(jd, jr); 658 break; 659 default: 660 ret = 0; 661 continue; 662 } 663 } 664 return ret; 665 } 666 667 static int 668 jit_inject(struct jit_buf_desc *jd, char *path) 669 { 670 int ret; 671 672 if (verbose > 0) 673 fprintf(stderr, "injecting: %s\n", path); 674 675 ret = jit_open(jd, path); 676 if (ret) 677 return -1; 678 679 ret = jit_process_dump(jd); 680 681 jit_close(jd); 682 683 if (verbose > 0) 684 fprintf(stderr, "injected: %s (%d)\n", path, ret); 685 686 return 0; 687 } 688 689 /* 690 * File must be with pattern .../jit-XXXX.dump 691 * where XXXX is the PID of the process which did the mmap() 692 * as captured in the RECORD_MMAP record 693 */ 694 static int 695 jit_detect(char *mmap_name, pid_t pid) 696 { 697 char *p; 698 char *end = NULL; 699 pid_t pid2; 700 701 if (verbose > 2) 702 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 703 /* 704 * get file name 705 */ 706 p = strrchr(mmap_name, '/'); 707 if (!p) 708 return -1; 709 710 /* 711 * match prefix 712 */ 713 if (strncmp(p, "/jit-", 5)) 714 return -1; 715 716 /* 717 * skip prefix 718 */ 719 p += 5; 720 721 /* 722 * must be followed by a pid 723 */ 724 if (!isdigit(*p)) 725 return -1; 726 727 pid2 = (int)strtol(p, &end, 10); 728 if (!end) 729 return -1; 730 731 /* 732 * pid does not match mmap pid 733 * pid==0 in system-wide mode (synthesized) 734 */ 735 if (pid && pid2 != pid) 736 return -1; 737 /* 738 * validate suffix 739 */ 740 if (strcmp(end, ".dump")) 741 return -1; 742 743 if (verbose > 0) 744 fprintf(stderr, "jit marker found: %s\n", mmap_name); 745 746 return 0; 747 } 748 749 int 750 jit_process(struct perf_session *session, 751 struct perf_data_file *output, 752 struct machine *machine, 753 char *filename, 754 pid_t pid, 755 u64 *nbytes) 756 { 757 struct perf_evsel *first; 758 struct jit_buf_desc jd; 759 int ret; 760 761 /* 762 * first, detect marker mmap (i.e., the jitdump mmap) 763 */ 764 if (jit_detect(filename, pid)) 765 return 0; 766 767 memset(&jd, 0, sizeof(jd)); 768 769 jd.session = session; 770 jd.output = output; 771 jd.machine = machine; 772 773 /* 774 * track sample_type to compute id_all layout 775 * perf sets the same sample type to all events as of now 776 */ 777 first = perf_evlist__first(session->evlist); 778 jd.sample_type = first->attr.sample_type; 779 780 *nbytes = 0; 781 782 ret = jit_inject(&jd, filename); 783 if (!ret) { 784 *nbytes = jd.bytes_written; 785 ret = 1; 786 } 787 788 return ret; 789 } 790