1 #include <sys/sysmacros.h> 2 #include <sys/types.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <inttypes.h> 9 #include <byteswap.h> 10 #include <sys/stat.h> 11 #include <sys/mman.h> 12 13 #include "util.h" 14 #include "event.h" 15 #include "debug.h" 16 #include "evlist.h" 17 #include "symbol.h" 18 #include "strlist.h" 19 #include <elf.h> 20 21 #include "tsc.h" 22 #include "session.h" 23 #include "jit.h" 24 #include "jitdump.h" 25 #include "genelf.h" 26 #include "../builtin.h" 27 28 struct jit_buf_desc { 29 struct perf_data_file *output; 30 struct perf_session *session; 31 struct machine *machine; 32 union jr_entry *entry; 33 void *buf; 34 uint64_t sample_type; 35 size_t bufsize; 36 FILE *in; 37 bool needs_bswap; /* handles cross-endianess */ 38 bool use_arch_timestamp; 39 void *debug_data; 40 size_t nr_debug_entries; 41 uint32_t code_load_count; 42 u64 bytes_written; 43 struct rb_root code_root; 44 char dir[PATH_MAX]; 45 }; 46 47 struct debug_line_info { 48 unsigned long vma; 49 unsigned int lineno; 50 /* The filename format is unspecified, absolute path, relative etc. */ 51 char const filename[0]; 52 }; 53 54 struct jit_tool { 55 struct perf_tool tool; 56 struct perf_data_file output; 57 struct perf_data_file input; 58 u64 bytes_written; 59 }; 60 61 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 62 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 63 64 static int 65 jit_emit_elf(char *filename, 66 const char *sym, 67 uint64_t code_addr, 68 const void *code, 69 int csize, 70 void *debug, 71 int nr_debug_entries) 72 { 73 int ret, fd; 74 75 if (verbose > 0) 76 fprintf(stderr, "write ELF image %s\n", filename); 77 78 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 79 if (fd == -1) { 80 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 81 return -1; 82 } 83 84 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries); 85 86 close(fd); 87 88 if (ret) 89 unlink(filename); 90 91 return ret; 92 } 93 94 static void 95 jit_close(struct jit_buf_desc *jd) 96 { 97 if (!(jd && jd->in)) 98 return; 99 funlockfile(jd->in); 100 fclose(jd->in); 101 jd->in = NULL; 102 } 103 104 static int 105 jit_validate_events(struct perf_session *session) 106 { 107 struct perf_evsel *evsel; 108 109 /* 110 * check that all events use CLOCK_MONOTONIC 111 */ 112 evlist__for_each_entry(session->evlist, evsel) { 113 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC) 114 return -1; 115 } 116 return 0; 117 } 118 119 static int 120 jit_open(struct jit_buf_desc *jd, const char *name) 121 { 122 struct jitheader header; 123 struct jr_prefix *prefix; 124 ssize_t bs, bsz = 0; 125 void *n, *buf = NULL; 126 int ret, retval = -1; 127 128 jd->in = fopen(name, "r"); 129 if (!jd->in) 130 return -1; 131 132 bsz = hmax(sizeof(header), sizeof(*prefix)); 133 134 buf = malloc(bsz); 135 if (!buf) 136 goto error; 137 138 /* 139 * protect from writer modifying the file while we are reading it 140 */ 141 flockfile(jd->in); 142 143 ret = fread(buf, sizeof(header), 1, jd->in); 144 if (ret != 1) 145 goto error; 146 147 memcpy(&header, buf, sizeof(header)); 148 149 if (header.magic != JITHEADER_MAGIC) { 150 if (header.magic != JITHEADER_MAGIC_SW) 151 goto error; 152 jd->needs_bswap = true; 153 } 154 155 if (jd->needs_bswap) { 156 header.version = bswap_32(header.version); 157 header.total_size = bswap_32(header.total_size); 158 header.pid = bswap_32(header.pid); 159 header.elf_mach = bswap_32(header.elf_mach); 160 header.timestamp = bswap_64(header.timestamp); 161 header.flags = bswap_64(header.flags); 162 } 163 164 jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP; 165 166 if (verbose > 2) 167 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n", 168 header.version, 169 header.total_size, 170 (unsigned long long)header.timestamp, 171 header.pid, 172 header.elf_mach, 173 jd->use_arch_timestamp); 174 175 if (header.flags & JITDUMP_FLAGS_RESERVED) { 176 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 177 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 178 goto error; 179 } 180 181 if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) { 182 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n"); 183 goto error; 184 } 185 186 /* 187 * validate event is using the correct clockid 188 */ 189 if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) { 190 pr_err("error, jitted code must be sampled with perf record -k 1\n"); 191 goto error; 192 } 193 194 bs = header.total_size - sizeof(header); 195 196 if (bs > bsz) { 197 n = realloc(buf, bs); 198 if (!n) 199 goto error; 200 bsz = bs; 201 buf = n; 202 /* read extra we do not know about */ 203 ret = fread(buf, bs - bsz, 1, jd->in); 204 if (ret != 1) 205 goto error; 206 } 207 /* 208 * keep dirname for generating files and mmap records 209 */ 210 strcpy(jd->dir, name); 211 dirname(jd->dir); 212 213 return 0; 214 error: 215 funlockfile(jd->in); 216 fclose(jd->in); 217 return retval; 218 } 219 220 static union jr_entry * 221 jit_get_next_entry(struct jit_buf_desc *jd) 222 { 223 struct jr_prefix *prefix; 224 union jr_entry *jr; 225 void *addr; 226 size_t bs, size; 227 int id, ret; 228 229 if (!(jd && jd->in)) 230 return NULL; 231 232 if (jd->buf == NULL) { 233 size_t sz = getpagesize(); 234 if (sz < sizeof(*prefix)) 235 sz = sizeof(*prefix); 236 237 jd->buf = malloc(sz); 238 if (jd->buf == NULL) 239 return NULL; 240 241 jd->bufsize = sz; 242 } 243 244 prefix = jd->buf; 245 246 /* 247 * file is still locked at this point 248 */ 249 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 250 if (ret != 1) 251 return NULL; 252 253 if (jd->needs_bswap) { 254 prefix->id = bswap_32(prefix->id); 255 prefix->total_size = bswap_32(prefix->total_size); 256 prefix->timestamp = bswap_64(prefix->timestamp); 257 } 258 id = prefix->id; 259 size = prefix->total_size; 260 261 bs = (size_t)size; 262 if (bs < sizeof(*prefix)) 263 return NULL; 264 265 if (id >= JIT_CODE_MAX) { 266 pr_warning("next_entry: unknown prefix %d, skipping\n", id); 267 return NULL; 268 } 269 if (bs > jd->bufsize) { 270 void *n; 271 n = realloc(jd->buf, bs); 272 if (!n) 273 return NULL; 274 jd->buf = n; 275 jd->bufsize = bs; 276 } 277 278 addr = ((void *)jd->buf) + sizeof(*prefix); 279 280 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 281 if (ret != 1) 282 return NULL; 283 284 jr = (union jr_entry *)jd->buf; 285 286 switch(id) { 287 case JIT_CODE_DEBUG_INFO: 288 if (jd->needs_bswap) { 289 uint64_t n; 290 jr->info.code_addr = bswap_64(jr->info.code_addr); 291 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 292 for (n = 0 ; n < jr->info.nr_entry; n++) { 293 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 294 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 295 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 296 } 297 } 298 break; 299 case JIT_CODE_CLOSE: 300 break; 301 case JIT_CODE_LOAD: 302 if (jd->needs_bswap) { 303 jr->load.pid = bswap_32(jr->load.pid); 304 jr->load.tid = bswap_32(jr->load.tid); 305 jr->load.vma = bswap_64(jr->load.vma); 306 jr->load.code_addr = bswap_64(jr->load.code_addr); 307 jr->load.code_size = bswap_64(jr->load.code_size); 308 jr->load.code_index= bswap_64(jr->load.code_index); 309 } 310 jd->code_load_count++; 311 break; 312 case JIT_CODE_MOVE: 313 if (jd->needs_bswap) { 314 jr->move.pid = bswap_32(jr->move.pid); 315 jr->move.tid = bswap_32(jr->move.tid); 316 jr->move.vma = bswap_64(jr->move.vma); 317 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 318 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 319 jr->move.code_size = bswap_64(jr->move.code_size); 320 jr->move.code_index = bswap_64(jr->move.code_index); 321 } 322 break; 323 case JIT_CODE_MAX: 324 default: 325 return NULL; 326 } 327 return jr; 328 } 329 330 static int 331 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 332 { 333 ssize_t size; 334 335 size = perf_data_file__write(jd->output, event, event->header.size); 336 if (size < 0) 337 return -1; 338 339 jd->bytes_written += size; 340 return 0; 341 } 342 343 static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) 344 { 345 struct perf_tsc_conversion tc; 346 347 if (!jd->use_arch_timestamp) 348 return timestamp; 349 350 tc.time_shift = jd->session->time_conv.time_shift; 351 tc.time_mult = jd->session->time_conv.time_mult; 352 tc.time_zero = jd->session->time_conv.time_zero; 353 354 if (!tc.time_mult) 355 return 0; 356 357 return tsc_to_perf_time(timestamp, &tc); 358 } 359 360 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 361 { 362 struct perf_sample sample; 363 union perf_event *event; 364 struct perf_tool *tool = jd->session->tool; 365 uint64_t code, addr; 366 uintptr_t uaddr; 367 char *filename; 368 struct stat st; 369 size_t size; 370 u16 idr_size; 371 const char *sym; 372 uint32_t count; 373 int ret, csize; 374 pid_t pid, tid; 375 struct { 376 u32 pid, tid; 377 u64 time; 378 } *id; 379 380 pid = jr->load.pid; 381 tid = jr->load.tid; 382 csize = jr->load.code_size; 383 addr = jr->load.code_addr; 384 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 385 code = (unsigned long)jr + jr->load.p.total_size - csize; 386 count = jr->load.code_index; 387 idr_size = jd->machine->id_hdr_size; 388 389 event = calloc(1, sizeof(*event) + idr_size); 390 if (!event) 391 return -1; 392 393 filename = event->mmap2.filename; 394 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 395 jd->dir, 396 pid, 397 count); 398 399 size++; /* for \0 */ 400 401 size = PERF_ALIGN(size, sizeof(u64)); 402 uaddr = (uintptr_t)code; 403 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries); 404 405 if (jd->debug_data && jd->nr_debug_entries) { 406 free(jd->debug_data); 407 jd->debug_data = NULL; 408 jd->nr_debug_entries = 0; 409 } 410 411 if (ret) { 412 free(event); 413 return -1; 414 } 415 if (stat(filename, &st)) 416 memset(&st, 0, sizeof(st)); 417 418 event->mmap2.header.type = PERF_RECORD_MMAP2; 419 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 420 event->mmap2.header.size = (sizeof(event->mmap2) - 421 (sizeof(event->mmap2.filename) - size) + idr_size); 422 423 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 424 event->mmap2.start = addr; 425 event->mmap2.len = csize; 426 event->mmap2.pid = pid; 427 event->mmap2.tid = tid; 428 event->mmap2.ino = st.st_ino; 429 event->mmap2.maj = major(st.st_dev); 430 event->mmap2.min = minor(st.st_dev); 431 event->mmap2.prot = st.st_mode; 432 event->mmap2.flags = MAP_SHARED; 433 event->mmap2.ino_generation = 1; 434 435 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 436 if (jd->sample_type & PERF_SAMPLE_TID) { 437 id->pid = pid; 438 id->tid = tid; 439 } 440 if (jd->sample_type & PERF_SAMPLE_TIME) 441 id->time = convert_timestamp(jd, jr->load.p.timestamp); 442 443 /* 444 * create pseudo sample to induce dso hit increment 445 * use first address as sample address 446 */ 447 memset(&sample, 0, sizeof(sample)); 448 sample.cpumode = PERF_RECORD_MISC_USER; 449 sample.pid = pid; 450 sample.tid = tid; 451 sample.time = id->time; 452 sample.ip = addr; 453 454 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 455 if (ret) 456 return ret; 457 458 ret = jit_inject_event(jd, event); 459 /* 460 * mark dso as use to generate buildid in the header 461 */ 462 if (!ret) 463 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 464 465 return ret; 466 } 467 468 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 469 { 470 struct perf_sample sample; 471 union perf_event *event; 472 struct perf_tool *tool = jd->session->tool; 473 char *filename; 474 size_t size; 475 struct stat st; 476 u16 idr_size; 477 int ret; 478 pid_t pid, tid; 479 struct { 480 u32 pid, tid; 481 u64 time; 482 } *id; 483 484 pid = jr->move.pid; 485 tid = jr->move.tid; 486 idr_size = jd->machine->id_hdr_size; 487 488 /* 489 * +16 to account for sample_id_all (hack) 490 */ 491 event = calloc(1, sizeof(*event) + 16); 492 if (!event) 493 return -1; 494 495 filename = event->mmap2.filename; 496 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 497 jd->dir, 498 pid, 499 jr->move.code_index); 500 501 size++; /* for \0 */ 502 503 if (stat(filename, &st)) 504 memset(&st, 0, sizeof(st)); 505 506 size = PERF_ALIGN(size, sizeof(u64)); 507 508 event->mmap2.header.type = PERF_RECORD_MMAP2; 509 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 510 event->mmap2.header.size = (sizeof(event->mmap2) - 511 (sizeof(event->mmap2.filename) - size) + idr_size); 512 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 513 event->mmap2.start = jr->move.new_code_addr; 514 event->mmap2.len = jr->move.code_size; 515 event->mmap2.pid = pid; 516 event->mmap2.tid = tid; 517 event->mmap2.ino = st.st_ino; 518 event->mmap2.maj = major(st.st_dev); 519 event->mmap2.min = minor(st.st_dev); 520 event->mmap2.prot = st.st_mode; 521 event->mmap2.flags = MAP_SHARED; 522 event->mmap2.ino_generation = 1; 523 524 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 525 if (jd->sample_type & PERF_SAMPLE_TID) { 526 id->pid = pid; 527 id->tid = tid; 528 } 529 if (jd->sample_type & PERF_SAMPLE_TIME) 530 id->time = convert_timestamp(jd, jr->load.p.timestamp); 531 532 /* 533 * create pseudo sample to induce dso hit increment 534 * use first address as sample address 535 */ 536 memset(&sample, 0, sizeof(sample)); 537 sample.cpumode = PERF_RECORD_MISC_USER; 538 sample.pid = pid; 539 sample.tid = tid; 540 sample.time = id->time; 541 sample.ip = jr->move.new_code_addr; 542 543 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 544 if (ret) 545 return ret; 546 547 ret = jit_inject_event(jd, event); 548 if (!ret) 549 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 550 551 return ret; 552 } 553 554 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 555 { 556 void *data; 557 size_t sz; 558 559 if (!(jd && jr)) 560 return -1; 561 562 sz = jr->prefix.total_size - sizeof(jr->info); 563 data = malloc(sz); 564 if (!data) 565 return -1; 566 567 memcpy(data, &jr->info.entries, sz); 568 569 jd->debug_data = data; 570 571 /* 572 * we must use nr_entry instead of size here because 573 * we cannot distinguish actual entry from padding otherwise 574 */ 575 jd->nr_debug_entries = jr->info.nr_entry; 576 577 return 0; 578 } 579 580 static int 581 jit_process_dump(struct jit_buf_desc *jd) 582 { 583 union jr_entry *jr; 584 int ret; 585 586 while ((jr = jit_get_next_entry(jd))) { 587 switch(jr->prefix.id) { 588 case JIT_CODE_LOAD: 589 ret = jit_repipe_code_load(jd, jr); 590 break; 591 case JIT_CODE_MOVE: 592 ret = jit_repipe_code_move(jd, jr); 593 break; 594 case JIT_CODE_DEBUG_INFO: 595 ret = jit_repipe_debug_info(jd, jr); 596 break; 597 default: 598 ret = 0; 599 continue; 600 } 601 } 602 return ret; 603 } 604 605 static int 606 jit_inject(struct jit_buf_desc *jd, char *path) 607 { 608 int ret; 609 610 if (verbose > 0) 611 fprintf(stderr, "injecting: %s\n", path); 612 613 ret = jit_open(jd, path); 614 if (ret) 615 return -1; 616 617 ret = jit_process_dump(jd); 618 619 jit_close(jd); 620 621 if (verbose > 0) 622 fprintf(stderr, "injected: %s (%d)\n", path, ret); 623 624 return 0; 625 } 626 627 /* 628 * File must be with pattern .../jit-XXXX.dump 629 * where XXXX is the PID of the process which did the mmap() 630 * as captured in the RECORD_MMAP record 631 */ 632 static int 633 jit_detect(char *mmap_name, pid_t pid) 634 { 635 char *p; 636 char *end = NULL; 637 pid_t pid2; 638 639 if (verbose > 2) 640 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 641 /* 642 * get file name 643 */ 644 p = strrchr(mmap_name, '/'); 645 if (!p) 646 return -1; 647 648 /* 649 * match prefix 650 */ 651 if (strncmp(p, "/jit-", 5)) 652 return -1; 653 654 /* 655 * skip prefix 656 */ 657 p += 5; 658 659 /* 660 * must be followed by a pid 661 */ 662 if (!isdigit(*p)) 663 return -1; 664 665 pid2 = (int)strtol(p, &end, 10); 666 if (!end) 667 return -1; 668 669 /* 670 * pid does not match mmap pid 671 * pid==0 in system-wide mode (synthesized) 672 */ 673 if (pid && pid2 != pid) 674 return -1; 675 /* 676 * validate suffix 677 */ 678 if (strcmp(end, ".dump")) 679 return -1; 680 681 if (verbose > 0) 682 fprintf(stderr, "jit marker found: %s\n", mmap_name); 683 684 return 0; 685 } 686 687 int 688 jit_process(struct perf_session *session, 689 struct perf_data_file *output, 690 struct machine *machine, 691 char *filename, 692 pid_t pid, 693 u64 *nbytes) 694 { 695 struct perf_evsel *first; 696 struct jit_buf_desc jd; 697 int ret; 698 699 /* 700 * first, detect marker mmap (i.e., the jitdump mmap) 701 */ 702 if (jit_detect(filename, pid)) 703 return 0; 704 705 memset(&jd, 0, sizeof(jd)); 706 707 jd.session = session; 708 jd.output = output; 709 jd.machine = machine; 710 711 /* 712 * track sample_type to compute id_all layout 713 * perf sets the same sample type to all events as of now 714 */ 715 first = perf_evlist__first(session->evlist); 716 jd.sample_type = first->attr.sample_type; 717 718 *nbytes = 0; 719 720 ret = jit_inject(&jd, filename); 721 if (!ret) { 722 *nbytes = jd.bytes_written; 723 ret = 1; 724 } 725 726 return ret; 727 } 728