1 #include <sys/types.h> 2 #include <stdio.h> 3 #include <stdlib.h> 4 #include <string.h> 5 #include <fcntl.h> 6 #include <unistd.h> 7 #include <inttypes.h> 8 #include <byteswap.h> 9 #include <sys/stat.h> 10 #include <sys/mman.h> 11 12 #include "util.h" 13 #include "event.h" 14 #include "debug.h" 15 #include "evlist.h" 16 #include "symbol.h" 17 #include "strlist.h" 18 #include <elf.h> 19 20 #include "tsc.h" 21 #include "session.h" 22 #include "jit.h" 23 #include "jitdump.h" 24 #include "genelf.h" 25 #include "../builtin.h" 26 27 struct jit_buf_desc { 28 struct perf_data_file *output; 29 struct perf_session *session; 30 struct machine *machine; 31 union jr_entry *entry; 32 void *buf; 33 uint64_t sample_type; 34 size_t bufsize; 35 FILE *in; 36 bool needs_bswap; /* handles cross-endianess */ 37 bool use_arch_timestamp; 38 void *debug_data; 39 size_t nr_debug_entries; 40 uint32_t code_load_count; 41 u64 bytes_written; 42 struct rb_root code_root; 43 char dir[PATH_MAX]; 44 }; 45 46 struct debug_line_info { 47 unsigned long vma; 48 unsigned int lineno; 49 /* The filename format is unspecified, absolute path, relative etc. */ 50 char const filename[0]; 51 }; 52 53 struct jit_tool { 54 struct perf_tool tool; 55 struct perf_data_file output; 56 struct perf_data_file input; 57 u64 bytes_written; 58 }; 59 60 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 61 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 62 63 static int 64 jit_emit_elf(char *filename, 65 const char *sym, 66 uint64_t code_addr, 67 const void *code, 68 int csize, 69 void *debug, 70 int nr_debug_entries) 71 { 72 int ret, fd; 73 74 if (verbose > 0) 75 fprintf(stderr, "write ELF image %s\n", filename); 76 77 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 78 if (fd == -1) { 79 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 80 return -1; 81 } 82 83 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries); 84 85 close(fd); 86 87 if (ret) 88 unlink(filename); 89 90 return ret; 91 } 92 93 static void 94 jit_close(struct jit_buf_desc *jd) 95 { 96 if (!(jd && jd->in)) 97 return; 98 funlockfile(jd->in); 99 fclose(jd->in); 100 jd->in = NULL; 101 } 102 103 static int 104 jit_validate_events(struct perf_session *session) 105 { 106 struct perf_evsel *evsel; 107 108 /* 109 * check that all events use CLOCK_MONOTONIC 110 */ 111 evlist__for_each(session->evlist, evsel) { 112 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC) 113 return -1; 114 } 115 return 0; 116 } 117 118 static int 119 jit_open(struct jit_buf_desc *jd, const char *name) 120 { 121 struct jitheader header; 122 struct jr_prefix *prefix; 123 ssize_t bs, bsz = 0; 124 void *n, *buf = NULL; 125 int ret, retval = -1; 126 127 jd->in = fopen(name, "r"); 128 if (!jd->in) 129 return -1; 130 131 bsz = hmax(sizeof(header), sizeof(*prefix)); 132 133 buf = malloc(bsz); 134 if (!buf) 135 goto error; 136 137 /* 138 * protect from writer modifying the file while we are reading it 139 */ 140 flockfile(jd->in); 141 142 ret = fread(buf, sizeof(header), 1, jd->in); 143 if (ret != 1) 144 goto error; 145 146 memcpy(&header, buf, sizeof(header)); 147 148 if (header.magic != JITHEADER_MAGIC) { 149 if (header.magic != JITHEADER_MAGIC_SW) 150 goto error; 151 jd->needs_bswap = true; 152 } 153 154 if (jd->needs_bswap) { 155 header.version = bswap_32(header.version); 156 header.total_size = bswap_32(header.total_size); 157 header.pid = bswap_32(header.pid); 158 header.elf_mach = bswap_32(header.elf_mach); 159 header.timestamp = bswap_64(header.timestamp); 160 header.flags = bswap_64(header.flags); 161 } 162 163 jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP; 164 165 if (verbose > 2) 166 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n", 167 header.version, 168 header.total_size, 169 (unsigned long long)header.timestamp, 170 header.pid, 171 header.elf_mach, 172 jd->use_arch_timestamp); 173 174 if (header.flags & JITDUMP_FLAGS_RESERVED) { 175 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 176 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 177 goto error; 178 } 179 180 if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) { 181 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n"); 182 goto error; 183 } 184 185 /* 186 * validate event is using the correct clockid 187 */ 188 if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) { 189 pr_err("error, jitted code must be sampled with perf record -k 1\n"); 190 goto error; 191 } 192 193 bs = header.total_size - sizeof(header); 194 195 if (bs > bsz) { 196 n = realloc(buf, bs); 197 if (!n) 198 goto error; 199 bsz = bs; 200 buf = n; 201 /* read extra we do not know about */ 202 ret = fread(buf, bs - bsz, 1, jd->in); 203 if (ret != 1) 204 goto error; 205 } 206 /* 207 * keep dirname for generating files and mmap records 208 */ 209 strcpy(jd->dir, name); 210 dirname(jd->dir); 211 212 return 0; 213 error: 214 funlockfile(jd->in); 215 fclose(jd->in); 216 return retval; 217 } 218 219 static union jr_entry * 220 jit_get_next_entry(struct jit_buf_desc *jd) 221 { 222 struct jr_prefix *prefix; 223 union jr_entry *jr; 224 void *addr; 225 size_t bs, size; 226 int id, ret; 227 228 if (!(jd && jd->in)) 229 return NULL; 230 231 if (jd->buf == NULL) { 232 size_t sz = getpagesize(); 233 if (sz < sizeof(*prefix)) 234 sz = sizeof(*prefix); 235 236 jd->buf = malloc(sz); 237 if (jd->buf == NULL) 238 return NULL; 239 240 jd->bufsize = sz; 241 } 242 243 prefix = jd->buf; 244 245 /* 246 * file is still locked at this point 247 */ 248 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 249 if (ret != 1) 250 return NULL; 251 252 if (jd->needs_bswap) { 253 prefix->id = bswap_32(prefix->id); 254 prefix->total_size = bswap_32(prefix->total_size); 255 prefix->timestamp = bswap_64(prefix->timestamp); 256 } 257 id = prefix->id; 258 size = prefix->total_size; 259 260 bs = (size_t)size; 261 if (bs < sizeof(*prefix)) 262 return NULL; 263 264 if (id >= JIT_CODE_MAX) { 265 pr_warning("next_entry: unknown prefix %d, skipping\n", id); 266 return NULL; 267 } 268 if (bs > jd->bufsize) { 269 void *n; 270 n = realloc(jd->buf, bs); 271 if (!n) 272 return NULL; 273 jd->buf = n; 274 jd->bufsize = bs; 275 } 276 277 addr = ((void *)jd->buf) + sizeof(*prefix); 278 279 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 280 if (ret != 1) 281 return NULL; 282 283 jr = (union jr_entry *)jd->buf; 284 285 switch(id) { 286 case JIT_CODE_DEBUG_INFO: 287 if (jd->needs_bswap) { 288 uint64_t n; 289 jr->info.code_addr = bswap_64(jr->info.code_addr); 290 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 291 for (n = 0 ; n < jr->info.nr_entry; n++) { 292 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 293 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 294 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 295 } 296 } 297 break; 298 case JIT_CODE_CLOSE: 299 break; 300 case JIT_CODE_LOAD: 301 if (jd->needs_bswap) { 302 jr->load.pid = bswap_32(jr->load.pid); 303 jr->load.tid = bswap_32(jr->load.tid); 304 jr->load.vma = bswap_64(jr->load.vma); 305 jr->load.code_addr = bswap_64(jr->load.code_addr); 306 jr->load.code_size = bswap_64(jr->load.code_size); 307 jr->load.code_index= bswap_64(jr->load.code_index); 308 } 309 jd->code_load_count++; 310 break; 311 case JIT_CODE_MOVE: 312 if (jd->needs_bswap) { 313 jr->move.pid = bswap_32(jr->move.pid); 314 jr->move.tid = bswap_32(jr->move.tid); 315 jr->move.vma = bswap_64(jr->move.vma); 316 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 317 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 318 jr->move.code_size = bswap_64(jr->move.code_size); 319 jr->move.code_index = bswap_64(jr->move.code_index); 320 } 321 break; 322 case JIT_CODE_MAX: 323 default: 324 return NULL; 325 } 326 return jr; 327 } 328 329 static int 330 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 331 { 332 ssize_t size; 333 334 size = perf_data_file__write(jd->output, event, event->header.size); 335 if (size < 0) 336 return -1; 337 338 jd->bytes_written += size; 339 return 0; 340 } 341 342 static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) 343 { 344 struct perf_tsc_conversion tc; 345 346 if (!jd->use_arch_timestamp) 347 return timestamp; 348 349 tc.time_shift = jd->session->time_conv.time_shift; 350 tc.time_mult = jd->session->time_conv.time_mult; 351 tc.time_zero = jd->session->time_conv.time_zero; 352 353 if (!tc.time_mult) 354 return 0; 355 356 return tsc_to_perf_time(timestamp, &tc); 357 } 358 359 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 360 { 361 struct perf_sample sample; 362 union perf_event *event; 363 struct perf_tool *tool = jd->session->tool; 364 uint64_t code, addr; 365 uintptr_t uaddr; 366 char *filename; 367 struct stat st; 368 size_t size; 369 u16 idr_size; 370 const char *sym; 371 uint32_t count; 372 int ret, csize; 373 pid_t pid, tid; 374 struct { 375 u32 pid, tid; 376 u64 time; 377 } *id; 378 379 pid = jr->load.pid; 380 tid = jr->load.tid; 381 csize = jr->load.code_size; 382 addr = jr->load.code_addr; 383 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 384 code = (unsigned long)jr + jr->load.p.total_size - csize; 385 count = jr->load.code_index; 386 idr_size = jd->machine->id_hdr_size; 387 388 event = calloc(1, sizeof(*event) + idr_size); 389 if (!event) 390 return -1; 391 392 filename = event->mmap2.filename; 393 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 394 jd->dir, 395 pid, 396 count); 397 398 size++; /* for \0 */ 399 400 size = PERF_ALIGN(size, sizeof(u64)); 401 uaddr = (uintptr_t)code; 402 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries); 403 404 if (jd->debug_data && jd->nr_debug_entries) { 405 free(jd->debug_data); 406 jd->debug_data = NULL; 407 jd->nr_debug_entries = 0; 408 } 409 410 if (ret) { 411 free(event); 412 return -1; 413 } 414 if (stat(filename, &st)) 415 memset(&st, 0, sizeof(st)); 416 417 event->mmap2.header.type = PERF_RECORD_MMAP2; 418 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 419 event->mmap2.header.size = (sizeof(event->mmap2) - 420 (sizeof(event->mmap2.filename) - size) + idr_size); 421 422 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 423 event->mmap2.start = addr; 424 event->mmap2.len = csize; 425 event->mmap2.pid = pid; 426 event->mmap2.tid = tid; 427 event->mmap2.ino = st.st_ino; 428 event->mmap2.maj = major(st.st_dev); 429 event->mmap2.min = minor(st.st_dev); 430 event->mmap2.prot = st.st_mode; 431 event->mmap2.flags = MAP_SHARED; 432 event->mmap2.ino_generation = 1; 433 434 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 435 if (jd->sample_type & PERF_SAMPLE_TID) { 436 id->pid = pid; 437 id->tid = tid; 438 } 439 if (jd->sample_type & PERF_SAMPLE_TIME) 440 id->time = convert_timestamp(jd, jr->load.p.timestamp); 441 442 /* 443 * create pseudo sample to induce dso hit increment 444 * use first address as sample address 445 */ 446 memset(&sample, 0, sizeof(sample)); 447 sample.cpumode = PERF_RECORD_MISC_USER; 448 sample.pid = pid; 449 sample.tid = tid; 450 sample.time = id->time; 451 sample.ip = addr; 452 453 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 454 if (ret) 455 return ret; 456 457 ret = jit_inject_event(jd, event); 458 /* 459 * mark dso as use to generate buildid in the header 460 */ 461 if (!ret) 462 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 463 464 return ret; 465 } 466 467 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 468 { 469 struct perf_sample sample; 470 union perf_event *event; 471 struct perf_tool *tool = jd->session->tool; 472 char *filename; 473 size_t size; 474 struct stat st; 475 u16 idr_size; 476 int ret; 477 pid_t pid, tid; 478 struct { 479 u32 pid, tid; 480 u64 time; 481 } *id; 482 483 pid = jr->move.pid; 484 tid = jr->move.tid; 485 idr_size = jd->machine->id_hdr_size; 486 487 /* 488 * +16 to account for sample_id_all (hack) 489 */ 490 event = calloc(1, sizeof(*event) + 16); 491 if (!event) 492 return -1; 493 494 filename = event->mmap2.filename; 495 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 496 jd->dir, 497 pid, 498 jr->move.code_index); 499 500 size++; /* for \0 */ 501 502 if (stat(filename, &st)) 503 memset(&st, 0, sizeof(st)); 504 505 size = PERF_ALIGN(size, sizeof(u64)); 506 507 event->mmap2.header.type = PERF_RECORD_MMAP2; 508 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 509 event->mmap2.header.size = (sizeof(event->mmap2) - 510 (sizeof(event->mmap2.filename) - size) + idr_size); 511 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 512 event->mmap2.start = jr->move.new_code_addr; 513 event->mmap2.len = jr->move.code_size; 514 event->mmap2.pid = pid; 515 event->mmap2.tid = tid; 516 event->mmap2.ino = st.st_ino; 517 event->mmap2.maj = major(st.st_dev); 518 event->mmap2.min = minor(st.st_dev); 519 event->mmap2.prot = st.st_mode; 520 event->mmap2.flags = MAP_SHARED; 521 event->mmap2.ino_generation = 1; 522 523 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 524 if (jd->sample_type & PERF_SAMPLE_TID) { 525 id->pid = pid; 526 id->tid = tid; 527 } 528 if (jd->sample_type & PERF_SAMPLE_TIME) 529 id->time = convert_timestamp(jd, jr->load.p.timestamp); 530 531 /* 532 * create pseudo sample to induce dso hit increment 533 * use first address as sample address 534 */ 535 memset(&sample, 0, sizeof(sample)); 536 sample.cpumode = PERF_RECORD_MISC_USER; 537 sample.pid = pid; 538 sample.tid = tid; 539 sample.time = id->time; 540 sample.ip = jr->move.new_code_addr; 541 542 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 543 if (ret) 544 return ret; 545 546 ret = jit_inject_event(jd, event); 547 if (!ret) 548 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 549 550 return ret; 551 } 552 553 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 554 { 555 void *data; 556 size_t sz; 557 558 if (!(jd && jr)) 559 return -1; 560 561 sz = jr->prefix.total_size - sizeof(jr->info); 562 data = malloc(sz); 563 if (!data) 564 return -1; 565 566 memcpy(data, &jr->info.entries, sz); 567 568 jd->debug_data = data; 569 570 /* 571 * we must use nr_entry instead of size here because 572 * we cannot distinguish actual entry from padding otherwise 573 */ 574 jd->nr_debug_entries = jr->info.nr_entry; 575 576 return 0; 577 } 578 579 static int 580 jit_process_dump(struct jit_buf_desc *jd) 581 { 582 union jr_entry *jr; 583 int ret; 584 585 while ((jr = jit_get_next_entry(jd))) { 586 switch(jr->prefix.id) { 587 case JIT_CODE_LOAD: 588 ret = jit_repipe_code_load(jd, jr); 589 break; 590 case JIT_CODE_MOVE: 591 ret = jit_repipe_code_move(jd, jr); 592 break; 593 case JIT_CODE_DEBUG_INFO: 594 ret = jit_repipe_debug_info(jd, jr); 595 break; 596 default: 597 ret = 0; 598 continue; 599 } 600 } 601 return ret; 602 } 603 604 static int 605 jit_inject(struct jit_buf_desc *jd, char *path) 606 { 607 int ret; 608 609 if (verbose > 0) 610 fprintf(stderr, "injecting: %s\n", path); 611 612 ret = jit_open(jd, path); 613 if (ret) 614 return -1; 615 616 ret = jit_process_dump(jd); 617 618 jit_close(jd); 619 620 if (verbose > 0) 621 fprintf(stderr, "injected: %s (%d)\n", path, ret); 622 623 return 0; 624 } 625 626 /* 627 * File must be with pattern .../jit-XXXX.dump 628 * where XXXX is the PID of the process which did the mmap() 629 * as captured in the RECORD_MMAP record 630 */ 631 static int 632 jit_detect(char *mmap_name, pid_t pid) 633 { 634 char *p; 635 char *end = NULL; 636 pid_t pid2; 637 638 if (verbose > 2) 639 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 640 /* 641 * get file name 642 */ 643 p = strrchr(mmap_name, '/'); 644 if (!p) 645 return -1; 646 647 /* 648 * match prefix 649 */ 650 if (strncmp(p, "/jit-", 5)) 651 return -1; 652 653 /* 654 * skip prefix 655 */ 656 p += 5; 657 658 /* 659 * must be followed by a pid 660 */ 661 if (!isdigit(*p)) 662 return -1; 663 664 pid2 = (int)strtol(p, &end, 10); 665 if (!end) 666 return -1; 667 668 /* 669 * pid does not match mmap pid 670 * pid==0 in system-wide mode (synthesized) 671 */ 672 if (pid && pid2 != pid) 673 return -1; 674 /* 675 * validate suffix 676 */ 677 if (strcmp(end, ".dump")) 678 return -1; 679 680 if (verbose > 0) 681 fprintf(stderr, "jit marker found: %s\n", mmap_name); 682 683 return 0; 684 } 685 686 int 687 jit_process(struct perf_session *session, 688 struct perf_data_file *output, 689 struct machine *machine, 690 char *filename, 691 pid_t pid, 692 u64 *nbytes) 693 { 694 struct perf_evsel *first; 695 struct jit_buf_desc jd; 696 int ret; 697 698 /* 699 * first, detect marker mmap (i.e., the jitdump mmap) 700 */ 701 if (jit_detect(filename, pid)) 702 return 0; 703 704 memset(&jd, 0, sizeof(jd)); 705 706 jd.session = session; 707 jd.output = output; 708 jd.machine = machine; 709 710 /* 711 * track sample_type to compute id_all layout 712 * perf sets the same sample type to all events as of now 713 */ 714 first = perf_evlist__first(session->evlist); 715 jd.sample_type = first->attr.sample_type; 716 717 *nbytes = 0; 718 719 ret = jit_inject(&jd, filename); 720 if (!ret) { 721 *nbytes = jd.bytes_written; 722 ret = 1; 723 } 724 725 return ret; 726 } 727