1 #include <sys/types.h> 2 #include <stdio.h> 3 #include <stdlib.h> 4 #include <string.h> 5 #include <fcntl.h> 6 #include <unistd.h> 7 #include <inttypes.h> 8 #include <byteswap.h> 9 #include <sys/stat.h> 10 #include <sys/mman.h> 11 12 #include "util.h" 13 #include "event.h" 14 #include "debug.h" 15 #include "evlist.h" 16 #include "symbol.h" 17 #include "strlist.h" 18 #include <elf.h> 19 20 #include "session.h" 21 #include "jit.h" 22 #include "jitdump.h" 23 #include "genelf.h" 24 #include "../builtin.h" 25 26 struct jit_buf_desc { 27 struct perf_data_file *output; 28 struct perf_session *session; 29 struct machine *machine; 30 union jr_entry *entry; 31 void *buf; 32 uint64_t sample_type; 33 size_t bufsize; 34 FILE *in; 35 bool needs_bswap; /* handles cross-endianess */ 36 void *debug_data; 37 size_t nr_debug_entries; 38 uint32_t code_load_count; 39 u64 bytes_written; 40 struct rb_root code_root; 41 char dir[PATH_MAX]; 42 }; 43 44 struct debug_line_info { 45 unsigned long vma; 46 unsigned int lineno; 47 /* The filename format is unspecified, absolute path, relative etc. */ 48 char const filename[0]; 49 }; 50 51 struct jit_tool { 52 struct perf_tool tool; 53 struct perf_data_file output; 54 struct perf_data_file input; 55 u64 bytes_written; 56 }; 57 58 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 59 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 60 61 static int 62 jit_emit_elf(char *filename, 63 const char *sym, 64 uint64_t code_addr, 65 const void *code, 66 int csize, 67 void *debug, 68 int nr_debug_entries) 69 { 70 int ret, fd; 71 72 if (verbose > 0) 73 fprintf(stderr, "write ELF image %s\n", filename); 74 75 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 76 if (fd == -1) { 77 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 78 return -1; 79 } 80 81 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries); 82 83 close(fd); 84 85 if (ret) 86 unlink(filename); 87 88 return ret; 89 } 90 91 static void 92 jit_close(struct jit_buf_desc *jd) 93 { 94 if (!(jd && jd->in)) 95 return; 96 funlockfile(jd->in); 97 fclose(jd->in); 98 jd->in = NULL; 99 } 100 101 static int 102 jit_validate_events(struct perf_session *session) 103 { 104 struct perf_evsel *evsel; 105 106 /* 107 * check that all events use CLOCK_MONOTONIC 108 */ 109 evlist__for_each(session->evlist, evsel) { 110 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC) 111 return -1; 112 } 113 return 0; 114 } 115 116 static int 117 jit_open(struct jit_buf_desc *jd, const char *name) 118 { 119 struct jitheader header; 120 struct jr_prefix *prefix; 121 ssize_t bs, bsz = 0; 122 void *n, *buf = NULL; 123 int ret, retval = -1; 124 125 jd->in = fopen(name, "r"); 126 if (!jd->in) 127 return -1; 128 129 bsz = hmax(sizeof(header), sizeof(*prefix)); 130 131 buf = malloc(bsz); 132 if (!buf) 133 goto error; 134 135 /* 136 * protect from writer modifying the file while we are reading it 137 */ 138 flockfile(jd->in); 139 140 ret = fread(buf, sizeof(header), 1, jd->in); 141 if (ret != 1) 142 goto error; 143 144 memcpy(&header, buf, sizeof(header)); 145 146 if (header.magic != JITHEADER_MAGIC) { 147 if (header.magic != JITHEADER_MAGIC_SW) 148 goto error; 149 jd->needs_bswap = true; 150 } 151 152 if (jd->needs_bswap) { 153 header.version = bswap_32(header.version); 154 header.total_size = bswap_32(header.total_size); 155 header.pid = bswap_32(header.pid); 156 header.elf_mach = bswap_32(header.elf_mach); 157 header.timestamp = bswap_64(header.timestamp); 158 header.flags = bswap_64(header.flags); 159 } 160 161 if (verbose > 2) 162 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\n", 163 header.version, 164 header.total_size, 165 (unsigned long long)header.timestamp, 166 header.pid, 167 header.elf_mach); 168 169 if (header.flags & JITDUMP_FLAGS_RESERVED) { 170 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 171 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 172 goto error; 173 } 174 175 /* 176 * validate event is using the correct clockid 177 */ 178 if (jit_validate_events(jd->session)) { 179 pr_err("error, jitted code must be sampled with perf record -k 1\n"); 180 goto error; 181 } 182 183 bs = header.total_size - sizeof(header); 184 185 if (bs > bsz) { 186 n = realloc(buf, bs); 187 if (!n) 188 goto error; 189 bsz = bs; 190 buf = n; 191 /* read extra we do not know about */ 192 ret = fread(buf, bs - bsz, 1, jd->in); 193 if (ret != 1) 194 goto error; 195 } 196 /* 197 * keep dirname for generating files and mmap records 198 */ 199 strcpy(jd->dir, name); 200 dirname(jd->dir); 201 202 return 0; 203 error: 204 funlockfile(jd->in); 205 fclose(jd->in); 206 return retval; 207 } 208 209 static union jr_entry * 210 jit_get_next_entry(struct jit_buf_desc *jd) 211 { 212 struct jr_prefix *prefix; 213 union jr_entry *jr; 214 void *addr; 215 size_t bs, size; 216 int id, ret; 217 218 if (!(jd && jd->in)) 219 return NULL; 220 221 if (jd->buf == NULL) { 222 size_t sz = getpagesize(); 223 if (sz < sizeof(*prefix)) 224 sz = sizeof(*prefix); 225 226 jd->buf = malloc(sz); 227 if (jd->buf == NULL) 228 return NULL; 229 230 jd->bufsize = sz; 231 } 232 233 prefix = jd->buf; 234 235 /* 236 * file is still locked at this point 237 */ 238 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 239 if (ret != 1) 240 return NULL; 241 242 if (jd->needs_bswap) { 243 prefix->id = bswap_32(prefix->id); 244 prefix->total_size = bswap_32(prefix->total_size); 245 prefix->timestamp = bswap_64(prefix->timestamp); 246 } 247 id = prefix->id; 248 size = prefix->total_size; 249 250 bs = (size_t)size; 251 if (bs < sizeof(*prefix)) 252 return NULL; 253 254 if (id >= JIT_CODE_MAX) { 255 pr_warning("next_entry: unknown prefix %d, skipping\n", id); 256 return NULL; 257 } 258 if (bs > jd->bufsize) { 259 void *n; 260 n = realloc(jd->buf, bs); 261 if (!n) 262 return NULL; 263 jd->buf = n; 264 jd->bufsize = bs; 265 } 266 267 addr = ((void *)jd->buf) + sizeof(*prefix); 268 269 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 270 if (ret != 1) 271 return NULL; 272 273 jr = (union jr_entry *)jd->buf; 274 275 switch(id) { 276 case JIT_CODE_DEBUG_INFO: 277 if (jd->needs_bswap) { 278 uint64_t n; 279 jr->info.code_addr = bswap_64(jr->info.code_addr); 280 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 281 for (n = 0 ; n < jr->info.nr_entry; n++) { 282 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 283 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 284 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 285 } 286 } 287 break; 288 case JIT_CODE_CLOSE: 289 break; 290 case JIT_CODE_LOAD: 291 if (jd->needs_bswap) { 292 jr->load.pid = bswap_32(jr->load.pid); 293 jr->load.tid = bswap_32(jr->load.tid); 294 jr->load.vma = bswap_64(jr->load.vma); 295 jr->load.code_addr = bswap_64(jr->load.code_addr); 296 jr->load.code_size = bswap_64(jr->load.code_size); 297 jr->load.code_index= bswap_64(jr->load.code_index); 298 } 299 jd->code_load_count++; 300 break; 301 case JIT_CODE_MOVE: 302 if (jd->needs_bswap) { 303 jr->move.pid = bswap_32(jr->move.pid); 304 jr->move.tid = bswap_32(jr->move.tid); 305 jr->move.vma = bswap_64(jr->move.vma); 306 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 307 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 308 jr->move.code_size = bswap_64(jr->move.code_size); 309 jr->move.code_index = bswap_64(jr->move.code_index); 310 } 311 break; 312 case JIT_CODE_MAX: 313 default: 314 return NULL; 315 } 316 return jr; 317 } 318 319 static int 320 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 321 { 322 ssize_t size; 323 324 size = perf_data_file__write(jd->output, event, event->header.size); 325 if (size < 0) 326 return -1; 327 328 jd->bytes_written += size; 329 return 0; 330 } 331 332 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 333 { 334 struct perf_sample sample; 335 union perf_event *event; 336 struct perf_tool *tool = jd->session->tool; 337 uint64_t code, addr; 338 uintptr_t uaddr; 339 char *filename; 340 struct stat st; 341 size_t size; 342 u16 idr_size; 343 const char *sym; 344 uint32_t count; 345 int ret, csize; 346 pid_t pid, tid; 347 struct { 348 u32 pid, tid; 349 u64 time; 350 } *id; 351 352 pid = jr->load.pid; 353 tid = jr->load.tid; 354 csize = jr->load.code_size; 355 addr = jr->load.code_addr; 356 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 357 code = (unsigned long)jr + jr->load.p.total_size - csize; 358 count = jr->load.code_index; 359 idr_size = jd->machine->id_hdr_size; 360 361 event = calloc(1, sizeof(*event) + idr_size); 362 if (!event) 363 return -1; 364 365 filename = event->mmap2.filename; 366 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 367 jd->dir, 368 pid, 369 count); 370 371 size++; /* for \0 */ 372 373 size = PERF_ALIGN(size, sizeof(u64)); 374 uaddr = (uintptr_t)code; 375 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries); 376 377 if (jd->debug_data && jd->nr_debug_entries) { 378 free(jd->debug_data); 379 jd->debug_data = NULL; 380 jd->nr_debug_entries = 0; 381 } 382 383 if (ret) { 384 free(event); 385 return -1; 386 } 387 if (stat(filename, &st)) 388 memset(&st, 0, sizeof(stat)); 389 390 event->mmap2.header.type = PERF_RECORD_MMAP2; 391 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 392 event->mmap2.header.size = (sizeof(event->mmap2) - 393 (sizeof(event->mmap2.filename) - size) + idr_size); 394 395 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 396 event->mmap2.start = addr; 397 event->mmap2.len = csize; 398 event->mmap2.pid = pid; 399 event->mmap2.tid = tid; 400 event->mmap2.ino = st.st_ino; 401 event->mmap2.maj = major(st.st_dev); 402 event->mmap2.min = minor(st.st_dev); 403 event->mmap2.prot = st.st_mode; 404 event->mmap2.flags = MAP_SHARED; 405 event->mmap2.ino_generation = 1; 406 407 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 408 if (jd->sample_type & PERF_SAMPLE_TID) { 409 id->pid = pid; 410 id->tid = tid; 411 } 412 if (jd->sample_type & PERF_SAMPLE_TIME) 413 id->time = jr->load.p.timestamp; 414 415 /* 416 * create pseudo sample to induce dso hit increment 417 * use first address as sample address 418 */ 419 memset(&sample, 0, sizeof(sample)); 420 sample.cpumode = PERF_RECORD_MISC_USER; 421 sample.pid = pid; 422 sample.tid = tid; 423 sample.time = id->time; 424 sample.ip = addr; 425 426 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 427 if (ret) 428 return ret; 429 430 ret = jit_inject_event(jd, event); 431 /* 432 * mark dso as use to generate buildid in the header 433 */ 434 if (!ret) 435 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 436 437 return ret; 438 } 439 440 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 441 { 442 struct perf_sample sample; 443 union perf_event *event; 444 struct perf_tool *tool = jd->session->tool; 445 char *filename; 446 size_t size; 447 struct stat st; 448 u16 idr_size; 449 int ret; 450 pid_t pid, tid; 451 struct { 452 u32 pid, tid; 453 u64 time; 454 } *id; 455 456 pid = jr->move.pid; 457 tid = jr->move.tid; 458 idr_size = jd->machine->id_hdr_size; 459 460 /* 461 * +16 to account for sample_id_all (hack) 462 */ 463 event = calloc(1, sizeof(*event) + 16); 464 if (!event) 465 return -1; 466 467 filename = event->mmap2.filename; 468 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 469 jd->dir, 470 pid, 471 jr->move.code_index); 472 473 size++; /* for \0 */ 474 475 if (stat(filename, &st)) 476 memset(&st, 0, sizeof(stat)); 477 478 size = PERF_ALIGN(size, sizeof(u64)); 479 480 event->mmap2.header.type = PERF_RECORD_MMAP2; 481 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 482 event->mmap2.header.size = (sizeof(event->mmap2) - 483 (sizeof(event->mmap2.filename) - size) + idr_size); 484 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 485 event->mmap2.start = jr->move.new_code_addr; 486 event->mmap2.len = jr->move.code_size; 487 event->mmap2.pid = pid; 488 event->mmap2.tid = tid; 489 event->mmap2.ino = st.st_ino; 490 event->mmap2.maj = major(st.st_dev); 491 event->mmap2.min = minor(st.st_dev); 492 event->mmap2.prot = st.st_mode; 493 event->mmap2.flags = MAP_SHARED; 494 event->mmap2.ino_generation = 1; 495 496 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 497 if (jd->sample_type & PERF_SAMPLE_TID) { 498 id->pid = pid; 499 id->tid = tid; 500 } 501 if (jd->sample_type & PERF_SAMPLE_TIME) 502 id->time = jr->load.p.timestamp; 503 504 /* 505 * create pseudo sample to induce dso hit increment 506 * use first address as sample address 507 */ 508 memset(&sample, 0, sizeof(sample)); 509 sample.cpumode = PERF_RECORD_MISC_USER; 510 sample.pid = pid; 511 sample.tid = tid; 512 sample.time = id->time; 513 sample.ip = jr->move.new_code_addr; 514 515 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 516 if (ret) 517 return ret; 518 519 ret = jit_inject_event(jd, event); 520 if (!ret) 521 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 522 523 return ret; 524 } 525 526 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 527 { 528 void *data; 529 size_t sz; 530 531 if (!(jd && jr)) 532 return -1; 533 534 sz = jr->prefix.total_size - sizeof(jr->info); 535 data = malloc(sz); 536 if (!data) 537 return -1; 538 539 memcpy(data, &jr->info.entries, sz); 540 541 jd->debug_data = data; 542 543 /* 544 * we must use nr_entry instead of size here because 545 * we cannot distinguish actual entry from padding otherwise 546 */ 547 jd->nr_debug_entries = jr->info.nr_entry; 548 549 return 0; 550 } 551 552 static int 553 jit_process_dump(struct jit_buf_desc *jd) 554 { 555 union jr_entry *jr; 556 int ret; 557 558 while ((jr = jit_get_next_entry(jd))) { 559 switch(jr->prefix.id) { 560 case JIT_CODE_LOAD: 561 ret = jit_repipe_code_load(jd, jr); 562 break; 563 case JIT_CODE_MOVE: 564 ret = jit_repipe_code_move(jd, jr); 565 break; 566 case JIT_CODE_DEBUG_INFO: 567 ret = jit_repipe_debug_info(jd, jr); 568 break; 569 default: 570 ret = 0; 571 continue; 572 } 573 } 574 return ret; 575 } 576 577 static int 578 jit_inject(struct jit_buf_desc *jd, char *path) 579 { 580 int ret; 581 582 if (verbose > 0) 583 fprintf(stderr, "injecting: %s\n", path); 584 585 ret = jit_open(jd, path); 586 if (ret) 587 return -1; 588 589 ret = jit_process_dump(jd); 590 591 jit_close(jd); 592 593 if (verbose > 0) 594 fprintf(stderr, "injected: %s (%d)\n", path, ret); 595 596 return 0; 597 } 598 599 /* 600 * File must be with pattern .../jit-XXXX.dump 601 * where XXXX is the PID of the process which did the mmap() 602 * as captured in the RECORD_MMAP record 603 */ 604 static int 605 jit_detect(char *mmap_name, pid_t pid) 606 { 607 char *p; 608 char *end = NULL; 609 pid_t pid2; 610 611 if (verbose > 2) 612 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 613 /* 614 * get file name 615 */ 616 p = strrchr(mmap_name, '/'); 617 if (!p) 618 return -1; 619 620 /* 621 * match prefix 622 */ 623 if (strncmp(p, "/jit-", 5)) 624 return -1; 625 626 /* 627 * skip prefix 628 */ 629 p += 5; 630 631 /* 632 * must be followed by a pid 633 */ 634 if (!isdigit(*p)) 635 return -1; 636 637 pid2 = (int)strtol(p, &end, 10); 638 if (!end) 639 return -1; 640 641 /* 642 * pid does not match mmap pid 643 * pid==0 in system-wide mode (synthesized) 644 */ 645 if (pid && pid2 != pid) 646 return -1; 647 /* 648 * validate suffix 649 */ 650 if (strcmp(end, ".dump")) 651 return -1; 652 653 if (verbose > 0) 654 fprintf(stderr, "jit marker found: %s\n", mmap_name); 655 656 return 0; 657 } 658 659 int 660 jit_process(struct perf_session *session, 661 struct perf_data_file *output, 662 struct machine *machine, 663 char *filename, 664 pid_t pid, 665 u64 *nbytes) 666 { 667 struct perf_evsel *first; 668 struct jit_buf_desc jd; 669 int ret; 670 671 /* 672 * first, detect marker mmap (i.e., the jitdump mmap) 673 */ 674 if (jit_detect(filename, pid)) 675 return 0; 676 677 memset(&jd, 0, sizeof(jd)); 678 679 jd.session = session; 680 jd.output = output; 681 jd.machine = machine; 682 683 /* 684 * track sample_type to compute id_all layout 685 * perf sets the same sample type to all events as of now 686 */ 687 first = perf_evlist__first(session->evlist); 688 jd.sample_type = first->attr.sample_type; 689 690 *nbytes = 0; 691 692 ret = jit_inject(&jd, filename); 693 if (!ret) { 694 *nbytes = jd.bytes_written; 695 ret = 1; 696 } 697 698 return ret; 699 } 700