1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * auxtrace.c: AUX area trace support 4 * Copyright (c) 2013-2015, Intel Corporation. 5 */ 6 7 #include <inttypes.h> 8 #include <sys/types.h> 9 #include <sys/mman.h> 10 #include <stdbool.h> 11 #include <string.h> 12 #include <limits.h> 13 #include <errno.h> 14 15 #include <linux/kernel.h> 16 #include <linux/perf_event.h> 17 #include <linux/types.h> 18 #include <linux/bitops.h> 19 #include <linux/log2.h> 20 #include <linux/string.h> 21 #include <linux/time64.h> 22 23 #include <sys/param.h> 24 #include <stdlib.h> 25 #include <stdio.h> 26 #include <linux/list.h> 27 #include <linux/zalloc.h> 28 29 #include "evlist.h" 30 #include "dso.h" 31 #include "map.h" 32 #include "pmu.h" 33 #include "evsel.h" 34 #include "cpumap.h" 35 #include "symbol.h" 36 #include "thread_map.h" 37 #include "asm/bug.h" 38 #include "auxtrace.h" 39 40 #include <linux/hash.h> 41 42 #include "event.h" 43 #include "record.h" 44 #include "session.h" 45 #include "debug.h" 46 #include <subcmd/parse-options.h> 47 48 #include "cs-etm.h" 49 #include "intel-pt.h" 50 #include "intel-bts.h" 51 #include "arm-spe.h" 52 #include "s390-cpumsf.h" 53 #include "util.h" 54 55 #include <linux/ctype.h> 56 #include "symbol/kallsyms.h" 57 58 static bool auxtrace__dont_decode(struct perf_session *session) 59 { 60 return !session->itrace_synth_opts || 61 session->itrace_synth_opts->dont_decode; 62 } 63 64 int auxtrace_mmap__mmap(struct auxtrace_mmap *mm, 65 struct auxtrace_mmap_params *mp, 66 void *userpg, int fd) 67 { 68 struct perf_event_mmap_page *pc = userpg; 69 70 WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n"); 71 72 mm->userpg = userpg; 73 mm->mask = mp->mask; 74 mm->len = mp->len; 75 mm->prev = 0; 76 mm->idx = mp->idx; 77 mm->tid = mp->tid; 78 mm->cpu = mp->cpu; 79 80 if (!mp->len) { 81 mm->base = NULL; 82 return 0; 83 } 84 85 #if BITS_PER_LONG != 64 && !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT) 86 pr_err("Cannot use AUX area tracing mmaps\n"); 87 return -1; 88 #endif 89 90 pc->aux_offset = mp->offset; 91 pc->aux_size = mp->len; 92 93 mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset); 94 if (mm->base == MAP_FAILED) { 95 pr_debug2("failed to mmap AUX area\n"); 96 mm->base = NULL; 97 return -1; 98 } 99 100 return 0; 101 } 102 103 void auxtrace_mmap__munmap(struct auxtrace_mmap *mm) 104 { 105 if (mm->base) { 106 munmap(mm->base, mm->len); 107 mm->base = NULL; 108 } 109 } 110 111 void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp, 112 off_t auxtrace_offset, 113 unsigned int auxtrace_pages, 114 bool auxtrace_overwrite) 115 { 116 if (auxtrace_pages) { 117 mp->offset = auxtrace_offset; 118 mp->len = auxtrace_pages * (size_t)page_size; 119 mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0; 120 mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE); 121 pr_debug2("AUX area mmap length %zu\n", mp->len); 122 } else { 123 mp->len = 0; 124 } 125 } 126 127 void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, 128 struct evlist *evlist, int idx, 129 bool per_cpu) 130 { 131 mp->idx = idx; 132 133 if (per_cpu) { 134 mp->cpu = evlist->core.cpus->map[idx]; 135 if (evlist->core.threads) 136 mp->tid = perf_thread_map__pid(evlist->core.threads, 0); 137 else 138 mp->tid = -1; 139 } else { 140 mp->cpu = -1; 141 mp->tid = perf_thread_map__pid(evlist->core.threads, idx); 142 } 143 } 144 145 #define AUXTRACE_INIT_NR_QUEUES 32 146 147 static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues) 148 { 149 struct auxtrace_queue *queue_array; 150 unsigned int max_nr_queues, i; 151 152 max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue); 153 if (nr_queues > max_nr_queues) 154 return NULL; 155 156 queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue)); 157 if (!queue_array) 158 return NULL; 159 160 for (i = 0; i < nr_queues; i++) { 161 INIT_LIST_HEAD(&queue_array[i].head); 162 queue_array[i].priv = NULL; 163 } 164 165 return queue_array; 166 } 167 168 int auxtrace_queues__init(struct auxtrace_queues *queues) 169 { 170 queues->nr_queues = AUXTRACE_INIT_NR_QUEUES; 171 queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues); 172 if (!queues->queue_array) 173 return -ENOMEM; 174 return 0; 175 } 176 177 static int auxtrace_queues__grow(struct auxtrace_queues *queues, 178 unsigned int new_nr_queues) 179 { 180 unsigned int nr_queues = queues->nr_queues; 181 struct auxtrace_queue *queue_array; 182 unsigned int i; 183 184 if (!nr_queues) 185 nr_queues = AUXTRACE_INIT_NR_QUEUES; 186 187 while (nr_queues && nr_queues < new_nr_queues) 188 nr_queues <<= 1; 189 190 if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues) 191 return -EINVAL; 192 193 queue_array = auxtrace_alloc_queue_array(nr_queues); 194 if (!queue_array) 195 return -ENOMEM; 196 197 for (i = 0; i < queues->nr_queues; i++) { 198 list_splice_tail(&queues->queue_array[i].head, 199 &queue_array[i].head); 200 queue_array[i].tid = queues->queue_array[i].tid; 201 queue_array[i].cpu = queues->queue_array[i].cpu; 202 queue_array[i].set = queues->queue_array[i].set; 203 queue_array[i].priv = queues->queue_array[i].priv; 204 } 205 206 queues->nr_queues = nr_queues; 207 queues->queue_array = queue_array; 208 209 return 0; 210 } 211 212 static void *auxtrace_copy_data(u64 size, struct perf_session *session) 213 { 214 int fd = perf_data__fd(session->data); 215 void *p; 216 ssize_t ret; 217 218 if (size > SSIZE_MAX) 219 return NULL; 220 221 p = malloc(size); 222 if (!p) 223 return NULL; 224 225 ret = readn(fd, p, size); 226 if (ret != (ssize_t)size) { 227 free(p); 228 return NULL; 229 } 230 231 return p; 232 } 233 234 static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues, 235 unsigned int idx, 236 struct auxtrace_buffer *buffer) 237 { 238 struct auxtrace_queue *queue; 239 int err; 240 241 if (idx >= queues->nr_queues) { 242 err = auxtrace_queues__grow(queues, idx + 1); 243 if (err) 244 return err; 245 } 246 247 queue = &queues->queue_array[idx]; 248 249 if (!queue->set) { 250 queue->set = true; 251 queue->tid = buffer->tid; 252 queue->cpu = buffer->cpu; 253 } else if (buffer->cpu != queue->cpu || buffer->tid != queue->tid) { 254 pr_err("auxtrace queue conflict: cpu %d, tid %d vs cpu %d, tid %d\n", 255 queue->cpu, queue->tid, buffer->cpu, buffer->tid); 256 return -EINVAL; 257 } 258 259 buffer->buffer_nr = queues->next_buffer_nr++; 260 261 list_add_tail(&buffer->list, &queue->head); 262 263 queues->new_data = true; 264 queues->populated = true; 265 266 return 0; 267 } 268 269 /* Limit buffers to 32MiB on 32-bit */ 270 #define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024) 271 272 static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues, 273 unsigned int idx, 274 struct auxtrace_buffer *buffer) 275 { 276 u64 sz = buffer->size; 277 bool consecutive = false; 278 struct auxtrace_buffer *b; 279 int err; 280 281 while (sz > BUFFER_LIMIT_FOR_32_BIT) { 282 b = memdup(buffer, sizeof(struct auxtrace_buffer)); 283 if (!b) 284 return -ENOMEM; 285 b->size = BUFFER_LIMIT_FOR_32_BIT; 286 b->consecutive = consecutive; 287 err = auxtrace_queues__queue_buffer(queues, idx, b); 288 if (err) { 289 auxtrace_buffer__free(b); 290 return err; 291 } 292 buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT; 293 sz -= BUFFER_LIMIT_FOR_32_BIT; 294 consecutive = true; 295 } 296 297 buffer->size = sz; 298 buffer->consecutive = consecutive; 299 300 return 0; 301 } 302 303 static bool filter_cpu(struct perf_session *session, int cpu) 304 { 305 unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap; 306 307 return cpu_bitmap && cpu != -1 && !test_bit(cpu, cpu_bitmap); 308 } 309 310 static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues, 311 struct perf_session *session, 312 unsigned int idx, 313 struct auxtrace_buffer *buffer, 314 struct auxtrace_buffer **buffer_ptr) 315 { 316 int err = -ENOMEM; 317 318 if (filter_cpu(session, buffer->cpu)) 319 return 0; 320 321 buffer = memdup(buffer, sizeof(*buffer)); 322 if (!buffer) 323 return -ENOMEM; 324 325 if (session->one_mmap) { 326 buffer->data = buffer->data_offset - session->one_mmap_offset + 327 session->one_mmap_addr; 328 } else if (perf_data__is_pipe(session->data)) { 329 buffer->data = auxtrace_copy_data(buffer->size, session); 330 if (!buffer->data) 331 goto out_free; 332 buffer->data_needs_freeing = true; 333 } else if (BITS_PER_LONG == 32 && 334 buffer->size > BUFFER_LIMIT_FOR_32_BIT) { 335 err = auxtrace_queues__split_buffer(queues, idx, buffer); 336 if (err) 337 goto out_free; 338 } 339 340 err = auxtrace_queues__queue_buffer(queues, idx, buffer); 341 if (err) 342 goto out_free; 343 344 /* FIXME: Doesn't work for split buffer */ 345 if (buffer_ptr) 346 *buffer_ptr = buffer; 347 348 return 0; 349 350 out_free: 351 auxtrace_buffer__free(buffer); 352 return err; 353 } 354 355 int auxtrace_queues__add_event(struct auxtrace_queues *queues, 356 struct perf_session *session, 357 union perf_event *event, off_t data_offset, 358 struct auxtrace_buffer **buffer_ptr) 359 { 360 struct auxtrace_buffer buffer = { 361 .pid = -1, 362 .tid = event->auxtrace.tid, 363 .cpu = event->auxtrace.cpu, 364 .data_offset = data_offset, 365 .offset = event->auxtrace.offset, 366 .reference = event->auxtrace.reference, 367 .size = event->auxtrace.size, 368 }; 369 unsigned int idx = event->auxtrace.idx; 370 371 return auxtrace_queues__add_buffer(queues, session, idx, &buffer, 372 buffer_ptr); 373 } 374 375 static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues, 376 struct perf_session *session, 377 off_t file_offset, size_t sz) 378 { 379 union perf_event *event; 380 int err; 381 char buf[PERF_SAMPLE_MAX_SIZE]; 382 383 err = perf_session__peek_event(session, file_offset, buf, 384 PERF_SAMPLE_MAX_SIZE, &event, NULL); 385 if (err) 386 return err; 387 388 if (event->header.type == PERF_RECORD_AUXTRACE) { 389 if (event->header.size < sizeof(struct perf_record_auxtrace) || 390 event->header.size != sz) { 391 err = -EINVAL; 392 goto out; 393 } 394 file_offset += event->header.size; 395 err = auxtrace_queues__add_event(queues, session, event, 396 file_offset, NULL); 397 } 398 out: 399 return err; 400 } 401 402 void auxtrace_queues__free(struct auxtrace_queues *queues) 403 { 404 unsigned int i; 405 406 for (i = 0; i < queues->nr_queues; i++) { 407 while (!list_empty(&queues->queue_array[i].head)) { 408 struct auxtrace_buffer *buffer; 409 410 buffer = list_entry(queues->queue_array[i].head.next, 411 struct auxtrace_buffer, list); 412 list_del_init(&buffer->list); 413 auxtrace_buffer__free(buffer); 414 } 415 } 416 417 zfree(&queues->queue_array); 418 queues->nr_queues = 0; 419 } 420 421 static void auxtrace_heapify(struct auxtrace_heap_item *heap_array, 422 unsigned int pos, unsigned int queue_nr, 423 u64 ordinal) 424 { 425 unsigned int parent; 426 427 while (pos) { 428 parent = (pos - 1) >> 1; 429 if (heap_array[parent].ordinal <= ordinal) 430 break; 431 heap_array[pos] = heap_array[parent]; 432 pos = parent; 433 } 434 heap_array[pos].queue_nr = queue_nr; 435 heap_array[pos].ordinal = ordinal; 436 } 437 438 int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr, 439 u64 ordinal) 440 { 441 struct auxtrace_heap_item *heap_array; 442 443 if (queue_nr >= heap->heap_sz) { 444 unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES; 445 446 while (heap_sz <= queue_nr) 447 heap_sz <<= 1; 448 heap_array = realloc(heap->heap_array, 449 heap_sz * sizeof(struct auxtrace_heap_item)); 450 if (!heap_array) 451 return -ENOMEM; 452 heap->heap_array = heap_array; 453 heap->heap_sz = heap_sz; 454 } 455 456 auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal); 457 458 return 0; 459 } 460 461 void auxtrace_heap__free(struct auxtrace_heap *heap) 462 { 463 zfree(&heap->heap_array); 464 heap->heap_cnt = 0; 465 heap->heap_sz = 0; 466 } 467 468 void auxtrace_heap__pop(struct auxtrace_heap *heap) 469 { 470 unsigned int pos, last, heap_cnt = heap->heap_cnt; 471 struct auxtrace_heap_item *heap_array; 472 473 if (!heap_cnt) 474 return; 475 476 heap->heap_cnt -= 1; 477 478 heap_array = heap->heap_array; 479 480 pos = 0; 481 while (1) { 482 unsigned int left, right; 483 484 left = (pos << 1) + 1; 485 if (left >= heap_cnt) 486 break; 487 right = left + 1; 488 if (right >= heap_cnt) { 489 heap_array[pos] = heap_array[left]; 490 return; 491 } 492 if (heap_array[left].ordinal < heap_array[right].ordinal) { 493 heap_array[pos] = heap_array[left]; 494 pos = left; 495 } else { 496 heap_array[pos] = heap_array[right]; 497 pos = right; 498 } 499 } 500 501 last = heap_cnt - 1; 502 auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr, 503 heap_array[last].ordinal); 504 } 505 506 size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr, 507 struct evlist *evlist) 508 { 509 if (itr) 510 return itr->info_priv_size(itr, evlist); 511 return 0; 512 } 513 514 static int auxtrace_not_supported(void) 515 { 516 pr_err("AUX area tracing is not supported on this architecture\n"); 517 return -EINVAL; 518 } 519 520 int auxtrace_record__info_fill(struct auxtrace_record *itr, 521 struct perf_session *session, 522 struct perf_record_auxtrace_info *auxtrace_info, 523 size_t priv_size) 524 { 525 if (itr) 526 return itr->info_fill(itr, session, auxtrace_info, priv_size); 527 return auxtrace_not_supported(); 528 } 529 530 void auxtrace_record__free(struct auxtrace_record *itr) 531 { 532 if (itr) 533 itr->free(itr); 534 } 535 536 int auxtrace_record__snapshot_start(struct auxtrace_record *itr) 537 { 538 if (itr && itr->snapshot_start) 539 return itr->snapshot_start(itr); 540 return 0; 541 } 542 543 int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit) 544 { 545 if (!on_exit && itr && itr->snapshot_finish) 546 return itr->snapshot_finish(itr); 547 return 0; 548 } 549 550 int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx, 551 struct auxtrace_mmap *mm, 552 unsigned char *data, u64 *head, u64 *old) 553 { 554 if (itr && itr->find_snapshot) 555 return itr->find_snapshot(itr, idx, mm, data, head, old); 556 return 0; 557 } 558 559 int auxtrace_record__options(struct auxtrace_record *itr, 560 struct evlist *evlist, 561 struct record_opts *opts) 562 { 563 if (itr) 564 return itr->recording_options(itr, evlist, opts); 565 return 0; 566 } 567 568 u64 auxtrace_record__reference(struct auxtrace_record *itr) 569 { 570 if (itr) 571 return itr->reference(itr); 572 return 0; 573 } 574 575 int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, 576 struct record_opts *opts, const char *str) 577 { 578 if (!str) 579 return 0; 580 581 /* PMU-agnostic options */ 582 switch (*str) { 583 case 'e': 584 opts->auxtrace_snapshot_on_exit = true; 585 str++; 586 break; 587 default: 588 break; 589 } 590 591 if (itr) 592 return itr->parse_snapshot_options(itr, opts, str); 593 594 pr_err("No AUX area tracing to snapshot\n"); 595 return -EINVAL; 596 } 597 598 struct auxtrace_record *__weak 599 auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err) 600 { 601 *err = 0; 602 return NULL; 603 } 604 605 static int auxtrace_index__alloc(struct list_head *head) 606 { 607 struct auxtrace_index *auxtrace_index; 608 609 auxtrace_index = malloc(sizeof(struct auxtrace_index)); 610 if (!auxtrace_index) 611 return -ENOMEM; 612 613 auxtrace_index->nr = 0; 614 INIT_LIST_HEAD(&auxtrace_index->list); 615 616 list_add_tail(&auxtrace_index->list, head); 617 618 return 0; 619 } 620 621 void auxtrace_index__free(struct list_head *head) 622 { 623 struct auxtrace_index *auxtrace_index, *n; 624 625 list_for_each_entry_safe(auxtrace_index, n, head, list) { 626 list_del_init(&auxtrace_index->list); 627 free(auxtrace_index); 628 } 629 } 630 631 static struct auxtrace_index *auxtrace_index__last(struct list_head *head) 632 { 633 struct auxtrace_index *auxtrace_index; 634 int err; 635 636 if (list_empty(head)) { 637 err = auxtrace_index__alloc(head); 638 if (err) 639 return NULL; 640 } 641 642 auxtrace_index = list_entry(head->prev, struct auxtrace_index, list); 643 644 if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) { 645 err = auxtrace_index__alloc(head); 646 if (err) 647 return NULL; 648 auxtrace_index = list_entry(head->prev, struct auxtrace_index, 649 list); 650 } 651 652 return auxtrace_index; 653 } 654 655 int auxtrace_index__auxtrace_event(struct list_head *head, 656 union perf_event *event, off_t file_offset) 657 { 658 struct auxtrace_index *auxtrace_index; 659 size_t nr; 660 661 auxtrace_index = auxtrace_index__last(head); 662 if (!auxtrace_index) 663 return -ENOMEM; 664 665 nr = auxtrace_index->nr; 666 auxtrace_index->entries[nr].file_offset = file_offset; 667 auxtrace_index->entries[nr].sz = event->header.size; 668 auxtrace_index->nr += 1; 669 670 return 0; 671 } 672 673 static int auxtrace_index__do_write(int fd, 674 struct auxtrace_index *auxtrace_index) 675 { 676 struct auxtrace_index_entry ent; 677 size_t i; 678 679 for (i = 0; i < auxtrace_index->nr; i++) { 680 ent.file_offset = auxtrace_index->entries[i].file_offset; 681 ent.sz = auxtrace_index->entries[i].sz; 682 if (writen(fd, &ent, sizeof(ent)) != sizeof(ent)) 683 return -errno; 684 } 685 return 0; 686 } 687 688 int auxtrace_index__write(int fd, struct list_head *head) 689 { 690 struct auxtrace_index *auxtrace_index; 691 u64 total = 0; 692 int err; 693 694 list_for_each_entry(auxtrace_index, head, list) 695 total += auxtrace_index->nr; 696 697 if (writen(fd, &total, sizeof(total)) != sizeof(total)) 698 return -errno; 699 700 list_for_each_entry(auxtrace_index, head, list) { 701 err = auxtrace_index__do_write(fd, auxtrace_index); 702 if (err) 703 return err; 704 } 705 706 return 0; 707 } 708 709 static int auxtrace_index__process_entry(int fd, struct list_head *head, 710 bool needs_swap) 711 { 712 struct auxtrace_index *auxtrace_index; 713 struct auxtrace_index_entry ent; 714 size_t nr; 715 716 if (readn(fd, &ent, sizeof(ent)) != sizeof(ent)) 717 return -1; 718 719 auxtrace_index = auxtrace_index__last(head); 720 if (!auxtrace_index) 721 return -1; 722 723 nr = auxtrace_index->nr; 724 if (needs_swap) { 725 auxtrace_index->entries[nr].file_offset = 726 bswap_64(ent.file_offset); 727 auxtrace_index->entries[nr].sz = bswap_64(ent.sz); 728 } else { 729 auxtrace_index->entries[nr].file_offset = ent.file_offset; 730 auxtrace_index->entries[nr].sz = ent.sz; 731 } 732 733 auxtrace_index->nr = nr + 1; 734 735 return 0; 736 } 737 738 int auxtrace_index__process(int fd, u64 size, struct perf_session *session, 739 bool needs_swap) 740 { 741 struct list_head *head = &session->auxtrace_index; 742 u64 nr; 743 744 if (readn(fd, &nr, sizeof(u64)) != sizeof(u64)) 745 return -1; 746 747 if (needs_swap) 748 nr = bswap_64(nr); 749 750 if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size) 751 return -1; 752 753 while (nr--) { 754 int err; 755 756 err = auxtrace_index__process_entry(fd, head, needs_swap); 757 if (err) 758 return -1; 759 } 760 761 return 0; 762 } 763 764 static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues, 765 struct perf_session *session, 766 struct auxtrace_index_entry *ent) 767 { 768 return auxtrace_queues__add_indexed_event(queues, session, 769 ent->file_offset, ent->sz); 770 } 771 772 int auxtrace_queues__process_index(struct auxtrace_queues *queues, 773 struct perf_session *session) 774 { 775 struct auxtrace_index *auxtrace_index; 776 struct auxtrace_index_entry *ent; 777 size_t i; 778 int err; 779 780 if (auxtrace__dont_decode(session)) 781 return 0; 782 783 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { 784 for (i = 0; i < auxtrace_index->nr; i++) { 785 ent = &auxtrace_index->entries[i]; 786 err = auxtrace_queues__process_index_entry(queues, 787 session, 788 ent); 789 if (err) 790 return err; 791 } 792 } 793 return 0; 794 } 795 796 struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue, 797 struct auxtrace_buffer *buffer) 798 { 799 if (buffer) { 800 if (list_is_last(&buffer->list, &queue->head)) 801 return NULL; 802 return list_entry(buffer->list.next, struct auxtrace_buffer, 803 list); 804 } else { 805 if (list_empty(&queue->head)) 806 return NULL; 807 return list_entry(queue->head.next, struct auxtrace_buffer, 808 list); 809 } 810 } 811 812 void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd) 813 { 814 size_t adj = buffer->data_offset & (page_size - 1); 815 size_t size = buffer->size + adj; 816 off_t file_offset = buffer->data_offset - adj; 817 void *addr; 818 819 if (buffer->data) 820 return buffer->data; 821 822 addr = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, file_offset); 823 if (addr == MAP_FAILED) 824 return NULL; 825 826 buffer->mmap_addr = addr; 827 buffer->mmap_size = size; 828 829 buffer->data = addr + adj; 830 831 return buffer->data; 832 } 833 834 void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer) 835 { 836 if (!buffer->data || !buffer->mmap_addr) 837 return; 838 munmap(buffer->mmap_addr, buffer->mmap_size); 839 buffer->mmap_addr = NULL; 840 buffer->mmap_size = 0; 841 buffer->data = NULL; 842 buffer->use_data = NULL; 843 } 844 845 void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer) 846 { 847 auxtrace_buffer__put_data(buffer); 848 if (buffer->data_needs_freeing) { 849 buffer->data_needs_freeing = false; 850 zfree(&buffer->data); 851 buffer->use_data = NULL; 852 buffer->size = 0; 853 } 854 } 855 856 void auxtrace_buffer__free(struct auxtrace_buffer *buffer) 857 { 858 auxtrace_buffer__drop_data(buffer); 859 free(buffer); 860 } 861 862 void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type, 863 int code, int cpu, pid_t pid, pid_t tid, u64 ip, 864 const char *msg, u64 timestamp) 865 { 866 size_t size; 867 868 memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error)); 869 870 auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR; 871 auxtrace_error->type = type; 872 auxtrace_error->code = code; 873 auxtrace_error->cpu = cpu; 874 auxtrace_error->pid = pid; 875 auxtrace_error->tid = tid; 876 auxtrace_error->fmt = 1; 877 auxtrace_error->ip = ip; 878 auxtrace_error->time = timestamp; 879 strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG); 880 881 size = (void *)auxtrace_error->msg - (void *)auxtrace_error + 882 strlen(auxtrace_error->msg) + 1; 883 auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64)); 884 } 885 886 int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr, 887 struct perf_tool *tool, 888 struct perf_session *session, 889 perf_event__handler_t process) 890 { 891 union perf_event *ev; 892 size_t priv_size; 893 int err; 894 895 pr_debug2("Synthesizing auxtrace information\n"); 896 priv_size = auxtrace_record__info_priv_size(itr, session->evlist); 897 ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size); 898 if (!ev) 899 return -ENOMEM; 900 901 ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO; 902 ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) + 903 priv_size; 904 err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info, 905 priv_size); 906 if (err) 907 goto out_free; 908 909 err = process(tool, ev, NULL, NULL); 910 out_free: 911 free(ev); 912 return err; 913 } 914 915 int perf_event__process_auxtrace_info(struct perf_session *session, 916 union perf_event *event) 917 { 918 enum auxtrace_type type = event->auxtrace_info.type; 919 920 if (dump_trace) 921 fprintf(stdout, " type: %u\n", type); 922 923 switch (type) { 924 case PERF_AUXTRACE_INTEL_PT: 925 return intel_pt_process_auxtrace_info(event, session); 926 case PERF_AUXTRACE_INTEL_BTS: 927 return intel_bts_process_auxtrace_info(event, session); 928 case PERF_AUXTRACE_ARM_SPE: 929 return arm_spe_process_auxtrace_info(event, session); 930 case PERF_AUXTRACE_CS_ETM: 931 return cs_etm__process_auxtrace_info(event, session); 932 case PERF_AUXTRACE_S390_CPUMSF: 933 return s390_cpumsf_process_auxtrace_info(event, session); 934 case PERF_AUXTRACE_UNKNOWN: 935 default: 936 return -EINVAL; 937 } 938 } 939 940 s64 perf_event__process_auxtrace(struct perf_session *session, 941 union perf_event *event) 942 { 943 s64 err; 944 945 if (dump_trace) 946 fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n", 947 event->auxtrace.size, event->auxtrace.offset, 948 event->auxtrace.reference, event->auxtrace.idx, 949 event->auxtrace.tid, event->auxtrace.cpu); 950 951 if (auxtrace__dont_decode(session)) 952 return event->auxtrace.size; 953 954 if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE) 955 return -EINVAL; 956 957 err = session->auxtrace->process_auxtrace_event(session, event, session->tool); 958 if (err < 0) 959 return err; 960 961 return event->auxtrace.size; 962 } 963 964 #define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS 965 #define PERF_ITRACE_DEFAULT_PERIOD 100000 966 #define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16 967 #define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024 968 #define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64 969 #define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024 970 971 void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, 972 bool no_sample) 973 { 974 synth_opts->branches = true; 975 synth_opts->transactions = true; 976 synth_opts->ptwrites = true; 977 synth_opts->pwr_events = true; 978 synth_opts->other_events = true; 979 synth_opts->errors = true; 980 if (no_sample) { 981 synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS; 982 synth_opts->period = 1; 983 synth_opts->calls = true; 984 } else { 985 synth_opts->instructions = true; 986 synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE; 987 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD; 988 } 989 synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ; 990 synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ; 991 synth_opts->initial_skip = 0; 992 } 993 994 /* 995 * Please check tools/perf/Documentation/perf-script.txt for information 996 * about the options parsed here, which is introduced after this cset, 997 * when support in 'perf script' for these options is introduced. 998 */ 999 int itrace_parse_synth_opts(const struct option *opt, const char *str, 1000 int unset) 1001 { 1002 struct itrace_synth_opts *synth_opts = opt->value; 1003 const char *p; 1004 char *endptr; 1005 bool period_type_set = false; 1006 bool period_set = false; 1007 1008 synth_opts->set = true; 1009 1010 if (unset) { 1011 synth_opts->dont_decode = true; 1012 return 0; 1013 } 1014 1015 if (!str) { 1016 itrace_synth_opts__set_default(synth_opts, 1017 synth_opts->default_no_sample); 1018 return 0; 1019 } 1020 1021 for (p = str; *p;) { 1022 switch (*p++) { 1023 case 'i': 1024 synth_opts->instructions = true; 1025 while (*p == ' ' || *p == ',') 1026 p += 1; 1027 if (isdigit(*p)) { 1028 synth_opts->period = strtoull(p, &endptr, 10); 1029 period_set = true; 1030 p = endptr; 1031 while (*p == ' ' || *p == ',') 1032 p += 1; 1033 switch (*p++) { 1034 case 'i': 1035 synth_opts->period_type = 1036 PERF_ITRACE_PERIOD_INSTRUCTIONS; 1037 period_type_set = true; 1038 break; 1039 case 't': 1040 synth_opts->period_type = 1041 PERF_ITRACE_PERIOD_TICKS; 1042 period_type_set = true; 1043 break; 1044 case 'm': 1045 synth_opts->period *= 1000; 1046 /* Fall through */ 1047 case 'u': 1048 synth_opts->period *= 1000; 1049 /* Fall through */ 1050 case 'n': 1051 if (*p++ != 's') 1052 goto out_err; 1053 synth_opts->period_type = 1054 PERF_ITRACE_PERIOD_NANOSECS; 1055 period_type_set = true; 1056 break; 1057 case '\0': 1058 goto out; 1059 default: 1060 goto out_err; 1061 } 1062 } 1063 break; 1064 case 'b': 1065 synth_opts->branches = true; 1066 break; 1067 case 'x': 1068 synth_opts->transactions = true; 1069 break; 1070 case 'w': 1071 synth_opts->ptwrites = true; 1072 break; 1073 case 'p': 1074 synth_opts->pwr_events = true; 1075 break; 1076 case 'o': 1077 synth_opts->other_events = true; 1078 break; 1079 case 'e': 1080 synth_opts->errors = true; 1081 break; 1082 case 'd': 1083 synth_opts->log = true; 1084 break; 1085 case 'c': 1086 synth_opts->branches = true; 1087 synth_opts->calls = true; 1088 break; 1089 case 'r': 1090 synth_opts->branches = true; 1091 synth_opts->returns = true; 1092 break; 1093 case 'g': 1094 synth_opts->callchain = true; 1095 synth_opts->callchain_sz = 1096 PERF_ITRACE_DEFAULT_CALLCHAIN_SZ; 1097 while (*p == ' ' || *p == ',') 1098 p += 1; 1099 if (isdigit(*p)) { 1100 unsigned int val; 1101 1102 val = strtoul(p, &endptr, 10); 1103 p = endptr; 1104 if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ) 1105 goto out_err; 1106 synth_opts->callchain_sz = val; 1107 } 1108 break; 1109 case 'l': 1110 synth_opts->last_branch = true; 1111 synth_opts->last_branch_sz = 1112 PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ; 1113 while (*p == ' ' || *p == ',') 1114 p += 1; 1115 if (isdigit(*p)) { 1116 unsigned int val; 1117 1118 val = strtoul(p, &endptr, 10); 1119 p = endptr; 1120 if (!val || 1121 val > PERF_ITRACE_MAX_LAST_BRANCH_SZ) 1122 goto out_err; 1123 synth_opts->last_branch_sz = val; 1124 } 1125 break; 1126 case 's': 1127 synth_opts->initial_skip = strtoul(p, &endptr, 10); 1128 if (p == endptr) 1129 goto out_err; 1130 p = endptr; 1131 break; 1132 case ' ': 1133 case ',': 1134 break; 1135 default: 1136 goto out_err; 1137 } 1138 } 1139 out: 1140 if (synth_opts->instructions) { 1141 if (!period_type_set) 1142 synth_opts->period_type = 1143 PERF_ITRACE_DEFAULT_PERIOD_TYPE; 1144 if (!period_set) 1145 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD; 1146 } 1147 1148 return 0; 1149 1150 out_err: 1151 pr_err("Bad Instruction Tracing options '%s'\n", str); 1152 return -EINVAL; 1153 } 1154 1155 static const char * const auxtrace_error_type_name[] = { 1156 [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace", 1157 }; 1158 1159 static const char *auxtrace_error_name(int type) 1160 { 1161 const char *error_type_name = NULL; 1162 1163 if (type < PERF_AUXTRACE_ERROR_MAX) 1164 error_type_name = auxtrace_error_type_name[type]; 1165 if (!error_type_name) 1166 error_type_name = "unknown AUX"; 1167 return error_type_name; 1168 } 1169 1170 size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) 1171 { 1172 struct perf_record_auxtrace_error *e = &event->auxtrace_error; 1173 unsigned long long nsecs = e->time; 1174 const char *msg = e->msg; 1175 int ret; 1176 1177 ret = fprintf(fp, " %s error type %u", 1178 auxtrace_error_name(e->type), e->type); 1179 1180 if (e->fmt && nsecs) { 1181 unsigned long secs = nsecs / NSEC_PER_SEC; 1182 1183 nsecs -= secs * NSEC_PER_SEC; 1184 ret += fprintf(fp, " time %lu.%09llu", secs, nsecs); 1185 } else { 1186 ret += fprintf(fp, " time 0"); 1187 } 1188 1189 if (!e->fmt) 1190 msg = (const char *)&e->time; 1191 1192 ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n", 1193 e->cpu, e->pid, e->tid, e->ip, e->code, msg); 1194 return ret; 1195 } 1196 1197 void perf_session__auxtrace_error_inc(struct perf_session *session, 1198 union perf_event *event) 1199 { 1200 struct perf_record_auxtrace_error *e = &event->auxtrace_error; 1201 1202 if (e->type < PERF_AUXTRACE_ERROR_MAX) 1203 session->evlist->stats.nr_auxtrace_errors[e->type] += 1; 1204 } 1205 1206 void events_stats__auxtrace_error_warn(const struct events_stats *stats) 1207 { 1208 int i; 1209 1210 for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) { 1211 if (!stats->nr_auxtrace_errors[i]) 1212 continue; 1213 ui__warning("%u %s errors\n", 1214 stats->nr_auxtrace_errors[i], 1215 auxtrace_error_name(i)); 1216 } 1217 } 1218 1219 int perf_event__process_auxtrace_error(struct perf_session *session, 1220 union perf_event *event) 1221 { 1222 if (auxtrace__dont_decode(session)) 1223 return 0; 1224 1225 perf_event__fprintf_auxtrace_error(event, stdout); 1226 return 0; 1227 } 1228 1229 static int __auxtrace_mmap__read(struct perf_mmap *map, 1230 struct auxtrace_record *itr, 1231 struct perf_tool *tool, process_auxtrace_t fn, 1232 bool snapshot, size_t snapshot_size) 1233 { 1234 struct auxtrace_mmap *mm = &map->auxtrace_mmap; 1235 u64 head, old = mm->prev, offset, ref; 1236 unsigned char *data = mm->base; 1237 size_t size, head_off, old_off, len1, len2, padding; 1238 union perf_event ev; 1239 void *data1, *data2; 1240 1241 if (snapshot) { 1242 head = auxtrace_mmap__read_snapshot_head(mm); 1243 if (auxtrace_record__find_snapshot(itr, mm->idx, mm, data, 1244 &head, &old)) 1245 return -1; 1246 } else { 1247 head = auxtrace_mmap__read_head(mm); 1248 } 1249 1250 if (old == head) 1251 return 0; 1252 1253 pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n", 1254 mm->idx, old, head, head - old); 1255 1256 if (mm->mask) { 1257 head_off = head & mm->mask; 1258 old_off = old & mm->mask; 1259 } else { 1260 head_off = head % mm->len; 1261 old_off = old % mm->len; 1262 } 1263 1264 if (head_off > old_off) 1265 size = head_off - old_off; 1266 else 1267 size = mm->len - (old_off - head_off); 1268 1269 if (snapshot && size > snapshot_size) 1270 size = snapshot_size; 1271 1272 ref = auxtrace_record__reference(itr); 1273 1274 if (head > old || size <= head || mm->mask) { 1275 offset = head - size; 1276 } else { 1277 /* 1278 * When the buffer size is not a power of 2, 'head' wraps at the 1279 * highest multiple of the buffer size, so we have to subtract 1280 * the remainder here. 1281 */ 1282 u64 rem = (0ULL - mm->len) % mm->len; 1283 1284 offset = head - size - rem; 1285 } 1286 1287 if (size > head_off) { 1288 len1 = size - head_off; 1289 data1 = &data[mm->len - len1]; 1290 len2 = head_off; 1291 data2 = &data[0]; 1292 } else { 1293 len1 = size; 1294 data1 = &data[head_off - len1]; 1295 len2 = 0; 1296 data2 = NULL; 1297 } 1298 1299 if (itr->alignment) { 1300 unsigned int unwanted = len1 % itr->alignment; 1301 1302 len1 -= unwanted; 1303 size -= unwanted; 1304 } 1305 1306 /* padding must be written by fn() e.g. record__process_auxtrace() */ 1307 padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1); 1308 if (padding) 1309 padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding; 1310 1311 memset(&ev, 0, sizeof(ev)); 1312 ev.auxtrace.header.type = PERF_RECORD_AUXTRACE; 1313 ev.auxtrace.header.size = sizeof(ev.auxtrace); 1314 ev.auxtrace.size = size + padding; 1315 ev.auxtrace.offset = offset; 1316 ev.auxtrace.reference = ref; 1317 ev.auxtrace.idx = mm->idx; 1318 ev.auxtrace.tid = mm->tid; 1319 ev.auxtrace.cpu = mm->cpu; 1320 1321 if (fn(tool, map, &ev, data1, len1, data2, len2)) 1322 return -1; 1323 1324 mm->prev = head; 1325 1326 if (!snapshot) { 1327 auxtrace_mmap__write_tail(mm, head); 1328 if (itr->read_finish) { 1329 int err; 1330 1331 err = itr->read_finish(itr, mm->idx); 1332 if (err < 0) 1333 return err; 1334 } 1335 } 1336 1337 return 1; 1338 } 1339 1340 int auxtrace_mmap__read(struct perf_mmap *map, struct auxtrace_record *itr, 1341 struct perf_tool *tool, process_auxtrace_t fn) 1342 { 1343 return __auxtrace_mmap__read(map, itr, tool, fn, false, 0); 1344 } 1345 1346 int auxtrace_mmap__read_snapshot(struct perf_mmap *map, 1347 struct auxtrace_record *itr, 1348 struct perf_tool *tool, process_auxtrace_t fn, 1349 size_t snapshot_size) 1350 { 1351 return __auxtrace_mmap__read(map, itr, tool, fn, true, snapshot_size); 1352 } 1353 1354 /** 1355 * struct auxtrace_cache - hash table to implement a cache 1356 * @hashtable: the hashtable 1357 * @sz: hashtable size (number of hlists) 1358 * @entry_size: size of an entry 1359 * @limit: limit the number of entries to this maximum, when reached the cache 1360 * is dropped and caching begins again with an empty cache 1361 * @cnt: current number of entries 1362 * @bits: hashtable size (@sz = 2^@bits) 1363 */ 1364 struct auxtrace_cache { 1365 struct hlist_head *hashtable; 1366 size_t sz; 1367 size_t entry_size; 1368 size_t limit; 1369 size_t cnt; 1370 unsigned int bits; 1371 }; 1372 1373 struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size, 1374 unsigned int limit_percent) 1375 { 1376 struct auxtrace_cache *c; 1377 struct hlist_head *ht; 1378 size_t sz, i; 1379 1380 c = zalloc(sizeof(struct auxtrace_cache)); 1381 if (!c) 1382 return NULL; 1383 1384 sz = 1UL << bits; 1385 1386 ht = calloc(sz, sizeof(struct hlist_head)); 1387 if (!ht) 1388 goto out_free; 1389 1390 for (i = 0; i < sz; i++) 1391 INIT_HLIST_HEAD(&ht[i]); 1392 1393 c->hashtable = ht; 1394 c->sz = sz; 1395 c->entry_size = entry_size; 1396 c->limit = (c->sz * limit_percent) / 100; 1397 c->bits = bits; 1398 1399 return c; 1400 1401 out_free: 1402 free(c); 1403 return NULL; 1404 } 1405 1406 static void auxtrace_cache__drop(struct auxtrace_cache *c) 1407 { 1408 struct auxtrace_cache_entry *entry; 1409 struct hlist_node *tmp; 1410 size_t i; 1411 1412 if (!c) 1413 return; 1414 1415 for (i = 0; i < c->sz; i++) { 1416 hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) { 1417 hlist_del(&entry->hash); 1418 auxtrace_cache__free_entry(c, entry); 1419 } 1420 } 1421 1422 c->cnt = 0; 1423 } 1424 1425 void auxtrace_cache__free(struct auxtrace_cache *c) 1426 { 1427 if (!c) 1428 return; 1429 1430 auxtrace_cache__drop(c); 1431 zfree(&c->hashtable); 1432 free(c); 1433 } 1434 1435 void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c) 1436 { 1437 return malloc(c->entry_size); 1438 } 1439 1440 void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused, 1441 void *entry) 1442 { 1443 free(entry); 1444 } 1445 1446 int auxtrace_cache__add(struct auxtrace_cache *c, u32 key, 1447 struct auxtrace_cache_entry *entry) 1448 { 1449 if (c->limit && ++c->cnt > c->limit) 1450 auxtrace_cache__drop(c); 1451 1452 entry->key = key; 1453 hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]); 1454 1455 return 0; 1456 } 1457 1458 void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key) 1459 { 1460 struct auxtrace_cache_entry *entry; 1461 struct hlist_head *hlist; 1462 1463 if (!c) 1464 return NULL; 1465 1466 hlist = &c->hashtable[hash_32(key, c->bits)]; 1467 hlist_for_each_entry(entry, hlist, hash) { 1468 if (entry->key == key) 1469 return entry; 1470 } 1471 1472 return NULL; 1473 } 1474 1475 static void addr_filter__free_str(struct addr_filter *filt) 1476 { 1477 zfree(&filt->str); 1478 filt->action = NULL; 1479 filt->sym_from = NULL; 1480 filt->sym_to = NULL; 1481 filt->filename = NULL; 1482 } 1483 1484 static struct addr_filter *addr_filter__new(void) 1485 { 1486 struct addr_filter *filt = zalloc(sizeof(*filt)); 1487 1488 if (filt) 1489 INIT_LIST_HEAD(&filt->list); 1490 1491 return filt; 1492 } 1493 1494 static void addr_filter__free(struct addr_filter *filt) 1495 { 1496 if (filt) 1497 addr_filter__free_str(filt); 1498 free(filt); 1499 } 1500 1501 static void addr_filters__add(struct addr_filters *filts, 1502 struct addr_filter *filt) 1503 { 1504 list_add_tail(&filt->list, &filts->head); 1505 filts->cnt += 1; 1506 } 1507 1508 static void addr_filters__del(struct addr_filters *filts, 1509 struct addr_filter *filt) 1510 { 1511 list_del_init(&filt->list); 1512 filts->cnt -= 1; 1513 } 1514 1515 void addr_filters__init(struct addr_filters *filts) 1516 { 1517 INIT_LIST_HEAD(&filts->head); 1518 filts->cnt = 0; 1519 } 1520 1521 void addr_filters__exit(struct addr_filters *filts) 1522 { 1523 struct addr_filter *filt, *n; 1524 1525 list_for_each_entry_safe(filt, n, &filts->head, list) { 1526 addr_filters__del(filts, filt); 1527 addr_filter__free(filt); 1528 } 1529 } 1530 1531 static int parse_num_or_str(char **inp, u64 *num, const char **str, 1532 const char *str_delim) 1533 { 1534 *inp += strspn(*inp, " "); 1535 1536 if (isdigit(**inp)) { 1537 char *endptr; 1538 1539 if (!num) 1540 return -EINVAL; 1541 errno = 0; 1542 *num = strtoull(*inp, &endptr, 0); 1543 if (errno) 1544 return -errno; 1545 if (endptr == *inp) 1546 return -EINVAL; 1547 *inp = endptr; 1548 } else { 1549 size_t n; 1550 1551 if (!str) 1552 return -EINVAL; 1553 *inp += strspn(*inp, " "); 1554 *str = *inp; 1555 n = strcspn(*inp, str_delim); 1556 if (!n) 1557 return -EINVAL; 1558 *inp += n; 1559 if (**inp) { 1560 **inp = '\0'; 1561 *inp += 1; 1562 } 1563 } 1564 return 0; 1565 } 1566 1567 static int parse_action(struct addr_filter *filt) 1568 { 1569 if (!strcmp(filt->action, "filter")) { 1570 filt->start = true; 1571 filt->range = true; 1572 } else if (!strcmp(filt->action, "start")) { 1573 filt->start = true; 1574 } else if (!strcmp(filt->action, "stop")) { 1575 filt->start = false; 1576 } else if (!strcmp(filt->action, "tracestop")) { 1577 filt->start = false; 1578 filt->range = true; 1579 filt->action += 5; /* Change 'tracestop' to 'stop' */ 1580 } else { 1581 return -EINVAL; 1582 } 1583 return 0; 1584 } 1585 1586 static int parse_sym_idx(char **inp, int *idx) 1587 { 1588 *idx = -1; 1589 1590 *inp += strspn(*inp, " "); 1591 1592 if (**inp != '#') 1593 return 0; 1594 1595 *inp += 1; 1596 1597 if (**inp == 'g' || **inp == 'G') { 1598 *inp += 1; 1599 *idx = 0; 1600 } else { 1601 unsigned long num; 1602 char *endptr; 1603 1604 errno = 0; 1605 num = strtoul(*inp, &endptr, 0); 1606 if (errno) 1607 return -errno; 1608 if (endptr == *inp || num > INT_MAX) 1609 return -EINVAL; 1610 *inp = endptr; 1611 *idx = num; 1612 } 1613 1614 return 0; 1615 } 1616 1617 static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx) 1618 { 1619 int err = parse_num_or_str(inp, num, str, " "); 1620 1621 if (!err && *str) 1622 err = parse_sym_idx(inp, idx); 1623 1624 return err; 1625 } 1626 1627 static int parse_one_filter(struct addr_filter *filt, const char **filter_inp) 1628 { 1629 char *fstr; 1630 int err; 1631 1632 filt->str = fstr = strdup(*filter_inp); 1633 if (!fstr) 1634 return -ENOMEM; 1635 1636 err = parse_num_or_str(&fstr, NULL, &filt->action, " "); 1637 if (err) 1638 goto out_err; 1639 1640 err = parse_action(filt); 1641 if (err) 1642 goto out_err; 1643 1644 err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from, 1645 &filt->sym_from_idx); 1646 if (err) 1647 goto out_err; 1648 1649 fstr += strspn(fstr, " "); 1650 1651 if (*fstr == '/') { 1652 fstr += 1; 1653 err = parse_addr_size(&fstr, &filt->size, &filt->sym_to, 1654 &filt->sym_to_idx); 1655 if (err) 1656 goto out_err; 1657 filt->range = true; 1658 } 1659 1660 fstr += strspn(fstr, " "); 1661 1662 if (*fstr == '@') { 1663 fstr += 1; 1664 err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,"); 1665 if (err) 1666 goto out_err; 1667 } 1668 1669 fstr += strspn(fstr, " ,"); 1670 1671 *filter_inp += fstr - filt->str; 1672 1673 return 0; 1674 1675 out_err: 1676 addr_filter__free_str(filt); 1677 1678 return err; 1679 } 1680 1681 int addr_filters__parse_bare_filter(struct addr_filters *filts, 1682 const char *filter) 1683 { 1684 struct addr_filter *filt; 1685 const char *fstr = filter; 1686 int err; 1687 1688 while (*fstr) { 1689 filt = addr_filter__new(); 1690 err = parse_one_filter(filt, &fstr); 1691 if (err) { 1692 addr_filter__free(filt); 1693 addr_filters__exit(filts); 1694 return err; 1695 } 1696 addr_filters__add(filts, filt); 1697 } 1698 1699 return 0; 1700 } 1701 1702 struct sym_args { 1703 const char *name; 1704 u64 start; 1705 u64 size; 1706 int idx; 1707 int cnt; 1708 bool started; 1709 bool global; 1710 bool selected; 1711 bool duplicate; 1712 bool near; 1713 }; 1714 1715 static bool kern_sym_match(struct sym_args *args, const char *name, char type) 1716 { 1717 /* A function with the same name, and global or the n'th found or any */ 1718 return kallsyms__is_function(type) && 1719 !strcmp(name, args->name) && 1720 ((args->global && isupper(type)) || 1721 (args->selected && ++(args->cnt) == args->idx) || 1722 (!args->global && !args->selected)); 1723 } 1724 1725 static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start) 1726 { 1727 struct sym_args *args = arg; 1728 1729 if (args->started) { 1730 if (!args->size) 1731 args->size = start - args->start; 1732 if (args->selected) { 1733 if (args->size) 1734 return 1; 1735 } else if (kern_sym_match(args, name, type)) { 1736 args->duplicate = true; 1737 return 1; 1738 } 1739 } else if (kern_sym_match(args, name, type)) { 1740 args->started = true; 1741 args->start = start; 1742 } 1743 1744 return 0; 1745 } 1746 1747 static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start) 1748 { 1749 struct sym_args *args = arg; 1750 1751 if (kern_sym_match(args, name, type)) { 1752 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", 1753 ++args->cnt, start, type, name); 1754 args->near = true; 1755 } else if (args->near) { 1756 args->near = false; 1757 pr_err("\t\twhich is near\t\t%s\n", name); 1758 } 1759 1760 return 0; 1761 } 1762 1763 static int sym_not_found_error(const char *sym_name, int idx) 1764 { 1765 if (idx > 0) { 1766 pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n", 1767 idx, sym_name); 1768 } else if (!idx) { 1769 pr_err("Global symbol '%s' not found.\n", sym_name); 1770 } else { 1771 pr_err("Symbol '%s' not found.\n", sym_name); 1772 } 1773 pr_err("Note that symbols must be functions.\n"); 1774 1775 return -EINVAL; 1776 } 1777 1778 static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx) 1779 { 1780 struct sym_args args = { 1781 .name = sym_name, 1782 .idx = idx, 1783 .global = !idx, 1784 .selected = idx > 0, 1785 }; 1786 int err; 1787 1788 *start = 0; 1789 *size = 0; 1790 1791 err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb); 1792 if (err < 0) { 1793 pr_err("Failed to parse /proc/kallsyms\n"); 1794 return err; 1795 } 1796 1797 if (args.duplicate) { 1798 pr_err("Multiple kernel symbols with name '%s'\n", sym_name); 1799 args.cnt = 0; 1800 kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb); 1801 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n", 1802 sym_name); 1803 pr_err("Or select a global symbol by inserting #0 or #g or #G\n"); 1804 return -EINVAL; 1805 } 1806 1807 if (!args.started) { 1808 pr_err("Kernel symbol lookup: "); 1809 return sym_not_found_error(sym_name, idx); 1810 } 1811 1812 *start = args.start; 1813 *size = args.size; 1814 1815 return 0; 1816 } 1817 1818 static int find_entire_kern_cb(void *arg, const char *name __maybe_unused, 1819 char type, u64 start) 1820 { 1821 struct sym_args *args = arg; 1822 1823 if (!kallsyms__is_function(type)) 1824 return 0; 1825 1826 if (!args->started) { 1827 args->started = true; 1828 args->start = start; 1829 } 1830 /* Don't know exactly where the kernel ends, so we add a page */ 1831 args->size = round_up(start, page_size) + page_size - args->start; 1832 1833 return 0; 1834 } 1835 1836 static int addr_filter__entire_kernel(struct addr_filter *filt) 1837 { 1838 struct sym_args args = { .started = false }; 1839 int err; 1840 1841 err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb); 1842 if (err < 0 || !args.started) { 1843 pr_err("Failed to parse /proc/kallsyms\n"); 1844 return err; 1845 } 1846 1847 filt->addr = args.start; 1848 filt->size = args.size; 1849 1850 return 0; 1851 } 1852 1853 static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size) 1854 { 1855 if (start + size >= filt->addr) 1856 return 0; 1857 1858 if (filt->sym_from) { 1859 pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n", 1860 filt->sym_to, start, filt->sym_from, filt->addr); 1861 } else { 1862 pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n", 1863 filt->sym_to, start, filt->addr); 1864 } 1865 1866 return -EINVAL; 1867 } 1868 1869 static int addr_filter__resolve_kernel_syms(struct addr_filter *filt) 1870 { 1871 bool no_size = false; 1872 u64 start, size; 1873 int err; 1874 1875 if (symbol_conf.kptr_restrict) { 1876 pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n"); 1877 return -EINVAL; 1878 } 1879 1880 if (filt->sym_from && !strcmp(filt->sym_from, "*")) 1881 return addr_filter__entire_kernel(filt); 1882 1883 if (filt->sym_from) { 1884 err = find_kern_sym(filt->sym_from, &start, &size, 1885 filt->sym_from_idx); 1886 if (err) 1887 return err; 1888 filt->addr = start; 1889 if (filt->range && !filt->size && !filt->sym_to) { 1890 filt->size = size; 1891 no_size = !size; 1892 } 1893 } 1894 1895 if (filt->sym_to) { 1896 err = find_kern_sym(filt->sym_to, &start, &size, 1897 filt->sym_to_idx); 1898 if (err) 1899 return err; 1900 1901 err = check_end_after_start(filt, start, size); 1902 if (err) 1903 return err; 1904 filt->size = start + size - filt->addr; 1905 no_size = !size; 1906 } 1907 1908 /* The very last symbol in kallsyms does not imply a particular size */ 1909 if (no_size) { 1910 pr_err("Cannot determine size of symbol '%s'\n", 1911 filt->sym_to ? filt->sym_to : filt->sym_from); 1912 return -EINVAL; 1913 } 1914 1915 return 0; 1916 } 1917 1918 static struct dso *load_dso(const char *name) 1919 { 1920 struct map *map; 1921 struct dso *dso; 1922 1923 map = dso__new_map(name); 1924 if (!map) 1925 return NULL; 1926 1927 if (map__load(map) < 0) 1928 pr_err("File '%s' not found or has no symbols.\n", name); 1929 1930 dso = dso__get(map->dso); 1931 1932 map__put(map); 1933 1934 return dso; 1935 } 1936 1937 static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt, 1938 int idx) 1939 { 1940 /* Same name, and global or the n'th found or any */ 1941 return !arch__compare_symbol_names(name, sym->name) && 1942 ((!idx && sym->binding == STB_GLOBAL) || 1943 (idx > 0 && ++*cnt == idx) || 1944 idx < 0); 1945 } 1946 1947 static void print_duplicate_syms(struct dso *dso, const char *sym_name) 1948 { 1949 struct symbol *sym; 1950 bool near = false; 1951 int cnt = 0; 1952 1953 pr_err("Multiple symbols with name '%s'\n", sym_name); 1954 1955 sym = dso__first_symbol(dso); 1956 while (sym) { 1957 if (dso_sym_match(sym, sym_name, &cnt, -1)) { 1958 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", 1959 ++cnt, sym->start, 1960 sym->binding == STB_GLOBAL ? 'g' : 1961 sym->binding == STB_LOCAL ? 'l' : 'w', 1962 sym->name); 1963 near = true; 1964 } else if (near) { 1965 near = false; 1966 pr_err("\t\twhich is near\t\t%s\n", sym->name); 1967 } 1968 sym = dso__next_symbol(sym); 1969 } 1970 1971 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n", 1972 sym_name); 1973 pr_err("Or select a global symbol by inserting #0 or #g or #G\n"); 1974 } 1975 1976 static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start, 1977 u64 *size, int idx) 1978 { 1979 struct symbol *sym; 1980 int cnt = 0; 1981 1982 *start = 0; 1983 *size = 0; 1984 1985 sym = dso__first_symbol(dso); 1986 while (sym) { 1987 if (*start) { 1988 if (!*size) 1989 *size = sym->start - *start; 1990 if (idx > 0) { 1991 if (*size) 1992 return 1; 1993 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) { 1994 print_duplicate_syms(dso, sym_name); 1995 return -EINVAL; 1996 } 1997 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) { 1998 *start = sym->start; 1999 *size = sym->end - sym->start; 2000 } 2001 sym = dso__next_symbol(sym); 2002 } 2003 2004 if (!*start) 2005 return sym_not_found_error(sym_name, idx); 2006 2007 return 0; 2008 } 2009 2010 static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso) 2011 { 2012 if (dso__data_file_size(dso, NULL)) { 2013 pr_err("Failed to determine filter for %s\nCannot determine file size.\n", 2014 filt->filename); 2015 return -EINVAL; 2016 } 2017 2018 filt->addr = 0; 2019 filt->size = dso->data.file_size; 2020 2021 return 0; 2022 } 2023 2024 static int addr_filter__resolve_syms(struct addr_filter *filt) 2025 { 2026 u64 start, size; 2027 struct dso *dso; 2028 int err = 0; 2029 2030 if (!filt->sym_from && !filt->sym_to) 2031 return 0; 2032 2033 if (!filt->filename) 2034 return addr_filter__resolve_kernel_syms(filt); 2035 2036 dso = load_dso(filt->filename); 2037 if (!dso) { 2038 pr_err("Failed to load symbols from: %s\n", filt->filename); 2039 return -EINVAL; 2040 } 2041 2042 if (filt->sym_from && !strcmp(filt->sym_from, "*")) { 2043 err = addr_filter__entire_dso(filt, dso); 2044 goto put_dso; 2045 } 2046 2047 if (filt->sym_from) { 2048 err = find_dso_sym(dso, filt->sym_from, &start, &size, 2049 filt->sym_from_idx); 2050 if (err) 2051 goto put_dso; 2052 filt->addr = start; 2053 if (filt->range && !filt->size && !filt->sym_to) 2054 filt->size = size; 2055 } 2056 2057 if (filt->sym_to) { 2058 err = find_dso_sym(dso, filt->sym_to, &start, &size, 2059 filt->sym_to_idx); 2060 if (err) 2061 goto put_dso; 2062 2063 err = check_end_after_start(filt, start, size); 2064 if (err) 2065 return err; 2066 2067 filt->size = start + size - filt->addr; 2068 } 2069 2070 put_dso: 2071 dso__put(dso); 2072 2073 return err; 2074 } 2075 2076 static char *addr_filter__to_str(struct addr_filter *filt) 2077 { 2078 char filename_buf[PATH_MAX]; 2079 const char *at = ""; 2080 const char *fn = ""; 2081 char *filter; 2082 int err; 2083 2084 if (filt->filename) { 2085 at = "@"; 2086 fn = realpath(filt->filename, filename_buf); 2087 if (!fn) 2088 return NULL; 2089 } 2090 2091 if (filt->range) { 2092 err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s", 2093 filt->action, filt->addr, filt->size, at, fn); 2094 } else { 2095 err = asprintf(&filter, "%s 0x%"PRIx64"%s%s", 2096 filt->action, filt->addr, at, fn); 2097 } 2098 2099 return err < 0 ? NULL : filter; 2100 } 2101 2102 static int parse_addr_filter(struct evsel *evsel, const char *filter, 2103 int max_nr) 2104 { 2105 struct addr_filters filts; 2106 struct addr_filter *filt; 2107 int err; 2108 2109 addr_filters__init(&filts); 2110 2111 err = addr_filters__parse_bare_filter(&filts, filter); 2112 if (err) 2113 goto out_exit; 2114 2115 if (filts.cnt > max_nr) { 2116 pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n", 2117 filts.cnt, max_nr); 2118 err = -EINVAL; 2119 goto out_exit; 2120 } 2121 2122 list_for_each_entry(filt, &filts.head, list) { 2123 char *new_filter; 2124 2125 err = addr_filter__resolve_syms(filt); 2126 if (err) 2127 goto out_exit; 2128 2129 new_filter = addr_filter__to_str(filt); 2130 if (!new_filter) { 2131 err = -ENOMEM; 2132 goto out_exit; 2133 } 2134 2135 if (perf_evsel__append_addr_filter(evsel, new_filter)) { 2136 err = -ENOMEM; 2137 goto out_exit; 2138 } 2139 } 2140 2141 out_exit: 2142 addr_filters__exit(&filts); 2143 2144 if (err) { 2145 pr_err("Failed to parse address filter: '%s'\n", filter); 2146 pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n"); 2147 pr_err("Where multiple filters are separated by space or comma.\n"); 2148 } 2149 2150 return err; 2151 } 2152 2153 static struct perf_pmu *perf_evsel__find_pmu(struct evsel *evsel) 2154 { 2155 struct perf_pmu *pmu = NULL; 2156 2157 while ((pmu = perf_pmu__scan(pmu)) != NULL) { 2158 if (pmu->type == evsel->core.attr.type) 2159 break; 2160 } 2161 2162 return pmu; 2163 } 2164 2165 static int perf_evsel__nr_addr_filter(struct evsel *evsel) 2166 { 2167 struct perf_pmu *pmu = perf_evsel__find_pmu(evsel); 2168 int nr_addr_filters = 0; 2169 2170 if (!pmu) 2171 return 0; 2172 2173 perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters); 2174 2175 return nr_addr_filters; 2176 } 2177 2178 int auxtrace_parse_filters(struct evlist *evlist) 2179 { 2180 struct evsel *evsel; 2181 char *filter; 2182 int err, max_nr; 2183 2184 evlist__for_each_entry(evlist, evsel) { 2185 filter = evsel->filter; 2186 max_nr = perf_evsel__nr_addr_filter(evsel); 2187 if (!filter || !max_nr) 2188 continue; 2189 evsel->filter = NULL; 2190 err = parse_addr_filter(evsel, filter, max_nr); 2191 free(filter); 2192 if (err) 2193 return err; 2194 pr_debug("Address filter: %s\n", evsel->filter); 2195 } 2196 2197 return 0; 2198 } 2199 2200 int auxtrace__process_event(struct perf_session *session, union perf_event *event, 2201 struct perf_sample *sample, struct perf_tool *tool) 2202 { 2203 if (!session->auxtrace) 2204 return 0; 2205 2206 return session->auxtrace->process_event(session, event, sample, tool); 2207 } 2208 2209 int auxtrace__flush_events(struct perf_session *session, struct perf_tool *tool) 2210 { 2211 if (!session->auxtrace) 2212 return 0; 2213 2214 return session->auxtrace->flush_events(session, tool); 2215 } 2216 2217 void auxtrace__free_events(struct perf_session *session) 2218 { 2219 if (!session->auxtrace) 2220 return; 2221 2222 return session->auxtrace->free_events(session); 2223 } 2224 2225 void auxtrace__free(struct perf_session *session) 2226 { 2227 if (!session->auxtrace) 2228 return; 2229 2230 return session->auxtrace->free(session); 2231 } 2232