Lines Matching refs:ptq
352 struct intel_pt_queue *ptq = data; in intel_pt_findnew_vmcs_info() local
353 struct intel_pt *pt = ptq->pt; in intel_pt_findnew_vmcs_info()
400 static int intel_pt_get_buffer(struct intel_pt_queue *ptq, in intel_pt_get_buffer() argument
408 int fd = perf_data__fd(ptq->pt->session->data); in intel_pt_get_buffer()
415 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode; in intel_pt_get_buffer()
417 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_buffer()
440 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq, in intel_pt_lookahead_drop_buffer() argument
443 if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer) in intel_pt_lookahead_drop_buffer()
453 struct intel_pt_queue *ptq = data; in intel_pt_lookahead() local
454 struct auxtrace_buffer *buffer = ptq->buffer; in intel_pt_lookahead()
455 struct auxtrace_buffer *old_buffer = ptq->old_buffer; in intel_pt_lookahead()
459 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_lookahead()
468 err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b); in intel_pt_lookahead()
473 intel_pt_lookahead_drop_buffer(ptq, old_buffer); in intel_pt_lookahead()
476 intel_pt_lookahead_drop_buffer(ptq, buffer); in intel_pt_lookahead()
486 intel_pt_lookahead_drop_buffer(ptq, buffer); in intel_pt_lookahead()
487 intel_pt_lookahead_drop_buffer(ptq, old_buffer); in intel_pt_lookahead()
498 struct intel_pt_queue *ptq = data; in intel_pt_get_trace() local
499 struct auxtrace_buffer *buffer = ptq->buffer; in intel_pt_get_trace()
500 struct auxtrace_buffer *old_buffer = ptq->old_buffer; in intel_pt_get_trace()
504 if (ptq->stop) { in intel_pt_get_trace()
509 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
519 ptq->buffer = buffer; in intel_pt_get_trace()
521 err = intel_pt_get_buffer(ptq, buffer, old_buffer, b); in intel_pt_get_trace()
525 if (ptq->step_through_buffers) in intel_pt_get_trace()
526 ptq->stop = true; in intel_pt_get_trace()
531 ptq->old_buffer = buffer; in intel_pt_get_trace()
673 static inline u8 intel_pt_nr_cpumode(struct intel_pt_queue *ptq, uint64_t ip, bool nr) in intel_pt_nr_cpumode() argument
681 return ip >= ptq->pt->kernel_start ? in intel_pt_nr_cpumode()
686 static inline u8 intel_pt_cpumode(struct intel_pt_queue *ptq, uint64_t from_ip, uint64_t to_ip) in intel_pt_cpumode() argument
690 return intel_pt_nr_cpumode(ptq, from_ip, ptq->state->from_nr); in intel_pt_cpumode()
691 return intel_pt_nr_cpumode(ptq, to_ip, ptq->state->to_nr); in intel_pt_cpumode()
694 static int intel_pt_get_guest(struct intel_pt_queue *ptq) in intel_pt_get_guest() argument
696 struct machines *machines = &ptq->pt->session->machines; in intel_pt_get_guest()
698 pid_t pid = ptq->pid <= 0 ? DEFAULT_GUEST_KERNEL_ID : ptq->pid; in intel_pt_get_guest()
700 if (ptq->guest_machine && pid == ptq->guest_machine->pid) in intel_pt_get_guest()
703 ptq->guest_machine = NULL; in intel_pt_get_guest()
704 thread__zput(ptq->unknown_guest_thread); in intel_pt_get_guest()
707 thread__zput(ptq->guest_thread); in intel_pt_get_guest()
708 ptq->guest_thread = machines__findnew_guest_code(machines, pid); in intel_pt_get_guest()
715 ptq->unknown_guest_thread = machine__idle_thread(machine); in intel_pt_get_guest()
716 if (!ptq->unknown_guest_thread) in intel_pt_get_guest()
719 ptq->guest_machine = machine; in intel_pt_get_guest()
751 struct intel_pt_queue *ptq = data; in intel_pt_walk_next_insn() local
752 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
772 nr = ptq->state->to_nr; in intel_pt_walk_next_insn()
773 cpumode = intel_pt_nr_cpumode(ptq, *ip, nr); in intel_pt_walk_next_insn()
776 if (ptq->pt->have_guest_sideband) { in intel_pt_walk_next_insn()
777 if (!ptq->guest_machine || ptq->guest_machine_pid != ptq->pid) { in intel_pt_walk_next_insn()
783 intel_pt_get_guest(ptq)) { in intel_pt_walk_next_insn()
788 machine = ptq->guest_machine; in intel_pt_walk_next_insn()
789 thread = ptq->guest_thread; in intel_pt_walk_next_insn()
796 thread = ptq->unknown_guest_thread; in intel_pt_walk_next_insn()
799 thread = ptq->thread; in intel_pt_walk_next_insn()
806 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
985 struct intel_pt_queue *ptq = data; in __intel_pt_pgd_ip() local
992 if (ptq->state->to_nr) { in __intel_pt_pgd_ip()
994 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
997 } else if (ip >= ptq->pt->kernel_start) { in __intel_pt_pgd_ip()
998 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
1003 thread = ptq->thread; in __intel_pt_pgd_ip()
1013 res = intel_pt_match_pgd_ip(ptq->pt, ip, offset, map__dso(al.map)->long_name); in __intel_pt_pgd_ip()
1296 struct intel_pt_queue *ptq; in intel_pt_alloc_queue() local
1298 ptq = zalloc(sizeof(struct intel_pt_queue)); in intel_pt_alloc_queue()
1299 if (!ptq) in intel_pt_alloc_queue()
1303 ptq->chain = intel_pt_alloc_chain(pt); in intel_pt_alloc_queue()
1304 if (!ptq->chain) in intel_pt_alloc_queue()
1311 ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt); in intel_pt_alloc_queue()
1312 if (!ptq->last_branch) in intel_pt_alloc_queue()
1316 ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); in intel_pt_alloc_queue()
1317 if (!ptq->event_buf) in intel_pt_alloc_queue()
1320 ptq->pt = pt; in intel_pt_alloc_queue()
1321 ptq->queue_nr = queue_nr; in intel_pt_alloc_queue()
1322 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
1323 ptq->pid = -1; in intel_pt_alloc_queue()
1324 ptq->tid = -1; in intel_pt_alloc_queue()
1325 ptq->cpu = -1; in intel_pt_alloc_queue()
1326 ptq->next_tid = -1; in intel_pt_alloc_queue()
1332 params.data = ptq; in intel_pt_alloc_queue()
1384 ptq->decoder = intel_pt_decoder_new(¶ms); in intel_pt_alloc_queue()
1385 if (!ptq->decoder) in intel_pt_alloc_queue()
1388 return ptq; in intel_pt_alloc_queue()
1391 zfree(&ptq->event_buf); in intel_pt_alloc_queue()
1392 zfree(&ptq->last_branch); in intel_pt_alloc_queue()
1393 zfree(&ptq->chain); in intel_pt_alloc_queue()
1394 free(ptq); in intel_pt_alloc_queue()
1400 struct intel_pt_queue *ptq = priv; in intel_pt_free_queue() local
1402 if (!ptq) in intel_pt_free_queue()
1404 thread__zput(ptq->thread); in intel_pt_free_queue()
1405 thread__zput(ptq->guest_thread); in intel_pt_free_queue()
1406 thread__zput(ptq->unknown_guest_thread); in intel_pt_free_queue()
1407 intel_pt_decoder_free(ptq->decoder); in intel_pt_free_queue()
1408 zfree(&ptq->event_buf); in intel_pt_free_queue()
1409 zfree(&ptq->last_branch); in intel_pt_free_queue()
1410 zfree(&ptq->chain); in intel_pt_free_queue()
1411 free(ptq); in intel_pt_free_queue()
1422 struct intel_pt_queue *ptq = queue->priv; in intel_pt_first_timestamp() local
1424 if (ptq && ptq->decoder) in intel_pt_first_timestamp()
1425 intel_pt_set_first_timestamp(ptq->decoder, timestamp); in intel_pt_first_timestamp()
1429 static int intel_pt_get_guest_from_sideband(struct intel_pt_queue *ptq) in intel_pt_get_guest_from_sideband() argument
1431 struct machines *machines = &ptq->pt->session->machines; in intel_pt_get_guest_from_sideband()
1433 pid_t machine_pid = ptq->pid; in intel_pt_get_guest_from_sideband()
1444 if (ptq->guest_machine != machine) { in intel_pt_get_guest_from_sideband()
1445 ptq->guest_machine = NULL; in intel_pt_get_guest_from_sideband()
1446 thread__zput(ptq->guest_thread); in intel_pt_get_guest_from_sideband()
1447 thread__zput(ptq->unknown_guest_thread); in intel_pt_get_guest_from_sideband()
1449 ptq->unknown_guest_thread = machine__find_thread(machine, 0, 0); in intel_pt_get_guest_from_sideband()
1450 if (!ptq->unknown_guest_thread) in intel_pt_get_guest_from_sideband()
1452 ptq->guest_machine = machine; in intel_pt_get_guest_from_sideband()
1455 vcpu = ptq->thread ? thread__guest_cpu(ptq->thread) : -1; in intel_pt_get_guest_from_sideband()
1461 if (ptq->guest_thread && thread__tid(ptq->guest_thread) != tid) in intel_pt_get_guest_from_sideband()
1462 thread__zput(ptq->guest_thread); in intel_pt_get_guest_from_sideband()
1464 if (!ptq->guest_thread) { in intel_pt_get_guest_from_sideband()
1465 ptq->guest_thread = machine__find_thread(machine, -1, tid); in intel_pt_get_guest_from_sideband()
1466 if (!ptq->guest_thread) in intel_pt_get_guest_from_sideband()
1470 ptq->guest_machine_pid = machine_pid; in intel_pt_get_guest_from_sideband()
1471 ptq->guest_pid = thread__pid(ptq->guest_thread); in intel_pt_get_guest_from_sideband()
1472 ptq->guest_tid = tid; in intel_pt_get_guest_from_sideband()
1473 ptq->vcpu = vcpu; in intel_pt_get_guest_from_sideband()
1481 struct intel_pt_queue *ptq = queue->priv; in intel_pt_set_pid_tid_cpu() local
1484 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
1485 if (ptq->tid == -1) in intel_pt_set_pid_tid_cpu()
1486 ptq->pid = -1; in intel_pt_set_pid_tid_cpu()
1487 thread__zput(ptq->thread); in intel_pt_set_pid_tid_cpu()
1490 if (!ptq->thread && ptq->tid != -1) in intel_pt_set_pid_tid_cpu()
1491 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
1493 if (ptq->thread) { in intel_pt_set_pid_tid_cpu()
1494 ptq->pid = thread__pid(ptq->thread); in intel_pt_set_pid_tid_cpu()
1496 ptq->cpu = thread__cpu(ptq->thread); in intel_pt_set_pid_tid_cpu()
1499 if (pt->have_guest_sideband && intel_pt_get_guest_from_sideband(ptq)) { in intel_pt_set_pid_tid_cpu()
1500 ptq->guest_machine_pid = 0; in intel_pt_set_pid_tid_cpu()
1501 ptq->guest_pid = -1; in intel_pt_set_pid_tid_cpu()
1502 ptq->guest_tid = -1; in intel_pt_set_pid_tid_cpu()
1503 ptq->vcpu = -1; in intel_pt_set_pid_tid_cpu()
1507 static void intel_pt_sample_flags(struct intel_pt_queue *ptq) in intel_pt_sample_flags() argument
1509 struct intel_pt *pt = ptq->pt; in intel_pt_sample_flags()
1511 ptq->insn_len = 0; in intel_pt_sample_flags()
1512 if (ptq->state->flags & INTEL_PT_ABORT_TX) { in intel_pt_sample_flags()
1513 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT; in intel_pt_sample_flags()
1514 } else if (ptq->state->flags & INTEL_PT_ASYNC) { in intel_pt_sample_flags()
1515 if (!ptq->state->to_ip) in intel_pt_sample_flags()
1516 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
1519 else if (ptq->state->from_nr && !ptq->state->to_nr) in intel_pt_sample_flags()
1520 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | in intel_pt_sample_flags()
1524 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | in intel_pt_sample_flags()
1528 if (ptq->state->from_ip) in intel_pt_sample_flags()
1529 ptq->flags = intel_pt_insn_type(ptq->state->insn_op); in intel_pt_sample_flags()
1531 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
1533 if (ptq->state->flags & INTEL_PT_IN_TX) in intel_pt_sample_flags()
1534 ptq->flags |= PERF_IP_FLAG_IN_TX; in intel_pt_sample_flags()
1535 ptq->insn_len = ptq->state->insn_len; in intel_pt_sample_flags()
1536 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ); in intel_pt_sample_flags()
1539 if (ptq->state->type & INTEL_PT_TRACE_BEGIN) in intel_pt_sample_flags()
1540 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN; in intel_pt_sample_flags()
1541 if (ptq->state->type & INTEL_PT_TRACE_END) in intel_pt_sample_flags()
1542 ptq->flags |= PERF_IP_FLAG_TRACE_END; in intel_pt_sample_flags()
1545 if (ptq->state->type & INTEL_PT_IFLAG_CHG) { in intel_pt_sample_flags()
1546 if (!ptq->state->from_iflag) in intel_pt_sample_flags()
1547 ptq->flags |= PERF_IP_FLAG_INTR_DISABLE; in intel_pt_sample_flags()
1548 if (ptq->state->from_iflag != ptq->state->to_iflag) in intel_pt_sample_flags()
1549 ptq->flags |= PERF_IP_FLAG_INTR_TOGGLE; in intel_pt_sample_flags()
1550 } else if (!ptq->state->to_iflag) { in intel_pt_sample_flags()
1551 ptq->flags |= PERF_IP_FLAG_INTR_DISABLE; in intel_pt_sample_flags()
1557 struct intel_pt_queue *ptq) in intel_pt_setup_time_range() argument
1562 ptq->sel_timestamp = pt->time_ranges[0].start; in intel_pt_setup_time_range()
1563 ptq->sel_idx = 0; in intel_pt_setup_time_range()
1565 if (ptq->sel_timestamp) { in intel_pt_setup_time_range()
1566 ptq->sel_start = true; in intel_pt_setup_time_range()
1568 ptq->sel_timestamp = pt->time_ranges[0].end; in intel_pt_setup_time_range()
1569 ptq->sel_start = false; in intel_pt_setup_time_range()
1577 struct intel_pt_queue *ptq = queue->priv; in intel_pt_setup_queue() local
1582 if (!ptq) { in intel_pt_setup_queue()
1583 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
1584 if (!ptq) in intel_pt_setup_queue()
1586 queue->priv = ptq; in intel_pt_setup_queue()
1589 ptq->cpu = queue->cpu; in intel_pt_setup_queue()
1590 ptq->tid = queue->tid; in intel_pt_setup_queue()
1592 ptq->cbr_seen = UINT_MAX; in intel_pt_setup_queue()
1596 ptq->step_through_buffers = true; in intel_pt_setup_queue()
1598 ptq->sync_switch = pt->sync_switch; in intel_pt_setup_queue()
1600 intel_pt_setup_time_range(pt, ptq); in intel_pt_setup_queue()
1603 if (!ptq->on_heap && in intel_pt_setup_queue()
1604 (!ptq->sync_switch || in intel_pt_setup_queue()
1605 ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) { in intel_pt_setup_queue()
1614 queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_setup_queue()
1616 if (ptq->sel_start && ptq->sel_timestamp) { in intel_pt_setup_queue()
1617 ret = intel_pt_fast_forward(ptq->decoder, in intel_pt_setup_queue()
1618 ptq->sel_timestamp); in intel_pt_setup_queue()
1624 state = intel_pt_decode(ptq->decoder); in intel_pt_setup_queue()
1637 ptq->timestamp = state->timestamp; in intel_pt_setup_queue()
1639 queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1640 ptq->state = state; in intel_pt_setup_queue()
1641 ptq->have_sample = true; in intel_pt_setup_queue()
1642 if (ptq->sel_start && ptq->sel_timestamp && in intel_pt_setup_queue()
1643 ptq->timestamp < ptq->sel_timestamp) in intel_pt_setup_queue()
1644 ptq->have_sample = false; in intel_pt_setup_queue()
1645 intel_pt_sample_flags(ptq); in intel_pt_setup_queue()
1646 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1649 ptq->on_heap = true; in intel_pt_setup_queue()
1685 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq, in intel_pt_prep_a_sample() argument
1692 sample->pid = ptq->pid; in intel_pt_prep_a_sample()
1693 sample->tid = ptq->tid; in intel_pt_prep_a_sample()
1695 if (ptq->pt->have_guest_sideband) { in intel_pt_prep_a_sample()
1696 if ((ptq->state->from_ip && ptq->state->from_nr) || in intel_pt_prep_a_sample()
1697 (ptq->state->to_ip && ptq->state->to_nr)) { in intel_pt_prep_a_sample()
1698 sample->pid = ptq->guest_pid; in intel_pt_prep_a_sample()
1699 sample->tid = ptq->guest_tid; in intel_pt_prep_a_sample()
1700 sample->machine_pid = ptq->guest_machine_pid; in intel_pt_prep_a_sample()
1701 sample->vcpu = ptq->vcpu; in intel_pt_prep_a_sample()
1705 sample->cpu = ptq->cpu; in intel_pt_prep_a_sample()
1706 sample->insn_len = ptq->insn_len; in intel_pt_prep_a_sample()
1707 memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ); in intel_pt_prep_a_sample()
1711 struct intel_pt_queue *ptq, in intel_pt_prep_b_sample() argument
1715 intel_pt_prep_a_sample(ptq, event, sample); in intel_pt_prep_b_sample()
1718 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_prep_b_sample()
1720 sample->ip = ptq->state->from_ip; in intel_pt_prep_b_sample()
1721 sample->addr = ptq->state->to_ip; in intel_pt_prep_b_sample()
1722 sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr); in intel_pt_prep_b_sample()
1724 sample->flags = ptq->flags; in intel_pt_prep_b_sample()
1763 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq) in intel_pt_synth_branch_sample() argument
1765 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample()
1766 union perf_event *event = ptq->event_buf; in intel_pt_synth_branch_sample()
1774 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
1780 intel_pt_prep_b_sample(pt, ptq, event, &sample); in intel_pt_synth_branch_sample()
1782 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1783 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1801 if (ptq->sample_ipc) in intel_pt_synth_branch_sample()
1802 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt; in intel_pt_synth_branch_sample()
1804 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt; in intel_pt_synth_branch_sample()
1805 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_branch_sample()
1806 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_branch_sample()
1814 struct intel_pt_queue *ptq, in intel_pt_prep_sample() argument
1818 intel_pt_prep_b_sample(pt, ptq, event, sample); in intel_pt_prep_sample()
1821 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain, in intel_pt_prep_sample()
1824 sample->callchain = ptq->chain; in intel_pt_prep_sample()
1828 thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch, in intel_pt_prep_sample()
1830 sample->branch_stack = ptq->last_branch; in intel_pt_prep_sample()
1834 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_instruction_sample() argument
1836 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample()
1837 union perf_event *event = ptq->event_buf; in intel_pt_synth_instruction_sample()
1843 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_instruction_sample()
1845 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1846 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1850 sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt; in intel_pt_synth_instruction_sample()
1852 if (ptq->sample_ipc) in intel_pt_synth_instruction_sample()
1853 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt; in intel_pt_synth_instruction_sample()
1855 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt; in intel_pt_synth_instruction_sample()
1856 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_instruction_sample()
1857 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_instruction_sample()
1860 ptq->last_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_synth_instruction_sample()
1866 static int intel_pt_synth_cycle_sample(struct intel_pt_queue *ptq) in intel_pt_synth_cycle_sample() argument
1868 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cycle_sample()
1869 union perf_event *event = ptq->event_buf; in intel_pt_synth_cycle_sample()
1873 if (ptq->sample_ipc) in intel_pt_synth_cycle_sample()
1874 period = ptq->ipc_cyc_cnt - ptq->last_cy_cyc_cnt; in intel_pt_synth_cycle_sample()
1879 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_cycle_sample()
1881 sample.id = ptq->pt->cycles_id; in intel_pt_synth_cycle_sample()
1882 sample.stream_id = ptq->pt->cycles_id; in intel_pt_synth_cycle_sample()
1886 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_cy_insn_cnt; in intel_pt_synth_cycle_sample()
1887 ptq->last_cy_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_cycle_sample()
1888 ptq->last_cy_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_cycle_sample()
1893 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_transaction_sample() argument
1895 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample()
1896 union perf_event *event = ptq->event_buf; in intel_pt_synth_transaction_sample()
1902 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_transaction_sample()
1904 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1905 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1912 struct intel_pt_queue *ptq, in intel_pt_prep_p_sample() argument
1916 intel_pt_prep_sample(pt, ptq, event, sample); in intel_pt_prep_p_sample()
1926 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq) in intel_pt_synth_ptwrite_sample() argument
1928 struct intel_pt *pt = ptq->pt; in intel_pt_synth_ptwrite_sample()
1929 union perf_event *event = ptq->event_buf; in intel_pt_synth_ptwrite_sample()
1936 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_ptwrite_sample()
1938 sample.id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1939 sample.stream_id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1942 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_ptwrite_sample()
1943 raw.payload = cpu_to_le64(ptq->state->ptw_payload); in intel_pt_synth_ptwrite_sample()
1952 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq) in intel_pt_synth_cbr_sample() argument
1954 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cbr_sample()
1955 union perf_event *event = ptq->event_buf; in intel_pt_synth_cbr_sample()
1963 ptq->cbr_seen = ptq->state->cbr; in intel_pt_synth_cbr_sample()
1965 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_cbr_sample()
1967 sample.id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1968 sample.stream_id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1970 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16); in intel_pt_synth_cbr_sample()
1982 static int intel_pt_synth_psb_sample(struct intel_pt_queue *ptq) in intel_pt_synth_psb_sample() argument
1984 struct intel_pt *pt = ptq->pt; in intel_pt_synth_psb_sample()
1985 union perf_event *event = ptq->event_buf; in intel_pt_synth_psb_sample()
1992 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_psb_sample()
1994 sample.id = ptq->pt->psb_id; in intel_pt_synth_psb_sample()
1995 sample.stream_id = ptq->pt->psb_id; in intel_pt_synth_psb_sample()
1999 raw.offset = ptq->state->psb_offset; in intel_pt_synth_psb_sample()
2008 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq) in intel_pt_synth_mwait_sample() argument
2010 struct intel_pt *pt = ptq->pt; in intel_pt_synth_mwait_sample()
2011 union perf_event *event = ptq->event_buf; in intel_pt_synth_mwait_sample()
2018 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_mwait_sample()
2020 sample.id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
2021 sample.stream_id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
2024 raw.payload = cpu_to_le64(ptq->state->mwait_payload); in intel_pt_synth_mwait_sample()
2033 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pwre_sample() argument
2035 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwre_sample()
2036 union perf_event *event = ptq->event_buf; in intel_pt_synth_pwre_sample()
2043 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwre_sample()
2045 sample.id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
2046 sample.stream_id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
2049 raw.payload = cpu_to_le64(ptq->state->pwre_payload); in intel_pt_synth_pwre_sample()
2058 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq) in intel_pt_synth_exstop_sample() argument
2060 struct intel_pt *pt = ptq->pt; in intel_pt_synth_exstop_sample()
2061 union perf_event *event = ptq->event_buf; in intel_pt_synth_exstop_sample()
2068 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_exstop_sample()
2070 sample.id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
2071 sample.stream_id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
2074 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_exstop_sample()
2083 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pwrx_sample() argument
2085 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwrx_sample()
2086 union perf_event *event = ptq->event_buf; in intel_pt_synth_pwrx_sample()
2093 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwrx_sample()
2095 sample.id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
2096 sample.stream_id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
2099 raw.payload = cpu_to_le64(ptq->state->pwrx_payload); in intel_pt_synth_pwrx_sample()
2235 static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evsel *evsel, u64 id) in intel_pt_do_synth_pebs_sample() argument
2237 const struct intel_pt_blk_items *items = &ptq->state->items; in intel_pt_do_synth_pebs_sample()
2239 union perf_event *event = ptq->event_buf; in intel_pt_do_synth_pebs_sample()
2240 struct intel_pt *pt = ptq->pt; in intel_pt_do_synth_pebs_sample()
2248 intel_pt_prep_a_sample(ptq, event, &sample); in intel_pt_do_synth_pebs_sample()
2262 sample.ip = ptq->state->from_ip; in intel_pt_do_synth_pebs_sample()
2264 cpumode = intel_pt_cpumode(ptq, sample.ip, 0); in intel_pt_do_synth_pebs_sample()
2276 timestamp = ptq->timestamp; in intel_pt_do_synth_pebs_sample()
2283 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain, in intel_pt_do_synth_pebs_sample()
2286 sample.callchain = ptq->chain; in intel_pt_do_synth_pebs_sample()
2309 intel_pt_add_lbrs(ptq->last_branch, items); in intel_pt_do_synth_pebs_sample()
2311 thread_stack__br_sample(ptq->thread, ptq->cpu, in intel_pt_do_synth_pebs_sample()
2312 ptq->last_branch, in intel_pt_do_synth_pebs_sample()
2315 ptq->last_branch->nr = 0; in intel_pt_do_synth_pebs_sample()
2317 sample.branch_stack = ptq->last_branch; in intel_pt_do_synth_pebs_sample()
2367 static int intel_pt_synth_single_pebs_sample(struct intel_pt_queue *ptq) in intel_pt_synth_single_pebs_sample() argument
2369 struct intel_pt *pt = ptq->pt; in intel_pt_synth_single_pebs_sample()
2373 return intel_pt_do_synth_pebs_sample(ptq, evsel, id); in intel_pt_synth_single_pebs_sample()
2376 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pebs_sample() argument
2378 const struct intel_pt_blk_items *items = &ptq->state->items; in intel_pt_synth_pebs_sample()
2380 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pebs_sample()
2387 return intel_pt_synth_single_pebs_sample(ptq); in intel_pt_synth_pebs_sample()
2391 pe = &ptq->pebs[hw_id]; in intel_pt_synth_pebs_sample()
2396 return intel_pt_synth_single_pebs_sample(ptq); in intel_pt_synth_pebs_sample()
2398 err = intel_pt_do_synth_pebs_sample(ptq, pe->evsel, pe->id); in intel_pt_synth_pebs_sample()
2406 static int intel_pt_synth_events_sample(struct intel_pt_queue *ptq) in intel_pt_synth_events_sample() argument
2408 struct intel_pt *pt = ptq->pt; in intel_pt_synth_events_sample()
2409 union perf_event *event = ptq->event_buf; in intel_pt_synth_events_sample()
2420 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_events_sample()
2422 sample.id = ptq->pt->evt_id; in intel_pt_synth_events_sample()
2423 sample.stream_id = ptq->pt->evt_id; in intel_pt_synth_events_sample()
2425 raw.cfe.type = ptq->state->cfe_type; in intel_pt_synth_events_sample()
2427 raw.cfe.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_events_sample()
2428 raw.cfe.vector = ptq->state->cfe_vector; in intel_pt_synth_events_sample()
2429 raw.cfe.evd_cnt = ptq->state->evd_cnt; in intel_pt_synth_events_sample()
2431 for (i = 0; i < ptq->state->evd_cnt; i++) { in intel_pt_synth_events_sample()
2433 raw.evd[i].evd_type = ptq->state->evd[i].type; in intel_pt_synth_events_sample()
2434 raw.evd[i].payload = ptq->state->evd[i].payload; in intel_pt_synth_events_sample()
2438 ptq->state->evd_cnt * sizeof(struct perf_synth_intel_evd); in intel_pt_synth_events_sample()
2445 static int intel_pt_synth_iflag_chg_sample(struct intel_pt_queue *ptq) in intel_pt_synth_iflag_chg_sample() argument
2447 struct intel_pt *pt = ptq->pt; in intel_pt_synth_iflag_chg_sample()
2448 union perf_event *event = ptq->event_buf; in intel_pt_synth_iflag_chg_sample()
2455 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_iflag_chg_sample()
2457 sample.id = ptq->pt->iflag_chg_id; in intel_pt_synth_iflag_chg_sample()
2458 sample.stream_id = ptq->pt->iflag_chg_id; in intel_pt_synth_iflag_chg_sample()
2461 raw.iflag = ptq->state->to_iflag; in intel_pt_synth_iflag_chg_sample()
2463 if (ptq->state->type & INTEL_PT_BRANCH) { in intel_pt_synth_iflag_chg_sample()
2465 raw.branch_ip = ptq->state->to_ip; in intel_pt_synth_iflag_chg_sample()
2469 sample.flags = ptq->flags; in intel_pt_synth_iflag_chg_sample()
2521 static int intel_ptq_synth_error(struct intel_pt_queue *ptq, in intel_ptq_synth_error() argument
2524 struct intel_pt *pt = ptq->pt; in intel_ptq_synth_error()
2525 u64 tm = ptq->timestamp; in intel_ptq_synth_error()
2527 pid_t pid = ptq->pid; in intel_ptq_synth_error()
2528 pid_t tid = ptq->tid; in intel_ptq_synth_error()
2534 machine_pid = ptq->guest_machine_pid; in intel_ptq_synth_error()
2535 vcpu = ptq->vcpu; in intel_ptq_synth_error()
2536 pid = ptq->guest_pid; in intel_ptq_synth_error()
2537 tid = ptq->guest_tid; in intel_ptq_synth_error()
2540 return intel_pt_synth_error(pt, state->err, ptq->cpu, pid, tid, in intel_ptq_synth_error()
2544 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
2547 pid_t tid = ptq->next_tid; in intel_pt_next_tid()
2553 intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid); in intel_pt_next_tid()
2555 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
2557 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
2560 ptq->next_tid = -1; in intel_pt_next_tid()
2565 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip) in intel_pt_is_switch_ip() argument
2567 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip()
2570 (ptq->flags & PERF_IP_FLAG_BRANCH) && in intel_pt_is_switch_ip()
2571 !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC | in intel_pt_is_switch_ip()
2578 static int intel_pt_sample(struct intel_pt_queue *ptq) in intel_pt_sample() argument
2580 const struct intel_pt_state *state = ptq->state; in intel_pt_sample()
2581 struct intel_pt *pt = ptq->pt; in intel_pt_sample()
2584 if (!ptq->have_sample) in intel_pt_sample()
2587 ptq->have_sample = false; in intel_pt_sample()
2590 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_sample()
2591 ptq->ipc_cyc_cnt = ptq->state->cycles; in intel_pt_sample()
2592 ptq->sample_ipc = true; in intel_pt_sample()
2594 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_sample()
2595 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt; in intel_pt_sample()
2596 ptq->sample_ipc = ptq->state->flags & INTEL_PT_SAMPLE_IPC; in intel_pt_sample()
2601 intel_pt_get_guest(ptq); in intel_pt_sample()
2608 err = intel_pt_synth_pebs_sample(ptq); in intel_pt_sample()
2615 err = intel_pt_synth_events_sample(ptq); in intel_pt_sample()
2620 err = intel_pt_synth_iflag_chg_sample(ptq); in intel_pt_sample()
2628 err = intel_pt_synth_psb_sample(ptq); in intel_pt_sample()
2632 if (ptq->state->cbr != ptq->cbr_seen) { in intel_pt_sample()
2633 err = intel_pt_synth_cbr_sample(ptq); in intel_pt_sample()
2639 err = intel_pt_synth_mwait_sample(ptq); in intel_pt_sample()
2644 err = intel_pt_synth_pwre_sample(ptq); in intel_pt_sample()
2649 err = intel_pt_synth_exstop_sample(ptq); in intel_pt_sample()
2654 err = intel_pt_synth_pwrx_sample(ptq); in intel_pt_sample()
2663 err = intel_pt_synth_instruction_sample(ptq); in intel_pt_sample()
2668 err = intel_pt_synth_cycle_sample(ptq); in intel_pt_sample()
2675 err = intel_pt_synth_transaction_sample(ptq); in intel_pt_sample()
2681 err = intel_pt_synth_ptwrite_sample(ptq); in intel_pt_sample()
2690 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, in intel_pt_sample()
2691 state->from_ip, state->to_ip, ptq->insn_len, in intel_pt_sample()
2696 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr); in intel_pt_sample()
2711 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2716 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2719 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2725 if (!ptq->sync_switch) in intel_pt_sample()
2728 if (intel_pt_is_switch_ip(ptq, state->to_ip)) { in intel_pt_sample()
2729 switch (ptq->switch_state) { in intel_pt_sample()
2733 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
2736 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
2739 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT; in intel_pt_sample()
2743 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_sample()
2744 } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) { in intel_pt_sample()
2745 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_sample()
2746 } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_sample()
2748 (ptq->flags & PERF_IP_FLAG_CALL)) { in intel_pt_sample()
2749 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
2818 struct intel_pt_queue *ptq = queue->priv; in intel_pt_enable_sync_switch() local
2820 if (ptq) in intel_pt_enable_sync_switch()
2821 ptq->sync_switch = true; in intel_pt_enable_sync_switch()
2833 struct intel_pt_queue *ptq = queue->priv; in intel_pt_disable_sync_switch() local
2835 if (ptq) { in intel_pt_disable_sync_switch()
2836 ptq->sync_switch = false; in intel_pt_disable_sync_switch()
2837 intel_pt_next_tid(pt, ptq); in intel_pt_disable_sync_switch()
2846 static bool intel_pt_next_time(struct intel_pt_queue *ptq) in intel_pt_next_time() argument
2848 struct intel_pt *pt = ptq->pt; in intel_pt_next_time()
2850 if (ptq->sel_start) { in intel_pt_next_time()
2852 ptq->sel_start = false; in intel_pt_next_time()
2853 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end; in intel_pt_next_time()
2855 } else if (ptq->sel_idx + 1 < pt->range_cnt) { in intel_pt_next_time()
2857 ptq->sel_start = true; in intel_pt_next_time()
2858 ptq->sel_idx += 1; in intel_pt_next_time()
2859 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start; in intel_pt_next_time()
2867 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp) in intel_pt_time_filter() argument
2872 if (ptq->sel_start) { in intel_pt_time_filter()
2873 if (ptq->timestamp >= ptq->sel_timestamp) { in intel_pt_time_filter()
2875 intel_pt_next_time(ptq); in intel_pt_time_filter()
2876 if (!ptq->sel_timestamp) { in intel_pt_time_filter()
2884 ptq->have_sample = false; in intel_pt_time_filter()
2885 if (ptq->sel_timestamp > *ff_timestamp) { in intel_pt_time_filter()
2886 if (ptq->sync_switch) { in intel_pt_time_filter()
2887 intel_pt_next_tid(ptq->pt, ptq); in intel_pt_time_filter()
2888 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_time_filter()
2890 *ff_timestamp = ptq->sel_timestamp; in intel_pt_time_filter()
2891 err = intel_pt_fast_forward(ptq->decoder, in intel_pt_time_filter()
2892 ptq->sel_timestamp); in intel_pt_time_filter()
2897 } else if (ptq->timestamp > ptq->sel_timestamp) { in intel_pt_time_filter()
2899 if (!intel_pt_next_time(ptq)) { in intel_pt_time_filter()
2901 ptq->have_sample = false; in intel_pt_time_filter()
2902 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_time_filter()
2914 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp) in intel_pt_run_decoder() argument
2916 const struct intel_pt_state *state = ptq->state; in intel_pt_run_decoder()
2917 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder()
2937 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_run_decoder()
2939 err = intel_pt_sample(ptq); in intel_pt_run_decoder()
2943 state = intel_pt_decode(ptq->decoder); in intel_pt_run_decoder()
2947 if (ptq->sync_switch && in intel_pt_run_decoder()
2949 ptq->sync_switch = false; in intel_pt_run_decoder()
2950 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
2952 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2954 err = intel_ptq_synth_error(ptq, state); in intel_pt_run_decoder()
2961 ptq->state = state; in intel_pt_run_decoder()
2962 ptq->have_sample = true; in intel_pt_run_decoder()
2963 intel_pt_sample_flags(ptq); in intel_pt_run_decoder()
2971 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2973 } else if (ptq->sync_switch && in intel_pt_run_decoder()
2974 ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_run_decoder()
2975 intel_pt_is_switch_ip(ptq, state->to_ip) && in intel_pt_run_decoder()
2976 ptq->next_tid == -1) { in intel_pt_run_decoder()
2979 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2980 } else if (state->timestamp > ptq->timestamp) { in intel_pt_run_decoder()
2981 ptq->timestamp = state->timestamp; in intel_pt_run_decoder()
2984 if (ptq->sel_timestamp) { in intel_pt_run_decoder()
2985 err = intel_pt_time_filter(ptq, &ff_timestamp); in intel_pt_run_decoder()
2990 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
2991 *timestamp = ptq->timestamp; in intel_pt_run_decoder()
3015 struct intel_pt_queue *ptq; in intel_pt_process_queues() local
3025 ptq = queue->priv; in intel_pt_process_queues()
3043 ret = intel_pt_run_decoder(ptq, &ts); in intel_pt_process_queues()
3055 ptq->on_heap = false; in intel_pt_process_queues()
3071 struct intel_pt_queue *ptq = queue->priv; in intel_pt_process_timeless_queues() local
3073 if (ptq && (tid == -1 || ptq->tid == tid)) { in intel_pt_process_timeless_queues()
3074 ptq->time = time_; in intel_pt_process_timeless_queues()
3076 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_queues()
3082 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq, in intel_pt_sample_set_pid_tid_cpu() argument
3086 struct machine *m = ptq->pt->machine; in intel_pt_sample_set_pid_tid_cpu()
3088 ptq->pid = sample->pid; in intel_pt_sample_set_pid_tid_cpu()
3089 ptq->tid = sample->tid; in intel_pt_sample_set_pid_tid_cpu()
3090 ptq->cpu = queue->cpu; in intel_pt_sample_set_pid_tid_cpu()
3093 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
3095 thread__zput(ptq->thread); in intel_pt_sample_set_pid_tid_cpu()
3097 if (ptq->tid == -1) in intel_pt_sample_set_pid_tid_cpu()
3100 if (ptq->pid == -1) { in intel_pt_sample_set_pid_tid_cpu()
3101 ptq->thread = machine__find_thread(m, -1, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
3102 if (ptq->thread) in intel_pt_sample_set_pid_tid_cpu()
3103 ptq->pid = thread__pid(ptq->thread); in intel_pt_sample_set_pid_tid_cpu()
3107 ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
3114 struct intel_pt_queue *ptq; in intel_pt_process_timeless_sample() local
3121 ptq = queue->priv; in intel_pt_process_timeless_sample()
3122 if (!ptq) in intel_pt_process_timeless_sample()
3125 ptq->stop = false; in intel_pt_process_timeless_sample()
3126 ptq->time = sample->time; in intel_pt_process_timeless_sample()
3127 intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample); in intel_pt_process_timeless_sample()
3128 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_sample()
3170 struct intel_pt_queue *ptq; in intel_pt_sync_switch() local
3176 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
3177 if (!ptq || !ptq->sync_switch) in intel_pt_sync_switch()
3180 switch (ptq->switch_state) { in intel_pt_sync_switch()
3185 ptq->next_tid = tid; in intel_pt_sync_switch()
3186 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP; in intel_pt_sync_switch()
3189 if (!ptq->on_heap) { in intel_pt_sync_switch()
3190 ptq->timestamp = perf_time_to_tsc(timestamp, in intel_pt_sync_switch()
3192 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
3193 ptq->timestamp); in intel_pt_sync_switch()
3196 ptq->on_heap = true; in intel_pt_sync_switch()
3198 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sync_switch()
3207 ptq->next_tid = -1; in intel_pt_sync_switch()
3246 struct intel_pt_queue *ptq; in intel_pt_context_switch_in() local
3248 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_context_switch_in()
3249 if (ptq && ptq->sync_switch) { in intel_pt_context_switch_in()
3250 ptq->next_tid = -1; in intel_pt_context_switch_in()
3251 switch (ptq->switch_state) { in intel_pt_context_switch_in()
3258 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_context_switch_in()
3364 struct intel_pt_queue *ptq; in intel_pt_process_aux_output_hw_id() local
3374 ptq = queue->priv; in intel_pt_process_aux_output_hw_id()
3376 ptq->pebs[hw_id].evsel = evsel; in intel_pt_process_aux_output_hw_id()
3377 ptq->pebs[hw_id].id = sample->id; in intel_pt_process_aux_output_hw_id()