100447ccdSAdrian Hunter /* 200447ccdSAdrian Hunter * thread-stack.c: Synthesize a thread's stack using call / return events 300447ccdSAdrian Hunter * Copyright (c) 2014, Intel Corporation. 400447ccdSAdrian Hunter * 500447ccdSAdrian Hunter * This program is free software; you can redistribute it and/or modify it 600447ccdSAdrian Hunter * under the terms and conditions of the GNU General Public License, 700447ccdSAdrian Hunter * version 2, as published by the Free Software Foundation. 800447ccdSAdrian Hunter * 900447ccdSAdrian Hunter * This program is distributed in the hope it will be useful, but WITHOUT 1000447ccdSAdrian Hunter * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1100447ccdSAdrian Hunter * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1200447ccdSAdrian Hunter * more details. 1300447ccdSAdrian Hunter * 1400447ccdSAdrian Hunter */ 1500447ccdSAdrian Hunter 1692a9e4f7SAdrian Hunter #include <linux/rbtree.h> 1792a9e4f7SAdrian Hunter #include <linux/list.h> 18256d92bcSAdrian Hunter #include <linux/log2.h> 19a43783aeSArnaldo Carvalho de Melo #include <errno.h> 2000447ccdSAdrian Hunter #include "thread.h" 2100447ccdSAdrian Hunter #include "event.h" 2292a9e4f7SAdrian Hunter #include "machine.h" 2300447ccdSAdrian Hunter #include "util.h" 2400447ccdSAdrian Hunter #include "debug.h" 2592a9e4f7SAdrian Hunter #include "symbol.h" 2692a9e4f7SAdrian Hunter #include "comm.h" 27451db126SChris Phlipot #include "call-path.h" 2800447ccdSAdrian Hunter #include "thread-stack.h" 2900447ccdSAdrian Hunter 3092a9e4f7SAdrian Hunter #define STACK_GROWTH 2048 3192a9e4f7SAdrian Hunter 3292a9e4f7SAdrian Hunter /** 3392a9e4f7SAdrian Hunter * struct thread_stack_entry - thread stack entry. 3492a9e4f7SAdrian Hunter * @ret_addr: return address 3592a9e4f7SAdrian Hunter * @timestamp: timestamp (if known) 3692a9e4f7SAdrian Hunter * @ref: external reference (e.g. db_id of sample) 3792a9e4f7SAdrian Hunter * @branch_count: the branch count when the entry was created 3892a9e4f7SAdrian Hunter * @cp: call path 3992a9e4f7SAdrian Hunter * @no_call: a 'call' was not seen 404d60e5e3SAdrian Hunter * @trace_end: a 'call' but trace ended 41f08046cbSAdrian Hunter * @non_call: a branch but not a 'call' to the start of a different symbol 4292a9e4f7SAdrian Hunter */ 4392a9e4f7SAdrian Hunter struct thread_stack_entry { 4492a9e4f7SAdrian Hunter u64 ret_addr; 4592a9e4f7SAdrian Hunter u64 timestamp; 4692a9e4f7SAdrian Hunter u64 ref; 4792a9e4f7SAdrian Hunter u64 branch_count; 4892a9e4f7SAdrian Hunter struct call_path *cp; 4992a9e4f7SAdrian Hunter bool no_call; 504d60e5e3SAdrian Hunter bool trace_end; 51f08046cbSAdrian Hunter bool non_call; 5292a9e4f7SAdrian Hunter }; 5392a9e4f7SAdrian Hunter 5492a9e4f7SAdrian Hunter /** 5592a9e4f7SAdrian Hunter * struct thread_stack - thread stack constructed from 'call' and 'return' 5692a9e4f7SAdrian Hunter * branch samples. 5792a9e4f7SAdrian Hunter * @stack: array that holds the stack 5892a9e4f7SAdrian Hunter * @cnt: number of entries in the stack 5992a9e4f7SAdrian Hunter * @sz: current maximum stack size 6092a9e4f7SAdrian Hunter * @trace_nr: current trace number 6192a9e4f7SAdrian Hunter * @branch_count: running branch count 6292a9e4f7SAdrian Hunter * @kernel_start: kernel start address 6392a9e4f7SAdrian Hunter * @last_time: last timestamp 6492a9e4f7SAdrian Hunter * @crp: call/return processor 6592a9e4f7SAdrian Hunter * @comm: current comm 66f6060ac6SAdrian Hunter * @arr_sz: size of array if this is the first element of an array 6792a9e4f7SAdrian Hunter */ 6800447ccdSAdrian Hunter struct thread_stack { 6900447ccdSAdrian Hunter struct thread_stack_entry *stack; 7000447ccdSAdrian Hunter size_t cnt; 7100447ccdSAdrian Hunter size_t sz; 7200447ccdSAdrian Hunter u64 trace_nr; 7392a9e4f7SAdrian Hunter u64 branch_count; 7492a9e4f7SAdrian Hunter u64 kernel_start; 7592a9e4f7SAdrian Hunter u64 last_time; 7692a9e4f7SAdrian Hunter struct call_return_processor *crp; 7792a9e4f7SAdrian Hunter struct comm *comm; 78f6060ac6SAdrian Hunter unsigned int arr_sz; 7900447ccdSAdrian Hunter }; 8000447ccdSAdrian Hunter 81256d92bcSAdrian Hunter /* 82256d92bcSAdrian Hunter * Assume pid == tid == 0 identifies the idle task as defined by 83256d92bcSAdrian Hunter * perf_session__register_idle_thread(). The idle task is really 1 task per cpu, 84256d92bcSAdrian Hunter * and therefore requires a stack for each cpu. 85256d92bcSAdrian Hunter */ 86256d92bcSAdrian Hunter static inline bool thread_stack__per_cpu(struct thread *thread) 87256d92bcSAdrian Hunter { 88256d92bcSAdrian Hunter return !(thread->tid || thread->pid_); 89256d92bcSAdrian Hunter } 90256d92bcSAdrian Hunter 9100447ccdSAdrian Hunter static int thread_stack__grow(struct thread_stack *ts) 9200447ccdSAdrian Hunter { 9300447ccdSAdrian Hunter struct thread_stack_entry *new_stack; 9400447ccdSAdrian Hunter size_t sz, new_sz; 9500447ccdSAdrian Hunter 9600447ccdSAdrian Hunter new_sz = ts->sz + STACK_GROWTH; 9700447ccdSAdrian Hunter sz = new_sz * sizeof(struct thread_stack_entry); 9800447ccdSAdrian Hunter 9900447ccdSAdrian Hunter new_stack = realloc(ts->stack, sz); 10000447ccdSAdrian Hunter if (!new_stack) 10100447ccdSAdrian Hunter return -ENOMEM; 10200447ccdSAdrian Hunter 10300447ccdSAdrian Hunter ts->stack = new_stack; 10400447ccdSAdrian Hunter ts->sz = new_sz; 10500447ccdSAdrian Hunter 10600447ccdSAdrian Hunter return 0; 10700447ccdSAdrian Hunter } 10800447ccdSAdrian Hunter 1092e9e8688SAdrian Hunter static int thread_stack__init(struct thread_stack *ts, struct thread *thread, 1102e9e8688SAdrian Hunter struct call_return_processor *crp) 1112e9e8688SAdrian Hunter { 1122e9e8688SAdrian Hunter int err; 1132e9e8688SAdrian Hunter 1142e9e8688SAdrian Hunter err = thread_stack__grow(ts); 1152e9e8688SAdrian Hunter if (err) 1162e9e8688SAdrian Hunter return err; 1172e9e8688SAdrian Hunter 1182e9e8688SAdrian Hunter if (thread->mg && thread->mg->machine) 1192e9e8688SAdrian Hunter ts->kernel_start = machine__kernel_start(thread->mg->machine); 1202e9e8688SAdrian Hunter else 1212e9e8688SAdrian Hunter ts->kernel_start = 1ULL << 63; 1222e9e8688SAdrian Hunter ts->crp = crp; 1232e9e8688SAdrian Hunter 1242e9e8688SAdrian Hunter return 0; 1252e9e8688SAdrian Hunter } 1262e9e8688SAdrian Hunter 127256d92bcSAdrian Hunter static struct thread_stack *thread_stack__new(struct thread *thread, int cpu, 12892a9e4f7SAdrian Hunter struct call_return_processor *crp) 12900447ccdSAdrian Hunter { 130139f42f3SAdrian Hunter struct thread_stack *ts = thread->ts, *new_ts; 131139f42f3SAdrian Hunter unsigned int old_sz = ts ? ts->arr_sz : 0; 132139f42f3SAdrian Hunter unsigned int new_sz = 1; 13300447ccdSAdrian Hunter 134256d92bcSAdrian Hunter if (thread_stack__per_cpu(thread) && cpu > 0) 135256d92bcSAdrian Hunter new_sz = roundup_pow_of_two(cpu + 1); 136256d92bcSAdrian Hunter 137139f42f3SAdrian Hunter if (!ts || new_sz > old_sz) { 138139f42f3SAdrian Hunter new_ts = calloc(new_sz, sizeof(*ts)); 139139f42f3SAdrian Hunter if (!new_ts) 14000447ccdSAdrian Hunter return NULL; 141139f42f3SAdrian Hunter if (ts) 142139f42f3SAdrian Hunter memcpy(new_ts, ts, old_sz * sizeof(*ts)); 143139f42f3SAdrian Hunter new_ts->arr_sz = new_sz; 144139f42f3SAdrian Hunter zfree(&thread->ts); 145139f42f3SAdrian Hunter thread->ts = new_ts; 146139f42f3SAdrian Hunter ts = new_ts; 14700447ccdSAdrian Hunter } 14800447ccdSAdrian Hunter 149256d92bcSAdrian Hunter if (thread_stack__per_cpu(thread) && cpu > 0 && 150256d92bcSAdrian Hunter (unsigned int)cpu < ts->arr_sz) 151256d92bcSAdrian Hunter ts += cpu; 152256d92bcSAdrian Hunter 153139f42f3SAdrian Hunter if (!ts->stack && 154139f42f3SAdrian Hunter thread_stack__init(ts, thread, crp)) 155139f42f3SAdrian Hunter return NULL; 156bd8e68acSAdrian Hunter 15700447ccdSAdrian Hunter return ts; 15800447ccdSAdrian Hunter } 15900447ccdSAdrian Hunter 160256d92bcSAdrian Hunter static struct thread_stack *thread__cpu_stack(struct thread *thread, int cpu) 161bd8e68acSAdrian Hunter { 162256d92bcSAdrian Hunter struct thread_stack *ts = thread->ts; 163256d92bcSAdrian Hunter 164256d92bcSAdrian Hunter if (cpu < 0) 165256d92bcSAdrian Hunter cpu = 0; 166256d92bcSAdrian Hunter 167256d92bcSAdrian Hunter if (!ts || (unsigned int)cpu >= ts->arr_sz) 168256d92bcSAdrian Hunter return NULL; 169256d92bcSAdrian Hunter 170256d92bcSAdrian Hunter ts += cpu; 171256d92bcSAdrian Hunter 172256d92bcSAdrian Hunter if (!ts->stack) 173256d92bcSAdrian Hunter return NULL; 174256d92bcSAdrian Hunter 175256d92bcSAdrian Hunter return ts; 176256d92bcSAdrian Hunter } 177256d92bcSAdrian Hunter 178256d92bcSAdrian Hunter static inline struct thread_stack *thread__stack(struct thread *thread, 179256d92bcSAdrian Hunter int cpu) 180256d92bcSAdrian Hunter { 181256d92bcSAdrian Hunter if (!thread) 182256d92bcSAdrian Hunter return NULL; 183256d92bcSAdrian Hunter 184256d92bcSAdrian Hunter if (thread_stack__per_cpu(thread)) 185256d92bcSAdrian Hunter return thread__cpu_stack(thread, cpu); 186256d92bcSAdrian Hunter 187256d92bcSAdrian Hunter return thread->ts; 188bd8e68acSAdrian Hunter } 189bd8e68acSAdrian Hunter 1904d60e5e3SAdrian Hunter static int thread_stack__push(struct thread_stack *ts, u64 ret_addr, 1914d60e5e3SAdrian Hunter bool trace_end) 19200447ccdSAdrian Hunter { 19300447ccdSAdrian Hunter int err = 0; 19400447ccdSAdrian Hunter 19500447ccdSAdrian Hunter if (ts->cnt == ts->sz) { 19600447ccdSAdrian Hunter err = thread_stack__grow(ts); 19700447ccdSAdrian Hunter if (err) { 19800447ccdSAdrian Hunter pr_warning("Out of memory: discarding thread stack\n"); 19900447ccdSAdrian Hunter ts->cnt = 0; 20000447ccdSAdrian Hunter } 20100447ccdSAdrian Hunter } 20200447ccdSAdrian Hunter 2034d60e5e3SAdrian Hunter ts->stack[ts->cnt].trace_end = trace_end; 20400447ccdSAdrian Hunter ts->stack[ts->cnt++].ret_addr = ret_addr; 20500447ccdSAdrian Hunter 20600447ccdSAdrian Hunter return err; 20700447ccdSAdrian Hunter } 20800447ccdSAdrian Hunter 20900447ccdSAdrian Hunter static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr) 21000447ccdSAdrian Hunter { 21100447ccdSAdrian Hunter size_t i; 21200447ccdSAdrian Hunter 21300447ccdSAdrian Hunter /* 21400447ccdSAdrian Hunter * In some cases there may be functions which are not seen to return. 21500447ccdSAdrian Hunter * For example when setjmp / longjmp has been used. Or the perf context 21600447ccdSAdrian Hunter * switch in the kernel which doesn't stop and start tracing in exactly 21700447ccdSAdrian Hunter * the same code path. When that happens the return address will be 21800447ccdSAdrian Hunter * further down the stack. If the return address is not found at all, 21900447ccdSAdrian Hunter * we assume the opposite (i.e. this is a return for a call that wasn't 22000447ccdSAdrian Hunter * seen for some reason) and leave the stack alone. 22100447ccdSAdrian Hunter */ 22200447ccdSAdrian Hunter for (i = ts->cnt; i; ) { 22300447ccdSAdrian Hunter if (ts->stack[--i].ret_addr == ret_addr) { 22400447ccdSAdrian Hunter ts->cnt = i; 22500447ccdSAdrian Hunter return; 22600447ccdSAdrian Hunter } 22700447ccdSAdrian Hunter } 22800447ccdSAdrian Hunter } 22900447ccdSAdrian Hunter 2304d60e5e3SAdrian Hunter static void thread_stack__pop_trace_end(struct thread_stack *ts) 2314d60e5e3SAdrian Hunter { 2324d60e5e3SAdrian Hunter size_t i; 2334d60e5e3SAdrian Hunter 2344d60e5e3SAdrian Hunter for (i = ts->cnt; i; ) { 2354d60e5e3SAdrian Hunter if (ts->stack[--i].trace_end) 2364d60e5e3SAdrian Hunter ts->cnt = i; 2374d60e5e3SAdrian Hunter else 2384d60e5e3SAdrian Hunter return; 2394d60e5e3SAdrian Hunter } 2404d60e5e3SAdrian Hunter } 2414d60e5e3SAdrian Hunter 24292a9e4f7SAdrian Hunter static bool thread_stack__in_kernel(struct thread_stack *ts) 24392a9e4f7SAdrian Hunter { 24492a9e4f7SAdrian Hunter if (!ts->cnt) 24592a9e4f7SAdrian Hunter return false; 24692a9e4f7SAdrian Hunter 24792a9e4f7SAdrian Hunter return ts->stack[ts->cnt - 1].cp->in_kernel; 24892a9e4f7SAdrian Hunter } 24992a9e4f7SAdrian Hunter 25092a9e4f7SAdrian Hunter static int thread_stack__call_return(struct thread *thread, 25192a9e4f7SAdrian Hunter struct thread_stack *ts, size_t idx, 25292a9e4f7SAdrian Hunter u64 timestamp, u64 ref, bool no_return) 25392a9e4f7SAdrian Hunter { 25492a9e4f7SAdrian Hunter struct call_return_processor *crp = ts->crp; 25592a9e4f7SAdrian Hunter struct thread_stack_entry *tse; 25692a9e4f7SAdrian Hunter struct call_return cr = { 25792a9e4f7SAdrian Hunter .thread = thread, 25892a9e4f7SAdrian Hunter .comm = ts->comm, 25992a9e4f7SAdrian Hunter .db_id = 0, 26092a9e4f7SAdrian Hunter }; 26192a9e4f7SAdrian Hunter 26292a9e4f7SAdrian Hunter tse = &ts->stack[idx]; 26392a9e4f7SAdrian Hunter cr.cp = tse->cp; 26492a9e4f7SAdrian Hunter cr.call_time = tse->timestamp; 26592a9e4f7SAdrian Hunter cr.return_time = timestamp; 26692a9e4f7SAdrian Hunter cr.branch_count = ts->branch_count - tse->branch_count; 26792a9e4f7SAdrian Hunter cr.call_ref = tse->ref; 26892a9e4f7SAdrian Hunter cr.return_ref = ref; 26992a9e4f7SAdrian Hunter if (tse->no_call) 27092a9e4f7SAdrian Hunter cr.flags |= CALL_RETURN_NO_CALL; 27192a9e4f7SAdrian Hunter if (no_return) 27292a9e4f7SAdrian Hunter cr.flags |= CALL_RETURN_NO_RETURN; 273f08046cbSAdrian Hunter if (tse->non_call) 274f08046cbSAdrian Hunter cr.flags |= CALL_RETURN_NON_CALL; 27592a9e4f7SAdrian Hunter 27692a9e4f7SAdrian Hunter return crp->process(&cr, crp->data); 27792a9e4f7SAdrian Hunter } 27892a9e4f7SAdrian Hunter 279a5499b37SAdrian Hunter static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts) 28092a9e4f7SAdrian Hunter { 28192a9e4f7SAdrian Hunter struct call_return_processor *crp = ts->crp; 28292a9e4f7SAdrian Hunter int err; 28392a9e4f7SAdrian Hunter 28492a9e4f7SAdrian Hunter if (!crp) { 28592a9e4f7SAdrian Hunter ts->cnt = 0; 28692a9e4f7SAdrian Hunter return 0; 28792a9e4f7SAdrian Hunter } 28892a9e4f7SAdrian Hunter 28992a9e4f7SAdrian Hunter while (ts->cnt) { 29092a9e4f7SAdrian Hunter err = thread_stack__call_return(thread, ts, --ts->cnt, 29192a9e4f7SAdrian Hunter ts->last_time, 0, true); 29292a9e4f7SAdrian Hunter if (err) { 29392a9e4f7SAdrian Hunter pr_err("Error flushing thread stack!\n"); 29492a9e4f7SAdrian Hunter ts->cnt = 0; 29592a9e4f7SAdrian Hunter return err; 29692a9e4f7SAdrian Hunter } 29792a9e4f7SAdrian Hunter } 29892a9e4f7SAdrian Hunter 29992a9e4f7SAdrian Hunter return 0; 30092a9e4f7SAdrian Hunter } 30192a9e4f7SAdrian Hunter 302a5499b37SAdrian Hunter int thread_stack__flush(struct thread *thread) 303a5499b37SAdrian Hunter { 304bd8e68acSAdrian Hunter struct thread_stack *ts = thread->ts; 305f6060ac6SAdrian Hunter unsigned int pos; 306f6060ac6SAdrian Hunter int err = 0; 307bd8e68acSAdrian Hunter 308f6060ac6SAdrian Hunter if (ts) { 309f6060ac6SAdrian Hunter for (pos = 0; pos < ts->arr_sz; pos++) { 310f6060ac6SAdrian Hunter int ret = __thread_stack__flush(thread, ts + pos); 311a5499b37SAdrian Hunter 312f6060ac6SAdrian Hunter if (ret) 313f6060ac6SAdrian Hunter err = ret; 314f6060ac6SAdrian Hunter } 315f6060ac6SAdrian Hunter } 316f6060ac6SAdrian Hunter 317f6060ac6SAdrian Hunter return err; 318a5499b37SAdrian Hunter } 319a5499b37SAdrian Hunter 320256d92bcSAdrian Hunter int thread_stack__event(struct thread *thread, int cpu, u32 flags, u64 from_ip, 32100447ccdSAdrian Hunter u64 to_ip, u16 insn_len, u64 trace_nr) 32200447ccdSAdrian Hunter { 323256d92bcSAdrian Hunter struct thread_stack *ts = thread__stack(thread, cpu); 324bd8e68acSAdrian Hunter 32500447ccdSAdrian Hunter if (!thread) 32600447ccdSAdrian Hunter return -EINVAL; 32700447ccdSAdrian Hunter 328bd8e68acSAdrian Hunter if (!ts) { 329256d92bcSAdrian Hunter ts = thread_stack__new(thread, cpu, NULL); 330bd8e68acSAdrian Hunter if (!ts) { 33100447ccdSAdrian Hunter pr_warning("Out of memory: no thread stack\n"); 33200447ccdSAdrian Hunter return -ENOMEM; 33300447ccdSAdrian Hunter } 334bd8e68acSAdrian Hunter ts->trace_nr = trace_nr; 33500447ccdSAdrian Hunter } 33600447ccdSAdrian Hunter 33700447ccdSAdrian Hunter /* 33800447ccdSAdrian Hunter * When the trace is discontinuous, the trace_nr changes. In that case 33900447ccdSAdrian Hunter * the stack might be completely invalid. Better to report nothing than 34092a9e4f7SAdrian Hunter * to report something misleading, so flush the stack. 34100447ccdSAdrian Hunter */ 342bd8e68acSAdrian Hunter if (trace_nr != ts->trace_nr) { 343bd8e68acSAdrian Hunter if (ts->trace_nr) 344bd8e68acSAdrian Hunter __thread_stack__flush(thread, ts); 345bd8e68acSAdrian Hunter ts->trace_nr = trace_nr; 34600447ccdSAdrian Hunter } 34700447ccdSAdrian Hunter 34892a9e4f7SAdrian Hunter /* Stop here if thread_stack__process() is in use */ 349bd8e68acSAdrian Hunter if (ts->crp) 35092a9e4f7SAdrian Hunter return 0; 35192a9e4f7SAdrian Hunter 35200447ccdSAdrian Hunter if (flags & PERF_IP_FLAG_CALL) { 35300447ccdSAdrian Hunter u64 ret_addr; 35400447ccdSAdrian Hunter 35500447ccdSAdrian Hunter if (!to_ip) 35600447ccdSAdrian Hunter return 0; 35700447ccdSAdrian Hunter ret_addr = from_ip + insn_len; 35800447ccdSAdrian Hunter if (ret_addr == to_ip) 35900447ccdSAdrian Hunter return 0; /* Zero-length calls are excluded */ 360bd8e68acSAdrian Hunter return thread_stack__push(ts, ret_addr, 3614d60e5e3SAdrian Hunter flags & PERF_IP_FLAG_TRACE_END); 3624d60e5e3SAdrian Hunter } else if (flags & PERF_IP_FLAG_TRACE_BEGIN) { 3634d60e5e3SAdrian Hunter /* 3644d60e5e3SAdrian Hunter * If the caller did not change the trace number (which would 3654d60e5e3SAdrian Hunter * have flushed the stack) then try to make sense of the stack. 3664d60e5e3SAdrian Hunter * Possibly, tracing began after returning to the current 3674d60e5e3SAdrian Hunter * address, so try to pop that. Also, do not expect a call made 3684d60e5e3SAdrian Hunter * when the trace ended, to return, so pop that. 3694d60e5e3SAdrian Hunter */ 370bd8e68acSAdrian Hunter thread_stack__pop(ts, to_ip); 371bd8e68acSAdrian Hunter thread_stack__pop_trace_end(ts); 3724d60e5e3SAdrian Hunter } else if ((flags & PERF_IP_FLAG_RETURN) && from_ip) { 373bd8e68acSAdrian Hunter thread_stack__pop(ts, to_ip); 37400447ccdSAdrian Hunter } 37500447ccdSAdrian Hunter 37600447ccdSAdrian Hunter return 0; 37700447ccdSAdrian Hunter } 37800447ccdSAdrian Hunter 379256d92bcSAdrian Hunter void thread_stack__set_trace_nr(struct thread *thread, int cpu, u64 trace_nr) 38092a9e4f7SAdrian Hunter { 381256d92bcSAdrian Hunter struct thread_stack *ts = thread__stack(thread, cpu); 382bd8e68acSAdrian Hunter 383bd8e68acSAdrian Hunter if (!ts) 38492a9e4f7SAdrian Hunter return; 38592a9e4f7SAdrian Hunter 386bd8e68acSAdrian Hunter if (trace_nr != ts->trace_nr) { 387bd8e68acSAdrian Hunter if (ts->trace_nr) 388bd8e68acSAdrian Hunter __thread_stack__flush(thread, ts); 389bd8e68acSAdrian Hunter ts->trace_nr = trace_nr; 39092a9e4f7SAdrian Hunter } 39192a9e4f7SAdrian Hunter } 39292a9e4f7SAdrian Hunter 393f6060ac6SAdrian Hunter static void __thread_stack__free(struct thread *thread, struct thread_stack *ts) 394f6060ac6SAdrian Hunter { 395f6060ac6SAdrian Hunter __thread_stack__flush(thread, ts); 396f6060ac6SAdrian Hunter zfree(&ts->stack); 397f6060ac6SAdrian Hunter } 398f6060ac6SAdrian Hunter 399f6060ac6SAdrian Hunter static void thread_stack__reset(struct thread *thread, struct thread_stack *ts) 400f6060ac6SAdrian Hunter { 401f6060ac6SAdrian Hunter unsigned int arr_sz = ts->arr_sz; 402f6060ac6SAdrian Hunter 403f6060ac6SAdrian Hunter __thread_stack__free(thread, ts); 404f6060ac6SAdrian Hunter memset(ts, 0, sizeof(*ts)); 405f6060ac6SAdrian Hunter ts->arr_sz = arr_sz; 406f6060ac6SAdrian Hunter } 407f6060ac6SAdrian Hunter 40800447ccdSAdrian Hunter void thread_stack__free(struct thread *thread) 40900447ccdSAdrian Hunter { 410bd8e68acSAdrian Hunter struct thread_stack *ts = thread->ts; 411f6060ac6SAdrian Hunter unsigned int pos; 412bd8e68acSAdrian Hunter 413bd8e68acSAdrian Hunter if (ts) { 414f6060ac6SAdrian Hunter for (pos = 0; pos < ts->arr_sz; pos++) 415f6060ac6SAdrian Hunter __thread_stack__free(thread, ts + pos); 41600447ccdSAdrian Hunter zfree(&thread->ts); 41700447ccdSAdrian Hunter } 41800447ccdSAdrian Hunter } 41900447ccdSAdrian Hunter 42024248306SAdrian Hunter static inline u64 callchain_context(u64 ip, u64 kernel_start) 42100447ccdSAdrian Hunter { 42224248306SAdrian Hunter return ip < kernel_start ? PERF_CONTEXT_USER : PERF_CONTEXT_KERNEL; 42324248306SAdrian Hunter } 42400447ccdSAdrian Hunter 425256d92bcSAdrian Hunter void thread_stack__sample(struct thread *thread, int cpu, 426256d92bcSAdrian Hunter struct ip_callchain *chain, 42724248306SAdrian Hunter size_t sz, u64 ip, u64 kernel_start) 42824248306SAdrian Hunter { 429256d92bcSAdrian Hunter struct thread_stack *ts = thread__stack(thread, cpu); 43024248306SAdrian Hunter u64 context = callchain_context(ip, kernel_start); 43124248306SAdrian Hunter u64 last_context; 43224248306SAdrian Hunter size_t i, j; 43300447ccdSAdrian Hunter 43424248306SAdrian Hunter if (sz < 2) { 43524248306SAdrian Hunter chain->nr = 0; 43624248306SAdrian Hunter return; 43724248306SAdrian Hunter } 43800447ccdSAdrian Hunter 43924248306SAdrian Hunter chain->ips[0] = context; 44024248306SAdrian Hunter chain->ips[1] = ip; 44124248306SAdrian Hunter 442bd8e68acSAdrian Hunter if (!ts) { 44324248306SAdrian Hunter chain->nr = 2; 44424248306SAdrian Hunter return; 44524248306SAdrian Hunter } 44624248306SAdrian Hunter 44724248306SAdrian Hunter last_context = context; 44824248306SAdrian Hunter 449bd8e68acSAdrian Hunter for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) { 450bd8e68acSAdrian Hunter ip = ts->stack[ts->cnt - j].ret_addr; 45124248306SAdrian Hunter context = callchain_context(ip, kernel_start); 45224248306SAdrian Hunter if (context != last_context) { 45324248306SAdrian Hunter if (i >= sz - 1) 45424248306SAdrian Hunter break; 45524248306SAdrian Hunter chain->ips[i++] = context; 45624248306SAdrian Hunter last_context = context; 45724248306SAdrian Hunter } 45824248306SAdrian Hunter chain->ips[i] = ip; 45924248306SAdrian Hunter } 46024248306SAdrian Hunter 46124248306SAdrian Hunter chain->nr = i; 46200447ccdSAdrian Hunter } 46392a9e4f7SAdrian Hunter 46492a9e4f7SAdrian Hunter struct call_return_processor * 46592a9e4f7SAdrian Hunter call_return_processor__new(int (*process)(struct call_return *cr, void *data), 46692a9e4f7SAdrian Hunter void *data) 46792a9e4f7SAdrian Hunter { 46892a9e4f7SAdrian Hunter struct call_return_processor *crp; 46992a9e4f7SAdrian Hunter 47092a9e4f7SAdrian Hunter crp = zalloc(sizeof(struct call_return_processor)); 47192a9e4f7SAdrian Hunter if (!crp) 47292a9e4f7SAdrian Hunter return NULL; 47392a9e4f7SAdrian Hunter crp->cpr = call_path_root__new(); 47492a9e4f7SAdrian Hunter if (!crp->cpr) 47592a9e4f7SAdrian Hunter goto out_free; 47692a9e4f7SAdrian Hunter crp->process = process; 47792a9e4f7SAdrian Hunter crp->data = data; 47892a9e4f7SAdrian Hunter return crp; 47992a9e4f7SAdrian Hunter 48092a9e4f7SAdrian Hunter out_free: 48192a9e4f7SAdrian Hunter free(crp); 48292a9e4f7SAdrian Hunter return NULL; 48392a9e4f7SAdrian Hunter } 48492a9e4f7SAdrian Hunter 48592a9e4f7SAdrian Hunter void call_return_processor__free(struct call_return_processor *crp) 48692a9e4f7SAdrian Hunter { 48792a9e4f7SAdrian Hunter if (crp) { 48892a9e4f7SAdrian Hunter call_path_root__free(crp->cpr); 48992a9e4f7SAdrian Hunter free(crp); 49092a9e4f7SAdrian Hunter } 49192a9e4f7SAdrian Hunter } 49292a9e4f7SAdrian Hunter 49392a9e4f7SAdrian Hunter static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr, 49492a9e4f7SAdrian Hunter u64 timestamp, u64 ref, struct call_path *cp, 4952dcde4e1SAdrian Hunter bool no_call, bool trace_end) 49692a9e4f7SAdrian Hunter { 49792a9e4f7SAdrian Hunter struct thread_stack_entry *tse; 49892a9e4f7SAdrian Hunter int err; 49992a9e4f7SAdrian Hunter 500e7a3a055SAdrian Hunter if (!cp) 501e7a3a055SAdrian Hunter return -ENOMEM; 502e7a3a055SAdrian Hunter 50392a9e4f7SAdrian Hunter if (ts->cnt == ts->sz) { 50492a9e4f7SAdrian Hunter err = thread_stack__grow(ts); 50592a9e4f7SAdrian Hunter if (err) 50692a9e4f7SAdrian Hunter return err; 50792a9e4f7SAdrian Hunter } 50892a9e4f7SAdrian Hunter 50992a9e4f7SAdrian Hunter tse = &ts->stack[ts->cnt++]; 51092a9e4f7SAdrian Hunter tse->ret_addr = ret_addr; 51192a9e4f7SAdrian Hunter tse->timestamp = timestamp; 51292a9e4f7SAdrian Hunter tse->ref = ref; 51392a9e4f7SAdrian Hunter tse->branch_count = ts->branch_count; 51492a9e4f7SAdrian Hunter tse->cp = cp; 51592a9e4f7SAdrian Hunter tse->no_call = no_call; 5162dcde4e1SAdrian Hunter tse->trace_end = trace_end; 517f08046cbSAdrian Hunter tse->non_call = false; 51892a9e4f7SAdrian Hunter 51992a9e4f7SAdrian Hunter return 0; 52092a9e4f7SAdrian Hunter } 52192a9e4f7SAdrian Hunter 52292a9e4f7SAdrian Hunter static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts, 52392a9e4f7SAdrian Hunter u64 ret_addr, u64 timestamp, u64 ref, 52492a9e4f7SAdrian Hunter struct symbol *sym) 52592a9e4f7SAdrian Hunter { 52692a9e4f7SAdrian Hunter int err; 52792a9e4f7SAdrian Hunter 52892a9e4f7SAdrian Hunter if (!ts->cnt) 52992a9e4f7SAdrian Hunter return 1; 53092a9e4f7SAdrian Hunter 53192a9e4f7SAdrian Hunter if (ts->cnt == 1) { 53292a9e4f7SAdrian Hunter struct thread_stack_entry *tse = &ts->stack[0]; 53392a9e4f7SAdrian Hunter 53492a9e4f7SAdrian Hunter if (tse->cp->sym == sym) 53592a9e4f7SAdrian Hunter return thread_stack__call_return(thread, ts, --ts->cnt, 53692a9e4f7SAdrian Hunter timestamp, ref, false); 53792a9e4f7SAdrian Hunter } 53892a9e4f7SAdrian Hunter 539f08046cbSAdrian Hunter if (ts->stack[ts->cnt - 1].ret_addr == ret_addr && 540f08046cbSAdrian Hunter !ts->stack[ts->cnt - 1].non_call) { 54192a9e4f7SAdrian Hunter return thread_stack__call_return(thread, ts, --ts->cnt, 54292a9e4f7SAdrian Hunter timestamp, ref, false); 54392a9e4f7SAdrian Hunter } else { 54492a9e4f7SAdrian Hunter size_t i = ts->cnt - 1; 54592a9e4f7SAdrian Hunter 54692a9e4f7SAdrian Hunter while (i--) { 547f08046cbSAdrian Hunter if (ts->stack[i].ret_addr != ret_addr || 548f08046cbSAdrian Hunter ts->stack[i].non_call) 54992a9e4f7SAdrian Hunter continue; 55092a9e4f7SAdrian Hunter i += 1; 55192a9e4f7SAdrian Hunter while (ts->cnt > i) { 55292a9e4f7SAdrian Hunter err = thread_stack__call_return(thread, ts, 55392a9e4f7SAdrian Hunter --ts->cnt, 55492a9e4f7SAdrian Hunter timestamp, ref, 55592a9e4f7SAdrian Hunter true); 55692a9e4f7SAdrian Hunter if (err) 55792a9e4f7SAdrian Hunter return err; 55892a9e4f7SAdrian Hunter } 55992a9e4f7SAdrian Hunter return thread_stack__call_return(thread, ts, --ts->cnt, 56092a9e4f7SAdrian Hunter timestamp, ref, false); 56192a9e4f7SAdrian Hunter } 56292a9e4f7SAdrian Hunter } 56392a9e4f7SAdrian Hunter 56492a9e4f7SAdrian Hunter return 1; 56592a9e4f7SAdrian Hunter } 56692a9e4f7SAdrian Hunter 567e0b89511SAdrian Hunter static int thread_stack__bottom(struct thread_stack *ts, 56892a9e4f7SAdrian Hunter struct perf_sample *sample, 56992a9e4f7SAdrian Hunter struct addr_location *from_al, 57092a9e4f7SAdrian Hunter struct addr_location *to_al, u64 ref) 57192a9e4f7SAdrian Hunter { 57292a9e4f7SAdrian Hunter struct call_path_root *cpr = ts->crp->cpr; 57392a9e4f7SAdrian Hunter struct call_path *cp; 57492a9e4f7SAdrian Hunter struct symbol *sym; 57592a9e4f7SAdrian Hunter u64 ip; 57692a9e4f7SAdrian Hunter 57792a9e4f7SAdrian Hunter if (sample->ip) { 57892a9e4f7SAdrian Hunter ip = sample->ip; 57992a9e4f7SAdrian Hunter sym = from_al->sym; 58092a9e4f7SAdrian Hunter } else if (sample->addr) { 58192a9e4f7SAdrian Hunter ip = sample->addr; 58292a9e4f7SAdrian Hunter sym = to_al->sym; 58392a9e4f7SAdrian Hunter } else { 58492a9e4f7SAdrian Hunter return 0; 58592a9e4f7SAdrian Hunter } 58692a9e4f7SAdrian Hunter 58792a9e4f7SAdrian Hunter cp = call_path__findnew(cpr, &cpr->call_path, sym, ip, 58892a9e4f7SAdrian Hunter ts->kernel_start); 58992a9e4f7SAdrian Hunter 590e0b89511SAdrian Hunter return thread_stack__push_cp(ts, ip, sample->time, ref, cp, 5912dcde4e1SAdrian Hunter true, false); 59292a9e4f7SAdrian Hunter } 59392a9e4f7SAdrian Hunter 59492a9e4f7SAdrian Hunter static int thread_stack__no_call_return(struct thread *thread, 59592a9e4f7SAdrian Hunter struct thread_stack *ts, 59692a9e4f7SAdrian Hunter struct perf_sample *sample, 59792a9e4f7SAdrian Hunter struct addr_location *from_al, 59892a9e4f7SAdrian Hunter struct addr_location *to_al, u64 ref) 59992a9e4f7SAdrian Hunter { 60092a9e4f7SAdrian Hunter struct call_path_root *cpr = ts->crp->cpr; 60190c2cda7SAdrian Hunter struct call_path *root = &cpr->call_path; 60290c2cda7SAdrian Hunter struct symbol *fsym = from_al->sym; 60390c2cda7SAdrian Hunter struct symbol *tsym = to_al->sym; 60492a9e4f7SAdrian Hunter struct call_path *cp, *parent; 60592a9e4f7SAdrian Hunter u64 ks = ts->kernel_start; 60690c2cda7SAdrian Hunter u64 addr = sample->addr; 60790c2cda7SAdrian Hunter u64 tm = sample->time; 60890c2cda7SAdrian Hunter u64 ip = sample->ip; 60992a9e4f7SAdrian Hunter int err; 61092a9e4f7SAdrian Hunter 61190c2cda7SAdrian Hunter if (ip >= ks && addr < ks) { 61292a9e4f7SAdrian Hunter /* Return to userspace, so pop all kernel addresses */ 61392a9e4f7SAdrian Hunter while (thread_stack__in_kernel(ts)) { 61492a9e4f7SAdrian Hunter err = thread_stack__call_return(thread, ts, --ts->cnt, 61590c2cda7SAdrian Hunter tm, ref, true); 61692a9e4f7SAdrian Hunter if (err) 61792a9e4f7SAdrian Hunter return err; 61892a9e4f7SAdrian Hunter } 61992a9e4f7SAdrian Hunter 62092a9e4f7SAdrian Hunter /* If the stack is empty, push the userspace address */ 62192a9e4f7SAdrian Hunter if (!ts->cnt) { 62290c2cda7SAdrian Hunter cp = call_path__findnew(cpr, root, tsym, addr, ks); 62390c2cda7SAdrian Hunter return thread_stack__push_cp(ts, 0, tm, ref, cp, true, 62490c2cda7SAdrian Hunter false); 62592a9e4f7SAdrian Hunter } 62690c2cda7SAdrian Hunter } else if (thread_stack__in_kernel(ts) && ip < ks) { 62792a9e4f7SAdrian Hunter /* Return to userspace, so pop all kernel addresses */ 62892a9e4f7SAdrian Hunter while (thread_stack__in_kernel(ts)) { 62992a9e4f7SAdrian Hunter err = thread_stack__call_return(thread, ts, --ts->cnt, 63090c2cda7SAdrian Hunter tm, ref, true); 63192a9e4f7SAdrian Hunter if (err) 63292a9e4f7SAdrian Hunter return err; 63392a9e4f7SAdrian Hunter } 63492a9e4f7SAdrian Hunter } 63592a9e4f7SAdrian Hunter 63692a9e4f7SAdrian Hunter if (ts->cnt) 63792a9e4f7SAdrian Hunter parent = ts->stack[ts->cnt - 1].cp; 63892a9e4f7SAdrian Hunter else 63990c2cda7SAdrian Hunter parent = root; 64092a9e4f7SAdrian Hunter 6411f35cd65SAdrian Hunter if (parent->sym == from_al->sym) { 6421f35cd65SAdrian Hunter /* 6431f35cd65SAdrian Hunter * At the bottom of the stack, assume the missing 'call' was 6441f35cd65SAdrian Hunter * before the trace started. So, pop the current symbol and push 6451f35cd65SAdrian Hunter * the 'to' symbol. 6461f35cd65SAdrian Hunter */ 6471f35cd65SAdrian Hunter if (ts->cnt == 1) { 6481f35cd65SAdrian Hunter err = thread_stack__call_return(thread, ts, --ts->cnt, 6491f35cd65SAdrian Hunter tm, ref, false); 6501f35cd65SAdrian Hunter if (err) 6511f35cd65SAdrian Hunter return err; 6521f35cd65SAdrian Hunter } 6531f35cd65SAdrian Hunter 6541f35cd65SAdrian Hunter if (!ts->cnt) { 6551f35cd65SAdrian Hunter cp = call_path__findnew(cpr, root, tsym, addr, ks); 6561f35cd65SAdrian Hunter 6571f35cd65SAdrian Hunter return thread_stack__push_cp(ts, addr, tm, ref, cp, 6581f35cd65SAdrian Hunter true, false); 6591f35cd65SAdrian Hunter } 6601f35cd65SAdrian Hunter 6611f35cd65SAdrian Hunter /* 6621f35cd65SAdrian Hunter * Otherwise assume the 'return' is being used as a jump (e.g. 6631f35cd65SAdrian Hunter * retpoline) and just push the 'to' symbol. 6641f35cd65SAdrian Hunter */ 6651f35cd65SAdrian Hunter cp = call_path__findnew(cpr, parent, tsym, addr, ks); 6661f35cd65SAdrian Hunter 6671f35cd65SAdrian Hunter err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false); 6681f35cd65SAdrian Hunter if (!err) 6691f35cd65SAdrian Hunter ts->stack[ts->cnt - 1].non_call = true; 6701f35cd65SAdrian Hunter 6711f35cd65SAdrian Hunter return err; 6721f35cd65SAdrian Hunter } 6731f35cd65SAdrian Hunter 6741f35cd65SAdrian Hunter /* 6751f35cd65SAdrian Hunter * Assume 'parent' has not yet returned, so push 'to', and then push and 6761f35cd65SAdrian Hunter * pop 'from'. 6771f35cd65SAdrian Hunter */ 6781f35cd65SAdrian Hunter 6791f35cd65SAdrian Hunter cp = call_path__findnew(cpr, parent, tsym, addr, ks); 68092a9e4f7SAdrian Hunter 68190c2cda7SAdrian Hunter err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false); 68292a9e4f7SAdrian Hunter if (err) 68392a9e4f7SAdrian Hunter return err; 68492a9e4f7SAdrian Hunter 6851f35cd65SAdrian Hunter cp = call_path__findnew(cpr, cp, fsym, ip, ks); 6861f35cd65SAdrian Hunter 6871f35cd65SAdrian Hunter err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false); 6881f35cd65SAdrian Hunter if (err) 6891f35cd65SAdrian Hunter return err; 6901f35cd65SAdrian Hunter 6911f35cd65SAdrian Hunter return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false); 69292a9e4f7SAdrian Hunter } 69392a9e4f7SAdrian Hunter 69492a9e4f7SAdrian Hunter static int thread_stack__trace_begin(struct thread *thread, 69592a9e4f7SAdrian Hunter struct thread_stack *ts, u64 timestamp, 69692a9e4f7SAdrian Hunter u64 ref) 69792a9e4f7SAdrian Hunter { 69892a9e4f7SAdrian Hunter struct thread_stack_entry *tse; 69992a9e4f7SAdrian Hunter int err; 70092a9e4f7SAdrian Hunter 70192a9e4f7SAdrian Hunter if (!ts->cnt) 70292a9e4f7SAdrian Hunter return 0; 70392a9e4f7SAdrian Hunter 70492a9e4f7SAdrian Hunter /* Pop trace end */ 70592a9e4f7SAdrian Hunter tse = &ts->stack[ts->cnt - 1]; 7062dcde4e1SAdrian Hunter if (tse->trace_end) { 70792a9e4f7SAdrian Hunter err = thread_stack__call_return(thread, ts, --ts->cnt, 70892a9e4f7SAdrian Hunter timestamp, ref, false); 70992a9e4f7SAdrian Hunter if (err) 71092a9e4f7SAdrian Hunter return err; 71192a9e4f7SAdrian Hunter } 71292a9e4f7SAdrian Hunter 71392a9e4f7SAdrian Hunter return 0; 71492a9e4f7SAdrian Hunter } 71592a9e4f7SAdrian Hunter 71692a9e4f7SAdrian Hunter static int thread_stack__trace_end(struct thread_stack *ts, 71792a9e4f7SAdrian Hunter struct perf_sample *sample, u64 ref) 71892a9e4f7SAdrian Hunter { 71992a9e4f7SAdrian Hunter struct call_path_root *cpr = ts->crp->cpr; 72092a9e4f7SAdrian Hunter struct call_path *cp; 72192a9e4f7SAdrian Hunter u64 ret_addr; 72292a9e4f7SAdrian Hunter 72392a9e4f7SAdrian Hunter /* No point having 'trace end' on the bottom of the stack */ 72492a9e4f7SAdrian Hunter if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref)) 72592a9e4f7SAdrian Hunter return 0; 72692a9e4f7SAdrian Hunter 72792a9e4f7SAdrian Hunter cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0, 72892a9e4f7SAdrian Hunter ts->kernel_start); 72992a9e4f7SAdrian Hunter 73092a9e4f7SAdrian Hunter ret_addr = sample->ip + sample->insn_len; 73192a9e4f7SAdrian Hunter 73292a9e4f7SAdrian Hunter return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp, 7332dcde4e1SAdrian Hunter false, true); 73492a9e4f7SAdrian Hunter } 73592a9e4f7SAdrian Hunter 73692a9e4f7SAdrian Hunter int thread_stack__process(struct thread *thread, struct comm *comm, 73792a9e4f7SAdrian Hunter struct perf_sample *sample, 73892a9e4f7SAdrian Hunter struct addr_location *from_al, 73992a9e4f7SAdrian Hunter struct addr_location *to_al, u64 ref, 74092a9e4f7SAdrian Hunter struct call_return_processor *crp) 74192a9e4f7SAdrian Hunter { 742256d92bcSAdrian Hunter struct thread_stack *ts = thread__stack(thread, sample->cpu); 74392a9e4f7SAdrian Hunter int err = 0; 74492a9e4f7SAdrian Hunter 74503b32cb2SAdrian Hunter if (ts && !ts->crp) { 74692a9e4f7SAdrian Hunter /* Supersede thread_stack__event() */ 747f6060ac6SAdrian Hunter thread_stack__reset(thread, ts); 74803b32cb2SAdrian Hunter ts = NULL; 74992a9e4f7SAdrian Hunter } 75003b32cb2SAdrian Hunter 75103b32cb2SAdrian Hunter if (!ts) { 752256d92bcSAdrian Hunter ts = thread_stack__new(thread, sample->cpu, crp); 753bd8e68acSAdrian Hunter if (!ts) 75492a9e4f7SAdrian Hunter return -ENOMEM; 75592a9e4f7SAdrian Hunter ts->comm = comm; 75692a9e4f7SAdrian Hunter } 75792a9e4f7SAdrian Hunter 75892a9e4f7SAdrian Hunter /* Flush stack on exec */ 75992a9e4f7SAdrian Hunter if (ts->comm != comm && thread->pid_ == thread->tid) { 760a5499b37SAdrian Hunter err = __thread_stack__flush(thread, ts); 76192a9e4f7SAdrian Hunter if (err) 76292a9e4f7SAdrian Hunter return err; 76392a9e4f7SAdrian Hunter ts->comm = comm; 76492a9e4f7SAdrian Hunter } 76592a9e4f7SAdrian Hunter 76692a9e4f7SAdrian Hunter /* If the stack is empty, put the current symbol on the stack */ 76792a9e4f7SAdrian Hunter if (!ts->cnt) { 768e0b89511SAdrian Hunter err = thread_stack__bottom(ts, sample, from_al, to_al, ref); 76992a9e4f7SAdrian Hunter if (err) 77092a9e4f7SAdrian Hunter return err; 77192a9e4f7SAdrian Hunter } 77292a9e4f7SAdrian Hunter 77392a9e4f7SAdrian Hunter ts->branch_count += 1; 77492a9e4f7SAdrian Hunter ts->last_time = sample->time; 77592a9e4f7SAdrian Hunter 77692a9e4f7SAdrian Hunter if (sample->flags & PERF_IP_FLAG_CALL) { 7772dcde4e1SAdrian Hunter bool trace_end = sample->flags & PERF_IP_FLAG_TRACE_END; 77892a9e4f7SAdrian Hunter struct call_path_root *cpr = ts->crp->cpr; 77992a9e4f7SAdrian Hunter struct call_path *cp; 78092a9e4f7SAdrian Hunter u64 ret_addr; 78192a9e4f7SAdrian Hunter 78292a9e4f7SAdrian Hunter if (!sample->ip || !sample->addr) 78392a9e4f7SAdrian Hunter return 0; 78492a9e4f7SAdrian Hunter 78592a9e4f7SAdrian Hunter ret_addr = sample->ip + sample->insn_len; 78692a9e4f7SAdrian Hunter if (ret_addr == sample->addr) 78792a9e4f7SAdrian Hunter return 0; /* Zero-length calls are excluded */ 78892a9e4f7SAdrian Hunter 78992a9e4f7SAdrian Hunter cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, 79092a9e4f7SAdrian Hunter to_al->sym, sample->addr, 79192a9e4f7SAdrian Hunter ts->kernel_start); 79292a9e4f7SAdrian Hunter err = thread_stack__push_cp(ts, ret_addr, sample->time, ref, 7932dcde4e1SAdrian Hunter cp, false, trace_end); 79492a9e4f7SAdrian Hunter } else if (sample->flags & PERF_IP_FLAG_RETURN) { 79592a9e4f7SAdrian Hunter if (!sample->ip || !sample->addr) 79692a9e4f7SAdrian Hunter return 0; 79792a9e4f7SAdrian Hunter 79892a9e4f7SAdrian Hunter err = thread_stack__pop_cp(thread, ts, sample->addr, 79992a9e4f7SAdrian Hunter sample->time, ref, from_al->sym); 80092a9e4f7SAdrian Hunter if (err) { 80192a9e4f7SAdrian Hunter if (err < 0) 80292a9e4f7SAdrian Hunter return err; 80392a9e4f7SAdrian Hunter err = thread_stack__no_call_return(thread, ts, sample, 80492a9e4f7SAdrian Hunter from_al, to_al, ref); 80592a9e4f7SAdrian Hunter } 80692a9e4f7SAdrian Hunter } else if (sample->flags & PERF_IP_FLAG_TRACE_BEGIN) { 80792a9e4f7SAdrian Hunter err = thread_stack__trace_begin(thread, ts, sample->time, ref); 80892a9e4f7SAdrian Hunter } else if (sample->flags & PERF_IP_FLAG_TRACE_END) { 80992a9e4f7SAdrian Hunter err = thread_stack__trace_end(ts, sample, ref); 810f08046cbSAdrian Hunter } else if (sample->flags & PERF_IP_FLAG_BRANCH && 811f08046cbSAdrian Hunter from_al->sym != to_al->sym && to_al->sym && 812f08046cbSAdrian Hunter to_al->addr == to_al->sym->start) { 813f08046cbSAdrian Hunter struct call_path_root *cpr = ts->crp->cpr; 814f08046cbSAdrian Hunter struct call_path *cp; 815f08046cbSAdrian Hunter 816f08046cbSAdrian Hunter /* 817f08046cbSAdrian Hunter * The compiler might optimize a call/ret combination by making 818f08046cbSAdrian Hunter * it a jmp. Make that visible by recording on the stack a 819f08046cbSAdrian Hunter * branch to the start of a different symbol. Note, that means 820f08046cbSAdrian Hunter * when a ret pops the stack, all jmps must be popped off first. 821f08046cbSAdrian Hunter */ 822f08046cbSAdrian Hunter cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, 823f08046cbSAdrian Hunter to_al->sym, sample->addr, 824f08046cbSAdrian Hunter ts->kernel_start); 825f08046cbSAdrian Hunter err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false, 826f08046cbSAdrian Hunter false); 827f08046cbSAdrian Hunter if (!err) 828f08046cbSAdrian Hunter ts->stack[ts->cnt - 1].non_call = true; 82992a9e4f7SAdrian Hunter } 83092a9e4f7SAdrian Hunter 83192a9e4f7SAdrian Hunter return err; 83292a9e4f7SAdrian Hunter } 833e216708dSAdrian Hunter 834256d92bcSAdrian Hunter size_t thread_stack__depth(struct thread *thread, int cpu) 835e216708dSAdrian Hunter { 836256d92bcSAdrian Hunter struct thread_stack *ts = thread__stack(thread, cpu); 837bd8e68acSAdrian Hunter 838bd8e68acSAdrian Hunter if (!ts) 839e216708dSAdrian Hunter return 0; 840bd8e68acSAdrian Hunter return ts->cnt; 841e216708dSAdrian Hunter } 842