118 static int thread_stack__grow(struct thread_stack *ts)  in thread_stack__grow()  argument
123 	new_sz = ts->sz + STACK_GROWTH;  in thread_stack__grow()
126 	new_stack = realloc(ts->stack, sz);  in thread_stack__grow()
130 	ts->stack = new_stack;  in thread_stack__grow()
131 	ts->sz = new_sz;  in thread_stack__grow()
136 static int thread_stack__init(struct thread_stack *ts, struct thread *thread,  in thread_stack__init()  argument
143 		err = thread_stack__grow(ts);  in thread_stack__init()
152 		ts->br_stack_rb = zalloc(sz);  in thread_stack__init()
153 		if (!ts->br_stack_rb)  in thread_stack__init()
155 		ts->br_stack_sz = br_stack_sz;  in thread_stack__init()
162 		ts->kernel_start = machine__kernel_start(machine);  in thread_stack__init()
164 			ts->rstate = X86_RETPOLINE_POSSIBLE;  in thread_stack__init()
166 		ts->kernel_start = 1ULL << 63;  in thread_stack__init()
168 	ts->crp = crp;  in thread_stack__init()
178 	struct thread_stack *ts = thread__ts(thread), *new_ts;  in thread_stack__new()  local
179 	unsigned int old_sz = ts ? ts->arr_sz : 0;  in thread_stack__new()
185 	if (!ts || new_sz > old_sz) {  in thread_stack__new()
186 		new_ts = calloc(new_sz, sizeof(*ts));  in thread_stack__new()
189 		if (ts)  in thread_stack__new()
190 			memcpy(new_ts, ts, old_sz * sizeof(*ts));  in thread_stack__new()
194 		ts = new_ts;  in thread_stack__new()
198 	    (unsigned int)cpu < ts->arr_sz)  in thread_stack__new()
199 		ts += cpu;  in thread_stack__new()
201 	if (!ts->stack &&  in thread_stack__new()
202 	    thread_stack__init(ts, thread, crp, callstack, br_stack_sz))  in thread_stack__new()
205 	return ts;  in thread_stack__new()
210 	struct thread_stack *ts = thread__ts(thread);  in thread__cpu_stack()  local
215 	if (!ts || (unsigned int)cpu >= ts->arr_sz)  in thread__cpu_stack()
218 	ts += cpu;  in thread__cpu_stack()
220 	if (!ts->stack)  in thread__cpu_stack()
223 	return ts;  in thread__cpu_stack()
238 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push()  argument
243 	if (ts->cnt == ts->sz) {  in thread_stack__push()
244 		err = thread_stack__grow(ts);  in thread_stack__push()
247 			ts->cnt = 0;  in thread_stack__push()
251 	ts->stack[ts->cnt].trace_end = trace_end;  in thread_stack__push()
252 	ts->stack[ts->cnt++].ret_addr = ret_addr;  in thread_stack__push()
257 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)  in thread_stack__pop()  argument
270 	for (i = ts->cnt; i; ) {  in thread_stack__pop()
271 		if (ts->stack[--i].ret_addr == ret_addr) {  in thread_stack__pop()
272 			ts->cnt = i;  in thread_stack__pop()
278 static void thread_stack__pop_trace_end(struct thread_stack *ts)  in thread_stack__pop_trace_end()  argument
282 	for (i = ts->cnt; i; ) {  in thread_stack__pop_trace_end()
283 		if (ts->stack[--i].trace_end)  in thread_stack__pop_trace_end()
284 			ts->cnt = i;  in thread_stack__pop_trace_end()
290 static bool thread_stack__in_kernel(struct thread_stack *ts)  in thread_stack__in_kernel()  argument
292 	if (!ts->cnt)  in thread_stack__in_kernel()
295 	return ts->stack[ts->cnt - 1].cp->in_kernel;  in thread_stack__in_kernel()
299 				     struct thread_stack *ts, size_t idx,  in thread_stack__call_return()  argument
302 	struct call_return_processor *crp = ts->crp;  in thread_stack__call_return()
306 		.comm = ts->comm,  in thread_stack__call_return()
311 	tse = &ts->stack[idx];  in thread_stack__call_return()
315 	cr.branch_count = ts->branch_count - tse->branch_count;  in thread_stack__call_return()
316 	cr.insn_count = ts->insn_count - tse->insn_count;  in thread_stack__call_return()
317 	cr.cyc_count = ts->cyc_count - tse->cyc_count;  in thread_stack__call_return()
338 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)  in __thread_stack__flush()  argument
340 	struct call_return_processor *crp = ts->crp;  in __thread_stack__flush()
344 		ts->cnt = 0;  in __thread_stack__flush()
345 		ts->br_stack_pos = 0;  in __thread_stack__flush()
346 		if (ts->br_stack_rb)  in __thread_stack__flush()
347 			ts->br_stack_rb->nr = 0;  in __thread_stack__flush()
351 	while (ts->cnt) {  in __thread_stack__flush()
352 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in __thread_stack__flush()
353 						ts->last_time, 0, true);  in __thread_stack__flush()
356 			ts->cnt = 0;  in __thread_stack__flush()
366 	struct thread_stack *ts = thread__ts(thread);  in thread_stack__flush()  local
370 	if (ts) {  in thread_stack__flush()
371 		for (pos = 0; pos < ts->arr_sz; pos++) {  in thread_stack__flush()
372 			int ret = __thread_stack__flush(thread, ts + pos);  in thread_stack__flush()
382 static void thread_stack__update_br_stack(struct thread_stack *ts, u32 flags,  in thread_stack__update_br_stack()  argument
385 	struct branch_stack *bs = ts->br_stack_rb;  in thread_stack__update_br_stack()
388 	if (!ts->br_stack_pos)  in thread_stack__update_br_stack()
389 		ts->br_stack_pos = ts->br_stack_sz;  in thread_stack__update_br_stack()
391 	ts->br_stack_pos -= 1;  in thread_stack__update_br_stack()
393 	be              = &bs->entries[ts->br_stack_pos];  in thread_stack__update_br_stack()
400 	be->flags.mispred = ts->mispred_all;  in thread_stack__update_br_stack()
402 	if (bs->nr < ts->br_stack_sz)  in thread_stack__update_br_stack()
410 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__event()  local
415 	if (!ts) {  in thread_stack__event()
416 		ts = thread_stack__new(thread, cpu, NULL, callstack, br_stack_sz);  in thread_stack__event()
417 		if (!ts) {  in thread_stack__event()
421 		ts->trace_nr = trace_nr;  in thread_stack__event()
422 		ts->mispred_all = mispred_all;  in thread_stack__event()
430 	if (trace_nr != ts->trace_nr) {  in thread_stack__event()
431 		if (ts->trace_nr)  in thread_stack__event()
432 			__thread_stack__flush(thread, ts);  in thread_stack__event()
433 		ts->trace_nr = trace_nr;  in thread_stack__event()
437 		thread_stack__update_br_stack(ts, flags, from_ip, to_ip);  in thread_stack__event()
443 	if (ts->crp || !callstack)  in thread_stack__event()
454 		return thread_stack__push(ts, ret_addr,  in thread_stack__event()
464 		thread_stack__pop(ts, to_ip);  in thread_stack__event()
465 		thread_stack__pop_trace_end(ts);  in thread_stack__event()
467 		thread_stack__pop(ts, to_ip);  in thread_stack__event()
475 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__set_trace_nr()  local
477 	if (!ts)  in thread_stack__set_trace_nr()
480 	if (trace_nr != ts->trace_nr) {  in thread_stack__set_trace_nr()
481 		if (ts->trace_nr)  in thread_stack__set_trace_nr()
482 			__thread_stack__flush(thread, ts);  in thread_stack__set_trace_nr()
483 		ts->trace_nr = trace_nr;  in thread_stack__set_trace_nr()
487 static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)  in __thread_stack__free()  argument
489 	__thread_stack__flush(thread, ts);  in __thread_stack__free()
490 	zfree(&ts->stack);  in __thread_stack__free()
491 	zfree(&ts->br_stack_rb);  in __thread_stack__free()
494 static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)  in thread_stack__reset()  argument
496 	unsigned int arr_sz = ts->arr_sz;  in thread_stack__reset()
498 	__thread_stack__free(thread, ts);  in thread_stack__reset()
499 	memset(ts, 0, sizeof(*ts));  in thread_stack__reset()
500 	ts->arr_sz = arr_sz;  in thread_stack__reset()
505 	struct thread_stack *ts = thread__ts(thread);  in thread_stack__free()  local
508 	if (ts) {  in thread_stack__free()
509 		for (pos = 0; pos < ts->arr_sz; pos++)  in thread_stack__free()
510 			__thread_stack__free(thread, ts + pos);  in thread_stack__free()
525 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__sample()  local
538 	if (!ts) {  in thread_stack__sample()
545 	for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {  in thread_stack__sample()
546 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample()
568 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__sample_late()  local
578 	if (!ts)  in thread_stack__sample_late()
586 	for (j = 1; j <= ts->cnt; j++) {  in thread_stack__sample_late()
587 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample_late()
596 	for (; nr < sz && j <= ts->cnt; nr++, j++) {  in thread_stack__sample_late()
597 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample_late()
620 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__br_sample()  local
628 	if (!ts)  in thread_stack__br_sample()
631 	src = ts->br_stack_rb;  in thread_stack__br_sample()
638 	nr = min(ts->br_stack_sz - ts->br_stack_pos, (unsigned int)dst->nr);  in thread_stack__br_sample()
639 	memcpy(be, &src->entries[ts->br_stack_pos], bsz * nr);  in thread_stack__br_sample()
641 	if (src->nr >= ts->br_stack_sz) {  in thread_stack__br_sample()
644 		nr = min(ts->br_stack_pos, sz);  in thread_stack__br_sample()
645 		memcpy(be, &src->entries[0], bsz * ts->br_stack_pos);  in thread_stack__br_sample()
682 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__br_sample_late()  local
690 	if (!ts)  in thread_stack__br_sample_late()
693 	src = ts->br_stack_rb;  in thread_stack__br_sample_late()
697 	spos = &src->entries[ts->br_stack_pos];  in thread_stack__br_sample_late()
698 	ssz  = &src->entries[ts->br_stack_sz];  in thread_stack__br_sample_late()
715 		if (src->nr >= ts->br_stack_sz) {  in thread_stack__br_sample_late()
739 		if (src->nr >= ts->br_stack_sz) {  in thread_stack__br_sample_late()
782 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push_cp()  argument
792 	if (ts->cnt == ts->sz) {  in thread_stack__push_cp()
793 		err = thread_stack__grow(ts);  in thread_stack__push_cp()
798 	tse = &ts->stack[ts->cnt++];  in thread_stack__push_cp()
802 	tse->branch_count = ts->branch_count;  in thread_stack__push_cp()
803 	tse->insn_count = ts->insn_count;  in thread_stack__push_cp()
804 	tse->cyc_count = ts->cyc_count;  in thread_stack__push_cp()
814 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_cp()  argument
820 	if (!ts->cnt)  in thread_stack__pop_cp()
823 	if (ts->cnt == 1) {  in thread_stack__pop_cp()
824 		struct thread_stack_entry *tse = &ts->stack[0];  in thread_stack__pop_cp()
827 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
831 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&  in thread_stack__pop_cp()
832 	    !ts->stack[ts->cnt - 1].non_call) {  in thread_stack__pop_cp()
833 		return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
836 		size_t i = ts->cnt - 1;  in thread_stack__pop_cp()
839 			if (ts->stack[i].ret_addr != ret_addr ||  in thread_stack__pop_cp()
840 			    ts->stack[i].non_call)  in thread_stack__pop_cp()
843 			while (ts->cnt > i) {  in thread_stack__pop_cp()
844 				err = thread_stack__call_return(thread, ts,  in thread_stack__pop_cp()
845 								--ts->cnt,  in thread_stack__pop_cp()
851 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
859 static int thread_stack__bottom(struct thread_stack *ts,  in thread_stack__bottom()  argument
864 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__bottom()
880 				ts->kernel_start);  in thread_stack__bottom()
882 	return thread_stack__push_cp(ts, ip, sample->time, ref, cp,  in thread_stack__bottom()
886 static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_ks()  argument
893 	while (thread_stack__in_kernel(ts)) {  in thread_stack__pop_ks()
894 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_ks()
904 					struct thread_stack *ts,  in thread_stack__no_call_return()  argument
909 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__no_call_return()
914 	u64 ks = ts->kernel_start;  in thread_stack__no_call_return()
922 		err = thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__no_call_return()
927 		if (!ts->cnt) {  in thread_stack__no_call_return()
929 			return thread_stack__push_cp(ts, 0, tm, ref, cp, true,  in thread_stack__no_call_return()
932 	} else if (thread_stack__in_kernel(ts) && ip < ks) {  in thread_stack__no_call_return()
934 		err = thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__no_call_return()
939 	if (ts->cnt)  in thread_stack__no_call_return()
940 		parent = ts->stack[ts->cnt - 1].cp;  in thread_stack__no_call_return()
950 		if (ts->cnt == 1) {  in thread_stack__no_call_return()
951 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
957 		if (!ts->cnt) {  in thread_stack__no_call_return()
960 			return thread_stack__push_cp(ts, addr, tm, ref, cp,  in thread_stack__no_call_return()
970 		err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);  in thread_stack__no_call_return()
972 			ts->stack[ts->cnt - 1].non_call = true;  in thread_stack__no_call_return()
984 	err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);  in thread_stack__no_call_return()
990 	err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);  in thread_stack__no_call_return()
994 	return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);  in thread_stack__no_call_return()
998 				     struct thread_stack *ts, u64 timestamp,  in thread_stack__trace_begin()  argument
1004 	if (!ts->cnt)  in thread_stack__trace_begin()
1008 	tse = &ts->stack[ts->cnt - 1];  in thread_stack__trace_begin()
1010 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__trace_begin()
1019 static int thread_stack__trace_end(struct thread_stack *ts,  in thread_stack__trace_end()  argument
1022 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__trace_end()
1027 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))  in thread_stack__trace_end()
1030 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,  in thread_stack__trace_end()
1031 				ts->kernel_start);  in thread_stack__trace_end()
1035 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,  in thread_stack__trace_end()
1049 static int thread_stack__x86_retpoline(struct thread_stack *ts,  in thread_stack__x86_retpoline()  argument
1053 	struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];  in thread_stack__x86_retpoline()
1054 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__x86_retpoline()
1070 		ts->cnt -= 1;  in thread_stack__x86_retpoline()
1071 		sym = ts->stack[ts->cnt - 2].cp->sym;  in thread_stack__x86_retpoline()
1078 			ts->cnt -= 1;  in thread_stack__x86_retpoline()
1086 		ts->cnt -= 1;  in thread_stack__x86_retpoline()
1090 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,  in thread_stack__x86_retpoline()
1091 				sample->addr, ts->kernel_start);  in thread_stack__x86_retpoline()
1096 	ts->stack[ts->cnt - 1].cp = cp;  in thread_stack__x86_retpoline()
1107 	struct thread_stack *ts = thread__stack(thread, sample->cpu);  in thread_stack__process()  local
1111 	if (ts && !ts->crp) {  in thread_stack__process()
1113 		thread_stack__reset(thread, ts);  in thread_stack__process()
1114 		ts = NULL;  in thread_stack__process()
1117 	if (!ts) {  in thread_stack__process()
1118 		ts = thread_stack__new(thread, sample->cpu, crp, true, 0);  in thread_stack__process()
1119 		if (!ts)  in thread_stack__process()
1121 		ts->comm = comm;  in thread_stack__process()
1124 	rstate = ts->rstate;  in thread_stack__process()
1126 		ts->rstate = X86_RETPOLINE_POSSIBLE;  in thread_stack__process()
1129 	if (ts->comm != comm && thread__pid(thread) == thread__tid(thread)) {  in thread_stack__process()
1130 		err = __thread_stack__flush(thread, ts);  in thread_stack__process()
1133 		ts->comm = comm;  in thread_stack__process()
1137 	if (!ts->cnt) {  in thread_stack__process()
1138 		err = thread_stack__bottom(ts, sample, from_al, to_al, ref);  in thread_stack__process()
1143 	ts->branch_count += 1;  in thread_stack__process()
1144 	ts->insn_count += sample->insn_cnt;  in thread_stack__process()
1145 	ts->cyc_count += sample->cyc_cnt;  in thread_stack__process()
1146 	ts->last_time = sample->time;  in thread_stack__process()
1150 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
1161 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
1163 					ts->kernel_start);  in thread_stack__process()
1164 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,  in thread_stack__process()
1174 			ts->rstate = X86_RETPOLINE_DETECTED;  in thread_stack__process()
1185 			return thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__process()
1192 		if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&  in thread_stack__process()
1193 		    ts->stack[ts->cnt - 1].ret_addr != sample->addr)  in thread_stack__process()
1194 			return thread_stack__x86_retpoline(ts, sample, to_al);  in thread_stack__process()
1196 		err = thread_stack__pop_cp(thread, ts, sample->addr,  in thread_stack__process()
1201 			err = thread_stack__no_call_return(thread, ts, sample,  in thread_stack__process()
1205 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);  in thread_stack__process()
1207 		err = thread_stack__trace_end(ts, sample, ref);  in thread_stack__process()
1211 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
1220 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
1222 					ts->kernel_start);  in thread_stack__process()
1223 		err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,  in thread_stack__process()
1226 			ts->stack[ts->cnt - 1].non_call = true;  in thread_stack__process()
1234 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__depth()  local
1236 	if (!ts)  in thread_stack__depth()
1238 	return ts->cnt;  in thread_stack__depth()