1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Stack tracing support
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 */
7 #include <linux/kernel.h>
8 #include <linux/efi.h>
9 #include <linux/export.h>
10 #include <linux/ftrace.h>
11 #include <linux/sched.h>
12 #include <linux/sched/debug.h>
13 #include <linux/sched/task_stack.h>
14 #include <linux/stacktrace.h>
15
16 #include <asm/efi.h>
17 #include <asm/irq.h>
18 #include <asm/stack_pointer.h>
19 #include <asm/stacktrace.h>
20
21 /*
22 * Start an unwind from a pt_regs.
23 *
24 * The unwind will begin at the PC within the regs.
25 *
26 * The regs must be on a stack currently owned by the calling task.
27 */
28 static __always_inline void
unwind_init_from_regs(struct unwind_state * state,struct pt_regs * regs)29 unwind_init_from_regs(struct unwind_state *state,
30 struct pt_regs *regs)
31 {
32 unwind_init_common(state, current);
33
34 state->fp = regs->regs[29];
35 state->pc = regs->pc;
36 }
37
38 /*
39 * Start an unwind from a caller.
40 *
41 * The unwind will begin at the caller of whichever function this is inlined
42 * into.
43 *
44 * The function which invokes this must be noinline.
45 */
46 static __always_inline void
unwind_init_from_caller(struct unwind_state * state)47 unwind_init_from_caller(struct unwind_state *state)
48 {
49 unwind_init_common(state, current);
50
51 state->fp = (unsigned long)__builtin_frame_address(1);
52 state->pc = (unsigned long)__builtin_return_address(0);
53 }
54
55 /*
56 * Start an unwind from a blocked task.
57 *
58 * The unwind will begin at the blocked tasks saved PC (i.e. the caller of
59 * cpu_switch_to()).
60 *
61 * The caller should ensure the task is blocked in cpu_switch_to() for the
62 * duration of the unwind, or the unwind will be bogus. It is never valid to
63 * call this for the current task.
64 */
65 static __always_inline void
unwind_init_from_task(struct unwind_state * state,struct task_struct * task)66 unwind_init_from_task(struct unwind_state *state,
67 struct task_struct *task)
68 {
69 unwind_init_common(state, task);
70
71 state->fp = thread_saved_fp(task);
72 state->pc = thread_saved_pc(task);
73 }
74
75 static __always_inline int
unwind_recover_return_address(struct unwind_state * state)76 unwind_recover_return_address(struct unwind_state *state)
77 {
78 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
79 if (state->task->ret_stack &&
80 (state->pc == (unsigned long)return_to_handler)) {
81 unsigned long orig_pc;
82 orig_pc = ftrace_graph_ret_addr(state->task, NULL, state->pc,
83 (void *)state->fp);
84 if (WARN_ON_ONCE(state->pc == orig_pc))
85 return -EINVAL;
86 state->pc = orig_pc;
87 }
88 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
89
90 #ifdef CONFIG_KRETPROBES
91 if (is_kretprobe_trampoline(state->pc)) {
92 state->pc = kretprobe_find_ret_addr(state->task,
93 (void *)state->fp,
94 &state->kr_cur);
95 }
96 #endif /* CONFIG_KRETPROBES */
97
98 return 0;
99 }
100
101 /*
102 * Unwind from one frame record (A) to the next frame record (B).
103 *
104 * We terminate early if the location of B indicates a malformed chain of frame
105 * records (e.g. a cycle), determined based on the location and fp value of A
106 * and the location (but not the fp value) of B.
107 */
108 static __always_inline int
unwind_next(struct unwind_state * state)109 unwind_next(struct unwind_state *state)
110 {
111 struct task_struct *tsk = state->task;
112 unsigned long fp = state->fp;
113 int err;
114
115 /* Final frame; nothing to unwind */
116 if (fp == (unsigned long)task_pt_regs(tsk)->stackframe)
117 return -ENOENT;
118
119 err = unwind_next_frame_record(state);
120 if (err)
121 return err;
122
123 state->pc = ptrauth_strip_kernel_insn_pac(state->pc);
124
125 return unwind_recover_return_address(state);
126 }
127
128 static __always_inline void
unwind(struct unwind_state * state,stack_trace_consume_fn consume_entry,void * cookie)129 unwind(struct unwind_state *state, stack_trace_consume_fn consume_entry,
130 void *cookie)
131 {
132 if (unwind_recover_return_address(state))
133 return;
134
135 while (1) {
136 int ret;
137
138 if (!consume_entry(cookie, state->pc))
139 break;
140 ret = unwind_next(state);
141 if (ret < 0)
142 break;
143 }
144 }
145
146 /*
147 * Per-cpu stacks are only accessible when unwinding the current task in a
148 * non-preemptible context.
149 */
150 #define STACKINFO_CPU(name) \
151 ({ \
152 ((task == current) && !preemptible()) \
153 ? stackinfo_get_##name() \
154 : stackinfo_get_unknown(); \
155 })
156
157 /*
158 * SDEI stacks are only accessible when unwinding the current task in an NMI
159 * context.
160 */
161 #define STACKINFO_SDEI(name) \
162 ({ \
163 ((task == current) && in_nmi()) \
164 ? stackinfo_get_sdei_##name() \
165 : stackinfo_get_unknown(); \
166 })
167
168 #define STACKINFO_EFI \
169 ({ \
170 ((task == current) && current_in_efi()) \
171 ? stackinfo_get_efi() \
172 : stackinfo_get_unknown(); \
173 })
174
arch_stack_walk(stack_trace_consume_fn consume_entry,void * cookie,struct task_struct * task,struct pt_regs * regs)175 noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
176 void *cookie, struct task_struct *task,
177 struct pt_regs *regs)
178 {
179 struct stack_info stacks[] = {
180 stackinfo_get_task(task),
181 STACKINFO_CPU(irq),
182 #if defined(CONFIG_VMAP_STACK)
183 STACKINFO_CPU(overflow),
184 #endif
185 #if defined(CONFIG_VMAP_STACK) && defined(CONFIG_ARM_SDE_INTERFACE)
186 STACKINFO_SDEI(normal),
187 STACKINFO_SDEI(critical),
188 #endif
189 #ifdef CONFIG_EFI
190 STACKINFO_EFI,
191 #endif
192 };
193 struct unwind_state state = {
194 .stacks = stacks,
195 .nr_stacks = ARRAY_SIZE(stacks),
196 };
197
198 if (regs) {
199 if (task != current)
200 return;
201 unwind_init_from_regs(&state, regs);
202 } else if (task == current) {
203 unwind_init_from_caller(&state);
204 } else {
205 unwind_init_from_task(&state, task);
206 }
207
208 unwind(&state, consume_entry, cookie);
209 }
210
dump_backtrace_entry(void * arg,unsigned long where)211 static bool dump_backtrace_entry(void *arg, unsigned long where)
212 {
213 char *loglvl = arg;
214 printk("%s %pSb\n", loglvl, (void *)where);
215 return true;
216 }
217
dump_backtrace(struct pt_regs * regs,struct task_struct * tsk,const char * loglvl)218 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
219 const char *loglvl)
220 {
221 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
222
223 if (regs && user_mode(regs))
224 return;
225
226 if (!tsk)
227 tsk = current;
228
229 if (!try_get_task_stack(tsk))
230 return;
231
232 printk("%sCall trace:\n", loglvl);
233 arch_stack_walk(dump_backtrace_entry, (void *)loglvl, tsk, regs);
234
235 put_task_stack(tsk);
236 }
237
show_stack(struct task_struct * tsk,unsigned long * sp,const char * loglvl)238 void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
239 {
240 dump_backtrace(NULL, tsk, loglvl);
241 barrier();
242 }
243