xref: /openbmc/linux/arch/x86/kernel/unwind_orc.c (revision b9f174c8)
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/objtool.h>
3 #include <linux/module.h>
4 #include <linux/sort.h>
5 #include <asm/ptrace.h>
6 #include <asm/stacktrace.h>
7 #include <asm/unwind.h>
8 #include <asm/orc_types.h>
9 #include <asm/orc_lookup.h>
10 #include <asm/orc_header.h>
11 
12 ORC_HEADER;
13 
14 #define orc_warn(fmt, ...) \
15 	printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
16 
17 #define orc_warn_current(args...)					\
18 ({									\
19 	static bool dumped_before;					\
20 	if (state->task == current && !state->error) {			\
21 		orc_warn(args);						\
22 		if (unwind_debug && !dumped_before) {			\
23 			dumped_before = true;				\
24 			unwind_dump(state);				\
25 		}							\
26 	}								\
27 })
28 
29 extern int __start_orc_unwind_ip[];
30 extern int __stop_orc_unwind_ip[];
31 extern struct orc_entry __start_orc_unwind[];
32 extern struct orc_entry __stop_orc_unwind[];
33 
34 static bool orc_init __ro_after_init;
35 static bool unwind_debug __ro_after_init;
36 static unsigned int lookup_num_blocks __ro_after_init;
37 
unwind_debug_cmdline(char * str)38 static int __init unwind_debug_cmdline(char *str)
39 {
40 	unwind_debug = true;
41 
42 	return 0;
43 }
44 early_param("unwind_debug", unwind_debug_cmdline);
45 
unwind_dump(struct unwind_state * state)46 static void unwind_dump(struct unwind_state *state)
47 {
48 	static bool dumped_before;
49 	unsigned long word, *sp;
50 	struct stack_info stack_info = {0};
51 	unsigned long visit_mask = 0;
52 
53 	if (dumped_before)
54 		return;
55 
56 	dumped_before = true;
57 
58 	printk_deferred("unwind stack type:%d next_sp:%p mask:0x%lx graph_idx:%d\n",
59 			state->stack_info.type, state->stack_info.next_sp,
60 			state->stack_mask, state->graph_idx);
61 
62 	for (sp = __builtin_frame_address(0); sp;
63 	     sp = PTR_ALIGN(stack_info.next_sp, sizeof(long))) {
64 		if (get_stack_info(sp, state->task, &stack_info, &visit_mask))
65 			break;
66 
67 		for (; sp < stack_info.end; sp++) {
68 
69 			word = READ_ONCE_NOCHECK(*sp);
70 
71 			printk_deferred("%0*lx: %0*lx (%pB)\n", BITS_PER_LONG/4,
72 					(unsigned long)sp, BITS_PER_LONG/4,
73 					word, (void *)word);
74 		}
75 	}
76 }
77 
orc_ip(const int * ip)78 static inline unsigned long orc_ip(const int *ip)
79 {
80 	return (unsigned long)ip + *ip;
81 }
82 
__orc_find(int * ip_table,struct orc_entry * u_table,unsigned int num_entries,unsigned long ip)83 static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
84 				    unsigned int num_entries, unsigned long ip)
85 {
86 	int *first = ip_table;
87 	int *last = ip_table + num_entries - 1;
88 	int *mid = first, *found = first;
89 
90 	if (!num_entries)
91 		return NULL;
92 
93 	/*
94 	 * Do a binary range search to find the rightmost duplicate of a given
95 	 * starting address.  Some entries are section terminators which are
96 	 * "weak" entries for ensuring there are no gaps.  They should be
97 	 * ignored when they conflict with a real entry.
98 	 */
99 	while (first <= last) {
100 		mid = first + ((last - first) / 2);
101 
102 		if (orc_ip(mid) <= ip) {
103 			found = mid;
104 			first = mid + 1;
105 		} else
106 			last = mid - 1;
107 	}
108 
109 	return u_table + (found - ip_table);
110 }
111 
112 #ifdef CONFIG_MODULES
orc_module_find(unsigned long ip)113 static struct orc_entry *orc_module_find(unsigned long ip)
114 {
115 	struct module *mod;
116 
117 	mod = __module_address(ip);
118 	if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip)
119 		return NULL;
120 	return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind,
121 			  mod->arch.num_orcs, ip);
122 }
123 #else
orc_module_find(unsigned long ip)124 static struct orc_entry *orc_module_find(unsigned long ip)
125 {
126 	return NULL;
127 }
128 #endif
129 
130 #ifdef CONFIG_DYNAMIC_FTRACE
131 static struct orc_entry *orc_find(unsigned long ip);
132 
133 /*
134  * Ftrace dynamic trampolines do not have orc entries of their own.
135  * But they are copies of the ftrace entries that are static and
136  * defined in ftrace_*.S, which do have orc entries.
137  *
138  * If the unwinder comes across a ftrace trampoline, then find the
139  * ftrace function that was used to create it, and use that ftrace
140  * function's orc entry, as the placement of the return code in
141  * the stack will be identical.
142  */
orc_ftrace_find(unsigned long ip)143 static struct orc_entry *orc_ftrace_find(unsigned long ip)
144 {
145 	struct ftrace_ops *ops;
146 	unsigned long tramp_addr, offset;
147 
148 	ops = ftrace_ops_trampoline(ip);
149 	if (!ops)
150 		return NULL;
151 
152 	/* Set tramp_addr to the start of the code copied by the trampoline */
153 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
154 		tramp_addr = (unsigned long)ftrace_regs_caller;
155 	else
156 		tramp_addr = (unsigned long)ftrace_caller;
157 
158 	/* Now place tramp_addr to the location within the trampoline ip is at */
159 	offset = ip - ops->trampoline;
160 	tramp_addr += offset;
161 
162 	/* Prevent unlikely recursion */
163 	if (ip == tramp_addr)
164 		return NULL;
165 
166 	return orc_find(tramp_addr);
167 }
168 #else
orc_ftrace_find(unsigned long ip)169 static struct orc_entry *orc_ftrace_find(unsigned long ip)
170 {
171 	return NULL;
172 }
173 #endif
174 
175 /*
176  * If we crash with IP==0, the last successfully executed instruction
177  * was probably an indirect function call with a NULL function pointer,
178  * and we don't have unwind information for NULL.
179  * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
180  * pointer into its parent and then continue normally from there.
181  */
182 static struct orc_entry null_orc_entry = {
183 	.sp_offset = sizeof(long),
184 	.sp_reg = ORC_REG_SP,
185 	.bp_reg = ORC_REG_UNDEFINED,
186 	.type = ORC_TYPE_CALL
187 };
188 
189 /* Fake frame pointer entry -- used as a fallback for generated code */
190 static struct orc_entry orc_fp_entry = {
191 	.type		= ORC_TYPE_CALL,
192 	.sp_reg		= ORC_REG_BP,
193 	.sp_offset	= 16,
194 	.bp_reg		= ORC_REG_PREV_SP,
195 	.bp_offset	= -16,
196 };
197 
orc_find(unsigned long ip)198 static struct orc_entry *orc_find(unsigned long ip)
199 {
200 	static struct orc_entry *orc;
201 
202 	if (ip == 0)
203 		return &null_orc_entry;
204 
205 	/* For non-init vmlinux addresses, use the fast lookup table: */
206 	if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) {
207 		unsigned int idx, start, stop;
208 
209 		idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE;
210 
211 		if (unlikely((idx >= lookup_num_blocks-1))) {
212 			orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n",
213 				 idx, lookup_num_blocks, (void *)ip);
214 			return NULL;
215 		}
216 
217 		start = orc_lookup[idx];
218 		stop = orc_lookup[idx + 1] + 1;
219 
220 		if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) ||
221 			     (__start_orc_unwind + stop > __stop_orc_unwind))) {
222 			orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n",
223 				 idx, lookup_num_blocks, start, stop, (void *)ip);
224 			return NULL;
225 		}
226 
227 		return __orc_find(__start_orc_unwind_ip + start,
228 				  __start_orc_unwind + start, stop - start, ip);
229 	}
230 
231 	/* vmlinux .init slow lookup: */
232 	if (is_kernel_inittext(ip))
233 		return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
234 				  __stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
235 
236 	/* Module lookup: */
237 	orc = orc_module_find(ip);
238 	if (orc)
239 		return orc;
240 
241 	return orc_ftrace_find(ip);
242 }
243 
244 #ifdef CONFIG_MODULES
245 
246 static DEFINE_MUTEX(sort_mutex);
247 static int *cur_orc_ip_table = __start_orc_unwind_ip;
248 static struct orc_entry *cur_orc_table = __start_orc_unwind;
249 
orc_sort_swap(void * _a,void * _b,int size)250 static void orc_sort_swap(void *_a, void *_b, int size)
251 {
252 	struct orc_entry *orc_a, *orc_b;
253 	int *a = _a, *b = _b, tmp;
254 	int delta = _b - _a;
255 
256 	/* Swap the .orc_unwind_ip entries: */
257 	tmp = *a;
258 	*a = *b + delta;
259 	*b = tmp - delta;
260 
261 	/* Swap the corresponding .orc_unwind entries: */
262 	orc_a = cur_orc_table + (a - cur_orc_ip_table);
263 	orc_b = cur_orc_table + (b - cur_orc_ip_table);
264 	swap(*orc_a, *orc_b);
265 }
266 
orc_sort_cmp(const void * _a,const void * _b)267 static int orc_sort_cmp(const void *_a, const void *_b)
268 {
269 	struct orc_entry *orc_a;
270 	const int *a = _a, *b = _b;
271 	unsigned long a_val = orc_ip(a);
272 	unsigned long b_val = orc_ip(b);
273 
274 	if (a_val > b_val)
275 		return 1;
276 	if (a_val < b_val)
277 		return -1;
278 
279 	/*
280 	 * The "weak" section terminator entries need to always be first
281 	 * to ensure the lookup code skips them in favor of real entries.
282 	 * These terminator entries exist to handle any gaps created by
283 	 * whitelisted .o files which didn't get objtool generation.
284 	 */
285 	orc_a = cur_orc_table + (a - cur_orc_ip_table);
286 	return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
287 }
288 
unwind_module_init(struct module * mod,void * _orc_ip,size_t orc_ip_size,void * _orc,size_t orc_size)289 void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
290 			void *_orc, size_t orc_size)
291 {
292 	int *orc_ip = _orc_ip;
293 	struct orc_entry *orc = _orc;
294 	unsigned int num_entries = orc_ip_size / sizeof(int);
295 
296 	WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 ||
297 		     orc_size % sizeof(*orc) != 0 ||
298 		     num_entries != orc_size / sizeof(*orc));
299 
300 	/*
301 	 * The 'cur_orc_*' globals allow the orc_sort_swap() callback to
302 	 * associate an .orc_unwind_ip table entry with its corresponding
303 	 * .orc_unwind entry so they can both be swapped.
304 	 */
305 	mutex_lock(&sort_mutex);
306 	cur_orc_ip_table = orc_ip;
307 	cur_orc_table = orc;
308 	sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
309 	mutex_unlock(&sort_mutex);
310 
311 	mod->arch.orc_unwind_ip = orc_ip;
312 	mod->arch.orc_unwind = orc;
313 	mod->arch.num_orcs = num_entries;
314 }
315 #endif
316 
unwind_init(void)317 void __init unwind_init(void)
318 {
319 	size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip;
320 	size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind;
321 	size_t num_entries = orc_ip_size / sizeof(int);
322 	struct orc_entry *orc;
323 	int i;
324 
325 	if (!num_entries || orc_ip_size % sizeof(int) != 0 ||
326 	    orc_size % sizeof(struct orc_entry) != 0 ||
327 	    num_entries != orc_size / sizeof(struct orc_entry)) {
328 		orc_warn("WARNING: Bad or missing .orc_unwind table.  Disabling unwinder.\n");
329 		return;
330 	}
331 
332 	/*
333 	 * Note, the orc_unwind and orc_unwind_ip tables were already
334 	 * sorted at build time via the 'sorttable' tool.
335 	 * It's ready for binary search straight away, no need to sort it.
336 	 */
337 
338 	/* Initialize the fast lookup table: */
339 	lookup_num_blocks = orc_lookup_end - orc_lookup;
340 	for (i = 0; i < lookup_num_blocks-1; i++) {
341 		orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
342 				 num_entries,
343 				 LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i));
344 		if (!orc) {
345 			orc_warn("WARNING: Corrupt .orc_unwind table.  Disabling unwinder.\n");
346 			return;
347 		}
348 
349 		orc_lookup[i] = orc - __start_orc_unwind;
350 	}
351 
352 	/* Initialize the ending block: */
353 	orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries,
354 			 LOOKUP_STOP_IP);
355 	if (!orc) {
356 		orc_warn("WARNING: Corrupt .orc_unwind table.  Disabling unwinder.\n");
357 		return;
358 	}
359 	orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind;
360 
361 	orc_init = true;
362 }
363 
unwind_get_return_address(struct unwind_state * state)364 unsigned long unwind_get_return_address(struct unwind_state *state)
365 {
366 	if (unwind_done(state))
367 		return 0;
368 
369 	return __kernel_text_address(state->ip) ? state->ip : 0;
370 }
371 EXPORT_SYMBOL_GPL(unwind_get_return_address);
372 
unwind_get_return_address_ptr(struct unwind_state * state)373 unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)
374 {
375 	if (unwind_done(state))
376 		return NULL;
377 
378 	if (state->regs)
379 		return &state->regs->ip;
380 
381 	if (state->sp)
382 		return (unsigned long *)state->sp - 1;
383 
384 	return NULL;
385 }
386 
stack_access_ok(struct unwind_state * state,unsigned long _addr,size_t len)387 static bool stack_access_ok(struct unwind_state *state, unsigned long _addr,
388 			    size_t len)
389 {
390 	struct stack_info *info = &state->stack_info;
391 	void *addr = (void *)_addr;
392 
393 	if (on_stack(info, addr, len))
394 		return true;
395 
396 	return !get_stack_info(addr, state->task, info, &state->stack_mask) &&
397 		on_stack(info, addr, len);
398 }
399 
deref_stack_reg(struct unwind_state * state,unsigned long addr,unsigned long * val)400 static bool deref_stack_reg(struct unwind_state *state, unsigned long addr,
401 			    unsigned long *val)
402 {
403 	if (!stack_access_ok(state, addr, sizeof(long)))
404 		return false;
405 
406 	*val = READ_ONCE_NOCHECK(*(unsigned long *)addr);
407 	return true;
408 }
409 
deref_stack_regs(struct unwind_state * state,unsigned long addr,unsigned long * ip,unsigned long * sp)410 static bool deref_stack_regs(struct unwind_state *state, unsigned long addr,
411 			     unsigned long *ip, unsigned long *sp)
412 {
413 	struct pt_regs *regs = (struct pt_regs *)addr;
414 
415 	/* x86-32 support will be more complicated due to the &regs->sp hack */
416 	BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32));
417 
418 	if (!stack_access_ok(state, addr, sizeof(struct pt_regs)))
419 		return false;
420 
421 	*ip = READ_ONCE_NOCHECK(regs->ip);
422 	*sp = READ_ONCE_NOCHECK(regs->sp);
423 	return true;
424 }
425 
deref_stack_iret_regs(struct unwind_state * state,unsigned long addr,unsigned long * ip,unsigned long * sp)426 static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr,
427 				  unsigned long *ip, unsigned long *sp)
428 {
429 	struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET;
430 
431 	if (!stack_access_ok(state, addr, IRET_FRAME_SIZE))
432 		return false;
433 
434 	*ip = READ_ONCE_NOCHECK(regs->ip);
435 	*sp = READ_ONCE_NOCHECK(regs->sp);
436 	return true;
437 }
438 
439 /*
440  * If state->regs is non-NULL, and points to a full pt_regs, just get the reg
441  * value from state->regs.
442  *
443  * Otherwise, if state->regs just points to IRET regs, and the previous frame
444  * had full regs, it's safe to get the value from the previous regs.  This can
445  * happen when early/late IRQ entry code gets interrupted by an NMI.
446  */
get_reg(struct unwind_state * state,unsigned int reg_off,unsigned long * val)447 static bool get_reg(struct unwind_state *state, unsigned int reg_off,
448 		    unsigned long *val)
449 {
450 	unsigned int reg = reg_off/8;
451 
452 	if (!state->regs)
453 		return false;
454 
455 	if (state->full_regs) {
456 		*val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]);
457 		return true;
458 	}
459 
460 	if (state->prev_regs) {
461 		*val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]);
462 		return true;
463 	}
464 
465 	return false;
466 }
467 
unwind_next_frame(struct unwind_state * state)468 bool unwind_next_frame(struct unwind_state *state)
469 {
470 	unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp;
471 	enum stack_type prev_type = state->stack_info.type;
472 	struct orc_entry *orc;
473 	bool indirect = false;
474 
475 	if (unwind_done(state))
476 		return false;
477 
478 	/* Don't let modules unload while we're reading their ORC data. */
479 	preempt_disable();
480 
481 	/* End-of-stack check for user tasks: */
482 	if (state->regs && user_mode(state->regs))
483 		goto the_end;
484 
485 	/*
486 	 * Find the orc_entry associated with the text address.
487 	 *
488 	 * For a call frame (as opposed to a signal frame), state->ip points to
489 	 * the instruction after the call.  That instruction's stack layout
490 	 * could be different from the call instruction's layout, for example
491 	 * if the call was to a noreturn function.  So get the ORC data for the
492 	 * call instruction itself.
493 	 */
494 	orc = orc_find(state->signal ? state->ip : state->ip - 1);
495 	if (!orc) {
496 		/*
497 		 * As a fallback, try to assume this code uses a frame pointer.
498 		 * This is useful for generated code, like BPF, which ORC
499 		 * doesn't know about.  This is just a guess, so the rest of
500 		 * the unwind is no longer considered reliable.
501 		 */
502 		orc = &orc_fp_entry;
503 		state->error = true;
504 	} else {
505 		if (orc->type == ORC_TYPE_UNDEFINED)
506 			goto err;
507 
508 		if (orc->type == ORC_TYPE_END_OF_STACK)
509 			goto the_end;
510 	}
511 
512 	state->signal = orc->signal;
513 
514 	/* Find the previous frame's stack: */
515 	switch (orc->sp_reg) {
516 	case ORC_REG_SP:
517 		sp = state->sp + orc->sp_offset;
518 		break;
519 
520 	case ORC_REG_BP:
521 		sp = state->bp + orc->sp_offset;
522 		break;
523 
524 	case ORC_REG_SP_INDIRECT:
525 		sp = state->sp;
526 		indirect = true;
527 		break;
528 
529 	case ORC_REG_BP_INDIRECT:
530 		sp = state->bp + orc->sp_offset;
531 		indirect = true;
532 		break;
533 
534 	case ORC_REG_R10:
535 		if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) {
536 			orc_warn_current("missing R10 value at %pB\n",
537 					 (void *)state->ip);
538 			goto err;
539 		}
540 		break;
541 
542 	case ORC_REG_R13:
543 		if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) {
544 			orc_warn_current("missing R13 value at %pB\n",
545 					 (void *)state->ip);
546 			goto err;
547 		}
548 		break;
549 
550 	case ORC_REG_DI:
551 		if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) {
552 			orc_warn_current("missing RDI value at %pB\n",
553 					 (void *)state->ip);
554 			goto err;
555 		}
556 		break;
557 
558 	case ORC_REG_DX:
559 		if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) {
560 			orc_warn_current("missing DX value at %pB\n",
561 					 (void *)state->ip);
562 			goto err;
563 		}
564 		break;
565 
566 	default:
567 		orc_warn("unknown SP base reg %d at %pB\n",
568 			 orc->sp_reg, (void *)state->ip);
569 		goto err;
570 	}
571 
572 	if (indirect) {
573 		if (!deref_stack_reg(state, sp, &sp))
574 			goto err;
575 
576 		if (orc->sp_reg == ORC_REG_SP_INDIRECT)
577 			sp += orc->sp_offset;
578 	}
579 
580 	/* Find IP, SP and possibly regs: */
581 	switch (orc->type) {
582 	case ORC_TYPE_CALL:
583 		ip_p = sp - sizeof(long);
584 
585 		if (!deref_stack_reg(state, ip_p, &state->ip))
586 			goto err;
587 
588 		state->ip = unwind_recover_ret_addr(state, state->ip,
589 						    (unsigned long *)ip_p);
590 		state->sp = sp;
591 		state->regs = NULL;
592 		state->prev_regs = NULL;
593 		break;
594 
595 	case ORC_TYPE_REGS:
596 		if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) {
597 			orc_warn_current("can't access registers at %pB\n",
598 					 (void *)orig_ip);
599 			goto err;
600 		}
601 		/*
602 		 * There is a small chance to interrupt at the entry of
603 		 * arch_rethook_trampoline() where the ORC info doesn't exist.
604 		 * That point is right after the RET to arch_rethook_trampoline()
605 		 * which was modified return address.
606 		 * At that point, the @addr_p of the unwind_recover_rethook()
607 		 * (this has to point the address of the stack entry storing
608 		 * the modified return address) must be "SP - (a stack entry)"
609 		 * because SP is incremented by the RET.
610 		 */
611 		state->ip = unwind_recover_rethook(state, state->ip,
612 				(unsigned long *)(state->sp - sizeof(long)));
613 		state->regs = (struct pt_regs *)sp;
614 		state->prev_regs = NULL;
615 		state->full_regs = true;
616 		break;
617 
618 	case ORC_TYPE_REGS_PARTIAL:
619 		if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) {
620 			orc_warn_current("can't access iret registers at %pB\n",
621 					 (void *)orig_ip);
622 			goto err;
623 		}
624 		/* See ORC_TYPE_REGS case comment. */
625 		state->ip = unwind_recover_rethook(state, state->ip,
626 				(unsigned long *)(state->sp - sizeof(long)));
627 
628 		if (state->full_regs)
629 			state->prev_regs = state->regs;
630 		state->regs = (void *)sp - IRET_FRAME_OFFSET;
631 		state->full_regs = false;
632 		break;
633 
634 	default:
635 		orc_warn("unknown .orc_unwind entry type %d at %pB\n",
636 			 orc->type, (void *)orig_ip);
637 		goto err;
638 	}
639 
640 	/* Find BP: */
641 	switch (orc->bp_reg) {
642 	case ORC_REG_UNDEFINED:
643 		if (get_reg(state, offsetof(struct pt_regs, bp), &tmp))
644 			state->bp = tmp;
645 		break;
646 
647 	case ORC_REG_PREV_SP:
648 		if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp))
649 			goto err;
650 		break;
651 
652 	case ORC_REG_BP:
653 		if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp))
654 			goto err;
655 		break;
656 
657 	default:
658 		orc_warn("unknown BP base reg %d for ip %pB\n",
659 			 orc->bp_reg, (void *)orig_ip);
660 		goto err;
661 	}
662 
663 	/* Prevent a recursive loop due to bad ORC data: */
664 	if (state->stack_info.type == prev_type &&
665 	    on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) &&
666 	    state->sp <= prev_sp) {
667 		orc_warn_current("stack going in the wrong direction? at %pB\n",
668 				 (void *)orig_ip);
669 		goto err;
670 	}
671 
672 	preempt_enable();
673 	return true;
674 
675 err:
676 	state->error = true;
677 
678 the_end:
679 	preempt_enable();
680 	state->stack_info.type = STACK_TYPE_UNKNOWN;
681 	return false;
682 }
683 EXPORT_SYMBOL_GPL(unwind_next_frame);
684 
__unwind_start(struct unwind_state * state,struct task_struct * task,struct pt_regs * regs,unsigned long * first_frame)685 void __unwind_start(struct unwind_state *state, struct task_struct *task,
686 		    struct pt_regs *regs, unsigned long *first_frame)
687 {
688 	memset(state, 0, sizeof(*state));
689 	state->task = task;
690 
691 	if (!orc_init)
692 		goto err;
693 
694 	/*
695 	 * Refuse to unwind the stack of a task while it's executing on another
696 	 * CPU.  This check is racy, but that's ok: the unwinder has other
697 	 * checks to prevent it from going off the rails.
698 	 */
699 	if (task_on_another_cpu(task))
700 		goto err;
701 
702 	if (regs) {
703 		if (user_mode(regs))
704 			goto the_end;
705 
706 		state->ip = regs->ip;
707 		state->sp = regs->sp;
708 		state->bp = regs->bp;
709 		state->regs = regs;
710 		state->full_regs = true;
711 		state->signal = true;
712 
713 	} else if (task == current) {
714 		asm volatile("lea (%%rip), %0\n\t"
715 			     "mov %%rsp, %1\n\t"
716 			     "mov %%rbp, %2\n\t"
717 			     : "=r" (state->ip), "=r" (state->sp),
718 			       "=r" (state->bp));
719 
720 	} else {
721 		struct inactive_task_frame *frame = (void *)task->thread.sp;
722 
723 		state->sp = task->thread.sp + sizeof(*frame);
724 		state->bp = READ_ONCE_NOCHECK(frame->bp);
725 		state->ip = READ_ONCE_NOCHECK(frame->ret_addr);
726 		state->signal = (void *)state->ip == ret_from_fork;
727 	}
728 
729 	if (get_stack_info((unsigned long *)state->sp, state->task,
730 			   &state->stack_info, &state->stack_mask)) {
731 		/*
732 		 * We weren't on a valid stack.  It's possible that
733 		 * we overflowed a valid stack into a guard page.
734 		 * See if the next page up is valid so that we can
735 		 * generate some kind of backtrace if this happens.
736 		 */
737 		void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp);
738 		state->error = true;
739 		if (get_stack_info(next_page, state->task, &state->stack_info,
740 				   &state->stack_mask))
741 			return;
742 	}
743 
744 	/*
745 	 * The caller can provide the address of the first frame directly
746 	 * (first_frame) or indirectly (regs->sp) to indicate which stack frame
747 	 * to start unwinding at.  Skip ahead until we reach it.
748 	 */
749 
750 	/* When starting from regs, skip the regs frame: */
751 	if (regs) {
752 		unwind_next_frame(state);
753 		return;
754 	}
755 
756 	/* Otherwise, skip ahead to the user-specified starting frame: */
757 	while (!unwind_done(state) &&
758 	       (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||
759 			state->sp <= (unsigned long)first_frame))
760 		unwind_next_frame(state);
761 
762 	return;
763 
764 err:
765 	state->error = true;
766 the_end:
767 	state->stack_info.type = STACK_TYPE_UNKNOWN;
768 }
769 EXPORT_SYMBOL_GPL(__unwind_start);
770