17c7900f8SJosh Poimboeuf #include <linux/sched.h> 27c7900f8SJosh Poimboeuf #include <asm/ptrace.h> 37c7900f8SJosh Poimboeuf #include <asm/bitops.h> 47c7900f8SJosh Poimboeuf #include <asm/stacktrace.h> 57c7900f8SJosh Poimboeuf #include <asm/unwind.h> 67c7900f8SJosh Poimboeuf 77c7900f8SJosh Poimboeuf #define FRAME_HEADER_SIZE (sizeof(long) * 2) 87c7900f8SJosh Poimboeuf 98b5e99f0SJosh Poimboeuf static void unwind_dump(struct unwind_state *state, unsigned long *sp) 108b5e99f0SJosh Poimboeuf { 118b5e99f0SJosh Poimboeuf static bool dumped_before = false; 128b5e99f0SJosh Poimboeuf bool prev_zero, zero = false; 138b5e99f0SJosh Poimboeuf unsigned long word; 148b5e99f0SJosh Poimboeuf 158b5e99f0SJosh Poimboeuf if (dumped_before) 168b5e99f0SJosh Poimboeuf return; 178b5e99f0SJosh Poimboeuf 188b5e99f0SJosh Poimboeuf dumped_before = true; 198b5e99f0SJosh Poimboeuf 208b5e99f0SJosh Poimboeuf printk_deferred("unwind stack type:%d next_sp:%p mask:%lx graph_idx:%d\n", 218b5e99f0SJosh Poimboeuf state->stack_info.type, state->stack_info.next_sp, 228b5e99f0SJosh Poimboeuf state->stack_mask, state->graph_idx); 238b5e99f0SJosh Poimboeuf 248b5e99f0SJosh Poimboeuf for (sp = state->orig_sp; sp < state->stack_info.end; sp++) { 258b5e99f0SJosh Poimboeuf word = READ_ONCE_NOCHECK(*sp); 268b5e99f0SJosh Poimboeuf 278b5e99f0SJosh Poimboeuf prev_zero = zero; 288b5e99f0SJosh Poimboeuf zero = word == 0; 298b5e99f0SJosh Poimboeuf 308b5e99f0SJosh Poimboeuf if (zero) { 318b5e99f0SJosh Poimboeuf if (!prev_zero) 328b5e99f0SJosh Poimboeuf printk_deferred("%p: %016x ...\n", sp, 0); 338b5e99f0SJosh Poimboeuf continue; 348b5e99f0SJosh Poimboeuf } 358b5e99f0SJosh Poimboeuf 368b5e99f0SJosh Poimboeuf printk_deferred("%p: %016lx (%pB)\n", sp, word, (void *)word); 378b5e99f0SJosh Poimboeuf } 388b5e99f0SJosh Poimboeuf } 398b5e99f0SJosh Poimboeuf 407c7900f8SJosh Poimboeuf unsigned long unwind_get_return_address(struct unwind_state *state) 417c7900f8SJosh Poimboeuf { 427c7900f8SJosh Poimboeuf unsigned long addr; 437c7900f8SJosh Poimboeuf unsigned long *addr_p = unwind_get_return_address_ptr(state); 447c7900f8SJosh Poimboeuf 457c7900f8SJosh Poimboeuf if (unwind_done(state)) 467c7900f8SJosh Poimboeuf return 0; 477c7900f8SJosh Poimboeuf 48946c1911SJosh Poimboeuf if (state->regs && user_mode(state->regs)) 49946c1911SJosh Poimboeuf return 0; 50946c1911SJosh Poimboeuf 517c7900f8SJosh Poimboeuf addr = ftrace_graph_ret_addr(state->task, &state->graph_idx, *addr_p, 527c7900f8SJosh Poimboeuf addr_p); 537c7900f8SJosh Poimboeuf 54*c280f773SJosh Poimboeuf return __kernel_text_address(addr) ? addr : 0; 557c7900f8SJosh Poimboeuf } 567c7900f8SJosh Poimboeuf EXPORT_SYMBOL_GPL(unwind_get_return_address); 577c7900f8SJosh Poimboeuf 5824d86f59SJosh Poimboeuf static size_t regs_size(struct pt_regs *regs) 5924d86f59SJosh Poimboeuf { 6024d86f59SJosh Poimboeuf /* x86_32 regs from kernel mode are two words shorter: */ 6124d86f59SJosh Poimboeuf if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs)) 6224d86f59SJosh Poimboeuf return sizeof(*regs) - 2*sizeof(long); 6324d86f59SJosh Poimboeuf 6424d86f59SJosh Poimboeuf return sizeof(*regs); 6524d86f59SJosh Poimboeuf } 6624d86f59SJosh Poimboeuf 67acb4608aSJosh Poimboeuf static bool is_last_task_frame(struct unwind_state *state) 68acb4608aSJosh Poimboeuf { 69acb4608aSJosh Poimboeuf unsigned long bp = (unsigned long)state->bp; 70acb4608aSJosh Poimboeuf unsigned long regs = (unsigned long)task_pt_regs(state->task); 71acb4608aSJosh Poimboeuf 728023e0e2SJosh Poimboeuf /* 738023e0e2SJosh Poimboeuf * We have to check for the last task frame at two different locations 748023e0e2SJosh Poimboeuf * because gcc can occasionally decide to realign the stack pointer and 758023e0e2SJosh Poimboeuf * change the offset of the stack frame by a word in the prologue of a 768023e0e2SJosh Poimboeuf * function called by head/entry code. 778023e0e2SJosh Poimboeuf */ 788023e0e2SJosh Poimboeuf return bp == regs - FRAME_HEADER_SIZE || 798023e0e2SJosh Poimboeuf bp == regs - FRAME_HEADER_SIZE - sizeof(long); 80acb4608aSJosh Poimboeuf } 81acb4608aSJosh Poimboeuf 82946c1911SJosh Poimboeuf /* 83946c1911SJosh Poimboeuf * This determines if the frame pointer actually contains an encoded pointer to 84946c1911SJosh Poimboeuf * pt_regs on the stack. See ENCODE_FRAME_POINTER. 85946c1911SJosh Poimboeuf */ 86946c1911SJosh Poimboeuf static struct pt_regs *decode_frame_pointer(unsigned long *bp) 87946c1911SJosh Poimboeuf { 88946c1911SJosh Poimboeuf unsigned long regs = (unsigned long)bp; 89946c1911SJosh Poimboeuf 90946c1911SJosh Poimboeuf if (!(regs & 0x1)) 91946c1911SJosh Poimboeuf return NULL; 92946c1911SJosh Poimboeuf 93946c1911SJosh Poimboeuf return (struct pt_regs *)(regs & ~0x1); 94946c1911SJosh Poimboeuf } 95946c1911SJosh Poimboeuf 967c7900f8SJosh Poimboeuf static bool update_stack_state(struct unwind_state *state, void *addr, 977c7900f8SJosh Poimboeuf size_t len) 987c7900f8SJosh Poimboeuf { 997c7900f8SJosh Poimboeuf struct stack_info *info = &state->stack_info; 1008b5e99f0SJosh Poimboeuf enum stack_type orig_type = info->type; 1017c7900f8SJosh Poimboeuf 1027c7900f8SJosh Poimboeuf /* 1037c7900f8SJosh Poimboeuf * If addr isn't on the current stack, switch to the next one. 1047c7900f8SJosh Poimboeuf * 1057c7900f8SJosh Poimboeuf * We may have to traverse multiple stacks to deal with the possibility 1067c7900f8SJosh Poimboeuf * that 'info->next_sp' could point to an empty stack and 'addr' could 1077c7900f8SJosh Poimboeuf * be on a subsequent stack. 1087c7900f8SJosh Poimboeuf */ 1097c7900f8SJosh Poimboeuf while (!on_stack(info, addr, len)) 1107c7900f8SJosh Poimboeuf if (get_stack_info(info->next_sp, state->task, info, 1117c7900f8SJosh Poimboeuf &state->stack_mask)) 1127c7900f8SJosh Poimboeuf return false; 1137c7900f8SJosh Poimboeuf 1148b5e99f0SJosh Poimboeuf if (!state->orig_sp || info->type != orig_type) 1158b5e99f0SJosh Poimboeuf state->orig_sp = addr; 1168b5e99f0SJosh Poimboeuf 1177c7900f8SJosh Poimboeuf return true; 1187c7900f8SJosh Poimboeuf } 1197c7900f8SJosh Poimboeuf 1207c7900f8SJosh Poimboeuf bool unwind_next_frame(struct unwind_state *state) 1217c7900f8SJosh Poimboeuf { 122946c1911SJosh Poimboeuf struct pt_regs *regs; 123946c1911SJosh Poimboeuf unsigned long *next_bp, *next_frame; 124946c1911SJosh Poimboeuf size_t next_len; 12524d86f59SJosh Poimboeuf enum stack_type prev_type = state->stack_info.type; 1267c7900f8SJosh Poimboeuf 1277c7900f8SJosh Poimboeuf if (unwind_done(state)) 1287c7900f8SJosh Poimboeuf return false; 1297c7900f8SJosh Poimboeuf 130946c1911SJosh Poimboeuf /* have we reached the end? */ 131946c1911SJosh Poimboeuf if (state->regs && user_mode(state->regs)) 132946c1911SJosh Poimboeuf goto the_end; 133946c1911SJosh Poimboeuf 134acb4608aSJosh Poimboeuf if (is_last_task_frame(state)) { 135acb4608aSJosh Poimboeuf regs = task_pt_regs(state->task); 136acb4608aSJosh Poimboeuf 137acb4608aSJosh Poimboeuf /* 138acb4608aSJosh Poimboeuf * kthreads (other than the boot CPU's idle thread) have some 139acb4608aSJosh Poimboeuf * partial regs at the end of their stack which were placed 140acb4608aSJosh Poimboeuf * there by copy_thread_tls(). But the regs don't have any 141acb4608aSJosh Poimboeuf * useful information, so we can skip them. 142acb4608aSJosh Poimboeuf * 143acb4608aSJosh Poimboeuf * This user_mode() check is slightly broader than a PF_KTHREAD 144acb4608aSJosh Poimboeuf * check because it also catches the awkward situation where a 145acb4608aSJosh Poimboeuf * newly forked kthread transitions into a user task by calling 146acb4608aSJosh Poimboeuf * do_execve(), which eventually clears PF_KTHREAD. 147acb4608aSJosh Poimboeuf */ 148acb4608aSJosh Poimboeuf if (!user_mode(regs)) 149acb4608aSJosh Poimboeuf goto the_end; 150acb4608aSJosh Poimboeuf 151acb4608aSJosh Poimboeuf /* 152acb4608aSJosh Poimboeuf * We're almost at the end, but not quite: there's still the 153acb4608aSJosh Poimboeuf * syscall regs frame. Entry code doesn't encode the regs 154acb4608aSJosh Poimboeuf * pointer for syscalls, so we have to set it manually. 155acb4608aSJosh Poimboeuf */ 156acb4608aSJosh Poimboeuf state->regs = regs; 157acb4608aSJosh Poimboeuf state->bp = NULL; 158acb4608aSJosh Poimboeuf return true; 159acb4608aSJosh Poimboeuf } 160acb4608aSJosh Poimboeuf 161946c1911SJosh Poimboeuf /* get the next frame pointer */ 162946c1911SJosh Poimboeuf if (state->regs) 163946c1911SJosh Poimboeuf next_bp = (unsigned long *)state->regs->bp; 164946c1911SJosh Poimboeuf else 1657c7900f8SJosh Poimboeuf next_bp = (unsigned long *)*state->bp; 1667c7900f8SJosh Poimboeuf 167946c1911SJosh Poimboeuf /* is the next frame pointer an encoded pointer to pt_regs? */ 168946c1911SJosh Poimboeuf regs = decode_frame_pointer(next_bp); 169946c1911SJosh Poimboeuf if (regs) { 170946c1911SJosh Poimboeuf next_frame = (unsigned long *)regs; 171946c1911SJosh Poimboeuf next_len = sizeof(*regs); 172946c1911SJosh Poimboeuf } else { 173946c1911SJosh Poimboeuf next_frame = next_bp; 174946c1911SJosh Poimboeuf next_len = FRAME_HEADER_SIZE; 175946c1911SJosh Poimboeuf } 1767c7900f8SJosh Poimboeuf 177946c1911SJosh Poimboeuf /* make sure the next frame's data is accessible */ 178c32c47c6SJosh Poimboeuf if (!update_stack_state(state, next_frame, next_len)) { 179c32c47c6SJosh Poimboeuf /* 180c32c47c6SJosh Poimboeuf * Don't warn on bad regs->bp. An interrupt in entry code 181c32c47c6SJosh Poimboeuf * might cause a false positive warning. 182c32c47c6SJosh Poimboeuf */ 183c32c47c6SJosh Poimboeuf if (state->regs) 184c32c47c6SJosh Poimboeuf goto the_end; 185c32c47c6SJosh Poimboeuf 186c32c47c6SJosh Poimboeuf goto bad_address; 187c32c47c6SJosh Poimboeuf } 188c32c47c6SJosh Poimboeuf 18924d86f59SJosh Poimboeuf /* Make sure it only unwinds up and doesn't overlap the last frame: */ 19024d86f59SJosh Poimboeuf if (state->stack_info.type == prev_type) { 19124d86f59SJosh Poimboeuf if (state->regs && (void *)next_frame < (void *)state->regs + regs_size(state->regs)) 19224d86f59SJosh Poimboeuf goto bad_address; 19324d86f59SJosh Poimboeuf 19424d86f59SJosh Poimboeuf if (state->bp && (void *)next_frame < (void *)state->bp + FRAME_HEADER_SIZE) 19524d86f59SJosh Poimboeuf goto bad_address; 19624d86f59SJosh Poimboeuf } 19724d86f59SJosh Poimboeuf 1987c7900f8SJosh Poimboeuf /* move to the next frame */ 199946c1911SJosh Poimboeuf if (regs) { 200946c1911SJosh Poimboeuf state->regs = regs; 201946c1911SJosh Poimboeuf state->bp = NULL; 202946c1911SJosh Poimboeuf } else { 2037c7900f8SJosh Poimboeuf state->bp = next_bp; 204946c1911SJosh Poimboeuf state->regs = NULL; 205946c1911SJosh Poimboeuf } 206946c1911SJosh Poimboeuf 2077c7900f8SJosh Poimboeuf return true; 208946c1911SJosh Poimboeuf 209c32c47c6SJosh Poimboeuf bad_address: 21024d86f59SJosh Poimboeuf if (state->regs) { 21124d86f59SJosh Poimboeuf printk_deferred_once(KERN_WARNING 21224d86f59SJosh Poimboeuf "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n", 21324d86f59SJosh Poimboeuf state->regs, state->task->comm, 21424d86f59SJosh Poimboeuf state->task->pid, next_frame); 2158b5e99f0SJosh Poimboeuf unwind_dump(state, (unsigned long *)state->regs); 21624d86f59SJosh Poimboeuf } else { 217c32c47c6SJosh Poimboeuf printk_deferred_once(KERN_WARNING 218c32c47c6SJosh Poimboeuf "WARNING: kernel stack frame pointer at %p in %s:%d has bad value %p\n", 219c32c47c6SJosh Poimboeuf state->bp, state->task->comm, 22024d86f59SJosh Poimboeuf state->task->pid, next_frame); 2218b5e99f0SJosh Poimboeuf unwind_dump(state, state->bp); 22224d86f59SJosh Poimboeuf } 223946c1911SJosh Poimboeuf the_end: 224946c1911SJosh Poimboeuf state->stack_info.type = STACK_TYPE_UNKNOWN; 225946c1911SJosh Poimboeuf return false; 2267c7900f8SJosh Poimboeuf } 2277c7900f8SJosh Poimboeuf EXPORT_SYMBOL_GPL(unwind_next_frame); 2287c7900f8SJosh Poimboeuf 2297c7900f8SJosh Poimboeuf void __unwind_start(struct unwind_state *state, struct task_struct *task, 2307c7900f8SJosh Poimboeuf struct pt_regs *regs, unsigned long *first_frame) 2317c7900f8SJosh Poimboeuf { 232946c1911SJosh Poimboeuf unsigned long *bp, *frame; 233946c1911SJosh Poimboeuf size_t len; 234946c1911SJosh Poimboeuf 2357c7900f8SJosh Poimboeuf memset(state, 0, sizeof(*state)); 2367c7900f8SJosh Poimboeuf state->task = task; 2377c7900f8SJosh Poimboeuf 2387c7900f8SJosh Poimboeuf /* don't even attempt to start from user mode regs */ 2397c7900f8SJosh Poimboeuf if (regs && user_mode(regs)) { 2407c7900f8SJosh Poimboeuf state->stack_info.type = STACK_TYPE_UNKNOWN; 2417c7900f8SJosh Poimboeuf return; 2427c7900f8SJosh Poimboeuf } 2437c7900f8SJosh Poimboeuf 2447c7900f8SJosh Poimboeuf /* set up the starting stack frame */ 245946c1911SJosh Poimboeuf bp = get_frame_pointer(task, regs); 246946c1911SJosh Poimboeuf regs = decode_frame_pointer(bp); 247946c1911SJosh Poimboeuf if (regs) { 248946c1911SJosh Poimboeuf state->regs = regs; 249946c1911SJosh Poimboeuf frame = (unsigned long *)regs; 250946c1911SJosh Poimboeuf len = sizeof(*regs); 251946c1911SJosh Poimboeuf } else { 252946c1911SJosh Poimboeuf state->bp = bp; 253946c1911SJosh Poimboeuf frame = bp; 254946c1911SJosh Poimboeuf len = FRAME_HEADER_SIZE; 255946c1911SJosh Poimboeuf } 2567c7900f8SJosh Poimboeuf 2577c7900f8SJosh Poimboeuf /* initialize stack info and make sure the frame data is accessible */ 258946c1911SJosh Poimboeuf get_stack_info(frame, state->task, &state->stack_info, 2597c7900f8SJosh Poimboeuf &state->stack_mask); 260946c1911SJosh Poimboeuf update_stack_state(state, frame, len); 2617c7900f8SJosh Poimboeuf 2627c7900f8SJosh Poimboeuf /* 2637c7900f8SJosh Poimboeuf * The caller can provide the address of the first frame directly 2647c7900f8SJosh Poimboeuf * (first_frame) or indirectly (regs->sp) to indicate which stack frame 2657c7900f8SJosh Poimboeuf * to start unwinding at. Skip ahead until we reach it. 2667c7900f8SJosh Poimboeuf */ 2677c7900f8SJosh Poimboeuf while (!unwind_done(state) && 2687c7900f8SJosh Poimboeuf (!on_stack(&state->stack_info, first_frame, sizeof(long)) || 2697c7900f8SJosh Poimboeuf state->bp < first_frame)) 2707c7900f8SJosh Poimboeuf unwind_next_frame(state); 2717c7900f8SJosh Poimboeuf } 2727c7900f8SJosh Poimboeuf EXPORT_SYMBOL_GPL(__unwind_start); 273