xref: /openbmc/linux/arch/riscv/kernel/traps.c (revision 6db6b729) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Regents of the University of California
4  */
5 
6 #include <linux/cpu.h>
7 #include <linux/kernel.h>
8 #include <linux/init.h>
9 #include <linux/sched.h>
10 #include <linux/sched/debug.h>
11 #include <linux/sched/signal.h>
12 #include <linux/signal.h>
13 #include <linux/kdebug.h>
14 #include <linux/uaccess.h>
15 #include <linux/kprobes.h>
16 #include <linux/uprobes.h>
17 #include <asm/uprobes.h>
18 #include <linux/mm.h>
19 #include <linux/module.h>
20 #include <linux/irq.h>
21 #include <linux/kexec.h>
22 #include <linux/entry-common.h>
23 
24 #include <asm/asm-prototypes.h>
25 #include <asm/bug.h>
26 #include <asm/cfi.h>
27 #include <asm/csr.h>
28 #include <asm/processor.h>
29 #include <asm/ptrace.h>
30 #include <asm/syscall.h>
31 #include <asm/thread_info.h>
32 #include <asm/vector.h>
33 #include <asm/irq_stack.h>
34 
35 int show_unhandled_signals = 1;
36 
37 static DEFINE_SPINLOCK(die_lock);
38 
39 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs)
40 {
41 	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
42 	const u16 *insns = (u16 *)instruction_pointer(regs);
43 	long bad;
44 	u16 val;
45 	int i;
46 
47 	for (i = -10; i < 2; i++) {
48 		bad = get_kernel_nofault(val, &insns[i]);
49 		if (!bad) {
50 			p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
51 		} else {
52 			printk("%sCode: Unable to access instruction at 0x%px.\n",
53 			       loglvl, &insns[i]);
54 			return;
55 		}
56 	}
57 	printk("%sCode: %s\n", loglvl, str);
58 }
59 
60 void die(struct pt_regs *regs, const char *str)
61 {
62 	static int die_counter;
63 	int ret;
64 	long cause;
65 	unsigned long flags;
66 
67 	oops_enter();
68 
69 	spin_lock_irqsave(&die_lock, flags);
70 	console_verbose();
71 	bust_spinlocks(1);
72 
73 	pr_emerg("%s [#%d]\n", str, ++die_counter);
74 	print_modules();
75 	if (regs) {
76 		show_regs(regs);
77 		dump_kernel_instr(KERN_EMERG, regs);
78 	}
79 
80 	cause = regs ? regs->cause : -1;
81 	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
82 
83 	if (kexec_should_crash(current))
84 		crash_kexec(regs);
85 
86 	bust_spinlocks(0);
87 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
88 	spin_unlock_irqrestore(&die_lock, flags);
89 	oops_exit();
90 
91 	if (in_interrupt())
92 		panic("Fatal exception in interrupt");
93 	if (panic_on_oops)
94 		panic("Fatal exception");
95 	if (ret != NOTIFY_STOP)
96 		make_task_dead(SIGSEGV);
97 }
98 
99 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
100 {
101 	struct task_struct *tsk = current;
102 
103 	if (show_unhandled_signals && unhandled_signal(tsk, signo)
104 	    && printk_ratelimit()) {
105 		pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
106 			tsk->comm, task_pid_nr(tsk), signo, code, addr);
107 		print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
108 		pr_cont("\n");
109 		__show_regs(regs);
110 	}
111 
112 	force_sig_fault(signo, code, (void __user *)addr);
113 }
114 
115 static void do_trap_error(struct pt_regs *regs, int signo, int code,
116 	unsigned long addr, const char *str)
117 {
118 	current->thread.bad_cause = regs->cause;
119 
120 	if (user_mode(regs)) {
121 		do_trap(regs, signo, code, addr);
122 	} else {
123 		if (!fixup_exception(regs))
124 			die(regs, str);
125 	}
126 }
127 
128 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
129 #define __trap_section __noinstr_section(".xip.traps")
130 #else
131 #define __trap_section noinstr
132 #endif
133 #define DO_ERROR_INFO(name, signo, code, str)					\
134 asmlinkage __visible __trap_section void name(struct pt_regs *regs)		\
135 {										\
136 	if (user_mode(regs)) {							\
137 		irqentry_enter_from_user_mode(regs);				\
138 		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
139 		irqentry_exit_to_user_mode(regs);				\
140 	} else {								\
141 		irqentry_state_t state = irqentry_nmi_enter(regs);		\
142 		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
143 		irqentry_nmi_exit(regs, state);					\
144 	}									\
145 }
146 
147 DO_ERROR_INFO(do_trap_unknown,
148 	SIGILL, ILL_ILLTRP, "unknown exception");
149 DO_ERROR_INFO(do_trap_insn_misaligned,
150 	SIGBUS, BUS_ADRALN, "instruction address misaligned");
151 DO_ERROR_INFO(do_trap_insn_fault,
152 	SIGSEGV, SEGV_ACCERR, "instruction access fault");
153 
154 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
155 {
156 	bool handled;
157 
158 	if (user_mode(regs)) {
159 		irqentry_enter_from_user_mode(regs);
160 
161 		local_irq_enable();
162 
163 		handled = riscv_v_first_use_handler(regs);
164 
165 		local_irq_disable();
166 
167 		if (!handled)
168 			do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
169 				      "Oops - illegal instruction");
170 
171 		irqentry_exit_to_user_mode(regs);
172 	} else {
173 		irqentry_state_t state = irqentry_nmi_enter(regs);
174 
175 		do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
176 			      "Oops - illegal instruction");
177 
178 		irqentry_nmi_exit(regs, state);
179 	}
180 }
181 
182 DO_ERROR_INFO(do_trap_load_fault,
183 	SIGSEGV, SEGV_ACCERR, "load access fault");
184 #ifndef CONFIG_RISCV_M_MODE
185 DO_ERROR_INFO(do_trap_load_misaligned,
186 	SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
187 DO_ERROR_INFO(do_trap_store_misaligned,
188 	SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned");
189 #else
190 int handle_misaligned_load(struct pt_regs *regs);
191 int handle_misaligned_store(struct pt_regs *regs);
192 
193 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
194 {
195 	if (user_mode(regs)) {
196 		irqentry_enter_from_user_mode(regs);
197 
198 		if (handle_misaligned_load(regs))
199 			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
200 			      "Oops - load address misaligned");
201 
202 		irqentry_exit_to_user_mode(regs);
203 	} else {
204 		irqentry_state_t state = irqentry_nmi_enter(regs);
205 
206 		if (handle_misaligned_load(regs))
207 			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
208 			      "Oops - load address misaligned");
209 
210 		irqentry_nmi_exit(regs, state);
211 	}
212 }
213 
214 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
215 {
216 	if (user_mode(regs)) {
217 		irqentry_enter_from_user_mode(regs);
218 
219 		if (handle_misaligned_store(regs))
220 			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
221 				"Oops - store (or AMO) address misaligned");
222 
223 		irqentry_exit_to_user_mode(regs);
224 	} else {
225 		irqentry_state_t state = irqentry_nmi_enter(regs);
226 
227 		if (handle_misaligned_store(regs))
228 			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
229 				"Oops - store (or AMO) address misaligned");
230 
231 		irqentry_nmi_exit(regs, state);
232 	}
233 }
234 #endif
235 DO_ERROR_INFO(do_trap_store_fault,
236 	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
237 DO_ERROR_INFO(do_trap_ecall_s,
238 	SIGILL, ILL_ILLTRP, "environment call from S-mode");
239 DO_ERROR_INFO(do_trap_ecall_m,
240 	SIGILL, ILL_ILLTRP, "environment call from M-mode");
241 
242 static inline unsigned long get_break_insn_length(unsigned long pc)
243 {
244 	bug_insn_t insn;
245 
246 	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
247 		return 0;
248 
249 	return GET_INSN_LENGTH(insn);
250 }
251 
252 static bool probe_single_step_handler(struct pt_regs *regs)
253 {
254 	bool user = user_mode(regs);
255 
256 	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
257 }
258 
259 static bool probe_breakpoint_handler(struct pt_regs *regs)
260 {
261 	bool user = user_mode(regs);
262 
263 	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
264 }
265 
266 void handle_break(struct pt_regs *regs)
267 {
268 	if (probe_single_step_handler(regs))
269 		return;
270 
271 	if (probe_breakpoint_handler(regs))
272 		return;
273 
274 	current->thread.bad_cause = regs->cause;
275 
276 	if (user_mode(regs))
277 		force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
278 #ifdef CONFIG_KGDB
279 	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
280 								== NOTIFY_STOP)
281 		return;
282 #endif
283 	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
284 		 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
285 		regs->epc += get_break_insn_length(regs->epc);
286 	else
287 		die(regs, "Kernel BUG");
288 }
289 
290 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
291 {
292 	if (user_mode(regs)) {
293 		irqentry_enter_from_user_mode(regs);
294 
295 		handle_break(regs);
296 
297 		irqentry_exit_to_user_mode(regs);
298 	} else {
299 		irqentry_state_t state = irqentry_nmi_enter(regs);
300 
301 		handle_break(regs);
302 
303 		irqentry_nmi_exit(regs, state);
304 	}
305 }
306 
307 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
308 {
309 	if (user_mode(regs)) {
310 		long syscall = regs->a7;
311 
312 		regs->epc += 4;
313 		regs->orig_a0 = regs->a0;
314 
315 		riscv_v_vstate_discard(regs);
316 
317 		syscall = syscall_enter_from_user_mode(regs, syscall);
318 
319 		if (syscall >= 0 && syscall < NR_syscalls)
320 			syscall_handler(regs, syscall);
321 		else if (syscall != -1)
322 			regs->a0 = -ENOSYS;
323 
324 		syscall_exit_to_user_mode(regs);
325 	} else {
326 		irqentry_state_t state = irqentry_nmi_enter(regs);
327 
328 		do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
329 			"Oops - environment call from U-mode");
330 
331 		irqentry_nmi_exit(regs, state);
332 	}
333 
334 }
335 
336 #ifdef CONFIG_MMU
337 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
338 {
339 	irqentry_state_t state = irqentry_enter(regs);
340 
341 	handle_page_fault(regs);
342 
343 	local_irq_disable();
344 
345 	irqentry_exit(regs, state);
346 }
347 #endif
348 
349 static void noinstr handle_riscv_irq(struct pt_regs *regs)
350 {
351 	struct pt_regs *old_regs;
352 
353 	irq_enter_rcu();
354 	old_regs = set_irq_regs(regs);
355 	handle_arch_irq(regs);
356 	set_irq_regs(old_regs);
357 	irq_exit_rcu();
358 }
359 
360 asmlinkage void noinstr do_irq(struct pt_regs *regs)
361 {
362 	irqentry_state_t state = irqentry_enter(regs);
363 #ifdef CONFIG_IRQ_STACKS
364 	if (on_thread_stack()) {
365 		ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
366 					+ IRQ_STACK_SIZE/sizeof(ulong);
367 		__asm__ __volatile(
368 		"addi	sp, sp, -"RISCV_SZPTR  "\n"
369 		REG_S"  ra, (sp)		\n"
370 		"addi	sp, sp, -"RISCV_SZPTR  "\n"
371 		REG_S"  s0, (sp)		\n"
372 		"addi	s0, sp, 2*"RISCV_SZPTR "\n"
373 		"move	sp, %[sp]		\n"
374 		"move	a0, %[regs]		\n"
375 		"call	handle_riscv_irq	\n"
376 		"addi	sp, s0, -2*"RISCV_SZPTR"\n"
377 		REG_L"  s0, (sp)		\n"
378 		"addi	sp, sp, "RISCV_SZPTR   "\n"
379 		REG_L"  ra, (sp)		\n"
380 		"addi	sp, sp, "RISCV_SZPTR   "\n"
381 		:
382 		: [sp] "r" (sp), [regs] "r" (regs)
383 		: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
384 		  "t0", "t1", "t2", "t3", "t4", "t5", "t6",
385 #ifndef CONFIG_FRAME_POINTER
386 		  "s0",
387 #endif
388 		  "memory");
389 	} else
390 #endif
391 		handle_riscv_irq(regs);
392 
393 	irqentry_exit(regs, state);
394 }
395 
396 #ifdef CONFIG_GENERIC_BUG
397 int is_valid_bugaddr(unsigned long pc)
398 {
399 	bug_insn_t insn;
400 
401 	if (pc < VMALLOC_START)
402 		return 0;
403 	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
404 		return 0;
405 	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
406 		return (insn == __BUG_INSN_32);
407 	else
408 		return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
409 }
410 #endif /* CONFIG_GENERIC_BUG */
411 
412 #ifdef CONFIG_VMAP_STACK
413 /*
414  * Extra stack space that allows us to provide panic messages when the kernel
415  * has overflowed its stack.
416  */
417 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
418 		overflow_stack)__aligned(16);
419 /*
420  * A temporary stack for use by handle_kernel_stack_overflow.  This is used so
421  * we can call into C code to get the per-hart overflow stack.  Usage of this
422  * stack must be protected by spin_shadow_stack.
423  */
424 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
425 
426 /*
427  * A pseudo spinlock to protect the shadow stack from being used by multiple
428  * harts concurrently.  This isn't a real spinlock because the lock side must
429  * be taken without a valid stack and only a single register, it's only taken
430  * while in the process of panicing anyway so the performance and error
431  * checking a proper spinlock gives us doesn't matter.
432  */
433 unsigned long spin_shadow_stack;
434 
435 asmlinkage unsigned long get_overflow_stack(void)
436 {
437 	return (unsigned long)this_cpu_ptr(overflow_stack) +
438 		OVERFLOW_STACK_SIZE;
439 }
440 
441 asmlinkage void handle_bad_stack(struct pt_regs *regs)
442 {
443 	unsigned long tsk_stk = (unsigned long)current->stack;
444 	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
445 
446 	/*
447 	 * We're done with the shadow stack by this point, as we're on the
448 	 * overflow stack.  Tell any other concurrent overflowing harts that
449 	 * they can proceed with panicing by releasing the pseudo-spinlock.
450 	 *
451 	 * This pairs with an amoswap.aq in handle_kernel_stack_overflow.
452 	 */
453 	smp_store_release(&spin_shadow_stack, 0);
454 
455 	console_verbose();
456 
457 	pr_emerg("Insufficient stack space to handle exception!\n");
458 	pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
459 			tsk_stk, tsk_stk + THREAD_SIZE);
460 	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
461 			ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
462 
463 	__show_regs(regs);
464 	panic("Kernel stack overflow");
465 
466 	for (;;)
467 		wait_for_interrupt();
468 }
469 #endif
470