1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/kernel.h> 8 #include <linux/init.h> 9 #include <linux/sched.h> 10 #include <linux/sched/debug.h> 11 #include <linux/sched/signal.h> 12 #include <linux/signal.h> 13 #include <linux/kdebug.h> 14 #include <linux/uaccess.h> 15 #include <linux/kprobes.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/irq.h> 19 #include <linux/kexec.h> 20 #include <linux/entry-common.h> 21 22 #include <asm/asm-prototypes.h> 23 #include <asm/bug.h> 24 #include <asm/csr.h> 25 #include <asm/processor.h> 26 #include <asm/ptrace.h> 27 #include <asm/syscall.h> 28 #include <asm/thread_info.h> 29 #include <asm/vector.h> 30 #include <asm/irq_stack.h> 31 32 int show_unhandled_signals = 1; 33 34 static DEFINE_SPINLOCK(die_lock); 35 36 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs) 37 { 38 char str[sizeof("0000 ") * 12 + 2 + 1], *p = str; 39 const u16 *insns = (u16 *)instruction_pointer(regs); 40 long bad; 41 u16 val; 42 int i; 43 44 for (i = -10; i < 2; i++) { 45 bad = get_kernel_nofault(val, &insns[i]); 46 if (!bad) { 47 p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val); 48 } else { 49 printk("%sCode: Unable to access instruction at 0x%px.\n", 50 loglvl, &insns[i]); 51 return; 52 } 53 } 54 printk("%sCode: %s\n", loglvl, str); 55 } 56 57 void die(struct pt_regs *regs, const char *str) 58 { 59 static int die_counter; 60 int ret; 61 long cause; 62 unsigned long flags; 63 64 oops_enter(); 65 66 spin_lock_irqsave(&die_lock, flags); 67 console_verbose(); 68 bust_spinlocks(1); 69 70 pr_emerg("%s [#%d]\n", str, ++die_counter); 71 print_modules(); 72 if (regs) { 73 show_regs(regs); 74 dump_kernel_instr(KERN_EMERG, regs); 75 } 76 77 cause = regs ? regs->cause : -1; 78 ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); 79 80 if (kexec_should_crash(current)) 81 crash_kexec(regs); 82 83 bust_spinlocks(0); 84 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 85 spin_unlock_irqrestore(&die_lock, flags); 86 oops_exit(); 87 88 if (in_interrupt()) 89 panic("Fatal exception in interrupt"); 90 if (panic_on_oops) 91 panic("Fatal exception"); 92 if (ret != NOTIFY_STOP) 93 make_task_dead(SIGSEGV); 94 } 95 96 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) 97 { 98 struct task_struct *tsk = current; 99 100 if (show_unhandled_signals && unhandled_signal(tsk, signo) 101 && printk_ratelimit()) { 102 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT, 103 tsk->comm, task_pid_nr(tsk), signo, code, addr); 104 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); 105 pr_cont("\n"); 106 __show_regs(regs); 107 } 108 109 force_sig_fault(signo, code, (void __user *)addr); 110 } 111 112 static void do_trap_error(struct pt_regs *regs, int signo, int code, 113 unsigned long addr, const char *str) 114 { 115 current->thread.bad_cause = regs->cause; 116 117 if (user_mode(regs)) { 118 do_trap(regs, signo, code, addr); 119 } else { 120 if (!fixup_exception(regs)) 121 die(regs, str); 122 } 123 } 124 125 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) 126 #define __trap_section __noinstr_section(".xip.traps") 127 #else 128 #define __trap_section noinstr 129 #endif 130 #define DO_ERROR_INFO(name, signo, code, str) \ 131 asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ 132 { \ 133 if (user_mode(regs)) { \ 134 irqentry_enter_from_user_mode(regs); \ 135 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ 136 irqentry_exit_to_user_mode(regs); \ 137 } else { \ 138 irqentry_state_t state = irqentry_nmi_enter(regs); \ 139 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ 140 irqentry_nmi_exit(regs, state); \ 141 } \ 142 } 143 144 DO_ERROR_INFO(do_trap_unknown, 145 SIGILL, ILL_ILLTRP, "unknown exception"); 146 DO_ERROR_INFO(do_trap_insn_misaligned, 147 SIGBUS, BUS_ADRALN, "instruction address misaligned"); 148 DO_ERROR_INFO(do_trap_insn_fault, 149 SIGSEGV, SEGV_ACCERR, "instruction access fault"); 150 151 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs) 152 { 153 bool handled; 154 155 if (user_mode(regs)) { 156 irqentry_enter_from_user_mode(regs); 157 158 local_irq_enable(); 159 160 handled = riscv_v_first_use_handler(regs); 161 162 local_irq_disable(); 163 164 if (!handled) 165 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, 166 "Oops - illegal instruction"); 167 168 irqentry_exit_to_user_mode(regs); 169 } else { 170 irqentry_state_t state = irqentry_nmi_enter(regs); 171 172 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, 173 "Oops - illegal instruction"); 174 175 irqentry_nmi_exit(regs, state); 176 } 177 } 178 179 DO_ERROR_INFO(do_trap_load_fault, 180 SIGSEGV, SEGV_ACCERR, "load access fault"); 181 #ifndef CONFIG_RISCV_M_MODE 182 DO_ERROR_INFO(do_trap_load_misaligned, 183 SIGBUS, BUS_ADRALN, "Oops - load address misaligned"); 184 DO_ERROR_INFO(do_trap_store_misaligned, 185 SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned"); 186 #else 187 int handle_misaligned_load(struct pt_regs *regs); 188 int handle_misaligned_store(struct pt_regs *regs); 189 190 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) 191 { 192 if (user_mode(regs)) { 193 irqentry_enter_from_user_mode(regs); 194 195 if (handle_misaligned_load(regs)) 196 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 197 "Oops - load address misaligned"); 198 199 irqentry_exit_to_user_mode(regs); 200 } else { 201 irqentry_state_t state = irqentry_nmi_enter(regs); 202 203 if (handle_misaligned_load(regs)) 204 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 205 "Oops - load address misaligned"); 206 207 irqentry_nmi_exit(regs, state); 208 } 209 } 210 211 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) 212 { 213 if (user_mode(regs)) { 214 irqentry_enter_from_user_mode(regs); 215 216 if (handle_misaligned_store(regs)) 217 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 218 "Oops - store (or AMO) address misaligned"); 219 220 irqentry_exit_to_user_mode(regs); 221 } else { 222 irqentry_state_t state = irqentry_nmi_enter(regs); 223 224 if (handle_misaligned_store(regs)) 225 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 226 "Oops - store (or AMO) address misaligned"); 227 228 irqentry_nmi_exit(regs, state); 229 } 230 } 231 #endif 232 DO_ERROR_INFO(do_trap_store_fault, 233 SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault"); 234 DO_ERROR_INFO(do_trap_ecall_s, 235 SIGILL, ILL_ILLTRP, "environment call from S-mode"); 236 DO_ERROR_INFO(do_trap_ecall_m, 237 SIGILL, ILL_ILLTRP, "environment call from M-mode"); 238 239 static inline unsigned long get_break_insn_length(unsigned long pc) 240 { 241 bug_insn_t insn; 242 243 if (get_kernel_nofault(insn, (bug_insn_t *)pc)) 244 return 0; 245 246 return GET_INSN_LENGTH(insn); 247 } 248 249 void handle_break(struct pt_regs *regs) 250 { 251 #ifdef CONFIG_KPROBES 252 if (kprobe_single_step_handler(regs)) 253 return; 254 255 if (kprobe_breakpoint_handler(regs)) 256 return; 257 #endif 258 #ifdef CONFIG_UPROBES 259 if (uprobe_single_step_handler(regs)) 260 return; 261 262 if (uprobe_breakpoint_handler(regs)) 263 return; 264 #endif 265 current->thread.bad_cause = regs->cause; 266 267 if (user_mode(regs)) 268 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); 269 #ifdef CONFIG_KGDB 270 else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP) 271 == NOTIFY_STOP) 272 return; 273 #endif 274 else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN) 275 regs->epc += get_break_insn_length(regs->epc); 276 else 277 die(regs, "Kernel BUG"); 278 } 279 280 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) 281 { 282 if (user_mode(regs)) { 283 irqentry_enter_from_user_mode(regs); 284 285 handle_break(regs); 286 287 irqentry_exit_to_user_mode(regs); 288 } else { 289 irqentry_state_t state = irqentry_nmi_enter(regs); 290 291 handle_break(regs); 292 293 irqentry_nmi_exit(regs, state); 294 } 295 } 296 297 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs) 298 { 299 if (user_mode(regs)) { 300 ulong syscall = regs->a7; 301 302 regs->epc += 4; 303 regs->orig_a0 = regs->a0; 304 305 riscv_v_vstate_discard(regs); 306 307 syscall = syscall_enter_from_user_mode(regs, syscall); 308 309 if (syscall < NR_syscalls) 310 syscall_handler(regs, syscall); 311 else 312 regs->a0 = -ENOSYS; 313 314 syscall_exit_to_user_mode(regs); 315 } else { 316 irqentry_state_t state = irqentry_nmi_enter(regs); 317 318 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc, 319 "Oops - environment call from U-mode"); 320 321 irqentry_nmi_exit(regs, state); 322 } 323 324 } 325 326 #ifdef CONFIG_MMU 327 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) 328 { 329 irqentry_state_t state = irqentry_enter(regs); 330 331 handle_page_fault(regs); 332 333 local_irq_disable(); 334 335 irqentry_exit(regs, state); 336 } 337 #endif 338 339 static void noinstr handle_riscv_irq(struct pt_regs *regs) 340 { 341 struct pt_regs *old_regs; 342 343 irq_enter_rcu(); 344 old_regs = set_irq_regs(regs); 345 handle_arch_irq(regs); 346 set_irq_regs(old_regs); 347 irq_exit_rcu(); 348 } 349 350 asmlinkage void noinstr do_irq(struct pt_regs *regs) 351 { 352 irqentry_state_t state = irqentry_enter(regs); 353 #ifdef CONFIG_IRQ_STACKS 354 if (on_thread_stack()) { 355 ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) 356 + IRQ_STACK_SIZE/sizeof(ulong); 357 __asm__ __volatile( 358 "addi sp, sp, -"RISCV_SZPTR "\n" 359 REG_S" ra, (sp) \n" 360 "addi sp, sp, -"RISCV_SZPTR "\n" 361 REG_S" s0, (sp) \n" 362 "addi s0, sp, 2*"RISCV_SZPTR "\n" 363 "move sp, %[sp] \n" 364 "move a0, %[regs] \n" 365 "call handle_riscv_irq \n" 366 "addi sp, s0, -2*"RISCV_SZPTR"\n" 367 REG_L" s0, (sp) \n" 368 "addi sp, sp, "RISCV_SZPTR "\n" 369 REG_L" ra, (sp) \n" 370 "addi sp, sp, "RISCV_SZPTR "\n" 371 : 372 : [sp] "r" (sp), [regs] "r" (regs) 373 : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", 374 "t0", "t1", "t2", "t3", "t4", "t5", "t6", 375 "memory"); 376 } else 377 #endif 378 handle_riscv_irq(regs); 379 380 irqentry_exit(regs, state); 381 } 382 383 #ifdef CONFIG_GENERIC_BUG 384 int is_valid_bugaddr(unsigned long pc) 385 { 386 bug_insn_t insn; 387 388 if (pc < VMALLOC_START) 389 return 0; 390 if (get_kernel_nofault(insn, (bug_insn_t *)pc)) 391 return 0; 392 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) 393 return (insn == __BUG_INSN_32); 394 else 395 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16); 396 } 397 #endif /* CONFIG_GENERIC_BUG */ 398 399 #ifdef CONFIG_VMAP_STACK 400 /* 401 * Extra stack space that allows us to provide panic messages when the kernel 402 * has overflowed its stack. 403 */ 404 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], 405 overflow_stack)__aligned(16); 406 /* 407 * A temporary stack for use by handle_kernel_stack_overflow. This is used so 408 * we can call into C code to get the per-hart overflow stack. Usage of this 409 * stack must be protected by spin_shadow_stack. 410 */ 411 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16); 412 413 /* 414 * A pseudo spinlock to protect the shadow stack from being used by multiple 415 * harts concurrently. This isn't a real spinlock because the lock side must 416 * be taken without a valid stack and only a single register, it's only taken 417 * while in the process of panicing anyway so the performance and error 418 * checking a proper spinlock gives us doesn't matter. 419 */ 420 unsigned long spin_shadow_stack; 421 422 asmlinkage unsigned long get_overflow_stack(void) 423 { 424 return (unsigned long)this_cpu_ptr(overflow_stack) + 425 OVERFLOW_STACK_SIZE; 426 } 427 428 asmlinkage void handle_bad_stack(struct pt_regs *regs) 429 { 430 unsigned long tsk_stk = (unsigned long)current->stack; 431 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 432 433 /* 434 * We're done with the shadow stack by this point, as we're on the 435 * overflow stack. Tell any other concurrent overflowing harts that 436 * they can proceed with panicing by releasing the pseudo-spinlock. 437 * 438 * This pairs with an amoswap.aq in handle_kernel_stack_overflow. 439 */ 440 smp_store_release(&spin_shadow_stack, 0); 441 442 console_verbose(); 443 444 pr_emerg("Insufficient stack space to handle exception!\n"); 445 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 446 tsk_stk, tsk_stk + THREAD_SIZE); 447 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 448 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 449 450 __show_regs(regs); 451 panic("Kernel stack overflow"); 452 453 for (;;) 454 wait_for_interrupt(); 455 } 456 #endif 457