1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/kernel.h> 8 #include <linux/init.h> 9 #include <linux/sched.h> 10 #include <linux/sched/debug.h> 11 #include <linux/sched/signal.h> 12 #include <linux/signal.h> 13 #include <linux/kdebug.h> 14 #include <linux/uaccess.h> 15 #include <linux/kprobes.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/irq.h> 19 #include <linux/kexec.h> 20 #include <linux/entry-common.h> 21 22 #include <asm/asm-prototypes.h> 23 #include <asm/bug.h> 24 #include <asm/csr.h> 25 #include <asm/processor.h> 26 #include <asm/ptrace.h> 27 #include <asm/syscall.h> 28 #include <asm/thread_info.h> 29 #include <asm/vector.h> 30 #include <asm/irq_stack.h> 31 32 int show_unhandled_signals = 1; 33 34 static DEFINE_SPINLOCK(die_lock); 35 36 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs) 37 { 38 char str[sizeof("0000 ") * 12 + 2 + 1], *p = str; 39 const u16 *insns = (u16 *)instruction_pointer(regs); 40 long bad; 41 u16 val; 42 int i; 43 44 for (i = -10; i < 2; i++) { 45 bad = get_kernel_nofault(val, &insns[i]); 46 if (!bad) { 47 p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val); 48 } else { 49 printk("%sCode: Unable to access instruction at 0x%px.\n", 50 loglvl, &insns[i]); 51 return; 52 } 53 } 54 printk("%sCode: %s\n", loglvl, str); 55 } 56 57 void die(struct pt_regs *regs, const char *str) 58 { 59 static int die_counter; 60 int ret; 61 long cause; 62 unsigned long flags; 63 64 oops_enter(); 65 66 spin_lock_irqsave(&die_lock, flags); 67 console_verbose(); 68 bust_spinlocks(1); 69 70 pr_emerg("%s [#%d]\n", str, ++die_counter); 71 print_modules(); 72 if (regs) { 73 show_regs(regs); 74 dump_kernel_instr(KERN_EMERG, regs); 75 } 76 77 cause = regs ? regs->cause : -1; 78 ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); 79 80 if (kexec_should_crash(current)) 81 crash_kexec(regs); 82 83 bust_spinlocks(0); 84 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 85 spin_unlock_irqrestore(&die_lock, flags); 86 oops_exit(); 87 88 if (in_interrupt()) 89 panic("Fatal exception in interrupt"); 90 if (panic_on_oops) 91 panic("Fatal exception"); 92 if (ret != NOTIFY_STOP) 93 make_task_dead(SIGSEGV); 94 } 95 96 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) 97 { 98 struct task_struct *tsk = current; 99 100 if (show_unhandled_signals && unhandled_signal(tsk, signo) 101 && printk_ratelimit()) { 102 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT, 103 tsk->comm, task_pid_nr(tsk), signo, code, addr); 104 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); 105 pr_cont("\n"); 106 __show_regs(regs); 107 } 108 109 force_sig_fault(signo, code, (void __user *)addr); 110 } 111 112 static void do_trap_error(struct pt_regs *regs, int signo, int code, 113 unsigned long addr, const char *str) 114 { 115 current->thread.bad_cause = regs->cause; 116 117 if (user_mode(regs)) { 118 do_trap(regs, signo, code, addr); 119 } else { 120 if (!fixup_exception(regs)) 121 die(regs, str); 122 } 123 } 124 125 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) 126 #define __trap_section __noinstr_section(".xip.traps") 127 #else 128 #define __trap_section noinstr 129 #endif 130 #define DO_ERROR_INFO(name, signo, code, str) \ 131 asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ 132 { \ 133 if (user_mode(regs)) { \ 134 irqentry_enter_from_user_mode(regs); \ 135 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ 136 irqentry_exit_to_user_mode(regs); \ 137 } else { \ 138 irqentry_state_t state = irqentry_nmi_enter(regs); \ 139 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ 140 irqentry_nmi_exit(regs, state); \ 141 } \ 142 } 143 144 DO_ERROR_INFO(do_trap_unknown, 145 SIGILL, ILL_ILLTRP, "unknown exception"); 146 DO_ERROR_INFO(do_trap_insn_misaligned, 147 SIGBUS, BUS_ADRALN, "instruction address misaligned"); 148 DO_ERROR_INFO(do_trap_insn_fault, 149 SIGSEGV, SEGV_ACCERR, "instruction access fault"); 150 151 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs) 152 { 153 if (user_mode(regs)) { 154 irqentry_enter_from_user_mode(regs); 155 156 local_irq_enable(); 157 158 if (!riscv_v_first_use_handler(regs)) 159 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, 160 "Oops - illegal instruction"); 161 162 irqentry_exit_to_user_mode(regs); 163 } else { 164 irqentry_state_t state = irqentry_nmi_enter(regs); 165 166 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, 167 "Oops - illegal instruction"); 168 169 irqentry_nmi_exit(regs, state); 170 } 171 } 172 173 DO_ERROR_INFO(do_trap_load_fault, 174 SIGSEGV, SEGV_ACCERR, "load access fault"); 175 #ifndef CONFIG_RISCV_M_MODE 176 DO_ERROR_INFO(do_trap_load_misaligned, 177 SIGBUS, BUS_ADRALN, "Oops - load address misaligned"); 178 DO_ERROR_INFO(do_trap_store_misaligned, 179 SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned"); 180 #else 181 int handle_misaligned_load(struct pt_regs *regs); 182 int handle_misaligned_store(struct pt_regs *regs); 183 184 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) 185 { 186 if (user_mode(regs)) { 187 irqentry_enter_from_user_mode(regs); 188 189 if (handle_misaligned_load(regs)) 190 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 191 "Oops - load address misaligned"); 192 193 irqentry_exit_to_user_mode(regs); 194 } else { 195 irqentry_state_t state = irqentry_nmi_enter(regs); 196 197 if (handle_misaligned_load(regs)) 198 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 199 "Oops - load address misaligned"); 200 201 irqentry_nmi_exit(regs, state); 202 } 203 } 204 205 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) 206 { 207 if (user_mode(regs)) { 208 irqentry_enter_from_user_mode(regs); 209 210 if (handle_misaligned_store(regs)) 211 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 212 "Oops - store (or AMO) address misaligned"); 213 214 irqentry_exit_to_user_mode(regs); 215 } else { 216 irqentry_state_t state = irqentry_nmi_enter(regs); 217 218 if (handle_misaligned_store(regs)) 219 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 220 "Oops - store (or AMO) address misaligned"); 221 222 irqentry_nmi_exit(regs, state); 223 } 224 } 225 #endif 226 DO_ERROR_INFO(do_trap_store_fault, 227 SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault"); 228 DO_ERROR_INFO(do_trap_ecall_s, 229 SIGILL, ILL_ILLTRP, "environment call from S-mode"); 230 DO_ERROR_INFO(do_trap_ecall_m, 231 SIGILL, ILL_ILLTRP, "environment call from M-mode"); 232 233 static inline unsigned long get_break_insn_length(unsigned long pc) 234 { 235 bug_insn_t insn; 236 237 if (get_kernel_nofault(insn, (bug_insn_t *)pc)) 238 return 0; 239 240 return GET_INSN_LENGTH(insn); 241 } 242 243 void handle_break(struct pt_regs *regs) 244 { 245 #ifdef CONFIG_KPROBES 246 if (kprobe_single_step_handler(regs)) 247 return; 248 249 if (kprobe_breakpoint_handler(regs)) 250 return; 251 #endif 252 #ifdef CONFIG_UPROBES 253 if (uprobe_single_step_handler(regs)) 254 return; 255 256 if (uprobe_breakpoint_handler(regs)) 257 return; 258 #endif 259 current->thread.bad_cause = regs->cause; 260 261 if (user_mode(regs)) 262 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); 263 #ifdef CONFIG_KGDB 264 else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP) 265 == NOTIFY_STOP) 266 return; 267 #endif 268 else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN) 269 regs->epc += get_break_insn_length(regs->epc); 270 else 271 die(regs, "Kernel BUG"); 272 } 273 274 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) 275 { 276 if (user_mode(regs)) { 277 irqentry_enter_from_user_mode(regs); 278 279 handle_break(regs); 280 281 irqentry_exit_to_user_mode(regs); 282 } else { 283 irqentry_state_t state = irqentry_nmi_enter(regs); 284 285 handle_break(regs); 286 287 irqentry_nmi_exit(regs, state); 288 } 289 } 290 291 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs) 292 { 293 if (user_mode(regs)) { 294 ulong syscall = regs->a7; 295 296 regs->epc += 4; 297 regs->orig_a0 = regs->a0; 298 299 syscall = syscall_enter_from_user_mode(regs, syscall); 300 301 if (syscall < NR_syscalls) 302 syscall_handler(regs, syscall); 303 else 304 regs->a0 = -ENOSYS; 305 306 syscall_exit_to_user_mode(regs); 307 } else { 308 irqentry_state_t state = irqentry_nmi_enter(regs); 309 310 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc, 311 "Oops - environment call from U-mode"); 312 313 irqentry_nmi_exit(regs, state); 314 } 315 316 } 317 318 #ifdef CONFIG_MMU 319 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) 320 { 321 irqentry_state_t state = irqentry_enter(regs); 322 323 handle_page_fault(regs); 324 325 local_irq_disable(); 326 327 irqentry_exit(regs, state); 328 } 329 #endif 330 331 static void noinstr handle_riscv_irq(struct pt_regs *regs) 332 { 333 struct pt_regs *old_regs; 334 335 irq_enter_rcu(); 336 old_regs = set_irq_regs(regs); 337 handle_arch_irq(regs); 338 set_irq_regs(old_regs); 339 irq_exit_rcu(); 340 } 341 342 asmlinkage void noinstr do_irq(struct pt_regs *regs) 343 { 344 irqentry_state_t state = irqentry_enter(regs); 345 #ifdef CONFIG_IRQ_STACKS 346 if (on_thread_stack()) { 347 ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) 348 + IRQ_STACK_SIZE/sizeof(ulong); 349 __asm__ __volatile( 350 "addi sp, sp, -"RISCV_SZPTR "\n" 351 REG_S" ra, (sp) \n" 352 "addi sp, sp, -"RISCV_SZPTR "\n" 353 REG_S" s0, (sp) \n" 354 "addi s0, sp, 2*"RISCV_SZPTR "\n" 355 "move sp, %[sp] \n" 356 "move a0, %[regs] \n" 357 "call handle_riscv_irq \n" 358 "addi sp, s0, -2*"RISCV_SZPTR"\n" 359 REG_L" s0, (sp) \n" 360 "addi sp, sp, "RISCV_SZPTR "\n" 361 REG_L" ra, (sp) \n" 362 "addi sp, sp, "RISCV_SZPTR "\n" 363 : 364 : [sp] "r" (sp), [regs] "r" (regs) 365 : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", 366 "t0", "t1", "t2", "t3", "t4", "t5", "t6", 367 "memory"); 368 } else 369 #endif 370 handle_riscv_irq(regs); 371 372 irqentry_exit(regs, state); 373 } 374 375 #ifdef CONFIG_GENERIC_BUG 376 int is_valid_bugaddr(unsigned long pc) 377 { 378 bug_insn_t insn; 379 380 if (pc < VMALLOC_START) 381 return 0; 382 if (get_kernel_nofault(insn, (bug_insn_t *)pc)) 383 return 0; 384 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) 385 return (insn == __BUG_INSN_32); 386 else 387 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16); 388 } 389 #endif /* CONFIG_GENERIC_BUG */ 390 391 #ifdef CONFIG_VMAP_STACK 392 /* 393 * Extra stack space that allows us to provide panic messages when the kernel 394 * has overflowed its stack. 395 */ 396 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], 397 overflow_stack)__aligned(16); 398 /* 399 * A temporary stack for use by handle_kernel_stack_overflow. This is used so 400 * we can call into C code to get the per-hart overflow stack. Usage of this 401 * stack must be protected by spin_shadow_stack. 402 */ 403 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16); 404 405 /* 406 * A pseudo spinlock to protect the shadow stack from being used by multiple 407 * harts concurrently. This isn't a real spinlock because the lock side must 408 * be taken without a valid stack and only a single register, it's only taken 409 * while in the process of panicing anyway so the performance and error 410 * checking a proper spinlock gives us doesn't matter. 411 */ 412 unsigned long spin_shadow_stack; 413 414 asmlinkage unsigned long get_overflow_stack(void) 415 { 416 return (unsigned long)this_cpu_ptr(overflow_stack) + 417 OVERFLOW_STACK_SIZE; 418 } 419 420 asmlinkage void handle_bad_stack(struct pt_regs *regs) 421 { 422 unsigned long tsk_stk = (unsigned long)current->stack; 423 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 424 425 /* 426 * We're done with the shadow stack by this point, as we're on the 427 * overflow stack. Tell any other concurrent overflowing harts that 428 * they can proceed with panicing by releasing the pseudo-spinlock. 429 * 430 * This pairs with an amoswap.aq in handle_kernel_stack_overflow. 431 */ 432 smp_store_release(&spin_shadow_stack, 0); 433 434 console_verbose(); 435 436 pr_emerg("Insufficient stack space to handle exception!\n"); 437 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 438 tsk_stk, tsk_stk + THREAD_SIZE); 439 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 440 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 441 442 __show_regs(regs); 443 panic("Kernel stack overflow"); 444 445 for (;;) 446 wait_for_interrupt(); 447 } 448 #endif 449