1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/kernel.h> 8 #include <linux/init.h> 9 #include <linux/sched.h> 10 #include <linux/sched/debug.h> 11 #include <linux/sched/signal.h> 12 #include <linux/signal.h> 13 #include <linux/kdebug.h> 14 #include <linux/uaccess.h> 15 #include <linux/kprobes.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/irq.h> 19 #include <linux/kexec.h> 20 #include <linux/entry-common.h> 21 22 #include <asm/asm-prototypes.h> 23 #include <asm/bug.h> 24 #include <asm/cfi.h> 25 #include <asm/csr.h> 26 #include <asm/processor.h> 27 #include <asm/ptrace.h> 28 #include <asm/syscall.h> 29 #include <asm/thread_info.h> 30 #include <asm/vector.h> 31 #include <asm/irq_stack.h> 32 33 int show_unhandled_signals = 1; 34 35 static DEFINE_SPINLOCK(die_lock); 36 37 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs) 38 { 39 char str[sizeof("0000 ") * 12 + 2 + 1], *p = str; 40 const u16 *insns = (u16 *)instruction_pointer(regs); 41 long bad; 42 u16 val; 43 int i; 44 45 for (i = -10; i < 2; i++) { 46 bad = get_kernel_nofault(val, &insns[i]); 47 if (!bad) { 48 p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val); 49 } else { 50 printk("%sCode: Unable to access instruction at 0x%px.\n", 51 loglvl, &insns[i]); 52 return; 53 } 54 } 55 printk("%sCode: %s\n", loglvl, str); 56 } 57 58 void die(struct pt_regs *regs, const char *str) 59 { 60 static int die_counter; 61 int ret; 62 long cause; 63 unsigned long flags; 64 65 oops_enter(); 66 67 spin_lock_irqsave(&die_lock, flags); 68 console_verbose(); 69 bust_spinlocks(1); 70 71 pr_emerg("%s [#%d]\n", str, ++die_counter); 72 print_modules(); 73 if (regs) { 74 show_regs(regs); 75 dump_kernel_instr(KERN_EMERG, regs); 76 } 77 78 cause = regs ? regs->cause : -1; 79 ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); 80 81 if (kexec_should_crash(current)) 82 crash_kexec(regs); 83 84 bust_spinlocks(0); 85 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 86 spin_unlock_irqrestore(&die_lock, flags); 87 oops_exit(); 88 89 if (in_interrupt()) 90 panic("Fatal exception in interrupt"); 91 if (panic_on_oops) 92 panic("Fatal exception"); 93 if (ret != NOTIFY_STOP) 94 make_task_dead(SIGSEGV); 95 } 96 97 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) 98 { 99 struct task_struct *tsk = current; 100 101 if (show_unhandled_signals && unhandled_signal(tsk, signo) 102 && printk_ratelimit()) { 103 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT, 104 tsk->comm, task_pid_nr(tsk), signo, code, addr); 105 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); 106 pr_cont("\n"); 107 __show_regs(regs); 108 } 109 110 force_sig_fault(signo, code, (void __user *)addr); 111 } 112 113 static void do_trap_error(struct pt_regs *regs, int signo, int code, 114 unsigned long addr, const char *str) 115 { 116 current->thread.bad_cause = regs->cause; 117 118 if (user_mode(regs)) { 119 do_trap(regs, signo, code, addr); 120 } else { 121 if (!fixup_exception(regs)) 122 die(regs, str); 123 } 124 } 125 126 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) 127 #define __trap_section __noinstr_section(".xip.traps") 128 #else 129 #define __trap_section noinstr 130 #endif 131 #define DO_ERROR_INFO(name, signo, code, str) \ 132 asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ 133 { \ 134 if (user_mode(regs)) { \ 135 irqentry_enter_from_user_mode(regs); \ 136 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ 137 irqentry_exit_to_user_mode(regs); \ 138 } else { \ 139 irqentry_state_t state = irqentry_nmi_enter(regs); \ 140 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ 141 irqentry_nmi_exit(regs, state); \ 142 } \ 143 } 144 145 DO_ERROR_INFO(do_trap_unknown, 146 SIGILL, ILL_ILLTRP, "unknown exception"); 147 DO_ERROR_INFO(do_trap_insn_misaligned, 148 SIGBUS, BUS_ADRALN, "instruction address misaligned"); 149 DO_ERROR_INFO(do_trap_insn_fault, 150 SIGSEGV, SEGV_ACCERR, "instruction access fault"); 151 152 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs) 153 { 154 bool handled; 155 156 if (user_mode(regs)) { 157 irqentry_enter_from_user_mode(regs); 158 159 local_irq_enable(); 160 161 handled = riscv_v_first_use_handler(regs); 162 163 local_irq_disable(); 164 165 if (!handled) 166 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, 167 "Oops - illegal instruction"); 168 169 irqentry_exit_to_user_mode(regs); 170 } else { 171 irqentry_state_t state = irqentry_nmi_enter(regs); 172 173 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, 174 "Oops - illegal instruction"); 175 176 irqentry_nmi_exit(regs, state); 177 } 178 } 179 180 DO_ERROR_INFO(do_trap_load_fault, 181 SIGSEGV, SEGV_ACCERR, "load access fault"); 182 #ifndef CONFIG_RISCV_M_MODE 183 DO_ERROR_INFO(do_trap_load_misaligned, 184 SIGBUS, BUS_ADRALN, "Oops - load address misaligned"); 185 DO_ERROR_INFO(do_trap_store_misaligned, 186 SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned"); 187 #else 188 int handle_misaligned_load(struct pt_regs *regs); 189 int handle_misaligned_store(struct pt_regs *regs); 190 191 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) 192 { 193 if (user_mode(regs)) { 194 irqentry_enter_from_user_mode(regs); 195 196 if (handle_misaligned_load(regs)) 197 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 198 "Oops - load address misaligned"); 199 200 irqentry_exit_to_user_mode(regs); 201 } else { 202 irqentry_state_t state = irqentry_nmi_enter(regs); 203 204 if (handle_misaligned_load(regs)) 205 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 206 "Oops - load address misaligned"); 207 208 irqentry_nmi_exit(regs, state); 209 } 210 } 211 212 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) 213 { 214 if (user_mode(regs)) { 215 irqentry_enter_from_user_mode(regs); 216 217 if (handle_misaligned_store(regs)) 218 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 219 "Oops - store (or AMO) address misaligned"); 220 221 irqentry_exit_to_user_mode(regs); 222 } else { 223 irqentry_state_t state = irqentry_nmi_enter(regs); 224 225 if (handle_misaligned_store(regs)) 226 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, 227 "Oops - store (or AMO) address misaligned"); 228 229 irqentry_nmi_exit(regs, state); 230 } 231 } 232 #endif 233 DO_ERROR_INFO(do_trap_store_fault, 234 SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault"); 235 DO_ERROR_INFO(do_trap_ecall_s, 236 SIGILL, ILL_ILLTRP, "environment call from S-mode"); 237 DO_ERROR_INFO(do_trap_ecall_m, 238 SIGILL, ILL_ILLTRP, "environment call from M-mode"); 239 240 static inline unsigned long get_break_insn_length(unsigned long pc) 241 { 242 bug_insn_t insn; 243 244 if (get_kernel_nofault(insn, (bug_insn_t *)pc)) 245 return 0; 246 247 return GET_INSN_LENGTH(insn); 248 } 249 250 void handle_break(struct pt_regs *regs) 251 { 252 #ifdef CONFIG_KPROBES 253 if (kprobe_single_step_handler(regs)) 254 return; 255 256 if (kprobe_breakpoint_handler(regs)) 257 return; 258 #endif 259 #ifdef CONFIG_UPROBES 260 if (uprobe_single_step_handler(regs)) 261 return; 262 263 if (uprobe_breakpoint_handler(regs)) 264 return; 265 #endif 266 current->thread.bad_cause = regs->cause; 267 268 if (user_mode(regs)) 269 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); 270 #ifdef CONFIG_KGDB 271 else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP) 272 == NOTIFY_STOP) 273 return; 274 #endif 275 else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN || 276 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) 277 regs->epc += get_break_insn_length(regs->epc); 278 else 279 die(regs, "Kernel BUG"); 280 } 281 282 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) 283 { 284 if (user_mode(regs)) { 285 irqentry_enter_from_user_mode(regs); 286 287 handle_break(regs); 288 289 irqentry_exit_to_user_mode(regs); 290 } else { 291 irqentry_state_t state = irqentry_nmi_enter(regs); 292 293 handle_break(regs); 294 295 irqentry_nmi_exit(regs, state); 296 } 297 } 298 299 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs) 300 { 301 if (user_mode(regs)) { 302 long syscall = regs->a7; 303 304 regs->epc += 4; 305 regs->orig_a0 = regs->a0; 306 307 riscv_v_vstate_discard(regs); 308 309 syscall = syscall_enter_from_user_mode(regs, syscall); 310 311 if (syscall >= 0 && syscall < NR_syscalls) 312 syscall_handler(regs, syscall); 313 else if (syscall != -1) 314 regs->a0 = -ENOSYS; 315 316 syscall_exit_to_user_mode(regs); 317 } else { 318 irqentry_state_t state = irqentry_nmi_enter(regs); 319 320 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc, 321 "Oops - environment call from U-mode"); 322 323 irqentry_nmi_exit(regs, state); 324 } 325 326 } 327 328 #ifdef CONFIG_MMU 329 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) 330 { 331 irqentry_state_t state = irqentry_enter(regs); 332 333 handle_page_fault(regs); 334 335 local_irq_disable(); 336 337 irqentry_exit(regs, state); 338 } 339 #endif 340 341 static void noinstr handle_riscv_irq(struct pt_regs *regs) 342 { 343 struct pt_regs *old_regs; 344 345 irq_enter_rcu(); 346 old_regs = set_irq_regs(regs); 347 handle_arch_irq(regs); 348 set_irq_regs(old_regs); 349 irq_exit_rcu(); 350 } 351 352 asmlinkage void noinstr do_irq(struct pt_regs *regs) 353 { 354 irqentry_state_t state = irqentry_enter(regs); 355 #ifdef CONFIG_IRQ_STACKS 356 if (on_thread_stack()) { 357 ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) 358 + IRQ_STACK_SIZE/sizeof(ulong); 359 __asm__ __volatile( 360 "addi sp, sp, -"RISCV_SZPTR "\n" 361 REG_S" ra, (sp) \n" 362 "addi sp, sp, -"RISCV_SZPTR "\n" 363 REG_S" s0, (sp) \n" 364 "addi s0, sp, 2*"RISCV_SZPTR "\n" 365 "move sp, %[sp] \n" 366 "move a0, %[regs] \n" 367 "call handle_riscv_irq \n" 368 "addi sp, s0, -2*"RISCV_SZPTR"\n" 369 REG_L" s0, (sp) \n" 370 "addi sp, sp, "RISCV_SZPTR "\n" 371 REG_L" ra, (sp) \n" 372 "addi sp, sp, "RISCV_SZPTR "\n" 373 : 374 : [sp] "r" (sp), [regs] "r" (regs) 375 : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", 376 "t0", "t1", "t2", "t3", "t4", "t5", "t6", 377 #ifndef CONFIG_FRAME_POINTER 378 "s0", 379 #endif 380 "memory"); 381 } else 382 #endif 383 handle_riscv_irq(regs); 384 385 irqentry_exit(regs, state); 386 } 387 388 #ifdef CONFIG_GENERIC_BUG 389 int is_valid_bugaddr(unsigned long pc) 390 { 391 bug_insn_t insn; 392 393 if (pc < VMALLOC_START) 394 return 0; 395 if (get_kernel_nofault(insn, (bug_insn_t *)pc)) 396 return 0; 397 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) 398 return (insn == __BUG_INSN_32); 399 else 400 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16); 401 } 402 #endif /* CONFIG_GENERIC_BUG */ 403 404 #ifdef CONFIG_VMAP_STACK 405 /* 406 * Extra stack space that allows us to provide panic messages when the kernel 407 * has overflowed its stack. 408 */ 409 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], 410 overflow_stack)__aligned(16); 411 /* 412 * A temporary stack for use by handle_kernel_stack_overflow. This is used so 413 * we can call into C code to get the per-hart overflow stack. Usage of this 414 * stack must be protected by spin_shadow_stack. 415 */ 416 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16); 417 418 /* 419 * A pseudo spinlock to protect the shadow stack from being used by multiple 420 * harts concurrently. This isn't a real spinlock because the lock side must 421 * be taken without a valid stack and only a single register, it's only taken 422 * while in the process of panicing anyway so the performance and error 423 * checking a proper spinlock gives us doesn't matter. 424 */ 425 unsigned long spin_shadow_stack; 426 427 asmlinkage unsigned long get_overflow_stack(void) 428 { 429 return (unsigned long)this_cpu_ptr(overflow_stack) + 430 OVERFLOW_STACK_SIZE; 431 } 432 433 asmlinkage void handle_bad_stack(struct pt_regs *regs) 434 { 435 unsigned long tsk_stk = (unsigned long)current->stack; 436 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 437 438 /* 439 * We're done with the shadow stack by this point, as we're on the 440 * overflow stack. Tell any other concurrent overflowing harts that 441 * they can proceed with panicing by releasing the pseudo-spinlock. 442 * 443 * This pairs with an amoswap.aq in handle_kernel_stack_overflow. 444 */ 445 smp_store_release(&spin_shadow_stack, 0); 446 447 console_verbose(); 448 449 pr_emerg("Insufficient stack space to handle exception!\n"); 450 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 451 tsk_stk, tsk_stk + THREAD_SIZE); 452 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 453 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 454 455 __show_regs(regs); 456 panic("Kernel stack overflow"); 457 458 for (;;) 459 wait_for_interrupt(); 460 } 461 #endif 462