1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/kernel/traps.c 4 * 5 * Copyright (C) 1995-2009 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9 #include <linux/bug.h> 10 #include <linux/context_tracking.h> 11 #include <linux/signal.h> 12 #include <linux/personality.h> 13 #include <linux/kallsyms.h> 14 #include <linux/kprobes.h> 15 #include <linux/spinlock.h> 16 #include <linux/uaccess.h> 17 #include <linux/hardirq.h> 18 #include <linux/kdebug.h> 19 #include <linux/module.h> 20 #include <linux/kexec.h> 21 #include <linux/delay.h> 22 #include <linux/init.h> 23 #include <linux/sched/signal.h> 24 #include <linux/sched/debug.h> 25 #include <linux/sched/task_stack.h> 26 #include <linux/sizes.h> 27 #include <linux/syscalls.h> 28 #include <linux/mm_types.h> 29 #include <linux/kasan.h> 30 31 #include <asm/atomic.h> 32 #include <asm/bug.h> 33 #include <asm/cpufeature.h> 34 #include <asm/daifflags.h> 35 #include <asm/debug-monitors.h> 36 #include <asm/esr.h> 37 #include <asm/insn.h> 38 #include <asm/kprobes.h> 39 #include <asm/traps.h> 40 #include <asm/smp.h> 41 #include <asm/stack_pointer.h> 42 #include <asm/stacktrace.h> 43 #include <asm/exception.h> 44 #include <asm/system_misc.h> 45 #include <asm/sysreg.h> 46 47 static const char *handler[]= { 48 "Synchronous Abort", 49 "IRQ", 50 "FIQ", 51 "Error" 52 }; 53 54 int show_unhandled_signals = 0; 55 56 static void dump_backtrace_entry(unsigned long where, const char *loglvl) 57 { 58 printk("%s %pS\n", loglvl, (void *)where); 59 } 60 61 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs) 62 { 63 unsigned long addr = instruction_pointer(regs); 64 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; 65 int i; 66 67 if (user_mode(regs)) 68 return; 69 70 for (i = -4; i < 1; i++) { 71 unsigned int val, bad; 72 73 bad = aarch64_insn_read(&((u32 *)addr)[i], &val); 74 75 if (!bad) 76 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); 77 else { 78 p += sprintf(p, "bad PC value"); 79 break; 80 } 81 } 82 83 printk("%sCode: %s\n", lvl, str); 84 } 85 86 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, 87 const char *loglvl) 88 { 89 struct stackframe frame; 90 int skip = 0; 91 92 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); 93 94 if (regs) { 95 if (user_mode(regs)) 96 return; 97 skip = 1; 98 } 99 100 if (!tsk) 101 tsk = current; 102 103 if (!try_get_task_stack(tsk)) 104 return; 105 106 if (tsk == current) { 107 start_backtrace(&frame, 108 (unsigned long)__builtin_frame_address(0), 109 (unsigned long)dump_backtrace); 110 } else { 111 /* 112 * task blocked in __switch_to 113 */ 114 start_backtrace(&frame, 115 thread_saved_fp(tsk), 116 thread_saved_pc(tsk)); 117 } 118 119 printk("%sCall trace:\n", loglvl); 120 do { 121 /* skip until specified stack frame */ 122 if (!skip) { 123 dump_backtrace_entry(frame.pc, loglvl); 124 } else if (frame.fp == regs->regs[29]) { 125 skip = 0; 126 /* 127 * Mostly, this is the case where this function is 128 * called in panic/abort. As exception handler's 129 * stack frame does not contain the corresponding pc 130 * at which an exception has taken place, use regs->pc 131 * instead. 132 */ 133 dump_backtrace_entry(regs->pc, loglvl); 134 } 135 } while (!unwind_frame(tsk, &frame)); 136 137 put_task_stack(tsk); 138 } 139 140 void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) 141 { 142 dump_backtrace(NULL, tsk, loglvl); 143 barrier(); 144 } 145 146 #ifdef CONFIG_PREEMPT 147 #define S_PREEMPT " PREEMPT" 148 #elif defined(CONFIG_PREEMPT_RT) 149 #define S_PREEMPT " PREEMPT_RT" 150 #else 151 #define S_PREEMPT "" 152 #endif 153 154 #define S_SMP " SMP" 155 156 static int __die(const char *str, int err, struct pt_regs *regs) 157 { 158 static int die_counter; 159 int ret; 160 161 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", 162 str, err, ++die_counter); 163 164 /* trap and error numbers are mostly meaningless on ARM */ 165 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV); 166 if (ret == NOTIFY_STOP) 167 return ret; 168 169 print_modules(); 170 show_regs(regs); 171 172 dump_kernel_instr(KERN_EMERG, regs); 173 174 return ret; 175 } 176 177 static DEFINE_RAW_SPINLOCK(die_lock); 178 179 /* 180 * This function is protected against re-entrancy. 181 */ 182 void die(const char *str, struct pt_regs *regs, int err) 183 { 184 int ret; 185 unsigned long flags; 186 187 raw_spin_lock_irqsave(&die_lock, flags); 188 189 oops_enter(); 190 191 console_verbose(); 192 bust_spinlocks(1); 193 ret = __die(str, err, regs); 194 195 if (regs && kexec_should_crash(current)) 196 crash_kexec(regs); 197 198 bust_spinlocks(0); 199 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 200 oops_exit(); 201 202 if (in_interrupt()) 203 panic("Fatal exception in interrupt"); 204 if (panic_on_oops) 205 panic("Fatal exception"); 206 207 raw_spin_unlock_irqrestore(&die_lock, flags); 208 209 if (ret != NOTIFY_STOP) 210 do_exit(SIGSEGV); 211 } 212 213 static void arm64_show_signal(int signo, const char *str) 214 { 215 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 216 DEFAULT_RATELIMIT_BURST); 217 struct task_struct *tsk = current; 218 unsigned int esr = tsk->thread.fault_code; 219 struct pt_regs *regs = task_pt_regs(tsk); 220 221 /* Leave if the signal won't be shown */ 222 if (!show_unhandled_signals || 223 !unhandled_signal(tsk, signo) || 224 !__ratelimit(&rs)) 225 return; 226 227 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk)); 228 if (esr) 229 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr); 230 231 pr_cont("%s", str); 232 print_vma_addr(KERN_CONT " in ", regs->pc); 233 pr_cont("\n"); 234 __show_regs(regs); 235 } 236 237 void arm64_force_sig_fault(int signo, int code, void __user *addr, 238 const char *str) 239 { 240 arm64_show_signal(signo, str); 241 if (signo == SIGKILL) 242 force_sig(SIGKILL); 243 else 244 force_sig_fault(signo, code, addr); 245 } 246 247 void arm64_force_sig_mceerr(int code, void __user *addr, short lsb, 248 const char *str) 249 { 250 arm64_show_signal(SIGBUS, str); 251 force_sig_mceerr(code, addr, lsb); 252 } 253 254 void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr, 255 const char *str) 256 { 257 arm64_show_signal(SIGTRAP, str); 258 force_sig_ptrace_errno_trap(errno, addr); 259 } 260 261 void arm64_notify_die(const char *str, struct pt_regs *regs, 262 int signo, int sicode, void __user *addr, 263 int err) 264 { 265 if (user_mode(regs)) { 266 WARN_ON(regs != current_pt_regs()); 267 current->thread.fault_address = 0; 268 current->thread.fault_code = err; 269 270 arm64_force_sig_fault(signo, sicode, addr, str); 271 } else { 272 die(str, regs, err); 273 } 274 } 275 276 #ifdef CONFIG_COMPAT 277 #define PSTATE_IT_1_0_SHIFT 25 278 #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT) 279 #define PSTATE_IT_7_2_SHIFT 10 280 #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT) 281 282 static u32 compat_get_it_state(struct pt_regs *regs) 283 { 284 u32 it, pstate = regs->pstate; 285 286 it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT; 287 it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2; 288 289 return it; 290 } 291 292 static void compat_set_it_state(struct pt_regs *regs, u32 it) 293 { 294 u32 pstate_it; 295 296 pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK; 297 pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK; 298 299 regs->pstate &= ~PSR_AA32_IT_MASK; 300 regs->pstate |= pstate_it; 301 } 302 303 static void advance_itstate(struct pt_regs *regs) 304 { 305 u32 it; 306 307 /* ARM mode */ 308 if (!(regs->pstate & PSR_AA32_T_BIT) || 309 !(regs->pstate & PSR_AA32_IT_MASK)) 310 return; 311 312 it = compat_get_it_state(regs); 313 314 /* 315 * If this is the last instruction of the block, wipe the IT 316 * state. Otherwise advance it. 317 */ 318 if (!(it & 7)) 319 it = 0; 320 else 321 it = (it & 0xe0) | ((it << 1) & 0x1f); 322 323 compat_set_it_state(regs, it); 324 } 325 #else 326 static void advance_itstate(struct pt_regs *regs) 327 { 328 } 329 #endif 330 331 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) 332 { 333 regs->pc += size; 334 335 /* 336 * If we were single stepping, we want to get the step exception after 337 * we return from the trap. 338 */ 339 if (user_mode(regs)) 340 user_fastforward_single_step(current); 341 342 if (compat_user_mode(regs)) 343 advance_itstate(regs); 344 else 345 regs->pstate &= ~PSR_BTYPE_MASK; 346 } 347 348 static LIST_HEAD(undef_hook); 349 static DEFINE_RAW_SPINLOCK(undef_lock); 350 351 void register_undef_hook(struct undef_hook *hook) 352 { 353 unsigned long flags; 354 355 raw_spin_lock_irqsave(&undef_lock, flags); 356 list_add(&hook->node, &undef_hook); 357 raw_spin_unlock_irqrestore(&undef_lock, flags); 358 } 359 360 void unregister_undef_hook(struct undef_hook *hook) 361 { 362 unsigned long flags; 363 364 raw_spin_lock_irqsave(&undef_lock, flags); 365 list_del(&hook->node); 366 raw_spin_unlock_irqrestore(&undef_lock, flags); 367 } 368 369 static int call_undef_hook(struct pt_regs *regs) 370 { 371 struct undef_hook *hook; 372 unsigned long flags; 373 u32 instr; 374 int (*fn)(struct pt_regs *regs, u32 instr) = NULL; 375 void __user *pc = (void __user *)instruction_pointer(regs); 376 377 if (!user_mode(regs)) { 378 __le32 instr_le; 379 if (get_kernel_nofault(instr_le, (__force __le32 *)pc)) 380 goto exit; 381 instr = le32_to_cpu(instr_le); 382 } else if (compat_thumb_mode(regs)) { 383 /* 16-bit Thumb instruction */ 384 __le16 instr_le; 385 if (get_user(instr_le, (__le16 __user *)pc)) 386 goto exit; 387 instr = le16_to_cpu(instr_le); 388 if (aarch32_insn_is_wide(instr)) { 389 u32 instr2; 390 391 if (get_user(instr_le, (__le16 __user *)(pc + 2))) 392 goto exit; 393 instr2 = le16_to_cpu(instr_le); 394 instr = (instr << 16) | instr2; 395 } 396 } else { 397 /* 32-bit ARM instruction */ 398 __le32 instr_le; 399 if (get_user(instr_le, (__le32 __user *)pc)) 400 goto exit; 401 instr = le32_to_cpu(instr_le); 402 } 403 404 raw_spin_lock_irqsave(&undef_lock, flags); 405 list_for_each_entry(hook, &undef_hook, node) 406 if ((instr & hook->instr_mask) == hook->instr_val && 407 (regs->pstate & hook->pstate_mask) == hook->pstate_val) 408 fn = hook->fn; 409 410 raw_spin_unlock_irqrestore(&undef_lock, flags); 411 exit: 412 return fn ? fn(regs, instr) : 1; 413 } 414 415 void force_signal_inject(int signal, int code, unsigned long address) 416 { 417 const char *desc; 418 struct pt_regs *regs = current_pt_regs(); 419 420 if (WARN_ON(!user_mode(regs))) 421 return; 422 423 switch (signal) { 424 case SIGILL: 425 desc = "undefined instruction"; 426 break; 427 case SIGSEGV: 428 desc = "illegal memory access"; 429 break; 430 default: 431 desc = "unknown or unrecoverable error"; 432 break; 433 } 434 435 /* Force signals we don't understand to SIGKILL */ 436 if (WARN_ON(signal != SIGKILL && 437 siginfo_layout(signal, code) != SIL_FAULT)) { 438 signal = SIGKILL; 439 } 440 441 arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0); 442 } 443 444 /* 445 * Set up process info to signal segmentation fault - called on access error. 446 */ 447 void arm64_notify_segfault(unsigned long addr) 448 { 449 int code; 450 451 mmap_read_lock(current->mm); 452 if (find_vma(current->mm, addr) == NULL) 453 code = SEGV_MAPERR; 454 else 455 code = SEGV_ACCERR; 456 mmap_read_unlock(current->mm); 457 458 force_signal_inject(SIGSEGV, code, addr); 459 } 460 461 void do_undefinstr(struct pt_regs *regs) 462 { 463 /* check for AArch32 breakpoint instructions */ 464 if (!aarch32_break_handler(regs)) 465 return; 466 467 if (call_undef_hook(regs) == 0) 468 return; 469 470 BUG_ON(!user_mode(regs)); 471 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 472 } 473 NOKPROBE_SYMBOL(do_undefinstr); 474 475 void do_bti(struct pt_regs *regs) 476 { 477 BUG_ON(!user_mode(regs)); 478 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 479 } 480 NOKPROBE_SYMBOL(do_bti); 481 482 #define __user_cache_maint(insn, address, res) \ 483 if (address >= user_addr_max()) { \ 484 res = -EFAULT; \ 485 } else { \ 486 uaccess_ttbr0_enable(); \ 487 asm volatile ( \ 488 "1: " insn ", %1\n" \ 489 " mov %w0, #0\n" \ 490 "2:\n" \ 491 " .pushsection .fixup,\"ax\"\n" \ 492 " .align 2\n" \ 493 "3: mov %w0, %w2\n" \ 494 " b 2b\n" \ 495 " .popsection\n" \ 496 _ASM_EXTABLE(1b, 3b) \ 497 : "=r" (res) \ 498 : "r" (address), "i" (-EFAULT)); \ 499 uaccess_ttbr0_disable(); \ 500 } 501 502 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) 503 { 504 unsigned long address; 505 int rt = ESR_ELx_SYS64_ISS_RT(esr); 506 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 507 int ret = 0; 508 509 address = untagged_addr(pt_regs_read_reg(regs, rt)); 510 511 switch (crm) { 512 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */ 513 __user_cache_maint("dc civac", address, ret); 514 break; 515 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ 516 __user_cache_maint("dc civac", address, ret); 517 break; 518 case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */ 519 __user_cache_maint("sys 3, c7, c13, 1", address, ret); 520 break; 521 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ 522 __user_cache_maint("sys 3, c7, c12, 1", address, ret); 523 break; 524 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ 525 __user_cache_maint("dc civac", address, ret); 526 break; 527 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */ 528 __user_cache_maint("ic ivau", address, ret); 529 break; 530 default: 531 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 532 return; 533 } 534 535 if (ret) 536 arm64_notify_segfault(address); 537 else 538 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 539 } 540 541 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) 542 { 543 int rt = ESR_ELx_SYS64_ISS_RT(esr); 544 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0); 545 546 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) { 547 /* Hide DIC so that we can trap the unnecessary maintenance...*/ 548 val &= ~BIT(CTR_DIC_SHIFT); 549 550 /* ... and fake IminLine to reduce the number of traps. */ 551 val &= ~CTR_IMINLINE_MASK; 552 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK; 553 } 554 555 pt_regs_write_reg(regs, rt, val); 556 557 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 558 } 559 560 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) 561 { 562 int rt = ESR_ELx_SYS64_ISS_RT(esr); 563 564 pt_regs_write_reg(regs, rt, arch_timer_read_counter()); 565 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 566 } 567 568 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) 569 { 570 int rt = ESR_ELx_SYS64_ISS_RT(esr); 571 572 pt_regs_write_reg(regs, rt, arch_timer_get_rate()); 573 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 574 } 575 576 static void mrs_handler(unsigned int esr, struct pt_regs *regs) 577 { 578 u32 sysreg, rt; 579 580 rt = ESR_ELx_SYS64_ISS_RT(esr); 581 sysreg = esr_sys64_to_sysreg(esr); 582 583 if (do_emulate_mrs(regs, sysreg, rt) != 0) 584 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 585 } 586 587 static void wfi_handler(unsigned int esr, struct pt_regs *regs) 588 { 589 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 590 } 591 592 struct sys64_hook { 593 unsigned int esr_mask; 594 unsigned int esr_val; 595 void (*handler)(unsigned int esr, struct pt_regs *regs); 596 }; 597 598 static const struct sys64_hook sys64_hooks[] = { 599 { 600 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK, 601 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL, 602 .handler = user_cache_maint_handler, 603 }, 604 { 605 /* Trap read access to CTR_EL0 */ 606 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 607 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ, 608 .handler = ctr_read_handler, 609 }, 610 { 611 /* Trap read access to CNTVCT_EL0 */ 612 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 613 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, 614 .handler = cntvct_read_handler, 615 }, 616 { 617 /* Trap read access to CNTFRQ_EL0 */ 618 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 619 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ, 620 .handler = cntfrq_read_handler, 621 }, 622 { 623 /* Trap read access to CPUID registers */ 624 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK, 625 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL, 626 .handler = mrs_handler, 627 }, 628 { 629 /* Trap WFI instructions executed in userspace */ 630 .esr_mask = ESR_ELx_WFx_MASK, 631 .esr_val = ESR_ELx_WFx_WFI_VAL, 632 .handler = wfi_handler, 633 }, 634 {}, 635 }; 636 637 #ifdef CONFIG_COMPAT 638 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs) 639 { 640 int cond; 641 642 /* Only a T32 instruction can trap without CV being set */ 643 if (!(esr & ESR_ELx_CV)) { 644 u32 it; 645 646 it = compat_get_it_state(regs); 647 if (!it) 648 return true; 649 650 cond = it >> 4; 651 } else { 652 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; 653 } 654 655 return aarch32_opcode_cond_checks[cond](regs->pstate); 656 } 657 658 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) 659 { 660 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT; 661 662 pt_regs_write_reg(regs, reg, arch_timer_get_rate()); 663 arm64_skip_faulting_instruction(regs, 4); 664 } 665 666 static const struct sys64_hook cp15_32_hooks[] = { 667 { 668 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK, 669 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ, 670 .handler = compat_cntfrq_read_handler, 671 }, 672 {}, 673 }; 674 675 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs) 676 { 677 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; 678 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; 679 u64 val = arch_timer_read_counter(); 680 681 pt_regs_write_reg(regs, rt, lower_32_bits(val)); 682 pt_regs_write_reg(regs, rt2, upper_32_bits(val)); 683 arm64_skip_faulting_instruction(regs, 4); 684 } 685 686 static const struct sys64_hook cp15_64_hooks[] = { 687 { 688 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, 689 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, 690 .handler = compat_cntvct_read_handler, 691 }, 692 {}, 693 }; 694 695 void do_cp15instr(unsigned int esr, struct pt_regs *regs) 696 { 697 const struct sys64_hook *hook, *hook_base; 698 699 if (!cp15_cond_valid(esr, regs)) { 700 /* 701 * There is no T16 variant of a CP access, so we 702 * always advance PC by 4 bytes. 703 */ 704 arm64_skip_faulting_instruction(regs, 4); 705 return; 706 } 707 708 switch (ESR_ELx_EC(esr)) { 709 case ESR_ELx_EC_CP15_32: 710 hook_base = cp15_32_hooks; 711 break; 712 case ESR_ELx_EC_CP15_64: 713 hook_base = cp15_64_hooks; 714 break; 715 default: 716 do_undefinstr(regs); 717 return; 718 } 719 720 for (hook = hook_base; hook->handler; hook++) 721 if ((hook->esr_mask & esr) == hook->esr_val) { 722 hook->handler(esr, regs); 723 return; 724 } 725 726 /* 727 * New cp15 instructions may previously have been undefined at 728 * EL0. Fall back to our usual undefined instruction handler 729 * so that we handle these consistently. 730 */ 731 do_undefinstr(regs); 732 } 733 NOKPROBE_SYMBOL(do_cp15instr); 734 #endif 735 736 void do_sysinstr(unsigned int esr, struct pt_regs *regs) 737 { 738 const struct sys64_hook *hook; 739 740 for (hook = sys64_hooks; hook->handler; hook++) 741 if ((hook->esr_mask & esr) == hook->esr_val) { 742 hook->handler(esr, regs); 743 return; 744 } 745 746 /* 747 * New SYS instructions may previously have been undefined at EL0. Fall 748 * back to our usual undefined instruction handler so that we handle 749 * these consistently. 750 */ 751 do_undefinstr(regs); 752 } 753 NOKPROBE_SYMBOL(do_sysinstr); 754 755 static const char *esr_class_str[] = { 756 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", 757 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", 758 [ESR_ELx_EC_WFx] = "WFI/WFE", 759 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", 760 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", 761 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", 762 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", 763 [ESR_ELx_EC_FP_ASIMD] = "ASIMD", 764 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", 765 [ESR_ELx_EC_PAC] = "PAC", 766 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", 767 [ESR_ELx_EC_BTI] = "BTI", 768 [ESR_ELx_EC_ILL] = "PSTATE.IL", 769 [ESR_ELx_EC_SVC32] = "SVC (AArch32)", 770 [ESR_ELx_EC_HVC32] = "HVC (AArch32)", 771 [ESR_ELx_EC_SMC32] = "SMC (AArch32)", 772 [ESR_ELx_EC_SVC64] = "SVC (AArch64)", 773 [ESR_ELx_EC_HVC64] = "HVC (AArch64)", 774 [ESR_ELx_EC_SMC64] = "SMC (AArch64)", 775 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", 776 [ESR_ELx_EC_SVE] = "SVE", 777 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB", 778 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", 779 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", 780 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", 781 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", 782 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", 783 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", 784 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", 785 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", 786 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", 787 [ESR_ELx_EC_SERROR] = "SError", 788 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", 789 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", 790 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", 791 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", 792 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", 793 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", 794 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", 795 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", 796 [ESR_ELx_EC_BRK64] = "BRK (AArch64)", 797 }; 798 799 const char *esr_get_class_string(u32 esr) 800 { 801 return esr_class_str[ESR_ELx_EC(esr)]; 802 } 803 804 /* 805 * bad_mode handles the impossible case in the exception vector. This is always 806 * fatal. 807 */ 808 asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) 809 { 810 console_verbose(); 811 812 pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", 813 handler[reason], smp_processor_id(), esr, 814 esr_get_class_string(esr)); 815 816 __show_regs(regs); 817 local_daif_mask(); 818 panic("bad mode"); 819 } 820 821 /* 822 * bad_el0_sync handles unexpected, but potentially recoverable synchronous 823 * exceptions taken from EL0. Unlike bad_mode, this returns. 824 */ 825 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) 826 { 827 void __user *pc = (void __user *)instruction_pointer(regs); 828 829 current->thread.fault_address = 0; 830 current->thread.fault_code = esr; 831 832 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc, 833 "Bad EL0 synchronous exception"); 834 } 835 836 #ifdef CONFIG_VMAP_STACK 837 838 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) 839 __aligned(16); 840 841 asmlinkage void handle_bad_stack(struct pt_regs *regs) 842 { 843 unsigned long tsk_stk = (unsigned long)current->stack; 844 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); 845 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 846 unsigned int esr = read_sysreg(esr_el1); 847 unsigned long far = read_sysreg(far_el1); 848 849 console_verbose(); 850 pr_emerg("Insufficient stack space to handle exception!"); 851 852 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr)); 853 pr_emerg("FAR: 0x%016lx\n", far); 854 855 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 856 tsk_stk, tsk_stk + THREAD_SIZE); 857 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", 858 irq_stk, irq_stk + THREAD_SIZE); 859 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 860 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 861 862 __show_regs(regs); 863 864 /* 865 * We use nmi_panic to limit the potential for recusive overflows, and 866 * to get a better stack trace. 867 */ 868 nmi_panic(NULL, "kernel stack overflow"); 869 cpu_park_loop(); 870 } 871 #endif 872 873 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr) 874 { 875 console_verbose(); 876 877 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n", 878 smp_processor_id(), esr, esr_get_class_string(esr)); 879 if (regs) 880 __show_regs(regs); 881 882 nmi_panic(regs, "Asynchronous SError Interrupt"); 883 884 cpu_park_loop(); 885 unreachable(); 886 } 887 888 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) 889 { 890 u32 aet = arm64_ras_serror_get_severity(esr); 891 892 switch (aet) { 893 case ESR_ELx_AET_CE: /* corrected error */ 894 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */ 895 /* 896 * The CPU can make progress. We may take UEO again as 897 * a more severe error. 898 */ 899 return false; 900 901 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */ 902 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */ 903 /* 904 * The CPU can't make progress. The exception may have 905 * been imprecise. 906 * 907 * Neoverse-N1 #1349291 means a non-KVM SError reported as 908 * Unrecoverable should be treated as Uncontainable. We 909 * call arm64_serror_panic() in both cases. 910 */ 911 return true; 912 913 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */ 914 default: 915 /* Error has been silently propagated */ 916 arm64_serror_panic(regs, esr); 917 } 918 } 919 920 asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr) 921 { 922 nmi_enter(); 923 924 /* non-RAS errors are not containable */ 925 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) 926 arm64_serror_panic(regs, esr); 927 928 nmi_exit(); 929 } 930 931 asmlinkage void enter_from_user_mode(void) 932 { 933 CT_WARN_ON(ct_state() != CONTEXT_USER); 934 user_exit_irqoff(); 935 } 936 NOKPROBE_SYMBOL(enter_from_user_mode); 937 938 void __pte_error(const char *file, int line, unsigned long val) 939 { 940 pr_err("%s:%d: bad pte %016lx.\n", file, line, val); 941 } 942 943 void __pmd_error(const char *file, int line, unsigned long val) 944 { 945 pr_err("%s:%d: bad pmd %016lx.\n", file, line, val); 946 } 947 948 void __pud_error(const char *file, int line, unsigned long val) 949 { 950 pr_err("%s:%d: bad pud %016lx.\n", file, line, val); 951 } 952 953 void __pgd_error(const char *file, int line, unsigned long val) 954 { 955 pr_err("%s:%d: bad pgd %016lx.\n", file, line, val); 956 } 957 958 /* GENERIC_BUG traps */ 959 960 int is_valid_bugaddr(unsigned long addr) 961 { 962 /* 963 * bug_handler() only called for BRK #BUG_BRK_IMM. 964 * So the answer is trivial -- any spurious instances with no 965 * bug table entry will be rejected by report_bug() and passed 966 * back to the debug-monitors code and handled as a fatal 967 * unexpected debug exception. 968 */ 969 return 1; 970 } 971 972 static int bug_handler(struct pt_regs *regs, unsigned int esr) 973 { 974 switch (report_bug(regs->pc, regs)) { 975 case BUG_TRAP_TYPE_BUG: 976 die("Oops - BUG", regs, 0); 977 break; 978 979 case BUG_TRAP_TYPE_WARN: 980 break; 981 982 default: 983 /* unknown/unrecognised bug trap type */ 984 return DBG_HOOK_ERROR; 985 } 986 987 /* If thread survives, skip over the BUG instruction and continue: */ 988 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 989 return DBG_HOOK_HANDLED; 990 } 991 992 static struct break_hook bug_break_hook = { 993 .fn = bug_handler, 994 .imm = BUG_BRK_IMM, 995 }; 996 997 #ifdef CONFIG_KASAN_SW_TAGS 998 999 #define KASAN_ESR_RECOVER 0x20 1000 #define KASAN_ESR_WRITE 0x10 1001 #define KASAN_ESR_SIZE_MASK 0x0f 1002 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK)) 1003 1004 static int kasan_handler(struct pt_regs *regs, unsigned int esr) 1005 { 1006 bool recover = esr & KASAN_ESR_RECOVER; 1007 bool write = esr & KASAN_ESR_WRITE; 1008 size_t size = KASAN_ESR_SIZE(esr); 1009 u64 addr = regs->regs[0]; 1010 u64 pc = regs->pc; 1011 1012 kasan_report(addr, size, write, pc); 1013 1014 /* 1015 * The instrumentation allows to control whether we can proceed after 1016 * a crash was detected. This is done by passing the -recover flag to 1017 * the compiler. Disabling recovery allows to generate more compact 1018 * code. 1019 * 1020 * Unfortunately disabling recovery doesn't work for the kernel right 1021 * now. KASAN reporting is disabled in some contexts (for example when 1022 * the allocator accesses slab object metadata; this is controlled by 1023 * current->kasan_depth). All these accesses are detected by the tool, 1024 * even though the reports for them are not printed. 1025 * 1026 * This is something that might be fixed at some point in the future. 1027 */ 1028 if (!recover) 1029 die("Oops - KASAN", regs, 0); 1030 1031 /* If thread survives, skip over the brk instruction and continue: */ 1032 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1033 return DBG_HOOK_HANDLED; 1034 } 1035 1036 static struct break_hook kasan_break_hook = { 1037 .fn = kasan_handler, 1038 .imm = KASAN_BRK_IMM, 1039 .mask = KASAN_BRK_MASK, 1040 }; 1041 #endif 1042 1043 /* 1044 * Initial handler for AArch64 BRK exceptions 1045 * This handler only used until debug_traps_init(). 1046 */ 1047 int __init early_brk64(unsigned long addr, unsigned int esr, 1048 struct pt_regs *regs) 1049 { 1050 #ifdef CONFIG_KASAN_SW_TAGS 1051 unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; 1052 1053 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) 1054 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; 1055 #endif 1056 return bug_handler(regs, esr) != DBG_HOOK_HANDLED; 1057 } 1058 1059 void __init trap_init(void) 1060 { 1061 register_kernel_break_hook(&bug_break_hook); 1062 #ifdef CONFIG_KASAN_SW_TAGS 1063 register_kernel_break_hook(&kasan_break_hook); 1064 #endif 1065 debug_traps_init(); 1066 } 1067