1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/kernel/traps.c 4 * 5 * Copyright (C) 1995-2009 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9 #include <linux/bug.h> 10 #include <linux/context_tracking.h> 11 #include <linux/signal.h> 12 #include <linux/personality.h> 13 #include <linux/kallsyms.h> 14 #include <linux/kprobes.h> 15 #include <linux/spinlock.h> 16 #include <linux/uaccess.h> 17 #include <linux/hardirq.h> 18 #include <linux/kdebug.h> 19 #include <linux/module.h> 20 #include <linux/kexec.h> 21 #include <linux/delay.h> 22 #include <linux/init.h> 23 #include <linux/sched/signal.h> 24 #include <linux/sched/debug.h> 25 #include <linux/sched/task_stack.h> 26 #include <linux/sizes.h> 27 #include <linux/syscalls.h> 28 #include <linux/mm_types.h> 29 #include <linux/kasan.h> 30 31 #include <asm/atomic.h> 32 #include <asm/bug.h> 33 #include <asm/cpufeature.h> 34 #include <asm/daifflags.h> 35 #include <asm/debug-monitors.h> 36 #include <asm/esr.h> 37 #include <asm/insn.h> 38 #include <asm/kprobes.h> 39 #include <asm/traps.h> 40 #include <asm/smp.h> 41 #include <asm/stack_pointer.h> 42 #include <asm/stacktrace.h> 43 #include <asm/exception.h> 44 #include <asm/system_misc.h> 45 #include <asm/sysreg.h> 46 47 static const char *handler[]= { 48 "Synchronous Abort", 49 "IRQ", 50 "FIQ", 51 "Error" 52 }; 53 54 int show_unhandled_signals = 0; 55 56 static void dump_backtrace_entry(unsigned long where) 57 { 58 printk(" %pS\n", (void *)where); 59 } 60 61 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs) 62 { 63 unsigned long addr = instruction_pointer(regs); 64 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; 65 int i; 66 67 if (user_mode(regs)) 68 return; 69 70 for (i = -4; i < 1; i++) { 71 unsigned int val, bad; 72 73 bad = aarch64_insn_read(&((u32 *)addr)[i], &val); 74 75 if (!bad) 76 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); 77 else { 78 p += sprintf(p, "bad PC value"); 79 break; 80 } 81 } 82 83 printk("%sCode: %s\n", lvl, str); 84 } 85 86 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) 87 { 88 struct stackframe frame; 89 int skip = 0; 90 91 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); 92 93 if (regs) { 94 if (user_mode(regs)) 95 return; 96 skip = 1; 97 } 98 99 if (!tsk) 100 tsk = current; 101 102 if (!try_get_task_stack(tsk)) 103 return; 104 105 if (tsk == current) { 106 start_backtrace(&frame, 107 (unsigned long)__builtin_frame_address(0), 108 (unsigned long)dump_backtrace); 109 } else { 110 /* 111 * task blocked in __switch_to 112 */ 113 start_backtrace(&frame, 114 thread_saved_fp(tsk), 115 thread_saved_pc(tsk)); 116 } 117 118 printk("Call trace:\n"); 119 do { 120 /* skip until specified stack frame */ 121 if (!skip) { 122 dump_backtrace_entry(frame.pc); 123 } else if (frame.fp == regs->regs[29]) { 124 skip = 0; 125 /* 126 * Mostly, this is the case where this function is 127 * called in panic/abort. As exception handler's 128 * stack frame does not contain the corresponding pc 129 * at which an exception has taken place, use regs->pc 130 * instead. 131 */ 132 dump_backtrace_entry(regs->pc); 133 } 134 } while (!unwind_frame(tsk, &frame)); 135 136 put_task_stack(tsk); 137 } 138 139 void show_stack(struct task_struct *tsk, unsigned long *sp) 140 { 141 dump_backtrace(NULL, tsk); 142 barrier(); 143 } 144 145 #ifdef CONFIG_PREEMPT 146 #define S_PREEMPT " PREEMPT" 147 #elif defined(CONFIG_PREEMPT_RT) 148 #define S_PREEMPT " PREEMPT_RT" 149 #else 150 #define S_PREEMPT "" 151 #endif 152 153 #define S_SMP " SMP" 154 155 static int __die(const char *str, int err, struct pt_regs *regs) 156 { 157 static int die_counter; 158 int ret; 159 160 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", 161 str, err, ++die_counter); 162 163 /* trap and error numbers are mostly meaningless on ARM */ 164 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV); 165 if (ret == NOTIFY_STOP) 166 return ret; 167 168 print_modules(); 169 show_regs(regs); 170 171 dump_kernel_instr(KERN_EMERG, regs); 172 173 return ret; 174 } 175 176 static DEFINE_RAW_SPINLOCK(die_lock); 177 178 /* 179 * This function is protected against re-entrancy. 180 */ 181 void die(const char *str, struct pt_regs *regs, int err) 182 { 183 int ret; 184 unsigned long flags; 185 186 raw_spin_lock_irqsave(&die_lock, flags); 187 188 oops_enter(); 189 190 console_verbose(); 191 bust_spinlocks(1); 192 ret = __die(str, err, regs); 193 194 if (regs && kexec_should_crash(current)) 195 crash_kexec(regs); 196 197 bust_spinlocks(0); 198 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 199 oops_exit(); 200 201 if (in_interrupt()) 202 panic("Fatal exception in interrupt"); 203 if (panic_on_oops) 204 panic("Fatal exception"); 205 206 raw_spin_unlock_irqrestore(&die_lock, flags); 207 208 if (ret != NOTIFY_STOP) 209 do_exit(SIGSEGV); 210 } 211 212 static void arm64_show_signal(int signo, const char *str) 213 { 214 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 215 DEFAULT_RATELIMIT_BURST); 216 struct task_struct *tsk = current; 217 unsigned int esr = tsk->thread.fault_code; 218 struct pt_regs *regs = task_pt_regs(tsk); 219 220 /* Leave if the signal won't be shown */ 221 if (!show_unhandled_signals || 222 !unhandled_signal(tsk, signo) || 223 !__ratelimit(&rs)) 224 return; 225 226 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk)); 227 if (esr) 228 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr); 229 230 pr_cont("%s", str); 231 print_vma_addr(KERN_CONT " in ", regs->pc); 232 pr_cont("\n"); 233 __show_regs(regs); 234 } 235 236 void arm64_force_sig_fault(int signo, int code, void __user *addr, 237 const char *str) 238 { 239 arm64_show_signal(signo, str); 240 if (signo == SIGKILL) 241 force_sig(SIGKILL); 242 else 243 force_sig_fault(signo, code, addr); 244 } 245 246 void arm64_force_sig_mceerr(int code, void __user *addr, short lsb, 247 const char *str) 248 { 249 arm64_show_signal(SIGBUS, str); 250 force_sig_mceerr(code, addr, lsb); 251 } 252 253 void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr, 254 const char *str) 255 { 256 arm64_show_signal(SIGTRAP, str); 257 force_sig_ptrace_errno_trap(errno, addr); 258 } 259 260 void arm64_notify_die(const char *str, struct pt_regs *regs, 261 int signo, int sicode, void __user *addr, 262 int err) 263 { 264 if (user_mode(regs)) { 265 WARN_ON(regs != current_pt_regs()); 266 current->thread.fault_address = 0; 267 current->thread.fault_code = err; 268 269 arm64_force_sig_fault(signo, sicode, addr, str); 270 } else { 271 die(str, regs, err); 272 } 273 } 274 275 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) 276 { 277 regs->pc += size; 278 279 /* 280 * If we were single stepping, we want to get the step exception after 281 * we return from the trap. 282 */ 283 if (user_mode(regs)) 284 user_fastforward_single_step(current); 285 } 286 287 static LIST_HEAD(undef_hook); 288 static DEFINE_RAW_SPINLOCK(undef_lock); 289 290 void register_undef_hook(struct undef_hook *hook) 291 { 292 unsigned long flags; 293 294 raw_spin_lock_irqsave(&undef_lock, flags); 295 list_add(&hook->node, &undef_hook); 296 raw_spin_unlock_irqrestore(&undef_lock, flags); 297 } 298 299 void unregister_undef_hook(struct undef_hook *hook) 300 { 301 unsigned long flags; 302 303 raw_spin_lock_irqsave(&undef_lock, flags); 304 list_del(&hook->node); 305 raw_spin_unlock_irqrestore(&undef_lock, flags); 306 } 307 308 static int call_undef_hook(struct pt_regs *regs) 309 { 310 struct undef_hook *hook; 311 unsigned long flags; 312 u32 instr; 313 int (*fn)(struct pt_regs *regs, u32 instr) = NULL; 314 void __user *pc = (void __user *)instruction_pointer(regs); 315 316 if (!user_mode(regs)) { 317 __le32 instr_le; 318 if (probe_kernel_address((__force __le32 *)pc, instr_le)) 319 goto exit; 320 instr = le32_to_cpu(instr_le); 321 } else if (compat_thumb_mode(regs)) { 322 /* 16-bit Thumb instruction */ 323 __le16 instr_le; 324 if (get_user(instr_le, (__le16 __user *)pc)) 325 goto exit; 326 instr = le16_to_cpu(instr_le); 327 if (aarch32_insn_is_wide(instr)) { 328 u32 instr2; 329 330 if (get_user(instr_le, (__le16 __user *)(pc + 2))) 331 goto exit; 332 instr2 = le16_to_cpu(instr_le); 333 instr = (instr << 16) | instr2; 334 } 335 } else { 336 /* 32-bit ARM instruction */ 337 __le32 instr_le; 338 if (get_user(instr_le, (__le32 __user *)pc)) 339 goto exit; 340 instr = le32_to_cpu(instr_le); 341 } 342 343 raw_spin_lock_irqsave(&undef_lock, flags); 344 list_for_each_entry(hook, &undef_hook, node) 345 if ((instr & hook->instr_mask) == hook->instr_val && 346 (regs->pstate & hook->pstate_mask) == hook->pstate_val) 347 fn = hook->fn; 348 349 raw_spin_unlock_irqrestore(&undef_lock, flags); 350 exit: 351 return fn ? fn(regs, instr) : 1; 352 } 353 354 void force_signal_inject(int signal, int code, unsigned long address) 355 { 356 const char *desc; 357 struct pt_regs *regs = current_pt_regs(); 358 359 if (WARN_ON(!user_mode(regs))) 360 return; 361 362 switch (signal) { 363 case SIGILL: 364 desc = "undefined instruction"; 365 break; 366 case SIGSEGV: 367 desc = "illegal memory access"; 368 break; 369 default: 370 desc = "unknown or unrecoverable error"; 371 break; 372 } 373 374 /* Force signals we don't understand to SIGKILL */ 375 if (WARN_ON(signal != SIGKILL && 376 siginfo_layout(signal, code) != SIL_FAULT)) { 377 signal = SIGKILL; 378 } 379 380 arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0); 381 } 382 383 /* 384 * Set up process info to signal segmentation fault - called on access error. 385 */ 386 void arm64_notify_segfault(unsigned long addr) 387 { 388 int code; 389 390 down_read(¤t->mm->mmap_sem); 391 if (find_vma(current->mm, addr) == NULL) 392 code = SEGV_MAPERR; 393 else 394 code = SEGV_ACCERR; 395 up_read(¤t->mm->mmap_sem); 396 397 force_signal_inject(SIGSEGV, code, addr); 398 } 399 400 void do_undefinstr(struct pt_regs *regs) 401 { 402 /* check for AArch32 breakpoint instructions */ 403 if (!aarch32_break_handler(regs)) 404 return; 405 406 if (call_undef_hook(regs) == 0) 407 return; 408 409 BUG_ON(!user_mode(regs)); 410 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 411 } 412 NOKPROBE_SYMBOL(do_undefinstr); 413 414 #define __user_cache_maint(insn, address, res) \ 415 if (address >= user_addr_max()) { \ 416 res = -EFAULT; \ 417 } else { \ 418 uaccess_ttbr0_enable(); \ 419 asm volatile ( \ 420 "1: " insn ", %1\n" \ 421 " mov %w0, #0\n" \ 422 "2:\n" \ 423 " .pushsection .fixup,\"ax\"\n" \ 424 " .align 2\n" \ 425 "3: mov %w0, %w2\n" \ 426 " b 2b\n" \ 427 " .popsection\n" \ 428 _ASM_EXTABLE(1b, 3b) \ 429 : "=r" (res) \ 430 : "r" (address), "i" (-EFAULT)); \ 431 uaccess_ttbr0_disable(); \ 432 } 433 434 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) 435 { 436 unsigned long address; 437 int rt = ESR_ELx_SYS64_ISS_RT(esr); 438 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 439 int ret = 0; 440 441 address = untagged_addr(pt_regs_read_reg(regs, rt)); 442 443 switch (crm) { 444 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */ 445 __user_cache_maint("dc civac", address, ret); 446 break; 447 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ 448 __user_cache_maint("dc civac", address, ret); 449 break; 450 case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */ 451 __user_cache_maint("sys 3, c7, c13, 1", address, ret); 452 break; 453 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ 454 __user_cache_maint("sys 3, c7, c12, 1", address, ret); 455 break; 456 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ 457 __user_cache_maint("dc civac", address, ret); 458 break; 459 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */ 460 __user_cache_maint("ic ivau", address, ret); 461 break; 462 default: 463 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 464 return; 465 } 466 467 if (ret) 468 arm64_notify_segfault(address); 469 else 470 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 471 } 472 473 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) 474 { 475 int rt = ESR_ELx_SYS64_ISS_RT(esr); 476 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0); 477 478 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) { 479 /* Hide DIC so that we can trap the unnecessary maintenance...*/ 480 val &= ~BIT(CTR_DIC_SHIFT); 481 482 /* ... and fake IminLine to reduce the number of traps. */ 483 val &= ~CTR_IMINLINE_MASK; 484 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK; 485 } 486 487 pt_regs_write_reg(regs, rt, val); 488 489 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 490 } 491 492 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) 493 { 494 int rt = ESR_ELx_SYS64_ISS_RT(esr); 495 496 pt_regs_write_reg(regs, rt, arch_timer_read_counter()); 497 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 498 } 499 500 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) 501 { 502 int rt = ESR_ELx_SYS64_ISS_RT(esr); 503 504 pt_regs_write_reg(regs, rt, arch_timer_get_rate()); 505 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 506 } 507 508 static void mrs_handler(unsigned int esr, struct pt_regs *regs) 509 { 510 u32 sysreg, rt; 511 512 rt = ESR_ELx_SYS64_ISS_RT(esr); 513 sysreg = esr_sys64_to_sysreg(esr); 514 515 if (do_emulate_mrs(regs, sysreg, rt) != 0) 516 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 517 } 518 519 static void wfi_handler(unsigned int esr, struct pt_regs *regs) 520 { 521 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 522 } 523 524 struct sys64_hook { 525 unsigned int esr_mask; 526 unsigned int esr_val; 527 void (*handler)(unsigned int esr, struct pt_regs *regs); 528 }; 529 530 static const struct sys64_hook sys64_hooks[] = { 531 { 532 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK, 533 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL, 534 .handler = user_cache_maint_handler, 535 }, 536 { 537 /* Trap read access to CTR_EL0 */ 538 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 539 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ, 540 .handler = ctr_read_handler, 541 }, 542 { 543 /* Trap read access to CNTVCT_EL0 */ 544 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 545 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, 546 .handler = cntvct_read_handler, 547 }, 548 { 549 /* Trap read access to CNTFRQ_EL0 */ 550 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 551 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ, 552 .handler = cntfrq_read_handler, 553 }, 554 { 555 /* Trap read access to CPUID registers */ 556 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK, 557 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL, 558 .handler = mrs_handler, 559 }, 560 { 561 /* Trap WFI instructions executed in userspace */ 562 .esr_mask = ESR_ELx_WFx_MASK, 563 .esr_val = ESR_ELx_WFx_WFI_VAL, 564 .handler = wfi_handler, 565 }, 566 {}, 567 }; 568 569 570 #ifdef CONFIG_COMPAT 571 #define PSTATE_IT_1_0_SHIFT 25 572 #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT) 573 #define PSTATE_IT_7_2_SHIFT 10 574 #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT) 575 576 static u32 compat_get_it_state(struct pt_regs *regs) 577 { 578 u32 it, pstate = regs->pstate; 579 580 it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT; 581 it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2; 582 583 return it; 584 } 585 586 static void compat_set_it_state(struct pt_regs *regs, u32 it) 587 { 588 u32 pstate_it; 589 590 pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK; 591 pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK; 592 593 regs->pstate &= ~PSR_AA32_IT_MASK; 594 regs->pstate |= pstate_it; 595 } 596 597 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs) 598 { 599 int cond; 600 601 /* Only a T32 instruction can trap without CV being set */ 602 if (!(esr & ESR_ELx_CV)) { 603 u32 it; 604 605 it = compat_get_it_state(regs); 606 if (!it) 607 return true; 608 609 cond = it >> 4; 610 } else { 611 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; 612 } 613 614 return aarch32_opcode_cond_checks[cond](regs->pstate); 615 } 616 617 static void advance_itstate(struct pt_regs *regs) 618 { 619 u32 it; 620 621 /* ARM mode */ 622 if (!(regs->pstate & PSR_AA32_T_BIT) || 623 !(regs->pstate & PSR_AA32_IT_MASK)) 624 return; 625 626 it = compat_get_it_state(regs); 627 628 /* 629 * If this is the last instruction of the block, wipe the IT 630 * state. Otherwise advance it. 631 */ 632 if (!(it & 7)) 633 it = 0; 634 else 635 it = (it & 0xe0) | ((it << 1) & 0x1f); 636 637 compat_set_it_state(regs, it); 638 } 639 640 static void arm64_compat_skip_faulting_instruction(struct pt_regs *regs, 641 unsigned int sz) 642 { 643 advance_itstate(regs); 644 arm64_skip_faulting_instruction(regs, sz); 645 } 646 647 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) 648 { 649 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT; 650 651 pt_regs_write_reg(regs, reg, arch_timer_get_rate()); 652 arm64_compat_skip_faulting_instruction(regs, 4); 653 } 654 655 static const struct sys64_hook cp15_32_hooks[] = { 656 { 657 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK, 658 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ, 659 .handler = compat_cntfrq_read_handler, 660 }, 661 {}, 662 }; 663 664 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs) 665 { 666 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; 667 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; 668 u64 val = arch_timer_read_counter(); 669 670 pt_regs_write_reg(regs, rt, lower_32_bits(val)); 671 pt_regs_write_reg(regs, rt2, upper_32_bits(val)); 672 arm64_compat_skip_faulting_instruction(regs, 4); 673 } 674 675 static const struct sys64_hook cp15_64_hooks[] = { 676 { 677 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, 678 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, 679 .handler = compat_cntvct_read_handler, 680 }, 681 {}, 682 }; 683 684 void do_cp15instr(unsigned int esr, struct pt_regs *regs) 685 { 686 const struct sys64_hook *hook, *hook_base; 687 688 if (!cp15_cond_valid(esr, regs)) { 689 /* 690 * There is no T16 variant of a CP access, so we 691 * always advance PC by 4 bytes. 692 */ 693 arm64_compat_skip_faulting_instruction(regs, 4); 694 return; 695 } 696 697 switch (ESR_ELx_EC(esr)) { 698 case ESR_ELx_EC_CP15_32: 699 hook_base = cp15_32_hooks; 700 break; 701 case ESR_ELx_EC_CP15_64: 702 hook_base = cp15_64_hooks; 703 break; 704 default: 705 do_undefinstr(regs); 706 return; 707 } 708 709 for (hook = hook_base; hook->handler; hook++) 710 if ((hook->esr_mask & esr) == hook->esr_val) { 711 hook->handler(esr, regs); 712 return; 713 } 714 715 /* 716 * New cp15 instructions may previously have been undefined at 717 * EL0. Fall back to our usual undefined instruction handler 718 * so that we handle these consistently. 719 */ 720 do_undefinstr(regs); 721 } 722 NOKPROBE_SYMBOL(do_cp15instr); 723 #endif 724 725 void do_sysinstr(unsigned int esr, struct pt_regs *regs) 726 { 727 const struct sys64_hook *hook; 728 729 for (hook = sys64_hooks; hook->handler; hook++) 730 if ((hook->esr_mask & esr) == hook->esr_val) { 731 hook->handler(esr, regs); 732 return; 733 } 734 735 /* 736 * New SYS instructions may previously have been undefined at EL0. Fall 737 * back to our usual undefined instruction handler so that we handle 738 * these consistently. 739 */ 740 do_undefinstr(regs); 741 } 742 NOKPROBE_SYMBOL(do_sysinstr); 743 744 static const char *esr_class_str[] = { 745 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", 746 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", 747 [ESR_ELx_EC_WFx] = "WFI/WFE", 748 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", 749 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", 750 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", 751 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", 752 [ESR_ELx_EC_FP_ASIMD] = "ASIMD", 753 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", 754 [ESR_ELx_EC_PAC] = "PAC", 755 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", 756 [ESR_ELx_EC_ILL] = "PSTATE.IL", 757 [ESR_ELx_EC_SVC32] = "SVC (AArch32)", 758 [ESR_ELx_EC_HVC32] = "HVC (AArch32)", 759 [ESR_ELx_EC_SMC32] = "SMC (AArch32)", 760 [ESR_ELx_EC_SVC64] = "SVC (AArch64)", 761 [ESR_ELx_EC_HVC64] = "HVC (AArch64)", 762 [ESR_ELx_EC_SMC64] = "SMC (AArch64)", 763 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", 764 [ESR_ELx_EC_SVE] = "SVE", 765 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB", 766 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", 767 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", 768 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", 769 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", 770 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", 771 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", 772 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", 773 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", 774 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", 775 [ESR_ELx_EC_SERROR] = "SError", 776 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", 777 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", 778 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", 779 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", 780 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", 781 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", 782 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", 783 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", 784 [ESR_ELx_EC_BRK64] = "BRK (AArch64)", 785 }; 786 787 const char *esr_get_class_string(u32 esr) 788 { 789 return esr_class_str[ESR_ELx_EC(esr)]; 790 } 791 792 /* 793 * bad_mode handles the impossible case in the exception vector. This is always 794 * fatal. 795 */ 796 asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) 797 { 798 console_verbose(); 799 800 pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", 801 handler[reason], smp_processor_id(), esr, 802 esr_get_class_string(esr)); 803 804 local_daif_mask(); 805 panic("bad mode"); 806 } 807 808 /* 809 * bad_el0_sync handles unexpected, but potentially recoverable synchronous 810 * exceptions taken from EL0. Unlike bad_mode, this returns. 811 */ 812 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) 813 { 814 void __user *pc = (void __user *)instruction_pointer(regs); 815 816 current->thread.fault_address = 0; 817 current->thread.fault_code = esr; 818 819 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc, 820 "Bad EL0 synchronous exception"); 821 } 822 823 #ifdef CONFIG_VMAP_STACK 824 825 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) 826 __aligned(16); 827 828 asmlinkage void handle_bad_stack(struct pt_regs *regs) 829 { 830 unsigned long tsk_stk = (unsigned long)current->stack; 831 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); 832 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 833 unsigned int esr = read_sysreg(esr_el1); 834 unsigned long far = read_sysreg(far_el1); 835 836 console_verbose(); 837 pr_emerg("Insufficient stack space to handle exception!"); 838 839 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr)); 840 pr_emerg("FAR: 0x%016lx\n", far); 841 842 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 843 tsk_stk, tsk_stk + THREAD_SIZE); 844 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", 845 irq_stk, irq_stk + THREAD_SIZE); 846 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 847 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 848 849 __show_regs(regs); 850 851 /* 852 * We use nmi_panic to limit the potential for recusive overflows, and 853 * to get a better stack trace. 854 */ 855 nmi_panic(NULL, "kernel stack overflow"); 856 cpu_park_loop(); 857 } 858 #endif 859 860 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr) 861 { 862 console_verbose(); 863 864 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n", 865 smp_processor_id(), esr, esr_get_class_string(esr)); 866 if (regs) 867 __show_regs(regs); 868 869 nmi_panic(regs, "Asynchronous SError Interrupt"); 870 871 cpu_park_loop(); 872 unreachable(); 873 } 874 875 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) 876 { 877 u32 aet = arm64_ras_serror_get_severity(esr); 878 879 switch (aet) { 880 case ESR_ELx_AET_CE: /* corrected error */ 881 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */ 882 /* 883 * The CPU can make progress. We may take UEO again as 884 * a more severe error. 885 */ 886 return false; 887 888 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */ 889 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */ 890 /* 891 * The CPU can't make progress. The exception may have 892 * been imprecise. 893 * 894 * Neoverse-N1 #1349291 means a non-KVM SError reported as 895 * Unrecoverable should be treated as Uncontainable. We 896 * call arm64_serror_panic() in both cases. 897 */ 898 return true; 899 900 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */ 901 default: 902 /* Error has been silently propagated */ 903 arm64_serror_panic(regs, esr); 904 } 905 } 906 907 asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr) 908 { 909 const bool was_in_nmi = in_nmi(); 910 911 if (!was_in_nmi) 912 nmi_enter(); 913 914 /* non-RAS errors are not containable */ 915 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) 916 arm64_serror_panic(regs, esr); 917 918 if (!was_in_nmi) 919 nmi_exit(); 920 } 921 922 asmlinkage void enter_from_user_mode(void) 923 { 924 CT_WARN_ON(ct_state() != CONTEXT_USER); 925 user_exit_irqoff(); 926 } 927 NOKPROBE_SYMBOL(enter_from_user_mode); 928 929 void __pte_error(const char *file, int line, unsigned long val) 930 { 931 pr_err("%s:%d: bad pte %016lx.\n", file, line, val); 932 } 933 934 void __pmd_error(const char *file, int line, unsigned long val) 935 { 936 pr_err("%s:%d: bad pmd %016lx.\n", file, line, val); 937 } 938 939 void __pud_error(const char *file, int line, unsigned long val) 940 { 941 pr_err("%s:%d: bad pud %016lx.\n", file, line, val); 942 } 943 944 void __pgd_error(const char *file, int line, unsigned long val) 945 { 946 pr_err("%s:%d: bad pgd %016lx.\n", file, line, val); 947 } 948 949 /* GENERIC_BUG traps */ 950 951 int is_valid_bugaddr(unsigned long addr) 952 { 953 /* 954 * bug_handler() only called for BRK #BUG_BRK_IMM. 955 * So the answer is trivial -- any spurious instances with no 956 * bug table entry will be rejected by report_bug() and passed 957 * back to the debug-monitors code and handled as a fatal 958 * unexpected debug exception. 959 */ 960 return 1; 961 } 962 963 static int bug_handler(struct pt_regs *regs, unsigned int esr) 964 { 965 switch (report_bug(regs->pc, regs)) { 966 case BUG_TRAP_TYPE_BUG: 967 die("Oops - BUG", regs, 0); 968 break; 969 970 case BUG_TRAP_TYPE_WARN: 971 break; 972 973 default: 974 /* unknown/unrecognised bug trap type */ 975 return DBG_HOOK_ERROR; 976 } 977 978 /* If thread survives, skip over the BUG instruction and continue: */ 979 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 980 return DBG_HOOK_HANDLED; 981 } 982 983 static struct break_hook bug_break_hook = { 984 .fn = bug_handler, 985 .imm = BUG_BRK_IMM, 986 }; 987 988 #ifdef CONFIG_KASAN_SW_TAGS 989 990 #define KASAN_ESR_RECOVER 0x20 991 #define KASAN_ESR_WRITE 0x10 992 #define KASAN_ESR_SIZE_MASK 0x0f 993 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK)) 994 995 static int kasan_handler(struct pt_regs *regs, unsigned int esr) 996 { 997 bool recover = esr & KASAN_ESR_RECOVER; 998 bool write = esr & KASAN_ESR_WRITE; 999 size_t size = KASAN_ESR_SIZE(esr); 1000 u64 addr = regs->regs[0]; 1001 u64 pc = regs->pc; 1002 1003 kasan_report(addr, size, write, pc); 1004 1005 /* 1006 * The instrumentation allows to control whether we can proceed after 1007 * a crash was detected. This is done by passing the -recover flag to 1008 * the compiler. Disabling recovery allows to generate more compact 1009 * code. 1010 * 1011 * Unfortunately disabling recovery doesn't work for the kernel right 1012 * now. KASAN reporting is disabled in some contexts (for example when 1013 * the allocator accesses slab object metadata; this is controlled by 1014 * current->kasan_depth). All these accesses are detected by the tool, 1015 * even though the reports for them are not printed. 1016 * 1017 * This is something that might be fixed at some point in the future. 1018 */ 1019 if (!recover) 1020 die("Oops - KASAN", regs, 0); 1021 1022 /* If thread survives, skip over the brk instruction and continue: */ 1023 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1024 return DBG_HOOK_HANDLED; 1025 } 1026 1027 static struct break_hook kasan_break_hook = { 1028 .fn = kasan_handler, 1029 .imm = KASAN_BRK_IMM, 1030 .mask = KASAN_BRK_MASK, 1031 }; 1032 #endif 1033 1034 /* 1035 * Initial handler for AArch64 BRK exceptions 1036 * This handler only used until debug_traps_init(). 1037 */ 1038 int __init early_brk64(unsigned long addr, unsigned int esr, 1039 struct pt_regs *regs) 1040 { 1041 #ifdef CONFIG_KASAN_SW_TAGS 1042 unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; 1043 1044 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) 1045 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; 1046 #endif 1047 return bug_handler(regs, esr) != DBG_HOOK_HANDLED; 1048 } 1049 1050 /* This registration must happen early, before debug_traps_init(). */ 1051 void __init trap_init(void) 1052 { 1053 register_kernel_break_hook(&bug_break_hook); 1054 #ifdef CONFIG_KASAN_SW_TAGS 1055 register_kernel_break_hook(&kasan_break_hook); 1056 #endif 1057 } 1058