1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/kernel/traps.c 4 * 5 * Copyright (C) 1995-2009 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9 #include <linux/bug.h> 10 #include <linux/signal.h> 11 #include <linux/personality.h> 12 #include <linux/kallsyms.h> 13 #include <linux/spinlock.h> 14 #include <linux/uaccess.h> 15 #include <linux/hardirq.h> 16 #include <linux/kdebug.h> 17 #include <linux/module.h> 18 #include <linux/kexec.h> 19 #include <linux/delay.h> 20 #include <linux/init.h> 21 #include <linux/sched/signal.h> 22 #include <linux/sched/debug.h> 23 #include <linux/sched/task_stack.h> 24 #include <linux/sizes.h> 25 #include <linux/syscalls.h> 26 #include <linux/mm_types.h> 27 #include <linux/kasan.h> 28 29 #include <asm/atomic.h> 30 #include <asm/bug.h> 31 #include <asm/cpufeature.h> 32 #include <asm/daifflags.h> 33 #include <asm/debug-monitors.h> 34 #include <asm/esr.h> 35 #include <asm/insn.h> 36 #include <asm/traps.h> 37 #include <asm/smp.h> 38 #include <asm/stack_pointer.h> 39 #include <asm/stacktrace.h> 40 #include <asm/exception.h> 41 #include <asm/system_misc.h> 42 #include <asm/sysreg.h> 43 44 static const char *handler[]= { 45 "Synchronous Abort", 46 "IRQ", 47 "FIQ", 48 "Error" 49 }; 50 51 int show_unhandled_signals = 0; 52 53 static void dump_backtrace_entry(unsigned long where) 54 { 55 printk(" %pS\n", (void *)where); 56 } 57 58 static void __dump_instr(const char *lvl, struct pt_regs *regs) 59 { 60 unsigned long addr = instruction_pointer(regs); 61 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; 62 int i; 63 64 for (i = -4; i < 1; i++) { 65 unsigned int val, bad; 66 67 bad = get_user(val, &((u32 *)addr)[i]); 68 69 if (!bad) 70 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); 71 else { 72 p += sprintf(p, "bad PC value"); 73 break; 74 } 75 } 76 printk("%sCode: %s\n", lvl, str); 77 } 78 79 static void dump_instr(const char *lvl, struct pt_regs *regs) 80 { 81 if (!user_mode(regs)) { 82 mm_segment_t fs = get_fs(); 83 set_fs(KERNEL_DS); 84 __dump_instr(lvl, regs); 85 set_fs(fs); 86 } else { 87 __dump_instr(lvl, regs); 88 } 89 } 90 91 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) 92 { 93 struct stackframe frame; 94 int skip = 0; 95 96 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); 97 98 if (regs) { 99 if (user_mode(regs)) 100 return; 101 skip = 1; 102 } 103 104 if (!tsk) 105 tsk = current; 106 107 if (!try_get_task_stack(tsk)) 108 return; 109 110 if (tsk == current) { 111 frame.fp = (unsigned long)__builtin_frame_address(0); 112 frame.pc = (unsigned long)dump_backtrace; 113 } else { 114 /* 115 * task blocked in __switch_to 116 */ 117 frame.fp = thread_saved_fp(tsk); 118 frame.pc = thread_saved_pc(tsk); 119 } 120 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 121 frame.graph = 0; 122 #endif 123 124 printk("Call trace:\n"); 125 do { 126 /* skip until specified stack frame */ 127 if (!skip) { 128 dump_backtrace_entry(frame.pc); 129 } else if (frame.fp == regs->regs[29]) { 130 skip = 0; 131 /* 132 * Mostly, this is the case where this function is 133 * called in panic/abort. As exception handler's 134 * stack frame does not contain the corresponding pc 135 * at which an exception has taken place, use regs->pc 136 * instead. 137 */ 138 dump_backtrace_entry(regs->pc); 139 } 140 } while (!unwind_frame(tsk, &frame)); 141 142 put_task_stack(tsk); 143 } 144 145 void show_stack(struct task_struct *tsk, unsigned long *sp) 146 { 147 dump_backtrace(NULL, tsk); 148 barrier(); 149 } 150 151 #ifdef CONFIG_PREEMPT 152 #define S_PREEMPT " PREEMPT" 153 #else 154 #define S_PREEMPT "" 155 #endif 156 #define S_SMP " SMP" 157 158 static int __die(const char *str, int err, struct pt_regs *regs) 159 { 160 static int die_counter; 161 int ret; 162 163 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", 164 str, err, ++die_counter); 165 166 /* trap and error numbers are mostly meaningless on ARM */ 167 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV); 168 if (ret == NOTIFY_STOP) 169 return ret; 170 171 print_modules(); 172 show_regs(regs); 173 174 if (!user_mode(regs)) 175 dump_instr(KERN_EMERG, regs); 176 177 return ret; 178 } 179 180 static DEFINE_RAW_SPINLOCK(die_lock); 181 182 /* 183 * This function is protected against re-entrancy. 184 */ 185 void die(const char *str, struct pt_regs *regs, int err) 186 { 187 int ret; 188 unsigned long flags; 189 190 raw_spin_lock_irqsave(&die_lock, flags); 191 192 oops_enter(); 193 194 console_verbose(); 195 bust_spinlocks(1); 196 ret = __die(str, err, regs); 197 198 if (regs && kexec_should_crash(current)) 199 crash_kexec(regs); 200 201 bust_spinlocks(0); 202 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 203 oops_exit(); 204 205 if (in_interrupt()) 206 panic("Fatal exception in interrupt"); 207 if (panic_on_oops) 208 panic("Fatal exception"); 209 210 raw_spin_unlock_irqrestore(&die_lock, flags); 211 212 if (ret != NOTIFY_STOP) 213 do_exit(SIGSEGV); 214 } 215 216 static void arm64_show_signal(int signo, const char *str) 217 { 218 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 219 DEFAULT_RATELIMIT_BURST); 220 struct task_struct *tsk = current; 221 unsigned int esr = tsk->thread.fault_code; 222 struct pt_regs *regs = task_pt_regs(tsk); 223 224 /* Leave if the signal won't be shown */ 225 if (!show_unhandled_signals || 226 !unhandled_signal(tsk, signo) || 227 !__ratelimit(&rs)) 228 return; 229 230 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk)); 231 if (esr) 232 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr); 233 234 pr_cont("%s", str); 235 print_vma_addr(KERN_CONT " in ", regs->pc); 236 pr_cont("\n"); 237 __show_regs(regs); 238 } 239 240 void arm64_force_sig_fault(int signo, int code, void __user *addr, 241 const char *str) 242 { 243 arm64_show_signal(signo, str); 244 if (signo == SIGKILL) 245 force_sig(SIGKILL, current); 246 else 247 force_sig_fault(signo, code, addr, current); 248 } 249 250 void arm64_force_sig_mceerr(int code, void __user *addr, short lsb, 251 const char *str) 252 { 253 arm64_show_signal(SIGBUS, str); 254 force_sig_mceerr(code, addr, lsb, current); 255 } 256 257 void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr, 258 const char *str) 259 { 260 arm64_show_signal(SIGTRAP, str); 261 force_sig_ptrace_errno_trap(errno, addr); 262 } 263 264 void arm64_notify_die(const char *str, struct pt_regs *regs, 265 int signo, int sicode, void __user *addr, 266 int err) 267 { 268 if (user_mode(regs)) { 269 WARN_ON(regs != current_pt_regs()); 270 current->thread.fault_address = 0; 271 current->thread.fault_code = err; 272 273 arm64_force_sig_fault(signo, sicode, addr, str); 274 } else { 275 die(str, regs, err); 276 } 277 } 278 279 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) 280 { 281 regs->pc += size; 282 283 /* 284 * If we were single stepping, we want to get the step exception after 285 * we return from the trap. 286 */ 287 if (user_mode(regs)) 288 user_fastforward_single_step(current); 289 } 290 291 static LIST_HEAD(undef_hook); 292 static DEFINE_RAW_SPINLOCK(undef_lock); 293 294 void register_undef_hook(struct undef_hook *hook) 295 { 296 unsigned long flags; 297 298 raw_spin_lock_irqsave(&undef_lock, flags); 299 list_add(&hook->node, &undef_hook); 300 raw_spin_unlock_irqrestore(&undef_lock, flags); 301 } 302 303 void unregister_undef_hook(struct undef_hook *hook) 304 { 305 unsigned long flags; 306 307 raw_spin_lock_irqsave(&undef_lock, flags); 308 list_del(&hook->node); 309 raw_spin_unlock_irqrestore(&undef_lock, flags); 310 } 311 312 static int call_undef_hook(struct pt_regs *regs) 313 { 314 struct undef_hook *hook; 315 unsigned long flags; 316 u32 instr; 317 int (*fn)(struct pt_regs *regs, u32 instr) = NULL; 318 void __user *pc = (void __user *)instruction_pointer(regs); 319 320 if (!user_mode(regs)) { 321 __le32 instr_le; 322 if (probe_kernel_address((__force __le32 *)pc, instr_le)) 323 goto exit; 324 instr = le32_to_cpu(instr_le); 325 } else if (compat_thumb_mode(regs)) { 326 /* 16-bit Thumb instruction */ 327 __le16 instr_le; 328 if (get_user(instr_le, (__le16 __user *)pc)) 329 goto exit; 330 instr = le16_to_cpu(instr_le); 331 if (aarch32_insn_is_wide(instr)) { 332 u32 instr2; 333 334 if (get_user(instr_le, (__le16 __user *)(pc + 2))) 335 goto exit; 336 instr2 = le16_to_cpu(instr_le); 337 instr = (instr << 16) | instr2; 338 } 339 } else { 340 /* 32-bit ARM instruction */ 341 __le32 instr_le; 342 if (get_user(instr_le, (__le32 __user *)pc)) 343 goto exit; 344 instr = le32_to_cpu(instr_le); 345 } 346 347 raw_spin_lock_irqsave(&undef_lock, flags); 348 list_for_each_entry(hook, &undef_hook, node) 349 if ((instr & hook->instr_mask) == hook->instr_val && 350 (regs->pstate & hook->pstate_mask) == hook->pstate_val) 351 fn = hook->fn; 352 353 raw_spin_unlock_irqrestore(&undef_lock, flags); 354 exit: 355 return fn ? fn(regs, instr) : 1; 356 } 357 358 void force_signal_inject(int signal, int code, unsigned long address) 359 { 360 const char *desc; 361 struct pt_regs *regs = current_pt_regs(); 362 363 if (WARN_ON(!user_mode(regs))) 364 return; 365 366 switch (signal) { 367 case SIGILL: 368 desc = "undefined instruction"; 369 break; 370 case SIGSEGV: 371 desc = "illegal memory access"; 372 break; 373 default: 374 desc = "unknown or unrecoverable error"; 375 break; 376 } 377 378 /* Force signals we don't understand to SIGKILL */ 379 if (WARN_ON(signal != SIGKILL && 380 siginfo_layout(signal, code) != SIL_FAULT)) { 381 signal = SIGKILL; 382 } 383 384 arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0); 385 } 386 387 /* 388 * Set up process info to signal segmentation fault - called on access error. 389 */ 390 void arm64_notify_segfault(unsigned long addr) 391 { 392 int code; 393 394 down_read(¤t->mm->mmap_sem); 395 if (find_vma(current->mm, addr) == NULL) 396 code = SEGV_MAPERR; 397 else 398 code = SEGV_ACCERR; 399 up_read(¤t->mm->mmap_sem); 400 401 force_signal_inject(SIGSEGV, code, addr); 402 } 403 404 asmlinkage void __exception do_undefinstr(struct pt_regs *regs) 405 { 406 /* check for AArch32 breakpoint instructions */ 407 if (!aarch32_break_handler(regs)) 408 return; 409 410 if (call_undef_hook(regs) == 0) 411 return; 412 413 BUG_ON(!user_mode(regs)); 414 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 415 } 416 417 #define __user_cache_maint(insn, address, res) \ 418 if (address >= user_addr_max()) { \ 419 res = -EFAULT; \ 420 } else { \ 421 uaccess_ttbr0_enable(); \ 422 asm volatile ( \ 423 "1: " insn ", %1\n" \ 424 " mov %w0, #0\n" \ 425 "2:\n" \ 426 " .pushsection .fixup,\"ax\"\n" \ 427 " .align 2\n" \ 428 "3: mov %w0, %w2\n" \ 429 " b 2b\n" \ 430 " .popsection\n" \ 431 _ASM_EXTABLE(1b, 3b) \ 432 : "=r" (res) \ 433 : "r" (address), "i" (-EFAULT)); \ 434 uaccess_ttbr0_disable(); \ 435 } 436 437 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) 438 { 439 unsigned long address; 440 int rt = ESR_ELx_SYS64_ISS_RT(esr); 441 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 442 int ret = 0; 443 444 address = untagged_addr(pt_regs_read_reg(regs, rt)); 445 446 switch (crm) { 447 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */ 448 __user_cache_maint("dc civac", address, ret); 449 break; 450 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ 451 __user_cache_maint("dc civac", address, ret); 452 break; 453 case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */ 454 __user_cache_maint("sys 3, c7, c13, 1", address, ret); 455 break; 456 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ 457 __user_cache_maint("sys 3, c7, c12, 1", address, ret); 458 break; 459 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ 460 __user_cache_maint("dc civac", address, ret); 461 break; 462 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */ 463 __user_cache_maint("ic ivau", address, ret); 464 break; 465 default: 466 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 467 return; 468 } 469 470 if (ret) 471 arm64_notify_segfault(address); 472 else 473 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 474 } 475 476 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) 477 { 478 int rt = ESR_ELx_SYS64_ISS_RT(esr); 479 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0); 480 481 pt_regs_write_reg(regs, rt, val); 482 483 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 484 } 485 486 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) 487 { 488 int rt = ESR_ELx_SYS64_ISS_RT(esr); 489 490 pt_regs_write_reg(regs, rt, arch_timer_read_counter()); 491 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 492 } 493 494 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) 495 { 496 int rt = ESR_ELx_SYS64_ISS_RT(esr); 497 498 pt_regs_write_reg(regs, rt, arch_timer_get_rate()); 499 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 500 } 501 502 static void mrs_handler(unsigned int esr, struct pt_regs *regs) 503 { 504 u32 sysreg, rt; 505 506 rt = ESR_ELx_SYS64_ISS_RT(esr); 507 sysreg = esr_sys64_to_sysreg(esr); 508 509 if (do_emulate_mrs(regs, sysreg, rt) != 0) 510 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 511 } 512 513 static void wfi_handler(unsigned int esr, struct pt_regs *regs) 514 { 515 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 516 } 517 518 struct sys64_hook { 519 unsigned int esr_mask; 520 unsigned int esr_val; 521 void (*handler)(unsigned int esr, struct pt_regs *regs); 522 }; 523 524 static struct sys64_hook sys64_hooks[] = { 525 { 526 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK, 527 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL, 528 .handler = user_cache_maint_handler, 529 }, 530 { 531 /* Trap read access to CTR_EL0 */ 532 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 533 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ, 534 .handler = ctr_read_handler, 535 }, 536 { 537 /* Trap read access to CNTVCT_EL0 */ 538 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 539 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, 540 .handler = cntvct_read_handler, 541 }, 542 { 543 /* Trap read access to CNTFRQ_EL0 */ 544 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 545 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ, 546 .handler = cntfrq_read_handler, 547 }, 548 { 549 /* Trap read access to CPUID registers */ 550 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK, 551 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL, 552 .handler = mrs_handler, 553 }, 554 { 555 /* Trap WFI instructions executed in userspace */ 556 .esr_mask = ESR_ELx_WFx_MASK, 557 .esr_val = ESR_ELx_WFx_WFI_VAL, 558 .handler = wfi_handler, 559 }, 560 {}, 561 }; 562 563 564 #ifdef CONFIG_COMPAT 565 #define PSTATE_IT_1_0_SHIFT 25 566 #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT) 567 #define PSTATE_IT_7_2_SHIFT 10 568 #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT) 569 570 static u32 compat_get_it_state(struct pt_regs *regs) 571 { 572 u32 it, pstate = regs->pstate; 573 574 it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT; 575 it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2; 576 577 return it; 578 } 579 580 static void compat_set_it_state(struct pt_regs *regs, u32 it) 581 { 582 u32 pstate_it; 583 584 pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK; 585 pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK; 586 587 regs->pstate &= ~PSR_AA32_IT_MASK; 588 regs->pstate |= pstate_it; 589 } 590 591 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs) 592 { 593 int cond; 594 595 /* Only a T32 instruction can trap without CV being set */ 596 if (!(esr & ESR_ELx_CV)) { 597 u32 it; 598 599 it = compat_get_it_state(regs); 600 if (!it) 601 return true; 602 603 cond = it >> 4; 604 } else { 605 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; 606 } 607 608 return aarch32_opcode_cond_checks[cond](regs->pstate); 609 } 610 611 static void advance_itstate(struct pt_regs *regs) 612 { 613 u32 it; 614 615 /* ARM mode */ 616 if (!(regs->pstate & PSR_AA32_T_BIT) || 617 !(regs->pstate & PSR_AA32_IT_MASK)) 618 return; 619 620 it = compat_get_it_state(regs); 621 622 /* 623 * If this is the last instruction of the block, wipe the IT 624 * state. Otherwise advance it. 625 */ 626 if (!(it & 7)) 627 it = 0; 628 else 629 it = (it & 0xe0) | ((it << 1) & 0x1f); 630 631 compat_set_it_state(regs, it); 632 } 633 634 static void arm64_compat_skip_faulting_instruction(struct pt_regs *regs, 635 unsigned int sz) 636 { 637 advance_itstate(regs); 638 arm64_skip_faulting_instruction(regs, sz); 639 } 640 641 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) 642 { 643 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT; 644 645 pt_regs_write_reg(regs, reg, arch_timer_get_rate()); 646 arm64_compat_skip_faulting_instruction(regs, 4); 647 } 648 649 static struct sys64_hook cp15_32_hooks[] = { 650 { 651 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK, 652 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ, 653 .handler = compat_cntfrq_read_handler, 654 }, 655 {}, 656 }; 657 658 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs) 659 { 660 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; 661 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; 662 u64 val = arch_timer_read_counter(); 663 664 pt_regs_write_reg(regs, rt, lower_32_bits(val)); 665 pt_regs_write_reg(regs, rt2, upper_32_bits(val)); 666 arm64_compat_skip_faulting_instruction(regs, 4); 667 } 668 669 static struct sys64_hook cp15_64_hooks[] = { 670 { 671 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, 672 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, 673 .handler = compat_cntvct_read_handler, 674 }, 675 {}, 676 }; 677 678 asmlinkage void __exception do_cp15instr(unsigned int esr, struct pt_regs *regs) 679 { 680 struct sys64_hook *hook, *hook_base; 681 682 if (!cp15_cond_valid(esr, regs)) { 683 /* 684 * There is no T16 variant of a CP access, so we 685 * always advance PC by 4 bytes. 686 */ 687 arm64_compat_skip_faulting_instruction(regs, 4); 688 return; 689 } 690 691 switch (ESR_ELx_EC(esr)) { 692 case ESR_ELx_EC_CP15_32: 693 hook_base = cp15_32_hooks; 694 break; 695 case ESR_ELx_EC_CP15_64: 696 hook_base = cp15_64_hooks; 697 break; 698 default: 699 do_undefinstr(regs); 700 return; 701 } 702 703 for (hook = hook_base; hook->handler; hook++) 704 if ((hook->esr_mask & esr) == hook->esr_val) { 705 hook->handler(esr, regs); 706 return; 707 } 708 709 /* 710 * New cp15 instructions may previously have been undefined at 711 * EL0. Fall back to our usual undefined instruction handler 712 * so that we handle these consistently. 713 */ 714 do_undefinstr(regs); 715 } 716 #endif 717 718 asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs) 719 { 720 struct sys64_hook *hook; 721 722 for (hook = sys64_hooks; hook->handler; hook++) 723 if ((hook->esr_mask & esr) == hook->esr_val) { 724 hook->handler(esr, regs); 725 return; 726 } 727 728 /* 729 * New SYS instructions may previously have been undefined at EL0. Fall 730 * back to our usual undefined instruction handler so that we handle 731 * these consistently. 732 */ 733 do_undefinstr(regs); 734 } 735 736 static const char *esr_class_str[] = { 737 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", 738 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", 739 [ESR_ELx_EC_WFx] = "WFI/WFE", 740 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", 741 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", 742 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", 743 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", 744 [ESR_ELx_EC_FP_ASIMD] = "ASIMD", 745 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", 746 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", 747 [ESR_ELx_EC_ILL] = "PSTATE.IL", 748 [ESR_ELx_EC_SVC32] = "SVC (AArch32)", 749 [ESR_ELx_EC_HVC32] = "HVC (AArch32)", 750 [ESR_ELx_EC_SMC32] = "SMC (AArch32)", 751 [ESR_ELx_EC_SVC64] = "SVC (AArch64)", 752 [ESR_ELx_EC_HVC64] = "HVC (AArch64)", 753 [ESR_ELx_EC_SMC64] = "SMC (AArch64)", 754 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", 755 [ESR_ELx_EC_SVE] = "SVE", 756 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", 757 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", 758 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", 759 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", 760 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", 761 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", 762 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", 763 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", 764 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", 765 [ESR_ELx_EC_SERROR] = "SError", 766 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", 767 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", 768 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", 769 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", 770 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", 771 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", 772 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", 773 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", 774 [ESR_ELx_EC_BRK64] = "BRK (AArch64)", 775 }; 776 777 const char *esr_get_class_string(u32 esr) 778 { 779 return esr_class_str[ESR_ELx_EC(esr)]; 780 } 781 782 /* 783 * bad_mode handles the impossible case in the exception vector. This is always 784 * fatal. 785 */ 786 asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) 787 { 788 console_verbose(); 789 790 pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", 791 handler[reason], smp_processor_id(), esr, 792 esr_get_class_string(esr)); 793 794 local_daif_mask(); 795 panic("bad mode"); 796 } 797 798 /* 799 * bad_el0_sync handles unexpected, but potentially recoverable synchronous 800 * exceptions taken from EL0. Unlike bad_mode, this returns. 801 */ 802 asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) 803 { 804 void __user *pc = (void __user *)instruction_pointer(regs); 805 806 current->thread.fault_address = 0; 807 current->thread.fault_code = esr; 808 809 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc, 810 "Bad EL0 synchronous exception"); 811 } 812 813 #ifdef CONFIG_VMAP_STACK 814 815 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) 816 __aligned(16); 817 818 asmlinkage void handle_bad_stack(struct pt_regs *regs) 819 { 820 unsigned long tsk_stk = (unsigned long)current->stack; 821 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); 822 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 823 unsigned int esr = read_sysreg(esr_el1); 824 unsigned long far = read_sysreg(far_el1); 825 826 console_verbose(); 827 pr_emerg("Insufficient stack space to handle exception!"); 828 829 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr)); 830 pr_emerg("FAR: 0x%016lx\n", far); 831 832 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 833 tsk_stk, tsk_stk + THREAD_SIZE); 834 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", 835 irq_stk, irq_stk + THREAD_SIZE); 836 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 837 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 838 839 __show_regs(regs); 840 841 /* 842 * We use nmi_panic to limit the potential for recusive overflows, and 843 * to get a better stack trace. 844 */ 845 nmi_panic(NULL, "kernel stack overflow"); 846 cpu_park_loop(); 847 } 848 #endif 849 850 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr) 851 { 852 console_verbose(); 853 854 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n", 855 smp_processor_id(), esr, esr_get_class_string(esr)); 856 if (regs) 857 __show_regs(regs); 858 859 nmi_panic(regs, "Asynchronous SError Interrupt"); 860 861 cpu_park_loop(); 862 unreachable(); 863 } 864 865 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) 866 { 867 u32 aet = arm64_ras_serror_get_severity(esr); 868 869 switch (aet) { 870 case ESR_ELx_AET_CE: /* corrected error */ 871 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */ 872 /* 873 * The CPU can make progress. We may take UEO again as 874 * a more severe error. 875 */ 876 return false; 877 878 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */ 879 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */ 880 /* 881 * The CPU can't make progress. The exception may have 882 * been imprecise. 883 */ 884 return true; 885 886 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */ 887 default: 888 /* Error has been silently propagated */ 889 arm64_serror_panic(regs, esr); 890 } 891 } 892 893 asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr) 894 { 895 const bool was_in_nmi = in_nmi(); 896 897 if (!was_in_nmi) 898 nmi_enter(); 899 900 /* non-RAS errors are not containable */ 901 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) 902 arm64_serror_panic(regs, esr); 903 904 if (!was_in_nmi) 905 nmi_exit(); 906 } 907 908 void __pte_error(const char *file, int line, unsigned long val) 909 { 910 pr_err("%s:%d: bad pte %016lx.\n", file, line, val); 911 } 912 913 void __pmd_error(const char *file, int line, unsigned long val) 914 { 915 pr_err("%s:%d: bad pmd %016lx.\n", file, line, val); 916 } 917 918 void __pud_error(const char *file, int line, unsigned long val) 919 { 920 pr_err("%s:%d: bad pud %016lx.\n", file, line, val); 921 } 922 923 void __pgd_error(const char *file, int line, unsigned long val) 924 { 925 pr_err("%s:%d: bad pgd %016lx.\n", file, line, val); 926 } 927 928 /* GENERIC_BUG traps */ 929 930 int is_valid_bugaddr(unsigned long addr) 931 { 932 /* 933 * bug_handler() only called for BRK #BUG_BRK_IMM. 934 * So the answer is trivial -- any spurious instances with no 935 * bug table entry will be rejected by report_bug() and passed 936 * back to the debug-monitors code and handled as a fatal 937 * unexpected debug exception. 938 */ 939 return 1; 940 } 941 942 static int bug_handler(struct pt_regs *regs, unsigned int esr) 943 { 944 switch (report_bug(regs->pc, regs)) { 945 case BUG_TRAP_TYPE_BUG: 946 die("Oops - BUG", regs, 0); 947 break; 948 949 case BUG_TRAP_TYPE_WARN: 950 break; 951 952 default: 953 /* unknown/unrecognised bug trap type */ 954 return DBG_HOOK_ERROR; 955 } 956 957 /* If thread survives, skip over the BUG instruction and continue: */ 958 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 959 return DBG_HOOK_HANDLED; 960 } 961 962 static struct break_hook bug_break_hook = { 963 .fn = bug_handler, 964 .imm = BUG_BRK_IMM, 965 }; 966 967 #ifdef CONFIG_KASAN_SW_TAGS 968 969 #define KASAN_ESR_RECOVER 0x20 970 #define KASAN_ESR_WRITE 0x10 971 #define KASAN_ESR_SIZE_MASK 0x0f 972 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK)) 973 974 static int kasan_handler(struct pt_regs *regs, unsigned int esr) 975 { 976 bool recover = esr & KASAN_ESR_RECOVER; 977 bool write = esr & KASAN_ESR_WRITE; 978 size_t size = KASAN_ESR_SIZE(esr); 979 u64 addr = regs->regs[0]; 980 u64 pc = regs->pc; 981 982 kasan_report(addr, size, write, pc); 983 984 /* 985 * The instrumentation allows to control whether we can proceed after 986 * a crash was detected. This is done by passing the -recover flag to 987 * the compiler. Disabling recovery allows to generate more compact 988 * code. 989 * 990 * Unfortunately disabling recovery doesn't work for the kernel right 991 * now. KASAN reporting is disabled in some contexts (for example when 992 * the allocator accesses slab object metadata; this is controlled by 993 * current->kasan_depth). All these accesses are detected by the tool, 994 * even though the reports for them are not printed. 995 * 996 * This is something that might be fixed at some point in the future. 997 */ 998 if (!recover) 999 die("Oops - KASAN", regs, 0); 1000 1001 /* If thread survives, skip over the brk instruction and continue: */ 1002 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1003 return DBG_HOOK_HANDLED; 1004 } 1005 1006 static struct break_hook kasan_break_hook = { 1007 .fn = kasan_handler, 1008 .imm = KASAN_BRK_IMM, 1009 .mask = KASAN_BRK_MASK, 1010 }; 1011 #endif 1012 1013 /* 1014 * Initial handler for AArch64 BRK exceptions 1015 * This handler only used until debug_traps_init(). 1016 */ 1017 int __init early_brk64(unsigned long addr, unsigned int esr, 1018 struct pt_regs *regs) 1019 { 1020 #ifdef CONFIG_KASAN_SW_TAGS 1021 unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; 1022 1023 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) 1024 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; 1025 #endif 1026 return bug_handler(regs, esr) != DBG_HOOK_HANDLED; 1027 } 1028 1029 /* This registration must happen early, before debug_traps_init(). */ 1030 void __init trap_init(void) 1031 { 1032 register_kernel_break_hook(&bug_break_hook); 1033 #ifdef CONFIG_KASAN_SW_TAGS 1034 register_kernel_break_hook(&kasan_break_hook); 1035 #endif 1036 } 1037