traps.c (b4adc83b07706042ad6e6a767f6c04636db69bcc) traps.c (8d56e5c5a99ce1d17d39ce5a8260e42c2a2d7682)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Based on arch/arm/kernel/traps.c
4 *
5 * Copyright (C) 1995-2009 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8

--- 228 unchanged lines hidden (view full) ---

237 make_task_dead(SIGSEGV);
238}
239
240static void arm64_show_signal(int signo, const char *str)
241{
242 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
243 DEFAULT_RATELIMIT_BURST);
244 struct task_struct *tsk = current;
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Based on arch/arm/kernel/traps.c
4 *
5 * Copyright (C) 1995-2009 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8

--- 228 unchanged lines hidden (view full) ---

237 make_task_dead(SIGSEGV);
238}
239
240static void arm64_show_signal(int signo, const char *str)
241{
242 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
243 DEFAULT_RATELIMIT_BURST);
244 struct task_struct *tsk = current;
245 unsigned int esr = tsk->thread.fault_code;
245 unsigned long esr = tsk->thread.fault_code;
246 struct pt_regs *regs = task_pt_regs(tsk);
247
248 /* Leave if the signal won't be shown */
249 if (!show_unhandled_signals ||
250 !unhandled_signal(tsk, signo) ||
251 !__ratelimit(&rs))
252 return;
253
254 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
255 if (esr)
246 struct pt_regs *regs = task_pt_regs(tsk);
247
248 /* Leave if the signal won't be shown */
249 if (!show_unhandled_signals ||
250 !unhandled_signal(tsk, signo) ||
251 !__ratelimit(&rs))
252 return;
253
254 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
255 if (esr)
256 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr);
256 pr_cont("%s, ESR 0x%016lx, ", esr_get_class_string(esr), esr);
257
258 pr_cont("%s", str);
259 print_vma_addr(KERN_CONT " in ", regs->pc);
260 pr_cont("\n");
261 __show_regs(regs);
262}
263
264void arm64_force_sig_fault(int signo, int code, unsigned long far,

--- 17 unchanged lines hidden (view full) ---

282 const char *str)
283{
284 arm64_show_signal(SIGTRAP, str);
285 force_sig_ptrace_errno_trap(errno, (void __user *)far);
286}
287
288void arm64_notify_die(const char *str, struct pt_regs *regs,
289 int signo, int sicode, unsigned long far,
257
258 pr_cont("%s", str);
259 print_vma_addr(KERN_CONT " in ", regs->pc);
260 pr_cont("\n");
261 __show_regs(regs);
262}
263
264void arm64_force_sig_fault(int signo, int code, unsigned long far,

--- 17 unchanged lines hidden (view full) ---

282 const char *str)
283{
284 arm64_show_signal(SIGTRAP, str);
285 force_sig_ptrace_errno_trap(errno, (void __user *)far);
286}
287
288void arm64_notify_die(const char *str, struct pt_regs *regs,
289 int signo, int sicode, unsigned long far,
290 int err)
290 unsigned long err)
291{
292 if (user_mode(regs)) {
293 WARN_ON(regs != current_pt_regs());
294 current->thread.fault_address = 0;
295 current->thread.fault_code = err;
296
297 arm64_force_sig_fault(signo, sicode, far, str);
298 } else {

--- 135 unchanged lines hidden (view full) ---

434 (regs->pstate & hook->pstate_mask) == hook->pstate_val)
435 fn = hook->fn;
436
437 raw_spin_unlock_irqrestore(&undef_lock, flags);
438exit:
439 return fn ? fn(regs, instr) : 1;
440}
441
291{
292 if (user_mode(regs)) {
293 WARN_ON(regs != current_pt_regs());
294 current->thread.fault_address = 0;
295 current->thread.fault_code = err;
296
297 arm64_force_sig_fault(signo, sicode, far, str);
298 } else {

--- 135 unchanged lines hidden (view full) ---

434 (regs->pstate & hook->pstate_mask) == hook->pstate_val)
435 fn = hook->fn;
436
437 raw_spin_unlock_irqrestore(&undef_lock, flags);
438exit:
439 return fn ? fn(regs, instr) : 1;
440}
441
442void force_signal_inject(int signal, int code, unsigned long address, unsigned int err)
442void force_signal_inject(int signal, int code, unsigned long address, unsigned long err)
443{
444 const char *desc;
445 struct pt_regs *regs = current_pt_regs();
446
447 if (WARN_ON(!user_mode(regs)))
448 return;
449
450 switch (signal) {

--- 50 unchanged lines hidden (view full) ---

501
502void do_bti(struct pt_regs *regs)
503{
504 BUG_ON(!user_mode(regs));
505 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
506}
507NOKPROBE_SYMBOL(do_bti);
508
443{
444 const char *desc;
445 struct pt_regs *regs = current_pt_regs();
446
447 if (WARN_ON(!user_mode(regs)))
448 return;
449
450 switch (signal) {

--- 50 unchanged lines hidden (view full) ---

501
502void do_bti(struct pt_regs *regs)
503{
504 BUG_ON(!user_mode(regs));
505 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
506}
507NOKPROBE_SYMBOL(do_bti);
508
509void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr)
509void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr)
510{
511 /*
512 * Unexpected FPAC exception or pointer authentication failure in
513 * the kernel: kill the task before it does any more harm.
514 */
515 BUG_ON(!user_mode(regs));
516 force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
517}

--- 9 unchanged lines hidden (view full) ---

527 " mov %w0, #0\n" \
528 "2:\n" \
529 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \
530 : "=r" (res) \
531 : "r" (address)); \
532 uaccess_ttbr0_disable(); \
533 }
534
510{
511 /*
512 * Unexpected FPAC exception or pointer authentication failure in
513 * the kernel: kill the task before it does any more harm.
514 */
515 BUG_ON(!user_mode(regs));
516 force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
517}

--- 9 unchanged lines hidden (view full) ---

527 " mov %w0, #0\n" \
528 "2:\n" \
529 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \
530 : "=r" (res) \
531 : "r" (address)); \
532 uaccess_ttbr0_disable(); \
533 }
534
535static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
535static void user_cache_maint_handler(unsigned long esr, struct pt_regs *regs)
536{
537 unsigned long tagged_address, address;
538 int rt = ESR_ELx_SYS64_ISS_RT(esr);
539 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
540 int ret = 0;
541
542 tagged_address = pt_regs_read_reg(regs, rt);
543 address = untagged_addr(tagged_address);

--- 23 unchanged lines hidden (view full) ---

567 }
568
569 if (ret)
570 arm64_notify_segfault(tagged_address);
571 else
572 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
573}
574
536{
537 unsigned long tagged_address, address;
538 int rt = ESR_ELx_SYS64_ISS_RT(esr);
539 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
540 int ret = 0;
541
542 tagged_address = pt_regs_read_reg(regs, rt);
543 address = untagged_addr(tagged_address);

--- 23 unchanged lines hidden (view full) ---

567 }
568
569 if (ret)
570 arm64_notify_segfault(tagged_address);
571 else
572 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
573}
574
575static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
575static void ctr_read_handler(unsigned long esr, struct pt_regs *regs)
576{
577 int rt = ESR_ELx_SYS64_ISS_RT(esr);
578 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
579
580 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) {
581 /* Hide DIC so that we can trap the unnecessary maintenance...*/
582 val &= ~BIT(CTR_DIC_SHIFT);
583
584 /* ... and fake IminLine to reduce the number of traps. */
585 val &= ~CTR_IMINLINE_MASK;
586 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK;
587 }
588
589 pt_regs_write_reg(regs, rt, val);
590
591 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
592}
593
576{
577 int rt = ESR_ELx_SYS64_ISS_RT(esr);
578 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
579
580 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) {
581 /* Hide DIC so that we can trap the unnecessary maintenance...*/
582 val &= ~BIT(CTR_DIC_SHIFT);
583
584 /* ... and fake IminLine to reduce the number of traps. */
585 val &= ~CTR_IMINLINE_MASK;
586 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK;
587 }
588
589 pt_regs_write_reg(regs, rt, val);
590
591 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
592}
593
594static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
594static void cntvct_read_handler(unsigned long esr, struct pt_regs *regs)
595{
596 int rt = ESR_ELx_SYS64_ISS_RT(esr);
597
598 pt_regs_write_reg(regs, rt, arch_timer_read_counter());
599 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
600}
601
595{
596 int rt = ESR_ELx_SYS64_ISS_RT(esr);
597
598 pt_regs_write_reg(regs, rt, arch_timer_read_counter());
599 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
600}
601
602static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
602static void cntfrq_read_handler(unsigned long esr, struct pt_regs *regs)
603{
604 int rt = ESR_ELx_SYS64_ISS_RT(esr);
605
606 pt_regs_write_reg(regs, rt, arch_timer_get_rate());
607 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
608}
609
603{
604 int rt = ESR_ELx_SYS64_ISS_RT(esr);
605
606 pt_regs_write_reg(regs, rt, arch_timer_get_rate());
607 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
608}
609
610static void mrs_handler(unsigned int esr, struct pt_regs *regs)
610static void mrs_handler(unsigned long esr, struct pt_regs *regs)
611{
612 u32 sysreg, rt;
613
614 rt = ESR_ELx_SYS64_ISS_RT(esr);
615 sysreg = esr_sys64_to_sysreg(esr);
616
617 if (do_emulate_mrs(regs, sysreg, rt) != 0)
618 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
619}
620
611{
612 u32 sysreg, rt;
613
614 rt = ESR_ELx_SYS64_ISS_RT(esr);
615 sysreg = esr_sys64_to_sysreg(esr);
616
617 if (do_emulate_mrs(regs, sysreg, rt) != 0)
618 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
619}
620
621static void wfi_handler(unsigned int esr, struct pt_regs *regs)
621static void wfi_handler(unsigned long esr, struct pt_regs *regs)
622{
623 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
624}
625
626struct sys64_hook {
622{
623 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
624}
625
626struct sys64_hook {
627 unsigned int esr_mask;
628 unsigned int esr_val;
629 void (*handler)(unsigned int esr, struct pt_regs *regs);
627 unsigned long esr_mask;
628 unsigned long esr_val;
629 void (*handler)(unsigned long esr, struct pt_regs *regs);
630};
631
632static const struct sys64_hook sys64_hooks[] = {
633 {
634 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
635 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
636 .handler = user_cache_maint_handler,
637 },

--- 32 unchanged lines hidden (view full) ---

670 .esr_mask = ESR_ELx_WFx_MASK,
671 .esr_val = ESR_ELx_WFx_WFI_VAL,
672 .handler = wfi_handler,
673 },
674 {},
675};
676
677#ifdef CONFIG_COMPAT
630};
631
632static const struct sys64_hook sys64_hooks[] = {
633 {
634 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
635 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
636 .handler = user_cache_maint_handler,
637 },

--- 32 unchanged lines hidden (view full) ---

670 .esr_mask = ESR_ELx_WFx_MASK,
671 .esr_val = ESR_ELx_WFx_WFI_VAL,
672 .handler = wfi_handler,
673 },
674 {},
675};
676
677#ifdef CONFIG_COMPAT
678static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
678static bool cp15_cond_valid(unsigned long esr, struct pt_regs *regs)
679{
680 int cond;
681
682 /* Only a T32 instruction can trap without CV being set */
683 if (!(esr & ESR_ELx_CV)) {
684 u32 it;
685
686 it = compat_get_it_state(regs);
687 if (!it)
688 return true;
689
690 cond = it >> 4;
691 } else {
692 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
693 }
694
695 return aarch32_opcode_cond_checks[cond](regs->pstate);
696}
697
679{
680 int cond;
681
682 /* Only a T32 instruction can trap without CV being set */
683 if (!(esr & ESR_ELx_CV)) {
684 u32 it;
685
686 it = compat_get_it_state(regs);
687 if (!it)
688 return true;
689
690 cond = it >> 4;
691 } else {
692 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
693 }
694
695 return aarch32_opcode_cond_checks[cond](regs->pstate);
696}
697
698static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
698static void compat_cntfrq_read_handler(unsigned long esr, struct pt_regs *regs)
699{
700 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
701
702 pt_regs_write_reg(regs, reg, arch_timer_get_rate());
703 arm64_skip_faulting_instruction(regs, 4);
704}
705
706static const struct sys64_hook cp15_32_hooks[] = {
707 {
708 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
709 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
710 .handler = compat_cntfrq_read_handler,
711 },
712 {},
713};
714
699{
700 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
701
702 pt_regs_write_reg(regs, reg, arch_timer_get_rate());
703 arm64_skip_faulting_instruction(regs, 4);
704}
705
706static const struct sys64_hook cp15_32_hooks[] = {
707 {
708 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
709 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
710 .handler = compat_cntfrq_read_handler,
711 },
712 {},
713};
714
715static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
715static void compat_cntvct_read_handler(unsigned long esr, struct pt_regs *regs)
716{
717 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
718 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
719 u64 val = arch_timer_read_counter();
720
721 pt_regs_write_reg(regs, rt, lower_32_bits(val));
722 pt_regs_write_reg(regs, rt2, upper_32_bits(val));
723 arm64_skip_faulting_instruction(regs, 4);

--- 8 unchanged lines hidden (view full) ---

732 {
733 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
734 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS,
735 .handler = compat_cntvct_read_handler,
736 },
737 {},
738};
739
716{
717 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
718 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
719 u64 val = arch_timer_read_counter();
720
721 pt_regs_write_reg(regs, rt, lower_32_bits(val));
722 pt_regs_write_reg(regs, rt2, upper_32_bits(val));
723 arm64_skip_faulting_instruction(regs, 4);

--- 8 unchanged lines hidden (view full) ---

732 {
733 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
734 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS,
735 .handler = compat_cntvct_read_handler,
736 },
737 {},
738};
739
740void do_cp15instr(unsigned int esr, struct pt_regs *regs)
740void do_cp15instr(unsigned long esr, struct pt_regs *regs)
741{
742 const struct sys64_hook *hook, *hook_base;
743
744 if (!cp15_cond_valid(esr, regs)) {
745 /*
746 * There is no T16 variant of a CP access, so we
747 * always advance PC by 4 bytes.
748 */

--- 24 unchanged lines hidden (view full) ---

773 * EL0. Fall back to our usual undefined instruction handler
774 * so that we handle these consistently.
775 */
776 do_undefinstr(regs);
777}
778NOKPROBE_SYMBOL(do_cp15instr);
779#endif
780
741{
742 const struct sys64_hook *hook, *hook_base;
743
744 if (!cp15_cond_valid(esr, regs)) {
745 /*
746 * There is no T16 variant of a CP access, so we
747 * always advance PC by 4 bytes.
748 */

--- 24 unchanged lines hidden (view full) ---

773 * EL0. Fall back to our usual undefined instruction handler
774 * so that we handle these consistently.
775 */
776 do_undefinstr(regs);
777}
778NOKPROBE_SYMBOL(do_cp15instr);
779#endif
780
781void do_sysinstr(unsigned int esr, struct pt_regs *regs)
781void do_sysinstr(unsigned long esr, struct pt_regs *regs)
782{
783 const struct sys64_hook *hook;
784
785 for (hook = sys64_hooks; hook->handler; hook++)
786 if ((hook->esr_mask & esr) == hook->esr_val) {
787 hook->handler(esr, regs);
788 return;
789 }

--- 26 unchanged lines hidden (view full) ---

816 [ESR_ELx_EC_SMC32] = "SMC (AArch32)",
817 [ESR_ELx_EC_SVC64] = "SVC (AArch64)",
818 [ESR_ELx_EC_HVC64] = "HVC (AArch64)",
819 [ESR_ELx_EC_SMC64] = "SMC (AArch64)",
820 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
821 [ESR_ELx_EC_SVE] = "SVE",
822 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB",
823 [ESR_ELx_EC_FPAC] = "FPAC",
782{
783 const struct sys64_hook *hook;
784
785 for (hook = sys64_hooks; hook->handler; hook++)
786 if ((hook->esr_mask & esr) == hook->esr_val) {
787 hook->handler(esr, regs);
788 return;
789 }

--- 26 unchanged lines hidden (view full) ---

816 [ESR_ELx_EC_SMC32] = "SMC (AArch32)",
817 [ESR_ELx_EC_SVC64] = "SVC (AArch64)",
818 [ESR_ELx_EC_HVC64] = "HVC (AArch64)",
819 [ESR_ELx_EC_SMC64] = "SMC (AArch64)",
820 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
821 [ESR_ELx_EC_SVE] = "SVE",
822 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB",
823 [ESR_ELx_EC_FPAC] = "FPAC",
824 [ESR_ELx_EC_SME] = "SME",
825 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
826 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
827 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
828 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
829 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
830 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
831 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
832 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",

--- 5 unchanged lines hidden (view full) ---

838 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
839 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
840 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
841 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
842 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
843 [ESR_ELx_EC_BRK64] = "BRK (AArch64)",
844};
845
824 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
825 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
826 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
827 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
828 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
829 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
830 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
831 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",

--- 5 unchanged lines hidden (view full) ---

837 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
838 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
839 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
840 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
841 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
842 [ESR_ELx_EC_BRK64] = "BRK (AArch64)",
843};
844
846const char *esr_get_class_string(u32 esr)
845const char *esr_get_class_string(unsigned long esr)
847{
848 return esr_class_str[ESR_ELx_EC(esr)];
849}
850
851/*
852 * bad_el0_sync handles unexpected, but potentially recoverable synchronous
853 * exceptions taken from EL0.
854 */
846{
847 return esr_class_str[ESR_ELx_EC(esr)];
848}
849
850/*
851 * bad_el0_sync handles unexpected, but potentially recoverable synchronous
852 * exceptions taken from EL0.
853 */
855void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
854void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr)
856{
857 unsigned long pc = instruction_pointer(regs);
858
859 current->thread.fault_address = 0;
860 current->thread.fault_code = esr;
861
862 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
863 "Bad EL0 synchronous exception");
864}
865
866#ifdef CONFIG_VMAP_STACK
867
868DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
869 __aligned(16);
870
855{
856 unsigned long pc = instruction_pointer(regs);
857
858 current->thread.fault_address = 0;
859 current->thread.fault_code = esr;
860
861 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
862 "Bad EL0 synchronous exception");
863}
864
865#ifdef CONFIG_VMAP_STACK
866
867DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
868 __aligned(16);
869
871void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far)
870void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far)
872{
873 unsigned long tsk_stk = (unsigned long)current->stack;
874 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
875 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
876
877 console_verbose();
878 pr_emerg("Insufficient stack space to handle exception!");
879
871{
872 unsigned long tsk_stk = (unsigned long)current->stack;
873 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
874 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
875
876 console_verbose();
877 pr_emerg("Insufficient stack space to handle exception!");
878
880 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
879 pr_emerg("ESR: 0x%016lx -- %s\n", esr, esr_get_class_string(esr));
881 pr_emerg("FAR: 0x%016lx\n", far);
882
883 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
884 tsk_stk, tsk_stk + THREAD_SIZE);
885 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
886 irq_stk, irq_stk + IRQ_STACK_SIZE);
887 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
888 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);

--- 4 unchanged lines hidden (view full) ---

893 * We use nmi_panic to limit the potential for recusive overflows, and
894 * to get a better stack trace.
895 */
896 nmi_panic(NULL, "kernel stack overflow");
897 cpu_park_loop();
898}
899#endif
900
880 pr_emerg("FAR: 0x%016lx\n", far);
881
882 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
883 tsk_stk, tsk_stk + THREAD_SIZE);
884 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
885 irq_stk, irq_stk + IRQ_STACK_SIZE);
886 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
887 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);

--- 4 unchanged lines hidden (view full) ---

892 * We use nmi_panic to limit the potential for recusive overflows, and
893 * to get a better stack trace.
894 */
895 nmi_panic(NULL, "kernel stack overflow");
896 cpu_park_loop();
897}
898#endif
899
901void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
900void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr)
902{
903 console_verbose();
904
901{
902 console_verbose();
903
905 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
904 pr_crit("SError Interrupt on CPU%d, code 0x%016lx -- %s\n",
906 smp_processor_id(), esr, esr_get_class_string(esr));
907 if (regs)
908 __show_regs(regs);
909
910 nmi_panic(regs, "Asynchronous SError Interrupt");
911
912 cpu_park_loop();
913 unreachable();
914}
915
905 smp_processor_id(), esr, esr_get_class_string(esr));
906 if (regs)
907 __show_regs(regs);
908
909 nmi_panic(regs, "Asynchronous SError Interrupt");
910
911 cpu_park_loop();
912 unreachable();
913}
914
916bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
915bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr)
917{
916{
918 u32 aet = arm64_ras_serror_get_severity(esr);
917 unsigned long aet = arm64_ras_serror_get_severity(esr);
919
920 switch (aet) {
921 case ESR_ELx_AET_CE: /* corrected error */
922 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */
923 /*
924 * The CPU can make progress. We may take UEO again as
925 * a more severe error.
926 */

--- 13 unchanged lines hidden (view full) ---

940
941 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */
942 default:
943 /* Error has been silently propagated */
944 arm64_serror_panic(regs, esr);
945 }
946}
947
918
919 switch (aet) {
920 case ESR_ELx_AET_CE: /* corrected error */
921 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */
922 /*
923 * The CPU can make progress. We may take UEO again as
924 * a more severe error.
925 */

--- 13 unchanged lines hidden (view full) ---

939
940 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */
941 default:
942 /* Error has been silently propagated */
943 arm64_serror_panic(regs, esr);
944 }
945}
946
948void do_serror(struct pt_regs *regs, unsigned int esr)
947void do_serror(struct pt_regs *regs, unsigned long esr)
949{
950 /* non-RAS errors are not containable */
951 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
952 arm64_serror_panic(regs, esr);
953}
954
955/* GENERIC_BUG traps */
956

--- 4 unchanged lines hidden (view full) ---

961 * So the answer is trivial -- any spurious instances with no
962 * bug table entry will be rejected by report_bug() and passed
963 * back to the debug-monitors code and handled as a fatal
964 * unexpected debug exception.
965 */
966 return 1;
967}
968
948{
949 /* non-RAS errors are not containable */
950 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
951 arm64_serror_panic(regs, esr);
952}
953
954/* GENERIC_BUG traps */
955

--- 4 unchanged lines hidden (view full) ---

960 * So the answer is trivial -- any spurious instances with no
961 * bug table entry will be rejected by report_bug() and passed
962 * back to the debug-monitors code and handled as a fatal
963 * unexpected debug exception.
964 */
965 return 1;
966}
967
969static int bug_handler(struct pt_regs *regs, unsigned int esr)
968static int bug_handler(struct pt_regs *regs, unsigned long esr)
970{
971 switch (report_bug(regs->pc, regs)) {
972 case BUG_TRAP_TYPE_BUG:
973 die("Oops - BUG", regs, 0);
974 break;
975
976 case BUG_TRAP_TYPE_WARN:
977 break;

--- 8 unchanged lines hidden (view full) ---

986 return DBG_HOOK_HANDLED;
987}
988
989static struct break_hook bug_break_hook = {
990 .fn = bug_handler,
991 .imm = BUG_BRK_IMM,
992};
993
969{
970 switch (report_bug(regs->pc, regs)) {
971 case BUG_TRAP_TYPE_BUG:
972 die("Oops - BUG", regs, 0);
973 break;
974
975 case BUG_TRAP_TYPE_WARN:
976 break;

--- 8 unchanged lines hidden (view full) ---

985 return DBG_HOOK_HANDLED;
986}
987
988static struct break_hook bug_break_hook = {
989 .fn = bug_handler,
990 .imm = BUG_BRK_IMM,
991};
992
994static int reserved_fault_handler(struct pt_regs *regs, unsigned int esr)
993static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr)
995{
996 pr_err("%s generated an invalid instruction at %pS!\n",
997 "Kernel text patching",
998 (void *)instruction_pointer(regs));
999
1000 /* We cannot handle this */
1001 return DBG_HOOK_ERROR;
1002}

--- 5 unchanged lines hidden (view full) ---

1008
1009#ifdef CONFIG_KASAN_SW_TAGS
1010
1011#define KASAN_ESR_RECOVER 0x20
1012#define KASAN_ESR_WRITE 0x10
1013#define KASAN_ESR_SIZE_MASK 0x0f
1014#define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK))
1015
994{
995 pr_err("%s generated an invalid instruction at %pS!\n",
996 "Kernel text patching",
997 (void *)instruction_pointer(regs));
998
999 /* We cannot handle this */
1000 return DBG_HOOK_ERROR;
1001}

--- 5 unchanged lines hidden (view full) ---

1007
1008#ifdef CONFIG_KASAN_SW_TAGS
1009
1010#define KASAN_ESR_RECOVER 0x20
1011#define KASAN_ESR_WRITE 0x10
1012#define KASAN_ESR_SIZE_MASK 0x0f
1013#define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK))
1014
1016static int kasan_handler(struct pt_regs *regs, unsigned int esr)
1015static int kasan_handler(struct pt_regs *regs, unsigned long esr)
1017{
1018 bool recover = esr & KASAN_ESR_RECOVER;
1019 bool write = esr & KASAN_ESR_WRITE;
1020 size_t size = KASAN_ESR_SIZE(esr);
1021 u64 addr = regs->regs[0];
1022 u64 pc = regs->pc;
1023
1024 kasan_report(addr, size, write, pc);

--- 26 unchanged lines hidden (view full) ---

1051 .mask = KASAN_BRK_MASK,
1052};
1053#endif
1054
1055/*
1056 * Initial handler for AArch64 BRK exceptions
1057 * This handler only used until debug_traps_init().
1058 */
1016{
1017 bool recover = esr & KASAN_ESR_RECOVER;
1018 bool write = esr & KASAN_ESR_WRITE;
1019 size_t size = KASAN_ESR_SIZE(esr);
1020 u64 addr = regs->regs[0];
1021 u64 pc = regs->pc;
1022
1023 kasan_report(addr, size, write, pc);

--- 26 unchanged lines hidden (view full) ---

1050 .mask = KASAN_BRK_MASK,
1051};
1052#endif
1053
1054/*
1055 * Initial handler for AArch64 BRK exceptions
1056 * This handler only used until debug_traps_init().
1057 */
1059int __init early_brk64(unsigned long addr, unsigned int esr,
1058int __init early_brk64(unsigned long addr, unsigned long esr,
1060 struct pt_regs *regs)
1061{
1062#ifdef CONFIG_KASAN_SW_TAGS
1059 struct pt_regs *regs)
1060{
1061#ifdef CONFIG_KASAN_SW_TAGS
1063 unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
1062 unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
1064
1065 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1066 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
1067#endif
1068 return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
1069}
1070
1071void __init trap_init(void)
1072{
1073 register_kernel_break_hook(&bug_break_hook);
1074 register_kernel_break_hook(&fault_break_hook);
1075#ifdef CONFIG_KASAN_SW_TAGS
1076 register_kernel_break_hook(&kasan_break_hook);
1077#endif
1078 debug_traps_init();
1079}
1063
1064 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1065 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
1066#endif
1067 return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
1068}
1069
1070void __init trap_init(void)
1071{
1072 register_kernel_break_hook(&bug_break_hook);
1073 register_kernel_break_hook(&fault_break_hook);
1074#ifdef CONFIG_KASAN_SW_TAGS
1075 register_kernel_break_hook(&kasan_break_hook);
1076#endif
1077 debug_traps_init();
1078}