fault.c (de4fb176622d54a82ea3ceb7362392aaf5ff0b5a) fault.c (8d56e5c5a99ce1d17d39ce5a8260e42c2a2d7682)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Based on arch/arm/mm/fault.c
4 *
5 * Copyright (C) 1995 Linus Torvalds
6 * Copyright (C) 1995-2004 Russell King
7 * Copyright (C) 2012 ARM Ltd.
8 */

--- 29 unchanged lines hidden (view full) ---

38#include <asm/mte.h>
39#include <asm/processor.h>
40#include <asm/sysreg.h>
41#include <asm/system_misc.h>
42#include <asm/tlbflush.h>
43#include <asm/traps.h>
44
45struct fault_info {
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Based on arch/arm/mm/fault.c
4 *
5 * Copyright (C) 1995 Linus Torvalds
6 * Copyright (C) 1995-2004 Russell King
7 * Copyright (C) 2012 ARM Ltd.
8 */

--- 29 unchanged lines hidden (view full) ---

38#include <asm/mte.h>
39#include <asm/processor.h>
40#include <asm/sysreg.h>
41#include <asm/system_misc.h>
42#include <asm/tlbflush.h>
43#include <asm/traps.h>
44
45struct fault_info {
46 int (*fn)(unsigned long far, unsigned int esr,
46 int (*fn)(unsigned long far, unsigned long esr,
47 struct pt_regs *regs);
48 int sig;
49 int code;
50 const char *name;
51};
52
53static const struct fault_info fault_info[];
54static struct fault_info debug_fault_info[];
55
47 struct pt_regs *regs);
48 int sig;
49 int code;
50 const char *name;
51};
52
53static const struct fault_info fault_info[];
54static struct fault_info debug_fault_info[];
55
56static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
56static inline const struct fault_info *esr_to_fault_info(unsigned long esr)
57{
58 return fault_info + (esr & ESR_ELx_FSC);
59}
60
57{
58 return fault_info + (esr & ESR_ELx_FSC);
59}
60
61static inline const struct fault_info *esr_to_debug_fault_info(unsigned int esr)
61static inline const struct fault_info *esr_to_debug_fault_info(unsigned long esr)
62{
63 return debug_fault_info + DBG_ESR_EVT(esr);
64}
65
62{
63 return debug_fault_info + DBG_ESR_EVT(esr);
64}
65
66static void data_abort_decode(unsigned int esr)
66static void data_abort_decode(unsigned long esr)
67{
68 pr_alert("Data abort info:\n");
69
70 if (esr & ESR_ELx_ISV) {
71 pr_alert(" Access size = %u byte(s)\n",
72 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
73 pr_alert(" SSE = %lu, SRT = %lu\n",
74 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,

--- 5 unchanged lines hidden (view full) ---

80 pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK);
81 }
82
83 pr_alert(" CM = %lu, WnR = %lu\n",
84 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
85 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
86}
87
67{
68 pr_alert("Data abort info:\n");
69
70 if (esr & ESR_ELx_ISV) {
71 pr_alert(" Access size = %u byte(s)\n",
72 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
73 pr_alert(" SSE = %lu, SRT = %lu\n",
74 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,

--- 5 unchanged lines hidden (view full) ---

80 pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK);
81 }
82
83 pr_alert(" CM = %lu, WnR = %lu\n",
84 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
85 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
86}
87
88static void mem_abort_decode(unsigned int esr)
88static void mem_abort_decode(unsigned long esr)
89{
90 pr_alert("Mem abort info:\n");
91
89{
90 pr_alert("Mem abort info:\n");
91
92 pr_alert(" ESR = 0x%08x\n", esr);
92 pr_alert(" ESR = 0x%016lx\n", esr);
93 pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
94 ESR_ELx_EC(esr), esr_get_class_string(esr),
95 (esr & ESR_ELx_IL) ? 32 : 16);
96 pr_alert(" SET = %lu, FnV = %lu\n",
97 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
98 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
99 pr_alert(" EA = %lu, S1PTW = %lu\n",
100 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
101 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
93 pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
94 ESR_ELx_EC(esr), esr_get_class_string(esr),
95 (esr & ESR_ELx_IL) ? 32 : 16);
96 pr_alert(" SET = %lu, FnV = %lu\n",
97 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
98 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
99 pr_alert(" EA = %lu, S1PTW = %lu\n",
100 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
101 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
102 pr_alert(" FSC = 0x%02x: %s\n", (esr & ESR_ELx_FSC),
102 pr_alert(" FSC = 0x%02lx: %s\n", (esr & ESR_ELx_FSC),
103 esr_to_fault_info(esr)->name);
104
105 if (esr_is_data_abort(esr))
106 data_abort_decode(esr);
107}
108
109static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
110{

--- 113 unchanged lines hidden (view full) ---

224 } while (pteval != old_pteval);
225
226 /* Invalidate a stale read-only entry */
227 if (dirty)
228 flush_tlb_page(vma, address);
229 return 1;
230}
231
103 esr_to_fault_info(esr)->name);
104
105 if (esr_is_data_abort(esr))
106 data_abort_decode(esr);
107}
108
109static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
110{

--- 113 unchanged lines hidden (view full) ---

224 } while (pteval != old_pteval);
225
226 /* Invalidate a stale read-only entry */
227 if (dirty)
228 flush_tlb_page(vma, address);
229 return 1;
230}
231
232static bool is_el1_instruction_abort(unsigned int esr)
232static bool is_el1_instruction_abort(unsigned long esr)
233{
234 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
235}
236
233{
234 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
235}
236
237static bool is_el1_data_abort(unsigned int esr)
237static bool is_el1_data_abort(unsigned long esr)
238{
239 return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
240}
241
238{
239 return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
240}
241
242static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr,
242static inline bool is_el1_permission_fault(unsigned long addr, unsigned long esr,
243 struct pt_regs *regs)
244{
243 struct pt_regs *regs)
244{
245 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
245 unsigned long fsc_type = esr & ESR_ELx_FSC_TYPE;
246
247 if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
248 return false;
249
250 if (fsc_type == ESR_ELx_FSC_PERM)
251 return true;
252
253 if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
254 return fsc_type == ESR_ELx_FSC_FAULT &&
255 (regs->pstate & PSR_PAN_BIT);
256
257 return false;
258}
259
260static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
246
247 if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
248 return false;
249
250 if (fsc_type == ESR_ELx_FSC_PERM)
251 return true;
252
253 if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
254 return fsc_type == ESR_ELx_FSC_FAULT &&
255 (regs->pstate & PSR_PAN_BIT);
256
257 return false;
258}
259
260static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
261 unsigned int esr,
261 unsigned long esr,
262 struct pt_regs *regs)
263{
264 unsigned long flags;
265 u64 par, dfsc;
266
267 if (!is_el1_data_abort(esr) ||
268 (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
269 return false;

--- 15 unchanged lines hidden (view full) ---

285 * If we got a different type of fault from the AT instruction,
286 * treat the translation fault as spurious.
287 */
288 dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
289 return (dfsc & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT;
290}
291
292static void die_kernel_fault(const char *msg, unsigned long addr,
262 struct pt_regs *regs)
263{
264 unsigned long flags;
265 u64 par, dfsc;
266
267 if (!is_el1_data_abort(esr) ||
268 (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
269 return false;

--- 15 unchanged lines hidden (view full) ---

285 * If we got a different type of fault from the AT instruction,
286 * treat the translation fault as spurious.
287 */
288 dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
289 return (dfsc & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT;
290}
291
292static void die_kernel_fault(const char *msg, unsigned long addr,
293 unsigned int esr, struct pt_regs *regs)
293 unsigned long esr, struct pt_regs *regs)
294{
295 bust_spinlocks(1);
296
297 pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
298 addr);
299
300 kasan_non_canonical_hook(addr);
301
302 mem_abort_decode(esr);
303
304 show_pte(addr);
305 die("Oops", regs, esr);
306 bust_spinlocks(0);
307 make_task_dead(SIGKILL);
308}
309
310#ifdef CONFIG_KASAN_HW_TAGS
294{
295 bust_spinlocks(1);
296
297 pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
298 addr);
299
300 kasan_non_canonical_hook(addr);
301
302 mem_abort_decode(esr);
303
304 show_pte(addr);
305 die("Oops", regs, esr);
306 bust_spinlocks(0);
307 make_task_dead(SIGKILL);
308}
309
310#ifdef CONFIG_KASAN_HW_TAGS
311static void report_tag_fault(unsigned long addr, unsigned int esr,
311static void report_tag_fault(unsigned long addr, unsigned long esr,
312 struct pt_regs *regs)
313{
314 /*
315 * SAS bits aren't set for all faults reported in EL1, so we can't
316 * find out access size.
317 */
318 bool is_write = !!(esr & ESR_ELx_WNR);
319 kasan_report(addr, 0, is_write, regs->pc);
320}
321#else
322/* Tag faults aren't enabled without CONFIG_KASAN_HW_TAGS. */
312 struct pt_regs *regs)
313{
314 /*
315 * SAS bits aren't set for all faults reported in EL1, so we can't
316 * find out access size.
317 */
318 bool is_write = !!(esr & ESR_ELx_WNR);
319 kasan_report(addr, 0, is_write, regs->pc);
320}
321#else
322/* Tag faults aren't enabled without CONFIG_KASAN_HW_TAGS. */
323static inline void report_tag_fault(unsigned long addr, unsigned int esr,
323static inline void report_tag_fault(unsigned long addr, unsigned long esr,
324 struct pt_regs *regs) { }
325#endif
326
324 struct pt_regs *regs) { }
325#endif
326
327static void do_tag_recovery(unsigned long addr, unsigned int esr,
327static void do_tag_recovery(unsigned long addr, unsigned long esr,
328 struct pt_regs *regs)
329{
330
331 report_tag_fault(addr, esr, regs);
332
333 /*
334 * Disable MTE Tag Checking on the local CPU for the current EL.
335 * It will be done lazily on the other CPUs when they will hit a
336 * tag fault.
337 */
338 sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_NONE);
339 isb();
340}
341
328 struct pt_regs *regs)
329{
330
331 report_tag_fault(addr, esr, regs);
332
333 /*
334 * Disable MTE Tag Checking on the local CPU for the current EL.
335 * It will be done lazily on the other CPUs when they will hit a
336 * tag fault.
337 */
338 sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_NONE);
339 isb();
340}
341
342static bool is_el1_mte_sync_tag_check_fault(unsigned int esr)
342static bool is_el1_mte_sync_tag_check_fault(unsigned long esr)
343{
343{
344 unsigned int fsc = esr & ESR_ELx_FSC;
344 unsigned long fsc = esr & ESR_ELx_FSC;
345
346 if (!is_el1_data_abort(esr))
347 return false;
348
349 if (fsc == ESR_ELx_FSC_MTE)
350 return true;
351
352 return false;
353}
354
345
346 if (!is_el1_data_abort(esr))
347 return false;
348
349 if (fsc == ESR_ELx_FSC_MTE)
350 return true;
351
352 return false;
353}
354
355static void __do_kernel_fault(unsigned long addr, unsigned int esr,
355static void __do_kernel_fault(unsigned long addr, unsigned long esr,
356 struct pt_regs *regs)
357{
358 const char *msg;
359
360 /*
361 * Are we prepared to handle this kernel fault?
362 * We are almost certainly not prepared to handle instruction faults.
363 */

--- 24 unchanged lines hidden (view full) ---

388 return;
389
390 msg = "paging request";
391 }
392
393 die_kernel_fault(msg, addr, esr, regs);
394}
395
356 struct pt_regs *regs)
357{
358 const char *msg;
359
360 /*
361 * Are we prepared to handle this kernel fault?
362 * We are almost certainly not prepared to handle instruction faults.
363 */

--- 24 unchanged lines hidden (view full) ---

388 return;
389
390 msg = "paging request";
391 }
392
393 die_kernel_fault(msg, addr, esr, regs);
394}
395
396static void set_thread_esr(unsigned long address, unsigned int esr)
396static void set_thread_esr(unsigned long address, unsigned long esr)
397{
398 current->thread.fault_address = address;
399
400 /*
401 * If the faulting address is in the kernel, we must sanitize the ESR.
402 * From userspace's point of view, kernel-only mappings don't exist
403 * at all, so we report them as level 0 translation faults.
404 * (This is not quite the way that "no mapping there at all" behaves:

--- 31 unchanged lines hidden (view full) ---

436 break;
437 default:
438 /*
439 * This should never happen (entry.S only brings us
440 * into this code for insn and data aborts from a lower
441 * exception level). Fail safe by not providing an ESR
442 * context record at all.
443 */
397{
398 current->thread.fault_address = address;
399
400 /*
401 * If the faulting address is in the kernel, we must sanitize the ESR.
402 * From userspace's point of view, kernel-only mappings don't exist
403 * at all, so we report them as level 0 translation faults.
404 * (This is not quite the way that "no mapping there at all" behaves:

--- 31 unchanged lines hidden (view full) ---

436 break;
437 default:
438 /*
439 * This should never happen (entry.S only brings us
440 * into this code for insn and data aborts from a lower
441 * exception level). Fail safe by not providing an ESR
442 * context record at all.
443 */
444 WARN(1, "ESR 0x%x is not DABT or IABT from EL0\n", esr);
444 WARN(1, "ESR 0x%lx is not DABT or IABT from EL0\n", esr);
445 esr = 0;
446 break;
447 }
448 }
449
450 current->thread.fault_code = esr;
451}
452
445 esr = 0;
446 break;
447 }
448 }
449
450 current->thread.fault_code = esr;
451}
452
453static void do_bad_area(unsigned long far, unsigned int esr,
453static void do_bad_area(unsigned long far, unsigned long esr,
454 struct pt_regs *regs)
455{
456 unsigned long addr = untagged_addr(far);
457
458 /*
459 * If we are in kernel mode at this point, we have no context to
460 * handle this fault with.
461 */

--- 34 unchanged lines hidden (view full) ---

496 * Check that the permissions on the VMA allow for the fault which
497 * occurred.
498 */
499 if (!(vma->vm_flags & vm_flags))
500 return VM_FAULT_BADACCESS;
501 return handle_mm_fault(vma, addr, mm_flags, regs);
502}
503
454 struct pt_regs *regs)
455{
456 unsigned long addr = untagged_addr(far);
457
458 /*
459 * If we are in kernel mode at this point, we have no context to
460 * handle this fault with.
461 */

--- 34 unchanged lines hidden (view full) ---

496 * Check that the permissions on the VMA allow for the fault which
497 * occurred.
498 */
499 if (!(vma->vm_flags & vm_flags))
500 return VM_FAULT_BADACCESS;
501 return handle_mm_fault(vma, addr, mm_flags, regs);
502}
503
504static bool is_el0_instruction_abort(unsigned int esr)
504static bool is_el0_instruction_abort(unsigned long esr)
505{
506 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
507}
508
509/*
510 * Note: not valid for EL1 DC IVAC, but we never use that such that it
511 * should fault. EL0 cannot issue DC IVAC (undef).
512 */
505{
506 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
507}
508
509/*
510 * Note: not valid for EL1 DC IVAC, but we never use that such that it
511 * should fault. EL0 cannot issue DC IVAC (undef).
512 */
513static bool is_write_abort(unsigned int esr)
513static bool is_write_abort(unsigned long esr)
514{
515 return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
516}
517
514{
515 return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
516}
517
518static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
518static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
519 struct pt_regs *regs)
520{
521 const struct fault_info *inf;
522 struct mm_struct *mm = current->mm;
523 vm_fault_t fault;
524 unsigned long vm_flags;
525 unsigned int mm_flags = FAULT_FLAG_DEFAULT;
526 unsigned long addr = untagged_addr(far);

--- 139 unchanged lines hidden (view full) ---

666 return 0;
667
668no_context:
669 __do_kernel_fault(addr, esr, regs);
670 return 0;
671}
672
673static int __kprobes do_translation_fault(unsigned long far,
519 struct pt_regs *regs)
520{
521 const struct fault_info *inf;
522 struct mm_struct *mm = current->mm;
523 vm_fault_t fault;
524 unsigned long vm_flags;
525 unsigned int mm_flags = FAULT_FLAG_DEFAULT;
526 unsigned long addr = untagged_addr(far);

--- 139 unchanged lines hidden (view full) ---

666 return 0;
667
668no_context:
669 __do_kernel_fault(addr, esr, regs);
670 return 0;
671}
672
673static int __kprobes do_translation_fault(unsigned long far,
674 unsigned int esr,
674 unsigned long esr,
675 struct pt_regs *regs)
676{
677 unsigned long addr = untagged_addr(far);
678
679 if (is_ttbr0_addr(addr))
680 return do_page_fault(far, esr, regs);
681
682 do_bad_area(far, esr, regs);
683 return 0;
684}
685
675 struct pt_regs *regs)
676{
677 unsigned long addr = untagged_addr(far);
678
679 if (is_ttbr0_addr(addr))
680 return do_page_fault(far, esr, regs);
681
682 do_bad_area(far, esr, regs);
683 return 0;
684}
685
686static int do_alignment_fault(unsigned long far, unsigned int esr,
686static int do_alignment_fault(unsigned long far, unsigned long esr,
687 struct pt_regs *regs)
688{
689 do_bad_area(far, esr, regs);
690 return 0;
691}
692
687 struct pt_regs *regs)
688{
689 do_bad_area(far, esr, regs);
690 return 0;
691}
692
693static int do_bad(unsigned long far, unsigned int esr, struct pt_regs *regs)
693static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
694{
695 return 1; /* "fault" */
696}
697
694{
695 return 1; /* "fault" */
696}
697
698static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs)
698static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
699{
700 const struct fault_info *inf;
701 unsigned long siaddr;
702
703 inf = esr_to_fault_info(esr);
704
705 if (user_mode(regs) && apei_claim_sea(regs) == 0) {
706 /*

--- 13 unchanged lines hidden (view full) ---

720 */
721 siaddr = untagged_addr(far);
722 }
723 arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
724
725 return 0;
726}
727
699{
700 const struct fault_info *inf;
701 unsigned long siaddr;
702
703 inf = esr_to_fault_info(esr);
704
705 if (user_mode(regs) && apei_claim_sea(regs) == 0) {
706 /*

--- 13 unchanged lines hidden (view full) ---

720 */
721 siaddr = untagged_addr(far);
722 }
723 arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
724
725 return 0;
726}
727
728static int do_tag_check_fault(unsigned long far, unsigned int esr,
728static int do_tag_check_fault(unsigned long far, unsigned long esr,
729 struct pt_regs *regs)
730{
731 /*
732 * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN
733 * for tag check faults. Set them to corresponding bits in the untagged
734 * address.
735 */
736 far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK);

--- 63 unchanged lines hidden (view full) ---

800 { do_bad, SIGKILL, SI_KERNEL, "unknown 58" },
801 { do_bad, SIGKILL, SI_KERNEL, "unknown 59" },
802 { do_bad, SIGKILL, SI_KERNEL, "unknown 60" },
803 { do_bad, SIGKILL, SI_KERNEL, "section domain fault" },
804 { do_bad, SIGKILL, SI_KERNEL, "page domain fault" },
805 { do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
806};
807
729 struct pt_regs *regs)
730{
731 /*
732 * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN
733 * for tag check faults. Set them to corresponding bits in the untagged
734 * address.
735 */
736 far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK);

--- 63 unchanged lines hidden (view full) ---

800 { do_bad, SIGKILL, SI_KERNEL, "unknown 58" },
801 { do_bad, SIGKILL, SI_KERNEL, "unknown 59" },
802 { do_bad, SIGKILL, SI_KERNEL, "unknown 60" },
803 { do_bad, SIGKILL, SI_KERNEL, "section domain fault" },
804 { do_bad, SIGKILL, SI_KERNEL, "page domain fault" },
805 { do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
806};
807
808void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs)
808void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs)
809{
810 const struct fault_info *inf = esr_to_fault_info(esr);
811 unsigned long addr = untagged_addr(far);
812
813 if (!inf->fn(far, esr, regs))
814 return;
815
816 if (!user_mode(regs))
817 die_kernel_fault(inf->name, addr, esr, regs);
818
819 /*
820 * At this point we have an unrecognized fault type whose tag bits may
821 * have been defined as UNKNOWN. Therefore we only expose the untagged
822 * address to the signal handler.
823 */
824 arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
825}
826NOKPROBE_SYMBOL(do_mem_abort);
827
809{
810 const struct fault_info *inf = esr_to_fault_info(esr);
811 unsigned long addr = untagged_addr(far);
812
813 if (!inf->fn(far, esr, regs))
814 return;
815
816 if (!user_mode(regs))
817 die_kernel_fault(inf->name, addr, esr, regs);
818
819 /*
820 * At this point we have an unrecognized fault type whose tag bits may
821 * have been defined as UNKNOWN. Therefore we only expose the untagged
822 * address to the signal handler.
823 */
824 arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
825}
826NOKPROBE_SYMBOL(do_mem_abort);
827
828void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
828void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
829{
830 arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
831 addr, esr);
832}
833NOKPROBE_SYMBOL(do_sp_pc_abort);
834
829{
830 arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
831 addr, esr);
832}
833NOKPROBE_SYMBOL(do_sp_pc_abort);
834
835int __init early_brk64(unsigned long addr, unsigned int esr,
835int __init early_brk64(unsigned long addr, unsigned long esr,
836 struct pt_regs *regs);
837
838/*
839 * __refdata because early_brk64 is __init, but the reference to it is
840 * clobbered at arch_initcall time.
841 * See traps.c and debug-monitors.c:debug_traps_init().
842 */
843static struct fault_info __refdata debug_fault_info[] = {
844 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
845 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
846 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
847 { do_bad, SIGKILL, SI_KERNEL, "unknown 3" },
848 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
849 { do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" },
850 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
851 { do_bad, SIGKILL, SI_KERNEL, "unknown 7" },
852};
853
854void __init hook_debug_fault_code(int nr,
836 struct pt_regs *regs);
837
838/*
839 * __refdata because early_brk64 is __init, but the reference to it is
840 * clobbered at arch_initcall time.
841 * See traps.c and debug-monitors.c:debug_traps_init().
842 */
843static struct fault_info __refdata debug_fault_info[] = {
844 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
845 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
846 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
847 { do_bad, SIGKILL, SI_KERNEL, "unknown 3" },
848 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
849 { do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" },
850 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
851 { do_bad, SIGKILL, SI_KERNEL, "unknown 7" },
852};
853
854void __init hook_debug_fault_code(int nr,
855 int (*fn)(unsigned long, unsigned int, struct pt_regs *),
855 int (*fn)(unsigned long, unsigned long, struct pt_regs *),
856 int sig, int code, const char *name)
857{
858 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
859
860 debug_fault_info[nr].fn = fn;
861 debug_fault_info[nr].sig = sig;
862 debug_fault_info[nr].code = code;
863 debug_fault_info[nr].name = name;

--- 16 unchanged lines hidden (view full) ---

880NOKPROBE_SYMBOL(debug_exception_enter);
881
882static void debug_exception_exit(struct pt_regs *regs)
883{
884 preempt_enable_no_resched();
885}
886NOKPROBE_SYMBOL(debug_exception_exit);
887
856 int sig, int code, const char *name)
857{
858 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
859
860 debug_fault_info[nr].fn = fn;
861 debug_fault_info[nr].sig = sig;
862 debug_fault_info[nr].code = code;
863 debug_fault_info[nr].name = name;

--- 16 unchanged lines hidden (view full) ---

880NOKPROBE_SYMBOL(debug_exception_enter);
881
882static void debug_exception_exit(struct pt_regs *regs)
883{
884 preempt_enable_no_resched();
885}
886NOKPROBE_SYMBOL(debug_exception_exit);
887
888void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
888void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
889 struct pt_regs *regs)
890{
891 const struct fault_info *inf = esr_to_debug_fault_info(esr);
892 unsigned long pc = instruction_pointer(regs);
893
894 debug_exception_enter(regs);
895
896 if (user_mode(regs) && !is_ttbr0_addr(pc))

--- 35 unchanged lines hidden ---
889 struct pt_regs *regs)
890{
891 const struct fault_info *inf = esr_to_debug_fault_info(esr);
892 unsigned long pc = instruction_pointer(regs);
893
894 debug_exception_enter(regs);
895
896 if (user_mode(regs) && !is_ttbr0_addr(pc))

--- 35 unchanged lines hidden ---