xref: /openbmc/linux/arch/arm64/mm/fault.c (revision 4f205687)
1 /*
2  * Based on arch/arm/mm/fault.c
3  *
4  * Copyright (C) 1995  Linus Torvalds
5  * Copyright (C) 1995-2004 Russell King
6  * Copyright (C) 2012 ARM Ltd.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #include <linux/module.h>
22 #include <linux/signal.h>
23 #include <linux/mm.h>
24 #include <linux/hardirq.h>
25 #include <linux/init.h>
26 #include <linux/kprobes.h>
27 #include <linux/uaccess.h>
28 #include <linux/page-flags.h>
29 #include <linux/sched.h>
30 #include <linux/highmem.h>
31 #include <linux/perf_event.h>
32 
33 #include <asm/cpufeature.h>
34 #include <asm/exception.h>
35 #include <asm/debug-monitors.h>
36 #include <asm/esr.h>
37 #include <asm/sysreg.h>
38 #include <asm/system_misc.h>
39 #include <asm/pgtable.h>
40 #include <asm/tlbflush.h>
41 
42 static const char *fault_name(unsigned int esr);
43 
44 /*
45  * Dump out the page tables associated with 'addr' in mm 'mm'.
46  */
47 void show_pte(struct mm_struct *mm, unsigned long addr)
48 {
49 	pgd_t *pgd;
50 
51 	if (!mm)
52 		mm = &init_mm;
53 
54 	pr_alert("pgd = %p\n", mm->pgd);
55 	pgd = pgd_offset(mm, addr);
56 	pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd));
57 
58 	do {
59 		pud_t *pud;
60 		pmd_t *pmd;
61 		pte_t *pte;
62 
63 		if (pgd_none(*pgd) || pgd_bad(*pgd))
64 			break;
65 
66 		pud = pud_offset(pgd, addr);
67 		printk(", *pud=%016llx", pud_val(*pud));
68 		if (pud_none(*pud) || pud_bad(*pud))
69 			break;
70 
71 		pmd = pmd_offset(pud, addr);
72 		printk(", *pmd=%016llx", pmd_val(*pmd));
73 		if (pmd_none(*pmd) || pmd_bad(*pmd))
74 			break;
75 
76 		pte = pte_offset_map(pmd, addr);
77 		printk(", *pte=%016llx", pte_val(*pte));
78 		pte_unmap(pte);
79 	} while(0);
80 
81 	printk("\n");
82 }
83 
84 #ifdef CONFIG_ARM64_HW_AFDBM
85 /*
86  * This function sets the access flags (dirty, accessed), as well as write
87  * permission, and only to a more permissive setting.
88  *
89  * It needs to cope with hardware update of the accessed/dirty state by other
90  * agents in the system and can safely skip the __sync_icache_dcache() call as,
91  * like set_pte_at(), the PTE is never changed from no-exec to exec here.
92  *
93  * Returns whether or not the PTE actually changed.
94  */
95 int ptep_set_access_flags(struct vm_area_struct *vma,
96 			  unsigned long address, pte_t *ptep,
97 			  pte_t entry, int dirty)
98 {
99 	pteval_t old_pteval;
100 	unsigned int tmp;
101 
102 	if (pte_same(*ptep, entry))
103 		return 0;
104 
105 	/* only preserve the access flags and write permission */
106 	pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY;
107 
108 	/*
109 	 * PTE_RDONLY is cleared by default in the asm below, so set it in
110 	 * back if necessary (read-only or clean PTE).
111 	 */
112 	if (!pte_write(entry) || !pte_sw_dirty(entry))
113 		pte_val(entry) |= PTE_RDONLY;
114 
115 	/*
116 	 * Setting the flags must be done atomically to avoid racing with the
117 	 * hardware update of the access/dirty state.
118 	 */
119 	asm volatile("//	ptep_set_access_flags\n"
120 	"	prfm	pstl1strm, %2\n"
121 	"1:	ldxr	%0, %2\n"
122 	"	and	%0, %0, %3		// clear PTE_RDONLY\n"
123 	"	orr	%0, %0, %4		// set flags\n"
124 	"	stxr	%w1, %0, %2\n"
125 	"	cbnz	%w1, 1b\n"
126 	: "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
127 	: "L" (~PTE_RDONLY), "r" (pte_val(entry)));
128 
129 	flush_tlb_fix_spurious_fault(vma, address);
130 	return 1;
131 }
132 #endif
133 
134 /*
135  * The kernel tried to access some page that wasn't present.
136  */
137 static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
138 			      unsigned int esr, struct pt_regs *regs)
139 {
140 	/*
141 	 * Are we prepared to handle this kernel fault?
142 	 */
143 	if (fixup_exception(regs))
144 		return;
145 
146 	/*
147 	 * No handler, we'll have to terminate things with extreme prejudice.
148 	 */
149 	bust_spinlocks(1);
150 	pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
151 		 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
152 		 "paging request", addr);
153 
154 	show_pte(mm, addr);
155 	die("Oops", regs, esr);
156 	bust_spinlocks(0);
157 	do_exit(SIGKILL);
158 }
159 
160 /*
161  * Something tried to access memory that isn't in our memory map. User mode
162  * accesses just cause a SIGSEGV
163  */
164 static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
165 			    unsigned int esr, unsigned int sig, int code,
166 			    struct pt_regs *regs)
167 {
168 	struct siginfo si;
169 
170 	if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) {
171 		pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n",
172 			tsk->comm, task_pid_nr(tsk), fault_name(esr), sig,
173 			addr, esr);
174 		show_pte(tsk->mm, addr);
175 		show_regs(regs);
176 	}
177 
178 	tsk->thread.fault_address = addr;
179 	tsk->thread.fault_code = esr;
180 	si.si_signo = sig;
181 	si.si_errno = 0;
182 	si.si_code = code;
183 	si.si_addr = (void __user *)addr;
184 	force_sig_info(sig, &si, tsk);
185 }
186 
187 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
188 {
189 	struct task_struct *tsk = current;
190 	struct mm_struct *mm = tsk->active_mm;
191 
192 	/*
193 	 * If we are in kernel mode at this point, we have no context to
194 	 * handle this fault with.
195 	 */
196 	if (user_mode(regs))
197 		__do_user_fault(tsk, addr, esr, SIGSEGV, SEGV_MAPERR, regs);
198 	else
199 		__do_kernel_fault(mm, addr, esr, regs);
200 }
201 
202 #define VM_FAULT_BADMAP		0x010000
203 #define VM_FAULT_BADACCESS	0x020000
204 
205 #define ESR_LNX_EXEC		(1 << 24)
206 
207 static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
208 			   unsigned int mm_flags, unsigned long vm_flags,
209 			   struct task_struct *tsk)
210 {
211 	struct vm_area_struct *vma;
212 	int fault;
213 
214 	vma = find_vma(mm, addr);
215 	fault = VM_FAULT_BADMAP;
216 	if (unlikely(!vma))
217 		goto out;
218 	if (unlikely(vma->vm_start > addr))
219 		goto check_stack;
220 
221 	/*
222 	 * Ok, we have a good vm_area for this memory access, so we can handle
223 	 * it.
224 	 */
225 good_area:
226 	/*
227 	 * Check that the permissions on the VMA allow for the fault which
228 	 * occurred. If we encountered a write or exec fault, we must have
229 	 * appropriate permissions, otherwise we allow any permission.
230 	 */
231 	if (!(vma->vm_flags & vm_flags)) {
232 		fault = VM_FAULT_BADACCESS;
233 		goto out;
234 	}
235 
236 	return handle_mm_fault(mm, vma, addr & PAGE_MASK, mm_flags);
237 
238 check_stack:
239 	if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
240 		goto good_area;
241 out:
242 	return fault;
243 }
244 
245 static inline int permission_fault(unsigned int esr)
246 {
247 	unsigned int ec       = (esr & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT;
248 	unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
249 
250 	return (ec == ESR_ELx_EC_DABT_CUR && fsc_type == ESR_ELx_FSC_PERM);
251 }
252 
253 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
254 				   struct pt_regs *regs)
255 {
256 	struct task_struct *tsk;
257 	struct mm_struct *mm;
258 	int fault, sig, code;
259 	unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
260 	unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
261 
262 	tsk = current;
263 	mm  = tsk->mm;
264 
265 	/*
266 	 * If we're in an interrupt or have no user context, we must not take
267 	 * the fault.
268 	 */
269 	if (faulthandler_disabled() || !mm)
270 		goto no_context;
271 
272 	if (user_mode(regs))
273 		mm_flags |= FAULT_FLAG_USER;
274 
275 	if (esr & ESR_LNX_EXEC) {
276 		vm_flags = VM_EXEC;
277 	} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
278 		vm_flags = VM_WRITE;
279 		mm_flags |= FAULT_FLAG_WRITE;
280 	}
281 
282 	if (permission_fault(esr) && (addr < USER_DS)) {
283 		/* regs->orig_addr_limit may be 0 if we entered from EL0 */
284 		if (regs->orig_addr_limit == KERNEL_DS)
285 			die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
286 
287 		if (!search_exception_tables(regs->pc))
288 			die("Accessing user space memory outside uaccess.h routines", regs, esr);
289 	}
290 
291 	/*
292 	 * As per x86, we may deadlock here. However, since the kernel only
293 	 * validly references user space from well defined areas of the code,
294 	 * we can bug out early if this is from code which shouldn't.
295 	 */
296 	if (!down_read_trylock(&mm->mmap_sem)) {
297 		if (!user_mode(regs) && !search_exception_tables(regs->pc))
298 			goto no_context;
299 retry:
300 		down_read(&mm->mmap_sem);
301 	} else {
302 		/*
303 		 * The above down_read_trylock() might have succeeded in which
304 		 * case, we'll have missed the might_sleep() from down_read().
305 		 */
306 		might_sleep();
307 #ifdef CONFIG_DEBUG_VM
308 		if (!user_mode(regs) && !search_exception_tables(regs->pc))
309 			goto no_context;
310 #endif
311 	}
312 
313 	fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
314 
315 	/*
316 	 * If we need to retry but a fatal signal is pending, handle the
317 	 * signal first. We do not need to release the mmap_sem because it
318 	 * would already be released in __lock_page_or_retry in mm/filemap.c.
319 	 */
320 	if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
321 		return 0;
322 
323 	/*
324 	 * Major/minor page fault accounting is only done on the initial
325 	 * attempt. If we go through a retry, it is extremely likely that the
326 	 * page will be found in page cache at that point.
327 	 */
328 
329 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
330 	if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
331 		if (fault & VM_FAULT_MAJOR) {
332 			tsk->maj_flt++;
333 			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
334 				      addr);
335 		} else {
336 			tsk->min_flt++;
337 			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
338 				      addr);
339 		}
340 		if (fault & VM_FAULT_RETRY) {
341 			/*
342 			 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
343 			 * starvation.
344 			 */
345 			mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
346 			mm_flags |= FAULT_FLAG_TRIED;
347 			goto retry;
348 		}
349 	}
350 
351 	up_read(&mm->mmap_sem);
352 
353 	/*
354 	 * Handle the "normal" case first - VM_FAULT_MAJOR
355 	 */
356 	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
357 			      VM_FAULT_BADACCESS))))
358 		return 0;
359 
360 	/*
361 	 * If we are in kernel mode at this point, we have no context to
362 	 * handle this fault with.
363 	 */
364 	if (!user_mode(regs))
365 		goto no_context;
366 
367 	if (fault & VM_FAULT_OOM) {
368 		/*
369 		 * We ran out of memory, call the OOM killer, and return to
370 		 * userspace (which will retry the fault, or kill us if we got
371 		 * oom-killed).
372 		 */
373 		pagefault_out_of_memory();
374 		return 0;
375 	}
376 
377 	if (fault & VM_FAULT_SIGBUS) {
378 		/*
379 		 * We had some memory, but were unable to successfully fix up
380 		 * this page fault.
381 		 */
382 		sig = SIGBUS;
383 		code = BUS_ADRERR;
384 	} else {
385 		/*
386 		 * Something tried to access memory that isn't in our memory
387 		 * map.
388 		 */
389 		sig = SIGSEGV;
390 		code = fault == VM_FAULT_BADACCESS ?
391 			SEGV_ACCERR : SEGV_MAPERR;
392 	}
393 
394 	__do_user_fault(tsk, addr, esr, sig, code, regs);
395 	return 0;
396 
397 no_context:
398 	__do_kernel_fault(mm, addr, esr, regs);
399 	return 0;
400 }
401 
402 /*
403  * First Level Translation Fault Handler
404  *
405  * We enter here because the first level page table doesn't contain a valid
406  * entry for the address.
407  *
408  * If the address is in kernel space (>= TASK_SIZE), then we are probably
409  * faulting in the vmalloc() area.
410  *
411  * If the init_task's first level page tables contains the relevant entry, we
412  * copy the it to this task.  If not, we send the process a signal, fixup the
413  * exception, or oops the kernel.
414  *
415  * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
416  * or a critical region, and should only copy the information from the master
417  * page table, nothing more.
418  */
419 static int __kprobes do_translation_fault(unsigned long addr,
420 					  unsigned int esr,
421 					  struct pt_regs *regs)
422 {
423 	if (addr < TASK_SIZE)
424 		return do_page_fault(addr, esr, regs);
425 
426 	do_bad_area(addr, esr, regs);
427 	return 0;
428 }
429 
430 static int do_alignment_fault(unsigned long addr, unsigned int esr,
431 			      struct pt_regs *regs)
432 {
433 	do_bad_area(addr, esr, regs);
434 	return 0;
435 }
436 
437 /*
438  * This abort handler always returns "fault".
439  */
440 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
441 {
442 	return 1;
443 }
444 
445 static const struct fault_info {
446 	int	(*fn)(unsigned long addr, unsigned int esr, struct pt_regs *regs);
447 	int	sig;
448 	int	code;
449 	const char *name;
450 } fault_info[] = {
451 	{ do_bad,		SIGBUS,  0,		"ttbr address size fault"	},
452 	{ do_bad,		SIGBUS,  0,		"level 1 address size fault"	},
453 	{ do_bad,		SIGBUS,  0,		"level 2 address size fault"	},
454 	{ do_bad,		SIGBUS,  0,		"level 3 address size fault"	},
455 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 0 translation fault"	},
456 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
457 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
458 	{ do_page_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
459 	{ do_bad,		SIGBUS,  0,		"unknown 8"			},
460 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
461 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
462 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
463 	{ do_bad,		SIGBUS,  0,		"unknown 12"			},
464 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
465 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
466 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
467 	{ do_bad,		SIGBUS,  0,		"synchronous external abort"	},
468 	{ do_bad,		SIGBUS,  0,		"unknown 17"			},
469 	{ do_bad,		SIGBUS,  0,		"unknown 18"			},
470 	{ do_bad,		SIGBUS,  0,		"unknown 19"			},
471 	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
472 	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
473 	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
474 	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
475 	{ do_bad,		SIGBUS,  0,		"synchronous parity error"	},
476 	{ do_bad,		SIGBUS,  0,		"unknown 25"			},
477 	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
478 	{ do_bad,		SIGBUS,  0,		"unknown 27"			},
479 	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
480 	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
481 	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
482 	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
483 	{ do_bad,		SIGBUS,  0,		"unknown 32"			},
484 	{ do_alignment_fault,	SIGBUS,  BUS_ADRALN,	"alignment fault"		},
485 	{ do_bad,		SIGBUS,  0,		"unknown 34"			},
486 	{ do_bad,		SIGBUS,  0,		"unknown 35"			},
487 	{ do_bad,		SIGBUS,  0,		"unknown 36"			},
488 	{ do_bad,		SIGBUS,  0,		"unknown 37"			},
489 	{ do_bad,		SIGBUS,  0,		"unknown 38"			},
490 	{ do_bad,		SIGBUS,  0,		"unknown 39"			},
491 	{ do_bad,		SIGBUS,  0,		"unknown 40"			},
492 	{ do_bad,		SIGBUS,  0,		"unknown 41"			},
493 	{ do_bad,		SIGBUS,  0,		"unknown 42"			},
494 	{ do_bad,		SIGBUS,  0,		"unknown 43"			},
495 	{ do_bad,		SIGBUS,  0,		"unknown 44"			},
496 	{ do_bad,		SIGBUS,  0,		"unknown 45"			},
497 	{ do_bad,		SIGBUS,  0,		"unknown 46"			},
498 	{ do_bad,		SIGBUS,  0,		"unknown 47"			},
499 	{ do_bad,		SIGBUS,  0,		"TLB conflict abort"		},
500 	{ do_bad,		SIGBUS,  0,		"unknown 49"			},
501 	{ do_bad,		SIGBUS,  0,		"unknown 50"			},
502 	{ do_bad,		SIGBUS,  0,		"unknown 51"			},
503 	{ do_bad,		SIGBUS,  0,		"implementation fault (lockdown abort)" },
504 	{ do_bad,		SIGBUS,  0,		"implementation fault (unsupported exclusive)" },
505 	{ do_bad,		SIGBUS,  0,		"unknown 54"			},
506 	{ do_bad,		SIGBUS,  0,		"unknown 55"			},
507 	{ do_bad,		SIGBUS,  0,		"unknown 56"			},
508 	{ do_bad,		SIGBUS,  0,		"unknown 57"			},
509 	{ do_bad,		SIGBUS,  0,		"unknown 58" 			},
510 	{ do_bad,		SIGBUS,  0,		"unknown 59"			},
511 	{ do_bad,		SIGBUS,  0,		"unknown 60"			},
512 	{ do_bad,		SIGBUS,  0,		"section domain fault"		},
513 	{ do_bad,		SIGBUS,  0,		"page domain fault"		},
514 	{ do_bad,		SIGBUS,  0,		"unknown 63"			},
515 };
516 
517 static const char *fault_name(unsigned int esr)
518 {
519 	const struct fault_info *inf = fault_info + (esr & 63);
520 	return inf->name;
521 }
522 
523 /*
524  * Dispatch a data abort to the relevant handler.
525  */
526 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
527 					 struct pt_regs *regs)
528 {
529 	const struct fault_info *inf = fault_info + (esr & 63);
530 	struct siginfo info;
531 
532 	if (!inf->fn(addr, esr, regs))
533 		return;
534 
535 	pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
536 		 inf->name, esr, addr);
537 
538 	info.si_signo = inf->sig;
539 	info.si_errno = 0;
540 	info.si_code  = inf->code;
541 	info.si_addr  = (void __user *)addr;
542 	arm64_notify_die("", regs, &info, esr);
543 }
544 
545 /*
546  * Handle stack alignment exceptions.
547  */
548 asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
549 					   unsigned int esr,
550 					   struct pt_regs *regs)
551 {
552 	struct siginfo info;
553 	struct task_struct *tsk = current;
554 
555 	if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS))
556 		pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n",
557 				    tsk->comm, task_pid_nr(tsk),
558 				    esr_get_class_string(esr), (void *)regs->pc,
559 				    (void *)regs->sp);
560 
561 	info.si_signo = SIGBUS;
562 	info.si_errno = 0;
563 	info.si_code  = BUS_ADRALN;
564 	info.si_addr  = (void __user *)addr;
565 	arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr);
566 }
567 
568 int __init early_brk64(unsigned long addr, unsigned int esr,
569 		       struct pt_regs *regs);
570 
571 /*
572  * __refdata because early_brk64 is __init, but the reference to it is
573  * clobbered at arch_initcall time.
574  * See traps.c and debug-monitors.c:debug_traps_init().
575  */
576 static struct fault_info __refdata debug_fault_info[] = {
577 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
578 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
579 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
580 	{ do_bad,	SIGBUS,		0,		"unknown 3"		},
581 	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
582 	{ do_bad,	SIGTRAP,	0,		"aarch32 vector catch"	},
583 	{ early_brk64,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
584 	{ do_bad,	SIGBUS,		0,		"unknown 7"		},
585 };
586 
587 void __init hook_debug_fault_code(int nr,
588 				  int (*fn)(unsigned long, unsigned int, struct pt_regs *),
589 				  int sig, int code, const char *name)
590 {
591 	BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
592 
593 	debug_fault_info[nr].fn		= fn;
594 	debug_fault_info[nr].sig	= sig;
595 	debug_fault_info[nr].code	= code;
596 	debug_fault_info[nr].name	= name;
597 }
598 
599 asmlinkage int __exception do_debug_exception(unsigned long addr,
600 					      unsigned int esr,
601 					      struct pt_regs *regs)
602 {
603 	const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
604 	struct siginfo info;
605 	int rv;
606 
607 	/*
608 	 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were
609 	 * already disabled to preserve the last enabled/disabled addresses.
610 	 */
611 	if (interrupts_enabled(regs))
612 		trace_hardirqs_off();
613 
614 	if (!inf->fn(addr, esr, regs)) {
615 		rv = 1;
616 	} else {
617 		pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
618 			 inf->name, esr, addr);
619 
620 		info.si_signo = inf->sig;
621 		info.si_errno = 0;
622 		info.si_code  = inf->code;
623 		info.si_addr  = (void __user *)addr;
624 		arm64_notify_die("", regs, &info, 0);
625 		rv = 0;
626 	}
627 
628 	if (interrupts_enabled(regs))
629 		trace_hardirqs_on();
630 
631 	return rv;
632 }
633 
634 #ifdef CONFIG_ARM64_PAN
635 void cpu_enable_pan(void *__unused)
636 {
637 	config_sctlr_el1(SCTLR_EL1_SPAN, 0);
638 }
639 #endif /* CONFIG_ARM64_PAN */
640 
641 #ifdef CONFIG_ARM64_UAO
642 /*
643  * Kernel threads have fs=KERNEL_DS by default, and don't need to call
644  * set_fs(), devtmpfs in particular relies on this behaviour.
645  * We need to enable the feature at runtime (instead of adding it to
646  * PSR_MODE_EL1h) as the feature may not be implemented by the cpu.
647  */
648 void cpu_enable_uao(void *__unused)
649 {
650 	asm(SET_PSTATE_UAO(1));
651 }
652 #endif /* CONFIG_ARM64_UAO */
653