1 /* 2 * Based on arch/arm/mm/fault.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 1995-2004 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include <linux/extable.h> 22 #include <linux/signal.h> 23 #include <linux/mm.h> 24 #include <linux/hardirq.h> 25 #include <linux/init.h> 26 #include <linux/kprobes.h> 27 #include <linux/uaccess.h> 28 #include <linux/page-flags.h> 29 #include <linux/sched/signal.h> 30 #include <linux/sched/debug.h> 31 #include <linux/highmem.h> 32 #include <linux/perf_event.h> 33 #include <linux/preempt.h> 34 35 #include <asm/bug.h> 36 #include <asm/cpufeature.h> 37 #include <asm/exception.h> 38 #include <asm/debug-monitors.h> 39 #include <asm/esr.h> 40 #include <asm/sysreg.h> 41 #include <asm/system_misc.h> 42 #include <asm/pgtable.h> 43 #include <asm/tlbflush.h> 44 45 struct fault_info { 46 int (*fn)(unsigned long addr, unsigned int esr, 47 struct pt_regs *regs); 48 int sig; 49 int code; 50 const char *name; 51 }; 52 53 static const struct fault_info fault_info[]; 54 55 static inline const struct fault_info *esr_to_fault_info(unsigned int esr) 56 { 57 return fault_info + (esr & 63); 58 } 59 60 #ifdef CONFIG_KPROBES 61 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 62 { 63 int ret = 0; 64 65 /* kprobe_running() needs smp_processor_id() */ 66 if (!user_mode(regs)) { 67 preempt_disable(); 68 if (kprobe_running() && kprobe_fault_handler(regs, esr)) 69 ret = 1; 70 preempt_enable(); 71 } 72 73 return ret; 74 } 75 #else 76 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 77 { 78 return 0; 79 } 80 #endif 81 82 /* 83 * Dump out the page tables associated with 'addr' in mm 'mm'. 84 */ 85 void show_pte(struct mm_struct *mm, unsigned long addr) 86 { 87 pgd_t *pgd; 88 89 if (!mm) 90 mm = &init_mm; 91 92 pr_alert("pgd = %p\n", mm->pgd); 93 pgd = pgd_offset(mm, addr); 94 pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd)); 95 96 do { 97 pud_t *pud; 98 pmd_t *pmd; 99 pte_t *pte; 100 101 if (pgd_none(*pgd) || pgd_bad(*pgd)) 102 break; 103 104 pud = pud_offset(pgd, addr); 105 pr_cont(", *pud=%016llx", pud_val(*pud)); 106 if (pud_none(*pud) || pud_bad(*pud)) 107 break; 108 109 pmd = pmd_offset(pud, addr); 110 pr_cont(", *pmd=%016llx", pmd_val(*pmd)); 111 if (pmd_none(*pmd) || pmd_bad(*pmd)) 112 break; 113 114 pte = pte_offset_map(pmd, addr); 115 pr_cont(", *pte=%016llx", pte_val(*pte)); 116 pte_unmap(pte); 117 } while(0); 118 119 pr_cont("\n"); 120 } 121 122 #ifdef CONFIG_ARM64_HW_AFDBM 123 /* 124 * This function sets the access flags (dirty, accessed), as well as write 125 * permission, and only to a more permissive setting. 126 * 127 * It needs to cope with hardware update of the accessed/dirty state by other 128 * agents in the system and can safely skip the __sync_icache_dcache() call as, 129 * like set_pte_at(), the PTE is never changed from no-exec to exec here. 130 * 131 * Returns whether or not the PTE actually changed. 132 */ 133 int ptep_set_access_flags(struct vm_area_struct *vma, 134 unsigned long address, pte_t *ptep, 135 pte_t entry, int dirty) 136 { 137 pteval_t old_pteval; 138 unsigned int tmp; 139 140 if (pte_same(*ptep, entry)) 141 return 0; 142 143 /* only preserve the access flags and write permission */ 144 pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; 145 146 /* 147 * PTE_RDONLY is cleared by default in the asm below, so set it in 148 * back if necessary (read-only or clean PTE). 149 */ 150 if (!pte_write(entry) || !pte_sw_dirty(entry)) 151 pte_val(entry) |= PTE_RDONLY; 152 153 /* 154 * Setting the flags must be done atomically to avoid racing with the 155 * hardware update of the access/dirty state. 156 */ 157 asm volatile("// ptep_set_access_flags\n" 158 " prfm pstl1strm, %2\n" 159 "1: ldxr %0, %2\n" 160 " and %0, %0, %3 // clear PTE_RDONLY\n" 161 " orr %0, %0, %4 // set flags\n" 162 " stxr %w1, %0, %2\n" 163 " cbnz %w1, 1b\n" 164 : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) 165 : "L" (~PTE_RDONLY), "r" (pte_val(entry))); 166 167 flush_tlb_fix_spurious_fault(vma, address); 168 return 1; 169 } 170 #endif 171 172 static bool is_el1_instruction_abort(unsigned int esr) 173 { 174 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; 175 } 176 177 /* 178 * The kernel tried to access some page that wasn't present. 179 */ 180 static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, 181 unsigned int esr, struct pt_regs *regs) 182 { 183 /* 184 * Are we prepared to handle this kernel fault? 185 * We are almost certainly not prepared to handle instruction faults. 186 */ 187 if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) 188 return; 189 190 /* 191 * No handler, we'll have to terminate things with extreme prejudice. 192 */ 193 bust_spinlocks(1); 194 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", 195 (addr < PAGE_SIZE) ? "NULL pointer dereference" : 196 "paging request", addr); 197 198 show_pte(mm, addr); 199 die("Oops", regs, esr); 200 bust_spinlocks(0); 201 do_exit(SIGKILL); 202 } 203 204 /* 205 * Something tried to access memory that isn't in our memory map. User mode 206 * accesses just cause a SIGSEGV 207 */ 208 static void __do_user_fault(struct task_struct *tsk, unsigned long addr, 209 unsigned int esr, unsigned int sig, int code, 210 struct pt_regs *regs) 211 { 212 struct siginfo si; 213 const struct fault_info *inf; 214 215 if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { 216 inf = esr_to_fault_info(esr); 217 pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n", 218 tsk->comm, task_pid_nr(tsk), inf->name, sig, 219 addr, esr); 220 show_pte(tsk->mm, addr); 221 show_regs(regs); 222 } 223 224 tsk->thread.fault_address = addr; 225 tsk->thread.fault_code = esr; 226 si.si_signo = sig; 227 si.si_errno = 0; 228 si.si_code = code; 229 si.si_addr = (void __user *)addr; 230 force_sig_info(sig, &si, tsk); 231 } 232 233 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) 234 { 235 struct task_struct *tsk = current; 236 struct mm_struct *mm = tsk->active_mm; 237 const struct fault_info *inf; 238 239 /* 240 * If we are in kernel mode at this point, we have no context to 241 * handle this fault with. 242 */ 243 if (user_mode(regs)) { 244 inf = esr_to_fault_info(esr); 245 __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs); 246 } else 247 __do_kernel_fault(mm, addr, esr, regs); 248 } 249 250 #define VM_FAULT_BADMAP 0x010000 251 #define VM_FAULT_BADACCESS 0x020000 252 253 static int __do_page_fault(struct mm_struct *mm, unsigned long addr, 254 unsigned int mm_flags, unsigned long vm_flags, 255 struct task_struct *tsk) 256 { 257 struct vm_area_struct *vma; 258 int fault; 259 260 vma = find_vma(mm, addr); 261 fault = VM_FAULT_BADMAP; 262 if (unlikely(!vma)) 263 goto out; 264 if (unlikely(vma->vm_start > addr)) 265 goto check_stack; 266 267 /* 268 * Ok, we have a good vm_area for this memory access, so we can handle 269 * it. 270 */ 271 good_area: 272 /* 273 * Check that the permissions on the VMA allow for the fault which 274 * occurred. 275 */ 276 if (!(vma->vm_flags & vm_flags)) { 277 fault = VM_FAULT_BADACCESS; 278 goto out; 279 } 280 281 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); 282 283 check_stack: 284 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) 285 goto good_area; 286 out: 287 return fault; 288 } 289 290 static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs) 291 { 292 unsigned int ec = ESR_ELx_EC(esr); 293 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; 294 295 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) 296 return false; 297 298 if (system_uses_ttbr0_pan()) 299 return fsc_type == ESR_ELx_FSC_FAULT && 300 (regs->pstate & PSR_PAN_BIT); 301 else 302 return fsc_type == ESR_ELx_FSC_PERM; 303 } 304 305 static bool is_el0_instruction_abort(unsigned int esr) 306 { 307 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; 308 } 309 310 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, 311 struct pt_regs *regs) 312 { 313 struct task_struct *tsk; 314 struct mm_struct *mm; 315 int fault, sig, code; 316 unsigned long vm_flags = VM_READ | VM_WRITE; 317 unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 318 319 if (notify_page_fault(regs, esr)) 320 return 0; 321 322 tsk = current; 323 mm = tsk->mm; 324 325 /* 326 * If we're in an interrupt or have no user context, we must not take 327 * the fault. 328 */ 329 if (faulthandler_disabled() || !mm) 330 goto no_context; 331 332 if (user_mode(regs)) 333 mm_flags |= FAULT_FLAG_USER; 334 335 if (is_el0_instruction_abort(esr)) { 336 vm_flags = VM_EXEC; 337 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { 338 vm_flags = VM_WRITE; 339 mm_flags |= FAULT_FLAG_WRITE; 340 } 341 342 if (addr < USER_DS && is_permission_fault(esr, regs)) { 343 /* regs->orig_addr_limit may be 0 if we entered from EL0 */ 344 if (regs->orig_addr_limit == KERNEL_DS) 345 die("Accessing user space memory with fs=KERNEL_DS", regs, esr); 346 347 if (is_el1_instruction_abort(esr)) 348 die("Attempting to execute userspace memory", regs, esr); 349 350 if (!search_exception_tables(regs->pc)) 351 die("Accessing user space memory outside uaccess.h routines", regs, esr); 352 } 353 354 /* 355 * As per x86, we may deadlock here. However, since the kernel only 356 * validly references user space from well defined areas of the code, 357 * we can bug out early if this is from code which shouldn't. 358 */ 359 if (!down_read_trylock(&mm->mmap_sem)) { 360 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 361 goto no_context; 362 retry: 363 down_read(&mm->mmap_sem); 364 } else { 365 /* 366 * The above down_read_trylock() might have succeeded in which 367 * case, we'll have missed the might_sleep() from down_read(). 368 */ 369 might_sleep(); 370 #ifdef CONFIG_DEBUG_VM 371 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 372 goto no_context; 373 #endif 374 } 375 376 fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); 377 378 /* 379 * If we need to retry but a fatal signal is pending, handle the 380 * signal first. We do not need to release the mmap_sem because it 381 * would already be released in __lock_page_or_retry in mm/filemap.c. 382 */ 383 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) 384 return 0; 385 386 /* 387 * Major/minor page fault accounting is only done on the initial 388 * attempt. If we go through a retry, it is extremely likely that the 389 * page will be found in page cache at that point. 390 */ 391 392 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 393 if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { 394 if (fault & VM_FAULT_MAJOR) { 395 tsk->maj_flt++; 396 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, 397 addr); 398 } else { 399 tsk->min_flt++; 400 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, 401 addr); 402 } 403 if (fault & VM_FAULT_RETRY) { 404 /* 405 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of 406 * starvation. 407 */ 408 mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; 409 mm_flags |= FAULT_FLAG_TRIED; 410 goto retry; 411 } 412 } 413 414 up_read(&mm->mmap_sem); 415 416 /* 417 * Handle the "normal" case first - VM_FAULT_MAJOR 418 */ 419 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | 420 VM_FAULT_BADACCESS)))) 421 return 0; 422 423 /* 424 * If we are in kernel mode at this point, we have no context to 425 * handle this fault with. 426 */ 427 if (!user_mode(regs)) 428 goto no_context; 429 430 if (fault & VM_FAULT_OOM) { 431 /* 432 * We ran out of memory, call the OOM killer, and return to 433 * userspace (which will retry the fault, or kill us if we got 434 * oom-killed). 435 */ 436 pagefault_out_of_memory(); 437 return 0; 438 } 439 440 if (fault & VM_FAULT_SIGBUS) { 441 /* 442 * We had some memory, but were unable to successfully fix up 443 * this page fault. 444 */ 445 sig = SIGBUS; 446 code = BUS_ADRERR; 447 } else { 448 /* 449 * Something tried to access memory that isn't in our memory 450 * map. 451 */ 452 sig = SIGSEGV; 453 code = fault == VM_FAULT_BADACCESS ? 454 SEGV_ACCERR : SEGV_MAPERR; 455 } 456 457 __do_user_fault(tsk, addr, esr, sig, code, regs); 458 return 0; 459 460 no_context: 461 __do_kernel_fault(mm, addr, esr, regs); 462 return 0; 463 } 464 465 /* 466 * First Level Translation Fault Handler 467 * 468 * We enter here because the first level page table doesn't contain a valid 469 * entry for the address. 470 * 471 * If the address is in kernel space (>= TASK_SIZE), then we are probably 472 * faulting in the vmalloc() area. 473 * 474 * If the init_task's first level page tables contains the relevant entry, we 475 * copy the it to this task. If not, we send the process a signal, fixup the 476 * exception, or oops the kernel. 477 * 478 * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt 479 * or a critical region, and should only copy the information from the master 480 * page table, nothing more. 481 */ 482 static int __kprobes do_translation_fault(unsigned long addr, 483 unsigned int esr, 484 struct pt_regs *regs) 485 { 486 if (addr < TASK_SIZE) 487 return do_page_fault(addr, esr, regs); 488 489 do_bad_area(addr, esr, regs); 490 return 0; 491 } 492 493 static int do_alignment_fault(unsigned long addr, unsigned int esr, 494 struct pt_regs *regs) 495 { 496 do_bad_area(addr, esr, regs); 497 return 0; 498 } 499 500 /* 501 * This abort handler always returns "fault". 502 */ 503 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) 504 { 505 return 1; 506 } 507 508 static const struct fault_info fault_info[] = { 509 { do_bad, SIGBUS, 0, "ttbr address size fault" }, 510 { do_bad, SIGBUS, 0, "level 1 address size fault" }, 511 { do_bad, SIGBUS, 0, "level 2 address size fault" }, 512 { do_bad, SIGBUS, 0, "level 3 address size fault" }, 513 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, 514 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, 515 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, 516 { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, 517 { do_bad, SIGBUS, 0, "unknown 8" }, 518 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, 519 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, 520 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, 521 { do_bad, SIGBUS, 0, "unknown 12" }, 522 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, 523 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, 524 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, 525 { do_bad, SIGBUS, 0, "synchronous external abort" }, 526 { do_bad, SIGBUS, 0, "unknown 17" }, 527 { do_bad, SIGBUS, 0, "unknown 18" }, 528 { do_bad, SIGBUS, 0, "unknown 19" }, 529 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 530 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 531 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 532 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 533 { do_bad, SIGBUS, 0, "synchronous parity error" }, 534 { do_bad, SIGBUS, 0, "unknown 25" }, 535 { do_bad, SIGBUS, 0, "unknown 26" }, 536 { do_bad, SIGBUS, 0, "unknown 27" }, 537 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 538 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 539 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 540 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 541 { do_bad, SIGBUS, 0, "unknown 32" }, 542 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, 543 { do_bad, SIGBUS, 0, "unknown 34" }, 544 { do_bad, SIGBUS, 0, "unknown 35" }, 545 { do_bad, SIGBUS, 0, "unknown 36" }, 546 { do_bad, SIGBUS, 0, "unknown 37" }, 547 { do_bad, SIGBUS, 0, "unknown 38" }, 548 { do_bad, SIGBUS, 0, "unknown 39" }, 549 { do_bad, SIGBUS, 0, "unknown 40" }, 550 { do_bad, SIGBUS, 0, "unknown 41" }, 551 { do_bad, SIGBUS, 0, "unknown 42" }, 552 { do_bad, SIGBUS, 0, "unknown 43" }, 553 { do_bad, SIGBUS, 0, "unknown 44" }, 554 { do_bad, SIGBUS, 0, "unknown 45" }, 555 { do_bad, SIGBUS, 0, "unknown 46" }, 556 { do_bad, SIGBUS, 0, "unknown 47" }, 557 { do_bad, SIGBUS, 0, "TLB conflict abort" }, 558 { do_bad, SIGBUS, 0, "unknown 49" }, 559 { do_bad, SIGBUS, 0, "unknown 50" }, 560 { do_bad, SIGBUS, 0, "unknown 51" }, 561 { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" }, 562 { do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" }, 563 { do_bad, SIGBUS, 0, "unknown 54" }, 564 { do_bad, SIGBUS, 0, "unknown 55" }, 565 { do_bad, SIGBUS, 0, "unknown 56" }, 566 { do_bad, SIGBUS, 0, "unknown 57" }, 567 { do_bad, SIGBUS, 0, "unknown 58" }, 568 { do_bad, SIGBUS, 0, "unknown 59" }, 569 { do_bad, SIGBUS, 0, "unknown 60" }, 570 { do_bad, SIGBUS, 0, "section domain fault" }, 571 { do_bad, SIGBUS, 0, "page domain fault" }, 572 { do_bad, SIGBUS, 0, "unknown 63" }, 573 }; 574 575 /* 576 * Dispatch a data abort to the relevant handler. 577 */ 578 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, 579 struct pt_regs *regs) 580 { 581 const struct fault_info *inf = esr_to_fault_info(esr); 582 struct siginfo info; 583 584 if (!inf->fn(addr, esr, regs)) 585 return; 586 587 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", 588 inf->name, esr, addr); 589 590 info.si_signo = inf->sig; 591 info.si_errno = 0; 592 info.si_code = inf->code; 593 info.si_addr = (void __user *)addr; 594 arm64_notify_die("", regs, &info, esr); 595 } 596 597 /* 598 * Handle stack alignment exceptions. 599 */ 600 asmlinkage void __exception do_sp_pc_abort(unsigned long addr, 601 unsigned int esr, 602 struct pt_regs *regs) 603 { 604 struct siginfo info; 605 struct task_struct *tsk = current; 606 607 if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) 608 pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", 609 tsk->comm, task_pid_nr(tsk), 610 esr_get_class_string(esr), (void *)regs->pc, 611 (void *)regs->sp); 612 613 info.si_signo = SIGBUS; 614 info.si_errno = 0; 615 info.si_code = BUS_ADRALN; 616 info.si_addr = (void __user *)addr; 617 arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); 618 } 619 620 int __init early_brk64(unsigned long addr, unsigned int esr, 621 struct pt_regs *regs); 622 623 /* 624 * __refdata because early_brk64 is __init, but the reference to it is 625 * clobbered at arch_initcall time. 626 * See traps.c and debug-monitors.c:debug_traps_init(). 627 */ 628 static struct fault_info __refdata debug_fault_info[] = { 629 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, 630 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, 631 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, 632 { do_bad, SIGBUS, 0, "unknown 3" }, 633 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, 634 { do_bad, SIGTRAP, 0, "aarch32 vector catch" }, 635 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, 636 { do_bad, SIGBUS, 0, "unknown 7" }, 637 }; 638 639 void __init hook_debug_fault_code(int nr, 640 int (*fn)(unsigned long, unsigned int, struct pt_regs *), 641 int sig, int code, const char *name) 642 { 643 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); 644 645 debug_fault_info[nr].fn = fn; 646 debug_fault_info[nr].sig = sig; 647 debug_fault_info[nr].code = code; 648 debug_fault_info[nr].name = name; 649 } 650 651 asmlinkage int __exception do_debug_exception(unsigned long addr, 652 unsigned int esr, 653 struct pt_regs *regs) 654 { 655 const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); 656 struct siginfo info; 657 int rv; 658 659 /* 660 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were 661 * already disabled to preserve the last enabled/disabled addresses. 662 */ 663 if (interrupts_enabled(regs)) 664 trace_hardirqs_off(); 665 666 if (!inf->fn(addr, esr, regs)) { 667 rv = 1; 668 } else { 669 pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", 670 inf->name, esr, addr); 671 672 info.si_signo = inf->sig; 673 info.si_errno = 0; 674 info.si_code = inf->code; 675 info.si_addr = (void __user *)addr; 676 arm64_notify_die("", regs, &info, 0); 677 rv = 0; 678 } 679 680 if (interrupts_enabled(regs)) 681 trace_hardirqs_on(); 682 683 return rv; 684 } 685 NOKPROBE_SYMBOL(do_debug_exception); 686 687 #ifdef CONFIG_ARM64_PAN 688 int cpu_enable_pan(void *__unused) 689 { 690 /* 691 * We modify PSTATE. This won't work from irq context as the PSTATE 692 * is discarded once we return from the exception. 693 */ 694 WARN_ON_ONCE(in_interrupt()); 695 696 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 697 asm(SET_PSTATE_PAN(1)); 698 return 0; 699 } 700 #endif /* CONFIG_ARM64_PAN */ 701