1 /* 2 * Based on arch/arm/mm/fault.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 1995-2004 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include <linux/extable.h> 22 #include <linux/signal.h> 23 #include <linux/mm.h> 24 #include <linux/hardirq.h> 25 #include <linux/init.h> 26 #include <linux/kprobes.h> 27 #include <linux/uaccess.h> 28 #include <linux/page-flags.h> 29 #include <linux/sched/signal.h> 30 #include <linux/sched/debug.h> 31 #include <linux/highmem.h> 32 #include <linux/perf_event.h> 33 #include <linux/preempt.h> 34 35 #include <asm/bug.h> 36 #include <asm/cpufeature.h> 37 #include <asm/exception.h> 38 #include <asm/debug-monitors.h> 39 #include <asm/esr.h> 40 #include <asm/sysreg.h> 41 #include <asm/system_misc.h> 42 #include <asm/pgtable.h> 43 #include <asm/tlbflush.h> 44 45 struct fault_info { 46 int (*fn)(unsigned long addr, unsigned int esr, 47 struct pt_regs *regs); 48 int sig; 49 int code; 50 const char *name; 51 }; 52 53 static const struct fault_info fault_info[]; 54 55 static inline const struct fault_info *esr_to_fault_info(unsigned int esr) 56 { 57 return fault_info + (esr & 63); 58 } 59 60 #ifdef CONFIG_KPROBES 61 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 62 { 63 int ret = 0; 64 65 /* kprobe_running() needs smp_processor_id() */ 66 if (!user_mode(regs)) { 67 preempt_disable(); 68 if (kprobe_running() && kprobe_fault_handler(regs, esr)) 69 ret = 1; 70 preempt_enable(); 71 } 72 73 return ret; 74 } 75 #else 76 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 77 { 78 return 0; 79 } 80 #endif 81 82 /* 83 * Dump out the page tables associated with 'addr' in mm 'mm'. 84 */ 85 void show_pte(struct mm_struct *mm, unsigned long addr) 86 { 87 pgd_t *pgd; 88 89 if (!mm) 90 mm = &init_mm; 91 92 pr_alert("pgd = %p\n", mm->pgd); 93 pgd = pgd_offset(mm, addr); 94 pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd)); 95 96 do { 97 pud_t *pud; 98 pmd_t *pmd; 99 pte_t *pte; 100 101 if (pgd_none(*pgd) || pgd_bad(*pgd)) 102 break; 103 104 pud = pud_offset(pgd, addr); 105 pr_cont(", *pud=%016llx", pud_val(*pud)); 106 if (pud_none(*pud) || pud_bad(*pud)) 107 break; 108 109 pmd = pmd_offset(pud, addr); 110 pr_cont(", *pmd=%016llx", pmd_val(*pmd)); 111 if (pmd_none(*pmd) || pmd_bad(*pmd)) 112 break; 113 114 pte = pte_offset_map(pmd, addr); 115 pr_cont(", *pte=%016llx", pte_val(*pte)); 116 pte_unmap(pte); 117 } while(0); 118 119 pr_cont("\n"); 120 } 121 122 #ifdef CONFIG_ARM64_HW_AFDBM 123 /* 124 * This function sets the access flags (dirty, accessed), as well as write 125 * permission, and only to a more permissive setting. 126 * 127 * It needs to cope with hardware update of the accessed/dirty state by other 128 * agents in the system and can safely skip the __sync_icache_dcache() call as, 129 * like set_pte_at(), the PTE is never changed from no-exec to exec here. 130 * 131 * Returns whether or not the PTE actually changed. 132 */ 133 int ptep_set_access_flags(struct vm_area_struct *vma, 134 unsigned long address, pte_t *ptep, 135 pte_t entry, int dirty) 136 { 137 pteval_t old_pteval; 138 unsigned int tmp; 139 140 if (pte_same(*ptep, entry)) 141 return 0; 142 143 /* only preserve the access flags and write permission */ 144 pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; 145 146 /* 147 * PTE_RDONLY is cleared by default in the asm below, so set it in 148 * back if necessary (read-only or clean PTE). 149 */ 150 if (!pte_write(entry) || !pte_sw_dirty(entry)) 151 pte_val(entry) |= PTE_RDONLY; 152 153 /* 154 * Setting the flags must be done atomically to avoid racing with the 155 * hardware update of the access/dirty state. 156 */ 157 asm volatile("// ptep_set_access_flags\n" 158 " prfm pstl1strm, %2\n" 159 "1: ldxr %0, %2\n" 160 " and %0, %0, %3 // clear PTE_RDONLY\n" 161 " orr %0, %0, %4 // set flags\n" 162 " stxr %w1, %0, %2\n" 163 " cbnz %w1, 1b\n" 164 : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) 165 : "L" (~PTE_RDONLY), "r" (pte_val(entry))); 166 167 flush_tlb_fix_spurious_fault(vma, address); 168 return 1; 169 } 170 #endif 171 172 static bool is_el1_instruction_abort(unsigned int esr) 173 { 174 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; 175 } 176 177 static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs, 178 unsigned long addr) 179 { 180 unsigned int ec = ESR_ELx_EC(esr); 181 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; 182 183 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) 184 return false; 185 186 if (fsc_type == ESR_ELx_FSC_PERM) 187 return true; 188 189 if (addr < USER_DS && system_uses_ttbr0_pan()) 190 return fsc_type == ESR_ELx_FSC_FAULT && 191 (regs->pstate & PSR_PAN_BIT); 192 193 return false; 194 } 195 196 /* 197 * The kernel tried to access some page that wasn't present. 198 */ 199 static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, 200 unsigned int esr, struct pt_regs *regs) 201 { 202 const char *msg; 203 204 /* 205 * Are we prepared to handle this kernel fault? 206 * We are almost certainly not prepared to handle instruction faults. 207 */ 208 if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) 209 return; 210 211 /* 212 * No handler, we'll have to terminate things with extreme prejudice. 213 */ 214 bust_spinlocks(1); 215 216 if (is_permission_fault(esr, regs, addr)) { 217 if (esr & ESR_ELx_WNR) 218 msg = "write to read-only memory"; 219 else 220 msg = "read from unreadable memory"; 221 } else if (addr < PAGE_SIZE) { 222 msg = "NULL pointer dereference"; 223 } else { 224 msg = "paging request"; 225 } 226 227 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, 228 addr); 229 230 show_pte(mm, addr); 231 die("Oops", regs, esr); 232 bust_spinlocks(0); 233 do_exit(SIGKILL); 234 } 235 236 /* 237 * Something tried to access memory that isn't in our memory map. User mode 238 * accesses just cause a SIGSEGV 239 */ 240 static void __do_user_fault(struct task_struct *tsk, unsigned long addr, 241 unsigned int esr, unsigned int sig, int code, 242 struct pt_regs *regs) 243 { 244 struct siginfo si; 245 const struct fault_info *inf; 246 247 if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { 248 inf = esr_to_fault_info(esr); 249 pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n", 250 tsk->comm, task_pid_nr(tsk), inf->name, sig, 251 addr, esr); 252 show_pte(tsk->mm, addr); 253 show_regs(regs); 254 } 255 256 tsk->thread.fault_address = addr; 257 tsk->thread.fault_code = esr; 258 si.si_signo = sig; 259 si.si_errno = 0; 260 si.si_code = code; 261 si.si_addr = (void __user *)addr; 262 force_sig_info(sig, &si, tsk); 263 } 264 265 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) 266 { 267 struct task_struct *tsk = current; 268 struct mm_struct *mm = tsk->active_mm; 269 const struct fault_info *inf; 270 271 /* 272 * If we are in kernel mode at this point, we have no context to 273 * handle this fault with. 274 */ 275 if (user_mode(regs)) { 276 inf = esr_to_fault_info(esr); 277 __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs); 278 } else 279 __do_kernel_fault(mm, addr, esr, regs); 280 } 281 282 #define VM_FAULT_BADMAP 0x010000 283 #define VM_FAULT_BADACCESS 0x020000 284 285 static int __do_page_fault(struct mm_struct *mm, unsigned long addr, 286 unsigned int mm_flags, unsigned long vm_flags, 287 struct task_struct *tsk) 288 { 289 struct vm_area_struct *vma; 290 int fault; 291 292 vma = find_vma(mm, addr); 293 fault = VM_FAULT_BADMAP; 294 if (unlikely(!vma)) 295 goto out; 296 if (unlikely(vma->vm_start > addr)) 297 goto check_stack; 298 299 /* 300 * Ok, we have a good vm_area for this memory access, so we can handle 301 * it. 302 */ 303 good_area: 304 /* 305 * Check that the permissions on the VMA allow for the fault which 306 * occurred. 307 */ 308 if (!(vma->vm_flags & vm_flags)) { 309 fault = VM_FAULT_BADACCESS; 310 goto out; 311 } 312 313 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); 314 315 check_stack: 316 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) 317 goto good_area; 318 out: 319 return fault; 320 } 321 322 static bool is_el0_instruction_abort(unsigned int esr) 323 { 324 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; 325 } 326 327 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, 328 struct pt_regs *regs) 329 { 330 struct task_struct *tsk; 331 struct mm_struct *mm; 332 int fault, sig, code; 333 unsigned long vm_flags = VM_READ | VM_WRITE; 334 unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 335 336 if (notify_page_fault(regs, esr)) 337 return 0; 338 339 tsk = current; 340 mm = tsk->mm; 341 342 /* 343 * If we're in an interrupt or have no user context, we must not take 344 * the fault. 345 */ 346 if (faulthandler_disabled() || !mm) 347 goto no_context; 348 349 if (user_mode(regs)) 350 mm_flags |= FAULT_FLAG_USER; 351 352 if (is_el0_instruction_abort(esr)) { 353 vm_flags = VM_EXEC; 354 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { 355 vm_flags = VM_WRITE; 356 mm_flags |= FAULT_FLAG_WRITE; 357 } 358 359 if (addr < USER_DS && is_permission_fault(esr, regs, addr)) { 360 /* regs->orig_addr_limit may be 0 if we entered from EL0 */ 361 if (regs->orig_addr_limit == KERNEL_DS) 362 die("Accessing user space memory with fs=KERNEL_DS", regs, esr); 363 364 if (is_el1_instruction_abort(esr)) 365 die("Attempting to execute userspace memory", regs, esr); 366 367 if (!search_exception_tables(regs->pc)) 368 die("Accessing user space memory outside uaccess.h routines", regs, esr); 369 } 370 371 /* 372 * As per x86, we may deadlock here. However, since the kernel only 373 * validly references user space from well defined areas of the code, 374 * we can bug out early if this is from code which shouldn't. 375 */ 376 if (!down_read_trylock(&mm->mmap_sem)) { 377 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 378 goto no_context; 379 retry: 380 down_read(&mm->mmap_sem); 381 } else { 382 /* 383 * The above down_read_trylock() might have succeeded in which 384 * case, we'll have missed the might_sleep() from down_read(). 385 */ 386 might_sleep(); 387 #ifdef CONFIG_DEBUG_VM 388 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 389 goto no_context; 390 #endif 391 } 392 393 fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); 394 395 /* 396 * If we need to retry but a fatal signal is pending, handle the 397 * signal first. We do not need to release the mmap_sem because it 398 * would already be released in __lock_page_or_retry in mm/filemap.c. 399 */ 400 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) 401 return 0; 402 403 /* 404 * Major/minor page fault accounting is only done on the initial 405 * attempt. If we go through a retry, it is extremely likely that the 406 * page will be found in page cache at that point. 407 */ 408 409 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 410 if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { 411 if (fault & VM_FAULT_MAJOR) { 412 tsk->maj_flt++; 413 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, 414 addr); 415 } else { 416 tsk->min_flt++; 417 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, 418 addr); 419 } 420 if (fault & VM_FAULT_RETRY) { 421 /* 422 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of 423 * starvation. 424 */ 425 mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; 426 mm_flags |= FAULT_FLAG_TRIED; 427 goto retry; 428 } 429 } 430 431 up_read(&mm->mmap_sem); 432 433 /* 434 * Handle the "normal" case first - VM_FAULT_MAJOR 435 */ 436 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | 437 VM_FAULT_BADACCESS)))) 438 return 0; 439 440 /* 441 * If we are in kernel mode at this point, we have no context to 442 * handle this fault with. 443 */ 444 if (!user_mode(regs)) 445 goto no_context; 446 447 if (fault & VM_FAULT_OOM) { 448 /* 449 * We ran out of memory, call the OOM killer, and return to 450 * userspace (which will retry the fault, or kill us if we got 451 * oom-killed). 452 */ 453 pagefault_out_of_memory(); 454 return 0; 455 } 456 457 if (fault & VM_FAULT_SIGBUS) { 458 /* 459 * We had some memory, but were unable to successfully fix up 460 * this page fault. 461 */ 462 sig = SIGBUS; 463 code = BUS_ADRERR; 464 } else { 465 /* 466 * Something tried to access memory that isn't in our memory 467 * map. 468 */ 469 sig = SIGSEGV; 470 code = fault == VM_FAULT_BADACCESS ? 471 SEGV_ACCERR : SEGV_MAPERR; 472 } 473 474 __do_user_fault(tsk, addr, esr, sig, code, regs); 475 return 0; 476 477 no_context: 478 __do_kernel_fault(mm, addr, esr, regs); 479 return 0; 480 } 481 482 /* 483 * First Level Translation Fault Handler 484 * 485 * We enter here because the first level page table doesn't contain a valid 486 * entry for the address. 487 * 488 * If the address is in kernel space (>= TASK_SIZE), then we are probably 489 * faulting in the vmalloc() area. 490 * 491 * If the init_task's first level page tables contains the relevant entry, we 492 * copy the it to this task. If not, we send the process a signal, fixup the 493 * exception, or oops the kernel. 494 * 495 * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt 496 * or a critical region, and should only copy the information from the master 497 * page table, nothing more. 498 */ 499 static int __kprobes do_translation_fault(unsigned long addr, 500 unsigned int esr, 501 struct pt_regs *regs) 502 { 503 if (addr < TASK_SIZE) 504 return do_page_fault(addr, esr, regs); 505 506 do_bad_area(addr, esr, regs); 507 return 0; 508 } 509 510 static int do_alignment_fault(unsigned long addr, unsigned int esr, 511 struct pt_regs *regs) 512 { 513 do_bad_area(addr, esr, regs); 514 return 0; 515 } 516 517 /* 518 * This abort handler always returns "fault". 519 */ 520 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) 521 { 522 return 1; 523 } 524 525 static const struct fault_info fault_info[] = { 526 { do_bad, SIGBUS, 0, "ttbr address size fault" }, 527 { do_bad, SIGBUS, 0, "level 1 address size fault" }, 528 { do_bad, SIGBUS, 0, "level 2 address size fault" }, 529 { do_bad, SIGBUS, 0, "level 3 address size fault" }, 530 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, 531 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, 532 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, 533 { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, 534 { do_bad, SIGBUS, 0, "unknown 8" }, 535 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, 536 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, 537 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, 538 { do_bad, SIGBUS, 0, "unknown 12" }, 539 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, 540 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, 541 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, 542 { do_bad, SIGBUS, 0, "synchronous external abort" }, 543 { do_bad, SIGBUS, 0, "unknown 17" }, 544 { do_bad, SIGBUS, 0, "unknown 18" }, 545 { do_bad, SIGBUS, 0, "unknown 19" }, 546 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 547 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 548 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 549 { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, 550 { do_bad, SIGBUS, 0, "synchronous parity error" }, 551 { do_bad, SIGBUS, 0, "unknown 25" }, 552 { do_bad, SIGBUS, 0, "unknown 26" }, 553 { do_bad, SIGBUS, 0, "unknown 27" }, 554 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 555 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 556 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 557 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, 558 { do_bad, SIGBUS, 0, "unknown 32" }, 559 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, 560 { do_bad, SIGBUS, 0, "unknown 34" }, 561 { do_bad, SIGBUS, 0, "unknown 35" }, 562 { do_bad, SIGBUS, 0, "unknown 36" }, 563 { do_bad, SIGBUS, 0, "unknown 37" }, 564 { do_bad, SIGBUS, 0, "unknown 38" }, 565 { do_bad, SIGBUS, 0, "unknown 39" }, 566 { do_bad, SIGBUS, 0, "unknown 40" }, 567 { do_bad, SIGBUS, 0, "unknown 41" }, 568 { do_bad, SIGBUS, 0, "unknown 42" }, 569 { do_bad, SIGBUS, 0, "unknown 43" }, 570 { do_bad, SIGBUS, 0, "unknown 44" }, 571 { do_bad, SIGBUS, 0, "unknown 45" }, 572 { do_bad, SIGBUS, 0, "unknown 46" }, 573 { do_bad, SIGBUS, 0, "unknown 47" }, 574 { do_bad, SIGBUS, 0, "TLB conflict abort" }, 575 { do_bad, SIGBUS, 0, "unknown 49" }, 576 { do_bad, SIGBUS, 0, "unknown 50" }, 577 { do_bad, SIGBUS, 0, "unknown 51" }, 578 { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" }, 579 { do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" }, 580 { do_bad, SIGBUS, 0, "unknown 54" }, 581 { do_bad, SIGBUS, 0, "unknown 55" }, 582 { do_bad, SIGBUS, 0, "unknown 56" }, 583 { do_bad, SIGBUS, 0, "unknown 57" }, 584 { do_bad, SIGBUS, 0, "unknown 58" }, 585 { do_bad, SIGBUS, 0, "unknown 59" }, 586 { do_bad, SIGBUS, 0, "unknown 60" }, 587 { do_bad, SIGBUS, 0, "section domain fault" }, 588 { do_bad, SIGBUS, 0, "page domain fault" }, 589 { do_bad, SIGBUS, 0, "unknown 63" }, 590 }; 591 592 /* 593 * Dispatch a data abort to the relevant handler. 594 */ 595 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, 596 struct pt_regs *regs) 597 { 598 const struct fault_info *inf = esr_to_fault_info(esr); 599 struct siginfo info; 600 601 if (!inf->fn(addr, esr, regs)) 602 return; 603 604 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", 605 inf->name, esr, addr); 606 607 info.si_signo = inf->sig; 608 info.si_errno = 0; 609 info.si_code = inf->code; 610 info.si_addr = (void __user *)addr; 611 arm64_notify_die("", regs, &info, esr); 612 } 613 614 /* 615 * Handle stack alignment exceptions. 616 */ 617 asmlinkage void __exception do_sp_pc_abort(unsigned long addr, 618 unsigned int esr, 619 struct pt_regs *regs) 620 { 621 struct siginfo info; 622 struct task_struct *tsk = current; 623 624 if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) 625 pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", 626 tsk->comm, task_pid_nr(tsk), 627 esr_get_class_string(esr), (void *)regs->pc, 628 (void *)regs->sp); 629 630 info.si_signo = SIGBUS; 631 info.si_errno = 0; 632 info.si_code = BUS_ADRALN; 633 info.si_addr = (void __user *)addr; 634 arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); 635 } 636 637 int __init early_brk64(unsigned long addr, unsigned int esr, 638 struct pt_regs *regs); 639 640 /* 641 * __refdata because early_brk64 is __init, but the reference to it is 642 * clobbered at arch_initcall time. 643 * See traps.c and debug-monitors.c:debug_traps_init(). 644 */ 645 static struct fault_info __refdata debug_fault_info[] = { 646 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, 647 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, 648 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, 649 { do_bad, SIGBUS, 0, "unknown 3" }, 650 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, 651 { do_bad, SIGTRAP, 0, "aarch32 vector catch" }, 652 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, 653 { do_bad, SIGBUS, 0, "unknown 7" }, 654 }; 655 656 void __init hook_debug_fault_code(int nr, 657 int (*fn)(unsigned long, unsigned int, struct pt_regs *), 658 int sig, int code, const char *name) 659 { 660 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); 661 662 debug_fault_info[nr].fn = fn; 663 debug_fault_info[nr].sig = sig; 664 debug_fault_info[nr].code = code; 665 debug_fault_info[nr].name = name; 666 } 667 668 asmlinkage int __exception do_debug_exception(unsigned long addr, 669 unsigned int esr, 670 struct pt_regs *regs) 671 { 672 const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); 673 struct siginfo info; 674 int rv; 675 676 /* 677 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were 678 * already disabled to preserve the last enabled/disabled addresses. 679 */ 680 if (interrupts_enabled(regs)) 681 trace_hardirqs_off(); 682 683 if (!inf->fn(addr, esr, regs)) { 684 rv = 1; 685 } else { 686 pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", 687 inf->name, esr, addr); 688 689 info.si_signo = inf->sig; 690 info.si_errno = 0; 691 info.si_code = inf->code; 692 info.si_addr = (void __user *)addr; 693 arm64_notify_die("", regs, &info, 0); 694 rv = 0; 695 } 696 697 if (interrupts_enabled(regs)) 698 trace_hardirqs_on(); 699 700 return rv; 701 } 702 NOKPROBE_SYMBOL(do_debug_exception); 703 704 #ifdef CONFIG_ARM64_PAN 705 int cpu_enable_pan(void *__unused) 706 { 707 /* 708 * We modify PSTATE. This won't work from irq context as the PSTATE 709 * is discarded once we return from the exception. 710 */ 711 WARN_ON_ONCE(in_interrupt()); 712 713 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 714 asm(SET_PSTATE_PAN(1)); 715 return 0; 716 } 717 #endif /* CONFIG_ARM64_PAN */ 718