1 /* 2 * Based on arch/arm/mm/fault.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 1995-2004 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include <linux/extable.h> 22 #include <linux/signal.h> 23 #include <linux/mm.h> 24 #include <linux/hardirq.h> 25 #include <linux/init.h> 26 #include <linux/kprobes.h> 27 #include <linux/uaccess.h> 28 #include <linux/page-flags.h> 29 #include <linux/sched/signal.h> 30 #include <linux/sched/debug.h> 31 #include <linux/highmem.h> 32 #include <linux/perf_event.h> 33 #include <linux/preempt.h> 34 #include <linux/hugetlb.h> 35 36 #include <asm/bug.h> 37 #include <asm/cpufeature.h> 38 #include <asm/exception.h> 39 #include <asm/debug-monitors.h> 40 #include <asm/esr.h> 41 #include <asm/sysreg.h> 42 #include <asm/system_misc.h> 43 #include <asm/pgtable.h> 44 #include <asm/tlbflush.h> 45 46 #include <acpi/ghes.h> 47 48 struct fault_info { 49 int (*fn)(unsigned long addr, unsigned int esr, 50 struct pt_regs *regs); 51 int sig; 52 int code; 53 const char *name; 54 }; 55 56 static const struct fault_info fault_info[]; 57 58 static inline const struct fault_info *esr_to_fault_info(unsigned int esr) 59 { 60 return fault_info + (esr & 63); 61 } 62 63 #ifdef CONFIG_KPROBES 64 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 65 { 66 int ret = 0; 67 68 /* kprobe_running() needs smp_processor_id() */ 69 if (!user_mode(regs)) { 70 preempt_disable(); 71 if (kprobe_running() && kprobe_fault_handler(regs, esr)) 72 ret = 1; 73 preempt_enable(); 74 } 75 76 return ret; 77 } 78 #else 79 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 80 { 81 return 0; 82 } 83 #endif 84 85 /* 86 * Dump out the page tables associated with 'addr' in the currently active mm. 87 */ 88 void show_pte(unsigned long addr) 89 { 90 struct mm_struct *mm; 91 pgd_t *pgd; 92 93 if (addr < TASK_SIZE) { 94 /* TTBR0 */ 95 mm = current->active_mm; 96 if (mm == &init_mm) { 97 pr_alert("[%016lx] user address but active_mm is swapper\n", 98 addr); 99 return; 100 } 101 } else if (addr >= VA_START) { 102 /* TTBR1 */ 103 mm = &init_mm; 104 } else { 105 pr_alert("[%016lx] address between user and kernel address ranges\n", 106 addr); 107 return; 108 } 109 110 pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgd = %p\n", 111 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, 112 VA_BITS, mm->pgd); 113 pgd = pgd_offset(mm, addr); 114 pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd)); 115 116 do { 117 pud_t *pud; 118 pmd_t *pmd; 119 pte_t *pte; 120 121 if (pgd_none(*pgd) || pgd_bad(*pgd)) 122 break; 123 124 pud = pud_offset(pgd, addr); 125 pr_cont(", *pud=%016llx", pud_val(*pud)); 126 if (pud_none(*pud) || pud_bad(*pud)) 127 break; 128 129 pmd = pmd_offset(pud, addr); 130 pr_cont(", *pmd=%016llx", pmd_val(*pmd)); 131 if (pmd_none(*pmd) || pmd_bad(*pmd)) 132 break; 133 134 pte = pte_offset_map(pmd, addr); 135 pr_cont(", *pte=%016llx", pte_val(*pte)); 136 pte_unmap(pte); 137 } while(0); 138 139 pr_cont("\n"); 140 } 141 142 #ifdef CONFIG_ARM64_HW_AFDBM 143 /* 144 * This function sets the access flags (dirty, accessed), as well as write 145 * permission, and only to a more permissive setting. 146 * 147 * It needs to cope with hardware update of the accessed/dirty state by other 148 * agents in the system and can safely skip the __sync_icache_dcache() call as, 149 * like set_pte_at(), the PTE is never changed from no-exec to exec here. 150 * 151 * Returns whether or not the PTE actually changed. 152 */ 153 int ptep_set_access_flags(struct vm_area_struct *vma, 154 unsigned long address, pte_t *ptep, 155 pte_t entry, int dirty) 156 { 157 pteval_t old_pteval; 158 unsigned int tmp; 159 160 if (pte_same(*ptep, entry)) 161 return 0; 162 163 /* only preserve the access flags and write permission */ 164 pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; 165 166 /* 167 * PTE_RDONLY is cleared by default in the asm below, so set it in 168 * back if necessary (read-only or clean PTE). 169 */ 170 if (!pte_write(entry) || !pte_sw_dirty(entry)) 171 pte_val(entry) |= PTE_RDONLY; 172 173 /* 174 * Setting the flags must be done atomically to avoid racing with the 175 * hardware update of the access/dirty state. 176 */ 177 asm volatile("// ptep_set_access_flags\n" 178 " prfm pstl1strm, %2\n" 179 "1: ldxr %0, %2\n" 180 " and %0, %0, %3 // clear PTE_RDONLY\n" 181 " orr %0, %0, %4 // set flags\n" 182 " stxr %w1, %0, %2\n" 183 " cbnz %w1, 1b\n" 184 : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) 185 : "L" (~PTE_RDONLY), "r" (pte_val(entry))); 186 187 flush_tlb_fix_spurious_fault(vma, address); 188 return 1; 189 } 190 #endif 191 192 static bool is_el1_instruction_abort(unsigned int esr) 193 { 194 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; 195 } 196 197 static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs, 198 unsigned long addr) 199 { 200 unsigned int ec = ESR_ELx_EC(esr); 201 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; 202 203 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) 204 return false; 205 206 if (fsc_type == ESR_ELx_FSC_PERM) 207 return true; 208 209 if (addr < USER_DS && system_uses_ttbr0_pan()) 210 return fsc_type == ESR_ELx_FSC_FAULT && 211 (regs->pstate & PSR_PAN_BIT); 212 213 return false; 214 } 215 216 /* 217 * The kernel tried to access some page that wasn't present. 218 */ 219 static void __do_kernel_fault(unsigned long addr, unsigned int esr, 220 struct pt_regs *regs) 221 { 222 const char *msg; 223 224 /* 225 * Are we prepared to handle this kernel fault? 226 * We are almost certainly not prepared to handle instruction faults. 227 */ 228 if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) 229 return; 230 231 /* 232 * No handler, we'll have to terminate things with extreme prejudice. 233 */ 234 bust_spinlocks(1); 235 236 if (is_permission_fault(esr, regs, addr)) { 237 if (esr & ESR_ELx_WNR) 238 msg = "write to read-only memory"; 239 else 240 msg = "read from unreadable memory"; 241 } else if (addr < PAGE_SIZE) { 242 msg = "NULL pointer dereference"; 243 } else { 244 msg = "paging request"; 245 } 246 247 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, 248 addr); 249 250 show_pte(addr); 251 die("Oops", regs, esr); 252 bust_spinlocks(0); 253 do_exit(SIGKILL); 254 } 255 256 /* 257 * Something tried to access memory that isn't in our memory map. User mode 258 * accesses just cause a SIGSEGV 259 */ 260 static void __do_user_fault(struct task_struct *tsk, unsigned long addr, 261 unsigned int esr, unsigned int sig, int code, 262 struct pt_regs *regs, int fault) 263 { 264 struct siginfo si; 265 const struct fault_info *inf; 266 unsigned int lsb = 0; 267 268 if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { 269 inf = esr_to_fault_info(esr); 270 pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x", 271 tsk->comm, task_pid_nr(tsk), inf->name, sig, 272 addr, esr); 273 print_vma_addr(KERN_CONT ", in ", regs->pc); 274 pr_cont("\n"); 275 __show_regs(regs); 276 } 277 278 tsk->thread.fault_address = addr; 279 tsk->thread.fault_code = esr; 280 si.si_signo = sig; 281 si.si_errno = 0; 282 si.si_code = code; 283 si.si_addr = (void __user *)addr; 284 /* 285 * Either small page or large page may be poisoned. 286 * In other words, VM_FAULT_HWPOISON_LARGE and 287 * VM_FAULT_HWPOISON are mutually exclusive. 288 */ 289 if (fault & VM_FAULT_HWPOISON_LARGE) 290 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); 291 else if (fault & VM_FAULT_HWPOISON) 292 lsb = PAGE_SHIFT; 293 si.si_addr_lsb = lsb; 294 295 force_sig_info(sig, &si, tsk); 296 } 297 298 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) 299 { 300 struct task_struct *tsk = current; 301 const struct fault_info *inf; 302 303 /* 304 * If we are in kernel mode at this point, we have no context to 305 * handle this fault with. 306 */ 307 if (user_mode(regs)) { 308 inf = esr_to_fault_info(esr); 309 __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs, 0); 310 } else 311 __do_kernel_fault(addr, esr, regs); 312 } 313 314 #define VM_FAULT_BADMAP 0x010000 315 #define VM_FAULT_BADACCESS 0x020000 316 317 static int __do_page_fault(struct mm_struct *mm, unsigned long addr, 318 unsigned int mm_flags, unsigned long vm_flags, 319 struct task_struct *tsk) 320 { 321 struct vm_area_struct *vma; 322 int fault; 323 324 vma = find_vma(mm, addr); 325 fault = VM_FAULT_BADMAP; 326 if (unlikely(!vma)) 327 goto out; 328 if (unlikely(vma->vm_start > addr)) 329 goto check_stack; 330 331 /* 332 * Ok, we have a good vm_area for this memory access, so we can handle 333 * it. 334 */ 335 good_area: 336 /* 337 * Check that the permissions on the VMA allow for the fault which 338 * occurred. 339 */ 340 if (!(vma->vm_flags & vm_flags)) { 341 fault = VM_FAULT_BADACCESS; 342 goto out; 343 } 344 345 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); 346 347 check_stack: 348 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) 349 goto good_area; 350 out: 351 return fault; 352 } 353 354 static bool is_el0_instruction_abort(unsigned int esr) 355 { 356 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; 357 } 358 359 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, 360 struct pt_regs *regs) 361 { 362 struct task_struct *tsk; 363 struct mm_struct *mm; 364 int fault, sig, code, major = 0; 365 unsigned long vm_flags = VM_READ | VM_WRITE; 366 unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 367 368 if (notify_page_fault(regs, esr)) 369 return 0; 370 371 tsk = current; 372 mm = tsk->mm; 373 374 /* 375 * If we're in an interrupt or have no user context, we must not take 376 * the fault. 377 */ 378 if (faulthandler_disabled() || !mm) 379 goto no_context; 380 381 if (user_mode(regs)) 382 mm_flags |= FAULT_FLAG_USER; 383 384 if (is_el0_instruction_abort(esr)) { 385 vm_flags = VM_EXEC; 386 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { 387 vm_flags = VM_WRITE; 388 mm_flags |= FAULT_FLAG_WRITE; 389 } 390 391 if (addr < USER_DS && is_permission_fault(esr, regs, addr)) { 392 /* regs->orig_addr_limit may be 0 if we entered from EL0 */ 393 if (regs->orig_addr_limit == KERNEL_DS) 394 die("Accessing user space memory with fs=KERNEL_DS", regs, esr); 395 396 if (is_el1_instruction_abort(esr)) 397 die("Attempting to execute userspace memory", regs, esr); 398 399 if (!search_exception_tables(regs->pc)) 400 die("Accessing user space memory outside uaccess.h routines", regs, esr); 401 } 402 403 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 404 405 /* 406 * As per x86, we may deadlock here. However, since the kernel only 407 * validly references user space from well defined areas of the code, 408 * we can bug out early if this is from code which shouldn't. 409 */ 410 if (!down_read_trylock(&mm->mmap_sem)) { 411 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 412 goto no_context; 413 retry: 414 down_read(&mm->mmap_sem); 415 } else { 416 /* 417 * The above down_read_trylock() might have succeeded in which 418 * case, we'll have missed the might_sleep() from down_read(). 419 */ 420 might_sleep(); 421 #ifdef CONFIG_DEBUG_VM 422 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 423 goto no_context; 424 #endif 425 } 426 427 fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); 428 major |= fault & VM_FAULT_MAJOR; 429 430 if (fault & VM_FAULT_RETRY) { 431 /* 432 * If we need to retry but a fatal signal is pending, 433 * handle the signal first. We do not need to release 434 * the mmap_sem because it would already be released 435 * in __lock_page_or_retry in mm/filemap.c. 436 */ 437 if (fatal_signal_pending(current)) 438 return 0; 439 440 /* 441 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of 442 * starvation. 443 */ 444 if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { 445 mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; 446 mm_flags |= FAULT_FLAG_TRIED; 447 goto retry; 448 } 449 } 450 up_read(&mm->mmap_sem); 451 452 /* 453 * Handle the "normal" (no error) case first. 454 */ 455 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | 456 VM_FAULT_BADACCESS)))) { 457 /* 458 * Major/minor page fault accounting is only done 459 * once. If we go through a retry, it is extremely 460 * likely that the page will be found in page cache at 461 * that point. 462 */ 463 if (major) { 464 tsk->maj_flt++; 465 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, 466 addr); 467 } else { 468 tsk->min_flt++; 469 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, 470 addr); 471 } 472 473 return 0; 474 } 475 476 /* 477 * If we are in kernel mode at this point, we have no context to 478 * handle this fault with. 479 */ 480 if (!user_mode(regs)) 481 goto no_context; 482 483 if (fault & VM_FAULT_OOM) { 484 /* 485 * We ran out of memory, call the OOM killer, and return to 486 * userspace (which will retry the fault, or kill us if we got 487 * oom-killed). 488 */ 489 pagefault_out_of_memory(); 490 return 0; 491 } 492 493 if (fault & VM_FAULT_SIGBUS) { 494 /* 495 * We had some memory, but were unable to successfully fix up 496 * this page fault. 497 */ 498 sig = SIGBUS; 499 code = BUS_ADRERR; 500 } else if (fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) { 501 sig = SIGBUS; 502 code = BUS_MCEERR_AR; 503 } else { 504 /* 505 * Something tried to access memory that isn't in our memory 506 * map. 507 */ 508 sig = SIGSEGV; 509 code = fault == VM_FAULT_BADACCESS ? 510 SEGV_ACCERR : SEGV_MAPERR; 511 } 512 513 __do_user_fault(tsk, addr, esr, sig, code, regs, fault); 514 return 0; 515 516 no_context: 517 __do_kernel_fault(addr, esr, regs); 518 return 0; 519 } 520 521 /* 522 * First Level Translation Fault Handler 523 * 524 * We enter here because the first level page table doesn't contain a valid 525 * entry for the address. 526 * 527 * If the address is in kernel space (>= TASK_SIZE), then we are probably 528 * faulting in the vmalloc() area. 529 * 530 * If the init_task's first level page tables contains the relevant entry, we 531 * copy the it to this task. If not, we send the process a signal, fixup the 532 * exception, or oops the kernel. 533 * 534 * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt 535 * or a critical region, and should only copy the information from the master 536 * page table, nothing more. 537 */ 538 static int __kprobes do_translation_fault(unsigned long addr, 539 unsigned int esr, 540 struct pt_regs *regs) 541 { 542 if (addr < TASK_SIZE) 543 return do_page_fault(addr, esr, regs); 544 545 do_bad_area(addr, esr, regs); 546 return 0; 547 } 548 549 static int do_alignment_fault(unsigned long addr, unsigned int esr, 550 struct pt_regs *regs) 551 { 552 do_bad_area(addr, esr, regs); 553 return 0; 554 } 555 556 /* 557 * This abort handler always returns "fault". 558 */ 559 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) 560 { 561 return 1; 562 } 563 564 /* 565 * This abort handler deals with Synchronous External Abort. 566 * It calls notifiers, and then returns "fault". 567 */ 568 static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs) 569 { 570 struct siginfo info; 571 const struct fault_info *inf; 572 int ret = 0; 573 574 inf = esr_to_fault_info(esr); 575 pr_err("Synchronous External Abort: %s (0x%08x) at 0x%016lx\n", 576 inf->name, esr, addr); 577 578 /* 579 * Synchronous aborts may interrupt code which had interrupts masked. 580 * Before calling out into the wider kernel tell the interested 581 * subsystems. 582 */ 583 if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) { 584 if (interrupts_enabled(regs)) 585 nmi_enter(); 586 587 ret = ghes_notify_sea(); 588 589 if (interrupts_enabled(regs)) 590 nmi_exit(); 591 } 592 593 info.si_signo = SIGBUS; 594 info.si_errno = 0; 595 info.si_code = 0; 596 if (esr & ESR_ELx_FnV) 597 info.si_addr = NULL; 598 else 599 info.si_addr = (void __user *)addr; 600 arm64_notify_die("", regs, &info, esr); 601 602 return ret; 603 } 604 605 static const struct fault_info fault_info[] = { 606 { do_bad, SIGBUS, 0, "ttbr address size fault" }, 607 { do_bad, SIGBUS, 0, "level 1 address size fault" }, 608 { do_bad, SIGBUS, 0, "level 2 address size fault" }, 609 { do_bad, SIGBUS, 0, "level 3 address size fault" }, 610 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, 611 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, 612 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, 613 { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, 614 { do_bad, SIGBUS, 0, "unknown 8" }, 615 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, 616 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, 617 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, 618 { do_bad, SIGBUS, 0, "unknown 12" }, 619 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, 620 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, 621 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, 622 { do_sea, SIGBUS, 0, "synchronous external abort" }, 623 { do_bad, SIGBUS, 0, "unknown 17" }, 624 { do_bad, SIGBUS, 0, "unknown 18" }, 625 { do_bad, SIGBUS, 0, "unknown 19" }, 626 { do_sea, SIGBUS, 0, "level 0 (translation table walk)" }, 627 { do_sea, SIGBUS, 0, "level 1 (translation table walk)" }, 628 { do_sea, SIGBUS, 0, "level 2 (translation table walk)" }, 629 { do_sea, SIGBUS, 0, "level 3 (translation table walk)" }, 630 { do_sea, SIGBUS, 0, "synchronous parity or ECC error" }, 631 { do_bad, SIGBUS, 0, "unknown 25" }, 632 { do_bad, SIGBUS, 0, "unknown 26" }, 633 { do_bad, SIGBUS, 0, "unknown 27" }, 634 { do_sea, SIGBUS, 0, "level 0 synchronous parity error (translation table walk)" }, 635 { do_sea, SIGBUS, 0, "level 1 synchronous parity error (translation table walk)" }, 636 { do_sea, SIGBUS, 0, "level 2 synchronous parity error (translation table walk)" }, 637 { do_sea, SIGBUS, 0, "level 3 synchronous parity error (translation table walk)" }, 638 { do_bad, SIGBUS, 0, "unknown 32" }, 639 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, 640 { do_bad, SIGBUS, 0, "unknown 34" }, 641 { do_bad, SIGBUS, 0, "unknown 35" }, 642 { do_bad, SIGBUS, 0, "unknown 36" }, 643 { do_bad, SIGBUS, 0, "unknown 37" }, 644 { do_bad, SIGBUS, 0, "unknown 38" }, 645 { do_bad, SIGBUS, 0, "unknown 39" }, 646 { do_bad, SIGBUS, 0, "unknown 40" }, 647 { do_bad, SIGBUS, 0, "unknown 41" }, 648 { do_bad, SIGBUS, 0, "unknown 42" }, 649 { do_bad, SIGBUS, 0, "unknown 43" }, 650 { do_bad, SIGBUS, 0, "unknown 44" }, 651 { do_bad, SIGBUS, 0, "unknown 45" }, 652 { do_bad, SIGBUS, 0, "unknown 46" }, 653 { do_bad, SIGBUS, 0, "unknown 47" }, 654 { do_bad, SIGBUS, 0, "TLB conflict abort" }, 655 { do_bad, SIGBUS, 0, "unknown 49" }, 656 { do_bad, SIGBUS, 0, "unknown 50" }, 657 { do_bad, SIGBUS, 0, "unknown 51" }, 658 { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" }, 659 { do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" }, 660 { do_bad, SIGBUS, 0, "unknown 54" }, 661 { do_bad, SIGBUS, 0, "unknown 55" }, 662 { do_bad, SIGBUS, 0, "unknown 56" }, 663 { do_bad, SIGBUS, 0, "unknown 57" }, 664 { do_bad, SIGBUS, 0, "unknown 58" }, 665 { do_bad, SIGBUS, 0, "unknown 59" }, 666 { do_bad, SIGBUS, 0, "unknown 60" }, 667 { do_bad, SIGBUS, 0, "section domain fault" }, 668 { do_bad, SIGBUS, 0, "page domain fault" }, 669 { do_bad, SIGBUS, 0, "unknown 63" }, 670 }; 671 672 /* 673 * Handle Synchronous External Aborts that occur in a guest kernel. 674 * 675 * The return value will be zero if the SEA was successfully handled 676 * and non-zero if there was an error processing the error or there was 677 * no error to process. 678 */ 679 int handle_guest_sea(phys_addr_t addr, unsigned int esr) 680 { 681 int ret = -ENOENT; 682 683 if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) 684 ret = ghes_notify_sea(); 685 686 return ret; 687 } 688 689 /* 690 * Dispatch a data abort to the relevant handler. 691 */ 692 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, 693 struct pt_regs *regs) 694 { 695 const struct fault_info *inf = esr_to_fault_info(esr); 696 struct siginfo info; 697 698 if (!inf->fn(addr, esr, regs)) 699 return; 700 701 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", 702 inf->name, esr, addr); 703 704 info.si_signo = inf->sig; 705 info.si_errno = 0; 706 info.si_code = inf->code; 707 info.si_addr = (void __user *)addr; 708 arm64_notify_die("", regs, &info, esr); 709 } 710 711 /* 712 * Handle stack alignment exceptions. 713 */ 714 asmlinkage void __exception do_sp_pc_abort(unsigned long addr, 715 unsigned int esr, 716 struct pt_regs *regs) 717 { 718 struct siginfo info; 719 struct task_struct *tsk = current; 720 721 if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) 722 pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", 723 tsk->comm, task_pid_nr(tsk), 724 esr_get_class_string(esr), (void *)regs->pc, 725 (void *)regs->sp); 726 727 info.si_signo = SIGBUS; 728 info.si_errno = 0; 729 info.si_code = BUS_ADRALN; 730 info.si_addr = (void __user *)addr; 731 arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); 732 } 733 734 int __init early_brk64(unsigned long addr, unsigned int esr, 735 struct pt_regs *regs); 736 737 /* 738 * __refdata because early_brk64 is __init, but the reference to it is 739 * clobbered at arch_initcall time. 740 * See traps.c and debug-monitors.c:debug_traps_init(). 741 */ 742 static struct fault_info __refdata debug_fault_info[] = { 743 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, 744 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, 745 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, 746 { do_bad, SIGBUS, 0, "unknown 3" }, 747 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, 748 { do_bad, SIGTRAP, 0, "aarch32 vector catch" }, 749 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, 750 { do_bad, SIGBUS, 0, "unknown 7" }, 751 }; 752 753 void __init hook_debug_fault_code(int nr, 754 int (*fn)(unsigned long, unsigned int, struct pt_regs *), 755 int sig, int code, const char *name) 756 { 757 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); 758 759 debug_fault_info[nr].fn = fn; 760 debug_fault_info[nr].sig = sig; 761 debug_fault_info[nr].code = code; 762 debug_fault_info[nr].name = name; 763 } 764 765 asmlinkage int __exception do_debug_exception(unsigned long addr, 766 unsigned int esr, 767 struct pt_regs *regs) 768 { 769 const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); 770 struct siginfo info; 771 int rv; 772 773 /* 774 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were 775 * already disabled to preserve the last enabled/disabled addresses. 776 */ 777 if (interrupts_enabled(regs)) 778 trace_hardirqs_off(); 779 780 if (!inf->fn(addr, esr, regs)) { 781 rv = 1; 782 } else { 783 pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", 784 inf->name, esr, addr); 785 786 info.si_signo = inf->sig; 787 info.si_errno = 0; 788 info.si_code = inf->code; 789 info.si_addr = (void __user *)addr; 790 arm64_notify_die("", regs, &info, 0); 791 rv = 0; 792 } 793 794 if (interrupts_enabled(regs)) 795 trace_hardirqs_on(); 796 797 return rv; 798 } 799 NOKPROBE_SYMBOL(do_debug_exception); 800 801 #ifdef CONFIG_ARM64_PAN 802 int cpu_enable_pan(void *__unused) 803 { 804 /* 805 * We modify PSTATE. This won't work from irq context as the PSTATE 806 * is discarded once we return from the exception. 807 */ 808 WARN_ON_ONCE(in_interrupt()); 809 810 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 811 asm(SET_PSTATE_PAN(1)); 812 return 0; 813 } 814 #endif /* CONFIG_ARM64_PAN */ 815