1 /* 2 * Based on arch/arm/mm/fault.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 1995-2004 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include <linux/extable.h> 22 #include <linux/signal.h> 23 #include <linux/mm.h> 24 #include <linux/hardirq.h> 25 #include <linux/init.h> 26 #include <linux/kprobes.h> 27 #include <linux/uaccess.h> 28 #include <linux/page-flags.h> 29 #include <linux/sched/signal.h> 30 #include <linux/sched/debug.h> 31 #include <linux/highmem.h> 32 #include <linux/perf_event.h> 33 #include <linux/preempt.h> 34 #include <linux/hugetlb.h> 35 36 #include <asm/bug.h> 37 #include <asm/cmpxchg.h> 38 #include <asm/cpufeature.h> 39 #include <asm/exception.h> 40 #include <asm/debug-monitors.h> 41 #include <asm/esr.h> 42 #include <asm/sysreg.h> 43 #include <asm/system_misc.h> 44 #include <asm/pgtable.h> 45 #include <asm/tlbflush.h> 46 47 #include <acpi/ghes.h> 48 49 struct fault_info { 50 int (*fn)(unsigned long addr, unsigned int esr, 51 struct pt_regs *regs); 52 int sig; 53 int code; 54 const char *name; 55 }; 56 57 static const struct fault_info fault_info[]; 58 59 static inline const struct fault_info *esr_to_fault_info(unsigned int esr) 60 { 61 return fault_info + (esr & 63); 62 } 63 64 #ifdef CONFIG_KPROBES 65 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 66 { 67 int ret = 0; 68 69 /* kprobe_running() needs smp_processor_id() */ 70 if (!user_mode(regs)) { 71 preempt_disable(); 72 if (kprobe_running() && kprobe_fault_handler(regs, esr)) 73 ret = 1; 74 preempt_enable(); 75 } 76 77 return ret; 78 } 79 #else 80 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 81 { 82 return 0; 83 } 84 #endif 85 86 static void data_abort_decode(unsigned int esr) 87 { 88 pr_alert("Data abort info:\n"); 89 90 if (esr & ESR_ELx_ISV) { 91 pr_alert(" Access size = %u byte(s)\n", 92 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT)); 93 pr_alert(" SSE = %lu, SRT = %lu\n", 94 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT, 95 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT); 96 pr_alert(" SF = %lu, AR = %lu\n", 97 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT, 98 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT); 99 } else { 100 pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK); 101 } 102 103 pr_alert(" CM = %lu, WnR = %lu\n", 104 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT, 105 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT); 106 } 107 108 static void mem_abort_decode(unsigned int esr) 109 { 110 pr_alert("Mem abort info:\n"); 111 112 pr_alert(" ESR = 0x%08x\n", esr); 113 pr_alert(" Exception class = %s, IL = %u bits\n", 114 esr_get_class_string(esr), 115 (esr & ESR_ELx_IL) ? 32 : 16); 116 pr_alert(" SET = %lu, FnV = %lu\n", 117 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT, 118 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT); 119 pr_alert(" EA = %lu, S1PTW = %lu\n", 120 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT, 121 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT); 122 123 if (esr_is_data_abort(esr)) 124 data_abort_decode(esr); 125 } 126 127 /* 128 * Dump out the page tables associated with 'addr' in the currently active mm. 129 */ 130 void show_pte(unsigned long addr) 131 { 132 struct mm_struct *mm; 133 pgd_t *pgd; 134 135 if (addr < TASK_SIZE) { 136 /* TTBR0 */ 137 mm = current->active_mm; 138 if (mm == &init_mm) { 139 pr_alert("[%016lx] user address but active_mm is swapper\n", 140 addr); 141 return; 142 } 143 } else if (addr >= VA_START) { 144 /* TTBR1 */ 145 mm = &init_mm; 146 } else { 147 pr_alert("[%016lx] address between user and kernel address ranges\n", 148 addr); 149 return; 150 } 151 152 pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgd = %p\n", 153 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, 154 VA_BITS, mm->pgd); 155 pgd = pgd_offset(mm, addr); 156 pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd)); 157 158 do { 159 pud_t *pud; 160 pmd_t *pmd; 161 pte_t *pte; 162 163 if (pgd_none(*pgd) || pgd_bad(*pgd)) 164 break; 165 166 pud = pud_offset(pgd, addr); 167 pr_cont(", *pud=%016llx", pud_val(*pud)); 168 if (pud_none(*pud) || pud_bad(*pud)) 169 break; 170 171 pmd = pmd_offset(pud, addr); 172 pr_cont(", *pmd=%016llx", pmd_val(*pmd)); 173 if (pmd_none(*pmd) || pmd_bad(*pmd)) 174 break; 175 176 pte = pte_offset_map(pmd, addr); 177 pr_cont(", *pte=%016llx", pte_val(*pte)); 178 pte_unmap(pte); 179 } while(0); 180 181 pr_cont("\n"); 182 } 183 184 /* 185 * This function sets the access flags (dirty, accessed), as well as write 186 * permission, and only to a more permissive setting. 187 * 188 * It needs to cope with hardware update of the accessed/dirty state by other 189 * agents in the system and can safely skip the __sync_icache_dcache() call as, 190 * like set_pte_at(), the PTE is never changed from no-exec to exec here. 191 * 192 * Returns whether or not the PTE actually changed. 193 */ 194 int ptep_set_access_flags(struct vm_area_struct *vma, 195 unsigned long address, pte_t *ptep, 196 pte_t entry, int dirty) 197 { 198 pteval_t old_pteval, pteval; 199 200 if (pte_same(*ptep, entry)) 201 return 0; 202 203 /* only preserve the access flags and write permission */ 204 pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; 205 206 /* 207 * Setting the flags must be done atomically to avoid racing with the 208 * hardware update of the access/dirty state. The PTE_RDONLY bit must 209 * be set to the most permissive (lowest value) of *ptep and entry 210 * (calculated as: a & b == ~(~a | ~b)). 211 */ 212 pte_val(entry) ^= PTE_RDONLY; 213 pteval = READ_ONCE(pte_val(*ptep)); 214 do { 215 old_pteval = pteval; 216 pteval ^= PTE_RDONLY; 217 pteval |= pte_val(entry); 218 pteval ^= PTE_RDONLY; 219 pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); 220 } while (pteval != old_pteval); 221 222 flush_tlb_fix_spurious_fault(vma, address); 223 return 1; 224 } 225 226 static bool is_el1_instruction_abort(unsigned int esr) 227 { 228 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; 229 } 230 231 static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs, 232 unsigned long addr) 233 { 234 unsigned int ec = ESR_ELx_EC(esr); 235 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; 236 237 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) 238 return false; 239 240 if (fsc_type == ESR_ELx_FSC_PERM) 241 return true; 242 243 if (addr < USER_DS && system_uses_ttbr0_pan()) 244 return fsc_type == ESR_ELx_FSC_FAULT && 245 (regs->pstate & PSR_PAN_BIT); 246 247 return false; 248 } 249 250 static void __do_kernel_fault(unsigned long addr, unsigned int esr, 251 struct pt_regs *regs) 252 { 253 const char *msg; 254 255 /* 256 * Are we prepared to handle this kernel fault? 257 * We are almost certainly not prepared to handle instruction faults. 258 */ 259 if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) 260 return; 261 262 bust_spinlocks(1); 263 264 if (is_permission_fault(esr, regs, addr)) { 265 if (esr & ESR_ELx_WNR) 266 msg = "write to read-only memory"; 267 else 268 msg = "read from unreadable memory"; 269 } else if (addr < PAGE_SIZE) { 270 msg = "NULL pointer dereference"; 271 } else { 272 msg = "paging request"; 273 } 274 275 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, 276 addr); 277 278 mem_abort_decode(esr); 279 280 show_pte(addr); 281 die("Oops", regs, esr); 282 bust_spinlocks(0); 283 do_exit(SIGKILL); 284 } 285 286 static void __do_user_fault(struct task_struct *tsk, unsigned long addr, 287 unsigned int esr, unsigned int sig, int code, 288 struct pt_regs *regs, int fault) 289 { 290 struct siginfo si; 291 const struct fault_info *inf; 292 unsigned int lsb = 0; 293 294 if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { 295 inf = esr_to_fault_info(esr); 296 pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x", 297 tsk->comm, task_pid_nr(tsk), inf->name, sig, 298 addr, esr); 299 print_vma_addr(KERN_CONT ", in ", regs->pc); 300 pr_cont("\n"); 301 __show_regs(regs); 302 } 303 304 tsk->thread.fault_address = addr; 305 tsk->thread.fault_code = esr; 306 si.si_signo = sig; 307 si.si_errno = 0; 308 si.si_code = code; 309 si.si_addr = (void __user *)addr; 310 /* 311 * Either small page or large page may be poisoned. 312 * In other words, VM_FAULT_HWPOISON_LARGE and 313 * VM_FAULT_HWPOISON are mutually exclusive. 314 */ 315 if (fault & VM_FAULT_HWPOISON_LARGE) 316 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); 317 else if (fault & VM_FAULT_HWPOISON) 318 lsb = PAGE_SHIFT; 319 si.si_addr_lsb = lsb; 320 321 force_sig_info(sig, &si, tsk); 322 } 323 324 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) 325 { 326 struct task_struct *tsk = current; 327 const struct fault_info *inf; 328 329 /* 330 * If we are in kernel mode at this point, we have no context to 331 * handle this fault with. 332 */ 333 if (user_mode(regs)) { 334 inf = esr_to_fault_info(esr); 335 __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs, 0); 336 } else 337 __do_kernel_fault(addr, esr, regs); 338 } 339 340 #define VM_FAULT_BADMAP 0x010000 341 #define VM_FAULT_BADACCESS 0x020000 342 343 static int __do_page_fault(struct mm_struct *mm, unsigned long addr, 344 unsigned int mm_flags, unsigned long vm_flags, 345 struct task_struct *tsk) 346 { 347 struct vm_area_struct *vma; 348 int fault; 349 350 vma = find_vma(mm, addr); 351 fault = VM_FAULT_BADMAP; 352 if (unlikely(!vma)) 353 goto out; 354 if (unlikely(vma->vm_start > addr)) 355 goto check_stack; 356 357 /* 358 * Ok, we have a good vm_area for this memory access, so we can handle 359 * it. 360 */ 361 good_area: 362 /* 363 * Check that the permissions on the VMA allow for the fault which 364 * occurred. 365 */ 366 if (!(vma->vm_flags & vm_flags)) { 367 fault = VM_FAULT_BADACCESS; 368 goto out; 369 } 370 371 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); 372 373 check_stack: 374 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) 375 goto good_area; 376 out: 377 return fault; 378 } 379 380 static bool is_el0_instruction_abort(unsigned int esr) 381 { 382 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; 383 } 384 385 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, 386 struct pt_regs *regs) 387 { 388 struct task_struct *tsk; 389 struct mm_struct *mm; 390 int fault, sig, code, major = 0; 391 unsigned long vm_flags = VM_READ | VM_WRITE; 392 unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 393 394 if (notify_page_fault(regs, esr)) 395 return 0; 396 397 tsk = current; 398 mm = tsk->mm; 399 400 /* 401 * If we're in an interrupt or have no user context, we must not take 402 * the fault. 403 */ 404 if (faulthandler_disabled() || !mm) 405 goto no_context; 406 407 if (user_mode(regs)) 408 mm_flags |= FAULT_FLAG_USER; 409 410 if (is_el0_instruction_abort(esr)) { 411 vm_flags = VM_EXEC; 412 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { 413 vm_flags = VM_WRITE; 414 mm_flags |= FAULT_FLAG_WRITE; 415 } 416 417 if (addr < USER_DS && is_permission_fault(esr, regs, addr)) { 418 /* regs->orig_addr_limit may be 0 if we entered from EL0 */ 419 if (regs->orig_addr_limit == KERNEL_DS) 420 die("Accessing user space memory with fs=KERNEL_DS", regs, esr); 421 422 if (is_el1_instruction_abort(esr)) 423 die("Attempting to execute userspace memory", regs, esr); 424 425 if (!search_exception_tables(regs->pc)) 426 die("Accessing user space memory outside uaccess.h routines", regs, esr); 427 } 428 429 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 430 431 /* 432 * As per x86, we may deadlock here. However, since the kernel only 433 * validly references user space from well defined areas of the code, 434 * we can bug out early if this is from code which shouldn't. 435 */ 436 if (!down_read_trylock(&mm->mmap_sem)) { 437 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 438 goto no_context; 439 retry: 440 down_read(&mm->mmap_sem); 441 } else { 442 /* 443 * The above down_read_trylock() might have succeeded in which 444 * case, we'll have missed the might_sleep() from down_read(). 445 */ 446 might_sleep(); 447 #ifdef CONFIG_DEBUG_VM 448 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 449 goto no_context; 450 #endif 451 } 452 453 fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); 454 major |= fault & VM_FAULT_MAJOR; 455 456 if (fault & VM_FAULT_RETRY) { 457 /* 458 * If we need to retry but a fatal signal is pending, 459 * handle the signal first. We do not need to release 460 * the mmap_sem because it would already be released 461 * in __lock_page_or_retry in mm/filemap.c. 462 */ 463 if (fatal_signal_pending(current)) { 464 if (!user_mode(regs)) 465 goto no_context; 466 return 0; 467 } 468 469 /* 470 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of 471 * starvation. 472 */ 473 if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { 474 mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; 475 mm_flags |= FAULT_FLAG_TRIED; 476 goto retry; 477 } 478 } 479 up_read(&mm->mmap_sem); 480 481 /* 482 * Handle the "normal" (no error) case first. 483 */ 484 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | 485 VM_FAULT_BADACCESS)))) { 486 /* 487 * Major/minor page fault accounting is only done 488 * once. If we go through a retry, it is extremely 489 * likely that the page will be found in page cache at 490 * that point. 491 */ 492 if (major) { 493 tsk->maj_flt++; 494 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, 495 addr); 496 } else { 497 tsk->min_flt++; 498 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, 499 addr); 500 } 501 502 return 0; 503 } 504 505 /* 506 * If we are in kernel mode at this point, we have no context to 507 * handle this fault with. 508 */ 509 if (!user_mode(regs)) 510 goto no_context; 511 512 if (fault & VM_FAULT_OOM) { 513 /* 514 * We ran out of memory, call the OOM killer, and return to 515 * userspace (which will retry the fault, or kill us if we got 516 * oom-killed). 517 */ 518 pagefault_out_of_memory(); 519 return 0; 520 } 521 522 if (fault & VM_FAULT_SIGBUS) { 523 /* 524 * We had some memory, but were unable to successfully fix up 525 * this page fault. 526 */ 527 sig = SIGBUS; 528 code = BUS_ADRERR; 529 } else if (fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) { 530 sig = SIGBUS; 531 code = BUS_MCEERR_AR; 532 } else { 533 /* 534 * Something tried to access memory that isn't in our memory 535 * map. 536 */ 537 sig = SIGSEGV; 538 code = fault == VM_FAULT_BADACCESS ? 539 SEGV_ACCERR : SEGV_MAPERR; 540 } 541 542 __do_user_fault(tsk, addr, esr, sig, code, regs, fault); 543 return 0; 544 545 no_context: 546 __do_kernel_fault(addr, esr, regs); 547 return 0; 548 } 549 550 static int __kprobes do_translation_fault(unsigned long addr, 551 unsigned int esr, 552 struct pt_regs *regs) 553 { 554 if (addr < TASK_SIZE) 555 return do_page_fault(addr, esr, regs); 556 557 do_bad_area(addr, esr, regs); 558 return 0; 559 } 560 561 static int do_alignment_fault(unsigned long addr, unsigned int esr, 562 struct pt_regs *regs) 563 { 564 do_bad_area(addr, esr, regs); 565 return 0; 566 } 567 568 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) 569 { 570 return 1; /* "fault" */ 571 } 572 573 static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs) 574 { 575 struct siginfo info; 576 const struct fault_info *inf; 577 578 inf = esr_to_fault_info(esr); 579 pr_err("Synchronous External Abort: %s (0x%08x) at 0x%016lx\n", 580 inf->name, esr, addr); 581 582 /* 583 * Synchronous aborts may interrupt code which had interrupts masked. 584 * Before calling out into the wider kernel tell the interested 585 * subsystems. 586 */ 587 if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) { 588 if (interrupts_enabled(regs)) 589 nmi_enter(); 590 591 ghes_notify_sea(); 592 593 if (interrupts_enabled(regs)) 594 nmi_exit(); 595 } 596 597 info.si_signo = SIGBUS; 598 info.si_errno = 0; 599 info.si_code = 0; 600 if (esr & ESR_ELx_FnV) 601 info.si_addr = NULL; 602 else 603 info.si_addr = (void __user *)addr; 604 arm64_notify_die("", regs, &info, esr); 605 606 return 0; 607 } 608 609 static const struct fault_info fault_info[] = { 610 { do_bad, SIGBUS, 0, "ttbr address size fault" }, 611 { do_bad, SIGBUS, 0, "level 1 address size fault" }, 612 { do_bad, SIGBUS, 0, "level 2 address size fault" }, 613 { do_bad, SIGBUS, 0, "level 3 address size fault" }, 614 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, 615 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, 616 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, 617 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, 618 { do_bad, SIGBUS, 0, "unknown 8" }, 619 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, 620 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, 621 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, 622 { do_bad, SIGBUS, 0, "unknown 12" }, 623 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, 624 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, 625 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, 626 { do_sea, SIGBUS, 0, "synchronous external abort" }, 627 { do_bad, SIGBUS, 0, "unknown 17" }, 628 { do_bad, SIGBUS, 0, "unknown 18" }, 629 { do_bad, SIGBUS, 0, "unknown 19" }, 630 { do_sea, SIGBUS, 0, "level 0 (translation table walk)" }, 631 { do_sea, SIGBUS, 0, "level 1 (translation table walk)" }, 632 { do_sea, SIGBUS, 0, "level 2 (translation table walk)" }, 633 { do_sea, SIGBUS, 0, "level 3 (translation table walk)" }, 634 { do_sea, SIGBUS, 0, "synchronous parity or ECC error" }, // Reserved when RAS is implemented 635 { do_bad, SIGBUS, 0, "unknown 25" }, 636 { do_bad, SIGBUS, 0, "unknown 26" }, 637 { do_bad, SIGBUS, 0, "unknown 27" }, 638 { do_sea, SIGBUS, 0, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 639 { do_sea, SIGBUS, 0, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 640 { do_sea, SIGBUS, 0, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 641 { do_sea, SIGBUS, 0, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 642 { do_bad, SIGBUS, 0, "unknown 32" }, 643 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, 644 { do_bad, SIGBUS, 0, "unknown 34" }, 645 { do_bad, SIGBUS, 0, "unknown 35" }, 646 { do_bad, SIGBUS, 0, "unknown 36" }, 647 { do_bad, SIGBUS, 0, "unknown 37" }, 648 { do_bad, SIGBUS, 0, "unknown 38" }, 649 { do_bad, SIGBUS, 0, "unknown 39" }, 650 { do_bad, SIGBUS, 0, "unknown 40" }, 651 { do_bad, SIGBUS, 0, "unknown 41" }, 652 { do_bad, SIGBUS, 0, "unknown 42" }, 653 { do_bad, SIGBUS, 0, "unknown 43" }, 654 { do_bad, SIGBUS, 0, "unknown 44" }, 655 { do_bad, SIGBUS, 0, "unknown 45" }, 656 { do_bad, SIGBUS, 0, "unknown 46" }, 657 { do_bad, SIGBUS, 0, "unknown 47" }, 658 { do_bad, SIGBUS, 0, "TLB conflict abort" }, 659 { do_bad, SIGBUS, 0, "Unsupported atomic hardware update fault" }, 660 { do_bad, SIGBUS, 0, "unknown 50" }, 661 { do_bad, SIGBUS, 0, "unknown 51" }, 662 { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" }, 663 { do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" }, 664 { do_bad, SIGBUS, 0, "unknown 54" }, 665 { do_bad, SIGBUS, 0, "unknown 55" }, 666 { do_bad, SIGBUS, 0, "unknown 56" }, 667 { do_bad, SIGBUS, 0, "unknown 57" }, 668 { do_bad, SIGBUS, 0, "unknown 58" }, 669 { do_bad, SIGBUS, 0, "unknown 59" }, 670 { do_bad, SIGBUS, 0, "unknown 60" }, 671 { do_bad, SIGBUS, 0, "section domain fault" }, 672 { do_bad, SIGBUS, 0, "page domain fault" }, 673 { do_bad, SIGBUS, 0, "unknown 63" }, 674 }; 675 676 int handle_guest_sea(phys_addr_t addr, unsigned int esr) 677 { 678 int ret = -ENOENT; 679 680 if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) 681 ret = ghes_notify_sea(); 682 683 return ret; 684 } 685 686 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, 687 struct pt_regs *regs) 688 { 689 const struct fault_info *inf = esr_to_fault_info(esr); 690 struct siginfo info; 691 692 if (!inf->fn(addr, esr, regs)) 693 return; 694 695 pr_alert("Unhandled fault: %s at 0x%016lx\n", 696 inf->name, addr); 697 698 mem_abort_decode(esr); 699 700 if (!user_mode(regs)) 701 show_pte(addr); 702 703 info.si_signo = inf->sig; 704 info.si_errno = 0; 705 info.si_code = inf->code; 706 info.si_addr = (void __user *)addr; 707 arm64_notify_die("", regs, &info, esr); 708 } 709 710 asmlinkage void __exception do_sp_pc_abort(unsigned long addr, 711 unsigned int esr, 712 struct pt_regs *regs) 713 { 714 struct siginfo info; 715 struct task_struct *tsk = current; 716 717 if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) 718 pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", 719 tsk->comm, task_pid_nr(tsk), 720 esr_get_class_string(esr), (void *)regs->pc, 721 (void *)regs->sp); 722 723 info.si_signo = SIGBUS; 724 info.si_errno = 0; 725 info.si_code = BUS_ADRALN; 726 info.si_addr = (void __user *)addr; 727 arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); 728 } 729 730 int __init early_brk64(unsigned long addr, unsigned int esr, 731 struct pt_regs *regs); 732 733 /* 734 * __refdata because early_brk64 is __init, but the reference to it is 735 * clobbered at arch_initcall time. 736 * See traps.c and debug-monitors.c:debug_traps_init(). 737 */ 738 static struct fault_info __refdata debug_fault_info[] = { 739 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, 740 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, 741 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, 742 { do_bad, SIGBUS, 0, "unknown 3" }, 743 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, 744 { do_bad, SIGTRAP, 0, "aarch32 vector catch" }, 745 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, 746 { do_bad, SIGBUS, 0, "unknown 7" }, 747 }; 748 749 void __init hook_debug_fault_code(int nr, 750 int (*fn)(unsigned long, unsigned int, struct pt_regs *), 751 int sig, int code, const char *name) 752 { 753 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); 754 755 debug_fault_info[nr].fn = fn; 756 debug_fault_info[nr].sig = sig; 757 debug_fault_info[nr].code = code; 758 debug_fault_info[nr].name = name; 759 } 760 761 asmlinkage int __exception do_debug_exception(unsigned long addr, 762 unsigned int esr, 763 struct pt_regs *regs) 764 { 765 const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); 766 struct siginfo info; 767 int rv; 768 769 /* 770 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were 771 * already disabled to preserve the last enabled/disabled addresses. 772 */ 773 if (interrupts_enabled(regs)) 774 trace_hardirqs_off(); 775 776 if (!inf->fn(addr, esr, regs)) { 777 rv = 1; 778 } else { 779 pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", 780 inf->name, esr, addr); 781 782 info.si_signo = inf->sig; 783 info.si_errno = 0; 784 info.si_code = inf->code; 785 info.si_addr = (void __user *)addr; 786 arm64_notify_die("", regs, &info, 0); 787 rv = 0; 788 } 789 790 if (interrupts_enabled(regs)) 791 trace_hardirqs_on(); 792 793 return rv; 794 } 795 NOKPROBE_SYMBOL(do_debug_exception); 796 797 #ifdef CONFIG_ARM64_PAN 798 int cpu_enable_pan(void *__unused) 799 { 800 /* 801 * We modify PSTATE. This won't work from irq context as the PSTATE 802 * is discarded once we return from the exception. 803 */ 804 WARN_ON_ONCE(in_interrupt()); 805 806 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 807 asm(SET_PSTATE_PAN(1)); 808 return 0; 809 } 810 #endif /* CONFIG_ARM64_PAN */ 811