1 /* 2 * Based on arch/arm/mm/fault.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 1995-2004 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include <linux/extable.h> 22 #include <linux/signal.h> 23 #include <linux/mm.h> 24 #include <linux/hardirq.h> 25 #include <linux/init.h> 26 #include <linux/kprobes.h> 27 #include <linux/uaccess.h> 28 #include <linux/page-flags.h> 29 #include <linux/sched/signal.h> 30 #include <linux/sched/debug.h> 31 #include <linux/highmem.h> 32 #include <linux/perf_event.h> 33 #include <linux/preempt.h> 34 #include <linux/hugetlb.h> 35 36 #include <asm/bug.h> 37 #include <asm/cmpxchg.h> 38 #include <asm/cpufeature.h> 39 #include <asm/exception.h> 40 #include <asm/debug-monitors.h> 41 #include <asm/esr.h> 42 #include <asm/sysreg.h> 43 #include <asm/system_misc.h> 44 #include <asm/pgtable.h> 45 #include <asm/tlbflush.h> 46 47 #include <acpi/ghes.h> 48 49 struct fault_info { 50 int (*fn)(unsigned long addr, unsigned int esr, 51 struct pt_regs *regs); 52 int sig; 53 int code; 54 const char *name; 55 }; 56 57 static const struct fault_info fault_info[]; 58 59 static inline const struct fault_info *esr_to_fault_info(unsigned int esr) 60 { 61 return fault_info + (esr & 63); 62 } 63 64 #ifdef CONFIG_KPROBES 65 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 66 { 67 int ret = 0; 68 69 /* kprobe_running() needs smp_processor_id() */ 70 if (!user_mode(regs)) { 71 preempt_disable(); 72 if (kprobe_running() && kprobe_fault_handler(regs, esr)) 73 ret = 1; 74 preempt_enable(); 75 } 76 77 return ret; 78 } 79 #else 80 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) 81 { 82 return 0; 83 } 84 #endif 85 86 static void data_abort_decode(unsigned int esr) 87 { 88 pr_alert("Data abort info:\n"); 89 90 if (esr & ESR_ELx_ISV) { 91 pr_alert(" Access size = %u byte(s)\n", 92 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT)); 93 pr_alert(" SSE = %lu, SRT = %lu\n", 94 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT, 95 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT); 96 pr_alert(" SF = %lu, AR = %lu\n", 97 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT, 98 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT); 99 } else { 100 pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK); 101 } 102 103 pr_alert(" CM = %lu, WnR = %lu\n", 104 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT, 105 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT); 106 } 107 108 static void mem_abort_decode(unsigned int esr) 109 { 110 pr_alert("Mem abort info:\n"); 111 112 pr_alert(" ESR = 0x%08x\n", esr); 113 pr_alert(" Exception class = %s, IL = %u bits\n", 114 esr_get_class_string(esr), 115 (esr & ESR_ELx_IL) ? 32 : 16); 116 pr_alert(" SET = %lu, FnV = %lu\n", 117 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT, 118 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT); 119 pr_alert(" EA = %lu, S1PTW = %lu\n", 120 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT, 121 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT); 122 123 if (esr_is_data_abort(esr)) 124 data_abort_decode(esr); 125 } 126 127 /* 128 * Dump out the page tables associated with 'addr' in the currently active mm. 129 */ 130 void show_pte(unsigned long addr) 131 { 132 struct mm_struct *mm; 133 pgd_t *pgdp; 134 pgd_t pgd; 135 136 if (addr < TASK_SIZE) { 137 /* TTBR0 */ 138 mm = current->active_mm; 139 if (mm == &init_mm) { 140 pr_alert("[%016lx] user address but active_mm is swapper\n", 141 addr); 142 return; 143 } 144 } else if (addr >= VA_START) { 145 /* TTBR1 */ 146 mm = &init_mm; 147 } else { 148 pr_alert("[%016lx] address between user and kernel address ranges\n", 149 addr); 150 return; 151 } 152 153 pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgdp = %p\n", 154 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, 155 VA_BITS, mm->pgd); 156 pgdp = pgd_offset(mm, addr); 157 pgd = READ_ONCE(*pgdp); 158 pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd)); 159 160 do { 161 pud_t *pudp, pud; 162 pmd_t *pmdp, pmd; 163 pte_t *ptep, pte; 164 165 if (pgd_none(pgd) || pgd_bad(pgd)) 166 break; 167 168 pudp = pud_offset(pgdp, addr); 169 pud = READ_ONCE(*pudp); 170 pr_cont(", pud=%016llx", pud_val(pud)); 171 if (pud_none(pud) || pud_bad(pud)) 172 break; 173 174 pmdp = pmd_offset(pudp, addr); 175 pmd = READ_ONCE(*pmdp); 176 pr_cont(", pmd=%016llx", pmd_val(pmd)); 177 if (pmd_none(pmd) || pmd_bad(pmd)) 178 break; 179 180 ptep = pte_offset_map(pmdp, addr); 181 pte = READ_ONCE(*ptep); 182 pr_cont(", pte=%016llx", pte_val(pte)); 183 pte_unmap(ptep); 184 } while(0); 185 186 pr_cont("\n"); 187 } 188 189 /* 190 * This function sets the access flags (dirty, accessed), as well as write 191 * permission, and only to a more permissive setting. 192 * 193 * It needs to cope with hardware update of the accessed/dirty state by other 194 * agents in the system and can safely skip the __sync_icache_dcache() call as, 195 * like set_pte_at(), the PTE is never changed from no-exec to exec here. 196 * 197 * Returns whether or not the PTE actually changed. 198 */ 199 int ptep_set_access_flags(struct vm_area_struct *vma, 200 unsigned long address, pte_t *ptep, 201 pte_t entry, int dirty) 202 { 203 pteval_t old_pteval, pteval; 204 pte_t pte = READ_ONCE(*ptep); 205 206 if (pte_same(pte, entry)) 207 return 0; 208 209 /* only preserve the access flags and write permission */ 210 pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; 211 212 /* 213 * Setting the flags must be done atomically to avoid racing with the 214 * hardware update of the access/dirty state. The PTE_RDONLY bit must 215 * be set to the most permissive (lowest value) of *ptep and entry 216 * (calculated as: a & b == ~(~a | ~b)). 217 */ 218 pte_val(entry) ^= PTE_RDONLY; 219 pteval = pte_val(pte); 220 do { 221 old_pteval = pteval; 222 pteval ^= PTE_RDONLY; 223 pteval |= pte_val(entry); 224 pteval ^= PTE_RDONLY; 225 pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); 226 } while (pteval != old_pteval); 227 228 flush_tlb_fix_spurious_fault(vma, address); 229 return 1; 230 } 231 232 static bool is_el1_instruction_abort(unsigned int esr) 233 { 234 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; 235 } 236 237 static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs, 238 unsigned long addr) 239 { 240 unsigned int ec = ESR_ELx_EC(esr); 241 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; 242 243 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) 244 return false; 245 246 if (fsc_type == ESR_ELx_FSC_PERM) 247 return true; 248 249 if (addr < TASK_SIZE && system_uses_ttbr0_pan()) 250 return fsc_type == ESR_ELx_FSC_FAULT && 251 (regs->pstate & PSR_PAN_BIT); 252 253 return false; 254 } 255 256 static void __do_kernel_fault(unsigned long addr, unsigned int esr, 257 struct pt_regs *regs) 258 { 259 const char *msg; 260 261 /* 262 * Are we prepared to handle this kernel fault? 263 * We are almost certainly not prepared to handle instruction faults. 264 */ 265 if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) 266 return; 267 268 bust_spinlocks(1); 269 270 if (is_permission_fault(esr, regs, addr)) { 271 if (esr & ESR_ELx_WNR) 272 msg = "write to read-only memory"; 273 else 274 msg = "read from unreadable memory"; 275 } else if (addr < PAGE_SIZE) { 276 msg = "NULL pointer dereference"; 277 } else { 278 msg = "paging request"; 279 } 280 281 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, 282 addr); 283 284 mem_abort_decode(esr); 285 286 show_pte(addr); 287 die("Oops", regs, esr); 288 bust_spinlocks(0); 289 do_exit(SIGKILL); 290 } 291 292 static void __do_user_fault(struct task_struct *tsk, unsigned long addr, 293 unsigned int esr, unsigned int sig, int code, 294 struct pt_regs *regs, int fault) 295 { 296 struct siginfo si; 297 const struct fault_info *inf; 298 unsigned int lsb = 0; 299 300 if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { 301 inf = esr_to_fault_info(esr); 302 pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x", 303 tsk->comm, task_pid_nr(tsk), inf->name, sig, 304 addr, esr); 305 print_vma_addr(KERN_CONT ", in ", regs->pc); 306 pr_cont("\n"); 307 __show_regs(regs); 308 } 309 310 tsk->thread.fault_address = addr; 311 tsk->thread.fault_code = esr; 312 si.si_signo = sig; 313 si.si_errno = 0; 314 si.si_code = code; 315 si.si_addr = (void __user *)addr; 316 /* 317 * Either small page or large page may be poisoned. 318 * In other words, VM_FAULT_HWPOISON_LARGE and 319 * VM_FAULT_HWPOISON are mutually exclusive. 320 */ 321 if (fault & VM_FAULT_HWPOISON_LARGE) 322 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); 323 else if (fault & VM_FAULT_HWPOISON) 324 lsb = PAGE_SHIFT; 325 si.si_addr_lsb = lsb; 326 327 force_sig_info(sig, &si, tsk); 328 } 329 330 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) 331 { 332 struct task_struct *tsk = current; 333 const struct fault_info *inf; 334 335 /* 336 * If we are in kernel mode at this point, we have no context to 337 * handle this fault with. 338 */ 339 if (user_mode(regs)) { 340 inf = esr_to_fault_info(esr); 341 __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs, 0); 342 } else 343 __do_kernel_fault(addr, esr, regs); 344 } 345 346 #define VM_FAULT_BADMAP 0x010000 347 #define VM_FAULT_BADACCESS 0x020000 348 349 static int __do_page_fault(struct mm_struct *mm, unsigned long addr, 350 unsigned int mm_flags, unsigned long vm_flags, 351 struct task_struct *tsk) 352 { 353 struct vm_area_struct *vma; 354 int fault; 355 356 vma = find_vma(mm, addr); 357 fault = VM_FAULT_BADMAP; 358 if (unlikely(!vma)) 359 goto out; 360 if (unlikely(vma->vm_start > addr)) 361 goto check_stack; 362 363 /* 364 * Ok, we have a good vm_area for this memory access, so we can handle 365 * it. 366 */ 367 good_area: 368 /* 369 * Check that the permissions on the VMA allow for the fault which 370 * occurred. 371 */ 372 if (!(vma->vm_flags & vm_flags)) { 373 fault = VM_FAULT_BADACCESS; 374 goto out; 375 } 376 377 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); 378 379 check_stack: 380 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) 381 goto good_area; 382 out: 383 return fault; 384 } 385 386 static bool is_el0_instruction_abort(unsigned int esr) 387 { 388 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; 389 } 390 391 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, 392 struct pt_regs *regs) 393 { 394 struct task_struct *tsk; 395 struct mm_struct *mm; 396 int fault, sig, code, major = 0; 397 unsigned long vm_flags = VM_READ | VM_WRITE; 398 unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 399 400 if (notify_page_fault(regs, esr)) 401 return 0; 402 403 tsk = current; 404 mm = tsk->mm; 405 406 /* 407 * If we're in an interrupt or have no user context, we must not take 408 * the fault. 409 */ 410 if (faulthandler_disabled() || !mm) 411 goto no_context; 412 413 if (user_mode(regs)) 414 mm_flags |= FAULT_FLAG_USER; 415 416 if (is_el0_instruction_abort(esr)) { 417 vm_flags = VM_EXEC; 418 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { 419 vm_flags = VM_WRITE; 420 mm_flags |= FAULT_FLAG_WRITE; 421 } 422 423 if (addr < TASK_SIZE && is_permission_fault(esr, regs, addr)) { 424 /* regs->orig_addr_limit may be 0 if we entered from EL0 */ 425 if (regs->orig_addr_limit == KERNEL_DS) 426 die("Accessing user space memory with fs=KERNEL_DS", regs, esr); 427 428 if (is_el1_instruction_abort(esr)) 429 die("Attempting to execute userspace memory", regs, esr); 430 431 if (!search_exception_tables(regs->pc)) 432 die("Accessing user space memory outside uaccess.h routines", regs, esr); 433 } 434 435 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 436 437 /* 438 * As per x86, we may deadlock here. However, since the kernel only 439 * validly references user space from well defined areas of the code, 440 * we can bug out early if this is from code which shouldn't. 441 */ 442 if (!down_read_trylock(&mm->mmap_sem)) { 443 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 444 goto no_context; 445 retry: 446 down_read(&mm->mmap_sem); 447 } else { 448 /* 449 * The above down_read_trylock() might have succeeded in which 450 * case, we'll have missed the might_sleep() from down_read(). 451 */ 452 might_sleep(); 453 #ifdef CONFIG_DEBUG_VM 454 if (!user_mode(regs) && !search_exception_tables(regs->pc)) 455 goto no_context; 456 #endif 457 } 458 459 fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); 460 major |= fault & VM_FAULT_MAJOR; 461 462 if (fault & VM_FAULT_RETRY) { 463 /* 464 * If we need to retry but a fatal signal is pending, 465 * handle the signal first. We do not need to release 466 * the mmap_sem because it would already be released 467 * in __lock_page_or_retry in mm/filemap.c. 468 */ 469 if (fatal_signal_pending(current)) { 470 if (!user_mode(regs)) 471 goto no_context; 472 return 0; 473 } 474 475 /* 476 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of 477 * starvation. 478 */ 479 if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { 480 mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; 481 mm_flags |= FAULT_FLAG_TRIED; 482 goto retry; 483 } 484 } 485 up_read(&mm->mmap_sem); 486 487 /* 488 * Handle the "normal" (no error) case first. 489 */ 490 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | 491 VM_FAULT_BADACCESS)))) { 492 /* 493 * Major/minor page fault accounting is only done 494 * once. If we go through a retry, it is extremely 495 * likely that the page will be found in page cache at 496 * that point. 497 */ 498 if (major) { 499 tsk->maj_flt++; 500 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, 501 addr); 502 } else { 503 tsk->min_flt++; 504 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, 505 addr); 506 } 507 508 return 0; 509 } 510 511 /* 512 * If we are in kernel mode at this point, we have no context to 513 * handle this fault with. 514 */ 515 if (!user_mode(regs)) 516 goto no_context; 517 518 if (fault & VM_FAULT_OOM) { 519 /* 520 * We ran out of memory, call the OOM killer, and return to 521 * userspace (which will retry the fault, or kill us if we got 522 * oom-killed). 523 */ 524 pagefault_out_of_memory(); 525 return 0; 526 } 527 528 if (fault & VM_FAULT_SIGBUS) { 529 /* 530 * We had some memory, but were unable to successfully fix up 531 * this page fault. 532 */ 533 sig = SIGBUS; 534 code = BUS_ADRERR; 535 } else if (fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) { 536 sig = SIGBUS; 537 code = BUS_MCEERR_AR; 538 } else { 539 /* 540 * Something tried to access memory that isn't in our memory 541 * map. 542 */ 543 sig = SIGSEGV; 544 code = fault == VM_FAULT_BADACCESS ? 545 SEGV_ACCERR : SEGV_MAPERR; 546 } 547 548 __do_user_fault(tsk, addr, esr, sig, code, regs, fault); 549 return 0; 550 551 no_context: 552 __do_kernel_fault(addr, esr, regs); 553 return 0; 554 } 555 556 static int __kprobes do_translation_fault(unsigned long addr, 557 unsigned int esr, 558 struct pt_regs *regs) 559 { 560 if (addr < TASK_SIZE) 561 return do_page_fault(addr, esr, regs); 562 563 do_bad_area(addr, esr, regs); 564 return 0; 565 } 566 567 static int do_alignment_fault(unsigned long addr, unsigned int esr, 568 struct pt_regs *regs) 569 { 570 do_bad_area(addr, esr, regs); 571 return 0; 572 } 573 574 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) 575 { 576 return 1; /* "fault" */ 577 } 578 579 static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs) 580 { 581 struct siginfo info; 582 const struct fault_info *inf; 583 584 inf = esr_to_fault_info(esr); 585 pr_err("Synchronous External Abort: %s (0x%08x) at 0x%016lx\n", 586 inf->name, esr, addr); 587 588 /* 589 * Synchronous aborts may interrupt code which had interrupts masked. 590 * Before calling out into the wider kernel tell the interested 591 * subsystems. 592 */ 593 if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) { 594 if (interrupts_enabled(regs)) 595 nmi_enter(); 596 597 ghes_notify_sea(); 598 599 if (interrupts_enabled(regs)) 600 nmi_exit(); 601 } 602 603 info.si_signo = SIGBUS; 604 info.si_errno = 0; 605 info.si_code = BUS_FIXME; 606 if (esr & ESR_ELx_FnV) 607 info.si_addr = NULL; 608 else 609 info.si_addr = (void __user *)addr; 610 arm64_notify_die("", regs, &info, esr); 611 612 return 0; 613 } 614 615 static const struct fault_info fault_info[] = { 616 { do_bad, SIGBUS, BUS_FIXME, "ttbr address size fault" }, 617 { do_bad, SIGBUS, BUS_FIXME, "level 1 address size fault" }, 618 { do_bad, SIGBUS, BUS_FIXME, "level 2 address size fault" }, 619 { do_bad, SIGBUS, BUS_FIXME, "level 3 address size fault" }, 620 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, 621 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, 622 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, 623 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, 624 { do_bad, SIGBUS, BUS_FIXME, "unknown 8" }, 625 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, 626 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, 627 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, 628 { do_bad, SIGBUS, BUS_FIXME, "unknown 12" }, 629 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, 630 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, 631 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, 632 { do_sea, SIGBUS, BUS_FIXME, "synchronous external abort" }, 633 { do_bad, SIGBUS, BUS_FIXME, "unknown 17" }, 634 { do_bad, SIGBUS, BUS_FIXME, "unknown 18" }, 635 { do_bad, SIGBUS, BUS_FIXME, "unknown 19" }, 636 { do_sea, SIGBUS, BUS_FIXME, "level 0 (translation table walk)" }, 637 { do_sea, SIGBUS, BUS_FIXME, "level 1 (translation table walk)" }, 638 { do_sea, SIGBUS, BUS_FIXME, "level 2 (translation table walk)" }, 639 { do_sea, SIGBUS, BUS_FIXME, "level 3 (translation table walk)" }, 640 { do_sea, SIGBUS, BUS_FIXME, "synchronous parity or ECC error" }, // Reserved when RAS is implemented 641 { do_bad, SIGBUS, BUS_FIXME, "unknown 25" }, 642 { do_bad, SIGBUS, BUS_FIXME, "unknown 26" }, 643 { do_bad, SIGBUS, BUS_FIXME, "unknown 27" }, 644 { do_sea, SIGBUS, BUS_FIXME, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 645 { do_sea, SIGBUS, BUS_FIXME, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 646 { do_sea, SIGBUS, BUS_FIXME, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 647 { do_sea, SIGBUS, BUS_FIXME, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented 648 { do_bad, SIGBUS, BUS_FIXME, "unknown 32" }, 649 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, 650 { do_bad, SIGBUS, BUS_FIXME, "unknown 34" }, 651 { do_bad, SIGBUS, BUS_FIXME, "unknown 35" }, 652 { do_bad, SIGBUS, BUS_FIXME, "unknown 36" }, 653 { do_bad, SIGBUS, BUS_FIXME, "unknown 37" }, 654 { do_bad, SIGBUS, BUS_FIXME, "unknown 38" }, 655 { do_bad, SIGBUS, BUS_FIXME, "unknown 39" }, 656 { do_bad, SIGBUS, BUS_FIXME, "unknown 40" }, 657 { do_bad, SIGBUS, BUS_FIXME, "unknown 41" }, 658 { do_bad, SIGBUS, BUS_FIXME, "unknown 42" }, 659 { do_bad, SIGBUS, BUS_FIXME, "unknown 43" }, 660 { do_bad, SIGBUS, BUS_FIXME, "unknown 44" }, 661 { do_bad, SIGBUS, BUS_FIXME, "unknown 45" }, 662 { do_bad, SIGBUS, BUS_FIXME, "unknown 46" }, 663 { do_bad, SIGBUS, BUS_FIXME, "unknown 47" }, 664 { do_bad, SIGBUS, BUS_FIXME, "TLB conflict abort" }, 665 { do_bad, SIGBUS, BUS_FIXME, "Unsupported atomic hardware update fault" }, 666 { do_bad, SIGBUS, BUS_FIXME, "unknown 50" }, 667 { do_bad, SIGBUS, BUS_FIXME, "unknown 51" }, 668 { do_bad, SIGBUS, BUS_FIXME, "implementation fault (lockdown abort)" }, 669 { do_bad, SIGBUS, BUS_FIXME, "implementation fault (unsupported exclusive)" }, 670 { do_bad, SIGBUS, BUS_FIXME, "unknown 54" }, 671 { do_bad, SIGBUS, BUS_FIXME, "unknown 55" }, 672 { do_bad, SIGBUS, BUS_FIXME, "unknown 56" }, 673 { do_bad, SIGBUS, BUS_FIXME, "unknown 57" }, 674 { do_bad, SIGBUS, BUS_FIXME, "unknown 58" }, 675 { do_bad, SIGBUS, BUS_FIXME, "unknown 59" }, 676 { do_bad, SIGBUS, BUS_FIXME, "unknown 60" }, 677 { do_bad, SIGBUS, BUS_FIXME, "section domain fault" }, 678 { do_bad, SIGBUS, BUS_FIXME, "page domain fault" }, 679 { do_bad, SIGBUS, BUS_FIXME, "unknown 63" }, 680 }; 681 682 int handle_guest_sea(phys_addr_t addr, unsigned int esr) 683 { 684 int ret = -ENOENT; 685 686 if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) 687 ret = ghes_notify_sea(); 688 689 return ret; 690 } 691 692 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, 693 struct pt_regs *regs) 694 { 695 const struct fault_info *inf = esr_to_fault_info(esr); 696 struct siginfo info; 697 698 if (!inf->fn(addr, esr, regs)) 699 return; 700 701 pr_alert("Unhandled fault: %s at 0x%016lx\n", 702 inf->name, addr); 703 704 mem_abort_decode(esr); 705 706 if (!user_mode(regs)) 707 show_pte(addr); 708 709 info.si_signo = inf->sig; 710 info.si_errno = 0; 711 info.si_code = inf->code; 712 info.si_addr = (void __user *)addr; 713 arm64_notify_die("", regs, &info, esr); 714 } 715 716 asmlinkage void __exception do_el0_irq_bp_hardening(void) 717 { 718 /* PC has already been checked in entry.S */ 719 arm64_apply_bp_hardening(); 720 } 721 722 asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, 723 unsigned int esr, 724 struct pt_regs *regs) 725 { 726 /* 727 * We've taken an instruction abort from userspace and not yet 728 * re-enabled IRQs. If the address is a kernel address, apply 729 * BP hardening prior to enabling IRQs and pre-emption. 730 */ 731 if (addr > TASK_SIZE) 732 arm64_apply_bp_hardening(); 733 734 local_irq_enable(); 735 do_mem_abort(addr, esr, regs); 736 } 737 738 739 asmlinkage void __exception do_sp_pc_abort(unsigned long addr, 740 unsigned int esr, 741 struct pt_regs *regs) 742 { 743 struct siginfo info; 744 struct task_struct *tsk = current; 745 746 if (user_mode(regs)) { 747 if (instruction_pointer(regs) > TASK_SIZE) 748 arm64_apply_bp_hardening(); 749 local_irq_enable(); 750 } 751 752 if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) 753 pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", 754 tsk->comm, task_pid_nr(tsk), 755 esr_get_class_string(esr), (void *)regs->pc, 756 (void *)regs->sp); 757 758 info.si_signo = SIGBUS; 759 info.si_errno = 0; 760 info.si_code = BUS_ADRALN; 761 info.si_addr = (void __user *)addr; 762 arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); 763 } 764 765 int __init early_brk64(unsigned long addr, unsigned int esr, 766 struct pt_regs *regs); 767 768 /* 769 * __refdata because early_brk64 is __init, but the reference to it is 770 * clobbered at arch_initcall time. 771 * See traps.c and debug-monitors.c:debug_traps_init(). 772 */ 773 static struct fault_info __refdata debug_fault_info[] = { 774 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, 775 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, 776 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, 777 { do_bad, SIGBUS, BUS_FIXME, "unknown 3" }, 778 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, 779 { do_bad, SIGTRAP, TRAP_FIXME, "aarch32 vector catch" }, 780 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, 781 { do_bad, SIGBUS, BUS_FIXME, "unknown 7" }, 782 }; 783 784 void __init hook_debug_fault_code(int nr, 785 int (*fn)(unsigned long, unsigned int, struct pt_regs *), 786 int sig, int code, const char *name) 787 { 788 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); 789 790 debug_fault_info[nr].fn = fn; 791 debug_fault_info[nr].sig = sig; 792 debug_fault_info[nr].code = code; 793 debug_fault_info[nr].name = name; 794 } 795 796 asmlinkage int __exception do_debug_exception(unsigned long addr, 797 unsigned int esr, 798 struct pt_regs *regs) 799 { 800 const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); 801 struct siginfo info; 802 int rv; 803 804 /* 805 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were 806 * already disabled to preserve the last enabled/disabled addresses. 807 */ 808 if (interrupts_enabled(regs)) 809 trace_hardirqs_off(); 810 811 if (user_mode(regs) && instruction_pointer(regs) > TASK_SIZE) 812 arm64_apply_bp_hardening(); 813 814 if (!inf->fn(addr, esr, regs)) { 815 rv = 1; 816 } else { 817 pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", 818 inf->name, esr, addr); 819 820 info.si_signo = inf->sig; 821 info.si_errno = 0; 822 info.si_code = inf->code; 823 info.si_addr = (void __user *)addr; 824 arm64_notify_die("", regs, &info, 0); 825 rv = 0; 826 } 827 828 if (interrupts_enabled(regs)) 829 trace_hardirqs_on(); 830 831 return rv; 832 } 833 NOKPROBE_SYMBOL(do_debug_exception); 834 835 #ifdef CONFIG_ARM64_PAN 836 int cpu_enable_pan(void *__unused) 837 { 838 /* 839 * We modify PSTATE. This won't work from irq context as the PSTATE 840 * is discarded once we return from the exception. 841 */ 842 WARN_ON_ONCE(in_interrupt()); 843 844 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 845 asm(SET_PSTATE_PAN(1)); 846 return 0; 847 } 848 #endif /* CONFIG_ARM64_PAN */ 849