114cf11afSPaul Mackerras /* 214cf11afSPaul Mackerras * PowerPC version 314cf11afSPaul Mackerras * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 414cf11afSPaul Mackerras * 514cf11afSPaul Mackerras * Derived from "arch/i386/mm/fault.c" 614cf11afSPaul Mackerras * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 714cf11afSPaul Mackerras * 814cf11afSPaul Mackerras * Modified by Cort Dougan and Paul Mackerras. 914cf11afSPaul Mackerras * 1014cf11afSPaul Mackerras * Modified for PPC64 by Dave Engebretsen (engebret@ibm.com) 1114cf11afSPaul Mackerras * 1214cf11afSPaul Mackerras * This program is free software; you can redistribute it and/or 1314cf11afSPaul Mackerras * modify it under the terms of the GNU General Public License 1414cf11afSPaul Mackerras * as published by the Free Software Foundation; either version 1514cf11afSPaul Mackerras * 2 of the License, or (at your option) any later version. 1614cf11afSPaul Mackerras */ 1714cf11afSPaul Mackerras 1814cf11afSPaul Mackerras #include <linux/signal.h> 1914cf11afSPaul Mackerras #include <linux/sched.h> 2068db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 2114cf11afSPaul Mackerras #include <linux/kernel.h> 2214cf11afSPaul Mackerras #include <linux/errno.h> 2314cf11afSPaul Mackerras #include <linux/string.h> 2414cf11afSPaul Mackerras #include <linux/types.h> 250e36b0d1SChristophe Leroy #include <linux/pagemap.h> 2614cf11afSPaul Mackerras #include <linux/ptrace.h> 2714cf11afSPaul Mackerras #include <linux/mman.h> 2814cf11afSPaul Mackerras #include <linux/mm.h> 2914cf11afSPaul Mackerras #include <linux/interrupt.h> 3014cf11afSPaul Mackerras #include <linux/highmem.h> 318a39b05fSPaul Gortmaker #include <linux/extable.h> 3214cf11afSPaul Mackerras #include <linux/kprobes.h> 331eeb66a1SChristoph Hellwig #include <linux/kdebug.h> 34cdd6c482SIngo Molnar #include <linux/perf_event.h> 3576462232SChristian Dietrich #include <linux/ratelimit.h> 36ba12eedeSLi Zhong #include <linux/context_tracking.h> 379d57472fSAnton Blanchard #include <linux/hugetlb.h> 3870ffdb93SDavid Hildenbrand #include <linux/uaccess.h> 3914cf11afSPaul Mackerras 4040900194SBrian King #include <asm/firmware.h> 4114cf11afSPaul Mackerras #include <asm/page.h> 4214cf11afSPaul Mackerras #include <asm/pgtable.h> 4314cf11afSPaul Mackerras #include <asm/mmu.h> 4414cf11afSPaul Mackerras #include <asm/mmu_context.h> 4514cf11afSPaul Mackerras #include <asm/siginfo.h> 46ae3a197eSDavid Howells #include <asm/debug.h> 479f90b997SChristoph Hellwig 48bb4be50eSBenjamin Herrenschmidt static inline bool notify_page_fault(struct pt_regs *regs) 494f9e87c0SAnil S Keshavamurthy { 50bb4be50eSBenjamin Herrenschmidt bool ret = false; 519f90b997SChristoph Hellwig 52bb4be50eSBenjamin Herrenschmidt #ifdef CONFIG_KPROBES 539f90b997SChristoph Hellwig /* kprobe_running() needs smp_processor_id() */ 549f90b997SChristoph Hellwig if (!user_mode(regs)) { 559f90b997SChristoph Hellwig preempt_disable(); 569f90b997SChristoph Hellwig if (kprobe_running() && kprobe_fault_handler(regs, 11)) 57bb4be50eSBenjamin Herrenschmidt ret = true; 589f90b997SChristoph Hellwig preempt_enable(); 594f9e87c0SAnil S Keshavamurthy } 60bb4be50eSBenjamin Herrenschmidt #endif /* CONFIG_KPROBES */ 61bb4be50eSBenjamin Herrenschmidt 62bb4be50eSBenjamin Herrenschmidt if (unlikely(debugger_fault_handler(regs))) 63bb4be50eSBenjamin Herrenschmidt ret = true; 644f9e87c0SAnil S Keshavamurthy 659f90b997SChristoph Hellwig return ret; 664f9e87c0SAnil S Keshavamurthy } 674f9e87c0SAnil S Keshavamurthy 6814cf11afSPaul Mackerras /* 690e36b0d1SChristophe Leroy * Check whether the instruction inst is a store using 7014cf11afSPaul Mackerras * an update addressing form which will update r1. 7114cf11afSPaul Mackerras */ 720e36b0d1SChristophe Leroy static bool store_updates_sp(unsigned int inst) 7314cf11afSPaul Mackerras { 7414cf11afSPaul Mackerras /* check for 1 in the rA field */ 7514cf11afSPaul Mackerras if (((inst >> 16) & 0x1f) != 1) 768f5ca0b3SBenjamin Herrenschmidt return false; 7714cf11afSPaul Mackerras /* check major opcode */ 7814cf11afSPaul Mackerras switch (inst >> 26) { 798a0b1120SChristophe Leroy case OP_STWU: 808a0b1120SChristophe Leroy case OP_STBU: 818a0b1120SChristophe Leroy case OP_STHU: 828a0b1120SChristophe Leroy case OP_STFSU: 838a0b1120SChristophe Leroy case OP_STFDU: 848f5ca0b3SBenjamin Herrenschmidt return true; 858a0b1120SChristophe Leroy case OP_STD: /* std or stdu */ 8614cf11afSPaul Mackerras return (inst & 3) == 1; 878a0b1120SChristophe Leroy case OP_31: 8814cf11afSPaul Mackerras /* check minor opcode */ 8914cf11afSPaul Mackerras switch ((inst >> 1) & 0x3ff) { 908a0b1120SChristophe Leroy case OP_31_XOP_STDUX: 918a0b1120SChristophe Leroy case OP_31_XOP_STWUX: 928a0b1120SChristophe Leroy case OP_31_XOP_STBUX: 938a0b1120SChristophe Leroy case OP_31_XOP_STHUX: 948a0b1120SChristophe Leroy case OP_31_XOP_STFSUX: 958a0b1120SChristophe Leroy case OP_31_XOP_STFDUX: 968f5ca0b3SBenjamin Herrenschmidt return true; 9714cf11afSPaul Mackerras } 9814cf11afSPaul Mackerras } 998f5ca0b3SBenjamin Herrenschmidt return false; 10014cf11afSPaul Mackerras } 1019be72573SBenjamin Herrenschmidt /* 1029be72573SBenjamin Herrenschmidt * do_page_fault error handling helpers 1039be72573SBenjamin Herrenschmidt */ 1049be72573SBenjamin Herrenschmidt 105c3350602SBenjamin Herrenschmidt static int 10699cd1302SRam Pai __bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code, 10799cd1302SRam Pai int pkey) 108c3350602SBenjamin Herrenschmidt { 109c3350602SBenjamin Herrenschmidt /* 110c3350602SBenjamin Herrenschmidt * If we are in kernel mode, bail out with a SEGV, this will 111c3350602SBenjamin Herrenschmidt * be caught by the assembly which will restore the non-volatile 112c3350602SBenjamin Herrenschmidt * registers before calling bad_page_fault() 113c3350602SBenjamin Herrenschmidt */ 114c3350602SBenjamin Herrenschmidt if (!user_mode(regs)) 115c3350602SBenjamin Herrenschmidt return SIGSEGV; 116c3350602SBenjamin Herrenschmidt 11799cd1302SRam Pai _exception_pkey(SIGSEGV, regs, si_code, address, pkey); 118c3350602SBenjamin Herrenschmidt 119c3350602SBenjamin Herrenschmidt return 0; 120c3350602SBenjamin Herrenschmidt } 121c3350602SBenjamin Herrenschmidt 122c3350602SBenjamin Herrenschmidt static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long address) 123c3350602SBenjamin Herrenschmidt { 12499cd1302SRam Pai return __bad_area_nosemaphore(regs, address, SEGV_MAPERR, 0); 125c3350602SBenjamin Herrenschmidt } 126c3350602SBenjamin Herrenschmidt 127*9f2ee693SEric W. Biederman static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code) 128c3350602SBenjamin Herrenschmidt { 129c3350602SBenjamin Herrenschmidt struct mm_struct *mm = current->mm; 130c3350602SBenjamin Herrenschmidt 131c3350602SBenjamin Herrenschmidt /* 132c3350602SBenjamin Herrenschmidt * Something tried to access memory that isn't in our memory map.. 133c3350602SBenjamin Herrenschmidt * Fix it, but check if it's kernel or user first.. 134c3350602SBenjamin Herrenschmidt */ 135c3350602SBenjamin Herrenschmidt up_read(&mm->mmap_sem); 136c3350602SBenjamin Herrenschmidt 137*9f2ee693SEric W. Biederman return __bad_area_nosemaphore(regs, address, si_code, 0); 138c3350602SBenjamin Herrenschmidt } 139c3350602SBenjamin Herrenschmidt 140c3350602SBenjamin Herrenschmidt static noinline int bad_area(struct pt_regs *regs, unsigned long address) 141c3350602SBenjamin Herrenschmidt { 142*9f2ee693SEric W. Biederman return __bad_area(regs, address, SEGV_MAPERR); 14399cd1302SRam Pai } 14499cd1302SRam Pai 14599cd1302SRam Pai static int bad_key_fault_exception(struct pt_regs *regs, unsigned long address, 14699cd1302SRam Pai int pkey) 14799cd1302SRam Pai { 14899cd1302SRam Pai return __bad_area_nosemaphore(regs, address, SEGV_PKUERR, pkey); 149c3350602SBenjamin Herrenschmidt } 150c3350602SBenjamin Herrenschmidt 151ecb101aeSJohn Sperbeck static noinline int bad_access(struct pt_regs *regs, unsigned long address) 152ecb101aeSJohn Sperbeck { 153*9f2ee693SEric W. Biederman return __bad_area(regs, address, SEGV_ACCERR); 154ecb101aeSJohn Sperbeck } 155ecb101aeSJohn Sperbeck 1563913fdd7SAnton Blanchard static int do_sigbus(struct pt_regs *regs, unsigned long address, 15750a7ca3cSSouptick Joarder vm_fault_t fault) 1589be72573SBenjamin Herrenschmidt { 1599be72573SBenjamin Herrenschmidt siginfo_t info; 1609be72573SBenjamin Herrenschmidt 16163af5262SAnton Blanchard if (!user_mode(regs)) 162b5c8f0fdSBenjamin Herrenschmidt return SIGBUS; 16363af5262SAnton Blanchard 16441ab5266SAnanth N Mavinakayanahalli current->thread.trap_nr = BUS_ADRERR; 1653eb0f519SEric W. Biederman clear_siginfo(&info); 1669be72573SBenjamin Herrenschmidt info.si_signo = SIGBUS; 1679be72573SBenjamin Herrenschmidt info.si_errno = 0; 1689be72573SBenjamin Herrenschmidt info.si_code = BUS_ADRERR; 1699be72573SBenjamin Herrenschmidt info.si_addr = (void __user *)address; 1703913fdd7SAnton Blanchard #ifdef CONFIG_MEMORY_FAILURE 1713913fdd7SAnton Blanchard if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { 172f654fc07SEric W. Biederman unsigned int lsb = 0; /* shutup gcc */ 173f654fc07SEric W. Biederman 1743913fdd7SAnton Blanchard pr_err("MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n", 1753913fdd7SAnton Blanchard current->comm, current->pid, address); 1769d57472fSAnton Blanchard 1779d57472fSAnton Blanchard if (fault & VM_FAULT_HWPOISON_LARGE) 1789d57472fSAnton Blanchard lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); 1799d57472fSAnton Blanchard if (fault & VM_FAULT_HWPOISON) 1809d57472fSAnton Blanchard lsb = PAGE_SHIFT; 181f654fc07SEric W. Biederman 182f654fc07SEric W. Biederman force_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, 183f654fc07SEric W. Biederman current); 184f654fc07SEric W. Biederman return 0; 185f654fc07SEric W. Biederman } 186f654fc07SEric W. Biederman 1873913fdd7SAnton Blanchard #endif 1889be72573SBenjamin Herrenschmidt force_sig_info(SIGBUS, &info, current); 189b5c8f0fdSBenjamin Herrenschmidt return 0; 1909be72573SBenjamin Herrenschmidt } 1919be72573SBenjamin Herrenschmidt 19250a7ca3cSSouptick Joarder static int mm_fault_error(struct pt_regs *regs, unsigned long addr, 19350a7ca3cSSouptick Joarder vm_fault_t fault) 1949be72573SBenjamin Herrenschmidt { 1959be72573SBenjamin Herrenschmidt /* 196b5c8f0fdSBenjamin Herrenschmidt * Kernel page fault interrupted by SIGKILL. We have no reason to 197b5c8f0fdSBenjamin Herrenschmidt * continue processing. 1989be72573SBenjamin Herrenschmidt */ 199b5c8f0fdSBenjamin Herrenschmidt if (fatal_signal_pending(current) && !user_mode(regs)) 200b5c8f0fdSBenjamin Herrenschmidt return SIGKILL; 2019be72573SBenjamin Herrenschmidt 2029be72573SBenjamin Herrenschmidt /* Out of memory */ 203c2d23f91SDavid Rientjes if (fault & VM_FAULT_OOM) { 204c2d23f91SDavid Rientjes /* 205c2d23f91SDavid Rientjes * We ran out of memory, or some other thing happened to us that 206c2d23f91SDavid Rientjes * made us unable to handle the page fault gracefully. 207c2d23f91SDavid Rientjes */ 208c2d23f91SDavid Rientjes if (!user_mode(regs)) 209b5c8f0fdSBenjamin Herrenschmidt return SIGSEGV; 210c2d23f91SDavid Rientjes pagefault_out_of_memory(); 211b5c8f0fdSBenjamin Herrenschmidt } else { 212b5c8f0fdSBenjamin Herrenschmidt if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON| 213b5c8f0fdSBenjamin Herrenschmidt VM_FAULT_HWPOISON_LARGE)) 2143913fdd7SAnton Blanchard return do_sigbus(regs, addr, fault); 215b5c8f0fdSBenjamin Herrenschmidt else if (fault & VM_FAULT_SIGSEGV) 216b5c8f0fdSBenjamin Herrenschmidt return bad_area_nosemaphore(regs, addr); 217b5c8f0fdSBenjamin Herrenschmidt else 2189be72573SBenjamin Herrenschmidt BUG(); 219b5c8f0fdSBenjamin Herrenschmidt } 220b5c8f0fdSBenjamin Herrenschmidt return 0; 2219be72573SBenjamin Herrenschmidt } 22214cf11afSPaul Mackerras 223d3ca5874SBenjamin Herrenschmidt /* Is this a bad kernel fault ? */ 224d3ca5874SBenjamin Herrenschmidt static bool bad_kernel_fault(bool is_exec, unsigned long error_code, 225d3ca5874SBenjamin Herrenschmidt unsigned long address) 226d3ca5874SBenjamin Herrenschmidt { 227d3ca5874SBenjamin Herrenschmidt if (is_exec && (error_code & (DSISR_NOEXEC_OR_G | DSISR_KEYFAULT))) { 228d3ca5874SBenjamin Herrenschmidt printk_ratelimited(KERN_CRIT "kernel tried to execute" 229d3ca5874SBenjamin Herrenschmidt " exec-protected page (%lx) -" 230d3ca5874SBenjamin Herrenschmidt "exploit attempt? (uid: %d)\n", 231d3ca5874SBenjamin Herrenschmidt address, from_kuid(&init_user_ns, 232d3ca5874SBenjamin Herrenschmidt current_uid())); 233d3ca5874SBenjamin Herrenschmidt } 234d3ca5874SBenjamin Herrenschmidt return is_exec || (address >= TASK_SIZE); 235d3ca5874SBenjamin Herrenschmidt } 236d3ca5874SBenjamin Herrenschmidt 2378f5ca0b3SBenjamin Herrenschmidt static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address, 2380e36b0d1SChristophe Leroy struct vm_area_struct *vma, unsigned int flags, 2390e36b0d1SChristophe Leroy bool *must_retry) 2408f5ca0b3SBenjamin Herrenschmidt { 2418f5ca0b3SBenjamin Herrenschmidt /* 2428f5ca0b3SBenjamin Herrenschmidt * N.B. The POWER/Open ABI allows programs to access up to 2438f5ca0b3SBenjamin Herrenschmidt * 288 bytes below the stack pointer. 2448f5ca0b3SBenjamin Herrenschmidt * The kernel signal delivery code writes up to about 1.5kB 2458f5ca0b3SBenjamin Herrenschmidt * below the stack pointer (r1) before decrementing it. 2468f5ca0b3SBenjamin Herrenschmidt * The exec code can write slightly over 640kB to the stack 2478f5ca0b3SBenjamin Herrenschmidt * before setting the user r1. Thus we allow the stack to 2488f5ca0b3SBenjamin Herrenschmidt * expand to 1MB without further checks. 2498f5ca0b3SBenjamin Herrenschmidt */ 2508f5ca0b3SBenjamin Herrenschmidt if (address + 0x100000 < vma->vm_end) { 2510e36b0d1SChristophe Leroy unsigned int __user *nip = (unsigned int __user *)regs->nip; 2528f5ca0b3SBenjamin Herrenschmidt /* get user regs even if this fault is in kernel mode */ 2538f5ca0b3SBenjamin Herrenschmidt struct pt_regs *uregs = current->thread.regs; 2548f5ca0b3SBenjamin Herrenschmidt if (uregs == NULL) 2558f5ca0b3SBenjamin Herrenschmidt return true; 2568f5ca0b3SBenjamin Herrenschmidt 2578f5ca0b3SBenjamin Herrenschmidt /* 2588f5ca0b3SBenjamin Herrenschmidt * A user-mode access to an address a long way below 2598f5ca0b3SBenjamin Herrenschmidt * the stack pointer is only valid if the instruction 2608f5ca0b3SBenjamin Herrenschmidt * is one which would update the stack pointer to the 2618f5ca0b3SBenjamin Herrenschmidt * address accessed if the instruction completed, 2628f5ca0b3SBenjamin Herrenschmidt * i.e. either stwu rs,n(r1) or stwux rs,r1,rb 2638f5ca0b3SBenjamin Herrenschmidt * (or the byte, halfword, float or double forms). 2648f5ca0b3SBenjamin Herrenschmidt * 2658f5ca0b3SBenjamin Herrenschmidt * If we don't check this then any write to the area 2668f5ca0b3SBenjamin Herrenschmidt * between the last mapped region and the stack will 2678f5ca0b3SBenjamin Herrenschmidt * expand the stack rather than segfaulting. 2688f5ca0b3SBenjamin Herrenschmidt */ 2690e36b0d1SChristophe Leroy if (address + 2048 >= uregs->gpr[1]) 2700e36b0d1SChristophe Leroy return false; 2710e36b0d1SChristophe Leroy 2720e36b0d1SChristophe Leroy if ((flags & FAULT_FLAG_WRITE) && (flags & FAULT_FLAG_USER) && 2730e36b0d1SChristophe Leroy access_ok(VERIFY_READ, nip, sizeof(*nip))) { 2740e36b0d1SChristophe Leroy unsigned int inst; 2750e36b0d1SChristophe Leroy int res; 2760e36b0d1SChristophe Leroy 2770e36b0d1SChristophe Leroy pagefault_disable(); 2780e36b0d1SChristophe Leroy res = __get_user_inatomic(inst, nip); 2790e36b0d1SChristophe Leroy pagefault_enable(); 2800e36b0d1SChristophe Leroy if (!res) 2810e36b0d1SChristophe Leroy return !store_updates_sp(inst); 2820e36b0d1SChristophe Leroy *must_retry = true; 2830e36b0d1SChristophe Leroy } 2848f5ca0b3SBenjamin Herrenschmidt return true; 2858f5ca0b3SBenjamin Herrenschmidt } 2868f5ca0b3SBenjamin Herrenschmidt return false; 2878f5ca0b3SBenjamin Herrenschmidt } 2888f5ca0b3SBenjamin Herrenschmidt 289bd0d63f8SBenjamin Herrenschmidt static bool access_error(bool is_write, bool is_exec, 290bd0d63f8SBenjamin Herrenschmidt struct vm_area_struct *vma) 291bd0d63f8SBenjamin Herrenschmidt { 292bd0d63f8SBenjamin Herrenschmidt /* 293bd0d63f8SBenjamin Herrenschmidt * Allow execution from readable areas if the MMU does not 294bd0d63f8SBenjamin Herrenschmidt * provide separate controls over reading and executing. 295bd0d63f8SBenjamin Herrenschmidt * 296bd0d63f8SBenjamin Herrenschmidt * Note: That code used to not be enabled for 4xx/BookE. 297bd0d63f8SBenjamin Herrenschmidt * It is now as I/D cache coherency for these is done at 298bd0d63f8SBenjamin Herrenschmidt * set_pte_at() time and I see no reason why the test 299bd0d63f8SBenjamin Herrenschmidt * below wouldn't be valid on those processors. This -may- 300bd0d63f8SBenjamin Herrenschmidt * break programs compiled with a really old ABI though. 301bd0d63f8SBenjamin Herrenschmidt */ 302bd0d63f8SBenjamin Herrenschmidt if (is_exec) { 303bd0d63f8SBenjamin Herrenschmidt return !(vma->vm_flags & VM_EXEC) && 304bd0d63f8SBenjamin Herrenschmidt (cpu_has_feature(CPU_FTR_NOEXECUTE) || 305bd0d63f8SBenjamin Herrenschmidt !(vma->vm_flags & (VM_READ | VM_WRITE))); 306bd0d63f8SBenjamin Herrenschmidt } 307bd0d63f8SBenjamin Herrenschmidt 308bd0d63f8SBenjamin Herrenschmidt if (is_write) { 309bd0d63f8SBenjamin Herrenschmidt if (unlikely(!(vma->vm_flags & VM_WRITE))) 310bd0d63f8SBenjamin Herrenschmidt return true; 311bd0d63f8SBenjamin Herrenschmidt return false; 312bd0d63f8SBenjamin Herrenschmidt } 313bd0d63f8SBenjamin Herrenschmidt 314bd0d63f8SBenjamin Herrenschmidt if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) 315bd0d63f8SBenjamin Herrenschmidt return true; 316f2ed480fSAneesh Kumar K.V /* 317f2ed480fSAneesh Kumar K.V * We should ideally do the vma pkey access check here. But in the 318f2ed480fSAneesh Kumar K.V * fault path, handle_mm_fault() also does the same check. To avoid 319f2ed480fSAneesh Kumar K.V * these multiple checks, we skip it here and handle access error due 320f2ed480fSAneesh Kumar K.V * to pkeys later. 321f2ed480fSAneesh Kumar K.V */ 322bd0d63f8SBenjamin Herrenschmidt return false; 323bd0d63f8SBenjamin Herrenschmidt } 324bd0d63f8SBenjamin Herrenschmidt 3253da02648SBenjamin Herrenschmidt #ifdef CONFIG_PPC_SMLPAR 3263da02648SBenjamin Herrenschmidt static inline void cmo_account_page_fault(void) 3273da02648SBenjamin Herrenschmidt { 3283da02648SBenjamin Herrenschmidt if (firmware_has_feature(FW_FEATURE_CMO)) { 3293da02648SBenjamin Herrenschmidt u32 page_ins; 3303da02648SBenjamin Herrenschmidt 3313da02648SBenjamin Herrenschmidt preempt_disable(); 3323da02648SBenjamin Herrenschmidt page_ins = be32_to_cpu(get_lppaca()->page_ins); 3333da02648SBenjamin Herrenschmidt page_ins += 1 << PAGE_FACTOR; 3343da02648SBenjamin Herrenschmidt get_lppaca()->page_ins = cpu_to_be32(page_ins); 3353da02648SBenjamin Herrenschmidt preempt_enable(); 3363da02648SBenjamin Herrenschmidt } 3373da02648SBenjamin Herrenschmidt } 3383da02648SBenjamin Herrenschmidt #else 3393da02648SBenjamin Herrenschmidt static inline void cmo_account_page_fault(void) { } 3403da02648SBenjamin Herrenschmidt #endif /* CONFIG_PPC_SMLPAR */ 3413da02648SBenjamin Herrenschmidt 3422865d08dSBenjamin Herrenschmidt #ifdef CONFIG_PPC_STD_MMU 3432865d08dSBenjamin Herrenschmidt static void sanity_check_fault(bool is_write, unsigned long error_code) 3442865d08dSBenjamin Herrenschmidt { 3452865d08dSBenjamin Herrenschmidt /* 3462865d08dSBenjamin Herrenschmidt * For hash translation mode, we should never get a 3472865d08dSBenjamin Herrenschmidt * PROTFAULT. Any update to pte to reduce access will result in us 3482865d08dSBenjamin Herrenschmidt * removing the hash page table entry, thus resulting in a DSISR_NOHPTE 3492865d08dSBenjamin Herrenschmidt * fault instead of DSISR_PROTFAULT. 3502865d08dSBenjamin Herrenschmidt * 3512865d08dSBenjamin Herrenschmidt * A pte update to relax the access will not result in a hash page table 3522865d08dSBenjamin Herrenschmidt * entry invalidate and hence can result in DSISR_PROTFAULT. 3532865d08dSBenjamin Herrenschmidt * ptep_set_access_flags() doesn't do a hpte flush. This is why we have 3542865d08dSBenjamin Herrenschmidt * the special !is_write in the below conditional. 3552865d08dSBenjamin Herrenschmidt * 3562865d08dSBenjamin Herrenschmidt * For platforms that doesn't supports coherent icache and do support 3572865d08dSBenjamin Herrenschmidt * per page noexec bit, we do setup things such that we do the 3582865d08dSBenjamin Herrenschmidt * sync between D/I cache via fault. But that is handled via low level 3592865d08dSBenjamin Herrenschmidt * hash fault code (hash_page_do_lazy_icache()) and we should not reach 3602865d08dSBenjamin Herrenschmidt * here in such case. 3612865d08dSBenjamin Herrenschmidt * 3622865d08dSBenjamin Herrenschmidt * For wrong access that can result in PROTFAULT, the above vma->vm_flags 3632865d08dSBenjamin Herrenschmidt * check should handle those and hence we should fall to the bad_area 3642865d08dSBenjamin Herrenschmidt * handling correctly. 3652865d08dSBenjamin Herrenschmidt * 3662865d08dSBenjamin Herrenschmidt * For embedded with per page exec support that doesn't support coherent 3672865d08dSBenjamin Herrenschmidt * icache we do get PROTFAULT and we handle that D/I cache sync in 3682865d08dSBenjamin Herrenschmidt * set_pte_at while taking the noexec/prot fault. Hence this is WARN_ON 3692865d08dSBenjamin Herrenschmidt * is conditional for server MMU. 3702865d08dSBenjamin Herrenschmidt * 3712865d08dSBenjamin Herrenschmidt * For radix, we can get prot fault for autonuma case, because radix 3722865d08dSBenjamin Herrenschmidt * page table will have them marked noaccess for user. 3732865d08dSBenjamin Herrenschmidt */ 3742865d08dSBenjamin Herrenschmidt if (!radix_enabled() && !is_write) 3752865d08dSBenjamin Herrenschmidt WARN_ON_ONCE(error_code & DSISR_PROTFAULT); 3762865d08dSBenjamin Herrenschmidt } 3772865d08dSBenjamin Herrenschmidt #else 3782865d08dSBenjamin Herrenschmidt static void sanity_check_fault(bool is_write, unsigned long error_code) { } 3792865d08dSBenjamin Herrenschmidt #endif /* CONFIG_PPC_STD_MMU */ 3802865d08dSBenjamin Herrenschmidt 38114cf11afSPaul Mackerras /* 38241b464e5SBenjamin Herrenschmidt * Define the correct "is_write" bit in error_code based 38341b464e5SBenjamin Herrenschmidt * on the processor family 38441b464e5SBenjamin Herrenschmidt */ 38541b464e5SBenjamin Herrenschmidt #if (defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 38641b464e5SBenjamin Herrenschmidt #define page_fault_is_write(__err) ((__err) & ESR_DST) 387f3d96e69SBenjamin Herrenschmidt #define page_fault_is_bad(__err) (0) 38841b464e5SBenjamin Herrenschmidt #else 38941b464e5SBenjamin Herrenschmidt #define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE) 390968159c0SChristophe Leroy #if defined(CONFIG_PPC_8xx) 3914915349bSChristophe Leroy #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G) 392f3d96e69SBenjamin Herrenschmidt #elif defined(CONFIG_PPC64) 393f3d96e69SBenjamin Herrenschmidt #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S) 394f3d96e69SBenjamin Herrenschmidt #else 395f3d96e69SBenjamin Herrenschmidt #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S) 396f3d96e69SBenjamin Herrenschmidt #endif 39741b464e5SBenjamin Herrenschmidt #endif 39841b464e5SBenjamin Herrenschmidt 39941b464e5SBenjamin Herrenschmidt /* 40014cf11afSPaul Mackerras * For 600- and 800-family processors, the error_code parameter is DSISR 40114cf11afSPaul Mackerras * for a data fault, SRR1 for an instruction fault. For 400-family processors 40214cf11afSPaul Mackerras * the error_code parameter is ESR for a data fault, 0 for an instruction 40314cf11afSPaul Mackerras * fault. 40414cf11afSPaul Mackerras * For 64-bit processors, the error_code parameter is 40514cf11afSPaul Mackerras * - DSISR for a non-SLB data access fault, 40614cf11afSPaul Mackerras * - SRR1 & 0x08000000 for a non-SLB instruction access fault 40714cf11afSPaul Mackerras * - 0 any SLB fault. 40814cf11afSPaul Mackerras * 40914cf11afSPaul Mackerras * The return value is 0 if the fault was handled, or the signal 41014cf11afSPaul Mackerras * number if this is a kernel fault that can't be handled here. 41114cf11afSPaul Mackerras */ 4127afad422SBenjamin Herrenschmidt static int __do_page_fault(struct pt_regs *regs, unsigned long address, 41314cf11afSPaul Mackerras unsigned long error_code) 41414cf11afSPaul Mackerras { 41514cf11afSPaul Mackerras struct vm_area_struct * vma; 41614cf11afSPaul Mackerras struct mm_struct *mm = current->mm; 4179be72573SBenjamin Herrenschmidt unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 418c433ec04SBenjamin Herrenschmidt int is_exec = TRAP(regs) == 0x400; 419da929f6aSChristophe Leroy int is_user = user_mode(regs); 42041b464e5SBenjamin Herrenschmidt int is_write = page_fault_is_write(error_code); 42150a7ca3cSSouptick Joarder vm_fault_t fault, major = 0; 4220e36b0d1SChristophe Leroy bool must_retry = false; 42314cf11afSPaul Mackerras 4249f90b997SChristoph Hellwig if (notify_page_fault(regs)) 42565d47fd4SBenjamin Herrenschmidt return 0; 42614cf11afSPaul Mackerras 427f3d96e69SBenjamin Herrenschmidt if (unlikely(page_fault_is_bad(error_code))) { 42865d47fd4SBenjamin Herrenschmidt if (is_user) { 429f3d96e69SBenjamin Herrenschmidt _exception(SIGBUS, regs, BUS_OBJERR, address); 43065d47fd4SBenjamin Herrenschmidt return 0; 43165d47fd4SBenjamin Herrenschmidt } 43265d47fd4SBenjamin Herrenschmidt return SIGBUS; 433e6c8290aSBenjamin Herrenschmidt } 434e6c8290aSBenjamin Herrenschmidt 4352865d08dSBenjamin Herrenschmidt /* Additional sanity check(s) */ 4362865d08dSBenjamin Herrenschmidt sanity_check_fault(is_write, error_code); 4372865d08dSBenjamin Herrenschmidt 438d7df2443SBenjamin Herrenschmidt /* 439d7df2443SBenjamin Herrenschmidt * The kernel should never take an execute fault nor should it 440d7df2443SBenjamin Herrenschmidt * take a page fault to a kernel address. 441d7df2443SBenjamin Herrenschmidt */ 442d3ca5874SBenjamin Herrenschmidt if (unlikely(!is_user && bad_kernel_fault(is_exec, error_code, address))) 44365d47fd4SBenjamin Herrenschmidt return SIGSEGV; 44414cf11afSPaul Mackerras 44511ccdd33SBenjamin Herrenschmidt /* 44611ccdd33SBenjamin Herrenschmidt * If we're in an interrupt, have no user context or are running 44711ccdd33SBenjamin Herrenschmidt * in a region with pagefaults disabled then we must not take the fault 44811ccdd33SBenjamin Herrenschmidt */ 44911ccdd33SBenjamin Herrenschmidt if (unlikely(faulthandler_disabled() || !mm)) { 45011ccdd33SBenjamin Herrenschmidt if (is_user) 45111ccdd33SBenjamin Herrenschmidt printk_ratelimited(KERN_ERR "Page fault in user mode" 45211ccdd33SBenjamin Herrenschmidt " with faulthandler_disabled()=%d" 45311ccdd33SBenjamin Herrenschmidt " mm=%p\n", 45411ccdd33SBenjamin Herrenschmidt faulthandler_disabled(), mm); 45511ccdd33SBenjamin Herrenschmidt return bad_area_nosemaphore(regs, address); 45611ccdd33SBenjamin Herrenschmidt } 45711ccdd33SBenjamin Herrenschmidt 458a546498fSBenjamin Herrenschmidt /* We restore the interrupt state now */ 459a546498fSBenjamin Herrenschmidt if (!arch_irq_disabled_regs(regs)) 460a546498fSBenjamin Herrenschmidt local_irq_enable(); 461a546498fSBenjamin Herrenschmidt 462a8b0ca17SPeter Zijlstra perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 4637dd1fcc2SPeter Zijlstra 46499cd1302SRam Pai if (error_code & DSISR_KEYFAULT) 46599cd1302SRam Pai return bad_key_fault_exception(regs, address, 46699cd1302SRam Pai get_mm_addr_key(mm, address)); 467e6c2a479SRam Pai 46869e044ddSAneesh Kumar K.V /* 46969e044ddSAneesh Kumar K.V * We want to do this outside mmap_sem, because reading code around nip 47069e044ddSAneesh Kumar K.V * can result in fault, which will cause a deadlock when called with 47169e044ddSAneesh Kumar K.V * mmap_sem held 47269e044ddSAneesh Kumar K.V */ 473da929f6aSChristophe Leroy if (is_user) 474759496baSJohannes Weiner flags |= FAULT_FLAG_USER; 475d2e0d2c5SBenjamin Herrenschmidt if (is_write) 476d2e0d2c5SBenjamin Herrenschmidt flags |= FAULT_FLAG_WRITE; 477d2e0d2c5SBenjamin Herrenschmidt if (is_exec) 478d2e0d2c5SBenjamin Herrenschmidt flags |= FAULT_FLAG_INSTRUCTION; 479759496baSJohannes Weiner 48014cf11afSPaul Mackerras /* When running in the kernel we expect faults to occur only to 48114cf11afSPaul Mackerras * addresses in user space. All other faults represent errors in the 482fc5266eaSAnton Blanchard * kernel and should generate an OOPS. Unfortunately, in the case of an 483fc5266eaSAnton Blanchard * erroneous fault occurring in a code path which already holds mmap_sem 48414cf11afSPaul Mackerras * we will deadlock attempting to validate the fault against the 48514cf11afSPaul Mackerras * address space. Luckily the kernel only validly references user 48614cf11afSPaul Mackerras * space from well defined areas of code, which are listed in the 48714cf11afSPaul Mackerras * exceptions table. 48814cf11afSPaul Mackerras * 48914cf11afSPaul Mackerras * As the vast majority of faults will be valid we will only perform 490fc5266eaSAnton Blanchard * the source reference check when there is a possibility of a deadlock. 49114cf11afSPaul Mackerras * Attempt to lock the address space, if we cannot we then validate the 49214cf11afSPaul Mackerras * source. If this is invalid we can skip the address space check, 49314cf11afSPaul Mackerras * thus avoiding the deadlock. 49414cf11afSPaul Mackerras */ 495b15021d9SBenjamin Herrenschmidt if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 496da929f6aSChristophe Leroy if (!is_user && !search_exception_tables(regs->nip)) 497c3350602SBenjamin Herrenschmidt return bad_area_nosemaphore(regs, address); 49814cf11afSPaul Mackerras 4999be72573SBenjamin Herrenschmidt retry: 50014cf11afSPaul Mackerras down_read(&mm->mmap_sem); 501a546498fSBenjamin Herrenschmidt } else { 502a546498fSBenjamin Herrenschmidt /* 503a546498fSBenjamin Herrenschmidt * The above down_read_trylock() might have succeeded in 504a546498fSBenjamin Herrenschmidt * which case we'll have missed the might_sleep() from 505a546498fSBenjamin Herrenschmidt * down_read(): 506a546498fSBenjamin Herrenschmidt */ 507a546498fSBenjamin Herrenschmidt might_sleep(); 50814cf11afSPaul Mackerras } 50914cf11afSPaul Mackerras 51014cf11afSPaul Mackerras vma = find_vma(mm, address); 511b15021d9SBenjamin Herrenschmidt if (unlikely(!vma)) 512c3350602SBenjamin Herrenschmidt return bad_area(regs, address); 513b15021d9SBenjamin Herrenschmidt if (likely(vma->vm_start <= address)) 51414cf11afSPaul Mackerras goto good_area; 515b15021d9SBenjamin Herrenschmidt if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) 516c3350602SBenjamin Herrenschmidt return bad_area(regs, address); 51714cf11afSPaul Mackerras 5188f5ca0b3SBenjamin Herrenschmidt /* The stack is being expanded, check if it's valid */ 5190e36b0d1SChristophe Leroy if (unlikely(bad_stack_expansion(regs, address, vma, flags, 5200e36b0d1SChristophe Leroy &must_retry))) { 5210e36b0d1SChristophe Leroy if (!must_retry) 522c3350602SBenjamin Herrenschmidt return bad_area(regs, address); 52314cf11afSPaul Mackerras 5240e36b0d1SChristophe Leroy up_read(&mm->mmap_sem); 5250e36b0d1SChristophe Leroy if (fault_in_pages_readable((const char __user *)regs->nip, 5260e36b0d1SChristophe Leroy sizeof(unsigned int))) 5270e36b0d1SChristophe Leroy return bad_area_nosemaphore(regs, address); 5280e36b0d1SChristophe Leroy goto retry; 5290e36b0d1SChristophe Leroy } 5300e36b0d1SChristophe Leroy 5318f5ca0b3SBenjamin Herrenschmidt /* Try to expand it */ 532b15021d9SBenjamin Herrenschmidt if (unlikely(expand_stack(vma, address))) 533c3350602SBenjamin Herrenschmidt return bad_area(regs, address); 53414cf11afSPaul Mackerras 53514cf11afSPaul Mackerras good_area: 536bd0d63f8SBenjamin Herrenschmidt if (unlikely(access_error(is_write, is_exec, vma))) 537ecb101aeSJohn Sperbeck return bad_access(regs, address); 53814cf11afSPaul Mackerras 53914cf11afSPaul Mackerras /* 54014cf11afSPaul Mackerras * If for any reason at all we couldn't handle the fault, 54114cf11afSPaul Mackerras * make sure we exit gracefully rather than endlessly redo 54214cf11afSPaul Mackerras * the fault. 54314cf11afSPaul Mackerras */ 544dcddffd4SKirill A. Shutemov fault = handle_mm_fault(vma, address, flags); 545e6c2a479SRam Pai 546e6c2a479SRam Pai #ifdef CONFIG_PPC_MEM_KEYS 547e6c2a479SRam Pai /* 548f2ed480fSAneesh Kumar K.V * we skipped checking for access error due to key earlier. 549f2ed480fSAneesh Kumar K.V * Check that using handle_mm_fault error return. 550e6c2a479SRam Pai */ 551e6c2a479SRam Pai if (unlikely(fault & VM_FAULT_SIGSEGV) && 552f2ed480fSAneesh Kumar K.V !arch_vma_access_permitted(vma, is_write, is_exec, 0)) { 553f2ed480fSAneesh Kumar K.V 554e6c2a479SRam Pai int pkey = vma_pkey(vma); 555e6c2a479SRam Pai 55699cd1302SRam Pai up_read(&mm->mmap_sem); 55799cd1302SRam Pai return bad_key_fault_exception(regs, address, pkey); 55899cd1302SRam Pai } 559e6c2a479SRam Pai #endif /* CONFIG_PPC_MEM_KEYS */ 560e6c2a479SRam Pai 561f43bb27eSBenjamin Herrenschmidt major |= fault & VM_FAULT_MAJOR; 56214c02e41SLaurent Dufour 56314c02e41SLaurent Dufour /* 56414c02e41SLaurent Dufour * Handle the retry right now, the mmap_sem has been released in that 56514c02e41SLaurent Dufour * case. 56614c02e41SLaurent Dufour */ 56714c02e41SLaurent Dufour if (unlikely(fault & VM_FAULT_RETRY)) { 56814c02e41SLaurent Dufour /* We retry only once */ 56914c02e41SLaurent Dufour if (flags & FAULT_FLAG_ALLOW_RETRY) { 57014c02e41SLaurent Dufour /* 57114c02e41SLaurent Dufour * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 57214c02e41SLaurent Dufour * of starvation. 57314c02e41SLaurent Dufour */ 57414c02e41SLaurent Dufour flags &= ~FAULT_FLAG_ALLOW_RETRY; 57514c02e41SLaurent Dufour flags |= FAULT_FLAG_TRIED; 57614c02e41SLaurent Dufour if (!fatal_signal_pending(current)) 57714c02e41SLaurent Dufour goto retry; 57814c02e41SLaurent Dufour } 579b5c8f0fdSBenjamin Herrenschmidt 580b5c8f0fdSBenjamin Herrenschmidt /* 581b5c8f0fdSBenjamin Herrenschmidt * User mode? Just return to handle the fatal exception otherwise 582b5c8f0fdSBenjamin Herrenschmidt * return to bad_page_fault 583b5c8f0fdSBenjamin Herrenschmidt */ 584b5c8f0fdSBenjamin Herrenschmidt return is_user ? 0 : SIGBUS; 585b5c8f0fdSBenjamin Herrenschmidt } 586b5c8f0fdSBenjamin Herrenschmidt 587819cdcdbSLaurent Dufour up_read(¤t->mm->mmap_sem); 58814c02e41SLaurent Dufour 589b5c8f0fdSBenjamin Herrenschmidt if (unlikely(fault & VM_FAULT_ERROR)) 590b5c8f0fdSBenjamin Herrenschmidt return mm_fault_error(regs, address, fault); 5919be72573SBenjamin Herrenschmidt 5929be72573SBenjamin Herrenschmidt /* 59314c02e41SLaurent Dufour * Major/minor page fault accounting. 5949be72573SBenjamin Herrenschmidt */ 595f43bb27eSBenjamin Herrenschmidt if (major) { 59683c54070SNick Piggin current->maj_flt++; 59704aafdc6SBenjamin Herrenschmidt perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); 5983da02648SBenjamin Herrenschmidt cmo_account_page_fault(); 599ac17dc8eSPeter Zijlstra } else { 60083c54070SNick Piggin current->min_flt++; 60104aafdc6SBenjamin Herrenschmidt perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); 602ac17dc8eSPeter Zijlstra } 60365d47fd4SBenjamin Herrenschmidt return 0; 60414cf11afSPaul Mackerras } 6057afad422SBenjamin Herrenschmidt NOKPROBE_SYMBOL(__do_page_fault); 6067afad422SBenjamin Herrenschmidt 6077afad422SBenjamin Herrenschmidt int do_page_fault(struct pt_regs *regs, unsigned long address, 6087afad422SBenjamin Herrenschmidt unsigned long error_code) 6097afad422SBenjamin Herrenschmidt { 6107afad422SBenjamin Herrenschmidt enum ctx_state prev_state = exception_enter(); 6117afad422SBenjamin Herrenschmidt int rc = __do_page_fault(regs, address, error_code); 612ba12eedeSLi Zhong exception_exit(prev_state); 613ba12eedeSLi Zhong return rc; 61414cf11afSPaul Mackerras } 61503465f89SNicholas Piggin NOKPROBE_SYMBOL(do_page_fault); 61614cf11afSPaul Mackerras 61714cf11afSPaul Mackerras /* 61814cf11afSPaul Mackerras * bad_page_fault is called when we have a bad access from the kernel. 61914cf11afSPaul Mackerras * It is called from the DSI and ISI handlers in head.S and from some 62014cf11afSPaul Mackerras * of the procedures in traps.c. 62114cf11afSPaul Mackerras */ 62214cf11afSPaul Mackerras void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) 62314cf11afSPaul Mackerras { 62414cf11afSPaul Mackerras const struct exception_table_entry *entry; 62514cf11afSPaul Mackerras 62614cf11afSPaul Mackerras /* Are we prepared to handle this fault? */ 62714cf11afSPaul Mackerras if ((entry = search_exception_tables(regs->nip)) != NULL) { 62861a92f70SNicholas Piggin regs->nip = extable_fixup(entry); 62914cf11afSPaul Mackerras return; 63014cf11afSPaul Mackerras } 63114cf11afSPaul Mackerras 63214cf11afSPaul Mackerras /* kernel has accessed a bad area */ 633723925b7SOlof Johansson 6342271db20SBenjamin Herrenschmidt switch (TRAP(regs)) { 635723925b7SOlof Johansson case 0x300: 636723925b7SOlof Johansson case 0x380: 637a416dd8dSMichael Ellerman printk(KERN_ALERT "Unable to handle kernel paging request for " 638a416dd8dSMichael Ellerman "data at address 0x%08lx\n", regs->dar); 639723925b7SOlof Johansson break; 640723925b7SOlof Johansson case 0x400: 641723925b7SOlof Johansson case 0x480: 642a416dd8dSMichael Ellerman printk(KERN_ALERT "Unable to handle kernel paging request for " 643a416dd8dSMichael Ellerman "instruction fetch\n"); 644723925b7SOlof Johansson break; 645eab861a7SAnton Blanchard case 0x600: 646eab861a7SAnton Blanchard printk(KERN_ALERT "Unable to handle kernel paging request for " 647eab861a7SAnton Blanchard "unaligned access at address 0x%08lx\n", regs->dar); 648eab861a7SAnton Blanchard break; 649723925b7SOlof Johansson default: 650a416dd8dSMichael Ellerman printk(KERN_ALERT "Unable to handle kernel paging request for " 651a416dd8dSMichael Ellerman "unknown fault\n"); 652a416dd8dSMichael Ellerman break; 653723925b7SOlof Johansson } 654723925b7SOlof Johansson printk(KERN_ALERT "Faulting instruction address: 0x%08lx\n", 655723925b7SOlof Johansson regs->nip); 656723925b7SOlof Johansson 657a70857e4SAaron Tomlin if (task_stack_end_corrupted(current)) 65828b54990SAnton Blanchard printk(KERN_ALERT "Thread overran stack, or stack corrupted\n"); 65928b54990SAnton Blanchard 66014cf11afSPaul Mackerras die("Kernel access of bad area", regs, sig); 66114cf11afSPaul Mackerras } 662