1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 1995 - 2000 by Ralf Baechle 7 */ 8 #include <linux/context_tracking.h> 9 #include <linux/signal.h> 10 #include <linux/sched.h> 11 #include <linux/interrupt.h> 12 #include <linux/kernel.h> 13 #include <linux/entry-common.h> 14 #include <linux/errno.h> 15 #include <linux/string.h> 16 #include <linux/types.h> 17 #include <linux/ptrace.h> 18 #include <linux/ratelimit.h> 19 #include <linux/mman.h> 20 #include <linux/mm.h> 21 #include <linux/smp.h> 22 #include <linux/kdebug.h> 23 #include <linux/kprobes.h> 24 #include <linux/perf_event.h> 25 #include <linux/uaccess.h> 26 27 #include <asm/branch.h> 28 #include <asm/mmu_context.h> 29 #include <asm/ptrace.h> 30 31 int show_unhandled_signals = 1; 32 33 static void __kprobes no_context(struct pt_regs *regs, unsigned long address) 34 { 35 const int field = sizeof(unsigned long) * 2; 36 37 /* Are we prepared to handle this kernel fault? */ 38 if (fixup_exception(regs)) 39 return; 40 41 /* 42 * Oops. The kernel tried to access some bad page. We'll have to 43 * terminate things with extreme prejudice. 44 */ 45 bust_spinlocks(1); 46 47 pr_alert("CPU %d Unable to handle kernel paging request at " 48 "virtual address %0*lx, era == %0*lx, ra == %0*lx\n", 49 raw_smp_processor_id(), field, address, field, regs->csr_era, 50 field, regs->regs[1]); 51 die("Oops", regs); 52 } 53 54 static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address) 55 { 56 /* 57 * We ran out of memory, call the OOM killer, and return the userspace 58 * (which will retry the fault, or kill us if we got oom-killed). 59 */ 60 if (!user_mode(regs)) { 61 no_context(regs, address); 62 return; 63 } 64 pagefault_out_of_memory(); 65 } 66 67 static void __kprobes do_sigbus(struct pt_regs *regs, 68 unsigned long write, unsigned long address, int si_code) 69 { 70 /* Kernel mode? Handle exceptions or die */ 71 if (!user_mode(regs)) { 72 no_context(regs, address); 73 return; 74 } 75 76 /* 77 * Send a sigbus, regardless of whether we were in kernel 78 * or user mode. 79 */ 80 current->thread.csr_badvaddr = address; 81 current->thread.trap_nr = read_csr_excode(); 82 force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); 83 } 84 85 static void __kprobes do_sigsegv(struct pt_regs *regs, 86 unsigned long write, unsigned long address, int si_code) 87 { 88 const int field = sizeof(unsigned long) * 2; 89 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); 90 91 /* Kernel mode? Handle exceptions or die */ 92 if (!user_mode(regs)) { 93 no_context(regs, address); 94 return; 95 } 96 97 /* User mode accesses just cause a SIGSEGV */ 98 current->thread.csr_badvaddr = address; 99 if (!write) 100 current->thread.error_code = 1; 101 else 102 current->thread.error_code = 2; 103 current->thread.trap_nr = read_csr_excode(); 104 105 if (show_unhandled_signals && 106 unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) { 107 pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n", 108 current->comm, 109 write ? "write access to" : "read access from", 110 field, address); 111 pr_info("era = %0*lx in", field, 112 (unsigned long) regs->csr_era); 113 print_vma_addr(KERN_CONT " ", regs->csr_era); 114 pr_cont("\n"); 115 pr_info("ra = %0*lx in", field, 116 (unsigned long) regs->regs[1]); 117 print_vma_addr(KERN_CONT " ", regs->regs[1]); 118 pr_cont("\n"); 119 } 120 force_sig_fault(SIGSEGV, si_code, (void __user *)address); 121 } 122 123 /* 124 * This routine handles page faults. It determines the address, 125 * and the problem, and then passes it off to one of the appropriate 126 * routines. 127 */ 128 static void __kprobes __do_page_fault(struct pt_regs *regs, 129 unsigned long write, unsigned long address) 130 { 131 int si_code = SEGV_MAPERR; 132 unsigned int flags = FAULT_FLAG_DEFAULT; 133 struct task_struct *tsk = current; 134 struct mm_struct *mm = tsk->mm; 135 struct vm_area_struct *vma = NULL; 136 vm_fault_t fault; 137 138 if (kprobe_page_fault(regs, current->thread.trap_nr)) 139 return; 140 141 /* 142 * We fault-in kernel-space virtual memory on-demand. The 143 * 'reference' page table is init_mm.pgd. 144 * 145 * NOTE! We MUST NOT take any locks for this case. We may 146 * be in an interrupt or a critical region, and should 147 * only copy the information from the master page table, 148 * nothing more. 149 */ 150 if (address & __UA_LIMIT) { 151 if (!user_mode(regs)) 152 no_context(regs, address); 153 else 154 do_sigsegv(regs, write, address, si_code); 155 return; 156 } 157 158 /* 159 * If we're in an interrupt or have no user 160 * context, we must not take the fault.. 161 */ 162 if (faulthandler_disabled() || !mm) { 163 do_sigsegv(regs, write, address, si_code); 164 return; 165 } 166 167 if (user_mode(regs)) 168 flags |= FAULT_FLAG_USER; 169 170 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 171 retry: 172 vma = lock_mm_and_find_vma(mm, address, regs); 173 if (unlikely(!vma)) 174 goto bad_area_nosemaphore; 175 goto good_area; 176 177 /* 178 * Something tried to access memory that isn't in our memory map.. 179 * Fix it, but check if it's kernel or user first.. 180 */ 181 bad_area: 182 mmap_read_unlock(mm); 183 bad_area_nosemaphore: 184 do_sigsegv(regs, write, address, si_code); 185 return; 186 187 /* 188 * Ok, we have a good vm_area for this memory access, so 189 * we can handle it.. 190 */ 191 good_area: 192 si_code = SEGV_ACCERR; 193 194 if (write) { 195 flags |= FAULT_FLAG_WRITE; 196 if (!(vma->vm_flags & VM_WRITE)) 197 goto bad_area; 198 } else { 199 if (!(vma->vm_flags & VM_READ) && address != exception_era(regs)) 200 goto bad_area; 201 if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) 202 goto bad_area; 203 } 204 205 /* 206 * If for any reason at all we couldn't handle the fault, 207 * make sure we exit gracefully rather than endlessly redo 208 * the fault. 209 */ 210 fault = handle_mm_fault(vma, address, flags, regs); 211 212 if (fault_signal_pending(fault, regs)) { 213 if (!user_mode(regs)) 214 no_context(regs, address); 215 return; 216 } 217 218 /* The fault is fully completed (including releasing mmap lock) */ 219 if (fault & VM_FAULT_COMPLETED) 220 return; 221 222 if (unlikely(fault & VM_FAULT_RETRY)) { 223 flags |= FAULT_FLAG_TRIED; 224 225 /* 226 * No need to mmap_read_unlock(mm) as we would 227 * have already released it in __lock_page_or_retry 228 * in mm/filemap.c. 229 */ 230 goto retry; 231 } 232 if (unlikely(fault & VM_FAULT_ERROR)) { 233 mmap_read_unlock(mm); 234 if (fault & VM_FAULT_OOM) { 235 do_out_of_memory(regs, address); 236 return; 237 } else if (fault & VM_FAULT_SIGSEGV) { 238 do_sigsegv(regs, write, address, si_code); 239 return; 240 } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { 241 do_sigbus(regs, write, address, si_code); 242 return; 243 } 244 BUG(); 245 } 246 247 mmap_read_unlock(mm); 248 } 249 250 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, 251 unsigned long write, unsigned long address) 252 { 253 irqentry_state_t state = irqentry_enter(regs); 254 255 /* Enable interrupt if enabled in parent context */ 256 if (likely(regs->csr_prmd & CSR_PRMD_PIE)) 257 local_irq_enable(); 258 259 __do_page_fault(regs, write, address); 260 261 local_irq_disable(); 262 263 irqentry_exit(regs, state); 264 } 265