1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 1995 - 2000 by Ralf Baechle 7 */ 8 #include <linux/context_tracking.h> 9 #include <linux/signal.h> 10 #include <linux/sched.h> 11 #include <linux/interrupt.h> 12 #include <linux/kernel.h> 13 #include <linux/entry-common.h> 14 #include <linux/errno.h> 15 #include <linux/string.h> 16 #include <linux/types.h> 17 #include <linux/ptrace.h> 18 #include <linux/ratelimit.h> 19 #include <linux/mman.h> 20 #include <linux/mm.h> 21 #include <linux/smp.h> 22 #include <linux/kdebug.h> 23 #include <linux/kprobes.h> 24 #include <linux/perf_event.h> 25 #include <linux/uaccess.h> 26 27 #include <asm/branch.h> 28 #include <asm/mmu_context.h> 29 #include <asm/ptrace.h> 30 31 int show_unhandled_signals = 1; 32 33 static void __kprobes no_context(struct pt_regs *regs, unsigned long address) 34 { 35 const int field = sizeof(unsigned long) * 2; 36 37 /* Are we prepared to handle this kernel fault? */ 38 if (fixup_exception(regs)) 39 return; 40 41 /* 42 * Oops. The kernel tried to access some bad page. We'll have to 43 * terminate things with extreme prejudice. 44 */ 45 bust_spinlocks(1); 46 47 pr_alert("CPU %d Unable to handle kernel paging request at " 48 "virtual address %0*lx, era == %0*lx, ra == %0*lx\n", 49 raw_smp_processor_id(), field, address, field, regs->csr_era, 50 field, regs->regs[1]); 51 die("Oops", regs); 52 } 53 54 static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address) 55 { 56 /* 57 * We ran out of memory, call the OOM killer, and return the userspace 58 * (which will retry the fault, or kill us if we got oom-killed). 59 */ 60 if (!user_mode(regs)) { 61 no_context(regs, address); 62 return; 63 } 64 pagefault_out_of_memory(); 65 } 66 67 static void __kprobes do_sigbus(struct pt_regs *regs, 68 unsigned long write, unsigned long address, int si_code) 69 { 70 /* Kernel mode? Handle exceptions or die */ 71 if (!user_mode(regs)) { 72 no_context(regs, address); 73 return; 74 } 75 76 /* 77 * Send a sigbus, regardless of whether we were in kernel 78 * or user mode. 79 */ 80 current->thread.csr_badvaddr = address; 81 current->thread.trap_nr = read_csr_excode(); 82 force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); 83 } 84 85 static void __kprobes do_sigsegv(struct pt_regs *regs, 86 unsigned long write, unsigned long address, int si_code) 87 { 88 const int field = sizeof(unsigned long) * 2; 89 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); 90 91 /* Kernel mode? Handle exceptions or die */ 92 if (!user_mode(regs)) { 93 no_context(regs, address); 94 return; 95 } 96 97 /* User mode accesses just cause a SIGSEGV */ 98 current->thread.csr_badvaddr = address; 99 if (!write) 100 current->thread.error_code = 1; 101 else 102 current->thread.error_code = 2; 103 current->thread.trap_nr = read_csr_excode(); 104 105 if (show_unhandled_signals && 106 unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) { 107 pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n", 108 current->comm, 109 write ? "write access to" : "read access from", 110 field, address); 111 pr_info("era = %0*lx in", field, 112 (unsigned long) regs->csr_era); 113 print_vma_addr(KERN_CONT " ", regs->csr_era); 114 pr_cont("\n"); 115 pr_info("ra = %0*lx in", field, 116 (unsigned long) regs->regs[1]); 117 print_vma_addr(KERN_CONT " ", regs->regs[1]); 118 pr_cont("\n"); 119 } 120 force_sig_fault(SIGSEGV, si_code, (void __user *)address); 121 } 122 123 /* 124 * This routine handles page faults. It determines the address, 125 * and the problem, and then passes it off to one of the appropriate 126 * routines. 127 */ 128 static void __kprobes __do_page_fault(struct pt_regs *regs, 129 unsigned long write, unsigned long address) 130 { 131 int si_code = SEGV_MAPERR; 132 unsigned int flags = FAULT_FLAG_DEFAULT; 133 struct task_struct *tsk = current; 134 struct mm_struct *mm = tsk->mm; 135 struct vm_area_struct *vma = NULL; 136 vm_fault_t fault; 137 138 if (kprobe_page_fault(regs, current->thread.trap_nr)) 139 return; 140 141 /* 142 * We fault-in kernel-space virtual memory on-demand. The 143 * 'reference' page table is init_mm.pgd. 144 * 145 * NOTE! We MUST NOT take any locks for this case. We may 146 * be in an interrupt or a critical region, and should 147 * only copy the information from the master page table, 148 * nothing more. 149 */ 150 if (address & __UA_LIMIT) { 151 if (!user_mode(regs)) 152 no_context(regs, address); 153 else 154 do_sigsegv(regs, write, address, si_code); 155 return; 156 } 157 158 /* 159 * If we're in an interrupt or have no user 160 * context, we must not take the fault.. 161 */ 162 if (faulthandler_disabled() || !mm) { 163 do_sigsegv(regs, write, address, si_code); 164 return; 165 } 166 167 if (user_mode(regs)) 168 flags |= FAULT_FLAG_USER; 169 170 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 171 retry: 172 mmap_read_lock(mm); 173 vma = find_vma(mm, address); 174 if (!vma) 175 goto bad_area; 176 if (vma->vm_start <= address) 177 goto good_area; 178 if (!(vma->vm_flags & VM_GROWSDOWN)) 179 goto bad_area; 180 if (!expand_stack(vma, address)) 181 goto good_area; 182 /* 183 * Something tried to access memory that isn't in our memory map.. 184 * Fix it, but check if it's kernel or user first.. 185 */ 186 bad_area: 187 mmap_read_unlock(mm); 188 do_sigsegv(regs, write, address, si_code); 189 return; 190 191 /* 192 * Ok, we have a good vm_area for this memory access, so 193 * we can handle it.. 194 */ 195 good_area: 196 si_code = SEGV_ACCERR; 197 198 if (write) { 199 flags |= FAULT_FLAG_WRITE; 200 if (!(vma->vm_flags & VM_WRITE)) 201 goto bad_area; 202 } else { 203 if (!(vma->vm_flags & VM_READ) && address != exception_era(regs)) 204 goto bad_area; 205 if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) 206 goto bad_area; 207 } 208 209 /* 210 * If for any reason at all we couldn't handle the fault, 211 * make sure we exit gracefully rather than endlessly redo 212 * the fault. 213 */ 214 fault = handle_mm_fault(vma, address, flags, regs); 215 216 if (fault_signal_pending(fault, regs)) { 217 if (!user_mode(regs)) 218 no_context(regs, address); 219 return; 220 } 221 222 /* The fault is fully completed (including releasing mmap lock) */ 223 if (fault & VM_FAULT_COMPLETED) 224 return; 225 226 if (unlikely(fault & VM_FAULT_RETRY)) { 227 flags |= FAULT_FLAG_TRIED; 228 229 /* 230 * No need to mmap_read_unlock(mm) as we would 231 * have already released it in __lock_page_or_retry 232 * in mm/filemap.c. 233 */ 234 goto retry; 235 } 236 if (unlikely(fault & VM_FAULT_ERROR)) { 237 mmap_read_unlock(mm); 238 if (fault & VM_FAULT_OOM) { 239 do_out_of_memory(regs, address); 240 return; 241 } else if (fault & VM_FAULT_SIGSEGV) { 242 do_sigsegv(regs, write, address, si_code); 243 return; 244 } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { 245 do_sigbus(regs, write, address, si_code); 246 return; 247 } 248 BUG(); 249 } 250 251 mmap_read_unlock(mm); 252 } 253 254 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, 255 unsigned long write, unsigned long address) 256 { 257 irqentry_state_t state = irqentry_enter(regs); 258 259 /* Enable interrupt if enabled in parent context */ 260 if (likely(regs->csr_prmd & CSR_PRMD_PIE)) 261 local_irq_enable(); 262 263 __do_page_fault(regs, write, address); 264 265 local_irq_disable(); 266 267 irqentry_exit(regs, state); 268 } 269