1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 1995 - 2000 by Ralf Baechle 7 */ 8 #include <linux/context_tracking.h> 9 #include <linux/signal.h> 10 #include <linux/sched.h> 11 #include <linux/interrupt.h> 12 #include <linux/kernel.h> 13 #include <linux/entry-common.h> 14 #include <linux/errno.h> 15 #include <linux/string.h> 16 #include <linux/types.h> 17 #include <linux/ptrace.h> 18 #include <linux/ratelimit.h> 19 #include <linux/mman.h> 20 #include <linux/mm.h> 21 #include <linux/smp.h> 22 #include <linux/kdebug.h> 23 #include <linux/perf_event.h> 24 #include <linux/uaccess.h> 25 #include <linux/kfence.h> 26 27 #include <asm/branch.h> 28 #include <asm/exception.h> 29 #include <asm/mmu_context.h> 30 #include <asm/ptrace.h> 31 32 int show_unhandled_signals = 1; 33 34 static void __kprobes no_context(struct pt_regs *regs, 35 unsigned long write, unsigned long address) 36 { 37 const int field = sizeof(unsigned long) * 2; 38 39 /* Are we prepared to handle this kernel fault? */ 40 if (fixup_exception(regs)) 41 return; 42 43 if (kfence_handle_page_fault(address, write, regs)) 44 return; 45 46 /* 47 * Oops. The kernel tried to access some bad page. We'll have to 48 * terminate things with extreme prejudice. 49 */ 50 bust_spinlocks(1); 51 52 pr_alert("CPU %d Unable to handle kernel paging request at " 53 "virtual address %0*lx, era == %0*lx, ra == %0*lx\n", 54 raw_smp_processor_id(), field, address, field, regs->csr_era, 55 field, regs->regs[1]); 56 die("Oops", regs); 57 } 58 59 static void __kprobes do_out_of_memory(struct pt_regs *regs, 60 unsigned long write, unsigned long address) 61 { 62 /* 63 * We ran out of memory, call the OOM killer, and return the userspace 64 * (which will retry the fault, or kill us if we got oom-killed). 65 */ 66 if (!user_mode(regs)) { 67 no_context(regs, write, address); 68 return; 69 } 70 pagefault_out_of_memory(); 71 } 72 73 static void __kprobes do_sigbus(struct pt_regs *regs, 74 unsigned long write, unsigned long address, int si_code) 75 { 76 /* Kernel mode? Handle exceptions or die */ 77 if (!user_mode(regs)) { 78 no_context(regs, write, address); 79 return; 80 } 81 82 /* 83 * Send a sigbus, regardless of whether we were in kernel 84 * or user mode. 85 */ 86 current->thread.csr_badvaddr = address; 87 current->thread.trap_nr = read_csr_excode(); 88 force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); 89 } 90 91 static void __kprobes do_sigsegv(struct pt_regs *regs, 92 unsigned long write, unsigned long address, int si_code) 93 { 94 const int field = sizeof(unsigned long) * 2; 95 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); 96 97 /* Kernel mode? Handle exceptions or die */ 98 if (!user_mode(regs)) { 99 no_context(regs, write, address); 100 return; 101 } 102 103 /* User mode accesses just cause a SIGSEGV */ 104 current->thread.csr_badvaddr = address; 105 if (!write) 106 current->thread.error_code = 1; 107 else 108 current->thread.error_code = 2; 109 current->thread.trap_nr = read_csr_excode(); 110 111 if (show_unhandled_signals && 112 unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) { 113 pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n", 114 current->comm, 115 write ? "write access to" : "read access from", 116 field, address); 117 pr_info("era = %0*lx in", field, 118 (unsigned long) regs->csr_era); 119 print_vma_addr(KERN_CONT " ", regs->csr_era); 120 pr_cont("\n"); 121 pr_info("ra = %0*lx in", field, 122 (unsigned long) regs->regs[1]); 123 print_vma_addr(KERN_CONT " ", regs->regs[1]); 124 pr_cont("\n"); 125 } 126 force_sig_fault(SIGSEGV, si_code, (void __user *)address); 127 } 128 129 /* 130 * This routine handles page faults. It determines the address, 131 * and the problem, and then passes it off to one of the appropriate 132 * routines. 133 */ 134 static void __kprobes __do_page_fault(struct pt_regs *regs, 135 unsigned long write, unsigned long address) 136 { 137 int si_code = SEGV_MAPERR; 138 unsigned int flags = FAULT_FLAG_DEFAULT; 139 struct task_struct *tsk = current; 140 struct mm_struct *mm = tsk->mm; 141 struct vm_area_struct *vma = NULL; 142 vm_fault_t fault; 143 144 if (kprobe_page_fault(regs, current->thread.trap_nr)) 145 return; 146 147 /* 148 * We fault-in kernel-space virtual memory on-demand. The 149 * 'reference' page table is init_mm.pgd. 150 * 151 * NOTE! We MUST NOT take any locks for this case. We may 152 * be in an interrupt or a critical region, and should 153 * only copy the information from the master page table, 154 * nothing more. 155 */ 156 if (address & __UA_LIMIT) { 157 if (!user_mode(regs)) 158 no_context(regs, write, address); 159 else 160 do_sigsegv(regs, write, address, si_code); 161 return; 162 } 163 164 /* 165 * If we're in an interrupt or have no user 166 * context, we must not take the fault.. 167 */ 168 if (faulthandler_disabled() || !mm) { 169 do_sigsegv(regs, write, address, si_code); 170 return; 171 } 172 173 if (user_mode(regs)) 174 flags |= FAULT_FLAG_USER; 175 176 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 177 retry: 178 vma = lock_mm_and_find_vma(mm, address, regs); 179 if (unlikely(!vma)) 180 goto bad_area_nosemaphore; 181 goto good_area; 182 183 /* 184 * Something tried to access memory that isn't in our memory map.. 185 * Fix it, but check if it's kernel or user first.. 186 */ 187 bad_area: 188 mmap_read_unlock(mm); 189 bad_area_nosemaphore: 190 do_sigsegv(regs, write, address, si_code); 191 return; 192 193 /* 194 * Ok, we have a good vm_area for this memory access, so 195 * we can handle it.. 196 */ 197 good_area: 198 si_code = SEGV_ACCERR; 199 200 if (write) { 201 flags |= FAULT_FLAG_WRITE; 202 if (!(vma->vm_flags & VM_WRITE)) 203 goto bad_area; 204 } else { 205 if (!(vma->vm_flags & VM_READ) && address != exception_era(regs)) 206 goto bad_area; 207 if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) 208 goto bad_area; 209 } 210 211 /* 212 * If for any reason at all we couldn't handle the fault, 213 * make sure we exit gracefully rather than endlessly redo 214 * the fault. 215 */ 216 fault = handle_mm_fault(vma, address, flags, regs); 217 218 if (fault_signal_pending(fault, regs)) { 219 if (!user_mode(regs)) 220 no_context(regs, write, address); 221 return; 222 } 223 224 /* The fault is fully completed (including releasing mmap lock) */ 225 if (fault & VM_FAULT_COMPLETED) 226 return; 227 228 if (unlikely(fault & VM_FAULT_RETRY)) { 229 flags |= FAULT_FLAG_TRIED; 230 231 /* 232 * No need to mmap_read_unlock(mm) as we would 233 * have already released it in __lock_page_or_retry 234 * in mm/filemap.c. 235 */ 236 goto retry; 237 } 238 if (unlikely(fault & VM_FAULT_ERROR)) { 239 mmap_read_unlock(mm); 240 if (fault & VM_FAULT_OOM) { 241 do_out_of_memory(regs, write, address); 242 return; 243 } else if (fault & VM_FAULT_SIGSEGV) { 244 do_sigsegv(regs, write, address, si_code); 245 return; 246 } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { 247 do_sigbus(regs, write, address, si_code); 248 return; 249 } 250 BUG(); 251 } 252 253 mmap_read_unlock(mm); 254 } 255 256 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, 257 unsigned long write, unsigned long address) 258 { 259 irqentry_state_t state = irqentry_enter(regs); 260 261 /* Enable interrupt if enabled in parent context */ 262 if (likely(regs->csr_prmd & CSR_PRMD_PIE)) 263 local_irq_enable(); 264 265 __do_page_fault(regs, write, address); 266 267 local_irq_disable(); 268 269 irqentry_exit(regs, state); 270 } 271