1 /* 2 * Page fault handler for SH with an MMU. 3 * 4 * Copyright (C) 1999 Niibe Yutaka 5 * Copyright (C) 2003 - 2012 Paul Mundt 6 * 7 * Based on linux/arch/i386/mm/fault.c: 8 * Copyright (C) 1995 Linus Torvalds 9 * 10 * This file is subject to the terms and conditions of the GNU General Public 11 * License. See the file "COPYING" in the main directory of this archive 12 * for more details. 13 */ 14 #include <linux/kernel.h> 15 #include <linux/mm.h> 16 #include <linux/sched/signal.h> 17 #include <linux/hardirq.h> 18 #include <linux/kprobes.h> 19 #include <linux/perf_event.h> 20 #include <linux/kdebug.h> 21 #include <linux/uaccess.h> 22 #include <asm/io_trapped.h> 23 #include <asm/mmu_context.h> 24 #include <asm/tlbflush.h> 25 #include <asm/traps.h> 26 27 static void 28 force_sig_info_fault(int si_signo, int si_code, unsigned long address) 29 { 30 force_sig_fault(si_signo, si_code, (void __user *)address); 31 } 32 33 /* 34 * This is useful to dump out the page tables associated with 35 * 'addr' in mm 'mm'. 36 */ 37 static void show_pte(struct mm_struct *mm, unsigned long addr) 38 { 39 pgd_t *pgd; 40 41 if (mm) { 42 pgd = mm->pgd; 43 } else { 44 pgd = get_TTB(); 45 46 if (unlikely(!pgd)) 47 pgd = swapper_pg_dir; 48 } 49 50 printk(KERN_ALERT "pgd = %p\n", pgd); 51 pgd += pgd_index(addr); 52 printk(KERN_ALERT "[%08lx] *pgd=%0*Lx", addr, 53 (u32)(sizeof(*pgd) * 2), (u64)pgd_val(*pgd)); 54 55 do { 56 pud_t *pud; 57 pmd_t *pmd; 58 pte_t *pte; 59 60 if (pgd_none(*pgd)) 61 break; 62 63 if (pgd_bad(*pgd)) { 64 printk("(bad)"); 65 break; 66 } 67 68 pud = pud_offset(pgd, addr); 69 if (PTRS_PER_PUD != 1) 70 printk(", *pud=%0*Lx", (u32)(sizeof(*pud) * 2), 71 (u64)pud_val(*pud)); 72 73 if (pud_none(*pud)) 74 break; 75 76 if (pud_bad(*pud)) { 77 printk("(bad)"); 78 break; 79 } 80 81 pmd = pmd_offset(pud, addr); 82 if (PTRS_PER_PMD != 1) 83 printk(", *pmd=%0*Lx", (u32)(sizeof(*pmd) * 2), 84 (u64)pmd_val(*pmd)); 85 86 if (pmd_none(*pmd)) 87 break; 88 89 if (pmd_bad(*pmd)) { 90 printk("(bad)"); 91 break; 92 } 93 94 /* We must not map this if we have highmem enabled */ 95 if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT))) 96 break; 97 98 pte = pte_offset_kernel(pmd, addr); 99 printk(", *pte=%0*Lx", (u32)(sizeof(*pte) * 2), 100 (u64)pte_val(*pte)); 101 } while (0); 102 103 printk("\n"); 104 } 105 106 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) 107 { 108 unsigned index = pgd_index(address); 109 pgd_t *pgd_k; 110 pud_t *pud, *pud_k; 111 pmd_t *pmd, *pmd_k; 112 113 pgd += index; 114 pgd_k = init_mm.pgd + index; 115 116 if (!pgd_present(*pgd_k)) 117 return NULL; 118 119 pud = pud_offset(pgd, address); 120 pud_k = pud_offset(pgd_k, address); 121 if (!pud_present(*pud_k)) 122 return NULL; 123 124 if (!pud_present(*pud)) 125 set_pud(pud, *pud_k); 126 127 pmd = pmd_offset(pud, address); 128 pmd_k = pmd_offset(pud_k, address); 129 if (!pmd_present(*pmd_k)) 130 return NULL; 131 132 if (!pmd_present(*pmd)) 133 set_pmd(pmd, *pmd_k); 134 else { 135 /* 136 * The page tables are fully synchronised so there must 137 * be another reason for the fault. Return NULL here to 138 * signal that we have not taken care of the fault. 139 */ 140 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k)); 141 return NULL; 142 } 143 144 return pmd_k; 145 } 146 147 #ifdef CONFIG_SH_STORE_QUEUES 148 #define __FAULT_ADDR_LIMIT P3_ADDR_MAX 149 #else 150 #define __FAULT_ADDR_LIMIT VMALLOC_END 151 #endif 152 153 /* 154 * Handle a fault on the vmalloc or module mapping area 155 */ 156 static noinline int vmalloc_fault(unsigned long address) 157 { 158 pgd_t *pgd_k; 159 pmd_t *pmd_k; 160 pte_t *pte_k; 161 162 /* Make sure we are in vmalloc/module/P3 area: */ 163 if (!(address >= VMALLOC_START && address < __FAULT_ADDR_LIMIT)) 164 return -1; 165 166 /* 167 * Synchronize this task's top level page-table 168 * with the 'reference' page table. 169 * 170 * Do _not_ use "current" here. We might be inside 171 * an interrupt in the middle of a task switch.. 172 */ 173 pgd_k = get_TTB(); 174 pmd_k = vmalloc_sync_one(pgd_k, address); 175 if (!pmd_k) 176 return -1; 177 178 pte_k = pte_offset_kernel(pmd_k, address); 179 if (!pte_present(*pte_k)) 180 return -1; 181 182 return 0; 183 } 184 185 static void 186 show_fault_oops(struct pt_regs *regs, unsigned long address) 187 { 188 if (!oops_may_print()) 189 return; 190 191 printk(KERN_ALERT "BUG: unable to handle kernel "); 192 if (address < PAGE_SIZE) 193 printk(KERN_CONT "NULL pointer dereference"); 194 else 195 printk(KERN_CONT "paging request"); 196 197 printk(KERN_CONT " at %08lx\n", address); 198 printk(KERN_ALERT "PC:"); 199 printk_address(regs->pc, 1); 200 201 show_pte(NULL, address); 202 } 203 204 static noinline void 205 no_context(struct pt_regs *regs, unsigned long error_code, 206 unsigned long address) 207 { 208 /* Are we prepared to handle this kernel fault? */ 209 if (fixup_exception(regs)) 210 return; 211 212 if (handle_trapped_io(regs, address)) 213 return; 214 215 /* 216 * Oops. The kernel tried to access some bad page. We'll have to 217 * terminate things with extreme prejudice. 218 */ 219 bust_spinlocks(1); 220 221 show_fault_oops(regs, address); 222 223 die("Oops", regs, error_code); 224 bust_spinlocks(0); 225 do_exit(SIGKILL); 226 } 227 228 static void 229 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, 230 unsigned long address, int si_code) 231 { 232 /* User mode accesses just cause a SIGSEGV */ 233 if (user_mode(regs)) { 234 /* 235 * It's possible to have interrupts off here: 236 */ 237 local_irq_enable(); 238 239 force_sig_info_fault(SIGSEGV, si_code, address); 240 241 return; 242 } 243 244 no_context(regs, error_code, address); 245 } 246 247 static noinline void 248 bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, 249 unsigned long address) 250 { 251 __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR); 252 } 253 254 static void 255 __bad_area(struct pt_regs *regs, unsigned long error_code, 256 unsigned long address, int si_code) 257 { 258 struct mm_struct *mm = current->mm; 259 260 /* 261 * Something tried to access memory that isn't in our memory map.. 262 * Fix it, but check if it's kernel or user first.. 263 */ 264 up_read(&mm->mmap_sem); 265 266 __bad_area_nosemaphore(regs, error_code, address, si_code); 267 } 268 269 static noinline void 270 bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address) 271 { 272 __bad_area(regs, error_code, address, SEGV_MAPERR); 273 } 274 275 static noinline void 276 bad_area_access_error(struct pt_regs *regs, unsigned long error_code, 277 unsigned long address) 278 { 279 __bad_area(regs, error_code, address, SEGV_ACCERR); 280 } 281 282 static void 283 do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address) 284 { 285 struct task_struct *tsk = current; 286 struct mm_struct *mm = tsk->mm; 287 288 up_read(&mm->mmap_sem); 289 290 /* Kernel mode? Handle exceptions or die: */ 291 if (!user_mode(regs)) 292 no_context(regs, error_code, address); 293 294 force_sig_info_fault(SIGBUS, BUS_ADRERR, address); 295 } 296 297 static noinline int 298 mm_fault_error(struct pt_regs *regs, unsigned long error_code, 299 unsigned long address, vm_fault_t fault) 300 { 301 /* 302 * Pagefault was interrupted by SIGKILL. We have no reason to 303 * continue pagefault. 304 */ 305 if (fatal_signal_pending(current)) { 306 if (!(fault & VM_FAULT_RETRY)) 307 up_read(¤t->mm->mmap_sem); 308 if (!user_mode(regs)) 309 no_context(regs, error_code, address); 310 return 1; 311 } 312 313 if (!(fault & VM_FAULT_ERROR)) 314 return 0; 315 316 if (fault & VM_FAULT_OOM) { 317 /* Kernel mode? Handle exceptions or die: */ 318 if (!user_mode(regs)) { 319 up_read(¤t->mm->mmap_sem); 320 no_context(regs, error_code, address); 321 return 1; 322 } 323 up_read(¤t->mm->mmap_sem); 324 325 /* 326 * We ran out of memory, call the OOM killer, and return the 327 * userspace (which will retry the fault, or kill us if we got 328 * oom-killed): 329 */ 330 pagefault_out_of_memory(); 331 } else { 332 if (fault & VM_FAULT_SIGBUS) 333 do_sigbus(regs, error_code, address); 334 else if (fault & VM_FAULT_SIGSEGV) 335 bad_area(regs, error_code, address); 336 else 337 BUG(); 338 } 339 340 return 1; 341 } 342 343 static inline int access_error(int error_code, struct vm_area_struct *vma) 344 { 345 if (error_code & FAULT_CODE_WRITE) { 346 /* write, present and write, not present: */ 347 if (unlikely(!(vma->vm_flags & VM_WRITE))) 348 return 1; 349 return 0; 350 } 351 352 /* ITLB miss on NX page */ 353 if (unlikely((error_code & FAULT_CODE_ITLB) && 354 !(vma->vm_flags & VM_EXEC))) 355 return 1; 356 357 /* read, not present: */ 358 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) 359 return 1; 360 361 return 0; 362 } 363 364 static int fault_in_kernel_space(unsigned long address) 365 { 366 return address >= TASK_SIZE; 367 } 368 369 /* 370 * This routine handles page faults. It determines the address, 371 * and the problem, and then passes it off to one of the appropriate 372 * routines. 373 */ 374 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, 375 unsigned long error_code, 376 unsigned long address) 377 { 378 unsigned long vec; 379 struct task_struct *tsk; 380 struct mm_struct *mm; 381 struct vm_area_struct * vma; 382 vm_fault_t fault; 383 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 384 385 tsk = current; 386 mm = tsk->mm; 387 vec = lookup_exception_vector(); 388 389 /* 390 * We fault-in kernel-space virtual memory on-demand. The 391 * 'reference' page table is init_mm.pgd. 392 * 393 * NOTE! We MUST NOT take any locks for this case. We may 394 * be in an interrupt or a critical region, and should 395 * only copy the information from the master page table, 396 * nothing more. 397 */ 398 if (unlikely(fault_in_kernel_space(address))) { 399 if (vmalloc_fault(address) >= 0) 400 return; 401 if (kprobe_page_fault(regs, vec)) 402 return; 403 404 bad_area_nosemaphore(regs, error_code, address); 405 return; 406 } 407 408 if (unlikely(kprobe_page_fault(regs, vec))) 409 return; 410 411 /* Only enable interrupts if they were on before the fault */ 412 if ((regs->sr & SR_IMASK) != SR_IMASK) 413 local_irq_enable(); 414 415 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 416 417 /* 418 * If we're in an interrupt, have no user context or are running 419 * with pagefaults disabled then we must not take the fault: 420 */ 421 if (unlikely(faulthandler_disabled() || !mm)) { 422 bad_area_nosemaphore(regs, error_code, address); 423 return; 424 } 425 426 retry: 427 down_read(&mm->mmap_sem); 428 429 vma = find_vma(mm, address); 430 if (unlikely(!vma)) { 431 bad_area(regs, error_code, address); 432 return; 433 } 434 if (likely(vma->vm_start <= address)) 435 goto good_area; 436 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { 437 bad_area(regs, error_code, address); 438 return; 439 } 440 if (unlikely(expand_stack(vma, address))) { 441 bad_area(regs, error_code, address); 442 return; 443 } 444 445 /* 446 * Ok, we have a good vm_area for this memory access, so 447 * we can handle it.. 448 */ 449 good_area: 450 if (unlikely(access_error(error_code, vma))) { 451 bad_area_access_error(regs, error_code, address); 452 return; 453 } 454 455 set_thread_fault_code(error_code); 456 457 if (user_mode(regs)) 458 flags |= FAULT_FLAG_USER; 459 if (error_code & FAULT_CODE_WRITE) 460 flags |= FAULT_FLAG_WRITE; 461 462 /* 463 * If for any reason at all we couldn't handle the fault, 464 * make sure we exit gracefully rather than endlessly redo 465 * the fault. 466 */ 467 fault = handle_mm_fault(vma, address, flags); 468 469 if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR))) 470 if (mm_fault_error(regs, error_code, address, fault)) 471 return; 472 473 if (flags & FAULT_FLAG_ALLOW_RETRY) { 474 if (fault & VM_FAULT_MAJOR) { 475 tsk->maj_flt++; 476 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 477 regs, address); 478 } else { 479 tsk->min_flt++; 480 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 481 regs, address); 482 } 483 if (fault & VM_FAULT_RETRY) { 484 flags &= ~FAULT_FLAG_ALLOW_RETRY; 485 flags |= FAULT_FLAG_TRIED; 486 487 /* 488 * No need to up_read(&mm->mmap_sem) as we would 489 * have already released it in __lock_page_or_retry 490 * in mm/filemap.c. 491 */ 492 goto retry; 493 } 494 } 495 496 up_read(&mm->mmap_sem); 497 } 498