1 /* 2 * arch/s390/mm/fault.c 3 * 4 * S390 version 5 * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation 6 * Author(s): Hartmut Penner (hp@de.ibm.com) 7 * Ulrich Weigand (uweigand@de.ibm.com) 8 * 9 * Derived from "arch/i386/mm/fault.c" 10 * Copyright (C) 1995 Linus Torvalds 11 */ 12 13 #include <linux/kernel_stat.h> 14 #include <linux/perf_event.h> 15 #include <linux/signal.h> 16 #include <linux/sched.h> 17 #include <linux/kernel.h> 18 #include <linux/errno.h> 19 #include <linux/string.h> 20 #include <linux/types.h> 21 #include <linux/ptrace.h> 22 #include <linux/mman.h> 23 #include <linux/mm.h> 24 #include <linux/compat.h> 25 #include <linux/smp.h> 26 #include <linux/kdebug.h> 27 #include <linux/init.h> 28 #include <linux/console.h> 29 #include <linux/module.h> 30 #include <linux/hardirq.h> 31 #include <linux/kprobes.h> 32 #include <linux/uaccess.h> 33 #include <linux/hugetlb.h> 34 #include <asm/asm-offsets.h> 35 #include <asm/system.h> 36 #include <asm/pgtable.h> 37 #include <asm/irq.h> 38 #include <asm/mmu_context.h> 39 #include <asm/compat.h> 40 #include "../kernel/entry.h" 41 42 #ifndef CONFIG_64BIT 43 #define __FAIL_ADDR_MASK 0x7ffff000 44 #define __SUBCODE_MASK 0x0200 45 #define __PF_RES_FIELD 0ULL 46 #else /* CONFIG_64BIT */ 47 #define __FAIL_ADDR_MASK -4096L 48 #define __SUBCODE_MASK 0x0600 49 #define __PF_RES_FIELD 0x8000000000000000ULL 50 #endif /* CONFIG_64BIT */ 51 52 #define VM_FAULT_BADCONTEXT 0x010000 53 #define VM_FAULT_BADMAP 0x020000 54 #define VM_FAULT_BADACCESS 0x040000 55 56 static unsigned long store_indication; 57 58 void fault_init(void) 59 { 60 if (test_facility(2) && test_facility(75)) 61 store_indication = 0xc00; 62 } 63 64 static inline int notify_page_fault(struct pt_regs *regs) 65 { 66 int ret = 0; 67 68 /* kprobe_running() needs smp_processor_id() */ 69 if (kprobes_built_in() && !user_mode(regs)) { 70 preempt_disable(); 71 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 72 ret = 1; 73 preempt_enable(); 74 } 75 return ret; 76 } 77 78 79 /* 80 * Unlock any spinlocks which will prevent us from getting the 81 * message out. 82 */ 83 void bust_spinlocks(int yes) 84 { 85 if (yes) { 86 oops_in_progress = 1; 87 } else { 88 int loglevel_save = console_loglevel; 89 console_unblank(); 90 oops_in_progress = 0; 91 /* 92 * OK, the message is on the console. Now we call printk() 93 * without oops_in_progress set so that printk will give klogd 94 * a poke. Hold onto your hats... 95 */ 96 console_loglevel = 15; 97 printk(" "); 98 console_loglevel = loglevel_save; 99 } 100 } 101 102 /* 103 * Returns the address space associated with the fault. 104 * Returns 0 for kernel space and 1 for user space. 105 */ 106 static inline int user_space_fault(unsigned long trans_exc_code) 107 { 108 /* 109 * The lowest two bits of the translation exception 110 * identification indicate which paging table was used. 111 */ 112 trans_exc_code &= 3; 113 if (trans_exc_code == 2) 114 /* Access via secondary space, set_fs setting decides */ 115 return current->thread.mm_segment.ar4; 116 if (user_mode == HOME_SPACE_MODE) 117 /* User space if the access has been done via home space. */ 118 return trans_exc_code == 3; 119 /* 120 * If the user space is not the home space the kernel runs in home 121 * space. Access via secondary space has already been covered, 122 * access via primary space or access register is from user space 123 * and access via home space is from the kernel. 124 */ 125 return trans_exc_code != 3; 126 } 127 128 static inline void report_user_fault(struct pt_regs *regs, long int_code, 129 int signr, unsigned long address) 130 { 131 if ((task_pid_nr(current) > 1) && !show_unhandled_signals) 132 return; 133 if (!unhandled_signal(current, signr)) 134 return; 135 if (!printk_ratelimit()) 136 return; 137 printk("User process fault: interruption code 0x%lX ", int_code); 138 print_vma_addr(KERN_CONT "in ", regs->psw.addr & PSW_ADDR_INSN); 139 printk("\n"); 140 printk("failing address: %lX\n", address); 141 show_regs(regs); 142 } 143 144 /* 145 * Send SIGSEGV to task. This is an external routine 146 * to keep the stack usage of do_page_fault small. 147 */ 148 static noinline void do_sigsegv(struct pt_regs *regs, long int_code, 149 int si_code, unsigned long trans_exc_code) 150 { 151 struct siginfo si; 152 unsigned long address; 153 154 address = trans_exc_code & __FAIL_ADDR_MASK; 155 current->thread.prot_addr = address; 156 current->thread.trap_no = int_code; 157 report_user_fault(regs, int_code, SIGSEGV, address); 158 si.si_signo = SIGSEGV; 159 si.si_code = si_code; 160 si.si_addr = (void __user *) address; 161 force_sig_info(SIGSEGV, &si, current); 162 } 163 164 static noinline void do_no_context(struct pt_regs *regs, long int_code, 165 unsigned long trans_exc_code) 166 { 167 const struct exception_table_entry *fixup; 168 unsigned long address; 169 170 /* Are we prepared to handle this kernel fault? */ 171 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN); 172 if (fixup) { 173 regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE; 174 return; 175 } 176 177 /* 178 * Oops. The kernel tried to access some bad page. We'll have to 179 * terminate things with extreme prejudice. 180 */ 181 address = trans_exc_code & __FAIL_ADDR_MASK; 182 if (!user_space_fault(trans_exc_code)) 183 printk(KERN_ALERT "Unable to handle kernel pointer dereference" 184 " at virtual kernel address %p\n", (void *)address); 185 else 186 printk(KERN_ALERT "Unable to handle kernel paging request" 187 " at virtual user address %p\n", (void *)address); 188 189 die("Oops", regs, int_code); 190 do_exit(SIGKILL); 191 } 192 193 static noinline void do_low_address(struct pt_regs *regs, long int_code, 194 unsigned long trans_exc_code) 195 { 196 /* Low-address protection hit in kernel mode means 197 NULL pointer write access in kernel mode. */ 198 if (regs->psw.mask & PSW_MASK_PSTATE) { 199 /* Low-address protection hit in user mode 'cannot happen'. */ 200 die ("Low-address protection", regs, int_code); 201 do_exit(SIGKILL); 202 } 203 204 do_no_context(regs, int_code, trans_exc_code); 205 } 206 207 static noinline void do_sigbus(struct pt_regs *regs, long int_code, 208 unsigned long trans_exc_code) 209 { 210 struct task_struct *tsk = current; 211 unsigned long address; 212 struct siginfo si; 213 214 /* 215 * Send a sigbus, regardless of whether we were in kernel 216 * or user mode. 217 */ 218 address = trans_exc_code & __FAIL_ADDR_MASK; 219 tsk->thread.prot_addr = address; 220 tsk->thread.trap_no = int_code; 221 si.si_signo = SIGBUS; 222 si.si_errno = 0; 223 si.si_code = BUS_ADRERR; 224 si.si_addr = (void __user *) address; 225 force_sig_info(SIGBUS, &si, tsk); 226 } 227 228 static noinline void do_fault_error(struct pt_regs *regs, long int_code, 229 unsigned long trans_exc_code, int fault) 230 { 231 int si_code; 232 233 switch (fault) { 234 case VM_FAULT_BADACCESS: 235 case VM_FAULT_BADMAP: 236 /* Bad memory access. Check if it is kernel or user space. */ 237 if (regs->psw.mask & PSW_MASK_PSTATE) { 238 /* User mode accesses just cause a SIGSEGV */ 239 si_code = (fault == VM_FAULT_BADMAP) ? 240 SEGV_MAPERR : SEGV_ACCERR; 241 do_sigsegv(regs, int_code, si_code, trans_exc_code); 242 return; 243 } 244 case VM_FAULT_BADCONTEXT: 245 do_no_context(regs, int_code, trans_exc_code); 246 break; 247 default: /* fault & VM_FAULT_ERROR */ 248 if (fault & VM_FAULT_OOM) { 249 if (!(regs->psw.mask & PSW_MASK_PSTATE)) 250 do_no_context(regs, int_code, trans_exc_code); 251 else 252 pagefault_out_of_memory(); 253 } else if (fault & VM_FAULT_SIGBUS) { 254 /* Kernel mode? Handle exceptions or die */ 255 if (!(regs->psw.mask & PSW_MASK_PSTATE)) 256 do_no_context(regs, int_code, trans_exc_code); 257 else 258 do_sigbus(regs, int_code, trans_exc_code); 259 } else 260 BUG(); 261 break; 262 } 263 } 264 265 /* 266 * This routine handles page faults. It determines the address, 267 * and the problem, and then passes it off to one of the appropriate 268 * routines. 269 * 270 * interruption code (int_code): 271 * 04 Protection -> Write-Protection (suprression) 272 * 10 Segment translation -> Not present (nullification) 273 * 11 Page translation -> Not present (nullification) 274 * 3b Region third trans. -> Not present (nullification) 275 */ 276 static inline int do_exception(struct pt_regs *regs, int access, 277 unsigned long trans_exc_code) 278 { 279 struct task_struct *tsk; 280 struct mm_struct *mm; 281 struct vm_area_struct *vma; 282 unsigned long address; 283 unsigned int flags; 284 int fault; 285 286 if (notify_page_fault(regs)) 287 return 0; 288 289 tsk = current; 290 mm = tsk->mm; 291 292 /* 293 * Verify that the fault happened in user space, that 294 * we are not in an interrupt and that there is a 295 * user context. 296 */ 297 fault = VM_FAULT_BADCONTEXT; 298 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 299 goto out; 300 301 address = trans_exc_code & __FAIL_ADDR_MASK; 302 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 303 flags = FAULT_FLAG_ALLOW_RETRY; 304 if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400) 305 flags |= FAULT_FLAG_WRITE; 306 down_read(&mm->mmap_sem); 307 308 #ifdef CONFIG_PGSTE 309 if (test_tsk_thread_flag(current, TIF_SIE) && S390_lowcore.gmap) { 310 address = gmap_fault(address, 311 (struct gmap *) S390_lowcore.gmap); 312 if (address == -EFAULT) { 313 fault = VM_FAULT_BADMAP; 314 goto out_up; 315 } 316 if (address == -ENOMEM) { 317 fault = VM_FAULT_OOM; 318 goto out_up; 319 } 320 } 321 #endif 322 323 retry: 324 fault = VM_FAULT_BADMAP; 325 vma = find_vma(mm, address); 326 if (!vma) 327 goto out_up; 328 329 if (unlikely(vma->vm_start > address)) { 330 if (!(vma->vm_flags & VM_GROWSDOWN)) 331 goto out_up; 332 if (expand_stack(vma, address)) 333 goto out_up; 334 } 335 336 /* 337 * Ok, we have a good vm_area for this memory access, so 338 * we can handle it.. 339 */ 340 fault = VM_FAULT_BADACCESS; 341 if (unlikely(!(vma->vm_flags & access))) 342 goto out_up; 343 344 if (is_vm_hugetlb_page(vma)) 345 address &= HPAGE_MASK; 346 /* 347 * If for any reason at all we couldn't handle the fault, 348 * make sure we exit gracefully rather than endlessly redo 349 * the fault. 350 */ 351 fault = handle_mm_fault(mm, vma, address, flags); 352 if (unlikely(fault & VM_FAULT_ERROR)) 353 goto out_up; 354 355 /* 356 * Major/minor page fault accounting is only done on the 357 * initial attempt. If we go through a retry, it is extremely 358 * likely that the page will be found in page cache at that point. 359 */ 360 if (flags & FAULT_FLAG_ALLOW_RETRY) { 361 if (fault & VM_FAULT_MAJOR) { 362 tsk->maj_flt++; 363 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 364 regs, address); 365 } else { 366 tsk->min_flt++; 367 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 368 regs, address); 369 } 370 if (fault & VM_FAULT_RETRY) { 371 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 372 * of starvation. */ 373 flags &= ~FAULT_FLAG_ALLOW_RETRY; 374 down_read(&mm->mmap_sem); 375 goto retry; 376 } 377 } 378 /* 379 * The instruction that caused the program check will 380 * be repeated. Don't signal single step via SIGTRAP. 381 */ 382 clear_tsk_thread_flag(tsk, TIF_PER_TRAP); 383 fault = 0; 384 out_up: 385 up_read(&mm->mmap_sem); 386 out: 387 return fault; 388 } 389 390 void __kprobes do_protection_exception(struct pt_regs *regs, long pgm_int_code, 391 unsigned long trans_exc_code) 392 { 393 int fault; 394 395 /* Protection exception is suppressing, decrement psw address. */ 396 regs->psw.addr -= (pgm_int_code >> 16); 397 /* 398 * Check for low-address protection. This needs to be treated 399 * as a special case because the translation exception code 400 * field is not guaranteed to contain valid data in this case. 401 */ 402 if (unlikely(!(trans_exc_code & 4))) { 403 do_low_address(regs, pgm_int_code, trans_exc_code); 404 return; 405 } 406 fault = do_exception(regs, VM_WRITE, trans_exc_code); 407 if (unlikely(fault)) 408 do_fault_error(regs, 4, trans_exc_code, fault); 409 } 410 411 void __kprobes do_dat_exception(struct pt_regs *regs, long pgm_int_code, 412 unsigned long trans_exc_code) 413 { 414 int access, fault; 415 416 access = VM_READ | VM_EXEC | VM_WRITE; 417 fault = do_exception(regs, access, trans_exc_code); 418 if (unlikely(fault)) 419 do_fault_error(regs, pgm_int_code & 255, trans_exc_code, fault); 420 } 421 422 #ifdef CONFIG_64BIT 423 void __kprobes do_asce_exception(struct pt_regs *regs, long pgm_int_code, 424 unsigned long trans_exc_code) 425 { 426 struct mm_struct *mm = current->mm; 427 struct vm_area_struct *vma; 428 429 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 430 goto no_context; 431 432 down_read(&mm->mmap_sem); 433 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK); 434 up_read(&mm->mmap_sem); 435 436 if (vma) { 437 update_mm(mm, current); 438 return; 439 } 440 441 /* User mode accesses just cause a SIGSEGV */ 442 if (regs->psw.mask & PSW_MASK_PSTATE) { 443 do_sigsegv(regs, pgm_int_code, SEGV_MAPERR, trans_exc_code); 444 return; 445 } 446 447 no_context: 448 do_no_context(regs, pgm_int_code, trans_exc_code); 449 } 450 #endif 451 452 int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write) 453 { 454 struct pt_regs regs; 455 int access, fault; 456 457 regs.psw.mask = psw_kernel_bits; 458 if (!irqs_disabled()) 459 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT; 460 regs.psw.addr = (unsigned long) __builtin_return_address(0); 461 regs.psw.addr |= PSW_ADDR_AMODE; 462 uaddr &= PAGE_MASK; 463 access = write ? VM_WRITE : VM_READ; 464 fault = do_exception(®s, access, uaddr | 2); 465 if (unlikely(fault)) { 466 if (fault & VM_FAULT_OOM) 467 return -EFAULT; 468 else if (fault & VM_FAULT_SIGBUS) 469 do_sigbus(®s, pgm_int_code, uaddr); 470 } 471 return fault ? -EFAULT : 0; 472 } 473 474 #ifdef CONFIG_PFAULT 475 /* 476 * 'pfault' pseudo page faults routines. 477 */ 478 static int pfault_disable; 479 480 static int __init nopfault(char *str) 481 { 482 pfault_disable = 1; 483 return 1; 484 } 485 486 __setup("nopfault", nopfault); 487 488 struct pfault_refbk { 489 u16 refdiagc; 490 u16 reffcode; 491 u16 refdwlen; 492 u16 refversn; 493 u64 refgaddr; 494 u64 refselmk; 495 u64 refcmpmk; 496 u64 reserved; 497 } __attribute__ ((packed, aligned(8))); 498 499 int pfault_init(void) 500 { 501 struct pfault_refbk refbk = { 502 .refdiagc = 0x258, 503 .reffcode = 0, 504 .refdwlen = 5, 505 .refversn = 2, 506 .refgaddr = __LC_CURRENT_PID, 507 .refselmk = 1ULL << 48, 508 .refcmpmk = 1ULL << 48, 509 .reserved = __PF_RES_FIELD }; 510 int rc; 511 512 if (!MACHINE_IS_VM || pfault_disable) 513 return -1; 514 asm volatile( 515 " diag %1,%0,0x258\n" 516 "0: j 2f\n" 517 "1: la %0,8\n" 518 "2:\n" 519 EX_TABLE(0b,1b) 520 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc"); 521 return rc; 522 } 523 524 void pfault_fini(void) 525 { 526 struct pfault_refbk refbk = { 527 .refdiagc = 0x258, 528 .reffcode = 1, 529 .refdwlen = 5, 530 .refversn = 2, 531 }; 532 533 if (!MACHINE_IS_VM || pfault_disable) 534 return; 535 asm volatile( 536 " diag %0,0,0x258\n" 537 "0:\n" 538 EX_TABLE(0b,0b) 539 : : "a" (&refbk), "m" (refbk) : "cc"); 540 } 541 542 static DEFINE_SPINLOCK(pfault_lock); 543 static LIST_HEAD(pfault_list); 544 545 static void pfault_interrupt(unsigned int ext_int_code, 546 unsigned int param32, unsigned long param64) 547 { 548 struct task_struct *tsk; 549 __u16 subcode; 550 pid_t pid; 551 552 /* 553 * Get the external interruption subcode & pfault 554 * initial/completion signal bit. VM stores this 555 * in the 'cpu address' field associated with the 556 * external interrupt. 557 */ 558 subcode = ext_int_code >> 16; 559 if ((subcode & 0xff00) != __SUBCODE_MASK) 560 return; 561 kstat_cpu(smp_processor_id()).irqs[EXTINT_PFL]++; 562 if (subcode & 0x0080) { 563 /* Get the token (= pid of the affected task). */ 564 pid = sizeof(void *) == 4 ? param32 : param64; 565 rcu_read_lock(); 566 tsk = find_task_by_pid_ns(pid, &init_pid_ns); 567 if (tsk) 568 get_task_struct(tsk); 569 rcu_read_unlock(); 570 if (!tsk) 571 return; 572 } else { 573 tsk = current; 574 } 575 spin_lock(&pfault_lock); 576 if (subcode & 0x0080) { 577 /* signal bit is set -> a page has been swapped in by VM */ 578 if (tsk->thread.pfault_wait == 1) { 579 /* Initial interrupt was faster than the completion 580 * interrupt. pfault_wait is valid. Set pfault_wait 581 * back to zero and wake up the process. This can 582 * safely be done because the task is still sleeping 583 * and can't produce new pfaults. */ 584 tsk->thread.pfault_wait = 0; 585 list_del(&tsk->thread.list); 586 wake_up_process(tsk); 587 } else { 588 /* Completion interrupt was faster than initial 589 * interrupt. Set pfault_wait to -1 so the initial 590 * interrupt doesn't put the task to sleep. */ 591 tsk->thread.pfault_wait = -1; 592 } 593 put_task_struct(tsk); 594 } else { 595 /* signal bit not set -> a real page is missing. */ 596 if (tsk->thread.pfault_wait == -1) { 597 /* Completion interrupt was faster than the initial 598 * interrupt (pfault_wait == -1). Set pfault_wait 599 * back to zero and exit. */ 600 tsk->thread.pfault_wait = 0; 601 } else { 602 /* Initial interrupt arrived before completion 603 * interrupt. Let the task sleep. */ 604 tsk->thread.pfault_wait = 1; 605 list_add(&tsk->thread.list, &pfault_list); 606 set_task_state(tsk, TASK_UNINTERRUPTIBLE); 607 set_tsk_need_resched(tsk); 608 } 609 } 610 spin_unlock(&pfault_lock); 611 } 612 613 static int __cpuinit pfault_cpu_notify(struct notifier_block *self, 614 unsigned long action, void *hcpu) 615 { 616 struct thread_struct *thread, *next; 617 struct task_struct *tsk; 618 619 switch (action) { 620 case CPU_DEAD: 621 case CPU_DEAD_FROZEN: 622 spin_lock_irq(&pfault_lock); 623 list_for_each_entry_safe(thread, next, &pfault_list, list) { 624 thread->pfault_wait = 0; 625 list_del(&thread->list); 626 tsk = container_of(thread, struct task_struct, thread); 627 wake_up_process(tsk); 628 } 629 spin_unlock_irq(&pfault_lock); 630 break; 631 default: 632 break; 633 } 634 return NOTIFY_OK; 635 } 636 637 static int __init pfault_irq_init(void) 638 { 639 int rc; 640 641 if (!MACHINE_IS_VM) 642 return 0; 643 rc = register_external_interrupt(0x2603, pfault_interrupt); 644 if (rc) 645 goto out_extint; 646 rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP; 647 if (rc) 648 goto out_pfault; 649 service_subclass_irq_register(); 650 hotcpu_notifier(pfault_cpu_notify, 0); 651 return 0; 652 653 out_pfault: 654 unregister_external_interrupt(0x2603, pfault_interrupt); 655 out_extint: 656 pfault_disable = 1; 657 return rc; 658 } 659 early_initcall(pfault_irq_init); 660 661 #endif /* CONFIG_PFAULT */ 662