1 /* 2 * S390 version 3 * Copyright IBM Corp. 1999 4 * Author(s): Hartmut Penner (hp@de.ibm.com) 5 * Ulrich Weigand (uweigand@de.ibm.com) 6 * 7 * Derived from "arch/i386/mm/fault.c" 8 * Copyright (C) 1995 Linus Torvalds 9 */ 10 11 #include <linux/kernel_stat.h> 12 #include <linux/perf_event.h> 13 #include <linux/signal.h> 14 #include <linux/sched.h> 15 #include <linux/kernel.h> 16 #include <linux/errno.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 #include <linux/ptrace.h> 20 #include <linux/mman.h> 21 #include <linux/mm.h> 22 #include <linux/compat.h> 23 #include <linux/smp.h> 24 #include <linux/kdebug.h> 25 #include <linux/init.h> 26 #include <linux/console.h> 27 #include <linux/module.h> 28 #include <linux/hardirq.h> 29 #include <linux/kprobes.h> 30 #include <linux/uaccess.h> 31 #include <linux/hugetlb.h> 32 #include <asm/asm-offsets.h> 33 #include <asm/pgtable.h> 34 #include <asm/irq.h> 35 #include <asm/mmu_context.h> 36 #include <asm/facility.h> 37 #include "../kernel/entry.h" 38 39 #ifndef CONFIG_64BIT 40 #define __FAIL_ADDR_MASK 0x7ffff000 41 #define __SUBCODE_MASK 0x0200 42 #define __PF_RES_FIELD 0ULL 43 #else /* CONFIG_64BIT */ 44 #define __FAIL_ADDR_MASK -4096L 45 #define __SUBCODE_MASK 0x0600 46 #define __PF_RES_FIELD 0x8000000000000000ULL 47 #endif /* CONFIG_64BIT */ 48 49 #define VM_FAULT_BADCONTEXT 0x010000 50 #define VM_FAULT_BADMAP 0x020000 51 #define VM_FAULT_BADACCESS 0x040000 52 #define VM_FAULT_SIGNAL 0x080000 53 54 static unsigned long store_indication __read_mostly; 55 56 #ifdef CONFIG_64BIT 57 static int __init fault_init(void) 58 { 59 if (test_facility(75)) 60 store_indication = 0xc00; 61 return 0; 62 } 63 early_initcall(fault_init); 64 #endif 65 66 static inline int notify_page_fault(struct pt_regs *regs) 67 { 68 int ret = 0; 69 70 /* kprobe_running() needs smp_processor_id() */ 71 if (kprobes_built_in() && !user_mode(regs)) { 72 preempt_disable(); 73 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 74 ret = 1; 75 preempt_enable(); 76 } 77 return ret; 78 } 79 80 81 /* 82 * Unlock any spinlocks which will prevent us from getting the 83 * message out. 84 */ 85 void bust_spinlocks(int yes) 86 { 87 if (yes) { 88 oops_in_progress = 1; 89 } else { 90 int loglevel_save = console_loglevel; 91 console_unblank(); 92 oops_in_progress = 0; 93 /* 94 * OK, the message is on the console. Now we call printk() 95 * without oops_in_progress set so that printk will give klogd 96 * a poke. Hold onto your hats... 97 */ 98 console_loglevel = 15; 99 printk(" "); 100 console_loglevel = loglevel_save; 101 } 102 } 103 104 /* 105 * Returns the address space associated with the fault. 106 * Returns 0 for kernel space and 1 for user space. 107 */ 108 static inline int user_space_fault(unsigned long trans_exc_code) 109 { 110 /* 111 * The lowest two bits of the translation exception 112 * identification indicate which paging table was used. 113 */ 114 trans_exc_code &= 3; 115 if (trans_exc_code == 2) 116 /* Access via secondary space, set_fs setting decides */ 117 return current->thread.mm_segment.ar4; 118 if (s390_user_mode == HOME_SPACE_MODE) 119 /* User space if the access has been done via home space. */ 120 return trans_exc_code == 3; 121 /* 122 * If the user space is not the home space the kernel runs in home 123 * space. Access via secondary space has already been covered, 124 * access via primary space or access register is from user space 125 * and access via home space is from the kernel. 126 */ 127 return trans_exc_code != 3; 128 } 129 130 static inline void report_user_fault(struct pt_regs *regs, long signr) 131 { 132 if ((task_pid_nr(current) > 1) && !show_unhandled_signals) 133 return; 134 if (!unhandled_signal(current, signr)) 135 return; 136 if (!printk_ratelimit()) 137 return; 138 printk(KERN_ALERT "User process fault: interruption code 0x%X ", 139 regs->int_code); 140 print_vma_addr(KERN_CONT "in ", regs->psw.addr & PSW_ADDR_INSN); 141 printk(KERN_CONT "\n"); 142 printk(KERN_ALERT "failing address: %lX\n", 143 regs->int_parm_long & __FAIL_ADDR_MASK); 144 show_regs(regs); 145 } 146 147 /* 148 * Send SIGSEGV to task. This is an external routine 149 * to keep the stack usage of do_page_fault small. 150 */ 151 static noinline void do_sigsegv(struct pt_regs *regs, int si_code) 152 { 153 struct siginfo si; 154 155 report_user_fault(regs, SIGSEGV); 156 si.si_signo = SIGSEGV; 157 si.si_code = si_code; 158 si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK); 159 force_sig_info(SIGSEGV, &si, current); 160 } 161 162 static noinline void do_no_context(struct pt_regs *regs) 163 { 164 const struct exception_table_entry *fixup; 165 unsigned long address; 166 167 /* Are we prepared to handle this kernel fault? */ 168 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN); 169 if (fixup) { 170 regs->psw.addr = extable_fixup(fixup) | PSW_ADDR_AMODE; 171 return; 172 } 173 174 /* 175 * Oops. The kernel tried to access some bad page. We'll have to 176 * terminate things with extreme prejudice. 177 */ 178 address = regs->int_parm_long & __FAIL_ADDR_MASK; 179 if (!user_space_fault(regs->int_parm_long)) 180 printk(KERN_ALERT "Unable to handle kernel pointer dereference" 181 " at virtual kernel address %p\n", (void *)address); 182 else 183 printk(KERN_ALERT "Unable to handle kernel paging request" 184 " at virtual user address %p\n", (void *)address); 185 186 die(regs, "Oops"); 187 do_exit(SIGKILL); 188 } 189 190 static noinline void do_low_address(struct pt_regs *regs) 191 { 192 /* Low-address protection hit in kernel mode means 193 NULL pointer write access in kernel mode. */ 194 if (regs->psw.mask & PSW_MASK_PSTATE) { 195 /* Low-address protection hit in user mode 'cannot happen'. */ 196 die (regs, "Low-address protection"); 197 do_exit(SIGKILL); 198 } 199 200 do_no_context(regs); 201 } 202 203 static noinline void do_sigbus(struct pt_regs *regs) 204 { 205 struct task_struct *tsk = current; 206 struct siginfo si; 207 208 /* 209 * Send a sigbus, regardless of whether we were in kernel 210 * or user mode. 211 */ 212 si.si_signo = SIGBUS; 213 si.si_errno = 0; 214 si.si_code = BUS_ADRERR; 215 si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK); 216 force_sig_info(SIGBUS, &si, tsk); 217 } 218 219 static noinline void do_fault_error(struct pt_regs *regs, int fault) 220 { 221 int si_code; 222 223 switch (fault) { 224 case VM_FAULT_BADACCESS: 225 case VM_FAULT_BADMAP: 226 /* Bad memory access. Check if it is kernel or user space. */ 227 if (user_mode(regs)) { 228 /* User mode accesses just cause a SIGSEGV */ 229 si_code = (fault == VM_FAULT_BADMAP) ? 230 SEGV_MAPERR : SEGV_ACCERR; 231 do_sigsegv(regs, si_code); 232 return; 233 } 234 case VM_FAULT_BADCONTEXT: 235 do_no_context(regs); 236 break; 237 case VM_FAULT_SIGNAL: 238 if (!user_mode(regs)) 239 do_no_context(regs); 240 break; 241 default: /* fault & VM_FAULT_ERROR */ 242 if (fault & VM_FAULT_OOM) { 243 if (!user_mode(regs)) 244 do_no_context(regs); 245 else 246 pagefault_out_of_memory(); 247 } else if (fault & VM_FAULT_SIGBUS) { 248 /* Kernel mode? Handle exceptions or die */ 249 if (!user_mode(regs)) 250 do_no_context(regs); 251 else 252 do_sigbus(regs); 253 } else 254 BUG(); 255 break; 256 } 257 } 258 259 /* 260 * This routine handles page faults. It determines the address, 261 * and the problem, and then passes it off to one of the appropriate 262 * routines. 263 * 264 * interruption code (int_code): 265 * 04 Protection -> Write-Protection (suprression) 266 * 10 Segment translation -> Not present (nullification) 267 * 11 Page translation -> Not present (nullification) 268 * 3b Region third trans. -> Not present (nullification) 269 */ 270 static inline int do_exception(struct pt_regs *regs, int access) 271 { 272 struct task_struct *tsk; 273 struct mm_struct *mm; 274 struct vm_area_struct *vma; 275 unsigned long trans_exc_code; 276 unsigned long address; 277 unsigned int flags; 278 int fault; 279 280 tsk = current; 281 /* 282 * The instruction that caused the program check has 283 * been nullified. Don't signal single step via SIGTRAP. 284 */ 285 clear_tsk_thread_flag(tsk, TIF_PER_TRAP); 286 287 if (notify_page_fault(regs)) 288 return 0; 289 290 mm = tsk->mm; 291 trans_exc_code = regs->int_parm_long; 292 293 /* 294 * Verify that the fault happened in user space, that 295 * we are not in an interrupt and that there is a 296 * user context. 297 */ 298 fault = VM_FAULT_BADCONTEXT; 299 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 300 goto out; 301 302 address = trans_exc_code & __FAIL_ADDR_MASK; 303 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 304 flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 305 if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400) 306 flags |= FAULT_FLAG_WRITE; 307 down_read(&mm->mmap_sem); 308 309 #ifdef CONFIG_PGSTE 310 if ((current->flags & PF_VCPU) && S390_lowcore.gmap) { 311 address = __gmap_fault(address, 312 (struct gmap *) S390_lowcore.gmap); 313 if (address == -EFAULT) { 314 fault = VM_FAULT_BADMAP; 315 goto out_up; 316 } 317 if (address == -ENOMEM) { 318 fault = VM_FAULT_OOM; 319 goto out_up; 320 } 321 } 322 #endif 323 324 retry: 325 fault = VM_FAULT_BADMAP; 326 vma = find_vma(mm, address); 327 if (!vma) 328 goto out_up; 329 330 if (unlikely(vma->vm_start > address)) { 331 if (!(vma->vm_flags & VM_GROWSDOWN)) 332 goto out_up; 333 if (expand_stack(vma, address)) 334 goto out_up; 335 } 336 337 /* 338 * Ok, we have a good vm_area for this memory access, so 339 * we can handle it.. 340 */ 341 fault = VM_FAULT_BADACCESS; 342 if (unlikely(!(vma->vm_flags & access))) 343 goto out_up; 344 345 if (is_vm_hugetlb_page(vma)) 346 address &= HPAGE_MASK; 347 /* 348 * If for any reason at all we couldn't handle the fault, 349 * make sure we exit gracefully rather than endlessly redo 350 * the fault. 351 */ 352 fault = handle_mm_fault(mm, vma, address, flags); 353 /* No reason to continue if interrupted by SIGKILL. */ 354 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { 355 fault = VM_FAULT_SIGNAL; 356 goto out; 357 } 358 if (unlikely(fault & VM_FAULT_ERROR)) 359 goto out_up; 360 361 /* 362 * Major/minor page fault accounting is only done on the 363 * initial attempt. If we go through a retry, it is extremely 364 * likely that the page will be found in page cache at that point. 365 */ 366 if (flags & FAULT_FLAG_ALLOW_RETRY) { 367 if (fault & VM_FAULT_MAJOR) { 368 tsk->maj_flt++; 369 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 370 regs, address); 371 } else { 372 tsk->min_flt++; 373 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 374 regs, address); 375 } 376 if (fault & VM_FAULT_RETRY) { 377 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 378 * of starvation. */ 379 flags &= ~FAULT_FLAG_ALLOW_RETRY; 380 flags |= FAULT_FLAG_TRIED; 381 down_read(&mm->mmap_sem); 382 goto retry; 383 } 384 } 385 fault = 0; 386 out_up: 387 up_read(&mm->mmap_sem); 388 out: 389 return fault; 390 } 391 392 void __kprobes do_protection_exception(struct pt_regs *regs) 393 { 394 unsigned long trans_exc_code; 395 int fault; 396 397 trans_exc_code = regs->int_parm_long; 398 /* Protection exception is suppressing, decrement psw address. */ 399 regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16); 400 /* 401 * Check for low-address protection. This needs to be treated 402 * as a special case because the translation exception code 403 * field is not guaranteed to contain valid data in this case. 404 */ 405 if (unlikely(!(trans_exc_code & 4))) { 406 do_low_address(regs); 407 return; 408 } 409 fault = do_exception(regs, VM_WRITE); 410 if (unlikely(fault)) 411 do_fault_error(regs, fault); 412 } 413 414 void __kprobes do_dat_exception(struct pt_regs *regs) 415 { 416 int access, fault; 417 418 access = VM_READ | VM_EXEC | VM_WRITE; 419 fault = do_exception(regs, access); 420 if (unlikely(fault)) 421 do_fault_error(regs, fault); 422 } 423 424 #ifdef CONFIG_64BIT 425 void __kprobes do_asce_exception(struct pt_regs *regs) 426 { 427 struct mm_struct *mm = current->mm; 428 struct vm_area_struct *vma; 429 unsigned long trans_exc_code; 430 431 /* 432 * The instruction that caused the program check has 433 * been nullified. Don't signal single step via SIGTRAP. 434 */ 435 clear_tsk_thread_flag(current, TIF_PER_TRAP); 436 437 trans_exc_code = regs->int_parm_long; 438 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 439 goto no_context; 440 441 down_read(&mm->mmap_sem); 442 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK); 443 up_read(&mm->mmap_sem); 444 445 if (vma) { 446 update_mm(mm, current); 447 return; 448 } 449 450 /* User mode accesses just cause a SIGSEGV */ 451 if (user_mode(regs)) { 452 do_sigsegv(regs, SEGV_MAPERR); 453 return; 454 } 455 456 no_context: 457 do_no_context(regs); 458 } 459 #endif 460 461 int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write) 462 { 463 struct pt_regs regs; 464 int access, fault; 465 466 /* Emulate a uaccess fault from kernel mode. */ 467 regs.psw.mask = psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_MCHECK; 468 if (!irqs_disabled()) 469 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT; 470 regs.psw.addr = (unsigned long) __builtin_return_address(0); 471 regs.psw.addr |= PSW_ADDR_AMODE; 472 regs.int_code = pgm_int_code; 473 regs.int_parm_long = (uaddr & PAGE_MASK) | 2; 474 access = write ? VM_WRITE : VM_READ; 475 fault = do_exception(®s, access); 476 /* 477 * Since the fault happened in kernel mode while performing a uaccess 478 * all we need to do now is emulating a fixup in case "fault" is not 479 * zero. 480 * For the calling uaccess functions this results always in -EFAULT. 481 */ 482 return fault ? -EFAULT : 0; 483 } 484 485 #ifdef CONFIG_PFAULT 486 /* 487 * 'pfault' pseudo page faults routines. 488 */ 489 static int pfault_disable; 490 491 static int __init nopfault(char *str) 492 { 493 pfault_disable = 1; 494 return 1; 495 } 496 497 __setup("nopfault", nopfault); 498 499 struct pfault_refbk { 500 u16 refdiagc; 501 u16 reffcode; 502 u16 refdwlen; 503 u16 refversn; 504 u64 refgaddr; 505 u64 refselmk; 506 u64 refcmpmk; 507 u64 reserved; 508 } __attribute__ ((packed, aligned(8))); 509 510 int pfault_init(void) 511 { 512 struct pfault_refbk refbk = { 513 .refdiagc = 0x258, 514 .reffcode = 0, 515 .refdwlen = 5, 516 .refversn = 2, 517 .refgaddr = __LC_CURRENT_PID, 518 .refselmk = 1ULL << 48, 519 .refcmpmk = 1ULL << 48, 520 .reserved = __PF_RES_FIELD }; 521 int rc; 522 523 if (pfault_disable) 524 return -1; 525 asm volatile( 526 " diag %1,%0,0x258\n" 527 "0: j 2f\n" 528 "1: la %0,8\n" 529 "2:\n" 530 EX_TABLE(0b,1b) 531 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc"); 532 return rc; 533 } 534 535 void pfault_fini(void) 536 { 537 struct pfault_refbk refbk = { 538 .refdiagc = 0x258, 539 .reffcode = 1, 540 .refdwlen = 5, 541 .refversn = 2, 542 }; 543 544 if (pfault_disable) 545 return; 546 asm volatile( 547 " diag %0,0,0x258\n" 548 "0:\n" 549 EX_TABLE(0b,0b) 550 : : "a" (&refbk), "m" (refbk) : "cc"); 551 } 552 553 static DEFINE_SPINLOCK(pfault_lock); 554 static LIST_HEAD(pfault_list); 555 556 static void pfault_interrupt(struct ext_code ext_code, 557 unsigned int param32, unsigned long param64) 558 { 559 struct task_struct *tsk; 560 __u16 subcode; 561 pid_t pid; 562 563 /* 564 * Get the external interruption subcode & pfault 565 * initial/completion signal bit. VM stores this 566 * in the 'cpu address' field associated with the 567 * external interrupt. 568 */ 569 subcode = ext_code.subcode; 570 if ((subcode & 0xff00) != __SUBCODE_MASK) 571 return; 572 inc_irq_stat(IRQEXT_PFL); 573 /* Get the token (= pid of the affected task). */ 574 pid = sizeof(void *) == 4 ? param32 : param64; 575 rcu_read_lock(); 576 tsk = find_task_by_pid_ns(pid, &init_pid_ns); 577 if (tsk) 578 get_task_struct(tsk); 579 rcu_read_unlock(); 580 if (!tsk) 581 return; 582 spin_lock(&pfault_lock); 583 if (subcode & 0x0080) { 584 /* signal bit is set -> a page has been swapped in by VM */ 585 if (tsk->thread.pfault_wait == 1) { 586 /* Initial interrupt was faster than the completion 587 * interrupt. pfault_wait is valid. Set pfault_wait 588 * back to zero and wake up the process. This can 589 * safely be done because the task is still sleeping 590 * and can't produce new pfaults. */ 591 tsk->thread.pfault_wait = 0; 592 list_del(&tsk->thread.list); 593 wake_up_process(tsk); 594 put_task_struct(tsk); 595 } else { 596 /* Completion interrupt was faster than initial 597 * interrupt. Set pfault_wait to -1 so the initial 598 * interrupt doesn't put the task to sleep. 599 * If the task is not running, ignore the completion 600 * interrupt since it must be a leftover of a PFAULT 601 * CANCEL operation which didn't remove all pending 602 * completion interrupts. */ 603 if (tsk->state == TASK_RUNNING) 604 tsk->thread.pfault_wait = -1; 605 } 606 } else { 607 /* signal bit not set -> a real page is missing. */ 608 if (WARN_ON_ONCE(tsk != current)) 609 goto out; 610 if (tsk->thread.pfault_wait == 1) { 611 /* Already on the list with a reference: put to sleep */ 612 __set_task_state(tsk, TASK_UNINTERRUPTIBLE); 613 set_tsk_need_resched(tsk); 614 } else if (tsk->thread.pfault_wait == -1) { 615 /* Completion interrupt was faster than the initial 616 * interrupt (pfault_wait == -1). Set pfault_wait 617 * back to zero and exit. */ 618 tsk->thread.pfault_wait = 0; 619 } else { 620 /* Initial interrupt arrived before completion 621 * interrupt. Let the task sleep. 622 * An extra task reference is needed since a different 623 * cpu may set the task state to TASK_RUNNING again 624 * before the scheduler is reached. */ 625 get_task_struct(tsk); 626 tsk->thread.pfault_wait = 1; 627 list_add(&tsk->thread.list, &pfault_list); 628 __set_task_state(tsk, TASK_UNINTERRUPTIBLE); 629 set_tsk_need_resched(tsk); 630 } 631 } 632 out: 633 spin_unlock(&pfault_lock); 634 put_task_struct(tsk); 635 } 636 637 static int __cpuinit pfault_cpu_notify(struct notifier_block *self, 638 unsigned long action, void *hcpu) 639 { 640 struct thread_struct *thread, *next; 641 struct task_struct *tsk; 642 643 switch (action & ~CPU_TASKS_FROZEN) { 644 case CPU_DEAD: 645 spin_lock_irq(&pfault_lock); 646 list_for_each_entry_safe(thread, next, &pfault_list, list) { 647 thread->pfault_wait = 0; 648 list_del(&thread->list); 649 tsk = container_of(thread, struct task_struct, thread); 650 wake_up_process(tsk); 651 put_task_struct(tsk); 652 } 653 spin_unlock_irq(&pfault_lock); 654 break; 655 default: 656 break; 657 } 658 return NOTIFY_OK; 659 } 660 661 static int __init pfault_irq_init(void) 662 { 663 int rc; 664 665 rc = register_external_interrupt(0x2603, pfault_interrupt); 666 if (rc) 667 goto out_extint; 668 rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP; 669 if (rc) 670 goto out_pfault; 671 service_subclass_irq_register(); 672 hotcpu_notifier(pfault_cpu_notify, 0); 673 return 0; 674 675 out_pfault: 676 unregister_external_interrupt(0x2603, pfault_interrupt); 677 out_extint: 678 pfault_disable = 1; 679 return rc; 680 } 681 early_initcall(pfault_irq_init); 682 683 #endif /* CONFIG_PFAULT */ 684