1 /* 2 * S390 version 3 * Copyright IBM Corp. 1999 4 * Author(s): Hartmut Penner (hp@de.ibm.com) 5 * Ulrich Weigand (uweigand@de.ibm.com) 6 * 7 * Derived from "arch/i386/mm/fault.c" 8 * Copyright (C) 1995 Linus Torvalds 9 */ 10 11 #include <linux/kernel_stat.h> 12 #include <linux/perf_event.h> 13 #include <linux/signal.h> 14 #include <linux/sched.h> 15 #include <linux/kernel.h> 16 #include <linux/errno.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 #include <linux/ptrace.h> 20 #include <linux/mman.h> 21 #include <linux/mm.h> 22 #include <linux/compat.h> 23 #include <linux/smp.h> 24 #include <linux/kdebug.h> 25 #include <linux/init.h> 26 #include <linux/console.h> 27 #include <linux/module.h> 28 #include <linux/hardirq.h> 29 #include <linux/kprobes.h> 30 #include <linux/uaccess.h> 31 #include <linux/hugetlb.h> 32 #include <asm/asm-offsets.h> 33 #include <asm/pgtable.h> 34 #include <asm/irq.h> 35 #include <asm/mmu_context.h> 36 #include <asm/facility.h> 37 #include "../kernel/entry.h" 38 39 #ifndef CONFIG_64BIT 40 #define __FAIL_ADDR_MASK 0x7ffff000 41 #define __SUBCODE_MASK 0x0200 42 #define __PF_RES_FIELD 0ULL 43 #else /* CONFIG_64BIT */ 44 #define __FAIL_ADDR_MASK -4096L 45 #define __SUBCODE_MASK 0x0600 46 #define __PF_RES_FIELD 0x8000000000000000ULL 47 #endif /* CONFIG_64BIT */ 48 49 #define VM_FAULT_BADCONTEXT 0x010000 50 #define VM_FAULT_BADMAP 0x020000 51 #define VM_FAULT_BADACCESS 0x040000 52 #define VM_FAULT_SIGNAL 0x080000 53 54 static unsigned long store_indication __read_mostly; 55 56 #ifdef CONFIG_64BIT 57 static int __init fault_init(void) 58 { 59 if (test_facility(75)) 60 store_indication = 0xc00; 61 return 0; 62 } 63 early_initcall(fault_init); 64 #endif 65 66 static inline int notify_page_fault(struct pt_regs *regs) 67 { 68 int ret = 0; 69 70 /* kprobe_running() needs smp_processor_id() */ 71 if (kprobes_built_in() && !user_mode(regs)) { 72 preempt_disable(); 73 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 74 ret = 1; 75 preempt_enable(); 76 } 77 return ret; 78 } 79 80 81 /* 82 * Unlock any spinlocks which will prevent us from getting the 83 * message out. 84 */ 85 void bust_spinlocks(int yes) 86 { 87 if (yes) { 88 oops_in_progress = 1; 89 } else { 90 int loglevel_save = console_loglevel; 91 console_unblank(); 92 oops_in_progress = 0; 93 /* 94 * OK, the message is on the console. Now we call printk() 95 * without oops_in_progress set so that printk will give klogd 96 * a poke. Hold onto your hats... 97 */ 98 console_loglevel = 15; 99 printk(" "); 100 console_loglevel = loglevel_save; 101 } 102 } 103 104 /* 105 * Returns the address space associated with the fault. 106 * Returns 0 for kernel space and 1 for user space. 107 */ 108 static inline int user_space_fault(unsigned long trans_exc_code) 109 { 110 /* 111 * The lowest two bits of the translation exception 112 * identification indicate which paging table was used. 113 */ 114 trans_exc_code &= 3; 115 if (trans_exc_code == 2) 116 /* Access via secondary space, set_fs setting decides */ 117 return current->thread.mm_segment.ar4; 118 if (s390_user_mode == HOME_SPACE_MODE) 119 /* User space if the access has been done via home space. */ 120 return trans_exc_code == 3; 121 /* 122 * If the user space is not the home space the kernel runs in home 123 * space. Access via secondary space has already been covered, 124 * access via primary space or access register is from user space 125 * and access via home space is from the kernel. 126 */ 127 return trans_exc_code != 3; 128 } 129 130 static inline void report_user_fault(struct pt_regs *regs, long signr) 131 { 132 if ((task_pid_nr(current) > 1) && !show_unhandled_signals) 133 return; 134 if (!unhandled_signal(current, signr)) 135 return; 136 if (!printk_ratelimit()) 137 return; 138 printk(KERN_ALERT "User process fault: interruption code 0x%X ", 139 regs->int_code); 140 print_vma_addr(KERN_CONT "in ", regs->psw.addr & PSW_ADDR_INSN); 141 printk(KERN_CONT "\n"); 142 printk(KERN_ALERT "failing address: %lX\n", 143 regs->int_parm_long & __FAIL_ADDR_MASK); 144 show_regs(regs); 145 } 146 147 /* 148 * Send SIGSEGV to task. This is an external routine 149 * to keep the stack usage of do_page_fault small. 150 */ 151 static noinline void do_sigsegv(struct pt_regs *regs, int si_code) 152 { 153 struct siginfo si; 154 155 report_user_fault(regs, SIGSEGV); 156 si.si_signo = SIGSEGV; 157 si.si_code = si_code; 158 si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK); 159 force_sig_info(SIGSEGV, &si, current); 160 } 161 162 static noinline void do_no_context(struct pt_regs *regs) 163 { 164 const struct exception_table_entry *fixup; 165 unsigned long address; 166 167 /* Are we prepared to handle this kernel fault? */ 168 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN); 169 if (fixup) { 170 regs->psw.addr = extable_fixup(fixup) | PSW_ADDR_AMODE; 171 return; 172 } 173 174 /* 175 * Oops. The kernel tried to access some bad page. We'll have to 176 * terminate things with extreme prejudice. 177 */ 178 address = regs->int_parm_long & __FAIL_ADDR_MASK; 179 if (!user_space_fault(regs->int_parm_long)) 180 printk(KERN_ALERT "Unable to handle kernel pointer dereference" 181 " at virtual kernel address %p\n", (void *)address); 182 else 183 printk(KERN_ALERT "Unable to handle kernel paging request" 184 " at virtual user address %p\n", (void *)address); 185 186 die(regs, "Oops"); 187 do_exit(SIGKILL); 188 } 189 190 static noinline void do_low_address(struct pt_regs *regs) 191 { 192 /* Low-address protection hit in kernel mode means 193 NULL pointer write access in kernel mode. */ 194 if (regs->psw.mask & PSW_MASK_PSTATE) { 195 /* Low-address protection hit in user mode 'cannot happen'. */ 196 die (regs, "Low-address protection"); 197 do_exit(SIGKILL); 198 } 199 200 do_no_context(regs); 201 } 202 203 static noinline void do_sigbus(struct pt_regs *regs) 204 { 205 struct task_struct *tsk = current; 206 struct siginfo si; 207 208 /* 209 * Send a sigbus, regardless of whether we were in kernel 210 * or user mode. 211 */ 212 si.si_signo = SIGBUS; 213 si.si_errno = 0; 214 si.si_code = BUS_ADRERR; 215 si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK); 216 force_sig_info(SIGBUS, &si, tsk); 217 } 218 219 static noinline void do_fault_error(struct pt_regs *regs, int fault) 220 { 221 int si_code; 222 223 switch (fault) { 224 case VM_FAULT_BADACCESS: 225 case VM_FAULT_BADMAP: 226 /* Bad memory access. Check if it is kernel or user space. */ 227 if (user_mode(regs)) { 228 /* User mode accesses just cause a SIGSEGV */ 229 si_code = (fault == VM_FAULT_BADMAP) ? 230 SEGV_MAPERR : SEGV_ACCERR; 231 do_sigsegv(regs, si_code); 232 return; 233 } 234 case VM_FAULT_BADCONTEXT: 235 do_no_context(regs); 236 break; 237 case VM_FAULT_SIGNAL: 238 if (!user_mode(regs)) 239 do_no_context(regs); 240 break; 241 default: /* fault & VM_FAULT_ERROR */ 242 if (fault & VM_FAULT_OOM) { 243 if (!user_mode(regs)) 244 do_no_context(regs); 245 else 246 pagefault_out_of_memory(); 247 } else if (fault & VM_FAULT_SIGBUS) { 248 /* Kernel mode? Handle exceptions or die */ 249 if (!user_mode(regs)) 250 do_no_context(regs); 251 else 252 do_sigbus(regs); 253 } else 254 BUG(); 255 break; 256 } 257 } 258 259 /* 260 * This routine handles page faults. It determines the address, 261 * and the problem, and then passes it off to one of the appropriate 262 * routines. 263 * 264 * interruption code (int_code): 265 * 04 Protection -> Write-Protection (suprression) 266 * 10 Segment translation -> Not present (nullification) 267 * 11 Page translation -> Not present (nullification) 268 * 3b Region third trans. -> Not present (nullification) 269 */ 270 static inline int do_exception(struct pt_regs *regs, int access) 271 { 272 struct task_struct *tsk; 273 struct mm_struct *mm; 274 struct vm_area_struct *vma; 275 unsigned long trans_exc_code; 276 unsigned long address; 277 unsigned int flags; 278 int fault; 279 280 tsk = current; 281 /* 282 * The instruction that caused the program check has 283 * been nullified. Don't signal single step via SIGTRAP. 284 */ 285 clear_tsk_thread_flag(tsk, TIF_PER_TRAP); 286 287 if (notify_page_fault(regs)) 288 return 0; 289 290 mm = tsk->mm; 291 trans_exc_code = regs->int_parm_long; 292 293 /* 294 * Verify that the fault happened in user space, that 295 * we are not in an interrupt and that there is a 296 * user context. 297 */ 298 fault = VM_FAULT_BADCONTEXT; 299 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 300 goto out; 301 302 address = trans_exc_code & __FAIL_ADDR_MASK; 303 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 304 flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 305 if (user_mode(regs)) 306 flags |= FAULT_FLAG_USER; 307 if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400) 308 flags |= FAULT_FLAG_WRITE; 309 down_read(&mm->mmap_sem); 310 311 #ifdef CONFIG_PGSTE 312 if ((current->flags & PF_VCPU) && S390_lowcore.gmap) { 313 address = __gmap_fault(address, 314 (struct gmap *) S390_lowcore.gmap); 315 if (address == -EFAULT) { 316 fault = VM_FAULT_BADMAP; 317 goto out_up; 318 } 319 if (address == -ENOMEM) { 320 fault = VM_FAULT_OOM; 321 goto out_up; 322 } 323 } 324 #endif 325 326 retry: 327 fault = VM_FAULT_BADMAP; 328 vma = find_vma(mm, address); 329 if (!vma) 330 goto out_up; 331 332 if (unlikely(vma->vm_start > address)) { 333 if (!(vma->vm_flags & VM_GROWSDOWN)) 334 goto out_up; 335 if (expand_stack(vma, address)) 336 goto out_up; 337 } 338 339 /* 340 * Ok, we have a good vm_area for this memory access, so 341 * we can handle it.. 342 */ 343 fault = VM_FAULT_BADACCESS; 344 if (unlikely(!(vma->vm_flags & access))) 345 goto out_up; 346 347 if (is_vm_hugetlb_page(vma)) 348 address &= HPAGE_MASK; 349 /* 350 * If for any reason at all we couldn't handle the fault, 351 * make sure we exit gracefully rather than endlessly redo 352 * the fault. 353 */ 354 fault = handle_mm_fault(mm, vma, address, flags); 355 /* No reason to continue if interrupted by SIGKILL. */ 356 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { 357 fault = VM_FAULT_SIGNAL; 358 goto out; 359 } 360 if (unlikely(fault & VM_FAULT_ERROR)) 361 goto out_up; 362 363 /* 364 * Major/minor page fault accounting is only done on the 365 * initial attempt. If we go through a retry, it is extremely 366 * likely that the page will be found in page cache at that point. 367 */ 368 if (flags & FAULT_FLAG_ALLOW_RETRY) { 369 if (fault & VM_FAULT_MAJOR) { 370 tsk->maj_flt++; 371 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 372 regs, address); 373 } else { 374 tsk->min_flt++; 375 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 376 regs, address); 377 } 378 if (fault & VM_FAULT_RETRY) { 379 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 380 * of starvation. */ 381 flags &= ~FAULT_FLAG_ALLOW_RETRY; 382 flags |= FAULT_FLAG_TRIED; 383 down_read(&mm->mmap_sem); 384 goto retry; 385 } 386 } 387 fault = 0; 388 out_up: 389 up_read(&mm->mmap_sem); 390 out: 391 return fault; 392 } 393 394 void __kprobes do_protection_exception(struct pt_regs *regs) 395 { 396 unsigned long trans_exc_code; 397 int fault; 398 399 trans_exc_code = regs->int_parm_long; 400 /* 401 * Protection exceptions are suppressing, decrement psw address. 402 * The exception to this rule are aborted transactions, for these 403 * the PSW already points to the correct location. 404 */ 405 if (!(regs->int_code & 0x200)) 406 regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16); 407 /* 408 * Check for low-address protection. This needs to be treated 409 * as a special case because the translation exception code 410 * field is not guaranteed to contain valid data in this case. 411 */ 412 if (unlikely(!(trans_exc_code & 4))) { 413 do_low_address(regs); 414 return; 415 } 416 fault = do_exception(regs, VM_WRITE); 417 if (unlikely(fault)) 418 do_fault_error(regs, fault); 419 } 420 421 void __kprobes do_dat_exception(struct pt_regs *regs) 422 { 423 int access, fault; 424 425 access = VM_READ | VM_EXEC | VM_WRITE; 426 fault = do_exception(regs, access); 427 if (unlikely(fault)) 428 do_fault_error(regs, fault); 429 } 430 431 #ifdef CONFIG_64BIT 432 void __kprobes do_asce_exception(struct pt_regs *regs) 433 { 434 struct mm_struct *mm = current->mm; 435 struct vm_area_struct *vma; 436 unsigned long trans_exc_code; 437 438 /* 439 * The instruction that caused the program check has 440 * been nullified. Don't signal single step via SIGTRAP. 441 */ 442 clear_tsk_thread_flag(current, TIF_PER_TRAP); 443 444 trans_exc_code = regs->int_parm_long; 445 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 446 goto no_context; 447 448 down_read(&mm->mmap_sem); 449 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK); 450 up_read(&mm->mmap_sem); 451 452 if (vma) { 453 update_mm(mm, current); 454 return; 455 } 456 457 /* User mode accesses just cause a SIGSEGV */ 458 if (user_mode(regs)) { 459 do_sigsegv(regs, SEGV_MAPERR); 460 return; 461 } 462 463 no_context: 464 do_no_context(regs); 465 } 466 #endif 467 468 int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write) 469 { 470 struct pt_regs regs; 471 int access, fault; 472 473 /* Emulate a uaccess fault from kernel mode. */ 474 regs.psw.mask = psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_MCHECK; 475 if (!irqs_disabled()) 476 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT; 477 regs.psw.addr = (unsigned long) __builtin_return_address(0); 478 regs.psw.addr |= PSW_ADDR_AMODE; 479 regs.int_code = pgm_int_code; 480 regs.int_parm_long = (uaddr & PAGE_MASK) | 2; 481 access = write ? VM_WRITE : VM_READ; 482 fault = do_exception(®s, access); 483 /* 484 * Since the fault happened in kernel mode while performing a uaccess 485 * all we need to do now is emulating a fixup in case "fault" is not 486 * zero. 487 * For the calling uaccess functions this results always in -EFAULT. 488 */ 489 return fault ? -EFAULT : 0; 490 } 491 492 #ifdef CONFIG_PFAULT 493 /* 494 * 'pfault' pseudo page faults routines. 495 */ 496 static int pfault_disable; 497 498 static int __init nopfault(char *str) 499 { 500 pfault_disable = 1; 501 return 1; 502 } 503 504 __setup("nopfault", nopfault); 505 506 struct pfault_refbk { 507 u16 refdiagc; 508 u16 reffcode; 509 u16 refdwlen; 510 u16 refversn; 511 u64 refgaddr; 512 u64 refselmk; 513 u64 refcmpmk; 514 u64 reserved; 515 } __attribute__ ((packed, aligned(8))); 516 517 int pfault_init(void) 518 { 519 struct pfault_refbk refbk = { 520 .refdiagc = 0x258, 521 .reffcode = 0, 522 .refdwlen = 5, 523 .refversn = 2, 524 .refgaddr = __LC_CURRENT_PID, 525 .refselmk = 1ULL << 48, 526 .refcmpmk = 1ULL << 48, 527 .reserved = __PF_RES_FIELD }; 528 int rc; 529 530 if (pfault_disable) 531 return -1; 532 asm volatile( 533 " diag %1,%0,0x258\n" 534 "0: j 2f\n" 535 "1: la %0,8\n" 536 "2:\n" 537 EX_TABLE(0b,1b) 538 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc"); 539 return rc; 540 } 541 542 void pfault_fini(void) 543 { 544 struct pfault_refbk refbk = { 545 .refdiagc = 0x258, 546 .reffcode = 1, 547 .refdwlen = 5, 548 .refversn = 2, 549 }; 550 551 if (pfault_disable) 552 return; 553 asm volatile( 554 " diag %0,0,0x258\n" 555 "0:\n" 556 EX_TABLE(0b,0b) 557 : : "a" (&refbk), "m" (refbk) : "cc"); 558 } 559 560 static DEFINE_SPINLOCK(pfault_lock); 561 static LIST_HEAD(pfault_list); 562 563 static void pfault_interrupt(struct ext_code ext_code, 564 unsigned int param32, unsigned long param64) 565 { 566 struct task_struct *tsk; 567 __u16 subcode; 568 pid_t pid; 569 570 /* 571 * Get the external interruption subcode & pfault 572 * initial/completion signal bit. VM stores this 573 * in the 'cpu address' field associated with the 574 * external interrupt. 575 */ 576 subcode = ext_code.subcode; 577 if ((subcode & 0xff00) != __SUBCODE_MASK) 578 return; 579 inc_irq_stat(IRQEXT_PFL); 580 /* Get the token (= pid of the affected task). */ 581 pid = sizeof(void *) == 4 ? param32 : param64; 582 rcu_read_lock(); 583 tsk = find_task_by_pid_ns(pid, &init_pid_ns); 584 if (tsk) 585 get_task_struct(tsk); 586 rcu_read_unlock(); 587 if (!tsk) 588 return; 589 spin_lock(&pfault_lock); 590 if (subcode & 0x0080) { 591 /* signal bit is set -> a page has been swapped in by VM */ 592 if (tsk->thread.pfault_wait == 1) { 593 /* Initial interrupt was faster than the completion 594 * interrupt. pfault_wait is valid. Set pfault_wait 595 * back to zero and wake up the process. This can 596 * safely be done because the task is still sleeping 597 * and can't produce new pfaults. */ 598 tsk->thread.pfault_wait = 0; 599 list_del(&tsk->thread.list); 600 wake_up_process(tsk); 601 put_task_struct(tsk); 602 } else { 603 /* Completion interrupt was faster than initial 604 * interrupt. Set pfault_wait to -1 so the initial 605 * interrupt doesn't put the task to sleep. 606 * If the task is not running, ignore the completion 607 * interrupt since it must be a leftover of a PFAULT 608 * CANCEL operation which didn't remove all pending 609 * completion interrupts. */ 610 if (tsk->state == TASK_RUNNING) 611 tsk->thread.pfault_wait = -1; 612 } 613 } else { 614 /* signal bit not set -> a real page is missing. */ 615 if (WARN_ON_ONCE(tsk != current)) 616 goto out; 617 if (tsk->thread.pfault_wait == 1) { 618 /* Already on the list with a reference: put to sleep */ 619 __set_task_state(tsk, TASK_UNINTERRUPTIBLE); 620 set_tsk_need_resched(tsk); 621 } else if (tsk->thread.pfault_wait == -1) { 622 /* Completion interrupt was faster than the initial 623 * interrupt (pfault_wait == -1). Set pfault_wait 624 * back to zero and exit. */ 625 tsk->thread.pfault_wait = 0; 626 } else { 627 /* Initial interrupt arrived before completion 628 * interrupt. Let the task sleep. 629 * An extra task reference is needed since a different 630 * cpu may set the task state to TASK_RUNNING again 631 * before the scheduler is reached. */ 632 get_task_struct(tsk); 633 tsk->thread.pfault_wait = 1; 634 list_add(&tsk->thread.list, &pfault_list); 635 __set_task_state(tsk, TASK_UNINTERRUPTIBLE); 636 set_tsk_need_resched(tsk); 637 } 638 } 639 out: 640 spin_unlock(&pfault_lock); 641 put_task_struct(tsk); 642 } 643 644 static int pfault_cpu_notify(struct notifier_block *self, unsigned long action, 645 void *hcpu) 646 { 647 struct thread_struct *thread, *next; 648 struct task_struct *tsk; 649 650 switch (action & ~CPU_TASKS_FROZEN) { 651 case CPU_DEAD: 652 spin_lock_irq(&pfault_lock); 653 list_for_each_entry_safe(thread, next, &pfault_list, list) { 654 thread->pfault_wait = 0; 655 list_del(&thread->list); 656 tsk = container_of(thread, struct task_struct, thread); 657 wake_up_process(tsk); 658 put_task_struct(tsk); 659 } 660 spin_unlock_irq(&pfault_lock); 661 break; 662 default: 663 break; 664 } 665 return NOTIFY_OK; 666 } 667 668 static int __init pfault_irq_init(void) 669 { 670 int rc; 671 672 rc = register_external_interrupt(0x2603, pfault_interrupt); 673 if (rc) 674 goto out_extint; 675 rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP; 676 if (rc) 677 goto out_pfault; 678 irq_subclass_register(IRQ_SUBCLASS_SERVICE_SIGNAL); 679 hotcpu_notifier(pfault_cpu_notify, 0); 680 return 0; 681 682 out_pfault: 683 unregister_external_interrupt(0x2603, pfault_interrupt); 684 out_extint: 685 pfault_disable = 1; 686 return rc; 687 } 688 early_initcall(pfault_irq_init); 689 690 #endif /* CONFIG_PFAULT */ 691