1 /* 2 * arch/s390/mm/fault.c 3 * 4 * S390 version 5 * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation 6 * Author(s): Hartmut Penner (hp@de.ibm.com) 7 * Ulrich Weigand (uweigand@de.ibm.com) 8 * 9 * Derived from "arch/i386/mm/fault.c" 10 * Copyright (C) 1995 Linus Torvalds 11 */ 12 13 #include <linux/perf_event.h> 14 #include <linux/signal.h> 15 #include <linux/sched.h> 16 #include <linux/kernel.h> 17 #include <linux/errno.h> 18 #include <linux/string.h> 19 #include <linux/types.h> 20 #include <linux/ptrace.h> 21 #include <linux/mman.h> 22 #include <linux/mm.h> 23 #include <linux/compat.h> 24 #include <linux/smp.h> 25 #include <linux/kdebug.h> 26 #include <linux/init.h> 27 #include <linux/console.h> 28 #include <linux/module.h> 29 #include <linux/hardirq.h> 30 #include <linux/kprobes.h> 31 #include <linux/uaccess.h> 32 #include <linux/hugetlb.h> 33 #include <asm/system.h> 34 #include <asm/pgtable.h> 35 #include <asm/s390_ext.h> 36 #include <asm/mmu_context.h> 37 #include <asm/compat.h> 38 #include "../kernel/entry.h" 39 40 #ifndef CONFIG_64BIT 41 #define __FAIL_ADDR_MASK 0x7ffff000 42 #define __SUBCODE_MASK 0x0200 43 #define __PF_RES_FIELD 0ULL 44 #else /* CONFIG_64BIT */ 45 #define __FAIL_ADDR_MASK -4096L 46 #define __SUBCODE_MASK 0x0600 47 #define __PF_RES_FIELD 0x8000000000000000ULL 48 #endif /* CONFIG_64BIT */ 49 50 #ifdef CONFIG_SYSCTL 51 extern int sysctl_userprocess_debug; 52 #endif 53 54 #define VM_FAULT_BADCONTEXT 0x010000 55 #define VM_FAULT_BADMAP 0x020000 56 #define VM_FAULT_BADACCESS 0x040000 57 58 static inline int notify_page_fault(struct pt_regs *regs) 59 { 60 int ret = 0; 61 62 #ifdef CONFIG_KPROBES 63 /* kprobe_running() needs smp_processor_id() */ 64 if (!user_mode(regs)) { 65 preempt_disable(); 66 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 67 ret = 1; 68 preempt_enable(); 69 } 70 #endif 71 return ret; 72 } 73 74 75 /* 76 * Unlock any spinlocks which will prevent us from getting the 77 * message out. 78 */ 79 void bust_spinlocks(int yes) 80 { 81 if (yes) { 82 oops_in_progress = 1; 83 } else { 84 int loglevel_save = console_loglevel; 85 console_unblank(); 86 oops_in_progress = 0; 87 /* 88 * OK, the message is on the console. Now we call printk() 89 * without oops_in_progress set so that printk will give klogd 90 * a poke. Hold onto your hats... 91 */ 92 console_loglevel = 15; 93 printk(" "); 94 console_loglevel = loglevel_save; 95 } 96 } 97 98 /* 99 * Returns the address space associated with the fault. 100 * Returns 0 for kernel space and 1 for user space. 101 */ 102 static inline int user_space_fault(unsigned long trans_exc_code) 103 { 104 /* 105 * The lowest two bits of the translation exception 106 * identification indicate which paging table was used. 107 */ 108 trans_exc_code &= 3; 109 if (trans_exc_code == 2) 110 /* Access via secondary space, set_fs setting decides */ 111 return current->thread.mm_segment.ar4; 112 if (user_mode == HOME_SPACE_MODE) 113 /* User space if the access has been done via home space. */ 114 return trans_exc_code == 3; 115 /* 116 * If the user space is not the home space the kernel runs in home 117 * space. Access via secondary space has already been covered, 118 * access via primary space or access register is from user space 119 * and access via home space is from the kernel. 120 */ 121 return trans_exc_code != 3; 122 } 123 124 /* 125 * Send SIGSEGV to task. This is an external routine 126 * to keep the stack usage of do_page_fault small. 127 */ 128 static noinline void do_sigsegv(struct pt_regs *regs, long int_code, 129 int si_code, unsigned long trans_exc_code) 130 { 131 struct siginfo si; 132 unsigned long address; 133 134 address = trans_exc_code & __FAIL_ADDR_MASK; 135 current->thread.prot_addr = address; 136 current->thread.trap_no = int_code; 137 #if defined(CONFIG_SYSCTL) || defined(CONFIG_PROCESS_DEBUG) 138 #if defined(CONFIG_SYSCTL) 139 if (sysctl_userprocess_debug) 140 #endif 141 { 142 printk("User process fault: interruption code 0x%lX\n", 143 int_code); 144 printk("failing address: %lX\n", address); 145 show_regs(regs); 146 } 147 #endif 148 si.si_signo = SIGSEGV; 149 si.si_code = si_code; 150 si.si_addr = (void __user *) address; 151 force_sig_info(SIGSEGV, &si, current); 152 } 153 154 static noinline void do_no_context(struct pt_regs *regs, long int_code, 155 unsigned long trans_exc_code) 156 { 157 const struct exception_table_entry *fixup; 158 unsigned long address; 159 160 /* Are we prepared to handle this kernel fault? */ 161 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN); 162 if (fixup) { 163 regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE; 164 return; 165 } 166 167 /* 168 * Oops. The kernel tried to access some bad page. We'll have to 169 * terminate things with extreme prejudice. 170 */ 171 address = trans_exc_code & __FAIL_ADDR_MASK; 172 if (!user_space_fault(trans_exc_code)) 173 printk(KERN_ALERT "Unable to handle kernel pointer dereference" 174 " at virtual kernel address %p\n", (void *)address); 175 else 176 printk(KERN_ALERT "Unable to handle kernel paging request" 177 " at virtual user address %p\n", (void *)address); 178 179 die("Oops", regs, int_code); 180 do_exit(SIGKILL); 181 } 182 183 static noinline void do_low_address(struct pt_regs *regs, long int_code, 184 unsigned long trans_exc_code) 185 { 186 /* Low-address protection hit in kernel mode means 187 NULL pointer write access in kernel mode. */ 188 if (regs->psw.mask & PSW_MASK_PSTATE) { 189 /* Low-address protection hit in user mode 'cannot happen'. */ 190 die ("Low-address protection", regs, int_code); 191 do_exit(SIGKILL); 192 } 193 194 do_no_context(regs, int_code, trans_exc_code); 195 } 196 197 static noinline void do_sigbus(struct pt_regs *regs, long int_code, 198 unsigned long trans_exc_code) 199 { 200 struct task_struct *tsk = current; 201 202 /* 203 * Send a sigbus, regardless of whether we were in kernel 204 * or user mode. 205 */ 206 tsk->thread.prot_addr = trans_exc_code & __FAIL_ADDR_MASK; 207 tsk->thread.trap_no = int_code; 208 force_sig(SIGBUS, tsk); 209 } 210 211 #ifdef CONFIG_S390_EXEC_PROTECT 212 static noinline int signal_return(struct pt_regs *regs, long int_code, 213 unsigned long trans_exc_code) 214 { 215 u16 instruction; 216 int rc; 217 218 rc = __get_user(instruction, (u16 __user *) regs->psw.addr); 219 220 if (!rc && instruction == 0x0a77) { 221 clear_tsk_thread_flag(current, TIF_SINGLE_STEP); 222 if (is_compat_task()) 223 sys32_sigreturn(); 224 else 225 sys_sigreturn(); 226 } else if (!rc && instruction == 0x0aad) { 227 clear_tsk_thread_flag(current, TIF_SINGLE_STEP); 228 if (is_compat_task()) 229 sys32_rt_sigreturn(); 230 else 231 sys_rt_sigreturn(); 232 } else 233 do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code); 234 return 0; 235 } 236 #endif /* CONFIG_S390_EXEC_PROTECT */ 237 238 static noinline void do_fault_error(struct pt_regs *regs, long int_code, 239 unsigned long trans_exc_code, int fault) 240 { 241 int si_code; 242 243 switch (fault) { 244 case VM_FAULT_BADACCESS: 245 #ifdef CONFIG_S390_EXEC_PROTECT 246 if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY && 247 (trans_exc_code & 3) == 0) { 248 signal_return(regs, int_code, trans_exc_code); 249 break; 250 } 251 #endif /* CONFIG_S390_EXEC_PROTECT */ 252 case VM_FAULT_BADMAP: 253 /* Bad memory access. Check if it is kernel or user space. */ 254 if (regs->psw.mask & PSW_MASK_PSTATE) { 255 /* User mode accesses just cause a SIGSEGV */ 256 si_code = (fault == VM_FAULT_BADMAP) ? 257 SEGV_MAPERR : SEGV_ACCERR; 258 do_sigsegv(regs, int_code, si_code, trans_exc_code); 259 return; 260 } 261 case VM_FAULT_BADCONTEXT: 262 do_no_context(regs, int_code, trans_exc_code); 263 break; 264 default: /* fault & VM_FAULT_ERROR */ 265 if (fault & VM_FAULT_OOM) 266 pagefault_out_of_memory(); 267 else if (fault & VM_FAULT_SIGBUS) { 268 do_sigbus(regs, int_code, trans_exc_code); 269 /* Kernel mode? Handle exceptions or die */ 270 if (!(regs->psw.mask & PSW_MASK_PSTATE)) 271 do_no_context(regs, int_code, trans_exc_code); 272 } else 273 BUG(); 274 break; 275 } 276 } 277 278 /* 279 * This routine handles page faults. It determines the address, 280 * and the problem, and then passes it off to one of the appropriate 281 * routines. 282 * 283 * interruption code (int_code): 284 * 04 Protection -> Write-Protection (suprression) 285 * 10 Segment translation -> Not present (nullification) 286 * 11 Page translation -> Not present (nullification) 287 * 3b Region third trans. -> Not present (nullification) 288 */ 289 static inline int do_exception(struct pt_regs *regs, int access, 290 unsigned long trans_exc_code) 291 { 292 struct task_struct *tsk; 293 struct mm_struct *mm; 294 struct vm_area_struct *vma; 295 unsigned long address; 296 int fault; 297 298 if (notify_page_fault(regs)) 299 return 0; 300 301 tsk = current; 302 mm = tsk->mm; 303 304 /* 305 * Verify that the fault happened in user space, that 306 * we are not in an interrupt and that there is a 307 * user context. 308 */ 309 fault = VM_FAULT_BADCONTEXT; 310 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 311 goto out; 312 313 address = trans_exc_code & __FAIL_ADDR_MASK; 314 /* 315 * When we get here, the fault happened in the current 316 * task's user address space, so we can switch on the 317 * interrupts again and then search the VMAs 318 */ 319 local_irq_enable(); 320 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); 321 down_read(&mm->mmap_sem); 322 323 fault = VM_FAULT_BADMAP; 324 vma = find_vma(mm, address); 325 if (!vma) 326 goto out_up; 327 328 if (unlikely(vma->vm_start > address)) { 329 if (!(vma->vm_flags & VM_GROWSDOWN)) 330 goto out_up; 331 if (expand_stack(vma, address)) 332 goto out_up; 333 } 334 335 /* 336 * Ok, we have a good vm_area for this memory access, so 337 * we can handle it.. 338 */ 339 fault = VM_FAULT_BADACCESS; 340 if (unlikely(!(vma->vm_flags & access))) 341 goto out_up; 342 343 if (is_vm_hugetlb_page(vma)) 344 address &= HPAGE_MASK; 345 /* 346 * If for any reason at all we couldn't handle the fault, 347 * make sure we exit gracefully rather than endlessly redo 348 * the fault. 349 */ 350 fault = handle_mm_fault(mm, vma, address, 351 (access == VM_WRITE) ? FAULT_FLAG_WRITE : 0); 352 if (unlikely(fault & VM_FAULT_ERROR)) 353 goto out_up; 354 355 if (fault & VM_FAULT_MAJOR) { 356 tsk->maj_flt++; 357 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, 358 regs, address); 359 } else { 360 tsk->min_flt++; 361 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, 362 regs, address); 363 } 364 /* 365 * The instruction that caused the program check will 366 * be repeated. Don't signal single step via SIGTRAP. 367 */ 368 clear_tsk_thread_flag(tsk, TIF_SINGLE_STEP); 369 fault = 0; 370 out_up: 371 up_read(&mm->mmap_sem); 372 out: 373 return fault; 374 } 375 376 void __kprobes do_protection_exception(struct pt_regs *regs, long int_code) 377 { 378 unsigned long trans_exc_code = S390_lowcore.trans_exc_code; 379 int fault; 380 381 /* Protection exception is supressing, decrement psw address. */ 382 regs->psw.addr -= (int_code >> 16); 383 /* 384 * Check for low-address protection. This needs to be treated 385 * as a special case because the translation exception code 386 * field is not guaranteed to contain valid data in this case. 387 */ 388 if (unlikely(!(trans_exc_code & 4))) { 389 do_low_address(regs, int_code, trans_exc_code); 390 return; 391 } 392 fault = do_exception(regs, VM_WRITE, trans_exc_code); 393 if (unlikely(fault)) 394 do_fault_error(regs, 4, trans_exc_code, fault); 395 } 396 397 void __kprobes do_dat_exception(struct pt_regs *regs, long int_code) 398 { 399 unsigned long trans_exc_code = S390_lowcore.trans_exc_code; 400 int access, fault; 401 402 access = VM_READ | VM_EXEC | VM_WRITE; 403 #ifdef CONFIG_S390_EXEC_PROTECT 404 if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY && 405 (trans_exc_code & 3) == 0) 406 access = VM_EXEC; 407 #endif 408 fault = do_exception(regs, access, trans_exc_code); 409 if (unlikely(fault)) 410 do_fault_error(regs, int_code & 255, trans_exc_code, fault); 411 } 412 413 #ifdef CONFIG_64BIT 414 void __kprobes do_asce_exception(struct pt_regs *regs, long int_code) 415 { 416 unsigned long trans_exc_code = S390_lowcore.trans_exc_code; 417 struct mm_struct *mm = current->mm; 418 struct vm_area_struct *vma; 419 420 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) 421 goto no_context; 422 423 local_irq_enable(); 424 425 down_read(&mm->mmap_sem); 426 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK); 427 up_read(&mm->mmap_sem); 428 429 if (vma) { 430 update_mm(mm, current); 431 return; 432 } 433 434 /* User mode accesses just cause a SIGSEGV */ 435 if (regs->psw.mask & PSW_MASK_PSTATE) { 436 do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code); 437 return; 438 } 439 440 no_context: 441 do_no_context(regs, int_code, trans_exc_code); 442 } 443 #endif 444 445 int __handle_fault(unsigned long uaddr, unsigned long int_code, int write_user) 446 { 447 struct pt_regs regs; 448 int access, fault; 449 450 regs.psw.mask = psw_kernel_bits; 451 if (!irqs_disabled()) 452 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT; 453 regs.psw.addr = (unsigned long) __builtin_return_address(0); 454 regs.psw.addr |= PSW_ADDR_AMODE; 455 uaddr &= PAGE_MASK; 456 access = write_user ? VM_WRITE : VM_READ; 457 fault = do_exception(®s, access, uaddr | 2); 458 if (unlikely(fault)) { 459 if (fault & VM_FAULT_OOM) { 460 pagefault_out_of_memory(); 461 fault = 0; 462 } else if (fault & VM_FAULT_SIGBUS) 463 do_sigbus(®s, int_code, uaddr); 464 } 465 return fault ? -EFAULT : 0; 466 } 467 468 #ifdef CONFIG_PFAULT 469 /* 470 * 'pfault' pseudo page faults routines. 471 */ 472 static ext_int_info_t ext_int_pfault; 473 static int pfault_disable = 0; 474 475 static int __init nopfault(char *str) 476 { 477 pfault_disable = 1; 478 return 1; 479 } 480 481 __setup("nopfault", nopfault); 482 483 typedef struct { 484 __u16 refdiagc; 485 __u16 reffcode; 486 __u16 refdwlen; 487 __u16 refversn; 488 __u64 refgaddr; 489 __u64 refselmk; 490 __u64 refcmpmk; 491 __u64 reserved; 492 } __attribute__ ((packed, aligned(8))) pfault_refbk_t; 493 494 int pfault_init(void) 495 { 496 pfault_refbk_t refbk = 497 { 0x258, 0, 5, 2, __LC_CURRENT, 1ULL << 48, 1ULL << 48, 498 __PF_RES_FIELD }; 499 int rc; 500 501 if (!MACHINE_IS_VM || pfault_disable) 502 return -1; 503 asm volatile( 504 " diag %1,%0,0x258\n" 505 "0: j 2f\n" 506 "1: la %0,8\n" 507 "2:\n" 508 EX_TABLE(0b,1b) 509 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc"); 510 __ctl_set_bit(0, 9); 511 return rc; 512 } 513 514 void pfault_fini(void) 515 { 516 pfault_refbk_t refbk = 517 { 0x258, 1, 5, 2, 0ULL, 0ULL, 0ULL, 0ULL }; 518 519 if (!MACHINE_IS_VM || pfault_disable) 520 return; 521 __ctl_clear_bit(0,9); 522 asm volatile( 523 " diag %0,0,0x258\n" 524 "0:\n" 525 EX_TABLE(0b,0b) 526 : : "a" (&refbk), "m" (refbk) : "cc"); 527 } 528 529 static void pfault_interrupt(__u16 int_code) 530 { 531 struct task_struct *tsk; 532 __u16 subcode; 533 534 /* 535 * Get the external interruption subcode & pfault 536 * initial/completion signal bit. VM stores this 537 * in the 'cpu address' field associated with the 538 * external interrupt. 539 */ 540 subcode = S390_lowcore.cpu_addr; 541 if ((subcode & 0xff00) != __SUBCODE_MASK) 542 return; 543 544 /* 545 * Get the token (= address of the task structure of the affected task). 546 */ 547 tsk = *(struct task_struct **) __LC_PFAULT_INTPARM; 548 549 if (subcode & 0x0080) { 550 /* signal bit is set -> a page has been swapped in by VM */ 551 if (xchg(&tsk->thread.pfault_wait, -1) != 0) { 552 /* Initial interrupt was faster than the completion 553 * interrupt. pfault_wait is valid. Set pfault_wait 554 * back to zero and wake up the process. This can 555 * safely be done because the task is still sleeping 556 * and can't produce new pfaults. */ 557 tsk->thread.pfault_wait = 0; 558 wake_up_process(tsk); 559 put_task_struct(tsk); 560 } 561 } else { 562 /* signal bit not set -> a real page is missing. */ 563 get_task_struct(tsk); 564 set_task_state(tsk, TASK_UNINTERRUPTIBLE); 565 if (xchg(&tsk->thread.pfault_wait, 1) != 0) { 566 /* Completion interrupt was faster than the initial 567 * interrupt (swapped in a -1 for pfault_wait). Set 568 * pfault_wait back to zero and exit. This can be 569 * done safely because tsk is running in kernel 570 * mode and can't produce new pfaults. */ 571 tsk->thread.pfault_wait = 0; 572 set_task_state(tsk, TASK_RUNNING); 573 put_task_struct(tsk); 574 } else 575 set_tsk_need_resched(tsk); 576 } 577 } 578 579 void __init pfault_irq_init(void) 580 { 581 if (!MACHINE_IS_VM) 582 return; 583 584 /* 585 * Try to get pfault pseudo page faults going. 586 */ 587 if (register_early_external_interrupt(0x2603, pfault_interrupt, 588 &ext_int_pfault) != 0) 589 panic("Couldn't request external interrupt 0x2603"); 590 591 if (pfault_init() == 0) 592 return; 593 594 /* Tough luck, no pfault. */ 595 pfault_disable = 1; 596 unregister_early_external_interrupt(0x2603, pfault_interrupt, 597 &ext_int_pfault); 598 } 599 #endif 600