1 /* 2 * arch/s390/mm/fault.c 3 * 4 * S390 version 5 * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation 6 * Author(s): Hartmut Penner (hp@de.ibm.com) 7 * Ulrich Weigand (uweigand@de.ibm.com) 8 * 9 * Derived from "arch/i386/mm/fault.c" 10 * Copyright (C) 1995 Linus Torvalds 11 */ 12 13 #include <linux/signal.h> 14 #include <linux/sched.h> 15 #include <linux/kernel.h> 16 #include <linux/errno.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 #include <linux/ptrace.h> 20 #include <linux/mman.h> 21 #include <linux/mm.h> 22 #include <linux/smp.h> 23 #include <linux/kdebug.h> 24 #include <linux/smp_lock.h> 25 #include <linux/init.h> 26 #include <linux/console.h> 27 #include <linux/module.h> 28 #include <linux/hardirq.h> 29 #include <linux/kprobes.h> 30 #include <linux/uaccess.h> 31 #include <asm/system.h> 32 #include <asm/pgtable.h> 33 #include <asm/s390_ext.h> 34 #include <asm/mmu_context.h> 35 #include "../kernel/entry.h" 36 37 #ifndef CONFIG_64BIT 38 #define __FAIL_ADDR_MASK 0x7ffff000 39 #define __FIXUP_MASK 0x7fffffff 40 #define __SUBCODE_MASK 0x0200 41 #define __PF_RES_FIELD 0ULL 42 #else /* CONFIG_64BIT */ 43 #define __FAIL_ADDR_MASK -4096L 44 #define __FIXUP_MASK ~0L 45 #define __SUBCODE_MASK 0x0600 46 #define __PF_RES_FIELD 0x8000000000000000ULL 47 #endif /* CONFIG_64BIT */ 48 49 #ifdef CONFIG_SYSCTL 50 extern int sysctl_userprocess_debug; 51 #endif 52 53 #ifdef CONFIG_KPROBES 54 static inline int notify_page_fault(struct pt_regs *regs, long err) 55 { 56 int ret = 0; 57 58 /* kprobe_running() needs smp_processor_id() */ 59 if (!user_mode(regs)) { 60 preempt_disable(); 61 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 62 ret = 1; 63 preempt_enable(); 64 } 65 66 return ret; 67 } 68 #else 69 static inline int notify_page_fault(struct pt_regs *regs, long err) 70 { 71 return 0; 72 } 73 #endif 74 75 76 /* 77 * Unlock any spinlocks which will prevent us from getting the 78 * message out. 79 */ 80 void bust_spinlocks(int yes) 81 { 82 if (yes) { 83 oops_in_progress = 1; 84 } else { 85 int loglevel_save = console_loglevel; 86 console_unblank(); 87 oops_in_progress = 0; 88 /* 89 * OK, the message is on the console. Now we call printk() 90 * without oops_in_progress set so that printk will give klogd 91 * a poke. Hold onto your hats... 92 */ 93 console_loglevel = 15; 94 printk(" "); 95 console_loglevel = loglevel_save; 96 } 97 } 98 99 /* 100 * Returns the address space associated with the fault. 101 * Returns 0 for kernel space, 1 for user space and 102 * 2 for code execution in user space with noexec=on. 103 */ 104 static inline int check_space(struct task_struct *tsk) 105 { 106 /* 107 * The lowest two bits of S390_lowcore.trans_exc_code 108 * indicate which paging table was used. 109 */ 110 int desc = S390_lowcore.trans_exc_code & 3; 111 112 if (desc == 3) /* Home Segment Table Descriptor */ 113 return switch_amode == 0; 114 if (desc == 2) /* Secondary Segment Table Descriptor */ 115 return tsk->thread.mm_segment.ar4; 116 #ifdef CONFIG_S390_SWITCH_AMODE 117 if (unlikely(desc == 1)) { /* STD determined via access register */ 118 /* %a0 always indicates primary space. */ 119 if (S390_lowcore.exc_access_id != 0) { 120 save_access_regs(tsk->thread.acrs); 121 /* 122 * An alet of 0 indicates primary space. 123 * An alet of 1 indicates secondary space. 124 * Any other alet values generate an 125 * alen-translation exception. 126 */ 127 if (tsk->thread.acrs[S390_lowcore.exc_access_id]) 128 return tsk->thread.mm_segment.ar4; 129 } 130 } 131 #endif 132 /* Primary Segment Table Descriptor */ 133 return switch_amode << s390_noexec; 134 } 135 136 /* 137 * Send SIGSEGV to task. This is an external routine 138 * to keep the stack usage of do_page_fault small. 139 */ 140 static void do_sigsegv(struct pt_regs *regs, unsigned long error_code, 141 int si_code, unsigned long address) 142 { 143 struct siginfo si; 144 145 #if defined(CONFIG_SYSCTL) || defined(CONFIG_PROCESS_DEBUG) 146 #if defined(CONFIG_SYSCTL) 147 if (sysctl_userprocess_debug) 148 #endif 149 { 150 printk("User process fault: interruption code 0x%lX\n", 151 error_code); 152 printk("failing address: %lX\n", address); 153 show_regs(regs); 154 } 155 #endif 156 si.si_signo = SIGSEGV; 157 si.si_code = si_code; 158 si.si_addr = (void __user *) address; 159 force_sig_info(SIGSEGV, &si, current); 160 } 161 162 static void do_no_context(struct pt_regs *regs, unsigned long error_code, 163 unsigned long address) 164 { 165 const struct exception_table_entry *fixup; 166 167 /* Are we prepared to handle this kernel fault? */ 168 fixup = search_exception_tables(regs->psw.addr & __FIXUP_MASK); 169 if (fixup) { 170 regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE; 171 return; 172 } 173 174 /* 175 * Oops. The kernel tried to access some bad page. We'll have to 176 * terminate things with extreme prejudice. 177 */ 178 if (check_space(current) == 0) 179 printk(KERN_ALERT "Unable to handle kernel pointer dereference" 180 " at virtual kernel address %p\n", (void *)address); 181 else 182 printk(KERN_ALERT "Unable to handle kernel paging request" 183 " at virtual user address %p\n", (void *)address); 184 185 die("Oops", regs, error_code); 186 do_exit(SIGKILL); 187 } 188 189 static void do_low_address(struct pt_regs *regs, unsigned long error_code) 190 { 191 /* Low-address protection hit in kernel mode means 192 NULL pointer write access in kernel mode. */ 193 if (regs->psw.mask & PSW_MASK_PSTATE) { 194 /* Low-address protection hit in user mode 'cannot happen'. */ 195 die ("Low-address protection", regs, error_code); 196 do_exit(SIGKILL); 197 } 198 199 do_no_context(regs, error_code, 0); 200 } 201 202 /* 203 * We ran out of memory, or some other thing happened to us that made 204 * us unable to handle the page fault gracefully. 205 */ 206 static int do_out_of_memory(struct pt_regs *regs, unsigned long error_code, 207 unsigned long address) 208 { 209 struct task_struct *tsk = current; 210 struct mm_struct *mm = tsk->mm; 211 212 up_read(&mm->mmap_sem); 213 if (is_global_init(tsk)) { 214 yield(); 215 down_read(&mm->mmap_sem); 216 return 1; 217 } 218 printk("VM: killing process %s\n", tsk->comm); 219 if (regs->psw.mask & PSW_MASK_PSTATE) 220 do_group_exit(SIGKILL); 221 do_no_context(regs, error_code, address); 222 return 0; 223 } 224 225 static void do_sigbus(struct pt_regs *regs, unsigned long error_code, 226 unsigned long address) 227 { 228 struct task_struct *tsk = current; 229 struct mm_struct *mm = tsk->mm; 230 231 up_read(&mm->mmap_sem); 232 /* 233 * Send a sigbus, regardless of whether we were in kernel 234 * or user mode. 235 */ 236 tsk->thread.prot_addr = address; 237 tsk->thread.trap_no = error_code; 238 force_sig(SIGBUS, tsk); 239 240 /* Kernel mode? Handle exceptions or die */ 241 if (!(regs->psw.mask & PSW_MASK_PSTATE)) 242 do_no_context(regs, error_code, address); 243 } 244 245 #ifdef CONFIG_S390_EXEC_PROTECT 246 static int signal_return(struct mm_struct *mm, struct pt_regs *regs, 247 unsigned long address, unsigned long error_code) 248 { 249 u16 instruction; 250 int rc; 251 #ifdef CONFIG_COMPAT 252 int compat; 253 #endif 254 255 pagefault_disable(); 256 rc = __get_user(instruction, (u16 __user *) regs->psw.addr); 257 pagefault_enable(); 258 if (rc) 259 return -EFAULT; 260 261 up_read(&mm->mmap_sem); 262 clear_tsk_thread_flag(current, TIF_SINGLE_STEP); 263 #ifdef CONFIG_COMPAT 264 compat = test_tsk_thread_flag(current, TIF_31BIT); 265 if (compat && instruction == 0x0a77) 266 sys32_sigreturn(); 267 else if (compat && instruction == 0x0aad) 268 sys32_rt_sigreturn(); 269 else 270 #endif 271 if (instruction == 0x0a77) 272 sys_sigreturn(); 273 else if (instruction == 0x0aad) 274 sys_rt_sigreturn(); 275 else { 276 current->thread.prot_addr = address; 277 current->thread.trap_no = error_code; 278 do_sigsegv(regs, error_code, SEGV_MAPERR, address); 279 } 280 return 0; 281 } 282 #endif /* CONFIG_S390_EXEC_PROTECT */ 283 284 /* 285 * This routine handles page faults. It determines the address, 286 * and the problem, and then passes it off to one of the appropriate 287 * routines. 288 * 289 * error_code: 290 * 04 Protection -> Write-Protection (suprression) 291 * 10 Segment translation -> Not present (nullification) 292 * 11 Page translation -> Not present (nullification) 293 * 3b Region third trans. -> Not present (nullification) 294 */ 295 static inline void 296 do_exception(struct pt_regs *regs, unsigned long error_code, int write) 297 { 298 struct task_struct *tsk; 299 struct mm_struct *mm; 300 struct vm_area_struct *vma; 301 unsigned long address; 302 int space; 303 int si_code; 304 int fault; 305 306 if (notify_page_fault(regs, error_code)) 307 return; 308 309 tsk = current; 310 mm = tsk->mm; 311 312 /* get the failing address and the affected space */ 313 address = S390_lowcore.trans_exc_code & __FAIL_ADDR_MASK; 314 space = check_space(tsk); 315 316 /* 317 * Verify that the fault happened in user space, that 318 * we are not in an interrupt and that there is a 319 * user context. 320 */ 321 if (unlikely(space == 0 || in_atomic() || !mm)) 322 goto no_context; 323 324 /* 325 * When we get here, the fault happened in the current 326 * task's user address space, so we can switch on the 327 * interrupts again and then search the VMAs 328 */ 329 local_irq_enable(); 330 331 down_read(&mm->mmap_sem); 332 333 si_code = SEGV_MAPERR; 334 vma = find_vma(mm, address); 335 if (!vma) 336 goto bad_area; 337 338 #ifdef CONFIG_S390_EXEC_PROTECT 339 if (unlikely((space == 2) && !(vma->vm_flags & VM_EXEC))) 340 if (!signal_return(mm, regs, address, error_code)) 341 /* 342 * signal_return() has done an up_read(&mm->mmap_sem) 343 * if it returns 0. 344 */ 345 return; 346 #endif 347 348 if (vma->vm_start <= address) 349 goto good_area; 350 if (!(vma->vm_flags & VM_GROWSDOWN)) 351 goto bad_area; 352 if (expand_stack(vma, address)) 353 goto bad_area; 354 /* 355 * Ok, we have a good vm_area for this memory access, so 356 * we can handle it.. 357 */ 358 good_area: 359 si_code = SEGV_ACCERR; 360 if (!write) { 361 /* page not present, check vm flags */ 362 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) 363 goto bad_area; 364 } else { 365 if (!(vma->vm_flags & VM_WRITE)) 366 goto bad_area; 367 } 368 369 survive: 370 /* 371 * If for any reason at all we couldn't handle the fault, 372 * make sure we exit gracefully rather than endlessly redo 373 * the fault. 374 */ 375 fault = handle_mm_fault(mm, vma, address, write); 376 if (unlikely(fault & VM_FAULT_ERROR)) { 377 if (fault & VM_FAULT_OOM) { 378 if (do_out_of_memory(regs, error_code, address)) 379 goto survive; 380 return; 381 } else if (fault & VM_FAULT_SIGBUS) { 382 do_sigbus(regs, error_code, address); 383 return; 384 } 385 BUG(); 386 } 387 if (fault & VM_FAULT_MAJOR) 388 tsk->maj_flt++; 389 else 390 tsk->min_flt++; 391 392 up_read(&mm->mmap_sem); 393 /* 394 * The instruction that caused the program check will 395 * be repeated. Don't signal single step via SIGTRAP. 396 */ 397 clear_tsk_thread_flag(tsk, TIF_SINGLE_STEP); 398 return; 399 400 /* 401 * Something tried to access memory that isn't in our memory map.. 402 * Fix it, but check if it's kernel or user first.. 403 */ 404 bad_area: 405 up_read(&mm->mmap_sem); 406 407 /* User mode accesses just cause a SIGSEGV */ 408 if (regs->psw.mask & PSW_MASK_PSTATE) { 409 tsk->thread.prot_addr = address; 410 tsk->thread.trap_no = error_code; 411 do_sigsegv(regs, error_code, si_code, address); 412 return; 413 } 414 415 no_context: 416 do_no_context(regs, error_code, address); 417 } 418 419 void __kprobes do_protection_exception(struct pt_regs *regs, 420 long error_code) 421 { 422 /* Protection exception is supressing, decrement psw address. */ 423 regs->psw.addr -= (error_code >> 16); 424 /* 425 * Check for low-address protection. This needs to be treated 426 * as a special case because the translation exception code 427 * field is not guaranteed to contain valid data in this case. 428 */ 429 if (unlikely(!(S390_lowcore.trans_exc_code & 4))) { 430 do_low_address(regs, error_code); 431 return; 432 } 433 do_exception(regs, 4, 1); 434 } 435 436 void __kprobes do_dat_exception(struct pt_regs *regs, long error_code) 437 { 438 do_exception(regs, error_code & 0xff, 0); 439 } 440 441 #ifdef CONFIG_64BIT 442 void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code) 443 { 444 struct mm_struct *mm; 445 struct vm_area_struct *vma; 446 unsigned long address; 447 int space; 448 449 mm = current->mm; 450 address = S390_lowcore.trans_exc_code & __FAIL_ADDR_MASK; 451 space = check_space(current); 452 453 if (unlikely(space == 0 || in_atomic() || !mm)) 454 goto no_context; 455 456 local_irq_enable(); 457 458 down_read(&mm->mmap_sem); 459 vma = find_vma(mm, address); 460 up_read(&mm->mmap_sem); 461 462 if (vma) { 463 update_mm(mm, current); 464 return; 465 } 466 467 /* User mode accesses just cause a SIGSEGV */ 468 if (regs->psw.mask & PSW_MASK_PSTATE) { 469 current->thread.prot_addr = address; 470 current->thread.trap_no = error_code; 471 do_sigsegv(regs, error_code, SEGV_MAPERR, address); 472 return; 473 } 474 475 no_context: 476 do_no_context(regs, error_code, address); 477 } 478 #endif 479 480 #ifdef CONFIG_PFAULT 481 /* 482 * 'pfault' pseudo page faults routines. 483 */ 484 static ext_int_info_t ext_int_pfault; 485 static int pfault_disable = 0; 486 487 static int __init nopfault(char *str) 488 { 489 pfault_disable = 1; 490 return 1; 491 } 492 493 __setup("nopfault", nopfault); 494 495 typedef struct { 496 __u16 refdiagc; 497 __u16 reffcode; 498 __u16 refdwlen; 499 __u16 refversn; 500 __u64 refgaddr; 501 __u64 refselmk; 502 __u64 refcmpmk; 503 __u64 reserved; 504 } __attribute__ ((packed, aligned(8))) pfault_refbk_t; 505 506 int pfault_init(void) 507 { 508 pfault_refbk_t refbk = 509 { 0x258, 0, 5, 2, __LC_CURRENT, 1ULL << 48, 1ULL << 48, 510 __PF_RES_FIELD }; 511 int rc; 512 513 if (!MACHINE_IS_VM || pfault_disable) 514 return -1; 515 asm volatile( 516 " diag %1,%0,0x258\n" 517 "0: j 2f\n" 518 "1: la %0,8\n" 519 "2:\n" 520 EX_TABLE(0b,1b) 521 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc"); 522 __ctl_set_bit(0, 9); 523 return rc; 524 } 525 526 void pfault_fini(void) 527 { 528 pfault_refbk_t refbk = 529 { 0x258, 1, 5, 2, 0ULL, 0ULL, 0ULL, 0ULL }; 530 531 if (!MACHINE_IS_VM || pfault_disable) 532 return; 533 __ctl_clear_bit(0,9); 534 asm volatile( 535 " diag %0,0,0x258\n" 536 "0:\n" 537 EX_TABLE(0b,0b) 538 : : "a" (&refbk), "m" (refbk) : "cc"); 539 } 540 541 static void pfault_interrupt(__u16 error_code) 542 { 543 struct task_struct *tsk; 544 __u16 subcode; 545 546 /* 547 * Get the external interruption subcode & pfault 548 * initial/completion signal bit. VM stores this 549 * in the 'cpu address' field associated with the 550 * external interrupt. 551 */ 552 subcode = S390_lowcore.cpu_addr; 553 if ((subcode & 0xff00) != __SUBCODE_MASK) 554 return; 555 556 /* 557 * Get the token (= address of the task structure of the affected task). 558 */ 559 tsk = *(struct task_struct **) __LC_PFAULT_INTPARM; 560 561 if (subcode & 0x0080) { 562 /* signal bit is set -> a page has been swapped in by VM */ 563 if (xchg(&tsk->thread.pfault_wait, -1) != 0) { 564 /* Initial interrupt was faster than the completion 565 * interrupt. pfault_wait is valid. Set pfault_wait 566 * back to zero and wake up the process. This can 567 * safely be done because the task is still sleeping 568 * and can't produce new pfaults. */ 569 tsk->thread.pfault_wait = 0; 570 wake_up_process(tsk); 571 put_task_struct(tsk); 572 } 573 } else { 574 /* signal bit not set -> a real page is missing. */ 575 get_task_struct(tsk); 576 set_task_state(tsk, TASK_UNINTERRUPTIBLE); 577 if (xchg(&tsk->thread.pfault_wait, 1) != 0) { 578 /* Completion interrupt was faster than the initial 579 * interrupt (swapped in a -1 for pfault_wait). Set 580 * pfault_wait back to zero and exit. This can be 581 * done safely because tsk is running in kernel 582 * mode and can't produce new pfaults. */ 583 tsk->thread.pfault_wait = 0; 584 set_task_state(tsk, TASK_RUNNING); 585 put_task_struct(tsk); 586 } else 587 set_tsk_need_resched(tsk); 588 } 589 } 590 591 void __init pfault_irq_init(void) 592 { 593 if (!MACHINE_IS_VM) 594 return; 595 596 /* 597 * Try to get pfault pseudo page faults going. 598 */ 599 if (register_early_external_interrupt(0x2603, pfault_interrupt, 600 &ext_int_pfault) != 0) 601 panic("Couldn't request external interrupt 0x2603"); 602 603 if (pfault_init() == 0) 604 return; 605 606 /* Tough luck, no pfault. */ 607 pfault_disable = 1; 608 unregister_early_external_interrupt(0x2603, pfault_interrupt, 609 &ext_int_pfault); 610 } 611 #endif 612