1 /* 2 * Copyright (C) 1995 Linus Torvalds 3 * Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs. 4 * Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar 5 */ 6 #include <linux/magic.h> /* STACK_END_MAGIC */ 7 #include <linux/sched.h> /* test_thread_flag(), ... */ 8 #include <linux/kdebug.h> /* oops_begin/end, ... */ 9 #include <linux/module.h> /* search_exception_table */ 10 #include <linux/bootmem.h> /* max_low_pfn */ 11 #include <linux/kprobes.h> /* NOKPROBE_SYMBOL, ... */ 12 #include <linux/mmiotrace.h> /* kmmio_handler, ... */ 13 #include <linux/perf_event.h> /* perf_sw_event */ 14 #include <linux/hugetlb.h> /* hstate_index_to_shift */ 15 #include <linux/prefetch.h> /* prefetchw */ 16 #include <linux/context_tracking.h> /* exception_enter(), ... */ 17 18 #include <asm/traps.h> /* dotraplinkage, ... */ 19 #include <asm/pgalloc.h> /* pgd_*(), ... */ 20 #include <asm/kmemcheck.h> /* kmemcheck_*(), ... */ 21 #include <asm/fixmap.h> /* VSYSCALL_ADDR */ 22 #include <asm/vsyscall.h> /* emulate_vsyscall */ 23 24 #define CREATE_TRACE_POINTS 25 #include <asm/trace/exceptions.h> 26 27 /* 28 * Page fault error code bits: 29 * 30 * bit 0 == 0: no page found 1: protection fault 31 * bit 1 == 0: read access 1: write access 32 * bit 2 == 0: kernel-mode access 1: user-mode access 33 * bit 3 == 1: use of reserved bit detected 34 * bit 4 == 1: fault was an instruction fetch 35 */ 36 enum x86_pf_error_code { 37 38 PF_PROT = 1 << 0, 39 PF_WRITE = 1 << 1, 40 PF_USER = 1 << 2, 41 PF_RSVD = 1 << 3, 42 PF_INSTR = 1 << 4, 43 }; 44 45 /* 46 * Returns 0 if mmiotrace is disabled, or if the fault is not 47 * handled by mmiotrace: 48 */ 49 static nokprobe_inline int 50 kmmio_fault(struct pt_regs *regs, unsigned long addr) 51 { 52 if (unlikely(is_kmmio_active())) 53 if (kmmio_handler(regs, addr) == 1) 54 return -1; 55 return 0; 56 } 57 58 static nokprobe_inline int kprobes_fault(struct pt_regs *regs) 59 { 60 int ret = 0; 61 62 /* kprobe_running() needs smp_processor_id() */ 63 if (kprobes_built_in() && !user_mode_vm(regs)) { 64 preempt_disable(); 65 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 66 ret = 1; 67 preempt_enable(); 68 } 69 70 return ret; 71 } 72 73 /* 74 * Prefetch quirks: 75 * 76 * 32-bit mode: 77 * 78 * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch. 79 * Check that here and ignore it. 80 * 81 * 64-bit mode: 82 * 83 * Sometimes the CPU reports invalid exceptions on prefetch. 84 * Check that here and ignore it. 85 * 86 * Opcode checker based on code by Richard Brunner. 87 */ 88 static inline int 89 check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr, 90 unsigned char opcode, int *prefetch) 91 { 92 unsigned char instr_hi = opcode & 0xf0; 93 unsigned char instr_lo = opcode & 0x0f; 94 95 switch (instr_hi) { 96 case 0x20: 97 case 0x30: 98 /* 99 * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes. 100 * In X86_64 long mode, the CPU will signal invalid 101 * opcode if some of these prefixes are present so 102 * X86_64 will never get here anyway 103 */ 104 return ((instr_lo & 7) == 0x6); 105 #ifdef CONFIG_X86_64 106 case 0x40: 107 /* 108 * In AMD64 long mode 0x40..0x4F are valid REX prefixes 109 * Need to figure out under what instruction mode the 110 * instruction was issued. Could check the LDT for lm, 111 * but for now it's good enough to assume that long 112 * mode only uses well known segments or kernel. 113 */ 114 return (!user_mode(regs) || user_64bit_mode(regs)); 115 #endif 116 case 0x60: 117 /* 0x64 thru 0x67 are valid prefixes in all modes. */ 118 return (instr_lo & 0xC) == 0x4; 119 case 0xF0: 120 /* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */ 121 return !instr_lo || (instr_lo>>1) == 1; 122 case 0x00: 123 /* Prefetch instruction is 0x0F0D or 0x0F18 */ 124 if (probe_kernel_address(instr, opcode)) 125 return 0; 126 127 *prefetch = (instr_lo == 0xF) && 128 (opcode == 0x0D || opcode == 0x18); 129 return 0; 130 default: 131 return 0; 132 } 133 } 134 135 static int 136 is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr) 137 { 138 unsigned char *max_instr; 139 unsigned char *instr; 140 int prefetch = 0; 141 142 /* 143 * If it was a exec (instruction fetch) fault on NX page, then 144 * do not ignore the fault: 145 */ 146 if (error_code & PF_INSTR) 147 return 0; 148 149 instr = (void *)convert_ip_to_linear(current, regs); 150 max_instr = instr + 15; 151 152 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE) 153 return 0; 154 155 while (instr < max_instr) { 156 unsigned char opcode; 157 158 if (probe_kernel_address(instr, opcode)) 159 break; 160 161 instr++; 162 163 if (!check_prefetch_opcode(regs, instr, opcode, &prefetch)) 164 break; 165 } 166 return prefetch; 167 } 168 169 static void 170 force_sig_info_fault(int si_signo, int si_code, unsigned long address, 171 struct task_struct *tsk, int fault) 172 { 173 unsigned lsb = 0; 174 siginfo_t info; 175 176 info.si_signo = si_signo; 177 info.si_errno = 0; 178 info.si_code = si_code; 179 info.si_addr = (void __user *)address; 180 if (fault & VM_FAULT_HWPOISON_LARGE) 181 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); 182 if (fault & VM_FAULT_HWPOISON) 183 lsb = PAGE_SHIFT; 184 info.si_addr_lsb = lsb; 185 186 force_sig_info(si_signo, &info, tsk); 187 } 188 189 DEFINE_SPINLOCK(pgd_lock); 190 LIST_HEAD(pgd_list); 191 192 #ifdef CONFIG_X86_32 193 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) 194 { 195 unsigned index = pgd_index(address); 196 pgd_t *pgd_k; 197 pud_t *pud, *pud_k; 198 pmd_t *pmd, *pmd_k; 199 200 pgd += index; 201 pgd_k = init_mm.pgd + index; 202 203 if (!pgd_present(*pgd_k)) 204 return NULL; 205 206 /* 207 * set_pgd(pgd, *pgd_k); here would be useless on PAE 208 * and redundant with the set_pmd() on non-PAE. As would 209 * set_pud. 210 */ 211 pud = pud_offset(pgd, address); 212 pud_k = pud_offset(pgd_k, address); 213 if (!pud_present(*pud_k)) 214 return NULL; 215 216 pmd = pmd_offset(pud, address); 217 pmd_k = pmd_offset(pud_k, address); 218 if (!pmd_present(*pmd_k)) 219 return NULL; 220 221 if (!pmd_present(*pmd)) 222 set_pmd(pmd, *pmd_k); 223 else 224 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k)); 225 226 return pmd_k; 227 } 228 229 void vmalloc_sync_all(void) 230 { 231 unsigned long address; 232 233 if (SHARED_KERNEL_PMD) 234 return; 235 236 for (address = VMALLOC_START & PMD_MASK; 237 address >= TASK_SIZE && address < FIXADDR_TOP; 238 address += PMD_SIZE) { 239 struct page *page; 240 241 spin_lock(&pgd_lock); 242 list_for_each_entry(page, &pgd_list, lru) { 243 spinlock_t *pgt_lock; 244 pmd_t *ret; 245 246 /* the pgt_lock only for Xen */ 247 pgt_lock = &pgd_page_get_mm(page)->page_table_lock; 248 249 spin_lock(pgt_lock); 250 ret = vmalloc_sync_one(page_address(page), address); 251 spin_unlock(pgt_lock); 252 253 if (!ret) 254 break; 255 } 256 spin_unlock(&pgd_lock); 257 } 258 } 259 260 /* 261 * 32-bit: 262 * 263 * Handle a fault on the vmalloc or module mapping area 264 */ 265 static noinline int vmalloc_fault(unsigned long address) 266 { 267 unsigned long pgd_paddr; 268 pmd_t *pmd_k; 269 pte_t *pte_k; 270 271 /* Make sure we are in vmalloc area: */ 272 if (!(address >= VMALLOC_START && address < VMALLOC_END)) 273 return -1; 274 275 WARN_ON_ONCE(in_nmi()); 276 277 /* 278 * Synchronize this task's top level page-table 279 * with the 'reference' page table. 280 * 281 * Do _not_ use "current" here. We might be inside 282 * an interrupt in the middle of a task switch.. 283 */ 284 pgd_paddr = read_cr3(); 285 pmd_k = vmalloc_sync_one(__va(pgd_paddr), address); 286 if (!pmd_k) 287 return -1; 288 289 pte_k = pte_offset_kernel(pmd_k, address); 290 if (!pte_present(*pte_k)) 291 return -1; 292 293 return 0; 294 } 295 NOKPROBE_SYMBOL(vmalloc_fault); 296 297 /* 298 * Did it hit the DOS screen memory VA from vm86 mode? 299 */ 300 static inline void 301 check_v8086_mode(struct pt_regs *regs, unsigned long address, 302 struct task_struct *tsk) 303 { 304 unsigned long bit; 305 306 if (!v8086_mode(regs)) 307 return; 308 309 bit = (address - 0xA0000) >> PAGE_SHIFT; 310 if (bit < 32) 311 tsk->thread.screen_bitmap |= 1 << bit; 312 } 313 314 static bool low_pfn(unsigned long pfn) 315 { 316 return pfn < max_low_pfn; 317 } 318 319 static void dump_pagetable(unsigned long address) 320 { 321 pgd_t *base = __va(read_cr3()); 322 pgd_t *pgd = &base[pgd_index(address)]; 323 pmd_t *pmd; 324 pte_t *pte; 325 326 #ifdef CONFIG_X86_PAE 327 printk("*pdpt = %016Lx ", pgd_val(*pgd)); 328 if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd)) 329 goto out; 330 #endif 331 pmd = pmd_offset(pud_offset(pgd, address), address); 332 printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd)); 333 334 /* 335 * We must not directly access the pte in the highpte 336 * case if the page table is located in highmem. 337 * And let's rather not kmap-atomic the pte, just in case 338 * it's allocated already: 339 */ 340 if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd)) 341 goto out; 342 343 pte = pte_offset_kernel(pmd, address); 344 printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte)); 345 out: 346 printk("\n"); 347 } 348 349 #else /* CONFIG_X86_64: */ 350 351 void vmalloc_sync_all(void) 352 { 353 sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END); 354 } 355 356 /* 357 * 64-bit: 358 * 359 * Handle a fault on the vmalloc area 360 * 361 * This assumes no large pages in there. 362 */ 363 static noinline int vmalloc_fault(unsigned long address) 364 { 365 pgd_t *pgd, *pgd_ref; 366 pud_t *pud, *pud_ref; 367 pmd_t *pmd, *pmd_ref; 368 pte_t *pte, *pte_ref; 369 370 /* Make sure we are in vmalloc area: */ 371 if (!(address >= VMALLOC_START && address < VMALLOC_END)) 372 return -1; 373 374 WARN_ON_ONCE(in_nmi()); 375 376 /* 377 * Copy kernel mappings over when needed. This can also 378 * happen within a race in page table update. In the later 379 * case just flush: 380 */ 381 pgd = pgd_offset(current->active_mm, address); 382 pgd_ref = pgd_offset_k(address); 383 if (pgd_none(*pgd_ref)) 384 return -1; 385 386 if (pgd_none(*pgd)) { 387 set_pgd(pgd, *pgd_ref); 388 arch_flush_lazy_mmu_mode(); 389 } else { 390 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref)); 391 } 392 393 /* 394 * Below here mismatches are bugs because these lower tables 395 * are shared: 396 */ 397 398 pud = pud_offset(pgd, address); 399 pud_ref = pud_offset(pgd_ref, address); 400 if (pud_none(*pud_ref)) 401 return -1; 402 403 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref)) 404 BUG(); 405 406 pmd = pmd_offset(pud, address); 407 pmd_ref = pmd_offset(pud_ref, address); 408 if (pmd_none(*pmd_ref)) 409 return -1; 410 411 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref)) 412 BUG(); 413 414 pte_ref = pte_offset_kernel(pmd_ref, address); 415 if (!pte_present(*pte_ref)) 416 return -1; 417 418 pte = pte_offset_kernel(pmd, address); 419 420 /* 421 * Don't use pte_page here, because the mappings can point 422 * outside mem_map, and the NUMA hash lookup cannot handle 423 * that: 424 */ 425 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref)) 426 BUG(); 427 428 return 0; 429 } 430 NOKPROBE_SYMBOL(vmalloc_fault); 431 432 #ifdef CONFIG_CPU_SUP_AMD 433 static const char errata93_warning[] = 434 KERN_ERR 435 "******* Your BIOS seems to not contain a fix for K8 errata #93\n" 436 "******* Working around it, but it may cause SEGVs or burn power.\n" 437 "******* Please consider a BIOS update.\n" 438 "******* Disabling USB legacy in the BIOS may also help.\n"; 439 #endif 440 441 /* 442 * No vm86 mode in 64-bit mode: 443 */ 444 static inline void 445 check_v8086_mode(struct pt_regs *regs, unsigned long address, 446 struct task_struct *tsk) 447 { 448 } 449 450 static int bad_address(void *p) 451 { 452 unsigned long dummy; 453 454 return probe_kernel_address((unsigned long *)p, dummy); 455 } 456 457 static void dump_pagetable(unsigned long address) 458 { 459 pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK); 460 pgd_t *pgd = base + pgd_index(address); 461 pud_t *pud; 462 pmd_t *pmd; 463 pte_t *pte; 464 465 if (bad_address(pgd)) 466 goto bad; 467 468 printk("PGD %lx ", pgd_val(*pgd)); 469 470 if (!pgd_present(*pgd)) 471 goto out; 472 473 pud = pud_offset(pgd, address); 474 if (bad_address(pud)) 475 goto bad; 476 477 printk("PUD %lx ", pud_val(*pud)); 478 if (!pud_present(*pud) || pud_large(*pud)) 479 goto out; 480 481 pmd = pmd_offset(pud, address); 482 if (bad_address(pmd)) 483 goto bad; 484 485 printk("PMD %lx ", pmd_val(*pmd)); 486 if (!pmd_present(*pmd) || pmd_large(*pmd)) 487 goto out; 488 489 pte = pte_offset_kernel(pmd, address); 490 if (bad_address(pte)) 491 goto bad; 492 493 printk("PTE %lx", pte_val(*pte)); 494 out: 495 printk("\n"); 496 return; 497 bad: 498 printk("BAD\n"); 499 } 500 501 #endif /* CONFIG_X86_64 */ 502 503 /* 504 * Workaround for K8 erratum #93 & buggy BIOS. 505 * 506 * BIOS SMM functions are required to use a specific workaround 507 * to avoid corruption of the 64bit RIP register on C stepping K8. 508 * 509 * A lot of BIOS that didn't get tested properly miss this. 510 * 511 * The OS sees this as a page fault with the upper 32bits of RIP cleared. 512 * Try to work around it here. 513 * 514 * Note we only handle faults in kernel here. 515 * Does nothing on 32-bit. 516 */ 517 static int is_errata93(struct pt_regs *regs, unsigned long address) 518 { 519 #if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD) 520 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD 521 || boot_cpu_data.x86 != 0xf) 522 return 0; 523 524 if (address != regs->ip) 525 return 0; 526 527 if ((address >> 32) != 0) 528 return 0; 529 530 address |= 0xffffffffUL << 32; 531 if ((address >= (u64)_stext && address <= (u64)_etext) || 532 (address >= MODULES_VADDR && address <= MODULES_END)) { 533 printk_once(errata93_warning); 534 regs->ip = address; 535 return 1; 536 } 537 #endif 538 return 0; 539 } 540 541 /* 542 * Work around K8 erratum #100 K8 in compat mode occasionally jumps 543 * to illegal addresses >4GB. 544 * 545 * We catch this in the page fault handler because these addresses 546 * are not reachable. Just detect this case and return. Any code 547 * segment in LDT is compatibility mode. 548 */ 549 static int is_errata100(struct pt_regs *regs, unsigned long address) 550 { 551 #ifdef CONFIG_X86_64 552 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32)) 553 return 1; 554 #endif 555 return 0; 556 } 557 558 static int is_f00f_bug(struct pt_regs *regs, unsigned long address) 559 { 560 #ifdef CONFIG_X86_F00F_BUG 561 unsigned long nr; 562 563 /* 564 * Pentium F0 0F C7 C8 bug workaround: 565 */ 566 if (boot_cpu_has_bug(X86_BUG_F00F)) { 567 nr = (address - idt_descr.address) >> 3; 568 569 if (nr == 6) { 570 do_invalid_op(regs, 0); 571 return 1; 572 } 573 } 574 #endif 575 return 0; 576 } 577 578 static const char nx_warning[] = KERN_CRIT 579 "kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n"; 580 581 static void 582 show_fault_oops(struct pt_regs *regs, unsigned long error_code, 583 unsigned long address) 584 { 585 if (!oops_may_print()) 586 return; 587 588 if (error_code & PF_INSTR) { 589 unsigned int level; 590 pgd_t *pgd; 591 pte_t *pte; 592 593 pgd = __va(read_cr3() & PHYSICAL_PAGE_MASK); 594 pgd += pgd_index(address); 595 596 pte = lookup_address_in_pgd(pgd, address, &level); 597 598 if (pte && pte_present(*pte) && !pte_exec(*pte)) 599 printk(nx_warning, from_kuid(&init_user_ns, current_uid())); 600 } 601 602 printk(KERN_ALERT "BUG: unable to handle kernel "); 603 if (address < PAGE_SIZE) 604 printk(KERN_CONT "NULL pointer dereference"); 605 else 606 printk(KERN_CONT "paging request"); 607 608 printk(KERN_CONT " at %p\n", (void *) address); 609 printk(KERN_ALERT "IP:"); 610 printk_address(regs->ip); 611 612 dump_pagetable(address); 613 } 614 615 static noinline void 616 pgtable_bad(struct pt_regs *regs, unsigned long error_code, 617 unsigned long address) 618 { 619 struct task_struct *tsk; 620 unsigned long flags; 621 int sig; 622 623 flags = oops_begin(); 624 tsk = current; 625 sig = SIGKILL; 626 627 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n", 628 tsk->comm, address); 629 dump_pagetable(address); 630 631 tsk->thread.cr2 = address; 632 tsk->thread.trap_nr = X86_TRAP_PF; 633 tsk->thread.error_code = error_code; 634 635 if (__die("Bad pagetable", regs, error_code)) 636 sig = 0; 637 638 oops_end(flags, regs, sig); 639 } 640 641 static noinline void 642 no_context(struct pt_regs *regs, unsigned long error_code, 643 unsigned long address, int signal, int si_code) 644 { 645 struct task_struct *tsk = current; 646 unsigned long *stackend; 647 unsigned long flags; 648 int sig; 649 650 /* Are we prepared to handle this kernel fault? */ 651 if (fixup_exception(regs)) { 652 /* 653 * Any interrupt that takes a fault gets the fixup. This makes 654 * the below recursive fault logic only apply to a faults from 655 * task context. 656 */ 657 if (in_interrupt()) 658 return; 659 660 /* 661 * Per the above we're !in_interrupt(), aka. task context. 662 * 663 * In this case we need to make sure we're not recursively 664 * faulting through the emulate_vsyscall() logic. 665 */ 666 if (current_thread_info()->sig_on_uaccess_error && signal) { 667 tsk->thread.trap_nr = X86_TRAP_PF; 668 tsk->thread.error_code = error_code | PF_USER; 669 tsk->thread.cr2 = address; 670 671 /* XXX: hwpoison faults will set the wrong code. */ 672 force_sig_info_fault(signal, si_code, address, tsk, 0); 673 } 674 675 /* 676 * Barring that, we can do the fixup and be happy. 677 */ 678 return; 679 } 680 681 /* 682 * 32-bit: 683 * 684 * Valid to do another page fault here, because if this fault 685 * had been triggered by is_prefetch fixup_exception would have 686 * handled it. 687 * 688 * 64-bit: 689 * 690 * Hall of shame of CPU/BIOS bugs. 691 */ 692 if (is_prefetch(regs, error_code, address)) 693 return; 694 695 if (is_errata93(regs, address)) 696 return; 697 698 /* 699 * Oops. The kernel tried to access some bad page. We'll have to 700 * terminate things with extreme prejudice: 701 */ 702 flags = oops_begin(); 703 704 show_fault_oops(regs, error_code, address); 705 706 stackend = end_of_stack(tsk); 707 if (tsk != &init_task && *stackend != STACK_END_MAGIC) 708 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); 709 710 tsk->thread.cr2 = address; 711 tsk->thread.trap_nr = X86_TRAP_PF; 712 tsk->thread.error_code = error_code; 713 714 sig = SIGKILL; 715 if (__die("Oops", regs, error_code)) 716 sig = 0; 717 718 /* Executive summary in case the body of the oops scrolled away */ 719 printk(KERN_DEFAULT "CR2: %016lx\n", address); 720 721 oops_end(flags, regs, sig); 722 } 723 724 /* 725 * Print out info about fatal segfaults, if the show_unhandled_signals 726 * sysctl is set: 727 */ 728 static inline void 729 show_signal_msg(struct pt_regs *regs, unsigned long error_code, 730 unsigned long address, struct task_struct *tsk) 731 { 732 if (!unhandled_signal(tsk, SIGSEGV)) 733 return; 734 735 if (!printk_ratelimit()) 736 return; 737 738 printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx", 739 task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG, 740 tsk->comm, task_pid_nr(tsk), address, 741 (void *)regs->ip, (void *)regs->sp, error_code); 742 743 print_vma_addr(KERN_CONT " in ", regs->ip); 744 745 printk(KERN_CONT "\n"); 746 } 747 748 static void 749 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, 750 unsigned long address, int si_code) 751 { 752 struct task_struct *tsk = current; 753 754 /* User mode accesses just cause a SIGSEGV */ 755 if (error_code & PF_USER) { 756 /* 757 * It's possible to have interrupts off here: 758 */ 759 local_irq_enable(); 760 761 /* 762 * Valid to do another page fault here because this one came 763 * from user space: 764 */ 765 if (is_prefetch(regs, error_code, address)) 766 return; 767 768 if (is_errata100(regs, address)) 769 return; 770 771 #ifdef CONFIG_X86_64 772 /* 773 * Instruction fetch faults in the vsyscall page might need 774 * emulation. 775 */ 776 if (unlikely((error_code & PF_INSTR) && 777 ((address & ~0xfff) == VSYSCALL_ADDR))) { 778 if (emulate_vsyscall(regs, address)) 779 return; 780 } 781 #endif 782 /* Kernel addresses are always protection faults: */ 783 if (address >= TASK_SIZE) 784 error_code |= PF_PROT; 785 786 if (likely(show_unhandled_signals)) 787 show_signal_msg(regs, error_code, address, tsk); 788 789 tsk->thread.cr2 = address; 790 tsk->thread.error_code = error_code; 791 tsk->thread.trap_nr = X86_TRAP_PF; 792 793 force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0); 794 795 return; 796 } 797 798 if (is_f00f_bug(regs, address)) 799 return; 800 801 no_context(regs, error_code, address, SIGSEGV, si_code); 802 } 803 804 static noinline void 805 bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, 806 unsigned long address) 807 { 808 __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR); 809 } 810 811 static void 812 __bad_area(struct pt_regs *regs, unsigned long error_code, 813 unsigned long address, int si_code) 814 { 815 struct mm_struct *mm = current->mm; 816 817 /* 818 * Something tried to access memory that isn't in our memory map.. 819 * Fix it, but check if it's kernel or user first.. 820 */ 821 up_read(&mm->mmap_sem); 822 823 __bad_area_nosemaphore(regs, error_code, address, si_code); 824 } 825 826 static noinline void 827 bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address) 828 { 829 __bad_area(regs, error_code, address, SEGV_MAPERR); 830 } 831 832 static noinline void 833 bad_area_access_error(struct pt_regs *regs, unsigned long error_code, 834 unsigned long address) 835 { 836 __bad_area(regs, error_code, address, SEGV_ACCERR); 837 } 838 839 static void 840 do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address, 841 unsigned int fault) 842 { 843 struct task_struct *tsk = current; 844 struct mm_struct *mm = tsk->mm; 845 int code = BUS_ADRERR; 846 847 up_read(&mm->mmap_sem); 848 849 /* Kernel mode? Handle exceptions or die: */ 850 if (!(error_code & PF_USER)) { 851 no_context(regs, error_code, address, SIGBUS, BUS_ADRERR); 852 return; 853 } 854 855 /* User-space => ok to do another page fault: */ 856 if (is_prefetch(regs, error_code, address)) 857 return; 858 859 tsk->thread.cr2 = address; 860 tsk->thread.error_code = error_code; 861 tsk->thread.trap_nr = X86_TRAP_PF; 862 863 #ifdef CONFIG_MEMORY_FAILURE 864 if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { 865 printk(KERN_ERR 866 "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n", 867 tsk->comm, tsk->pid, address); 868 code = BUS_MCEERR_AR; 869 } 870 #endif 871 force_sig_info_fault(SIGBUS, code, address, tsk, fault); 872 } 873 874 static noinline void 875 mm_fault_error(struct pt_regs *regs, unsigned long error_code, 876 unsigned long address, unsigned int fault) 877 { 878 if (fatal_signal_pending(current) && !(error_code & PF_USER)) { 879 up_read(¤t->mm->mmap_sem); 880 no_context(regs, error_code, address, 0, 0); 881 return; 882 } 883 884 if (fault & VM_FAULT_OOM) { 885 /* Kernel mode? Handle exceptions or die: */ 886 if (!(error_code & PF_USER)) { 887 up_read(¤t->mm->mmap_sem); 888 no_context(regs, error_code, address, 889 SIGSEGV, SEGV_MAPERR); 890 return; 891 } 892 893 up_read(¤t->mm->mmap_sem); 894 895 /* 896 * We ran out of memory, call the OOM killer, and return the 897 * userspace (which will retry the fault, or kill us if we got 898 * oom-killed): 899 */ 900 pagefault_out_of_memory(); 901 } else { 902 if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON| 903 VM_FAULT_HWPOISON_LARGE)) 904 do_sigbus(regs, error_code, address, fault); 905 else 906 BUG(); 907 } 908 } 909 910 static int spurious_fault_check(unsigned long error_code, pte_t *pte) 911 { 912 if ((error_code & PF_WRITE) && !pte_write(*pte)) 913 return 0; 914 915 if ((error_code & PF_INSTR) && !pte_exec(*pte)) 916 return 0; 917 918 return 1; 919 } 920 921 /* 922 * Handle a spurious fault caused by a stale TLB entry. 923 * 924 * This allows us to lazily refresh the TLB when increasing the 925 * permissions of a kernel page (RO -> RW or NX -> X). Doing it 926 * eagerly is very expensive since that implies doing a full 927 * cross-processor TLB flush, even if no stale TLB entries exist 928 * on other processors. 929 * 930 * There are no security implications to leaving a stale TLB when 931 * increasing the permissions on a page. 932 */ 933 static noinline int 934 spurious_fault(unsigned long error_code, unsigned long address) 935 { 936 pgd_t *pgd; 937 pud_t *pud; 938 pmd_t *pmd; 939 pte_t *pte; 940 int ret; 941 942 /* Reserved-bit violation or user access to kernel space? */ 943 if (error_code & (PF_USER | PF_RSVD)) 944 return 0; 945 946 pgd = init_mm.pgd + pgd_index(address); 947 if (!pgd_present(*pgd)) 948 return 0; 949 950 pud = pud_offset(pgd, address); 951 if (!pud_present(*pud)) 952 return 0; 953 954 if (pud_large(*pud)) 955 return spurious_fault_check(error_code, (pte_t *) pud); 956 957 pmd = pmd_offset(pud, address); 958 if (!pmd_present(*pmd)) 959 return 0; 960 961 if (pmd_large(*pmd)) 962 return spurious_fault_check(error_code, (pte_t *) pmd); 963 964 pte = pte_offset_kernel(pmd, address); 965 if (!pte_present(*pte)) 966 return 0; 967 968 ret = spurious_fault_check(error_code, pte); 969 if (!ret) 970 return 0; 971 972 /* 973 * Make sure we have permissions in PMD. 974 * If not, then there's a bug in the page tables: 975 */ 976 ret = spurious_fault_check(error_code, (pte_t *) pmd); 977 WARN_ONCE(!ret, "PMD has incorrect permission bits\n"); 978 979 return ret; 980 } 981 NOKPROBE_SYMBOL(spurious_fault); 982 983 int show_unhandled_signals = 1; 984 985 static inline int 986 access_error(unsigned long error_code, struct vm_area_struct *vma) 987 { 988 if (error_code & PF_WRITE) { 989 /* write, present and write, not present: */ 990 if (unlikely(!(vma->vm_flags & VM_WRITE))) 991 return 1; 992 return 0; 993 } 994 995 /* read, present: */ 996 if (unlikely(error_code & PF_PROT)) 997 return 1; 998 999 /* read, not present: */ 1000 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) 1001 return 1; 1002 1003 return 0; 1004 } 1005 1006 static int fault_in_kernel_space(unsigned long address) 1007 { 1008 return address >= TASK_SIZE_MAX; 1009 } 1010 1011 static inline bool smap_violation(int error_code, struct pt_regs *regs) 1012 { 1013 if (!IS_ENABLED(CONFIG_X86_SMAP)) 1014 return false; 1015 1016 if (!static_cpu_has(X86_FEATURE_SMAP)) 1017 return false; 1018 1019 if (error_code & PF_USER) 1020 return false; 1021 1022 if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC)) 1023 return false; 1024 1025 return true; 1026 } 1027 1028 /* 1029 * This routine handles page faults. It determines the address, 1030 * and the problem, and then passes it off to one of the appropriate 1031 * routines. 1032 * 1033 * This function must have noinline because both callers 1034 * {,trace_}do_page_fault() have notrace on. Having this an actual function 1035 * guarantees there's a function trace entry. 1036 */ 1037 static noinline void 1038 __do_page_fault(struct pt_regs *regs, unsigned long error_code, 1039 unsigned long address) 1040 { 1041 struct vm_area_struct *vma; 1042 struct task_struct *tsk; 1043 struct mm_struct *mm; 1044 int fault; 1045 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 1046 1047 tsk = current; 1048 mm = tsk->mm; 1049 1050 /* 1051 * Detect and handle instructions that would cause a page fault for 1052 * both a tracked kernel page and a userspace page. 1053 */ 1054 if (kmemcheck_active(regs)) 1055 kmemcheck_hide(regs); 1056 prefetchw(&mm->mmap_sem); 1057 1058 if (unlikely(kmmio_fault(regs, address))) 1059 return; 1060 1061 /* 1062 * We fault-in kernel-space virtual memory on-demand. The 1063 * 'reference' page table is init_mm.pgd. 1064 * 1065 * NOTE! We MUST NOT take any locks for this case. We may 1066 * be in an interrupt or a critical region, and should 1067 * only copy the information from the master page table, 1068 * nothing more. 1069 * 1070 * This verifies that the fault happens in kernel space 1071 * (error_code & 4) == 0, and that the fault was not a 1072 * protection error (error_code & 9) == 0. 1073 */ 1074 if (unlikely(fault_in_kernel_space(address))) { 1075 if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) { 1076 if (vmalloc_fault(address) >= 0) 1077 return; 1078 1079 if (kmemcheck_fault(regs, address, error_code)) 1080 return; 1081 } 1082 1083 /* Can handle a stale RO->RW TLB: */ 1084 if (spurious_fault(error_code, address)) 1085 return; 1086 1087 /* kprobes don't want to hook the spurious faults: */ 1088 if (kprobes_fault(regs)) 1089 return; 1090 /* 1091 * Don't take the mm semaphore here. If we fixup a prefetch 1092 * fault we could otherwise deadlock: 1093 */ 1094 bad_area_nosemaphore(regs, error_code, address); 1095 1096 return; 1097 } 1098 1099 /* kprobes don't want to hook the spurious faults: */ 1100 if (unlikely(kprobes_fault(regs))) 1101 return; 1102 1103 if (unlikely(error_code & PF_RSVD)) 1104 pgtable_bad(regs, error_code, address); 1105 1106 if (unlikely(smap_violation(error_code, regs))) { 1107 bad_area_nosemaphore(regs, error_code, address); 1108 return; 1109 } 1110 1111 /* 1112 * If we're in an interrupt, have no user context or are running 1113 * in an atomic region then we must not take the fault: 1114 */ 1115 if (unlikely(in_atomic() || !mm)) { 1116 bad_area_nosemaphore(regs, error_code, address); 1117 return; 1118 } 1119 1120 /* 1121 * It's safe to allow irq's after cr2 has been saved and the 1122 * vmalloc fault has been handled. 1123 * 1124 * User-mode registers count as a user access even for any 1125 * potential system fault or CPU buglet: 1126 */ 1127 if (user_mode_vm(regs)) { 1128 local_irq_enable(); 1129 error_code |= PF_USER; 1130 flags |= FAULT_FLAG_USER; 1131 } else { 1132 if (regs->flags & X86_EFLAGS_IF) 1133 local_irq_enable(); 1134 } 1135 1136 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 1137 1138 if (error_code & PF_WRITE) 1139 flags |= FAULT_FLAG_WRITE; 1140 1141 /* 1142 * When running in the kernel we expect faults to occur only to 1143 * addresses in user space. All other faults represent errors in 1144 * the kernel and should generate an OOPS. Unfortunately, in the 1145 * case of an erroneous fault occurring in a code path which already 1146 * holds mmap_sem we will deadlock attempting to validate the fault 1147 * against the address space. Luckily the kernel only validly 1148 * references user space from well defined areas of code, which are 1149 * listed in the exceptions table. 1150 * 1151 * As the vast majority of faults will be valid we will only perform 1152 * the source reference check when there is a possibility of a 1153 * deadlock. Attempt to lock the address space, if we cannot we then 1154 * validate the source. If this is invalid we can skip the address 1155 * space check, thus avoiding the deadlock: 1156 */ 1157 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 1158 if ((error_code & PF_USER) == 0 && 1159 !search_exception_tables(regs->ip)) { 1160 bad_area_nosemaphore(regs, error_code, address); 1161 return; 1162 } 1163 retry: 1164 down_read(&mm->mmap_sem); 1165 } else { 1166 /* 1167 * The above down_read_trylock() might have succeeded in 1168 * which case we'll have missed the might_sleep() from 1169 * down_read(): 1170 */ 1171 might_sleep(); 1172 } 1173 1174 vma = find_vma(mm, address); 1175 if (unlikely(!vma)) { 1176 bad_area(regs, error_code, address); 1177 return; 1178 } 1179 if (likely(vma->vm_start <= address)) 1180 goto good_area; 1181 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { 1182 bad_area(regs, error_code, address); 1183 return; 1184 } 1185 if (error_code & PF_USER) { 1186 /* 1187 * Accessing the stack below %sp is always a bug. 1188 * The large cushion allows instructions like enter 1189 * and pusha to work. ("enter $65535, $31" pushes 1190 * 32 pointers and then decrements %sp by 65535.) 1191 */ 1192 if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) { 1193 bad_area(regs, error_code, address); 1194 return; 1195 } 1196 } 1197 if (unlikely(expand_stack(vma, address))) { 1198 bad_area(regs, error_code, address); 1199 return; 1200 } 1201 1202 /* 1203 * Ok, we have a good vm_area for this memory access, so 1204 * we can handle it.. 1205 */ 1206 good_area: 1207 if (unlikely(access_error(error_code, vma))) { 1208 bad_area_access_error(regs, error_code, address); 1209 return; 1210 } 1211 1212 /* 1213 * If for any reason at all we couldn't handle the fault, 1214 * make sure we exit gracefully rather than endlessly redo 1215 * the fault: 1216 */ 1217 fault = handle_mm_fault(mm, vma, address, flags); 1218 1219 /* 1220 * If we need to retry but a fatal signal is pending, handle the 1221 * signal first. We do not need to release the mmap_sem because it 1222 * would already be released in __lock_page_or_retry in mm/filemap.c. 1223 */ 1224 if (unlikely((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))) 1225 return; 1226 1227 if (unlikely(fault & VM_FAULT_ERROR)) { 1228 mm_fault_error(regs, error_code, address, fault); 1229 return; 1230 } 1231 1232 /* 1233 * Major/minor page fault accounting is only done on the 1234 * initial attempt. If we go through a retry, it is extremely 1235 * likely that the page will be found in page cache at that point. 1236 */ 1237 if (flags & FAULT_FLAG_ALLOW_RETRY) { 1238 if (fault & VM_FAULT_MAJOR) { 1239 tsk->maj_flt++; 1240 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 1241 regs, address); 1242 } else { 1243 tsk->min_flt++; 1244 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 1245 regs, address); 1246 } 1247 if (fault & VM_FAULT_RETRY) { 1248 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 1249 * of starvation. */ 1250 flags &= ~FAULT_FLAG_ALLOW_RETRY; 1251 flags |= FAULT_FLAG_TRIED; 1252 goto retry; 1253 } 1254 } 1255 1256 check_v8086_mode(regs, address, tsk); 1257 1258 up_read(&mm->mmap_sem); 1259 } 1260 NOKPROBE_SYMBOL(__do_page_fault); 1261 1262 dotraplinkage void notrace 1263 do_page_fault(struct pt_regs *regs, unsigned long error_code) 1264 { 1265 unsigned long address = read_cr2(); /* Get the faulting address */ 1266 enum ctx_state prev_state; 1267 1268 /* 1269 * We must have this function tagged with __kprobes, notrace and call 1270 * read_cr2() before calling anything else. To avoid calling any kind 1271 * of tracing machinery before we've observed the CR2 value. 1272 * 1273 * exception_{enter,exit}() contain all sorts of tracepoints. 1274 */ 1275 1276 prev_state = exception_enter(); 1277 __do_page_fault(regs, error_code, address); 1278 exception_exit(prev_state); 1279 } 1280 NOKPROBE_SYMBOL(do_page_fault); 1281 1282 #ifdef CONFIG_TRACING 1283 static nokprobe_inline void 1284 trace_page_fault_entries(unsigned long address, struct pt_regs *regs, 1285 unsigned long error_code) 1286 { 1287 if (user_mode(regs)) 1288 trace_page_fault_user(address, regs, error_code); 1289 else 1290 trace_page_fault_kernel(address, regs, error_code); 1291 } 1292 1293 dotraplinkage void notrace 1294 trace_do_page_fault(struct pt_regs *regs, unsigned long error_code) 1295 { 1296 /* 1297 * The exception_enter and tracepoint processing could 1298 * trigger another page faults (user space callchain 1299 * reading) and destroy the original cr2 value, so read 1300 * the faulting address now. 1301 */ 1302 unsigned long address = read_cr2(); 1303 enum ctx_state prev_state; 1304 1305 prev_state = exception_enter(); 1306 trace_page_fault_entries(address, regs, error_code); 1307 __do_page_fault(regs, error_code, address); 1308 exception_exit(prev_state); 1309 } 1310 NOKPROBE_SYMBOL(trace_do_page_fault); 1311 #endif /* CONFIG_TRACING */ 1312