xref: /openbmc/linux/arch/x86/mm/fault.c (revision e8f6f3b4)
1 /*
2  *  Copyright (C) 1995  Linus Torvalds
3  *  Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
4  *  Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
5  */
6 #include <linux/sched.h>		/* test_thread_flag(), ...	*/
7 #include <linux/kdebug.h>		/* oops_begin/end, ...		*/
8 #include <linux/module.h>		/* search_exception_table	*/
9 #include <linux/bootmem.h>		/* max_low_pfn			*/
10 #include <linux/kprobes.h>		/* NOKPROBE_SYMBOL, ...		*/
11 #include <linux/mmiotrace.h>		/* kmmio_handler, ...		*/
12 #include <linux/perf_event.h>		/* perf_sw_event		*/
13 #include <linux/hugetlb.h>		/* hstate_index_to_shift	*/
14 #include <linux/prefetch.h>		/* prefetchw			*/
15 #include <linux/context_tracking.h>	/* exception_enter(), ...	*/
16 
17 #include <asm/traps.h>			/* dotraplinkage, ...		*/
18 #include <asm/pgalloc.h>		/* pgd_*(), ...			*/
19 #include <asm/kmemcheck.h>		/* kmemcheck_*(), ...		*/
20 #include <asm/fixmap.h>			/* VSYSCALL_ADDR		*/
21 #include <asm/vsyscall.h>		/* emulate_vsyscall		*/
22 
23 #define CREATE_TRACE_POINTS
24 #include <asm/trace/exceptions.h>
25 
26 /*
27  * Page fault error code bits:
28  *
29  *   bit 0 ==	 0: no page found	1: protection fault
30  *   bit 1 ==	 0: read access		1: write access
31  *   bit 2 ==	 0: kernel-mode access	1: user-mode access
32  *   bit 3 ==				1: use of reserved bit detected
33  *   bit 4 ==				1: fault was an instruction fetch
34  */
35 enum x86_pf_error_code {
36 
37 	PF_PROT		=		1 << 0,
38 	PF_WRITE	=		1 << 1,
39 	PF_USER		=		1 << 2,
40 	PF_RSVD		=		1 << 3,
41 	PF_INSTR	=		1 << 4,
42 };
43 
44 /*
45  * Returns 0 if mmiotrace is disabled, or if the fault is not
46  * handled by mmiotrace:
47  */
48 static nokprobe_inline int
49 kmmio_fault(struct pt_regs *regs, unsigned long addr)
50 {
51 	if (unlikely(is_kmmio_active()))
52 		if (kmmio_handler(regs, addr) == 1)
53 			return -1;
54 	return 0;
55 }
56 
57 static nokprobe_inline int kprobes_fault(struct pt_regs *regs)
58 {
59 	int ret = 0;
60 
61 	/* kprobe_running() needs smp_processor_id() */
62 	if (kprobes_built_in() && !user_mode_vm(regs)) {
63 		preempt_disable();
64 		if (kprobe_running() && kprobe_fault_handler(regs, 14))
65 			ret = 1;
66 		preempt_enable();
67 	}
68 
69 	return ret;
70 }
71 
72 /*
73  * Prefetch quirks:
74  *
75  * 32-bit mode:
76  *
77  *   Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
78  *   Check that here and ignore it.
79  *
80  * 64-bit mode:
81  *
82  *   Sometimes the CPU reports invalid exceptions on prefetch.
83  *   Check that here and ignore it.
84  *
85  * Opcode checker based on code by Richard Brunner.
86  */
87 static inline int
88 check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
89 		      unsigned char opcode, int *prefetch)
90 {
91 	unsigned char instr_hi = opcode & 0xf0;
92 	unsigned char instr_lo = opcode & 0x0f;
93 
94 	switch (instr_hi) {
95 	case 0x20:
96 	case 0x30:
97 		/*
98 		 * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
99 		 * In X86_64 long mode, the CPU will signal invalid
100 		 * opcode if some of these prefixes are present so
101 		 * X86_64 will never get here anyway
102 		 */
103 		return ((instr_lo & 7) == 0x6);
104 #ifdef CONFIG_X86_64
105 	case 0x40:
106 		/*
107 		 * In AMD64 long mode 0x40..0x4F are valid REX prefixes
108 		 * Need to figure out under what instruction mode the
109 		 * instruction was issued. Could check the LDT for lm,
110 		 * but for now it's good enough to assume that long
111 		 * mode only uses well known segments or kernel.
112 		 */
113 		return (!user_mode(regs) || user_64bit_mode(regs));
114 #endif
115 	case 0x60:
116 		/* 0x64 thru 0x67 are valid prefixes in all modes. */
117 		return (instr_lo & 0xC) == 0x4;
118 	case 0xF0:
119 		/* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
120 		return !instr_lo || (instr_lo>>1) == 1;
121 	case 0x00:
122 		/* Prefetch instruction is 0x0F0D or 0x0F18 */
123 		if (probe_kernel_address(instr, opcode))
124 			return 0;
125 
126 		*prefetch = (instr_lo == 0xF) &&
127 			(opcode == 0x0D || opcode == 0x18);
128 		return 0;
129 	default:
130 		return 0;
131 	}
132 }
133 
134 static int
135 is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
136 {
137 	unsigned char *max_instr;
138 	unsigned char *instr;
139 	int prefetch = 0;
140 
141 	/*
142 	 * If it was a exec (instruction fetch) fault on NX page, then
143 	 * do not ignore the fault:
144 	 */
145 	if (error_code & PF_INSTR)
146 		return 0;
147 
148 	instr = (void *)convert_ip_to_linear(current, regs);
149 	max_instr = instr + 15;
150 
151 	if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
152 		return 0;
153 
154 	while (instr < max_instr) {
155 		unsigned char opcode;
156 
157 		if (probe_kernel_address(instr, opcode))
158 			break;
159 
160 		instr++;
161 
162 		if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
163 			break;
164 	}
165 	return prefetch;
166 }
167 
168 static void
169 force_sig_info_fault(int si_signo, int si_code, unsigned long address,
170 		     struct task_struct *tsk, int fault)
171 {
172 	unsigned lsb = 0;
173 	siginfo_t info;
174 
175 	info.si_signo	= si_signo;
176 	info.si_errno	= 0;
177 	info.si_code	= si_code;
178 	info.si_addr	= (void __user *)address;
179 	if (fault & VM_FAULT_HWPOISON_LARGE)
180 		lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
181 	if (fault & VM_FAULT_HWPOISON)
182 		lsb = PAGE_SHIFT;
183 	info.si_addr_lsb = lsb;
184 
185 	force_sig_info(si_signo, &info, tsk);
186 }
187 
188 DEFINE_SPINLOCK(pgd_lock);
189 LIST_HEAD(pgd_list);
190 
191 #ifdef CONFIG_X86_32
192 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
193 {
194 	unsigned index = pgd_index(address);
195 	pgd_t *pgd_k;
196 	pud_t *pud, *pud_k;
197 	pmd_t *pmd, *pmd_k;
198 
199 	pgd += index;
200 	pgd_k = init_mm.pgd + index;
201 
202 	if (!pgd_present(*pgd_k))
203 		return NULL;
204 
205 	/*
206 	 * set_pgd(pgd, *pgd_k); here would be useless on PAE
207 	 * and redundant with the set_pmd() on non-PAE. As would
208 	 * set_pud.
209 	 */
210 	pud = pud_offset(pgd, address);
211 	pud_k = pud_offset(pgd_k, address);
212 	if (!pud_present(*pud_k))
213 		return NULL;
214 
215 	pmd = pmd_offset(pud, address);
216 	pmd_k = pmd_offset(pud_k, address);
217 	if (!pmd_present(*pmd_k))
218 		return NULL;
219 
220 	if (!pmd_present(*pmd))
221 		set_pmd(pmd, *pmd_k);
222 	else
223 		BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
224 
225 	return pmd_k;
226 }
227 
228 void vmalloc_sync_all(void)
229 {
230 	unsigned long address;
231 
232 	if (SHARED_KERNEL_PMD)
233 		return;
234 
235 	for (address = VMALLOC_START & PMD_MASK;
236 	     address >= TASK_SIZE && address < FIXADDR_TOP;
237 	     address += PMD_SIZE) {
238 		struct page *page;
239 
240 		spin_lock(&pgd_lock);
241 		list_for_each_entry(page, &pgd_list, lru) {
242 			spinlock_t *pgt_lock;
243 			pmd_t *ret;
244 
245 			/* the pgt_lock only for Xen */
246 			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
247 
248 			spin_lock(pgt_lock);
249 			ret = vmalloc_sync_one(page_address(page), address);
250 			spin_unlock(pgt_lock);
251 
252 			if (!ret)
253 				break;
254 		}
255 		spin_unlock(&pgd_lock);
256 	}
257 }
258 
259 /*
260  * 32-bit:
261  *
262  *   Handle a fault on the vmalloc or module mapping area
263  */
264 static noinline int vmalloc_fault(unsigned long address)
265 {
266 	unsigned long pgd_paddr;
267 	pmd_t *pmd_k;
268 	pte_t *pte_k;
269 
270 	/* Make sure we are in vmalloc area: */
271 	if (!(address >= VMALLOC_START && address < VMALLOC_END))
272 		return -1;
273 
274 	WARN_ON_ONCE(in_nmi());
275 
276 	/*
277 	 * Synchronize this task's top level page-table
278 	 * with the 'reference' page table.
279 	 *
280 	 * Do _not_ use "current" here. We might be inside
281 	 * an interrupt in the middle of a task switch..
282 	 */
283 	pgd_paddr = read_cr3();
284 	pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
285 	if (!pmd_k)
286 		return -1;
287 
288 	pte_k = pte_offset_kernel(pmd_k, address);
289 	if (!pte_present(*pte_k))
290 		return -1;
291 
292 	return 0;
293 }
294 NOKPROBE_SYMBOL(vmalloc_fault);
295 
296 /*
297  * Did it hit the DOS screen memory VA from vm86 mode?
298  */
299 static inline void
300 check_v8086_mode(struct pt_regs *regs, unsigned long address,
301 		 struct task_struct *tsk)
302 {
303 	unsigned long bit;
304 
305 	if (!v8086_mode(regs))
306 		return;
307 
308 	bit = (address - 0xA0000) >> PAGE_SHIFT;
309 	if (bit < 32)
310 		tsk->thread.screen_bitmap |= 1 << bit;
311 }
312 
313 static bool low_pfn(unsigned long pfn)
314 {
315 	return pfn < max_low_pfn;
316 }
317 
318 static void dump_pagetable(unsigned long address)
319 {
320 	pgd_t *base = __va(read_cr3());
321 	pgd_t *pgd = &base[pgd_index(address)];
322 	pmd_t *pmd;
323 	pte_t *pte;
324 
325 #ifdef CONFIG_X86_PAE
326 	printk("*pdpt = %016Lx ", pgd_val(*pgd));
327 	if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
328 		goto out;
329 #endif
330 	pmd = pmd_offset(pud_offset(pgd, address), address);
331 	printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
332 
333 	/*
334 	 * We must not directly access the pte in the highpte
335 	 * case if the page table is located in highmem.
336 	 * And let's rather not kmap-atomic the pte, just in case
337 	 * it's allocated already:
338 	 */
339 	if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
340 		goto out;
341 
342 	pte = pte_offset_kernel(pmd, address);
343 	printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
344 out:
345 	printk("\n");
346 }
347 
348 #else /* CONFIG_X86_64: */
349 
350 void vmalloc_sync_all(void)
351 {
352 	sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END, 0);
353 }
354 
355 /*
356  * 64-bit:
357  *
358  *   Handle a fault on the vmalloc area
359  *
360  * This assumes no large pages in there.
361  */
362 static noinline int vmalloc_fault(unsigned long address)
363 {
364 	pgd_t *pgd, *pgd_ref;
365 	pud_t *pud, *pud_ref;
366 	pmd_t *pmd, *pmd_ref;
367 	pte_t *pte, *pte_ref;
368 
369 	/* Make sure we are in vmalloc area: */
370 	if (!(address >= VMALLOC_START && address < VMALLOC_END))
371 		return -1;
372 
373 	WARN_ON_ONCE(in_nmi());
374 
375 	/*
376 	 * Copy kernel mappings over when needed. This can also
377 	 * happen within a race in page table update. In the later
378 	 * case just flush:
379 	 */
380 	pgd = pgd_offset(current->active_mm, address);
381 	pgd_ref = pgd_offset_k(address);
382 	if (pgd_none(*pgd_ref))
383 		return -1;
384 
385 	if (pgd_none(*pgd)) {
386 		set_pgd(pgd, *pgd_ref);
387 		arch_flush_lazy_mmu_mode();
388 	} else {
389 		BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
390 	}
391 
392 	/*
393 	 * Below here mismatches are bugs because these lower tables
394 	 * are shared:
395 	 */
396 
397 	pud = pud_offset(pgd, address);
398 	pud_ref = pud_offset(pgd_ref, address);
399 	if (pud_none(*pud_ref))
400 		return -1;
401 
402 	if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
403 		BUG();
404 
405 	pmd = pmd_offset(pud, address);
406 	pmd_ref = pmd_offset(pud_ref, address);
407 	if (pmd_none(*pmd_ref))
408 		return -1;
409 
410 	if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
411 		BUG();
412 
413 	pte_ref = pte_offset_kernel(pmd_ref, address);
414 	if (!pte_present(*pte_ref))
415 		return -1;
416 
417 	pte = pte_offset_kernel(pmd, address);
418 
419 	/*
420 	 * Don't use pte_page here, because the mappings can point
421 	 * outside mem_map, and the NUMA hash lookup cannot handle
422 	 * that:
423 	 */
424 	if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
425 		BUG();
426 
427 	return 0;
428 }
429 NOKPROBE_SYMBOL(vmalloc_fault);
430 
431 #ifdef CONFIG_CPU_SUP_AMD
432 static const char errata93_warning[] =
433 KERN_ERR
434 "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
435 "******* Working around it, but it may cause SEGVs or burn power.\n"
436 "******* Please consider a BIOS update.\n"
437 "******* Disabling USB legacy in the BIOS may also help.\n";
438 #endif
439 
440 /*
441  * No vm86 mode in 64-bit mode:
442  */
443 static inline void
444 check_v8086_mode(struct pt_regs *regs, unsigned long address,
445 		 struct task_struct *tsk)
446 {
447 }
448 
449 static int bad_address(void *p)
450 {
451 	unsigned long dummy;
452 
453 	return probe_kernel_address((unsigned long *)p, dummy);
454 }
455 
456 static void dump_pagetable(unsigned long address)
457 {
458 	pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
459 	pgd_t *pgd = base + pgd_index(address);
460 	pud_t *pud;
461 	pmd_t *pmd;
462 	pte_t *pte;
463 
464 	if (bad_address(pgd))
465 		goto bad;
466 
467 	printk("PGD %lx ", pgd_val(*pgd));
468 
469 	if (!pgd_present(*pgd))
470 		goto out;
471 
472 	pud = pud_offset(pgd, address);
473 	if (bad_address(pud))
474 		goto bad;
475 
476 	printk("PUD %lx ", pud_val(*pud));
477 	if (!pud_present(*pud) || pud_large(*pud))
478 		goto out;
479 
480 	pmd = pmd_offset(pud, address);
481 	if (bad_address(pmd))
482 		goto bad;
483 
484 	printk("PMD %lx ", pmd_val(*pmd));
485 	if (!pmd_present(*pmd) || pmd_large(*pmd))
486 		goto out;
487 
488 	pte = pte_offset_kernel(pmd, address);
489 	if (bad_address(pte))
490 		goto bad;
491 
492 	printk("PTE %lx", pte_val(*pte));
493 out:
494 	printk("\n");
495 	return;
496 bad:
497 	printk("BAD\n");
498 }
499 
500 #endif /* CONFIG_X86_64 */
501 
502 /*
503  * Workaround for K8 erratum #93 & buggy BIOS.
504  *
505  * BIOS SMM functions are required to use a specific workaround
506  * to avoid corruption of the 64bit RIP register on C stepping K8.
507  *
508  * A lot of BIOS that didn't get tested properly miss this.
509  *
510  * The OS sees this as a page fault with the upper 32bits of RIP cleared.
511  * Try to work around it here.
512  *
513  * Note we only handle faults in kernel here.
514  * Does nothing on 32-bit.
515  */
516 static int is_errata93(struct pt_regs *regs, unsigned long address)
517 {
518 #if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
519 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
520 	    || boot_cpu_data.x86 != 0xf)
521 		return 0;
522 
523 	if (address != regs->ip)
524 		return 0;
525 
526 	if ((address >> 32) != 0)
527 		return 0;
528 
529 	address |= 0xffffffffUL << 32;
530 	if ((address >= (u64)_stext && address <= (u64)_etext) ||
531 	    (address >= MODULES_VADDR && address <= MODULES_END)) {
532 		printk_once(errata93_warning);
533 		regs->ip = address;
534 		return 1;
535 	}
536 #endif
537 	return 0;
538 }
539 
540 /*
541  * Work around K8 erratum #100 K8 in compat mode occasionally jumps
542  * to illegal addresses >4GB.
543  *
544  * We catch this in the page fault handler because these addresses
545  * are not reachable. Just detect this case and return.  Any code
546  * segment in LDT is compatibility mode.
547  */
548 static int is_errata100(struct pt_regs *regs, unsigned long address)
549 {
550 #ifdef CONFIG_X86_64
551 	if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
552 		return 1;
553 #endif
554 	return 0;
555 }
556 
557 static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
558 {
559 #ifdef CONFIG_X86_F00F_BUG
560 	unsigned long nr;
561 
562 	/*
563 	 * Pentium F0 0F C7 C8 bug workaround:
564 	 */
565 	if (boot_cpu_has_bug(X86_BUG_F00F)) {
566 		nr = (address - idt_descr.address) >> 3;
567 
568 		if (nr == 6) {
569 			do_invalid_op(regs, 0);
570 			return 1;
571 		}
572 	}
573 #endif
574 	return 0;
575 }
576 
577 static const char nx_warning[] = KERN_CRIT
578 "kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
579 static const char smep_warning[] = KERN_CRIT
580 "unable to execute userspace code (SMEP?) (uid: %d)\n";
581 
582 static void
583 show_fault_oops(struct pt_regs *regs, unsigned long error_code,
584 		unsigned long address)
585 {
586 	if (!oops_may_print())
587 		return;
588 
589 	if (error_code & PF_INSTR) {
590 		unsigned int level;
591 		pgd_t *pgd;
592 		pte_t *pte;
593 
594 		pgd = __va(read_cr3() & PHYSICAL_PAGE_MASK);
595 		pgd += pgd_index(address);
596 
597 		pte = lookup_address_in_pgd(pgd, address, &level);
598 
599 		if (pte && pte_present(*pte) && !pte_exec(*pte))
600 			printk(nx_warning, from_kuid(&init_user_ns, current_uid()));
601 		if (pte && pte_present(*pte) && pte_exec(*pte) &&
602 				(pgd_flags(*pgd) & _PAGE_USER) &&
603 				(read_cr4() & X86_CR4_SMEP))
604 			printk(smep_warning, from_kuid(&init_user_ns, current_uid()));
605 	}
606 
607 	printk(KERN_ALERT "BUG: unable to handle kernel ");
608 	if (address < PAGE_SIZE)
609 		printk(KERN_CONT "NULL pointer dereference");
610 	else
611 		printk(KERN_CONT "paging request");
612 
613 	printk(KERN_CONT " at %p\n", (void *) address);
614 	printk(KERN_ALERT "IP:");
615 	printk_address(regs->ip);
616 
617 	dump_pagetable(address);
618 }
619 
620 static noinline void
621 pgtable_bad(struct pt_regs *regs, unsigned long error_code,
622 	    unsigned long address)
623 {
624 	struct task_struct *tsk;
625 	unsigned long flags;
626 	int sig;
627 
628 	flags = oops_begin();
629 	tsk = current;
630 	sig = SIGKILL;
631 
632 	printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
633 	       tsk->comm, address);
634 	dump_pagetable(address);
635 
636 	tsk->thread.cr2		= address;
637 	tsk->thread.trap_nr	= X86_TRAP_PF;
638 	tsk->thread.error_code	= error_code;
639 
640 	if (__die("Bad pagetable", regs, error_code))
641 		sig = 0;
642 
643 	oops_end(flags, regs, sig);
644 }
645 
646 static noinline void
647 no_context(struct pt_regs *regs, unsigned long error_code,
648 	   unsigned long address, int signal, int si_code)
649 {
650 	struct task_struct *tsk = current;
651 	unsigned long flags;
652 	int sig;
653 
654 	/* Are we prepared to handle this kernel fault? */
655 	if (fixup_exception(regs)) {
656 		/*
657 		 * Any interrupt that takes a fault gets the fixup. This makes
658 		 * the below recursive fault logic only apply to a faults from
659 		 * task context.
660 		 */
661 		if (in_interrupt())
662 			return;
663 
664 		/*
665 		 * Per the above we're !in_interrupt(), aka. task context.
666 		 *
667 		 * In this case we need to make sure we're not recursively
668 		 * faulting through the emulate_vsyscall() logic.
669 		 */
670 		if (current_thread_info()->sig_on_uaccess_error && signal) {
671 			tsk->thread.trap_nr = X86_TRAP_PF;
672 			tsk->thread.error_code = error_code | PF_USER;
673 			tsk->thread.cr2 = address;
674 
675 			/* XXX: hwpoison faults will set the wrong code. */
676 			force_sig_info_fault(signal, si_code, address, tsk, 0);
677 		}
678 
679 		/*
680 		 * Barring that, we can do the fixup and be happy.
681 		 */
682 		return;
683 	}
684 
685 	/*
686 	 * 32-bit:
687 	 *
688 	 *   Valid to do another page fault here, because if this fault
689 	 *   had been triggered by is_prefetch fixup_exception would have
690 	 *   handled it.
691 	 *
692 	 * 64-bit:
693 	 *
694 	 *   Hall of shame of CPU/BIOS bugs.
695 	 */
696 	if (is_prefetch(regs, error_code, address))
697 		return;
698 
699 	if (is_errata93(regs, address))
700 		return;
701 
702 	/*
703 	 * Oops. The kernel tried to access some bad page. We'll have to
704 	 * terminate things with extreme prejudice:
705 	 */
706 	flags = oops_begin();
707 
708 	show_fault_oops(regs, error_code, address);
709 
710 	if (task_stack_end_corrupted(tsk))
711 		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
712 
713 	tsk->thread.cr2		= address;
714 	tsk->thread.trap_nr	= X86_TRAP_PF;
715 	tsk->thread.error_code	= error_code;
716 
717 	sig = SIGKILL;
718 	if (__die("Oops", regs, error_code))
719 		sig = 0;
720 
721 	/* Executive summary in case the body of the oops scrolled away */
722 	printk(KERN_DEFAULT "CR2: %016lx\n", address);
723 
724 	oops_end(flags, regs, sig);
725 }
726 
727 /*
728  * Print out info about fatal segfaults, if the show_unhandled_signals
729  * sysctl is set:
730  */
731 static inline void
732 show_signal_msg(struct pt_regs *regs, unsigned long error_code,
733 		unsigned long address, struct task_struct *tsk)
734 {
735 	if (!unhandled_signal(tsk, SIGSEGV))
736 		return;
737 
738 	if (!printk_ratelimit())
739 		return;
740 
741 	printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
742 		task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
743 		tsk->comm, task_pid_nr(tsk), address,
744 		(void *)regs->ip, (void *)regs->sp, error_code);
745 
746 	print_vma_addr(KERN_CONT " in ", regs->ip);
747 
748 	printk(KERN_CONT "\n");
749 }
750 
751 static void
752 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
753 		       unsigned long address, int si_code)
754 {
755 	struct task_struct *tsk = current;
756 
757 	/* User mode accesses just cause a SIGSEGV */
758 	if (error_code & PF_USER) {
759 		/*
760 		 * It's possible to have interrupts off here:
761 		 */
762 		local_irq_enable();
763 
764 		/*
765 		 * Valid to do another page fault here because this one came
766 		 * from user space:
767 		 */
768 		if (is_prefetch(regs, error_code, address))
769 			return;
770 
771 		if (is_errata100(regs, address))
772 			return;
773 
774 #ifdef CONFIG_X86_64
775 		/*
776 		 * Instruction fetch faults in the vsyscall page might need
777 		 * emulation.
778 		 */
779 		if (unlikely((error_code & PF_INSTR) &&
780 			     ((address & ~0xfff) == VSYSCALL_ADDR))) {
781 			if (emulate_vsyscall(regs, address))
782 				return;
783 		}
784 #endif
785 		/* Kernel addresses are always protection faults: */
786 		if (address >= TASK_SIZE)
787 			error_code |= PF_PROT;
788 
789 		if (likely(show_unhandled_signals))
790 			show_signal_msg(regs, error_code, address, tsk);
791 
792 		tsk->thread.cr2		= address;
793 		tsk->thread.error_code	= error_code;
794 		tsk->thread.trap_nr	= X86_TRAP_PF;
795 
796 		force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0);
797 
798 		return;
799 	}
800 
801 	if (is_f00f_bug(regs, address))
802 		return;
803 
804 	no_context(regs, error_code, address, SIGSEGV, si_code);
805 }
806 
807 static noinline void
808 bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
809 		     unsigned long address)
810 {
811 	__bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
812 }
813 
814 static void
815 __bad_area(struct pt_regs *regs, unsigned long error_code,
816 	   unsigned long address, int si_code)
817 {
818 	struct mm_struct *mm = current->mm;
819 
820 	/*
821 	 * Something tried to access memory that isn't in our memory map..
822 	 * Fix it, but check if it's kernel or user first..
823 	 */
824 	up_read(&mm->mmap_sem);
825 
826 	__bad_area_nosemaphore(regs, error_code, address, si_code);
827 }
828 
829 static noinline void
830 bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
831 {
832 	__bad_area(regs, error_code, address, SEGV_MAPERR);
833 }
834 
835 static noinline void
836 bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
837 		      unsigned long address)
838 {
839 	__bad_area(regs, error_code, address, SEGV_ACCERR);
840 }
841 
842 static void
843 do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
844 	  unsigned int fault)
845 {
846 	struct task_struct *tsk = current;
847 	int code = BUS_ADRERR;
848 
849 	/* Kernel mode? Handle exceptions or die: */
850 	if (!(error_code & PF_USER)) {
851 		no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
852 		return;
853 	}
854 
855 	/* User-space => ok to do another page fault: */
856 	if (is_prefetch(regs, error_code, address))
857 		return;
858 
859 	tsk->thread.cr2		= address;
860 	tsk->thread.error_code	= error_code;
861 	tsk->thread.trap_nr	= X86_TRAP_PF;
862 
863 #ifdef CONFIG_MEMORY_FAILURE
864 	if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
865 		printk(KERN_ERR
866 	"MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
867 			tsk->comm, tsk->pid, address);
868 		code = BUS_MCEERR_AR;
869 	}
870 #endif
871 	force_sig_info_fault(SIGBUS, code, address, tsk, fault);
872 }
873 
874 static noinline void
875 mm_fault_error(struct pt_regs *regs, unsigned long error_code,
876 	       unsigned long address, unsigned int fault)
877 {
878 	if (fatal_signal_pending(current) && !(error_code & PF_USER)) {
879 		no_context(regs, error_code, address, 0, 0);
880 		return;
881 	}
882 
883 	if (fault & VM_FAULT_OOM) {
884 		/* Kernel mode? Handle exceptions or die: */
885 		if (!(error_code & PF_USER)) {
886 			no_context(regs, error_code, address,
887 				   SIGSEGV, SEGV_MAPERR);
888 			return;
889 		}
890 
891 		/*
892 		 * We ran out of memory, call the OOM killer, and return the
893 		 * userspace (which will retry the fault, or kill us if we got
894 		 * oom-killed):
895 		 */
896 		pagefault_out_of_memory();
897 	} else {
898 		if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
899 			     VM_FAULT_HWPOISON_LARGE))
900 			do_sigbus(regs, error_code, address, fault);
901 		else
902 			BUG();
903 	}
904 }
905 
906 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
907 {
908 	if ((error_code & PF_WRITE) && !pte_write(*pte))
909 		return 0;
910 
911 	if ((error_code & PF_INSTR) && !pte_exec(*pte))
912 		return 0;
913 
914 	return 1;
915 }
916 
917 /*
918  * Handle a spurious fault caused by a stale TLB entry.
919  *
920  * This allows us to lazily refresh the TLB when increasing the
921  * permissions of a kernel page (RO -> RW or NX -> X).  Doing it
922  * eagerly is very expensive since that implies doing a full
923  * cross-processor TLB flush, even if no stale TLB entries exist
924  * on other processors.
925  *
926  * Spurious faults may only occur if the TLB contains an entry with
927  * fewer permission than the page table entry.  Non-present (P = 0)
928  * and reserved bit (R = 1) faults are never spurious.
929  *
930  * There are no security implications to leaving a stale TLB when
931  * increasing the permissions on a page.
932  *
933  * Returns non-zero if a spurious fault was handled, zero otherwise.
934  *
935  * See Intel Developer's Manual Vol 3 Section 4.10.4.3, bullet 3
936  * (Optional Invalidation).
937  */
938 static noinline int
939 spurious_fault(unsigned long error_code, unsigned long address)
940 {
941 	pgd_t *pgd;
942 	pud_t *pud;
943 	pmd_t *pmd;
944 	pte_t *pte;
945 	int ret;
946 
947 	/*
948 	 * Only writes to RO or instruction fetches from NX may cause
949 	 * spurious faults.
950 	 *
951 	 * These could be from user or supervisor accesses but the TLB
952 	 * is only lazily flushed after a kernel mapping protection
953 	 * change, so user accesses are not expected to cause spurious
954 	 * faults.
955 	 */
956 	if (error_code != (PF_WRITE | PF_PROT)
957 	    && error_code != (PF_INSTR | PF_PROT))
958 		return 0;
959 
960 	pgd = init_mm.pgd + pgd_index(address);
961 	if (!pgd_present(*pgd))
962 		return 0;
963 
964 	pud = pud_offset(pgd, address);
965 	if (!pud_present(*pud))
966 		return 0;
967 
968 	if (pud_large(*pud))
969 		return spurious_fault_check(error_code, (pte_t *) pud);
970 
971 	pmd = pmd_offset(pud, address);
972 	if (!pmd_present(*pmd))
973 		return 0;
974 
975 	if (pmd_large(*pmd))
976 		return spurious_fault_check(error_code, (pte_t *) pmd);
977 
978 	pte = pte_offset_kernel(pmd, address);
979 	if (!pte_present(*pte))
980 		return 0;
981 
982 	ret = spurious_fault_check(error_code, pte);
983 	if (!ret)
984 		return 0;
985 
986 	/*
987 	 * Make sure we have permissions in PMD.
988 	 * If not, then there's a bug in the page tables:
989 	 */
990 	ret = spurious_fault_check(error_code, (pte_t *) pmd);
991 	WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
992 
993 	return ret;
994 }
995 NOKPROBE_SYMBOL(spurious_fault);
996 
997 int show_unhandled_signals = 1;
998 
999 static inline int
1000 access_error(unsigned long error_code, struct vm_area_struct *vma)
1001 {
1002 	if (error_code & PF_WRITE) {
1003 		/* write, present and write, not present: */
1004 		if (unlikely(!(vma->vm_flags & VM_WRITE)))
1005 			return 1;
1006 		return 0;
1007 	}
1008 
1009 	/* read, present: */
1010 	if (unlikely(error_code & PF_PROT))
1011 		return 1;
1012 
1013 	/* read, not present: */
1014 	if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
1015 		return 1;
1016 
1017 	return 0;
1018 }
1019 
1020 static int fault_in_kernel_space(unsigned long address)
1021 {
1022 	return address >= TASK_SIZE_MAX;
1023 }
1024 
1025 static inline bool smap_violation(int error_code, struct pt_regs *regs)
1026 {
1027 	if (!IS_ENABLED(CONFIG_X86_SMAP))
1028 		return false;
1029 
1030 	if (!static_cpu_has(X86_FEATURE_SMAP))
1031 		return false;
1032 
1033 	if (error_code & PF_USER)
1034 		return false;
1035 
1036 	if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC))
1037 		return false;
1038 
1039 	return true;
1040 }
1041 
1042 /*
1043  * This routine handles page faults.  It determines the address,
1044  * and the problem, and then passes it off to one of the appropriate
1045  * routines.
1046  *
1047  * This function must have noinline because both callers
1048  * {,trace_}do_page_fault() have notrace on. Having this an actual function
1049  * guarantees there's a function trace entry.
1050  */
1051 static noinline void
1052 __do_page_fault(struct pt_regs *regs, unsigned long error_code,
1053 		unsigned long address)
1054 {
1055 	struct vm_area_struct *vma;
1056 	struct task_struct *tsk;
1057 	struct mm_struct *mm;
1058 	int fault, major = 0;
1059 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1060 
1061 	tsk = current;
1062 	mm = tsk->mm;
1063 
1064 	/*
1065 	 * Detect and handle instructions that would cause a page fault for
1066 	 * both a tracked kernel page and a userspace page.
1067 	 */
1068 	if (kmemcheck_active(regs))
1069 		kmemcheck_hide(regs);
1070 	prefetchw(&mm->mmap_sem);
1071 
1072 	if (unlikely(kmmio_fault(regs, address)))
1073 		return;
1074 
1075 	/*
1076 	 * We fault-in kernel-space virtual memory on-demand. The
1077 	 * 'reference' page table is init_mm.pgd.
1078 	 *
1079 	 * NOTE! We MUST NOT take any locks for this case. We may
1080 	 * be in an interrupt or a critical region, and should
1081 	 * only copy the information from the master page table,
1082 	 * nothing more.
1083 	 *
1084 	 * This verifies that the fault happens in kernel space
1085 	 * (error_code & 4) == 0, and that the fault was not a
1086 	 * protection error (error_code & 9) == 0.
1087 	 */
1088 	if (unlikely(fault_in_kernel_space(address))) {
1089 		if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
1090 			if (vmalloc_fault(address) >= 0)
1091 				return;
1092 
1093 			if (kmemcheck_fault(regs, address, error_code))
1094 				return;
1095 		}
1096 
1097 		/* Can handle a stale RO->RW TLB: */
1098 		if (spurious_fault(error_code, address))
1099 			return;
1100 
1101 		/* kprobes don't want to hook the spurious faults: */
1102 		if (kprobes_fault(regs))
1103 			return;
1104 		/*
1105 		 * Don't take the mm semaphore here. If we fixup a prefetch
1106 		 * fault we could otherwise deadlock:
1107 		 */
1108 		bad_area_nosemaphore(regs, error_code, address);
1109 
1110 		return;
1111 	}
1112 
1113 	/* kprobes don't want to hook the spurious faults: */
1114 	if (unlikely(kprobes_fault(regs)))
1115 		return;
1116 
1117 	if (unlikely(error_code & PF_RSVD))
1118 		pgtable_bad(regs, error_code, address);
1119 
1120 	if (unlikely(smap_violation(error_code, regs))) {
1121 		bad_area_nosemaphore(regs, error_code, address);
1122 		return;
1123 	}
1124 
1125 	/*
1126 	 * If we're in an interrupt, have no user context or are running
1127 	 * in an atomic region then we must not take the fault:
1128 	 */
1129 	if (unlikely(in_atomic() || !mm)) {
1130 		bad_area_nosemaphore(regs, error_code, address);
1131 		return;
1132 	}
1133 
1134 	/*
1135 	 * It's safe to allow irq's after cr2 has been saved and the
1136 	 * vmalloc fault has been handled.
1137 	 *
1138 	 * User-mode registers count as a user access even for any
1139 	 * potential system fault or CPU buglet:
1140 	 */
1141 	if (user_mode_vm(regs)) {
1142 		local_irq_enable();
1143 		error_code |= PF_USER;
1144 		flags |= FAULT_FLAG_USER;
1145 	} else {
1146 		if (regs->flags & X86_EFLAGS_IF)
1147 			local_irq_enable();
1148 	}
1149 
1150 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
1151 
1152 	if (error_code & PF_WRITE)
1153 		flags |= FAULT_FLAG_WRITE;
1154 
1155 	/*
1156 	 * When running in the kernel we expect faults to occur only to
1157 	 * addresses in user space.  All other faults represent errors in
1158 	 * the kernel and should generate an OOPS.  Unfortunately, in the
1159 	 * case of an erroneous fault occurring in a code path which already
1160 	 * holds mmap_sem we will deadlock attempting to validate the fault
1161 	 * against the address space.  Luckily the kernel only validly
1162 	 * references user space from well defined areas of code, which are
1163 	 * listed in the exceptions table.
1164 	 *
1165 	 * As the vast majority of faults will be valid we will only perform
1166 	 * the source reference check when there is a possibility of a
1167 	 * deadlock. Attempt to lock the address space, if we cannot we then
1168 	 * validate the source. If this is invalid we can skip the address
1169 	 * space check, thus avoiding the deadlock:
1170 	 */
1171 	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
1172 		if ((error_code & PF_USER) == 0 &&
1173 		    !search_exception_tables(regs->ip)) {
1174 			bad_area_nosemaphore(regs, error_code, address);
1175 			return;
1176 		}
1177 retry:
1178 		down_read(&mm->mmap_sem);
1179 	} else {
1180 		/*
1181 		 * The above down_read_trylock() might have succeeded in
1182 		 * which case we'll have missed the might_sleep() from
1183 		 * down_read():
1184 		 */
1185 		might_sleep();
1186 	}
1187 
1188 	vma = find_vma(mm, address);
1189 	if (unlikely(!vma)) {
1190 		bad_area(regs, error_code, address);
1191 		return;
1192 	}
1193 	if (likely(vma->vm_start <= address))
1194 		goto good_area;
1195 	if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
1196 		bad_area(regs, error_code, address);
1197 		return;
1198 	}
1199 	if (error_code & PF_USER) {
1200 		/*
1201 		 * Accessing the stack below %sp is always a bug.
1202 		 * The large cushion allows instructions like enter
1203 		 * and pusha to work. ("enter $65535, $31" pushes
1204 		 * 32 pointers and then decrements %sp by 65535.)
1205 		 */
1206 		if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
1207 			bad_area(regs, error_code, address);
1208 			return;
1209 		}
1210 	}
1211 	if (unlikely(expand_stack(vma, address))) {
1212 		bad_area(regs, error_code, address);
1213 		return;
1214 	}
1215 
1216 	/*
1217 	 * Ok, we have a good vm_area for this memory access, so
1218 	 * we can handle it..
1219 	 */
1220 good_area:
1221 	if (unlikely(access_error(error_code, vma))) {
1222 		bad_area_access_error(regs, error_code, address);
1223 		return;
1224 	}
1225 
1226 	/*
1227 	 * If for any reason at all we couldn't handle the fault,
1228 	 * make sure we exit gracefully rather than endlessly redo
1229 	 * the fault.  Since we never set FAULT_FLAG_RETRY_NOWAIT, if
1230 	 * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked.
1231 	 */
1232 	fault = handle_mm_fault(mm, vma, address, flags);
1233 	major |= fault & VM_FAULT_MAJOR;
1234 
1235 	/*
1236 	 * If we need to retry the mmap_sem has already been released,
1237 	 * and if there is a fatal signal pending there is no guarantee
1238 	 * that we made any progress. Handle this case first.
1239 	 */
1240 	if (unlikely(fault & VM_FAULT_RETRY)) {
1241 		/* Retry at most once */
1242 		if (flags & FAULT_FLAG_ALLOW_RETRY) {
1243 			flags &= ~FAULT_FLAG_ALLOW_RETRY;
1244 			flags |= FAULT_FLAG_TRIED;
1245 			if (!fatal_signal_pending(tsk))
1246 				goto retry;
1247 		}
1248 
1249 		/* User mode? Just return to handle the fatal exception */
1250 		if (flags & FAULT_FLAG_USER)
1251 			return;
1252 
1253 		/* Not returning to user mode? Handle exceptions or die: */
1254 		no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
1255 		return;
1256 	}
1257 
1258 	up_read(&mm->mmap_sem);
1259 	if (unlikely(fault & VM_FAULT_ERROR)) {
1260 		mm_fault_error(regs, error_code, address, fault);
1261 		return;
1262 	}
1263 
1264 	/*
1265 	 * Major/minor page fault accounting. If any of the events
1266 	 * returned VM_FAULT_MAJOR, we account it as a major fault.
1267 	 */
1268 	if (major) {
1269 		tsk->maj_flt++;
1270 		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
1271 	} else {
1272 		tsk->min_flt++;
1273 		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
1274 	}
1275 
1276 	check_v8086_mode(regs, address, tsk);
1277 }
1278 NOKPROBE_SYMBOL(__do_page_fault);
1279 
1280 dotraplinkage void notrace
1281 do_page_fault(struct pt_regs *regs, unsigned long error_code)
1282 {
1283 	unsigned long address = read_cr2(); /* Get the faulting address */
1284 	enum ctx_state prev_state;
1285 
1286 	/*
1287 	 * We must have this function tagged with __kprobes, notrace and call
1288 	 * read_cr2() before calling anything else. To avoid calling any kind
1289 	 * of tracing machinery before we've observed the CR2 value.
1290 	 *
1291 	 * exception_{enter,exit}() contain all sorts of tracepoints.
1292 	 */
1293 
1294 	prev_state = exception_enter();
1295 	__do_page_fault(regs, error_code, address);
1296 	exception_exit(prev_state);
1297 }
1298 NOKPROBE_SYMBOL(do_page_fault);
1299 
1300 #ifdef CONFIG_TRACING
1301 static nokprobe_inline void
1302 trace_page_fault_entries(unsigned long address, struct pt_regs *regs,
1303 			 unsigned long error_code)
1304 {
1305 	if (user_mode(regs))
1306 		trace_page_fault_user(address, regs, error_code);
1307 	else
1308 		trace_page_fault_kernel(address, regs, error_code);
1309 }
1310 
1311 dotraplinkage void notrace
1312 trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
1313 {
1314 	/*
1315 	 * The exception_enter and tracepoint processing could
1316 	 * trigger another page faults (user space callchain
1317 	 * reading) and destroy the original cr2 value, so read
1318 	 * the faulting address now.
1319 	 */
1320 	unsigned long address = read_cr2();
1321 	enum ctx_state prev_state;
1322 
1323 	prev_state = exception_enter();
1324 	trace_page_fault_entries(address, regs, error_code);
1325 	__do_page_fault(regs, error_code, address);
1326 	exception_exit(prev_state);
1327 }
1328 NOKPROBE_SYMBOL(trace_do_page_fault);
1329 #endif /* CONFIG_TRACING */
1330