xref: /openbmc/linux/arch/x86/entry/entry_64.S (revision fa5e5c409213265da8a188b4a5e4e641b1382eb4)
1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */
2905a36a2SIngo Molnar/*
3905a36a2SIngo Molnar *  linux/arch/x86_64/entry.S
4905a36a2SIngo Molnar *
5905a36a2SIngo Molnar *  Copyright (C) 1991, 1992  Linus Torvalds
6905a36a2SIngo Molnar *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
7905a36a2SIngo Molnar *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
84d732138SIngo Molnar *
9905a36a2SIngo Molnar * entry.S contains the system-call and fault low-level handling routines.
10905a36a2SIngo Molnar *
11cb1aaebeSMauro Carvalho Chehab * Some of this is documented in Documentation/x86/entry_64.rst
12905a36a2SIngo Molnar *
13905a36a2SIngo Molnar * A note on terminology:
14905a36a2SIngo Molnar * - iret frame:	Architecture defined interrupt frame from SS to RIP
15905a36a2SIngo Molnar *			at the top of the kernel process stack.
16905a36a2SIngo Molnar *
17905a36a2SIngo Molnar * Some macro usage:
186dcc5627SJiri Slaby * - SYM_FUNC_START/END:Define functions in the symbol table.
194d732138SIngo Molnar * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
204d732138SIngo Molnar * - idtentry:		Define exception entry points.
21905a36a2SIngo Molnar */
22905a36a2SIngo Molnar#include <linux/linkage.h>
23905a36a2SIngo Molnar#include <asm/segment.h>
24905a36a2SIngo Molnar#include <asm/cache.h>
25905a36a2SIngo Molnar#include <asm/errno.h>
26905a36a2SIngo Molnar#include <asm/asm-offsets.h>
27905a36a2SIngo Molnar#include <asm/msr.h>
28905a36a2SIngo Molnar#include <asm/unistd.h>
29905a36a2SIngo Molnar#include <asm/thread_info.h>
30905a36a2SIngo Molnar#include <asm/hw_irq.h>
31905a36a2SIngo Molnar#include <asm/page_types.h>
32905a36a2SIngo Molnar#include <asm/irqflags.h>
33905a36a2SIngo Molnar#include <asm/paravirt.h>
34905a36a2SIngo Molnar#include <asm/percpu.h>
35905a36a2SIngo Molnar#include <asm/asm.h>
36905a36a2SIngo Molnar#include <asm/smap.h>
37905a36a2SIngo Molnar#include <asm/pgtable_types.h>
38784d5699SAl Viro#include <asm/export.h>
398c1f7558SJosh Poimboeuf#include <asm/frame.h>
40cfa82a00SThomas Gleixner#include <asm/trapnr.h>
412641f08bSDavid Woodhouse#include <asm/nospec-branch.h>
42905a36a2SIngo Molnar#include <linux/err.h>
43905a36a2SIngo Molnar
446fd166aaSPeter Zijlstra#include "calling.h"
456fd166aaSPeter Zijlstra
46905a36a2SIngo Molnar.code64
47905a36a2SIngo Molnar.section .entry.text, "ax"
48905a36a2SIngo Molnar
49905a36a2SIngo Molnar#ifdef CONFIG_PARAVIRT
50bc7b11c0SJiri SlabySYM_CODE_START(native_usergs_sysret64)
518c1f7558SJosh Poimboeuf	UNWIND_HINT_EMPTY
52905a36a2SIngo Molnar	swapgs
53905a36a2SIngo Molnar	sysretq
54bc7b11c0SJiri SlabySYM_CODE_END(native_usergs_sysret64)
55905a36a2SIngo Molnar#endif /* CONFIG_PARAVIRT */
56905a36a2SIngo Molnar
57ca37e57bSAndy Lutomirski.macro TRACE_IRQS_FLAGS flags:req
58905a36a2SIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS
59a368d7fdSJan Beulich	btl	$9, \flags		/* interrupts off? */
60905a36a2SIngo Molnar	jnc	1f
61905a36a2SIngo Molnar	TRACE_IRQS_ON
62905a36a2SIngo Molnar1:
63905a36a2SIngo Molnar#endif
64905a36a2SIngo Molnar.endm
65905a36a2SIngo Molnar
66ca37e57bSAndy Lutomirski.macro TRACE_IRQS_IRETQ
67ca37e57bSAndy Lutomirski	TRACE_IRQS_FLAGS EFLAGS(%rsp)
68ca37e57bSAndy Lutomirski.endm
69ca37e57bSAndy Lutomirski
70905a36a2SIngo Molnar/*
71905a36a2SIngo Molnar * When dynamic function tracer is enabled it will add a breakpoint
72905a36a2SIngo Molnar * to all locations that it is about to modify, sync CPUs, update
73905a36a2SIngo Molnar * all the code, sync CPUs, then remove the breakpoints. In this time
74905a36a2SIngo Molnar * if lockdep is enabled, it might jump back into the debug handler
75905a36a2SIngo Molnar * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
76905a36a2SIngo Molnar *
77905a36a2SIngo Molnar * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
78905a36a2SIngo Molnar * make sure the stack pointer does not get reset back to the top
79905a36a2SIngo Molnar * of the debug stack, and instead just reuses the current stack.
80905a36a2SIngo Molnar */
81905a36a2SIngo Molnar#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)
82905a36a2SIngo Molnar
83905a36a2SIngo Molnar.macro TRACE_IRQS_OFF_DEBUG
84905a36a2SIngo Molnar	call	debug_stack_set_zero
85905a36a2SIngo Molnar	TRACE_IRQS_OFF
86905a36a2SIngo Molnar	call	debug_stack_reset
87905a36a2SIngo Molnar.endm
88905a36a2SIngo Molnar
89905a36a2SIngo Molnar.macro TRACE_IRQS_ON_DEBUG
90905a36a2SIngo Molnar	call	debug_stack_set_zero
91905a36a2SIngo Molnar	TRACE_IRQS_ON
92905a36a2SIngo Molnar	call	debug_stack_reset
93905a36a2SIngo Molnar.endm
94905a36a2SIngo Molnar
95905a36a2SIngo Molnar.macro TRACE_IRQS_IRETQ_DEBUG
966709812fSJan Beulich	btl	$9, EFLAGS(%rsp)		/* interrupts off? */
97905a36a2SIngo Molnar	jnc	1f
98905a36a2SIngo Molnar	TRACE_IRQS_ON_DEBUG
99905a36a2SIngo Molnar1:
100905a36a2SIngo Molnar.endm
101905a36a2SIngo Molnar
102905a36a2SIngo Molnar#else
103905a36a2SIngo Molnar# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
104905a36a2SIngo Molnar# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
105905a36a2SIngo Molnar# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
106905a36a2SIngo Molnar#endif
107905a36a2SIngo Molnar
108905a36a2SIngo Molnar/*
1094d732138SIngo Molnar * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
110905a36a2SIngo Molnar *
111fda57b22SAndy Lutomirski * This is the only entry point used for 64-bit system calls.  The
112fda57b22SAndy Lutomirski * hardware interface is reasonably well designed and the register to
113fda57b22SAndy Lutomirski * argument mapping Linux uses fits well with the registers that are
114fda57b22SAndy Lutomirski * available when SYSCALL is used.
115fda57b22SAndy Lutomirski *
116fda57b22SAndy Lutomirski * SYSCALL instructions can be found inlined in libc implementations as
117fda57b22SAndy Lutomirski * well as some other programs and libraries.  There are also a handful
118fda57b22SAndy Lutomirski * of SYSCALL instructions in the vDSO used, for example, as a
119fda57b22SAndy Lutomirski * clock_gettimeofday fallback.
120fda57b22SAndy Lutomirski *
1214d732138SIngo Molnar * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
122905a36a2SIngo Molnar * then loads new ss, cs, and rip from previously programmed MSRs.
123905a36a2SIngo Molnar * rflags gets masked by a value from another MSR (so CLD and CLAC
124905a36a2SIngo Molnar * are not needed). SYSCALL does not save anything on the stack
125905a36a2SIngo Molnar * and does not change rsp.
126905a36a2SIngo Molnar *
127905a36a2SIngo Molnar * Registers on entry:
128905a36a2SIngo Molnar * rax  system call number
129905a36a2SIngo Molnar * rcx  return address
130905a36a2SIngo Molnar * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
131905a36a2SIngo Molnar * rdi  arg0
132905a36a2SIngo Molnar * rsi  arg1
133905a36a2SIngo Molnar * rdx  arg2
134905a36a2SIngo Molnar * r10  arg3 (needs to be moved to rcx to conform to C ABI)
135905a36a2SIngo Molnar * r8   arg4
136905a36a2SIngo Molnar * r9   arg5
137905a36a2SIngo Molnar * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
138905a36a2SIngo Molnar *
139905a36a2SIngo Molnar * Only called from user space.
140905a36a2SIngo Molnar *
141905a36a2SIngo Molnar * When user can change pt_regs->foo always force IRET. That is because
142905a36a2SIngo Molnar * it deals with uncanonical addresses better. SYSRET has trouble
143905a36a2SIngo Molnar * with them due to bugs in both AMD and Intel CPUs.
144905a36a2SIngo Molnar */
145905a36a2SIngo Molnar
146bc7b11c0SJiri SlabySYM_CODE_START(entry_SYSCALL_64)
1478c1f7558SJosh Poimboeuf	UNWIND_HINT_EMPTY
148905a36a2SIngo Molnar	/*
149905a36a2SIngo Molnar	 * Interrupts are off on entry.
150905a36a2SIngo Molnar	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
151905a36a2SIngo Molnar	 * it is too small to ever cause noticeable irq latency.
152905a36a2SIngo Molnar	 */
153905a36a2SIngo Molnar
1548a9949bcSAndy Lutomirski	swapgs
155bf904d27SAndy Lutomirski	/* tss.sp2 is scratch space. */
15698f05b51SAndy Lutomirski	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
157bf904d27SAndy Lutomirski	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
158905a36a2SIngo Molnar	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
159905a36a2SIngo Molnar
160905a36a2SIngo Molnar	/* Construct struct pt_regs on stack */
161905a36a2SIngo Molnar	pushq	$__USER_DS				/* pt_regs->ss */
16298f05b51SAndy Lutomirski	pushq	PER_CPU_VAR(cpu_tss_rw + TSS_sp2)	/* pt_regs->sp */
163905a36a2SIngo Molnar	pushq	%r11					/* pt_regs->flags */
164905a36a2SIngo Molnar	pushq	$__USER_CS				/* pt_regs->cs */
165905a36a2SIngo Molnar	pushq	%rcx					/* pt_regs->ip */
16626ba4e57SJiri SlabySYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
167905a36a2SIngo Molnar	pushq	%rax					/* pt_regs->orig_ax */
16830907fd1SDominik Brodowski
16930907fd1SDominik Brodowski	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
170905a36a2SIngo Molnar
1711e423bffSAndy Lutomirski	/* IRQs are off. */
172dfe64506SLinus Torvalds	movq	%rax, %rdi
173dfe64506SLinus Torvalds	movq	%rsp, %rsi
1741e423bffSAndy Lutomirski	call	do_syscall_64		/* returns with IRQs disabled */
1751e423bffSAndy Lutomirski
176905a36a2SIngo Molnar	/*
177905a36a2SIngo Molnar	 * Try to use SYSRET instead of IRET if we're returning to
1788a055d7fSAndy Lutomirski	 * a completely clean 64-bit userspace context.  If we're not,
1798a055d7fSAndy Lutomirski	 * go to the slow exit path.
180905a36a2SIngo Molnar	 */
181905a36a2SIngo Molnar	movq	RCX(%rsp), %rcx
182905a36a2SIngo Molnar	movq	RIP(%rsp), %r11
1838a055d7fSAndy Lutomirski
1848a055d7fSAndy Lutomirski	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
1858a055d7fSAndy Lutomirski	jne	swapgs_restore_regs_and_return_to_usermode
186905a36a2SIngo Molnar
187905a36a2SIngo Molnar	/*
188905a36a2SIngo Molnar	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
189905a36a2SIngo Molnar	 * in kernel space.  This essentially lets the user take over
190905a36a2SIngo Molnar	 * the kernel, since userspace controls RSP.
191905a36a2SIngo Molnar	 *
192905a36a2SIngo Molnar	 * If width of "canonical tail" ever becomes variable, this will need
193905a36a2SIngo Molnar	 * to be updated to remain correct on both old and new CPUs.
194361b4b58SKirill A. Shutemov	 *
195cbe0317bSKirill A. Shutemov	 * Change top bits to match most significant bit (47th or 56th bit
196cbe0317bSKirill A. Shutemov	 * depending on paging mode) in the address.
197905a36a2SIngo Molnar	 */
19809e61a77SKirill A. Shutemov#ifdef CONFIG_X86_5LEVEL
19939b95522SKirill A. Shutemov	ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
20039b95522SKirill A. Shutemov		"shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
20109e61a77SKirill A. Shutemov#else
202905a36a2SIngo Molnar	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
203905a36a2SIngo Molnar	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
20409e61a77SKirill A. Shutemov#endif
2054d732138SIngo Molnar
206905a36a2SIngo Molnar	/* If this changed %rcx, it was not canonical */
207905a36a2SIngo Molnar	cmpq	%rcx, %r11
2088a055d7fSAndy Lutomirski	jne	swapgs_restore_regs_and_return_to_usermode
209905a36a2SIngo Molnar
210905a36a2SIngo Molnar	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
2118a055d7fSAndy Lutomirski	jne	swapgs_restore_regs_and_return_to_usermode
212905a36a2SIngo Molnar
213905a36a2SIngo Molnar	movq	R11(%rsp), %r11
214905a36a2SIngo Molnar	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
2158a055d7fSAndy Lutomirski	jne	swapgs_restore_regs_and_return_to_usermode
216905a36a2SIngo Molnar
217905a36a2SIngo Molnar	/*
2183e035305SBorislav Petkov	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
2193e035305SBorislav Petkov	 * restore RF properly. If the slowpath sets it for whatever reason, we
2203e035305SBorislav Petkov	 * need to restore it correctly.
2213e035305SBorislav Petkov	 *
2223e035305SBorislav Petkov	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
2233e035305SBorislav Petkov	 * trap from userspace immediately after SYSRET.  This would cause an
2243e035305SBorislav Petkov	 * infinite loop whenever #DB happens with register state that satisfies
2253e035305SBorislav Petkov	 * the opportunistic SYSRET conditions.  For example, single-stepping
2263e035305SBorislav Petkov	 * this user code:
227905a36a2SIngo Molnar	 *
228905a36a2SIngo Molnar	 *           movq	$stuck_here, %rcx
229905a36a2SIngo Molnar	 *           pushfq
230905a36a2SIngo Molnar	 *           popq %r11
231905a36a2SIngo Molnar	 *   stuck_here:
232905a36a2SIngo Molnar	 *
233905a36a2SIngo Molnar	 * would never get past 'stuck_here'.
234905a36a2SIngo Molnar	 */
235905a36a2SIngo Molnar	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
2368a055d7fSAndy Lutomirski	jnz	swapgs_restore_regs_and_return_to_usermode
237905a36a2SIngo Molnar
238905a36a2SIngo Molnar	/* nothing to check for RSP */
239905a36a2SIngo Molnar
240905a36a2SIngo Molnar	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
2418a055d7fSAndy Lutomirski	jne	swapgs_restore_regs_and_return_to_usermode
242905a36a2SIngo Molnar
243905a36a2SIngo Molnar	/*
244905a36a2SIngo Molnar	 * We win! This label is here just for ease of understanding
245905a36a2SIngo Molnar	 * perf profiles. Nothing jumps here.
246905a36a2SIngo Molnar	 */
247905a36a2SIngo Molnarsyscall_return_via_sysret:
248905a36a2SIngo Molnar	/* rcx and r11 are already restored (see code above) */
249502af0d7SDominik Brodowski	POP_REGS pop_rdi=0 skip_r11rcx=1
2503e3b9293SAndy Lutomirski
2513e3b9293SAndy Lutomirski	/*
2523e3b9293SAndy Lutomirski	 * Now all regs are restored except RSP and RDI.
2533e3b9293SAndy Lutomirski	 * Save old stack pointer and switch to trampoline stack.
2543e3b9293SAndy Lutomirski	 */
2553e3b9293SAndy Lutomirski	movq	%rsp, %rdi
256c482feefSAndy Lutomirski	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
2571fb14363SJosh Poimboeuf	UNWIND_HINT_EMPTY
2583e3b9293SAndy Lutomirski
2593e3b9293SAndy Lutomirski	pushq	RSP-RDI(%rdi)	/* RSP */
2603e3b9293SAndy Lutomirski	pushq	(%rdi)		/* RDI */
2613e3b9293SAndy Lutomirski
2623e3b9293SAndy Lutomirski	/*
2633e3b9293SAndy Lutomirski	 * We are on the trampoline stack.  All regs except RDI are live.
2643e3b9293SAndy Lutomirski	 * We can do future final exit work right here.
2653e3b9293SAndy Lutomirski	 */
266afaef01cSAlexander Popov	STACKLEAK_ERASE_NOCLOBBER
267afaef01cSAlexander Popov
2686fd166aaSPeter Zijlstra	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
2693e3b9293SAndy Lutomirski
2704fbb3910SAndy Lutomirski	popq	%rdi
2713e3b9293SAndy Lutomirski	popq	%rsp
272905a36a2SIngo Molnar	USERGS_SYSRET64
273bc7b11c0SJiri SlabySYM_CODE_END(entry_SYSCALL_64)
274905a36a2SIngo Molnar
275905a36a2SIngo Molnar/*
2760100301bSBrian Gerst * %rdi: prev task
2770100301bSBrian Gerst * %rsi: next task
2780100301bSBrian Gerst */
279b9f6976bSThomas Gleixner.pushsection .text, "ax"
28096c64806SJosh PoimboeufSYM_FUNC_START(__switch_to_asm)
2810100301bSBrian Gerst	/*
2820100301bSBrian Gerst	 * Save callee-saved registers
2830100301bSBrian Gerst	 * This must match the order in inactive_task_frame
2840100301bSBrian Gerst	 */
2850100301bSBrian Gerst	pushq	%rbp
2860100301bSBrian Gerst	pushq	%rbx
2870100301bSBrian Gerst	pushq	%r12
2880100301bSBrian Gerst	pushq	%r13
2890100301bSBrian Gerst	pushq	%r14
2900100301bSBrian Gerst	pushq	%r15
2910100301bSBrian Gerst
2920100301bSBrian Gerst	/* switch stack */
2930100301bSBrian Gerst	movq	%rsp, TASK_threadsp(%rdi)
2940100301bSBrian Gerst	movq	TASK_threadsp(%rsi), %rsp
2950100301bSBrian Gerst
296050e9baaSLinus Torvalds#ifdef CONFIG_STACKPROTECTOR
2970100301bSBrian Gerst	movq	TASK_stack_canary(%rsi), %rbx
298e6401c13SAndy Lutomirski	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
2990100301bSBrian Gerst#endif
3000100301bSBrian Gerst
301c995efd5SDavid Woodhouse#ifdef CONFIG_RETPOLINE
302c995efd5SDavid Woodhouse	/*
303c995efd5SDavid Woodhouse	 * When switching from a shallower to a deeper call stack
304c995efd5SDavid Woodhouse	 * the RSB may either underflow or use entries populated
305c995efd5SDavid Woodhouse	 * with userspace addresses. On CPUs where those concerns
306c995efd5SDavid Woodhouse	 * exist, overwrite the RSB with entries which capture
307c995efd5SDavid Woodhouse	 * speculative execution to prevent attack.
308c995efd5SDavid Woodhouse	 */
309d1c99108SDavid Woodhouse	FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
310c995efd5SDavid Woodhouse#endif
311c995efd5SDavid Woodhouse
3120100301bSBrian Gerst	/* restore callee-saved registers */
3130100301bSBrian Gerst	popq	%r15
3140100301bSBrian Gerst	popq	%r14
3150100301bSBrian Gerst	popq	%r13
3160100301bSBrian Gerst	popq	%r12
3170100301bSBrian Gerst	popq	%rbx
3180100301bSBrian Gerst	popq	%rbp
3190100301bSBrian Gerst
3200100301bSBrian Gerst	jmp	__switch_to
32196c64806SJosh PoimboeufSYM_FUNC_END(__switch_to_asm)
322b9f6976bSThomas Gleixner.popsection
3230100301bSBrian Gerst
3240100301bSBrian Gerst/*
325905a36a2SIngo Molnar * A newly forked process directly context switches into this address.
326905a36a2SIngo Molnar *
3270100301bSBrian Gerst * rax: prev task we switched from
328616d2483SBrian Gerst * rbx: kernel thread func (NULL for user thread)
329616d2483SBrian Gerst * r12: kernel thread arg
330905a36a2SIngo Molnar */
331b9f6976bSThomas Gleixner.pushsection .text, "ax"
332bc7b11c0SJiri SlabySYM_CODE_START(ret_from_fork)
3338c1f7558SJosh Poimboeuf	UNWIND_HINT_EMPTY
3340100301bSBrian Gerst	movq	%rax, %rdi
3354d732138SIngo Molnar	call	schedule_tail			/* rdi: 'prev' task parameter */
336905a36a2SIngo Molnar
337616d2483SBrian Gerst	testq	%rbx, %rbx			/* from kernel_thread? */
338616d2483SBrian Gerst	jnz	1f				/* kernel threads are uncommon */
339905a36a2SIngo Molnar
340616d2483SBrian Gerst2:
3418c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS
342ebd57499SJosh Poimboeuf	movq	%rsp, %rdi
34324d978b7SAndy Lutomirski	call	syscall_return_slowpath	/* returns with IRQs disabled */
3448a055d7fSAndy Lutomirski	jmp	swapgs_restore_regs_and_return_to_usermode
345616d2483SBrian Gerst
346616d2483SBrian Gerst1:
347616d2483SBrian Gerst	/* kernel thread */
348d31a5802SJosh Poimboeuf	UNWIND_HINT_EMPTY
349616d2483SBrian Gerst	movq	%r12, %rdi
35034fdce69SPeter Zijlstra	CALL_NOSPEC rbx
351616d2483SBrian Gerst	/*
352616d2483SBrian Gerst	 * A kernel thread is allowed to return here after successfully
353616d2483SBrian Gerst	 * calling do_execve().  Exit to userspace to complete the execve()
354616d2483SBrian Gerst	 * syscall.
355616d2483SBrian Gerst	 */
356616d2483SBrian Gerst	movq	$0, RAX(%rsp)
357616d2483SBrian Gerst	jmp	2b
358bc7b11c0SJiri SlabySYM_CODE_END(ret_from_fork)
359b9f6976bSThomas Gleixner.popsection
360905a36a2SIngo Molnar
3611d3e53e8SAndy Lutomirski.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
3621d3e53e8SAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY
363e17f8234SBoris Ostrovsky	pushq %rax
364e17f8234SBoris Ostrovsky	SAVE_FLAGS(CLBR_RAX)
365e17f8234SBoris Ostrovsky	testl $X86_EFLAGS_IF, %eax
3661d3e53e8SAndy Lutomirski	jz .Lokay_\@
3671d3e53e8SAndy Lutomirski	ud2
3681d3e53e8SAndy Lutomirski.Lokay_\@:
369e17f8234SBoris Ostrovsky	popq %rax
3701d3e53e8SAndy Lutomirski#endif
3711d3e53e8SAndy Lutomirski.endm
3721d3e53e8SAndy Lutomirski
3731d3e53e8SAndy Lutomirski/*
3741d3e53e8SAndy Lutomirski * Enters the IRQ stack if we're not already using it.  NMI-safe.  Clobbers
3751d3e53e8SAndy Lutomirski * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
3761d3e53e8SAndy Lutomirski * Requires kernel GSBASE.
3771d3e53e8SAndy Lutomirski *
3781d3e53e8SAndy Lutomirski * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
3791d3e53e8SAndy Lutomirski */
3802ba64741SDominik Brodowski.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0
3811d3e53e8SAndy Lutomirski	DEBUG_ENTRY_ASSERT_IRQS_OFF
3822ba64741SDominik Brodowski
3832ba64741SDominik Brodowski	.if \save_ret
3842ba64741SDominik Brodowski	/*
3852ba64741SDominik Brodowski	 * If save_ret is set, the original stack contains one additional
3862ba64741SDominik Brodowski	 * entry -- the return address. Therefore, move the address one
3872ba64741SDominik Brodowski	 * entry below %rsp to \old_rsp.
3882ba64741SDominik Brodowski	 */
3892ba64741SDominik Brodowski	leaq	8(%rsp), \old_rsp
3902ba64741SDominik Brodowski	.else
3911d3e53e8SAndy Lutomirski	movq	%rsp, \old_rsp
3922ba64741SDominik Brodowski	.endif
3938c1f7558SJosh Poimboeuf
3948c1f7558SJosh Poimboeuf	.if \regs
3958c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS base=\old_rsp
3968c1f7558SJosh Poimboeuf	.endif
3978c1f7558SJosh Poimboeuf
3981d3e53e8SAndy Lutomirski	incl	PER_CPU_VAR(irq_count)
39929955909SAndy Lutomirski	jnz	.Lirq_stack_push_old_rsp_\@
4001d3e53e8SAndy Lutomirski
4011d3e53e8SAndy Lutomirski	/*
4021d3e53e8SAndy Lutomirski	 * Right now, if we just incremented irq_count to zero, we've
4031d3e53e8SAndy Lutomirski	 * claimed the IRQ stack but we haven't switched to it yet.
4041d3e53e8SAndy Lutomirski	 *
4051d3e53e8SAndy Lutomirski	 * If anything is added that can interrupt us here without using IST,
4061d3e53e8SAndy Lutomirski	 * it must be *extremely* careful to limit its stack usage.  This
4071d3e53e8SAndy Lutomirski	 * could include kprobes and a hypothetical future IST-less #DB
4081d3e53e8SAndy Lutomirski	 * handler.
40929955909SAndy Lutomirski	 *
41029955909SAndy Lutomirski	 * The OOPS unwinder relies on the word at the top of the IRQ
41129955909SAndy Lutomirski	 * stack linking back to the previous RSP for the entire time we're
41229955909SAndy Lutomirski	 * on the IRQ stack.  For this to work reliably, we need to write
41329955909SAndy Lutomirski	 * it before we actually move ourselves to the IRQ stack.
4141d3e53e8SAndy Lutomirski	 */
4151d3e53e8SAndy Lutomirski
416e6401c13SAndy Lutomirski	movq	\old_rsp, PER_CPU_VAR(irq_stack_backing_store + IRQ_STACK_SIZE - 8)
417758a2e31SThomas Gleixner	movq	PER_CPU_VAR(hardirq_stack_ptr), %rsp
41829955909SAndy Lutomirski
41929955909SAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY
42029955909SAndy Lutomirski	/*
42129955909SAndy Lutomirski	 * If the first movq above becomes wrong due to IRQ stack layout
42229955909SAndy Lutomirski	 * changes, the only way we'll notice is if we try to unwind right
42329955909SAndy Lutomirski	 * here.  Assert that we set up the stack right to catch this type
42429955909SAndy Lutomirski	 * of bug quickly.
42529955909SAndy Lutomirski	 */
42629955909SAndy Lutomirski	cmpq	-8(%rsp), \old_rsp
42729955909SAndy Lutomirski	je	.Lirq_stack_okay\@
42829955909SAndy Lutomirski	ud2
42929955909SAndy Lutomirski	.Lirq_stack_okay\@:
43029955909SAndy Lutomirski#endif
43129955909SAndy Lutomirski
43229955909SAndy Lutomirski.Lirq_stack_push_old_rsp_\@:
4331d3e53e8SAndy Lutomirski	pushq	\old_rsp
4348c1f7558SJosh Poimboeuf
4358c1f7558SJosh Poimboeuf	.if \regs
4368c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS indirect=1
4378c1f7558SJosh Poimboeuf	.endif
4382ba64741SDominik Brodowski
4392ba64741SDominik Brodowski	.if \save_ret
4402ba64741SDominik Brodowski	/*
4412ba64741SDominik Brodowski	 * Push the return address to the stack. This return address can
4422ba64741SDominik Brodowski	 * be found at the "real" original RSP, which was offset by 8 at
4432ba64741SDominik Brodowski	 * the beginning of this macro.
4442ba64741SDominik Brodowski	 */
4452ba64741SDominik Brodowski	pushq	-8(\old_rsp)
4462ba64741SDominik Brodowski	.endif
4471d3e53e8SAndy Lutomirski.endm
4481d3e53e8SAndy Lutomirski
4491d3e53e8SAndy Lutomirski/*
4501d3e53e8SAndy Lutomirski * Undoes ENTER_IRQ_STACK.
4511d3e53e8SAndy Lutomirski */
4528c1f7558SJosh Poimboeuf.macro LEAVE_IRQ_STACK regs=1
4531d3e53e8SAndy Lutomirski	DEBUG_ENTRY_ASSERT_IRQS_OFF
4541d3e53e8SAndy Lutomirski	/* We need to be off the IRQ stack before decrementing irq_count. */
4551d3e53e8SAndy Lutomirski	popq	%rsp
4561d3e53e8SAndy Lutomirski
4578c1f7558SJosh Poimboeuf	.if \regs
4588c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS
4598c1f7558SJosh Poimboeuf	.endif
4608c1f7558SJosh Poimboeuf
4611d3e53e8SAndy Lutomirski	/*
4621d3e53e8SAndy Lutomirski	 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
4631d3e53e8SAndy Lutomirski	 * the irq stack but we're not on it.
4641d3e53e8SAndy Lutomirski	 */
4651d3e53e8SAndy Lutomirski
4661d3e53e8SAndy Lutomirski	decl	PER_CPU_VAR(irq_count)
4671d3e53e8SAndy Lutomirski.endm
4681d3e53e8SAndy Lutomirski
469cfa82a00SThomas Gleixner/**
470cfa82a00SThomas Gleixner * idtentry_body - Macro to emit code calling the C function
471cfa82a00SThomas Gleixner * @cfunc:		C function to be called
472cfa82a00SThomas Gleixner * @has_error_code:	Hardware pushed error code on stack
473cfa82a00SThomas Gleixner */
474e2dcb5f1SThomas Gleixner.macro idtentry_body cfunc has_error_code:req
475cfa82a00SThomas Gleixner
476cfa82a00SThomas Gleixner	call	error_entry
477cfa82a00SThomas Gleixner	UNWIND_HINT_REGS
478cfa82a00SThomas Gleixner
479cfa82a00SThomas Gleixner	movq	%rsp, %rdi			/* pt_regs pointer into 1st argument*/
480cfa82a00SThomas Gleixner
481cfa82a00SThomas Gleixner	.if \has_error_code == 1
482cfa82a00SThomas Gleixner		movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
483cfa82a00SThomas Gleixner		movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
484cfa82a00SThomas Gleixner	.endif
485cfa82a00SThomas Gleixner
486cfa82a00SThomas Gleixner	call	\cfunc
487cfa82a00SThomas Gleixner
488424c7d0aSThomas Gleixner	jmp	error_return
489cfa82a00SThomas Gleixner.endm
490cfa82a00SThomas Gleixner
491cfa82a00SThomas Gleixner/**
492cfa82a00SThomas Gleixner * idtentry - Macro to generate entry stubs for simple IDT entries
493cfa82a00SThomas Gleixner * @vector:		Vector number
494cfa82a00SThomas Gleixner * @asmsym:		ASM symbol for the entry point
495cfa82a00SThomas Gleixner * @cfunc:		C function to be called
496cfa82a00SThomas Gleixner * @has_error_code:	Hardware pushed error code on stack
497cfa82a00SThomas Gleixner *
498cfa82a00SThomas Gleixner * The macro emits code to set up the kernel context for straight forward
499cfa82a00SThomas Gleixner * and simple IDT entries. No IST stack, no paranoid entry checks.
500cfa82a00SThomas Gleixner */
501e2dcb5f1SThomas Gleixner.macro idtentry vector asmsym cfunc has_error_code:req
502cfa82a00SThomas GleixnerSYM_CODE_START(\asmsym)
503cfa82a00SThomas Gleixner	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
504cfa82a00SThomas Gleixner	ASM_CLAC
505cfa82a00SThomas Gleixner
506cfa82a00SThomas Gleixner	.if \has_error_code == 0
507cfa82a00SThomas Gleixner		pushq	$-1			/* ORIG_RAX: no syscall to restart */
508cfa82a00SThomas Gleixner	.endif
509cfa82a00SThomas Gleixner
510cfa82a00SThomas Gleixner	.if \vector == X86_TRAP_BP
511cfa82a00SThomas Gleixner		/*
512cfa82a00SThomas Gleixner		 * If coming from kernel space, create a 6-word gap to allow the
513cfa82a00SThomas Gleixner		 * int3 handler to emulate a call instruction.
514cfa82a00SThomas Gleixner		 */
515cfa82a00SThomas Gleixner		testb	$3, CS-ORIG_RAX(%rsp)
516cfa82a00SThomas Gleixner		jnz	.Lfrom_usermode_no_gap_\@
517cfa82a00SThomas Gleixner		.rept	6
518cfa82a00SThomas Gleixner		pushq	5*8(%rsp)
519cfa82a00SThomas Gleixner		.endr
520cfa82a00SThomas Gleixner		UNWIND_HINT_IRET_REGS offset=8
521cfa82a00SThomas Gleixner.Lfrom_usermode_no_gap_\@:
522cfa82a00SThomas Gleixner	.endif
523cfa82a00SThomas Gleixner
524e2dcb5f1SThomas Gleixner	idtentry_body \cfunc \has_error_code
525cfa82a00SThomas Gleixner
526cfa82a00SThomas Gleixner_ASM_NOKPROBE(\asmsym)
527cfa82a00SThomas GleixnerSYM_CODE_END(\asmsym)
528cfa82a00SThomas Gleixner.endm
529cfa82a00SThomas Gleixner
530cfa82a00SThomas Gleixner/*
5310bf7c314SThomas Gleixner * Interrupt entry/exit.
5320bf7c314SThomas Gleixner *
5330bf7c314SThomas Gleixner + The interrupt stubs push (vector) onto the stack, which is the error_code
5340bf7c314SThomas Gleixner * position of idtentry exceptions, and jump to one of the two idtentry points
5350bf7c314SThomas Gleixner * (common/spurious).
5360bf7c314SThomas Gleixner *
5370bf7c314SThomas Gleixner * common_interrupt is a hotpath, align it to a cache line
5380bf7c314SThomas Gleixner */
5390bf7c314SThomas Gleixner.macro idtentry_irq vector cfunc
5400bf7c314SThomas Gleixner	.p2align CONFIG_X86_L1_CACHE_SHIFT
5410bf7c314SThomas Gleixner	idtentry \vector asm_\cfunc \cfunc has_error_code=1
5420bf7c314SThomas Gleixner.endm
5430bf7c314SThomas Gleixner
5440bf7c314SThomas Gleixner/*
545cfa82a00SThomas Gleixner * MCE and DB exceptions
546cfa82a00SThomas Gleixner */
547cfa82a00SThomas Gleixner#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + (x) * 8)
548cfa82a00SThomas Gleixner
549cfa82a00SThomas Gleixner/**
550cfa82a00SThomas Gleixner * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB
551cfa82a00SThomas Gleixner * @vector:		Vector number
552cfa82a00SThomas Gleixner * @asmsym:		ASM symbol for the entry point
553cfa82a00SThomas Gleixner * @cfunc:		C function to be called
554cfa82a00SThomas Gleixner *
555cfa82a00SThomas Gleixner * The macro emits code to set up the kernel context for #MC and #DB
556cfa82a00SThomas Gleixner *
557cfa82a00SThomas Gleixner * If the entry comes from user space it uses the normal entry path
558cfa82a00SThomas Gleixner * including the return to user space work and preemption checks on
559cfa82a00SThomas Gleixner * exit.
560cfa82a00SThomas Gleixner *
561cfa82a00SThomas Gleixner * If hits in kernel mode then it needs to go through the paranoid
562cfa82a00SThomas Gleixner * entry as the exception can hit any random state. No preemption
563cfa82a00SThomas Gleixner * check on exit to keep the paranoid path simple.
564cfa82a00SThomas Gleixner *
565cfa82a00SThomas Gleixner * If the trap is #DB then the interrupt stack entry in the IST is
566cfa82a00SThomas Gleixner * moved to the second stack, so a potential recursion will have a
567cfa82a00SThomas Gleixner * fresh IST.
568cfa82a00SThomas Gleixner */
569cfa82a00SThomas Gleixner.macro idtentry_mce_db vector asmsym cfunc
570cfa82a00SThomas GleixnerSYM_CODE_START(\asmsym)
571cfa82a00SThomas Gleixner	UNWIND_HINT_IRET_REGS
572cfa82a00SThomas Gleixner	ASM_CLAC
573cfa82a00SThomas Gleixner
574cfa82a00SThomas Gleixner	pushq	$-1			/* ORIG_RAX: no syscall to restart */
575cfa82a00SThomas Gleixner
576cfa82a00SThomas Gleixner	/*
577cfa82a00SThomas Gleixner	 * If the entry is from userspace, switch stacks and treat it as
578cfa82a00SThomas Gleixner	 * a normal entry.
579cfa82a00SThomas Gleixner	 */
580cfa82a00SThomas Gleixner	testb	$3, CS-ORIG_RAX(%rsp)
581cfa82a00SThomas Gleixner	jnz	.Lfrom_usermode_switch_stack_\@
582cfa82a00SThomas Gleixner
583cfa82a00SThomas Gleixner	/*
584cfa82a00SThomas Gleixner	 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
585cfa82a00SThomas Gleixner	 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
586cfa82a00SThomas Gleixner	 */
587cfa82a00SThomas Gleixner	call	paranoid_entry
588cfa82a00SThomas Gleixner
589cfa82a00SThomas Gleixner	UNWIND_HINT_REGS
590cfa82a00SThomas Gleixner
591cfa82a00SThomas Gleixner	.if \vector == X86_TRAP_DB
592cfa82a00SThomas Gleixner		TRACE_IRQS_OFF_DEBUG
593cfa82a00SThomas Gleixner	.else
594cfa82a00SThomas Gleixner		TRACE_IRQS_OFF
595cfa82a00SThomas Gleixner	.endif
596cfa82a00SThomas Gleixner
597cfa82a00SThomas Gleixner	movq	%rsp, %rdi		/* pt_regs pointer */
598cfa82a00SThomas Gleixner
599cfa82a00SThomas Gleixner	.if \vector == X86_TRAP_DB
600cfa82a00SThomas Gleixner		subq	$DB_STACK_OFFSET, CPU_TSS_IST(IST_INDEX_DB)
601cfa82a00SThomas Gleixner	.endif
602cfa82a00SThomas Gleixner
603cfa82a00SThomas Gleixner	call	\cfunc
604cfa82a00SThomas Gleixner
605cfa82a00SThomas Gleixner	.if \vector == X86_TRAP_DB
606cfa82a00SThomas Gleixner		addq	$DB_STACK_OFFSET, CPU_TSS_IST(IST_INDEX_DB)
607cfa82a00SThomas Gleixner	.endif
608cfa82a00SThomas Gleixner
609cfa82a00SThomas Gleixner	jmp	paranoid_exit
610cfa82a00SThomas Gleixner
611cfa82a00SThomas Gleixner	/* Switch to the regular task stack and use the noist entry point */
612cfa82a00SThomas Gleixner.Lfrom_usermode_switch_stack_\@:
613e2dcb5f1SThomas Gleixner	idtentry_body noist_\cfunc, has_error_code=0
614cfa82a00SThomas Gleixner
615cfa82a00SThomas Gleixner_ASM_NOKPROBE(\asmsym)
616cfa82a00SThomas GleixnerSYM_CODE_END(\asmsym)
617cfa82a00SThomas Gleixner.endm
618cfa82a00SThomas Gleixner
619cfa82a00SThomas Gleixner/*
620cfa82a00SThomas Gleixner * Double fault entry. Straight paranoid. No checks from which context
621cfa82a00SThomas Gleixner * this comes because for the espfix induced #DF this would do the wrong
622cfa82a00SThomas Gleixner * thing.
623cfa82a00SThomas Gleixner */
624cfa82a00SThomas Gleixner.macro idtentry_df vector asmsym cfunc
625cfa82a00SThomas GleixnerSYM_CODE_START(\asmsym)
626cfa82a00SThomas Gleixner	UNWIND_HINT_IRET_REGS offset=8
627cfa82a00SThomas Gleixner	ASM_CLAC
628cfa82a00SThomas Gleixner
629cfa82a00SThomas Gleixner	/*
630cfa82a00SThomas Gleixner	 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
631cfa82a00SThomas Gleixner	 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
632cfa82a00SThomas Gleixner	 */
633cfa82a00SThomas Gleixner	call	paranoid_entry
634cfa82a00SThomas Gleixner	UNWIND_HINT_REGS
635cfa82a00SThomas Gleixner
636cfa82a00SThomas Gleixner	movq	%rsp, %rdi		/* pt_regs pointer into first argument */
637cfa82a00SThomas Gleixner	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
638cfa82a00SThomas Gleixner	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
639cfa82a00SThomas Gleixner	call	\cfunc
640cfa82a00SThomas Gleixner
641cfa82a00SThomas Gleixner	jmp	paranoid_exit
642cfa82a00SThomas Gleixner
643cfa82a00SThomas Gleixner_ASM_NOKPROBE(\asmsym)
644cfa82a00SThomas GleixnerSYM_CODE_END(\asmsym)
645cfa82a00SThomas Gleixner.endm
646cfa82a00SThomas Gleixner
647905a36a2SIngo Molnar/*
64853aaf262SThomas Gleixner * Include the defines which emit the idt entries which are shared
64953aaf262SThomas Gleixner * shared between 32 and 64 bit.
65053aaf262SThomas Gleixner */
65153aaf262SThomas Gleixner#include <asm/idtentry.h>
65253aaf262SThomas Gleixner
65353aaf262SThomas Gleixner/*
654f3d415eaSDominik Brodowski * Interrupt entry helper function.
655905a36a2SIngo Molnar *
656f3d415eaSDominik Brodowski * Entry runs with interrupts off. Stack layout at entry:
657f3d415eaSDominik Brodowski * +----------------------------------------------------+
658f3d415eaSDominik Brodowski * | regs->ss						|
659f3d415eaSDominik Brodowski * | regs->rsp						|
660f3d415eaSDominik Brodowski * | regs->eflags					|
661f3d415eaSDominik Brodowski * | regs->cs						|
662f3d415eaSDominik Brodowski * | regs->ip						|
663f3d415eaSDominik Brodowski * +----------------------------------------------------+
664f3d415eaSDominik Brodowski * | regs->orig_ax = ~(interrupt number)		|
665f3d415eaSDominik Brodowski * +----------------------------------------------------+
666f3d415eaSDominik Brodowski * | return address					|
667f3d415eaSDominik Brodowski * +----------------------------------------------------+
668905a36a2SIngo Molnar */
669bc7b11c0SJiri SlabySYM_CODE_START(interrupt_entry)
67081b67439SJosh Poimboeuf	UNWIND_HINT_IRET_REGS offset=16
671f3d415eaSDominik Brodowski	ASM_CLAC
672905a36a2SIngo Molnar	cld
6737f2590a1SAndy Lutomirski
674f3d415eaSDominik Brodowski	testb	$3, CS-ORIG_RAX+8(%rsp)
6757f2590a1SAndy Lutomirski	jz	1f
6767f2590a1SAndy Lutomirski	SWAPGS
67718ec54fdSJosh Poimboeuf	FENCE_SWAPGS_USER_ENTRY
678f3d415eaSDominik Brodowski	/*
679f3d415eaSDominik Brodowski	 * Switch to the thread stack. The IRET frame and orig_ax are
680f3d415eaSDominik Brodowski	 * on the stack, as well as the return address. RDI..R12 are
681f3d415eaSDominik Brodowski	 * not (yet) on the stack and space has not (yet) been
682f3d415eaSDominik Brodowski	 * allocated for them.
683f3d415eaSDominik Brodowski	 */
68490a6acc4SDominik Brodowski	pushq	%rdi
685f3d415eaSDominik Brodowski
68690a6acc4SDominik Brodowski	/* Need to switch before accessing the thread stack. */
68790a6acc4SDominik Brodowski	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
68890a6acc4SDominik Brodowski	movq	%rsp, %rdi
68990a6acc4SDominik Brodowski	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
690f3d415eaSDominik Brodowski
691f3d415eaSDominik Brodowski	 /*
692f3d415eaSDominik Brodowski	  * We have RDI, return address, and orig_ax on the stack on
693f3d415eaSDominik Brodowski	  * top of the IRET frame. That means offset=24
694f3d415eaSDominik Brodowski	  */
695f3d415eaSDominik Brodowski	UNWIND_HINT_IRET_REGS base=%rdi offset=24
69690a6acc4SDominik Brodowski
69790a6acc4SDominik Brodowski	pushq	7*8(%rdi)		/* regs->ss */
69890a6acc4SDominik Brodowski	pushq	6*8(%rdi)		/* regs->rsp */
69990a6acc4SDominik Brodowski	pushq	5*8(%rdi)		/* regs->eflags */
70090a6acc4SDominik Brodowski	pushq	4*8(%rdi)		/* regs->cs */
70190a6acc4SDominik Brodowski	pushq	3*8(%rdi)		/* regs->ip */
70281b67439SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
70390a6acc4SDominik Brodowski	pushq	2*8(%rdi)		/* regs->orig_ax */
70490a6acc4SDominik Brodowski	pushq	8(%rdi)			/* return address */
70590a6acc4SDominik Brodowski
70690a6acc4SDominik Brodowski	movq	(%rdi), %rdi
70764dbc122SJosh Poimboeuf	jmp	2f
7087f2590a1SAndy Lutomirski1:
70918ec54fdSJosh Poimboeuf	FENCE_SWAPGS_KERNEL_ENTRY
71018ec54fdSJosh Poimboeuf2:
7110e34d226SDominik Brodowski	PUSH_AND_CLEAR_REGS save_ret=1
7120e34d226SDominik Brodowski	ENCODE_FRAME_POINTER 8
713905a36a2SIngo Molnar
7142ba64741SDominik Brodowski	testb	$3, CS+8(%rsp)
715905a36a2SIngo Molnar	jz	1f
71602bc7768SAndy Lutomirski
71702bc7768SAndy Lutomirski	/*
7187f2590a1SAndy Lutomirski	 * IRQ from user mode.
7197f2590a1SAndy Lutomirski	 *
720f1075053SAndy Lutomirski	 * We need to tell lockdep that IRQs are off.  We can't do this until
721f1075053SAndy Lutomirski	 * we fix gsbase, and we should do it before enter_from_user_mode
722f3d415eaSDominik Brodowski	 * (which can take locks).  Since TRACE_IRQS_OFF is idempotent,
723f1075053SAndy Lutomirski	 * the simplest way to handle it is to just call it twice if
724f1075053SAndy Lutomirski	 * we enter from user mode.  There's no reason to optimize this since
725f1075053SAndy Lutomirski	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
726f1075053SAndy Lutomirski	 */
727f1075053SAndy Lutomirski	TRACE_IRQS_OFF
728f1075053SAndy Lutomirski
729478dc89cSAndy Lutomirski	CALL_enter_from_user_mode
73002bc7768SAndy Lutomirski
731905a36a2SIngo Molnar1:
7322ba64741SDominik Brodowski	ENTER_IRQ_STACK old_rsp=%rdi save_ret=1
733905a36a2SIngo Molnar	/* We entered an interrupt context - irqs are off: */
734905a36a2SIngo Molnar	TRACE_IRQS_OFF
735905a36a2SIngo Molnar
7362ba64741SDominik Brodowski	ret
737bc7b11c0SJiri SlabySYM_CODE_END(interrupt_entry)
738a50480cbSAndrea Righi_ASM_NOKPROBE(interrupt_entry)
7392ba64741SDominik Brodowski
740*fa5e5c40SThomas GleixnerSYM_CODE_START_LOCAL(common_interrupt_return)
741905a36a2SIngo Molnarret_from_intr:
7422140a994SJan Beulich	DISABLE_INTERRUPTS(CLBR_ANY)
743905a36a2SIngo Molnar	TRACE_IRQS_OFF
744905a36a2SIngo Molnar
7451d3e53e8SAndy Lutomirski	LEAVE_IRQ_STACK
746905a36a2SIngo Molnar
747905a36a2SIngo Molnar	testb	$3, CS(%rsp)
748905a36a2SIngo Molnar	jz	retint_kernel
74902bc7768SAndy Lutomirski
750905a36a2SIngo Molnar	/* Interrupt came from user space */
75130a2441cSJiri Slaby.Lretint_user:
75202bc7768SAndy Lutomirski	mov	%rsp,%rdi
75302bc7768SAndy Lutomirski	call	prepare_exit_to_usermode
75426c4ef9cSAndy Lutomirski
75526ba4e57SJiri SlabySYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
75626c4ef9cSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY
75726c4ef9cSAndy Lutomirski	/* Assert that pt_regs indicates user mode. */
7581e4c4f61SBorislav Petkov	testb	$3, CS(%rsp)
75926c4ef9cSAndy Lutomirski	jnz	1f
76026c4ef9cSAndy Lutomirski	ud2
76126c4ef9cSAndy Lutomirski1:
76226c4ef9cSAndy Lutomirski#endif
763502af0d7SDominik Brodowski	POP_REGS pop_rdi=0
7643e3b9293SAndy Lutomirski
7653e3b9293SAndy Lutomirski	/*
7663e3b9293SAndy Lutomirski	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
7673e3b9293SAndy Lutomirski	 * Save old stack pointer and switch to trampoline stack.
7683e3b9293SAndy Lutomirski	 */
7693e3b9293SAndy Lutomirski	movq	%rsp, %rdi
770c482feefSAndy Lutomirski	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
7711fb14363SJosh Poimboeuf	UNWIND_HINT_EMPTY
7723e3b9293SAndy Lutomirski
7733e3b9293SAndy Lutomirski	/* Copy the IRET frame to the trampoline stack. */
7743e3b9293SAndy Lutomirski	pushq	6*8(%rdi)	/* SS */
7753e3b9293SAndy Lutomirski	pushq	5*8(%rdi)	/* RSP */
7763e3b9293SAndy Lutomirski	pushq	4*8(%rdi)	/* EFLAGS */
7773e3b9293SAndy Lutomirski	pushq	3*8(%rdi)	/* CS */
7783e3b9293SAndy Lutomirski	pushq	2*8(%rdi)	/* RIP */
7793e3b9293SAndy Lutomirski
7803e3b9293SAndy Lutomirski	/* Push user RDI on the trampoline stack. */
7813e3b9293SAndy Lutomirski	pushq	(%rdi)
7823e3b9293SAndy Lutomirski
7833e3b9293SAndy Lutomirski	/*
7843e3b9293SAndy Lutomirski	 * We are on the trampoline stack.  All regs except RDI are live.
7853e3b9293SAndy Lutomirski	 * We can do future final exit work right here.
7863e3b9293SAndy Lutomirski	 */
787afaef01cSAlexander Popov	STACKLEAK_ERASE_NOCLOBBER
7883e3b9293SAndy Lutomirski
7896fd166aaSPeter Zijlstra	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
7908a09317bSDave Hansen
7913e3b9293SAndy Lutomirski	/* Restore RDI. */
7923e3b9293SAndy Lutomirski	popq	%rdi
7933e3b9293SAndy Lutomirski	SWAPGS
79426c4ef9cSAndy Lutomirski	INTERRUPT_RETURN
79526c4ef9cSAndy Lutomirski
796905a36a2SIngo Molnar
797905a36a2SIngo Molnar/* Returning to kernel space */
798905a36a2SIngo Molnarretint_kernel:
79948593975SThomas Gleixner#ifdef CONFIG_PREEMPTION
800905a36a2SIngo Molnar	/* Interrupts are off */
801905a36a2SIngo Molnar	/* Check if we need preemption */
8026709812fSJan Beulich	btl	$9, EFLAGS(%rsp)		/* were interrupts off? */
803905a36a2SIngo Molnar	jnc	1f
804b5b447b6SValentin Schneider	cmpl	$0, PER_CPU_VAR(__preempt_count)
805905a36a2SIngo Molnar	jnz	1f
806905a36a2SIngo Molnar	call	preempt_schedule_irq
807905a36a2SIngo Molnar1:
808905a36a2SIngo Molnar#endif
809905a36a2SIngo Molnar	/*
810905a36a2SIngo Molnar	 * The iretq could re-enable interrupts:
811905a36a2SIngo Molnar	 */
812905a36a2SIngo Molnar	TRACE_IRQS_IRETQ
813905a36a2SIngo Molnar
81426ba4e57SJiri SlabySYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
81526c4ef9cSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY
81626c4ef9cSAndy Lutomirski	/* Assert that pt_regs indicates kernel mode. */
8171e4c4f61SBorislav Petkov	testb	$3, CS(%rsp)
81826c4ef9cSAndy Lutomirski	jz	1f
81926c4ef9cSAndy Lutomirski	ud2
82026c4ef9cSAndy Lutomirski1:
82126c4ef9cSAndy Lutomirski#endif
822502af0d7SDominik Brodowski	POP_REGS
823e872045bSAndy Lutomirski	addq	$8, %rsp	/* skip regs->orig_ax */
82410bcc80eSMathieu Desnoyers	/*
82510bcc80eSMathieu Desnoyers	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
82610bcc80eSMathieu Desnoyers	 * when returning from IPI handler.
82710bcc80eSMathieu Desnoyers	 */
828905a36a2SIngo Molnar	INTERRUPT_RETURN
829905a36a2SIngo Molnar
830cc66936eSJiri SlabySYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)
8318c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
832905a36a2SIngo Molnar	/*
833905a36a2SIngo Molnar	 * Are we returning to a stack segment from the LDT?  Note: in
834905a36a2SIngo Molnar	 * 64-bit mode SS:RSP on the exception stack is always valid.
835905a36a2SIngo Molnar	 */
836905a36a2SIngo Molnar#ifdef CONFIG_X86_ESPFIX64
837905a36a2SIngo Molnar	testb	$4, (SS-RIP)(%rsp)
838905a36a2SIngo Molnar	jnz	native_irq_return_ldt
839905a36a2SIngo Molnar#endif
840905a36a2SIngo Molnar
841cc66936eSJiri SlabySYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
842905a36a2SIngo Molnar	/*
843905a36a2SIngo Molnar	 * This may fault.  Non-paranoid faults on return to userspace are
844905a36a2SIngo Molnar	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
845c29c775aSThomas Gleixner	 * Double-faults due to espfix64 are handled in exc_double_fault.
846905a36a2SIngo Molnar	 * Other faults here are fatal.
847905a36a2SIngo Molnar	 */
848905a36a2SIngo Molnar	iretq
849905a36a2SIngo Molnar
850905a36a2SIngo Molnar#ifdef CONFIG_X86_ESPFIX64
851905a36a2SIngo Molnarnative_irq_return_ldt:
85285063facSAndy Lutomirski	/*
85385063facSAndy Lutomirski	 * We are running with user GSBASE.  All GPRs contain their user
85485063facSAndy Lutomirski	 * values.  We have a percpu ESPFIX stack that is eight slots
85585063facSAndy Lutomirski	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
85685063facSAndy Lutomirski	 * of the ESPFIX stack.
85785063facSAndy Lutomirski	 *
85885063facSAndy Lutomirski	 * We clobber RAX and RDI in this code.  We stash RDI on the
85985063facSAndy Lutomirski	 * normal stack and RAX on the ESPFIX stack.
86085063facSAndy Lutomirski	 *
86185063facSAndy Lutomirski	 * The ESPFIX stack layout we set up looks like this:
86285063facSAndy Lutomirski	 *
86385063facSAndy Lutomirski	 * --- top of ESPFIX stack ---
86485063facSAndy Lutomirski	 * SS
86585063facSAndy Lutomirski	 * RSP
86685063facSAndy Lutomirski	 * RFLAGS
86785063facSAndy Lutomirski	 * CS
86885063facSAndy Lutomirski	 * RIP  <-- RSP points here when we're done
86985063facSAndy Lutomirski	 * RAX  <-- espfix_waddr points here
87085063facSAndy Lutomirski	 * --- bottom of ESPFIX stack ---
87185063facSAndy Lutomirski	 */
87285063facSAndy Lutomirski
87385063facSAndy Lutomirski	pushq	%rdi				/* Stash user RDI */
8748a09317bSDave Hansen	SWAPGS					/* to kernel GS */
8758a09317bSDave Hansen	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */
8768a09317bSDave Hansen
877905a36a2SIngo Molnar	movq	PER_CPU_VAR(espfix_waddr), %rdi
87885063facSAndy Lutomirski	movq	%rax, (0*8)(%rdi)		/* user RAX */
87985063facSAndy Lutomirski	movq	(1*8)(%rsp), %rax		/* user RIP */
880905a36a2SIngo Molnar	movq	%rax, (1*8)(%rdi)
88185063facSAndy Lutomirski	movq	(2*8)(%rsp), %rax		/* user CS */
882905a36a2SIngo Molnar	movq	%rax, (2*8)(%rdi)
88385063facSAndy Lutomirski	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
884905a36a2SIngo Molnar	movq	%rax, (3*8)(%rdi)
88585063facSAndy Lutomirski	movq	(5*8)(%rsp), %rax		/* user SS */
886905a36a2SIngo Molnar	movq	%rax, (5*8)(%rdi)
88785063facSAndy Lutomirski	movq	(4*8)(%rsp), %rax		/* user RSP */
888905a36a2SIngo Molnar	movq	%rax, (4*8)(%rdi)
88985063facSAndy Lutomirski	/* Now RAX == RSP. */
89085063facSAndy Lutomirski
89185063facSAndy Lutomirski	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */
89285063facSAndy Lutomirski
89385063facSAndy Lutomirski	/*
89485063facSAndy Lutomirski	 * espfix_stack[31:16] == 0.  The page tables are set up such that
89585063facSAndy Lutomirski	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
89685063facSAndy Lutomirski	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
89785063facSAndy Lutomirski	 * the same page.  Set up RSP so that RSP[31:16] contains the
89885063facSAndy Lutomirski	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
89985063facSAndy Lutomirski	 * still points to an RO alias of the ESPFIX stack.
90085063facSAndy Lutomirski	 */
901905a36a2SIngo Molnar	orq	PER_CPU_VAR(espfix_stack), %rax
9028a09317bSDave Hansen
9036fd166aaSPeter Zijlstra	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
9048a09317bSDave Hansen	SWAPGS					/* to user GS */
9058a09317bSDave Hansen	popq	%rdi				/* Restore user RDI */
9068a09317bSDave Hansen
907905a36a2SIngo Molnar	movq	%rax, %rsp
9088c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS offset=8
90985063facSAndy Lutomirski
91085063facSAndy Lutomirski	/*
91185063facSAndy Lutomirski	 * At this point, we cannot write to the stack any more, but we can
91285063facSAndy Lutomirski	 * still read.
91385063facSAndy Lutomirski	 */
91485063facSAndy Lutomirski	popq	%rax				/* Restore user RAX */
91585063facSAndy Lutomirski
91685063facSAndy Lutomirski	/*
91785063facSAndy Lutomirski	 * RSP now points to an ordinary IRET frame, except that the page
91885063facSAndy Lutomirski	 * is read-only and RSP[31:16] are preloaded with the userspace
91985063facSAndy Lutomirski	 * values.  We can now IRET back to userspace.
92085063facSAndy Lutomirski	 */
921905a36a2SIngo Molnar	jmp	native_irq_return_iret
922905a36a2SIngo Molnar#endif
923*fa5e5c40SThomas GleixnerSYM_CODE_END(common_interrupt_return)
924*fa5e5c40SThomas Gleixner_ASM_NOKPROBE(common_interrupt_return)
925905a36a2SIngo Molnar
926905a36a2SIngo Molnar/*
927905a36a2SIngo Molnar * APIC interrupts.
928905a36a2SIngo Molnar */
929905a36a2SIngo Molnar.macro apicinterrupt3 num sym do_sym
930bc7b11c0SJiri SlabySYM_CODE_START(\sym)
9318c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
932905a36a2SIngo Molnar	pushq	$~(\num)
9333aa99fc3SDominik Brodowski	call	interrupt_entry
9343aa99fc3SDominik Brodowski	UNWIND_HINT_REGS indirect=1
9353aa99fc3SDominik Brodowski	call	\do_sym	/* rdi points to pt_regs */
936905a36a2SIngo Molnar	jmp	ret_from_intr
937bc7b11c0SJiri SlabySYM_CODE_END(\sym)
938a50480cbSAndrea Righi_ASM_NOKPROBE(\sym)
939905a36a2SIngo Molnar.endm
940905a36a2SIngo Molnar
941469f0023SAlexander Potapenko/* Make sure APIC interrupt handlers end up in the irqentry section: */
942469f0023SAlexander Potapenko#define PUSH_SECTION_IRQENTRY	.pushsection .irqentry.text, "ax"
943469f0023SAlexander Potapenko#define POP_SECTION_IRQENTRY	.popsection
944469f0023SAlexander Potapenko
945905a36a2SIngo Molnar.macro apicinterrupt num sym do_sym
946469f0023SAlexander PotapenkoPUSH_SECTION_IRQENTRY
947905a36a2SIngo Molnarapicinterrupt3 \num \sym \do_sym
948469f0023SAlexander PotapenkoPOP_SECTION_IRQENTRY
949905a36a2SIngo Molnar.endm
950905a36a2SIngo Molnar
951905a36a2SIngo Molnar#ifdef CONFIG_SMP
9524d732138SIngo Molnarapicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
9534d732138SIngo Molnarapicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
954905a36a2SIngo Molnar#endif
955905a36a2SIngo Molnar
956905a36a2SIngo Molnar#ifdef CONFIG_X86_UV
9574d732138SIngo Molnarapicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
958905a36a2SIngo Molnar#endif
9594d732138SIngo Molnar
9604d732138SIngo Molnarapicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
9614d732138SIngo Molnarapicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
962905a36a2SIngo Molnar
963905a36a2SIngo Molnar#ifdef CONFIG_HAVE_KVM
9644d732138SIngo Molnarapicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
9654d732138SIngo Molnarapicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
966210f84b0SWincy Vanapicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
967905a36a2SIngo Molnar#endif
968905a36a2SIngo Molnar
969905a36a2SIngo Molnar#ifdef CONFIG_X86_MCE_THRESHOLD
9704d732138SIngo Molnarapicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
971905a36a2SIngo Molnar#endif
972905a36a2SIngo Molnar
9739dda1658SIngo Molnar#ifdef CONFIG_X86_MCE_AMD
9744d732138SIngo Molnarapicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
9759dda1658SIngo Molnar#endif
9769dda1658SIngo Molnar
977905a36a2SIngo Molnar#ifdef CONFIG_X86_THERMAL_VECTOR
9784d732138SIngo Molnarapicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
979905a36a2SIngo Molnar#endif
980905a36a2SIngo Molnar
981905a36a2SIngo Molnar#ifdef CONFIG_SMP
9824d732138SIngo Molnarapicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
9834d732138SIngo Molnarapicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
9844d732138SIngo Molnarapicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
985905a36a2SIngo Molnar#endif
986905a36a2SIngo Molnar
9874d732138SIngo Molnarapicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
988633260faSThomas Gleixnerapicinterrupt SPURIOUS_APIC_VECTOR		spurious_apic_interrupt		smp_spurious_apic_interrupt
989905a36a2SIngo Molnar
990905a36a2SIngo Molnar#ifdef CONFIG_IRQ_WORK
9914d732138SIngo Molnarapicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
992905a36a2SIngo Molnar#endif
993905a36a2SIngo Molnar
994905a36a2SIngo Molnar/*
9954d732138SIngo Molnar * Reload gs selector with exception handling
9964d732138SIngo Molnar * edi:  new selector
997b9f6976bSThomas Gleixner *
998b9f6976bSThomas Gleixner * Is in entry.text as it shouldn't be instrumented.
9994d732138SIngo Molnar */
1000410367e3SThomas GleixnerSYM_FUNC_START(asm_load_gs_index)
10018c1f7558SJosh Poimboeuf	FRAME_BEGIN
1002c9317202SThomas Gleixner	swapgs
100342c748bbSBorislav Petkov.Lgs_change:
1004905a36a2SIngo Molnar	movl	%edi, %gs
100596e5d28aSBorislav Petkov2:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
1006c9317202SThomas Gleixner	swapgs
10078c1f7558SJosh Poimboeuf	FRAME_END
1008905a36a2SIngo Molnar	ret
1009410367e3SThomas GleixnerSYM_FUNC_END(asm_load_gs_index)
1010410367e3SThomas GleixnerEXPORT_SYMBOL(asm_load_gs_index)
1011905a36a2SIngo Molnar
101298ededb6SJiri Slaby	_ASM_EXTABLE(.Lgs_change, .Lbad_gs)
1013905a36a2SIngo Molnar	.section .fixup, "ax"
1014905a36a2SIngo Molnar	/* running with kernelgs */
1015ef77e688SJiri SlabySYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs)
1016c9317202SThomas Gleixner	swapgs					/* switch back to user gs */
1017b038c842SAndy Lutomirski.macro ZAP_GS
1018b038c842SAndy Lutomirski	/* This can't be a string because the preprocessor needs to see it. */
1019b038c842SAndy Lutomirski	movl $__USER_DS, %eax
1020b038c842SAndy Lutomirski	movl %eax, %gs
1021b038c842SAndy Lutomirski.endm
1022b038c842SAndy Lutomirski	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
1023905a36a2SIngo Molnar	xorl	%eax, %eax
1024905a36a2SIngo Molnar	movl	%eax, %gs
1025905a36a2SIngo Molnar	jmp	2b
1026ef77e688SJiri SlabySYM_CODE_END(.Lbad_gs)
1027905a36a2SIngo Molnar	.previous
1028905a36a2SIngo Molnar
1029931b9414SThomas Gleixner/*
1030931b9414SThomas Gleixner * rdi: New stack pointer points to the top word of the stack
1031931b9414SThomas Gleixner * rsi: Function pointer
1032931b9414SThomas Gleixner * rdx: Function argument (can be NULL if none)
1033931b9414SThomas Gleixner */
1034931b9414SThomas GleixnerSYM_FUNC_START(asm_call_on_stack)
1035931b9414SThomas Gleixner	/*
1036931b9414SThomas Gleixner	 * Save the frame pointer unconditionally. This allows the ORC
1037931b9414SThomas Gleixner	 * unwinder to handle the stack switch.
1038931b9414SThomas Gleixner	 */
1039931b9414SThomas Gleixner	pushq		%rbp
1040931b9414SThomas Gleixner	mov		%rsp, %rbp
1041931b9414SThomas Gleixner
1042931b9414SThomas Gleixner	/*
1043931b9414SThomas Gleixner	 * The unwinder relies on the word at the top of the new stack
1044931b9414SThomas Gleixner	 * page linking back to the previous RSP.
1045931b9414SThomas Gleixner	 */
1046931b9414SThomas Gleixner	mov		%rsp, (%rdi)
1047931b9414SThomas Gleixner	mov		%rdi, %rsp
1048931b9414SThomas Gleixner	/* Move the argument to the right place */
1049931b9414SThomas Gleixner	mov		%rdx, %rdi
1050931b9414SThomas Gleixner
1051931b9414SThomas Gleixner1:
1052931b9414SThomas Gleixner	.pushsection .discard.instr_begin
1053931b9414SThomas Gleixner	.long 1b - .
1054931b9414SThomas Gleixner	.popsection
1055931b9414SThomas Gleixner
1056931b9414SThomas Gleixner	CALL_NOSPEC	rsi
1057931b9414SThomas Gleixner
1058931b9414SThomas Gleixner2:
1059931b9414SThomas Gleixner	.pushsection .discard.instr_end
1060931b9414SThomas Gleixner	.long 2b - .
1061931b9414SThomas Gleixner	.popsection
1062931b9414SThomas Gleixner
1063931b9414SThomas Gleixner	/* Restore the previous stack pointer from RBP. */
1064931b9414SThomas Gleixner	leaveq
1065931b9414SThomas Gleixner	ret
1066931b9414SThomas GleixnerSYM_FUNC_END(asm_call_on_stack)
1067931b9414SThomas Gleixner
106828c11b0fSJuergen Gross#ifdef CONFIG_XEN_PV
1069905a36a2SIngo Molnar/*
1070905a36a2SIngo Molnar * A note on the "critical region" in our callback handler.
1071905a36a2SIngo Molnar * We want to avoid stacking callback handlers due to events occurring
1072905a36a2SIngo Molnar * during handling of the last event. To do this, we keep events disabled
1073905a36a2SIngo Molnar * until we've done all processing. HOWEVER, we must enable events before
1074905a36a2SIngo Molnar * popping the stack frame (can't be done atomically) and so it would still
1075905a36a2SIngo Molnar * be possible to get enough handler activations to overflow the stack.
1076905a36a2SIngo Molnar * Although unlikely, bugs of that kind are hard to track down, so we'd
1077905a36a2SIngo Molnar * like to avoid the possibility.
1078905a36a2SIngo Molnar * So, on entry to the handler we detect whether we interrupted an
1079905a36a2SIngo Molnar * existing activation in its critical region -- if so, we pop the current
1080905a36a2SIngo Molnar * activation and restart the handler using the previous one.
10812f6474e4SThomas Gleixner *
10822f6474e4SThomas Gleixner * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
1083905a36a2SIngo Molnar */
10842f6474e4SThomas GleixnerSYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)
10854d732138SIngo Molnar
1086905a36a2SIngo Molnar/*
1087905a36a2SIngo Molnar * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
1088905a36a2SIngo Molnar * see the correct pointer to the pt_regs
1089905a36a2SIngo Molnar */
10908c1f7558SJosh Poimboeuf	UNWIND_HINT_FUNC
10914d732138SIngo Molnar	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
10928c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS
10931d3e53e8SAndy Lutomirski
10942f6474e4SThomas Gleixner	call	xen_pv_evtchn_do_upcall
10951d3e53e8SAndy Lutomirski
10962f6474e4SThomas Gleixner	jmp	error_return
10972f6474e4SThomas GleixnerSYM_CODE_END(exc_xen_hypervisor_callback)
1098905a36a2SIngo Molnar
1099905a36a2SIngo Molnar/*
1100905a36a2SIngo Molnar * Hypervisor uses this for application faults while it executes.
1101905a36a2SIngo Molnar * We get here for two reasons:
1102905a36a2SIngo Molnar *  1. Fault while reloading DS, ES, FS or GS
1103905a36a2SIngo Molnar *  2. Fault while executing IRET
1104905a36a2SIngo Molnar * Category 1 we do not need to fix up as Xen has already reloaded all segment
1105905a36a2SIngo Molnar * registers that could be reloaded and zeroed the others.
1106905a36a2SIngo Molnar * Category 2 we fix up by killing the current process. We cannot use the
1107905a36a2SIngo Molnar * normal Linux return path in this case because if we use the IRET hypercall
1108905a36a2SIngo Molnar * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1109905a36a2SIngo Molnar * We distinguish between categories by comparing each saved segment register
1110905a36a2SIngo Molnar * with its current contents: any discrepancy means we in category 1.
1111905a36a2SIngo Molnar */
1112bc7b11c0SJiri SlabySYM_CODE_START(xen_failsafe_callback)
11138c1f7558SJosh Poimboeuf	UNWIND_HINT_EMPTY
1114905a36a2SIngo Molnar	movl	%ds, %ecx
1115905a36a2SIngo Molnar	cmpw	%cx, 0x10(%rsp)
1116905a36a2SIngo Molnar	jne	1f
1117905a36a2SIngo Molnar	movl	%es, %ecx
1118905a36a2SIngo Molnar	cmpw	%cx, 0x18(%rsp)
1119905a36a2SIngo Molnar	jne	1f
1120905a36a2SIngo Molnar	movl	%fs, %ecx
1121905a36a2SIngo Molnar	cmpw	%cx, 0x20(%rsp)
1122905a36a2SIngo Molnar	jne	1f
1123905a36a2SIngo Molnar	movl	%gs, %ecx
1124905a36a2SIngo Molnar	cmpw	%cx, 0x28(%rsp)
1125905a36a2SIngo Molnar	jne	1f
1126905a36a2SIngo Molnar	/* All segments match their saved values => Category 2 (Bad IRET). */
1127905a36a2SIngo Molnar	movq	(%rsp), %rcx
1128905a36a2SIngo Molnar	movq	8(%rsp), %r11
1129905a36a2SIngo Molnar	addq	$0x30, %rsp
1130905a36a2SIngo Molnar	pushq	$0				/* RIP */
11318c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS offset=8
1132be4c11afSThomas Gleixner	jmp	asm_exc_general_protection
1133905a36a2SIngo Molnar1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1134905a36a2SIngo Molnar	movq	(%rsp), %rcx
1135905a36a2SIngo Molnar	movq	8(%rsp), %r11
1136905a36a2SIngo Molnar	addq	$0x30, %rsp
11378c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
1138905a36a2SIngo Molnar	pushq	$-1 /* orig_ax = -1 => not a system call */
11393f01daecSDominik Brodowski	PUSH_AND_CLEAR_REGS
1140946c1911SJosh Poimboeuf	ENCODE_FRAME_POINTER
1141e88d9741SThomas Gleixner	jmp	error_return
1142bc7b11c0SJiri SlabySYM_CODE_END(xen_failsafe_callback)
114328c11b0fSJuergen Gross#endif /* CONFIG_XEN_PV */
1144905a36a2SIngo Molnar
114528c11b0fSJuergen Gross#ifdef CONFIG_XEN_PVHVM
1146905a36a2SIngo Molnarapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1147905a36a2SIngo Molnar	xen_hvm_callback_vector xen_evtchn_do_upcall
114828c11b0fSJuergen Gross#endif
1149905a36a2SIngo Molnar
1150905a36a2SIngo Molnar
1151905a36a2SIngo Molnar#if IS_ENABLED(CONFIG_HYPERV)
1152905a36a2SIngo Molnarapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1153905a36a2SIngo Molnar	hyperv_callback_vector hyperv_vector_handler
115493286261SVitaly Kuznetsov
115593286261SVitaly Kuznetsovapicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \
115693286261SVitaly Kuznetsov	hyperv_reenlightenment_vector hyperv_reenlightenment_intr
1157248e742aSMichael Kelley
1158248e742aSMichael Kelleyapicinterrupt3 HYPERV_STIMER0_VECTOR \
1159248e742aSMichael Kelley	hv_stimer0_callback_vector hv_stimer0_vector_handler
1160905a36a2SIngo Molnar#endif /* CONFIG_HYPERV */
1161905a36a2SIngo Molnar
1162498ad393SZhao Yakui#if IS_ENABLED(CONFIG_ACRN_GUEST)
1163498ad393SZhao Yakuiapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1164498ad393SZhao Yakui	acrn_hv_callback_vector acrn_hv_vector_handler
1165498ad393SZhao Yakui#endif
1166498ad393SZhao Yakui
1167905a36a2SIngo Molnar/*
11689e809d15SDominik Brodowski * Save all registers in pt_regs, and switch gs if needed.
1169905a36a2SIngo Molnar * Use slow, but surefire "are we in kernel?" check.
1170905a36a2SIngo Molnar * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
1171905a36a2SIngo Molnar */
1172ef1e0315SJiri SlabySYM_CODE_START_LOCAL(paranoid_entry)
11738c1f7558SJosh Poimboeuf	UNWIND_HINT_FUNC
1174905a36a2SIngo Molnar	cld
11759e809d15SDominik Brodowski	PUSH_AND_CLEAR_REGS save_ret=1
11769e809d15SDominik Brodowski	ENCODE_FRAME_POINTER 8
1177905a36a2SIngo Molnar	movl	$1, %ebx
1178905a36a2SIngo Molnar	movl	$MSR_GS_BASE, %ecx
1179905a36a2SIngo Molnar	rdmsr
1180905a36a2SIngo Molnar	testl	%edx, %edx
1181905a36a2SIngo Molnar	js	1f				/* negative -> in kernel */
1182905a36a2SIngo Molnar	SWAPGS
1183905a36a2SIngo Molnar	xorl	%ebx, %ebx
11848a09317bSDave Hansen
11858a09317bSDave Hansen1:
118616561f27SDave Hansen	/*
118716561f27SDave Hansen	 * Always stash CR3 in %r14.  This value will be restored,
1188ae852495SAndy Lutomirski	 * verbatim, at exit.  Needed if paranoid_entry interrupted
1189ae852495SAndy Lutomirski	 * another entry that already switched to the user CR3 value
1190ae852495SAndy Lutomirski	 * but has not yet returned to userspace.
119116561f27SDave Hansen	 *
119216561f27SDave Hansen	 * This is also why CS (stashed in the "iret frame" by the
119316561f27SDave Hansen	 * hardware at entry) can not be used: this may be a return
1194ae852495SAndy Lutomirski	 * to kernel code, but with a user CR3 value.
119516561f27SDave Hansen	 */
11968a09317bSDave Hansen	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
11978a09317bSDave Hansen
119818ec54fdSJosh Poimboeuf	/*
119918ec54fdSJosh Poimboeuf	 * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
120018ec54fdSJosh Poimboeuf	 * unconditional CR3 write, even in the PTI case.  So do an lfence
120118ec54fdSJosh Poimboeuf	 * to prevent GS speculation, regardless of whether PTI is enabled.
120218ec54fdSJosh Poimboeuf	 */
120318ec54fdSJosh Poimboeuf	FENCE_SWAPGS_KERNEL_ENTRY
120418ec54fdSJosh Poimboeuf
12058a09317bSDave Hansen	ret
1206ef1e0315SJiri SlabySYM_CODE_END(paranoid_entry)
1207905a36a2SIngo Molnar
1208905a36a2SIngo Molnar/*
1209905a36a2SIngo Molnar * "Paranoid" exit path from exception stack.  This is invoked
1210905a36a2SIngo Molnar * only on return from non-NMI IST interrupts that came
1211905a36a2SIngo Molnar * from kernel space.
1212905a36a2SIngo Molnar *
1213905a36a2SIngo Molnar * We may be returning to very strange contexts (e.g. very early
1214905a36a2SIngo Molnar * in syscall entry), so checking for preemption here would
1215905a36a2SIngo Molnar * be complicated.  Fortunately, we there's no good reason
1216905a36a2SIngo Molnar * to try to handle preemption here.
12174d732138SIngo Molnar *
12184d732138SIngo Molnar * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1219905a36a2SIngo Molnar */
1220ef1e0315SJiri SlabySYM_CODE_START_LOCAL(paranoid_exit)
12218c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS
12222140a994SJan Beulich	DISABLE_INTERRUPTS(CLBR_ANY)
1223905a36a2SIngo Molnar	TRACE_IRQS_OFF_DEBUG
1224905a36a2SIngo Molnar	testl	%ebx, %ebx			/* swapgs needed? */
1225e5317832SAndy Lutomirski	jnz	.Lparanoid_exit_no_swapgs
1226905a36a2SIngo Molnar	TRACE_IRQS_IRETQ
122716561f27SDave Hansen	/* Always restore stashed CR3 value (see paranoid_entry) */
122821e94459SPeter Zijlstra	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1229905a36a2SIngo Molnar	SWAPGS_UNSAFE_STACK
123045c08383SThomas Gleixner	jmp	restore_regs_and_return_to_kernel
1231e5317832SAndy Lutomirski.Lparanoid_exit_no_swapgs:
1232905a36a2SIngo Molnar	TRACE_IRQS_IRETQ_DEBUG
123316561f27SDave Hansen	/* Always restore stashed CR3 value (see paranoid_entry) */
1234e4865757SIngo Molnar	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1235e5317832SAndy Lutomirski	jmp restore_regs_and_return_to_kernel
1236ef1e0315SJiri SlabySYM_CODE_END(paranoid_exit)
1237905a36a2SIngo Molnar
1238905a36a2SIngo Molnar/*
12399e809d15SDominik Brodowski * Save all registers in pt_regs, and switch GS if needed.
1240905a36a2SIngo Molnar */
1241ef1e0315SJiri SlabySYM_CODE_START_LOCAL(error_entry)
12429e809d15SDominik Brodowski	UNWIND_HINT_FUNC
1243905a36a2SIngo Molnar	cld
12449e809d15SDominik Brodowski	PUSH_AND_CLEAR_REGS save_ret=1
12459e809d15SDominik Brodowski	ENCODE_FRAME_POINTER 8
1246905a36a2SIngo Molnar	testb	$3, CS+8(%rsp)
1247cb6f64edSAndy Lutomirski	jz	.Lerror_kernelspace
1248539f5113SAndy Lutomirski
1249cb6f64edSAndy Lutomirski	/*
1250cb6f64edSAndy Lutomirski	 * We entered from user mode or we're pretending to have entered
1251cb6f64edSAndy Lutomirski	 * from user mode due to an IRET fault.
1252cb6f64edSAndy Lutomirski	 */
1253905a36a2SIngo Molnar	SWAPGS
125418ec54fdSJosh Poimboeuf	FENCE_SWAPGS_USER_ENTRY
12558a09317bSDave Hansen	/* We have user CR3.  Change to kernel CR3. */
12568a09317bSDave Hansen	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1257539f5113SAndy Lutomirski
1258cb6f64edSAndy Lutomirski.Lerror_entry_from_usermode_after_swapgs:
12597f2590a1SAndy Lutomirski	/* Put us onto the real thread stack. */
12607f2590a1SAndy Lutomirski	popq	%r12				/* save return addr in %12 */
12617f2590a1SAndy Lutomirski	movq	%rsp, %rdi			/* arg0 = pt_regs pointer */
12627f2590a1SAndy Lutomirski	call	sync_regs
12637f2590a1SAndy Lutomirski	movq	%rax, %rsp			/* switch stack */
12647f2590a1SAndy Lutomirski	ENCODE_FRAME_POINTER
12657f2590a1SAndy Lutomirski	pushq	%r12
1266f1075053SAndy Lutomirski	ret
126702bc7768SAndy Lutomirski
126818ec54fdSJosh Poimboeuf.Lerror_entry_done_lfence:
126918ec54fdSJosh Poimboeuf	FENCE_SWAPGS_KERNEL_ENTRY
1270cb6f64edSAndy Lutomirski.Lerror_entry_done:
1271905a36a2SIngo Molnar	ret
1272905a36a2SIngo Molnar
1273905a36a2SIngo Molnar	/*
1274905a36a2SIngo Molnar	 * There are two places in the kernel that can potentially fault with
1275905a36a2SIngo Molnar	 * usergs. Handle them here.  B stepping K8s sometimes report a
1276905a36a2SIngo Molnar	 * truncated RIP for IRET exceptions returning to compat mode. Check
1277905a36a2SIngo Molnar	 * for these here too.
1278905a36a2SIngo Molnar	 */
1279cb6f64edSAndy Lutomirski.Lerror_kernelspace:
1280905a36a2SIngo Molnar	leaq	native_irq_return_iret(%rip), %rcx
1281905a36a2SIngo Molnar	cmpq	%rcx, RIP+8(%rsp)
1282cb6f64edSAndy Lutomirski	je	.Lerror_bad_iret
1283905a36a2SIngo Molnar	movl	%ecx, %eax			/* zero extend */
1284905a36a2SIngo Molnar	cmpq	%rax, RIP+8(%rsp)
1285cb6f64edSAndy Lutomirski	je	.Lbstep_iret
128642c748bbSBorislav Petkov	cmpq	$.Lgs_change, RIP+8(%rsp)
128718ec54fdSJosh Poimboeuf	jne	.Lerror_entry_done_lfence
1288539f5113SAndy Lutomirski
1289539f5113SAndy Lutomirski	/*
129042c748bbSBorislav Petkov	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
1291539f5113SAndy Lutomirski	 * gsbase and proceed.  We'll fix up the exception and land in
129242c748bbSBorislav Petkov	 * .Lgs_change's error handler with kernel gsbase.
1293539f5113SAndy Lutomirski	 */
12942fa5f04fSWanpeng Li	SWAPGS
129518ec54fdSJosh Poimboeuf	FENCE_SWAPGS_USER_ENTRY
12962fa5f04fSWanpeng Li	jmp .Lerror_entry_done
1297905a36a2SIngo Molnar
1298cb6f64edSAndy Lutomirski.Lbstep_iret:
1299905a36a2SIngo Molnar	/* Fix truncated RIP */
1300905a36a2SIngo Molnar	movq	%rcx, RIP+8(%rsp)
1301905a36a2SIngo Molnar	/* fall through */
1302905a36a2SIngo Molnar
1303cb6f64edSAndy Lutomirski.Lerror_bad_iret:
1304539f5113SAndy Lutomirski	/*
13058a09317bSDave Hansen	 * We came from an IRET to user mode, so we have user
13068a09317bSDave Hansen	 * gsbase and CR3.  Switch to kernel gsbase and CR3:
1307539f5113SAndy Lutomirski	 */
1308905a36a2SIngo Molnar	SWAPGS
130918ec54fdSJosh Poimboeuf	FENCE_SWAPGS_USER_ENTRY
13108a09317bSDave Hansen	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1311539f5113SAndy Lutomirski
1312539f5113SAndy Lutomirski	/*
1313539f5113SAndy Lutomirski	 * Pretend that the exception came from user mode: set up pt_regs
1314b3681dd5SAndy Lutomirski	 * as if we faulted immediately after IRET.
1315539f5113SAndy Lutomirski	 */
1316905a36a2SIngo Molnar	mov	%rsp, %rdi
1317905a36a2SIngo Molnar	call	fixup_bad_iret
1318905a36a2SIngo Molnar	mov	%rax, %rsp
1319cb6f64edSAndy Lutomirski	jmp	.Lerror_entry_from_usermode_after_swapgs
1320ef1e0315SJiri SlabySYM_CODE_END(error_entry)
1321905a36a2SIngo Molnar
1322424c7d0aSThomas GleixnerSYM_CODE_START_LOCAL(error_return)
1323424c7d0aSThomas Gleixner	UNWIND_HINT_REGS
1324424c7d0aSThomas Gleixner	DEBUG_ENTRY_ASSERT_IRQS_OFF
1325424c7d0aSThomas Gleixner	testb	$3, CS(%rsp)
1326424c7d0aSThomas Gleixner	jz	restore_regs_and_return_to_kernel
1327424c7d0aSThomas Gleixner	jmp	swapgs_restore_regs_and_return_to_usermode
1328424c7d0aSThomas GleixnerSYM_CODE_END(error_return)
1329424c7d0aSThomas Gleixner
1330929bacecSAndy Lutomirski/*
1331929bacecSAndy Lutomirski * Runs on exception stack.  Xen PV does not go through this path at all,
1332929bacecSAndy Lutomirski * so we can use real assembly here.
13338a09317bSDave Hansen *
13348a09317bSDave Hansen * Registers:
13358a09317bSDave Hansen *	%r14: Used to save/restore the CR3 of the interrupted context
13368a09317bSDave Hansen *	      when PAGE_TABLE_ISOLATION is in use.  Do not clobber.
1337929bacecSAndy Lutomirski */
13386271fef0SThomas GleixnerSYM_CODE_START(asm_exc_nmi)
13398c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
1340929bacecSAndy Lutomirski
1341fc57a7c6SAndy Lutomirski	/*
1342905a36a2SIngo Molnar	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
1343905a36a2SIngo Molnar	 * the iretq it performs will take us out of NMI context.
1344905a36a2SIngo Molnar	 * This means that we can have nested NMIs where the next
1345905a36a2SIngo Molnar	 * NMI is using the top of the stack of the previous NMI. We
1346905a36a2SIngo Molnar	 * can't let it execute because the nested NMI will corrupt the
1347905a36a2SIngo Molnar	 * stack of the previous NMI. NMI handlers are not re-entrant
1348905a36a2SIngo Molnar	 * anyway.
1349905a36a2SIngo Molnar	 *
1350905a36a2SIngo Molnar	 * To handle this case we do the following:
1351905a36a2SIngo Molnar	 *  Check the a special location on the stack that contains
1352905a36a2SIngo Molnar	 *  a variable that is set when NMIs are executing.
1353905a36a2SIngo Molnar	 *  The interrupted task's stack is also checked to see if it
1354905a36a2SIngo Molnar	 *  is an NMI stack.
1355905a36a2SIngo Molnar	 *  If the variable is not set and the stack is not the NMI
1356905a36a2SIngo Molnar	 *  stack then:
1357905a36a2SIngo Molnar	 *    o Set the special variable on the stack
13580b22930eSAndy Lutomirski	 *    o Copy the interrupt frame into an "outermost" location on the
13590b22930eSAndy Lutomirski	 *      stack
13600b22930eSAndy Lutomirski	 *    o Copy the interrupt frame into an "iret" location on the stack
1361905a36a2SIngo Molnar	 *    o Continue processing the NMI
1362905a36a2SIngo Molnar	 *  If the variable is set or the previous stack is the NMI stack:
13630b22930eSAndy Lutomirski	 *    o Modify the "iret" location to jump to the repeat_nmi
1364905a36a2SIngo Molnar	 *    o return back to the first NMI
1365905a36a2SIngo Molnar	 *
1366905a36a2SIngo Molnar	 * Now on exit of the first NMI, we first clear the stack variable
1367905a36a2SIngo Molnar	 * The NMI stack will tell any nested NMIs at that point that it is
1368905a36a2SIngo Molnar	 * nested. Then we pop the stack normally with iret, and if there was
1369905a36a2SIngo Molnar	 * a nested NMI that updated the copy interrupt stack frame, a
1370905a36a2SIngo Molnar	 * jump will be made to the repeat_nmi code that will handle the second
1371905a36a2SIngo Molnar	 * NMI.
13729b6e6a83SAndy Lutomirski	 *
13739b6e6a83SAndy Lutomirski	 * However, espfix prevents us from directly returning to userspace
13749b6e6a83SAndy Lutomirski	 * with a single IRET instruction.  Similarly, IRET to user mode
13759b6e6a83SAndy Lutomirski	 * can fault.  We therefore handle NMIs from user space like
13769b6e6a83SAndy Lutomirski	 * other IST entries.
1377905a36a2SIngo Molnar	 */
1378905a36a2SIngo Molnar
1379e93c1730SAndy Lutomirski	ASM_CLAC
1380e93c1730SAndy Lutomirski
1381905a36a2SIngo Molnar	/* Use %rdx as our temp variable throughout */
1382905a36a2SIngo Molnar	pushq	%rdx
1383905a36a2SIngo Molnar
13849b6e6a83SAndy Lutomirski	testb	$3, CS-RIP+8(%rsp)
13859b6e6a83SAndy Lutomirski	jz	.Lnmi_from_kernel
1386905a36a2SIngo Molnar
1387905a36a2SIngo Molnar	/*
13889b6e6a83SAndy Lutomirski	 * NMI from user mode.  We need to run on the thread stack, but we
13899b6e6a83SAndy Lutomirski	 * can't go through the normal entry paths: NMIs are masked, and
13909b6e6a83SAndy Lutomirski	 * we don't want to enable interrupts, because then we'll end
13919b6e6a83SAndy Lutomirski	 * up in an awkward situation in which IRQs are on but NMIs
13929b6e6a83SAndy Lutomirski	 * are off.
139383c133cfSAndy Lutomirski	 *
139483c133cfSAndy Lutomirski	 * We also must not push anything to the stack before switching
139583c133cfSAndy Lutomirski	 * stacks lest we corrupt the "NMI executing" variable.
13969b6e6a83SAndy Lutomirski	 */
13979b6e6a83SAndy Lutomirski
1398929bacecSAndy Lutomirski	swapgs
13999b6e6a83SAndy Lutomirski	cld
140018ec54fdSJosh Poimboeuf	FENCE_SWAPGS_USER_ENTRY
14018a09317bSDave Hansen	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
14029b6e6a83SAndy Lutomirski	movq	%rsp, %rdx
14039b6e6a83SAndy Lutomirski	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
14048c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS base=%rdx offset=8
14059b6e6a83SAndy Lutomirski	pushq	5*8(%rdx)	/* pt_regs->ss */
14069b6e6a83SAndy Lutomirski	pushq	4*8(%rdx)	/* pt_regs->rsp */
14079b6e6a83SAndy Lutomirski	pushq	3*8(%rdx)	/* pt_regs->flags */
14089b6e6a83SAndy Lutomirski	pushq	2*8(%rdx)	/* pt_regs->cs */
14099b6e6a83SAndy Lutomirski	pushq	1*8(%rdx)	/* pt_regs->rip */
14108c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
14119b6e6a83SAndy Lutomirski	pushq   $-1		/* pt_regs->orig_ax */
141230907fd1SDominik Brodowski	PUSH_AND_CLEAR_REGS rdx=(%rdx)
1413946c1911SJosh Poimboeuf	ENCODE_FRAME_POINTER
14149b6e6a83SAndy Lutomirski
14159b6e6a83SAndy Lutomirski	/*
14169b6e6a83SAndy Lutomirski	 * At this point we no longer need to worry about stack damage
14179b6e6a83SAndy Lutomirski	 * due to nesting -- we're on the normal thread stack and we're
14189b6e6a83SAndy Lutomirski	 * done with the NMI stack.
14199b6e6a83SAndy Lutomirski	 */
14209b6e6a83SAndy Lutomirski
14219b6e6a83SAndy Lutomirski	movq	%rsp, %rdi
14229b6e6a83SAndy Lutomirski	movq	$-1, %rsi
14236271fef0SThomas Gleixner	call	exc_nmi
14249b6e6a83SAndy Lutomirski
14259b6e6a83SAndy Lutomirski	/*
14269b6e6a83SAndy Lutomirski	 * Return back to user mode.  We must *not* do the normal exit
1427946c1911SJosh Poimboeuf	 * work, because we don't want to enable interrupts.
14289b6e6a83SAndy Lutomirski	 */
14298a055d7fSAndy Lutomirski	jmp	swapgs_restore_regs_and_return_to_usermode
14309b6e6a83SAndy Lutomirski
14319b6e6a83SAndy Lutomirski.Lnmi_from_kernel:
14329b6e6a83SAndy Lutomirski	/*
14330b22930eSAndy Lutomirski	 * Here's what our stack frame will look like:
14340b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14350b22930eSAndy Lutomirski	 * | original SS                                             |
14360b22930eSAndy Lutomirski	 * | original Return RSP                                     |
14370b22930eSAndy Lutomirski	 * | original RFLAGS                                         |
14380b22930eSAndy Lutomirski	 * | original CS                                             |
14390b22930eSAndy Lutomirski	 * | original RIP                                            |
14400b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14410b22930eSAndy Lutomirski	 * | temp storage for rdx                                    |
14420b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14430b22930eSAndy Lutomirski	 * | "NMI executing" variable                                |
14440b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14450b22930eSAndy Lutomirski	 * | iret SS          } Copied from "outermost" frame        |
14460b22930eSAndy Lutomirski	 * | iret Return RSP  } on each loop iteration; overwritten  |
14470b22930eSAndy Lutomirski	 * | iret RFLAGS      } by a nested NMI to force another     |
14480b22930eSAndy Lutomirski	 * | iret CS          } iteration if needed.                 |
14490b22930eSAndy Lutomirski	 * | iret RIP         }                                      |
14500b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14510b22930eSAndy Lutomirski	 * | outermost SS          } initialized in first_nmi;       |
14520b22930eSAndy Lutomirski	 * | outermost Return RSP  } will not be changed before      |
14530b22930eSAndy Lutomirski	 * | outermost RFLAGS      } NMI processing is done.         |
14540b22930eSAndy Lutomirski	 * | outermost CS          } Copied to "iret" frame on each  |
14550b22930eSAndy Lutomirski	 * | outermost RIP         } iteration.                      |
14560b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14570b22930eSAndy Lutomirski	 * | pt_regs                                                 |
14580b22930eSAndy Lutomirski	 * +---------------------------------------------------------+
14590b22930eSAndy Lutomirski	 *
14600b22930eSAndy Lutomirski	 * The "original" frame is used by hardware.  Before re-enabling
14610b22930eSAndy Lutomirski	 * NMIs, we need to be done with it, and we need to leave enough
14620b22930eSAndy Lutomirski	 * space for the asm code here.
14630b22930eSAndy Lutomirski	 *
14640b22930eSAndy Lutomirski	 * We return by executing IRET while RSP points to the "iret" frame.
14650b22930eSAndy Lutomirski	 * That will either return for real or it will loop back into NMI
14660b22930eSAndy Lutomirski	 * processing.
14670b22930eSAndy Lutomirski	 *
14680b22930eSAndy Lutomirski	 * The "outermost" frame is copied to the "iret" frame on each
14690b22930eSAndy Lutomirski	 * iteration of the loop, so each iteration starts with the "iret"
14700b22930eSAndy Lutomirski	 * frame pointing to the final return target.
14710b22930eSAndy Lutomirski	 */
14720b22930eSAndy Lutomirski
14730b22930eSAndy Lutomirski	/*
14740b22930eSAndy Lutomirski	 * Determine whether we're a nested NMI.
14750b22930eSAndy Lutomirski	 *
1476a27507caSAndy Lutomirski	 * If we interrupted kernel code between repeat_nmi and
1477a27507caSAndy Lutomirski	 * end_repeat_nmi, then we are a nested NMI.  We must not
1478a27507caSAndy Lutomirski	 * modify the "iret" frame because it's being written by
1479a27507caSAndy Lutomirski	 * the outer NMI.  That's okay; the outer NMI handler is
14806271fef0SThomas Gleixner	 * about to about to call exc_nmi() anyway, so we can just
1481a27507caSAndy Lutomirski	 * resume the outer NMI.
1482a27507caSAndy Lutomirski	 */
1483a27507caSAndy Lutomirski
1484a27507caSAndy Lutomirski	movq	$repeat_nmi, %rdx
1485a27507caSAndy Lutomirski	cmpq	8(%rsp), %rdx
1486a27507caSAndy Lutomirski	ja	1f
1487a27507caSAndy Lutomirski	movq	$end_repeat_nmi, %rdx
1488a27507caSAndy Lutomirski	cmpq	8(%rsp), %rdx
1489a27507caSAndy Lutomirski	ja	nested_nmi_out
1490a27507caSAndy Lutomirski1:
1491a27507caSAndy Lutomirski
1492a27507caSAndy Lutomirski	/*
1493a27507caSAndy Lutomirski	 * Now check "NMI executing".  If it's set, then we're nested.
14940b22930eSAndy Lutomirski	 * This will not detect if we interrupted an outer NMI just
14950b22930eSAndy Lutomirski	 * before IRET.
1496905a36a2SIngo Molnar	 */
1497905a36a2SIngo Molnar	cmpl	$1, -8(%rsp)
1498905a36a2SIngo Molnar	je	nested_nmi
1499905a36a2SIngo Molnar
1500905a36a2SIngo Molnar	/*
15010b22930eSAndy Lutomirski	 * Now test if the previous stack was an NMI stack.  This covers
15020b22930eSAndy Lutomirski	 * the case where we interrupt an outer NMI after it clears
1503810bc075SAndy Lutomirski	 * "NMI executing" but before IRET.  We need to be careful, though:
1504810bc075SAndy Lutomirski	 * there is one case in which RSP could point to the NMI stack
1505810bc075SAndy Lutomirski	 * despite there being no NMI active: naughty userspace controls
1506810bc075SAndy Lutomirski	 * RSP at the very beginning of the SYSCALL targets.  We can
1507810bc075SAndy Lutomirski	 * pull a fast one on naughty userspace, though: we program
1508810bc075SAndy Lutomirski	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
1509810bc075SAndy Lutomirski	 * if it controls the kernel's RSP.  We set DF before we clear
1510810bc075SAndy Lutomirski	 * "NMI executing".
1511905a36a2SIngo Molnar	 */
1512905a36a2SIngo Molnar	lea	6*8(%rsp), %rdx
1513905a36a2SIngo Molnar	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
1514905a36a2SIngo Molnar	cmpq	%rdx, 4*8(%rsp)
1515905a36a2SIngo Molnar	/* If the stack pointer is above the NMI stack, this is a normal NMI */
1516905a36a2SIngo Molnar	ja	first_nmi
15174d732138SIngo Molnar
1518905a36a2SIngo Molnar	subq	$EXCEPTION_STKSZ, %rdx
1519905a36a2SIngo Molnar	cmpq	%rdx, 4*8(%rsp)
1520905a36a2SIngo Molnar	/* If it is below the NMI stack, it is a normal NMI */
1521905a36a2SIngo Molnar	jb	first_nmi
1522810bc075SAndy Lutomirski
1523810bc075SAndy Lutomirski	/* Ah, it is within the NMI stack. */
1524810bc075SAndy Lutomirski
1525810bc075SAndy Lutomirski	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
1526810bc075SAndy Lutomirski	jz	first_nmi	/* RSP was user controlled. */
1527810bc075SAndy Lutomirski
1528810bc075SAndy Lutomirski	/* This is a nested NMI. */
1529905a36a2SIngo Molnar
1530905a36a2SIngo Molnarnested_nmi:
1531905a36a2SIngo Molnar	/*
15320b22930eSAndy Lutomirski	 * Modify the "iret" frame to point to repeat_nmi, forcing another
15330b22930eSAndy Lutomirski	 * iteration of NMI handling.
1534905a36a2SIngo Molnar	 */
153523a781e9SAndy Lutomirski	subq	$8, %rsp
1536905a36a2SIngo Molnar	leaq	-10*8(%rsp), %rdx
1537905a36a2SIngo Molnar	pushq	$__KERNEL_DS
1538905a36a2SIngo Molnar	pushq	%rdx
1539905a36a2SIngo Molnar	pushfq
1540905a36a2SIngo Molnar	pushq	$__KERNEL_CS
1541905a36a2SIngo Molnar	pushq	$repeat_nmi
1542905a36a2SIngo Molnar
1543905a36a2SIngo Molnar	/* Put stack back */
1544905a36a2SIngo Molnar	addq	$(6*8), %rsp
1545905a36a2SIngo Molnar
1546905a36a2SIngo Molnarnested_nmi_out:
1547905a36a2SIngo Molnar	popq	%rdx
1548905a36a2SIngo Molnar
15490b22930eSAndy Lutomirski	/* We are returning to kernel mode, so this cannot result in a fault. */
1550929bacecSAndy Lutomirski	iretq
1551905a36a2SIngo Molnar
1552905a36a2SIngo Molnarfirst_nmi:
15530b22930eSAndy Lutomirski	/* Restore rdx. */
1554905a36a2SIngo Molnar	movq	(%rsp), %rdx
1555905a36a2SIngo Molnar
155636f1a77bSAndy Lutomirski	/* Make room for "NMI executing". */
155736f1a77bSAndy Lutomirski	pushq	$0
1558905a36a2SIngo Molnar
15590b22930eSAndy Lutomirski	/* Leave room for the "iret" frame */
1560905a36a2SIngo Molnar	subq	$(5*8), %rsp
1561905a36a2SIngo Molnar
15620b22930eSAndy Lutomirski	/* Copy the "original" frame to the "outermost" frame */
1563905a36a2SIngo Molnar	.rept 5
1564905a36a2SIngo Molnar	pushq	11*8(%rsp)
1565905a36a2SIngo Molnar	.endr
15668c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
1567905a36a2SIngo Molnar
1568905a36a2SIngo Molnar	/* Everything up to here is safe from nested NMIs */
1569905a36a2SIngo Molnar
1570a97439aaSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY
1571a97439aaSAndy Lutomirski	/*
1572a97439aaSAndy Lutomirski	 * For ease of testing, unmask NMIs right away.  Disabled by
1573a97439aaSAndy Lutomirski	 * default because IRET is very expensive.
1574a97439aaSAndy Lutomirski	 */
1575a97439aaSAndy Lutomirski	pushq	$0		/* SS */
1576a97439aaSAndy Lutomirski	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
1577a97439aaSAndy Lutomirski	addq	$8, (%rsp)	/* Fix up RSP */
1578a97439aaSAndy Lutomirski	pushfq			/* RFLAGS */
1579a97439aaSAndy Lutomirski	pushq	$__KERNEL_CS	/* CS */
1580a97439aaSAndy Lutomirski	pushq	$1f		/* RIP */
1581929bacecSAndy Lutomirski	iretq			/* continues at repeat_nmi below */
15828c1f7558SJosh Poimboeuf	UNWIND_HINT_IRET_REGS
1583a97439aaSAndy Lutomirski1:
1584a97439aaSAndy Lutomirski#endif
1585a97439aaSAndy Lutomirski
15860b22930eSAndy Lutomirskirepeat_nmi:
1587905a36a2SIngo Molnar	/*
1588905a36a2SIngo Molnar	 * If there was a nested NMI, the first NMI's iret will return
1589905a36a2SIngo Molnar	 * here. But NMIs are still enabled and we can take another
1590905a36a2SIngo Molnar	 * nested NMI. The nested NMI checks the interrupted RIP to see
1591905a36a2SIngo Molnar	 * if it is between repeat_nmi and end_repeat_nmi, and if so
1592905a36a2SIngo Molnar	 * it will just return, as we are about to repeat an NMI anyway.
1593905a36a2SIngo Molnar	 * This makes it safe to copy to the stack frame that a nested
1594905a36a2SIngo Molnar	 * NMI will update.
15950b22930eSAndy Lutomirski	 *
15960b22930eSAndy Lutomirski	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
15970b22930eSAndy Lutomirski	 * we're repeating an NMI, gsbase has the same value that it had on
15980b22930eSAndy Lutomirski	 * the first iteration.  paranoid_entry will load the kernel
15996271fef0SThomas Gleixner	 * gsbase if needed before we call exc_nmi().  "NMI executing"
160036f1a77bSAndy Lutomirski	 * is zero.
1601905a36a2SIngo Molnar	 */
160236f1a77bSAndy Lutomirski	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1603905a36a2SIngo Molnar
16040b22930eSAndy Lutomirski	/*
16050b22930eSAndy Lutomirski	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
16060b22930eSAndy Lutomirski	 * here must not modify the "iret" frame while we're writing to
16070b22930eSAndy Lutomirski	 * it or it will end up containing garbage.
16080b22930eSAndy Lutomirski	 */
1609905a36a2SIngo Molnar	addq	$(10*8), %rsp
1610905a36a2SIngo Molnar	.rept 5
1611905a36a2SIngo Molnar	pushq	-6*8(%rsp)
1612905a36a2SIngo Molnar	.endr
1613905a36a2SIngo Molnar	subq	$(5*8), %rsp
1614905a36a2SIngo Molnarend_repeat_nmi:
1615905a36a2SIngo Molnar
1616905a36a2SIngo Molnar	/*
16170b22930eSAndy Lutomirski	 * Everything below this point can be preempted by a nested NMI.
16180b22930eSAndy Lutomirski	 * If this happens, then the inner NMI will change the "iret"
16190b22930eSAndy Lutomirski	 * frame to point back to repeat_nmi.
1620905a36a2SIngo Molnar	 */
1621905a36a2SIngo Molnar	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1622905a36a2SIngo Molnar
1623905a36a2SIngo Molnar	/*
1624905a36a2SIngo Molnar	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1625905a36a2SIngo Molnar	 * as we should not be calling schedule in NMI context.
1626905a36a2SIngo Molnar	 * Even with normal interrupts enabled. An NMI should not be
1627905a36a2SIngo Molnar	 * setting NEED_RESCHED or anything that normal interrupts and
1628905a36a2SIngo Molnar	 * exceptions might do.
1629905a36a2SIngo Molnar	 */
1630905a36a2SIngo Molnar	call	paranoid_entry
16318c1f7558SJosh Poimboeuf	UNWIND_HINT_REGS
1632905a36a2SIngo Molnar
16336271fef0SThomas Gleixner	/* paranoidentry exc_nmi(), 0; without TRACE_IRQS_OFF */
1634905a36a2SIngo Molnar	movq	%rsp, %rdi
1635905a36a2SIngo Molnar	movq	$-1, %rsi
16366271fef0SThomas Gleixner	call	exc_nmi
1637905a36a2SIngo Molnar
163816561f27SDave Hansen	/* Always restore stashed CR3 value (see paranoid_entry) */
163921e94459SPeter Zijlstra	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
16408a09317bSDave Hansen
1641905a36a2SIngo Molnar	testl	%ebx, %ebx			/* swapgs needed? */
1642905a36a2SIngo Molnar	jnz	nmi_restore
1643905a36a2SIngo Molnarnmi_swapgs:
1644905a36a2SIngo Molnar	SWAPGS_UNSAFE_STACK
1645905a36a2SIngo Molnarnmi_restore:
1646502af0d7SDominik Brodowski	POP_REGS
16470b22930eSAndy Lutomirski
1648471ee483SAndy Lutomirski	/*
1649471ee483SAndy Lutomirski	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
1650471ee483SAndy Lutomirski	 * at the "iret" frame.
1651471ee483SAndy Lutomirski	 */
1652471ee483SAndy Lutomirski	addq	$6*8, %rsp
1653905a36a2SIngo Molnar
1654810bc075SAndy Lutomirski	/*
1655810bc075SAndy Lutomirski	 * Clear "NMI executing".  Set DF first so that we can easily
1656810bc075SAndy Lutomirski	 * distinguish the remaining code between here and IRET from
1657929bacecSAndy Lutomirski	 * the SYSCALL entry and exit paths.
1658929bacecSAndy Lutomirski	 *
1659929bacecSAndy Lutomirski	 * We arguably should just inspect RIP instead, but I (Andy) wrote
1660929bacecSAndy Lutomirski	 * this code when I had the misapprehension that Xen PV supported
1661929bacecSAndy Lutomirski	 * NMIs, and Xen PV would break that approach.
1662810bc075SAndy Lutomirski	 */
1663810bc075SAndy Lutomirski	std
1664810bc075SAndy Lutomirski	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
16650b22930eSAndy Lutomirski
16660b22930eSAndy Lutomirski	/*
1667929bacecSAndy Lutomirski	 * iretq reads the "iret" frame and exits the NMI stack in a
1668929bacecSAndy Lutomirski	 * single instruction.  We are returning to kernel mode, so this
1669929bacecSAndy Lutomirski	 * cannot result in a fault.  Similarly, we don't need to worry
1670929bacecSAndy Lutomirski	 * about espfix64 on the way back to kernel mode.
16710b22930eSAndy Lutomirski	 */
1672929bacecSAndy Lutomirski	iretq
16736271fef0SThomas GleixnerSYM_CODE_END(asm_exc_nmi)
1674905a36a2SIngo Molnar
1675dffb3f9dSAndy Lutomirski#ifndef CONFIG_IA32_EMULATION
1676dffb3f9dSAndy Lutomirski/*
1677dffb3f9dSAndy Lutomirski * This handles SYSCALL from 32-bit code.  There is no way to program
1678dffb3f9dSAndy Lutomirski * MSRs to fully disable 32-bit SYSCALL.
1679dffb3f9dSAndy Lutomirski */
1680bc7b11c0SJiri SlabySYM_CODE_START(ignore_sysret)
16818c1f7558SJosh Poimboeuf	UNWIND_HINT_EMPTY
1682905a36a2SIngo Molnar	mov	$-ENOSYS, %eax
1683b2b1d94cSJan Beulich	sysretl
1684bc7b11c0SJiri SlabySYM_CODE_END(ignore_sysret)
1685dffb3f9dSAndy Lutomirski#endif
16862deb4be2SAndy Lutomirski
1687b9f6976bSThomas Gleixner.pushsection .text, "ax"
1688bc7b11c0SJiri SlabySYM_CODE_START(rewind_stack_do_exit)
16898c1f7558SJosh Poimboeuf	UNWIND_HINT_FUNC
16902deb4be2SAndy Lutomirski	/* Prevent any naive code from trying to unwind to our caller. */
16912deb4be2SAndy Lutomirski	xorl	%ebp, %ebp
16922deb4be2SAndy Lutomirski
16932deb4be2SAndy Lutomirski	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
16948c1f7558SJosh Poimboeuf	leaq	-PTREGS_SIZE(%rax), %rsp
1695f977df7bSJann Horn	UNWIND_HINT_REGS
16962deb4be2SAndy Lutomirski
16972deb4be2SAndy Lutomirski	call	do_exit
1698bc7b11c0SJiri SlabySYM_CODE_END(rewind_stack_do_exit)
1699b9f6976bSThomas Gleixner.popsection
1700