xref: /openbmc/linux/arch/x86/entry/entry_64.S (revision 800e26b8)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  linux/arch/x86_64/entry.S
4 *
5 *  Copyright (C) 1991, 1992  Linus Torvalds
6 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
7 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
8 *
9 * entry.S contains the system-call and fault low-level handling routines.
10 *
11 * Some of this is documented in Documentation/x86/entry_64.rst
12 *
13 * A note on terminology:
14 * - iret frame:	Architecture defined interrupt frame from SS to RIP
15 *			at the top of the kernel process stack.
16 *
17 * Some macro usage:
18 * - SYM_FUNC_START/END:Define functions in the symbol table.
19 * - idtentry:		Define exception entry points.
20 */
21#include <linux/linkage.h>
22#include <asm/segment.h>
23#include <asm/cache.h>
24#include <asm/errno.h>
25#include <asm/asm-offsets.h>
26#include <asm/msr.h>
27#include <asm/unistd.h>
28#include <asm/thread_info.h>
29#include <asm/hw_irq.h>
30#include <asm/page_types.h>
31#include <asm/irqflags.h>
32#include <asm/paravirt.h>
33#include <asm/percpu.h>
34#include <asm/asm.h>
35#include <asm/smap.h>
36#include <asm/pgtable_types.h>
37#include <asm/export.h>
38#include <asm/frame.h>
39#include <asm/trapnr.h>
40#include <asm/nospec-branch.h>
41#include <linux/err.h>
42
43#include "calling.h"
44
45.code64
46.section .entry.text, "ax"
47
48#ifdef CONFIG_PARAVIRT
49SYM_CODE_START(native_usergs_sysret64)
50	UNWIND_HINT_EMPTY
51	swapgs
52	sysretq
53SYM_CODE_END(native_usergs_sysret64)
54#endif /* CONFIG_PARAVIRT */
55
56/*
57 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
58 *
59 * This is the only entry point used for 64-bit system calls.  The
60 * hardware interface is reasonably well designed and the register to
61 * argument mapping Linux uses fits well with the registers that are
62 * available when SYSCALL is used.
63 *
64 * SYSCALL instructions can be found inlined in libc implementations as
65 * well as some other programs and libraries.  There are also a handful
66 * of SYSCALL instructions in the vDSO used, for example, as a
67 * clock_gettimeofday fallback.
68 *
69 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
70 * then loads new ss, cs, and rip from previously programmed MSRs.
71 * rflags gets masked by a value from another MSR (so CLD and CLAC
72 * are not needed). SYSCALL does not save anything on the stack
73 * and does not change rsp.
74 *
75 * Registers on entry:
76 * rax  system call number
77 * rcx  return address
78 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
79 * rdi  arg0
80 * rsi  arg1
81 * rdx  arg2
82 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
83 * r8   arg4
84 * r9   arg5
85 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
86 *
87 * Only called from user space.
88 *
89 * When user can change pt_regs->foo always force IRET. That is because
90 * it deals with uncanonical addresses better. SYSRET has trouble
91 * with them due to bugs in both AMD and Intel CPUs.
92 */
93
94SYM_CODE_START(entry_SYSCALL_64)
95	UNWIND_HINT_EMPTY
96
97	swapgs
98	/* tss.sp2 is scratch space. */
99	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
100	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
101	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
102
103	/* Construct struct pt_regs on stack */
104	pushq	$__USER_DS				/* pt_regs->ss */
105	pushq	PER_CPU_VAR(cpu_tss_rw + TSS_sp2)	/* pt_regs->sp */
106	pushq	%r11					/* pt_regs->flags */
107	pushq	$__USER_CS				/* pt_regs->cs */
108	pushq	%rcx					/* pt_regs->ip */
109SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
110	pushq	%rax					/* pt_regs->orig_ax */
111
112	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
113
114	/* IRQs are off. */
115	movq	%rax, %rdi
116	movq	%rsp, %rsi
117	call	do_syscall_64		/* returns with IRQs disabled */
118
119	/*
120	 * Try to use SYSRET instead of IRET if we're returning to
121	 * a completely clean 64-bit userspace context.  If we're not,
122	 * go to the slow exit path.
123	 */
124	movq	RCX(%rsp), %rcx
125	movq	RIP(%rsp), %r11
126
127	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
128	jne	swapgs_restore_regs_and_return_to_usermode
129
130	/*
131	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
132	 * in kernel space.  This essentially lets the user take over
133	 * the kernel, since userspace controls RSP.
134	 *
135	 * If width of "canonical tail" ever becomes variable, this will need
136	 * to be updated to remain correct on both old and new CPUs.
137	 *
138	 * Change top bits to match most significant bit (47th or 56th bit
139	 * depending on paging mode) in the address.
140	 */
141#ifdef CONFIG_X86_5LEVEL
142	ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
143		"shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
144#else
145	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
146	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
147#endif
148
149	/* If this changed %rcx, it was not canonical */
150	cmpq	%rcx, %r11
151	jne	swapgs_restore_regs_and_return_to_usermode
152
153	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
154	jne	swapgs_restore_regs_and_return_to_usermode
155
156	movq	R11(%rsp), %r11
157	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
158	jne	swapgs_restore_regs_and_return_to_usermode
159
160	/*
161	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
162	 * restore RF properly. If the slowpath sets it for whatever reason, we
163	 * need to restore it correctly.
164	 *
165	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
166	 * trap from userspace immediately after SYSRET.  This would cause an
167	 * infinite loop whenever #DB happens with register state that satisfies
168	 * the opportunistic SYSRET conditions.  For example, single-stepping
169	 * this user code:
170	 *
171	 *           movq	$stuck_here, %rcx
172	 *           pushfq
173	 *           popq %r11
174	 *   stuck_here:
175	 *
176	 * would never get past 'stuck_here'.
177	 */
178	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
179	jnz	swapgs_restore_regs_and_return_to_usermode
180
181	/* nothing to check for RSP */
182
183	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
184	jne	swapgs_restore_regs_and_return_to_usermode
185
186	/*
187	 * We win! This label is here just for ease of understanding
188	 * perf profiles. Nothing jumps here.
189	 */
190syscall_return_via_sysret:
191	/* rcx and r11 are already restored (see code above) */
192	POP_REGS pop_rdi=0 skip_r11rcx=1
193
194	/*
195	 * Now all regs are restored except RSP and RDI.
196	 * Save old stack pointer and switch to trampoline stack.
197	 */
198	movq	%rsp, %rdi
199	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
200	UNWIND_HINT_EMPTY
201
202	pushq	RSP-RDI(%rdi)	/* RSP */
203	pushq	(%rdi)		/* RDI */
204
205	/*
206	 * We are on the trampoline stack.  All regs except RDI are live.
207	 * We can do future final exit work right here.
208	 */
209	STACKLEAK_ERASE_NOCLOBBER
210
211	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
212
213	popq	%rdi
214	popq	%rsp
215	USERGS_SYSRET64
216SYM_CODE_END(entry_SYSCALL_64)
217
218/*
219 * %rdi: prev task
220 * %rsi: next task
221 */
222.pushsection .text, "ax"
223SYM_FUNC_START(__switch_to_asm)
224	/*
225	 * Save callee-saved registers
226	 * This must match the order in inactive_task_frame
227	 */
228	pushq	%rbp
229	pushq	%rbx
230	pushq	%r12
231	pushq	%r13
232	pushq	%r14
233	pushq	%r15
234
235	/* switch stack */
236	movq	%rsp, TASK_threadsp(%rdi)
237	movq	TASK_threadsp(%rsi), %rsp
238
239#ifdef CONFIG_STACKPROTECTOR
240	movq	TASK_stack_canary(%rsi), %rbx
241	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
242#endif
243
244#ifdef CONFIG_RETPOLINE
245	/*
246	 * When switching from a shallower to a deeper call stack
247	 * the RSB may either underflow or use entries populated
248	 * with userspace addresses. On CPUs where those concerns
249	 * exist, overwrite the RSB with entries which capture
250	 * speculative execution to prevent attack.
251	 */
252	FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
253#endif
254
255	/* restore callee-saved registers */
256	popq	%r15
257	popq	%r14
258	popq	%r13
259	popq	%r12
260	popq	%rbx
261	popq	%rbp
262
263	jmp	__switch_to
264SYM_FUNC_END(__switch_to_asm)
265.popsection
266
267/*
268 * A newly forked process directly context switches into this address.
269 *
270 * rax: prev task we switched from
271 * rbx: kernel thread func (NULL for user thread)
272 * r12: kernel thread arg
273 */
274.pushsection .text, "ax"
275SYM_CODE_START(ret_from_fork)
276	UNWIND_HINT_EMPTY
277	movq	%rax, %rdi
278	call	schedule_tail			/* rdi: 'prev' task parameter */
279
280	testq	%rbx, %rbx			/* from kernel_thread? */
281	jnz	1f				/* kernel threads are uncommon */
282
2832:
284	UNWIND_HINT_REGS
285	movq	%rsp, %rdi
286	call	syscall_return_slowpath	/* returns with IRQs disabled */
287	jmp	swapgs_restore_regs_and_return_to_usermode
288
2891:
290	/* kernel thread */
291	UNWIND_HINT_EMPTY
292	movq	%r12, %rdi
293	CALL_NOSPEC rbx
294	/*
295	 * A kernel thread is allowed to return here after successfully
296	 * calling do_execve().  Exit to userspace to complete the execve()
297	 * syscall.
298	 */
299	movq	$0, RAX(%rsp)
300	jmp	2b
301SYM_CODE_END(ret_from_fork)
302.popsection
303
304.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
305#ifdef CONFIG_DEBUG_ENTRY
306	pushq %rax
307	SAVE_FLAGS(CLBR_RAX)
308	testl $X86_EFLAGS_IF, %eax
309	jz .Lokay_\@
310	ud2
311.Lokay_\@:
312	popq %rax
313#endif
314.endm
315
316/**
317 * idtentry_body - Macro to emit code calling the C function
318 * @cfunc:		C function to be called
319 * @has_error_code:	Hardware pushed error code on stack
320 */
321.macro idtentry_body cfunc has_error_code:req
322
323	call	error_entry
324	UNWIND_HINT_REGS
325
326	movq	%rsp, %rdi			/* pt_regs pointer into 1st argument*/
327
328	.if \has_error_code == 1
329		movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
330		movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
331	.endif
332
333	call	\cfunc
334
335	jmp	error_return
336.endm
337
338/**
339 * idtentry - Macro to generate entry stubs for simple IDT entries
340 * @vector:		Vector number
341 * @asmsym:		ASM symbol for the entry point
342 * @cfunc:		C function to be called
343 * @has_error_code:	Hardware pushed error code on stack
344 *
345 * The macro emits code to set up the kernel context for straight forward
346 * and simple IDT entries. No IST stack, no paranoid entry checks.
347 */
348.macro idtentry vector asmsym cfunc has_error_code:req
349SYM_CODE_START(\asmsym)
350	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
351	ASM_CLAC
352
353	.if \has_error_code == 0
354		pushq	$-1			/* ORIG_RAX: no syscall to restart */
355	.endif
356
357	.if \vector == X86_TRAP_BP
358		/*
359		 * If coming from kernel space, create a 6-word gap to allow the
360		 * int3 handler to emulate a call instruction.
361		 */
362		testb	$3, CS-ORIG_RAX(%rsp)
363		jnz	.Lfrom_usermode_no_gap_\@
364		.rept	6
365		pushq	5*8(%rsp)
366		.endr
367		UNWIND_HINT_IRET_REGS offset=8
368.Lfrom_usermode_no_gap_\@:
369	.endif
370
371	idtentry_body \cfunc \has_error_code
372
373_ASM_NOKPROBE(\asmsym)
374SYM_CODE_END(\asmsym)
375.endm
376
377/*
378 * Interrupt entry/exit.
379 *
380 + The interrupt stubs push (vector) onto the stack, which is the error_code
381 * position of idtentry exceptions, and jump to one of the two idtentry points
382 * (common/spurious).
383 *
384 * common_interrupt is a hotpath, align it to a cache line
385 */
386.macro idtentry_irq vector cfunc
387	.p2align CONFIG_X86_L1_CACHE_SHIFT
388	idtentry \vector asm_\cfunc \cfunc has_error_code=1
389.endm
390
391/*
392 * System vectors which invoke their handlers directly and are not
393 * going through the regular common device interrupt handling code.
394 */
395.macro idtentry_sysvec vector cfunc
396	idtentry \vector asm_\cfunc \cfunc has_error_code=0
397.endm
398
399/**
400 * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB
401 * @vector:		Vector number
402 * @asmsym:		ASM symbol for the entry point
403 * @cfunc:		C function to be called
404 *
405 * The macro emits code to set up the kernel context for #MC and #DB
406 *
407 * If the entry comes from user space it uses the normal entry path
408 * including the return to user space work and preemption checks on
409 * exit.
410 *
411 * If hits in kernel mode then it needs to go through the paranoid
412 * entry as the exception can hit any random state. No preemption
413 * check on exit to keep the paranoid path simple.
414 */
415.macro idtentry_mce_db vector asmsym cfunc
416SYM_CODE_START(\asmsym)
417	UNWIND_HINT_IRET_REGS
418	ASM_CLAC
419
420	pushq	$-1			/* ORIG_RAX: no syscall to restart */
421
422	/*
423	 * If the entry is from userspace, switch stacks and treat it as
424	 * a normal entry.
425	 */
426	testb	$3, CS-ORIG_RAX(%rsp)
427	jnz	.Lfrom_usermode_switch_stack_\@
428
429	/*
430	 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
431	 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
432	 */
433	call	paranoid_entry
434
435	UNWIND_HINT_REGS
436
437	movq	%rsp, %rdi		/* pt_regs pointer */
438
439	call	\cfunc
440
441	jmp	paranoid_exit
442
443	/* Switch to the regular task stack and use the noist entry point */
444.Lfrom_usermode_switch_stack_\@:
445	idtentry_body noist_\cfunc, has_error_code=0
446
447_ASM_NOKPROBE(\asmsym)
448SYM_CODE_END(\asmsym)
449.endm
450
451/*
452 * Double fault entry. Straight paranoid. No checks from which context
453 * this comes because for the espfix induced #DF this would do the wrong
454 * thing.
455 */
456.macro idtentry_df vector asmsym cfunc
457SYM_CODE_START(\asmsym)
458	UNWIND_HINT_IRET_REGS offset=8
459	ASM_CLAC
460
461	/*
462	 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
463	 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
464	 */
465	call	paranoid_entry
466	UNWIND_HINT_REGS
467
468	movq	%rsp, %rdi		/* pt_regs pointer into first argument */
469	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
470	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
471	call	\cfunc
472
473	jmp	paranoid_exit
474
475_ASM_NOKPROBE(\asmsym)
476SYM_CODE_END(\asmsym)
477.endm
478
479/*
480 * Include the defines which emit the idt entries which are shared
481 * shared between 32 and 64 bit and emit the __irqentry_text_* markers
482 * so the stacktrace boundary checks work.
483 */
484	.align 16
485	.globl __irqentry_text_start
486__irqentry_text_start:
487
488#include <asm/idtentry.h>
489
490	.align 16
491	.globl __irqentry_text_end
492__irqentry_text_end:
493
494SYM_CODE_START_LOCAL(common_interrupt_return)
495SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
496#ifdef CONFIG_DEBUG_ENTRY
497	/* Assert that pt_regs indicates user mode. */
498	testb	$3, CS(%rsp)
499	jnz	1f
500	ud2
5011:
502#endif
503	POP_REGS pop_rdi=0
504
505	/*
506	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
507	 * Save old stack pointer and switch to trampoline stack.
508	 */
509	movq	%rsp, %rdi
510	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
511	UNWIND_HINT_EMPTY
512
513	/* Copy the IRET frame to the trampoline stack. */
514	pushq	6*8(%rdi)	/* SS */
515	pushq	5*8(%rdi)	/* RSP */
516	pushq	4*8(%rdi)	/* EFLAGS */
517	pushq	3*8(%rdi)	/* CS */
518	pushq	2*8(%rdi)	/* RIP */
519
520	/* Push user RDI on the trampoline stack. */
521	pushq	(%rdi)
522
523	/*
524	 * We are on the trampoline stack.  All regs except RDI are live.
525	 * We can do future final exit work right here.
526	 */
527	STACKLEAK_ERASE_NOCLOBBER
528
529	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
530
531	/* Restore RDI. */
532	popq	%rdi
533	SWAPGS
534	INTERRUPT_RETURN
535
536
537SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
538#ifdef CONFIG_DEBUG_ENTRY
539	/* Assert that pt_regs indicates kernel mode. */
540	testb	$3, CS(%rsp)
541	jz	1f
542	ud2
5431:
544#endif
545	POP_REGS
546	addq	$8, %rsp	/* skip regs->orig_ax */
547	/*
548	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
549	 * when returning from IPI handler.
550	 */
551	INTERRUPT_RETURN
552
553SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)
554	UNWIND_HINT_IRET_REGS
555	/*
556	 * Are we returning to a stack segment from the LDT?  Note: in
557	 * 64-bit mode SS:RSP on the exception stack is always valid.
558	 */
559#ifdef CONFIG_X86_ESPFIX64
560	testb	$4, (SS-RIP)(%rsp)
561	jnz	native_irq_return_ldt
562#endif
563
564SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
565	/*
566	 * This may fault.  Non-paranoid faults on return to userspace are
567	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
568	 * Double-faults due to espfix64 are handled in exc_double_fault.
569	 * Other faults here are fatal.
570	 */
571	iretq
572
573#ifdef CONFIG_X86_ESPFIX64
574native_irq_return_ldt:
575	/*
576	 * We are running with user GSBASE.  All GPRs contain their user
577	 * values.  We have a percpu ESPFIX stack that is eight slots
578	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
579	 * of the ESPFIX stack.
580	 *
581	 * We clobber RAX and RDI in this code.  We stash RDI on the
582	 * normal stack and RAX on the ESPFIX stack.
583	 *
584	 * The ESPFIX stack layout we set up looks like this:
585	 *
586	 * --- top of ESPFIX stack ---
587	 * SS
588	 * RSP
589	 * RFLAGS
590	 * CS
591	 * RIP  <-- RSP points here when we're done
592	 * RAX  <-- espfix_waddr points here
593	 * --- bottom of ESPFIX stack ---
594	 */
595
596	pushq	%rdi				/* Stash user RDI */
597	SWAPGS					/* to kernel GS */
598	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */
599
600	movq	PER_CPU_VAR(espfix_waddr), %rdi
601	movq	%rax, (0*8)(%rdi)		/* user RAX */
602	movq	(1*8)(%rsp), %rax		/* user RIP */
603	movq	%rax, (1*8)(%rdi)
604	movq	(2*8)(%rsp), %rax		/* user CS */
605	movq	%rax, (2*8)(%rdi)
606	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
607	movq	%rax, (3*8)(%rdi)
608	movq	(5*8)(%rsp), %rax		/* user SS */
609	movq	%rax, (5*8)(%rdi)
610	movq	(4*8)(%rsp), %rax		/* user RSP */
611	movq	%rax, (4*8)(%rdi)
612	/* Now RAX == RSP. */
613
614	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */
615
616	/*
617	 * espfix_stack[31:16] == 0.  The page tables are set up such that
618	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
619	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
620	 * the same page.  Set up RSP so that RSP[31:16] contains the
621	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
622	 * still points to an RO alias of the ESPFIX stack.
623	 */
624	orq	PER_CPU_VAR(espfix_stack), %rax
625
626	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
627	SWAPGS					/* to user GS */
628	popq	%rdi				/* Restore user RDI */
629
630	movq	%rax, %rsp
631	UNWIND_HINT_IRET_REGS offset=8
632
633	/*
634	 * At this point, we cannot write to the stack any more, but we can
635	 * still read.
636	 */
637	popq	%rax				/* Restore user RAX */
638
639	/*
640	 * RSP now points to an ordinary IRET frame, except that the page
641	 * is read-only and RSP[31:16] are preloaded with the userspace
642	 * values.  We can now IRET back to userspace.
643	 */
644	jmp	native_irq_return_iret
645#endif
646SYM_CODE_END(common_interrupt_return)
647_ASM_NOKPROBE(common_interrupt_return)
648
649/*
650 * Reload gs selector with exception handling
651 * edi:  new selector
652 *
653 * Is in entry.text as it shouldn't be instrumented.
654 */
655SYM_FUNC_START(asm_load_gs_index)
656	FRAME_BEGIN
657	swapgs
658.Lgs_change:
659	movl	%edi, %gs
6602:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
661	swapgs
662	FRAME_END
663	ret
664SYM_FUNC_END(asm_load_gs_index)
665EXPORT_SYMBOL(asm_load_gs_index)
666
667	_ASM_EXTABLE(.Lgs_change, .Lbad_gs)
668	.section .fixup, "ax"
669	/* running with kernelgs */
670SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs)
671	swapgs					/* switch back to user gs */
672.macro ZAP_GS
673	/* This can't be a string because the preprocessor needs to see it. */
674	movl $__USER_DS, %eax
675	movl %eax, %gs
676.endm
677	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
678	xorl	%eax, %eax
679	movl	%eax, %gs
680	jmp	2b
681SYM_CODE_END(.Lbad_gs)
682	.previous
683
684/*
685 * rdi: New stack pointer points to the top word of the stack
686 * rsi: Function pointer
687 * rdx: Function argument (can be NULL if none)
688 */
689SYM_FUNC_START(asm_call_on_stack)
690	/*
691	 * Save the frame pointer unconditionally. This allows the ORC
692	 * unwinder to handle the stack switch.
693	 */
694	pushq		%rbp
695	mov		%rsp, %rbp
696
697	/*
698	 * The unwinder relies on the word at the top of the new stack
699	 * page linking back to the previous RSP.
700	 */
701	mov		%rsp, (%rdi)
702	mov		%rdi, %rsp
703	/* Move the argument to the right place */
704	mov		%rdx, %rdi
705
7061:
707	.pushsection .discard.instr_begin
708	.long 1b - .
709	.popsection
710
711	CALL_NOSPEC	rsi
712
7132:
714	.pushsection .discard.instr_end
715	.long 2b - .
716	.popsection
717
718	/* Restore the previous stack pointer from RBP. */
719	leaveq
720	ret
721SYM_FUNC_END(asm_call_on_stack)
722
723#ifdef CONFIG_XEN_PV
724/*
725 * A note on the "critical region" in our callback handler.
726 * We want to avoid stacking callback handlers due to events occurring
727 * during handling of the last event. To do this, we keep events disabled
728 * until we've done all processing. HOWEVER, we must enable events before
729 * popping the stack frame (can't be done atomically) and so it would still
730 * be possible to get enough handler activations to overflow the stack.
731 * Although unlikely, bugs of that kind are hard to track down, so we'd
732 * like to avoid the possibility.
733 * So, on entry to the handler we detect whether we interrupted an
734 * existing activation in its critical region -- if so, we pop the current
735 * activation and restart the handler using the previous one.
736 *
737 * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
738 */
739SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)
740
741/*
742 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
743 * see the correct pointer to the pt_regs
744 */
745	UNWIND_HINT_FUNC
746	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
747	UNWIND_HINT_REGS
748
749	call	xen_pv_evtchn_do_upcall
750
751	jmp	error_return
752SYM_CODE_END(exc_xen_hypervisor_callback)
753
754/*
755 * Hypervisor uses this for application faults while it executes.
756 * We get here for two reasons:
757 *  1. Fault while reloading DS, ES, FS or GS
758 *  2. Fault while executing IRET
759 * Category 1 we do not need to fix up as Xen has already reloaded all segment
760 * registers that could be reloaded and zeroed the others.
761 * Category 2 we fix up by killing the current process. We cannot use the
762 * normal Linux return path in this case because if we use the IRET hypercall
763 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
764 * We distinguish between categories by comparing each saved segment register
765 * with its current contents: any discrepancy means we in category 1.
766 */
767SYM_CODE_START(xen_failsafe_callback)
768	UNWIND_HINT_EMPTY
769	movl	%ds, %ecx
770	cmpw	%cx, 0x10(%rsp)
771	jne	1f
772	movl	%es, %ecx
773	cmpw	%cx, 0x18(%rsp)
774	jne	1f
775	movl	%fs, %ecx
776	cmpw	%cx, 0x20(%rsp)
777	jne	1f
778	movl	%gs, %ecx
779	cmpw	%cx, 0x28(%rsp)
780	jne	1f
781	/* All segments match their saved values => Category 2 (Bad IRET). */
782	movq	(%rsp), %rcx
783	movq	8(%rsp), %r11
784	addq	$0x30, %rsp
785	pushq	$0				/* RIP */
786	UNWIND_HINT_IRET_REGS offset=8
787	jmp	asm_exc_general_protection
7881:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
789	movq	(%rsp), %rcx
790	movq	8(%rsp), %r11
791	addq	$0x30, %rsp
792	UNWIND_HINT_IRET_REGS
793	pushq	$-1 /* orig_ax = -1 => not a system call */
794	PUSH_AND_CLEAR_REGS
795	ENCODE_FRAME_POINTER
796	jmp	error_return
797SYM_CODE_END(xen_failsafe_callback)
798#endif /* CONFIG_XEN_PV */
799
800/*
801 * Save all registers in pt_regs, and switch gs if needed.
802 * Use slow, but surefire "are we in kernel?" check.
803 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
804 */
805SYM_CODE_START_LOCAL(paranoid_entry)
806	UNWIND_HINT_FUNC
807	cld
808	PUSH_AND_CLEAR_REGS save_ret=1
809	ENCODE_FRAME_POINTER 8
810	movl	$1, %ebx
811	movl	$MSR_GS_BASE, %ecx
812	rdmsr
813	testl	%edx, %edx
814	js	1f				/* negative -> in kernel */
815	SWAPGS
816	xorl	%ebx, %ebx
817
8181:
819	/*
820	 * Always stash CR3 in %r14.  This value will be restored,
821	 * verbatim, at exit.  Needed if paranoid_entry interrupted
822	 * another entry that already switched to the user CR3 value
823	 * but has not yet returned to userspace.
824	 *
825	 * This is also why CS (stashed in the "iret frame" by the
826	 * hardware at entry) can not be used: this may be a return
827	 * to kernel code, but with a user CR3 value.
828	 */
829	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
830
831	/*
832	 * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
833	 * unconditional CR3 write, even in the PTI case.  So do an lfence
834	 * to prevent GS speculation, regardless of whether PTI is enabled.
835	 */
836	FENCE_SWAPGS_KERNEL_ENTRY
837
838	ret
839SYM_CODE_END(paranoid_entry)
840
841/*
842 * "Paranoid" exit path from exception stack.  This is invoked
843 * only on return from non-NMI IST interrupts that came
844 * from kernel space.
845 *
846 * We may be returning to very strange contexts (e.g. very early
847 * in syscall entry), so checking for preemption here would
848 * be complicated.  Fortunately, we there's no good reason
849 * to try to handle preemption here.
850 *
851 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
852 */
853SYM_CODE_START_LOCAL(paranoid_exit)
854	UNWIND_HINT_REGS
855	testl	%ebx, %ebx			/* swapgs needed? */
856	jnz	.Lparanoid_exit_no_swapgs
857	/* Always restore stashed CR3 value (see paranoid_entry) */
858	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
859	SWAPGS_UNSAFE_STACK
860	jmp	restore_regs_and_return_to_kernel
861.Lparanoid_exit_no_swapgs:
862	/* Always restore stashed CR3 value (see paranoid_entry) */
863	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
864	jmp restore_regs_and_return_to_kernel
865SYM_CODE_END(paranoid_exit)
866
867/*
868 * Save all registers in pt_regs, and switch GS if needed.
869 */
870SYM_CODE_START_LOCAL(error_entry)
871	UNWIND_HINT_FUNC
872	cld
873	PUSH_AND_CLEAR_REGS save_ret=1
874	ENCODE_FRAME_POINTER 8
875	testb	$3, CS+8(%rsp)
876	jz	.Lerror_kernelspace
877
878	/*
879	 * We entered from user mode or we're pretending to have entered
880	 * from user mode due to an IRET fault.
881	 */
882	SWAPGS
883	FENCE_SWAPGS_USER_ENTRY
884	/* We have user CR3.  Change to kernel CR3. */
885	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
886
887.Lerror_entry_from_usermode_after_swapgs:
888	/* Put us onto the real thread stack. */
889	popq	%r12				/* save return addr in %12 */
890	movq	%rsp, %rdi			/* arg0 = pt_regs pointer */
891	call	sync_regs
892	movq	%rax, %rsp			/* switch stack */
893	ENCODE_FRAME_POINTER
894	pushq	%r12
895	ret
896
897.Lerror_entry_done_lfence:
898	FENCE_SWAPGS_KERNEL_ENTRY
899.Lerror_entry_done:
900	ret
901
902	/*
903	 * There are two places in the kernel that can potentially fault with
904	 * usergs. Handle them here.  B stepping K8s sometimes report a
905	 * truncated RIP for IRET exceptions returning to compat mode. Check
906	 * for these here too.
907	 */
908.Lerror_kernelspace:
909	leaq	native_irq_return_iret(%rip), %rcx
910	cmpq	%rcx, RIP+8(%rsp)
911	je	.Lerror_bad_iret
912	movl	%ecx, %eax			/* zero extend */
913	cmpq	%rax, RIP+8(%rsp)
914	je	.Lbstep_iret
915	cmpq	$.Lgs_change, RIP+8(%rsp)
916	jne	.Lerror_entry_done_lfence
917
918	/*
919	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
920	 * gsbase and proceed.  We'll fix up the exception and land in
921	 * .Lgs_change's error handler with kernel gsbase.
922	 */
923	SWAPGS
924	FENCE_SWAPGS_USER_ENTRY
925	jmp .Lerror_entry_done
926
927.Lbstep_iret:
928	/* Fix truncated RIP */
929	movq	%rcx, RIP+8(%rsp)
930	/* fall through */
931
932.Lerror_bad_iret:
933	/*
934	 * We came from an IRET to user mode, so we have user
935	 * gsbase and CR3.  Switch to kernel gsbase and CR3:
936	 */
937	SWAPGS
938	FENCE_SWAPGS_USER_ENTRY
939	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
940
941	/*
942	 * Pretend that the exception came from user mode: set up pt_regs
943	 * as if we faulted immediately after IRET.
944	 */
945	mov	%rsp, %rdi
946	call	fixup_bad_iret
947	mov	%rax, %rsp
948	jmp	.Lerror_entry_from_usermode_after_swapgs
949SYM_CODE_END(error_entry)
950
951SYM_CODE_START_LOCAL(error_return)
952	UNWIND_HINT_REGS
953	DEBUG_ENTRY_ASSERT_IRQS_OFF
954	testb	$3, CS(%rsp)
955	jz	restore_regs_and_return_to_kernel
956	jmp	swapgs_restore_regs_and_return_to_usermode
957SYM_CODE_END(error_return)
958
959/*
960 * Runs on exception stack.  Xen PV does not go through this path at all,
961 * so we can use real assembly here.
962 *
963 * Registers:
964 *	%r14: Used to save/restore the CR3 of the interrupted context
965 *	      when PAGE_TABLE_ISOLATION is in use.  Do not clobber.
966 */
967SYM_CODE_START(asm_exc_nmi)
968	UNWIND_HINT_IRET_REGS
969
970	/*
971	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
972	 * the iretq it performs will take us out of NMI context.
973	 * This means that we can have nested NMIs where the next
974	 * NMI is using the top of the stack of the previous NMI. We
975	 * can't let it execute because the nested NMI will corrupt the
976	 * stack of the previous NMI. NMI handlers are not re-entrant
977	 * anyway.
978	 *
979	 * To handle this case we do the following:
980	 *  Check the a special location on the stack that contains
981	 *  a variable that is set when NMIs are executing.
982	 *  The interrupted task's stack is also checked to see if it
983	 *  is an NMI stack.
984	 *  If the variable is not set and the stack is not the NMI
985	 *  stack then:
986	 *    o Set the special variable on the stack
987	 *    o Copy the interrupt frame into an "outermost" location on the
988	 *      stack
989	 *    o Copy the interrupt frame into an "iret" location on the stack
990	 *    o Continue processing the NMI
991	 *  If the variable is set or the previous stack is the NMI stack:
992	 *    o Modify the "iret" location to jump to the repeat_nmi
993	 *    o return back to the first NMI
994	 *
995	 * Now on exit of the first NMI, we first clear the stack variable
996	 * The NMI stack will tell any nested NMIs at that point that it is
997	 * nested. Then we pop the stack normally with iret, and if there was
998	 * a nested NMI that updated the copy interrupt stack frame, a
999	 * jump will be made to the repeat_nmi code that will handle the second
1000	 * NMI.
1001	 *
1002	 * However, espfix prevents us from directly returning to userspace
1003	 * with a single IRET instruction.  Similarly, IRET to user mode
1004	 * can fault.  We therefore handle NMIs from user space like
1005	 * other IST entries.
1006	 */
1007
1008	ASM_CLAC
1009
1010	/* Use %rdx as our temp variable throughout */
1011	pushq	%rdx
1012
1013	testb	$3, CS-RIP+8(%rsp)
1014	jz	.Lnmi_from_kernel
1015
1016	/*
1017	 * NMI from user mode.  We need to run on the thread stack, but we
1018	 * can't go through the normal entry paths: NMIs are masked, and
1019	 * we don't want to enable interrupts, because then we'll end
1020	 * up in an awkward situation in which IRQs are on but NMIs
1021	 * are off.
1022	 *
1023	 * We also must not push anything to the stack before switching
1024	 * stacks lest we corrupt the "NMI executing" variable.
1025	 */
1026
1027	swapgs
1028	cld
1029	FENCE_SWAPGS_USER_ENTRY
1030	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1031	movq	%rsp, %rdx
1032	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1033	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1034	pushq	5*8(%rdx)	/* pt_regs->ss */
1035	pushq	4*8(%rdx)	/* pt_regs->rsp */
1036	pushq	3*8(%rdx)	/* pt_regs->flags */
1037	pushq	2*8(%rdx)	/* pt_regs->cs */
1038	pushq	1*8(%rdx)	/* pt_regs->rip */
1039	UNWIND_HINT_IRET_REGS
1040	pushq   $-1		/* pt_regs->orig_ax */
1041	PUSH_AND_CLEAR_REGS rdx=(%rdx)
1042	ENCODE_FRAME_POINTER
1043
1044	/*
1045	 * At this point we no longer need to worry about stack damage
1046	 * due to nesting -- we're on the normal thread stack and we're
1047	 * done with the NMI stack.
1048	 */
1049
1050	movq	%rsp, %rdi
1051	movq	$-1, %rsi
1052	call	exc_nmi
1053
1054	/*
1055	 * Return back to user mode.  We must *not* do the normal exit
1056	 * work, because we don't want to enable interrupts.
1057	 */
1058	jmp	swapgs_restore_regs_and_return_to_usermode
1059
1060.Lnmi_from_kernel:
1061	/*
1062	 * Here's what our stack frame will look like:
1063	 * +---------------------------------------------------------+
1064	 * | original SS                                             |
1065	 * | original Return RSP                                     |
1066	 * | original RFLAGS                                         |
1067	 * | original CS                                             |
1068	 * | original RIP                                            |
1069	 * +---------------------------------------------------------+
1070	 * | temp storage for rdx                                    |
1071	 * +---------------------------------------------------------+
1072	 * | "NMI executing" variable                                |
1073	 * +---------------------------------------------------------+
1074	 * | iret SS          } Copied from "outermost" frame        |
1075	 * | iret Return RSP  } on each loop iteration; overwritten  |
1076	 * | iret RFLAGS      } by a nested NMI to force another     |
1077	 * | iret CS          } iteration if needed.                 |
1078	 * | iret RIP         }                                      |
1079	 * +---------------------------------------------------------+
1080	 * | outermost SS          } initialized in first_nmi;       |
1081	 * | outermost Return RSP  } will not be changed before      |
1082	 * | outermost RFLAGS      } NMI processing is done.         |
1083	 * | outermost CS          } Copied to "iret" frame on each  |
1084	 * | outermost RIP         } iteration.                      |
1085	 * +---------------------------------------------------------+
1086	 * | pt_regs                                                 |
1087	 * +---------------------------------------------------------+
1088	 *
1089	 * The "original" frame is used by hardware.  Before re-enabling
1090	 * NMIs, we need to be done with it, and we need to leave enough
1091	 * space for the asm code here.
1092	 *
1093	 * We return by executing IRET while RSP points to the "iret" frame.
1094	 * That will either return for real or it will loop back into NMI
1095	 * processing.
1096	 *
1097	 * The "outermost" frame is copied to the "iret" frame on each
1098	 * iteration of the loop, so each iteration starts with the "iret"
1099	 * frame pointing to the final return target.
1100	 */
1101
1102	/*
1103	 * Determine whether we're a nested NMI.
1104	 *
1105	 * If we interrupted kernel code between repeat_nmi and
1106	 * end_repeat_nmi, then we are a nested NMI.  We must not
1107	 * modify the "iret" frame because it's being written by
1108	 * the outer NMI.  That's okay; the outer NMI handler is
1109	 * about to about to call exc_nmi() anyway, so we can just
1110	 * resume the outer NMI.
1111	 */
1112
1113	movq	$repeat_nmi, %rdx
1114	cmpq	8(%rsp), %rdx
1115	ja	1f
1116	movq	$end_repeat_nmi, %rdx
1117	cmpq	8(%rsp), %rdx
1118	ja	nested_nmi_out
11191:
1120
1121	/*
1122	 * Now check "NMI executing".  If it's set, then we're nested.
1123	 * This will not detect if we interrupted an outer NMI just
1124	 * before IRET.
1125	 */
1126	cmpl	$1, -8(%rsp)
1127	je	nested_nmi
1128
1129	/*
1130	 * Now test if the previous stack was an NMI stack.  This covers
1131	 * the case where we interrupt an outer NMI after it clears
1132	 * "NMI executing" but before IRET.  We need to be careful, though:
1133	 * there is one case in which RSP could point to the NMI stack
1134	 * despite there being no NMI active: naughty userspace controls
1135	 * RSP at the very beginning of the SYSCALL targets.  We can
1136	 * pull a fast one on naughty userspace, though: we program
1137	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
1138	 * if it controls the kernel's RSP.  We set DF before we clear
1139	 * "NMI executing".
1140	 */
1141	lea	6*8(%rsp), %rdx
1142	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
1143	cmpq	%rdx, 4*8(%rsp)
1144	/* If the stack pointer is above the NMI stack, this is a normal NMI */
1145	ja	first_nmi
1146
1147	subq	$EXCEPTION_STKSZ, %rdx
1148	cmpq	%rdx, 4*8(%rsp)
1149	/* If it is below the NMI stack, it is a normal NMI */
1150	jb	first_nmi
1151
1152	/* Ah, it is within the NMI stack. */
1153
1154	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
1155	jz	first_nmi	/* RSP was user controlled. */
1156
1157	/* This is a nested NMI. */
1158
1159nested_nmi:
1160	/*
1161	 * Modify the "iret" frame to point to repeat_nmi, forcing another
1162	 * iteration of NMI handling.
1163	 */
1164	subq	$8, %rsp
1165	leaq	-10*8(%rsp), %rdx
1166	pushq	$__KERNEL_DS
1167	pushq	%rdx
1168	pushfq
1169	pushq	$__KERNEL_CS
1170	pushq	$repeat_nmi
1171
1172	/* Put stack back */
1173	addq	$(6*8), %rsp
1174
1175nested_nmi_out:
1176	popq	%rdx
1177
1178	/* We are returning to kernel mode, so this cannot result in a fault. */
1179	iretq
1180
1181first_nmi:
1182	/* Restore rdx. */
1183	movq	(%rsp), %rdx
1184
1185	/* Make room for "NMI executing". */
1186	pushq	$0
1187
1188	/* Leave room for the "iret" frame */
1189	subq	$(5*8), %rsp
1190
1191	/* Copy the "original" frame to the "outermost" frame */
1192	.rept 5
1193	pushq	11*8(%rsp)
1194	.endr
1195	UNWIND_HINT_IRET_REGS
1196
1197	/* Everything up to here is safe from nested NMIs */
1198
1199#ifdef CONFIG_DEBUG_ENTRY
1200	/*
1201	 * For ease of testing, unmask NMIs right away.  Disabled by
1202	 * default because IRET is very expensive.
1203	 */
1204	pushq	$0		/* SS */
1205	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
1206	addq	$8, (%rsp)	/* Fix up RSP */
1207	pushfq			/* RFLAGS */
1208	pushq	$__KERNEL_CS	/* CS */
1209	pushq	$1f		/* RIP */
1210	iretq			/* continues at repeat_nmi below */
1211	UNWIND_HINT_IRET_REGS
12121:
1213#endif
1214
1215repeat_nmi:
1216	/*
1217	 * If there was a nested NMI, the first NMI's iret will return
1218	 * here. But NMIs are still enabled and we can take another
1219	 * nested NMI. The nested NMI checks the interrupted RIP to see
1220	 * if it is between repeat_nmi and end_repeat_nmi, and if so
1221	 * it will just return, as we are about to repeat an NMI anyway.
1222	 * This makes it safe to copy to the stack frame that a nested
1223	 * NMI will update.
1224	 *
1225	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
1226	 * we're repeating an NMI, gsbase has the same value that it had on
1227	 * the first iteration.  paranoid_entry will load the kernel
1228	 * gsbase if needed before we call exc_nmi().  "NMI executing"
1229	 * is zero.
1230	 */
1231	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1232
1233	/*
1234	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
1235	 * here must not modify the "iret" frame while we're writing to
1236	 * it or it will end up containing garbage.
1237	 */
1238	addq	$(10*8), %rsp
1239	.rept 5
1240	pushq	-6*8(%rsp)
1241	.endr
1242	subq	$(5*8), %rsp
1243end_repeat_nmi:
1244
1245	/*
1246	 * Everything below this point can be preempted by a nested NMI.
1247	 * If this happens, then the inner NMI will change the "iret"
1248	 * frame to point back to repeat_nmi.
1249	 */
1250	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1251
1252	/*
1253	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1254	 * as we should not be calling schedule in NMI context.
1255	 * Even with normal interrupts enabled. An NMI should not be
1256	 * setting NEED_RESCHED or anything that normal interrupts and
1257	 * exceptions might do.
1258	 */
1259	call	paranoid_entry
1260	UNWIND_HINT_REGS
1261
1262	movq	%rsp, %rdi
1263	movq	$-1, %rsi
1264	call	exc_nmi
1265
1266	/* Always restore stashed CR3 value (see paranoid_entry) */
1267	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1268
1269	testl	%ebx, %ebx			/* swapgs needed? */
1270	jnz	nmi_restore
1271nmi_swapgs:
1272	SWAPGS_UNSAFE_STACK
1273nmi_restore:
1274	POP_REGS
1275
1276	/*
1277	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
1278	 * at the "iret" frame.
1279	 */
1280	addq	$6*8, %rsp
1281
1282	/*
1283	 * Clear "NMI executing".  Set DF first so that we can easily
1284	 * distinguish the remaining code between here and IRET from
1285	 * the SYSCALL entry and exit paths.
1286	 *
1287	 * We arguably should just inspect RIP instead, but I (Andy) wrote
1288	 * this code when I had the misapprehension that Xen PV supported
1289	 * NMIs, and Xen PV would break that approach.
1290	 */
1291	std
1292	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1293
1294	/*
1295	 * iretq reads the "iret" frame and exits the NMI stack in a
1296	 * single instruction.  We are returning to kernel mode, so this
1297	 * cannot result in a fault.  Similarly, we don't need to worry
1298	 * about espfix64 on the way back to kernel mode.
1299	 */
1300	iretq
1301SYM_CODE_END(asm_exc_nmi)
1302
1303#ifndef CONFIG_IA32_EMULATION
1304/*
1305 * This handles SYSCALL from 32-bit code.  There is no way to program
1306 * MSRs to fully disable 32-bit SYSCALL.
1307 */
1308SYM_CODE_START(ignore_sysret)
1309	UNWIND_HINT_EMPTY
1310	mov	$-ENOSYS, %eax
1311	sysretl
1312SYM_CODE_END(ignore_sysret)
1313#endif
1314
1315.pushsection .text, "ax"
1316SYM_CODE_START(rewind_stack_do_exit)
1317	UNWIND_HINT_FUNC
1318	/* Prevent any naive code from trying to unwind to our caller. */
1319	xorl	%ebp, %ebp
1320
1321	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
1322	leaq	-PTREGS_SIZE(%rax), %rsp
1323	UNWIND_HINT_REGS
1324
1325	call	do_exit
1326SYM_CODE_END(rewind_stack_do_exit)
1327.popsection
1328