xref: /openbmc/linux/arch/x86/entry/entry_64.S (revision 37002bc6)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  linux/arch/x86_64/entry.S
4 *
5 *  Copyright (C) 1991, 1992  Linus Torvalds
6 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
7 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
8 *
9 * entry.S contains the system-call and fault low-level handling routines.
10 *
11 * Some of this is documented in Documentation/arch/x86/entry_64.rst
12 *
13 * A note on terminology:
14 * - iret frame:	Architecture defined interrupt frame from SS to RIP
15 *			at the top of the kernel process stack.
16 *
17 * Some macro usage:
18 * - SYM_FUNC_START/END:Define functions in the symbol table.
19 * - idtentry:		Define exception entry points.
20 */
21#include <linux/linkage.h>
22#include <asm/segment.h>
23#include <asm/cache.h>
24#include <asm/errno.h>
25#include <asm/asm-offsets.h>
26#include <asm/msr.h>
27#include <asm/unistd.h>
28#include <asm/thread_info.h>
29#include <asm/hw_irq.h>
30#include <asm/page_types.h>
31#include <asm/irqflags.h>
32#include <asm/paravirt.h>
33#include <asm/percpu.h>
34#include <asm/asm.h>
35#include <asm/smap.h>
36#include <asm/pgtable_types.h>
37#include <asm/export.h>
38#include <asm/frame.h>
39#include <asm/trapnr.h>
40#include <asm/nospec-branch.h>
41#include <asm/fsgsbase.h>
42#include <linux/err.h>
43
44#include "calling.h"
45
46.code64
47.section .entry.text, "ax"
48
49/*
50 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
51 *
52 * This is the only entry point used for 64-bit system calls.  The
53 * hardware interface is reasonably well designed and the register to
54 * argument mapping Linux uses fits well with the registers that are
55 * available when SYSCALL is used.
56 *
57 * SYSCALL instructions can be found inlined in libc implementations as
58 * well as some other programs and libraries.  There are also a handful
59 * of SYSCALL instructions in the vDSO used, for example, as a
60 * clock_gettimeofday fallback.
61 *
62 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
63 * then loads new ss, cs, and rip from previously programmed MSRs.
64 * rflags gets masked by a value from another MSR (so CLD and CLAC
65 * are not needed). SYSCALL does not save anything on the stack
66 * and does not change rsp.
67 *
68 * Registers on entry:
69 * rax  system call number
70 * rcx  return address
71 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
72 * rdi  arg0
73 * rsi  arg1
74 * rdx  arg2
75 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
76 * r8   arg4
77 * r9   arg5
78 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
79 *
80 * Only called from user space.
81 *
82 * When user can change pt_regs->foo always force IRET. That is because
83 * it deals with uncanonical addresses better. SYSRET has trouble
84 * with them due to bugs in both AMD and Intel CPUs.
85 */
86
87SYM_CODE_START(entry_SYSCALL_64)
88	UNWIND_HINT_ENTRY
89	ENDBR
90
91	swapgs
92	/* tss.sp2 is scratch space. */
93	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
94	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
95	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
96
97SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
98	ANNOTATE_NOENDBR
99
100	/* Construct struct pt_regs on stack */
101	pushq	$__USER_DS				/* pt_regs->ss */
102	pushq	PER_CPU_VAR(cpu_tss_rw + TSS_sp2)	/* pt_regs->sp */
103	pushq	%r11					/* pt_regs->flags */
104	pushq	$__USER_CS				/* pt_regs->cs */
105	pushq	%rcx					/* pt_regs->ip */
106SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
107	pushq	%rax					/* pt_regs->orig_ax */
108
109	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
110
111	/* IRQs are off. */
112	movq	%rsp, %rdi
113	/* Sign extend the lower 32bit as syscall numbers are treated as int */
114	movslq	%eax, %rsi
115
116	/* clobbers %rax, make sure it is after saving the syscall nr */
117	IBRS_ENTER
118	UNTRAIN_RET
119
120	call	do_syscall_64		/* returns with IRQs disabled */
121
122	/*
123	 * Try to use SYSRET instead of IRET if we're returning to
124	 * a completely clean 64-bit userspace context.  If we're not,
125	 * go to the slow exit path.
126	 * In the Xen PV case we must use iret anyway.
127	 */
128
129	ALTERNATIVE "", "jmp	swapgs_restore_regs_and_return_to_usermode", \
130		X86_FEATURE_XENPV
131
132	movq	RCX(%rsp), %rcx
133	movq	RIP(%rsp), %r11
134
135	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
136	jne	swapgs_restore_regs_and_return_to_usermode
137
138	/*
139	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
140	 * in kernel space.  This essentially lets the user take over
141	 * the kernel, since userspace controls RSP.
142	 *
143	 * If width of "canonical tail" ever becomes variable, this will need
144	 * to be updated to remain correct on both old and new CPUs.
145	 *
146	 * Change top bits to match most significant bit (47th or 56th bit
147	 * depending on paging mode) in the address.
148	 */
149#ifdef CONFIG_X86_5LEVEL
150	ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
151		"shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
152#else
153	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
154	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
155#endif
156
157	/* If this changed %rcx, it was not canonical */
158	cmpq	%rcx, %r11
159	jne	swapgs_restore_regs_and_return_to_usermode
160
161	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
162	jne	swapgs_restore_regs_and_return_to_usermode
163
164	movq	R11(%rsp), %r11
165	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
166	jne	swapgs_restore_regs_and_return_to_usermode
167
168	/*
169	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
170	 * restore RF properly. If the slowpath sets it for whatever reason, we
171	 * need to restore it correctly.
172	 *
173	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
174	 * trap from userspace immediately after SYSRET.  This would cause an
175	 * infinite loop whenever #DB happens with register state that satisfies
176	 * the opportunistic SYSRET conditions.  For example, single-stepping
177	 * this user code:
178	 *
179	 *           movq	$stuck_here, %rcx
180	 *           pushfq
181	 *           popq %r11
182	 *   stuck_here:
183	 *
184	 * would never get past 'stuck_here'.
185	 */
186	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
187	jnz	swapgs_restore_regs_and_return_to_usermode
188
189	/* nothing to check for RSP */
190
191	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
192	jne	swapgs_restore_regs_and_return_to_usermode
193
194	/*
195	 * We win! This label is here just for ease of understanding
196	 * perf profiles. Nothing jumps here.
197	 */
198syscall_return_via_sysret:
199	IBRS_EXIT
200	POP_REGS pop_rdi=0
201
202	/*
203	 * Now all regs are restored except RSP and RDI.
204	 * Save old stack pointer and switch to trampoline stack.
205	 */
206	movq	%rsp, %rdi
207	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
208	UNWIND_HINT_END_OF_STACK
209
210	pushq	RSP-RDI(%rdi)	/* RSP */
211	pushq	(%rdi)		/* RDI */
212
213	/*
214	 * We are on the trampoline stack.  All regs except RDI are live.
215	 * We can do future final exit work right here.
216	 */
217	STACKLEAK_ERASE_NOCLOBBER
218
219	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
220
221	popq	%rdi
222	popq	%rsp
223SYM_INNER_LABEL(entry_SYSRETQ_unsafe_stack, SYM_L_GLOBAL)
224	ANNOTATE_NOENDBR
225	swapgs
226	sysretq
227SYM_INNER_LABEL(entry_SYSRETQ_end, SYM_L_GLOBAL)
228	ANNOTATE_NOENDBR
229	int3
230SYM_CODE_END(entry_SYSCALL_64)
231
232/*
233 * %rdi: prev task
234 * %rsi: next task
235 */
236.pushsection .text, "ax"
237SYM_FUNC_START(__switch_to_asm)
238	/*
239	 * Save callee-saved registers
240	 * This must match the order in inactive_task_frame
241	 */
242	pushq	%rbp
243	pushq	%rbx
244	pushq	%r12
245	pushq	%r13
246	pushq	%r14
247	pushq	%r15
248
249	/* switch stack */
250	movq	%rsp, TASK_threadsp(%rdi)
251	movq	TASK_threadsp(%rsi), %rsp
252
253#ifdef CONFIG_STACKPROTECTOR
254	movq	TASK_stack_canary(%rsi), %rbx
255	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + FIXED_stack_canary
256#endif
257
258	/*
259	 * When switching from a shallower to a deeper call stack
260	 * the RSB may either underflow or use entries populated
261	 * with userspace addresses. On CPUs where those concerns
262	 * exist, overwrite the RSB with entries which capture
263	 * speculative execution to prevent attack.
264	 */
265	FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
266
267	/* restore callee-saved registers */
268	popq	%r15
269	popq	%r14
270	popq	%r13
271	popq	%r12
272	popq	%rbx
273	popq	%rbp
274
275	jmp	__switch_to
276SYM_FUNC_END(__switch_to_asm)
277.popsection
278
279/*
280 * A newly forked process directly context switches into this address.
281 *
282 * rax: prev task we switched from
283 * rbx: kernel thread func (NULL for user thread)
284 * r12: kernel thread arg
285 */
286.pushsection .text, "ax"
287SYM_CODE_START(ret_from_fork_asm)
288	UNWIND_HINT_REGS
289	ANNOTATE_NOENDBR // copy_thread
290	CALL_DEPTH_ACCOUNT
291
292	movq	%rax, %rdi		/* prev */
293	movq	%rsp, %rsi		/* regs */
294	movq	%rbx, %rdx		/* fn */
295	movq	%r12, %rcx		/* fn_arg */
296	call	ret_from_fork
297
298	jmp	swapgs_restore_regs_and_return_to_usermode
299SYM_CODE_END(ret_from_fork_asm)
300.popsection
301
302.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
303#ifdef CONFIG_DEBUG_ENTRY
304	pushq %rax
305	SAVE_FLAGS
306	testl $X86_EFLAGS_IF, %eax
307	jz .Lokay_\@
308	ud2
309.Lokay_\@:
310	popq %rax
311#endif
312.endm
313
314SYM_CODE_START(xen_error_entry)
315	ANNOTATE_NOENDBR
316	UNWIND_HINT_FUNC
317	PUSH_AND_CLEAR_REGS save_ret=1
318	ENCODE_FRAME_POINTER 8
319	UNTRAIN_RET_FROM_CALL
320	RET
321SYM_CODE_END(xen_error_entry)
322
323/**
324 * idtentry_body - Macro to emit code calling the C function
325 * @cfunc:		C function to be called
326 * @has_error_code:	Hardware pushed error code on stack
327 */
328.macro idtentry_body cfunc has_error_code:req
329
330	/*
331	 * Call error_entry() and switch to the task stack if from userspace.
332	 *
333	 * When in XENPV, it is already in the task stack, and it can't fault
334	 * for native_iret() nor native_load_gs_index() since XENPV uses its
335	 * own pvops for IRET and load_gs_index().  And it doesn't need to
336	 * switch the CR3.  So it can skip invoking error_entry().
337	 */
338	ALTERNATIVE "call error_entry; movq %rax, %rsp", \
339		    "call xen_error_entry", X86_FEATURE_XENPV
340
341	ENCODE_FRAME_POINTER
342	UNWIND_HINT_REGS
343
344	movq	%rsp, %rdi			/* pt_regs pointer into 1st argument*/
345
346	.if \has_error_code == 1
347		movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
348		movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
349	.endif
350
351	call	\cfunc
352
353	/* For some configurations \cfunc ends up being a noreturn. */
354	REACHABLE
355
356	jmp	error_return
357.endm
358
359/**
360 * idtentry - Macro to generate entry stubs for simple IDT entries
361 * @vector:		Vector number
362 * @asmsym:		ASM symbol for the entry point
363 * @cfunc:		C function to be called
364 * @has_error_code:	Hardware pushed error code on stack
365 *
366 * The macro emits code to set up the kernel context for straight forward
367 * and simple IDT entries. No IST stack, no paranoid entry checks.
368 */
369.macro idtentry vector asmsym cfunc has_error_code:req
370SYM_CODE_START(\asmsym)
371
372	.if \vector == X86_TRAP_BP
373		/* #BP advances %rip to the next instruction */
374		UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 signal=0
375	.else
376		UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8
377	.endif
378
379	ENDBR
380	ASM_CLAC
381	cld
382
383	.if \has_error_code == 0
384		pushq	$-1			/* ORIG_RAX: no syscall to restart */
385	.endif
386
387	.if \vector == X86_TRAP_BP
388		/*
389		 * If coming from kernel space, create a 6-word gap to allow the
390		 * int3 handler to emulate a call instruction.
391		 */
392		testb	$3, CS-ORIG_RAX(%rsp)
393		jnz	.Lfrom_usermode_no_gap_\@
394		.rept	6
395		pushq	5*8(%rsp)
396		.endr
397		UNWIND_HINT_IRET_REGS offset=8
398.Lfrom_usermode_no_gap_\@:
399	.endif
400
401	idtentry_body \cfunc \has_error_code
402
403_ASM_NOKPROBE(\asmsym)
404SYM_CODE_END(\asmsym)
405.endm
406
407/*
408 * Interrupt entry/exit.
409 *
410 + The interrupt stubs push (vector) onto the stack, which is the error_code
411 * position of idtentry exceptions, and jump to one of the two idtentry points
412 * (common/spurious).
413 *
414 * common_interrupt is a hotpath, align it to a cache line
415 */
416.macro idtentry_irq vector cfunc
417	.p2align CONFIG_X86_L1_CACHE_SHIFT
418	idtentry \vector asm_\cfunc \cfunc has_error_code=1
419.endm
420
421/*
422 * System vectors which invoke their handlers directly and are not
423 * going through the regular common device interrupt handling code.
424 */
425.macro idtentry_sysvec vector cfunc
426	idtentry \vector asm_\cfunc \cfunc has_error_code=0
427.endm
428
429/**
430 * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB
431 * @vector:		Vector number
432 * @asmsym:		ASM symbol for the entry point
433 * @cfunc:		C function to be called
434 *
435 * The macro emits code to set up the kernel context for #MC and #DB
436 *
437 * If the entry comes from user space it uses the normal entry path
438 * including the return to user space work and preemption checks on
439 * exit.
440 *
441 * If hits in kernel mode then it needs to go through the paranoid
442 * entry as the exception can hit any random state. No preemption
443 * check on exit to keep the paranoid path simple.
444 */
445.macro idtentry_mce_db vector asmsym cfunc
446SYM_CODE_START(\asmsym)
447	UNWIND_HINT_IRET_ENTRY
448	ENDBR
449	ASM_CLAC
450	cld
451
452	pushq	$-1			/* ORIG_RAX: no syscall to restart */
453
454	/*
455	 * If the entry is from userspace, switch stacks and treat it as
456	 * a normal entry.
457	 */
458	testb	$3, CS-ORIG_RAX(%rsp)
459	jnz	.Lfrom_usermode_switch_stack_\@
460
461	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
462	call	paranoid_entry
463
464	UNWIND_HINT_REGS
465
466	movq	%rsp, %rdi		/* pt_regs pointer */
467
468	call	\cfunc
469
470	jmp	paranoid_exit
471
472	/* Switch to the regular task stack and use the noist entry point */
473.Lfrom_usermode_switch_stack_\@:
474	idtentry_body noist_\cfunc, has_error_code=0
475
476_ASM_NOKPROBE(\asmsym)
477SYM_CODE_END(\asmsym)
478.endm
479
480#ifdef CONFIG_AMD_MEM_ENCRYPT
481/**
482 * idtentry_vc - Macro to generate entry stub for #VC
483 * @vector:		Vector number
484 * @asmsym:		ASM symbol for the entry point
485 * @cfunc:		C function to be called
486 *
487 * The macro emits code to set up the kernel context for #VC. The #VC handler
488 * runs on an IST stack and needs to be able to cause nested #VC exceptions.
489 *
490 * To make this work the #VC entry code tries its best to pretend it doesn't use
491 * an IST stack by switching to the task stack if coming from user-space (which
492 * includes early SYSCALL entry path) or back to the stack in the IRET frame if
493 * entered from kernel-mode.
494 *
495 * If entered from kernel-mode the return stack is validated first, and if it is
496 * not safe to use (e.g. because it points to the entry stack) the #VC handler
497 * will switch to a fall-back stack (VC2) and call a special handler function.
498 *
499 * The macro is only used for one vector, but it is planned to be extended in
500 * the future for the #HV exception.
501 */
502.macro idtentry_vc vector asmsym cfunc
503SYM_CODE_START(\asmsym)
504	UNWIND_HINT_IRET_ENTRY
505	ENDBR
506	ASM_CLAC
507	cld
508
509	/*
510	 * If the entry is from userspace, switch stacks and treat it as
511	 * a normal entry.
512	 */
513	testb	$3, CS-ORIG_RAX(%rsp)
514	jnz	.Lfrom_usermode_switch_stack_\@
515
516	/*
517	 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
518	 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
519	 */
520	call	paranoid_entry
521
522	UNWIND_HINT_REGS
523
524	/*
525	 * Switch off the IST stack to make it free for nested exceptions. The
526	 * vc_switch_off_ist() function will switch back to the interrupted
527	 * stack if it is safe to do so. If not it switches to the VC fall-back
528	 * stack.
529	 */
530	movq	%rsp, %rdi		/* pt_regs pointer */
531	call	vc_switch_off_ist
532	movq	%rax, %rsp		/* Switch to new stack */
533
534	ENCODE_FRAME_POINTER
535	UNWIND_HINT_REGS
536
537	/* Update pt_regs */
538	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
539	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
540
541	movq	%rsp, %rdi		/* pt_regs pointer */
542
543	call	kernel_\cfunc
544
545	/*
546	 * No need to switch back to the IST stack. The current stack is either
547	 * identical to the stack in the IRET frame or the VC fall-back stack,
548	 * so it is definitely mapped even with PTI enabled.
549	 */
550	jmp	paranoid_exit
551
552	/* Switch to the regular task stack */
553.Lfrom_usermode_switch_stack_\@:
554	idtentry_body user_\cfunc, has_error_code=1
555
556_ASM_NOKPROBE(\asmsym)
557SYM_CODE_END(\asmsym)
558.endm
559#endif
560
561/*
562 * Double fault entry. Straight paranoid. No checks from which context
563 * this comes because for the espfix induced #DF this would do the wrong
564 * thing.
565 */
566.macro idtentry_df vector asmsym cfunc
567SYM_CODE_START(\asmsym)
568	UNWIND_HINT_IRET_ENTRY offset=8
569	ENDBR
570	ASM_CLAC
571	cld
572
573	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
574	call	paranoid_entry
575	UNWIND_HINT_REGS
576
577	movq	%rsp, %rdi		/* pt_regs pointer into first argument */
578	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
579	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
580	call	\cfunc
581
582	/* For some configurations \cfunc ends up being a noreturn. */
583	REACHABLE
584
585	jmp	paranoid_exit
586
587_ASM_NOKPROBE(\asmsym)
588SYM_CODE_END(\asmsym)
589.endm
590
591/*
592 * Include the defines which emit the idt entries which are shared
593 * shared between 32 and 64 bit and emit the __irqentry_text_* markers
594 * so the stacktrace boundary checks work.
595 */
596	__ALIGN
597	.globl __irqentry_text_start
598__irqentry_text_start:
599
600#include <asm/idtentry.h>
601
602	__ALIGN
603	.globl __irqentry_text_end
604__irqentry_text_end:
605	ANNOTATE_NOENDBR
606
607SYM_CODE_START_LOCAL(common_interrupt_return)
608SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
609	IBRS_EXIT
610#ifdef CONFIG_DEBUG_ENTRY
611	/* Assert that pt_regs indicates user mode. */
612	testb	$3, CS(%rsp)
613	jnz	1f
614	ud2
6151:
616#endif
617#ifdef CONFIG_XEN_PV
618	ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
619#endif
620
621	POP_REGS pop_rdi=0
622
623	/*
624	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
625	 * Save old stack pointer and switch to trampoline stack.
626	 */
627	movq	%rsp, %rdi
628	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
629	UNWIND_HINT_END_OF_STACK
630
631	/* Copy the IRET frame to the trampoline stack. */
632	pushq	6*8(%rdi)	/* SS */
633	pushq	5*8(%rdi)	/* RSP */
634	pushq	4*8(%rdi)	/* EFLAGS */
635	pushq	3*8(%rdi)	/* CS */
636	pushq	2*8(%rdi)	/* RIP */
637
638	/* Push user RDI on the trampoline stack. */
639	pushq	(%rdi)
640
641	/*
642	 * We are on the trampoline stack.  All regs except RDI are live.
643	 * We can do future final exit work right here.
644	 */
645	STACKLEAK_ERASE_NOCLOBBER
646
647	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
648
649	/* Restore RDI. */
650	popq	%rdi
651	swapgs
652	jmp	.Lnative_iret
653
654
655SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
656#ifdef CONFIG_DEBUG_ENTRY
657	/* Assert that pt_regs indicates kernel mode. */
658	testb	$3, CS(%rsp)
659	jz	1f
660	ud2
6611:
662#endif
663	POP_REGS
664	addq	$8, %rsp	/* skip regs->orig_ax */
665	/*
666	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
667	 * when returning from IPI handler.
668	 */
669#ifdef CONFIG_XEN_PV
670SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)
671	ANNOTATE_NOENDBR
672	.byte 0xe9
673	.long .Lnative_iret - (. + 4)
674#endif
675
676.Lnative_iret:
677	UNWIND_HINT_IRET_REGS
678	/*
679	 * Are we returning to a stack segment from the LDT?  Note: in
680	 * 64-bit mode SS:RSP on the exception stack is always valid.
681	 */
682#ifdef CONFIG_X86_ESPFIX64
683	testb	$4, (SS-RIP)(%rsp)
684	jnz	native_irq_return_ldt
685#endif
686
687SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
688	ANNOTATE_NOENDBR // exc_double_fault
689	/*
690	 * This may fault.  Non-paranoid faults on return to userspace are
691	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
692	 * Double-faults due to espfix64 are handled in exc_double_fault.
693	 * Other faults here are fatal.
694	 */
695	iretq
696
697#ifdef CONFIG_X86_ESPFIX64
698native_irq_return_ldt:
699	/*
700	 * We are running with user GSBASE.  All GPRs contain their user
701	 * values.  We have a percpu ESPFIX stack that is eight slots
702	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
703	 * of the ESPFIX stack.
704	 *
705	 * We clobber RAX and RDI in this code.  We stash RDI on the
706	 * normal stack and RAX on the ESPFIX stack.
707	 *
708	 * The ESPFIX stack layout we set up looks like this:
709	 *
710	 * --- top of ESPFIX stack ---
711	 * SS
712	 * RSP
713	 * RFLAGS
714	 * CS
715	 * RIP  <-- RSP points here when we're done
716	 * RAX  <-- espfix_waddr points here
717	 * --- bottom of ESPFIX stack ---
718	 */
719
720	pushq	%rdi				/* Stash user RDI */
721	swapgs					/* to kernel GS */
722	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */
723
724	movq	PER_CPU_VAR(espfix_waddr), %rdi
725	movq	%rax, (0*8)(%rdi)		/* user RAX */
726	movq	(1*8)(%rsp), %rax		/* user RIP */
727	movq	%rax, (1*8)(%rdi)
728	movq	(2*8)(%rsp), %rax		/* user CS */
729	movq	%rax, (2*8)(%rdi)
730	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
731	movq	%rax, (3*8)(%rdi)
732	movq	(5*8)(%rsp), %rax		/* user SS */
733	movq	%rax, (5*8)(%rdi)
734	movq	(4*8)(%rsp), %rax		/* user RSP */
735	movq	%rax, (4*8)(%rdi)
736	/* Now RAX == RSP. */
737
738	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */
739
740	/*
741	 * espfix_stack[31:16] == 0.  The page tables are set up such that
742	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
743	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
744	 * the same page.  Set up RSP so that RSP[31:16] contains the
745	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
746	 * still points to an RO alias of the ESPFIX stack.
747	 */
748	orq	PER_CPU_VAR(espfix_stack), %rax
749
750	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
751	swapgs					/* to user GS */
752	popq	%rdi				/* Restore user RDI */
753
754	movq	%rax, %rsp
755	UNWIND_HINT_IRET_REGS offset=8
756
757	/*
758	 * At this point, we cannot write to the stack any more, but we can
759	 * still read.
760	 */
761	popq	%rax				/* Restore user RAX */
762
763	/*
764	 * RSP now points to an ordinary IRET frame, except that the page
765	 * is read-only and RSP[31:16] are preloaded with the userspace
766	 * values.  We can now IRET back to userspace.
767	 */
768	jmp	native_irq_return_iret
769#endif
770SYM_CODE_END(common_interrupt_return)
771_ASM_NOKPROBE(common_interrupt_return)
772
773/*
774 * Reload gs selector with exception handling
775 *  di:  new selector
776 *
777 * Is in entry.text as it shouldn't be instrumented.
778 */
779SYM_FUNC_START(asm_load_gs_index)
780	FRAME_BEGIN
781	swapgs
782.Lgs_change:
783	ANNOTATE_NOENDBR // error_entry
784	movl	%edi, %gs
7852:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
786	swapgs
787	FRAME_END
788	RET
789
790	/* running with kernelgs */
791.Lbad_gs:
792	swapgs					/* switch back to user gs */
793.macro ZAP_GS
794	/* This can't be a string because the preprocessor needs to see it. */
795	movl $__USER_DS, %eax
796	movl %eax, %gs
797.endm
798	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
799	xorl	%eax, %eax
800	movl	%eax, %gs
801	jmp	2b
802
803	_ASM_EXTABLE(.Lgs_change, .Lbad_gs)
804
805SYM_FUNC_END(asm_load_gs_index)
806EXPORT_SYMBOL(asm_load_gs_index)
807
808#ifdef CONFIG_XEN_PV
809/*
810 * A note on the "critical region" in our callback handler.
811 * We want to avoid stacking callback handlers due to events occurring
812 * during handling of the last event. To do this, we keep events disabled
813 * until we've done all processing. HOWEVER, we must enable events before
814 * popping the stack frame (can't be done atomically) and so it would still
815 * be possible to get enough handler activations to overflow the stack.
816 * Although unlikely, bugs of that kind are hard to track down, so we'd
817 * like to avoid the possibility.
818 * So, on entry to the handler we detect whether we interrupted an
819 * existing activation in its critical region -- if so, we pop the current
820 * activation and restart the handler using the previous one.
821 *
822 * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
823 */
824	__FUNC_ALIGN
825SYM_CODE_START_LOCAL_NOALIGN(exc_xen_hypervisor_callback)
826
827/*
828 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
829 * see the correct pointer to the pt_regs
830 */
831	UNWIND_HINT_FUNC
832	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
833	UNWIND_HINT_REGS
834
835	call	xen_pv_evtchn_do_upcall
836
837	jmp	error_return
838SYM_CODE_END(exc_xen_hypervisor_callback)
839
840/*
841 * Hypervisor uses this for application faults while it executes.
842 * We get here for two reasons:
843 *  1. Fault while reloading DS, ES, FS or GS
844 *  2. Fault while executing IRET
845 * Category 1 we do not need to fix up as Xen has already reloaded all segment
846 * registers that could be reloaded and zeroed the others.
847 * Category 2 we fix up by killing the current process. We cannot use the
848 * normal Linux return path in this case because if we use the IRET hypercall
849 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
850 * We distinguish between categories by comparing each saved segment register
851 * with its current contents: any discrepancy means we in category 1.
852 */
853	__FUNC_ALIGN
854SYM_CODE_START_NOALIGN(xen_failsafe_callback)
855	UNWIND_HINT_UNDEFINED
856	ENDBR
857	movl	%ds, %ecx
858	cmpw	%cx, 0x10(%rsp)
859	jne	1f
860	movl	%es, %ecx
861	cmpw	%cx, 0x18(%rsp)
862	jne	1f
863	movl	%fs, %ecx
864	cmpw	%cx, 0x20(%rsp)
865	jne	1f
866	movl	%gs, %ecx
867	cmpw	%cx, 0x28(%rsp)
868	jne	1f
869	/* All segments match their saved values => Category 2 (Bad IRET). */
870	movq	(%rsp), %rcx
871	movq	8(%rsp), %r11
872	addq	$0x30, %rsp
873	pushq	$0				/* RIP */
874	UNWIND_HINT_IRET_REGS offset=8
875	jmp	asm_exc_general_protection
8761:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
877	movq	(%rsp), %rcx
878	movq	8(%rsp), %r11
879	addq	$0x30, %rsp
880	UNWIND_HINT_IRET_REGS
881	pushq	$-1 /* orig_ax = -1 => not a system call */
882	PUSH_AND_CLEAR_REGS
883	ENCODE_FRAME_POINTER
884	jmp	error_return
885SYM_CODE_END(xen_failsafe_callback)
886#endif /* CONFIG_XEN_PV */
887
888/*
889 * Save all registers in pt_regs. Return GSBASE related information
890 * in EBX depending on the availability of the FSGSBASE instructions:
891 *
892 * FSGSBASE	R/EBX
893 *     N        0 -> SWAPGS on exit
894 *              1 -> no SWAPGS on exit
895 *
896 *     Y        GSBASE value at entry, must be restored in paranoid_exit
897 *
898 * R14 - old CR3
899 * R15 - old SPEC_CTRL
900 */
901SYM_CODE_START(paranoid_entry)
902	ANNOTATE_NOENDBR
903	UNWIND_HINT_FUNC
904	PUSH_AND_CLEAR_REGS save_ret=1
905	ENCODE_FRAME_POINTER 8
906
907	/*
908	 * Always stash CR3 in %r14.  This value will be restored,
909	 * verbatim, at exit.  Needed if paranoid_entry interrupted
910	 * another entry that already switched to the user CR3 value
911	 * but has not yet returned to userspace.
912	 *
913	 * This is also why CS (stashed in the "iret frame" by the
914	 * hardware at entry) can not be used: this may be a return
915	 * to kernel code, but with a user CR3 value.
916	 *
917	 * Switching CR3 does not depend on kernel GSBASE so it can
918	 * be done before switching to the kernel GSBASE. This is
919	 * required for FSGSBASE because the kernel GSBASE has to
920	 * be retrieved from a kernel internal table.
921	 */
922	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
923
924	/*
925	 * Handling GSBASE depends on the availability of FSGSBASE.
926	 *
927	 * Without FSGSBASE the kernel enforces that negative GSBASE
928	 * values indicate kernel GSBASE. With FSGSBASE no assumptions
929	 * can be made about the GSBASE value when entering from user
930	 * space.
931	 */
932	ALTERNATIVE "jmp .Lparanoid_entry_checkgs", "", X86_FEATURE_FSGSBASE
933
934	/*
935	 * Read the current GSBASE and store it in %rbx unconditionally,
936	 * retrieve and set the current CPUs kernel GSBASE. The stored value
937	 * has to be restored in paranoid_exit unconditionally.
938	 *
939	 * The unconditional write to GS base below ensures that no subsequent
940	 * loads based on a mispredicted GS base can happen, therefore no LFENCE
941	 * is needed here.
942	 */
943	SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
944	jmp .Lparanoid_gsbase_done
945
946.Lparanoid_entry_checkgs:
947	/* EBX = 1 -> kernel GSBASE active, no restore required */
948	movl	$1, %ebx
949
950	/*
951	 * The kernel-enforced convention is a negative GSBASE indicates
952	 * a kernel value. No SWAPGS needed on entry and exit.
953	 */
954	movl	$MSR_GS_BASE, %ecx
955	rdmsr
956	testl	%edx, %edx
957	js	.Lparanoid_kernel_gsbase
958
959	/* EBX = 0 -> SWAPGS required on exit */
960	xorl	%ebx, %ebx
961	swapgs
962.Lparanoid_kernel_gsbase:
963	FENCE_SWAPGS_KERNEL_ENTRY
964.Lparanoid_gsbase_done:
965
966	/*
967	 * Once we have CR3 and %GS setup save and set SPEC_CTRL. Just like
968	 * CR3 above, keep the old value in a callee saved register.
969	 */
970	IBRS_ENTER save_reg=%r15
971	UNTRAIN_RET_FROM_CALL
972
973	RET
974SYM_CODE_END(paranoid_entry)
975
976/*
977 * "Paranoid" exit path from exception stack.  This is invoked
978 * only on return from non-NMI IST interrupts that came
979 * from kernel space.
980 *
981 * We may be returning to very strange contexts (e.g. very early
982 * in syscall entry), so checking for preemption here would
983 * be complicated.  Fortunately, there's no good reason to try
984 * to handle preemption here.
985 *
986 * R/EBX contains the GSBASE related information depending on the
987 * availability of the FSGSBASE instructions:
988 *
989 * FSGSBASE	R/EBX
990 *     N        0 -> SWAPGS on exit
991 *              1 -> no SWAPGS on exit
992 *
993 *     Y        User space GSBASE, must be restored unconditionally
994 *
995 * R14 - old CR3
996 * R15 - old SPEC_CTRL
997 */
998SYM_CODE_START_LOCAL(paranoid_exit)
999	UNWIND_HINT_REGS
1000
1001	/*
1002	 * Must restore IBRS state before both CR3 and %GS since we need access
1003	 * to the per-CPU x86_spec_ctrl_shadow variable.
1004	 */
1005	IBRS_EXIT save_reg=%r15
1006
1007	/*
1008	 * The order of operations is important. RESTORE_CR3 requires
1009	 * kernel GSBASE.
1010	 *
1011	 * NB to anyone to try to optimize this code: this code does
1012	 * not execute at all for exceptions from user mode. Those
1013	 * exceptions go through error_return instead.
1014	 */
1015	RESTORE_CR3	scratch_reg=%rax save_reg=%r14
1016
1017	/* Handle the three GSBASE cases */
1018	ALTERNATIVE "jmp .Lparanoid_exit_checkgs", "", X86_FEATURE_FSGSBASE
1019
1020	/* With FSGSBASE enabled, unconditionally restore GSBASE */
1021	wrgsbase	%rbx
1022	jmp		restore_regs_and_return_to_kernel
1023
1024.Lparanoid_exit_checkgs:
1025	/* On non-FSGSBASE systems, conditionally do SWAPGS */
1026	testl		%ebx, %ebx
1027	jnz		restore_regs_and_return_to_kernel
1028
1029	/* We are returning to a context with user GSBASE */
1030	swapgs
1031	jmp		restore_regs_and_return_to_kernel
1032SYM_CODE_END(paranoid_exit)
1033
1034/*
1035 * Switch GS and CR3 if needed.
1036 */
1037SYM_CODE_START(error_entry)
1038	ANNOTATE_NOENDBR
1039	UNWIND_HINT_FUNC
1040
1041	PUSH_AND_CLEAR_REGS save_ret=1
1042	ENCODE_FRAME_POINTER 8
1043
1044	testb	$3, CS+8(%rsp)
1045	jz	.Lerror_kernelspace
1046
1047	/*
1048	 * We entered from user mode or we're pretending to have entered
1049	 * from user mode due to an IRET fault.
1050	 */
1051	swapgs
1052	FENCE_SWAPGS_USER_ENTRY
1053	/* We have user CR3.  Change to kernel CR3. */
1054	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1055	IBRS_ENTER
1056	UNTRAIN_RET_FROM_CALL
1057
1058	leaq	8(%rsp), %rdi			/* arg0 = pt_regs pointer */
1059	/* Put us onto the real thread stack. */
1060	jmp	sync_regs
1061
1062	/*
1063	 * There are two places in the kernel that can potentially fault with
1064	 * usergs. Handle them here.  B stepping K8s sometimes report a
1065	 * truncated RIP for IRET exceptions returning to compat mode. Check
1066	 * for these here too.
1067	 */
1068.Lerror_kernelspace:
1069	leaq	native_irq_return_iret(%rip), %rcx
1070	cmpq	%rcx, RIP+8(%rsp)
1071	je	.Lerror_bad_iret
1072	movl	%ecx, %eax			/* zero extend */
1073	cmpq	%rax, RIP+8(%rsp)
1074	je	.Lbstep_iret
1075	cmpq	$.Lgs_change, RIP+8(%rsp)
1076	jne	.Lerror_entry_done_lfence
1077
1078	/*
1079	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
1080	 * gsbase and proceed.  We'll fix up the exception and land in
1081	 * .Lgs_change's error handler with kernel gsbase.
1082	 */
1083	swapgs
1084
1085	/*
1086	 * Issue an LFENCE to prevent GS speculation, regardless of whether it is a
1087	 * kernel or user gsbase.
1088	 */
1089.Lerror_entry_done_lfence:
1090	FENCE_SWAPGS_KERNEL_ENTRY
1091	CALL_DEPTH_ACCOUNT
1092	leaq	8(%rsp), %rax			/* return pt_regs pointer */
1093	VALIDATE_UNRET_END
1094	RET
1095
1096.Lbstep_iret:
1097	/* Fix truncated RIP */
1098	movq	%rcx, RIP+8(%rsp)
1099	/* fall through */
1100
1101.Lerror_bad_iret:
1102	/*
1103	 * We came from an IRET to user mode, so we have user
1104	 * gsbase and CR3.  Switch to kernel gsbase and CR3:
1105	 */
1106	swapgs
1107	FENCE_SWAPGS_USER_ENTRY
1108	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1109	IBRS_ENTER
1110	UNTRAIN_RET_FROM_CALL
1111
1112	/*
1113	 * Pretend that the exception came from user mode: set up pt_regs
1114	 * as if we faulted immediately after IRET.
1115	 */
1116	leaq	8(%rsp), %rdi			/* arg0 = pt_regs pointer */
1117	call	fixup_bad_iret
1118	mov	%rax, %rdi
1119	jmp	sync_regs
1120SYM_CODE_END(error_entry)
1121
1122SYM_CODE_START_LOCAL(error_return)
1123	UNWIND_HINT_REGS
1124	DEBUG_ENTRY_ASSERT_IRQS_OFF
1125	testb	$3, CS(%rsp)
1126	jz	restore_regs_and_return_to_kernel
1127	jmp	swapgs_restore_regs_and_return_to_usermode
1128SYM_CODE_END(error_return)
1129
1130/*
1131 * Runs on exception stack.  Xen PV does not go through this path at all,
1132 * so we can use real assembly here.
1133 *
1134 * Registers:
1135 *	%r14: Used to save/restore the CR3 of the interrupted context
1136 *	      when PAGE_TABLE_ISOLATION is in use.  Do not clobber.
1137 */
1138SYM_CODE_START(asm_exc_nmi)
1139	UNWIND_HINT_IRET_ENTRY
1140	ENDBR
1141
1142	/*
1143	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
1144	 * the iretq it performs will take us out of NMI context.
1145	 * This means that we can have nested NMIs where the next
1146	 * NMI is using the top of the stack of the previous NMI. We
1147	 * can't let it execute because the nested NMI will corrupt the
1148	 * stack of the previous NMI. NMI handlers are not re-entrant
1149	 * anyway.
1150	 *
1151	 * To handle this case we do the following:
1152	 *  Check the a special location on the stack that contains
1153	 *  a variable that is set when NMIs are executing.
1154	 *  The interrupted task's stack is also checked to see if it
1155	 *  is an NMI stack.
1156	 *  If the variable is not set and the stack is not the NMI
1157	 *  stack then:
1158	 *    o Set the special variable on the stack
1159	 *    o Copy the interrupt frame into an "outermost" location on the
1160	 *      stack
1161	 *    o Copy the interrupt frame into an "iret" location on the stack
1162	 *    o Continue processing the NMI
1163	 *  If the variable is set or the previous stack is the NMI stack:
1164	 *    o Modify the "iret" location to jump to the repeat_nmi
1165	 *    o return back to the first NMI
1166	 *
1167	 * Now on exit of the first NMI, we first clear the stack variable
1168	 * The NMI stack will tell any nested NMIs at that point that it is
1169	 * nested. Then we pop the stack normally with iret, and if there was
1170	 * a nested NMI that updated the copy interrupt stack frame, a
1171	 * jump will be made to the repeat_nmi code that will handle the second
1172	 * NMI.
1173	 *
1174	 * However, espfix prevents us from directly returning to userspace
1175	 * with a single IRET instruction.  Similarly, IRET to user mode
1176	 * can fault.  We therefore handle NMIs from user space like
1177	 * other IST entries.
1178	 */
1179
1180	ASM_CLAC
1181	cld
1182
1183	/* Use %rdx as our temp variable throughout */
1184	pushq	%rdx
1185
1186	testb	$3, CS-RIP+8(%rsp)
1187	jz	.Lnmi_from_kernel
1188
1189	/*
1190	 * NMI from user mode.  We need to run on the thread stack, but we
1191	 * can't go through the normal entry paths: NMIs are masked, and
1192	 * we don't want to enable interrupts, because then we'll end
1193	 * up in an awkward situation in which IRQs are on but NMIs
1194	 * are off.
1195	 *
1196	 * We also must not push anything to the stack before switching
1197	 * stacks lest we corrupt the "NMI executing" variable.
1198	 */
1199
1200	swapgs
1201	FENCE_SWAPGS_USER_ENTRY
1202	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1203	movq	%rsp, %rdx
1204	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
1205	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1206	pushq	5*8(%rdx)	/* pt_regs->ss */
1207	pushq	4*8(%rdx)	/* pt_regs->rsp */
1208	pushq	3*8(%rdx)	/* pt_regs->flags */
1209	pushq	2*8(%rdx)	/* pt_regs->cs */
1210	pushq	1*8(%rdx)	/* pt_regs->rip */
1211	UNWIND_HINT_IRET_REGS
1212	pushq   $-1		/* pt_regs->orig_ax */
1213	PUSH_AND_CLEAR_REGS rdx=(%rdx)
1214	ENCODE_FRAME_POINTER
1215
1216	IBRS_ENTER
1217	UNTRAIN_RET
1218
1219	/*
1220	 * At this point we no longer need to worry about stack damage
1221	 * due to nesting -- we're on the normal thread stack and we're
1222	 * done with the NMI stack.
1223	 */
1224
1225	movq	%rsp, %rdi
1226	movq	$-1, %rsi
1227	call	exc_nmi
1228
1229	/*
1230	 * Return back to user mode.  We must *not* do the normal exit
1231	 * work, because we don't want to enable interrupts.
1232	 */
1233	jmp	swapgs_restore_regs_and_return_to_usermode
1234
1235.Lnmi_from_kernel:
1236	/*
1237	 * Here's what our stack frame will look like:
1238	 * +---------------------------------------------------------+
1239	 * | original SS                                             |
1240	 * | original Return RSP                                     |
1241	 * | original RFLAGS                                         |
1242	 * | original CS                                             |
1243	 * | original RIP                                            |
1244	 * +---------------------------------------------------------+
1245	 * | temp storage for rdx                                    |
1246	 * +---------------------------------------------------------+
1247	 * | "NMI executing" variable                                |
1248	 * +---------------------------------------------------------+
1249	 * | iret SS          } Copied from "outermost" frame        |
1250	 * | iret Return RSP  } on each loop iteration; overwritten  |
1251	 * | iret RFLAGS      } by a nested NMI to force another     |
1252	 * | iret CS          } iteration if needed.                 |
1253	 * | iret RIP         }                                      |
1254	 * +---------------------------------------------------------+
1255	 * | outermost SS          } initialized in first_nmi;       |
1256	 * | outermost Return RSP  } will not be changed before      |
1257	 * | outermost RFLAGS      } NMI processing is done.         |
1258	 * | outermost CS          } Copied to "iret" frame on each  |
1259	 * | outermost RIP         } iteration.                      |
1260	 * +---------------------------------------------------------+
1261	 * | pt_regs                                                 |
1262	 * +---------------------------------------------------------+
1263	 *
1264	 * The "original" frame is used by hardware.  Before re-enabling
1265	 * NMIs, we need to be done with it, and we need to leave enough
1266	 * space for the asm code here.
1267	 *
1268	 * We return by executing IRET while RSP points to the "iret" frame.
1269	 * That will either return for real or it will loop back into NMI
1270	 * processing.
1271	 *
1272	 * The "outermost" frame is copied to the "iret" frame on each
1273	 * iteration of the loop, so each iteration starts with the "iret"
1274	 * frame pointing to the final return target.
1275	 */
1276
1277	/*
1278	 * Determine whether we're a nested NMI.
1279	 *
1280	 * If we interrupted kernel code between repeat_nmi and
1281	 * end_repeat_nmi, then we are a nested NMI.  We must not
1282	 * modify the "iret" frame because it's being written by
1283	 * the outer NMI.  That's okay; the outer NMI handler is
1284	 * about to about to call exc_nmi() anyway, so we can just
1285	 * resume the outer NMI.
1286	 */
1287
1288	movq	$repeat_nmi, %rdx
1289	cmpq	8(%rsp), %rdx
1290	ja	1f
1291	movq	$end_repeat_nmi, %rdx
1292	cmpq	8(%rsp), %rdx
1293	ja	nested_nmi_out
12941:
1295
1296	/*
1297	 * Now check "NMI executing".  If it's set, then we're nested.
1298	 * This will not detect if we interrupted an outer NMI just
1299	 * before IRET.
1300	 */
1301	cmpl	$1, -8(%rsp)
1302	je	nested_nmi
1303
1304	/*
1305	 * Now test if the previous stack was an NMI stack.  This covers
1306	 * the case where we interrupt an outer NMI after it clears
1307	 * "NMI executing" but before IRET.  We need to be careful, though:
1308	 * there is one case in which RSP could point to the NMI stack
1309	 * despite there being no NMI active: naughty userspace controls
1310	 * RSP at the very beginning of the SYSCALL targets.  We can
1311	 * pull a fast one on naughty userspace, though: we program
1312	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
1313	 * if it controls the kernel's RSP.  We set DF before we clear
1314	 * "NMI executing".
1315	 */
1316	lea	6*8(%rsp), %rdx
1317	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
1318	cmpq	%rdx, 4*8(%rsp)
1319	/* If the stack pointer is above the NMI stack, this is a normal NMI */
1320	ja	first_nmi
1321
1322	subq	$EXCEPTION_STKSZ, %rdx
1323	cmpq	%rdx, 4*8(%rsp)
1324	/* If it is below the NMI stack, it is a normal NMI */
1325	jb	first_nmi
1326
1327	/* Ah, it is within the NMI stack. */
1328
1329	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
1330	jz	first_nmi	/* RSP was user controlled. */
1331
1332	/* This is a nested NMI. */
1333
1334nested_nmi:
1335	/*
1336	 * Modify the "iret" frame to point to repeat_nmi, forcing another
1337	 * iteration of NMI handling.
1338	 */
1339	subq	$8, %rsp
1340	leaq	-10*8(%rsp), %rdx
1341	pushq	$__KERNEL_DS
1342	pushq	%rdx
1343	pushfq
1344	pushq	$__KERNEL_CS
1345	pushq	$repeat_nmi
1346
1347	/* Put stack back */
1348	addq	$(6*8), %rsp
1349
1350nested_nmi_out:
1351	popq	%rdx
1352
1353	/* We are returning to kernel mode, so this cannot result in a fault. */
1354	iretq
1355
1356first_nmi:
1357	/* Restore rdx. */
1358	movq	(%rsp), %rdx
1359
1360	/* Make room for "NMI executing". */
1361	pushq	$0
1362
1363	/* Leave room for the "iret" frame */
1364	subq	$(5*8), %rsp
1365
1366	/* Copy the "original" frame to the "outermost" frame */
1367	.rept 5
1368	pushq	11*8(%rsp)
1369	.endr
1370	UNWIND_HINT_IRET_REGS
1371
1372	/* Everything up to here is safe from nested NMIs */
1373
1374#ifdef CONFIG_DEBUG_ENTRY
1375	/*
1376	 * For ease of testing, unmask NMIs right away.  Disabled by
1377	 * default because IRET is very expensive.
1378	 */
1379	pushq	$0		/* SS */
1380	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
1381	addq	$8, (%rsp)	/* Fix up RSP */
1382	pushfq			/* RFLAGS */
1383	pushq	$__KERNEL_CS	/* CS */
1384	pushq	$1f		/* RIP */
1385	iretq			/* continues at repeat_nmi below */
1386	UNWIND_HINT_IRET_REGS
13871:
1388#endif
1389
1390repeat_nmi:
1391	ANNOTATE_NOENDBR // this code
1392	/*
1393	 * If there was a nested NMI, the first NMI's iret will return
1394	 * here. But NMIs are still enabled and we can take another
1395	 * nested NMI. The nested NMI checks the interrupted RIP to see
1396	 * if it is between repeat_nmi and end_repeat_nmi, and if so
1397	 * it will just return, as we are about to repeat an NMI anyway.
1398	 * This makes it safe to copy to the stack frame that a nested
1399	 * NMI will update.
1400	 *
1401	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
1402	 * we're repeating an NMI, gsbase has the same value that it had on
1403	 * the first iteration.  paranoid_entry will load the kernel
1404	 * gsbase if needed before we call exc_nmi().  "NMI executing"
1405	 * is zero.
1406	 */
1407	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1408
1409	/*
1410	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
1411	 * here must not modify the "iret" frame while we're writing to
1412	 * it or it will end up containing garbage.
1413	 */
1414	addq	$(10*8), %rsp
1415	.rept 5
1416	pushq	-6*8(%rsp)
1417	.endr
1418	subq	$(5*8), %rsp
1419end_repeat_nmi:
1420	ANNOTATE_NOENDBR // this code
1421
1422	/*
1423	 * Everything below this point can be preempted by a nested NMI.
1424	 * If this happens, then the inner NMI will change the "iret"
1425	 * frame to point back to repeat_nmi.
1426	 */
1427	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1428
1429	/*
1430	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1431	 * as we should not be calling schedule in NMI context.
1432	 * Even with normal interrupts enabled. An NMI should not be
1433	 * setting NEED_RESCHED or anything that normal interrupts and
1434	 * exceptions might do.
1435	 */
1436	call	paranoid_entry
1437	UNWIND_HINT_REGS
1438
1439	movq	%rsp, %rdi
1440	movq	$-1, %rsi
1441	call	exc_nmi
1442
1443	/* Always restore stashed SPEC_CTRL value (see paranoid_entry) */
1444	IBRS_EXIT save_reg=%r15
1445
1446	/* Always restore stashed CR3 value (see paranoid_entry) */
1447	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1448
1449	/*
1450	 * The above invocation of paranoid_entry stored the GSBASE
1451	 * related information in R/EBX depending on the availability
1452	 * of FSGSBASE.
1453	 *
1454	 * If FSGSBASE is enabled, restore the saved GSBASE value
1455	 * unconditionally, otherwise take the conditional SWAPGS path.
1456	 */
1457	ALTERNATIVE "jmp nmi_no_fsgsbase", "", X86_FEATURE_FSGSBASE
1458
1459	wrgsbase	%rbx
1460	jmp	nmi_restore
1461
1462nmi_no_fsgsbase:
1463	/* EBX == 0 -> invoke SWAPGS */
1464	testl	%ebx, %ebx
1465	jnz	nmi_restore
1466
1467nmi_swapgs:
1468	swapgs
1469
1470nmi_restore:
1471	POP_REGS
1472
1473	/*
1474	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
1475	 * at the "iret" frame.
1476	 */
1477	addq	$6*8, %rsp
1478
1479	/*
1480	 * Clear "NMI executing".  Set DF first so that we can easily
1481	 * distinguish the remaining code between here and IRET from
1482	 * the SYSCALL entry and exit paths.
1483	 *
1484	 * We arguably should just inspect RIP instead, but I (Andy) wrote
1485	 * this code when I had the misapprehension that Xen PV supported
1486	 * NMIs, and Xen PV would break that approach.
1487	 */
1488	std
1489	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1490
1491	/*
1492	 * iretq reads the "iret" frame and exits the NMI stack in a
1493	 * single instruction.  We are returning to kernel mode, so this
1494	 * cannot result in a fault.  Similarly, we don't need to worry
1495	 * about espfix64 on the way back to kernel mode.
1496	 */
1497	iretq
1498SYM_CODE_END(asm_exc_nmi)
1499
1500#ifndef CONFIG_IA32_EMULATION
1501/*
1502 * This handles SYSCALL from 32-bit code.  There is no way to program
1503 * MSRs to fully disable 32-bit SYSCALL.
1504 */
1505SYM_CODE_START(ignore_sysret)
1506	UNWIND_HINT_END_OF_STACK
1507	ENDBR
1508	mov	$-ENOSYS, %eax
1509	sysretl
1510SYM_CODE_END(ignore_sysret)
1511#endif
1512
1513.pushsection .text, "ax"
1514	__FUNC_ALIGN
1515SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead)
1516	UNWIND_HINT_FUNC
1517	/* Prevent any naive code from trying to unwind to our caller. */
1518	xorl	%ebp, %ebp
1519
1520	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax
1521	leaq	-PTREGS_SIZE(%rax), %rsp
1522	UNWIND_HINT_REGS
1523
1524	call	make_task_dead
1525SYM_CODE_END(rewind_stack_and_make_dead)
1526.popsection
1527