xref: /openbmc/linux/arch/x86/xen/xen-asm.S (revision aa74c44b) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Asm versions of Xen pv-ops, suitable for direct use.
4 *
5 * We only bother with direct forms (ie, vcpu in percpu data) of the
6 * operations here; the indirect forms are better handled in C.
7 */
8
9#include <asm/errno.h>
10#include <asm/asm-offsets.h>
11#include <asm/percpu.h>
12#include <asm/processor-flags.h>
13#include <asm/segment.h>
14#include <asm/thread_info.h>
15#include <asm/asm.h>
16#include <asm/frame.h>
17#include <asm/unwind_hints.h>
18
19#include <xen/interface/xen.h>
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <../entry/calling.h>
24
25.pushsection .noinstr.text, "ax"
26/*
27 * Disabling events is simply a matter of making the event mask
28 * non-zero.
29 */
30SYM_FUNC_START(xen_irq_disable_direct)
31	movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
32	RET
33SYM_FUNC_END(xen_irq_disable_direct)
34
35/*
36 * Force an event check by making a hypercall, but preserve regs
37 * before making the call.
38 */
39SYM_FUNC_START(check_events)
40	FRAME_BEGIN
41	push %rax
42	push %rcx
43	push %rdx
44	push %rsi
45	push %rdi
46	push %r8
47	push %r9
48	push %r10
49	push %r11
50	call xen_force_evtchn_callback
51	pop %r11
52	pop %r10
53	pop %r9
54	pop %r8
55	pop %rdi
56	pop %rsi
57	pop %rdx
58	pop %rcx
59	pop %rax
60	FRAME_END
61	RET
62SYM_FUNC_END(check_events)
63
64/*
65 * Enable events.  This clears the event mask and tests the pending
66 * event status with one and operation.  If there are pending events,
67 * then enter the hypervisor to get them handled.
68 */
69SYM_FUNC_START(xen_irq_enable_direct)
70	FRAME_BEGIN
71	/* Unmask events */
72	movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
73
74	/*
75	 * Preempt here doesn't matter because that will deal with any
76	 * pending interrupts.  The pending check may end up being run
77	 * on the wrong CPU, but that doesn't hurt.
78	 */
79
80	/* Test for pending */
81	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
82	jz 1f
83
84	call check_events
851:
86	FRAME_END
87	RET
88SYM_FUNC_END(xen_irq_enable_direct)
89
90/*
91 * (xen_)save_fl is used to get the current interrupt enable status.
92 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
93 * may be set in the return value.  We take advantage of this by
94 * making sure that X86_EFLAGS_IF has the right value (and other bits
95 * in that byte are 0), but other bits in the return value are
96 * undefined.  We need to toggle the state of the bit, because Xen and
97 * x86 use opposite senses (mask vs enable).
98 */
99SYM_FUNC_START(xen_save_fl_direct)
100	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
101	setz %ah
102	addb %ah, %ah
103	RET
104SYM_FUNC_END(xen_save_fl_direct)
105
106SYM_FUNC_START(xen_read_cr2)
107	FRAME_BEGIN
108	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
109	_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
110	FRAME_END
111	RET
112SYM_FUNC_END(xen_read_cr2);
113
114SYM_FUNC_START(xen_read_cr2_direct)
115	FRAME_BEGIN
116	_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
117	FRAME_END
118	RET
119SYM_FUNC_END(xen_read_cr2_direct);
120.popsection
121
122.macro xen_pv_trap name
123SYM_CODE_START(xen_\name)
124	UNWIND_HINT_EMPTY
125	pop %rcx
126	pop %r11
127	jmp  \name
128SYM_CODE_END(xen_\name)
129_ASM_NOKPROBE(xen_\name)
130.endm
131
132xen_pv_trap asm_exc_divide_error
133xen_pv_trap asm_xenpv_exc_debug
134xen_pv_trap asm_exc_int3
135xen_pv_trap asm_xenpv_exc_nmi
136xen_pv_trap asm_exc_overflow
137xen_pv_trap asm_exc_bounds
138xen_pv_trap asm_exc_invalid_op
139xen_pv_trap asm_exc_device_not_available
140xen_pv_trap asm_xenpv_exc_double_fault
141xen_pv_trap asm_exc_coproc_segment_overrun
142xen_pv_trap asm_exc_invalid_tss
143xen_pv_trap asm_exc_segment_not_present
144xen_pv_trap asm_exc_stack_segment
145xen_pv_trap asm_exc_general_protection
146xen_pv_trap asm_exc_page_fault
147xen_pv_trap asm_exc_spurious_interrupt_bug
148xen_pv_trap asm_exc_coprocessor_error
149xen_pv_trap asm_exc_alignment_check
150#ifdef CONFIG_X86_MCE
151xen_pv_trap asm_xenpv_exc_machine_check
152#endif /* CONFIG_X86_MCE */
153xen_pv_trap asm_exc_simd_coprocessor_error
154#ifdef CONFIG_IA32_EMULATION
155xen_pv_trap entry_INT80_compat
156#endif
157xen_pv_trap asm_exc_xen_unknown_trap
158xen_pv_trap asm_exc_xen_hypervisor_callback
159
160	__INIT
161SYM_CODE_START(xen_early_idt_handler_array)
162	i = 0
163	.rept NUM_EXCEPTION_VECTORS
164	UNWIND_HINT_EMPTY
165	pop %rcx
166	pop %r11
167	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
168	i = i + 1
169	.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
170	.endr
171SYM_CODE_END(xen_early_idt_handler_array)
172	__FINIT
173
174hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
175/*
176 * Xen64 iret frame:
177 *
178 *	ss
179 *	rsp
180 *	rflags
181 *	cs
182 *	rip		<-- standard iret frame
183 *
184 *	flags
185 *
186 *	rcx		}
187 *	r11		}<-- pushed by hypercall page
188 * rsp->rax		}
189 */
190SYM_CODE_START(xen_iret)
191	UNWIND_HINT_EMPTY
192	pushq $0
193	jmp hypercall_iret
194SYM_CODE_END(xen_iret)
195
196/*
197 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
198 * also the kernel stack.  Reusing swapgs_restore_regs_and_return_to_usermode()
199 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
200 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
201 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
202 * frame at the same address is useless.
203 */
204SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
205	UNWIND_HINT_REGS
206	POP_REGS
207
208	/* stackleak_erase() can work safely on the kernel stack. */
209	STACKLEAK_ERASE_NOCLOBBER
210
211	addq	$8, %rsp	/* skip regs->orig_ax */
212	jmp xen_iret
213SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
214
215/*
216 * Xen handles syscall callbacks much like ordinary exceptions, which
217 * means we have:
218 * - kernel gs
219 * - kernel rsp
220 * - an iret-like stack frame on the stack (including rcx and r11):
221 *	ss
222 *	rsp
223 *	rflags
224 *	cs
225 *	rip
226 *	r11
227 * rsp->rcx
228 */
229
230/* Normal 64-bit system call target */
231SYM_CODE_START(xen_syscall_target)
232	UNWIND_HINT_EMPTY
233	popq %rcx
234	popq %r11
235
236	/*
237	 * Neither Xen nor the kernel really knows what the old SS and
238	 * CS were.  The kernel expects __USER_DS and __USER_CS, so
239	 * report those values even though Xen will guess its own values.
240	 */
241	movq $__USER_DS, 4*8(%rsp)
242	movq $__USER_CS, 1*8(%rsp)
243
244	jmp entry_SYSCALL_64_after_hwframe
245SYM_CODE_END(xen_syscall_target)
246
247#ifdef CONFIG_IA32_EMULATION
248
249/* 32-bit compat syscall target */
250SYM_CODE_START(xen_syscall32_target)
251	UNWIND_HINT_EMPTY
252	popq %rcx
253	popq %r11
254
255	/*
256	 * Neither Xen nor the kernel really knows what the old SS and
257	 * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
258	 * report those values even though Xen will guess its own values.
259	 */
260	movq $__USER32_DS, 4*8(%rsp)
261	movq $__USER32_CS, 1*8(%rsp)
262
263	jmp entry_SYSCALL_compat_after_hwframe
264SYM_CODE_END(xen_syscall32_target)
265
266/* 32-bit compat sysenter target */
267SYM_CODE_START(xen_sysenter_target)
268	UNWIND_HINT_EMPTY
269	/*
270	 * NB: Xen is polite and clears TF from EFLAGS for us.  This means
271	 * that we don't need to guard against single step exceptions here.
272	 */
273	popq %rcx
274	popq %r11
275
276	/*
277	 * Neither Xen nor the kernel really knows what the old SS and
278	 * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
279	 * report those values even though Xen will guess its own values.
280	 */
281	movq $__USER32_DS, 4*8(%rsp)
282	movq $__USER32_CS, 1*8(%rsp)
283
284	jmp entry_SYSENTER_compat_after_hwframe
285SYM_CODE_END(xen_sysenter_target)
286
287#else /* !CONFIG_IA32_EMULATION */
288
289SYM_CODE_START(xen_syscall32_target)
290SYM_CODE_START(xen_sysenter_target)
291	UNWIND_HINT_EMPTY
292	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
293	mov $-ENOSYS, %rax
294	pushq $0
295	jmp hypercall_iret
296SYM_CODE_END(xen_sysenter_target)
297SYM_CODE_END(xen_syscall32_target)
298
299#endif	/* CONFIG_IA32_EMULATION */
300