xref: /openbmc/linux/arch/x86/kvm/vmx/vmenter.S (revision e149ca29) 
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/linkage.h>
3#include <asm/asm.h>
4#include <asm/bitsperlong.h>
5#include <asm/kvm_vcpu_regs.h>
6#include <asm/nospec-branch.h>
7
8#define WORD_SIZE (BITS_PER_LONG / 8)
9
10#define VCPU_RAX	__VCPU_REGS_RAX * WORD_SIZE
11#define VCPU_RCX	__VCPU_REGS_RCX * WORD_SIZE
12#define VCPU_RDX	__VCPU_REGS_RDX * WORD_SIZE
13#define VCPU_RBX	__VCPU_REGS_RBX * WORD_SIZE
14/* Intentionally omit RSP as it's context switched by hardware */
15#define VCPU_RBP	__VCPU_REGS_RBP * WORD_SIZE
16#define VCPU_RSI	__VCPU_REGS_RSI * WORD_SIZE
17#define VCPU_RDI	__VCPU_REGS_RDI * WORD_SIZE
18
19#ifdef CONFIG_X86_64
20#define VCPU_R8		__VCPU_REGS_R8  * WORD_SIZE
21#define VCPU_R9		__VCPU_REGS_R9  * WORD_SIZE
22#define VCPU_R10	__VCPU_REGS_R10 * WORD_SIZE
23#define VCPU_R11	__VCPU_REGS_R11 * WORD_SIZE
24#define VCPU_R12	__VCPU_REGS_R12 * WORD_SIZE
25#define VCPU_R13	__VCPU_REGS_R13 * WORD_SIZE
26#define VCPU_R14	__VCPU_REGS_R14 * WORD_SIZE
27#define VCPU_R15	__VCPU_REGS_R15 * WORD_SIZE
28#endif
29
30	.text
31
32/**
33 * vmx_vmenter - VM-Enter the current loaded VMCS
34 *
35 * %RFLAGS.ZF:	!VMCS.LAUNCHED, i.e. controls VMLAUNCH vs. VMRESUME
36 *
37 * Returns:
38 *	%RFLAGS.CF is set on VM-Fail Invalid
39 *	%RFLAGS.ZF is set on VM-Fail Valid
40 *	%RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
41 *
42 * Note that VMRESUME/VMLAUNCH fall-through and return directly if
43 * they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
44 * to vmx_vmexit.
45 */
46SYM_FUNC_START(vmx_vmenter)
47	/* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
48	je 2f
49
501:	vmresume
51	ret
52
532:	vmlaunch
54	ret
55
563:	cmpb $0, kvm_rebooting
57	je 4f
58	ret
594:	ud2
60
61	_ASM_EXTABLE(1b, 3b)
62	_ASM_EXTABLE(2b, 3b)
63
64SYM_FUNC_END(vmx_vmenter)
65
66/**
67 * vmx_vmexit - Handle a VMX VM-Exit
68 *
69 * Returns:
70 *	%RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
71 *
72 * This is vmx_vmenter's partner in crime.  On a VM-Exit, control will jump
73 * here after hardware loads the host's state, i.e. this is the destination
74 * referred to by VMCS.HOST_RIP.
75 */
76SYM_FUNC_START(vmx_vmexit)
77#ifdef CONFIG_RETPOLINE
78	ALTERNATIVE "jmp .Lvmexit_skip_rsb", "", X86_FEATURE_RETPOLINE
79	/* Preserve guest's RAX, it's used to stuff the RSB. */
80	push %_ASM_AX
81
82	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
83	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
84
85	pop %_ASM_AX
86.Lvmexit_skip_rsb:
87#endif
88	ret
89SYM_FUNC_END(vmx_vmexit)
90
91/**
92 * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
93 * @vmx:	struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
94 * @regs:	unsigned long * (to guest registers)
95 * @launched:	%true if the VMCS has been launched
96 *
97 * Returns:
98 *	0 on VM-Exit, 1 on VM-Fail
99 */
100SYM_FUNC_START(__vmx_vcpu_run)
101	push %_ASM_BP
102	mov  %_ASM_SP, %_ASM_BP
103#ifdef CONFIG_X86_64
104	push %r15
105	push %r14
106	push %r13
107	push %r12
108#else
109	push %edi
110	push %esi
111#endif
112	push %_ASM_BX
113
114	/*
115	 * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
116	 * @regs is needed after VM-Exit to save the guest's register values.
117	 */
118	push %_ASM_ARG2
119
120	/* Copy @launched to BL, _ASM_ARG3 is volatile. */
121	mov %_ASM_ARG3B, %bl
122
123	/* Adjust RSP to account for the CALL to vmx_vmenter(). */
124	lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
125	call vmx_update_host_rsp
126
127	/* Load @regs to RAX. */
128	mov (%_ASM_SP), %_ASM_AX
129
130	/* Check if vmlaunch or vmresume is needed */
131	cmpb $0, %bl
132
133	/* Load guest registers.  Don't clobber flags. */
134	mov VCPU_RCX(%_ASM_AX), %_ASM_CX
135	mov VCPU_RDX(%_ASM_AX), %_ASM_DX
136	mov VCPU_RBX(%_ASM_AX), %_ASM_BX
137	mov VCPU_RBP(%_ASM_AX), %_ASM_BP
138	mov VCPU_RSI(%_ASM_AX), %_ASM_SI
139	mov VCPU_RDI(%_ASM_AX), %_ASM_DI
140#ifdef CONFIG_X86_64
141	mov VCPU_R8 (%_ASM_AX),  %r8
142	mov VCPU_R9 (%_ASM_AX),  %r9
143	mov VCPU_R10(%_ASM_AX), %r10
144	mov VCPU_R11(%_ASM_AX), %r11
145	mov VCPU_R12(%_ASM_AX), %r12
146	mov VCPU_R13(%_ASM_AX), %r13
147	mov VCPU_R14(%_ASM_AX), %r14
148	mov VCPU_R15(%_ASM_AX), %r15
149#endif
150	/* Load guest RAX.  This kills the @regs pointer! */
151	mov VCPU_RAX(%_ASM_AX), %_ASM_AX
152
153	/* Enter guest mode */
154	call vmx_vmenter
155
156	/* Jump on VM-Fail. */
157	jbe 2f
158
159	/* Temporarily save guest's RAX. */
160	push %_ASM_AX
161
162	/* Reload @regs to RAX. */
163	mov WORD_SIZE(%_ASM_SP), %_ASM_AX
164
165	/* Save all guest registers, including RAX from the stack */
166	__ASM_SIZE(pop) VCPU_RAX(%_ASM_AX)
167	mov %_ASM_CX,   VCPU_RCX(%_ASM_AX)
168	mov %_ASM_DX,   VCPU_RDX(%_ASM_AX)
169	mov %_ASM_BX,   VCPU_RBX(%_ASM_AX)
170	mov %_ASM_BP,   VCPU_RBP(%_ASM_AX)
171	mov %_ASM_SI,   VCPU_RSI(%_ASM_AX)
172	mov %_ASM_DI,   VCPU_RDI(%_ASM_AX)
173#ifdef CONFIG_X86_64
174	mov %r8,  VCPU_R8 (%_ASM_AX)
175	mov %r9,  VCPU_R9 (%_ASM_AX)
176	mov %r10, VCPU_R10(%_ASM_AX)
177	mov %r11, VCPU_R11(%_ASM_AX)
178	mov %r12, VCPU_R12(%_ASM_AX)
179	mov %r13, VCPU_R13(%_ASM_AX)
180	mov %r14, VCPU_R14(%_ASM_AX)
181	mov %r15, VCPU_R15(%_ASM_AX)
182#endif
183
184	/* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */
185	xor %eax, %eax
186
187	/*
188	 * Clear all general purpose registers except RSP and RAX to prevent
189	 * speculative use of the guest's values, even those that are reloaded
190	 * via the stack.  In theory, an L1 cache miss when restoring registers
191	 * could lead to speculative execution with the guest's values.
192	 * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
193	 * free.  RSP and RAX are exempt as RSP is restored by hardware during
194	 * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail.
195	 */
1961:	xor %ecx, %ecx
197	xor %edx, %edx
198	xor %ebx, %ebx
199	xor %ebp, %ebp
200	xor %esi, %esi
201	xor %edi, %edi
202#ifdef CONFIG_X86_64
203	xor %r8d,  %r8d
204	xor %r9d,  %r9d
205	xor %r10d, %r10d
206	xor %r11d, %r11d
207	xor %r12d, %r12d
208	xor %r13d, %r13d
209	xor %r14d, %r14d
210	xor %r15d, %r15d
211#endif
212
213	/* "POP" @regs. */
214	add $WORD_SIZE, %_ASM_SP
215	pop %_ASM_BX
216
217#ifdef CONFIG_X86_64
218	pop %r12
219	pop %r13
220	pop %r14
221	pop %r15
222#else
223	pop %esi
224	pop %edi
225#endif
226	pop %_ASM_BP
227	ret
228
229	/* VM-Fail.  Out-of-line to avoid a taken Jcc after VM-Exit. */
2302:	mov $1, %eax
231	jmp 1b
232SYM_FUNC_END(__vmx_vcpu_run)
233
234/**
235 * vmread_error_trampoline - Trampoline from inline asm to vmread_error()
236 * @field:	VMCS field encoding that failed
237 * @fault:	%true if the VMREAD faulted, %false if it failed
238
239 * Save and restore volatile registers across a call to vmread_error().  Note,
240 * all parameters are passed on the stack.
241 */
242SYM_FUNC_START(vmread_error_trampoline)
243	push %_ASM_BP
244	mov  %_ASM_SP, %_ASM_BP
245
246	push %_ASM_AX
247	push %_ASM_CX
248	push %_ASM_DX
249#ifdef CONFIG_X86_64
250	push %rdi
251	push %rsi
252	push %r8
253	push %r9
254	push %r10
255	push %r11
256#endif
257#ifdef CONFIG_X86_64
258	/* Load @field and @fault to arg1 and arg2 respectively. */
259	mov 3*WORD_SIZE(%rbp), %_ASM_ARG2
260	mov 2*WORD_SIZE(%rbp), %_ASM_ARG1
261#else
262	/* Parameters are passed on the stack for 32-bit (see asmlinkage). */
263	push 3*WORD_SIZE(%ebp)
264	push 2*WORD_SIZE(%ebp)
265#endif
266
267	call vmread_error
268
269#ifndef CONFIG_X86_64
270	add $8, %esp
271#endif
272
273	/* Zero out @fault, which will be popped into the result register. */
274	_ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP)
275
276#ifdef CONFIG_X86_64
277	pop %r11
278	pop %r10
279	pop %r9
280	pop %r8
281	pop %rsi
282	pop %rdi
283#endif
284	pop %_ASM_DX
285	pop %_ASM_CX
286	pop %_ASM_AX
287	pop %_ASM_BP
288
289	ret
290SYM_FUNC_END(vmread_error_trampoline)
291