xref: /openbmc/linux/arch/x86/kvm/vmx/vmenter.S (revision 36d26912) 
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/linkage.h>
3#include <asm/asm.h>
4#include <asm/bitsperlong.h>
5#include <asm/kvm_vcpu_regs.h>
6#include <asm/nospec-branch.h>
7
8#define WORD_SIZE (BITS_PER_LONG / 8)
9
10#define VCPU_RAX	__VCPU_REGS_RAX * WORD_SIZE
11#define VCPU_RCX	__VCPU_REGS_RCX * WORD_SIZE
12#define VCPU_RDX	__VCPU_REGS_RDX * WORD_SIZE
13#define VCPU_RBX	__VCPU_REGS_RBX * WORD_SIZE
14/* Intentionally omit RSP as it's context switched by hardware */
15#define VCPU_RBP	__VCPU_REGS_RBP * WORD_SIZE
16#define VCPU_RSI	__VCPU_REGS_RSI * WORD_SIZE
17#define VCPU_RDI	__VCPU_REGS_RDI * WORD_SIZE
18
19#ifdef CONFIG_X86_64
20#define VCPU_R8		__VCPU_REGS_R8  * WORD_SIZE
21#define VCPU_R9		__VCPU_REGS_R9  * WORD_SIZE
22#define VCPU_R10	__VCPU_REGS_R10 * WORD_SIZE
23#define VCPU_R11	__VCPU_REGS_R11 * WORD_SIZE
24#define VCPU_R12	__VCPU_REGS_R12 * WORD_SIZE
25#define VCPU_R13	__VCPU_REGS_R13 * WORD_SIZE
26#define VCPU_R14	__VCPU_REGS_R14 * WORD_SIZE
27#define VCPU_R15	__VCPU_REGS_R15 * WORD_SIZE
28#endif
29
30.section .noinstr.text, "ax"
31
32/**
33 * vmx_vmenter - VM-Enter the current loaded VMCS
34 *
35 * %RFLAGS.ZF:	!VMCS.LAUNCHED, i.e. controls VMLAUNCH vs. VMRESUME
36 *
37 * Returns:
38 *	%RFLAGS.CF is set on VM-Fail Invalid
39 *	%RFLAGS.ZF is set on VM-Fail Valid
40 *	%RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
41 *
42 * Note that VMRESUME/VMLAUNCH fall-through and return directly if
43 * they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
44 * to vmx_vmexit.
45 */
46SYM_FUNC_START(vmx_vmenter)
47	/* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
48	je 2f
49
501:	vmresume
51	ret
52
532:	vmlaunch
54	ret
55
563:	cmpb $0, kvm_rebooting
57	je 4f
58	ret
594:	ud2
60
61	_ASM_EXTABLE(1b, 3b)
62	_ASM_EXTABLE(2b, 3b)
63
64SYM_FUNC_END(vmx_vmenter)
65
66/**
67 * vmx_vmexit - Handle a VMX VM-Exit
68 *
69 * Returns:
70 *	%RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
71 *
72 * This is vmx_vmenter's partner in crime.  On a VM-Exit, control will jump
73 * here after hardware loads the host's state, i.e. this is the destination
74 * referred to by VMCS.HOST_RIP.
75 */
76SYM_FUNC_START(vmx_vmexit)
77#ifdef CONFIG_RETPOLINE
78	ALTERNATIVE "jmp .Lvmexit_skip_rsb", "", X86_FEATURE_RETPOLINE
79	/* Preserve guest's RAX, it's used to stuff the RSB. */
80	push %_ASM_AX
81
82	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
83	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
84
85	/* Clear RFLAGS.CF and RFLAGS.ZF to preserve VM-Exit, i.e. !VM-Fail. */
86	or $1, %_ASM_AX
87
88	pop %_ASM_AX
89.Lvmexit_skip_rsb:
90#endif
91	ret
92SYM_FUNC_END(vmx_vmexit)
93
94/**
95 * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
96 * @vmx:	struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
97 * @regs:	unsigned long * (to guest registers)
98 * @launched:	%true if the VMCS has been launched
99 *
100 * Returns:
101 *	0 on VM-Exit, 1 on VM-Fail
102 */
103SYM_FUNC_START(__vmx_vcpu_run)
104	push %_ASM_BP
105	mov  %_ASM_SP, %_ASM_BP
106#ifdef CONFIG_X86_64
107	push %r15
108	push %r14
109	push %r13
110	push %r12
111#else
112	push %edi
113	push %esi
114#endif
115	push %_ASM_BX
116
117	/*
118	 * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
119	 * @regs is needed after VM-Exit to save the guest's register values.
120	 */
121	push %_ASM_ARG2
122
123	/* Copy @launched to BL, _ASM_ARG3 is volatile. */
124	mov %_ASM_ARG3B, %bl
125
126	/* Adjust RSP to account for the CALL to vmx_vmenter(). */
127	lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
128	call vmx_update_host_rsp
129
130	/* Load @regs to RAX. */
131	mov (%_ASM_SP), %_ASM_AX
132
133	/* Check if vmlaunch or vmresume is needed */
134	cmpb $0, %bl
135
136	/* Load guest registers.  Don't clobber flags. */
137	mov VCPU_RCX(%_ASM_AX), %_ASM_CX
138	mov VCPU_RDX(%_ASM_AX), %_ASM_DX
139	mov VCPU_RBX(%_ASM_AX), %_ASM_BX
140	mov VCPU_RBP(%_ASM_AX), %_ASM_BP
141	mov VCPU_RSI(%_ASM_AX), %_ASM_SI
142	mov VCPU_RDI(%_ASM_AX), %_ASM_DI
143#ifdef CONFIG_X86_64
144	mov VCPU_R8 (%_ASM_AX),  %r8
145	mov VCPU_R9 (%_ASM_AX),  %r9
146	mov VCPU_R10(%_ASM_AX), %r10
147	mov VCPU_R11(%_ASM_AX), %r11
148	mov VCPU_R12(%_ASM_AX), %r12
149	mov VCPU_R13(%_ASM_AX), %r13
150	mov VCPU_R14(%_ASM_AX), %r14
151	mov VCPU_R15(%_ASM_AX), %r15
152#endif
153	/* Load guest RAX.  This kills the @regs pointer! */
154	mov VCPU_RAX(%_ASM_AX), %_ASM_AX
155
156	/* Enter guest mode */
157	call vmx_vmenter
158
159	/* Jump on VM-Fail. */
160	jbe 2f
161
162	/* Temporarily save guest's RAX. */
163	push %_ASM_AX
164
165	/* Reload @regs to RAX. */
166	mov WORD_SIZE(%_ASM_SP), %_ASM_AX
167
168	/* Save all guest registers, including RAX from the stack */
169	pop           VCPU_RAX(%_ASM_AX)
170	mov %_ASM_CX, VCPU_RCX(%_ASM_AX)
171	mov %_ASM_DX, VCPU_RDX(%_ASM_AX)
172	mov %_ASM_BX, VCPU_RBX(%_ASM_AX)
173	mov %_ASM_BP, VCPU_RBP(%_ASM_AX)
174	mov %_ASM_SI, VCPU_RSI(%_ASM_AX)
175	mov %_ASM_DI, VCPU_RDI(%_ASM_AX)
176#ifdef CONFIG_X86_64
177	mov %r8,  VCPU_R8 (%_ASM_AX)
178	mov %r9,  VCPU_R9 (%_ASM_AX)
179	mov %r10, VCPU_R10(%_ASM_AX)
180	mov %r11, VCPU_R11(%_ASM_AX)
181	mov %r12, VCPU_R12(%_ASM_AX)
182	mov %r13, VCPU_R13(%_ASM_AX)
183	mov %r14, VCPU_R14(%_ASM_AX)
184	mov %r15, VCPU_R15(%_ASM_AX)
185#endif
186
187	/* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */
188	xor %eax, %eax
189
190	/*
191	 * Clear all general purpose registers except RSP and RAX to prevent
192	 * speculative use of the guest's values, even those that are reloaded
193	 * via the stack.  In theory, an L1 cache miss when restoring registers
194	 * could lead to speculative execution with the guest's values.
195	 * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
196	 * free.  RSP and RAX are exempt as RSP is restored by hardware during
197	 * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail.
198	 */
1991:	xor %ecx, %ecx
200	xor %edx, %edx
201	xor %ebx, %ebx
202	xor %ebp, %ebp
203	xor %esi, %esi
204	xor %edi, %edi
205#ifdef CONFIG_X86_64
206	xor %r8d,  %r8d
207	xor %r9d,  %r9d
208	xor %r10d, %r10d
209	xor %r11d, %r11d
210	xor %r12d, %r12d
211	xor %r13d, %r13d
212	xor %r14d, %r14d
213	xor %r15d, %r15d
214#endif
215
216	/* "POP" @regs. */
217	add $WORD_SIZE, %_ASM_SP
218	pop %_ASM_BX
219
220#ifdef CONFIG_X86_64
221	pop %r12
222	pop %r13
223	pop %r14
224	pop %r15
225#else
226	pop %esi
227	pop %edi
228#endif
229	pop %_ASM_BP
230	ret
231
232	/* VM-Fail.  Out-of-line to avoid a taken Jcc after VM-Exit. */
2332:	mov $1, %eax
234	jmp 1b
235SYM_FUNC_END(__vmx_vcpu_run)
236
237
238.section .text, "ax"
239
240/**
241 * vmread_error_trampoline - Trampoline from inline asm to vmread_error()
242 * @field:	VMCS field encoding that failed
243 * @fault:	%true if the VMREAD faulted, %false if it failed
244
245 * Save and restore volatile registers across a call to vmread_error().  Note,
246 * all parameters are passed on the stack.
247 */
248SYM_FUNC_START(vmread_error_trampoline)
249	push %_ASM_BP
250	mov  %_ASM_SP, %_ASM_BP
251
252	push %_ASM_AX
253	push %_ASM_CX
254	push %_ASM_DX
255#ifdef CONFIG_X86_64
256	push %rdi
257	push %rsi
258	push %r8
259	push %r9
260	push %r10
261	push %r11
262#endif
263#ifdef CONFIG_X86_64
264	/* Load @field and @fault to arg1 and arg2 respectively. */
265	mov 3*WORD_SIZE(%rbp), %_ASM_ARG2
266	mov 2*WORD_SIZE(%rbp), %_ASM_ARG1
267#else
268	/* Parameters are passed on the stack for 32-bit (see asmlinkage). */
269	push 3*WORD_SIZE(%ebp)
270	push 2*WORD_SIZE(%ebp)
271#endif
272
273	call vmread_error
274
275#ifndef CONFIG_X86_64
276	add $8, %esp
277#endif
278
279	/* Zero out @fault, which will be popped into the result register. */
280	_ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP)
281
282#ifdef CONFIG_X86_64
283	pop %r11
284	pop %r10
285	pop %r9
286	pop %r8
287	pop %rsi
288	pop %rdi
289#endif
290	pop %_ASM_DX
291	pop %_ASM_CX
292	pop %_ASM_AX
293	pop %_ASM_BP
294
295	ret
296SYM_FUNC_END(vmread_error_trampoline)
297