xref: /openbmc/linux/arch/arm64/kvm/hyp/hyp-entry.S (revision ae40e94f) 
1/*
2 * Copyright (C) 2015-2018 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/arm-smccc.h>
19#include <linux/linkage.h>
20
21#include <asm/alternative.h>
22#include <asm/assembler.h>
23#include <asm/cpufeature.h>
24#include <asm/kvm_arm.h>
25#include <asm/kvm_asm.h>
26#include <asm/kvm_mmu.h>
27#include <asm/mmu.h>
28
29	.text
30	.pushsection	.hyp.text, "ax"
31
32.macro do_el2_call
33	/*
34	 * Shuffle the parameters before calling the function
35	 * pointed to in x0. Assumes parameters in x[1,2,3].
36	 */
37	str	lr, [sp, #-16]!
38	mov	lr, x0
39	mov	x0, x1
40	mov	x1, x2
41	mov	x2, x3
42	blr	lr
43	ldr	lr, [sp], #16
44.endm
45
46el1_sync:				// Guest trapped into EL2
47
48	mrs	x0, esr_el2
49	lsr	x0, x0, #ESR_ELx_EC_SHIFT
50	cmp	x0, #ESR_ELx_EC_HVC64
51	ccmp	x0, #ESR_ELx_EC_HVC32, #4, ne
52	b.ne	el1_trap
53
54	mrs	x1, vttbr_el2		// If vttbr is valid, the guest
55	cbnz	x1, el1_hvc_guest	// called HVC
56
57	/* Here, we're pretty sure the host called HVC. */
58	ldp	x0, x1, [sp], #16
59
60	/* Check for a stub HVC call */
61	cmp	x0, #HVC_STUB_HCALL_NR
62	b.hs	1f
63
64	/*
65	 * Compute the idmap address of __kvm_handle_stub_hvc and
66	 * jump there. Since we use kimage_voffset, do not use the
67	 * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
68	 * (by loading it from the constant pool).
69	 *
70	 * Preserve x0-x4, which may contain stub parameters.
71	 */
72	ldr	x5, =__kvm_handle_stub_hvc
73	ldr_l	x6, kimage_voffset
74
75	/* x5 = __pa(x5) */
76	sub	x5, x5, x6
77	br	x5
78
791:
80	/*
81	 * Perform the EL2 call
82	 */
83	kern_hyp_va	x0
84	do_el2_call
85
86	eret
87	sb
88
89el1_hvc_guest:
90	/*
91	 * Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
92	 * The workaround has already been applied on the host,
93	 * so let's quickly get back to the guest. We don't bother
94	 * restoring x1, as it can be clobbered anyway.
95	 */
96	ldr	x1, [sp]				// Guest's x0
97	eor	w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
98	cbz	w1, wa_epilogue
99
100	/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
101	eor	w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
102			  ARM_SMCCC_ARCH_WORKAROUND_2)
103	cbnz	w1, el1_trap
104
105#ifdef CONFIG_ARM64_SSBD
106alternative_cb	arm64_enable_wa2_handling
107	b	wa2_end
108alternative_cb_end
109	get_vcpu_ptr	x2, x0
110	ldr	x0, [x2, #VCPU_WORKAROUND_FLAGS]
111
112	// Sanitize the argument and update the guest flags
113	ldr	x1, [sp, #8]			// Guest's x1
114	clz	w1, w1				// Murphy's device:
115	lsr	w1, w1, #5			// w1 = !!w1 without using
116	eor	w1, w1, #1			// the flags...
117	bfi	x0, x1, #VCPU_WORKAROUND_2_FLAG_SHIFT, #1
118	str	x0, [x2, #VCPU_WORKAROUND_FLAGS]
119
120	/* Check that we actually need to perform the call */
121	hyp_ldr_this_cpu x0, arm64_ssbd_callback_required, x2
122	cbz	x0, wa2_end
123
124	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_2
125	smc	#0
126
127	/* Don't leak data from the SMC call */
128	mov	x3, xzr
129wa2_end:
130	mov	x2, xzr
131	mov	x1, xzr
132#endif
133
134wa_epilogue:
135	mov	x0, xzr
136	add	sp, sp, #16
137	eret
138	sb
139
140el1_trap:
141	get_vcpu_ptr	x1, x0
142	mov	x0, #ARM_EXCEPTION_TRAP
143	b	__guest_exit
144
145el1_irq:
146	get_vcpu_ptr	x1, x0
147	mov	x0, #ARM_EXCEPTION_IRQ
148	b	__guest_exit
149
150el1_error:
151	get_vcpu_ptr	x1, x0
152	mov	x0, #ARM_EXCEPTION_EL1_SERROR
153	b	__guest_exit
154
155el2_sync:
156	/* Check for illegal exception return, otherwise panic */
157	mrs	x0, spsr_el2
158
159	/* if this was something else, then panic! */
160	tst	x0, #PSR_IL_BIT
161	b.eq	__hyp_panic
162
163	/* Let's attempt a recovery from the illegal exception return */
164	get_vcpu_ptr	x1, x0
165	mov	x0, #ARM_EXCEPTION_IL
166	b	__guest_exit
167
168
169el2_error:
170	ldp	x0, x1, [sp], #16
171
172	/*
173	 * Only two possibilities:
174	 * 1) Either we come from the exit path, having just unmasked
175	 *    PSTATE.A: change the return code to an EL2 fault, and
176	 *    carry on, as we're already in a sane state to handle it.
177	 * 2) Or we come from anywhere else, and that's a bug: we panic.
178	 *
179	 * For (1), x0 contains the original return code and x1 doesn't
180	 * contain anything meaningful at that stage. We can reuse them
181	 * as temp registers.
182	 * For (2), who cares?
183	 */
184	mrs	x0, elr_el2
185	adr	x1, abort_guest_exit_start
186	cmp	x0, x1
187	adr	x1, abort_guest_exit_end
188	ccmp	x0, x1, #4, ne
189	b.ne	__hyp_panic
190	mov	x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
191	eret
192	sb
193
194ENTRY(__hyp_do_panic)
195	mov	lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
196		      PSR_MODE_EL1h)
197	msr	spsr_el2, lr
198	ldr	lr, =panic
199	msr	elr_el2, lr
200	eret
201	sb
202ENDPROC(__hyp_do_panic)
203
204ENTRY(__hyp_panic)
205	get_host_ctxt x0, x1
206	b	hyp_panic
207ENDPROC(__hyp_panic)
208
209.macro invalid_vector	label, target = __hyp_panic
210	.align	2
211\label:
212	b \target
213ENDPROC(\label)
214.endm
215
216	/* None of these should ever happen */
217	invalid_vector	el2t_sync_invalid
218	invalid_vector	el2t_irq_invalid
219	invalid_vector	el2t_fiq_invalid
220	invalid_vector	el2t_error_invalid
221	invalid_vector	el2h_sync_invalid
222	invalid_vector	el2h_irq_invalid
223	invalid_vector	el2h_fiq_invalid
224	invalid_vector	el1_fiq_invalid
225
226	.ltorg
227
228	.align 11
229
230.macro valid_vect target
231	.align 7
232	stp	x0, x1, [sp, #-16]!
233	b	\target
234.endm
235
236.macro invalid_vect target
237	.align 7
238	b	\target
239	ldp	x0, x1, [sp], #16
240	b	\target
241.endm
242
243ENTRY(__kvm_hyp_vector)
244	invalid_vect	el2t_sync_invalid	// Synchronous EL2t
245	invalid_vect	el2t_irq_invalid	// IRQ EL2t
246	invalid_vect	el2t_fiq_invalid	// FIQ EL2t
247	invalid_vect	el2t_error_invalid	// Error EL2t
248
249	valid_vect	el2_sync		// Synchronous EL2h
250	invalid_vect	el2h_irq_invalid	// IRQ EL2h
251	invalid_vect	el2h_fiq_invalid	// FIQ EL2h
252	valid_vect	el2_error		// Error EL2h
253
254	valid_vect	el1_sync		// Synchronous 64-bit EL1
255	valid_vect	el1_irq			// IRQ 64-bit EL1
256	invalid_vect	el1_fiq_invalid		// FIQ 64-bit EL1
257	valid_vect	el1_error		// Error 64-bit EL1
258
259	valid_vect	el1_sync		// Synchronous 32-bit EL1
260	valid_vect	el1_irq			// IRQ 32-bit EL1
261	invalid_vect	el1_fiq_invalid		// FIQ 32-bit EL1
262	valid_vect	el1_error		// Error 32-bit EL1
263ENDPROC(__kvm_hyp_vector)
264
265#ifdef CONFIG_KVM_INDIRECT_VECTORS
266.macro hyp_ventry
267	.align 7
2681:	.rept 27
269	nop
270	.endr
271/*
272 * The default sequence is to directly branch to the KVM vectors,
273 * using the computed offset. This applies for VHE as well as
274 * !ARM64_HARDEN_EL2_VECTORS.
275 *
276 * For ARM64_HARDEN_EL2_VECTORS configurations, this gets replaced
277 * with:
278 *
279 * stp	x0, x1, [sp, #-16]!
280 * movz	x0, #(addr & 0xffff)
281 * movk	x0, #((addr >> 16) & 0xffff), lsl #16
282 * movk	x0, #((addr >> 32) & 0xffff), lsl #32
283 * br	x0
284 *
285 * Where addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + 4.
286 * See kvm_patch_vector_branch for details.
287 */
288alternative_cb	kvm_patch_vector_branch
289	b	__kvm_hyp_vector + (1b - 0b)
290	nop
291	nop
292	nop
293	nop
294alternative_cb_end
295.endm
296
297.macro generate_vectors
2980:
299	.rept 16
300	hyp_ventry
301	.endr
302	.org 0b + SZ_2K		// Safety measure
303.endm
304
305	.align	11
306ENTRY(__bp_harden_hyp_vecs_start)
307	.rept BP_HARDEN_EL2_SLOTS
308	generate_vectors
309	.endr
310ENTRY(__bp_harden_hyp_vecs_end)
311
312	.popsection
313
314ENTRY(__smccc_workaround_1_smc_start)
315	sub	sp, sp, #(8 * 4)
316	stp	x2, x3, [sp, #(8 * 0)]
317	stp	x0, x1, [sp, #(8 * 2)]
318	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_1
319	smc	#0
320	ldp	x2, x3, [sp, #(8 * 0)]
321	ldp	x0, x1, [sp, #(8 * 2)]
322	add	sp, sp, #(8 * 4)
323ENTRY(__smccc_workaround_1_smc_end)
324#endif
325