1/* SPDX-License-Identifier: GPL-2.0-only */
2#include <asm/asm-offsets.h>
3#include <asm/cache.h>
4#include <asm/code-patching-asm.h>
5#include <asm/exception-64s.h>
6#include <asm/export.h>
7#include <asm/kvm_asm.h>
8#include <asm/kvm_book3s_asm.h>
9#include <asm/mmu.h>
10#include <asm/ppc_asm.h>
11#include <asm/ptrace.h>
12#include <asm/reg.h>
13#include <asm/ultravisor-api.h>
14
15/*
16 * These are branched to from interrupt handlers in exception-64s.S which set
17 * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero.
18 */
19
20/*
21 * This is a hcall, so register convention is as
22 * Documentation/powerpc/papr_hcalls.rst.
23 *
24 * This may also be a syscall from PR-KVM userspace that is to be
25 * reflected to the PR guest kernel, so registers may be set up for
26 * a system call rather than hcall. We don't currently clobber
27 * anything here, but the 0xc00 handler has already clobbered CTR
28 * and CR0, so PR-KVM can not support a guest kernel that preserves
29 * those registers across its system calls.
30 *
31 * The state of registers is as kvmppc_interrupt, except CFAR is not
32 * saved, R13 is not in SCRATCH0, and R10 does not contain the trap.
33 */
34.global	kvmppc_hcall
35.balign IFETCH_ALIGN_BYTES
36kvmppc_hcall:
37#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
38	lbz	r10,HSTATE_IN_GUEST(r13)
39	cmpwi	r10,KVM_GUEST_MODE_HV_P9
40	beq	kvmppc_p9_exit_hcall
41#endif
42	ld	r10,PACA_EXGEN+EX_R13(r13)
43	SET_SCRATCH0(r10)
44	li	r10,0xc00
45	/* Now we look like kvmppc_interrupt */
46	li	r11,PACA_EXGEN
47	b	.Lgot_save_area
48
49/*
50 * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that
51 * call convention:
52 *
53 * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area
54 * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts
55 * guest R13 also saved in SCRATCH0
56 * R13		= PACA
57 * R11		= (H)SRR0
58 * R12		= (H)SRR1
59 * R9		= guest CR
60 * PPR is set to medium
61 *
62 * With the addition for KVM:
63 * R10		= trap vector
64 */
65.global	kvmppc_interrupt
66.balign IFETCH_ALIGN_BYTES
67kvmppc_interrupt:
68#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
69	std	r10,HSTATE_SCRATCH0(r13)
70	lbz	r10,HSTATE_IN_GUEST(r13)
71	cmpwi	r10,KVM_GUEST_MODE_HV_P9
72	beq	kvmppc_p9_exit_interrupt
73	ld	r10,HSTATE_SCRATCH0(r13)
74#endif
75	li	r11,PACA_EXGEN
76	cmpdi	r10,0x200
77	bgt+	.Lgot_save_area
78	li	r11,PACA_EXMC
79	beq	.Lgot_save_area
80	li	r11,PACA_EXNMI
81.Lgot_save_area:
82	add	r11,r11,r13
83BEGIN_FTR_SECTION
84	ld	r12,EX_CFAR(r11)
85	std	r12,HSTATE_CFAR(r13)
86END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
87	ld	r12,EX_CTR(r11)
88	mtctr	r12
89BEGIN_FTR_SECTION
90	ld	r12,EX_PPR(r11)
91	std	r12,HSTATE_PPR(r13)
92END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
93	ld	r12,EX_R12(r11)
94	std	r12,HSTATE_SCRATCH0(r13)
95	sldi	r12,r9,32
96	or	r12,r12,r10
97	ld	r9,EX_R9(r11)
98	ld	r10,EX_R10(r11)
99	ld	r11,EX_R11(r11)
100
101	/*
102	 * Hcalls and other interrupts come here after normalising register
103	 * contents and save locations:
104	 *
105	 * R12		= (guest CR << 32) | interrupt vector
106	 * R13		= PACA
107	 * guest R12 saved in shadow HSTATE_SCRATCH0
108	 * guest R13 saved in SPRN_SCRATCH0
109	 */
110	std	r9,HSTATE_SCRATCH2(r13)
111	lbz	r9,HSTATE_IN_GUEST(r13)
112	cmpwi	r9,KVM_GUEST_MODE_SKIP
113	beq-	.Lmaybe_skip
114.Lno_skip:
115#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
116#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
117	cmpwi	r9,KVM_GUEST_MODE_GUEST
118	beq	kvmppc_interrupt_pr
119#endif
120	b	kvmppc_interrupt_hv
121#else
122	b	kvmppc_interrupt_pr
123#endif
124
125/*
126 * "Skip" interrupts are part of a trick KVM uses a with hash guests to load
127 * the faulting instruction in guest memory from the the hypervisor without
128 * walking page tables.
129 *
130 * When the guest takes a fault that requires the hypervisor to load the
131 * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
132 * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
133 * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
134 * but loads and stores will access the guest context. This is used to load
135 * the faulting instruction using the faulting guest effective address.
136 *
137 * However the guest context may not be able to translate, or it may cause a
138 * machine check or other issue, which results in a fault in the host
139 * (even with KVM-HV).
140 *
141 * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
142 * are (or are likely) caused by that load, the instruction is skipped by
143 * just returning with the PC advanced +4, where it is noticed the load did
144 * not execute and it goes to the slow path which walks the page tables to
145 * read guest memory.
146 */
147.Lmaybe_skip:
148	cmpwi	r12,BOOK3S_INTERRUPT_MACHINE_CHECK
149	beq	1f
150	cmpwi	r12,BOOK3S_INTERRUPT_DATA_STORAGE
151	beq	1f
152	cmpwi	r12,BOOK3S_INTERRUPT_DATA_SEGMENT
153	beq	1f
154#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
155	/* HSRR interrupts get 2 added to interrupt number */
156	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
157	beq	2f
158#endif
159	b	.Lno_skip
1601:	mfspr	r9,SPRN_SRR0
161	addi	r9,r9,4
162	mtspr	SPRN_SRR0,r9
163	ld	r12,HSTATE_SCRATCH0(r13)
164	ld	r9,HSTATE_SCRATCH2(r13)
165	GET_SCRATCH0(r13)
166	RFI_TO_KERNEL
167#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1682:	mfspr	r9,SPRN_HSRR0
169	addi	r9,r9,4
170	mtspr	SPRN_HSRR0,r9
171	ld	r12,HSTATE_SCRATCH0(r13)
172	ld	r9,HSTATE_SCRATCH2(r13)
173	GET_SCRATCH0(r13)
174	HRFI_TO_KERNEL
175#endif
176
177#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
178
179/* Stack frame offsets for kvmppc_p9_enter_guest */
180#define SFS			(144 + STACK_FRAME_MIN_SIZE)
181#define STACK_SLOT_NVGPRS	(SFS - 144)	/* 18 gprs */
182
183/*
184 * void kvmppc_p9_enter_guest(struct vcpu *vcpu);
185 *
186 * Enter the guest on a ISAv3.0 or later system.
187 */
188.balign	IFETCH_ALIGN_BYTES
189_GLOBAL(kvmppc_p9_enter_guest)
190EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest)
191	mflr	r0
192	std	r0,PPC_LR_STKOFF(r1)
193	stdu	r1,-SFS(r1)
194
195	std	r1,HSTATE_HOST_R1(r13)
196
197	mfcr	r4
198	stw	r4,SFS+8(r1)
199
200	reg = 14
201	.rept	18
202	std	reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
203	reg = reg + 1
204	.endr
205
206	ld	r4,VCPU_LR(r3)
207	mtlr	r4
208	ld	r4,VCPU_CTR(r3)
209	mtctr	r4
210	ld	r4,VCPU_XER(r3)
211	mtspr	SPRN_XER,r4
212
213	ld	r1,VCPU_CR(r3)
214
215BEGIN_FTR_SECTION
216	ld	r4,VCPU_CFAR(r3)
217	mtspr	SPRN_CFAR,r4
218END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
219BEGIN_FTR_SECTION
220	ld	r4,VCPU_PPR(r3)
221	mtspr	SPRN_PPR,r4
222END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
223
224	reg = 4
225	.rept	28
226	ld	reg,__VCPU_GPR(reg)(r3)
227	reg = reg + 1
228	.endr
229
230	ld	r4,VCPU_KVM(r3)
231	lbz	r4,KVM_SECURE_GUEST(r4)
232	cmpdi	r4,0
233	ld	r4,VCPU_GPR(R4)(r3)
234	bne	.Lret_to_ultra
235
236	mtcr	r1
237
238	ld	r0,VCPU_GPR(R0)(r3)
239	ld	r1,VCPU_GPR(R1)(r3)
240	ld	r2,VCPU_GPR(R2)(r3)
241	ld	r3,VCPU_GPR(R3)(r3)
242
243	HRFI_TO_GUEST
244	b	.
245
246	/*
247	 * Use UV_RETURN ultracall to return control back to the Ultravisor
248	 * after processing an hypercall or interrupt that was forwarded
249	 * (a.k.a. reflected) to the Hypervisor.
250	 *
251	 * All registers have already been reloaded except the ucall requires:
252	 *   R0 = hcall result
253	 *   R2 = SRR1, so UV can detect a synthesized interrupt (if any)
254	 *   R3 = UV_RETURN
255	 */
256.Lret_to_ultra:
257	mtcr	r1
258	ld	r1,VCPU_GPR(R1)(r3)
259
260	ld	r0,VCPU_GPR(R3)(r3)
261	mfspr	r2,SPRN_SRR1
262	LOAD_REG_IMMEDIATE(r3, UV_RETURN)
263	sc	2
264
265/*
266 * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from
267 * above if the interrupt was taken for a guest that was entered via
268 * kvmppc_p9_enter_guest().
269 *
270 * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs
271 * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re-
272 * establishes the host stack and registers to return from the
273 * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers
274 * (SPRs and FP, VEC, etc).
275 */
276.balign	IFETCH_ALIGN_BYTES
277kvmppc_p9_exit_hcall:
278	mfspr	r11,SPRN_SRR0
279	mfspr	r12,SPRN_SRR1
280	li	r10,0xc00
281	std	r10,HSTATE_SCRATCH0(r13)
282
283.balign	IFETCH_ALIGN_BYTES
284kvmppc_p9_exit_interrupt:
285	/*
286	 * If set to KVM_GUEST_MODE_HV_P9 but we're still in the
287	 * hypervisor, that means we can't return from the entry stack.
288	 */
289	rldicl. r10,r12,64-MSR_HV_LG,63
290	bne-	kvmppc_p9_bad_interrupt
291
292	std     r1,HSTATE_SCRATCH1(r13)
293	std     r3,HSTATE_SCRATCH2(r13)
294	ld	r1,HSTATE_HOST_R1(r13)
295	ld	r3,HSTATE_KVM_VCPU(r13)
296
297	std	r9,VCPU_CR(r3)
298
2991:
300	std	r11,VCPU_PC(r3)
301	std	r12,VCPU_MSR(r3)
302
303	reg = 14
304	.rept	18
305	std	reg,__VCPU_GPR(reg)(r3)
306	reg = reg + 1
307	.endr
308
309	/* r1, r3, r9-r13 are saved to vcpu by C code */
310	std	r0,VCPU_GPR(R0)(r3)
311	std	r2,VCPU_GPR(R2)(r3)
312	reg = 4
313	.rept	5
314	std	reg,__VCPU_GPR(reg)(r3)
315	reg = reg + 1
316	.endr
317
318	ld	r2,PACATOC(r13)
319
320	mflr	r4
321	std	r4,VCPU_LR(r3)
322	mfspr	r4,SPRN_XER
323	std	r4,VCPU_XER(r3)
324
325	reg = 14
326	.rept	18
327	ld	reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
328	reg = reg + 1
329	.endr
330
331	lwz	r4,SFS+8(r1)
332	mtcr	r4
333
334	/*
335	 * Flush the link stack here, before executing the first blr on the
336	 * way out of the guest.
337	 *
338	 * The link stack won't match coming out of the guest anyway so the
339	 * only cost is the flush itself. The call clobbers r0.
340	 */
3411:	nop
342	patch_site 1b patch__call_kvm_flush_link_stack_p9
343
344	addi	r1,r1,SFS
345	ld	r0,PPC_LR_STKOFF(r1)
346	mtlr	r0
347	blr
348
349/*
350 * Took an interrupt somewhere right before HRFID to guest, so registers are
351 * in a bad way. Return things hopefully enough to run host virtual code and
352 * run the Linux interrupt handler (SRESET or MCE) to print something useful.
353 *
354 * We could be really clever and save all host registers in known locations
355 * before setting HSTATE_IN_GUEST, then restoring them all here, and setting
356 * return address to a fixup that sets them up again. But that's a lot of
357 * effort for a small bit of code. Lots of other things to do first.
358 */
359kvmppc_p9_bad_interrupt:
360BEGIN_MMU_FTR_SECTION
361	/*
362	 * Hash host doesn't try to recover MMU (requires host SLB reload)
363	 */
364	b	.
365END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
366	/*
367	 * Clean up guest registers to give host a chance to run.
368	 */
369	li	r10,0
370	mtspr	SPRN_AMR,r10
371	mtspr	SPRN_IAMR,r10
372	mtspr	SPRN_CIABR,r10
373	mtspr	SPRN_DAWRX0,r10
374BEGIN_FTR_SECTION
375	mtspr	SPRN_DAWRX1,r10
376END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
377
378	/*
379	 * Switch to host MMU mode (don't have the real host PID but we aren't
380	 * going back to userspace).
381	 */
382	hwsync
383	isync
384
385	mtspr	SPRN_PID,r10
386
387	ld	r10, HSTATE_KVM_VCPU(r13)
388	ld	r10, VCPU_KVM(r10)
389	lwz	r10, KVM_HOST_LPID(r10)
390	mtspr	SPRN_LPID,r10
391
392	ld	r10, HSTATE_KVM_VCPU(r13)
393	ld	r10, VCPU_KVM(r10)
394	ld	r10, KVM_HOST_LPCR(r10)
395	mtspr	SPRN_LPCR,r10
396
397	isync
398
399	/*
400	 * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear
401	 * MSR_RI in r12 ([H]SRR1) so the handler won't try to return.
402	 */
403	li	r10,KVM_GUEST_MODE_NONE
404	stb	r10,HSTATE_IN_GUEST(r13)
405	li	r10,MSR_RI
406	andc	r12,r12,r10
407
408	/*
409	 * Go back to interrupt handler. MCE and SRESET have their specific
410	 * PACA save area so they should be used directly. They set up their
411	 * own stack. The other handlers all use EXGEN. They will use the
412	 * guest r1 if it looks like a kernel stack, so just load the
413	 * emergency stack and go to program check for all other interrupts.
414	 */
415	ld	r10,HSTATE_SCRATCH0(r13)
416	cmpwi	r10,BOOK3S_INTERRUPT_MACHINE_CHECK
417	beq	machine_check_common
418
419	cmpwi	r10,BOOK3S_INTERRUPT_SYSTEM_RESET
420	beq	system_reset_common
421
422	b	.
423#endif
424