1/* SPDX-License-Identifier: GPL-2.0-only */
2#include <asm/asm-offsets.h>
3#include <asm/cache.h>
4#include <asm/code-patching-asm.h>
5#include <asm/exception-64s.h>
6#include <asm/export.h>
7#include <asm/kvm_asm.h>
8#include <asm/kvm_book3s_asm.h>
9#include <asm/mmu.h>
10#include <asm/ppc_asm.h>
11#include <asm/ptrace.h>
12#include <asm/reg.h>
13#include <asm/ultravisor-api.h>
14
15/*
16 * These are branched to from interrupt handlers in exception-64s.S which set
17 * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero.
18 */
19
20/*
21 * This is a hcall, so register convention is as
22 * Documentation/powerpc/papr_hcalls.rst.
23 *
24 * This may also be a syscall from PR-KVM userspace that is to be
25 * reflected to the PR guest kernel, so registers may be set up for
26 * a system call rather than hcall. We don't currently clobber
27 * anything here, but the 0xc00 handler has already clobbered CTR
28 * and CR0, so PR-KVM can not support a guest kernel that preserves
29 * those registers across its system calls.
30 *
31 * The state of registers is as kvmppc_interrupt, except CFAR is not
32 * saved, R13 is not in SCRATCH0, and R10 does not contain the trap.
33 */
34.global	kvmppc_hcall
35.balign IFETCH_ALIGN_BYTES
36kvmppc_hcall:
37#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
38	lbz	r10,HSTATE_IN_GUEST(r13)
39	cmpwi	r10,KVM_GUEST_MODE_HV_FAST
40	beq	kvmppc_p9_exit_hcall
41#endif
42	ld	r10,PACA_EXGEN+EX_R13(r13)
43	SET_SCRATCH0(r10)
44	li	r10,0xc00
45	/* Now we look like kvmppc_interrupt */
46	li	r11,PACA_EXGEN
47	b	.Lgot_save_area
48
49/*
50 * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that
51 * call convention:
52 *
53 * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area
54 * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts
55 * guest R13 also saved in SCRATCH0
56 * R13		= PACA
57 * R11		= (H)SRR0
58 * R12		= (H)SRR1
59 * R9		= guest CR
60 * PPR is set to medium
61 *
62 * With the addition for KVM:
63 * R10		= trap vector
64 */
65.global	kvmppc_interrupt
66.balign IFETCH_ALIGN_BYTES
67kvmppc_interrupt:
68#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
69	std	r10,HSTATE_SCRATCH0(r13)
70	lbz	r10,HSTATE_IN_GUEST(r13)
71	cmpwi	r10,KVM_GUEST_MODE_HV_FAST
72	beq	kvmppc_p9_exit_interrupt
73	ld	r10,HSTATE_SCRATCH0(r13)
74#endif
75	li	r11,PACA_EXGEN
76	cmpdi	r10,0x200
77	bgt+	.Lgot_save_area
78	li	r11,PACA_EXMC
79	beq	.Lgot_save_area
80	li	r11,PACA_EXNMI
81.Lgot_save_area:
82	add	r11,r11,r13
83BEGIN_FTR_SECTION
84	ld	r12,EX_CFAR(r11)
85	std	r12,HSTATE_CFAR(r13)
86END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
87	ld	r12,EX_CTR(r11)
88	mtctr	r12
89BEGIN_FTR_SECTION
90	ld	r12,EX_PPR(r11)
91	std	r12,HSTATE_PPR(r13)
92END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
93	ld	r12,EX_R12(r11)
94	std	r12,HSTATE_SCRATCH0(r13)
95	sldi	r12,r9,32
96	or	r12,r12,r10
97	ld	r9,EX_R9(r11)
98	ld	r10,EX_R10(r11)
99	ld	r11,EX_R11(r11)
100
101	/*
102	 * Hcalls and other interrupts come here after normalising register
103	 * contents and save locations:
104	 *
105	 * R12		= (guest CR << 32) | interrupt vector
106	 * R13		= PACA
107	 * guest R12 saved in shadow HSTATE_SCRATCH0
108	 * guest R13 saved in SPRN_SCRATCH0
109	 */
110	std	r9,HSTATE_SCRATCH2(r13)
111	lbz	r9,HSTATE_IN_GUEST(r13)
112	cmpwi	r9,KVM_GUEST_MODE_SKIP
113	beq-	.Lmaybe_skip
114.Lno_skip:
115#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
116#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
117	cmpwi	r9,KVM_GUEST_MODE_GUEST
118	beq	kvmppc_interrupt_pr
119#endif
120	b	kvmppc_interrupt_hv
121#else
122	b	kvmppc_interrupt_pr
123#endif
124
125/*
126 * "Skip" interrupts are part of a trick KVM uses a with hash guests to load
127 * the faulting instruction in guest memory from the the hypervisor without
128 * walking page tables.
129 *
130 * When the guest takes a fault that requires the hypervisor to load the
131 * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
132 * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
133 * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
134 * but loads and stores will access the guest context. This is used to load
135 * the faulting instruction using the faulting guest effective address.
136 *
137 * However the guest context may not be able to translate, or it may cause a
138 * machine check or other issue, which results in a fault in the host
139 * (even with KVM-HV).
140 *
141 * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
142 * are (or are likely) caused by that load, the instruction is skipped by
143 * just returning with the PC advanced +4, where it is noticed the load did
144 * not execute and it goes to the slow path which walks the page tables to
145 * read guest memory.
146 */
147.Lmaybe_skip:
148	cmpwi	r12,BOOK3S_INTERRUPT_MACHINE_CHECK
149	beq	1f
150	cmpwi	r12,BOOK3S_INTERRUPT_DATA_STORAGE
151	beq	1f
152	cmpwi	r12,BOOK3S_INTERRUPT_DATA_SEGMENT
153	beq	1f
154#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
155	/* HSRR interrupts get 2 added to interrupt number */
156	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
157	beq	2f
158#endif
159	b	.Lno_skip
1601:	mfspr	r9,SPRN_SRR0
161	addi	r9,r9,4
162	mtspr	SPRN_SRR0,r9
163	ld	r12,HSTATE_SCRATCH0(r13)
164	ld	r9,HSTATE_SCRATCH2(r13)
165	GET_SCRATCH0(r13)
166	RFI_TO_KERNEL
167#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1682:	mfspr	r9,SPRN_HSRR0
169	addi	r9,r9,4
170	mtspr	SPRN_HSRR0,r9
171	ld	r12,HSTATE_SCRATCH0(r13)
172	ld	r9,HSTATE_SCRATCH2(r13)
173	GET_SCRATCH0(r13)
174	HRFI_TO_KERNEL
175#endif
176
177#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
178
179/* Stack frame offsets for kvmppc_p9_enter_guest */
180#define SFS			(144 + STACK_FRAME_MIN_SIZE)
181#define STACK_SLOT_NVGPRS	(SFS - 144)	/* 18 gprs */
182
183/*
184 * void kvmppc_p9_enter_guest(struct vcpu *vcpu);
185 *
186 * Enter the guest on a ISAv3.0 or later system where we have exactly
187 * one vcpu per vcore, and both the host and guest are radix, and threads
188 * are set to "indepdent mode".
189 */
190.balign	IFETCH_ALIGN_BYTES
191_GLOBAL(kvmppc_p9_enter_guest)
192EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest)
193	mflr	r0
194	std	r0,PPC_LR_STKOFF(r1)
195	stdu	r1,-SFS(r1)
196
197	std	r1,HSTATE_HOST_R1(r13)
198
199	mfcr	r4
200	stw	r4,SFS+8(r1)
201
202	reg = 14
203	.rept	18
204	std	reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
205	reg = reg + 1
206	.endr
207
208	ld	r4,VCPU_LR(r3)
209	mtlr	r4
210	ld	r4,VCPU_CTR(r3)
211	mtctr	r4
212	ld	r4,VCPU_XER(r3)
213	mtspr	SPRN_XER,r4
214
215	ld	r1,VCPU_CR(r3)
216
217BEGIN_FTR_SECTION
218	ld	r4,VCPU_CFAR(r3)
219	mtspr	SPRN_CFAR,r4
220END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
221BEGIN_FTR_SECTION
222	ld	r4,VCPU_PPR(r3)
223	mtspr	SPRN_PPR,r4
224END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
225
226	reg = 4
227	.rept	28
228	ld	reg,__VCPU_GPR(reg)(r3)
229	reg = reg + 1
230	.endr
231
232	ld	r4,VCPU_KVM(r3)
233	lbz	r4,KVM_SECURE_GUEST(r4)
234	cmpdi	r4,0
235	ld	r4,VCPU_GPR(R4)(r3)
236	bne	.Lret_to_ultra
237
238	mtcr	r1
239
240	ld	r0,VCPU_GPR(R0)(r3)
241	ld	r1,VCPU_GPR(R1)(r3)
242	ld	r2,VCPU_GPR(R2)(r3)
243	ld	r3,VCPU_GPR(R3)(r3)
244
245	HRFI_TO_GUEST
246	b	.
247
248	/*
249	 * Use UV_RETURN ultracall to return control back to the Ultravisor
250	 * after processing an hypercall or interrupt that was forwarded
251	 * (a.k.a. reflected) to the Hypervisor.
252	 *
253	 * All registers have already been reloaded except the ucall requires:
254	 *   R0 = hcall result
255	 *   R2 = SRR1, so UV can detect a synthesized interrupt (if any)
256	 *   R3 = UV_RETURN
257	 */
258.Lret_to_ultra:
259	mtcr	r1
260	ld	r1,VCPU_GPR(R1)(r3)
261
262	ld	r0,VCPU_GPR(R3)(r3)
263	mfspr	r2,SPRN_SRR1
264	LOAD_REG_IMMEDIATE(r3, UV_RETURN)
265	sc	2
266
267/*
268 * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from
269 * above if the interrupt was taken for a guest that was entered via
270 * kvmppc_p9_enter_guest().
271 *
272 * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs
273 * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re-
274 * establishes the host stack and registers to return from the
275 * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers
276 * (SPRs and FP, VEC, etc).
277 */
278.balign	IFETCH_ALIGN_BYTES
279kvmppc_p9_exit_hcall:
280	mfspr	r11,SPRN_SRR0
281	mfspr	r12,SPRN_SRR1
282	li	r10,0xc00
283	std	r10,HSTATE_SCRATCH0(r13)
284
285.balign	IFETCH_ALIGN_BYTES
286kvmppc_p9_exit_interrupt:
287	/*
288	 * If set to KVM_GUEST_MODE_HV_FAST but we're still in the
289	 * hypervisor, that means we can't return from the entry stack.
290	 */
291	rldicl. r10,r12,64-MSR_HV_LG,63
292	bne-	kvmppc_p9_bad_interrupt
293
294	std     r1,HSTATE_SCRATCH1(r13)
295	std     r3,HSTATE_SCRATCH2(r13)
296	ld	r1,HSTATE_HOST_R1(r13)
297	ld	r3,HSTATE_KVM_VCPU(r13)
298
299	std	r9,VCPU_CR(r3)
300
3011:
302	std	r11,VCPU_PC(r3)
303	std	r12,VCPU_MSR(r3)
304
305	reg = 14
306	.rept	18
307	std	reg,__VCPU_GPR(reg)(r3)
308	reg = reg + 1
309	.endr
310
311	/* r1, r3, r9-r13 are saved to vcpu by C code */
312	std	r0,VCPU_GPR(R0)(r3)
313	std	r2,VCPU_GPR(R2)(r3)
314	reg = 4
315	.rept	5
316	std	reg,__VCPU_GPR(reg)(r3)
317	reg = reg + 1
318	.endr
319
320	ld	r2,PACATOC(r13)
321
322	mflr	r4
323	std	r4,VCPU_LR(r3)
324	mfspr	r4,SPRN_XER
325	std	r4,VCPU_XER(r3)
326
327	reg = 14
328	.rept	18
329	ld	reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
330	reg = reg + 1
331	.endr
332
333	lwz	r4,SFS+8(r1)
334	mtcr	r4
335
336	/*
337	 * Flush the link stack here, before executing the first blr on the
338	 * way out of the guest.
339	 *
340	 * The link stack won't match coming out of the guest anyway so the
341	 * only cost is the flush itself. The call clobbers r0.
342	 */
3431:	nop
344	patch_site 1b patch__call_kvm_flush_link_stack_p9
345
346	addi	r1,r1,SFS
347	ld	r0,PPC_LR_STKOFF(r1)
348	mtlr	r0
349	blr
350
351/*
352 * Took an interrupt somewhere right before HRFID to guest, so registers are
353 * in a bad way. Return things hopefully enough to run host virtual code and
354 * run the Linux interrupt handler (SRESET or MCE) to print something useful.
355 *
356 * We could be really clever and save all host registers in known locations
357 * before setting HSTATE_IN_GUEST, then restoring them all here, and setting
358 * return address to a fixup that sets them up again. But that's a lot of
359 * effort for a small bit of code. Lots of other things to do first.
360 */
361kvmppc_p9_bad_interrupt:
362	/*
363	 * Clean up guest registers to give host a chance to run.
364	 */
365	li	r10,0
366	mtspr	SPRN_AMR,r10
367	mtspr	SPRN_IAMR,r10
368	mtspr	SPRN_CIABR,r10
369	mtspr	SPRN_DAWRX0,r10
370BEGIN_FTR_SECTION
371	mtspr	SPRN_DAWRX1,r10
372END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
373	mtspr	SPRN_PID,r10
374
375	/*
376	 * Switch to host MMU mode
377	 */
378	ld	r10, HSTATE_KVM_VCPU(r13)
379	ld	r10, VCPU_KVM(r10)
380	lwz	r10, KVM_HOST_LPID(r10)
381	mtspr	SPRN_LPID,r10
382
383	ld	r10, HSTATE_KVM_VCPU(r13)
384	ld	r10, VCPU_KVM(r10)
385	ld	r10, KVM_HOST_LPCR(r10)
386	mtspr	SPRN_LPCR,r10
387
388	/*
389	 * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear
390	 * MSR_RI in r12 ([H]SRR1) so the handler won't try to return.
391	 */
392	li	r10,KVM_GUEST_MODE_NONE
393	stb	r10,HSTATE_IN_GUEST(r13)
394	li	r10,MSR_RI
395	andc	r12,r12,r10
396
397	/*
398	 * Go back to interrupt handler. MCE and SRESET have their specific
399	 * PACA save area so they should be used directly. They set up their
400	 * own stack. The other handlers all use EXGEN. They will use the
401	 * guest r1 if it looks like a kernel stack, so just load the
402	 * emergency stack and go to program check for all other interrupts.
403	 */
404	ld	r10,HSTATE_SCRATCH0(r13)
405	cmpwi	r10,BOOK3S_INTERRUPT_MACHINE_CHECK
406	beq	machine_check_common
407
408	cmpwi	r10,BOOK3S_INTERRUPT_SYSTEM_RESET
409	beq	system_reset_common
410
411	b	.
412#endif
413