1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
14 *
15 * Copyright SUSE Linux Products GmbH 2010
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20/* Real mode helpers */
21
22#if defined(CONFIG_PPC_BOOK3S_64)
23
24#define GET_SHADOW_VCPU(reg)    \
25	mr	reg, r13
26
27#elif defined(CONFIG_PPC_BOOK3S_32)
28
29#define GET_SHADOW_VCPU(reg)    			\
30	tophys(reg, r2);       			\
31	lwz     reg, (THREAD + THREAD_KVM_SVCPU)(reg);	\
32	tophys(reg, reg)
33
34#endif
35
36/* Disable for nested KVM */
37#define USE_QUICK_LAST_INST
38
39
40/* Get helper functions for subarch specific functionality */
41
42#if defined(CONFIG_PPC_BOOK3S_64)
43#include "book3s_64_slb.S"
44#elif defined(CONFIG_PPC_BOOK3S_32)
45#include "book3s_32_sr.S"
46#endif
47
48/******************************************************************************
49 *                                                                            *
50 *                               Entry code                                   *
51 *                                                                            *
52 *****************************************************************************/
53
54.global kvmppc_handler_trampoline_enter
55kvmppc_handler_trampoline_enter:
56
57	/* Required state:
58	 *
59	 * MSR = ~IR|DR
60	 * R1 = host R1
61	 * R2 = host R2
62	 * R4 = guest shadow MSR
63	 * R5 = normal host MSR
64	 * R6 = current host MSR (EE, IR, DR off)
65	 * LR = highmem guest exit code
66	 * all other volatile GPRS = free
67	 * SVCPU[CR] = guest CR
68	 * SVCPU[XER] = guest XER
69	 * SVCPU[CTR] = guest CTR
70	 * SVCPU[LR] = guest LR
71	 */
72
73	/* r3 = shadow vcpu */
74	GET_SHADOW_VCPU(r3)
75
76	/* Save guest exit handler address and MSR */
77	mflr	r0
78	PPC_STL	r0, HSTATE_VMHANDLER(r3)
79	PPC_STL	r5, HSTATE_HOST_MSR(r3)
80
81	/* Save R1/R2 in the PACA (64-bit) or shadow_vcpu (32-bit) */
82	PPC_STL	r1, HSTATE_HOST_R1(r3)
83	PPC_STL	r2, HSTATE_HOST_R2(r3)
84
85	/* Activate guest mode, so faults get handled by KVM */
86	li	r11, KVM_GUEST_MODE_GUEST
87	stb	r11, HSTATE_IN_GUEST(r3)
88
89	/* Switch to guest segment. This is subarch specific. */
90	LOAD_GUEST_SEGMENTS
91
92#ifdef CONFIG_PPC_BOOK3S_64
93BEGIN_FTR_SECTION
94	/* Save host FSCR */
95	mfspr	r8, SPRN_FSCR
96	std	r8, HSTATE_HOST_FSCR(r13)
97	/* Set FSCR during guest execution */
98	ld	r9, SVCPU_SHADOW_FSCR(r13)
99	mtspr	SPRN_FSCR, r9
100END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
101
102	/* Some guests may need to have dcbz set to 32 byte length.
103	 *
104	 * Usually we ensure that by patching the guest's instructions
105	 * to trap on dcbz and emulate it in the hypervisor.
106	 *
107	 * If we can, we should tell the CPU to use 32 byte dcbz though,
108	 * because that's a lot faster.
109	 */
110	lbz	r0, HSTATE_RESTORE_HID5(r3)
111	cmpwi	r0, 0
112	beq	no_dcbz32_on
113
114	mfspr   r0,SPRN_HID5
115	ori     r0, r0, 0x80		/* XXX HID5_dcbz32 = 0x80 */
116	mtspr   SPRN_HID5,r0
117no_dcbz32_on:
118
119#endif /* CONFIG_PPC_BOOK3S_64 */
120
121	/* Enter guest */
122
123	PPC_LL	r8, SVCPU_CTR(r3)
124	PPC_LL	r9, SVCPU_LR(r3)
125	lwz	r10, SVCPU_CR(r3)
126	PPC_LL	r11, SVCPU_XER(r3)
127
128	mtctr	r8
129	mtlr	r9
130	mtcr	r10
131	mtxer	r11
132
133	/* Move SRR0 and SRR1 into the respective regs */
134	PPC_LL  r9, SVCPU_PC(r3)
135	/* First clear RI in our current MSR value */
136	li	r0, MSR_RI
137	andc	r6, r6, r0
138
139	PPC_LL	r0, SVCPU_R0(r3)
140	PPC_LL	r1, SVCPU_R1(r3)
141	PPC_LL	r2, SVCPU_R2(r3)
142	PPC_LL	r5, SVCPU_R5(r3)
143	PPC_LL	r7, SVCPU_R7(r3)
144	PPC_LL	r8, SVCPU_R8(r3)
145	PPC_LL	r10, SVCPU_R10(r3)
146	PPC_LL	r11, SVCPU_R11(r3)
147	PPC_LL	r12, SVCPU_R12(r3)
148	PPC_LL	r13, SVCPU_R13(r3)
149
150	MTMSR_EERI(r6)
151	mtsrr0	r9
152	mtsrr1	r4
153
154	PPC_LL	r4, SVCPU_R4(r3)
155	PPC_LL	r6, SVCPU_R6(r3)
156	PPC_LL	r9, SVCPU_R9(r3)
157	PPC_LL	r3, (SVCPU_R3)(r3)
158
159	RFI_TO_GUEST
160kvmppc_handler_trampoline_enter_end:
161
162
163
164/******************************************************************************
165 *                                                                            *
166 *                               Exit code                                    *
167 *                                                                            *
168 *****************************************************************************/
169
170.global kvmppc_interrupt_pr
171kvmppc_interrupt_pr:
172	/* 64-bit entry. Register usage at this point:
173	 *
174	 * SPRG_SCRATCH0   = guest R13
175	 * R12             = (guest CR << 32) | exit handler id
176	 * R13             = PACA
177	 * HSTATE.SCRATCH0 = guest R12
178	 * HSTATE.SCRATCH1 = guest CTR if RELOCATABLE
179	 */
180#ifdef CONFIG_PPC64
181	/* Match 32-bit entry */
182#ifdef CONFIG_RELOCATABLE
183	std	r9, HSTATE_SCRATCH2(r13)
184	ld	r9, HSTATE_SCRATCH1(r13)
185	mtctr	r9
186	ld	r9, HSTATE_SCRATCH2(r13)
187#endif
188	rotldi	r12, r12, 32		  /* Flip R12 halves for stw */
189	stw	r12, HSTATE_SCRATCH1(r13) /* CR is now in the low half */
190	srdi	r12, r12, 32		  /* shift trap into low half */
191#endif
192
193.global kvmppc_handler_trampoline_exit
194kvmppc_handler_trampoline_exit:
195	/* Register usage at this point:
196	 *
197	 * SPRG_SCRATCH0   = guest R13
198	 * R12             = exit handler id
199	 * R13             = shadow vcpu (32-bit) or PACA (64-bit)
200	 * HSTATE.SCRATCH0 = guest R12
201	 * HSTATE.SCRATCH1 = guest CR
202	 */
203
204	/* Save registers */
205
206	PPC_STL	r0, SVCPU_R0(r13)
207	PPC_STL	r1, SVCPU_R1(r13)
208	PPC_STL	r2, SVCPU_R2(r13)
209	PPC_STL	r3, SVCPU_R3(r13)
210	PPC_STL	r4, SVCPU_R4(r13)
211	PPC_STL	r5, SVCPU_R5(r13)
212	PPC_STL	r6, SVCPU_R6(r13)
213	PPC_STL	r7, SVCPU_R7(r13)
214	PPC_STL	r8, SVCPU_R8(r13)
215	PPC_STL	r9, SVCPU_R9(r13)
216	PPC_STL	r10, SVCPU_R10(r13)
217	PPC_STL	r11, SVCPU_R11(r13)
218
219	/* Restore R1/R2 so we can handle faults */
220	PPC_LL	r1, HSTATE_HOST_R1(r13)
221	PPC_LL	r2, HSTATE_HOST_R2(r13)
222
223	/* Save guest PC and MSR */
224#ifdef CONFIG_PPC64
225BEGIN_FTR_SECTION
226	andi.	r0, r12, 0x2
227	cmpwi	cr1, r0, 0
228	beq	1f
229	mfspr	r3,SPRN_HSRR0
230	mfspr	r4,SPRN_HSRR1
231	andi.	r12,r12,0x3ffd
232	b	2f
233END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
234#endif
2351:	mfsrr0	r3
236	mfsrr1	r4
2372:
238	PPC_STL	r3, SVCPU_PC(r13)
239	PPC_STL	r4, SVCPU_SHADOW_SRR1(r13)
240
241	/* Get scratch'ed off registers */
242	GET_SCRATCH0(r9)
243	PPC_LL	r8, HSTATE_SCRATCH0(r13)
244	lwz	r7, HSTATE_SCRATCH1(r13)
245
246	PPC_STL	r9, SVCPU_R13(r13)
247	PPC_STL	r8, SVCPU_R12(r13)
248	stw	r7, SVCPU_CR(r13)
249
250	/* Save more register state  */
251
252	mfxer	r5
253	mfdar	r6
254	mfdsisr	r7
255	mfctr	r8
256	mflr	r9
257
258	PPC_STL	r5, SVCPU_XER(r13)
259	PPC_STL	r6, SVCPU_FAULT_DAR(r13)
260	stw	r7, SVCPU_FAULT_DSISR(r13)
261	PPC_STL	r8, SVCPU_CTR(r13)
262	PPC_STL	r9, SVCPU_LR(r13)
263
264	/*
265	 * In order for us to easily get the last instruction,
266	 * we got the #vmexit at, we exploit the fact that the
267	 * virtual layout is still the same here, so we can just
268	 * ld from the guest's PC address
269	 */
270
271	/* We only load the last instruction when it's safe */
272	cmpwi	r12, BOOK3S_INTERRUPT_DATA_STORAGE
273	beq	ld_last_inst
274	cmpwi	r12, BOOK3S_INTERRUPT_PROGRAM
275	beq	ld_last_inst
276	cmpwi	r12, BOOK3S_INTERRUPT_SYSCALL
277	beq	ld_last_prev_inst
278	cmpwi	r12, BOOK3S_INTERRUPT_ALIGNMENT
279	beq-	ld_last_inst
280#ifdef CONFIG_PPC64
281BEGIN_FTR_SECTION
282	cmpwi	r12, BOOK3S_INTERRUPT_H_EMUL_ASSIST
283	beq-	ld_last_inst
284END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
285BEGIN_FTR_SECTION
286	cmpwi	r12, BOOK3S_INTERRUPT_FAC_UNAVAIL
287	beq-	ld_last_inst
288END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
289#endif
290
291	b	no_ld_last_inst
292
293ld_last_prev_inst:
294	addi	r3, r3, -4
295
296ld_last_inst:
297	/* Save off the guest instruction we're at */
298
299	/* In case lwz faults */
300	li	r0, KVM_INST_FETCH_FAILED
301
302#ifdef USE_QUICK_LAST_INST
303
304	/* Set guest mode to 'jump over instruction' so if lwz faults
305	 * we'll just continue at the next IP. */
306	li	r9, KVM_GUEST_MODE_SKIP
307	stb	r9, HSTATE_IN_GUEST(r13)
308
309	/*    1) enable paging for data */
310	mfmsr	r9
311	ori	r11, r9, MSR_DR			/* Enable paging for data */
312	mtmsr	r11
313	sync
314	/*    2) fetch the instruction */
315	lwz	r0, 0(r3)
316	/*    3) disable paging again */
317	mtmsr	r9
318	sync
319
320#endif
321	stw	r0, SVCPU_LAST_INST(r13)
322
323no_ld_last_inst:
324
325	/* Unset guest mode */
326	li	r9, KVM_GUEST_MODE_NONE
327	stb	r9, HSTATE_IN_GUEST(r13)
328
329	/* Switch back to host MMU */
330	LOAD_HOST_SEGMENTS
331
332#ifdef CONFIG_PPC_BOOK3S_64
333
334	lbz	r5, HSTATE_RESTORE_HID5(r13)
335	cmpwi	r5, 0
336	beq	no_dcbz32_off
337
338	li	r4, 0
339	mfspr   r5,SPRN_HID5
340	rldimi  r5,r4,6,56
341	mtspr   SPRN_HID5,r5
342
343no_dcbz32_off:
344
345BEGIN_FTR_SECTION
346	/* Save guest FSCR on a FAC_UNAVAIL interrupt */
347	cmpwi	r12, BOOK3S_INTERRUPT_FAC_UNAVAIL
348	bne+	no_fscr_save
349	mfspr	r7, SPRN_FSCR
350	std	r7, SVCPU_SHADOW_FSCR(r13)
351no_fscr_save:
352	/* Restore host FSCR */
353	ld	r8, HSTATE_HOST_FSCR(r13)
354	mtspr	SPRN_FSCR, r8
355END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
356
357#endif /* CONFIG_PPC_BOOK3S_64 */
358
359	/*
360	 * For some interrupts, we need to call the real Linux
361	 * handler, so it can do work for us. This has to happen
362	 * as if the interrupt arrived from the kernel though,
363	 * so let's fake it here where most state is restored.
364	 *
365	 * Having set up SRR0/1 with the address where we want
366	 * to continue with relocation on (potentially in module
367	 * space), we either just go straight there with rfi[d],
368	 * or we jump to an interrupt handler if there is an
369	 * interrupt to be handled first.  In the latter case,
370	 * the rfi[d] at the end of the interrupt handler will
371	 * get us back to where we want to continue.
372	 */
373
374	/* Register usage at this point:
375	 *
376	 * R1       = host R1
377	 * R2       = host R2
378	 * R10      = raw exit handler id
379	 * R12      = exit handler id
380	 * R13      = shadow vcpu (32-bit) or PACA (64-bit)
381	 * SVCPU.*  = guest *
382	 *
383	 */
384
385	PPC_LL	r6, HSTATE_HOST_MSR(r13)
386	PPC_LL	r8, HSTATE_VMHANDLER(r13)
387
388#ifdef CONFIG_PPC64
389BEGIN_FTR_SECTION
390	beq	cr1, 1f
391	mtspr	SPRN_HSRR1, r6
392	mtspr	SPRN_HSRR0, r8
393END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
394#endif
3951:	/* Restore host msr -> SRR1 */
396	mtsrr1	r6
397	/* Load highmem handler address */
398	mtsrr0	r8
399
400	/* RFI into the highmem handler, or jump to interrupt handler */
401	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
402	beqa	BOOK3S_INTERRUPT_EXTERNAL
403	cmpwi	r12, BOOK3S_INTERRUPT_DECREMENTER
404	beqa	BOOK3S_INTERRUPT_DECREMENTER
405	cmpwi	r12, BOOK3S_INTERRUPT_PERFMON
406	beqa	BOOK3S_INTERRUPT_PERFMON
407	cmpwi	r12, BOOK3S_INTERRUPT_DOORBELL
408	beqa	BOOK3S_INTERRUPT_DOORBELL
409
410	RFI_TO_KERNEL
411kvmppc_handler_trampoline_exit_end:
412