1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20#include <asm/ppc_asm.h>
21#include <asm/kvm_asm.h>
22#include <asm/reg.h>
23#include <asm/mmu.h>
24#include <asm/page.h>
25#include <asm/ptrace.h>
26#include <asm/hvcall.h>
27#include <asm/asm-offsets.h>
28#include <asm/exception-64s.h>
29#include <asm/kvm_book3s_asm.h>
30#include <asm/mmu-hash64.h>
31
32/*****************************************************************************
33 *                                                                           *
34 *        Real Mode handlers that need to be in the linear mapping           *
35 *                                                                           *
36 ****************************************************************************/
37
38	.globl	kvmppc_skip_interrupt
39kvmppc_skip_interrupt:
40	mfspr	r13,SPRN_SRR0
41	addi	r13,r13,4
42	mtspr	SPRN_SRR0,r13
43	GET_SCRATCH0(r13)
44	rfid
45	b	.
46
47	.globl	kvmppc_skip_Hinterrupt
48kvmppc_skip_Hinterrupt:
49	mfspr	r13,SPRN_HSRR0
50	addi	r13,r13,4
51	mtspr	SPRN_HSRR0,r13
52	GET_SCRATCH0(r13)
53	hrfid
54	b	.
55
56/*
57 * Call kvmppc_hv_entry in real mode.
58 * Must be called with interrupts hard-disabled.
59 *
60 * Input Registers:
61 *
62 * LR = return address to continue at after eventually re-enabling MMU
63 */
64_GLOBAL(kvmppc_hv_entry_trampoline)
65	mfmsr	r10
66	LOAD_REG_ADDR(r5, kvmppc_hv_entry)
67	li	r0,MSR_RI
68	andc	r0,r10,r0
69	li	r6,MSR_IR | MSR_DR
70	andc	r6,r10,r6
71	mtmsrd	r0,1		/* clear RI in MSR */
72	mtsrr0	r5
73	mtsrr1	r6
74	RFI
75
76/******************************************************************************
77 *                                                                            *
78 *                               Entry code                                   *
79 *                                                                            *
80 *****************************************************************************/
81
82/*
83 * We come in here when wakened from nap mode on a secondary hw thread.
84 * Relocation is off and most register values are lost.
85 * r13 points to the PACA.
86 */
87	.globl	kvm_start_guest
88kvm_start_guest:
89	ld	r1,PACAEMERGSP(r13)
90	subi	r1,r1,STACK_FRAME_OVERHEAD
91	ld	r2,PACATOC(r13)
92
93	li	r0,KVM_HWTHREAD_IN_KVM
94	stb	r0,HSTATE_HWTHREAD_STATE(r13)
95
96	/* NV GPR values from power7_idle() will no longer be valid */
97	li	r0,1
98	stb	r0,PACA_NAPSTATELOST(r13)
99
100	/* were we napping due to cede? */
101	lbz	r0,HSTATE_NAPPING(r13)
102	cmpwi	r0,0
103	bne	kvm_end_cede
104
105	/*
106	 * We weren't napping due to cede, so this must be a secondary
107	 * thread being woken up to run a guest, or being woken up due
108	 * to a stray IPI.  (Or due to some machine check or hypervisor
109	 * maintenance interrupt while the core is in KVM.)
110	 */
111
112	/* Check the wake reason in SRR1 to see why we got here */
113	mfspr	r3,SPRN_SRR1
114	rlwinm	r3,r3,44-31,0x7		/* extract wake reason field */
115	cmpwi	r3,4			/* was it an external interrupt? */
116	bne	27f			/* if not */
117	ld	r5,HSTATE_XICS_PHYS(r13)
118	li	r7,XICS_XIRR		/* if it was an external interrupt, */
119	lwzcix	r8,r5,r7		/* get and ack the interrupt */
120	sync
121	clrldi.	r9,r8,40		/* get interrupt source ID. */
122	beq	28f			/* none there? */
123	cmpwi	r9,XICS_IPI		/* was it an IPI? */
124	bne	29f
125	li	r0,0xff
126	li	r6,XICS_MFRR
127	stbcix	r0,r5,r6		/* clear IPI */
128	stwcix	r8,r5,r7		/* EOI the interrupt */
129	sync				/* order loading of vcpu after that */
130
131	/* get vcpu pointer, NULL if we have no vcpu to run */
132	ld	r4,HSTATE_KVM_VCPU(r13)
133	cmpdi	r4,0
134	/* if we have no vcpu to run, go back to sleep */
135	beq	kvm_no_guest
136	b	kvmppc_hv_entry
137
13827:	/* XXX should handle hypervisor maintenance interrupts etc. here */
139	b	kvm_no_guest
14028:	/* SRR1 said external but ICP said nope?? */
141	b	kvm_no_guest
14229:	/* External non-IPI interrupt to offline secondary thread? help?? */
143	stw	r8,HSTATE_SAVED_XIRR(r13)
144	b	kvm_no_guest
145
146.global kvmppc_hv_entry
147kvmppc_hv_entry:
148
149	/* Required state:
150	 *
151	 * R4 = vcpu pointer
152	 * MSR = ~IR|DR
153	 * R13 = PACA
154	 * R1 = host R1
155	 * all other volatile GPRS = free
156	 */
157	mflr	r0
158	std	r0, HSTATE_VMHANDLER(r13)
159
160	/* Set partition DABR */
161	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
162	li	r5,3
163	ld	r6,VCPU_DABR(r4)
164	mtspr	SPRN_DABRX,r5
165	mtspr	SPRN_DABR,r6
166BEGIN_FTR_SECTION
167	isync
168END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
169
170	/* Load guest PMU registers */
171	/* R4 is live here (vcpu pointer) */
172	li	r3, 1
173	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
174	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
175	isync
176	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
177	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
178	lwz	r6, VCPU_PMC + 8(r4)
179	lwz	r7, VCPU_PMC + 12(r4)
180	lwz	r8, VCPU_PMC + 16(r4)
181	lwz	r9, VCPU_PMC + 20(r4)
182BEGIN_FTR_SECTION
183	lwz	r10, VCPU_PMC + 24(r4)
184	lwz	r11, VCPU_PMC + 28(r4)
185END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
186	mtspr	SPRN_PMC1, r3
187	mtspr	SPRN_PMC2, r5
188	mtspr	SPRN_PMC3, r6
189	mtspr	SPRN_PMC4, r7
190	mtspr	SPRN_PMC5, r8
191	mtspr	SPRN_PMC6, r9
192BEGIN_FTR_SECTION
193	mtspr	SPRN_PMC7, r10
194	mtspr	SPRN_PMC8, r11
195END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
196	ld	r3, VCPU_MMCR(r4)
197	ld	r5, VCPU_MMCR + 8(r4)
198	ld	r6, VCPU_MMCR + 16(r4)
199	mtspr	SPRN_MMCR1, r5
200	mtspr	SPRN_MMCRA, r6
201	mtspr	SPRN_MMCR0, r3
202	isync
203
204	/* Load up FP, VMX and VSX registers */
205	bl	kvmppc_load_fp
206
207	ld	r14, VCPU_GPR(R14)(r4)
208	ld	r15, VCPU_GPR(R15)(r4)
209	ld	r16, VCPU_GPR(R16)(r4)
210	ld	r17, VCPU_GPR(R17)(r4)
211	ld	r18, VCPU_GPR(R18)(r4)
212	ld	r19, VCPU_GPR(R19)(r4)
213	ld	r20, VCPU_GPR(R20)(r4)
214	ld	r21, VCPU_GPR(R21)(r4)
215	ld	r22, VCPU_GPR(R22)(r4)
216	ld	r23, VCPU_GPR(R23)(r4)
217	ld	r24, VCPU_GPR(R24)(r4)
218	ld	r25, VCPU_GPR(R25)(r4)
219	ld	r26, VCPU_GPR(R26)(r4)
220	ld	r27, VCPU_GPR(R27)(r4)
221	ld	r28, VCPU_GPR(R28)(r4)
222	ld	r29, VCPU_GPR(R29)(r4)
223	ld	r30, VCPU_GPR(R30)(r4)
224	ld	r31, VCPU_GPR(R31)(r4)
225
226BEGIN_FTR_SECTION
227	/* Switch DSCR to guest value */
228	ld	r5, VCPU_DSCR(r4)
229	mtspr	SPRN_DSCR, r5
230END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
231
232	/*
233	 * Set the decrementer to the guest decrementer.
234	 */
235	ld	r8,VCPU_DEC_EXPIRES(r4)
236	mftb	r7
237	subf	r3,r7,r8
238	mtspr	SPRN_DEC,r3
239	stw	r3,VCPU_DEC(r4)
240
241	ld	r5, VCPU_SPRG0(r4)
242	ld	r6, VCPU_SPRG1(r4)
243	ld	r7, VCPU_SPRG2(r4)
244	ld	r8, VCPU_SPRG3(r4)
245	mtspr	SPRN_SPRG0, r5
246	mtspr	SPRN_SPRG1, r6
247	mtspr	SPRN_SPRG2, r7
248	mtspr	SPRN_SPRG3, r8
249
250	/* Save R1 in the PACA */
251	std	r1, HSTATE_HOST_R1(r13)
252
253	/* Increment yield count if they have a VPA */
254	ld	r3, VCPU_VPA(r4)
255	cmpdi	r3, 0
256	beq	25f
257	lwz	r5, LPPACA_YIELDCOUNT(r3)
258	addi	r5, r5, 1
259	stw	r5, LPPACA_YIELDCOUNT(r3)
260	li	r6, 1
261	stb	r6, VCPU_VPA_DIRTY(r4)
26225:
263	/* Load up DAR and DSISR */
264	ld	r5, VCPU_DAR(r4)
265	lwz	r6, VCPU_DSISR(r4)
266	mtspr	SPRN_DAR, r5
267	mtspr	SPRN_DSISR, r6
268
269BEGIN_FTR_SECTION
270	/* Restore AMR and UAMOR, set AMOR to all 1s */
271	ld	r5,VCPU_AMR(r4)
272	ld	r6,VCPU_UAMOR(r4)
273	li	r7,-1
274	mtspr	SPRN_AMR,r5
275	mtspr	SPRN_UAMOR,r6
276	mtspr	SPRN_AMOR,r7
277END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
278
279	/* Clear out SLB */
280	li	r6,0
281	slbmte	r6,r6
282	slbia
283	ptesync
284
285BEGIN_FTR_SECTION
286	b	30f
287END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
288	/*
289	 * POWER7 host -> guest partition switch code.
290	 * We don't have to lock against concurrent tlbies,
291	 * but we do have to coordinate across hardware threads.
292	 */
293	/* Increment entry count iff exit count is zero. */
294	ld	r5,HSTATE_KVM_VCORE(r13)
295	addi	r9,r5,VCORE_ENTRY_EXIT
29621:	lwarx	r3,0,r9
297	cmpwi	r3,0x100		/* any threads starting to exit? */
298	bge	secondary_too_late	/* if so we're too late to the party */
299	addi	r3,r3,1
300	stwcx.	r3,0,r9
301	bne	21b
302
303	/* Primary thread switches to guest partition. */
304	ld	r9,VCPU_KVM(r4)		/* pointer to struct kvm */
305	lwz	r6,VCPU_PTID(r4)
306	cmpwi	r6,0
307	bne	20f
308	ld	r6,KVM_SDR1(r9)
309	lwz	r7,KVM_LPID(r9)
310	li	r0,LPID_RSVD		/* switch to reserved LPID */
311	mtspr	SPRN_LPID,r0
312	ptesync
313	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
314	mtspr	SPRN_LPID,r7
315	isync
316
317	/* See if we need to flush the TLB */
318	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
319	clrldi	r7,r6,64-6		/* extract bit number (6 bits) */
320	srdi	r6,r6,6			/* doubleword number */
321	sldi	r6,r6,3			/* address offset */
322	add	r6,r6,r9
323	addi	r6,r6,KVM_NEED_FLUSH	/* dword in kvm->arch.need_tlb_flush */
324	li	r0,1
325	sld	r0,r0,r7
326	ld	r7,0(r6)
327	and.	r7,r7,r0
328	beq	22f
32923:	ldarx	r7,0,r6			/* if set, clear the bit */
330	andc	r7,r7,r0
331	stdcx.	r7,0,r6
332	bne	23b
333	li	r6,128			/* and flush the TLB */
334	mtctr	r6
335	li	r7,0x800		/* IS field = 0b10 */
336	ptesync
33728:	tlbiel	r7
338	addi	r7,r7,0x1000
339	bdnz	28b
340	ptesync
341
34222:	li	r0,1
343	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
344	b	10f
345
346	/* Secondary threads wait for primary to have done partition switch */
34720:	lbz	r0,VCORE_IN_GUEST(r5)
348	cmpwi	r0,0
349	beq	20b
350
351	/* Set LPCR and RMOR. */
35210:	ld	r8,KVM_LPCR(r9)
353	mtspr	SPRN_LPCR,r8
354	ld	r8,KVM_RMOR(r9)
355	mtspr	SPRN_RMOR,r8
356	isync
357
358	/* Check if HDEC expires soon */
359	mfspr	r3,SPRN_HDEC
360	cmpwi	r3,10
361	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
362	mr	r9,r4
363	blt	hdec_soon
364
365	/* Save purr/spurr */
366	mfspr	r5,SPRN_PURR
367	mfspr	r6,SPRN_SPURR
368	std	r5,HSTATE_PURR(r13)
369	std	r6,HSTATE_SPURR(r13)
370	ld	r7,VCPU_PURR(r4)
371	ld	r8,VCPU_SPURR(r4)
372	mtspr	SPRN_PURR,r7
373	mtspr	SPRN_SPURR,r8
374	b	31f
375
376	/*
377	 * PPC970 host -> guest partition switch code.
378	 * We have to lock against concurrent tlbies,
379	 * using native_tlbie_lock to lock against host tlbies
380	 * and kvm->arch.tlbie_lock to lock against guest tlbies.
381	 * We also have to invalidate the TLB since its
382	 * entries aren't tagged with the LPID.
383	 */
38430:	ld	r9,VCPU_KVM(r4)		/* pointer to struct kvm */
385
386	/* first take native_tlbie_lock */
387	.section ".toc","aw"
388toc_tlbie_lock:
389	.tc	native_tlbie_lock[TC],native_tlbie_lock
390	.previous
391	ld	r3,toc_tlbie_lock@toc(2)
392	lwz	r8,PACA_LOCK_TOKEN(r13)
39324:	lwarx	r0,0,r3
394	cmpwi	r0,0
395	bne	24b
396	stwcx.	r8,0,r3
397	bne	24b
398	isync
399
400	ld	r7,KVM_LPCR(r9)		/* use kvm->arch.lpcr to store HID4 */
401	li	r0,0x18f
402	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
403	or	r0,r7,r0
404	ptesync
405	sync
406	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
407	isync
408	li	r0,0
409	stw	r0,0(r3)		/* drop native_tlbie_lock */
410
411	/* invalidate the whole TLB */
412	li	r0,256
413	mtctr	r0
414	li	r6,0
41525:	tlbiel	r6
416	addi	r6,r6,0x1000
417	bdnz	25b
418	ptesync
419
420	/* Take the guest's tlbie_lock */
421	addi	r3,r9,KVM_TLBIE_LOCK
42224:	lwarx	r0,0,r3
423	cmpwi	r0,0
424	bne	24b
425	stwcx.	r8,0,r3
426	bne	24b
427	isync
428	ld	r6,KVM_SDR1(r9)
429	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
430
431	/* Set up HID4 with the guest's LPID etc. */
432	sync
433	mtspr	SPRN_HID4,r7
434	isync
435
436	/* drop the guest's tlbie_lock */
437	li	r0,0
438	stw	r0,0(r3)
439
440	/* Check if HDEC expires soon */
441	mfspr	r3,SPRN_HDEC
442	cmpwi	r3,10
443	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
444	mr	r9,r4
445	blt	hdec_soon
446
447	/* Enable HDEC interrupts */
448	mfspr	r0,SPRN_HID0
449	li	r3,1
450	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
451	sync
452	mtspr	SPRN_HID0,r0
453	mfspr	r0,SPRN_HID0
454	mfspr	r0,SPRN_HID0
455	mfspr	r0,SPRN_HID0
456	mfspr	r0,SPRN_HID0
457	mfspr	r0,SPRN_HID0
458	mfspr	r0,SPRN_HID0
459
460	/* Load up guest SLB entries */
46131:	lwz	r5,VCPU_SLB_MAX(r4)
462	cmpwi	r5,0
463	beq	9f
464	mtctr	r5
465	addi	r6,r4,VCPU_SLB
4661:	ld	r8,VCPU_SLB_E(r6)
467	ld	r9,VCPU_SLB_V(r6)
468	slbmte	r9,r8
469	addi	r6,r6,VCPU_SLB_SIZE
470	bdnz	1b
4719:
472
473	/* Restore state of CTRL run bit; assume 1 on entry */
474	lwz	r5,VCPU_CTRL(r4)
475	andi.	r5,r5,1
476	bne	4f
477	mfspr	r6,SPRN_CTRLF
478	clrrdi	r6,r6,1
479	mtspr	SPRN_CTRLT,r6
4804:
481	ld	r6, VCPU_CTR(r4)
482	lwz	r7, VCPU_XER(r4)
483
484	mtctr	r6
485	mtxer	r7
486
487	ld	r10, VCPU_PC(r4)
488	ld	r11, VCPU_MSR(r4)
489kvmppc_cede_reentry:		/* r4 = vcpu, r13 = paca */
490	ld	r6, VCPU_SRR0(r4)
491	ld	r7, VCPU_SRR1(r4)
492
493	/* r11 = vcpu->arch.msr & ~MSR_HV */
494	rldicl	r11, r11, 63 - MSR_HV_LG, 1
495	rotldi	r11, r11, 1 + MSR_HV_LG
496	ori	r11, r11, MSR_ME
497
498	/* Check if we can deliver an external or decrementer interrupt now */
499	ld	r0,VCPU_PENDING_EXC(r4)
500	lis	r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
501	and	r0,r0,r8
502	cmpdi	cr1,r0,0
503	andi.	r0,r11,MSR_EE
504	beq	cr1,11f
505BEGIN_FTR_SECTION
506	mfspr	r8,SPRN_LPCR
507	ori	r8,r8,LPCR_MER
508	mtspr	SPRN_LPCR,r8
509	isync
510END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
511	beq	5f
512	li	r0,BOOK3S_INTERRUPT_EXTERNAL
51312:	mr	r6,r10
514	mr	r10,r0
515	mr	r7,r11
516	li	r11,(MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
517	rotldi	r11,r11,63
518	b	5f
51911:	beq	5f
520	mfspr	r0,SPRN_DEC
521	cmpwi	r0,0
522	li	r0,BOOK3S_INTERRUPT_DECREMENTER
523	blt	12b
524
525	/* Move SRR0 and SRR1 into the respective regs */
5265:	mtspr	SPRN_SRR0, r6
527	mtspr	SPRN_SRR1, r7
528
529fast_guest_return:
530	li	r0,0
531	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
532	mtspr	SPRN_HSRR0,r10
533	mtspr	SPRN_HSRR1,r11
534
535	/* Activate guest mode, so faults get handled by KVM */
536	li	r9, KVM_GUEST_MODE_GUEST
537	stb	r9, HSTATE_IN_GUEST(r13)
538
539	/* Enter guest */
540
541BEGIN_FTR_SECTION
542	ld	r5, VCPU_CFAR(r4)
543	mtspr	SPRN_CFAR, r5
544END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
545
546	ld	r5, VCPU_LR(r4)
547	lwz	r6, VCPU_CR(r4)
548	mtlr	r5
549	mtcr	r6
550
551	ld	r0, VCPU_GPR(R0)(r4)
552	ld	r1, VCPU_GPR(R1)(r4)
553	ld	r2, VCPU_GPR(R2)(r4)
554	ld	r3, VCPU_GPR(R3)(r4)
555	ld	r5, VCPU_GPR(R5)(r4)
556	ld	r6, VCPU_GPR(R6)(r4)
557	ld	r7, VCPU_GPR(R7)(r4)
558	ld	r8, VCPU_GPR(R8)(r4)
559	ld	r9, VCPU_GPR(R9)(r4)
560	ld	r10, VCPU_GPR(R10)(r4)
561	ld	r11, VCPU_GPR(R11)(r4)
562	ld	r12, VCPU_GPR(R12)(r4)
563	ld	r13, VCPU_GPR(R13)(r4)
564
565	ld	r4, VCPU_GPR(R4)(r4)
566
567	hrfid
568	b	.
569
570/******************************************************************************
571 *                                                                            *
572 *                               Exit code                                    *
573 *                                                                            *
574 *****************************************************************************/
575
576/*
577 * We come here from the first-level interrupt handlers.
578 */
579	.globl	kvmppc_interrupt
580kvmppc_interrupt:
581	/*
582	 * Register contents:
583	 * R12		= interrupt vector
584	 * R13		= PACA
585	 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
586	 * guest R13 saved in SPRN_SCRATCH0
587	 */
588	/* abuse host_r2 as third scratch area; we get r2 from PACATOC(r13) */
589	std	r9, HSTATE_HOST_R2(r13)
590	ld	r9, HSTATE_KVM_VCPU(r13)
591
592	/* Save registers */
593
594	std	r0, VCPU_GPR(R0)(r9)
595	std	r1, VCPU_GPR(R1)(r9)
596	std	r2, VCPU_GPR(R2)(r9)
597	std	r3, VCPU_GPR(R3)(r9)
598	std	r4, VCPU_GPR(R4)(r9)
599	std	r5, VCPU_GPR(R5)(r9)
600	std	r6, VCPU_GPR(R6)(r9)
601	std	r7, VCPU_GPR(R7)(r9)
602	std	r8, VCPU_GPR(R8)(r9)
603	ld	r0, HSTATE_HOST_R2(r13)
604	std	r0, VCPU_GPR(R9)(r9)
605	std	r10, VCPU_GPR(R10)(r9)
606	std	r11, VCPU_GPR(R11)(r9)
607	ld	r3, HSTATE_SCRATCH0(r13)
608	lwz	r4, HSTATE_SCRATCH1(r13)
609	std	r3, VCPU_GPR(R12)(r9)
610	stw	r4, VCPU_CR(r9)
611BEGIN_FTR_SECTION
612	ld	r3, HSTATE_CFAR(r13)
613	std	r3, VCPU_CFAR(r9)
614END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
615
616	/* Restore R1/R2 so we can handle faults */
617	ld	r1, HSTATE_HOST_R1(r13)
618	ld	r2, PACATOC(r13)
619
620	mfspr	r10, SPRN_SRR0
621	mfspr	r11, SPRN_SRR1
622	std	r10, VCPU_SRR0(r9)
623	std	r11, VCPU_SRR1(r9)
624	andi.	r0, r12, 2		/* need to read HSRR0/1? */
625	beq	1f
626	mfspr	r10, SPRN_HSRR0
627	mfspr	r11, SPRN_HSRR1
628	clrrdi	r12, r12, 2
6291:	std	r10, VCPU_PC(r9)
630	std	r11, VCPU_MSR(r9)
631
632	GET_SCRATCH0(r3)
633	mflr	r4
634	std	r3, VCPU_GPR(R13)(r9)
635	std	r4, VCPU_LR(r9)
636
637	/* Unset guest mode */
638	li	r0, KVM_GUEST_MODE_NONE
639	stb	r0, HSTATE_IN_GUEST(r13)
640
641	stw	r12,VCPU_TRAP(r9)
642
643	/* Save HEIR (HV emulation assist reg) in last_inst
644	   if this is an HEI (HV emulation interrupt, e40) */
645	li	r3,KVM_INST_FETCH_FAILED
646BEGIN_FTR_SECTION
647	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
648	bne	11f
649	mfspr	r3,SPRN_HEIR
650END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
65111:	stw	r3,VCPU_LAST_INST(r9)
652
653	/* these are volatile across C function calls */
654	mfctr	r3
655	mfxer	r4
656	std	r3, VCPU_CTR(r9)
657	stw	r4, VCPU_XER(r9)
658
659BEGIN_FTR_SECTION
660	/* If this is a page table miss then see if it's theirs or ours */
661	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
662	beq	kvmppc_hdsi
663	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
664	beq	kvmppc_hisi
665END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
666
667	/* See if this is a leftover HDEC interrupt */
668	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
669	bne	2f
670	mfspr	r3,SPRN_HDEC
671	cmpwi	r3,0
672	bge	ignore_hdec
6732:
674	/* See if this is an hcall we can handle in real mode */
675	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
676	beq	hcall_try_real_mode
677
678	/* Only handle external interrupts here on arch 206 and later */
679BEGIN_FTR_SECTION
680	b	ext_interrupt_to_host
681END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
682
683	/* External interrupt ? */
684	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
685	bne+	ext_interrupt_to_host
686
687	/* External interrupt, first check for host_ipi. If this is
688	 * set, we know the host wants us out so let's do it now
689	 */
690do_ext_interrupt:
691	lbz	r0, HSTATE_HOST_IPI(r13)
692	cmpwi	r0, 0
693	bne	ext_interrupt_to_host
694
695	/* Now read the interrupt from the ICP */
696	ld	r5, HSTATE_XICS_PHYS(r13)
697	li	r7, XICS_XIRR
698	cmpdi	r5, 0
699	beq-	ext_interrupt_to_host
700	lwzcix	r3, r5, r7
701	rlwinm.	r0, r3, 0, 0xffffff
702	sync
703	beq	3f		/* if nothing pending in the ICP */
704
705	/* We found something in the ICP...
706	 *
707	 * If it's not an IPI, stash it in the PACA and return to
708	 * the host, we don't (yet) handle directing real external
709	 * interrupts directly to the guest
710	 */
711	cmpwi	r0, XICS_IPI
712	bne	ext_stash_for_host
713
714	/* It's an IPI, clear the MFRR and EOI it */
715	li	r0, 0xff
716	li	r6, XICS_MFRR
717	stbcix	r0, r5, r6		/* clear the IPI */
718	stwcix	r3, r5, r7		/* EOI it */
719	sync
720
721	/* We need to re-check host IPI now in case it got set in the
722	 * meantime. If it's clear, we bounce the interrupt to the
723	 * guest
724	 */
725	lbz	r0, HSTATE_HOST_IPI(r13)
726	cmpwi	r0, 0
727	bne-	1f
728
729	/* Allright, looks like an IPI for the guest, we need to set MER */
7303:
731	/* Check if any CPU is heading out to the host, if so head out too */
732	ld	r5, HSTATE_KVM_VCORE(r13)
733	lwz	r0, VCORE_ENTRY_EXIT(r5)
734	cmpwi	r0, 0x100
735	bge	ext_interrupt_to_host
736
737	/* See if there is a pending interrupt for the guest */
738	mfspr	r8, SPRN_LPCR
739	ld	r0, VCPU_PENDING_EXC(r9)
740	/* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
741	rldicl.	r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
742	rldimi	r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
743	beq	2f
744
745	/* And if the guest EE is set, we can deliver immediately, else
746	 * we return to the guest with MER set
747	 */
748	andi.	r0, r11, MSR_EE
749	beq	2f
750	mtspr	SPRN_SRR0, r10
751	mtspr	SPRN_SRR1, r11
752	li	r10, BOOK3S_INTERRUPT_EXTERNAL
753	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
754	rotldi	r11, r11, 63
7552:	mr	r4, r9
756	mtspr	SPRN_LPCR, r8
757	b	fast_guest_return
758
759	/* We raced with the host, we need to resend that IPI, bummer */
7601:	li	r0, IPI_PRIORITY
761	stbcix	r0, r5, r6		/* set the IPI */
762	sync
763	b	ext_interrupt_to_host
764
765ext_stash_for_host:
766	/* It's not an IPI and it's for the host, stash it in the PACA
767	 * before exit, it will be picked up by the host ICP driver
768	 */
769	stw	r3, HSTATE_SAVED_XIRR(r13)
770ext_interrupt_to_host:
771
772guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
773	/* Save DEC */
774	mfspr	r5,SPRN_DEC
775	mftb	r6
776	extsw	r5,r5
777	add	r5,r5,r6
778	std	r5,VCPU_DEC_EXPIRES(r9)
779
780	/* Save more register state  */
781	mfdar	r6
782	mfdsisr	r7
783	std	r6, VCPU_DAR(r9)
784	stw	r7, VCPU_DSISR(r9)
785BEGIN_FTR_SECTION
786	/* don't overwrite fault_dar/fault_dsisr if HDSI */
787	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
788	beq	6f
789END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
790	std	r6, VCPU_FAULT_DAR(r9)
791	stw	r7, VCPU_FAULT_DSISR(r9)
792
793	/* See if it is a machine check */
794	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
795	beq	machine_check_realmode
796mc_cont:
797
798	/* Save guest CTRL register, set runlatch to 1 */
7996:	mfspr	r6,SPRN_CTRLF
800	stw	r6,VCPU_CTRL(r9)
801	andi.	r0,r6,1
802	bne	4f
803	ori	r6,r6,1
804	mtspr	SPRN_CTRLT,r6
8054:
806	/* Read the guest SLB and save it away */
807	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
808	mtctr	r0
809	li	r6,0
810	addi	r7,r9,VCPU_SLB
811	li	r5,0
8121:	slbmfee	r8,r6
813	andis.	r0,r8,SLB_ESID_V@h
814	beq	2f
815	add	r8,r8,r6		/* put index in */
816	slbmfev	r3,r6
817	std	r8,VCPU_SLB_E(r7)
818	std	r3,VCPU_SLB_V(r7)
819	addi	r7,r7,VCPU_SLB_SIZE
820	addi	r5,r5,1
8212:	addi	r6,r6,1
822	bdnz	1b
823	stw	r5,VCPU_SLB_MAX(r9)
824
825	/*
826	 * Save the guest PURR/SPURR
827	 */
828BEGIN_FTR_SECTION
829	mfspr	r5,SPRN_PURR
830	mfspr	r6,SPRN_SPURR
831	ld	r7,VCPU_PURR(r9)
832	ld	r8,VCPU_SPURR(r9)
833	std	r5,VCPU_PURR(r9)
834	std	r6,VCPU_SPURR(r9)
835	subf	r5,r7,r5
836	subf	r6,r8,r6
837
838	/*
839	 * Restore host PURR/SPURR and add guest times
840	 * so that the time in the guest gets accounted.
841	 */
842	ld	r3,HSTATE_PURR(r13)
843	ld	r4,HSTATE_SPURR(r13)
844	add	r3,r3,r5
845	add	r4,r4,r6
846	mtspr	SPRN_PURR,r3
847	mtspr	SPRN_SPURR,r4
848END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
849
850	/* Clear out SLB */
851	li	r5,0
852	slbmte	r5,r5
853	slbia
854	ptesync
855
856hdec_soon:			/* r9 = vcpu, r12 = trap, r13 = paca */
857BEGIN_FTR_SECTION
858	b	32f
859END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
860	/*
861	 * POWER7 guest -> host partition switch code.
862	 * We don't have to lock against tlbies but we do
863	 * have to coordinate the hardware threads.
864	 */
865	/* Increment the threads-exiting-guest count in the 0xff00
866	   bits of vcore->entry_exit_count */
867	lwsync
868	ld	r5,HSTATE_KVM_VCORE(r13)
869	addi	r6,r5,VCORE_ENTRY_EXIT
87041:	lwarx	r3,0,r6
871	addi	r0,r3,0x100
872	stwcx.	r0,0,r6
873	bne	41b
874	lwsync
875
876	/*
877	 * At this point we have an interrupt that we have to pass
878	 * up to the kernel or qemu; we can't handle it in real mode.
879	 * Thus we have to do a partition switch, so we have to
880	 * collect the other threads, if we are the first thread
881	 * to take an interrupt.  To do this, we set the HDEC to 0,
882	 * which causes an HDEC interrupt in all threads within 2ns
883	 * because the HDEC register is shared between all 4 threads.
884	 * However, we don't need to bother if this is an HDEC
885	 * interrupt, since the other threads will already be on their
886	 * way here in that case.
887	 */
888	cmpwi	r3,0x100	/* Are we the first here? */
889	bge	43f
890	cmpwi	r3,1		/* Are any other threads in the guest? */
891	ble	43f
892	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
893	beq	40f
894	li	r0,0
895	mtspr	SPRN_HDEC,r0
89640:
897	/*
898	 * Send an IPI to any napping threads, since an HDEC interrupt
899	 * doesn't wake CPUs up from nap.
900	 */
901	lwz	r3,VCORE_NAPPING_THREADS(r5)
902	lwz	r4,VCPU_PTID(r9)
903	li	r0,1
904	sld	r0,r0,r4
905	andc.	r3,r3,r0		/* no sense IPI'ing ourselves */
906	beq	43f
907	mulli	r4,r4,PACA_SIZE		/* get paca for thread 0 */
908	subf	r6,r4,r13
90942:	andi.	r0,r3,1
910	beq	44f
911	ld	r8,HSTATE_XICS_PHYS(r6)	/* get thread's XICS reg addr */
912	li	r0,IPI_PRIORITY
913	li	r7,XICS_MFRR
914	stbcix	r0,r7,r8		/* trigger the IPI */
91544:	srdi.	r3,r3,1
916	addi	r6,r6,PACA_SIZE
917	bne	42b
918
919	/* Secondary threads wait for primary to do partition switch */
92043:	ld	r4,VCPU_KVM(r9)		/* pointer to struct kvm */
921	ld	r5,HSTATE_KVM_VCORE(r13)
922	lwz	r3,VCPU_PTID(r9)
923	cmpwi	r3,0
924	beq	15f
925	HMT_LOW
92613:	lbz	r3,VCORE_IN_GUEST(r5)
927	cmpwi	r3,0
928	bne	13b
929	HMT_MEDIUM
930	b	16f
931
932	/* Primary thread waits for all the secondaries to exit guest */
93315:	lwz	r3,VCORE_ENTRY_EXIT(r5)
934	srwi	r0,r3,8
935	clrldi	r3,r3,56
936	cmpw	r3,r0
937	bne	15b
938	isync
939
940	/* Primary thread switches back to host partition */
941	ld	r6,KVM_HOST_SDR1(r4)
942	lwz	r7,KVM_HOST_LPID(r4)
943	li	r8,LPID_RSVD		/* switch to reserved LPID */
944	mtspr	SPRN_LPID,r8
945	ptesync
946	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
947	mtspr	SPRN_LPID,r7
948	isync
949	li	r0,0
950	stb	r0,VCORE_IN_GUEST(r5)
951	lis	r8,0x7fff		/* MAX_INT@h */
952	mtspr	SPRN_HDEC,r8
953
95416:	ld	r8,KVM_HOST_LPCR(r4)
955	mtspr	SPRN_LPCR,r8
956	isync
957	b	33f
958
959	/*
960	 * PPC970 guest -> host partition switch code.
961	 * We have to lock against concurrent tlbies, and
962	 * we have to flush the whole TLB.
963	 */
96432:	ld	r4,VCPU_KVM(r9)		/* pointer to struct kvm */
965
966	/* Take the guest's tlbie_lock */
967	lwz	r8,PACA_LOCK_TOKEN(r13)
968	addi	r3,r4,KVM_TLBIE_LOCK
96924:	lwarx	r0,0,r3
970	cmpwi	r0,0
971	bne	24b
972	stwcx.	r8,0,r3
973	bne	24b
974	isync
975
976	ld	r7,KVM_HOST_LPCR(r4)	/* use kvm->arch.host_lpcr for HID4 */
977	li	r0,0x18f
978	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
979	or	r0,r7,r0
980	ptesync
981	sync
982	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
983	isync
984	li	r0,0
985	stw	r0,0(r3)		/* drop guest tlbie_lock */
986
987	/* invalidate the whole TLB */
988	li	r0,256
989	mtctr	r0
990	li	r6,0
99125:	tlbiel	r6
992	addi	r6,r6,0x1000
993	bdnz	25b
994	ptesync
995
996	/* take native_tlbie_lock */
997	ld	r3,toc_tlbie_lock@toc(2)
99824:	lwarx	r0,0,r3
999	cmpwi	r0,0
1000	bne	24b
1001	stwcx.	r8,0,r3
1002	bne	24b
1003	isync
1004
1005	ld	r6,KVM_HOST_SDR1(r4)
1006	mtspr	SPRN_SDR1,r6		/* switch to host page table */
1007
1008	/* Set up host HID4 value */
1009	sync
1010	mtspr	SPRN_HID4,r7
1011	isync
1012	li	r0,0
1013	stw	r0,0(r3)		/* drop native_tlbie_lock */
1014
1015	lis	r8,0x7fff		/* MAX_INT@h */
1016	mtspr	SPRN_HDEC,r8
1017
1018	/* Disable HDEC interrupts */
1019	mfspr	r0,SPRN_HID0
1020	li	r3,0
1021	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
1022	sync
1023	mtspr	SPRN_HID0,r0
1024	mfspr	r0,SPRN_HID0
1025	mfspr	r0,SPRN_HID0
1026	mfspr	r0,SPRN_HID0
1027	mfspr	r0,SPRN_HID0
1028	mfspr	r0,SPRN_HID0
1029	mfspr	r0,SPRN_HID0
1030
1031	/* load host SLB entries */
103233:	ld	r8,PACA_SLBSHADOWPTR(r13)
1033
1034	.rept	SLB_NUM_BOLTED
1035	ld	r5,SLBSHADOW_SAVEAREA(r8)
1036	ld	r6,SLBSHADOW_SAVEAREA+8(r8)
1037	andis.	r7,r5,SLB_ESID_V@h
1038	beq	1f
1039	slbmte	r6,r5
10401:	addi	r8,r8,16
1041	.endr
1042
1043	/* Save and reset AMR and UAMOR before turning on the MMU */
1044BEGIN_FTR_SECTION
1045	mfspr	r5,SPRN_AMR
1046	mfspr	r6,SPRN_UAMOR
1047	std	r5,VCPU_AMR(r9)
1048	std	r6,VCPU_UAMOR(r9)
1049	li	r6,0
1050	mtspr	SPRN_AMR,r6
1051END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1052
1053	/* Switch DSCR back to host value */
1054BEGIN_FTR_SECTION
1055	mfspr	r8, SPRN_DSCR
1056	ld	r7, HSTATE_DSCR(r13)
1057	std	r8, VCPU_DSCR(r7)
1058	mtspr	SPRN_DSCR, r7
1059END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1060
1061	/* Save non-volatile GPRs */
1062	std	r14, VCPU_GPR(R14)(r9)
1063	std	r15, VCPU_GPR(R15)(r9)
1064	std	r16, VCPU_GPR(R16)(r9)
1065	std	r17, VCPU_GPR(R17)(r9)
1066	std	r18, VCPU_GPR(R18)(r9)
1067	std	r19, VCPU_GPR(R19)(r9)
1068	std	r20, VCPU_GPR(R20)(r9)
1069	std	r21, VCPU_GPR(R21)(r9)
1070	std	r22, VCPU_GPR(R22)(r9)
1071	std	r23, VCPU_GPR(R23)(r9)
1072	std	r24, VCPU_GPR(R24)(r9)
1073	std	r25, VCPU_GPR(R25)(r9)
1074	std	r26, VCPU_GPR(R26)(r9)
1075	std	r27, VCPU_GPR(R27)(r9)
1076	std	r28, VCPU_GPR(R28)(r9)
1077	std	r29, VCPU_GPR(R29)(r9)
1078	std	r30, VCPU_GPR(R30)(r9)
1079	std	r31, VCPU_GPR(R31)(r9)
1080
1081	/* Save SPRGs */
1082	mfspr	r3, SPRN_SPRG0
1083	mfspr	r4, SPRN_SPRG1
1084	mfspr	r5, SPRN_SPRG2
1085	mfspr	r6, SPRN_SPRG3
1086	std	r3, VCPU_SPRG0(r9)
1087	std	r4, VCPU_SPRG1(r9)
1088	std	r5, VCPU_SPRG2(r9)
1089	std	r6, VCPU_SPRG3(r9)
1090
1091	/* save FP state */
1092	mr	r3, r9
1093	bl	.kvmppc_save_fp
1094
1095	/* Increment yield count if they have a VPA */
1096	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
1097	cmpdi	r8, 0
1098	beq	25f
1099	lwz	r3, LPPACA_YIELDCOUNT(r8)
1100	addi	r3, r3, 1
1101	stw	r3, LPPACA_YIELDCOUNT(r8)
1102	li	r3, 1
1103	stb	r3, VCPU_VPA_DIRTY(r9)
110425:
1105	/* Save PMU registers if requested */
1106	/* r8 and cr0.eq are live here */
1107	li	r3, 1
1108	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
1109	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
1110	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
1111	mfspr	r6, SPRN_MMCRA
1112BEGIN_FTR_SECTION
1113	/* On P7, clear MMCRA in order to disable SDAR updates */
1114	li	r7, 0
1115	mtspr	SPRN_MMCRA, r7
1116END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1117	isync
1118	beq	21f			/* if no VPA, save PMU stuff anyway */
1119	lbz	r7, LPPACA_PMCINUSE(r8)
1120	cmpwi	r7, 0			/* did they ask for PMU stuff to be saved? */
1121	bne	21f
1122	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
1123	b	22f
112421:	mfspr	r5, SPRN_MMCR1
1125	std	r4, VCPU_MMCR(r9)
1126	std	r5, VCPU_MMCR + 8(r9)
1127	std	r6, VCPU_MMCR + 16(r9)
1128	mfspr	r3, SPRN_PMC1
1129	mfspr	r4, SPRN_PMC2
1130	mfspr	r5, SPRN_PMC3
1131	mfspr	r6, SPRN_PMC4
1132	mfspr	r7, SPRN_PMC5
1133	mfspr	r8, SPRN_PMC6
1134BEGIN_FTR_SECTION
1135	mfspr	r10, SPRN_PMC7
1136	mfspr	r11, SPRN_PMC8
1137END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1138	stw	r3, VCPU_PMC(r9)
1139	stw	r4, VCPU_PMC + 4(r9)
1140	stw	r5, VCPU_PMC + 8(r9)
1141	stw	r6, VCPU_PMC + 12(r9)
1142	stw	r7, VCPU_PMC + 16(r9)
1143	stw	r8, VCPU_PMC + 20(r9)
1144BEGIN_FTR_SECTION
1145	stw	r10, VCPU_PMC + 24(r9)
1146	stw	r11, VCPU_PMC + 28(r9)
1147END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
114822:
1149
1150	/* Secondary threads go off to take a nap on POWER7 */
1151BEGIN_FTR_SECTION
1152	lwz	r0,VCPU_PTID(r9)
1153	cmpwi	r0,0
1154	bne	secondary_nap
1155END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1156
1157	/* Restore host DABR and DABRX */
1158	ld	r5,HSTATE_DABR(r13)
1159	li	r6,7
1160	mtspr	SPRN_DABR,r5
1161	mtspr	SPRN_DABRX,r6
1162
1163	/* Restore SPRG3 */
1164	ld	r3,PACA_SPRG3(r13)
1165	mtspr	SPRN_SPRG3,r3
1166
1167	/*
1168	 * Reload DEC.  HDEC interrupts were disabled when
1169	 * we reloaded the host's LPCR value.
1170	 */
1171	ld	r3, HSTATE_DECEXP(r13)
1172	mftb	r4
1173	subf	r4, r4, r3
1174	mtspr	SPRN_DEC, r4
1175
1176	/* Reload the host's PMU registers */
1177	ld	r3, PACALPPACAPTR(r13)	/* is the host using the PMU? */
1178	lbz	r4, LPPACA_PMCINUSE(r3)
1179	cmpwi	r4, 0
1180	beq	23f			/* skip if not */
1181	lwz	r3, HSTATE_PMC(r13)
1182	lwz	r4, HSTATE_PMC + 4(r13)
1183	lwz	r5, HSTATE_PMC + 8(r13)
1184	lwz	r6, HSTATE_PMC + 12(r13)
1185	lwz	r8, HSTATE_PMC + 16(r13)
1186	lwz	r9, HSTATE_PMC + 20(r13)
1187BEGIN_FTR_SECTION
1188	lwz	r10, HSTATE_PMC + 24(r13)
1189	lwz	r11, HSTATE_PMC + 28(r13)
1190END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1191	mtspr	SPRN_PMC1, r3
1192	mtspr	SPRN_PMC2, r4
1193	mtspr	SPRN_PMC3, r5
1194	mtspr	SPRN_PMC4, r6
1195	mtspr	SPRN_PMC5, r8
1196	mtspr	SPRN_PMC6, r9
1197BEGIN_FTR_SECTION
1198	mtspr	SPRN_PMC7, r10
1199	mtspr	SPRN_PMC8, r11
1200END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1201	ld	r3, HSTATE_MMCR(r13)
1202	ld	r4, HSTATE_MMCR + 8(r13)
1203	ld	r5, HSTATE_MMCR + 16(r13)
1204	mtspr	SPRN_MMCR1, r4
1205	mtspr	SPRN_MMCRA, r5
1206	mtspr	SPRN_MMCR0, r3
1207	isync
120823:
1209	/*
1210	 * For external and machine check interrupts, we need
1211	 * to call the Linux handler to process the interrupt.
1212	 * We do that by jumping to absolute address 0x500 for
1213	 * external interrupts, or the machine_check_fwnmi label
1214	 * for machine checks (since firmware might have patched
1215	 * the vector area at 0x200).  The [h]rfid at the end of the
1216	 * handler will return to the book3s_hv_interrupts.S code.
1217	 * For other interrupts we do the rfid to get back
1218	 * to the book3s_hv_interrupts.S code here.
1219	 */
1220	ld	r8, HSTATE_VMHANDLER(r13)
1221	ld	r7, HSTATE_HOST_MSR(r13)
1222
1223	cmpwi	cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1224	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
1225BEGIN_FTR_SECTION
1226	beq	11f
1227END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1228
1229	/* RFI into the highmem handler, or branch to interrupt handler */
1230	mfmsr	r6
1231	li	r0, MSR_RI
1232	andc	r6, r6, r0
1233	mtmsrd	r6, 1			/* Clear RI in MSR */
1234	mtsrr0	r8
1235	mtsrr1	r7
1236	beqa	0x500			/* external interrupt (PPC970) */
1237	beq	cr1, 13f		/* machine check */
1238	RFI
1239
1240	/* On POWER7, we have external interrupts set to use HSRR0/1 */
124111:	mtspr	SPRN_HSRR0, r8
1242	mtspr	SPRN_HSRR1, r7
1243	ba	0x500
1244
124513:	b	machine_check_fwnmi
1246
1247/*
1248 * Check whether an HDSI is an HPTE not found fault or something else.
1249 * If it is an HPTE not found fault that is due to the guest accessing
1250 * a page that they have mapped but which we have paged out, then
1251 * we continue on with the guest exit path.  In all other cases,
1252 * reflect the HDSI to the guest as a DSI.
1253 */
1254kvmppc_hdsi:
1255	mfspr	r4, SPRN_HDAR
1256	mfspr	r6, SPRN_HDSISR
1257	/* HPTE not found fault or protection fault? */
1258	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1259	beq	1f			/* if not, send it to the guest */
1260	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
1261	beq	3f
1262	clrrdi	r0, r4, 28
1263	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1264	bne	1f			/* if no SLB entry found */
12654:	std	r4, VCPU_FAULT_DAR(r9)
1266	stw	r6, VCPU_FAULT_DSISR(r9)
1267
1268	/* Search the hash table. */
1269	mr	r3, r9			/* vcpu pointer */
1270	li	r7, 1			/* data fault */
1271	bl	.kvmppc_hpte_hv_fault
1272	ld	r9, HSTATE_KVM_VCPU(r13)
1273	ld	r10, VCPU_PC(r9)
1274	ld	r11, VCPU_MSR(r9)
1275	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1276	cmpdi	r3, 0			/* retry the instruction */
1277	beq	6f
1278	cmpdi	r3, -1			/* handle in kernel mode */
1279	beq	guest_exit_cont
1280	cmpdi	r3, -2			/* MMIO emulation; need instr word */
1281	beq	2f
1282
1283	/* Synthesize a DSI for the guest */
1284	ld	r4, VCPU_FAULT_DAR(r9)
1285	mr	r6, r3
12861:	mtspr	SPRN_DAR, r4
1287	mtspr	SPRN_DSISR, r6
1288	mtspr	SPRN_SRR0, r10
1289	mtspr	SPRN_SRR1, r11
1290	li	r10, BOOK3S_INTERRUPT_DATA_STORAGE
1291	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
1292	rotldi	r11, r11, 63
1293fast_interrupt_c_return:
12946:	ld	r7, VCPU_CTR(r9)
1295	lwz	r8, VCPU_XER(r9)
1296	mtctr	r7
1297	mtxer	r8
1298	mr	r4, r9
1299	b	fast_guest_return
1300
13013:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
1302	ld	r5, KVM_VRMA_SLB_V(r5)
1303	b	4b
1304
1305	/* If this is for emulated MMIO, load the instruction word */
13062:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */
1307
1308	/* Set guest mode to 'jump over instruction' so if lwz faults
1309	 * we'll just continue at the next IP. */
1310	li	r0, KVM_GUEST_MODE_SKIP
1311	stb	r0, HSTATE_IN_GUEST(r13)
1312
1313	/* Do the access with MSR:DR enabled */
1314	mfmsr	r3
1315	ori	r4, r3, MSR_DR		/* Enable paging for data */
1316	mtmsrd	r4
1317	lwz	r8, 0(r10)
1318	mtmsrd	r3
1319
1320	/* Store the result */
1321	stw	r8, VCPU_LAST_INST(r9)
1322
1323	/* Unset guest mode. */
1324	li	r0, KVM_GUEST_MODE_NONE
1325	stb	r0, HSTATE_IN_GUEST(r13)
1326	b	guest_exit_cont
1327
1328/*
1329 * Similarly for an HISI, reflect it to the guest as an ISI unless
1330 * it is an HPTE not found fault for a page that we have paged out.
1331 */
1332kvmppc_hisi:
1333	andis.	r0, r11, SRR1_ISI_NOPT@h
1334	beq	1f
1335	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
1336	beq	3f
1337	clrrdi	r0, r10, 28
1338	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1339	bne	1f			/* if no SLB entry found */
13404:
1341	/* Search the hash table. */
1342	mr	r3, r9			/* vcpu pointer */
1343	mr	r4, r10
1344	mr	r6, r11
1345	li	r7, 0			/* instruction fault */
1346	bl	.kvmppc_hpte_hv_fault
1347	ld	r9, HSTATE_KVM_VCPU(r13)
1348	ld	r10, VCPU_PC(r9)
1349	ld	r11, VCPU_MSR(r9)
1350	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1351	cmpdi	r3, 0			/* retry the instruction */
1352	beq	fast_interrupt_c_return
1353	cmpdi	r3, -1			/* handle in kernel mode */
1354	beq	guest_exit_cont
1355
1356	/* Synthesize an ISI for the guest */
1357	mr	r11, r3
13581:	mtspr	SPRN_SRR0, r10
1359	mtspr	SPRN_SRR1, r11
1360	li	r10, BOOK3S_INTERRUPT_INST_STORAGE
1361	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
1362	rotldi	r11, r11, 63
1363	b	fast_interrupt_c_return
1364
13653:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
1366	ld	r5, KVM_VRMA_SLB_V(r6)
1367	b	4b
1368
1369/*
1370 * Try to handle an hcall in real mode.
1371 * Returns to the guest if we handle it, or continues on up to
1372 * the kernel if we can't (i.e. if we don't have a handler for
1373 * it, or if the handler returns H_TOO_HARD).
1374 */
1375	.globl	hcall_try_real_mode
1376hcall_try_real_mode:
1377	ld	r3,VCPU_GPR(R3)(r9)
1378	andi.	r0,r11,MSR_PR
1379	bne	guest_exit_cont
1380	clrrdi	r3,r3,2
1381	cmpldi	r3,hcall_real_table_end - hcall_real_table
1382	bge	guest_exit_cont
1383	LOAD_REG_ADDR(r4, hcall_real_table)
1384	lwzx	r3,r3,r4
1385	cmpwi	r3,0
1386	beq	guest_exit_cont
1387	add	r3,r3,r4
1388	mtctr	r3
1389	mr	r3,r9		/* get vcpu pointer */
1390	ld	r4,VCPU_GPR(R4)(r9)
1391	bctrl
1392	cmpdi	r3,H_TOO_HARD
1393	beq	hcall_real_fallback
1394	ld	r4,HSTATE_KVM_VCPU(r13)
1395	std	r3,VCPU_GPR(R3)(r4)
1396	ld	r10,VCPU_PC(r4)
1397	ld	r11,VCPU_MSR(r4)
1398	b	fast_guest_return
1399
1400	/* We've attempted a real mode hcall, but it's punted it back
1401	 * to userspace.  We need to restore some clobbered volatiles
1402	 * before resuming the pass-it-to-qemu path */
1403hcall_real_fallback:
1404	li	r12,BOOK3S_INTERRUPT_SYSCALL
1405	ld	r9, HSTATE_KVM_VCPU(r13)
1406
1407	b	guest_exit_cont
1408
1409	.globl	hcall_real_table
1410hcall_real_table:
1411	.long	0		/* 0 - unused */
1412	.long	.kvmppc_h_remove - hcall_real_table
1413	.long	.kvmppc_h_enter - hcall_real_table
1414	.long	.kvmppc_h_read - hcall_real_table
1415	.long	0		/* 0x10 - H_CLEAR_MOD */
1416	.long	0		/* 0x14 - H_CLEAR_REF */
1417	.long	.kvmppc_h_protect - hcall_real_table
1418	.long	0		/* 0x1c - H_GET_TCE */
1419	.long	.kvmppc_h_put_tce - hcall_real_table
1420	.long	0		/* 0x24 - H_SET_SPRG0 */
1421	.long	.kvmppc_h_set_dabr - hcall_real_table
1422	.long	0		/* 0x2c */
1423	.long	0		/* 0x30 */
1424	.long	0		/* 0x34 */
1425	.long	0		/* 0x38 */
1426	.long	0		/* 0x3c */
1427	.long	0		/* 0x40 */
1428	.long	0		/* 0x44 */
1429	.long	0		/* 0x48 */
1430	.long	0		/* 0x4c */
1431	.long	0		/* 0x50 */
1432	.long	0		/* 0x54 */
1433	.long	0		/* 0x58 */
1434	.long	0		/* 0x5c */
1435	.long	0		/* 0x60 */
1436#ifdef CONFIG_KVM_XICS
1437	.long	.kvmppc_rm_h_eoi - hcall_real_table
1438	.long	.kvmppc_rm_h_cppr - hcall_real_table
1439	.long	.kvmppc_rm_h_ipi - hcall_real_table
1440	.long	0		/* 0x70 - H_IPOLL */
1441	.long	.kvmppc_rm_h_xirr - hcall_real_table
1442#else
1443	.long	0		/* 0x64 - H_EOI */
1444	.long	0		/* 0x68 - H_CPPR */
1445	.long	0		/* 0x6c - H_IPI */
1446	.long	0		/* 0x70 - H_IPOLL */
1447	.long	0		/* 0x74 - H_XIRR */
1448#endif
1449	.long	0		/* 0x78 */
1450	.long	0		/* 0x7c */
1451	.long	0		/* 0x80 */
1452	.long	0		/* 0x84 */
1453	.long	0		/* 0x88 */
1454	.long	0		/* 0x8c */
1455	.long	0		/* 0x90 */
1456	.long	0		/* 0x94 */
1457	.long	0		/* 0x98 */
1458	.long	0		/* 0x9c */
1459	.long	0		/* 0xa0 */
1460	.long	0		/* 0xa4 */
1461	.long	0		/* 0xa8 */
1462	.long	0		/* 0xac */
1463	.long	0		/* 0xb0 */
1464	.long	0		/* 0xb4 */
1465	.long	0		/* 0xb8 */
1466	.long	0		/* 0xbc */
1467	.long	0		/* 0xc0 */
1468	.long	0		/* 0xc4 */
1469	.long	0		/* 0xc8 */
1470	.long	0		/* 0xcc */
1471	.long	0		/* 0xd0 */
1472	.long	0		/* 0xd4 */
1473	.long	0		/* 0xd8 */
1474	.long	0		/* 0xdc */
1475	.long	.kvmppc_h_cede - hcall_real_table
1476	.long	0		/* 0xe4 */
1477	.long	0		/* 0xe8 */
1478	.long	0		/* 0xec */
1479	.long	0		/* 0xf0 */
1480	.long	0		/* 0xf4 */
1481	.long	0		/* 0xf8 */
1482	.long	0		/* 0xfc */
1483	.long	0		/* 0x100 */
1484	.long	0		/* 0x104 */
1485	.long	0		/* 0x108 */
1486	.long	0		/* 0x10c */
1487	.long	0		/* 0x110 */
1488	.long	0		/* 0x114 */
1489	.long	0		/* 0x118 */
1490	.long	0		/* 0x11c */
1491	.long	0		/* 0x120 */
1492	.long	.kvmppc_h_bulk_remove - hcall_real_table
1493hcall_real_table_end:
1494
1495ignore_hdec:
1496	mr	r4,r9
1497	b	fast_guest_return
1498
1499_GLOBAL(kvmppc_h_set_dabr)
1500	std	r4,VCPU_DABR(r3)
1501	/* Work around P7 bug where DABR can get corrupted on mtspr */
15021:	mtspr	SPRN_DABR,r4
1503	mfspr	r5, SPRN_DABR
1504	cmpd	r4, r5
1505	bne	1b
1506	isync
1507	li	r3,0
1508	blr
1509
1510_GLOBAL(kvmppc_h_cede)
1511	ori	r11,r11,MSR_EE
1512	std	r11,VCPU_MSR(r3)
1513	li	r0,1
1514	stb	r0,VCPU_CEDED(r3)
1515	sync			/* order setting ceded vs. testing prodded */
1516	lbz	r5,VCPU_PRODDED(r3)
1517	cmpwi	r5,0
1518	bne	kvm_cede_prodded
1519	li	r0,0		/* set trap to 0 to say hcall is handled */
1520	stw	r0,VCPU_TRAP(r3)
1521	li	r0,H_SUCCESS
1522	std	r0,VCPU_GPR(R3)(r3)
1523BEGIN_FTR_SECTION
1524	b	kvm_cede_exit	/* just send it up to host on 970 */
1525END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
1526
1527	/*
1528	 * Set our bit in the bitmask of napping threads unless all the
1529	 * other threads are already napping, in which case we send this
1530	 * up to the host.
1531	 */
1532	ld	r5,HSTATE_KVM_VCORE(r13)
1533	lwz	r6,VCPU_PTID(r3)
1534	lwz	r8,VCORE_ENTRY_EXIT(r5)
1535	clrldi	r8,r8,56
1536	li	r0,1
1537	sld	r0,r0,r6
1538	addi	r6,r5,VCORE_NAPPING_THREADS
153931:	lwarx	r4,0,r6
1540	or	r4,r4,r0
1541	PPC_POPCNTW(R7,R4)
1542	cmpw	r7,r8
1543	bge	kvm_cede_exit
1544	stwcx.	r4,0,r6
1545	bne	31b
1546	li	r0,1
1547	stb	r0,HSTATE_NAPPING(r13)
1548	/* order napping_threads update vs testing entry_exit_count */
1549	lwsync
1550	mr	r4,r3
1551	lwz	r7,VCORE_ENTRY_EXIT(r5)
1552	cmpwi	r7,0x100
1553	bge	33f		/* another thread already exiting */
1554
1555/*
1556 * Although not specifically required by the architecture, POWER7
1557 * preserves the following registers in nap mode, even if an SMT mode
1558 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
1559 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
1560 */
1561	/* Save non-volatile GPRs */
1562	std	r14, VCPU_GPR(R14)(r3)
1563	std	r15, VCPU_GPR(R15)(r3)
1564	std	r16, VCPU_GPR(R16)(r3)
1565	std	r17, VCPU_GPR(R17)(r3)
1566	std	r18, VCPU_GPR(R18)(r3)
1567	std	r19, VCPU_GPR(R19)(r3)
1568	std	r20, VCPU_GPR(R20)(r3)
1569	std	r21, VCPU_GPR(R21)(r3)
1570	std	r22, VCPU_GPR(R22)(r3)
1571	std	r23, VCPU_GPR(R23)(r3)
1572	std	r24, VCPU_GPR(R24)(r3)
1573	std	r25, VCPU_GPR(R25)(r3)
1574	std	r26, VCPU_GPR(R26)(r3)
1575	std	r27, VCPU_GPR(R27)(r3)
1576	std	r28, VCPU_GPR(R28)(r3)
1577	std	r29, VCPU_GPR(R29)(r3)
1578	std	r30, VCPU_GPR(R30)(r3)
1579	std	r31, VCPU_GPR(R31)(r3)
1580
1581	/* save FP state */
1582	bl	.kvmppc_save_fp
1583
1584	/*
1585	 * Take a nap until a decrementer or external interrupt occurs,
1586	 * with PECE1 (wake on decr) and PECE0 (wake on external) set in LPCR
1587	 */
1588	li	r0,1
1589	stb	r0,HSTATE_HWTHREAD_REQ(r13)
1590	mfspr	r5,SPRN_LPCR
1591	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
1592	mtspr	SPRN_LPCR,r5
1593	isync
1594	li	r0, 0
1595	std	r0, HSTATE_SCRATCH0(r13)
1596	ptesync
1597	ld	r0, HSTATE_SCRATCH0(r13)
15981:	cmpd	r0, r0
1599	bne	1b
1600	nap
1601	b	.
1602
1603kvm_end_cede:
1604	/* get vcpu pointer */
1605	ld	r4, HSTATE_KVM_VCPU(r13)
1606
1607	/* Woken by external or decrementer interrupt */
1608	ld	r1, HSTATE_HOST_R1(r13)
1609
1610	/* load up FP state */
1611	bl	kvmppc_load_fp
1612
1613	/* Load NV GPRS */
1614	ld	r14, VCPU_GPR(R14)(r4)
1615	ld	r15, VCPU_GPR(R15)(r4)
1616	ld	r16, VCPU_GPR(R16)(r4)
1617	ld	r17, VCPU_GPR(R17)(r4)
1618	ld	r18, VCPU_GPR(R18)(r4)
1619	ld	r19, VCPU_GPR(R19)(r4)
1620	ld	r20, VCPU_GPR(R20)(r4)
1621	ld	r21, VCPU_GPR(R21)(r4)
1622	ld	r22, VCPU_GPR(R22)(r4)
1623	ld	r23, VCPU_GPR(R23)(r4)
1624	ld	r24, VCPU_GPR(R24)(r4)
1625	ld	r25, VCPU_GPR(R25)(r4)
1626	ld	r26, VCPU_GPR(R26)(r4)
1627	ld	r27, VCPU_GPR(R27)(r4)
1628	ld	r28, VCPU_GPR(R28)(r4)
1629	ld	r29, VCPU_GPR(R29)(r4)
1630	ld	r30, VCPU_GPR(R30)(r4)
1631	ld	r31, VCPU_GPR(R31)(r4)
1632
1633	/* clear our bit in vcore->napping_threads */
163433:	ld	r5,HSTATE_KVM_VCORE(r13)
1635	lwz	r3,VCPU_PTID(r4)
1636	li	r0,1
1637	sld	r0,r0,r3
1638	addi	r6,r5,VCORE_NAPPING_THREADS
163932:	lwarx	r7,0,r6
1640	andc	r7,r7,r0
1641	stwcx.	r7,0,r6
1642	bne	32b
1643	li	r0,0
1644	stb	r0,HSTATE_NAPPING(r13)
1645
1646	/* Check the wake reason in SRR1 to see why we got here */
1647	mfspr	r3, SPRN_SRR1
1648	rlwinm	r3, r3, 44-31, 0x7	/* extract wake reason field */
1649	cmpwi	r3, 4			/* was it an external interrupt? */
1650	li	r12, BOOK3S_INTERRUPT_EXTERNAL
1651	mr	r9, r4
1652	ld	r10, VCPU_PC(r9)
1653	ld	r11, VCPU_MSR(r9)
1654	beq	do_ext_interrupt	/* if so */
1655
1656	/* see if any other thread is already exiting */
1657	lwz	r0,VCORE_ENTRY_EXIT(r5)
1658	cmpwi	r0,0x100
1659	blt	kvmppc_cede_reentry	/* if not go back to guest */
1660
1661	/* some threads are exiting, so go to the guest exit path */
1662	b	hcall_real_fallback
1663
1664	/* cede when already previously prodded case */
1665kvm_cede_prodded:
1666	li	r0,0
1667	stb	r0,VCPU_PRODDED(r3)
1668	sync			/* order testing prodded vs. clearing ceded */
1669	stb	r0,VCPU_CEDED(r3)
1670	li	r3,H_SUCCESS
1671	blr
1672
1673	/* we've ceded but we want to give control to the host */
1674kvm_cede_exit:
1675	b	hcall_real_fallback
1676
1677	/* Try to handle a machine check in real mode */
1678machine_check_realmode:
1679	mr	r3, r9		/* get vcpu pointer */
1680	bl	.kvmppc_realmode_machine_check
1681	nop
1682	cmpdi	r3, 0		/* continue exiting from guest? */
1683	ld	r9, HSTATE_KVM_VCPU(r13)
1684	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1685	beq	mc_cont
1686	/* If not, deliver a machine check.  SRR0/1 are already set */
1687	li	r10, BOOK3S_INTERRUPT_MACHINE_CHECK
1688	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
1689	rotldi	r11, r11, 63
1690	b	fast_interrupt_c_return
1691
1692secondary_too_late:
1693	ld	r5,HSTATE_KVM_VCORE(r13)
1694	HMT_LOW
169513:	lbz	r3,VCORE_IN_GUEST(r5)
1696	cmpwi	r3,0
1697	bne	13b
1698	HMT_MEDIUM
1699	ld	r11,PACA_SLBSHADOWPTR(r13)
1700
1701	.rept	SLB_NUM_BOLTED
1702	ld	r5,SLBSHADOW_SAVEAREA(r11)
1703	ld	r6,SLBSHADOW_SAVEAREA+8(r11)
1704	andis.	r7,r5,SLB_ESID_V@h
1705	beq	1f
1706	slbmte	r6,r5
17071:	addi	r11,r11,16
1708	.endr
1709
1710secondary_nap:
1711	/* Clear our vcpu pointer so we don't come back in early */
1712	li	r0, 0
1713	std	r0, HSTATE_KVM_VCPU(r13)
1714	lwsync
1715	/* Clear any pending IPI - assume we're a secondary thread */
1716	ld	r5, HSTATE_XICS_PHYS(r13)
1717	li	r7, XICS_XIRR
1718	lwzcix	r3, r5, r7		/* ack any pending interrupt */
1719	rlwinm.	r0, r3, 0, 0xffffff	/* any pending? */
1720	beq	37f
1721	sync
1722	li	r0, 0xff
1723	li	r6, XICS_MFRR
1724	stbcix	r0, r5, r6		/* clear the IPI */
1725	stwcix	r3, r5, r7		/* EOI it */
172637:	sync
1727
1728	/* increment the nap count and then go to nap mode */
1729	ld	r4, HSTATE_KVM_VCORE(r13)
1730	addi	r4, r4, VCORE_NAP_COUNT
1731	lwsync				/* make previous updates visible */
173251:	lwarx	r3, 0, r4
1733	addi	r3, r3, 1
1734	stwcx.	r3, 0, r4
1735	bne	51b
1736
1737kvm_no_guest:
1738	li	r0, KVM_HWTHREAD_IN_NAP
1739	stb	r0, HSTATE_HWTHREAD_STATE(r13)
1740
1741	li	r3, LPCR_PECE0
1742	mfspr	r4, SPRN_LPCR
1743	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
1744	mtspr	SPRN_LPCR, r4
1745	isync
1746	std	r0, HSTATE_SCRATCH0(r13)
1747	ptesync
1748	ld	r0, HSTATE_SCRATCH0(r13)
17491:	cmpd	r0, r0
1750	bne	1b
1751	nap
1752	b	.
1753
1754/*
1755 * Save away FP, VMX and VSX registers.
1756 * r3 = vcpu pointer
1757 */
1758_GLOBAL(kvmppc_save_fp)
1759	mfmsr	r5
1760	ori	r8,r5,MSR_FP
1761#ifdef CONFIG_ALTIVEC
1762BEGIN_FTR_SECTION
1763	oris	r8,r8,MSR_VEC@h
1764END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1765#endif
1766#ifdef CONFIG_VSX
1767BEGIN_FTR_SECTION
1768	oris	r8,r8,MSR_VSX@h
1769END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1770#endif
1771	mtmsrd	r8
1772	isync
1773#ifdef CONFIG_VSX
1774BEGIN_FTR_SECTION
1775	reg = 0
1776	.rept	32
1777	li	r6,reg*16+VCPU_VSRS
1778	STXVD2X(reg,R6,R3)
1779	reg = reg + 1
1780	.endr
1781FTR_SECTION_ELSE
1782#endif
1783	reg = 0
1784	.rept	32
1785	stfd	reg,reg*8+VCPU_FPRS(r3)
1786	reg = reg + 1
1787	.endr
1788#ifdef CONFIG_VSX
1789ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
1790#endif
1791	mffs	fr0
1792	stfd	fr0,VCPU_FPSCR(r3)
1793
1794#ifdef CONFIG_ALTIVEC
1795BEGIN_FTR_SECTION
1796	reg = 0
1797	.rept	32
1798	li	r6,reg*16+VCPU_VRS
1799	stvx	reg,r6,r3
1800	reg = reg + 1
1801	.endr
1802	mfvscr	vr0
1803	li	r6,VCPU_VSCR
1804	stvx	vr0,r6,r3
1805END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1806#endif
1807	mfspr	r6,SPRN_VRSAVE
1808	stw	r6,VCPU_VRSAVE(r3)
1809	mtmsrd	r5
1810	isync
1811	blr
1812
1813/*
1814 * Load up FP, VMX and VSX registers
1815 * r4 = vcpu pointer
1816 */
1817	.globl	kvmppc_load_fp
1818kvmppc_load_fp:
1819	mfmsr	r9
1820	ori	r8,r9,MSR_FP
1821#ifdef CONFIG_ALTIVEC
1822BEGIN_FTR_SECTION
1823	oris	r8,r8,MSR_VEC@h
1824END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1825#endif
1826#ifdef CONFIG_VSX
1827BEGIN_FTR_SECTION
1828	oris	r8,r8,MSR_VSX@h
1829END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1830#endif
1831	mtmsrd	r8
1832	isync
1833	lfd	fr0,VCPU_FPSCR(r4)
1834	MTFSF_L(fr0)
1835#ifdef CONFIG_VSX
1836BEGIN_FTR_SECTION
1837	reg = 0
1838	.rept	32
1839	li	r7,reg*16+VCPU_VSRS
1840	LXVD2X(reg,R7,R4)
1841	reg = reg + 1
1842	.endr
1843FTR_SECTION_ELSE
1844#endif
1845	reg = 0
1846	.rept	32
1847	lfd	reg,reg*8+VCPU_FPRS(r4)
1848	reg = reg + 1
1849	.endr
1850#ifdef CONFIG_VSX
1851ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
1852#endif
1853
1854#ifdef CONFIG_ALTIVEC
1855BEGIN_FTR_SECTION
1856	li	r7,VCPU_VSCR
1857	lvx	vr0,r7,r4
1858	mtvscr	vr0
1859	reg = 0
1860	.rept	32
1861	li	r7,reg*16+VCPU_VRS
1862	lvx	reg,r7,r4
1863	reg = reg + 1
1864	.endr
1865END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1866#endif
1867	lwz	r7,VCPU_VRSAVE(r4)
1868	mtspr	SPRN_VRSAVE,r7
1869	blr
1870