1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20#include <asm/ppc_asm.h>
21#include <asm/kvm_asm.h>
22#include <asm/reg.h>
23#include <asm/mmu.h>
24#include <asm/page.h>
25#include <asm/ptrace.h>
26#include <asm/hvcall.h>
27#include <asm/asm-offsets.h>
28#include <asm/exception-64s.h>
29#include <asm/kvm_book3s_asm.h>
30#include <asm/mmu-hash64.h>
31#include <asm/tm.h>
32
33#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
34
35#ifdef __LITTLE_ENDIAN__
36#error Need to fix lppaca and SLB shadow accesses in little endian mode
37#endif
38
39/* Values in HSTATE_NAPPING(r13) */
40#define NAPPING_CEDE	1
41#define NAPPING_NOVCPU	2
42
43/*
44 * Call kvmppc_hv_entry in real mode.
45 * Must be called with interrupts hard-disabled.
46 *
47 * Input Registers:
48 *
49 * LR = return address to continue at after eventually re-enabling MMU
50 */
51_GLOBAL(kvmppc_hv_entry_trampoline)
52	mflr	r0
53	std	r0, PPC_LR_STKOFF(r1)
54	stdu	r1, -112(r1)
55	mfmsr	r10
56	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
57	li	r0,MSR_RI
58	andc	r0,r10,r0
59	li	r6,MSR_IR | MSR_DR
60	andc	r6,r10,r6
61	mtmsrd	r0,1		/* clear RI in MSR */
62	mtsrr0	r5
63	mtsrr1	r6
64	RFI
65
66kvmppc_call_hv_entry:
67	ld	r4, HSTATE_KVM_VCPU(r13)
68	bl	kvmppc_hv_entry
69
70	/* Back from guest - restore host state and return to caller */
71
72BEGIN_FTR_SECTION
73	/* Restore host DABR and DABRX */
74	ld	r5,HSTATE_DABR(r13)
75	li	r6,7
76	mtspr	SPRN_DABR,r5
77	mtspr	SPRN_DABRX,r6
78END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
79
80	/* Restore SPRG3 */
81	ld	r3,PACA_SPRG_VDSO(r13)
82	mtspr	SPRN_SPRG_VDSO_WRITE,r3
83
84	/* Reload the host's PMU registers */
85	ld	r3, PACALPPACAPTR(r13)	/* is the host using the PMU? */
86	lbz	r4, LPPACA_PMCINUSE(r3)
87	cmpwi	r4, 0
88	beq	23f			/* skip if not */
89	lwz	r3, HSTATE_PMC(r13)
90	lwz	r4, HSTATE_PMC + 4(r13)
91	lwz	r5, HSTATE_PMC + 8(r13)
92	lwz	r6, HSTATE_PMC + 12(r13)
93	lwz	r8, HSTATE_PMC + 16(r13)
94	lwz	r9, HSTATE_PMC + 20(r13)
95BEGIN_FTR_SECTION
96	lwz	r10, HSTATE_PMC + 24(r13)
97	lwz	r11, HSTATE_PMC + 28(r13)
98END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
99	mtspr	SPRN_PMC1, r3
100	mtspr	SPRN_PMC2, r4
101	mtspr	SPRN_PMC3, r5
102	mtspr	SPRN_PMC4, r6
103	mtspr	SPRN_PMC5, r8
104	mtspr	SPRN_PMC6, r9
105BEGIN_FTR_SECTION
106	mtspr	SPRN_PMC7, r10
107	mtspr	SPRN_PMC8, r11
108END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
109	ld	r3, HSTATE_MMCR(r13)
110	ld	r4, HSTATE_MMCR + 8(r13)
111	ld	r5, HSTATE_MMCR + 16(r13)
112	ld	r6, HSTATE_MMCR + 24(r13)
113	ld	r7, HSTATE_MMCR + 32(r13)
114	mtspr	SPRN_MMCR1, r4
115	mtspr	SPRN_MMCRA, r5
116	mtspr	SPRN_SIAR, r6
117	mtspr	SPRN_SDAR, r7
118BEGIN_FTR_SECTION
119	ld	r8, HSTATE_MMCR + 40(r13)
120	ld	r9, HSTATE_MMCR + 48(r13)
121	mtspr	SPRN_MMCR2, r8
122	mtspr	SPRN_SIER, r9
123END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
124	mtspr	SPRN_MMCR0, r3
125	isync
12623:
127
128	/*
129	 * Reload DEC.  HDEC interrupts were disabled when
130	 * we reloaded the host's LPCR value.
131	 */
132	ld	r3, HSTATE_DECEXP(r13)
133	mftb	r4
134	subf	r4, r4, r3
135	mtspr	SPRN_DEC, r4
136
137	/*
138	 * For external and machine check interrupts, we need
139	 * to call the Linux handler to process the interrupt.
140	 * We do that by jumping to absolute address 0x500 for
141	 * external interrupts, or the machine_check_fwnmi label
142	 * for machine checks (since firmware might have patched
143	 * the vector area at 0x200).  The [h]rfid at the end of the
144	 * handler will return to the book3s_hv_interrupts.S code.
145	 * For other interrupts we do the rfid to get back
146	 * to the book3s_hv_interrupts.S code here.
147	 */
148	ld	r8, 112+PPC_LR_STKOFF(r1)
149	addi	r1, r1, 112
150	ld	r7, HSTATE_HOST_MSR(r13)
151
152	cmpwi	cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
153	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
154BEGIN_FTR_SECTION
155	beq	11f
156END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
157
158	/* RFI into the highmem handler, or branch to interrupt handler */
159	mfmsr	r6
160	li	r0, MSR_RI
161	andc	r6, r6, r0
162	mtmsrd	r6, 1			/* Clear RI in MSR */
163	mtsrr0	r8
164	mtsrr1	r7
165	beqa	0x500			/* external interrupt (PPC970) */
166	beq	cr1, 13f		/* machine check */
167	RFI
168
169	/* On POWER7, we have external interrupts set to use HSRR0/1 */
17011:	mtspr	SPRN_HSRR0, r8
171	mtspr	SPRN_HSRR1, r7
172	ba	0x500
173
17413:	b	machine_check_fwnmi
175
176kvmppc_primary_no_guest:
177	/* We handle this much like a ceded vcpu */
178	/* set our bit in napping_threads */
179	ld	r5, HSTATE_KVM_VCORE(r13)
180	lbz	r7, HSTATE_PTID(r13)
181	li	r0, 1
182	sld	r0, r0, r7
183	addi	r6, r5, VCORE_NAPPING_THREADS
1841:	lwarx	r3, 0, r6
185	or	r3, r3, r0
186	stwcx.	r3, 0, r6
187	bne	1b
188	/* order napping_threads update vs testing entry_exit_count */
189	isync
190	li	r12, 0
191	lwz	r7, VCORE_ENTRY_EXIT(r5)
192	cmpwi	r7, 0x100
193	bge	kvm_novcpu_exit	/* another thread already exiting */
194	li	r3, NAPPING_NOVCPU
195	stb	r3, HSTATE_NAPPING(r13)
196	li	r3, 1
197	stb	r3, HSTATE_HWTHREAD_REQ(r13)
198
199	b	kvm_do_nap
200
201kvm_novcpu_wakeup:
202	ld	r1, HSTATE_HOST_R1(r13)
203	ld	r5, HSTATE_KVM_VCORE(r13)
204	li	r0, 0
205	stb	r0, HSTATE_NAPPING(r13)
206	stb	r0, HSTATE_HWTHREAD_REQ(r13)
207
208	/* check the wake reason */
209	bl	kvmppc_check_wake_reason
210
211	/* see if any other thread is already exiting */
212	lwz	r0, VCORE_ENTRY_EXIT(r5)
213	cmpwi	r0, 0x100
214	bge	kvm_novcpu_exit
215
216	/* clear our bit in napping_threads */
217	lbz	r7, HSTATE_PTID(r13)
218	li	r0, 1
219	sld	r0, r0, r7
220	addi	r6, r5, VCORE_NAPPING_THREADS
2214:	lwarx	r7, 0, r6
222	andc	r7, r7, r0
223	stwcx.	r7, 0, r6
224	bne	4b
225
226	/* See if the wake reason means we need to exit */
227	cmpdi	r3, 0
228	bge	kvm_novcpu_exit
229
230	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
231	ld	r4, HSTATE_KVM_VCPU(r13)
232	cmpdi	r4, 0
233	bne	kvmppc_got_guest
234
235kvm_novcpu_exit:
236	b	hdec_soon
237
238/*
239 * We come in here when wakened from nap mode.
240 * Relocation is off and most register values are lost.
241 * r13 points to the PACA.
242 */
243	.globl	kvm_start_guest
244kvm_start_guest:
245	ld	r2,PACATOC(r13)
246
247	li	r0,KVM_HWTHREAD_IN_KVM
248	stb	r0,HSTATE_HWTHREAD_STATE(r13)
249
250	/* NV GPR values from power7_idle() will no longer be valid */
251	li	r0,1
252	stb	r0,PACA_NAPSTATELOST(r13)
253
254	/* were we napping due to cede? */
255	lbz	r0,HSTATE_NAPPING(r13)
256	cmpwi	r0,NAPPING_CEDE
257	beq	kvm_end_cede
258	cmpwi	r0,NAPPING_NOVCPU
259	beq	kvm_novcpu_wakeup
260
261	ld	r1,PACAEMERGSP(r13)
262	subi	r1,r1,STACK_FRAME_OVERHEAD
263
264	/*
265	 * We weren't napping due to cede, so this must be a secondary
266	 * thread being woken up to run a guest, or being woken up due
267	 * to a stray IPI.  (Or due to some machine check or hypervisor
268	 * maintenance interrupt while the core is in KVM.)
269	 */
270
271	/* Check the wake reason in SRR1 to see why we got here */
272	bl	kvmppc_check_wake_reason
273	cmpdi	r3, 0
274	bge	kvm_no_guest
275
276	/* get vcpu pointer, NULL if we have no vcpu to run */
277	ld	r4,HSTATE_KVM_VCPU(r13)
278	cmpdi	r4,0
279	/* if we have no vcpu to run, go back to sleep */
280	beq	kvm_no_guest
281
282	/* Set HSTATE_DSCR(r13) to something sensible */
283	LOAD_REG_ADDR(r6, dscr_default)
284	ld	r6, 0(r6)
285	std	r6, HSTATE_DSCR(r13)
286
287	bl	kvmppc_hv_entry
288
289	/* Back from the guest, go back to nap */
290	/* Clear our vcpu pointer so we don't come back in early */
291	li	r0, 0
292	std	r0, HSTATE_KVM_VCPU(r13)
293	/*
294	 * Make sure we clear HSTATE_KVM_VCPU(r13) before incrementing
295	 * the nap_count, because once the increment to nap_count is
296	 * visible we could be given another vcpu.
297	 */
298	lwsync
299
300	/* increment the nap count and then go to nap mode */
301	ld	r4, HSTATE_KVM_VCORE(r13)
302	addi	r4, r4, VCORE_NAP_COUNT
30351:	lwarx	r3, 0, r4
304	addi	r3, r3, 1
305	stwcx.	r3, 0, r4
306	bne	51b
307
308kvm_no_guest:
309	li	r0, KVM_HWTHREAD_IN_NAP
310	stb	r0, HSTATE_HWTHREAD_STATE(r13)
311kvm_do_nap:
312	li	r3, LPCR_PECE0
313	mfspr	r4, SPRN_LPCR
314	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
315	mtspr	SPRN_LPCR, r4
316	isync
317	std	r0, HSTATE_SCRATCH0(r13)
318	ptesync
319	ld	r0, HSTATE_SCRATCH0(r13)
3201:	cmpd	r0, r0
321	bne	1b
322	nap
323	b	.
324
325/******************************************************************************
326 *                                                                            *
327 *                               Entry code                                   *
328 *                                                                            *
329 *****************************************************************************/
330
331.global kvmppc_hv_entry
332kvmppc_hv_entry:
333
334	/* Required state:
335	 *
336	 * R4 = vcpu pointer (or NULL)
337	 * MSR = ~IR|DR
338	 * R13 = PACA
339	 * R1 = host R1
340	 * all other volatile GPRS = free
341	 */
342	mflr	r0
343	std	r0, PPC_LR_STKOFF(r1)
344	stdu	r1, -112(r1)
345
346	/* Save R1 in the PACA */
347	std	r1, HSTATE_HOST_R1(r13)
348
349	li	r6, KVM_GUEST_MODE_HOST_HV
350	stb	r6, HSTATE_IN_GUEST(r13)
351
352	/* Clear out SLB */
353	li	r6,0
354	slbmte	r6,r6
355	slbia
356	ptesync
357
358BEGIN_FTR_SECTION
359	b	30f
360END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
361	/*
362	 * POWER7 host -> guest partition switch code.
363	 * We don't have to lock against concurrent tlbies,
364	 * but we do have to coordinate across hardware threads.
365	 */
366	/* Increment entry count iff exit count is zero. */
367	ld	r5,HSTATE_KVM_VCORE(r13)
368	addi	r9,r5,VCORE_ENTRY_EXIT
36921:	lwarx	r3,0,r9
370	cmpwi	r3,0x100		/* any threads starting to exit? */
371	bge	secondary_too_late	/* if so we're too late to the party */
372	addi	r3,r3,1
373	stwcx.	r3,0,r9
374	bne	21b
375
376	/* Primary thread switches to guest partition. */
377	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
378	lbz	r6,HSTATE_PTID(r13)
379	cmpwi	r6,0
380	bne	20f
381	ld	r6,KVM_SDR1(r9)
382	lwz	r7,KVM_LPID(r9)
383	li	r0,LPID_RSVD		/* switch to reserved LPID */
384	mtspr	SPRN_LPID,r0
385	ptesync
386	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
387	mtspr	SPRN_LPID,r7
388	isync
389
390	/* See if we need to flush the TLB */
391	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
392	clrldi	r7,r6,64-6		/* extract bit number (6 bits) */
393	srdi	r6,r6,6			/* doubleword number */
394	sldi	r6,r6,3			/* address offset */
395	add	r6,r6,r9
396	addi	r6,r6,KVM_NEED_FLUSH	/* dword in kvm->arch.need_tlb_flush */
397	li	r0,1
398	sld	r0,r0,r7
399	ld	r7,0(r6)
400	and.	r7,r7,r0
401	beq	22f
40223:	ldarx	r7,0,r6			/* if set, clear the bit */
403	andc	r7,r7,r0
404	stdcx.	r7,0,r6
405	bne	23b
406	/* Flush the TLB of any entries for this LPID */
407	/* use arch 2.07S as a proxy for POWER8 */
408BEGIN_FTR_SECTION
409	li	r6,512			/* POWER8 has 512 sets */
410FTR_SECTION_ELSE
411	li	r6,128			/* POWER7 has 128 sets */
412ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
413	mtctr	r6
414	li	r7,0x800		/* IS field = 0b10 */
415	ptesync
41628:	tlbiel	r7
417	addi	r7,r7,0x1000
418	bdnz	28b
419	ptesync
420
421	/* Add timebase offset onto timebase */
42222:	ld	r8,VCORE_TB_OFFSET(r5)
423	cmpdi	r8,0
424	beq	37f
425	mftb	r6		/* current host timebase */
426	add	r8,r8,r6
427	mtspr	SPRN_TBU40,r8	/* update upper 40 bits */
428	mftb	r7		/* check if lower 24 bits overflowed */
429	clrldi	r6,r6,40
430	clrldi	r7,r7,40
431	cmpld	r7,r6
432	bge	37f
433	addis	r8,r8,0x100	/* if so, increment upper 40 bits */
434	mtspr	SPRN_TBU40,r8
435
436	/* Load guest PCR value to select appropriate compat mode */
43737:	ld	r7, VCORE_PCR(r5)
438	cmpdi	r7, 0
439	beq	38f
440	mtspr	SPRN_PCR, r7
44138:
442
443BEGIN_FTR_SECTION
444	/* DPDES is shared between threads */
445	ld	r8, VCORE_DPDES(r5)
446	mtspr	SPRN_DPDES, r8
447END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
448
449	li	r0,1
450	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
451	b	10f
452
453	/* Secondary threads wait for primary to have done partition switch */
45420:	lbz	r0,VCORE_IN_GUEST(r5)
455	cmpwi	r0,0
456	beq	20b
457
458	/* Set LPCR and RMOR. */
45910:	ld	r8,VCORE_LPCR(r5)
460	mtspr	SPRN_LPCR,r8
461	ld	r8,KVM_RMOR(r9)
462	mtspr	SPRN_RMOR,r8
463	isync
464
465	/* Check if HDEC expires soon */
466	mfspr	r3,SPRN_HDEC
467	cmpwi	r3,512		/* 1 microsecond */
468	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
469	blt	hdec_soon
470	b	31f
471
472	/*
473	 * PPC970 host -> guest partition switch code.
474	 * We have to lock against concurrent tlbies,
475	 * using native_tlbie_lock to lock against host tlbies
476	 * and kvm->arch.tlbie_lock to lock against guest tlbies.
477	 * We also have to invalidate the TLB since its
478	 * entries aren't tagged with the LPID.
479	 */
48030:	ld	r5,HSTATE_KVM_VCORE(r13)
481	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
482
483	/* first take native_tlbie_lock */
484	.section ".toc","aw"
485toc_tlbie_lock:
486	.tc	native_tlbie_lock[TC],native_tlbie_lock
487	.previous
488	ld	r3,toc_tlbie_lock@toc(2)
489#ifdef __BIG_ENDIAN__
490	lwz	r8,PACA_LOCK_TOKEN(r13)
491#else
492	lwz	r8,PACAPACAINDEX(r13)
493#endif
49424:	lwarx	r0,0,r3
495	cmpwi	r0,0
496	bne	24b
497	stwcx.	r8,0,r3
498	bne	24b
499	isync
500
501	ld	r5,HSTATE_KVM_VCORE(r13)
502	ld	r7,VCORE_LPCR(r5)	/* use vcore->lpcr to store HID4 */
503	li	r0,0x18f
504	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
505	or	r0,r7,r0
506	ptesync
507	sync
508	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
509	isync
510	li	r0,0
511	stw	r0,0(r3)		/* drop native_tlbie_lock */
512
513	/* invalidate the whole TLB */
514	li	r0,256
515	mtctr	r0
516	li	r6,0
51725:	tlbiel	r6
518	addi	r6,r6,0x1000
519	bdnz	25b
520	ptesync
521
522	/* Take the guest's tlbie_lock */
523	addi	r3,r9,KVM_TLBIE_LOCK
52424:	lwarx	r0,0,r3
525	cmpwi	r0,0
526	bne	24b
527	stwcx.	r8,0,r3
528	bne	24b
529	isync
530	ld	r6,KVM_SDR1(r9)
531	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
532
533	/* Set up HID4 with the guest's LPID etc. */
534	sync
535	mtspr	SPRN_HID4,r7
536	isync
537
538	/* drop the guest's tlbie_lock */
539	li	r0,0
540	stw	r0,0(r3)
541
542	/* Check if HDEC expires soon */
543	mfspr	r3,SPRN_HDEC
544	cmpwi	r3,10
545	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
546	blt	hdec_soon
547
548	/* Enable HDEC interrupts */
549	mfspr	r0,SPRN_HID0
550	li	r3,1
551	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
552	sync
553	mtspr	SPRN_HID0,r0
554	mfspr	r0,SPRN_HID0
555	mfspr	r0,SPRN_HID0
556	mfspr	r0,SPRN_HID0
557	mfspr	r0,SPRN_HID0
558	mfspr	r0,SPRN_HID0
559	mfspr	r0,SPRN_HID0
56031:
561	/* Do we have a guest vcpu to run? */
562	cmpdi	r4, 0
563	beq	kvmppc_primary_no_guest
564kvmppc_got_guest:
565
566	/* Load up guest SLB entries */
567	lwz	r5,VCPU_SLB_MAX(r4)
568	cmpwi	r5,0
569	beq	9f
570	mtctr	r5
571	addi	r6,r4,VCPU_SLB
5721:	ld	r8,VCPU_SLB_E(r6)
573	ld	r9,VCPU_SLB_V(r6)
574	slbmte	r9,r8
575	addi	r6,r6,VCPU_SLB_SIZE
576	bdnz	1b
5779:
578	/* Increment yield count if they have a VPA */
579	ld	r3, VCPU_VPA(r4)
580	cmpdi	r3, 0
581	beq	25f
582	lwz	r5, LPPACA_YIELDCOUNT(r3)
583	addi	r5, r5, 1
584	stw	r5, LPPACA_YIELDCOUNT(r3)
585	li	r6, 1
586	stb	r6, VCPU_VPA_DIRTY(r4)
58725:
588
589BEGIN_FTR_SECTION
590	/* Save purr/spurr */
591	mfspr	r5,SPRN_PURR
592	mfspr	r6,SPRN_SPURR
593	std	r5,HSTATE_PURR(r13)
594	std	r6,HSTATE_SPURR(r13)
595	ld	r7,VCPU_PURR(r4)
596	ld	r8,VCPU_SPURR(r4)
597	mtspr	SPRN_PURR,r7
598	mtspr	SPRN_SPURR,r8
599END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
600
601BEGIN_FTR_SECTION
602	/* Set partition DABR */
603	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
604	lwz	r5,VCPU_DABRX(r4)
605	ld	r6,VCPU_DABR(r4)
606	mtspr	SPRN_DABRX,r5
607	mtspr	SPRN_DABR,r6
608 BEGIN_FTR_SECTION_NESTED(89)
609	isync
610 END_FTR_SECTION_NESTED(CPU_FTR_ARCH_206, CPU_FTR_ARCH_206, 89)
611END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
612
613#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
614BEGIN_FTR_SECTION
615	b	skip_tm
616END_FTR_SECTION_IFCLR(CPU_FTR_TM)
617
618	/* Turn on TM/FP/VSX/VMX so we can restore them. */
619	mfmsr	r5
620	li	r6, MSR_TM >> 32
621	sldi	r6, r6, 32
622	or	r5, r5, r6
623	ori	r5, r5, MSR_FP
624	oris	r5, r5, (MSR_VEC | MSR_VSX)@h
625	mtmsrd	r5
626
627	/*
628	 * The user may change these outside of a transaction, so they must
629	 * always be context switched.
630	 */
631	ld	r5, VCPU_TFHAR(r4)
632	ld	r6, VCPU_TFIAR(r4)
633	ld	r7, VCPU_TEXASR(r4)
634	mtspr	SPRN_TFHAR, r5
635	mtspr	SPRN_TFIAR, r6
636	mtspr	SPRN_TEXASR, r7
637
638	ld	r5, VCPU_MSR(r4)
639	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
640	beq	skip_tm	/* TM not active in guest */
641
642	/* Make sure the failure summary is set, otherwise we'll program check
643	 * when we trechkpt.  It's possible that this might have been not set
644	 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
645	 * host.
646	 */
647	oris	r7, r7, (TEXASR_FS)@h
648	mtspr	SPRN_TEXASR, r7
649
650	/*
651	 * We need to load up the checkpointed state for the guest.
652	 * We need to do this early as it will blow away any GPRs, VSRs and
653	 * some SPRs.
654	 */
655
656	mr	r31, r4
657	addi	r3, r31, VCPU_FPRS_TM
658	bl	.load_fp_state
659	addi	r3, r31, VCPU_VRS_TM
660	bl	.load_vr_state
661	mr	r4, r31
662	lwz	r7, VCPU_VRSAVE_TM(r4)
663	mtspr	SPRN_VRSAVE, r7
664
665	ld	r5, VCPU_LR_TM(r4)
666	lwz	r6, VCPU_CR_TM(r4)
667	ld	r7, VCPU_CTR_TM(r4)
668	ld	r8, VCPU_AMR_TM(r4)
669	ld	r9, VCPU_TAR_TM(r4)
670	mtlr	r5
671	mtcr	r6
672	mtctr	r7
673	mtspr	SPRN_AMR, r8
674	mtspr	SPRN_TAR, r9
675
676	/*
677	 * Load up PPR and DSCR values but don't put them in the actual SPRs
678	 * till the last moment to avoid running with userspace PPR and DSCR for
679	 * too long.
680	 */
681	ld	r29, VCPU_DSCR_TM(r4)
682	ld	r30, VCPU_PPR_TM(r4)
683
684	std	r2, PACATMSCRATCH(r13) /* Save TOC */
685
686	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
687	li	r5, 0
688	mtmsrd	r5, 1
689
690	/* Load GPRs r0-r28 */
691	reg = 0
692	.rept	29
693	ld	reg, VCPU_GPRS_TM(reg)(r31)
694	reg = reg + 1
695	.endr
696
697	mtspr	SPRN_DSCR, r29
698	mtspr	SPRN_PPR, r30
699
700	/* Load final GPRs */
701	ld	29, VCPU_GPRS_TM(29)(r31)
702	ld	30, VCPU_GPRS_TM(30)(r31)
703	ld	31, VCPU_GPRS_TM(31)(r31)
704
705	/* TM checkpointed state is now setup.  All GPRs are now volatile. */
706	TRECHKPT
707
708	/* Now let's get back the state we need. */
709	HMT_MEDIUM
710	GET_PACA(r13)
711	ld	r29, HSTATE_DSCR(r13)
712	mtspr	SPRN_DSCR, r29
713	ld	r4, HSTATE_KVM_VCPU(r13)
714	ld	r1, HSTATE_HOST_R1(r13)
715	ld	r2, PACATMSCRATCH(r13)
716
717	/* Set the MSR RI since we have our registers back. */
718	li	r5, MSR_RI
719	mtmsrd	r5, 1
720skip_tm:
721#endif
722
723	/* Load guest PMU registers */
724	/* R4 is live here (vcpu pointer) */
725	li	r3, 1
726	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
727	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
728	isync
729	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
730	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
731	lwz	r6, VCPU_PMC + 8(r4)
732	lwz	r7, VCPU_PMC + 12(r4)
733	lwz	r8, VCPU_PMC + 16(r4)
734	lwz	r9, VCPU_PMC + 20(r4)
735BEGIN_FTR_SECTION
736	lwz	r10, VCPU_PMC + 24(r4)
737	lwz	r11, VCPU_PMC + 28(r4)
738END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
739	mtspr	SPRN_PMC1, r3
740	mtspr	SPRN_PMC2, r5
741	mtspr	SPRN_PMC3, r6
742	mtspr	SPRN_PMC4, r7
743	mtspr	SPRN_PMC5, r8
744	mtspr	SPRN_PMC6, r9
745BEGIN_FTR_SECTION
746	mtspr	SPRN_PMC7, r10
747	mtspr	SPRN_PMC8, r11
748END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
749	ld	r3, VCPU_MMCR(r4)
750	ld	r5, VCPU_MMCR + 8(r4)
751	ld	r6, VCPU_MMCR + 16(r4)
752	ld	r7, VCPU_SIAR(r4)
753	ld	r8, VCPU_SDAR(r4)
754	mtspr	SPRN_MMCR1, r5
755	mtspr	SPRN_MMCRA, r6
756	mtspr	SPRN_SIAR, r7
757	mtspr	SPRN_SDAR, r8
758BEGIN_FTR_SECTION
759	ld	r5, VCPU_MMCR + 24(r4)
760	ld	r6, VCPU_SIER(r4)
761	lwz	r7, VCPU_PMC + 24(r4)
762	lwz	r8, VCPU_PMC + 28(r4)
763	ld	r9, VCPU_MMCR + 32(r4)
764	mtspr	SPRN_MMCR2, r5
765	mtspr	SPRN_SIER, r6
766	mtspr	SPRN_SPMC1, r7
767	mtspr	SPRN_SPMC2, r8
768	mtspr	SPRN_MMCRS, r9
769END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
770	mtspr	SPRN_MMCR0, r3
771	isync
772
773	/* Load up FP, VMX and VSX registers */
774	bl	kvmppc_load_fp
775
776	ld	r14, VCPU_GPR(R14)(r4)
777	ld	r15, VCPU_GPR(R15)(r4)
778	ld	r16, VCPU_GPR(R16)(r4)
779	ld	r17, VCPU_GPR(R17)(r4)
780	ld	r18, VCPU_GPR(R18)(r4)
781	ld	r19, VCPU_GPR(R19)(r4)
782	ld	r20, VCPU_GPR(R20)(r4)
783	ld	r21, VCPU_GPR(R21)(r4)
784	ld	r22, VCPU_GPR(R22)(r4)
785	ld	r23, VCPU_GPR(R23)(r4)
786	ld	r24, VCPU_GPR(R24)(r4)
787	ld	r25, VCPU_GPR(R25)(r4)
788	ld	r26, VCPU_GPR(R26)(r4)
789	ld	r27, VCPU_GPR(R27)(r4)
790	ld	r28, VCPU_GPR(R28)(r4)
791	ld	r29, VCPU_GPR(R29)(r4)
792	ld	r30, VCPU_GPR(R30)(r4)
793	ld	r31, VCPU_GPR(R31)(r4)
794
795BEGIN_FTR_SECTION
796	/* Switch DSCR to guest value */
797	ld	r5, VCPU_DSCR(r4)
798	mtspr	SPRN_DSCR, r5
799END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
800
801BEGIN_FTR_SECTION
802	/* Skip next section on POWER7 or PPC970 */
803	b	8f
804END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
805	/* Turn on TM so we can access TFHAR/TFIAR/TEXASR */
806	mfmsr	r8
807	li	r0, 1
808	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
809	mtmsrd	r8
810
811	/* Load up POWER8-specific registers */
812	ld	r5, VCPU_IAMR(r4)
813	lwz	r6, VCPU_PSPB(r4)
814	ld	r7, VCPU_FSCR(r4)
815	mtspr	SPRN_IAMR, r5
816	mtspr	SPRN_PSPB, r6
817	mtspr	SPRN_FSCR, r7
818	ld	r5, VCPU_DAWR(r4)
819	ld	r6, VCPU_DAWRX(r4)
820	ld	r7, VCPU_CIABR(r4)
821	ld	r8, VCPU_TAR(r4)
822	mtspr	SPRN_DAWR, r5
823	mtspr	SPRN_DAWRX, r6
824	mtspr	SPRN_CIABR, r7
825	mtspr	SPRN_TAR, r8
826	ld	r5, VCPU_IC(r4)
827	ld	r6, VCPU_VTB(r4)
828	mtspr	SPRN_IC, r5
829	mtspr	SPRN_VTB, r6
830	ld	r8, VCPU_EBBHR(r4)
831	mtspr	SPRN_EBBHR, r8
832	ld	r5, VCPU_EBBRR(r4)
833	ld	r6, VCPU_BESCR(r4)
834	ld	r7, VCPU_CSIGR(r4)
835	ld	r8, VCPU_TACR(r4)
836	mtspr	SPRN_EBBRR, r5
837	mtspr	SPRN_BESCR, r6
838	mtspr	SPRN_CSIGR, r7
839	mtspr	SPRN_TACR, r8
840	ld	r5, VCPU_TCSCR(r4)
841	ld	r6, VCPU_ACOP(r4)
842	lwz	r7, VCPU_GUEST_PID(r4)
843	ld	r8, VCPU_WORT(r4)
844	mtspr	SPRN_TCSCR, r5
845	mtspr	SPRN_ACOP, r6
846	mtspr	SPRN_PID, r7
847	mtspr	SPRN_WORT, r8
8488:
849
850	/*
851	 * Set the decrementer to the guest decrementer.
852	 */
853	ld	r8,VCPU_DEC_EXPIRES(r4)
854	/* r8 is a host timebase value here, convert to guest TB */
855	ld	r5,HSTATE_KVM_VCORE(r13)
856	ld	r6,VCORE_TB_OFFSET(r5)
857	add	r8,r8,r6
858	mftb	r7
859	subf	r3,r7,r8
860	mtspr	SPRN_DEC,r3
861	stw	r3,VCPU_DEC(r4)
862
863	ld	r5, VCPU_SPRG0(r4)
864	ld	r6, VCPU_SPRG1(r4)
865	ld	r7, VCPU_SPRG2(r4)
866	ld	r8, VCPU_SPRG3(r4)
867	mtspr	SPRN_SPRG0, r5
868	mtspr	SPRN_SPRG1, r6
869	mtspr	SPRN_SPRG2, r7
870	mtspr	SPRN_SPRG3, r8
871
872	/* Load up DAR and DSISR */
873	ld	r5, VCPU_DAR(r4)
874	lwz	r6, VCPU_DSISR(r4)
875	mtspr	SPRN_DAR, r5
876	mtspr	SPRN_DSISR, r6
877
878BEGIN_FTR_SECTION
879	/* Restore AMR and UAMOR, set AMOR to all 1s */
880	ld	r5,VCPU_AMR(r4)
881	ld	r6,VCPU_UAMOR(r4)
882	li	r7,-1
883	mtspr	SPRN_AMR,r5
884	mtspr	SPRN_UAMOR,r6
885	mtspr	SPRN_AMOR,r7
886END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
887
888	/* Restore state of CTRL run bit; assume 1 on entry */
889	lwz	r5,VCPU_CTRL(r4)
890	andi.	r5,r5,1
891	bne	4f
892	mfspr	r6,SPRN_CTRLF
893	clrrdi	r6,r6,1
894	mtspr	SPRN_CTRLT,r6
8954:
896	ld	r6, VCPU_CTR(r4)
897	lwz	r7, VCPU_XER(r4)
898
899	mtctr	r6
900	mtxer	r7
901
902kvmppc_cede_reentry:		/* r4 = vcpu, r13 = paca */
903	ld	r10, VCPU_PC(r4)
904	ld	r11, VCPU_MSR(r4)
905	ld	r6, VCPU_SRR0(r4)
906	ld	r7, VCPU_SRR1(r4)
907	mtspr	SPRN_SRR0, r6
908	mtspr	SPRN_SRR1, r7
909
910deliver_guest_interrupt:
911	/* r11 = vcpu->arch.msr & ~MSR_HV */
912	rldicl	r11, r11, 63 - MSR_HV_LG, 1
913	rotldi	r11, r11, 1 + MSR_HV_LG
914	ori	r11, r11, MSR_ME
915
916	/* Check if we can deliver an external or decrementer interrupt now */
917	ld	r0, VCPU_PENDING_EXC(r4)
918	rldicl	r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
919	cmpdi	cr1, r0, 0
920	andi.	r8, r11, MSR_EE
921BEGIN_FTR_SECTION
922	mfspr	r8, SPRN_LPCR
923	/* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
924	rldimi	r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
925	mtspr	SPRN_LPCR, r8
926	isync
927END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
928	beq	5f
929	li	r0, BOOK3S_INTERRUPT_EXTERNAL
930	bne	cr1, 12f
931	mfspr	r0, SPRN_DEC
932	cmpwi	r0, 0
933	li	r0, BOOK3S_INTERRUPT_DECREMENTER
934	bge	5f
935
93612:	mtspr	SPRN_SRR0, r10
937	mr	r10,r0
938	mtspr	SPRN_SRR1, r11
939	mr	r9, r4
940	bl	kvmppc_msr_interrupt
9415:
942
943/*
944 * Required state:
945 * R4 = vcpu
946 * R10: value for HSRR0
947 * R11: value for HSRR1
948 * R13 = PACA
949 */
950fast_guest_return:
951	li	r0,0
952	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
953	mtspr	SPRN_HSRR0,r10
954	mtspr	SPRN_HSRR1,r11
955
956	/* Activate guest mode, so faults get handled by KVM */
957	li	r9, KVM_GUEST_MODE_GUEST_HV
958	stb	r9, HSTATE_IN_GUEST(r13)
959
960	/* Enter guest */
961
962BEGIN_FTR_SECTION
963	ld	r5, VCPU_CFAR(r4)
964	mtspr	SPRN_CFAR, r5
965END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
966BEGIN_FTR_SECTION
967	ld	r0, VCPU_PPR(r4)
968END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
969
970	ld	r5, VCPU_LR(r4)
971	lwz	r6, VCPU_CR(r4)
972	mtlr	r5
973	mtcr	r6
974
975	ld	r1, VCPU_GPR(R1)(r4)
976	ld	r2, VCPU_GPR(R2)(r4)
977	ld	r3, VCPU_GPR(R3)(r4)
978	ld	r5, VCPU_GPR(R5)(r4)
979	ld	r6, VCPU_GPR(R6)(r4)
980	ld	r7, VCPU_GPR(R7)(r4)
981	ld	r8, VCPU_GPR(R8)(r4)
982	ld	r9, VCPU_GPR(R9)(r4)
983	ld	r10, VCPU_GPR(R10)(r4)
984	ld	r11, VCPU_GPR(R11)(r4)
985	ld	r12, VCPU_GPR(R12)(r4)
986	ld	r13, VCPU_GPR(R13)(r4)
987
988BEGIN_FTR_SECTION
989	mtspr	SPRN_PPR, r0
990END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
991	ld	r0, VCPU_GPR(R0)(r4)
992	ld	r4, VCPU_GPR(R4)(r4)
993
994	hrfid
995	b	.
996
997/******************************************************************************
998 *                                                                            *
999 *                               Exit code                                    *
1000 *                                                                            *
1001 *****************************************************************************/
1002
1003/*
1004 * We come here from the first-level interrupt handlers.
1005 */
1006	.globl	kvmppc_interrupt_hv
1007kvmppc_interrupt_hv:
1008	/*
1009	 * Register contents:
1010	 * R12		= interrupt vector
1011	 * R13		= PACA
1012	 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
1013	 * guest R13 saved in SPRN_SCRATCH0
1014	 */
1015	std	r9, HSTATE_SCRATCH2(r13)
1016
1017	lbz	r9, HSTATE_IN_GUEST(r13)
1018	cmpwi	r9, KVM_GUEST_MODE_HOST_HV
1019	beq	kvmppc_bad_host_intr
1020#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1021	cmpwi	r9, KVM_GUEST_MODE_GUEST
1022	ld	r9, HSTATE_SCRATCH2(r13)
1023	beq	kvmppc_interrupt_pr
1024#endif
1025	/* We're now back in the host but in guest MMU context */
1026	li	r9, KVM_GUEST_MODE_HOST_HV
1027	stb	r9, HSTATE_IN_GUEST(r13)
1028
1029	ld	r9, HSTATE_KVM_VCPU(r13)
1030
1031	/* Save registers */
1032
1033	std	r0, VCPU_GPR(R0)(r9)
1034	std	r1, VCPU_GPR(R1)(r9)
1035	std	r2, VCPU_GPR(R2)(r9)
1036	std	r3, VCPU_GPR(R3)(r9)
1037	std	r4, VCPU_GPR(R4)(r9)
1038	std	r5, VCPU_GPR(R5)(r9)
1039	std	r6, VCPU_GPR(R6)(r9)
1040	std	r7, VCPU_GPR(R7)(r9)
1041	std	r8, VCPU_GPR(R8)(r9)
1042	ld	r0, HSTATE_SCRATCH2(r13)
1043	std	r0, VCPU_GPR(R9)(r9)
1044	std	r10, VCPU_GPR(R10)(r9)
1045	std	r11, VCPU_GPR(R11)(r9)
1046	ld	r3, HSTATE_SCRATCH0(r13)
1047	lwz	r4, HSTATE_SCRATCH1(r13)
1048	std	r3, VCPU_GPR(R12)(r9)
1049	stw	r4, VCPU_CR(r9)
1050BEGIN_FTR_SECTION
1051	ld	r3, HSTATE_CFAR(r13)
1052	std	r3, VCPU_CFAR(r9)
1053END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1054BEGIN_FTR_SECTION
1055	ld	r4, HSTATE_PPR(r13)
1056	std	r4, VCPU_PPR(r9)
1057END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1058
1059	/* Restore R1/R2 so we can handle faults */
1060	ld	r1, HSTATE_HOST_R1(r13)
1061	ld	r2, PACATOC(r13)
1062
1063	mfspr	r10, SPRN_SRR0
1064	mfspr	r11, SPRN_SRR1
1065	std	r10, VCPU_SRR0(r9)
1066	std	r11, VCPU_SRR1(r9)
1067	andi.	r0, r12, 2		/* need to read HSRR0/1? */
1068	beq	1f
1069	mfspr	r10, SPRN_HSRR0
1070	mfspr	r11, SPRN_HSRR1
1071	clrrdi	r12, r12, 2
10721:	std	r10, VCPU_PC(r9)
1073	std	r11, VCPU_MSR(r9)
1074
1075	GET_SCRATCH0(r3)
1076	mflr	r4
1077	std	r3, VCPU_GPR(R13)(r9)
1078	std	r4, VCPU_LR(r9)
1079
1080	stw	r12,VCPU_TRAP(r9)
1081
1082	/* Save HEIR (HV emulation assist reg) in last_inst
1083	   if this is an HEI (HV emulation interrupt, e40) */
1084	li	r3,KVM_INST_FETCH_FAILED
1085BEGIN_FTR_SECTION
1086	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1087	bne	11f
1088	mfspr	r3,SPRN_HEIR
1089END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
109011:	stw	r3,VCPU_LAST_INST(r9)
1091
1092	/* these are volatile across C function calls */
1093	mfctr	r3
1094	mfxer	r4
1095	std	r3, VCPU_CTR(r9)
1096	stw	r4, VCPU_XER(r9)
1097
1098BEGIN_FTR_SECTION
1099	/* If this is a page table miss then see if it's theirs or ours */
1100	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1101	beq	kvmppc_hdsi
1102	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1103	beq	kvmppc_hisi
1104END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1105
1106	/* See if this is a leftover HDEC interrupt */
1107	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1108	bne	2f
1109	mfspr	r3,SPRN_HDEC
1110	cmpwi	r3,0
1111	bge	ignore_hdec
11122:
1113	/* See if this is an hcall we can handle in real mode */
1114	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
1115	beq	hcall_try_real_mode
1116
1117	/* Only handle external interrupts here on arch 206 and later */
1118BEGIN_FTR_SECTION
1119	b	ext_interrupt_to_host
1120END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
1121
1122	/* External interrupt ? */
1123	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
1124	bne+	ext_interrupt_to_host
1125
1126	/* External interrupt, first check for host_ipi. If this is
1127	 * set, we know the host wants us out so let's do it now
1128	 */
1129	bl	kvmppc_read_intr
1130	cmpdi	r3, 0
1131	bgt	ext_interrupt_to_host
1132
1133	/* Check if any CPU is heading out to the host, if so head out too */
1134	ld	r5, HSTATE_KVM_VCORE(r13)
1135	lwz	r0, VCORE_ENTRY_EXIT(r5)
1136	cmpwi	r0, 0x100
1137	bge	ext_interrupt_to_host
1138
1139	/* Return to guest after delivering any pending interrupt */
1140	mr	r4, r9
1141	b	deliver_guest_interrupt
1142
1143ext_interrupt_to_host:
1144
1145guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
1146	/* Save more register state  */
1147	mfdar	r6
1148	mfdsisr	r7
1149	std	r6, VCPU_DAR(r9)
1150	stw	r7, VCPU_DSISR(r9)
1151BEGIN_FTR_SECTION
1152	/* don't overwrite fault_dar/fault_dsisr if HDSI */
1153	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1154	beq	6f
1155END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1156	std	r6, VCPU_FAULT_DAR(r9)
1157	stw	r7, VCPU_FAULT_DSISR(r9)
1158
1159	/* See if it is a machine check */
1160	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1161	beq	machine_check_realmode
1162mc_cont:
1163
1164	/* Save guest CTRL register, set runlatch to 1 */
11656:	mfspr	r6,SPRN_CTRLF
1166	stw	r6,VCPU_CTRL(r9)
1167	andi.	r0,r6,1
1168	bne	4f
1169	ori	r6,r6,1
1170	mtspr	SPRN_CTRLT,r6
11714:
1172	/* Read the guest SLB and save it away */
1173	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
1174	mtctr	r0
1175	li	r6,0
1176	addi	r7,r9,VCPU_SLB
1177	li	r5,0
11781:	slbmfee	r8,r6
1179	andis.	r0,r8,SLB_ESID_V@h
1180	beq	2f
1181	add	r8,r8,r6		/* put index in */
1182	slbmfev	r3,r6
1183	std	r8,VCPU_SLB_E(r7)
1184	std	r3,VCPU_SLB_V(r7)
1185	addi	r7,r7,VCPU_SLB_SIZE
1186	addi	r5,r5,1
11872:	addi	r6,r6,1
1188	bdnz	1b
1189	stw	r5,VCPU_SLB_MAX(r9)
1190
1191	/*
1192	 * Save the guest PURR/SPURR
1193	 */
1194BEGIN_FTR_SECTION
1195	mfspr	r5,SPRN_PURR
1196	mfspr	r6,SPRN_SPURR
1197	ld	r7,VCPU_PURR(r9)
1198	ld	r8,VCPU_SPURR(r9)
1199	std	r5,VCPU_PURR(r9)
1200	std	r6,VCPU_SPURR(r9)
1201	subf	r5,r7,r5
1202	subf	r6,r8,r6
1203
1204	/*
1205	 * Restore host PURR/SPURR and add guest times
1206	 * so that the time in the guest gets accounted.
1207	 */
1208	ld	r3,HSTATE_PURR(r13)
1209	ld	r4,HSTATE_SPURR(r13)
1210	add	r3,r3,r5
1211	add	r4,r4,r6
1212	mtspr	SPRN_PURR,r3
1213	mtspr	SPRN_SPURR,r4
1214END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
1215
1216	/* Save DEC */
1217	mfspr	r5,SPRN_DEC
1218	mftb	r6
1219	extsw	r5,r5
1220	add	r5,r5,r6
1221	/* r5 is a guest timebase value here, convert to host TB */
1222	ld	r3,HSTATE_KVM_VCORE(r13)
1223	ld	r4,VCORE_TB_OFFSET(r3)
1224	subf	r5,r4,r5
1225	std	r5,VCPU_DEC_EXPIRES(r9)
1226
1227BEGIN_FTR_SECTION
1228	b	8f
1229END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1230	/* Save POWER8-specific registers */
1231	mfspr	r5, SPRN_IAMR
1232	mfspr	r6, SPRN_PSPB
1233	mfspr	r7, SPRN_FSCR
1234	std	r5, VCPU_IAMR(r9)
1235	stw	r6, VCPU_PSPB(r9)
1236	std	r7, VCPU_FSCR(r9)
1237	mfspr	r5, SPRN_IC
1238	mfspr	r6, SPRN_VTB
1239	mfspr	r7, SPRN_TAR
1240	std	r5, VCPU_IC(r9)
1241	std	r6, VCPU_VTB(r9)
1242	std	r7, VCPU_TAR(r9)
1243	mfspr	r8, SPRN_EBBHR
1244	std	r8, VCPU_EBBHR(r9)
1245	mfspr	r5, SPRN_EBBRR
1246	mfspr	r6, SPRN_BESCR
1247	mfspr	r7, SPRN_CSIGR
1248	mfspr	r8, SPRN_TACR
1249	std	r5, VCPU_EBBRR(r9)
1250	std	r6, VCPU_BESCR(r9)
1251	std	r7, VCPU_CSIGR(r9)
1252	std	r8, VCPU_TACR(r9)
1253	mfspr	r5, SPRN_TCSCR
1254	mfspr	r6, SPRN_ACOP
1255	mfspr	r7, SPRN_PID
1256	mfspr	r8, SPRN_WORT
1257	std	r5, VCPU_TCSCR(r9)
1258	std	r6, VCPU_ACOP(r9)
1259	stw	r7, VCPU_GUEST_PID(r9)
1260	std	r8, VCPU_WORT(r9)
12618:
1262
1263	/* Save and reset AMR and UAMOR before turning on the MMU */
1264BEGIN_FTR_SECTION
1265	mfspr	r5,SPRN_AMR
1266	mfspr	r6,SPRN_UAMOR
1267	std	r5,VCPU_AMR(r9)
1268	std	r6,VCPU_UAMOR(r9)
1269	li	r6,0
1270	mtspr	SPRN_AMR,r6
1271END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1272
1273	/* Switch DSCR back to host value */
1274BEGIN_FTR_SECTION
1275	mfspr	r8, SPRN_DSCR
1276	ld	r7, HSTATE_DSCR(r13)
1277	std	r8, VCPU_DSCR(r9)
1278	mtspr	SPRN_DSCR, r7
1279END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1280
1281	/* Save non-volatile GPRs */
1282	std	r14, VCPU_GPR(R14)(r9)
1283	std	r15, VCPU_GPR(R15)(r9)
1284	std	r16, VCPU_GPR(R16)(r9)
1285	std	r17, VCPU_GPR(R17)(r9)
1286	std	r18, VCPU_GPR(R18)(r9)
1287	std	r19, VCPU_GPR(R19)(r9)
1288	std	r20, VCPU_GPR(R20)(r9)
1289	std	r21, VCPU_GPR(R21)(r9)
1290	std	r22, VCPU_GPR(R22)(r9)
1291	std	r23, VCPU_GPR(R23)(r9)
1292	std	r24, VCPU_GPR(R24)(r9)
1293	std	r25, VCPU_GPR(R25)(r9)
1294	std	r26, VCPU_GPR(R26)(r9)
1295	std	r27, VCPU_GPR(R27)(r9)
1296	std	r28, VCPU_GPR(R28)(r9)
1297	std	r29, VCPU_GPR(R29)(r9)
1298	std	r30, VCPU_GPR(R30)(r9)
1299	std	r31, VCPU_GPR(R31)(r9)
1300
1301	/* Save SPRGs */
1302	mfspr	r3, SPRN_SPRG0
1303	mfspr	r4, SPRN_SPRG1
1304	mfspr	r5, SPRN_SPRG2
1305	mfspr	r6, SPRN_SPRG3
1306	std	r3, VCPU_SPRG0(r9)
1307	std	r4, VCPU_SPRG1(r9)
1308	std	r5, VCPU_SPRG2(r9)
1309	std	r6, VCPU_SPRG3(r9)
1310
1311	/* save FP state */
1312	mr	r3, r9
1313	bl	kvmppc_save_fp
1314
1315	/* Increment yield count if they have a VPA */
1316	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
1317	cmpdi	r8, 0
1318	beq	25f
1319	lwz	r3, LPPACA_YIELDCOUNT(r8)
1320	addi	r3, r3, 1
1321	stw	r3, LPPACA_YIELDCOUNT(r8)
1322	li	r3, 1
1323	stb	r3, VCPU_VPA_DIRTY(r9)
132425:
1325	/* Save PMU registers if requested */
1326	/* r8 and cr0.eq are live here */
1327	li	r3, 1
1328	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
1329	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
1330	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
1331	mfspr	r6, SPRN_MMCRA
1332BEGIN_FTR_SECTION
1333	/* On P7, clear MMCRA in order to disable SDAR updates */
1334	li	r7, 0
1335	mtspr	SPRN_MMCRA, r7
1336END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1337	isync
1338	beq	21f			/* if no VPA, save PMU stuff anyway */
1339	lbz	r7, LPPACA_PMCINUSE(r8)
1340	cmpwi	r7, 0			/* did they ask for PMU stuff to be saved? */
1341	bne	21f
1342	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
1343	b	22f
134421:	mfspr	r5, SPRN_MMCR1
1345	mfspr	r7, SPRN_SIAR
1346	mfspr	r8, SPRN_SDAR
1347	std	r4, VCPU_MMCR(r9)
1348	std	r5, VCPU_MMCR + 8(r9)
1349	std	r6, VCPU_MMCR + 16(r9)
1350	std	r7, VCPU_SIAR(r9)
1351	std	r8, VCPU_SDAR(r9)
1352	mfspr	r3, SPRN_PMC1
1353	mfspr	r4, SPRN_PMC2
1354	mfspr	r5, SPRN_PMC3
1355	mfspr	r6, SPRN_PMC4
1356	mfspr	r7, SPRN_PMC5
1357	mfspr	r8, SPRN_PMC6
1358BEGIN_FTR_SECTION
1359	mfspr	r10, SPRN_PMC7
1360	mfspr	r11, SPRN_PMC8
1361END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1362	stw	r3, VCPU_PMC(r9)
1363	stw	r4, VCPU_PMC + 4(r9)
1364	stw	r5, VCPU_PMC + 8(r9)
1365	stw	r6, VCPU_PMC + 12(r9)
1366	stw	r7, VCPU_PMC + 16(r9)
1367	stw	r8, VCPU_PMC + 20(r9)
1368BEGIN_FTR_SECTION
1369	stw	r10, VCPU_PMC + 24(r9)
1370	stw	r11, VCPU_PMC + 28(r9)
1371END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1372BEGIN_FTR_SECTION
1373	mfspr	r4, SPRN_MMCR2
1374	mfspr	r5, SPRN_SIER
1375	mfspr	r6, SPRN_SPMC1
1376	mfspr	r7, SPRN_SPMC2
1377	mfspr	r8, SPRN_MMCRS
1378	std	r4, VCPU_MMCR + 24(r9)
1379	std	r5, VCPU_SIER(r9)
1380	stw	r6, VCPU_PMC + 24(r9)
1381	stw	r7, VCPU_PMC + 28(r9)
1382	std	r8, VCPU_MMCR + 32(r9)
1383	lis	r4, 0x8000
1384	mtspr	SPRN_MMCRS, r4
1385END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
138622:
1387	/* Clear out SLB */
1388	li	r5,0
1389	slbmte	r5,r5
1390	slbia
1391	ptesync
1392
1393hdec_soon:			/* r12 = trap, r13 = paca */
1394BEGIN_FTR_SECTION
1395	b	32f
1396END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1397	/*
1398	 * POWER7 guest -> host partition switch code.
1399	 * We don't have to lock against tlbies but we do
1400	 * have to coordinate the hardware threads.
1401	 */
1402	/* Increment the threads-exiting-guest count in the 0xff00
1403	   bits of vcore->entry_exit_count */
1404	ld	r5,HSTATE_KVM_VCORE(r13)
1405	addi	r6,r5,VCORE_ENTRY_EXIT
140641:	lwarx	r3,0,r6
1407	addi	r0,r3,0x100
1408	stwcx.	r0,0,r6
1409	bne	41b
1410	isync		/* order stwcx. vs. reading napping_threads */
1411
1412	/*
1413	 * At this point we have an interrupt that we have to pass
1414	 * up to the kernel or qemu; we can't handle it in real mode.
1415	 * Thus we have to do a partition switch, so we have to
1416	 * collect the other threads, if we are the first thread
1417	 * to take an interrupt.  To do this, we set the HDEC to 0,
1418	 * which causes an HDEC interrupt in all threads within 2ns
1419	 * because the HDEC register is shared between all 4 threads.
1420	 * However, we don't need to bother if this is an HDEC
1421	 * interrupt, since the other threads will already be on their
1422	 * way here in that case.
1423	 */
1424	cmpwi	r3,0x100	/* Are we the first here? */
1425	bge	43f
1426	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1427	beq	40f
1428	li	r0,0
1429	mtspr	SPRN_HDEC,r0
143040:
1431	/*
1432	 * Send an IPI to any napping threads, since an HDEC interrupt
1433	 * doesn't wake CPUs up from nap.
1434	 */
1435	lwz	r3,VCORE_NAPPING_THREADS(r5)
1436	lbz	r4,HSTATE_PTID(r13)
1437	li	r0,1
1438	sld	r0,r0,r4
1439	andc.	r3,r3,r0		/* no sense IPI'ing ourselves */
1440	beq	43f
1441	/* Order entry/exit update vs. IPIs */
1442	sync
1443	mulli	r4,r4,PACA_SIZE		/* get paca for thread 0 */
1444	subf	r6,r4,r13
144542:	andi.	r0,r3,1
1446	beq	44f
1447	ld	r8,HSTATE_XICS_PHYS(r6)	/* get thread's XICS reg addr */
1448	li	r0,IPI_PRIORITY
1449	li	r7,XICS_MFRR
1450	stbcix	r0,r7,r8		/* trigger the IPI */
145144:	srdi.	r3,r3,1
1452	addi	r6,r6,PACA_SIZE
1453	bne	42b
1454
1455secondary_too_late:
1456	/* Secondary threads wait for primary to do partition switch */
145743:	ld	r5,HSTATE_KVM_VCORE(r13)
1458	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
1459	lbz	r3,HSTATE_PTID(r13)
1460	cmpwi	r3,0
1461	beq	15f
1462	HMT_LOW
146313:	lbz	r3,VCORE_IN_GUEST(r5)
1464	cmpwi	r3,0
1465	bne	13b
1466	HMT_MEDIUM
1467	b	16f
1468
1469	/* Primary thread waits for all the secondaries to exit guest */
147015:	lwz	r3,VCORE_ENTRY_EXIT(r5)
1471	srwi	r0,r3,8
1472	clrldi	r3,r3,56
1473	cmpw	r3,r0
1474	bne	15b
1475	isync
1476
1477	/* Primary thread switches back to host partition */
1478	ld	r6,KVM_HOST_SDR1(r4)
1479	lwz	r7,KVM_HOST_LPID(r4)
1480	li	r8,LPID_RSVD		/* switch to reserved LPID */
1481	mtspr	SPRN_LPID,r8
1482	ptesync
1483	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
1484	mtspr	SPRN_LPID,r7
1485	isync
1486
1487BEGIN_FTR_SECTION
1488	/* DPDES is shared between threads */
1489	mfspr	r7, SPRN_DPDES
1490	std	r7, VCORE_DPDES(r5)
1491	/* clear DPDES so we don't get guest doorbells in the host */
1492	li	r8, 0
1493	mtspr	SPRN_DPDES, r8
1494END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1495
1496	/* Subtract timebase offset from timebase */
1497	ld	r8,VCORE_TB_OFFSET(r5)
1498	cmpdi	r8,0
1499	beq	17f
1500	mftb	r6			/* current guest timebase */
1501	subf	r8,r8,r6
1502	mtspr	SPRN_TBU40,r8		/* update upper 40 bits */
1503	mftb	r7			/* check if lower 24 bits overflowed */
1504	clrldi	r6,r6,40
1505	clrldi	r7,r7,40
1506	cmpld	r7,r6
1507	bge	17f
1508	addis	r8,r8,0x100		/* if so, increment upper 40 bits */
1509	mtspr	SPRN_TBU40,r8
1510
1511	/* Reset PCR */
151217:	ld	r0, VCORE_PCR(r5)
1513	cmpdi	r0, 0
1514	beq	18f
1515	li	r0, 0
1516	mtspr	SPRN_PCR, r0
151718:
1518	/* Signal secondary CPUs to continue */
1519	stb	r0,VCORE_IN_GUEST(r5)
1520	lis	r8,0x7fff		/* MAX_INT@h */
1521	mtspr	SPRN_HDEC,r8
1522
152316:	ld	r8,KVM_HOST_LPCR(r4)
1524	mtspr	SPRN_LPCR,r8
1525	isync
1526	b	33f
1527
1528	/*
1529	 * PPC970 guest -> host partition switch code.
1530	 * We have to lock against concurrent tlbies, and
1531	 * we have to flush the whole TLB.
1532	 */
153332:	ld	r5,HSTATE_KVM_VCORE(r13)
1534	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
1535
1536	/* Take the guest's tlbie_lock */
1537#ifdef __BIG_ENDIAN__
1538	lwz	r8,PACA_LOCK_TOKEN(r13)
1539#else
1540	lwz	r8,PACAPACAINDEX(r13)
1541#endif
1542	addi	r3,r4,KVM_TLBIE_LOCK
154324:	lwarx	r0,0,r3
1544	cmpwi	r0,0
1545	bne	24b
1546	stwcx.	r8,0,r3
1547	bne	24b
1548	isync
1549
1550	ld	r7,KVM_HOST_LPCR(r4)	/* use kvm->arch.host_lpcr for HID4 */
1551	li	r0,0x18f
1552	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
1553	or	r0,r7,r0
1554	ptesync
1555	sync
1556	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
1557	isync
1558	li	r0,0
1559	stw	r0,0(r3)		/* drop guest tlbie_lock */
1560
1561	/* invalidate the whole TLB */
1562	li	r0,256
1563	mtctr	r0
1564	li	r6,0
156525:	tlbiel	r6
1566	addi	r6,r6,0x1000
1567	bdnz	25b
1568	ptesync
1569
1570	/* take native_tlbie_lock */
1571	ld	r3,toc_tlbie_lock@toc(2)
157224:	lwarx	r0,0,r3
1573	cmpwi	r0,0
1574	bne	24b
1575	stwcx.	r8,0,r3
1576	bne	24b
1577	isync
1578
1579	ld	r6,KVM_HOST_SDR1(r4)
1580	mtspr	SPRN_SDR1,r6		/* switch to host page table */
1581
1582	/* Set up host HID4 value */
1583	sync
1584	mtspr	SPRN_HID4,r7
1585	isync
1586	li	r0,0
1587	stw	r0,0(r3)		/* drop native_tlbie_lock */
1588
1589	lis	r8,0x7fff		/* MAX_INT@h */
1590	mtspr	SPRN_HDEC,r8
1591
1592	/* Disable HDEC interrupts */
1593	mfspr	r0,SPRN_HID0
1594	li	r3,0
1595	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
1596	sync
1597	mtspr	SPRN_HID0,r0
1598	mfspr	r0,SPRN_HID0
1599	mfspr	r0,SPRN_HID0
1600	mfspr	r0,SPRN_HID0
1601	mfspr	r0,SPRN_HID0
1602	mfspr	r0,SPRN_HID0
1603	mfspr	r0,SPRN_HID0
1604
1605	/* load host SLB entries */
160633:	ld	r8,PACA_SLBSHADOWPTR(r13)
1607
1608	.rept	SLB_NUM_BOLTED
1609	ld	r5,SLBSHADOW_SAVEAREA(r8)
1610	ld	r6,SLBSHADOW_SAVEAREA+8(r8)
1611	andis.	r7,r5,SLB_ESID_V@h
1612	beq	1f
1613	slbmte	r6,r5
16141:	addi	r8,r8,16
1615	.endr
1616
1617	/* Unset guest mode */
1618	li	r0, KVM_GUEST_MODE_NONE
1619	stb	r0, HSTATE_IN_GUEST(r13)
1620
1621	ld	r0, 112+PPC_LR_STKOFF(r1)
1622	addi	r1, r1, 112
1623	mtlr	r0
1624	blr
1625
1626/*
1627 * Check whether an HDSI is an HPTE not found fault or something else.
1628 * If it is an HPTE not found fault that is due to the guest accessing
1629 * a page that they have mapped but which we have paged out, then
1630 * we continue on with the guest exit path.  In all other cases,
1631 * reflect the HDSI to the guest as a DSI.
1632 */
1633kvmppc_hdsi:
1634	mfspr	r4, SPRN_HDAR
1635	mfspr	r6, SPRN_HDSISR
1636	/* HPTE not found fault or protection fault? */
1637	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1638	beq	1f			/* if not, send it to the guest */
1639	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
1640	beq	3f
1641	clrrdi	r0, r4, 28
1642	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1643	bne	1f			/* if no SLB entry found */
16444:	std	r4, VCPU_FAULT_DAR(r9)
1645	stw	r6, VCPU_FAULT_DSISR(r9)
1646
1647	/* Search the hash table. */
1648	mr	r3, r9			/* vcpu pointer */
1649	li	r7, 1			/* data fault */
1650	bl	.kvmppc_hpte_hv_fault
1651	ld	r9, HSTATE_KVM_VCPU(r13)
1652	ld	r10, VCPU_PC(r9)
1653	ld	r11, VCPU_MSR(r9)
1654	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1655	cmpdi	r3, 0			/* retry the instruction */
1656	beq	6f
1657	cmpdi	r3, -1			/* handle in kernel mode */
1658	beq	guest_exit_cont
1659	cmpdi	r3, -2			/* MMIO emulation; need instr word */
1660	beq	2f
1661
1662	/* Synthesize a DSI for the guest */
1663	ld	r4, VCPU_FAULT_DAR(r9)
1664	mr	r6, r3
16651:	mtspr	SPRN_DAR, r4
1666	mtspr	SPRN_DSISR, r6
1667	mtspr	SPRN_SRR0, r10
1668	mtspr	SPRN_SRR1, r11
1669	li	r10, BOOK3S_INTERRUPT_DATA_STORAGE
1670	bl	kvmppc_msr_interrupt
1671fast_interrupt_c_return:
16726:	ld	r7, VCPU_CTR(r9)
1673	lwz	r8, VCPU_XER(r9)
1674	mtctr	r7
1675	mtxer	r8
1676	mr	r4, r9
1677	b	fast_guest_return
1678
16793:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
1680	ld	r5, KVM_VRMA_SLB_V(r5)
1681	b	4b
1682
1683	/* If this is for emulated MMIO, load the instruction word */
16842:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */
1685
1686	/* Set guest mode to 'jump over instruction' so if lwz faults
1687	 * we'll just continue at the next IP. */
1688	li	r0, KVM_GUEST_MODE_SKIP
1689	stb	r0, HSTATE_IN_GUEST(r13)
1690
1691	/* Do the access with MSR:DR enabled */
1692	mfmsr	r3
1693	ori	r4, r3, MSR_DR		/* Enable paging for data */
1694	mtmsrd	r4
1695	lwz	r8, 0(r10)
1696	mtmsrd	r3
1697
1698	/* Store the result */
1699	stw	r8, VCPU_LAST_INST(r9)
1700
1701	/* Unset guest mode. */
1702	li	r0, KVM_GUEST_MODE_HOST_HV
1703	stb	r0, HSTATE_IN_GUEST(r13)
1704	b	guest_exit_cont
1705
1706/*
1707 * Similarly for an HISI, reflect it to the guest as an ISI unless
1708 * it is an HPTE not found fault for a page that we have paged out.
1709 */
1710kvmppc_hisi:
1711	andis.	r0, r11, SRR1_ISI_NOPT@h
1712	beq	1f
1713	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
1714	beq	3f
1715	clrrdi	r0, r10, 28
1716	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1717	bne	1f			/* if no SLB entry found */
17184:
1719	/* Search the hash table. */
1720	mr	r3, r9			/* vcpu pointer */
1721	mr	r4, r10
1722	mr	r6, r11
1723	li	r7, 0			/* instruction fault */
1724	bl	.kvmppc_hpte_hv_fault
1725	ld	r9, HSTATE_KVM_VCPU(r13)
1726	ld	r10, VCPU_PC(r9)
1727	ld	r11, VCPU_MSR(r9)
1728	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1729	cmpdi	r3, 0			/* retry the instruction */
1730	beq	fast_interrupt_c_return
1731	cmpdi	r3, -1			/* handle in kernel mode */
1732	beq	guest_exit_cont
1733
1734	/* Synthesize an ISI for the guest */
1735	mr	r11, r3
17361:	mtspr	SPRN_SRR0, r10
1737	mtspr	SPRN_SRR1, r11
1738	li	r10, BOOK3S_INTERRUPT_INST_STORAGE
1739	bl	kvmppc_msr_interrupt
1740	b	fast_interrupt_c_return
1741
17423:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
1743	ld	r5, KVM_VRMA_SLB_V(r6)
1744	b	4b
1745
1746/*
1747 * Try to handle an hcall in real mode.
1748 * Returns to the guest if we handle it, or continues on up to
1749 * the kernel if we can't (i.e. if we don't have a handler for
1750 * it, or if the handler returns H_TOO_HARD).
1751 */
1752	.globl	hcall_try_real_mode
1753hcall_try_real_mode:
1754	ld	r3,VCPU_GPR(R3)(r9)
1755	andi.	r0,r11,MSR_PR
1756	/* sc 1 from userspace - reflect to guest syscall */
1757	bne	sc_1_fast_return
1758	clrrdi	r3,r3,2
1759	cmpldi	r3,hcall_real_table_end - hcall_real_table
1760	bge	guest_exit_cont
1761	LOAD_REG_ADDR(r4, hcall_real_table)
1762	lwax	r3,r3,r4
1763	cmpwi	r3,0
1764	beq	guest_exit_cont
1765	add	r3,r3,r4
1766	mtctr	r3
1767	mr	r3,r9		/* get vcpu pointer */
1768	ld	r4,VCPU_GPR(R4)(r9)
1769	bctrl
1770	cmpdi	r3,H_TOO_HARD
1771	beq	hcall_real_fallback
1772	ld	r4,HSTATE_KVM_VCPU(r13)
1773	std	r3,VCPU_GPR(R3)(r4)
1774	ld	r10,VCPU_PC(r4)
1775	ld	r11,VCPU_MSR(r4)
1776	b	fast_guest_return
1777
1778sc_1_fast_return:
1779	mtspr	SPRN_SRR0,r10
1780	mtspr	SPRN_SRR1,r11
1781	li	r10, BOOK3S_INTERRUPT_SYSCALL
1782	bl	kvmppc_msr_interrupt
1783	mr	r4,r9
1784	b	fast_guest_return
1785
1786	/* We've attempted a real mode hcall, but it's punted it back
1787	 * to userspace.  We need to restore some clobbered volatiles
1788	 * before resuming the pass-it-to-qemu path */
1789hcall_real_fallback:
1790	li	r12,BOOK3S_INTERRUPT_SYSCALL
1791	ld	r9, HSTATE_KVM_VCPU(r13)
1792
1793	b	guest_exit_cont
1794
1795	.globl	hcall_real_table
1796hcall_real_table:
1797	.long	0		/* 0 - unused */
1798	.long	.kvmppc_h_remove - hcall_real_table
1799	.long	.kvmppc_h_enter - hcall_real_table
1800	.long	.kvmppc_h_read - hcall_real_table
1801	.long	0		/* 0x10 - H_CLEAR_MOD */
1802	.long	0		/* 0x14 - H_CLEAR_REF */
1803	.long	.kvmppc_h_protect - hcall_real_table
1804	.long	.kvmppc_h_get_tce - hcall_real_table
1805	.long	.kvmppc_h_put_tce - hcall_real_table
1806	.long	0		/* 0x24 - H_SET_SPRG0 */
1807	.long	.kvmppc_h_set_dabr - hcall_real_table
1808	.long	0		/* 0x2c */
1809	.long	0		/* 0x30 */
1810	.long	0		/* 0x34 */
1811	.long	0		/* 0x38 */
1812	.long	0		/* 0x3c */
1813	.long	0		/* 0x40 */
1814	.long	0		/* 0x44 */
1815	.long	0		/* 0x48 */
1816	.long	0		/* 0x4c */
1817	.long	0		/* 0x50 */
1818	.long	0		/* 0x54 */
1819	.long	0		/* 0x58 */
1820	.long	0		/* 0x5c */
1821	.long	0		/* 0x60 */
1822#ifdef CONFIG_KVM_XICS
1823	.long	.kvmppc_rm_h_eoi - hcall_real_table
1824	.long	.kvmppc_rm_h_cppr - hcall_real_table
1825	.long	.kvmppc_rm_h_ipi - hcall_real_table
1826	.long	0		/* 0x70 - H_IPOLL */
1827	.long	.kvmppc_rm_h_xirr - hcall_real_table
1828#else
1829	.long	0		/* 0x64 - H_EOI */
1830	.long	0		/* 0x68 - H_CPPR */
1831	.long	0		/* 0x6c - H_IPI */
1832	.long	0		/* 0x70 - H_IPOLL */
1833	.long	0		/* 0x74 - H_XIRR */
1834#endif
1835	.long	0		/* 0x78 */
1836	.long	0		/* 0x7c */
1837	.long	0		/* 0x80 */
1838	.long	0		/* 0x84 */
1839	.long	0		/* 0x88 */
1840	.long	0		/* 0x8c */
1841	.long	0		/* 0x90 */
1842	.long	0		/* 0x94 */
1843	.long	0		/* 0x98 */
1844	.long	0		/* 0x9c */
1845	.long	0		/* 0xa0 */
1846	.long	0		/* 0xa4 */
1847	.long	0		/* 0xa8 */
1848	.long	0		/* 0xac */
1849	.long	0		/* 0xb0 */
1850	.long	0		/* 0xb4 */
1851	.long	0		/* 0xb8 */
1852	.long	0		/* 0xbc */
1853	.long	0		/* 0xc0 */
1854	.long	0		/* 0xc4 */
1855	.long	0		/* 0xc8 */
1856	.long	0		/* 0xcc */
1857	.long	0		/* 0xd0 */
1858	.long	0		/* 0xd4 */
1859	.long	0		/* 0xd8 */
1860	.long	0		/* 0xdc */
1861	.long	.kvmppc_h_cede - hcall_real_table
1862	.long	0		/* 0xe4 */
1863	.long	0		/* 0xe8 */
1864	.long	0		/* 0xec */
1865	.long	0		/* 0xf0 */
1866	.long	0		/* 0xf4 */
1867	.long	0		/* 0xf8 */
1868	.long	0		/* 0xfc */
1869	.long	0		/* 0x100 */
1870	.long	0		/* 0x104 */
1871	.long	0		/* 0x108 */
1872	.long	0		/* 0x10c */
1873	.long	0		/* 0x110 */
1874	.long	0		/* 0x114 */
1875	.long	0		/* 0x118 */
1876	.long	0		/* 0x11c */
1877	.long	0		/* 0x120 */
1878	.long	.kvmppc_h_bulk_remove - hcall_real_table
1879	.long	0		/* 0x128 */
1880	.long	0		/* 0x12c */
1881	.long	0		/* 0x130 */
1882	.long	.kvmppc_h_set_xdabr - hcall_real_table
1883hcall_real_table_end:
1884
1885ignore_hdec:
1886	mr	r4,r9
1887	b	fast_guest_return
1888
1889_GLOBAL(kvmppc_h_set_xdabr)
1890	andi.	r0, r5, DABRX_USER | DABRX_KERNEL
1891	beq	6f
1892	li	r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
1893	andc.	r0, r5, r0
1894	beq	3f
18956:	li	r3, H_PARAMETER
1896	blr
1897
1898_GLOBAL(kvmppc_h_set_dabr)
1899	li	r5, DABRX_USER | DABRX_KERNEL
19003:
1901BEGIN_FTR_SECTION
1902	b	2f
1903END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1904	std	r4,VCPU_DABR(r3)
1905	stw	r5, VCPU_DABRX(r3)
1906	mtspr	SPRN_DABRX, r5
1907	/* Work around P7 bug where DABR can get corrupted on mtspr */
19081:	mtspr	SPRN_DABR,r4
1909	mfspr	r5, SPRN_DABR
1910	cmpd	r4, r5
1911	bne	1b
1912	isync
1913	li	r3,0
1914	blr
1915
1916	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
19172:	rlwimi	r5, r4, 5, DAWRX_DR | DAWRX_DW
1918	rlwimi	r5, r4, 1, DAWRX_WT
1919	clrrdi	r4, r4, 3
1920	std	r4, VCPU_DAWR(r3)
1921	std	r5, VCPU_DAWRX(r3)
1922	mtspr	SPRN_DAWR, r4
1923	mtspr	SPRN_DAWRX, r5
1924	li	r3, 0
1925	blr
1926
1927_GLOBAL(kvmppc_h_cede)
1928	ori	r11,r11,MSR_EE
1929	std	r11,VCPU_MSR(r3)
1930	li	r0,1
1931	stb	r0,VCPU_CEDED(r3)
1932	sync			/* order setting ceded vs. testing prodded */
1933	lbz	r5,VCPU_PRODDED(r3)
1934	cmpwi	r5,0
1935	bne	kvm_cede_prodded
1936	li	r0,0		/* set trap to 0 to say hcall is handled */
1937	stw	r0,VCPU_TRAP(r3)
1938	li	r0,H_SUCCESS
1939	std	r0,VCPU_GPR(R3)(r3)
1940BEGIN_FTR_SECTION
1941	b	kvm_cede_exit	/* just send it up to host on 970 */
1942END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
1943
1944	/*
1945	 * Set our bit in the bitmask of napping threads unless all the
1946	 * other threads are already napping, in which case we send this
1947	 * up to the host.
1948	 */
1949	ld	r5,HSTATE_KVM_VCORE(r13)
1950	lbz	r6,HSTATE_PTID(r13)
1951	lwz	r8,VCORE_ENTRY_EXIT(r5)
1952	clrldi	r8,r8,56
1953	li	r0,1
1954	sld	r0,r0,r6
1955	addi	r6,r5,VCORE_NAPPING_THREADS
195631:	lwarx	r4,0,r6
1957	or	r4,r4,r0
1958	PPC_POPCNTW(R7,R4)
1959	cmpw	r7,r8
1960	bge	kvm_cede_exit
1961	stwcx.	r4,0,r6
1962	bne	31b
1963	/* order napping_threads update vs testing entry_exit_count */
1964	isync
1965	li	r0,NAPPING_CEDE
1966	stb	r0,HSTATE_NAPPING(r13)
1967	lwz	r7,VCORE_ENTRY_EXIT(r5)
1968	cmpwi	r7,0x100
1969	bge	33f		/* another thread already exiting */
1970
1971/*
1972 * Although not specifically required by the architecture, POWER7
1973 * preserves the following registers in nap mode, even if an SMT mode
1974 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
1975 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
1976 */
1977	/* Save non-volatile GPRs */
1978	std	r14, VCPU_GPR(R14)(r3)
1979	std	r15, VCPU_GPR(R15)(r3)
1980	std	r16, VCPU_GPR(R16)(r3)
1981	std	r17, VCPU_GPR(R17)(r3)
1982	std	r18, VCPU_GPR(R18)(r3)
1983	std	r19, VCPU_GPR(R19)(r3)
1984	std	r20, VCPU_GPR(R20)(r3)
1985	std	r21, VCPU_GPR(R21)(r3)
1986	std	r22, VCPU_GPR(R22)(r3)
1987	std	r23, VCPU_GPR(R23)(r3)
1988	std	r24, VCPU_GPR(R24)(r3)
1989	std	r25, VCPU_GPR(R25)(r3)
1990	std	r26, VCPU_GPR(R26)(r3)
1991	std	r27, VCPU_GPR(R27)(r3)
1992	std	r28, VCPU_GPR(R28)(r3)
1993	std	r29, VCPU_GPR(R29)(r3)
1994	std	r30, VCPU_GPR(R30)(r3)
1995	std	r31, VCPU_GPR(R31)(r3)
1996
1997	/* save FP state */
1998	bl	kvmppc_save_fp
1999
2000	/*
2001	 * Take a nap until a decrementer or external or doobell interrupt
2002	 * occurs, with PECE1, PECE0 and PECEDP set in LPCR
2003	 */
2004	li	r0,1
2005	stb	r0,HSTATE_HWTHREAD_REQ(r13)
2006	mfspr	r5,SPRN_LPCR
2007	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
2008BEGIN_FTR_SECTION
2009	oris	r5,r5,LPCR_PECEDP@h
2010END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2011	mtspr	SPRN_LPCR,r5
2012	isync
2013	li	r0, 0
2014	std	r0, HSTATE_SCRATCH0(r13)
2015	ptesync
2016	ld	r0, HSTATE_SCRATCH0(r13)
20171:	cmpd	r0, r0
2018	bne	1b
2019	nap
2020	b	.
2021
202233:	mr	r4, r3
2023	li	r3, 0
2024	li	r12, 0
2025	b	34f
2026
2027kvm_end_cede:
2028	/* get vcpu pointer */
2029	ld	r4, HSTATE_KVM_VCPU(r13)
2030
2031	/* Woken by external or decrementer interrupt */
2032	ld	r1, HSTATE_HOST_R1(r13)
2033
2034	/* load up FP state */
2035	bl	kvmppc_load_fp
2036
2037	/* Load NV GPRS */
2038	ld	r14, VCPU_GPR(R14)(r4)
2039	ld	r15, VCPU_GPR(R15)(r4)
2040	ld	r16, VCPU_GPR(R16)(r4)
2041	ld	r17, VCPU_GPR(R17)(r4)
2042	ld	r18, VCPU_GPR(R18)(r4)
2043	ld	r19, VCPU_GPR(R19)(r4)
2044	ld	r20, VCPU_GPR(R20)(r4)
2045	ld	r21, VCPU_GPR(R21)(r4)
2046	ld	r22, VCPU_GPR(R22)(r4)
2047	ld	r23, VCPU_GPR(R23)(r4)
2048	ld	r24, VCPU_GPR(R24)(r4)
2049	ld	r25, VCPU_GPR(R25)(r4)
2050	ld	r26, VCPU_GPR(R26)(r4)
2051	ld	r27, VCPU_GPR(R27)(r4)
2052	ld	r28, VCPU_GPR(R28)(r4)
2053	ld	r29, VCPU_GPR(R29)(r4)
2054	ld	r30, VCPU_GPR(R30)(r4)
2055	ld	r31, VCPU_GPR(R31)(r4)
2056
2057	/* Check the wake reason in SRR1 to see why we got here */
2058	bl	kvmppc_check_wake_reason
2059
2060	/* clear our bit in vcore->napping_threads */
206134:	ld	r5,HSTATE_KVM_VCORE(r13)
2062	lbz	r7,HSTATE_PTID(r13)
2063	li	r0,1
2064	sld	r0,r0,r7
2065	addi	r6,r5,VCORE_NAPPING_THREADS
206632:	lwarx	r7,0,r6
2067	andc	r7,r7,r0
2068	stwcx.	r7,0,r6
2069	bne	32b
2070	li	r0,0
2071	stb	r0,HSTATE_NAPPING(r13)
2072
2073	/* See if the wake reason means we need to exit */
2074	stw	r12, VCPU_TRAP(r4)
2075	mr	r9, r4
2076	cmpdi	r3, 0
2077	bgt	guest_exit_cont
2078
2079	/* see if any other thread is already exiting */
2080	lwz	r0,VCORE_ENTRY_EXIT(r5)
2081	cmpwi	r0,0x100
2082	bge	guest_exit_cont
2083
2084	b	kvmppc_cede_reentry	/* if not go back to guest */
2085
2086	/* cede when already previously prodded case */
2087kvm_cede_prodded:
2088	li	r0,0
2089	stb	r0,VCPU_PRODDED(r3)
2090	sync			/* order testing prodded vs. clearing ceded */
2091	stb	r0,VCPU_CEDED(r3)
2092	li	r3,H_SUCCESS
2093	blr
2094
2095	/* we've ceded but we want to give control to the host */
2096kvm_cede_exit:
2097	b	hcall_real_fallback
2098
2099	/* Try to handle a machine check in real mode */
2100machine_check_realmode:
2101	mr	r3, r9		/* get vcpu pointer */
2102	bl	.kvmppc_realmode_machine_check
2103	nop
2104	cmpdi	r3, 0		/* continue exiting from guest? */
2105	ld	r9, HSTATE_KVM_VCPU(r13)
2106	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2107	beq	mc_cont
2108	/* If not, deliver a machine check.  SRR0/1 are already set */
2109	li	r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2110	bl	kvmppc_msr_interrupt
2111	b	fast_interrupt_c_return
2112
2113/*
2114 * Check the reason we woke from nap, and take appropriate action.
2115 * Returns:
2116 *	0 if nothing needs to be done
2117 *	1 if something happened that needs to be handled by the host
2118 *	-1 if there was a guest wakeup (IPI)
2119 *
2120 * Also sets r12 to the interrupt vector for any interrupt that needs
2121 * to be handled now by the host (0x500 for external interrupt), or zero.
2122 */
2123kvmppc_check_wake_reason:
2124	mfspr	r6, SPRN_SRR1
2125BEGIN_FTR_SECTION
2126	rlwinm	r6, r6, 45-31, 0xf	/* extract wake reason field (P8) */
2127FTR_SECTION_ELSE
2128	rlwinm	r6, r6, 45-31, 0xe	/* P7 wake reason field is 3 bits */
2129ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2130	cmpwi	r6, 8			/* was it an external interrupt? */
2131	li	r12, BOOK3S_INTERRUPT_EXTERNAL
2132	beq	kvmppc_read_intr	/* if so, see what it was */
2133	li	r3, 0
2134	li	r12, 0
2135	cmpwi	r6, 6			/* was it the decrementer? */
2136	beq	0f
2137BEGIN_FTR_SECTION
2138	cmpwi	r6, 5			/* privileged doorbell? */
2139	beq	0f
2140	cmpwi	r6, 3			/* hypervisor doorbell? */
2141	beq	3f
2142END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2143	li	r3, 1			/* anything else, return 1 */
21440:	blr
2145
2146	/* hypervisor doorbell */
21473:	li	r12, BOOK3S_INTERRUPT_H_DOORBELL
2148	li	r3, 1
2149	blr
2150
2151/*
2152 * Determine what sort of external interrupt is pending (if any).
2153 * Returns:
2154 *	0 if no interrupt is pending
2155 *	1 if an interrupt is pending that needs to be handled by the host
2156 *	-1 if there was a guest wakeup IPI (which has now been cleared)
2157 */
2158kvmppc_read_intr:
2159	/* see if a host IPI is pending */
2160	li	r3, 1
2161	lbz	r0, HSTATE_HOST_IPI(r13)
2162	cmpwi	r0, 0
2163	bne	1f
2164
2165	/* Now read the interrupt from the ICP */
2166	ld	r6, HSTATE_XICS_PHYS(r13)
2167	li	r7, XICS_XIRR
2168	cmpdi	r6, 0
2169	beq-	1f
2170	lwzcix	r0, r6, r7
2171	rlwinm.	r3, r0, 0, 0xffffff
2172	sync
2173	beq	1f			/* if nothing pending in the ICP */
2174
2175	/* We found something in the ICP...
2176	 *
2177	 * If it's not an IPI, stash it in the PACA and return to
2178	 * the host, we don't (yet) handle directing real external
2179	 * interrupts directly to the guest
2180	 */
2181	cmpwi	r3, XICS_IPI		/* if there is, is it an IPI? */
2182	bne	42f
2183
2184	/* It's an IPI, clear the MFRR and EOI it */
2185	li	r3, 0xff
2186	li	r8, XICS_MFRR
2187	stbcix	r3, r6, r8		/* clear the IPI */
2188	stwcix	r0, r6, r7		/* EOI it */
2189	sync
2190
2191	/* We need to re-check host IPI now in case it got set in the
2192	 * meantime. If it's clear, we bounce the interrupt to the
2193	 * guest
2194	 */
2195	lbz	r0, HSTATE_HOST_IPI(r13)
2196	cmpwi	r0, 0
2197	bne-	43f
2198
2199	/* OK, it's an IPI for us */
2200	li	r3, -1
22011:	blr
2202
220342:	/* It's not an IPI and it's for the host, stash it in the PACA
2204	 * before exit, it will be picked up by the host ICP driver
2205	 */
2206	stw	r0, HSTATE_SAVED_XIRR(r13)
2207	li	r3, 1
2208	b	1b
2209
221043:	/* We raced with the host, we need to resend that IPI, bummer */
2211	li	r0, IPI_PRIORITY
2212	stbcix	r0, r6, r8		/* set the IPI */
2213	sync
2214	li	r3, 1
2215	b	1b
2216
2217/*
2218 * Save away FP, VMX and VSX registers.
2219 * r3 = vcpu pointer
2220 * N.B. r30 and r31 are volatile across this function,
2221 * thus it is not callable from C.
2222 */
2223kvmppc_save_fp:
2224	mflr	r30
2225	mr	r31,r3
2226	mfmsr	r5
2227	ori	r8,r5,MSR_FP
2228#ifdef CONFIG_ALTIVEC
2229BEGIN_FTR_SECTION
2230	oris	r8,r8,MSR_VEC@h
2231END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2232#endif
2233#ifdef CONFIG_VSX
2234BEGIN_FTR_SECTION
2235	oris	r8,r8,MSR_VSX@h
2236END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2237#endif
2238	mtmsrd	r8
2239	isync
2240	addi	r3,r3,VCPU_FPRS
2241	bl	.store_fp_state
2242#ifdef CONFIG_ALTIVEC
2243BEGIN_FTR_SECTION
2244	addi	r3,r31,VCPU_VRS
2245	bl	.store_vr_state
2246END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2247#endif
2248	mfspr	r6,SPRN_VRSAVE
2249	stw	r6,VCPU_VRSAVE(r31)
2250	mtlr	r30
2251	blr
2252
2253/*
2254 * Load up FP, VMX and VSX registers
2255 * r4 = vcpu pointer
2256 * N.B. r30 and r31 are volatile across this function,
2257 * thus it is not callable from C.
2258 */
2259kvmppc_load_fp:
2260	mflr	r30
2261	mr	r31,r4
2262	mfmsr	r9
2263	ori	r8,r9,MSR_FP
2264#ifdef CONFIG_ALTIVEC
2265BEGIN_FTR_SECTION
2266	oris	r8,r8,MSR_VEC@h
2267END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2268#endif
2269#ifdef CONFIG_VSX
2270BEGIN_FTR_SECTION
2271	oris	r8,r8,MSR_VSX@h
2272END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2273#endif
2274	mtmsrd	r8
2275	isync
2276	addi	r3,r4,VCPU_FPRS
2277	bl	.load_fp_state
2278#ifdef CONFIG_ALTIVEC
2279BEGIN_FTR_SECTION
2280	addi	r3,r31,VCPU_VRS
2281	bl	.load_vr_state
2282END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2283#endif
2284	lwz	r7,VCPU_VRSAVE(r31)
2285	mtspr	SPRN_VRSAVE,r7
2286	mtlr	r30
2287	mr	r4,r31
2288	blr
2289
2290/*
2291 * We come here if we get any exception or interrupt while we are
2292 * executing host real mode code while in guest MMU context.
2293 * For now just spin, but we should do something better.
2294 */
2295kvmppc_bad_host_intr:
2296	b	.
2297
2298/*
2299 * This mimics the MSR transition on IRQ delivery.  The new guest MSR is taken
2300 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2301 *   r11 has the guest MSR value (in/out)
2302 *   r9 has a vcpu pointer (in)
2303 *   r0 is used as a scratch register
2304 */
2305kvmppc_msr_interrupt:
2306	rldicl	r0, r11, 64 - MSR_TS_S_LG, 62
2307	cmpwi	r0, 2 /* Check if we are in transactional state..  */
2308	ld	r11, VCPU_INTR_MSR(r9)
2309	bne	1f
2310	/* ... if transactional, change to suspended */
2311	li	r0, 1
23121:	rldimi	r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2313	blr
2314