1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20#include <asm/ppc_asm.h>
21#include <asm/kvm_asm.h>
22#include <asm/reg.h>
23#include <asm/mmu.h>
24#include <asm/page.h>
25#include <asm/ptrace.h>
26#include <asm/hvcall.h>
27#include <asm/asm-offsets.h>
28#include <asm/exception-64s.h>
29#include <asm/kvm_book3s_asm.h>
30#include <asm/mmu-hash64.h>
31#include <asm/tm.h>
32
33#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
34
35/* Values in HSTATE_NAPPING(r13) */
36#define NAPPING_CEDE	1
37#define NAPPING_NOVCPU	2
38
39/*
40 * Call kvmppc_hv_entry in real mode.
41 * Must be called with interrupts hard-disabled.
42 *
43 * Input Registers:
44 *
45 * LR = return address to continue at after eventually re-enabling MMU
46 */
47_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
48	mflr	r0
49	std	r0, PPC_LR_STKOFF(r1)
50	stdu	r1, -112(r1)
51	mfmsr	r10
52	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
53	li	r0,MSR_RI
54	andc	r0,r10,r0
55	li	r6,MSR_IR | MSR_DR
56	andc	r6,r10,r6
57	mtmsrd	r0,1		/* clear RI in MSR */
58	mtsrr0	r5
59	mtsrr1	r6
60	RFI
61
62kvmppc_call_hv_entry:
63	ld	r4, HSTATE_KVM_VCPU(r13)
64	bl	kvmppc_hv_entry
65
66	/* Back from guest - restore host state and return to caller */
67
68BEGIN_FTR_SECTION
69	/* Restore host DABR and DABRX */
70	ld	r5,HSTATE_DABR(r13)
71	li	r6,7
72	mtspr	SPRN_DABR,r5
73	mtspr	SPRN_DABRX,r6
74END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
75
76	/* Restore SPRG3 */
77	ld	r3,PACA_SPRG_VDSO(r13)
78	mtspr	SPRN_SPRG_VDSO_WRITE,r3
79
80	/* Reload the host's PMU registers */
81	ld	r3, PACALPPACAPTR(r13)	/* is the host using the PMU? */
82	lbz	r4, LPPACA_PMCINUSE(r3)
83	cmpwi	r4, 0
84	beq	23f			/* skip if not */
85BEGIN_FTR_SECTION
86	ld	r3, HSTATE_MMCR0(r13)
87	andi.	r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
88	cmpwi	r4, MMCR0_PMAO
89	beql	kvmppc_fix_pmao
90END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
91	lwz	r3, HSTATE_PMC1(r13)
92	lwz	r4, HSTATE_PMC2(r13)
93	lwz	r5, HSTATE_PMC3(r13)
94	lwz	r6, HSTATE_PMC4(r13)
95	lwz	r8, HSTATE_PMC5(r13)
96	lwz	r9, HSTATE_PMC6(r13)
97	mtspr	SPRN_PMC1, r3
98	mtspr	SPRN_PMC2, r4
99	mtspr	SPRN_PMC3, r5
100	mtspr	SPRN_PMC4, r6
101	mtspr	SPRN_PMC5, r8
102	mtspr	SPRN_PMC6, r9
103	ld	r3, HSTATE_MMCR0(r13)
104	ld	r4, HSTATE_MMCR1(r13)
105	ld	r5, HSTATE_MMCRA(r13)
106	ld	r6, HSTATE_SIAR(r13)
107	ld	r7, HSTATE_SDAR(r13)
108	mtspr	SPRN_MMCR1, r4
109	mtspr	SPRN_MMCRA, r5
110	mtspr	SPRN_SIAR, r6
111	mtspr	SPRN_SDAR, r7
112BEGIN_FTR_SECTION
113	ld	r8, HSTATE_MMCR2(r13)
114	ld	r9, HSTATE_SIER(r13)
115	mtspr	SPRN_MMCR2, r8
116	mtspr	SPRN_SIER, r9
117END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
118	mtspr	SPRN_MMCR0, r3
119	isync
12023:
121
122	/*
123	 * Reload DEC.  HDEC interrupts were disabled when
124	 * we reloaded the host's LPCR value.
125	 */
126	ld	r3, HSTATE_DECEXP(r13)
127	mftb	r4
128	subf	r4, r4, r3
129	mtspr	SPRN_DEC, r4
130
131	/*
132	 * For external and machine check interrupts, we need
133	 * to call the Linux handler to process the interrupt.
134	 * We do that by jumping to absolute address 0x500 for
135	 * external interrupts, or the machine_check_fwnmi label
136	 * for machine checks (since firmware might have patched
137	 * the vector area at 0x200).  The [h]rfid at the end of the
138	 * handler will return to the book3s_hv_interrupts.S code.
139	 * For other interrupts we do the rfid to get back
140	 * to the book3s_hv_interrupts.S code here.
141	 */
142	ld	r8, 112+PPC_LR_STKOFF(r1)
143	addi	r1, r1, 112
144	ld	r7, HSTATE_HOST_MSR(r13)
145
146	cmpwi	cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
147	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
148	beq	11f
149	cmpwi	cr2, r12, BOOK3S_INTERRUPT_HMI
150	beq	cr2, 14f			/* HMI check */
151
152	/* RFI into the highmem handler, or branch to interrupt handler */
153	mfmsr	r6
154	li	r0, MSR_RI
155	andc	r6, r6, r0
156	mtmsrd	r6, 1			/* Clear RI in MSR */
157	mtsrr0	r8
158	mtsrr1	r7
159	beq	cr1, 13f		/* machine check */
160	RFI
161
162	/* On POWER7, we have external interrupts set to use HSRR0/1 */
16311:	mtspr	SPRN_HSRR0, r8
164	mtspr	SPRN_HSRR1, r7
165	ba	0x500
166
16713:	b	machine_check_fwnmi
168
16914:	mtspr	SPRN_HSRR0, r8
170	mtspr	SPRN_HSRR1, r7
171	b	hmi_exception_after_realmode
172
173kvmppc_primary_no_guest:
174	/* We handle this much like a ceded vcpu */
175	/* set our bit in napping_threads */
176	ld	r5, HSTATE_KVM_VCORE(r13)
177	lbz	r7, HSTATE_PTID(r13)
178	li	r0, 1
179	sld	r0, r0, r7
180	addi	r6, r5, VCORE_NAPPING_THREADS
1811:	lwarx	r3, 0, r6
182	or	r3, r3, r0
183	stwcx.	r3, 0, r6
184	bne	1b
185	/* order napping_threads update vs testing entry_exit_count */
186	isync
187	li	r12, 0
188	lwz	r7, VCORE_ENTRY_EXIT(r5)
189	cmpwi	r7, 0x100
190	bge	kvm_novcpu_exit	/* another thread already exiting */
191	li	r3, NAPPING_NOVCPU
192	stb	r3, HSTATE_NAPPING(r13)
193
194	b	kvm_do_nap
195
196kvm_novcpu_wakeup:
197	ld	r1, HSTATE_HOST_R1(r13)
198	ld	r5, HSTATE_KVM_VCORE(r13)
199	li	r0, 0
200	stb	r0, HSTATE_NAPPING(r13)
201	stb	r0, HSTATE_HWTHREAD_REQ(r13)
202
203	/* check the wake reason */
204	bl	kvmppc_check_wake_reason
205
206	/* see if any other thread is already exiting */
207	lwz	r0, VCORE_ENTRY_EXIT(r5)
208	cmpwi	r0, 0x100
209	bge	kvm_novcpu_exit
210
211	/* clear our bit in napping_threads */
212	lbz	r7, HSTATE_PTID(r13)
213	li	r0, 1
214	sld	r0, r0, r7
215	addi	r6, r5, VCORE_NAPPING_THREADS
2164:	lwarx	r7, 0, r6
217	andc	r7, r7, r0
218	stwcx.	r7, 0, r6
219	bne	4b
220
221	/* See if the wake reason means we need to exit */
222	cmpdi	r3, 0
223	bge	kvm_novcpu_exit
224
225	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
226	ld	r4, HSTATE_KVM_VCPU(r13)
227	cmpdi	r4, 0
228	bne	kvmppc_got_guest
229
230kvm_novcpu_exit:
231	b	hdec_soon
232
233/*
234 * We come in here when wakened from nap mode.
235 * Relocation is off and most register values are lost.
236 * r13 points to the PACA.
237 */
238	.globl	kvm_start_guest
239kvm_start_guest:
240
241	/* Set runlatch bit the minute you wake up from nap */
242	mfspr	r1, SPRN_CTRLF
243	ori 	r1, r1, 1
244	mtspr	SPRN_CTRLT, r1
245
246	ld	r2,PACATOC(r13)
247
248	li	r0,KVM_HWTHREAD_IN_KVM
249	stb	r0,HSTATE_HWTHREAD_STATE(r13)
250
251	/* NV GPR values from power7_idle() will no longer be valid */
252	li	r0,1
253	stb	r0,PACA_NAPSTATELOST(r13)
254
255	/* were we napping due to cede? */
256	lbz	r0,HSTATE_NAPPING(r13)
257	cmpwi	r0,NAPPING_CEDE
258	beq	kvm_end_cede
259	cmpwi	r0,NAPPING_NOVCPU
260	beq	kvm_novcpu_wakeup
261
262	ld	r1,PACAEMERGSP(r13)
263	subi	r1,r1,STACK_FRAME_OVERHEAD
264
265	/*
266	 * We weren't napping due to cede, so this must be a secondary
267	 * thread being woken up to run a guest, or being woken up due
268	 * to a stray IPI.  (Or due to some machine check or hypervisor
269	 * maintenance interrupt while the core is in KVM.)
270	 */
271
272	/* Check the wake reason in SRR1 to see why we got here */
273	bl	kvmppc_check_wake_reason
274	cmpdi	r3, 0
275	bge	kvm_no_guest
276
277	/* get vcpu pointer, NULL if we have no vcpu to run */
278	ld	r4,HSTATE_KVM_VCPU(r13)
279	cmpdi	r4,0
280	/* if we have no vcpu to run, go back to sleep */
281	beq	kvm_no_guest
282
283kvm_secondary_got_guest:
284
285	/* Set HSTATE_DSCR(r13) to something sensible */
286	ld	r6, PACA_DSCR(r13)
287	std	r6, HSTATE_DSCR(r13)
288
289	bl	kvmppc_hv_entry
290
291	/* Back from the guest, go back to nap */
292	/* Clear our vcpu pointer so we don't come back in early */
293	li	r0, 0
294	std	r0, HSTATE_KVM_VCPU(r13)
295	/*
296	 * Make sure we clear HSTATE_KVM_VCPU(r13) before incrementing
297	 * the nap_count, because once the increment to nap_count is
298	 * visible we could be given another vcpu.
299	 */
300	lwsync
301
302	/* increment the nap count and then go to nap mode */
303	ld	r4, HSTATE_KVM_VCORE(r13)
304	addi	r4, r4, VCORE_NAP_COUNT
30551:	lwarx	r3, 0, r4
306	addi	r3, r3, 1
307	stwcx.	r3, 0, r4
308	bne	51b
309
310/*
311 * At this point we have finished executing in the guest.
312 * We need to wait for hwthread_req to become zero, since
313 * we may not turn on the MMU while hwthread_req is non-zero.
314 * While waiting we also need to check if we get given a vcpu to run.
315 */
316kvm_no_guest:
317	lbz	r3, HSTATE_HWTHREAD_REQ(r13)
318	cmpwi	r3, 0
319	bne	53f
320	HMT_MEDIUM
321	li	r0, KVM_HWTHREAD_IN_KERNEL
322	stb	r0, HSTATE_HWTHREAD_STATE(r13)
323	/* need to recheck hwthread_req after a barrier, to avoid race */
324	sync
325	lbz	r3, HSTATE_HWTHREAD_REQ(r13)
326	cmpwi	r3, 0
327	bne	54f
328/*
329 * We jump to power7_wakeup_loss, which will return to the caller
330 * of power7_nap in the powernv cpu offline loop.  The value we
331 * put in r3 becomes the return value for power7_nap.
332 */
333	li	r3, LPCR_PECE0
334	mfspr	r4, SPRN_LPCR
335	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
336	mtspr	SPRN_LPCR, r4
337	li	r3, 0
338	b	power7_wakeup_loss
339
34053:	HMT_LOW
341	ld	r4, HSTATE_KVM_VCPU(r13)
342	cmpdi	r4, 0
343	beq	kvm_no_guest
344	HMT_MEDIUM
345	b	kvm_secondary_got_guest
346
34754:	li	r0, KVM_HWTHREAD_IN_KVM
348	stb	r0, HSTATE_HWTHREAD_STATE(r13)
349	b	kvm_no_guest
350
351/******************************************************************************
352 *                                                                            *
353 *                               Entry code                                   *
354 *                                                                            *
355 *****************************************************************************/
356
357.global kvmppc_hv_entry
358kvmppc_hv_entry:
359
360	/* Required state:
361	 *
362	 * R4 = vcpu pointer (or NULL)
363	 * MSR = ~IR|DR
364	 * R13 = PACA
365	 * R1 = host R1
366	 * R2 = TOC
367	 * all other volatile GPRS = free
368	 */
369	mflr	r0
370	std	r0, PPC_LR_STKOFF(r1)
371	stdu	r1, -112(r1)
372
373	/* Save R1 in the PACA */
374	std	r1, HSTATE_HOST_R1(r13)
375
376	li	r6, KVM_GUEST_MODE_HOST_HV
377	stb	r6, HSTATE_IN_GUEST(r13)
378
379	/* Clear out SLB */
380	li	r6,0
381	slbmte	r6,r6
382	slbia
383	ptesync
384
385	/*
386	 * POWER7/POWER8 host -> guest partition switch code.
387	 * We don't have to lock against concurrent tlbies,
388	 * but we do have to coordinate across hardware threads.
389	 */
390	/* Increment entry count iff exit count is zero. */
391	ld	r5,HSTATE_KVM_VCORE(r13)
392	addi	r9,r5,VCORE_ENTRY_EXIT
39321:	lwarx	r3,0,r9
394	cmpwi	r3,0x100		/* any threads starting to exit? */
395	bge	secondary_too_late	/* if so we're too late to the party */
396	addi	r3,r3,1
397	stwcx.	r3,0,r9
398	bne	21b
399
400	/* Primary thread switches to guest partition. */
401	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
402	lbz	r6,HSTATE_PTID(r13)
403	cmpwi	r6,0
404	bne	20f
405	ld	r6,KVM_SDR1(r9)
406	lwz	r7,KVM_LPID(r9)
407	li	r0,LPID_RSVD		/* switch to reserved LPID */
408	mtspr	SPRN_LPID,r0
409	ptesync
410	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
411	mtspr	SPRN_LPID,r7
412	isync
413
414	/* See if we need to flush the TLB */
415	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
416	clrldi	r7,r6,64-6		/* extract bit number (6 bits) */
417	srdi	r6,r6,6			/* doubleword number */
418	sldi	r6,r6,3			/* address offset */
419	add	r6,r6,r9
420	addi	r6,r6,KVM_NEED_FLUSH	/* dword in kvm->arch.need_tlb_flush */
421	li	r0,1
422	sld	r0,r0,r7
423	ld	r7,0(r6)
424	and.	r7,r7,r0
425	beq	22f
42623:	ldarx	r7,0,r6			/* if set, clear the bit */
427	andc	r7,r7,r0
428	stdcx.	r7,0,r6
429	bne	23b
430	/* Flush the TLB of any entries for this LPID */
431	/* use arch 2.07S as a proxy for POWER8 */
432BEGIN_FTR_SECTION
433	li	r6,512			/* POWER8 has 512 sets */
434FTR_SECTION_ELSE
435	li	r6,128			/* POWER7 has 128 sets */
436ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
437	mtctr	r6
438	li	r7,0x800		/* IS field = 0b10 */
439	ptesync
44028:	tlbiel	r7
441	addi	r7,r7,0x1000
442	bdnz	28b
443	ptesync
444
445	/* Add timebase offset onto timebase */
44622:	ld	r8,VCORE_TB_OFFSET(r5)
447	cmpdi	r8,0
448	beq	37f
449	mftb	r6		/* current host timebase */
450	add	r8,r8,r6
451	mtspr	SPRN_TBU40,r8	/* update upper 40 bits */
452	mftb	r7		/* check if lower 24 bits overflowed */
453	clrldi	r6,r6,40
454	clrldi	r7,r7,40
455	cmpld	r7,r6
456	bge	37f
457	addis	r8,r8,0x100	/* if so, increment upper 40 bits */
458	mtspr	SPRN_TBU40,r8
459
460	/* Load guest PCR value to select appropriate compat mode */
46137:	ld	r7, VCORE_PCR(r5)
462	cmpdi	r7, 0
463	beq	38f
464	mtspr	SPRN_PCR, r7
46538:
466
467BEGIN_FTR_SECTION
468	/* DPDES is shared between threads */
469	ld	r8, VCORE_DPDES(r5)
470	mtspr	SPRN_DPDES, r8
471END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
472
473	li	r0,1
474	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
475	b	10f
476
477	/* Secondary threads wait for primary to have done partition switch */
47820:	lbz	r0,VCORE_IN_GUEST(r5)
479	cmpwi	r0,0
480	beq	20b
481
482	/* Set LPCR and RMOR. */
48310:	ld	r8,VCORE_LPCR(r5)
484	mtspr	SPRN_LPCR,r8
485	ld	r8,KVM_RMOR(r9)
486	mtspr	SPRN_RMOR,r8
487	isync
488
489	/* Check if HDEC expires soon */
490	mfspr	r3,SPRN_HDEC
491	cmpwi	r3,512		/* 1 microsecond */
492	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
493	blt	hdec_soon
494
495	/* Do we have a guest vcpu to run? */
496	cmpdi	r4, 0
497	beq	kvmppc_primary_no_guest
498kvmppc_got_guest:
499
500	/* Load up guest SLB entries */
501	lwz	r5,VCPU_SLB_MAX(r4)
502	cmpwi	r5,0
503	beq	9f
504	mtctr	r5
505	addi	r6,r4,VCPU_SLB
5061:	ld	r8,VCPU_SLB_E(r6)
507	ld	r9,VCPU_SLB_V(r6)
508	slbmte	r9,r8
509	addi	r6,r6,VCPU_SLB_SIZE
510	bdnz	1b
5119:
512	/* Increment yield count if they have a VPA */
513	ld	r3, VCPU_VPA(r4)
514	cmpdi	r3, 0
515	beq	25f
516	li	r6, LPPACA_YIELDCOUNT
517	LWZX_BE	r5, r3, r6
518	addi	r5, r5, 1
519	STWX_BE	r5, r3, r6
520	li	r6, 1
521	stb	r6, VCPU_VPA_DIRTY(r4)
52225:
523
524	/* Save purr/spurr */
525	mfspr	r5,SPRN_PURR
526	mfspr	r6,SPRN_SPURR
527	std	r5,HSTATE_PURR(r13)
528	std	r6,HSTATE_SPURR(r13)
529	ld	r7,VCPU_PURR(r4)
530	ld	r8,VCPU_SPURR(r4)
531	mtspr	SPRN_PURR,r7
532	mtspr	SPRN_SPURR,r8
533
534BEGIN_FTR_SECTION
535	/* Set partition DABR */
536	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
537	lwz	r5,VCPU_DABRX(r4)
538	ld	r6,VCPU_DABR(r4)
539	mtspr	SPRN_DABRX,r5
540	mtspr	SPRN_DABR,r6
541	isync
542END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
543
544#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
545BEGIN_FTR_SECTION
546	b	skip_tm
547END_FTR_SECTION_IFCLR(CPU_FTR_TM)
548
549	/* Turn on TM/FP/VSX/VMX so we can restore them. */
550	mfmsr	r5
551	li	r6, MSR_TM >> 32
552	sldi	r6, r6, 32
553	or	r5, r5, r6
554	ori	r5, r5, MSR_FP
555	oris	r5, r5, (MSR_VEC | MSR_VSX)@h
556	mtmsrd	r5
557
558	/*
559	 * The user may change these outside of a transaction, so they must
560	 * always be context switched.
561	 */
562	ld	r5, VCPU_TFHAR(r4)
563	ld	r6, VCPU_TFIAR(r4)
564	ld	r7, VCPU_TEXASR(r4)
565	mtspr	SPRN_TFHAR, r5
566	mtspr	SPRN_TFIAR, r6
567	mtspr	SPRN_TEXASR, r7
568
569	ld	r5, VCPU_MSR(r4)
570	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
571	beq	skip_tm	/* TM not active in guest */
572
573	/* Make sure the failure summary is set, otherwise we'll program check
574	 * when we trechkpt.  It's possible that this might have been not set
575	 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
576	 * host.
577	 */
578	oris	r7, r7, (TEXASR_FS)@h
579	mtspr	SPRN_TEXASR, r7
580
581	/*
582	 * We need to load up the checkpointed state for the guest.
583	 * We need to do this early as it will blow away any GPRs, VSRs and
584	 * some SPRs.
585	 */
586
587	mr	r31, r4
588	addi	r3, r31, VCPU_FPRS_TM
589	bl	load_fp_state
590	addi	r3, r31, VCPU_VRS_TM
591	bl	load_vr_state
592	mr	r4, r31
593	lwz	r7, VCPU_VRSAVE_TM(r4)
594	mtspr	SPRN_VRSAVE, r7
595
596	ld	r5, VCPU_LR_TM(r4)
597	lwz	r6, VCPU_CR_TM(r4)
598	ld	r7, VCPU_CTR_TM(r4)
599	ld	r8, VCPU_AMR_TM(r4)
600	ld	r9, VCPU_TAR_TM(r4)
601	mtlr	r5
602	mtcr	r6
603	mtctr	r7
604	mtspr	SPRN_AMR, r8
605	mtspr	SPRN_TAR, r9
606
607	/*
608	 * Load up PPR and DSCR values but don't put them in the actual SPRs
609	 * till the last moment to avoid running with userspace PPR and DSCR for
610	 * too long.
611	 */
612	ld	r29, VCPU_DSCR_TM(r4)
613	ld	r30, VCPU_PPR_TM(r4)
614
615	std	r2, PACATMSCRATCH(r13) /* Save TOC */
616
617	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
618	li	r5, 0
619	mtmsrd	r5, 1
620
621	/* Load GPRs r0-r28 */
622	reg = 0
623	.rept	29
624	ld	reg, VCPU_GPRS_TM(reg)(r31)
625	reg = reg + 1
626	.endr
627
628	mtspr	SPRN_DSCR, r29
629	mtspr	SPRN_PPR, r30
630
631	/* Load final GPRs */
632	ld	29, VCPU_GPRS_TM(29)(r31)
633	ld	30, VCPU_GPRS_TM(30)(r31)
634	ld	31, VCPU_GPRS_TM(31)(r31)
635
636	/* TM checkpointed state is now setup.  All GPRs are now volatile. */
637	TRECHKPT
638
639	/* Now let's get back the state we need. */
640	HMT_MEDIUM
641	GET_PACA(r13)
642	ld	r29, HSTATE_DSCR(r13)
643	mtspr	SPRN_DSCR, r29
644	ld	r4, HSTATE_KVM_VCPU(r13)
645	ld	r1, HSTATE_HOST_R1(r13)
646	ld	r2, PACATMSCRATCH(r13)
647
648	/* Set the MSR RI since we have our registers back. */
649	li	r5, MSR_RI
650	mtmsrd	r5, 1
651skip_tm:
652#endif
653
654	/* Load guest PMU registers */
655	/* R4 is live here (vcpu pointer) */
656	li	r3, 1
657	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
658	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
659	isync
660BEGIN_FTR_SECTION
661	ld	r3, VCPU_MMCR(r4)
662	andi.	r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
663	cmpwi	r5, MMCR0_PMAO
664	beql	kvmppc_fix_pmao
665END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
666	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
667	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
668	lwz	r6, VCPU_PMC + 8(r4)
669	lwz	r7, VCPU_PMC + 12(r4)
670	lwz	r8, VCPU_PMC + 16(r4)
671	lwz	r9, VCPU_PMC + 20(r4)
672	mtspr	SPRN_PMC1, r3
673	mtspr	SPRN_PMC2, r5
674	mtspr	SPRN_PMC3, r6
675	mtspr	SPRN_PMC4, r7
676	mtspr	SPRN_PMC5, r8
677	mtspr	SPRN_PMC6, r9
678	ld	r3, VCPU_MMCR(r4)
679	ld	r5, VCPU_MMCR + 8(r4)
680	ld	r6, VCPU_MMCR + 16(r4)
681	ld	r7, VCPU_SIAR(r4)
682	ld	r8, VCPU_SDAR(r4)
683	mtspr	SPRN_MMCR1, r5
684	mtspr	SPRN_MMCRA, r6
685	mtspr	SPRN_SIAR, r7
686	mtspr	SPRN_SDAR, r8
687BEGIN_FTR_SECTION
688	ld	r5, VCPU_MMCR + 24(r4)
689	ld	r6, VCPU_SIER(r4)
690	lwz	r7, VCPU_PMC + 24(r4)
691	lwz	r8, VCPU_PMC + 28(r4)
692	ld	r9, VCPU_MMCR + 32(r4)
693	mtspr	SPRN_MMCR2, r5
694	mtspr	SPRN_SIER, r6
695	mtspr	SPRN_SPMC1, r7
696	mtspr	SPRN_SPMC2, r8
697	mtspr	SPRN_MMCRS, r9
698END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
699	mtspr	SPRN_MMCR0, r3
700	isync
701
702	/* Load up FP, VMX and VSX registers */
703	bl	kvmppc_load_fp
704
705	ld	r14, VCPU_GPR(R14)(r4)
706	ld	r15, VCPU_GPR(R15)(r4)
707	ld	r16, VCPU_GPR(R16)(r4)
708	ld	r17, VCPU_GPR(R17)(r4)
709	ld	r18, VCPU_GPR(R18)(r4)
710	ld	r19, VCPU_GPR(R19)(r4)
711	ld	r20, VCPU_GPR(R20)(r4)
712	ld	r21, VCPU_GPR(R21)(r4)
713	ld	r22, VCPU_GPR(R22)(r4)
714	ld	r23, VCPU_GPR(R23)(r4)
715	ld	r24, VCPU_GPR(R24)(r4)
716	ld	r25, VCPU_GPR(R25)(r4)
717	ld	r26, VCPU_GPR(R26)(r4)
718	ld	r27, VCPU_GPR(R27)(r4)
719	ld	r28, VCPU_GPR(R28)(r4)
720	ld	r29, VCPU_GPR(R29)(r4)
721	ld	r30, VCPU_GPR(R30)(r4)
722	ld	r31, VCPU_GPR(R31)(r4)
723
724	/* Switch DSCR to guest value */
725	ld	r5, VCPU_DSCR(r4)
726	mtspr	SPRN_DSCR, r5
727
728BEGIN_FTR_SECTION
729	/* Skip next section on POWER7 */
730	b	8f
731END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
732	/* Turn on TM so we can access TFHAR/TFIAR/TEXASR */
733	mfmsr	r8
734	li	r0, 1
735	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
736	mtmsrd	r8
737
738	/* Load up POWER8-specific registers */
739	ld	r5, VCPU_IAMR(r4)
740	lwz	r6, VCPU_PSPB(r4)
741	ld	r7, VCPU_FSCR(r4)
742	mtspr	SPRN_IAMR, r5
743	mtspr	SPRN_PSPB, r6
744	mtspr	SPRN_FSCR, r7
745	ld	r5, VCPU_DAWR(r4)
746	ld	r6, VCPU_DAWRX(r4)
747	ld	r7, VCPU_CIABR(r4)
748	ld	r8, VCPU_TAR(r4)
749	mtspr	SPRN_DAWR, r5
750	mtspr	SPRN_DAWRX, r6
751	mtspr	SPRN_CIABR, r7
752	mtspr	SPRN_TAR, r8
753	ld	r5, VCPU_IC(r4)
754	ld	r6, VCPU_VTB(r4)
755	mtspr	SPRN_IC, r5
756	mtspr	SPRN_VTB, r6
757	ld	r8, VCPU_EBBHR(r4)
758	mtspr	SPRN_EBBHR, r8
759	ld	r5, VCPU_EBBRR(r4)
760	ld	r6, VCPU_BESCR(r4)
761	ld	r7, VCPU_CSIGR(r4)
762	ld	r8, VCPU_TACR(r4)
763	mtspr	SPRN_EBBRR, r5
764	mtspr	SPRN_BESCR, r6
765	mtspr	SPRN_CSIGR, r7
766	mtspr	SPRN_TACR, r8
767	ld	r5, VCPU_TCSCR(r4)
768	ld	r6, VCPU_ACOP(r4)
769	lwz	r7, VCPU_GUEST_PID(r4)
770	ld	r8, VCPU_WORT(r4)
771	mtspr	SPRN_TCSCR, r5
772	mtspr	SPRN_ACOP, r6
773	mtspr	SPRN_PID, r7
774	mtspr	SPRN_WORT, r8
7758:
776
777	/*
778	 * Set the decrementer to the guest decrementer.
779	 */
780	ld	r8,VCPU_DEC_EXPIRES(r4)
781	/* r8 is a host timebase value here, convert to guest TB */
782	ld	r5,HSTATE_KVM_VCORE(r13)
783	ld	r6,VCORE_TB_OFFSET(r5)
784	add	r8,r8,r6
785	mftb	r7
786	subf	r3,r7,r8
787	mtspr	SPRN_DEC,r3
788	stw	r3,VCPU_DEC(r4)
789
790	ld	r5, VCPU_SPRG0(r4)
791	ld	r6, VCPU_SPRG1(r4)
792	ld	r7, VCPU_SPRG2(r4)
793	ld	r8, VCPU_SPRG3(r4)
794	mtspr	SPRN_SPRG0, r5
795	mtspr	SPRN_SPRG1, r6
796	mtspr	SPRN_SPRG2, r7
797	mtspr	SPRN_SPRG3, r8
798
799	/* Load up DAR and DSISR */
800	ld	r5, VCPU_DAR(r4)
801	lwz	r6, VCPU_DSISR(r4)
802	mtspr	SPRN_DAR, r5
803	mtspr	SPRN_DSISR, r6
804
805	/* Restore AMR and UAMOR, set AMOR to all 1s */
806	ld	r5,VCPU_AMR(r4)
807	ld	r6,VCPU_UAMOR(r4)
808	li	r7,-1
809	mtspr	SPRN_AMR,r5
810	mtspr	SPRN_UAMOR,r6
811	mtspr	SPRN_AMOR,r7
812
813	/* Restore state of CTRL run bit; assume 1 on entry */
814	lwz	r5,VCPU_CTRL(r4)
815	andi.	r5,r5,1
816	bne	4f
817	mfspr	r6,SPRN_CTRLF
818	clrrdi	r6,r6,1
819	mtspr	SPRN_CTRLT,r6
8204:
821	ld	r6, VCPU_CTR(r4)
822	lwz	r7, VCPU_XER(r4)
823
824	mtctr	r6
825	mtxer	r7
826
827kvmppc_cede_reentry:		/* r4 = vcpu, r13 = paca */
828	ld	r10, VCPU_PC(r4)
829	ld	r11, VCPU_MSR(r4)
830	ld	r6, VCPU_SRR0(r4)
831	ld	r7, VCPU_SRR1(r4)
832	mtspr	SPRN_SRR0, r6
833	mtspr	SPRN_SRR1, r7
834
835deliver_guest_interrupt:
836	/* r11 = vcpu->arch.msr & ~MSR_HV */
837	rldicl	r11, r11, 63 - MSR_HV_LG, 1
838	rotldi	r11, r11, 1 + MSR_HV_LG
839	ori	r11, r11, MSR_ME
840
841	/* Check if we can deliver an external or decrementer interrupt now */
842	ld	r0, VCPU_PENDING_EXC(r4)
843	rldicl	r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
844	cmpdi	cr1, r0, 0
845	andi.	r8, r11, MSR_EE
846	mfspr	r8, SPRN_LPCR
847	/* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
848	rldimi	r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
849	mtspr	SPRN_LPCR, r8
850	isync
851	beq	5f
852	li	r0, BOOK3S_INTERRUPT_EXTERNAL
853	bne	cr1, 12f
854	mfspr	r0, SPRN_DEC
855	cmpwi	r0, 0
856	li	r0, BOOK3S_INTERRUPT_DECREMENTER
857	bge	5f
858
85912:	mtspr	SPRN_SRR0, r10
860	mr	r10,r0
861	mtspr	SPRN_SRR1, r11
862	mr	r9, r4
863	bl	kvmppc_msr_interrupt
8645:
865
866/*
867 * Required state:
868 * R4 = vcpu
869 * R10: value for HSRR0
870 * R11: value for HSRR1
871 * R13 = PACA
872 */
873fast_guest_return:
874	li	r0,0
875	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
876	mtspr	SPRN_HSRR0,r10
877	mtspr	SPRN_HSRR1,r11
878
879	/* Activate guest mode, so faults get handled by KVM */
880	li	r9, KVM_GUEST_MODE_GUEST_HV
881	stb	r9, HSTATE_IN_GUEST(r13)
882
883	/* Enter guest */
884
885BEGIN_FTR_SECTION
886	ld	r5, VCPU_CFAR(r4)
887	mtspr	SPRN_CFAR, r5
888END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
889BEGIN_FTR_SECTION
890	ld	r0, VCPU_PPR(r4)
891END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
892
893	ld	r5, VCPU_LR(r4)
894	lwz	r6, VCPU_CR(r4)
895	mtlr	r5
896	mtcr	r6
897
898	ld	r1, VCPU_GPR(R1)(r4)
899	ld	r2, VCPU_GPR(R2)(r4)
900	ld	r3, VCPU_GPR(R3)(r4)
901	ld	r5, VCPU_GPR(R5)(r4)
902	ld	r6, VCPU_GPR(R6)(r4)
903	ld	r7, VCPU_GPR(R7)(r4)
904	ld	r8, VCPU_GPR(R8)(r4)
905	ld	r9, VCPU_GPR(R9)(r4)
906	ld	r10, VCPU_GPR(R10)(r4)
907	ld	r11, VCPU_GPR(R11)(r4)
908	ld	r12, VCPU_GPR(R12)(r4)
909	ld	r13, VCPU_GPR(R13)(r4)
910
911BEGIN_FTR_SECTION
912	mtspr	SPRN_PPR, r0
913END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
914	ld	r0, VCPU_GPR(R0)(r4)
915	ld	r4, VCPU_GPR(R4)(r4)
916
917	hrfid
918	b	.
919
920/******************************************************************************
921 *                                                                            *
922 *                               Exit code                                    *
923 *                                                                            *
924 *****************************************************************************/
925
926/*
927 * We come here from the first-level interrupt handlers.
928 */
929	.globl	kvmppc_interrupt_hv
930kvmppc_interrupt_hv:
931	/*
932	 * Register contents:
933	 * R12		= interrupt vector
934	 * R13		= PACA
935	 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
936	 * guest R13 saved in SPRN_SCRATCH0
937	 */
938	std	r9, HSTATE_SCRATCH2(r13)
939
940	lbz	r9, HSTATE_IN_GUEST(r13)
941	cmpwi	r9, KVM_GUEST_MODE_HOST_HV
942	beq	kvmppc_bad_host_intr
943#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
944	cmpwi	r9, KVM_GUEST_MODE_GUEST
945	ld	r9, HSTATE_SCRATCH2(r13)
946	beq	kvmppc_interrupt_pr
947#endif
948	/* We're now back in the host but in guest MMU context */
949	li	r9, KVM_GUEST_MODE_HOST_HV
950	stb	r9, HSTATE_IN_GUEST(r13)
951
952	ld	r9, HSTATE_KVM_VCPU(r13)
953
954	/* Save registers */
955
956	std	r0, VCPU_GPR(R0)(r9)
957	std	r1, VCPU_GPR(R1)(r9)
958	std	r2, VCPU_GPR(R2)(r9)
959	std	r3, VCPU_GPR(R3)(r9)
960	std	r4, VCPU_GPR(R4)(r9)
961	std	r5, VCPU_GPR(R5)(r9)
962	std	r6, VCPU_GPR(R6)(r9)
963	std	r7, VCPU_GPR(R7)(r9)
964	std	r8, VCPU_GPR(R8)(r9)
965	ld	r0, HSTATE_SCRATCH2(r13)
966	std	r0, VCPU_GPR(R9)(r9)
967	std	r10, VCPU_GPR(R10)(r9)
968	std	r11, VCPU_GPR(R11)(r9)
969	ld	r3, HSTATE_SCRATCH0(r13)
970	lwz	r4, HSTATE_SCRATCH1(r13)
971	std	r3, VCPU_GPR(R12)(r9)
972	stw	r4, VCPU_CR(r9)
973BEGIN_FTR_SECTION
974	ld	r3, HSTATE_CFAR(r13)
975	std	r3, VCPU_CFAR(r9)
976END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
977BEGIN_FTR_SECTION
978	ld	r4, HSTATE_PPR(r13)
979	std	r4, VCPU_PPR(r9)
980END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
981
982	/* Restore R1/R2 so we can handle faults */
983	ld	r1, HSTATE_HOST_R1(r13)
984	ld	r2, PACATOC(r13)
985
986	mfspr	r10, SPRN_SRR0
987	mfspr	r11, SPRN_SRR1
988	std	r10, VCPU_SRR0(r9)
989	std	r11, VCPU_SRR1(r9)
990	andi.	r0, r12, 2		/* need to read HSRR0/1? */
991	beq	1f
992	mfspr	r10, SPRN_HSRR0
993	mfspr	r11, SPRN_HSRR1
994	clrrdi	r12, r12, 2
9951:	std	r10, VCPU_PC(r9)
996	std	r11, VCPU_MSR(r9)
997
998	GET_SCRATCH0(r3)
999	mflr	r4
1000	std	r3, VCPU_GPR(R13)(r9)
1001	std	r4, VCPU_LR(r9)
1002
1003	stw	r12,VCPU_TRAP(r9)
1004
1005	/* Save HEIR (HV emulation assist reg) in emul_inst
1006	   if this is an HEI (HV emulation interrupt, e40) */
1007	li	r3,KVM_INST_FETCH_FAILED
1008	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1009	bne	11f
1010	mfspr	r3,SPRN_HEIR
101111:	stw	r3,VCPU_HEIR(r9)
1012
1013	/* these are volatile across C function calls */
1014	mfctr	r3
1015	mfxer	r4
1016	std	r3, VCPU_CTR(r9)
1017	stw	r4, VCPU_XER(r9)
1018
1019	/* If this is a page table miss then see if it's theirs or ours */
1020	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1021	beq	kvmppc_hdsi
1022	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1023	beq	kvmppc_hisi
1024
1025	/* See if this is a leftover HDEC interrupt */
1026	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1027	bne	2f
1028	mfspr	r3,SPRN_HDEC
1029	cmpwi	r3,0
1030	bge	ignore_hdec
10312:
1032	/* See if this is an hcall we can handle in real mode */
1033	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
1034	beq	hcall_try_real_mode
1035
1036	/* External interrupt ? */
1037	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
1038	bne+	ext_interrupt_to_host
1039
1040	/* External interrupt, first check for host_ipi. If this is
1041	 * set, we know the host wants us out so let's do it now
1042	 */
1043	bl	kvmppc_read_intr
1044	cmpdi	r3, 0
1045	bgt	ext_interrupt_to_host
1046
1047	/* Check if any CPU is heading out to the host, if so head out too */
1048	ld	r5, HSTATE_KVM_VCORE(r13)
1049	lwz	r0, VCORE_ENTRY_EXIT(r5)
1050	cmpwi	r0, 0x100
1051	bge	ext_interrupt_to_host
1052
1053	/* Return to guest after delivering any pending interrupt */
1054	mr	r4, r9
1055	b	deliver_guest_interrupt
1056
1057ext_interrupt_to_host:
1058
1059guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
1060	/* Save more register state  */
1061	mfdar	r6
1062	mfdsisr	r7
1063	std	r6, VCPU_DAR(r9)
1064	stw	r7, VCPU_DSISR(r9)
1065	/* don't overwrite fault_dar/fault_dsisr if HDSI */
1066	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1067	beq	6f
1068	std	r6, VCPU_FAULT_DAR(r9)
1069	stw	r7, VCPU_FAULT_DSISR(r9)
1070
1071	/* See if it is a machine check */
1072	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1073	beq	machine_check_realmode
1074mc_cont:
1075
1076	/* Save guest CTRL register, set runlatch to 1 */
10776:	mfspr	r6,SPRN_CTRLF
1078	stw	r6,VCPU_CTRL(r9)
1079	andi.	r0,r6,1
1080	bne	4f
1081	ori	r6,r6,1
1082	mtspr	SPRN_CTRLT,r6
10834:
1084	/* Read the guest SLB and save it away */
1085	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
1086	mtctr	r0
1087	li	r6,0
1088	addi	r7,r9,VCPU_SLB
1089	li	r5,0
10901:	slbmfee	r8,r6
1091	andis.	r0,r8,SLB_ESID_V@h
1092	beq	2f
1093	add	r8,r8,r6		/* put index in */
1094	slbmfev	r3,r6
1095	std	r8,VCPU_SLB_E(r7)
1096	std	r3,VCPU_SLB_V(r7)
1097	addi	r7,r7,VCPU_SLB_SIZE
1098	addi	r5,r5,1
10992:	addi	r6,r6,1
1100	bdnz	1b
1101	stw	r5,VCPU_SLB_MAX(r9)
1102
1103	/*
1104	 * Save the guest PURR/SPURR
1105	 */
1106	mfspr	r5,SPRN_PURR
1107	mfspr	r6,SPRN_SPURR
1108	ld	r7,VCPU_PURR(r9)
1109	ld	r8,VCPU_SPURR(r9)
1110	std	r5,VCPU_PURR(r9)
1111	std	r6,VCPU_SPURR(r9)
1112	subf	r5,r7,r5
1113	subf	r6,r8,r6
1114
1115	/*
1116	 * Restore host PURR/SPURR and add guest times
1117	 * so that the time in the guest gets accounted.
1118	 */
1119	ld	r3,HSTATE_PURR(r13)
1120	ld	r4,HSTATE_SPURR(r13)
1121	add	r3,r3,r5
1122	add	r4,r4,r6
1123	mtspr	SPRN_PURR,r3
1124	mtspr	SPRN_SPURR,r4
1125
1126	/* Save DEC */
1127	mfspr	r5,SPRN_DEC
1128	mftb	r6
1129	extsw	r5,r5
1130	add	r5,r5,r6
1131	/* r5 is a guest timebase value here, convert to host TB */
1132	ld	r3,HSTATE_KVM_VCORE(r13)
1133	ld	r4,VCORE_TB_OFFSET(r3)
1134	subf	r5,r4,r5
1135	std	r5,VCPU_DEC_EXPIRES(r9)
1136
1137BEGIN_FTR_SECTION
1138	b	8f
1139END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1140	/* Save POWER8-specific registers */
1141	mfspr	r5, SPRN_IAMR
1142	mfspr	r6, SPRN_PSPB
1143	mfspr	r7, SPRN_FSCR
1144	std	r5, VCPU_IAMR(r9)
1145	stw	r6, VCPU_PSPB(r9)
1146	std	r7, VCPU_FSCR(r9)
1147	mfspr	r5, SPRN_IC
1148	mfspr	r6, SPRN_VTB
1149	mfspr	r7, SPRN_TAR
1150	std	r5, VCPU_IC(r9)
1151	std	r6, VCPU_VTB(r9)
1152	std	r7, VCPU_TAR(r9)
1153	mfspr	r8, SPRN_EBBHR
1154	std	r8, VCPU_EBBHR(r9)
1155	mfspr	r5, SPRN_EBBRR
1156	mfspr	r6, SPRN_BESCR
1157	mfspr	r7, SPRN_CSIGR
1158	mfspr	r8, SPRN_TACR
1159	std	r5, VCPU_EBBRR(r9)
1160	std	r6, VCPU_BESCR(r9)
1161	std	r7, VCPU_CSIGR(r9)
1162	std	r8, VCPU_TACR(r9)
1163	mfspr	r5, SPRN_TCSCR
1164	mfspr	r6, SPRN_ACOP
1165	mfspr	r7, SPRN_PID
1166	mfspr	r8, SPRN_WORT
1167	std	r5, VCPU_TCSCR(r9)
1168	std	r6, VCPU_ACOP(r9)
1169	stw	r7, VCPU_GUEST_PID(r9)
1170	std	r8, VCPU_WORT(r9)
11718:
1172
1173	/* Save and reset AMR and UAMOR before turning on the MMU */
1174	mfspr	r5,SPRN_AMR
1175	mfspr	r6,SPRN_UAMOR
1176	std	r5,VCPU_AMR(r9)
1177	std	r6,VCPU_UAMOR(r9)
1178	li	r6,0
1179	mtspr	SPRN_AMR,r6
1180
1181	/* Switch DSCR back to host value */
1182	mfspr	r8, SPRN_DSCR
1183	ld	r7, HSTATE_DSCR(r13)
1184	std	r8, VCPU_DSCR(r9)
1185	mtspr	SPRN_DSCR, r7
1186
1187	/* Save non-volatile GPRs */
1188	std	r14, VCPU_GPR(R14)(r9)
1189	std	r15, VCPU_GPR(R15)(r9)
1190	std	r16, VCPU_GPR(R16)(r9)
1191	std	r17, VCPU_GPR(R17)(r9)
1192	std	r18, VCPU_GPR(R18)(r9)
1193	std	r19, VCPU_GPR(R19)(r9)
1194	std	r20, VCPU_GPR(R20)(r9)
1195	std	r21, VCPU_GPR(R21)(r9)
1196	std	r22, VCPU_GPR(R22)(r9)
1197	std	r23, VCPU_GPR(R23)(r9)
1198	std	r24, VCPU_GPR(R24)(r9)
1199	std	r25, VCPU_GPR(R25)(r9)
1200	std	r26, VCPU_GPR(R26)(r9)
1201	std	r27, VCPU_GPR(R27)(r9)
1202	std	r28, VCPU_GPR(R28)(r9)
1203	std	r29, VCPU_GPR(R29)(r9)
1204	std	r30, VCPU_GPR(R30)(r9)
1205	std	r31, VCPU_GPR(R31)(r9)
1206
1207	/* Save SPRGs */
1208	mfspr	r3, SPRN_SPRG0
1209	mfspr	r4, SPRN_SPRG1
1210	mfspr	r5, SPRN_SPRG2
1211	mfspr	r6, SPRN_SPRG3
1212	std	r3, VCPU_SPRG0(r9)
1213	std	r4, VCPU_SPRG1(r9)
1214	std	r5, VCPU_SPRG2(r9)
1215	std	r6, VCPU_SPRG3(r9)
1216
1217	/* save FP state */
1218	mr	r3, r9
1219	bl	kvmppc_save_fp
1220
1221#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1222BEGIN_FTR_SECTION
1223	b	2f
1224END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1225	/* Turn on TM. */
1226	mfmsr	r8
1227	li	r0, 1
1228	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
1229	mtmsrd	r8
1230
1231	ld	r5, VCPU_MSR(r9)
1232	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
1233	beq	1f	/* TM not active in guest. */
1234
1235	li	r3, TM_CAUSE_KVM_RESCHED
1236
1237	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
1238	li	r5, 0
1239	mtmsrd	r5, 1
1240
1241	/* All GPRs are volatile at this point. */
1242	TRECLAIM(R3)
1243
1244	/* Temporarily store r13 and r9 so we have some regs to play with */
1245	SET_SCRATCH0(r13)
1246	GET_PACA(r13)
1247	std	r9, PACATMSCRATCH(r13)
1248	ld	r9, HSTATE_KVM_VCPU(r13)
1249
1250	/* Get a few more GPRs free. */
1251	std	r29, VCPU_GPRS_TM(29)(r9)
1252	std	r30, VCPU_GPRS_TM(30)(r9)
1253	std	r31, VCPU_GPRS_TM(31)(r9)
1254
1255	/* Save away PPR and DSCR soon so don't run with user values. */
1256	mfspr	r31, SPRN_PPR
1257	HMT_MEDIUM
1258	mfspr	r30, SPRN_DSCR
1259	ld	r29, HSTATE_DSCR(r13)
1260	mtspr	SPRN_DSCR, r29
1261
1262	/* Save all but r9, r13 & r29-r31 */
1263	reg = 0
1264	.rept	29
1265	.if (reg != 9) && (reg != 13)
1266	std	reg, VCPU_GPRS_TM(reg)(r9)
1267	.endif
1268	reg = reg + 1
1269	.endr
1270	/* ... now save r13 */
1271	GET_SCRATCH0(r4)
1272	std	r4, VCPU_GPRS_TM(13)(r9)
1273	/* ... and save r9 */
1274	ld	r4, PACATMSCRATCH(r13)
1275	std	r4, VCPU_GPRS_TM(9)(r9)
1276
1277	/* Reload stack pointer and TOC. */
1278	ld	r1, HSTATE_HOST_R1(r13)
1279	ld	r2, PACATOC(r13)
1280
1281	/* Set MSR RI now we have r1 and r13 back. */
1282	li	r5, MSR_RI
1283	mtmsrd	r5, 1
1284
1285	/* Save away checkpinted SPRs. */
1286	std	r31, VCPU_PPR_TM(r9)
1287	std	r30, VCPU_DSCR_TM(r9)
1288	mflr	r5
1289	mfcr	r6
1290	mfctr	r7
1291	mfspr	r8, SPRN_AMR
1292	mfspr	r10, SPRN_TAR
1293	std	r5, VCPU_LR_TM(r9)
1294	stw	r6, VCPU_CR_TM(r9)
1295	std	r7, VCPU_CTR_TM(r9)
1296	std	r8, VCPU_AMR_TM(r9)
1297	std	r10, VCPU_TAR_TM(r9)
1298
1299	/* Restore r12 as trap number. */
1300	lwz	r12, VCPU_TRAP(r9)
1301
1302	/* Save FP/VSX. */
1303	addi	r3, r9, VCPU_FPRS_TM
1304	bl	store_fp_state
1305	addi	r3, r9, VCPU_VRS_TM
1306	bl	store_vr_state
1307	mfspr	r6, SPRN_VRSAVE
1308	stw	r6, VCPU_VRSAVE_TM(r9)
13091:
1310	/*
1311	 * We need to save these SPRs after the treclaim so that the software
1312	 * error code is recorded correctly in the TEXASR.  Also the user may
1313	 * change these outside of a transaction, so they must always be
1314	 * context switched.
1315	 */
1316	mfspr	r5, SPRN_TFHAR
1317	mfspr	r6, SPRN_TFIAR
1318	mfspr	r7, SPRN_TEXASR
1319	std	r5, VCPU_TFHAR(r9)
1320	std	r6, VCPU_TFIAR(r9)
1321	std	r7, VCPU_TEXASR(r9)
13222:
1323#endif
1324
1325	/* Increment yield count if they have a VPA */
1326	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
1327	cmpdi	r8, 0
1328	beq	25f
1329	li	r4, LPPACA_YIELDCOUNT
1330	LWZX_BE	r3, r8, r4
1331	addi	r3, r3, 1
1332	STWX_BE	r3, r8, r4
1333	li	r3, 1
1334	stb	r3, VCPU_VPA_DIRTY(r9)
133525:
1336	/* Save PMU registers if requested */
1337	/* r8 and cr0.eq are live here */
1338BEGIN_FTR_SECTION
1339	/*
1340	 * POWER8 seems to have a hardware bug where setting
1341	 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
1342	 * when some counters are already negative doesn't seem
1343	 * to cause a performance monitor alert (and hence interrupt).
1344	 * The effect of this is that when saving the PMU state,
1345	 * if there is no PMU alert pending when we read MMCR0
1346	 * before freezing the counters, but one becomes pending
1347	 * before we read the counters, we lose it.
1348	 * To work around this, we need a way to freeze the counters
1349	 * before reading MMCR0.  Normally, freezing the counters
1350	 * is done by writing MMCR0 (to set MMCR0[FC]) which
1351	 * unavoidably writes MMCR0[PMA0] as well.  On POWER8,
1352	 * we can also freeze the counters using MMCR2, by writing
1353	 * 1s to all the counter freeze condition bits (there are
1354	 * 9 bits each for 6 counters).
1355	 */
1356	li	r3, -1			/* set all freeze bits */
1357	clrrdi	r3, r3, 10
1358	mfspr	r10, SPRN_MMCR2
1359	mtspr	SPRN_MMCR2, r3
1360	isync
1361END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1362	li	r3, 1
1363	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
1364	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
1365	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
1366	mfspr	r6, SPRN_MMCRA
1367	/* Clear MMCRA in order to disable SDAR updates */
1368	li	r7, 0
1369	mtspr	SPRN_MMCRA, r7
1370	isync
1371	beq	21f			/* if no VPA, save PMU stuff anyway */
1372	lbz	r7, LPPACA_PMCINUSE(r8)
1373	cmpwi	r7, 0			/* did they ask for PMU stuff to be saved? */
1374	bne	21f
1375	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
1376	b	22f
137721:	mfspr	r5, SPRN_MMCR1
1378	mfspr	r7, SPRN_SIAR
1379	mfspr	r8, SPRN_SDAR
1380	std	r4, VCPU_MMCR(r9)
1381	std	r5, VCPU_MMCR + 8(r9)
1382	std	r6, VCPU_MMCR + 16(r9)
1383BEGIN_FTR_SECTION
1384	std	r10, VCPU_MMCR + 24(r9)
1385END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1386	std	r7, VCPU_SIAR(r9)
1387	std	r8, VCPU_SDAR(r9)
1388	mfspr	r3, SPRN_PMC1
1389	mfspr	r4, SPRN_PMC2
1390	mfspr	r5, SPRN_PMC3
1391	mfspr	r6, SPRN_PMC4
1392	mfspr	r7, SPRN_PMC5
1393	mfspr	r8, SPRN_PMC6
1394	stw	r3, VCPU_PMC(r9)
1395	stw	r4, VCPU_PMC + 4(r9)
1396	stw	r5, VCPU_PMC + 8(r9)
1397	stw	r6, VCPU_PMC + 12(r9)
1398	stw	r7, VCPU_PMC + 16(r9)
1399	stw	r8, VCPU_PMC + 20(r9)
1400BEGIN_FTR_SECTION
1401	mfspr	r5, SPRN_SIER
1402	mfspr	r6, SPRN_SPMC1
1403	mfspr	r7, SPRN_SPMC2
1404	mfspr	r8, SPRN_MMCRS
1405	std	r5, VCPU_SIER(r9)
1406	stw	r6, VCPU_PMC + 24(r9)
1407	stw	r7, VCPU_PMC + 28(r9)
1408	std	r8, VCPU_MMCR + 32(r9)
1409	lis	r4, 0x8000
1410	mtspr	SPRN_MMCRS, r4
1411END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
141222:
1413	/* Clear out SLB */
1414	li	r5,0
1415	slbmte	r5,r5
1416	slbia
1417	ptesync
1418
1419hdec_soon:			/* r12 = trap, r13 = paca */
1420	/*
1421	 * POWER7/POWER8 guest -> host partition switch code.
1422	 * We don't have to lock against tlbies but we do
1423	 * have to coordinate the hardware threads.
1424	 */
1425	/* Increment the threads-exiting-guest count in the 0xff00
1426	   bits of vcore->entry_exit_count */
1427	ld	r5,HSTATE_KVM_VCORE(r13)
1428	addi	r6,r5,VCORE_ENTRY_EXIT
142941:	lwarx	r3,0,r6
1430	addi	r0,r3,0x100
1431	stwcx.	r0,0,r6
1432	bne	41b
1433	isync		/* order stwcx. vs. reading napping_threads */
1434
1435	/*
1436	 * At this point we have an interrupt that we have to pass
1437	 * up to the kernel or qemu; we can't handle it in real mode.
1438	 * Thus we have to do a partition switch, so we have to
1439	 * collect the other threads, if we are the first thread
1440	 * to take an interrupt.  To do this, we set the HDEC to 0,
1441	 * which causes an HDEC interrupt in all threads within 2ns
1442	 * because the HDEC register is shared between all 4 threads.
1443	 * However, we don't need to bother if this is an HDEC
1444	 * interrupt, since the other threads will already be on their
1445	 * way here in that case.
1446	 */
1447	cmpwi	r3,0x100	/* Are we the first here? */
1448	bge	43f
1449	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1450	beq	40f
1451	li	r0,0
1452	mtspr	SPRN_HDEC,r0
145340:
1454	/*
1455	 * Send an IPI to any napping threads, since an HDEC interrupt
1456	 * doesn't wake CPUs up from nap.
1457	 */
1458	lwz	r3,VCORE_NAPPING_THREADS(r5)
1459	lbz	r4,HSTATE_PTID(r13)
1460	li	r0,1
1461	sld	r0,r0,r4
1462	andc.	r3,r3,r0		/* no sense IPI'ing ourselves */
1463	beq	43f
1464	/* Order entry/exit update vs. IPIs */
1465	sync
1466	mulli	r4,r4,PACA_SIZE		/* get paca for thread 0 */
1467	subf	r6,r4,r13
146842:	andi.	r0,r3,1
1469	beq	44f
1470	ld	r8,HSTATE_XICS_PHYS(r6)	/* get thread's XICS reg addr */
1471	li	r0,IPI_PRIORITY
1472	li	r7,XICS_MFRR
1473	stbcix	r0,r7,r8		/* trigger the IPI */
147444:	srdi.	r3,r3,1
1475	addi	r6,r6,PACA_SIZE
1476	bne	42b
1477
1478secondary_too_late:
1479	/* Secondary threads wait for primary to do partition switch */
148043:	ld	r5,HSTATE_KVM_VCORE(r13)
1481	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
1482	lbz	r3,HSTATE_PTID(r13)
1483	cmpwi	r3,0
1484	beq	15f
1485	HMT_LOW
148613:	lbz	r3,VCORE_IN_GUEST(r5)
1487	cmpwi	r3,0
1488	bne	13b
1489	HMT_MEDIUM
1490	b	16f
1491
1492	/* Primary thread waits for all the secondaries to exit guest */
149315:	lwz	r3,VCORE_ENTRY_EXIT(r5)
1494	srwi	r0,r3,8
1495	clrldi	r3,r3,56
1496	cmpw	r3,r0
1497	bne	15b
1498	isync
1499
1500	/* Primary thread switches back to host partition */
1501	ld	r6,KVM_HOST_SDR1(r4)
1502	lwz	r7,KVM_HOST_LPID(r4)
1503	li	r8,LPID_RSVD		/* switch to reserved LPID */
1504	mtspr	SPRN_LPID,r8
1505	ptesync
1506	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
1507	mtspr	SPRN_LPID,r7
1508	isync
1509
1510BEGIN_FTR_SECTION
1511	/* DPDES is shared between threads */
1512	mfspr	r7, SPRN_DPDES
1513	std	r7, VCORE_DPDES(r5)
1514	/* clear DPDES so we don't get guest doorbells in the host */
1515	li	r8, 0
1516	mtspr	SPRN_DPDES, r8
1517END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1518
1519	/* Subtract timebase offset from timebase */
1520	ld	r8,VCORE_TB_OFFSET(r5)
1521	cmpdi	r8,0
1522	beq	17f
1523	mftb	r6			/* current guest timebase */
1524	subf	r8,r8,r6
1525	mtspr	SPRN_TBU40,r8		/* update upper 40 bits */
1526	mftb	r7			/* check if lower 24 bits overflowed */
1527	clrldi	r6,r6,40
1528	clrldi	r7,r7,40
1529	cmpld	r7,r6
1530	bge	17f
1531	addis	r8,r8,0x100		/* if so, increment upper 40 bits */
1532	mtspr	SPRN_TBU40,r8
1533
1534	/* Reset PCR */
153517:	ld	r0, VCORE_PCR(r5)
1536	cmpdi	r0, 0
1537	beq	18f
1538	li	r0, 0
1539	mtspr	SPRN_PCR, r0
154018:
1541	/* Signal secondary CPUs to continue */
1542	stb	r0,VCORE_IN_GUEST(r5)
1543	lis	r8,0x7fff		/* MAX_INT@h */
1544	mtspr	SPRN_HDEC,r8
1545
154616:	ld	r8,KVM_HOST_LPCR(r4)
1547	mtspr	SPRN_LPCR,r8
1548	isync
1549
1550	/* load host SLB entries */
1551	ld	r8,PACA_SLBSHADOWPTR(r13)
1552
1553	.rept	SLB_NUM_BOLTED
1554	li	r3, SLBSHADOW_SAVEAREA
1555	LDX_BE	r5, r8, r3
1556	addi	r3, r3, 8
1557	LDX_BE	r6, r8, r3
1558	andis.	r7,r5,SLB_ESID_V@h
1559	beq	1f
1560	slbmte	r6,r5
15611:	addi	r8,r8,16
1562	.endr
1563
1564	/* Unset guest mode */
1565	li	r0, KVM_GUEST_MODE_NONE
1566	stb	r0, HSTATE_IN_GUEST(r13)
1567
1568	ld	r0, 112+PPC_LR_STKOFF(r1)
1569	addi	r1, r1, 112
1570	mtlr	r0
1571	blr
1572
1573/*
1574 * Check whether an HDSI is an HPTE not found fault or something else.
1575 * If it is an HPTE not found fault that is due to the guest accessing
1576 * a page that they have mapped but which we have paged out, then
1577 * we continue on with the guest exit path.  In all other cases,
1578 * reflect the HDSI to the guest as a DSI.
1579 */
1580kvmppc_hdsi:
1581	mfspr	r4, SPRN_HDAR
1582	mfspr	r6, SPRN_HDSISR
1583	/* HPTE not found fault or protection fault? */
1584	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1585	beq	1f			/* if not, send it to the guest */
1586	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
1587	beq	3f
1588	clrrdi	r0, r4, 28
1589	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1590	bne	1f			/* if no SLB entry found */
15914:	std	r4, VCPU_FAULT_DAR(r9)
1592	stw	r6, VCPU_FAULT_DSISR(r9)
1593
1594	/* Search the hash table. */
1595	mr	r3, r9			/* vcpu pointer */
1596	li	r7, 1			/* data fault */
1597	bl	kvmppc_hpte_hv_fault
1598	ld	r9, HSTATE_KVM_VCPU(r13)
1599	ld	r10, VCPU_PC(r9)
1600	ld	r11, VCPU_MSR(r9)
1601	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1602	cmpdi	r3, 0			/* retry the instruction */
1603	beq	6f
1604	cmpdi	r3, -1			/* handle in kernel mode */
1605	beq	guest_exit_cont
1606	cmpdi	r3, -2			/* MMIO emulation; need instr word */
1607	beq	2f
1608
1609	/* Synthesize a DSI for the guest */
1610	ld	r4, VCPU_FAULT_DAR(r9)
1611	mr	r6, r3
16121:	mtspr	SPRN_DAR, r4
1613	mtspr	SPRN_DSISR, r6
1614	mtspr	SPRN_SRR0, r10
1615	mtspr	SPRN_SRR1, r11
1616	li	r10, BOOK3S_INTERRUPT_DATA_STORAGE
1617	bl	kvmppc_msr_interrupt
1618fast_interrupt_c_return:
16196:	ld	r7, VCPU_CTR(r9)
1620	lwz	r8, VCPU_XER(r9)
1621	mtctr	r7
1622	mtxer	r8
1623	mr	r4, r9
1624	b	fast_guest_return
1625
16263:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
1627	ld	r5, KVM_VRMA_SLB_V(r5)
1628	b	4b
1629
1630	/* If this is for emulated MMIO, load the instruction word */
16312:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */
1632
1633	/* Set guest mode to 'jump over instruction' so if lwz faults
1634	 * we'll just continue at the next IP. */
1635	li	r0, KVM_GUEST_MODE_SKIP
1636	stb	r0, HSTATE_IN_GUEST(r13)
1637
1638	/* Do the access with MSR:DR enabled */
1639	mfmsr	r3
1640	ori	r4, r3, MSR_DR		/* Enable paging for data */
1641	mtmsrd	r4
1642	lwz	r8, 0(r10)
1643	mtmsrd	r3
1644
1645	/* Store the result */
1646	stw	r8, VCPU_LAST_INST(r9)
1647
1648	/* Unset guest mode. */
1649	li	r0, KVM_GUEST_MODE_HOST_HV
1650	stb	r0, HSTATE_IN_GUEST(r13)
1651	b	guest_exit_cont
1652
1653/*
1654 * Similarly for an HISI, reflect it to the guest as an ISI unless
1655 * it is an HPTE not found fault for a page that we have paged out.
1656 */
1657kvmppc_hisi:
1658	andis.	r0, r11, SRR1_ISI_NOPT@h
1659	beq	1f
1660	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
1661	beq	3f
1662	clrrdi	r0, r10, 28
1663	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1664	bne	1f			/* if no SLB entry found */
16654:
1666	/* Search the hash table. */
1667	mr	r3, r9			/* vcpu pointer */
1668	mr	r4, r10
1669	mr	r6, r11
1670	li	r7, 0			/* instruction fault */
1671	bl	kvmppc_hpte_hv_fault
1672	ld	r9, HSTATE_KVM_VCPU(r13)
1673	ld	r10, VCPU_PC(r9)
1674	ld	r11, VCPU_MSR(r9)
1675	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1676	cmpdi	r3, 0			/* retry the instruction */
1677	beq	fast_interrupt_c_return
1678	cmpdi	r3, -1			/* handle in kernel mode */
1679	beq	guest_exit_cont
1680
1681	/* Synthesize an ISI for the guest */
1682	mr	r11, r3
16831:	mtspr	SPRN_SRR0, r10
1684	mtspr	SPRN_SRR1, r11
1685	li	r10, BOOK3S_INTERRUPT_INST_STORAGE
1686	bl	kvmppc_msr_interrupt
1687	b	fast_interrupt_c_return
1688
16893:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
1690	ld	r5, KVM_VRMA_SLB_V(r6)
1691	b	4b
1692
1693/*
1694 * Try to handle an hcall in real mode.
1695 * Returns to the guest if we handle it, or continues on up to
1696 * the kernel if we can't (i.e. if we don't have a handler for
1697 * it, or if the handler returns H_TOO_HARD).
1698 */
1699	.globl	hcall_try_real_mode
1700hcall_try_real_mode:
1701	ld	r3,VCPU_GPR(R3)(r9)
1702	andi.	r0,r11,MSR_PR
1703	/* sc 1 from userspace - reflect to guest syscall */
1704	bne	sc_1_fast_return
1705	clrrdi	r3,r3,2
1706	cmpldi	r3,hcall_real_table_end - hcall_real_table
1707	bge	guest_exit_cont
1708	/* See if this hcall is enabled for in-kernel handling */
1709	ld	r4, VCPU_KVM(r9)
1710	srdi	r0, r3, 8	/* r0 = (r3 / 4) >> 6 */
1711	sldi	r0, r0, 3	/* index into kvm->arch.enabled_hcalls[] */
1712	add	r4, r4, r0
1713	ld	r0, KVM_ENABLED_HCALLS(r4)
1714	rlwinm	r4, r3, 32-2, 0x3f	/* r4 = (r3 / 4) & 0x3f */
1715	srd	r0, r0, r4
1716	andi.	r0, r0, 1
1717	beq	guest_exit_cont
1718	/* Get pointer to handler, if any, and call it */
1719	LOAD_REG_ADDR(r4, hcall_real_table)
1720	lwax	r3,r3,r4
1721	cmpwi	r3,0
1722	beq	guest_exit_cont
1723	add	r12,r3,r4
1724	mtctr	r12
1725	mr	r3,r9		/* get vcpu pointer */
1726	ld	r4,VCPU_GPR(R4)(r9)
1727	bctrl
1728	cmpdi	r3,H_TOO_HARD
1729	beq	hcall_real_fallback
1730	ld	r4,HSTATE_KVM_VCPU(r13)
1731	std	r3,VCPU_GPR(R3)(r4)
1732	ld	r10,VCPU_PC(r4)
1733	ld	r11,VCPU_MSR(r4)
1734	b	fast_guest_return
1735
1736sc_1_fast_return:
1737	mtspr	SPRN_SRR0,r10
1738	mtspr	SPRN_SRR1,r11
1739	li	r10, BOOK3S_INTERRUPT_SYSCALL
1740	bl	kvmppc_msr_interrupt
1741	mr	r4,r9
1742	b	fast_guest_return
1743
1744	/* We've attempted a real mode hcall, but it's punted it back
1745	 * to userspace.  We need to restore some clobbered volatiles
1746	 * before resuming the pass-it-to-qemu path */
1747hcall_real_fallback:
1748	li	r12,BOOK3S_INTERRUPT_SYSCALL
1749	ld	r9, HSTATE_KVM_VCPU(r13)
1750
1751	b	guest_exit_cont
1752
1753	.globl	hcall_real_table
1754hcall_real_table:
1755	.long	0		/* 0 - unused */
1756	.long	DOTSYM(kvmppc_h_remove) - hcall_real_table
1757	.long	DOTSYM(kvmppc_h_enter) - hcall_real_table
1758	.long	DOTSYM(kvmppc_h_read) - hcall_real_table
1759	.long	0		/* 0x10 - H_CLEAR_MOD */
1760	.long	0		/* 0x14 - H_CLEAR_REF */
1761	.long	DOTSYM(kvmppc_h_protect) - hcall_real_table
1762	.long	DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1763	.long	DOTSYM(kvmppc_h_put_tce) - hcall_real_table
1764	.long	0		/* 0x24 - H_SET_SPRG0 */
1765	.long	DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1766	.long	0		/* 0x2c */
1767	.long	0		/* 0x30 */
1768	.long	0		/* 0x34 */
1769	.long	0		/* 0x38 */
1770	.long	0		/* 0x3c */
1771	.long	0		/* 0x40 */
1772	.long	0		/* 0x44 */
1773	.long	0		/* 0x48 */
1774	.long	0		/* 0x4c */
1775	.long	0		/* 0x50 */
1776	.long	0		/* 0x54 */
1777	.long	0		/* 0x58 */
1778	.long	0		/* 0x5c */
1779	.long	0		/* 0x60 */
1780#ifdef CONFIG_KVM_XICS
1781	.long	DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
1782	.long	DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
1783	.long	DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1784	.long	0		/* 0x70 - H_IPOLL */
1785	.long	DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1786#else
1787	.long	0		/* 0x64 - H_EOI */
1788	.long	0		/* 0x68 - H_CPPR */
1789	.long	0		/* 0x6c - H_IPI */
1790	.long	0		/* 0x70 - H_IPOLL */
1791	.long	0		/* 0x74 - H_XIRR */
1792#endif
1793	.long	0		/* 0x78 */
1794	.long	0		/* 0x7c */
1795	.long	0		/* 0x80 */
1796	.long	0		/* 0x84 */
1797	.long	0		/* 0x88 */
1798	.long	0		/* 0x8c */
1799	.long	0		/* 0x90 */
1800	.long	0		/* 0x94 */
1801	.long	0		/* 0x98 */
1802	.long	0		/* 0x9c */
1803	.long	0		/* 0xa0 */
1804	.long	0		/* 0xa4 */
1805	.long	0		/* 0xa8 */
1806	.long	0		/* 0xac */
1807	.long	0		/* 0xb0 */
1808	.long	0		/* 0xb4 */
1809	.long	0		/* 0xb8 */
1810	.long	0		/* 0xbc */
1811	.long	0		/* 0xc0 */
1812	.long	0		/* 0xc4 */
1813	.long	0		/* 0xc8 */
1814	.long	0		/* 0xcc */
1815	.long	0		/* 0xd0 */
1816	.long	0		/* 0xd4 */
1817	.long	0		/* 0xd8 */
1818	.long	0		/* 0xdc */
1819	.long	DOTSYM(kvmppc_h_cede) - hcall_real_table
1820	.long	DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1821	.long	0		/* 0xe8 */
1822	.long	0		/* 0xec */
1823	.long	0		/* 0xf0 */
1824	.long	0		/* 0xf4 */
1825	.long	0		/* 0xf8 */
1826	.long	0		/* 0xfc */
1827	.long	0		/* 0x100 */
1828	.long	0		/* 0x104 */
1829	.long	0		/* 0x108 */
1830	.long	0		/* 0x10c */
1831	.long	0		/* 0x110 */
1832	.long	0		/* 0x114 */
1833	.long	0		/* 0x118 */
1834	.long	0		/* 0x11c */
1835	.long	0		/* 0x120 */
1836	.long	DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1837	.long	0		/* 0x128 */
1838	.long	0		/* 0x12c */
1839	.long	0		/* 0x130 */
1840	.long	DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1841	.globl	hcall_real_table_end
1842hcall_real_table_end:
1843
1844ignore_hdec:
1845	mr	r4,r9
1846	b	fast_guest_return
1847
1848_GLOBAL(kvmppc_h_set_xdabr)
1849	andi.	r0, r5, DABRX_USER | DABRX_KERNEL
1850	beq	6f
1851	li	r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
1852	andc.	r0, r5, r0
1853	beq	3f
18546:	li	r3, H_PARAMETER
1855	blr
1856
1857_GLOBAL(kvmppc_h_set_dabr)
1858	li	r5, DABRX_USER | DABRX_KERNEL
18593:
1860BEGIN_FTR_SECTION
1861	b	2f
1862END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1863	std	r4,VCPU_DABR(r3)
1864	stw	r5, VCPU_DABRX(r3)
1865	mtspr	SPRN_DABRX, r5
1866	/* Work around P7 bug where DABR can get corrupted on mtspr */
18671:	mtspr	SPRN_DABR,r4
1868	mfspr	r5, SPRN_DABR
1869	cmpd	r4, r5
1870	bne	1b
1871	isync
1872	li	r3,0
1873	blr
1874
1875	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
18762:	rlwimi	r5, r4, 5, DAWRX_DR | DAWRX_DW
1877	rlwimi	r5, r4, 1, DAWRX_WT
1878	clrrdi	r4, r4, 3
1879	std	r4, VCPU_DAWR(r3)
1880	std	r5, VCPU_DAWRX(r3)
1881	mtspr	SPRN_DAWR, r4
1882	mtspr	SPRN_DAWRX, r5
1883	li	r3, 0
1884	blr
1885
1886_GLOBAL(kvmppc_h_cede)
1887	ori	r11,r11,MSR_EE
1888	std	r11,VCPU_MSR(r3)
1889	li	r0,1
1890	stb	r0,VCPU_CEDED(r3)
1891	sync			/* order setting ceded vs. testing prodded */
1892	lbz	r5,VCPU_PRODDED(r3)
1893	cmpwi	r5,0
1894	bne	kvm_cede_prodded
1895	li	r0,0		/* set trap to 0 to say hcall is handled */
1896	stw	r0,VCPU_TRAP(r3)
1897	li	r0,H_SUCCESS
1898	std	r0,VCPU_GPR(R3)(r3)
1899
1900	/*
1901	 * Set our bit in the bitmask of napping threads unless all the
1902	 * other threads are already napping, in which case we send this
1903	 * up to the host.
1904	 */
1905	ld	r5,HSTATE_KVM_VCORE(r13)
1906	lbz	r6,HSTATE_PTID(r13)
1907	lwz	r8,VCORE_ENTRY_EXIT(r5)
1908	clrldi	r8,r8,56
1909	li	r0,1
1910	sld	r0,r0,r6
1911	addi	r6,r5,VCORE_NAPPING_THREADS
191231:	lwarx	r4,0,r6
1913	or	r4,r4,r0
1914	PPC_POPCNTW(R7,R4)
1915	cmpw	r7,r8
1916	bge	kvm_cede_exit
1917	stwcx.	r4,0,r6
1918	bne	31b
1919	/* order napping_threads update vs testing entry_exit_count */
1920	isync
1921	li	r0,NAPPING_CEDE
1922	stb	r0,HSTATE_NAPPING(r13)
1923	lwz	r7,VCORE_ENTRY_EXIT(r5)
1924	cmpwi	r7,0x100
1925	bge	33f		/* another thread already exiting */
1926
1927/*
1928 * Although not specifically required by the architecture, POWER7
1929 * preserves the following registers in nap mode, even if an SMT mode
1930 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
1931 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
1932 */
1933	/* Save non-volatile GPRs */
1934	std	r14, VCPU_GPR(R14)(r3)
1935	std	r15, VCPU_GPR(R15)(r3)
1936	std	r16, VCPU_GPR(R16)(r3)
1937	std	r17, VCPU_GPR(R17)(r3)
1938	std	r18, VCPU_GPR(R18)(r3)
1939	std	r19, VCPU_GPR(R19)(r3)
1940	std	r20, VCPU_GPR(R20)(r3)
1941	std	r21, VCPU_GPR(R21)(r3)
1942	std	r22, VCPU_GPR(R22)(r3)
1943	std	r23, VCPU_GPR(R23)(r3)
1944	std	r24, VCPU_GPR(R24)(r3)
1945	std	r25, VCPU_GPR(R25)(r3)
1946	std	r26, VCPU_GPR(R26)(r3)
1947	std	r27, VCPU_GPR(R27)(r3)
1948	std	r28, VCPU_GPR(R28)(r3)
1949	std	r29, VCPU_GPR(R29)(r3)
1950	std	r30, VCPU_GPR(R30)(r3)
1951	std	r31, VCPU_GPR(R31)(r3)
1952
1953	/* save FP state */
1954	bl	kvmppc_save_fp
1955
1956	/*
1957	 * Take a nap until a decrementer or external or doobell interrupt
1958	 * occurs, with PECE1, PECE0 and PECEDP set in LPCR. Also clear the
1959	 * runlatch bit before napping.
1960	 */
1961kvm_do_nap:
1962	mfspr	r2, SPRN_CTRLF
1963	clrrdi	r2, r2, 1
1964	mtspr	SPRN_CTRLT, r2
1965
1966	li	r0,1
1967	stb	r0,HSTATE_HWTHREAD_REQ(r13)
1968	mfspr	r5,SPRN_LPCR
1969	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
1970BEGIN_FTR_SECTION
1971	oris	r5,r5,LPCR_PECEDP@h
1972END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1973	mtspr	SPRN_LPCR,r5
1974	isync
1975	li	r0, 0
1976	std	r0, HSTATE_SCRATCH0(r13)
1977	ptesync
1978	ld	r0, HSTATE_SCRATCH0(r13)
19791:	cmpd	r0, r0
1980	bne	1b
1981	nap
1982	b	.
1983
198433:	mr	r4, r3
1985	li	r3, 0
1986	li	r12, 0
1987	b	34f
1988
1989kvm_end_cede:
1990	/* get vcpu pointer */
1991	ld	r4, HSTATE_KVM_VCPU(r13)
1992
1993	/* Woken by external or decrementer interrupt */
1994	ld	r1, HSTATE_HOST_R1(r13)
1995
1996	/* load up FP state */
1997	bl	kvmppc_load_fp
1998
1999	/* Load NV GPRS */
2000	ld	r14, VCPU_GPR(R14)(r4)
2001	ld	r15, VCPU_GPR(R15)(r4)
2002	ld	r16, VCPU_GPR(R16)(r4)
2003	ld	r17, VCPU_GPR(R17)(r4)
2004	ld	r18, VCPU_GPR(R18)(r4)
2005	ld	r19, VCPU_GPR(R19)(r4)
2006	ld	r20, VCPU_GPR(R20)(r4)
2007	ld	r21, VCPU_GPR(R21)(r4)
2008	ld	r22, VCPU_GPR(R22)(r4)
2009	ld	r23, VCPU_GPR(R23)(r4)
2010	ld	r24, VCPU_GPR(R24)(r4)
2011	ld	r25, VCPU_GPR(R25)(r4)
2012	ld	r26, VCPU_GPR(R26)(r4)
2013	ld	r27, VCPU_GPR(R27)(r4)
2014	ld	r28, VCPU_GPR(R28)(r4)
2015	ld	r29, VCPU_GPR(R29)(r4)
2016	ld	r30, VCPU_GPR(R30)(r4)
2017	ld	r31, VCPU_GPR(R31)(r4)
2018
2019	/* Check the wake reason in SRR1 to see why we got here */
2020	bl	kvmppc_check_wake_reason
2021
2022	/* clear our bit in vcore->napping_threads */
202334:	ld	r5,HSTATE_KVM_VCORE(r13)
2024	lbz	r7,HSTATE_PTID(r13)
2025	li	r0,1
2026	sld	r0,r0,r7
2027	addi	r6,r5,VCORE_NAPPING_THREADS
202832:	lwarx	r7,0,r6
2029	andc	r7,r7,r0
2030	stwcx.	r7,0,r6
2031	bne	32b
2032	li	r0,0
2033	stb	r0,HSTATE_NAPPING(r13)
2034
2035	/* See if the wake reason means we need to exit */
2036	stw	r12, VCPU_TRAP(r4)
2037	mr	r9, r4
2038	cmpdi	r3, 0
2039	bgt	guest_exit_cont
2040
2041	/* see if any other thread is already exiting */
2042	lwz	r0,VCORE_ENTRY_EXIT(r5)
2043	cmpwi	r0,0x100
2044	bge	guest_exit_cont
2045
2046	b	kvmppc_cede_reentry	/* if not go back to guest */
2047
2048	/* cede when already previously prodded case */
2049kvm_cede_prodded:
2050	li	r0,0
2051	stb	r0,VCPU_PRODDED(r3)
2052	sync			/* order testing prodded vs. clearing ceded */
2053	stb	r0,VCPU_CEDED(r3)
2054	li	r3,H_SUCCESS
2055	blr
2056
2057	/* we've ceded but we want to give control to the host */
2058kvm_cede_exit:
2059	b	hcall_real_fallback
2060
2061	/* Try to handle a machine check in real mode */
2062machine_check_realmode:
2063	mr	r3, r9		/* get vcpu pointer */
2064	bl	kvmppc_realmode_machine_check
2065	nop
2066	cmpdi	r3, 0		/* Did we handle MCE ? */
2067	ld	r9, HSTATE_KVM_VCPU(r13)
2068	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2069	/*
2070	 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
2071	 * machine check interrupt (set HSRR0 to 0x200). And for handled
2072	 * errors (no-fatal), just go back to guest execution with current
2073	 * HSRR0 instead of exiting guest. This new approach will inject
2074	 * machine check to guest for fatal error causing guest to crash.
2075	 *
2076	 * The old code used to return to host for unhandled errors which
2077	 * was causing guest to hang with soft lockups inside guest and
2078	 * makes it difficult to recover guest instance.
2079	 */
2080	ld	r10, VCPU_PC(r9)
2081	ld	r11, VCPU_MSR(r9)
2082	bne	2f	/* Continue guest execution. */
2083	/* If not, deliver a machine check.  SRR0/1 are already set */
2084	li	r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2085	ld	r11, VCPU_MSR(r9)
2086	bl	kvmppc_msr_interrupt
20872:	b	fast_interrupt_c_return
2088
2089/*
2090 * Check the reason we woke from nap, and take appropriate action.
2091 * Returns:
2092 *	0 if nothing needs to be done
2093 *	1 if something happened that needs to be handled by the host
2094 *	-1 if there was a guest wakeup (IPI)
2095 *
2096 * Also sets r12 to the interrupt vector for any interrupt that needs
2097 * to be handled now by the host (0x500 for external interrupt), or zero.
2098 */
2099kvmppc_check_wake_reason:
2100	mfspr	r6, SPRN_SRR1
2101BEGIN_FTR_SECTION
2102	rlwinm	r6, r6, 45-31, 0xf	/* extract wake reason field (P8) */
2103FTR_SECTION_ELSE
2104	rlwinm	r6, r6, 45-31, 0xe	/* P7 wake reason field is 3 bits */
2105ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2106	cmpwi	r6, 8			/* was it an external interrupt? */
2107	li	r12, BOOK3S_INTERRUPT_EXTERNAL
2108	beq	kvmppc_read_intr	/* if so, see what it was */
2109	li	r3, 0
2110	li	r12, 0
2111	cmpwi	r6, 6			/* was it the decrementer? */
2112	beq	0f
2113BEGIN_FTR_SECTION
2114	cmpwi	r6, 5			/* privileged doorbell? */
2115	beq	0f
2116	cmpwi	r6, 3			/* hypervisor doorbell? */
2117	beq	3f
2118END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2119	li	r3, 1			/* anything else, return 1 */
21200:	blr
2121
2122	/* hypervisor doorbell */
21233:	li	r12, BOOK3S_INTERRUPT_H_DOORBELL
2124	li	r3, 1
2125	blr
2126
2127/*
2128 * Determine what sort of external interrupt is pending (if any).
2129 * Returns:
2130 *	0 if no interrupt is pending
2131 *	1 if an interrupt is pending that needs to be handled by the host
2132 *	-1 if there was a guest wakeup IPI (which has now been cleared)
2133 */
2134kvmppc_read_intr:
2135	/* see if a host IPI is pending */
2136	li	r3, 1
2137	lbz	r0, HSTATE_HOST_IPI(r13)
2138	cmpwi	r0, 0
2139	bne	1f
2140
2141	/* Now read the interrupt from the ICP */
2142	ld	r6, HSTATE_XICS_PHYS(r13)
2143	li	r7, XICS_XIRR
2144	cmpdi	r6, 0
2145	beq-	1f
2146	lwzcix	r0, r6, r7
2147	/*
2148	 * Save XIRR for later. Since we get in in reverse endian on LE
2149	 * systems, save it byte reversed and fetch it back in host endian.
2150	 */
2151	li	r3, HSTATE_SAVED_XIRR
2152	STWX_BE	r0, r3, r13
2153#ifdef __LITTLE_ENDIAN__
2154	lwz	r3, HSTATE_SAVED_XIRR(r13)
2155#else
2156	mr	r3, r0
2157#endif
2158	rlwinm.	r3, r3, 0, 0xffffff
2159	sync
2160	beq	1f			/* if nothing pending in the ICP */
2161
2162	/* We found something in the ICP...
2163	 *
2164	 * If it's not an IPI, stash it in the PACA and return to
2165	 * the host, we don't (yet) handle directing real external
2166	 * interrupts directly to the guest
2167	 */
2168	cmpwi	r3, XICS_IPI		/* if there is, is it an IPI? */
2169	bne	42f
2170
2171	/* It's an IPI, clear the MFRR and EOI it */
2172	li	r3, 0xff
2173	li	r8, XICS_MFRR
2174	stbcix	r3, r6, r8		/* clear the IPI */
2175	stwcix	r0, r6, r7		/* EOI it */
2176	sync
2177
2178	/* We need to re-check host IPI now in case it got set in the
2179	 * meantime. If it's clear, we bounce the interrupt to the
2180	 * guest
2181	 */
2182	lbz	r0, HSTATE_HOST_IPI(r13)
2183	cmpwi	r0, 0
2184	bne-	43f
2185
2186	/* OK, it's an IPI for us */
2187	li	r3, -1
21881:	blr
2189
219042:	/* It's not an IPI and it's for the host. We saved a copy of XIRR in
2191	 * the PACA earlier, it will be picked up by the host ICP driver
2192	 */
2193	li	r3, 1
2194	b	1b
2195
219643:	/* We raced with the host, we need to resend that IPI, bummer */
2197	li	r0, IPI_PRIORITY
2198	stbcix	r0, r6, r8		/* set the IPI */
2199	sync
2200	li	r3, 1
2201	b	1b
2202
2203/*
2204 * Save away FP, VMX and VSX registers.
2205 * r3 = vcpu pointer
2206 * N.B. r30 and r31 are volatile across this function,
2207 * thus it is not callable from C.
2208 */
2209kvmppc_save_fp:
2210	mflr	r30
2211	mr	r31,r3
2212	mfmsr	r5
2213	ori	r8,r5,MSR_FP
2214#ifdef CONFIG_ALTIVEC
2215BEGIN_FTR_SECTION
2216	oris	r8,r8,MSR_VEC@h
2217END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2218#endif
2219#ifdef CONFIG_VSX
2220BEGIN_FTR_SECTION
2221	oris	r8,r8,MSR_VSX@h
2222END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2223#endif
2224	mtmsrd	r8
2225	addi	r3,r3,VCPU_FPRS
2226	bl	store_fp_state
2227#ifdef CONFIG_ALTIVEC
2228BEGIN_FTR_SECTION
2229	addi	r3,r31,VCPU_VRS
2230	bl	store_vr_state
2231END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2232#endif
2233	mfspr	r6,SPRN_VRSAVE
2234	stw	r6,VCPU_VRSAVE(r31)
2235	mtlr	r30
2236	blr
2237
2238/*
2239 * Load up FP, VMX and VSX registers
2240 * r4 = vcpu pointer
2241 * N.B. r30 and r31 are volatile across this function,
2242 * thus it is not callable from C.
2243 */
2244kvmppc_load_fp:
2245	mflr	r30
2246	mr	r31,r4
2247	mfmsr	r9
2248	ori	r8,r9,MSR_FP
2249#ifdef CONFIG_ALTIVEC
2250BEGIN_FTR_SECTION
2251	oris	r8,r8,MSR_VEC@h
2252END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2253#endif
2254#ifdef CONFIG_VSX
2255BEGIN_FTR_SECTION
2256	oris	r8,r8,MSR_VSX@h
2257END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2258#endif
2259	mtmsrd	r8
2260	addi	r3,r4,VCPU_FPRS
2261	bl	load_fp_state
2262#ifdef CONFIG_ALTIVEC
2263BEGIN_FTR_SECTION
2264	addi	r3,r31,VCPU_VRS
2265	bl	load_vr_state
2266END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2267#endif
2268	lwz	r7,VCPU_VRSAVE(r31)
2269	mtspr	SPRN_VRSAVE,r7
2270	mtlr	r30
2271	mr	r4,r31
2272	blr
2273
2274/*
2275 * We come here if we get any exception or interrupt while we are
2276 * executing host real mode code while in guest MMU context.
2277 * For now just spin, but we should do something better.
2278 */
2279kvmppc_bad_host_intr:
2280	b	.
2281
2282/*
2283 * This mimics the MSR transition on IRQ delivery.  The new guest MSR is taken
2284 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2285 *   r11 has the guest MSR value (in/out)
2286 *   r9 has a vcpu pointer (in)
2287 *   r0 is used as a scratch register
2288 */
2289kvmppc_msr_interrupt:
2290	rldicl	r0, r11, 64 - MSR_TS_S_LG, 62
2291	cmpwi	r0, 2 /* Check if we are in transactional state..  */
2292	ld	r11, VCPU_INTR_MSR(r9)
2293	bne	1f
2294	/* ... if transactional, change to suspended */
2295	li	r0, 1
22961:	rldimi	r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2297	blr
2298
2299/*
2300 * This works around a hardware bug on POWER8E processors, where
2301 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2302 * performance monitor interrupt.  Instead, when we need to have
2303 * an interrupt pending, we have to arrange for a counter to overflow.
2304 */
2305kvmppc_fix_pmao:
2306	li	r3, 0
2307	mtspr	SPRN_MMCR2, r3
2308	lis	r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2309	ori	r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2310	mtspr	SPRN_MMCR0, r3
2311	lis	r3, 0x7fff
2312	ori	r3, r3, 0xffff
2313	mtspr	SPRN_PMC6, r3
2314	isync
2315	blr
2316