1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 *
4 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
5 *
6 * Derived from book3s_rmhandlers.S and other files, which are:
7 *
8 * Copyright SUSE Linux Products GmbH 2009
9 *
10 * Authors: Alexander Graf <agraf@suse.de>
11 */
12
13#include <linux/linkage.h>
14#include <linux/objtool.h>
15#include <asm/ppc_asm.h>
16#include <asm/code-patching-asm.h>
17#include <asm/kvm_asm.h>
18#include <asm/reg.h>
19#include <asm/mmu.h>
20#include <asm/page.h>
21#include <asm/ptrace.h>
22#include <asm/hvcall.h>
23#include <asm/asm-offsets.h>
24#include <asm/exception-64s.h>
25#include <asm/kvm_book3s_asm.h>
26#include <asm/book3s/64/mmu-hash.h>
27#include <asm/export.h>
28#include <asm/tm.h>
29#include <asm/opal.h>
30#include <asm/thread_info.h>
31#include <asm/asm-compat.h>
32#include <asm/feature-fixups.h>
33#include <asm/cpuidle.h>
34
35/* Values in HSTATE_NAPPING(r13) */
36#define NAPPING_CEDE	1
37#define NAPPING_NOVCPU	2
38#define NAPPING_UNSPLIT	3
39
40/* Stack frame offsets for kvmppc_hv_entry */
41#define SFS			160
42#define STACK_SLOT_TRAP		(SFS-4)
43#define STACK_SLOT_TID		(SFS-16)
44#define STACK_SLOT_PSSCR	(SFS-24)
45#define STACK_SLOT_PID		(SFS-32)
46#define STACK_SLOT_IAMR		(SFS-40)
47#define STACK_SLOT_CIABR	(SFS-48)
48#define STACK_SLOT_DAWR0	(SFS-56)
49#define STACK_SLOT_DAWRX0	(SFS-64)
50#define STACK_SLOT_HFSCR	(SFS-72)
51#define STACK_SLOT_AMR		(SFS-80)
52#define STACK_SLOT_UAMOR	(SFS-88)
53#define STACK_SLOT_FSCR		(SFS-96)
54
55/*
56 * Use the last LPID (all implemented LPID bits = 1) for partition switching.
57 * This is reserved in the LPID allocator. POWER7 only implements 0x3ff, but
58 * we write 0xfff into the LPID SPR anyway, which seems to work and just
59 * ignores the top bits.
60 */
61#define   LPID_RSVD		0xfff
62
63/*
64 * Call kvmppc_hv_entry in real mode.
65 * Must be called with interrupts hard-disabled.
66 *
67 * Input Registers:
68 *
69 * LR = return address to continue at after eventually re-enabling MMU
70 */
71_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
72	mflr	r0
73	std	r0, PPC_LR_STKOFF(r1)
74	stdu	r1, -112(r1)
75	mfmsr	r10
76	std	r10, HSTATE_HOST_MSR(r13)
77	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
78	li	r0,MSR_RI
79	andc	r0,r10,r0
80	li	r6,MSR_IR | MSR_DR
81	andc	r6,r10,r6
82	mtmsrd	r0,1		/* clear RI in MSR */
83	mtsrr0	r5
84	mtsrr1	r6
85	RFI_TO_KERNEL
86
87kvmppc_call_hv_entry:
88	ld	r4, HSTATE_KVM_VCPU(r13)
89	bl	kvmppc_hv_entry
90
91	/* Back from guest - restore host state and return to caller */
92
93BEGIN_FTR_SECTION
94	/* Restore host DABR and DABRX */
95	ld	r5,HSTATE_DABR(r13)
96	li	r6,7
97	mtspr	SPRN_DABR,r5
98	mtspr	SPRN_DABRX,r6
99END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
100
101	/* Restore SPRG3 */
102	ld	r3,PACA_SPRG_VDSO(r13)
103	mtspr	SPRN_SPRG_VDSO_WRITE,r3
104
105	/* Reload the host's PMU registers */
106	bl	kvmhv_load_host_pmu
107
108	/*
109	 * Reload DEC.  HDEC interrupts were disabled when
110	 * we reloaded the host's LPCR value.
111	 */
112	ld	r3, HSTATE_DECEXP(r13)
113	mftb	r4
114	subf	r4, r4, r3
115	mtspr	SPRN_DEC, r4
116
117	/* hwthread_req may have got set by cede or no vcpu, so clear it */
118	li	r0, 0
119	stb	r0, HSTATE_HWTHREAD_REQ(r13)
120
121	/*
122	 * For external interrupts we need to call the Linux
123	 * handler to process the interrupt. We do that by jumping
124	 * to absolute address 0x500 for external interrupts.
125	 * The [h]rfid at the end of the handler will return to
126	 * the book3s_hv_interrupts.S code. For other interrupts
127	 * we do the rfid to get back to the book3s_hv_interrupts.S
128	 * code here.
129	 */
130	ld	r8, 112+PPC_LR_STKOFF(r1)
131	addi	r1, r1, 112
132	ld	r7, HSTATE_HOST_MSR(r13)
133
134	/* Return the trap number on this thread as the return value */
135	mr	r3, r12
136
137	/* RFI into the highmem handler */
138	mfmsr	r6
139	li	r0, MSR_RI
140	andc	r6, r6, r0
141	mtmsrd	r6, 1			/* Clear RI in MSR */
142	mtsrr0	r8
143	mtsrr1	r7
144	RFI_TO_KERNEL
145
146kvmppc_primary_no_guest:
147	/* We handle this much like a ceded vcpu */
148	/* put the HDEC into the DEC, since HDEC interrupts don't wake us */
149	/* HDEC may be larger than DEC for arch >= v3.00, but since the */
150	/* HDEC value came from DEC in the first place, it will fit */
151	mfspr	r3, SPRN_HDEC
152	mtspr	SPRN_DEC, r3
153	/*
154	 * Make sure the primary has finished the MMU switch.
155	 * We should never get here on a secondary thread, but
156	 * check it for robustness' sake.
157	 */
158	ld	r5, HSTATE_KVM_VCORE(r13)
15965:	lbz	r0, VCORE_IN_GUEST(r5)
160	cmpwi	r0, 0
161	beq	65b
162	/* Set LPCR. */
163	ld	r8,VCORE_LPCR(r5)
164	mtspr	SPRN_LPCR,r8
165	isync
166	/* set our bit in napping_threads */
167	ld	r5, HSTATE_KVM_VCORE(r13)
168	lbz	r7, HSTATE_PTID(r13)
169	li	r0, 1
170	sld	r0, r0, r7
171	addi	r6, r5, VCORE_NAPPING_THREADS
1721:	lwarx	r3, 0, r6
173	or	r3, r3, r0
174	stwcx.	r3, 0, r6
175	bne	1b
176	/* order napping_threads update vs testing entry_exit_map */
177	isync
178	li	r12, 0
179	lwz	r7, VCORE_ENTRY_EXIT(r5)
180	cmpwi	r7, 0x100
181	bge	kvm_novcpu_exit	/* another thread already exiting */
182	li	r3, NAPPING_NOVCPU
183	stb	r3, HSTATE_NAPPING(r13)
184
185	li	r3, 0		/* Don't wake on privileged (OS) doorbell */
186	b	kvm_do_nap
187
188/*
189 * kvm_novcpu_wakeup
190 *	Entered from kvm_start_guest if kvm_hstate.napping is set
191 *	to NAPPING_NOVCPU
192 *		r2 = kernel TOC
193 *		r13 = paca
194 */
195kvm_novcpu_wakeup:
196	ld	r1, HSTATE_HOST_R1(r13)
197	ld	r5, HSTATE_KVM_VCORE(r13)
198	li	r0, 0
199	stb	r0, HSTATE_NAPPING(r13)
200
201	/* check the wake reason */
202	bl	kvmppc_check_wake_reason
203
204	/*
205	 * Restore volatile registers since we could have called
206	 * a C routine in kvmppc_check_wake_reason.
207	 *	r5 = VCORE
208	 */
209	ld	r5, HSTATE_KVM_VCORE(r13)
210
211	/* see if any other thread is already exiting */
212	lwz	r0, VCORE_ENTRY_EXIT(r5)
213	cmpwi	r0, 0x100
214	bge	kvm_novcpu_exit
215
216	/* clear our bit in napping_threads */
217	lbz	r7, HSTATE_PTID(r13)
218	li	r0, 1
219	sld	r0, r0, r7
220	addi	r6, r5, VCORE_NAPPING_THREADS
2214:	lwarx	r7, 0, r6
222	andc	r7, r7, r0
223	stwcx.	r7, 0, r6
224	bne	4b
225
226	/* See if the wake reason means we need to exit */
227	cmpdi	r3, 0
228	bge	kvm_novcpu_exit
229
230	/* See if our timeslice has expired (HDEC is negative) */
231	mfspr	r0, SPRN_HDEC
232	extsw	r0, r0
233	li	r12, BOOK3S_INTERRUPT_HV_DECREMENTER
234	cmpdi	r0, 0
235	blt	kvm_novcpu_exit
236
237	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
238	ld	r4, HSTATE_KVM_VCPU(r13)
239	cmpdi	r4, 0
240	beq	kvmppc_primary_no_guest
241
242#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
243	addi	r3, r4, VCPU_TB_RMENTRY
244	bl	kvmhv_start_timing
245#endif
246	b	kvmppc_got_guest
247
248kvm_novcpu_exit:
249#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
250	ld	r4, HSTATE_KVM_VCPU(r13)
251	cmpdi	r4, 0
252	beq	13f
253	addi	r3, r4, VCPU_TB_RMEXIT
254	bl	kvmhv_accumulate_time
255#endif
25613:	mr	r3, r12
257	stw	r12, STACK_SLOT_TRAP(r1)
258	bl	kvmhv_commence_exit
259	nop
260	b	kvmhv_switch_to_host
261
262/*
263 * We come in here when wakened from Linux offline idle code.
264 * Relocation is off
265 * r3 contains the SRR1 wakeup value, SRR1 is trashed.
266 */
267_GLOBAL(idle_kvm_start_guest)
268	mfcr	r5
269	mflr	r0
270	std	r5, 8(r1)	// Save CR in caller's frame
271	std	r0, 16(r1)	// Save LR in caller's frame
272	// Create frame on emergency stack
273	ld	r4, PACAEMERGSP(r13)
274	stdu	r1, -SWITCH_FRAME_SIZE(r4)
275	// Switch to new frame on emergency stack
276	mr	r1, r4
277	std	r3, 32(r1)	// Save SRR1 wakeup value
278	SAVE_NVGPRS(r1)
279
280	/*
281	 * Could avoid this and pass it through in r3. For now,
282	 * code expects it to be in SRR1.
283	 */
284	mtspr	SPRN_SRR1,r3
285
286	li	r0,0
287	stb	r0,PACA_FTRACE_ENABLED(r13)
288
289	li	r0,KVM_HWTHREAD_IN_KVM
290	stb	r0,HSTATE_HWTHREAD_STATE(r13)
291
292	/* kvm cede / napping does not come through here */
293	lbz	r0,HSTATE_NAPPING(r13)
294	twnei	r0,0
295
296	b	1f
297
298kvm_unsplit_wakeup:
299	li	r0, 0
300	stb	r0, HSTATE_NAPPING(r13)
301
3021:
303
304	/*
305	 * We weren't napping due to cede, so this must be a secondary
306	 * thread being woken up to run a guest, or being woken up due
307	 * to a stray IPI.  (Or due to some machine check or hypervisor
308	 * maintenance interrupt while the core is in KVM.)
309	 */
310
311	/* Check the wake reason in SRR1 to see why we got here */
312	bl	kvmppc_check_wake_reason
313	/*
314	 * kvmppc_check_wake_reason could invoke a C routine, but we
315	 * have no volatile registers to restore when we return.
316	 */
317
318	cmpdi	r3, 0
319	bge	kvm_no_guest
320
321	/* get vcore pointer, NULL if we have nothing to run */
322	ld	r5,HSTATE_KVM_VCORE(r13)
323	cmpdi	r5,0
324	/* if we have no vcore to run, go back to sleep */
325	beq	kvm_no_guest
326
327kvm_secondary_got_guest:
328
329	// About to go to guest, clear saved SRR1
330	li	r0, 0
331	std	r0, 32(r1)
332
333	/* Set HSTATE_DSCR(r13) to something sensible */
334	ld	r6, PACA_DSCR_DEFAULT(r13)
335	std	r6, HSTATE_DSCR(r13)
336
337	/* On thread 0 of a subcore, set HDEC to max */
338	lbz	r4, HSTATE_PTID(r13)
339	cmpwi	r4, 0
340	bne	63f
341	lis	r6,0x7fff		/* MAX_INT@h */
342	mtspr	SPRN_HDEC, r6
343	/* and set per-LPAR registers, if doing dynamic micro-threading */
344	ld	r6, HSTATE_SPLIT_MODE(r13)
345	cmpdi	r6, 0
346	beq	63f
347	ld	r0, KVM_SPLIT_RPR(r6)
348	mtspr	SPRN_RPR, r0
349	ld	r0, KVM_SPLIT_PMMAR(r6)
350	mtspr	SPRN_PMMAR, r0
351	ld	r0, KVM_SPLIT_LDBAR(r6)
352	mtspr	SPRN_LDBAR, r0
353	isync
35463:
355	/* Order load of vcpu after load of vcore */
356	lwsync
357	ld	r4, HSTATE_KVM_VCPU(r13)
358	bl	kvmppc_hv_entry
359
360	/* Back from the guest, go back to nap */
361	/* Clear our vcpu and vcore pointers so we don't come back in early */
362	li	r0, 0
363	std	r0, HSTATE_KVM_VCPU(r13)
364	/*
365	 * Once we clear HSTATE_KVM_VCORE(r13), the code in
366	 * kvmppc_run_core() is going to assume that all our vcpu
367	 * state is visible in memory.  This lwsync makes sure
368	 * that that is true.
369	 */
370	lwsync
371	std	r0, HSTATE_KVM_VCORE(r13)
372
373	/*
374	 * All secondaries exiting guest will fall through this path.
375	 * Before proceeding, just check for HMI interrupt and
376	 * invoke opal hmi handler. By now we are sure that the
377	 * primary thread on this core/subcore has already made partition
378	 * switch/TB resync and we are good to call opal hmi handler.
379	 */
380	cmpwi	r12, BOOK3S_INTERRUPT_HMI
381	bne	kvm_no_guest
382
383	li	r3,0			/* NULL argument */
384	bl	hmi_exception_realmode
385/*
386 * At this point we have finished executing in the guest.
387 * We need to wait for hwthread_req to become zero, since
388 * we may not turn on the MMU while hwthread_req is non-zero.
389 * While waiting we also need to check if we get given a vcpu to run.
390 */
391kvm_no_guest:
392	lbz	r3, HSTATE_HWTHREAD_REQ(r13)
393	cmpwi	r3, 0
394	bne	53f
395	HMT_MEDIUM
396	li	r0, KVM_HWTHREAD_IN_KERNEL
397	stb	r0, HSTATE_HWTHREAD_STATE(r13)
398	/* need to recheck hwthread_req after a barrier, to avoid race */
399	sync
400	lbz	r3, HSTATE_HWTHREAD_REQ(r13)
401	cmpwi	r3, 0
402	bne	54f
403
404	/*
405	 * Jump to idle_return_gpr_loss, which returns to the
406	 * idle_kvm_start_guest caller.
407	 */
408	li	r3, LPCR_PECE0
409	mfspr	r4, SPRN_LPCR
410	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
411	mtspr	SPRN_LPCR, r4
412	// Return SRR1 wakeup value, or 0 if we went into the guest
413	ld	r3, 32(r1)
414	REST_NVGPRS(r1)
415	ld	r1, 0(r1)	// Switch back to caller stack
416	ld	r0, 16(r1)	// Reload LR
417	ld	r5, 8(r1)	// Reload CR
418	mtlr	r0
419	mtcr	r5
420	blr
421
42253:
423	HMT_LOW
424	ld	r5, HSTATE_KVM_VCORE(r13)
425	cmpdi	r5, 0
426	bne	60f
427	ld	r3, HSTATE_SPLIT_MODE(r13)
428	cmpdi	r3, 0
429	beq	kvm_no_guest
430	lbz	r0, KVM_SPLIT_DO_NAP(r3)
431	cmpwi	r0, 0
432	beq	kvm_no_guest
433	HMT_MEDIUM
434	b	kvm_unsplit_nap
43560:	HMT_MEDIUM
436	b	kvm_secondary_got_guest
437
43854:	li	r0, KVM_HWTHREAD_IN_KVM
439	stb	r0, HSTATE_HWTHREAD_STATE(r13)
440	b	kvm_no_guest
441
442/*
443 * Here the primary thread is trying to return the core to
444 * whole-core mode, so we need to nap.
445 */
446kvm_unsplit_nap:
447	/*
448	 * When secondaries are napping in kvm_unsplit_nap() with
449	 * hwthread_req = 1, HMI goes ignored even though subcores are
450	 * already exited the guest. Hence HMI keeps waking up secondaries
451	 * from nap in a loop and secondaries always go back to nap since
452	 * no vcore is assigned to them. This makes impossible for primary
453	 * thread to get hold of secondary threads resulting into a soft
454	 * lockup in KVM path.
455	 *
456	 * Let us check if HMI is pending and handle it before we go to nap.
457	 */
458	cmpwi	r12, BOOK3S_INTERRUPT_HMI
459	bne	55f
460	li	r3, 0			/* NULL argument */
461	bl	hmi_exception_realmode
46255:
463	/*
464	 * Ensure that secondary doesn't nap when it has
465	 * its vcore pointer set.
466	 */
467	sync		/* matches smp_mb() before setting split_info.do_nap */
468	ld	r0, HSTATE_KVM_VCORE(r13)
469	cmpdi	r0, 0
470	bne	kvm_no_guest
471	/* clear any pending message */
472BEGIN_FTR_SECTION
473	lis	r6, (PPC_DBELL_SERVER << (63-36))@h
474	PPC_MSGCLR(6)
475END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
476	/* Set kvm_split_mode.napped[tid] = 1 */
477	ld	r3, HSTATE_SPLIT_MODE(r13)
478	li	r0, 1
479	lhz	r4, PACAPACAINDEX(r13)
480	clrldi	r4, r4, 61	/* micro-threading => P8 => 8 threads/core */
481	addi	r4, r4, KVM_SPLIT_NAPPED
482	stbx	r0, r3, r4
483	/* Check the do_nap flag again after setting napped[] */
484	sync
485	lbz	r0, KVM_SPLIT_DO_NAP(r3)
486	cmpwi	r0, 0
487	beq	57f
488	li	r3, NAPPING_UNSPLIT
489	stb	r3, HSTATE_NAPPING(r13)
490	li	r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
491	mfspr	r5, SPRN_LPCR
492	rlwimi	r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
493	b	kvm_nap_sequence
494
49557:	li	r0, 0
496	stbx	r0, r3, r4
497	b	kvm_no_guest
498
499/******************************************************************************
500 *                                                                            *
501 *                               Entry code                                   *
502 *                                                                            *
503 *****************************************************************************/
504
505.global kvmppc_hv_entry
506kvmppc_hv_entry:
507
508	/* Required state:
509	 *
510	 * R4 = vcpu pointer (or NULL)
511	 * MSR = ~IR|DR
512	 * R13 = PACA
513	 * R1 = host R1
514	 * R2 = TOC
515	 * all other volatile GPRS = free
516	 * Does not preserve non-volatile GPRs or CR fields
517	 */
518	mflr	r0
519	std	r0, PPC_LR_STKOFF(r1)
520	stdu	r1, -SFS(r1)
521
522	/* Save R1 in the PACA */
523	std	r1, HSTATE_HOST_R1(r13)
524
525	li	r6, KVM_GUEST_MODE_HOST_HV
526	stb	r6, HSTATE_IN_GUEST(r13)
527
528#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
529	/* Store initial timestamp */
530	cmpdi	r4, 0
531	beq	1f
532	addi	r3, r4, VCPU_TB_RMENTRY
533	bl	kvmhv_start_timing
5341:
535#endif
536
537	ld	r5, HSTATE_KVM_VCORE(r13)
538	ld	r9, VCORE_KVM(r5)	/* pointer to struct kvm */
539
540	/*
541	 * POWER7/POWER8 host -> guest partition switch code.
542	 * We don't have to lock against concurrent tlbies,
543	 * but we do have to coordinate across hardware threads.
544	 */
545	/* Set bit in entry map iff exit map is zero. */
546	li	r7, 1
547	lbz	r6, HSTATE_PTID(r13)
548	sld	r7, r7, r6
549	addi	r8, r5, VCORE_ENTRY_EXIT
55021:	lwarx	r3, 0, r8
551	cmpwi	r3, 0x100		/* any threads starting to exit? */
552	bge	secondary_too_late	/* if so we're too late to the party */
553	or	r3, r3, r7
554	stwcx.	r3, 0, r8
555	bne	21b
556
557	/* Primary thread switches to guest partition. */
558	cmpwi	r6,0
559	bne	10f
560
561	lwz	r7,KVM_LPID(r9)
562	ld	r6,KVM_SDR1(r9)
563	li	r0,LPID_RSVD		/* switch to reserved LPID */
564	mtspr	SPRN_LPID,r0
565	ptesync
566	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
567	mtspr	SPRN_LPID,r7
568	isync
569
570	/* See if we need to flush the TLB. */
571	mr	r3, r9			/* kvm pointer */
572	lhz	r4, PACAPACAINDEX(r13)	/* physical cpu number */
573	li	r5, 0			/* nested vcpu pointer */
574	bl	kvmppc_check_need_tlb_flush
575	nop
576	ld	r5, HSTATE_KVM_VCORE(r13)
577
578	/* Add timebase offset onto timebase */
57922:	ld	r8,VCORE_TB_OFFSET(r5)
580	cmpdi	r8,0
581	beq	37f
582	std	r8, VCORE_TB_OFFSET_APPL(r5)
583	mftb	r6		/* current host timebase */
584	add	r8,r8,r6
585	mtspr	SPRN_TBU40,r8	/* update upper 40 bits */
586	mftb	r7		/* check if lower 24 bits overflowed */
587	clrldi	r6,r6,40
588	clrldi	r7,r7,40
589	cmpld	r7,r6
590	bge	37f
591	addis	r8,r8,0x100	/* if so, increment upper 40 bits */
592	mtspr	SPRN_TBU40,r8
593
594	/* Load guest PCR value to select appropriate compat mode */
59537:	ld	r7, VCORE_PCR(r5)
596	LOAD_REG_IMMEDIATE(r6, PCR_MASK)
597	cmpld	r7, r6
598	beq	38f
599	or	r7, r7, r6
600	mtspr	SPRN_PCR, r7
60138:
602
603BEGIN_FTR_SECTION
604	/* DPDES and VTB are shared between threads */
605	ld	r8, VCORE_DPDES(r5)
606	ld	r7, VCORE_VTB(r5)
607	mtspr	SPRN_DPDES, r8
608	mtspr	SPRN_VTB, r7
609END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
610
611	/* Mark the subcore state as inside guest */
612	bl	kvmppc_subcore_enter_guest
613	nop
614	ld	r5, HSTATE_KVM_VCORE(r13)
615	ld	r4, HSTATE_KVM_VCPU(r13)
616	li	r0,1
617	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
618
619	/* Do we have a guest vcpu to run? */
62010:	cmpdi	r4, 0
621	beq	kvmppc_primary_no_guest
622kvmppc_got_guest:
623	/* Increment yield count if they have a VPA */
624	ld	r3, VCPU_VPA(r4)
625	cmpdi	r3, 0
626	beq	25f
627	li	r6, LPPACA_YIELDCOUNT
628	LWZX_BE	r5, r3, r6
629	addi	r5, r5, 1
630	STWX_BE	r5, r3, r6
631	li	r6, 1
632	stb	r6, VCPU_VPA_DIRTY(r4)
63325:
634
635	/* Save purr/spurr */
636	mfspr	r5,SPRN_PURR
637	mfspr	r6,SPRN_SPURR
638	std	r5,HSTATE_PURR(r13)
639	std	r6,HSTATE_SPURR(r13)
640	ld	r7,VCPU_PURR(r4)
641	ld	r8,VCPU_SPURR(r4)
642	mtspr	SPRN_PURR,r7
643	mtspr	SPRN_SPURR,r8
644
645	/* Save host values of some registers */
646BEGIN_FTR_SECTION
647	mfspr	r5, SPRN_CIABR
648	mfspr	r6, SPRN_DAWR0
649	mfspr	r7, SPRN_DAWRX0
650	mfspr	r8, SPRN_IAMR
651	std	r5, STACK_SLOT_CIABR(r1)
652	std	r6, STACK_SLOT_DAWR0(r1)
653	std	r7, STACK_SLOT_DAWRX0(r1)
654	std	r8, STACK_SLOT_IAMR(r1)
655	mfspr	r5, SPRN_FSCR
656	std	r5, STACK_SLOT_FSCR(r1)
657END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
658
659	mfspr	r5, SPRN_AMR
660	std	r5, STACK_SLOT_AMR(r1)
661	mfspr	r6, SPRN_UAMOR
662	std	r6, STACK_SLOT_UAMOR(r1)
663
664BEGIN_FTR_SECTION
665	/* Set partition DABR */
666	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
667	lwz	r5,VCPU_DABRX(r4)
668	ld	r6,VCPU_DABR(r4)
669	mtspr	SPRN_DABRX,r5
670	mtspr	SPRN_DABR,r6
671	isync
672END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
673
674#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
675BEGIN_FTR_SECTION
676	b	91f
677END_FTR_SECTION_IFCLR(CPU_FTR_TM)
678	/*
679	 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
680	 */
681	mr      r3, r4
682	ld      r4, VCPU_MSR(r3)
683	li	r5, 0			/* don't preserve non-vol regs */
684	bl	kvmppc_restore_tm_hv
685	nop
686	ld	r4, HSTATE_KVM_VCPU(r13)
68791:
688#endif
689
690	/* Load guest PMU registers; r4 = vcpu pointer here */
691	mr	r3, r4
692	bl	kvmhv_load_guest_pmu
693
694	/* Load up FP, VMX and VSX registers */
695	ld	r4, HSTATE_KVM_VCPU(r13)
696	bl	kvmppc_load_fp
697
698	ld	r14, VCPU_GPR(R14)(r4)
699	ld	r15, VCPU_GPR(R15)(r4)
700	ld	r16, VCPU_GPR(R16)(r4)
701	ld	r17, VCPU_GPR(R17)(r4)
702	ld	r18, VCPU_GPR(R18)(r4)
703	ld	r19, VCPU_GPR(R19)(r4)
704	ld	r20, VCPU_GPR(R20)(r4)
705	ld	r21, VCPU_GPR(R21)(r4)
706	ld	r22, VCPU_GPR(R22)(r4)
707	ld	r23, VCPU_GPR(R23)(r4)
708	ld	r24, VCPU_GPR(R24)(r4)
709	ld	r25, VCPU_GPR(R25)(r4)
710	ld	r26, VCPU_GPR(R26)(r4)
711	ld	r27, VCPU_GPR(R27)(r4)
712	ld	r28, VCPU_GPR(R28)(r4)
713	ld	r29, VCPU_GPR(R29)(r4)
714	ld	r30, VCPU_GPR(R30)(r4)
715	ld	r31, VCPU_GPR(R31)(r4)
716
717	/* Switch DSCR to guest value */
718	ld	r5, VCPU_DSCR(r4)
719	mtspr	SPRN_DSCR, r5
720
721BEGIN_FTR_SECTION
722	/* Skip next section on POWER7 */
723	b	8f
724END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
725	/* Load up POWER8-specific registers */
726	ld	r5, VCPU_IAMR(r4)
727	lwz	r6, VCPU_PSPB(r4)
728	ld	r7, VCPU_FSCR(r4)
729	mtspr	SPRN_IAMR, r5
730	mtspr	SPRN_PSPB, r6
731	mtspr	SPRN_FSCR, r7
732	/*
733	 * Handle broken DAWR case by not writing it. This means we
734	 * can still store the DAWR register for migration.
735	 */
736	LOAD_REG_ADDR(r5, dawr_force_enable)
737	lbz	r5, 0(r5)
738	cmpdi	r5, 0
739	beq	1f
740	ld	r5, VCPU_DAWR0(r4)
741	ld	r6, VCPU_DAWRX0(r4)
742	mtspr	SPRN_DAWR0, r5
743	mtspr	SPRN_DAWRX0, r6
7441:
745	ld	r7, VCPU_CIABR(r4)
746	ld	r8, VCPU_TAR(r4)
747	mtspr	SPRN_CIABR, r7
748	mtspr	SPRN_TAR, r8
749	ld	r5, VCPU_IC(r4)
750	ld	r8, VCPU_EBBHR(r4)
751	mtspr	SPRN_IC, r5
752	mtspr	SPRN_EBBHR, r8
753	ld	r5, VCPU_EBBRR(r4)
754	ld	r6, VCPU_BESCR(r4)
755	lwz	r7, VCPU_GUEST_PID(r4)
756	ld	r8, VCPU_WORT(r4)
757	mtspr	SPRN_EBBRR, r5
758	mtspr	SPRN_BESCR, r6
759	mtspr	SPRN_PID, r7
760	mtspr	SPRN_WORT, r8
761	/* POWER8-only registers */
762	ld	r5, VCPU_TCSCR(r4)
763	ld	r6, VCPU_ACOP(r4)
764	ld	r7, VCPU_CSIGR(r4)
765	ld	r8, VCPU_TACR(r4)
766	mtspr	SPRN_TCSCR, r5
767	mtspr	SPRN_ACOP, r6
768	mtspr	SPRN_CSIGR, r7
769	mtspr	SPRN_TACR, r8
770	nop
7718:
772
773	ld	r5, VCPU_SPRG0(r4)
774	ld	r6, VCPU_SPRG1(r4)
775	ld	r7, VCPU_SPRG2(r4)
776	ld	r8, VCPU_SPRG3(r4)
777	mtspr	SPRN_SPRG0, r5
778	mtspr	SPRN_SPRG1, r6
779	mtspr	SPRN_SPRG2, r7
780	mtspr	SPRN_SPRG3, r8
781
782	/* Load up DAR and DSISR */
783	ld	r5, VCPU_DAR(r4)
784	lwz	r6, VCPU_DSISR(r4)
785	mtspr	SPRN_DAR, r5
786	mtspr	SPRN_DSISR, r6
787
788	/* Restore AMR and UAMOR, set AMOR to all 1s */
789	ld	r5,VCPU_AMR(r4)
790	ld	r6,VCPU_UAMOR(r4)
791	mtspr	SPRN_AMR,r5
792	mtspr	SPRN_UAMOR,r6
793
794	/* Restore state of CTRL run bit; the host currently has it set to 1 */
795	lwz	r5,VCPU_CTRL(r4)
796	andi.	r5,r5,1
797	bne	4f
798	li	r6,0
799	mtspr	SPRN_CTRLT,r6
8004:
801	/* Secondary threads wait for primary to have done partition switch */
802	ld	r5, HSTATE_KVM_VCORE(r13)
803	lbz	r6, HSTATE_PTID(r13)
804	cmpwi	r6, 0
805	beq	21f
806	lbz	r0, VCORE_IN_GUEST(r5)
807	cmpwi	r0, 0
808	bne	21f
809	HMT_LOW
81020:	lwz	r3, VCORE_ENTRY_EXIT(r5)
811	cmpwi	r3, 0x100
812	bge	no_switch_exit
813	lbz	r0, VCORE_IN_GUEST(r5)
814	cmpwi	r0, 0
815	beq	20b
816	HMT_MEDIUM
81721:
818	/* Set LPCR. */
819	ld	r8,VCORE_LPCR(r5)
820	mtspr	SPRN_LPCR,r8
821	isync
822
823	/*
824	 * Set the decrementer to the guest decrementer.
825	 */
826	ld	r8,VCPU_DEC_EXPIRES(r4)
827	mftb	r7
828	subf	r3,r7,r8
829	mtspr	SPRN_DEC,r3
830
831	/* Check if HDEC expires soon */
832	mfspr	r3, SPRN_HDEC
833	extsw	r3, r3
834	cmpdi	r3, 512		/* 1 microsecond */
835	blt	hdec_soon
836
837	/* Clear out and reload the SLB */
838	li	r6, 0
839	slbmte	r6, r6
840	PPC_SLBIA(6)
841	ptesync
842
843	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
844	lwz	r5,VCPU_SLB_MAX(r4)
845	cmpwi	r5,0
846	beq	9f
847	mtctr	r5
848	addi	r6,r4,VCPU_SLB
8491:	ld	r8,VCPU_SLB_E(r6)
850	ld	r9,VCPU_SLB_V(r6)
851	slbmte	r9,r8
852	addi	r6,r6,VCPU_SLB_SIZE
853	bdnz	1b
8549:
855
856deliver_guest_interrupt:	/* r4 = vcpu, r13 = paca */
857	/* Check if we can deliver an external or decrementer interrupt now */
858	ld	r0, VCPU_PENDING_EXC(r4)
859	cmpdi	r0, 0
860	beq	71f
861	mr	r3, r4
862	bl	kvmppc_guest_entry_inject_int
863	ld	r4, HSTATE_KVM_VCPU(r13)
86471:
865	ld	r6, VCPU_SRR0(r4)
866	ld	r7, VCPU_SRR1(r4)
867	mtspr	SPRN_SRR0, r6
868	mtspr	SPRN_SRR1, r7
869
870	ld	r10, VCPU_PC(r4)
871	ld	r11, VCPU_MSR(r4)
872	/* r11 = vcpu->arch.msr & ~MSR_HV */
873	rldicl	r11, r11, 63 - MSR_HV_LG, 1
874	rotldi	r11, r11, 1 + MSR_HV_LG
875	ori	r11, r11, MSR_ME
876
877	ld	r6, VCPU_CTR(r4)
878	ld	r7, VCPU_XER(r4)
879	mtctr	r6
880	mtxer	r7
881
882/*
883 * Required state:
884 * R4 = vcpu
885 * R10: value for HSRR0
886 * R11: value for HSRR1
887 * R13 = PACA
888 */
889fast_guest_return:
890	li	r0,0
891	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
892	mtspr	SPRN_HSRR0,r10
893	mtspr	SPRN_HSRR1,r11
894
895	/* Activate guest mode, so faults get handled by KVM */
896	li	r9, KVM_GUEST_MODE_GUEST_HV
897	stb	r9, HSTATE_IN_GUEST(r13)
898
899#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
900	/* Accumulate timing */
901	addi	r3, r4, VCPU_TB_GUEST
902	bl	kvmhv_accumulate_time
903#endif
904
905	/* Enter guest */
906
907BEGIN_FTR_SECTION
908	ld	r5, VCPU_CFAR(r4)
909	mtspr	SPRN_CFAR, r5
910END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
911BEGIN_FTR_SECTION
912	ld	r0, VCPU_PPR(r4)
913END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
914
915	ld	r5, VCPU_LR(r4)
916	mtlr	r5
917
918	ld	r1, VCPU_GPR(R1)(r4)
919	ld	r5, VCPU_GPR(R5)(r4)
920	ld	r8, VCPU_GPR(R8)(r4)
921	ld	r9, VCPU_GPR(R9)(r4)
922	ld	r10, VCPU_GPR(R10)(r4)
923	ld	r11, VCPU_GPR(R11)(r4)
924	ld	r12, VCPU_GPR(R12)(r4)
925	ld	r13, VCPU_GPR(R13)(r4)
926
927BEGIN_FTR_SECTION
928	mtspr	SPRN_PPR, r0
929END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
930
931	ld	r6, VCPU_GPR(R6)(r4)
932	ld	r7, VCPU_GPR(R7)(r4)
933
934	ld	r0, VCPU_CR(r4)
935	mtcr	r0
936
937	ld	r0, VCPU_GPR(R0)(r4)
938	ld	r2, VCPU_GPR(R2)(r4)
939	ld	r3, VCPU_GPR(R3)(r4)
940	ld	r4, VCPU_GPR(R4)(r4)
941	HRFI_TO_GUEST
942	b	.
943
944secondary_too_late:
945	li	r12, 0
946	stw	r12, STACK_SLOT_TRAP(r1)
947	cmpdi	r4, 0
948	beq	11f
949	stw	r12, VCPU_TRAP(r4)
950#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
951	addi	r3, r4, VCPU_TB_RMEXIT
952	bl	kvmhv_accumulate_time
953#endif
95411:	b	kvmhv_switch_to_host
955
956no_switch_exit:
957	HMT_MEDIUM
958	li	r12, 0
959	b	12f
960hdec_soon:
961	li	r12, BOOK3S_INTERRUPT_HV_DECREMENTER
96212:	stw	r12, VCPU_TRAP(r4)
963	mr	r9, r4
964#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
965	addi	r3, r4, VCPU_TB_RMEXIT
966	bl	kvmhv_accumulate_time
967#endif
968	b	guest_bypass
969
970/******************************************************************************
971 *                                                                            *
972 *                               Exit code                                    *
973 *                                                                            *
974 *****************************************************************************/
975
976/*
977 * We come here from the first-level interrupt handlers.
978 */
979	.globl	kvmppc_interrupt_hv
980kvmppc_interrupt_hv:
981	/*
982	 * Register contents:
983	 * R9		= HSTATE_IN_GUEST
984	 * R12		= (guest CR << 32) | interrupt vector
985	 * R13		= PACA
986	 * guest R12 saved in shadow VCPU SCRATCH0
987	 * guest R13 saved in SPRN_SCRATCH0
988	 * guest R9 saved in HSTATE_SCRATCH2
989	 */
990	/* We're now back in the host but in guest MMU context */
991	cmpwi	r9,KVM_GUEST_MODE_HOST_HV
992	beq	kvmppc_bad_host_intr
993	li	r9, KVM_GUEST_MODE_HOST_HV
994	stb	r9, HSTATE_IN_GUEST(r13)
995
996	ld	r9, HSTATE_KVM_VCPU(r13)
997
998	/* Save registers */
999
1000	std	r0, VCPU_GPR(R0)(r9)
1001	std	r1, VCPU_GPR(R1)(r9)
1002	std	r2, VCPU_GPR(R2)(r9)
1003	std	r3, VCPU_GPR(R3)(r9)
1004	std	r4, VCPU_GPR(R4)(r9)
1005	std	r5, VCPU_GPR(R5)(r9)
1006	std	r6, VCPU_GPR(R6)(r9)
1007	std	r7, VCPU_GPR(R7)(r9)
1008	std	r8, VCPU_GPR(R8)(r9)
1009	ld	r0, HSTATE_SCRATCH2(r13)
1010	std	r0, VCPU_GPR(R9)(r9)
1011	std	r10, VCPU_GPR(R10)(r9)
1012	std	r11, VCPU_GPR(R11)(r9)
1013	ld	r3, HSTATE_SCRATCH0(r13)
1014	std	r3, VCPU_GPR(R12)(r9)
1015	/* CR is in the high half of r12 */
1016	srdi	r4, r12, 32
1017	std	r4, VCPU_CR(r9)
1018BEGIN_FTR_SECTION
1019	ld	r3, HSTATE_CFAR(r13)
1020	std	r3, VCPU_CFAR(r9)
1021END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1022BEGIN_FTR_SECTION
1023	ld	r4, HSTATE_PPR(r13)
1024	std	r4, VCPU_PPR(r9)
1025END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1026
1027	/* Restore R1/R2 so we can handle faults */
1028	ld	r1, HSTATE_HOST_R1(r13)
1029	LOAD_PACA_TOC()
1030
1031	mfspr	r10, SPRN_SRR0
1032	mfspr	r11, SPRN_SRR1
1033	std	r10, VCPU_SRR0(r9)
1034	std	r11, VCPU_SRR1(r9)
1035	/* trap is in the low half of r12, clear CR from the high half */
1036	clrldi	r12, r12, 32
1037	andi.	r0, r12, 2		/* need to read HSRR0/1? */
1038	beq	1f
1039	mfspr	r10, SPRN_HSRR0
1040	mfspr	r11, SPRN_HSRR1
1041	clrrdi	r12, r12, 2
10421:	std	r10, VCPU_PC(r9)
1043	std	r11, VCPU_MSR(r9)
1044
1045	GET_SCRATCH0(r3)
1046	mflr	r4
1047	std	r3, VCPU_GPR(R13)(r9)
1048	std	r4, VCPU_LR(r9)
1049
1050	stw	r12,VCPU_TRAP(r9)
1051
1052	/*
1053	 * Now that we have saved away SRR0/1 and HSRR0/1,
1054	 * interrupts are recoverable in principle, so set MSR_RI.
1055	 * This becomes important for relocation-on interrupts from
1056	 * the guest, which we can get in radix mode on POWER9.
1057	 */
1058	li	r0, MSR_RI
1059	mtmsrd	r0, 1
1060
1061#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1062	addi	r3, r9, VCPU_TB_RMINTR
1063	mr	r4, r9
1064	bl	kvmhv_accumulate_time
1065	ld	r5, VCPU_GPR(R5)(r9)
1066	ld	r6, VCPU_GPR(R6)(r9)
1067	ld	r7, VCPU_GPR(R7)(r9)
1068	ld	r8, VCPU_GPR(R8)(r9)
1069#endif
1070
1071	/* Save HEIR (HV emulation assist reg) in emul_inst
1072	   if this is an HEI (HV emulation interrupt, e40) */
1073	li	r3,KVM_INST_FETCH_FAILED
1074	stw	r3,VCPU_LAST_INST(r9)
1075	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1076	bne	11f
1077	mfspr	r3,SPRN_HEIR
107811:	stw	r3,VCPU_HEIR(r9)
1079
1080	/* these are volatile across C function calls */
1081	mfctr	r3
1082	mfxer	r4
1083	std	r3, VCPU_CTR(r9)
1084	std	r4, VCPU_XER(r9)
1085
1086	/* Save more register state  */
1087	mfdar	r3
1088	mfdsisr	r4
1089	std	r3, VCPU_DAR(r9)
1090	stw	r4, VCPU_DSISR(r9)
1091
1092	/* If this is a page table miss then see if it's theirs or ours */
1093	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1094	beq	kvmppc_hdsi
1095	std	r3, VCPU_FAULT_DAR(r9)
1096	stw	r4, VCPU_FAULT_DSISR(r9)
1097	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1098	beq	kvmppc_hisi
1099
1100	/* See if this is a leftover HDEC interrupt */
1101	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1102	bne	2f
1103	mfspr	r3,SPRN_HDEC
1104	extsw	r3, r3
1105	cmpdi	r3,0
1106	mr	r4,r9
1107	bge	fast_guest_return
11082:
1109	/* See if this is an hcall we can handle in real mode */
1110	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
1111	beq	hcall_try_real_mode
1112
1113	/* Hypervisor doorbell - exit only if host IPI flag set */
1114	cmpwi	r12, BOOK3S_INTERRUPT_H_DOORBELL
1115	bne	3f
1116	lbz	r0, HSTATE_HOST_IPI(r13)
1117	cmpwi	r0, 0
1118	beq	maybe_reenter_guest
1119	b	guest_exit_cont
11203:
1121	/* If it's a hypervisor facility unavailable interrupt, save HFSCR */
1122	cmpwi	r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL
1123	bne	14f
1124	mfspr	r3, SPRN_HFSCR
1125	std	r3, VCPU_HFSCR(r9)
1126	b	guest_exit_cont
112714:
1128	/* External interrupt ? */
1129	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
1130	beq	kvmppc_guest_external
1131	/* See if it is a machine check */
1132	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1133	beq	machine_check_realmode
1134	/* Or a hypervisor maintenance interrupt */
1135	cmpwi	r12, BOOK3S_INTERRUPT_HMI
1136	beq	hmi_realmode
1137
1138guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
1139
1140#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1141	addi	r3, r9, VCPU_TB_RMEXIT
1142	mr	r4, r9
1143	bl	kvmhv_accumulate_time
1144#endif
1145
1146	/*
1147	 * Possibly flush the link stack here, before we do a blr in
1148	 * kvmhv_switch_to_host.
1149	 */
11501:	nop
1151	patch_site 1b patch__call_kvm_flush_link_stack
1152
1153	/* For hash guest, read the guest SLB and save it away */
1154	li	r5, 0
1155	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
1156	mtctr	r0
1157	li	r6,0
1158	addi	r7,r9,VCPU_SLB
11591:	slbmfee	r8,r6
1160	andis.	r0,r8,SLB_ESID_V@h
1161	beq	2f
1162	add	r8,r8,r6		/* put index in */
1163	slbmfev	r3,r6
1164	std	r8,VCPU_SLB_E(r7)
1165	std	r3,VCPU_SLB_V(r7)
1166	addi	r7,r7,VCPU_SLB_SIZE
1167	addi	r5,r5,1
11682:	addi	r6,r6,1
1169	bdnz	1b
1170	/* Finally clear out the SLB */
1171	li	r0,0
1172	slbmte	r0,r0
1173	PPC_SLBIA(6)
1174	ptesync
1175	stw	r5,VCPU_SLB_MAX(r9)
1176
1177	/* load host SLB entries */
1178	ld	r8,PACA_SLBSHADOWPTR(r13)
1179
1180	.rept	SLB_NUM_BOLTED
1181	li	r3, SLBSHADOW_SAVEAREA
1182	LDX_BE	r5, r8, r3
1183	addi	r3, r3, 8
1184	LDX_BE	r6, r8, r3
1185	andis.	r7,r5,SLB_ESID_V@h
1186	beq	1f
1187	slbmte	r6,r5
11881:	addi	r8,r8,16
1189	.endr
1190
1191guest_bypass:
1192	stw	r12, STACK_SLOT_TRAP(r1)
1193
1194	/* Save DEC */
1195	/* Do this before kvmhv_commence_exit so we know TB is guest TB */
1196	ld	r3, HSTATE_KVM_VCORE(r13)
1197	mfspr	r5,SPRN_DEC
1198	mftb	r6
1199	extsw	r5,r5
120016:	add	r5,r5,r6
1201	std	r5,VCPU_DEC_EXPIRES(r9)
1202
1203	/* Increment exit count, poke other threads to exit */
1204	mr 	r3, r12
1205	bl	kvmhv_commence_exit
1206	nop
1207	ld	r9, HSTATE_KVM_VCPU(r13)
1208
1209	/* Stop others sending VCPU interrupts to this physical CPU */
1210	li	r0, -1
1211	stw	r0, VCPU_CPU(r9)
1212	stw	r0, VCPU_THREAD_CPU(r9)
1213
1214	/* Save guest CTRL register, set runlatch to 1 if it was clear */
1215	mfspr	r6,SPRN_CTRLF
1216	stw	r6,VCPU_CTRL(r9)
1217	andi.	r0,r6,1
1218	bne	4f
1219	li	r6,1
1220	mtspr	SPRN_CTRLT,r6
12214:
1222	/*
1223	 * Save the guest PURR/SPURR
1224	 */
1225	mfspr	r5,SPRN_PURR
1226	mfspr	r6,SPRN_SPURR
1227	ld	r7,VCPU_PURR(r9)
1228	ld	r8,VCPU_SPURR(r9)
1229	std	r5,VCPU_PURR(r9)
1230	std	r6,VCPU_SPURR(r9)
1231	subf	r5,r7,r5
1232	subf	r6,r8,r6
1233
1234	/*
1235	 * Restore host PURR/SPURR and add guest times
1236	 * so that the time in the guest gets accounted.
1237	 */
1238	ld	r3,HSTATE_PURR(r13)
1239	ld	r4,HSTATE_SPURR(r13)
1240	add	r3,r3,r5
1241	add	r4,r4,r6
1242	mtspr	SPRN_PURR,r3
1243	mtspr	SPRN_SPURR,r4
1244
1245BEGIN_FTR_SECTION
1246	b	8f
1247END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1248	/* Save POWER8-specific registers */
1249	mfspr	r5, SPRN_IAMR
1250	mfspr	r6, SPRN_PSPB
1251	mfspr	r7, SPRN_FSCR
1252	std	r5, VCPU_IAMR(r9)
1253	stw	r6, VCPU_PSPB(r9)
1254	std	r7, VCPU_FSCR(r9)
1255	mfspr	r5, SPRN_IC
1256	mfspr	r7, SPRN_TAR
1257	std	r5, VCPU_IC(r9)
1258	std	r7, VCPU_TAR(r9)
1259	mfspr	r8, SPRN_EBBHR
1260	std	r8, VCPU_EBBHR(r9)
1261	mfspr	r5, SPRN_EBBRR
1262	mfspr	r6, SPRN_BESCR
1263	mfspr	r7, SPRN_PID
1264	mfspr	r8, SPRN_WORT
1265	std	r5, VCPU_EBBRR(r9)
1266	std	r6, VCPU_BESCR(r9)
1267	stw	r7, VCPU_GUEST_PID(r9)
1268	std	r8, VCPU_WORT(r9)
1269	mfspr	r5, SPRN_TCSCR
1270	mfspr	r6, SPRN_ACOP
1271	mfspr	r7, SPRN_CSIGR
1272	mfspr	r8, SPRN_TACR
1273	std	r5, VCPU_TCSCR(r9)
1274	std	r6, VCPU_ACOP(r9)
1275	std	r7, VCPU_CSIGR(r9)
1276	std	r8, VCPU_TACR(r9)
1277BEGIN_FTR_SECTION
1278	ld	r5, STACK_SLOT_FSCR(r1)
1279	mtspr	SPRN_FSCR, r5
1280END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1281	/*
1282	 * Restore various registers to 0, where non-zero values
1283	 * set by the guest could disrupt the host.
1284	 */
1285	li	r0, 0
1286	mtspr	SPRN_PSPB, r0
1287	mtspr	SPRN_WORT, r0
1288	mtspr	SPRN_TCSCR, r0
1289	/* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
1290	li	r0, 1
1291	sldi	r0, r0, 31
1292	mtspr	SPRN_MMCRS, r0
1293
1294	/* Save and restore AMR, IAMR and UAMOR before turning on the MMU */
1295	ld	r8, STACK_SLOT_IAMR(r1)
1296	mtspr	SPRN_IAMR, r8
1297
12988:	/* Power7 jumps back in here */
1299	mfspr	r5,SPRN_AMR
1300	mfspr	r6,SPRN_UAMOR
1301	std	r5,VCPU_AMR(r9)
1302	std	r6,VCPU_UAMOR(r9)
1303	ld	r5,STACK_SLOT_AMR(r1)
1304	ld	r6,STACK_SLOT_UAMOR(r1)
1305	mtspr	SPRN_AMR, r5
1306	mtspr	SPRN_UAMOR, r6
1307
1308	/* Switch DSCR back to host value */
1309	mfspr	r8, SPRN_DSCR
1310	ld	r7, HSTATE_DSCR(r13)
1311	std	r8, VCPU_DSCR(r9)
1312	mtspr	SPRN_DSCR, r7
1313
1314	/* Save non-volatile GPRs */
1315	std	r14, VCPU_GPR(R14)(r9)
1316	std	r15, VCPU_GPR(R15)(r9)
1317	std	r16, VCPU_GPR(R16)(r9)
1318	std	r17, VCPU_GPR(R17)(r9)
1319	std	r18, VCPU_GPR(R18)(r9)
1320	std	r19, VCPU_GPR(R19)(r9)
1321	std	r20, VCPU_GPR(R20)(r9)
1322	std	r21, VCPU_GPR(R21)(r9)
1323	std	r22, VCPU_GPR(R22)(r9)
1324	std	r23, VCPU_GPR(R23)(r9)
1325	std	r24, VCPU_GPR(R24)(r9)
1326	std	r25, VCPU_GPR(R25)(r9)
1327	std	r26, VCPU_GPR(R26)(r9)
1328	std	r27, VCPU_GPR(R27)(r9)
1329	std	r28, VCPU_GPR(R28)(r9)
1330	std	r29, VCPU_GPR(R29)(r9)
1331	std	r30, VCPU_GPR(R30)(r9)
1332	std	r31, VCPU_GPR(R31)(r9)
1333
1334	/* Save SPRGs */
1335	mfspr	r3, SPRN_SPRG0
1336	mfspr	r4, SPRN_SPRG1
1337	mfspr	r5, SPRN_SPRG2
1338	mfspr	r6, SPRN_SPRG3
1339	std	r3, VCPU_SPRG0(r9)
1340	std	r4, VCPU_SPRG1(r9)
1341	std	r5, VCPU_SPRG2(r9)
1342	std	r6, VCPU_SPRG3(r9)
1343
1344	/* save FP state */
1345	mr	r3, r9
1346	bl	kvmppc_save_fp
1347
1348#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1349BEGIN_FTR_SECTION
1350	b	91f
1351END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1352	/*
1353	 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
1354	 */
1355	mr      r3, r9
1356	ld      r4, VCPU_MSR(r3)
1357	li	r5, 0			/* don't preserve non-vol regs */
1358	bl	kvmppc_save_tm_hv
1359	nop
1360	ld	r9, HSTATE_KVM_VCPU(r13)
136191:
1362#endif
1363
1364	/* Increment yield count if they have a VPA */
1365	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
1366	cmpdi	r8, 0
1367	beq	25f
1368	li	r4, LPPACA_YIELDCOUNT
1369	LWZX_BE	r3, r8, r4
1370	addi	r3, r3, 1
1371	STWX_BE	r3, r8, r4
1372	li	r3, 1
1373	stb	r3, VCPU_VPA_DIRTY(r9)
137425:
1375	/* Save PMU registers if requested */
1376	/* r8 and cr0.eq are live here */
1377	mr	r3, r9
1378	li	r4, 1
1379	beq	21f			/* if no VPA, save PMU stuff anyway */
1380	lbz	r4, LPPACA_PMCINUSE(r8)
138121:	bl	kvmhv_save_guest_pmu
1382	ld	r9, HSTATE_KVM_VCPU(r13)
1383
1384	/* Restore host values of some registers */
1385BEGIN_FTR_SECTION
1386	ld	r5, STACK_SLOT_CIABR(r1)
1387	ld	r6, STACK_SLOT_DAWR0(r1)
1388	ld	r7, STACK_SLOT_DAWRX0(r1)
1389	mtspr	SPRN_CIABR, r5
1390	/*
1391	 * If the DAWR doesn't work, it's ok to write these here as
1392	 * this value should always be zero
1393	*/
1394	mtspr	SPRN_DAWR0, r6
1395	mtspr	SPRN_DAWRX0, r7
1396END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1397
1398	/*
1399	 * POWER7/POWER8 guest -> host partition switch code.
1400	 * We don't have to lock against tlbies but we do
1401	 * have to coordinate the hardware threads.
1402	 * Here STACK_SLOT_TRAP(r1) contains the trap number.
1403	 */
1404kvmhv_switch_to_host:
1405	/* Secondary threads wait for primary to do partition switch */
1406	ld	r5,HSTATE_KVM_VCORE(r13)
1407	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
1408	lbz	r3,HSTATE_PTID(r13)
1409	cmpwi	r3,0
1410	beq	15f
1411	HMT_LOW
141213:	lbz	r3,VCORE_IN_GUEST(r5)
1413	cmpwi	r3,0
1414	bne	13b
1415	HMT_MEDIUM
1416	b	16f
1417
1418	/* Primary thread waits for all the secondaries to exit guest */
141915:	lwz	r3,VCORE_ENTRY_EXIT(r5)
1420	rlwinm	r0,r3,32-8,0xff
1421	clrldi	r3,r3,56
1422	cmpw	r3,r0
1423	bne	15b
1424	isync
1425
1426	/* Did we actually switch to the guest at all? */
1427	lbz	r6, VCORE_IN_GUEST(r5)
1428	cmpwi	r6, 0
1429	beq	19f
1430
1431	/* Primary thread switches back to host partition */
1432	lwz	r7,KVM_HOST_LPID(r4)
1433	ld	r6,KVM_HOST_SDR1(r4)
1434	li	r8,LPID_RSVD		/* switch to reserved LPID */
1435	mtspr	SPRN_LPID,r8
1436	ptesync
1437	mtspr	SPRN_SDR1,r6		/* switch to host page table */
1438	mtspr	SPRN_LPID,r7
1439	isync
1440
1441BEGIN_FTR_SECTION
1442	/* DPDES and VTB are shared between threads */
1443	mfspr	r7, SPRN_DPDES
1444	mfspr	r8, SPRN_VTB
1445	std	r7, VCORE_DPDES(r5)
1446	std	r8, VCORE_VTB(r5)
1447	/* clear DPDES so we don't get guest doorbells in the host */
1448	li	r8, 0
1449	mtspr	SPRN_DPDES, r8
1450END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1451
1452	/* Subtract timebase offset from timebase */
1453	ld	r8, VCORE_TB_OFFSET_APPL(r5)
1454	cmpdi	r8,0
1455	beq	17f
1456	li	r0, 0
1457	std	r0, VCORE_TB_OFFSET_APPL(r5)
1458	mftb	r6			/* current guest timebase */
1459	subf	r8,r8,r6
1460	mtspr	SPRN_TBU40,r8		/* update upper 40 bits */
1461	mftb	r7			/* check if lower 24 bits overflowed */
1462	clrldi	r6,r6,40
1463	clrldi	r7,r7,40
1464	cmpld	r7,r6
1465	bge	17f
1466	addis	r8,r8,0x100		/* if so, increment upper 40 bits */
1467	mtspr	SPRN_TBU40,r8
1468
146917:
1470	/*
1471	 * If this is an HMI, we called kvmppc_realmode_hmi_handler
1472	 * above, which may or may not have already called
1473	 * kvmppc_subcore_exit_guest.  Fortunately, all that
1474	 * kvmppc_subcore_exit_guest does is clear a flag, so calling
1475	 * it again here is benign even if kvmppc_realmode_hmi_handler
1476	 * has already called it.
1477	 */
1478	bl	kvmppc_subcore_exit_guest
1479	nop
148030:	ld	r5,HSTATE_KVM_VCORE(r13)
1481	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
1482
1483	/* Reset PCR */
1484	ld	r0, VCORE_PCR(r5)
1485	LOAD_REG_IMMEDIATE(r6, PCR_MASK)
1486	cmpld	r0, r6
1487	beq	18f
1488	mtspr	SPRN_PCR, r6
148918:
1490	/* Signal secondary CPUs to continue */
1491	li	r0, 0
1492	stb	r0,VCORE_IN_GUEST(r5)
149319:	lis	r8,0x7fff		/* MAX_INT@h */
1494	mtspr	SPRN_HDEC,r8
1495
149616:	ld	r8,KVM_HOST_LPCR(r4)
1497	mtspr	SPRN_LPCR,r8
1498	isync
1499
1500#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1501	/* Finish timing, if we have a vcpu */
1502	ld	r4, HSTATE_KVM_VCPU(r13)
1503	cmpdi	r4, 0
1504	li	r3, 0
1505	beq	2f
1506	bl	kvmhv_accumulate_time
15072:
1508#endif
1509	/* Unset guest mode */
1510	li	r0, KVM_GUEST_MODE_NONE
1511	stb	r0, HSTATE_IN_GUEST(r13)
1512
1513	lwz	r12, STACK_SLOT_TRAP(r1)	/* return trap # in r12 */
1514	ld	r0, SFS+PPC_LR_STKOFF(r1)
1515	addi	r1, r1, SFS
1516	mtlr	r0
1517	blr
1518
1519.balign 32
1520.global kvm_flush_link_stack
1521kvm_flush_link_stack:
1522	/* Save LR into r0 */
1523	mflr	r0
1524
1525	/* Flush the link stack. On Power8 it's up to 32 entries in size. */
1526	.rept 32
1527	ANNOTATE_INTRA_FUNCTION_CALL
1528	bl	.+4
1529	.endr
1530
1531	/* And on Power9 it's up to 64. */
1532BEGIN_FTR_SECTION
1533	.rept 32
1534	ANNOTATE_INTRA_FUNCTION_CALL
1535	bl	.+4
1536	.endr
1537END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1538
1539	/* Restore LR */
1540	mtlr	r0
1541	blr
1542
1543kvmppc_guest_external:
1544	/* External interrupt, first check for host_ipi. If this is
1545	 * set, we know the host wants us out so let's do it now
1546	 */
1547	bl	kvmppc_read_intr
1548
1549	/*
1550	 * Restore the active volatile registers after returning from
1551	 * a C function.
1552	 */
1553	ld	r9, HSTATE_KVM_VCPU(r13)
1554	li	r12, BOOK3S_INTERRUPT_EXTERNAL
1555
1556	/*
1557	 * kvmppc_read_intr return codes:
1558	 *
1559	 * Exit to host (r3 > 0)
1560	 *   1 An interrupt is pending that needs to be handled by the host
1561	 *     Exit guest and return to host by branching to guest_exit_cont
1562	 *
1563	 *   2 Passthrough that needs completion in the host
1564	 *     Exit guest and return to host by branching to guest_exit_cont
1565	 *     However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD
1566	 *     to indicate to the host to complete handling the interrupt
1567	 *
1568	 * Before returning to guest, we check if any CPU is heading out
1569	 * to the host and if so, we head out also. If no CPUs are heading
1570	 * check return values <= 0.
1571	 *
1572	 * Return to guest (r3 <= 0)
1573	 *  0 No external interrupt is pending
1574	 * -1 A guest wakeup IPI (which has now been cleared)
1575	 *    In either case, we return to guest to deliver any pending
1576	 *    guest interrupts.
1577	 *
1578	 * -2 A PCI passthrough external interrupt was handled
1579	 *    (interrupt was delivered directly to guest)
1580	 *    Return to guest to deliver any pending guest interrupts.
1581	 */
1582
1583	cmpdi	r3, 1
1584	ble	1f
1585
1586	/* Return code = 2 */
1587	li	r12, BOOK3S_INTERRUPT_HV_RM_HARD
1588	stw	r12, VCPU_TRAP(r9)
1589	b	guest_exit_cont
1590
15911:	/* Return code <= 1 */
1592	cmpdi	r3, 0
1593	bgt	guest_exit_cont
1594
1595	/* Return code <= 0 */
1596maybe_reenter_guest:
1597	ld	r5, HSTATE_KVM_VCORE(r13)
1598	lwz	r0, VCORE_ENTRY_EXIT(r5)
1599	cmpwi	r0, 0x100
1600	mr	r4, r9
1601	blt	deliver_guest_interrupt
1602	b	guest_exit_cont
1603
1604/*
1605 * Check whether an HDSI is an HPTE not found fault or something else.
1606 * If it is an HPTE not found fault that is due to the guest accessing
1607 * a page that they have mapped but which we have paged out, then
1608 * we continue on with the guest exit path.  In all other cases,
1609 * reflect the HDSI to the guest as a DSI.
1610 */
1611kvmppc_hdsi:
1612	mfspr	r4, SPRN_HDAR
1613	mfspr	r6, SPRN_HDSISR
1614	/* HPTE not found fault or protection fault? */
1615	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1616	beq	1f			/* if not, send it to the guest */
1617	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
1618	beq	3f
1619	clrrdi	r0, r4, 28
1620	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1621	li	r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1622	bne	7f			/* if no SLB entry found */
16234:	std	r4, VCPU_FAULT_DAR(r9)
1624	stw	r6, VCPU_FAULT_DSISR(r9)
1625
1626	/* Search the hash table. */
1627	mr	r3, r9			/* vcpu pointer */
1628	li	r7, 1			/* data fault */
1629	bl	kvmppc_hpte_hv_fault
1630	ld	r9, HSTATE_KVM_VCPU(r13)
1631	ld	r10, VCPU_PC(r9)
1632	ld	r11, VCPU_MSR(r9)
1633	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1634	cmpdi	r3, 0			/* retry the instruction */
1635	beq	6f
1636	cmpdi	r3, -1			/* handle in kernel mode */
1637	beq	guest_exit_cont
1638	cmpdi	r3, -2			/* MMIO emulation; need instr word */
1639	beq	2f
1640
1641	/* Synthesize a DSI (or DSegI) for the guest */
1642	ld	r4, VCPU_FAULT_DAR(r9)
1643	mr	r6, r3
16441:	li	r0, BOOK3S_INTERRUPT_DATA_STORAGE
1645	mtspr	SPRN_DSISR, r6
16467:	mtspr	SPRN_DAR, r4
1647	mtspr	SPRN_SRR0, r10
1648	mtspr	SPRN_SRR1, r11
1649	mr	r10, r0
1650	bl	kvmppc_msr_interrupt
1651fast_interrupt_c_return:
16526:	ld	r7, VCPU_CTR(r9)
1653	ld	r8, VCPU_XER(r9)
1654	mtctr	r7
1655	mtxer	r8
1656	mr	r4, r9
1657	b	fast_guest_return
1658
16593:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
1660	ld	r5, KVM_VRMA_SLB_V(r5)
1661	b	4b
1662
1663	/* If this is for emulated MMIO, load the instruction word */
16642:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */
1665
1666	/* Set guest mode to 'jump over instruction' so if lwz faults
1667	 * we'll just continue at the next IP. */
1668	li	r0, KVM_GUEST_MODE_SKIP
1669	stb	r0, HSTATE_IN_GUEST(r13)
1670
1671	/* Do the access with MSR:DR enabled */
1672	mfmsr	r3
1673	ori	r4, r3, MSR_DR		/* Enable paging for data */
1674	mtmsrd	r4
1675	lwz	r8, 0(r10)
1676	mtmsrd	r3
1677
1678	/* Store the result */
1679	stw	r8, VCPU_LAST_INST(r9)
1680
1681	/* Unset guest mode. */
1682	li	r0, KVM_GUEST_MODE_HOST_HV
1683	stb	r0, HSTATE_IN_GUEST(r13)
1684	b	guest_exit_cont
1685
1686/*
1687 * Similarly for an HISI, reflect it to the guest as an ISI unless
1688 * it is an HPTE not found fault for a page that we have paged out.
1689 */
1690kvmppc_hisi:
1691	andis.	r0, r11, SRR1_ISI_NOPT@h
1692	beq	1f
1693	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
1694	beq	3f
1695	clrrdi	r0, r10, 28
1696	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1697	li	r0, BOOK3S_INTERRUPT_INST_SEGMENT
1698	bne	7f			/* if no SLB entry found */
16994:
1700	/* Search the hash table. */
1701	mr	r3, r9			/* vcpu pointer */
1702	mr	r4, r10
1703	mr	r6, r11
1704	li	r7, 0			/* instruction fault */
1705	bl	kvmppc_hpte_hv_fault
1706	ld	r9, HSTATE_KVM_VCPU(r13)
1707	ld	r10, VCPU_PC(r9)
1708	ld	r11, VCPU_MSR(r9)
1709	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1710	cmpdi	r3, 0			/* retry the instruction */
1711	beq	fast_interrupt_c_return
1712	cmpdi	r3, -1			/* handle in kernel mode */
1713	beq	guest_exit_cont
1714
1715	/* Synthesize an ISI (or ISegI) for the guest */
1716	mr	r11, r3
17171:	li	r0, BOOK3S_INTERRUPT_INST_STORAGE
17187:	mtspr	SPRN_SRR0, r10
1719	mtspr	SPRN_SRR1, r11
1720	mr	r10, r0
1721	bl	kvmppc_msr_interrupt
1722	b	fast_interrupt_c_return
1723
17243:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
1725	ld	r5, KVM_VRMA_SLB_V(r6)
1726	b	4b
1727
1728/*
1729 * Try to handle an hcall in real mode.
1730 * Returns to the guest if we handle it, or continues on up to
1731 * the kernel if we can't (i.e. if we don't have a handler for
1732 * it, or if the handler returns H_TOO_HARD).
1733 *
1734 * r5 - r8 contain hcall args,
1735 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1736 */
1737hcall_try_real_mode:
1738	ld	r3,VCPU_GPR(R3)(r9)
1739	andi.	r0,r11,MSR_PR
1740	/* sc 1 from userspace - reflect to guest syscall */
1741	bne	sc_1_fast_return
1742	clrrdi	r3,r3,2
1743	cmpldi	r3,hcall_real_table_end - hcall_real_table
1744	bge	guest_exit_cont
1745	/* See if this hcall is enabled for in-kernel handling */
1746	ld	r4, VCPU_KVM(r9)
1747	srdi	r0, r3, 8	/* r0 = (r3 / 4) >> 6 */
1748	sldi	r0, r0, 3	/* index into kvm->arch.enabled_hcalls[] */
1749	add	r4, r4, r0
1750	ld	r0, KVM_ENABLED_HCALLS(r4)
1751	rlwinm	r4, r3, 32-2, 0x3f	/* r4 = (r3 / 4) & 0x3f */
1752	srd	r0, r0, r4
1753	andi.	r0, r0, 1
1754	beq	guest_exit_cont
1755	/* Get pointer to handler, if any, and call it */
1756	LOAD_REG_ADDR(r4, hcall_real_table)
1757	lwax	r3,r3,r4
1758	cmpwi	r3,0
1759	beq	guest_exit_cont
1760	add	r12,r3,r4
1761	mtctr	r12
1762	mr	r3,r9		/* get vcpu pointer */
1763	ld	r4,VCPU_GPR(R4)(r9)
1764	bctrl
1765	cmpdi	r3,H_TOO_HARD
1766	beq	hcall_real_fallback
1767	ld	r4,HSTATE_KVM_VCPU(r13)
1768	std	r3,VCPU_GPR(R3)(r4)
1769	ld	r10,VCPU_PC(r4)
1770	ld	r11,VCPU_MSR(r4)
1771	b	fast_guest_return
1772
1773sc_1_fast_return:
1774	mtspr	SPRN_SRR0,r10
1775	mtspr	SPRN_SRR1,r11
1776	li	r10, BOOK3S_INTERRUPT_SYSCALL
1777	bl	kvmppc_msr_interrupt
1778	mr	r4,r9
1779	b	fast_guest_return
1780
1781	/* We've attempted a real mode hcall, but it's punted it back
1782	 * to userspace.  We need to restore some clobbered volatiles
1783	 * before resuming the pass-it-to-qemu path */
1784hcall_real_fallback:
1785	li	r12,BOOK3S_INTERRUPT_SYSCALL
1786	ld	r9, HSTATE_KVM_VCPU(r13)
1787
1788	b	guest_exit_cont
1789
1790	.globl	hcall_real_table
1791hcall_real_table:
1792	.long	0		/* 0 - unused */
1793	.long	DOTSYM(kvmppc_h_remove) - hcall_real_table
1794	.long	DOTSYM(kvmppc_h_enter) - hcall_real_table
1795	.long	DOTSYM(kvmppc_h_read) - hcall_real_table
1796	.long	DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1797	.long	DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1798	.long	DOTSYM(kvmppc_h_protect) - hcall_real_table
1799	.long	0		/* 0x1c */
1800	.long	0		/* 0x20 */
1801	.long	0		/* 0x24 - H_SET_SPRG0 */
1802	.long	DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1803	.long	DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table
1804	.long	0		/* 0x30 */
1805	.long	0		/* 0x34 */
1806	.long	0		/* 0x38 */
1807	.long	0		/* 0x3c */
1808	.long	0		/* 0x40 */
1809	.long	0		/* 0x44 */
1810	.long	0		/* 0x48 */
1811	.long	0		/* 0x4c */
1812	.long	0		/* 0x50 */
1813	.long	0		/* 0x54 */
1814	.long	0		/* 0x58 */
1815	.long	0		/* 0x5c */
1816	.long	0		/* 0x60 */
1817#ifdef CONFIG_KVM_XICS
1818	.long	DOTSYM(xics_rm_h_eoi) - hcall_real_table
1819	.long	DOTSYM(xics_rm_h_cppr) - hcall_real_table
1820	.long	DOTSYM(xics_rm_h_ipi) - hcall_real_table
1821	.long	0		/* 0x70 - H_IPOLL */
1822	.long	DOTSYM(xics_rm_h_xirr) - hcall_real_table
1823#else
1824	.long	0		/* 0x64 - H_EOI */
1825	.long	0		/* 0x68 - H_CPPR */
1826	.long	0		/* 0x6c - H_IPI */
1827	.long	0		/* 0x70 - H_IPOLL */
1828	.long	0		/* 0x74 - H_XIRR */
1829#endif
1830	.long	0		/* 0x78 */
1831	.long	0		/* 0x7c */
1832	.long	0		/* 0x80 */
1833	.long	0		/* 0x84 */
1834	.long	0		/* 0x88 */
1835	.long	0		/* 0x8c */
1836	.long	0		/* 0x90 */
1837	.long	0		/* 0x94 */
1838	.long	0		/* 0x98 */
1839	.long	0		/* 0x9c */
1840	.long	0		/* 0xa0 */
1841	.long	0		/* 0xa4 */
1842	.long	0		/* 0xa8 */
1843	.long	0		/* 0xac */
1844	.long	0		/* 0xb0 */
1845	.long	0		/* 0xb4 */
1846	.long	0		/* 0xb8 */
1847	.long	0		/* 0xbc */
1848	.long	0		/* 0xc0 */
1849	.long	0		/* 0xc4 */
1850	.long	0		/* 0xc8 */
1851	.long	0		/* 0xcc */
1852	.long	0		/* 0xd0 */
1853	.long	0		/* 0xd4 */
1854	.long	0		/* 0xd8 */
1855	.long	0		/* 0xdc */
1856	.long	DOTSYM(kvmppc_h_cede) - hcall_real_table
1857	.long	DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1858	.long	0		/* 0xe8 */
1859	.long	0		/* 0xec */
1860	.long	0		/* 0xf0 */
1861	.long	0		/* 0xf4 */
1862	.long	0		/* 0xf8 */
1863	.long	0		/* 0xfc */
1864	.long	0		/* 0x100 */
1865	.long	0		/* 0x104 */
1866	.long	0		/* 0x108 */
1867	.long	0		/* 0x10c */
1868	.long	0		/* 0x110 */
1869	.long	0		/* 0x114 */
1870	.long	0		/* 0x118 */
1871	.long	0		/* 0x11c */
1872	.long	0		/* 0x120 */
1873	.long	DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1874	.long	0		/* 0x128 */
1875	.long	0		/* 0x12c */
1876	.long	0		/* 0x130 */
1877	.long	DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1878	.long	0		/* 0x138 */
1879	.long	0		/* 0x13c */
1880	.long	0		/* 0x140 */
1881	.long	0		/* 0x144 */
1882	.long	0		/* 0x148 */
1883	.long	0		/* 0x14c */
1884	.long	0		/* 0x150 */
1885	.long	0		/* 0x154 */
1886	.long	0		/* 0x158 */
1887	.long	0		/* 0x15c */
1888	.long	0		/* 0x160 */
1889	.long	0		/* 0x164 */
1890	.long	0		/* 0x168 */
1891	.long	0		/* 0x16c */
1892	.long	0		/* 0x170 */
1893	.long	0		/* 0x174 */
1894	.long	0		/* 0x178 */
1895	.long	0		/* 0x17c */
1896	.long	0		/* 0x180 */
1897	.long	0		/* 0x184 */
1898	.long	0		/* 0x188 */
1899	.long	0		/* 0x18c */
1900	.long	0		/* 0x190 */
1901	.long	0		/* 0x194 */
1902	.long	0		/* 0x198 */
1903	.long	0		/* 0x19c */
1904	.long	0		/* 0x1a0 */
1905	.long	0		/* 0x1a4 */
1906	.long	0		/* 0x1a8 */
1907	.long	0		/* 0x1ac */
1908	.long	0		/* 0x1b0 */
1909	.long	0		/* 0x1b4 */
1910	.long	0		/* 0x1b8 */
1911	.long	0		/* 0x1bc */
1912	.long	0		/* 0x1c0 */
1913	.long	0		/* 0x1c4 */
1914	.long	0		/* 0x1c8 */
1915	.long	0		/* 0x1cc */
1916	.long	0		/* 0x1d0 */
1917	.long	0		/* 0x1d4 */
1918	.long	0		/* 0x1d8 */
1919	.long	0		/* 0x1dc */
1920	.long	0		/* 0x1e0 */
1921	.long	0		/* 0x1e4 */
1922	.long	0		/* 0x1e8 */
1923	.long	0		/* 0x1ec */
1924	.long	0		/* 0x1f0 */
1925	.long	0		/* 0x1f4 */
1926	.long	0		/* 0x1f8 */
1927	.long	0		/* 0x1fc */
1928	.long	0		/* 0x200 */
1929	.long	0		/* 0x204 */
1930	.long	0		/* 0x208 */
1931	.long	0		/* 0x20c */
1932	.long	0		/* 0x210 */
1933	.long	0		/* 0x214 */
1934	.long	0		/* 0x218 */
1935	.long	0		/* 0x21c */
1936	.long	0		/* 0x220 */
1937	.long	0		/* 0x224 */
1938	.long	0		/* 0x228 */
1939	.long	0		/* 0x22c */
1940	.long	0		/* 0x230 */
1941	.long	0		/* 0x234 */
1942	.long	0		/* 0x238 */
1943	.long	0		/* 0x23c */
1944	.long	0		/* 0x240 */
1945	.long	0		/* 0x244 */
1946	.long	0		/* 0x248 */
1947	.long	0		/* 0x24c */
1948	.long	0		/* 0x250 */
1949	.long	0		/* 0x254 */
1950	.long	0		/* 0x258 */
1951	.long	0		/* 0x25c */
1952	.long	0		/* 0x260 */
1953	.long	0		/* 0x264 */
1954	.long	0		/* 0x268 */
1955	.long	0		/* 0x26c */
1956	.long	0		/* 0x270 */
1957	.long	0		/* 0x274 */
1958	.long	0		/* 0x278 */
1959	.long	0		/* 0x27c */
1960	.long	0		/* 0x280 */
1961	.long	0		/* 0x284 */
1962	.long	0		/* 0x288 */
1963	.long	0		/* 0x28c */
1964	.long	0		/* 0x290 */
1965	.long	0		/* 0x294 */
1966	.long	0		/* 0x298 */
1967	.long	0		/* 0x29c */
1968	.long	0		/* 0x2a0 */
1969	.long	0		/* 0x2a4 */
1970	.long	0		/* 0x2a8 */
1971	.long	0		/* 0x2ac */
1972	.long	0		/* 0x2b0 */
1973	.long	0		/* 0x2b4 */
1974	.long	0		/* 0x2b8 */
1975	.long	0		/* 0x2bc */
1976	.long	0		/* 0x2c0 */
1977	.long	0		/* 0x2c4 */
1978	.long	0		/* 0x2c8 */
1979	.long	0		/* 0x2cc */
1980	.long	0		/* 0x2d0 */
1981	.long	0		/* 0x2d4 */
1982	.long	0		/* 0x2d8 */
1983	.long	0		/* 0x2dc */
1984	.long	0		/* 0x2e0 */
1985	.long	0		/* 0x2e4 */
1986	.long	0		/* 0x2e8 */
1987	.long	0		/* 0x2ec */
1988	.long	0		/* 0x2f0 */
1989	.long	0		/* 0x2f4 */
1990	.long	0		/* 0x2f8 */
1991#ifdef CONFIG_KVM_XICS
1992	.long	DOTSYM(xics_rm_h_xirr_x) - hcall_real_table
1993#else
1994	.long	0		/* 0x2fc - H_XIRR_X*/
1995#endif
1996	.long	DOTSYM(kvmppc_rm_h_random) - hcall_real_table
1997	.globl	hcall_real_table_end
1998hcall_real_table_end:
1999
2000_GLOBAL_TOC(kvmppc_h_set_xdabr)
2001EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr)
2002	andi.	r0, r5, DABRX_USER | DABRX_KERNEL
2003	beq	6f
2004	li	r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2005	andc.	r0, r5, r0
2006	beq	3f
20076:	li	r3, H_PARAMETER
2008	blr
2009
2010_GLOBAL_TOC(kvmppc_h_set_dabr)
2011EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr)
2012	li	r5, DABRX_USER | DABRX_KERNEL
20133:
2014BEGIN_FTR_SECTION
2015	b	2f
2016END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2017	std	r4,VCPU_DABR(r3)
2018	stw	r5, VCPU_DABRX(r3)
2019	mtspr	SPRN_DABRX, r5
2020	/* Work around P7 bug where DABR can get corrupted on mtspr */
20211:	mtspr	SPRN_DABR,r4
2022	mfspr	r5, SPRN_DABR
2023	cmpd	r4, r5
2024	bne	1b
2025	isync
2026	li	r3,0
2027	blr
2028
20292:
2030	LOAD_REG_ADDR(r11, dawr_force_enable)
2031	lbz	r11, 0(r11)
2032	cmpdi	r11, 0
2033	bne	3f
2034	li	r3, H_HARDWARE
2035	blr
20363:
2037	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2038	rlwimi	r5, r4, 5, DAWRX_DR | DAWRX_DW
2039	rlwimi	r5, r4, 2, DAWRX_WT
2040	clrrdi	r4, r4, 3
2041	std	r4, VCPU_DAWR0(r3)
2042	std	r5, VCPU_DAWRX0(r3)
2043	/*
2044	 * If came in through the real mode hcall handler then it is necessary
2045	 * to write the registers since the return path won't. Otherwise it is
2046	 * sufficient to store then in the vcpu struct as they will be loaded
2047	 * next time the vcpu is run.
2048	 */
2049	mfmsr	r6
2050	andi.	r6, r6, MSR_DR		/* in real mode? */
2051	bne	4f
2052	mtspr	SPRN_DAWR0, r4
2053	mtspr	SPRN_DAWRX0, r5
20544:	li	r3, 0
2055	blr
2056
2057_GLOBAL(kvmppc_h_cede)		/* r3 = vcpu pointer, r11 = msr, r13 = paca */
2058	ori	r11,r11,MSR_EE
2059	std	r11,VCPU_MSR(r3)
2060	li	r0,1
2061	stb	r0,VCPU_CEDED(r3)
2062	sync			/* order setting ceded vs. testing prodded */
2063	lbz	r5,VCPU_PRODDED(r3)
2064	cmpwi	r5,0
2065	bne	kvm_cede_prodded
2066	li	r12,0		/* set trap to 0 to say hcall is handled */
2067	stw	r12,VCPU_TRAP(r3)
2068	li	r0,H_SUCCESS
2069	std	r0,VCPU_GPR(R3)(r3)
2070
2071	/*
2072	 * Set our bit in the bitmask of napping threads unless all the
2073	 * other threads are already napping, in which case we send this
2074	 * up to the host.
2075	 */
2076	ld	r5,HSTATE_KVM_VCORE(r13)
2077	lbz	r6,HSTATE_PTID(r13)
2078	lwz	r8,VCORE_ENTRY_EXIT(r5)
2079	clrldi	r8,r8,56
2080	li	r0,1
2081	sld	r0,r0,r6
2082	addi	r6,r5,VCORE_NAPPING_THREADS
208331:	lwarx	r4,0,r6
2084	or	r4,r4,r0
2085	cmpw	r4,r8
2086	beq	kvm_cede_exit
2087	stwcx.	r4,0,r6
2088	bne	31b
2089	/* order napping_threads update vs testing entry_exit_map */
2090	isync
2091	li	r0,NAPPING_CEDE
2092	stb	r0,HSTATE_NAPPING(r13)
2093	lwz	r7,VCORE_ENTRY_EXIT(r5)
2094	cmpwi	r7,0x100
2095	bge	33f		/* another thread already exiting */
2096
2097/*
2098 * Although not specifically required by the architecture, POWER7
2099 * preserves the following registers in nap mode, even if an SMT mode
2100 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2101 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2102 */
2103	/* Save non-volatile GPRs */
2104	std	r14, VCPU_GPR(R14)(r3)
2105	std	r15, VCPU_GPR(R15)(r3)
2106	std	r16, VCPU_GPR(R16)(r3)
2107	std	r17, VCPU_GPR(R17)(r3)
2108	std	r18, VCPU_GPR(R18)(r3)
2109	std	r19, VCPU_GPR(R19)(r3)
2110	std	r20, VCPU_GPR(R20)(r3)
2111	std	r21, VCPU_GPR(R21)(r3)
2112	std	r22, VCPU_GPR(R22)(r3)
2113	std	r23, VCPU_GPR(R23)(r3)
2114	std	r24, VCPU_GPR(R24)(r3)
2115	std	r25, VCPU_GPR(R25)(r3)
2116	std	r26, VCPU_GPR(R26)(r3)
2117	std	r27, VCPU_GPR(R27)(r3)
2118	std	r28, VCPU_GPR(R28)(r3)
2119	std	r29, VCPU_GPR(R29)(r3)
2120	std	r30, VCPU_GPR(R30)(r3)
2121	std	r31, VCPU_GPR(R31)(r3)
2122
2123	/* save FP state */
2124	bl	kvmppc_save_fp
2125
2126#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2127BEGIN_FTR_SECTION
2128	b	91f
2129END_FTR_SECTION_IFCLR(CPU_FTR_TM)
2130	/*
2131	 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
2132	 */
2133	ld	r3, HSTATE_KVM_VCPU(r13)
2134	ld      r4, VCPU_MSR(r3)
2135	li	r5, 0			/* don't preserve non-vol regs */
2136	bl	kvmppc_save_tm_hv
2137	nop
213891:
2139#endif
2140
2141	/*
2142	 * Set DEC to the smaller of DEC and HDEC, so that we wake
2143	 * no later than the end of our timeslice (HDEC interrupts
2144	 * don't wake us from nap).
2145	 */
2146	mfspr	r3, SPRN_DEC
2147	mfspr	r4, SPRN_HDEC
2148	mftb	r5
2149	extsw	r3, r3
2150	extsw	r4, r4
2151	cmpd	r3, r4
2152	ble	67f
2153	mtspr	SPRN_DEC, r4
215467:
2155	/* save expiry time of guest decrementer */
2156	add	r3, r3, r5
2157	ld	r4, HSTATE_KVM_VCPU(r13)
2158	std	r3, VCPU_DEC_EXPIRES(r4)
2159
2160#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2161	ld	r4, HSTATE_KVM_VCPU(r13)
2162	addi	r3, r4, VCPU_TB_CEDE
2163	bl	kvmhv_accumulate_time
2164#endif
2165
2166	lis	r3, LPCR_PECEDP@h	/* Do wake on privileged doorbell */
2167
2168	/* Go back to host stack */
2169	ld	r1, HSTATE_HOST_R1(r13)
2170
2171	/*
2172	 * Take a nap until a decrementer or external or doobell interrupt
2173	 * occurs, with PECE1 and PECE0 set in LPCR.
2174	 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2175	 * Also clear the runlatch bit before napping.
2176	 */
2177kvm_do_nap:
2178	li	r0,0
2179	mtspr	SPRN_CTRLT, r0
2180
2181	li	r0,1
2182	stb	r0,HSTATE_HWTHREAD_REQ(r13)
2183	mfspr	r5,SPRN_LPCR
2184	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
2185BEGIN_FTR_SECTION
2186	ori	r5, r5, LPCR_PECEDH
2187	rlwimi	r5, r3, 0, LPCR_PECEDP
2188END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2189
2190kvm_nap_sequence:		/* desired LPCR value in r5 */
2191	li	r3, PNV_THREAD_NAP
2192	mtspr	SPRN_LPCR,r5
2193	isync
2194
2195	bl	isa206_idle_insn_mayloss
2196
2197	li	r0,1
2198	mtspr	SPRN_CTRLT, r0
2199
2200	mtspr	SPRN_SRR1, r3
2201
2202	li	r0, 0
2203	stb	r0, PACA_FTRACE_ENABLED(r13)
2204
2205	li	r0, KVM_HWTHREAD_IN_KVM
2206	stb	r0, HSTATE_HWTHREAD_STATE(r13)
2207
2208	lbz	r0, HSTATE_NAPPING(r13)
2209	cmpwi	r0, NAPPING_CEDE
2210	beq	kvm_end_cede
2211	cmpwi	r0, NAPPING_NOVCPU
2212	beq	kvm_novcpu_wakeup
2213	cmpwi	r0, NAPPING_UNSPLIT
2214	beq	kvm_unsplit_wakeup
2215	twi	31,0,0 /* Nap state must not be zero */
2216
221733:	mr	r4, r3
2218	li	r3, 0
2219	li	r12, 0
2220	b	34f
2221
2222kvm_end_cede:
2223	/* Woken by external or decrementer interrupt */
2224
2225	/* get vcpu pointer */
2226	ld	r4, HSTATE_KVM_VCPU(r13)
2227
2228#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2229	addi	r3, r4, VCPU_TB_RMINTR
2230	bl	kvmhv_accumulate_time
2231#endif
2232
2233#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2234BEGIN_FTR_SECTION
2235	b	91f
2236END_FTR_SECTION_IFCLR(CPU_FTR_TM)
2237	/*
2238	 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
2239	 */
2240	mr      r3, r4
2241	ld      r4, VCPU_MSR(r3)
2242	li	r5, 0			/* don't preserve non-vol regs */
2243	bl	kvmppc_restore_tm_hv
2244	nop
2245	ld	r4, HSTATE_KVM_VCPU(r13)
224691:
2247#endif
2248
2249	/* load up FP state */
2250	bl	kvmppc_load_fp
2251
2252	/* Restore guest decrementer */
2253	ld	r3, VCPU_DEC_EXPIRES(r4)
2254	mftb	r7
2255	subf	r3, r7, r3
2256	mtspr	SPRN_DEC, r3
2257
2258	/* Load NV GPRS */
2259	ld	r14, VCPU_GPR(R14)(r4)
2260	ld	r15, VCPU_GPR(R15)(r4)
2261	ld	r16, VCPU_GPR(R16)(r4)
2262	ld	r17, VCPU_GPR(R17)(r4)
2263	ld	r18, VCPU_GPR(R18)(r4)
2264	ld	r19, VCPU_GPR(R19)(r4)
2265	ld	r20, VCPU_GPR(R20)(r4)
2266	ld	r21, VCPU_GPR(R21)(r4)
2267	ld	r22, VCPU_GPR(R22)(r4)
2268	ld	r23, VCPU_GPR(R23)(r4)
2269	ld	r24, VCPU_GPR(R24)(r4)
2270	ld	r25, VCPU_GPR(R25)(r4)
2271	ld	r26, VCPU_GPR(R26)(r4)
2272	ld	r27, VCPU_GPR(R27)(r4)
2273	ld	r28, VCPU_GPR(R28)(r4)
2274	ld	r29, VCPU_GPR(R29)(r4)
2275	ld	r30, VCPU_GPR(R30)(r4)
2276	ld	r31, VCPU_GPR(R31)(r4)
2277
2278	/* Check the wake reason in SRR1 to see why we got here */
2279	bl	kvmppc_check_wake_reason
2280
2281	/*
2282	 * Restore volatile registers since we could have called a
2283	 * C routine in kvmppc_check_wake_reason
2284	 *	r4 = VCPU
2285	 * r3 tells us whether we need to return to host or not
2286	 * WARNING: it gets checked further down:
2287	 * should not modify r3 until this check is done.
2288	 */
2289	ld	r4, HSTATE_KVM_VCPU(r13)
2290
2291	/* clear our bit in vcore->napping_threads */
229234:	ld	r5,HSTATE_KVM_VCORE(r13)
2293	lbz	r7,HSTATE_PTID(r13)
2294	li	r0,1
2295	sld	r0,r0,r7
2296	addi	r6,r5,VCORE_NAPPING_THREADS
229732:	lwarx	r7,0,r6
2298	andc	r7,r7,r0
2299	stwcx.	r7,0,r6
2300	bne	32b
2301	li	r0,0
2302	stb	r0,HSTATE_NAPPING(r13)
2303
2304	/* See if the wake reason saved in r3 means we need to exit */
2305	stw	r12, VCPU_TRAP(r4)
2306	mr	r9, r4
2307	cmpdi	r3, 0
2308	bgt	guest_exit_cont
2309	b	maybe_reenter_guest
2310
2311	/* cede when already previously prodded case */
2312kvm_cede_prodded:
2313	li	r0,0
2314	stb	r0,VCPU_PRODDED(r3)
2315	sync			/* order testing prodded vs. clearing ceded */
2316	stb	r0,VCPU_CEDED(r3)
2317	li	r3,H_SUCCESS
2318	blr
2319
2320	/* we've ceded but we want to give control to the host */
2321kvm_cede_exit:
2322	ld	r9, HSTATE_KVM_VCPU(r13)
2323	b	guest_exit_cont
2324
2325	/* Try to do machine check recovery in real mode */
2326machine_check_realmode:
2327	mr	r3, r9		/* get vcpu pointer */
2328	bl	kvmppc_realmode_machine_check
2329	nop
2330	/* all machine checks go to virtual mode for further handling */
2331	ld	r9, HSTATE_KVM_VCPU(r13)
2332	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2333	b	guest_exit_cont
2334
2335/*
2336 * Call C code to handle a HMI in real mode.
2337 * Only the primary thread does the call, secondary threads are handled
2338 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns.
2339 * r9 points to the vcpu on entry
2340 */
2341hmi_realmode:
2342	lbz	r0, HSTATE_PTID(r13)
2343	cmpwi	r0, 0
2344	bne	guest_exit_cont
2345	bl	kvmppc_realmode_hmi_handler
2346	ld	r9, HSTATE_KVM_VCPU(r13)
2347	li	r12, BOOK3S_INTERRUPT_HMI
2348	b	guest_exit_cont
2349
2350/*
2351 * Check the reason we woke from nap, and take appropriate action.
2352 * Returns (in r3):
2353 *	0 if nothing needs to be done
2354 *	1 if something happened that needs to be handled by the host
2355 *	-1 if there was a guest wakeup (IPI or msgsnd)
2356 *	-2 if we handled a PCI passthrough interrupt (returned by
2357 *		kvmppc_read_intr only)
2358 *
2359 * Also sets r12 to the interrupt vector for any interrupt that needs
2360 * to be handled now by the host (0x500 for external interrupt), or zero.
2361 * Modifies all volatile registers (since it may call a C function).
2362 * This routine calls kvmppc_read_intr, a C function, if an external
2363 * interrupt is pending.
2364 */
2365SYM_FUNC_START_LOCAL(kvmppc_check_wake_reason)
2366	mfspr	r6, SPRN_SRR1
2367BEGIN_FTR_SECTION
2368	rlwinm	r6, r6, 45-31, 0xf	/* extract wake reason field (P8) */
2369FTR_SECTION_ELSE
2370	rlwinm	r6, r6, 45-31, 0xe	/* P7 wake reason field is 3 bits */
2371ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2372	cmpwi	r6, 8			/* was it an external interrupt? */
2373	beq	7f			/* if so, see what it was */
2374	li	r3, 0
2375	li	r12, 0
2376	cmpwi	r6, 6			/* was it the decrementer? */
2377	beq	0f
2378BEGIN_FTR_SECTION
2379	cmpwi	r6, 5			/* privileged doorbell? */
2380	beq	0f
2381	cmpwi	r6, 3			/* hypervisor doorbell? */
2382	beq	3f
2383END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2384	cmpwi	r6, 0xa			/* Hypervisor maintenance ? */
2385	beq	4f
2386	li	r3, 1			/* anything else, return 1 */
23870:	blr
2388
2389	/* hypervisor doorbell */
23903:	li	r12, BOOK3S_INTERRUPT_H_DOORBELL
2391
2392	/*
2393	 * Clear the doorbell as we will invoke the handler
2394	 * explicitly in the guest exit path.
2395	 */
2396	lis	r6, (PPC_DBELL_SERVER << (63-36))@h
2397	PPC_MSGCLR(6)
2398	/* see if it's a host IPI */
2399	li	r3, 1
2400	lbz	r0, HSTATE_HOST_IPI(r13)
2401	cmpwi	r0, 0
2402	bnelr
2403	/* if not, return -1 */
2404	li	r3, -1
2405	blr
2406
2407	/* Woken up due to Hypervisor maintenance interrupt */
24084:	li	r12, BOOK3S_INTERRUPT_HMI
2409	li	r3, 1
2410	blr
2411
2412	/* external interrupt - create a stack frame so we can call C */
24137:	mflr	r0
2414	std	r0, PPC_LR_STKOFF(r1)
2415	stdu	r1, -PPC_MIN_STKFRM(r1)
2416	bl	kvmppc_read_intr
2417	nop
2418	li	r12, BOOK3S_INTERRUPT_EXTERNAL
2419	cmpdi	r3, 1
2420	ble	1f
2421
2422	/*
2423	 * Return code of 2 means PCI passthrough interrupt, but
2424	 * we need to return back to host to complete handling the
2425	 * interrupt. Trap reason is expected in r12 by guest
2426	 * exit code.
2427	 */
2428	li	r12, BOOK3S_INTERRUPT_HV_RM_HARD
24291:
2430	ld	r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
2431	addi	r1, r1, PPC_MIN_STKFRM
2432	mtlr	r0
2433	blr
2434SYM_FUNC_END(kvmppc_check_wake_reason)
2435
2436/*
2437 * Save away FP, VMX and VSX registers.
2438 * r3 = vcpu pointer
2439 * N.B. r30 and r31 are volatile across this function,
2440 * thus it is not callable from C.
2441 */
2442SYM_FUNC_START_LOCAL(kvmppc_save_fp)
2443	mflr	r30
2444	mr	r31,r3
2445	mfmsr	r5
2446	ori	r8,r5,MSR_FP
2447#ifdef CONFIG_ALTIVEC
2448BEGIN_FTR_SECTION
2449	oris	r8,r8,MSR_VEC@h
2450END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2451#endif
2452#ifdef CONFIG_VSX
2453BEGIN_FTR_SECTION
2454	oris	r8,r8,MSR_VSX@h
2455END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2456#endif
2457	mtmsrd	r8
2458	addi	r3,r3,VCPU_FPRS
2459	bl	store_fp_state
2460#ifdef CONFIG_ALTIVEC
2461BEGIN_FTR_SECTION
2462	addi	r3,r31,VCPU_VRS
2463	bl	store_vr_state
2464END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2465#endif
2466	mfspr	r6,SPRN_VRSAVE
2467	stw	r6,VCPU_VRSAVE(r31)
2468	mtlr	r30
2469	blr
2470SYM_FUNC_END(kvmppc_save_fp)
2471
2472/*
2473 * Load up FP, VMX and VSX registers
2474 * r4 = vcpu pointer
2475 * N.B. r30 and r31 are volatile across this function,
2476 * thus it is not callable from C.
2477 */
2478SYM_FUNC_START_LOCAL(kvmppc_load_fp)
2479	mflr	r30
2480	mr	r31,r4
2481	mfmsr	r9
2482	ori	r8,r9,MSR_FP
2483#ifdef CONFIG_ALTIVEC
2484BEGIN_FTR_SECTION
2485	oris	r8,r8,MSR_VEC@h
2486END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2487#endif
2488#ifdef CONFIG_VSX
2489BEGIN_FTR_SECTION
2490	oris	r8,r8,MSR_VSX@h
2491END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2492#endif
2493	mtmsrd	r8
2494	addi	r3,r4,VCPU_FPRS
2495	bl	load_fp_state
2496#ifdef CONFIG_ALTIVEC
2497BEGIN_FTR_SECTION
2498	addi	r3,r31,VCPU_VRS
2499	bl	load_vr_state
2500END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2501#endif
2502	lwz	r7,VCPU_VRSAVE(r31)
2503	mtspr	SPRN_VRSAVE,r7
2504	mtlr	r30
2505	mr	r4,r31
2506	blr
2507SYM_FUNC_END(kvmppc_load_fp)
2508
2509#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2510/*
2511 * Save transactional state and TM-related registers.
2512 * Called with r3 pointing to the vcpu struct and r4 containing
2513 * the guest MSR value.
2514 * r5 is non-zero iff non-volatile register state needs to be maintained.
2515 * If r5 == 0, this can modify all checkpointed registers, but
2516 * restores r1 and r2 before exit.
2517 */
2518_GLOBAL_TOC(kvmppc_save_tm_hv)
2519EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv)
2520	/* See if we need to handle fake suspend mode */
2521BEGIN_FTR_SECTION
2522	b	__kvmppc_save_tm
2523END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
2524
2525	lbz	r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */
2526	cmpwi	r0, 0
2527	beq	__kvmppc_save_tm
2528
2529	/* The following code handles the fake_suspend = 1 case */
2530	mflr	r0
2531	std	r0, PPC_LR_STKOFF(r1)
2532	stdu	r1, -TM_FRAME_SIZE(r1)
2533
2534	/* Turn on TM. */
2535	mfmsr	r8
2536	li	r0, 1
2537	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
2538	mtmsrd	r8
2539
2540	rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */
2541	beq	4f
2542BEGIN_FTR_SECTION
2543	bl	pnv_power9_force_smt4_catch
2544END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
2545	nop
2546
2547	/*
2548	 * It's possible that treclaim. may modify registers, if we have lost
2549	 * track of fake-suspend state in the guest due to it using rfscv.
2550	 * Save and restore registers in case this occurs.
2551	 */
2552	mfspr	r3, SPRN_DSCR
2553	mfspr	r4, SPRN_XER
2554	mfspr	r5, SPRN_AMR
2555	/* SPRN_TAR would need to be saved here if the kernel ever used it */
2556	mfcr	r12
2557	SAVE_NVGPRS(r1)
2558	SAVE_GPR(2, r1)
2559	SAVE_GPR(3, r1)
2560	SAVE_GPR(4, r1)
2561	SAVE_GPR(5, r1)
2562	stw	r12, 8(r1)
2563	std	r1, HSTATE_HOST_R1(r13)
2564
2565	/* We have to treclaim here because that's the only way to do S->N */
2566	li	r3, TM_CAUSE_KVM_RESCHED
2567	TRECLAIM(R3)
2568
2569	GET_PACA(r13)
2570	ld	r1, HSTATE_HOST_R1(r13)
2571	REST_GPR(2, r1)
2572	REST_GPR(3, r1)
2573	REST_GPR(4, r1)
2574	REST_GPR(5, r1)
2575	lwz	r12, 8(r1)
2576	REST_NVGPRS(r1)
2577	mtspr	SPRN_DSCR, r3
2578	mtspr	SPRN_XER, r4
2579	mtspr	SPRN_AMR, r5
2580	mtcr	r12
2581	HMT_MEDIUM
2582
2583	/*
2584	 * We were in fake suspend, so we are not going to save the
2585	 * register state as the guest checkpointed state (since
2586	 * we already have it), therefore we can now use any volatile GPR.
2587	 * In fact treclaim in fake suspend state doesn't modify
2588	 * any registers.
2589	 */
2590
2591BEGIN_FTR_SECTION
2592	bl	pnv_power9_force_smt4_release
2593END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
2594	nop
2595
25964:
2597	mfspr	r3, SPRN_PSSCR
2598	/* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */
2599	li	r0, PSSCR_FAKE_SUSPEND
2600	andc	r3, r3, r0
2601	mtspr	SPRN_PSSCR, r3
2602
2603	/* Don't save TEXASR, use value from last exit in real suspend state */
2604	ld	r9, HSTATE_KVM_VCPU(r13)
2605	mfspr	r5, SPRN_TFHAR
2606	mfspr	r6, SPRN_TFIAR
2607	std	r5, VCPU_TFHAR(r9)
2608	std	r6, VCPU_TFIAR(r9)
2609
2610	addi	r1, r1, TM_FRAME_SIZE
2611	ld	r0, PPC_LR_STKOFF(r1)
2612	mtlr	r0
2613	blr
2614
2615/*
2616 * Restore transactional state and TM-related registers.
2617 * Called with r3 pointing to the vcpu struct
2618 * and r4 containing the guest MSR value.
2619 * r5 is non-zero iff non-volatile register state needs to be maintained.
2620 * This potentially modifies all checkpointed registers.
2621 * It restores r1 and r2 from the PACA.
2622 */
2623_GLOBAL_TOC(kvmppc_restore_tm_hv)
2624EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv)
2625	/*
2626	 * If we are doing TM emulation for the guest on a POWER9 DD2,
2627	 * then we don't actually do a trechkpt -- we either set up
2628	 * fake-suspend mode, or emulate a TM rollback.
2629	 */
2630BEGIN_FTR_SECTION
2631	b	__kvmppc_restore_tm
2632END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
2633	mflr	r0
2634	std	r0, PPC_LR_STKOFF(r1)
2635
2636	li	r0, 0
2637	stb	r0, HSTATE_FAKE_SUSPEND(r13)
2638
2639	/* Turn on TM so we can restore TM SPRs */
2640	mfmsr	r5
2641	li	r0, 1
2642	rldimi	r5, r0, MSR_TM_LG, 63-MSR_TM_LG
2643	mtmsrd	r5
2644
2645	/*
2646	 * The user may change these outside of a transaction, so they must
2647	 * always be context switched.
2648	 */
2649	ld	r5, VCPU_TFHAR(r3)
2650	ld	r6, VCPU_TFIAR(r3)
2651	ld	r7, VCPU_TEXASR(r3)
2652	mtspr	SPRN_TFHAR, r5
2653	mtspr	SPRN_TFIAR, r6
2654	mtspr	SPRN_TEXASR, r7
2655
2656	rldicl. r5, r4, 64 - MSR_TS_S_LG, 62
2657	beqlr		/* TM not active in guest */
2658
2659	/* Make sure the failure summary is set */
2660	oris	r7, r7, (TEXASR_FS)@h
2661	mtspr	SPRN_TEXASR, r7
2662
2663	cmpwi	r5, 1		/* check for suspended state */
2664	bgt	10f
2665	stb	r5, HSTATE_FAKE_SUSPEND(r13)
2666	b	9f		/* and return */
266710:	stdu	r1, -PPC_MIN_STKFRM(r1)
2668	/* guest is in transactional state, so simulate rollback */
2669	bl	kvmhv_emulate_tm_rollback
2670	nop
2671	addi	r1, r1, PPC_MIN_STKFRM
26729:	ld	r0, PPC_LR_STKOFF(r1)
2673	mtlr	r0
2674	blr
2675#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
2676
2677/*
2678 * We come here if we get any exception or interrupt while we are
2679 * executing host real mode code while in guest MMU context.
2680 * r12 is (CR << 32) | vector
2681 * r13 points to our PACA
2682 * r12 is saved in HSTATE_SCRATCH0(r13)
2683 * r9 is saved in HSTATE_SCRATCH2(r13)
2684 * r13 is saved in HSPRG1
2685 * cfar is saved in HSTATE_CFAR(r13)
2686 * ppr is saved in HSTATE_PPR(r13)
2687 */
2688kvmppc_bad_host_intr:
2689	/*
2690	 * Switch to the emergency stack, but start half-way down in
2691	 * case we were already on it.
2692	 */
2693	mr	r9, r1
2694	std	r1, PACAR1(r13)
2695	ld	r1, PACAEMERGSP(r13)
2696	subi	r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE
2697	std	r9, 0(r1)
2698	std	r0, GPR0(r1)
2699	std	r9, GPR1(r1)
2700	std	r2, GPR2(r1)
2701	SAVE_GPRS(3, 8, r1)
2702	srdi	r0, r12, 32
2703	clrldi	r12, r12, 32
2704	std	r0, _CCR(r1)
2705	std	r12, _TRAP(r1)
2706	andi.	r0, r12, 2
2707	beq	1f
2708	mfspr	r3, SPRN_HSRR0
2709	mfspr	r4, SPRN_HSRR1
2710	mfspr	r5, SPRN_HDAR
2711	mfspr	r6, SPRN_HDSISR
2712	b	2f
27131:	mfspr	r3, SPRN_SRR0
2714	mfspr	r4, SPRN_SRR1
2715	mfspr	r5, SPRN_DAR
2716	mfspr	r6, SPRN_DSISR
27172:	std	r3, _NIP(r1)
2718	std	r4, _MSR(r1)
2719	std	r5, _DAR(r1)
2720	std	r6, _DSISR(r1)
2721	ld	r9, HSTATE_SCRATCH2(r13)
2722	ld	r12, HSTATE_SCRATCH0(r13)
2723	GET_SCRATCH0(r0)
2724	SAVE_GPRS(9, 12, r1)
2725	std	r0, GPR13(r1)
2726	SAVE_NVGPRS(r1)
2727	ld	r5, HSTATE_CFAR(r13)
2728	std	r5, ORIG_GPR3(r1)
2729	mflr	r3
2730	mfctr	r4
2731	mfxer	r5
2732	lbz	r6, PACAIRQSOFTMASK(r13)
2733	std	r3, _LINK(r1)
2734	std	r4, _CTR(r1)
2735	std	r5, _XER(r1)
2736	std	r6, SOFTE(r1)
2737	LOAD_PACA_TOC()
2738	LOAD_REG_IMMEDIATE(3, STACK_FRAME_REGS_MARKER)
2739	std	r3, STACK_INT_FRAME_MARKER(r1)
2740
2741	/*
2742	 * XXX On POWER7 and POWER8, we just spin here since we don't
2743	 * know what the other threads are doing (and we don't want to
2744	 * coordinate with them) - but at least we now have register state
2745	 * in memory that we might be able to look at from another CPU.
2746	 */
2747	b	.
2748
2749/*
2750 * This mimics the MSR transition on IRQ delivery.  The new guest MSR is taken
2751 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2752 *   r11 has the guest MSR value (in/out)
2753 *   r9 has a vcpu pointer (in)
2754 *   r0 is used as a scratch register
2755 */
2756SYM_FUNC_START_LOCAL(kvmppc_msr_interrupt)
2757	rldicl	r0, r11, 64 - MSR_TS_S_LG, 62
2758	cmpwi	r0, 2 /* Check if we are in transactional state..  */
2759	ld	r11, VCPU_INTR_MSR(r9)
2760	bne	1f
2761	/* ... if transactional, change to suspended */
2762	li	r0, 1
27631:	rldimi	r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2764	blr
2765SYM_FUNC_END(kvmppc_msr_interrupt)
2766
2767/*
2768 * void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu)
2769 *
2770 * Load up guest PMU state.  R3 points to the vcpu struct.
2771 */
2772SYM_FUNC_START_LOCAL(kvmhv_load_guest_pmu)
2773	mr	r4, r3
2774	mflr	r0
2775	li	r3, 1
2776	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
2777	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
2778	isync
2779BEGIN_FTR_SECTION
2780	ld	r3, VCPU_MMCR(r4)
2781	andi.	r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
2782	cmpwi	r5, MMCR0_PMAO
2783	beql	kvmppc_fix_pmao
2784END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
2785	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
2786	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
2787	lwz	r6, VCPU_PMC + 8(r4)
2788	lwz	r7, VCPU_PMC + 12(r4)
2789	lwz	r8, VCPU_PMC + 16(r4)
2790	lwz	r9, VCPU_PMC + 20(r4)
2791	mtspr	SPRN_PMC1, r3
2792	mtspr	SPRN_PMC2, r5
2793	mtspr	SPRN_PMC3, r6
2794	mtspr	SPRN_PMC4, r7
2795	mtspr	SPRN_PMC5, r8
2796	mtspr	SPRN_PMC6, r9
2797	ld	r3, VCPU_MMCR(r4)
2798	ld	r5, VCPU_MMCR + 8(r4)
2799	ld	r6, VCPU_MMCRA(r4)
2800	ld	r7, VCPU_SIAR(r4)
2801	ld	r8, VCPU_SDAR(r4)
2802	mtspr	SPRN_MMCR1, r5
2803	mtspr	SPRN_MMCRA, r6
2804	mtspr	SPRN_SIAR, r7
2805	mtspr	SPRN_SDAR, r8
2806BEGIN_FTR_SECTION
2807	ld	r5, VCPU_MMCR + 16(r4)
2808	ld	r6, VCPU_SIER(r4)
2809	mtspr	SPRN_MMCR2, r5
2810	mtspr	SPRN_SIER, r6
2811	lwz	r7, VCPU_PMC + 24(r4)
2812	lwz	r8, VCPU_PMC + 28(r4)
2813	ld	r9, VCPU_MMCRS(r4)
2814	mtspr	SPRN_SPMC1, r7
2815	mtspr	SPRN_SPMC2, r8
2816	mtspr	SPRN_MMCRS, r9
2817END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2818	mtspr	SPRN_MMCR0, r3
2819	isync
2820	mtlr	r0
2821	blr
2822SYM_FUNC_END(kvmhv_load_guest_pmu)
2823
2824/*
2825 * void kvmhv_load_host_pmu(void)
2826 *
2827 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu.
2828 */
2829SYM_FUNC_START_LOCAL(kvmhv_load_host_pmu)
2830	mflr	r0
2831	lbz	r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */
2832	cmpwi	r4, 0
2833	beq	23f			/* skip if not */
2834BEGIN_FTR_SECTION
2835	ld	r3, HSTATE_MMCR0(r13)
2836	andi.	r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
2837	cmpwi	r4, MMCR0_PMAO
2838	beql	kvmppc_fix_pmao
2839END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
2840	lwz	r3, HSTATE_PMC1(r13)
2841	lwz	r4, HSTATE_PMC2(r13)
2842	lwz	r5, HSTATE_PMC3(r13)
2843	lwz	r6, HSTATE_PMC4(r13)
2844	lwz	r8, HSTATE_PMC5(r13)
2845	lwz	r9, HSTATE_PMC6(r13)
2846	mtspr	SPRN_PMC1, r3
2847	mtspr	SPRN_PMC2, r4
2848	mtspr	SPRN_PMC3, r5
2849	mtspr	SPRN_PMC4, r6
2850	mtspr	SPRN_PMC5, r8
2851	mtspr	SPRN_PMC6, r9
2852	ld	r3, HSTATE_MMCR0(r13)
2853	ld	r4, HSTATE_MMCR1(r13)
2854	ld	r5, HSTATE_MMCRA(r13)
2855	ld	r6, HSTATE_SIAR(r13)
2856	ld	r7, HSTATE_SDAR(r13)
2857	mtspr	SPRN_MMCR1, r4
2858	mtspr	SPRN_MMCRA, r5
2859	mtspr	SPRN_SIAR, r6
2860	mtspr	SPRN_SDAR, r7
2861BEGIN_FTR_SECTION
2862	ld	r8, HSTATE_MMCR2(r13)
2863	ld	r9, HSTATE_SIER(r13)
2864	mtspr	SPRN_MMCR2, r8
2865	mtspr	SPRN_SIER, r9
2866END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2867	mtspr	SPRN_MMCR0, r3
2868	isync
2869	mtlr	r0
287023:	blr
2871SYM_FUNC_END(kvmhv_load_host_pmu)
2872
2873/*
2874 * void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use)
2875 *
2876 * Save guest PMU state into the vcpu struct.
2877 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA)
2878 */
2879SYM_FUNC_START_LOCAL(kvmhv_save_guest_pmu)
2880	mr	r9, r3
2881	mr	r8, r4
2882BEGIN_FTR_SECTION
2883	/*
2884	 * POWER8 seems to have a hardware bug where setting
2885	 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
2886	 * when some counters are already negative doesn't seem
2887	 * to cause a performance monitor alert (and hence interrupt).
2888	 * The effect of this is that when saving the PMU state,
2889	 * if there is no PMU alert pending when we read MMCR0
2890	 * before freezing the counters, but one becomes pending
2891	 * before we read the counters, we lose it.
2892	 * To work around this, we need a way to freeze the counters
2893	 * before reading MMCR0.  Normally, freezing the counters
2894	 * is done by writing MMCR0 (to set MMCR0[FC]) which
2895	 * unavoidably writes MMCR0[PMA0] as well.  On POWER8,
2896	 * we can also freeze the counters using MMCR2, by writing
2897	 * 1s to all the counter freeze condition bits (there are
2898	 * 9 bits each for 6 counters).
2899	 */
2900	li	r3, -1			/* set all freeze bits */
2901	clrrdi	r3, r3, 10
2902	mfspr	r10, SPRN_MMCR2
2903	mtspr	SPRN_MMCR2, r3
2904	isync
2905END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2906	li	r3, 1
2907	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
2908	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
2909	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
2910	mfspr	r6, SPRN_MMCRA
2911	/* Clear MMCRA in order to disable SDAR updates */
2912	li	r7, 0
2913	mtspr	SPRN_MMCRA, r7
2914	isync
2915	cmpwi	r8, 0			/* did they ask for PMU stuff to be saved? */
2916	bne	21f
2917	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
2918	b	22f
291921:	mfspr	r5, SPRN_MMCR1
2920	mfspr	r7, SPRN_SIAR
2921	mfspr	r8, SPRN_SDAR
2922	std	r4, VCPU_MMCR(r9)
2923	std	r5, VCPU_MMCR + 8(r9)
2924	std	r6, VCPU_MMCRA(r9)
2925BEGIN_FTR_SECTION
2926	std	r10, VCPU_MMCR + 16(r9)
2927END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2928	std	r7, VCPU_SIAR(r9)
2929	std	r8, VCPU_SDAR(r9)
2930	mfspr	r3, SPRN_PMC1
2931	mfspr	r4, SPRN_PMC2
2932	mfspr	r5, SPRN_PMC3
2933	mfspr	r6, SPRN_PMC4
2934	mfspr	r7, SPRN_PMC5
2935	mfspr	r8, SPRN_PMC6
2936	stw	r3, VCPU_PMC(r9)
2937	stw	r4, VCPU_PMC + 4(r9)
2938	stw	r5, VCPU_PMC + 8(r9)
2939	stw	r6, VCPU_PMC + 12(r9)
2940	stw	r7, VCPU_PMC + 16(r9)
2941	stw	r8, VCPU_PMC + 20(r9)
2942BEGIN_FTR_SECTION
2943	mfspr	r5, SPRN_SIER
2944	std	r5, VCPU_SIER(r9)
2945	mfspr	r6, SPRN_SPMC1
2946	mfspr	r7, SPRN_SPMC2
2947	mfspr	r8, SPRN_MMCRS
2948	stw	r6, VCPU_PMC + 24(r9)
2949	stw	r7, VCPU_PMC + 28(r9)
2950	std	r8, VCPU_MMCRS(r9)
2951	lis	r4, 0x8000
2952	mtspr	SPRN_MMCRS, r4
2953END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
295422:	blr
2955SYM_FUNC_END(kvmhv_save_guest_pmu)
2956
2957/*
2958 * This works around a hardware bug on POWER8E processors, where
2959 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2960 * performance monitor interrupt.  Instead, when we need to have
2961 * an interrupt pending, we have to arrange for a counter to overflow.
2962 */
2963kvmppc_fix_pmao:
2964	li	r3, 0
2965	mtspr	SPRN_MMCR2, r3
2966	lis	r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2967	ori	r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2968	mtspr	SPRN_MMCR0, r3
2969	lis	r3, 0x7fff
2970	ori	r3, r3, 0xffff
2971	mtspr	SPRN_PMC6, r3
2972	isync
2973	blr
2974
2975#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2976/*
2977 * Start timing an activity
2978 * r3 = pointer to time accumulation struct, r4 = vcpu
2979 */
2980kvmhv_start_timing:
2981	ld	r5, HSTATE_KVM_VCORE(r13)
2982	ld	r6, VCORE_TB_OFFSET_APPL(r5)
2983	mftb	r5
2984	subf	r5, r6, r5	/* subtract current timebase offset */
2985	std	r3, VCPU_CUR_ACTIVITY(r4)
2986	std	r5, VCPU_ACTIVITY_START(r4)
2987	blr
2988
2989/*
2990 * Accumulate time to one activity and start another.
2991 * r3 = pointer to new time accumulation struct, r4 = vcpu
2992 */
2993kvmhv_accumulate_time:
2994	ld	r5, HSTATE_KVM_VCORE(r13)
2995	ld	r8, VCORE_TB_OFFSET_APPL(r5)
2996	ld	r5, VCPU_CUR_ACTIVITY(r4)
2997	ld	r6, VCPU_ACTIVITY_START(r4)
2998	std	r3, VCPU_CUR_ACTIVITY(r4)
2999	mftb	r7
3000	subf	r7, r8, r7	/* subtract current timebase offset */
3001	std	r7, VCPU_ACTIVITY_START(r4)
3002	cmpdi	r5, 0
3003	beqlr
3004	subf	r3, r6, r7
3005	ld	r8, TAS_SEQCOUNT(r5)
3006	cmpdi	r8, 0
3007	addi	r8, r8, 1
3008	std	r8, TAS_SEQCOUNT(r5)
3009	lwsync
3010	ld	r7, TAS_TOTAL(r5)
3011	add	r7, r7, r3
3012	std	r7, TAS_TOTAL(r5)
3013	ld	r6, TAS_MIN(r5)
3014	ld	r7, TAS_MAX(r5)
3015	beq	3f
3016	cmpd	r3, r6
3017	bge	1f
30183:	std	r3, TAS_MIN(r5)
30191:	cmpd	r3, r7
3020	ble	2f
3021	std	r3, TAS_MAX(r5)
30222:	lwsync
3023	addi	r8, r8, 1
3024	std	r8, TAS_SEQCOUNT(r5)
3025	blr
3026#endif
3027