xref: /openbmc/linux/arch/arm64/kvm/hyp/nvhe/switch.c (revision f4c3b83b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2015 - ARM Ltd
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <hyp/adjust_pc.h>
8 #include <hyp/switch.h>
9 #include <hyp/sysreg-sr.h>
10 
11 #include <linux/arm-smccc.h>
12 #include <linux/kvm_host.h>
13 #include <linux/types.h>
14 #include <linux/jump_label.h>
15 #include <uapi/linux/psci.h>
16 
17 #include <kvm/arm_psci.h>
18 
19 #include <asm/barrier.h>
20 #include <asm/cpufeature.h>
21 #include <asm/kprobes.h>
22 #include <asm/kvm_asm.h>
23 #include <asm/kvm_emulate.h>
24 #include <asm/kvm_hyp.h>
25 #include <asm/kvm_mmu.h>
26 #include <asm/fpsimd.h>
27 #include <asm/debug-monitors.h>
28 #include <asm/processor.h>
29 #include <asm/thread_info.h>
30 
31 /* Non-VHE specific context */
32 DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
33 DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
34 DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
35 
36 static void __activate_traps(struct kvm_vcpu *vcpu)
37 {
38 	u64 val;
39 
40 	___activate_traps(vcpu);
41 	__activate_traps_common(vcpu);
42 
43 	val = CPTR_EL2_DEFAULT;
44 	val |= CPTR_EL2_TTA | CPTR_EL2_TZ | CPTR_EL2_TAM;
45 	if (!update_fp_enabled(vcpu)) {
46 		val |= CPTR_EL2_TFP;
47 		__activate_traps_fpsimd32(vcpu);
48 	}
49 
50 	write_sysreg(val, cptr_el2);
51 	write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2);
52 
53 	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
54 		struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
55 
56 		isb();
57 		/*
58 		 * At this stage, and thanks to the above isb(), S2 is
59 		 * configured and enabled. We can now restore the guest's S1
60 		 * configuration: SCTLR, and only then TCR.
61 		 */
62 		write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1),	SYS_SCTLR);
63 		isb();
64 		write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1),	SYS_TCR);
65 	}
66 }
67 
68 static void __deactivate_traps(struct kvm_vcpu *vcpu)
69 {
70 	extern char __kvm_hyp_host_vector[];
71 	u64 mdcr_el2;
72 
73 	___deactivate_traps(vcpu);
74 
75 	mdcr_el2 = read_sysreg(mdcr_el2);
76 
77 	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
78 		u64 val;
79 
80 		/*
81 		 * Set the TCR and SCTLR registers in the exact opposite
82 		 * sequence as __activate_traps (first prevent walks,
83 		 * then force the MMU on). A generous sprinkling of isb()
84 		 * ensure that things happen in this exact order.
85 		 */
86 		val = read_sysreg_el1(SYS_TCR);
87 		write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR);
88 		isb();
89 		val = read_sysreg_el1(SYS_SCTLR);
90 		write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR);
91 		isb();
92 	}
93 
94 	__deactivate_traps_common();
95 
96 	mdcr_el2 &= MDCR_EL2_HPMN_MASK;
97 	mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
98 
99 	write_sysreg(mdcr_el2, mdcr_el2);
100 	if (is_protected_kvm_enabled())
101 		write_sysreg(HCR_HOST_NVHE_PROTECTED_FLAGS, hcr_el2);
102 	else
103 		write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2);
104 	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
105 	write_sysreg(__kvm_hyp_host_vector, vbar_el2);
106 }
107 
108 static void __load_host_stage2(void)
109 {
110 	write_sysreg(0, vttbr_el2);
111 }
112 
113 /* Save VGICv3 state on non-VHE systems */
114 static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
115 {
116 	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
117 		__vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
118 		__vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
119 	}
120 }
121 
122 /* Restore VGICv3 state on non_VEH systems */
123 static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
124 {
125 	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
126 		__vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
127 		__vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
128 	}
129 }
130 
131 /**
132  * Disable host events, enable guest events
133  */
134 static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt)
135 {
136 	struct kvm_host_data *host;
137 	struct kvm_pmu_events *pmu;
138 
139 	host = container_of(host_ctxt, struct kvm_host_data, host_ctxt);
140 	pmu = &host->pmu_events;
141 
142 	if (pmu->events_host)
143 		write_sysreg(pmu->events_host, pmcntenclr_el0);
144 
145 	if (pmu->events_guest)
146 		write_sysreg(pmu->events_guest, pmcntenset_el0);
147 
148 	return (pmu->events_host || pmu->events_guest);
149 }
150 
151 /**
152  * Disable guest events, enable host events
153  */
154 static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt)
155 {
156 	struct kvm_host_data *host;
157 	struct kvm_pmu_events *pmu;
158 
159 	host = container_of(host_ctxt, struct kvm_host_data, host_ctxt);
160 	pmu = &host->pmu_events;
161 
162 	if (pmu->events_guest)
163 		write_sysreg(pmu->events_guest, pmcntenclr_el0);
164 
165 	if (pmu->events_host)
166 		write_sysreg(pmu->events_host, pmcntenset_el0);
167 }
168 
169 /* Switch to the guest for legacy non-VHE systems */
170 int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
171 {
172 	struct kvm_cpu_context *host_ctxt;
173 	struct kvm_cpu_context *guest_ctxt;
174 	bool pmu_switch_needed;
175 	u64 exit_code;
176 
177 	/*
178 	 * Having IRQs masked via PMR when entering the guest means the GIC
179 	 * will not signal the CPU of interrupts of lower priority, and the
180 	 * only way to get out will be via guest exceptions.
181 	 * Naturally, we want to avoid this.
182 	 */
183 	if (system_uses_irq_prio_masking()) {
184 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
185 		pmr_sync();
186 	}
187 
188 	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
189 	host_ctxt->__hyp_running_vcpu = vcpu;
190 	guest_ctxt = &vcpu->arch.ctxt;
191 
192 	pmu_switch_needed = __pmu_switch_to_guest(host_ctxt);
193 
194 	__sysreg_save_state_nvhe(host_ctxt);
195 
196 	__adjust_pc(vcpu);
197 
198 	/*
199 	 * We must restore the 32-bit state before the sysregs, thanks
200 	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
201 	 *
202 	 * Also, and in order to be able to deal with erratum #1319537 (A57)
203 	 * and #1319367 (A72), we must ensure that all VM-related sysreg are
204 	 * restored before we enable S2 translation.
205 	 */
206 	__sysreg32_restore_state(vcpu);
207 	__sysreg_restore_state_nvhe(guest_ctxt);
208 
209 	__load_guest_stage2(kern_hyp_va(vcpu->arch.hw_mmu));
210 	__activate_traps(vcpu);
211 
212 	__hyp_vgic_restore_state(vcpu);
213 	__timer_enable_traps(vcpu);
214 
215 	__debug_switch_to_guest(vcpu);
216 
217 	do {
218 		/* Jump in the fire! */
219 		exit_code = __guest_enter(vcpu);
220 
221 		/* And we're baaack! */
222 	} while (fixup_guest_exit(vcpu, &exit_code));
223 
224 	__sysreg_save_state_nvhe(guest_ctxt);
225 	__sysreg32_save_state(vcpu);
226 	__timer_disable_traps(vcpu);
227 	__hyp_vgic_save_state(vcpu);
228 
229 	__deactivate_traps(vcpu);
230 	__load_host_stage2();
231 
232 	__sysreg_restore_state_nvhe(host_ctxt);
233 
234 	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
235 		__fpsimd_save_fpexc32(vcpu);
236 
237 	/*
238 	 * This must come after restoring the host sysregs, since a non-VHE
239 	 * system may enable SPE here and make use of the TTBRs.
240 	 */
241 	__debug_switch_to_host(vcpu);
242 
243 	if (pmu_switch_needed)
244 		__pmu_switch_to_host(host_ctxt);
245 
246 	/* Returning to host will clear PSR.I, remask PMR if needed */
247 	if (system_uses_irq_prio_masking())
248 		gic_write_pmr(GIC_PRIO_IRQOFF);
249 
250 	host_ctxt->__hyp_running_vcpu = NULL;
251 
252 	return exit_code;
253 }
254 
255 void __noreturn hyp_panic(void)
256 {
257 	u64 spsr = read_sysreg_el2(SYS_SPSR);
258 	u64 elr = read_sysreg_el2(SYS_ELR);
259 	u64 par = read_sysreg_par();
260 	bool restore_host = true;
261 	struct kvm_cpu_context *host_ctxt;
262 	struct kvm_vcpu *vcpu;
263 
264 	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
265 	vcpu = host_ctxt->__hyp_running_vcpu;
266 
267 	if (vcpu) {
268 		__timer_disable_traps(vcpu);
269 		__deactivate_traps(vcpu);
270 		__load_host_stage2();
271 		__sysreg_restore_state_nvhe(host_ctxt);
272 	}
273 
274 	__hyp_do_panic(restore_host, spsr, elr, par);
275 	unreachable();
276 }
277 
278 asmlinkage void kvm_unexpected_el2_exception(void)
279 {
280 	return __kvm_unexpected_el2_exception();
281 }
282