1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <linux/arm-smccc.h>
8 #include <linux/preempt.h>
9 #include <linux/kvm_host.h>
10 #include <linux/uaccess.h>
11 #include <linux/wait.h>
12
13 #include <asm/cputype.h>
14 #include <asm/kvm_emulate.h>
15
16 #include <kvm/arm_psci.h>
17 #include <kvm/arm_hypercalls.h>
18
19 /*
20 * This is an implementation of the Power State Coordination Interface
21 * as described in ARM document number ARM DEN 0022A.
22 */
23
24 #define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
25
psci_affinity_mask(unsigned long affinity_level)26 static unsigned long psci_affinity_mask(unsigned long affinity_level)
27 {
28 if (affinity_level <= 3)
29 return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level);
30
31 return 0;
32 }
33
kvm_psci_vcpu_suspend(struct kvm_vcpu * vcpu)34 static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu)
35 {
36 /*
37 * NOTE: For simplicity, we make VCPU suspend emulation to be
38 * same-as WFI (Wait-for-interrupt) emulation.
39 *
40 * This means for KVM the wakeup events are interrupts and
41 * this is consistent with intended use of StateID as described
42 * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A).
43 *
44 * Further, we also treat power-down request to be same as
45 * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2
46 * specification (ARM DEN 0022A). This means all suspend states
47 * for KVM will preserve the register state.
48 */
49 kvm_vcpu_wfi(vcpu);
50
51 return PSCI_RET_SUCCESS;
52 }
53
kvm_psci_valid_affinity(struct kvm_vcpu * vcpu,unsigned long affinity)54 static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu,
55 unsigned long affinity)
56 {
57 return !(affinity & ~MPIDR_HWID_BITMASK);
58 }
59
kvm_psci_vcpu_on(struct kvm_vcpu * source_vcpu)60 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
61 {
62 struct vcpu_reset_state *reset_state;
63 struct kvm *kvm = source_vcpu->kvm;
64 struct kvm_vcpu *vcpu = NULL;
65 int ret = PSCI_RET_SUCCESS;
66 unsigned long cpu_id;
67
68 cpu_id = smccc_get_arg1(source_vcpu);
69 if (!kvm_psci_valid_affinity(source_vcpu, cpu_id))
70 return PSCI_RET_INVALID_PARAMS;
71
72 vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
73
74 /*
75 * Make sure the caller requested a valid CPU and that the CPU is
76 * turned off.
77 */
78 if (!vcpu)
79 return PSCI_RET_INVALID_PARAMS;
80
81 spin_lock(&vcpu->arch.mp_state_lock);
82 if (!kvm_arm_vcpu_stopped(vcpu)) {
83 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
84 ret = PSCI_RET_ALREADY_ON;
85 else
86 ret = PSCI_RET_INVALID_PARAMS;
87
88 goto out_unlock;
89 }
90
91 reset_state = &vcpu->arch.reset_state;
92
93 reset_state->pc = smccc_get_arg2(source_vcpu);
94
95 /* Propagate caller endianness */
96 reset_state->be = kvm_vcpu_is_be(source_vcpu);
97
98 /*
99 * NOTE: We always update r0 (or x0) because for PSCI v0.1
100 * the general purpose registers are undefined upon CPU_ON.
101 */
102 reset_state->r0 = smccc_get_arg3(source_vcpu);
103
104 reset_state->reset = true;
105 kvm_make_request(KVM_REQ_VCPU_RESET, vcpu);
106
107 /*
108 * Make sure the reset request is observed if the RUNNABLE mp_state is
109 * observed.
110 */
111 smp_wmb();
112
113 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
114 kvm_vcpu_wake_up(vcpu);
115
116 out_unlock:
117 spin_unlock(&vcpu->arch.mp_state_lock);
118 return ret;
119 }
120
kvm_psci_vcpu_affinity_info(struct kvm_vcpu * vcpu)121 static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
122 {
123 int matching_cpus = 0;
124 unsigned long i, mpidr;
125 unsigned long target_affinity;
126 unsigned long target_affinity_mask;
127 unsigned long lowest_affinity_level;
128 struct kvm *kvm = vcpu->kvm;
129 struct kvm_vcpu *tmp;
130
131 target_affinity = smccc_get_arg1(vcpu);
132 lowest_affinity_level = smccc_get_arg2(vcpu);
133
134 if (!kvm_psci_valid_affinity(vcpu, target_affinity))
135 return PSCI_RET_INVALID_PARAMS;
136
137 /* Determine target affinity mask */
138 target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
139 if (!target_affinity_mask)
140 return PSCI_RET_INVALID_PARAMS;
141
142 /* Ignore other bits of target affinity */
143 target_affinity &= target_affinity_mask;
144
145 /*
146 * If one or more VCPU matching target affinity are running
147 * then ON else OFF
148 */
149 kvm_for_each_vcpu(i, tmp, kvm) {
150 mpidr = kvm_vcpu_get_mpidr_aff(tmp);
151 if ((mpidr & target_affinity_mask) == target_affinity) {
152 matching_cpus++;
153 if (!kvm_arm_vcpu_stopped(tmp))
154 return PSCI_0_2_AFFINITY_LEVEL_ON;
155 }
156 }
157
158 if (!matching_cpus)
159 return PSCI_RET_INVALID_PARAMS;
160
161 return PSCI_0_2_AFFINITY_LEVEL_OFF;
162 }
163
kvm_prepare_system_event(struct kvm_vcpu * vcpu,u32 type,u64 flags)164 static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags)
165 {
166 unsigned long i;
167 struct kvm_vcpu *tmp;
168
169 /*
170 * The KVM ABI specifies that a system event exit may call KVM_RUN
171 * again and may perform shutdown/reboot at a later time that when the
172 * actual request is made. Since we are implementing PSCI and a
173 * caller of PSCI reboot and shutdown expects that the system shuts
174 * down or reboots immediately, let's make sure that VCPUs are not run
175 * after this call is handled and before the VCPUs have been
176 * re-initialized.
177 */
178 kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
179 spin_lock(&tmp->arch.mp_state_lock);
180 WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
181 spin_unlock(&tmp->arch.mp_state_lock);
182 }
183 kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
184
185 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
186 vcpu->run->system_event.type = type;
187 vcpu->run->system_event.ndata = 1;
188 vcpu->run->system_event.data[0] = flags;
189 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
190 }
191
kvm_psci_system_off(struct kvm_vcpu * vcpu)192 static void kvm_psci_system_off(struct kvm_vcpu *vcpu)
193 {
194 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0);
195 }
196
kvm_psci_system_reset(struct kvm_vcpu * vcpu)197 static void kvm_psci_system_reset(struct kvm_vcpu *vcpu)
198 {
199 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0);
200 }
201
kvm_psci_system_reset2(struct kvm_vcpu * vcpu)202 static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu)
203 {
204 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET,
205 KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2);
206 }
207
kvm_psci_system_suspend(struct kvm_vcpu * vcpu)208 static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu)
209 {
210 struct kvm_run *run = vcpu->run;
211
212 memset(&run->system_event, 0, sizeof(vcpu->run->system_event));
213 run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND;
214 run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
215 }
216
kvm_psci_narrow_to_32bit(struct kvm_vcpu * vcpu)217 static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu)
218 {
219 int i;
220
221 /*
222 * Zero the input registers' upper 32 bits. They will be fully
223 * zeroed on exit, so we're fine changing them in place.
224 */
225 for (i = 1; i < 4; i++)
226 vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i)));
227 }
228
kvm_psci_check_allowed_function(struct kvm_vcpu * vcpu,u32 fn)229 static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn)
230 {
231 /*
232 * Prevent 32 bit guests from calling 64 bit PSCI functions.
233 */
234 if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu))
235 return PSCI_RET_NOT_SUPPORTED;
236
237 return 0;
238 }
239
kvm_psci_0_2_call(struct kvm_vcpu * vcpu)240 static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
241 {
242 u32 psci_fn = smccc_get_function(vcpu);
243 unsigned long val;
244 int ret = 1;
245
246 switch (psci_fn) {
247 case PSCI_0_2_FN_PSCI_VERSION:
248 /*
249 * Bits[31:16] = Major Version = 0
250 * Bits[15:0] = Minor Version = 2
251 */
252 val = KVM_ARM_PSCI_0_2;
253 break;
254 case PSCI_0_2_FN_CPU_SUSPEND:
255 case PSCI_0_2_FN64_CPU_SUSPEND:
256 val = kvm_psci_vcpu_suspend(vcpu);
257 break;
258 case PSCI_0_2_FN_CPU_OFF:
259 kvm_arm_vcpu_power_off(vcpu);
260 val = PSCI_RET_SUCCESS;
261 break;
262 case PSCI_0_2_FN_CPU_ON:
263 kvm_psci_narrow_to_32bit(vcpu);
264 fallthrough;
265 case PSCI_0_2_FN64_CPU_ON:
266 val = kvm_psci_vcpu_on(vcpu);
267 break;
268 case PSCI_0_2_FN_AFFINITY_INFO:
269 kvm_psci_narrow_to_32bit(vcpu);
270 fallthrough;
271 case PSCI_0_2_FN64_AFFINITY_INFO:
272 val = kvm_psci_vcpu_affinity_info(vcpu);
273 break;
274 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
275 /*
276 * Trusted OS is MP hence does not require migration
277 * or
278 * Trusted OS is not present
279 */
280 val = PSCI_0_2_TOS_MP;
281 break;
282 case PSCI_0_2_FN_SYSTEM_OFF:
283 kvm_psci_system_off(vcpu);
284 /*
285 * We shouldn't be going back to guest VCPU after
286 * receiving SYSTEM_OFF request.
287 *
288 * If user space accidentally/deliberately resumes
289 * guest VCPU after SYSTEM_OFF request then guest
290 * VCPU should see internal failure from PSCI return
291 * value. To achieve this, we preload r0 (or x0) with
292 * PSCI return value INTERNAL_FAILURE.
293 */
294 val = PSCI_RET_INTERNAL_FAILURE;
295 ret = 0;
296 break;
297 case PSCI_0_2_FN_SYSTEM_RESET:
298 kvm_psci_system_reset(vcpu);
299 /*
300 * Same reason as SYSTEM_OFF for preloading r0 (or x0)
301 * with PSCI return value INTERNAL_FAILURE.
302 */
303 val = PSCI_RET_INTERNAL_FAILURE;
304 ret = 0;
305 break;
306 default:
307 val = PSCI_RET_NOT_SUPPORTED;
308 break;
309 }
310
311 smccc_set_retval(vcpu, val, 0, 0, 0);
312 return ret;
313 }
314
kvm_psci_1_x_call(struct kvm_vcpu * vcpu,u32 minor)315 static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor)
316 {
317 unsigned long val = PSCI_RET_NOT_SUPPORTED;
318 u32 psci_fn = smccc_get_function(vcpu);
319 struct kvm *kvm = vcpu->kvm;
320 u32 arg;
321 int ret = 1;
322
323 switch(psci_fn) {
324 case PSCI_0_2_FN_PSCI_VERSION:
325 val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1;
326 break;
327 case PSCI_1_0_FN_PSCI_FEATURES:
328 arg = smccc_get_arg1(vcpu);
329 val = kvm_psci_check_allowed_function(vcpu, arg);
330 if (val)
331 break;
332
333 val = PSCI_RET_NOT_SUPPORTED;
334
335 switch(arg) {
336 case PSCI_0_2_FN_PSCI_VERSION:
337 case PSCI_0_2_FN_CPU_SUSPEND:
338 case PSCI_0_2_FN64_CPU_SUSPEND:
339 case PSCI_0_2_FN_CPU_OFF:
340 case PSCI_0_2_FN_CPU_ON:
341 case PSCI_0_2_FN64_CPU_ON:
342 case PSCI_0_2_FN_AFFINITY_INFO:
343 case PSCI_0_2_FN64_AFFINITY_INFO:
344 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
345 case PSCI_0_2_FN_SYSTEM_OFF:
346 case PSCI_0_2_FN_SYSTEM_RESET:
347 case PSCI_1_0_FN_PSCI_FEATURES:
348 case ARM_SMCCC_VERSION_FUNC_ID:
349 val = 0;
350 break;
351 case PSCI_1_0_FN_SYSTEM_SUSPEND:
352 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
353 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags))
354 val = 0;
355 break;
356 case PSCI_1_1_FN_SYSTEM_RESET2:
357 case PSCI_1_1_FN64_SYSTEM_RESET2:
358 if (minor >= 1)
359 val = 0;
360 break;
361 }
362 break;
363 case PSCI_1_0_FN_SYSTEM_SUSPEND:
364 kvm_psci_narrow_to_32bit(vcpu);
365 fallthrough;
366 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
367 /*
368 * Return directly to userspace without changing the vCPU's
369 * registers. Userspace depends on reading the SMCCC parameters
370 * to implement SYSTEM_SUSPEND.
371 */
372 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) {
373 kvm_psci_system_suspend(vcpu);
374 return 0;
375 }
376 break;
377 case PSCI_1_1_FN_SYSTEM_RESET2:
378 kvm_psci_narrow_to_32bit(vcpu);
379 fallthrough;
380 case PSCI_1_1_FN64_SYSTEM_RESET2:
381 if (minor >= 1) {
382 arg = smccc_get_arg1(vcpu);
383
384 if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET ||
385 arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) {
386 kvm_psci_system_reset2(vcpu);
387 vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE);
388 return 0;
389 }
390
391 val = PSCI_RET_INVALID_PARAMS;
392 break;
393 }
394 break;
395 default:
396 return kvm_psci_0_2_call(vcpu);
397 }
398
399 smccc_set_retval(vcpu, val, 0, 0, 0);
400 return ret;
401 }
402
kvm_psci_0_1_call(struct kvm_vcpu * vcpu)403 static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
404 {
405 u32 psci_fn = smccc_get_function(vcpu);
406 unsigned long val;
407
408 switch (psci_fn) {
409 case KVM_PSCI_FN_CPU_OFF:
410 kvm_arm_vcpu_power_off(vcpu);
411 val = PSCI_RET_SUCCESS;
412 break;
413 case KVM_PSCI_FN_CPU_ON:
414 val = kvm_psci_vcpu_on(vcpu);
415 break;
416 default:
417 val = PSCI_RET_NOT_SUPPORTED;
418 break;
419 }
420
421 smccc_set_retval(vcpu, val, 0, 0, 0);
422 return 1;
423 }
424
425 /**
426 * kvm_psci_call - handle PSCI call if r0 value is in range
427 * @vcpu: Pointer to the VCPU struct
428 *
429 * Handle PSCI calls from guests through traps from HVC instructions.
430 * The calling convention is similar to SMC calls to the secure world
431 * where the function number is placed in r0.
432 *
433 * This function returns: > 0 (success), 0 (success but exit to user
434 * space), and < 0 (errors)
435 *
436 * Errors:
437 * -EINVAL: Unrecognized PSCI function
438 */
kvm_psci_call(struct kvm_vcpu * vcpu)439 int kvm_psci_call(struct kvm_vcpu *vcpu)
440 {
441 u32 psci_fn = smccc_get_function(vcpu);
442 int version = kvm_psci_version(vcpu);
443 unsigned long val;
444
445 val = kvm_psci_check_allowed_function(vcpu, psci_fn);
446 if (val) {
447 smccc_set_retval(vcpu, val, 0, 0, 0);
448 return 1;
449 }
450
451 switch (version) {
452 case KVM_ARM_PSCI_1_1:
453 return kvm_psci_1_x_call(vcpu, 1);
454 case KVM_ARM_PSCI_1_0:
455 return kvm_psci_1_x_call(vcpu, 0);
456 case KVM_ARM_PSCI_0_2:
457 return kvm_psci_0_2_call(vcpu);
458 case KVM_ARM_PSCI_0_1:
459 return kvm_psci_0_1_call(vcpu);
460 default:
461 WARN_ONCE(1, "Unknown PSCI version %d", version);
462 smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0);
463 return 1;
464 }
465 }
466