1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/arm-smccc.h> 8 #include <linux/preempt.h> 9 #include <linux/kvm_host.h> 10 #include <linux/uaccess.h> 11 #include <linux/wait.h> 12 13 #include <asm/cputype.h> 14 #include <asm/kvm_emulate.h> 15 16 #include <kvm/arm_psci.h> 17 #include <kvm/arm_hypercalls.h> 18 19 /* 20 * This is an implementation of the Power State Coordination Interface 21 * as described in ARM document number ARM DEN 0022A. 22 */ 23 24 #define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) 25 26 static unsigned long psci_affinity_mask(unsigned long affinity_level) 27 { 28 if (affinity_level <= 3) 29 return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level); 30 31 return 0; 32 } 33 34 static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) 35 { 36 /* 37 * NOTE: For simplicity, we make VCPU suspend emulation to be 38 * same-as WFI (Wait-for-interrupt) emulation. 39 * 40 * This means for KVM the wakeup events are interrupts and 41 * this is consistent with intended use of StateID as described 42 * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A). 43 * 44 * Further, we also treat power-down request to be same as 45 * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2 46 * specification (ARM DEN 0022A). This means all suspend states 47 * for KVM will preserve the register state. 48 */ 49 kvm_vcpu_wfi(vcpu); 50 51 return PSCI_RET_SUCCESS; 52 } 53 54 static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, 55 unsigned long affinity) 56 { 57 return !(affinity & ~MPIDR_HWID_BITMASK); 58 } 59 60 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) 61 { 62 struct vcpu_reset_state *reset_state; 63 struct kvm *kvm = source_vcpu->kvm; 64 struct kvm_vcpu *vcpu = NULL; 65 unsigned long cpu_id; 66 67 cpu_id = smccc_get_arg1(source_vcpu); 68 if (!kvm_psci_valid_affinity(source_vcpu, cpu_id)) 69 return PSCI_RET_INVALID_PARAMS; 70 71 vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id); 72 73 /* 74 * Make sure the caller requested a valid CPU and that the CPU is 75 * turned off. 76 */ 77 if (!vcpu) 78 return PSCI_RET_INVALID_PARAMS; 79 if (!kvm_arm_vcpu_stopped(vcpu)) { 80 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) 81 return PSCI_RET_ALREADY_ON; 82 else 83 return PSCI_RET_INVALID_PARAMS; 84 } 85 86 reset_state = &vcpu->arch.reset_state; 87 88 reset_state->pc = smccc_get_arg2(source_vcpu); 89 90 /* Propagate caller endianness */ 91 reset_state->be = kvm_vcpu_is_be(source_vcpu); 92 93 /* 94 * NOTE: We always update r0 (or x0) because for PSCI v0.1 95 * the general purpose registers are undefined upon CPU_ON. 96 */ 97 reset_state->r0 = smccc_get_arg3(source_vcpu); 98 99 WRITE_ONCE(reset_state->reset, true); 100 kvm_make_request(KVM_REQ_VCPU_RESET, vcpu); 101 102 /* 103 * Make sure the reset request is observed if the RUNNABLE mp_state is 104 * observed. 105 */ 106 smp_wmb(); 107 108 vcpu->arch.mp_state.mp_state = KVM_MP_STATE_RUNNABLE; 109 kvm_vcpu_wake_up(vcpu); 110 111 return PSCI_RET_SUCCESS; 112 } 113 114 static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) 115 { 116 int matching_cpus = 0; 117 unsigned long i, mpidr; 118 unsigned long target_affinity; 119 unsigned long target_affinity_mask; 120 unsigned long lowest_affinity_level; 121 struct kvm *kvm = vcpu->kvm; 122 struct kvm_vcpu *tmp; 123 124 target_affinity = smccc_get_arg1(vcpu); 125 lowest_affinity_level = smccc_get_arg2(vcpu); 126 127 if (!kvm_psci_valid_affinity(vcpu, target_affinity)) 128 return PSCI_RET_INVALID_PARAMS; 129 130 /* Determine target affinity mask */ 131 target_affinity_mask = psci_affinity_mask(lowest_affinity_level); 132 if (!target_affinity_mask) 133 return PSCI_RET_INVALID_PARAMS; 134 135 /* Ignore other bits of target affinity */ 136 target_affinity &= target_affinity_mask; 137 138 /* 139 * If one or more VCPU matching target affinity are running 140 * then ON else OFF 141 */ 142 kvm_for_each_vcpu(i, tmp, kvm) { 143 mpidr = kvm_vcpu_get_mpidr_aff(tmp); 144 if ((mpidr & target_affinity_mask) == target_affinity) { 145 matching_cpus++; 146 if (!kvm_arm_vcpu_stopped(tmp)) 147 return PSCI_0_2_AFFINITY_LEVEL_ON; 148 } 149 } 150 151 if (!matching_cpus) 152 return PSCI_RET_INVALID_PARAMS; 153 154 return PSCI_0_2_AFFINITY_LEVEL_OFF; 155 } 156 157 static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) 158 { 159 unsigned long i; 160 struct kvm_vcpu *tmp; 161 162 /* 163 * The KVM ABI specifies that a system event exit may call KVM_RUN 164 * again and may perform shutdown/reboot at a later time that when the 165 * actual request is made. Since we are implementing PSCI and a 166 * caller of PSCI reboot and shutdown expects that the system shuts 167 * down or reboots immediately, let's make sure that VCPUs are not run 168 * after this call is handled and before the VCPUs have been 169 * re-initialized. 170 */ 171 kvm_for_each_vcpu(i, tmp, vcpu->kvm) 172 tmp->arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; 173 kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); 174 175 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); 176 vcpu->run->system_event.type = type; 177 vcpu->run->system_event.ndata = 1; 178 vcpu->run->system_event.data[0] = flags; 179 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 180 } 181 182 static void kvm_psci_system_off(struct kvm_vcpu *vcpu) 183 { 184 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0); 185 } 186 187 static void kvm_psci_system_reset(struct kvm_vcpu *vcpu) 188 { 189 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0); 190 } 191 192 static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu) 193 { 194 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 195 KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2); 196 } 197 198 static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) 199 { 200 struct kvm_run *run = vcpu->run; 201 202 memset(&run->system_event, 0, sizeof(vcpu->run->system_event)); 203 run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND; 204 run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 205 } 206 207 static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) 208 { 209 int i; 210 211 /* 212 * Zero the input registers' upper 32 bits. They will be fully 213 * zeroed on exit, so we're fine changing them in place. 214 */ 215 for (i = 1; i < 4; i++) 216 vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); 217 } 218 219 static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) 220 { 221 /* 222 * Prevent 32 bit guests from calling 64 bit PSCI functions. 223 */ 224 if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu)) 225 return PSCI_RET_NOT_SUPPORTED; 226 227 return 0; 228 } 229 230 static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) 231 { 232 struct kvm *kvm = vcpu->kvm; 233 u32 psci_fn = smccc_get_function(vcpu); 234 unsigned long val; 235 int ret = 1; 236 237 switch (psci_fn) { 238 case PSCI_0_2_FN_PSCI_VERSION: 239 /* 240 * Bits[31:16] = Major Version = 0 241 * Bits[15:0] = Minor Version = 2 242 */ 243 val = KVM_ARM_PSCI_0_2; 244 break; 245 case PSCI_0_2_FN_CPU_SUSPEND: 246 case PSCI_0_2_FN64_CPU_SUSPEND: 247 val = kvm_psci_vcpu_suspend(vcpu); 248 break; 249 case PSCI_0_2_FN_CPU_OFF: 250 kvm_arm_vcpu_power_off(vcpu); 251 val = PSCI_RET_SUCCESS; 252 break; 253 case PSCI_0_2_FN_CPU_ON: 254 kvm_psci_narrow_to_32bit(vcpu); 255 fallthrough; 256 case PSCI_0_2_FN64_CPU_ON: 257 mutex_lock(&kvm->lock); 258 val = kvm_psci_vcpu_on(vcpu); 259 mutex_unlock(&kvm->lock); 260 break; 261 case PSCI_0_2_FN_AFFINITY_INFO: 262 kvm_psci_narrow_to_32bit(vcpu); 263 fallthrough; 264 case PSCI_0_2_FN64_AFFINITY_INFO: 265 val = kvm_psci_vcpu_affinity_info(vcpu); 266 break; 267 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 268 /* 269 * Trusted OS is MP hence does not require migration 270 * or 271 * Trusted OS is not present 272 */ 273 val = PSCI_0_2_TOS_MP; 274 break; 275 case PSCI_0_2_FN_SYSTEM_OFF: 276 kvm_psci_system_off(vcpu); 277 /* 278 * We shouldn't be going back to guest VCPU after 279 * receiving SYSTEM_OFF request. 280 * 281 * If user space accidentally/deliberately resumes 282 * guest VCPU after SYSTEM_OFF request then guest 283 * VCPU should see internal failure from PSCI return 284 * value. To achieve this, we preload r0 (or x0) with 285 * PSCI return value INTERNAL_FAILURE. 286 */ 287 val = PSCI_RET_INTERNAL_FAILURE; 288 ret = 0; 289 break; 290 case PSCI_0_2_FN_SYSTEM_RESET: 291 kvm_psci_system_reset(vcpu); 292 /* 293 * Same reason as SYSTEM_OFF for preloading r0 (or x0) 294 * with PSCI return value INTERNAL_FAILURE. 295 */ 296 val = PSCI_RET_INTERNAL_FAILURE; 297 ret = 0; 298 break; 299 default: 300 val = PSCI_RET_NOT_SUPPORTED; 301 break; 302 } 303 304 smccc_set_retval(vcpu, val, 0, 0, 0); 305 return ret; 306 } 307 308 static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) 309 { 310 unsigned long val = PSCI_RET_NOT_SUPPORTED; 311 u32 psci_fn = smccc_get_function(vcpu); 312 struct kvm *kvm = vcpu->kvm; 313 u32 arg; 314 int ret = 1; 315 316 switch(psci_fn) { 317 case PSCI_0_2_FN_PSCI_VERSION: 318 val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1; 319 break; 320 case PSCI_1_0_FN_PSCI_FEATURES: 321 arg = smccc_get_arg1(vcpu); 322 val = kvm_psci_check_allowed_function(vcpu, arg); 323 if (val) 324 break; 325 326 val = PSCI_RET_NOT_SUPPORTED; 327 328 switch(arg) { 329 case PSCI_0_2_FN_PSCI_VERSION: 330 case PSCI_0_2_FN_CPU_SUSPEND: 331 case PSCI_0_2_FN64_CPU_SUSPEND: 332 case PSCI_0_2_FN_CPU_OFF: 333 case PSCI_0_2_FN_CPU_ON: 334 case PSCI_0_2_FN64_CPU_ON: 335 case PSCI_0_2_FN_AFFINITY_INFO: 336 case PSCI_0_2_FN64_AFFINITY_INFO: 337 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 338 case PSCI_0_2_FN_SYSTEM_OFF: 339 case PSCI_0_2_FN_SYSTEM_RESET: 340 case PSCI_1_0_FN_PSCI_FEATURES: 341 case ARM_SMCCC_VERSION_FUNC_ID: 342 val = 0; 343 break; 344 case PSCI_1_0_FN_SYSTEM_SUSPEND: 345 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 346 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) 347 val = 0; 348 break; 349 case PSCI_1_1_FN_SYSTEM_RESET2: 350 case PSCI_1_1_FN64_SYSTEM_RESET2: 351 if (minor >= 1) 352 val = 0; 353 break; 354 } 355 break; 356 case PSCI_1_0_FN_SYSTEM_SUSPEND: 357 kvm_psci_narrow_to_32bit(vcpu); 358 fallthrough; 359 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 360 /* 361 * Return directly to userspace without changing the vCPU's 362 * registers. Userspace depends on reading the SMCCC parameters 363 * to implement SYSTEM_SUSPEND. 364 */ 365 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) { 366 kvm_psci_system_suspend(vcpu); 367 return 0; 368 } 369 break; 370 case PSCI_1_1_FN_SYSTEM_RESET2: 371 kvm_psci_narrow_to_32bit(vcpu); 372 fallthrough; 373 case PSCI_1_1_FN64_SYSTEM_RESET2: 374 if (minor >= 1) { 375 arg = smccc_get_arg1(vcpu); 376 377 if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET || 378 arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) { 379 kvm_psci_system_reset2(vcpu); 380 vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE); 381 return 0; 382 } 383 384 val = PSCI_RET_INVALID_PARAMS; 385 break; 386 } 387 break; 388 default: 389 return kvm_psci_0_2_call(vcpu); 390 } 391 392 smccc_set_retval(vcpu, val, 0, 0, 0); 393 return ret; 394 } 395 396 static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) 397 { 398 struct kvm *kvm = vcpu->kvm; 399 u32 psci_fn = smccc_get_function(vcpu); 400 unsigned long val; 401 402 switch (psci_fn) { 403 case KVM_PSCI_FN_CPU_OFF: 404 kvm_arm_vcpu_power_off(vcpu); 405 val = PSCI_RET_SUCCESS; 406 break; 407 case KVM_PSCI_FN_CPU_ON: 408 mutex_lock(&kvm->lock); 409 val = kvm_psci_vcpu_on(vcpu); 410 mutex_unlock(&kvm->lock); 411 break; 412 default: 413 val = PSCI_RET_NOT_SUPPORTED; 414 break; 415 } 416 417 smccc_set_retval(vcpu, val, 0, 0, 0); 418 return 1; 419 } 420 421 /** 422 * kvm_psci_call - handle PSCI call if r0 value is in range 423 * @vcpu: Pointer to the VCPU struct 424 * 425 * Handle PSCI calls from guests through traps from HVC instructions. 426 * The calling convention is similar to SMC calls to the secure world 427 * where the function number is placed in r0. 428 * 429 * This function returns: > 0 (success), 0 (success but exit to user 430 * space), and < 0 (errors) 431 * 432 * Errors: 433 * -EINVAL: Unrecognized PSCI function 434 */ 435 int kvm_psci_call(struct kvm_vcpu *vcpu) 436 { 437 u32 psci_fn = smccc_get_function(vcpu); 438 unsigned long val; 439 440 val = kvm_psci_check_allowed_function(vcpu, psci_fn); 441 if (val) { 442 smccc_set_retval(vcpu, val, 0, 0, 0); 443 return 1; 444 } 445 446 switch (kvm_psci_version(vcpu)) { 447 case KVM_ARM_PSCI_1_1: 448 return kvm_psci_1_x_call(vcpu, 1); 449 case KVM_ARM_PSCI_1_0: 450 return kvm_psci_1_x_call(vcpu, 0); 451 case KVM_ARM_PSCI_0_2: 452 return kvm_psci_0_2_call(vcpu); 453 case KVM_ARM_PSCI_0_1: 454 return kvm_psci_0_1_call(vcpu); 455 default: 456 return -EINVAL; 457 } 458 } 459