1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - Google LLC 4 * Author: David Brazdil <dbrazdil@google.com> 5 */ 6 7 #include <asm/kvm_asm.h> 8 #include <asm/kvm_hyp.h> 9 #include <asm/kvm_mmu.h> 10 #include <linux/arm-smccc.h> 11 #include <linux/kvm_host.h> 12 #include <uapi/linux/psci.h> 13 14 #include <nvhe/trap_handler.h> 15 16 void kvm_hyp_cpu_entry(unsigned long r0); 17 void kvm_hyp_cpu_resume(unsigned long r0); 18 19 void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); 20 21 /* Config options set by the host. */ 22 struct kvm_host_psci_config __ro_after_init kvm_host_psci_config; 23 s64 __ro_after_init hyp_physvirt_offset; 24 25 #define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset) 26 27 #define INVALID_CPU_ID UINT_MAX 28 29 struct psci_boot_args { 30 atomic_t lock; 31 unsigned long pc; 32 unsigned long r0; 33 }; 34 35 #define PSCI_BOOT_ARGS_UNLOCKED 0 36 #define PSCI_BOOT_ARGS_LOCKED 1 37 38 #define PSCI_BOOT_ARGS_INIT \ 39 ((struct psci_boot_args){ \ 40 .lock = ATOMIC_INIT(PSCI_BOOT_ARGS_UNLOCKED), \ 41 }) 42 43 static DEFINE_PER_CPU(struct psci_boot_args, cpu_on_args) = PSCI_BOOT_ARGS_INIT; 44 static DEFINE_PER_CPU(struct psci_boot_args, suspend_args) = PSCI_BOOT_ARGS_INIT; 45 46 #define is_psci_0_1(what, func_id) \ 47 (kvm_host_psci_config.psci_0_1_ ## what ## _implemented && \ 48 (func_id) == kvm_host_psci_config.function_ids_0_1.what) 49 50 static bool is_psci_0_1_call(u64 func_id) 51 { 52 return (is_psci_0_1(cpu_suspend, func_id) || 53 is_psci_0_1(cpu_on, func_id) || 54 is_psci_0_1(cpu_off, func_id) || 55 is_psci_0_1(migrate, func_id)); 56 } 57 58 static bool is_psci_0_2_call(u64 func_id) 59 { 60 /* SMCCC reserves IDs 0x00-1F with the given 32/64-bit base for PSCI. */ 61 return (PSCI_0_2_FN(0) <= func_id && func_id <= PSCI_0_2_FN(31)) || 62 (PSCI_0_2_FN64(0) <= func_id && func_id <= PSCI_0_2_FN64(31)); 63 } 64 65 static unsigned long psci_call(unsigned long fn, unsigned long arg0, 66 unsigned long arg1, unsigned long arg2) 67 { 68 struct arm_smccc_res res; 69 70 arm_smccc_1_1_smc(fn, arg0, arg1, arg2, &res); 71 return res.a0; 72 } 73 74 static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt) 75 { 76 return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1), 77 cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3)); 78 } 79 80 static unsigned int find_cpu_id(u64 mpidr) 81 { 82 unsigned int i; 83 84 /* Reject invalid MPIDRs */ 85 if (mpidr & ~MPIDR_HWID_BITMASK) 86 return INVALID_CPU_ID; 87 88 for (i = 0; i < NR_CPUS; i++) { 89 if (cpu_logical_map(i) == mpidr) 90 return i; 91 } 92 93 return INVALID_CPU_ID; 94 } 95 96 static __always_inline bool try_acquire_boot_args(struct psci_boot_args *args) 97 { 98 return atomic_cmpxchg_acquire(&args->lock, 99 PSCI_BOOT_ARGS_UNLOCKED, 100 PSCI_BOOT_ARGS_LOCKED) == 101 PSCI_BOOT_ARGS_UNLOCKED; 102 } 103 104 static __always_inline void release_boot_args(struct psci_boot_args *args) 105 { 106 atomic_set_release(&args->lock, PSCI_BOOT_ARGS_UNLOCKED); 107 } 108 109 static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) 110 { 111 DECLARE_REG(u64, mpidr, host_ctxt, 1); 112 DECLARE_REG(unsigned long, pc, host_ctxt, 2); 113 DECLARE_REG(unsigned long, r0, host_ctxt, 3); 114 115 unsigned int cpu_id; 116 struct psci_boot_args *boot_args; 117 struct kvm_nvhe_init_params *init_params; 118 int ret; 119 120 /* 121 * Find the logical CPU ID for the given MPIDR. The search set is 122 * the set of CPUs that were online at the point of KVM initialization. 123 * Booting other CPUs is rejected because their cpufeatures were not 124 * checked against the finalized capabilities. This could be relaxed 125 * by doing the feature checks in hyp. 126 */ 127 cpu_id = find_cpu_id(mpidr); 128 if (cpu_id == INVALID_CPU_ID) 129 return PSCI_RET_INVALID_PARAMS; 130 131 boot_args = per_cpu_ptr(&cpu_on_args, cpu_id); 132 init_params = per_cpu_ptr(&kvm_init_params, cpu_id); 133 134 /* Check if the target CPU is already being booted. */ 135 if (!try_acquire_boot_args(boot_args)) 136 return PSCI_RET_ALREADY_ON; 137 138 boot_args->pc = pc; 139 boot_args->r0 = r0; 140 wmb(); 141 142 ret = psci_call(func_id, mpidr, 143 __hyp_pa(&kvm_hyp_cpu_entry), 144 __hyp_pa(init_params)); 145 146 /* If successful, the lock will be released by the target CPU. */ 147 if (ret != PSCI_RET_SUCCESS) 148 release_boot_args(boot_args); 149 150 return ret; 151 } 152 153 static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) 154 { 155 DECLARE_REG(u64, power_state, host_ctxt, 1); 156 DECLARE_REG(unsigned long, pc, host_ctxt, 2); 157 DECLARE_REG(unsigned long, r0, host_ctxt, 3); 158 159 struct psci_boot_args *boot_args; 160 struct kvm_nvhe_init_params *init_params; 161 162 boot_args = this_cpu_ptr(&suspend_args); 163 init_params = this_cpu_ptr(&kvm_init_params); 164 165 /* 166 * No need to acquire a lock before writing to boot_args because a core 167 * can only suspend itself. Racy CPU_ON calls use a separate struct. 168 */ 169 boot_args->pc = pc; 170 boot_args->r0 = r0; 171 172 /* 173 * Will either return if shallow sleep state, or wake up into the entry 174 * point if it is a deep sleep state. 175 */ 176 return psci_call(func_id, power_state, 177 __hyp_pa(&kvm_hyp_cpu_resume), 178 __hyp_pa(init_params)); 179 } 180 181 static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) 182 { 183 DECLARE_REG(unsigned long, pc, host_ctxt, 1); 184 DECLARE_REG(unsigned long, r0, host_ctxt, 2); 185 186 struct psci_boot_args *boot_args; 187 struct kvm_nvhe_init_params *init_params; 188 189 boot_args = this_cpu_ptr(&suspend_args); 190 init_params = this_cpu_ptr(&kvm_init_params); 191 192 /* 193 * No need to acquire a lock before writing to boot_args because a core 194 * can only suspend itself. Racy CPU_ON calls use a separate struct. 195 */ 196 boot_args->pc = pc; 197 boot_args->r0 = r0; 198 199 /* Will only return on error. */ 200 return psci_call(func_id, 201 __hyp_pa(&kvm_hyp_cpu_resume), 202 __hyp_pa(init_params), 0); 203 } 204 205 asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on) 206 { 207 struct psci_boot_args *boot_args; 208 struct kvm_cpu_context *host_ctxt; 209 210 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 211 212 if (is_cpu_on) 213 boot_args = this_cpu_ptr(&cpu_on_args); 214 else 215 boot_args = this_cpu_ptr(&suspend_args); 216 217 cpu_reg(host_ctxt, 0) = boot_args->r0; 218 write_sysreg_el2(boot_args->pc, SYS_ELR); 219 220 if (is_cpu_on) 221 release_boot_args(boot_args); 222 223 __host_enter(host_ctxt); 224 } 225 226 static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 227 { 228 if (is_psci_0_1(cpu_off, func_id) || is_psci_0_1(migrate, func_id)) 229 return psci_forward(host_ctxt); 230 if (is_psci_0_1(cpu_on, func_id)) 231 return psci_cpu_on(func_id, host_ctxt); 232 if (is_psci_0_1(cpu_suspend, func_id)) 233 return psci_cpu_suspend(func_id, host_ctxt); 234 235 return PSCI_RET_NOT_SUPPORTED; 236 } 237 238 static unsigned long psci_0_2_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 239 { 240 switch (func_id) { 241 case PSCI_0_2_FN_PSCI_VERSION: 242 case PSCI_0_2_FN_CPU_OFF: 243 case PSCI_0_2_FN64_AFFINITY_INFO: 244 case PSCI_0_2_FN64_MIGRATE: 245 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 246 case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU: 247 return psci_forward(host_ctxt); 248 /* 249 * SYSTEM_OFF/RESET should not return according to the spec. 250 * Allow it so as to stay robust to broken firmware. 251 */ 252 case PSCI_0_2_FN_SYSTEM_OFF: 253 case PSCI_0_2_FN_SYSTEM_RESET: 254 return psci_forward(host_ctxt); 255 case PSCI_0_2_FN64_CPU_SUSPEND: 256 return psci_cpu_suspend(func_id, host_ctxt); 257 case PSCI_0_2_FN64_CPU_ON: 258 return psci_cpu_on(func_id, host_ctxt); 259 default: 260 return PSCI_RET_NOT_SUPPORTED; 261 } 262 } 263 264 static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 265 { 266 switch (func_id) { 267 case PSCI_1_0_FN_PSCI_FEATURES: 268 case PSCI_1_0_FN_SET_SUSPEND_MODE: 269 case PSCI_1_1_FN64_SYSTEM_RESET2: 270 return psci_forward(host_ctxt); 271 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 272 return psci_system_suspend(func_id, host_ctxt); 273 default: 274 return psci_0_2_handler(func_id, host_ctxt); 275 } 276 } 277 278 bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt) 279 { 280 DECLARE_REG(u64, func_id, host_ctxt, 0); 281 unsigned long ret; 282 283 switch (kvm_host_psci_config.version) { 284 case PSCI_VERSION(0, 1): 285 if (!is_psci_0_1_call(func_id)) 286 return false; 287 ret = psci_0_1_handler(func_id, host_ctxt); 288 break; 289 case PSCI_VERSION(0, 2): 290 if (!is_psci_0_2_call(func_id)) 291 return false; 292 ret = psci_0_2_handler(func_id, host_ctxt); 293 break; 294 default: 295 if (!is_psci_0_2_call(func_id)) 296 return false; 297 ret = psci_1_0_handler(func_id, host_ctxt); 298 break; 299 } 300 301 cpu_reg(host_ctxt, 0) = ret; 302 cpu_reg(host_ctxt, 1) = 0; 303 cpu_reg(host_ctxt, 2) = 0; 304 cpu_reg(host_ctxt, 3) = 0; 305 return true; 306 } 307