1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - Google LLC 4 * Author: David Brazdil <dbrazdil@google.com> 5 */ 6 7 #include <asm/kvm_asm.h> 8 #include <asm/kvm_hyp.h> 9 #include <asm/kvm_mmu.h> 10 #include <kvm/arm_hypercalls.h> 11 #include <linux/arm-smccc.h> 12 #include <linux/kvm_host.h> 13 #include <linux/psci.h> 14 #include <kvm/arm_psci.h> 15 #include <uapi/linux/psci.h> 16 17 #include <nvhe/trap_handler.h> 18 19 void kvm_hyp_cpu_entry(unsigned long r0); 20 void kvm_hyp_cpu_resume(unsigned long r0); 21 22 void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); 23 24 /* Config options set by the host. */ 25 __ro_after_init u32 kvm_host_psci_version; 26 __ro_after_init struct psci_0_1_function_ids kvm_host_psci_0_1_function_ids; 27 __ro_after_init s64 hyp_physvirt_offset; 28 29 #define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset) 30 31 #define INVALID_CPU_ID UINT_MAX 32 33 struct psci_boot_args { 34 atomic_t lock; 35 unsigned long pc; 36 unsigned long r0; 37 }; 38 39 #define PSCI_BOOT_ARGS_UNLOCKED 0 40 #define PSCI_BOOT_ARGS_LOCKED 1 41 42 #define PSCI_BOOT_ARGS_INIT \ 43 ((struct psci_boot_args){ \ 44 .lock = ATOMIC_INIT(PSCI_BOOT_ARGS_UNLOCKED), \ 45 }) 46 47 static DEFINE_PER_CPU(struct psci_boot_args, cpu_on_args) = PSCI_BOOT_ARGS_INIT; 48 static DEFINE_PER_CPU(struct psci_boot_args, suspend_args) = PSCI_BOOT_ARGS_INIT; 49 50 static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt) 51 { 52 DECLARE_REG(u64, func_id, host_ctxt, 0); 53 54 return func_id; 55 } 56 57 static bool is_psci_0_1_call(u64 func_id) 58 { 59 return (func_id == kvm_host_psci_0_1_function_ids.cpu_suspend) || 60 (func_id == kvm_host_psci_0_1_function_ids.cpu_on) || 61 (func_id == kvm_host_psci_0_1_function_ids.cpu_off) || 62 (func_id == kvm_host_psci_0_1_function_ids.migrate); 63 } 64 65 static bool is_psci_0_2_call(u64 func_id) 66 { 67 /* SMCCC reserves IDs 0x00-1F with the given 32/64-bit base for PSCI. */ 68 return (PSCI_0_2_FN(0) <= func_id && func_id <= PSCI_0_2_FN(31)) || 69 (PSCI_0_2_FN64(0) <= func_id && func_id <= PSCI_0_2_FN64(31)); 70 } 71 72 static bool is_psci_call(u64 func_id) 73 { 74 switch (kvm_host_psci_version) { 75 case PSCI_VERSION(0, 1): 76 return is_psci_0_1_call(func_id); 77 default: 78 return is_psci_0_2_call(func_id); 79 } 80 } 81 82 static unsigned long psci_call(unsigned long fn, unsigned long arg0, 83 unsigned long arg1, unsigned long arg2) 84 { 85 struct arm_smccc_res res; 86 87 arm_smccc_1_1_smc(fn, arg0, arg1, arg2, &res); 88 return res.a0; 89 } 90 91 static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt) 92 { 93 return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1), 94 cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3)); 95 } 96 97 static __noreturn unsigned long psci_forward_noreturn(struct kvm_cpu_context *host_ctxt) 98 { 99 psci_forward(host_ctxt); 100 hyp_panic(); /* unreachable */ 101 } 102 103 static unsigned int find_cpu_id(u64 mpidr) 104 { 105 unsigned int i; 106 107 /* Reject invalid MPIDRs */ 108 if (mpidr & ~MPIDR_HWID_BITMASK) 109 return INVALID_CPU_ID; 110 111 for (i = 0; i < NR_CPUS; i++) { 112 if (cpu_logical_map(i) == mpidr) 113 return i; 114 } 115 116 return INVALID_CPU_ID; 117 } 118 119 static __always_inline bool try_acquire_boot_args(struct psci_boot_args *args) 120 { 121 return atomic_cmpxchg_acquire(&args->lock, 122 PSCI_BOOT_ARGS_UNLOCKED, 123 PSCI_BOOT_ARGS_LOCKED) == 124 PSCI_BOOT_ARGS_UNLOCKED; 125 } 126 127 static __always_inline void release_boot_args(struct psci_boot_args *args) 128 { 129 atomic_set_release(&args->lock, PSCI_BOOT_ARGS_UNLOCKED); 130 } 131 132 static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) 133 { 134 DECLARE_REG(u64, mpidr, host_ctxt, 1); 135 DECLARE_REG(unsigned long, pc, host_ctxt, 2); 136 DECLARE_REG(unsigned long, r0, host_ctxt, 3); 137 138 unsigned int cpu_id; 139 struct psci_boot_args *boot_args; 140 struct kvm_nvhe_init_params *init_params; 141 int ret; 142 143 /* 144 * Find the logical CPU ID for the given MPIDR. The search set is 145 * the set of CPUs that were online at the point of KVM initialization. 146 * Booting other CPUs is rejected because their cpufeatures were not 147 * checked against the finalized capabilities. This could be relaxed 148 * by doing the feature checks in hyp. 149 */ 150 cpu_id = find_cpu_id(mpidr); 151 if (cpu_id == INVALID_CPU_ID) 152 return PSCI_RET_INVALID_PARAMS; 153 154 boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id); 155 init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id); 156 157 /* Check if the target CPU is already being booted. */ 158 if (!try_acquire_boot_args(boot_args)) 159 return PSCI_RET_ALREADY_ON; 160 161 boot_args->pc = pc; 162 boot_args->r0 = r0; 163 wmb(); 164 165 ret = psci_call(func_id, mpidr, 166 __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_entry)), 167 __hyp_pa(init_params)); 168 169 /* If successful, the lock will be released by the target CPU. */ 170 if (ret != PSCI_RET_SUCCESS) 171 release_boot_args(boot_args); 172 173 return ret; 174 } 175 176 static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) 177 { 178 DECLARE_REG(u64, power_state, host_ctxt, 1); 179 DECLARE_REG(unsigned long, pc, host_ctxt, 2); 180 DECLARE_REG(unsigned long, r0, host_ctxt, 3); 181 182 struct psci_boot_args *boot_args; 183 struct kvm_nvhe_init_params *init_params; 184 185 boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args)); 186 init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params)); 187 188 /* 189 * No need to acquire a lock before writing to boot_args because a core 190 * can only suspend itself. Racy CPU_ON calls use a separate struct. 191 */ 192 boot_args->pc = pc; 193 boot_args->r0 = r0; 194 195 /* 196 * Will either return if shallow sleep state, or wake up into the entry 197 * point if it is a deep sleep state. 198 */ 199 return psci_call(func_id, power_state, 200 __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)), 201 __hyp_pa(init_params)); 202 } 203 204 static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) 205 { 206 DECLARE_REG(unsigned long, pc, host_ctxt, 1); 207 DECLARE_REG(unsigned long, r0, host_ctxt, 2); 208 209 struct psci_boot_args *boot_args; 210 struct kvm_nvhe_init_params *init_params; 211 212 boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args)); 213 init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params)); 214 215 /* 216 * No need to acquire a lock before writing to boot_args because a core 217 * can only suspend itself. Racy CPU_ON calls use a separate struct. 218 */ 219 boot_args->pc = pc; 220 boot_args->r0 = r0; 221 222 /* Will only return on error. */ 223 return psci_call(func_id, 224 __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)), 225 __hyp_pa(init_params), 0); 226 } 227 228 asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on) 229 { 230 struct psci_boot_args *boot_args; 231 struct kvm_cpu_context *host_ctxt; 232 233 host_ctxt = &this_cpu_ptr(hyp_symbol_addr(kvm_host_data))->host_ctxt; 234 235 if (is_cpu_on) 236 boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args)); 237 else 238 boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args)); 239 240 cpu_reg(host_ctxt, 0) = boot_args->r0; 241 write_sysreg_el2(boot_args->pc, SYS_ELR); 242 243 if (is_cpu_on) 244 release_boot_args(boot_args); 245 246 __host_enter(host_ctxt); 247 } 248 249 static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 250 { 251 if ((func_id == kvm_host_psci_0_1_function_ids.cpu_off) || 252 (func_id == kvm_host_psci_0_1_function_ids.migrate)) 253 return psci_forward(host_ctxt); 254 else if (func_id == kvm_host_psci_0_1_function_ids.cpu_on) 255 return psci_cpu_on(func_id, host_ctxt); 256 else if (func_id == kvm_host_psci_0_1_function_ids.cpu_suspend) 257 return psci_cpu_suspend(func_id, host_ctxt); 258 else 259 return PSCI_RET_NOT_SUPPORTED; 260 } 261 262 static unsigned long psci_0_2_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 263 { 264 switch (func_id) { 265 case PSCI_0_2_FN_PSCI_VERSION: 266 case PSCI_0_2_FN_CPU_OFF: 267 case PSCI_0_2_FN64_AFFINITY_INFO: 268 case PSCI_0_2_FN64_MIGRATE: 269 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 270 case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU: 271 return psci_forward(host_ctxt); 272 case PSCI_0_2_FN_SYSTEM_OFF: 273 case PSCI_0_2_FN_SYSTEM_RESET: 274 psci_forward_noreturn(host_ctxt); 275 unreachable(); 276 case PSCI_0_2_FN64_CPU_SUSPEND: 277 return psci_cpu_suspend(func_id, host_ctxt); 278 case PSCI_0_2_FN64_CPU_ON: 279 return psci_cpu_on(func_id, host_ctxt); 280 default: 281 return PSCI_RET_NOT_SUPPORTED; 282 } 283 } 284 285 static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 286 { 287 switch (func_id) { 288 case PSCI_1_0_FN_PSCI_FEATURES: 289 case PSCI_1_0_FN_SET_SUSPEND_MODE: 290 case PSCI_1_1_FN64_SYSTEM_RESET2: 291 return psci_forward(host_ctxt); 292 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 293 return psci_system_suspend(func_id, host_ctxt); 294 default: 295 return psci_0_2_handler(func_id, host_ctxt); 296 } 297 } 298 299 bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt) 300 { 301 u64 func_id = get_psci_func_id(host_ctxt); 302 unsigned long ret; 303 304 if (!is_psci_call(func_id)) 305 return false; 306 307 switch (kvm_host_psci_version) { 308 case PSCI_VERSION(0, 1): 309 ret = psci_0_1_handler(func_id, host_ctxt); 310 break; 311 case PSCI_VERSION(0, 2): 312 ret = psci_0_2_handler(func_id, host_ctxt); 313 break; 314 default: 315 ret = psci_1_0_handler(func_id, host_ctxt); 316 break; 317 } 318 319 cpu_reg(host_ctxt, 0) = ret; 320 cpu_reg(host_ctxt, 1) = 0; 321 cpu_reg(host_ctxt, 2) = 0; 322 cpu_reg(host_ctxt, 3) = 0; 323 return true; 324 } 325