1eeeee719SDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only
2eeeee719SDavid Brazdil /*
3eeeee719SDavid Brazdil * Copyright (C) 2020 - Google LLC
4eeeee719SDavid Brazdil * Author: David Brazdil <dbrazdil@google.com>
5eeeee719SDavid Brazdil */
6eeeee719SDavid Brazdil
7eeeee719SDavid Brazdil #include <asm/kvm_asm.h>
8eeeee719SDavid Brazdil #include <asm/kvm_hyp.h>
9eeeee719SDavid Brazdil #include <asm/kvm_mmu.h>
10eeeee719SDavid Brazdil #include <linux/arm-smccc.h>
11cdf36719SDavid Brazdil #include <linux/kvm_host.h>
12eeeee719SDavid Brazdil #include <uapi/linux/psci.h>
13eeeee719SDavid Brazdil
14e7596040SQuentin Perret #include <nvhe/memory.h>
15eeeee719SDavid Brazdil #include <nvhe/trap_handler.h>
16eeeee719SDavid Brazdil
17cdf36719SDavid Brazdil void kvm_hyp_cpu_entry(unsigned long r0);
18abf16336SDavid Brazdil void kvm_hyp_cpu_resume(unsigned long r0);
19cdf36719SDavid Brazdil
20cdf36719SDavid Brazdil void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
21cdf36719SDavid Brazdil
22eeeee719SDavid Brazdil /* Config options set by the host. */
23ff367fe4SDavid Brazdil struct kvm_host_psci_config __ro_after_init kvm_host_psci_config;
24eeeee719SDavid Brazdil
25cdf36719SDavid Brazdil #define INVALID_CPU_ID UINT_MAX
26cdf36719SDavid Brazdil
27cdf36719SDavid Brazdil struct psci_boot_args {
28cdf36719SDavid Brazdil atomic_t lock;
29cdf36719SDavid Brazdil unsigned long pc;
30cdf36719SDavid Brazdil unsigned long r0;
31cdf36719SDavid Brazdil };
32cdf36719SDavid Brazdil
33cdf36719SDavid Brazdil #define PSCI_BOOT_ARGS_UNLOCKED 0
34cdf36719SDavid Brazdil #define PSCI_BOOT_ARGS_LOCKED 1
35cdf36719SDavid Brazdil
36cdf36719SDavid Brazdil #define PSCI_BOOT_ARGS_INIT \
37cdf36719SDavid Brazdil ((struct psci_boot_args){ \
38cdf36719SDavid Brazdil .lock = ATOMIC_INIT(PSCI_BOOT_ARGS_UNLOCKED), \
39cdf36719SDavid Brazdil })
40cdf36719SDavid Brazdil
41cdf36719SDavid Brazdil static DEFINE_PER_CPU(struct psci_boot_args, cpu_on_args) = PSCI_BOOT_ARGS_INIT;
42abf16336SDavid Brazdil static DEFINE_PER_CPU(struct psci_boot_args, suspend_args) = PSCI_BOOT_ARGS_INIT;
43cdf36719SDavid Brazdil
44767c973fSMarc Zyngier #define is_psci_0_1(what, func_id) \
45767c973fSMarc Zyngier (kvm_host_psci_config.psci_0_1_ ## what ## _implemented && \
46767c973fSMarc Zyngier (func_id) == kvm_host_psci_config.function_ids_0_1.what)
47ff367fe4SDavid Brazdil
is_psci_0_1_call(u64 func_id)48eeeee719SDavid Brazdil static bool is_psci_0_1_call(u64 func_id)
49eeeee719SDavid Brazdil {
50767c973fSMarc Zyngier return (is_psci_0_1(cpu_suspend, func_id) ||
51767c973fSMarc Zyngier is_psci_0_1(cpu_on, func_id) ||
52767c973fSMarc Zyngier is_psci_0_1(cpu_off, func_id) ||
53767c973fSMarc Zyngier is_psci_0_1(migrate, func_id));
54eeeee719SDavid Brazdil }
55eeeee719SDavid Brazdil
is_psci_0_2_call(u64 func_id)56eeeee719SDavid Brazdil static bool is_psci_0_2_call(u64 func_id)
57eeeee719SDavid Brazdil {
58eeeee719SDavid Brazdil /* SMCCC reserves IDs 0x00-1F with the given 32/64-bit base for PSCI. */
59eeeee719SDavid Brazdil return (PSCI_0_2_FN(0) <= func_id && func_id <= PSCI_0_2_FN(31)) ||
60eeeee719SDavid Brazdil (PSCI_0_2_FN64(0) <= func_id && func_id <= PSCI_0_2_FN64(31));
61eeeee719SDavid Brazdil }
62eeeee719SDavid Brazdil
psci_call(unsigned long fn,unsigned long arg0,unsigned long arg1,unsigned long arg2)631fd12b7eSDavid Brazdil static unsigned long psci_call(unsigned long fn, unsigned long arg0,
641fd12b7eSDavid Brazdil unsigned long arg1, unsigned long arg2)
651fd12b7eSDavid Brazdil {
661fd12b7eSDavid Brazdil struct arm_smccc_res res;
671fd12b7eSDavid Brazdil
681fd12b7eSDavid Brazdil arm_smccc_1_1_smc(fn, arg0, arg1, arg2, &res);
691fd12b7eSDavid Brazdil return res.a0;
701fd12b7eSDavid Brazdil }
711fd12b7eSDavid Brazdil
psci_forward(struct kvm_cpu_context * host_ctxt)721fd12b7eSDavid Brazdil static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt)
731fd12b7eSDavid Brazdil {
741fd12b7eSDavid Brazdil return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1),
751fd12b7eSDavid Brazdil cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3));
761fd12b7eSDavid Brazdil }
771fd12b7eSDavid Brazdil
find_cpu_id(u64 mpidr)78cdf36719SDavid Brazdil static unsigned int find_cpu_id(u64 mpidr)
79cdf36719SDavid Brazdil {
80cdf36719SDavid Brazdil unsigned int i;
81cdf36719SDavid Brazdil
82cdf36719SDavid Brazdil /* Reject invalid MPIDRs */
83cdf36719SDavid Brazdil if (mpidr & ~MPIDR_HWID_BITMASK)
84cdf36719SDavid Brazdil return INVALID_CPU_ID;
85cdf36719SDavid Brazdil
86cdf36719SDavid Brazdil for (i = 0; i < NR_CPUS; i++) {
87cdf36719SDavid Brazdil if (cpu_logical_map(i) == mpidr)
88cdf36719SDavid Brazdil return i;
89cdf36719SDavid Brazdil }
90cdf36719SDavid Brazdil
91cdf36719SDavid Brazdil return INVALID_CPU_ID;
92cdf36719SDavid Brazdil }
93cdf36719SDavid Brazdil
try_acquire_boot_args(struct psci_boot_args * args)94cdf36719SDavid Brazdil static __always_inline bool try_acquire_boot_args(struct psci_boot_args *args)
95cdf36719SDavid Brazdil {
96cdf36719SDavid Brazdil return atomic_cmpxchg_acquire(&args->lock,
97cdf36719SDavid Brazdil PSCI_BOOT_ARGS_UNLOCKED,
98cdf36719SDavid Brazdil PSCI_BOOT_ARGS_LOCKED) ==
99cdf36719SDavid Brazdil PSCI_BOOT_ARGS_UNLOCKED;
100cdf36719SDavid Brazdil }
101cdf36719SDavid Brazdil
release_boot_args(struct psci_boot_args * args)102cdf36719SDavid Brazdil static __always_inline void release_boot_args(struct psci_boot_args *args)
103cdf36719SDavid Brazdil {
104cdf36719SDavid Brazdil atomic_set_release(&args->lock, PSCI_BOOT_ARGS_UNLOCKED);
105cdf36719SDavid Brazdil }
106cdf36719SDavid Brazdil
psci_cpu_on(u64 func_id,struct kvm_cpu_context * host_ctxt)107cdf36719SDavid Brazdil static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt)
108cdf36719SDavid Brazdil {
109cdf36719SDavid Brazdil DECLARE_REG(u64, mpidr, host_ctxt, 1);
110cdf36719SDavid Brazdil DECLARE_REG(unsigned long, pc, host_ctxt, 2);
111cdf36719SDavid Brazdil DECLARE_REG(unsigned long, r0, host_ctxt, 3);
112cdf36719SDavid Brazdil
113cdf36719SDavid Brazdil unsigned int cpu_id;
114cdf36719SDavid Brazdil struct psci_boot_args *boot_args;
115cdf36719SDavid Brazdil struct kvm_nvhe_init_params *init_params;
116cdf36719SDavid Brazdil int ret;
117cdf36719SDavid Brazdil
118cdf36719SDavid Brazdil /*
119cdf36719SDavid Brazdil * Find the logical CPU ID for the given MPIDR. The search set is
120cdf36719SDavid Brazdil * the set of CPUs that were online at the point of KVM initialization.
121cdf36719SDavid Brazdil * Booting other CPUs is rejected because their cpufeatures were not
122cdf36719SDavid Brazdil * checked against the finalized capabilities. This could be relaxed
123cdf36719SDavid Brazdil * by doing the feature checks in hyp.
124cdf36719SDavid Brazdil */
125cdf36719SDavid Brazdil cpu_id = find_cpu_id(mpidr);
126cdf36719SDavid Brazdil if (cpu_id == INVALID_CPU_ID)
127cdf36719SDavid Brazdil return PSCI_RET_INVALID_PARAMS;
128cdf36719SDavid Brazdil
129247bc166SDavid Brazdil boot_args = per_cpu_ptr(&cpu_on_args, cpu_id);
130247bc166SDavid Brazdil init_params = per_cpu_ptr(&kvm_init_params, cpu_id);
131cdf36719SDavid Brazdil
132cdf36719SDavid Brazdil /* Check if the target CPU is already being booted. */
133cdf36719SDavid Brazdil if (!try_acquire_boot_args(boot_args))
134cdf36719SDavid Brazdil return PSCI_RET_ALREADY_ON;
135cdf36719SDavid Brazdil
136cdf36719SDavid Brazdil boot_args->pc = pc;
137cdf36719SDavid Brazdil boot_args->r0 = r0;
138cdf36719SDavid Brazdil wmb();
139cdf36719SDavid Brazdil
140cdf36719SDavid Brazdil ret = psci_call(func_id, mpidr,
141247bc166SDavid Brazdil __hyp_pa(&kvm_hyp_cpu_entry),
142cdf36719SDavid Brazdil __hyp_pa(init_params));
143cdf36719SDavid Brazdil
144cdf36719SDavid Brazdil /* If successful, the lock will be released by the target CPU. */
145cdf36719SDavid Brazdil if (ret != PSCI_RET_SUCCESS)
146cdf36719SDavid Brazdil release_boot_args(boot_args);
147cdf36719SDavid Brazdil
148cdf36719SDavid Brazdil return ret;
149cdf36719SDavid Brazdil }
150cdf36719SDavid Brazdil
psci_cpu_suspend(u64 func_id,struct kvm_cpu_context * host_ctxt)151abf16336SDavid Brazdil static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
152abf16336SDavid Brazdil {
153abf16336SDavid Brazdil DECLARE_REG(u64, power_state, host_ctxt, 1);
154abf16336SDavid Brazdil DECLARE_REG(unsigned long, pc, host_ctxt, 2);
155abf16336SDavid Brazdil DECLARE_REG(unsigned long, r0, host_ctxt, 3);
156abf16336SDavid Brazdil
157abf16336SDavid Brazdil struct psci_boot_args *boot_args;
158abf16336SDavid Brazdil struct kvm_nvhe_init_params *init_params;
159abf16336SDavid Brazdil
160247bc166SDavid Brazdil boot_args = this_cpu_ptr(&suspend_args);
161247bc166SDavid Brazdil init_params = this_cpu_ptr(&kvm_init_params);
162abf16336SDavid Brazdil
163abf16336SDavid Brazdil /*
164abf16336SDavid Brazdil * No need to acquire a lock before writing to boot_args because a core
165abf16336SDavid Brazdil * can only suspend itself. Racy CPU_ON calls use a separate struct.
166abf16336SDavid Brazdil */
167abf16336SDavid Brazdil boot_args->pc = pc;
168abf16336SDavid Brazdil boot_args->r0 = r0;
169abf16336SDavid Brazdil
170abf16336SDavid Brazdil /*
171abf16336SDavid Brazdil * Will either return if shallow sleep state, or wake up into the entry
172abf16336SDavid Brazdil * point if it is a deep sleep state.
173abf16336SDavid Brazdil */
174abf16336SDavid Brazdil return psci_call(func_id, power_state,
175247bc166SDavid Brazdil __hyp_pa(&kvm_hyp_cpu_resume),
176abf16336SDavid Brazdil __hyp_pa(init_params));
177abf16336SDavid Brazdil }
178abf16336SDavid Brazdil
psci_system_suspend(u64 func_id,struct kvm_cpu_context * host_ctxt)179d945f8d9SDavid Brazdil static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
180d945f8d9SDavid Brazdil {
181d945f8d9SDavid Brazdil DECLARE_REG(unsigned long, pc, host_ctxt, 1);
182d945f8d9SDavid Brazdil DECLARE_REG(unsigned long, r0, host_ctxt, 2);
183d945f8d9SDavid Brazdil
184d945f8d9SDavid Brazdil struct psci_boot_args *boot_args;
185d945f8d9SDavid Brazdil struct kvm_nvhe_init_params *init_params;
186d945f8d9SDavid Brazdil
187247bc166SDavid Brazdil boot_args = this_cpu_ptr(&suspend_args);
188247bc166SDavid Brazdil init_params = this_cpu_ptr(&kvm_init_params);
189d945f8d9SDavid Brazdil
190d945f8d9SDavid Brazdil /*
191d945f8d9SDavid Brazdil * No need to acquire a lock before writing to boot_args because a core
192d945f8d9SDavid Brazdil * can only suspend itself. Racy CPU_ON calls use a separate struct.
193d945f8d9SDavid Brazdil */
194d945f8d9SDavid Brazdil boot_args->pc = pc;
195d945f8d9SDavid Brazdil boot_args->r0 = r0;
196d945f8d9SDavid Brazdil
197d945f8d9SDavid Brazdil /* Will only return on error. */
198d945f8d9SDavid Brazdil return psci_call(func_id,
199247bc166SDavid Brazdil __hyp_pa(&kvm_hyp_cpu_resume),
200d945f8d9SDavid Brazdil __hyp_pa(init_params), 0);
201d945f8d9SDavid Brazdil }
202d945f8d9SDavid Brazdil
__kvm_host_psci_cpu_entry(bool is_cpu_on)203dcf89d11SMostafa Saleh asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on)
204cdf36719SDavid Brazdil {
205cdf36719SDavid Brazdil struct psci_boot_args *boot_args;
206cdf36719SDavid Brazdil struct kvm_cpu_context *host_ctxt;
207cdf36719SDavid Brazdil
208247bc166SDavid Brazdil host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
209abf16336SDavid Brazdil
210abf16336SDavid Brazdil if (is_cpu_on)
211247bc166SDavid Brazdil boot_args = this_cpu_ptr(&cpu_on_args);
212abf16336SDavid Brazdil else
213247bc166SDavid Brazdil boot_args = this_cpu_ptr(&suspend_args);
214cdf36719SDavid Brazdil
215cdf36719SDavid Brazdil cpu_reg(host_ctxt, 0) = boot_args->r0;
216cdf36719SDavid Brazdil write_sysreg_el2(boot_args->pc, SYS_ELR);
217abf16336SDavid Brazdil
218abf16336SDavid Brazdil if (is_cpu_on)
219cdf36719SDavid Brazdil release_boot_args(boot_args);
220cdf36719SDavid Brazdil
221cdf36719SDavid Brazdil __host_enter(host_ctxt);
222cdf36719SDavid Brazdil }
223cdf36719SDavid Brazdil
psci_0_1_handler(u64 func_id,struct kvm_cpu_context * host_ctxt)224eeeee719SDavid Brazdil static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context *host_ctxt)
225eeeee719SDavid Brazdil {
226767c973fSMarc Zyngier if (is_psci_0_1(cpu_off, func_id) || is_psci_0_1(migrate, func_id))
2271fd12b7eSDavid Brazdil return psci_forward(host_ctxt);
228767c973fSMarc Zyngier if (is_psci_0_1(cpu_on, func_id))
229cdf36719SDavid Brazdil return psci_cpu_on(func_id, host_ctxt);
230767c973fSMarc Zyngier if (is_psci_0_1(cpu_suspend, func_id))
231abf16336SDavid Brazdil return psci_cpu_suspend(func_id, host_ctxt);
232767c973fSMarc Zyngier
233eeeee719SDavid Brazdil return PSCI_RET_NOT_SUPPORTED;
234eeeee719SDavid Brazdil }
235eeeee719SDavid Brazdil
psci_0_2_handler(u64 func_id,struct kvm_cpu_context * host_ctxt)236eeeee719SDavid Brazdil static unsigned long psci_0_2_handler(u64 func_id, struct kvm_cpu_context *host_ctxt)
237eeeee719SDavid Brazdil {
238eeeee719SDavid Brazdil switch (func_id) {
2391fd12b7eSDavid Brazdil case PSCI_0_2_FN_PSCI_VERSION:
2401fd12b7eSDavid Brazdil case PSCI_0_2_FN_CPU_OFF:
2411fd12b7eSDavid Brazdil case PSCI_0_2_FN64_AFFINITY_INFO:
2421fd12b7eSDavid Brazdil case PSCI_0_2_FN64_MIGRATE:
2431fd12b7eSDavid Brazdil case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
2441fd12b7eSDavid Brazdil case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU:
2451fd12b7eSDavid Brazdil return psci_forward(host_ctxt);
2462c91ef39SDavid Brazdil /*
2472c91ef39SDavid Brazdil * SYSTEM_OFF/RESET should not return according to the spec.
2482c91ef39SDavid Brazdil * Allow it so as to stay robust to broken firmware.
2492c91ef39SDavid Brazdil */
2501fd12b7eSDavid Brazdil case PSCI_0_2_FN_SYSTEM_OFF:
2511fd12b7eSDavid Brazdil case PSCI_0_2_FN_SYSTEM_RESET:
2522c91ef39SDavid Brazdil return psci_forward(host_ctxt);
253abf16336SDavid Brazdil case PSCI_0_2_FN64_CPU_SUSPEND:
254abf16336SDavid Brazdil return psci_cpu_suspend(func_id, host_ctxt);
255cdf36719SDavid Brazdil case PSCI_0_2_FN64_CPU_ON:
256cdf36719SDavid Brazdil return psci_cpu_on(func_id, host_ctxt);
257eeeee719SDavid Brazdil default:
258eeeee719SDavid Brazdil return PSCI_RET_NOT_SUPPORTED;
259eeeee719SDavid Brazdil }
260eeeee719SDavid Brazdil }
261eeeee719SDavid Brazdil
psci_1_0_handler(u64 func_id,struct kvm_cpu_context * host_ctxt)262eeeee719SDavid Brazdil static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ctxt)
263eeeee719SDavid Brazdil {
264eeeee719SDavid Brazdil switch (func_id) {
2651fd12b7eSDavid Brazdil case PSCI_1_0_FN_PSCI_FEATURES:
2661fd12b7eSDavid Brazdil case PSCI_1_0_FN_SET_SUSPEND_MODE:
2671fd12b7eSDavid Brazdil case PSCI_1_1_FN64_SYSTEM_RESET2:
2681fd12b7eSDavid Brazdil return psci_forward(host_ctxt);
269d945f8d9SDavid Brazdil case PSCI_1_0_FN64_SYSTEM_SUSPEND:
270d945f8d9SDavid Brazdil return psci_system_suspend(func_id, host_ctxt);
271eeeee719SDavid Brazdil default:
272eeeee719SDavid Brazdil return psci_0_2_handler(func_id, host_ctxt);
273eeeee719SDavid Brazdil }
274eeeee719SDavid Brazdil }
275eeeee719SDavid Brazdil
kvm_host_psci_handler(struct kvm_cpu_context * host_ctxt,u32 func_id)276*373beef0SJean-Philippe Brucker bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
277eeeee719SDavid Brazdil {
278eeeee719SDavid Brazdil unsigned long ret;
279eeeee719SDavid Brazdil
280ff367fe4SDavid Brazdil switch (kvm_host_psci_config.version) {
281eeeee719SDavid Brazdil case PSCI_VERSION(0, 1):
282767c973fSMarc Zyngier if (!is_psci_0_1_call(func_id))
283767c973fSMarc Zyngier return false;
284eeeee719SDavid Brazdil ret = psci_0_1_handler(func_id, host_ctxt);
285eeeee719SDavid Brazdil break;
286eeeee719SDavid Brazdil case PSCI_VERSION(0, 2):
287767c973fSMarc Zyngier if (!is_psci_0_2_call(func_id))
288767c973fSMarc Zyngier return false;
289eeeee719SDavid Brazdil ret = psci_0_2_handler(func_id, host_ctxt);
290eeeee719SDavid Brazdil break;
291eeeee719SDavid Brazdil default:
292767c973fSMarc Zyngier if (!is_psci_0_2_call(func_id))
293767c973fSMarc Zyngier return false;
294eeeee719SDavid Brazdil ret = psci_1_0_handler(func_id, host_ctxt);
295eeeee719SDavid Brazdil break;
296eeeee719SDavid Brazdil }
297eeeee719SDavid Brazdil
298eeeee719SDavid Brazdil cpu_reg(host_ctxt, 0) = ret;
299eeeee719SDavid Brazdil cpu_reg(host_ctxt, 1) = 0;
300eeeee719SDavid Brazdil cpu_reg(host_ctxt, 2) = 0;
301eeeee719SDavid Brazdil cpu_reg(host_ctxt, 3) = 0;
302eeeee719SDavid Brazdil return true;
303eeeee719SDavid Brazdil }
304