1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/arm-smccc.h> 8#include <linux/linkage.h> 9 10#include <asm/alternative.h> 11#include <asm/assembler.h> 12#include <asm/el2_setup.h> 13#include <asm/kvm_arm.h> 14#include <asm/kvm_asm.h> 15#include <asm/kvm_mmu.h> 16#include <asm/pgtable-hwdef.h> 17#include <asm/sysreg.h> 18#include <asm/virt.h> 19 20 .text 21 .pushsection .hyp.idmap.text, "ax" 22 23 .align 11 24 25SYM_CODE_START(__kvm_hyp_init) 26 ventry __invalid // Synchronous EL2t 27 ventry __invalid // IRQ EL2t 28 ventry __invalid // FIQ EL2t 29 ventry __invalid // Error EL2t 30 31 ventry __invalid // Synchronous EL2h 32 ventry __invalid // IRQ EL2h 33 ventry __invalid // FIQ EL2h 34 ventry __invalid // Error EL2h 35 36 ventry __do_hyp_init // Synchronous 64-bit EL1 37 ventry __invalid // IRQ 64-bit EL1 38 ventry __invalid // FIQ 64-bit EL1 39 ventry __invalid // Error 64-bit EL1 40 41 ventry __invalid // Synchronous 32-bit EL1 42 ventry __invalid // IRQ 32-bit EL1 43 ventry __invalid // FIQ 32-bit EL1 44 ventry __invalid // Error 32-bit EL1 45 46__invalid: 47 b . 48 49 /* 50 * x0: SMCCC function ID 51 * x1: struct kvm_nvhe_init_params PA 52 */ 53__do_hyp_init: 54 /* Check for a stub HVC call */ 55 cmp x0, #HVC_STUB_HCALL_NR 56 b.lo __kvm_handle_stub_hvc 57 58 // We only actively check bits [24:31], and everything 59 // else has to be zero, which we check at build time. 60#if (KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) & 0xFFFFFFFF00FFFFFF) 61#error Unexpected __KVM_HOST_SMCCC_FUNC___kvm_hyp_init value 62#endif 63 64 ror x0, x0, #24 65 eor x0, x0, #((KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) >> 24) & 0xF) 66 ror x0, x0, #4 67 eor x0, x0, #((KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) >> 28) & 0xF) 68 cbz x0, 1f 69 mov x0, #SMCCC_RET_NOT_SUPPORTED 70 eret 71 721: mov x0, x1 73 mov x4, lr 74 bl ___kvm_hyp_init 75 mov lr, x4 76 77 /* Hello, World! */ 78 mov x0, #SMCCC_RET_SUCCESS 79 eret 80SYM_CODE_END(__kvm_hyp_init) 81 82/* 83 * Initialize the hypervisor in EL2. 84 * 85 * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers 86 * and leave x4 for the caller. 87 * 88 * x0: struct kvm_nvhe_init_params PA 89 */ 90SYM_CODE_START_LOCAL(___kvm_hyp_init) 91alternative_if ARM64_KVM_PROTECTED_MODE 92 mov_q x1, HCR_HOST_NVHE_PROTECTED_FLAGS 93 msr hcr_el2, x1 94alternative_else_nop_endif 95 96 ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] 97 msr tpidr_el2, x1 98 99 ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA] 100 mov sp, x1 101 102 ldr x1, [x0, #NVHE_INIT_MAIR_EL2] 103 msr mair_el2, x1 104 105 ldr x1, [x0, #NVHE_INIT_PGD_PA] 106 phys_to_ttbr x2, x1 107alternative_if ARM64_HAS_CNP 108 orr x2, x2, #TTBR_CNP_BIT 109alternative_else_nop_endif 110 msr ttbr0_el2, x2 111 112 /* 113 * Set the PS bits in TCR_EL2. 114 */ 115 ldr x1, [x0, #NVHE_INIT_TCR_EL2] 116 tcr_compute_pa_size x1, #TCR_EL2_PS_SHIFT, x2, x3 117 msr tcr_el2, x1 118 119 isb 120 121 /* Invalidate the stale TLBs from Bootloader */ 122 tlbi alle2 123 dsb sy 124 125 /* 126 * Preserve all the RES1 bits while setting the default flags, 127 * as well as the EE bit on BE. Drop the A flag since the compiler 128 * is allowed to generate unaligned accesses. 129 */ 130 mov_q x0, (SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A)) 131CPU_BE( orr x0, x0, #SCTLR_ELx_EE) 132alternative_if ARM64_HAS_ADDRESS_AUTH 133 mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \ 134 SCTLR_ELx_ENDA | SCTLR_ELx_ENDB) 135 orr x0, x0, x1 136alternative_else_nop_endif 137 msr sctlr_el2, x0 138 isb 139 140 /* Set the host vector */ 141 ldr x0, =__kvm_hyp_host_vector 142 kimg_hyp_va x0, x1 143 msr vbar_el2, x0 144 145 ret 146SYM_CODE_END(___kvm_hyp_init) 147 148/* 149 * PSCI CPU_ON entry point 150 * 151 * x0: struct kvm_nvhe_init_params PA 152 */ 153SYM_CODE_START(kvm_hyp_cpu_entry) 154 mov x1, #1 // is_cpu_on = true 155 b __kvm_hyp_init_cpu 156SYM_CODE_END(kvm_hyp_cpu_entry) 157 158/* 159 * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point 160 * 161 * x0: struct kvm_nvhe_init_params PA 162 */ 163SYM_CODE_START(kvm_hyp_cpu_resume) 164 mov x1, #0 // is_cpu_on = false 165 b __kvm_hyp_init_cpu 166SYM_CODE_END(kvm_hyp_cpu_resume) 167 168/* 169 * Common code for CPU entry points. Initializes EL2 state and 170 * installs the hypervisor before handing over to a C handler. 171 * 172 * x0: struct kvm_nvhe_init_params PA 173 * x1: bool is_cpu_on 174 */ 175SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) 176 mov x28, x0 // Stash arguments 177 mov x29, x1 178 179 /* Check that the core was booted in EL2. */ 180 mrs x0, CurrentEL 181 cmp x0, #CurrentEL_EL2 182 b.eq 2f 183 184 /* The core booted in EL1. KVM cannot be initialized on it. */ 1851: wfe 186 wfi 187 b 1b 188 1892: msr SPsel, #1 // We want to use SP_EL{1,2} 190 191 /* Initialize EL2 CPU state to sane values. */ 192 init_el2_state nvhe // Clobbers x0..x2 193 194 /* Enable MMU, set vectors and stack. */ 195 mov x0, x28 196 bl ___kvm_hyp_init // Clobbers x0..x3 197 198 /* Leave idmap. */ 199 mov x0, x29 200 ldr x1, =kvm_host_psci_cpu_entry 201 kimg_hyp_va x1, x2 202 br x1 203SYM_CODE_END(__kvm_hyp_init_cpu) 204 205SYM_CODE_START(__kvm_handle_stub_hvc) 206 cmp x0, #HVC_SOFT_RESTART 207 b.ne 1f 208 209 /* This is where we're about to jump, staying at EL2 */ 210 msr elr_el2, x1 211 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT | PSR_MODE_EL2h) 212 msr spsr_el2, x0 213 214 /* Shuffle the arguments, and don't come back */ 215 mov x0, x2 216 mov x1, x3 217 mov x2, x4 218 b reset 219 2201: cmp x0, #HVC_RESET_VECTORS 221 b.ne 1f 222 223 /* 224 * Set the HVC_RESET_VECTORS return code before entering the common 225 * path so that we do not clobber x0-x2 in case we are coming via 226 * HVC_SOFT_RESTART. 227 */ 228 mov x0, xzr 229reset: 230 /* Reset kvm back to the hyp stub. */ 231 mrs x5, sctlr_el2 232 mov_q x6, SCTLR_ELx_FLAGS 233 bic x5, x5, x6 // Clear SCTL_M and etc 234 pre_disable_mmu_workaround 235 msr sctlr_el2, x5 236 isb 237 238alternative_if ARM64_KVM_PROTECTED_MODE 239 mov_q x5, HCR_HOST_NVHE_FLAGS 240 msr hcr_el2, x5 241alternative_else_nop_endif 242 243 /* Install stub vectors */ 244 adr_l x5, __hyp_stub_vectors 245 msr vbar_el2, x5 246 eret 247 2481: /* Bad stub call */ 249 mov_q x0, HVC_STUB_ERR 250 eret 251 252SYM_CODE_END(__kvm_handle_stub_hvc) 253 254 .popsection 255