1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2015-2018 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/arm-smccc.h> 8#include <linux/linkage.h> 9 10#include <asm/alternative.h> 11#include <asm/assembler.h> 12#include <asm/cpufeature.h> 13#include <asm/kvm_arm.h> 14#include <asm/kvm_asm.h> 15#include <asm/kvm_mmu.h> 16#include <asm/mmu.h> 17 18.macro save_caller_saved_regs_vect 19 /* x0 and x1 were saved in the vector entry */ 20 stp x2, x3, [sp, #-16]! 21 stp x4, x5, [sp, #-16]! 22 stp x6, x7, [sp, #-16]! 23 stp x8, x9, [sp, #-16]! 24 stp x10, x11, [sp, #-16]! 25 stp x12, x13, [sp, #-16]! 26 stp x14, x15, [sp, #-16]! 27 stp x16, x17, [sp, #-16]! 28.endm 29 30.macro restore_caller_saved_regs_vect 31 ldp x16, x17, [sp], #16 32 ldp x14, x15, [sp], #16 33 ldp x12, x13, [sp], #16 34 ldp x10, x11, [sp], #16 35 ldp x8, x9, [sp], #16 36 ldp x6, x7, [sp], #16 37 ldp x4, x5, [sp], #16 38 ldp x2, x3, [sp], #16 39 ldp x0, x1, [sp], #16 40.endm 41 42 .text 43 44.macro do_el2_call 45 /* 46 * Shuffle the parameters before calling the function 47 * pointed to in x0. Assumes parameters in x[1,2,3]. 48 */ 49 str lr, [sp, #-16]! 50 mov lr, x0 51 mov x0, x1 52 mov x1, x2 53 mov x2, x3 54 blr lr 55 ldr lr, [sp], #16 56.endm 57 58el1_sync: // Guest trapped into EL2 59 60 mrs x0, esr_el2 61 lsr x0, x0, #ESR_ELx_EC_SHIFT 62 cmp x0, #ESR_ELx_EC_HVC64 63 ccmp x0, #ESR_ELx_EC_HVC32, #4, ne 64 b.ne el1_trap 65 66#ifdef __KVM_NVHE_HYPERVISOR__ 67 mrs x1, vttbr_el2 // If vttbr is valid, the guest 68 cbnz x1, el1_hvc_guest // called HVC 69 70 /* Here, we're pretty sure the host called HVC. */ 71 ldp x0, x1, [sp], #16 72 73 /* Check for a stub HVC call */ 74 cmp x0, #HVC_STUB_HCALL_NR 75 b.hs 1f 76 77 /* 78 * Compute the idmap address of __kvm_handle_stub_hvc and 79 * jump there. Since we use kimage_voffset, do not use the 80 * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead 81 * (by loading it from the constant pool). 82 * 83 * Preserve x0-x4, which may contain stub parameters. 84 */ 85 ldr x5, =__kvm_handle_stub_hvc 86 ldr_l x6, kimage_voffset 87 88 /* x5 = __pa(x5) */ 89 sub x5, x5, x6 90 br x5 91 921: 93 /* 94 * Perform the EL2 call 95 */ 96 kern_hyp_va x0 97 do_el2_call 98 99 eret 100 sb 101#endif /* __KVM_NVHE_HYPERVISOR__ */ 102 103el1_hvc_guest: 104 /* 105 * Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1. 106 * The workaround has already been applied on the host, 107 * so let's quickly get back to the guest. We don't bother 108 * restoring x1, as it can be clobbered anyway. 109 */ 110 ldr x1, [sp] // Guest's x0 111 eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1 112 cbz w1, wa_epilogue 113 114 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */ 115 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \ 116 ARM_SMCCC_ARCH_WORKAROUND_2) 117 cbnz w1, el1_trap 118 119#ifdef CONFIG_ARM64_SSBD 120alternative_cb arm64_enable_wa2_handling 121 b wa2_end 122alternative_cb_end 123 get_vcpu_ptr x2, x0 124 ldr x0, [x2, #VCPU_WORKAROUND_FLAGS] 125 126 // Sanitize the argument and update the guest flags 127 ldr x1, [sp, #8] // Guest's x1 128 clz w1, w1 // Murphy's device: 129 lsr w1, w1, #5 // w1 = !!w1 without using 130 eor w1, w1, #1 // the flags... 131 bfi x0, x1, #VCPU_WORKAROUND_2_FLAG_SHIFT, #1 132 str x0, [x2, #VCPU_WORKAROUND_FLAGS] 133 134 /* Check that we actually need to perform the call */ 135 hyp_ldr_this_cpu x0, arm64_ssbd_callback_required, x2 136 cbz x0, wa2_end 137 138 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 139 smc #0 140 141 /* Don't leak data from the SMC call */ 142 mov x3, xzr 143wa2_end: 144 mov x2, xzr 145 mov x1, xzr 146#endif 147 148wa_epilogue: 149 mov x0, xzr 150 add sp, sp, #16 151 eret 152 sb 153 154el1_trap: 155 get_vcpu_ptr x1, x0 156 mov x0, #ARM_EXCEPTION_TRAP 157 b __guest_exit 158 159el1_irq: 160 get_vcpu_ptr x1, x0 161 mov x0, #ARM_EXCEPTION_IRQ 162 b __guest_exit 163 164el1_error: 165 get_vcpu_ptr x1, x0 166 mov x0, #ARM_EXCEPTION_EL1_SERROR 167 b __guest_exit 168 169el2_sync: 170 /* Check for illegal exception return */ 171 mrs x0, spsr_el2 172 tbnz x0, #20, 1f 173 174 save_caller_saved_regs_vect 175 stp x29, x30, [sp, #-16]! 176 bl kvm_unexpected_el2_exception 177 ldp x29, x30, [sp], #16 178 restore_caller_saved_regs_vect 179 180 eret 181 1821: 183 /* Let's attempt a recovery from the illegal exception return */ 184 get_vcpu_ptr x1, x0 185 mov x0, #ARM_EXCEPTION_IL 186 b __guest_exit 187 188 189el2_error: 190 save_caller_saved_regs_vect 191 stp x29, x30, [sp, #-16]! 192 193 bl kvm_unexpected_el2_exception 194 195 ldp x29, x30, [sp], #16 196 restore_caller_saved_regs_vect 197 198 eret 199 sb 200 201#ifdef __KVM_NVHE_HYPERVISOR__ 202SYM_FUNC_START(__hyp_do_panic) 203 mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 204 PSR_MODE_EL1h) 205 msr spsr_el2, lr 206 ldr lr, =panic 207 msr elr_el2, lr 208 eret 209 sb 210SYM_FUNC_END(__hyp_do_panic) 211#endif 212 213SYM_CODE_START(__hyp_panic) 214 get_host_ctxt x0, x1 215 b hyp_panic 216SYM_CODE_END(__hyp_panic) 217 218.macro invalid_vector label, target = __hyp_panic 219 .align 2 220SYM_CODE_START(\label) 221 b \target 222SYM_CODE_END(\label) 223.endm 224 225 /* None of these should ever happen */ 226 invalid_vector el2t_sync_invalid 227 invalid_vector el2t_irq_invalid 228 invalid_vector el2t_fiq_invalid 229 invalid_vector el2t_error_invalid 230 invalid_vector el2h_sync_invalid 231 invalid_vector el2h_irq_invalid 232 invalid_vector el2h_fiq_invalid 233 invalid_vector el1_fiq_invalid 234 235 .ltorg 236 237 .align 11 238 239.macro check_preamble_length start, end 240/* kvm_patch_vector_branch() generates code that jumps over the preamble. */ 241.if ((\end-\start) != KVM_VECTOR_PREAMBLE) 242 .error "KVM vector preamble length mismatch" 243.endif 244.endm 245 246.macro valid_vect target 247 .align 7 248661: 249 esb 250 stp x0, x1, [sp, #-16]! 251662: 252 b \target 253 254check_preamble_length 661b, 662b 255.endm 256 257.macro invalid_vect target 258 .align 7 259661: 260 b \target 261 nop 262662: 263 ldp x0, x1, [sp], #16 264 b \target 265 266check_preamble_length 661b, 662b 267.endm 268 269SYM_CODE_START(__kvm_hyp_vector) 270 invalid_vect el2t_sync_invalid // Synchronous EL2t 271 invalid_vect el2t_irq_invalid // IRQ EL2t 272 invalid_vect el2t_fiq_invalid // FIQ EL2t 273 invalid_vect el2t_error_invalid // Error EL2t 274 275 valid_vect el2_sync // Synchronous EL2h 276 invalid_vect el2h_irq_invalid // IRQ EL2h 277 invalid_vect el2h_fiq_invalid // FIQ EL2h 278 valid_vect el2_error // Error EL2h 279 280 valid_vect el1_sync // Synchronous 64-bit EL1 281 valid_vect el1_irq // IRQ 64-bit EL1 282 invalid_vect el1_fiq_invalid // FIQ 64-bit EL1 283 valid_vect el1_error // Error 64-bit EL1 284 285 valid_vect el1_sync // Synchronous 32-bit EL1 286 valid_vect el1_irq // IRQ 32-bit EL1 287 invalid_vect el1_fiq_invalid // FIQ 32-bit EL1 288 valid_vect el1_error // Error 32-bit EL1 289SYM_CODE_END(__kvm_hyp_vector) 290 291#ifdef CONFIG_KVM_INDIRECT_VECTORS 292.macro hyp_ventry 293 .align 7 2941: esb 295 .rept 26 296 nop 297 .endr 298/* 299 * The default sequence is to directly branch to the KVM vectors, 300 * using the computed offset. This applies for VHE as well as 301 * !ARM64_HARDEN_EL2_VECTORS. The first vector must always run the preamble. 302 * 303 * For ARM64_HARDEN_EL2_VECTORS configurations, this gets replaced 304 * with: 305 * 306 * stp x0, x1, [sp, #-16]! 307 * movz x0, #(addr & 0xffff) 308 * movk x0, #((addr >> 16) & 0xffff), lsl #16 309 * movk x0, #((addr >> 32) & 0xffff), lsl #32 310 * br x0 311 * 312 * Where: 313 * addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + KVM_VECTOR_PREAMBLE. 314 * See kvm_patch_vector_branch for details. 315 */ 316alternative_cb kvm_patch_vector_branch 317 stp x0, x1, [sp, #-16]! 318 b __kvm_hyp_vector + (1b - 0b + KVM_VECTOR_PREAMBLE) 319 nop 320 nop 321 nop 322alternative_cb_end 323.endm 324 325.macro generate_vectors 3260: 327 .rept 16 328 hyp_ventry 329 .endr 330 .org 0b + SZ_2K // Safety measure 331.endm 332 333 .align 11 334SYM_CODE_START(__bp_harden_hyp_vecs) 335 .rept BP_HARDEN_EL2_SLOTS 336 generate_vectors 337 .endr 3381: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ 339 .org 1b 340SYM_CODE_END(__bp_harden_hyp_vecs) 341#endif 342