1/* 2 * Copyright (C) 2015-2018 - ARM Ltd 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18#include <linux/arm-smccc.h> 19#include <linux/linkage.h> 20 21#include <asm/alternative.h> 22#include <asm/assembler.h> 23#include <asm/cpufeature.h> 24#include <asm/kvm_arm.h> 25#include <asm/kvm_asm.h> 26#include <asm/kvm_mmu.h> 27#include <asm/mmu.h> 28 29 .text 30 .pushsection .hyp.text, "ax" 31 32.macro do_el2_call 33 /* 34 * Shuffle the parameters before calling the function 35 * pointed to in x0. Assumes parameters in x[1,2,3]. 36 */ 37 str lr, [sp, #-16]! 38 mov lr, x0 39 mov x0, x1 40 mov x1, x2 41 mov x2, x3 42 blr lr 43 ldr lr, [sp], #16 44.endm 45 46ENTRY(__vhe_hyp_call) 47 do_el2_call 48 /* 49 * We used to rely on having an exception return to get 50 * an implicit isb. In the E2H case, we don't have it anymore. 51 * rather than changing all the leaf functions, just do it here 52 * before returning to the rest of the kernel. 53 */ 54 isb 55 ret 56ENDPROC(__vhe_hyp_call) 57 58el1_sync: // Guest trapped into EL2 59 60 mrs x0, esr_el2 61 lsr x0, x0, #ESR_ELx_EC_SHIFT 62 cmp x0, #ESR_ELx_EC_HVC64 63 ccmp x0, #ESR_ELx_EC_HVC32, #4, ne 64 b.ne el1_trap 65 66 mrs x1, vttbr_el2 // If vttbr is valid, the guest 67 cbnz x1, el1_hvc_guest // called HVC 68 69 /* Here, we're pretty sure the host called HVC. */ 70 ldp x0, x1, [sp], #16 71 72 /* Check for a stub HVC call */ 73 cmp x0, #HVC_STUB_HCALL_NR 74 b.hs 1f 75 76 /* 77 * Compute the idmap address of __kvm_handle_stub_hvc and 78 * jump there. Since we use kimage_voffset, do not use the 79 * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead 80 * (by loading it from the constant pool). 81 * 82 * Preserve x0-x4, which may contain stub parameters. 83 */ 84 ldr x5, =__kvm_handle_stub_hvc 85 ldr_l x6, kimage_voffset 86 87 /* x5 = __pa(x5) */ 88 sub x5, x5, x6 89 br x5 90 911: 92 /* 93 * Perform the EL2 call 94 */ 95 kern_hyp_va x0 96 do_el2_call 97 98 eret 99 100el1_hvc_guest: 101 /* 102 * Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1. 103 * The workaround has already been applied on the host, 104 * so let's quickly get back to the guest. We don't bother 105 * restoring x1, as it can be clobbered anyway. 106 */ 107 ldr x1, [sp] // Guest's x0 108 eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1 109 cbz w1, wa_epilogue 110 111 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */ 112 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \ 113 ARM_SMCCC_ARCH_WORKAROUND_2) 114 cbnz w1, el1_trap 115 116#ifdef CONFIG_ARM64_SSBD 117alternative_cb arm64_enable_wa2_handling 118 b wa2_end 119alternative_cb_end 120 get_vcpu_ptr x2, x0 121 ldr x0, [x2, #VCPU_WORKAROUND_FLAGS] 122 123 // Sanitize the argument and update the guest flags 124 ldr x1, [sp, #8] // Guest's x1 125 clz w1, w1 // Murphy's device: 126 lsr w1, w1, #5 // w1 = !!w1 without using 127 eor w1, w1, #1 // the flags... 128 bfi x0, x1, #VCPU_WORKAROUND_2_FLAG_SHIFT, #1 129 str x0, [x2, #VCPU_WORKAROUND_FLAGS] 130 131 /* Check that we actually need to perform the call */ 132 hyp_ldr_this_cpu x0, arm64_ssbd_callback_required, x2 133 cbz x0, wa2_end 134 135 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 136 smc #0 137 138 /* Don't leak data from the SMC call */ 139 mov x3, xzr 140wa2_end: 141 mov x2, xzr 142 mov x1, xzr 143#endif 144 145wa_epilogue: 146 mov x0, xzr 147 add sp, sp, #16 148 eret 149 150el1_trap: 151 get_vcpu_ptr x1, x0 152 mov x0, #ARM_EXCEPTION_TRAP 153 b __guest_exit 154 155el1_irq: 156 get_vcpu_ptr x1, x0 157 mov x0, #ARM_EXCEPTION_IRQ 158 b __guest_exit 159 160el1_error: 161 get_vcpu_ptr x1, x0 162 mov x0, #ARM_EXCEPTION_EL1_SERROR 163 b __guest_exit 164 165el2_sync: 166 /* Check for illegal exception return, otherwise panic */ 167 mrs x0, spsr_el2 168 169 /* if this was something else, then panic! */ 170 tst x0, #PSR_IL_BIT 171 b.eq __hyp_panic 172 173 /* Let's attempt a recovery from the illegal exception return */ 174 get_vcpu_ptr x1, x0 175 mov x0, #ARM_EXCEPTION_IL 176 b __guest_exit 177 178 179el2_error: 180 ldp x0, x1, [sp], #16 181 182 /* 183 * Only two possibilities: 184 * 1) Either we come from the exit path, having just unmasked 185 * PSTATE.A: change the return code to an EL2 fault, and 186 * carry on, as we're already in a sane state to handle it. 187 * 2) Or we come from anywhere else, and that's a bug: we panic. 188 * 189 * For (1), x0 contains the original return code and x1 doesn't 190 * contain anything meaningful at that stage. We can reuse them 191 * as temp registers. 192 * For (2), who cares? 193 */ 194 mrs x0, elr_el2 195 adr x1, abort_guest_exit_start 196 cmp x0, x1 197 adr x1, abort_guest_exit_end 198 ccmp x0, x1, #4, ne 199 b.ne __hyp_panic 200 mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT) 201 eret 202 203ENTRY(__hyp_do_panic) 204 mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 205 PSR_MODE_EL1h) 206 msr spsr_el2, lr 207 ldr lr, =panic 208 msr elr_el2, lr 209 eret 210ENDPROC(__hyp_do_panic) 211 212ENTRY(__hyp_panic) 213 get_host_ctxt x0, x1 214 b hyp_panic 215ENDPROC(__hyp_panic) 216 217.macro invalid_vector label, target = __hyp_panic 218 .align 2 219\label: 220 b \target 221ENDPROC(\label) 222.endm 223 224 /* None of these should ever happen */ 225 invalid_vector el2t_sync_invalid 226 invalid_vector el2t_irq_invalid 227 invalid_vector el2t_fiq_invalid 228 invalid_vector el2t_error_invalid 229 invalid_vector el2h_sync_invalid 230 invalid_vector el2h_irq_invalid 231 invalid_vector el2h_fiq_invalid 232 invalid_vector el1_fiq_invalid 233 234 .ltorg 235 236 .align 11 237 238.macro valid_vect target 239 .align 7 240 stp x0, x1, [sp, #-16]! 241 b \target 242.endm 243 244.macro invalid_vect target 245 .align 7 246 b \target 247 ldp x0, x1, [sp], #16 248 b \target 249.endm 250 251ENTRY(__kvm_hyp_vector) 252 invalid_vect el2t_sync_invalid // Synchronous EL2t 253 invalid_vect el2t_irq_invalid // IRQ EL2t 254 invalid_vect el2t_fiq_invalid // FIQ EL2t 255 invalid_vect el2t_error_invalid // Error EL2t 256 257 valid_vect el2_sync // Synchronous EL2h 258 invalid_vect el2h_irq_invalid // IRQ EL2h 259 invalid_vect el2h_fiq_invalid // FIQ EL2h 260 valid_vect el2_error // Error EL2h 261 262 valid_vect el1_sync // Synchronous 64-bit EL1 263 valid_vect el1_irq // IRQ 64-bit EL1 264 invalid_vect el1_fiq_invalid // FIQ 64-bit EL1 265 valid_vect el1_error // Error 64-bit EL1 266 267 valid_vect el1_sync // Synchronous 32-bit EL1 268 valid_vect el1_irq // IRQ 32-bit EL1 269 invalid_vect el1_fiq_invalid // FIQ 32-bit EL1 270 valid_vect el1_error // Error 32-bit EL1 271ENDPROC(__kvm_hyp_vector) 272 273#ifdef CONFIG_KVM_INDIRECT_VECTORS 274.macro hyp_ventry 275 .align 7 2761: .rept 27 277 nop 278 .endr 279/* 280 * The default sequence is to directly branch to the KVM vectors, 281 * using the computed offset. This applies for VHE as well as 282 * !ARM64_HARDEN_EL2_VECTORS. 283 * 284 * For ARM64_HARDEN_EL2_VECTORS configurations, this gets replaced 285 * with: 286 * 287 * stp x0, x1, [sp, #-16]! 288 * movz x0, #(addr & 0xffff) 289 * movk x0, #((addr >> 16) & 0xffff), lsl #16 290 * movk x0, #((addr >> 32) & 0xffff), lsl #32 291 * br x0 292 * 293 * Where addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + 4. 294 * See kvm_patch_vector_branch for details. 295 */ 296alternative_cb kvm_patch_vector_branch 297 b __kvm_hyp_vector + (1b - 0b) 298 nop 299 nop 300 nop 301 nop 302alternative_cb_end 303.endm 304 305.macro generate_vectors 3060: 307 .rept 16 308 hyp_ventry 309 .endr 310 .org 0b + SZ_2K // Safety measure 311.endm 312 313 .align 11 314ENTRY(__bp_harden_hyp_vecs_start) 315 .rept BP_HARDEN_EL2_SLOTS 316 generate_vectors 317 .endr 318ENTRY(__bp_harden_hyp_vecs_end) 319 320 .popsection 321 322ENTRY(__smccc_workaround_1_smc_start) 323 sub sp, sp, #(8 * 4) 324 stp x2, x3, [sp, #(8 * 0)] 325 stp x0, x1, [sp, #(8 * 2)] 326 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1 327 smc #0 328 ldp x2, x3, [sp, #(8 * 0)] 329 ldp x0, x1, [sp, #(8 * 2)] 330 add sp, sp, #(8 * 4) 331ENTRY(__smccc_workaround_1_smc_end) 332#endif 333