1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2015-2018 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/arm-smccc.h> 8#include <linux/linkage.h> 9 10#include <asm/alternative.h> 11#include <asm/assembler.h> 12#include <asm/cpufeature.h> 13#include <asm/kvm_arm.h> 14#include <asm/kvm_asm.h> 15#include <asm/mmu.h> 16#include <asm/spectre.h> 17 18.macro save_caller_saved_regs_vect 19 /* x0 and x1 were saved in the vector entry */ 20 stp x2, x3, [sp, #-16]! 21 stp x4, x5, [sp, #-16]! 22 stp x6, x7, [sp, #-16]! 23 stp x8, x9, [sp, #-16]! 24 stp x10, x11, [sp, #-16]! 25 stp x12, x13, [sp, #-16]! 26 stp x14, x15, [sp, #-16]! 27 stp x16, x17, [sp, #-16]! 28.endm 29 30.macro restore_caller_saved_regs_vect 31 ldp x16, x17, [sp], #16 32 ldp x14, x15, [sp], #16 33 ldp x12, x13, [sp], #16 34 ldp x10, x11, [sp], #16 35 ldp x8, x9, [sp], #16 36 ldp x6, x7, [sp], #16 37 ldp x4, x5, [sp], #16 38 ldp x2, x3, [sp], #16 39 ldp x0, x1, [sp], #16 40.endm 41 42 .text 43 44el1_sync: // Guest trapped into EL2 45 46 mrs x0, esr_el2 47 ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH 48 cmp x0, #ESR_ELx_EC_HVC64 49 ccmp x0, #ESR_ELx_EC_HVC32, #4, ne 50 b.ne el1_trap 51 52 /* 53 * Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1. 54 * The workaround has already been applied on the host, 55 * so let's quickly get back to the guest. We don't bother 56 * restoring x1, as it can be clobbered anyway. 57 */ 58 ldr x1, [sp] // Guest's x0 59 eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1 60 cbz w1, wa_epilogue 61 62 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */ 63 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \ 64 ARM_SMCCC_ARCH_WORKAROUND_2) 65 cbz w1, wa_epilogue 66 67 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \ 68 ARM_SMCCC_ARCH_WORKAROUND_3) 69 cbnz w1, el1_trap 70 71wa_epilogue: 72 mov x0, xzr 73 add sp, sp, #16 74 eret 75 sb 76 77el1_trap: 78 get_vcpu_ptr x1, x0 79 mov x0, #ARM_EXCEPTION_TRAP 80 b __guest_exit 81 82el1_irq: 83el1_fiq: 84 get_vcpu_ptr x1, x0 85 mov x0, #ARM_EXCEPTION_IRQ 86 b __guest_exit 87 88el1_error: 89 get_vcpu_ptr x1, x0 90 mov x0, #ARM_EXCEPTION_EL1_SERROR 91 b __guest_exit 92 93el2_sync: 94 /* Check for illegal exception return */ 95 mrs x0, spsr_el2 96 tbnz x0, #20, 1f 97 98 save_caller_saved_regs_vect 99 stp x29, x30, [sp, #-16]! 100 bl kvm_unexpected_el2_exception 101 ldp x29, x30, [sp], #16 102 restore_caller_saved_regs_vect 103 104 eret 105 1061: 107 /* Let's attempt a recovery from the illegal exception return */ 108 get_vcpu_ptr x1, x0 109 mov x0, #ARM_EXCEPTION_IL 110 b __guest_exit 111 112 113el2_error: 114 save_caller_saved_regs_vect 115 stp x29, x30, [sp, #-16]! 116 117 bl kvm_unexpected_el2_exception 118 119 ldp x29, x30, [sp], #16 120 restore_caller_saved_regs_vect 121 122 eret 123 sb 124 125.macro invalid_vector label, target = __guest_exit_panic 126 .align 2 127SYM_CODE_START_LOCAL(\label) 128 b \target 129SYM_CODE_END(\label) 130.endm 131 132 /* None of these should ever happen */ 133 invalid_vector el2t_sync_invalid 134 invalid_vector el2t_irq_invalid 135 invalid_vector el2t_fiq_invalid 136 invalid_vector el2t_error_invalid 137 invalid_vector el2h_irq_invalid 138 invalid_vector el2h_fiq_invalid 139 140 .ltorg 141 142 .align 11 143 144.macro check_preamble_length start, end 145/* kvm_patch_vector_branch() generates code that jumps over the preamble. */ 146.if ((\end-\start) != KVM_VECTOR_PREAMBLE) 147 .error "KVM vector preamble length mismatch" 148.endif 149.endm 150 151.macro valid_vect target 152 .align 7 153661: 154 esb 155 stp x0, x1, [sp, #-16]! 156662: 157 /* 158 * spectre vectors __bp_harden_hyp_vecs generate br instructions at runtime 159 * that jump at offset 8 at __kvm_hyp_vector. 160 * As hyp .text is guarded section, it needs bti j. 161 */ 162 bti j 163 b \target 164 165check_preamble_length 661b, 662b 166.endm 167 168.macro invalid_vect target 169 .align 7 170661: 171 nop 172 stp x0, x1, [sp, #-16]! 173662: 174 /* Check valid_vect */ 175 bti j 176 b \target 177 178check_preamble_length 661b, 662b 179.endm 180 181SYM_CODE_START(__kvm_hyp_vector) 182 invalid_vect el2t_sync_invalid // Synchronous EL2t 183 invalid_vect el2t_irq_invalid // IRQ EL2t 184 invalid_vect el2t_fiq_invalid // FIQ EL2t 185 invalid_vect el2t_error_invalid // Error EL2t 186 187 valid_vect el2_sync // Synchronous EL2h 188 invalid_vect el2h_irq_invalid // IRQ EL2h 189 invalid_vect el2h_fiq_invalid // FIQ EL2h 190 valid_vect el2_error // Error EL2h 191 192 valid_vect el1_sync // Synchronous 64-bit EL1 193 valid_vect el1_irq // IRQ 64-bit EL1 194 valid_vect el1_fiq // FIQ 64-bit EL1 195 valid_vect el1_error // Error 64-bit EL1 196 197 valid_vect el1_sync // Synchronous 32-bit EL1 198 valid_vect el1_irq // IRQ 32-bit EL1 199 valid_vect el1_fiq // FIQ 32-bit EL1 200 valid_vect el1_error // Error 32-bit EL1 201SYM_CODE_END(__kvm_hyp_vector) 202 203.macro spectrev2_smccc_wa1_smc 204 sub sp, sp, #(8 * 4) 205 stp x2, x3, [sp, #(8 * 0)] 206 stp x0, x1, [sp, #(8 * 2)] 207 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_wa3 208 /* Patched to mov WA3 when supported */ 209 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1 210 alternative_cb_end 211 smc #0 212 ldp x2, x3, [sp, #(8 * 0)] 213 add sp, sp, #(8 * 2) 214.endm 215 216.macro hyp_ventry indirect, spectrev2 217 .align 7 2181: esb 219 .if \spectrev2 != 0 220 spectrev2_smccc_wa1_smc 221 .else 222 stp x0, x1, [sp, #-16]! 223 mitigate_spectre_bhb_loop x0 224 mitigate_spectre_bhb_clear_insn 225 .endif 226 .if \indirect != 0 227 alternative_cb ARM64_ALWAYS_SYSTEM, kvm_patch_vector_branch 228 /* 229 * For ARM64_SPECTRE_V3A configurations, these NOPs get replaced with: 230 * 231 * movz x0, #(addr & 0xffff) 232 * movk x0, #((addr >> 16) & 0xffff), lsl #16 233 * movk x0, #((addr >> 32) & 0xffff), lsl #32 234 * br x0 235 * 236 * Where: 237 * addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + KVM_VECTOR_PREAMBLE. 238 * See kvm_patch_vector_branch for details. 239 */ 240 nop 241 nop 242 nop 243 nop 244 alternative_cb_end 245 .endif 246 b __kvm_hyp_vector + (1b - 0b + KVM_VECTOR_PREAMBLE) 247.endm 248 249.macro generate_vectors indirect, spectrev2 2500: 251 .rept 16 252 hyp_ventry \indirect, \spectrev2 253 .endr 254 .org 0b + SZ_2K // Safety measure 255.endm 256 257 .align 11 258SYM_CODE_START(__bp_harden_hyp_vecs) 259 generate_vectors indirect = 0, spectrev2 = 1 // HYP_VECTOR_SPECTRE_DIRECT 260 generate_vectors indirect = 1, spectrev2 = 0 // HYP_VECTOR_INDIRECT 261 generate_vectors indirect = 1, spectrev2 = 1 // HYP_VECTOR_SPECTRE_INDIRECT 2621: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ 263 .org 1b 264SYM_CODE_END(__bp_harden_hyp_vecs) 265