1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __KVM_X86_VMX_INSN_H 3 #define __KVM_X86_VMX_INSN_H 4 5 #include <linux/nospec.h> 6 7 #include <asm/vmx.h> 8 9 #include "hyperv.h" 10 #include "vmcs.h" 11 #include "../x86.h" 12 13 void vmread_error(unsigned long field, bool fault); 14 void vmwrite_error(unsigned long field, unsigned long value); 15 void vmclear_error(struct vmcs *vmcs, u64 phys_addr); 16 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr); 17 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva); 18 void invept_error(unsigned long ext, u64 eptp, gpa_t gpa); 19 20 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 21 /* 22 * The VMREAD error trampoline _always_ uses the stack to pass parameters, even 23 * for 64-bit targets. Preserving all registers allows the VMREAD inline asm 24 * blob to avoid clobbering GPRs, which in turn allows the compiler to better 25 * optimize sequences of VMREADs. 26 * 27 * Declare the trampoline as an opaque label as it's not safe to call from C 28 * code; there is no way to tell the compiler to pass params on the stack for 29 * 64-bit targets. 30 * 31 * void vmread_error_trampoline(unsigned long field, bool fault); 32 */ 33 extern unsigned long vmread_error_trampoline; 34 #endif 35 36 static __always_inline void vmcs_check16(unsigned long field) 37 { 38 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 39 "16-bit accessor invalid for 64-bit field"); 40 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 41 "16-bit accessor invalid for 64-bit high field"); 42 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 43 "16-bit accessor invalid for 32-bit high field"); 44 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 45 "16-bit accessor invalid for natural width field"); 46 } 47 48 static __always_inline void vmcs_check32(unsigned long field) 49 { 50 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 51 "32-bit accessor invalid for 16-bit field"); 52 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 53 "32-bit accessor invalid for 64-bit field"); 54 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 55 "32-bit accessor invalid for 64-bit high field"); 56 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 57 "32-bit accessor invalid for natural width field"); 58 } 59 60 static __always_inline void vmcs_check64(unsigned long field) 61 { 62 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 63 "64-bit accessor invalid for 16-bit field"); 64 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 65 "64-bit accessor invalid for 64-bit high field"); 66 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 67 "64-bit accessor invalid for 32-bit field"); 68 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 69 "64-bit accessor invalid for natural width field"); 70 } 71 72 static __always_inline void vmcs_checkl(unsigned long field) 73 { 74 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 75 "Natural width accessor invalid for 16-bit field"); 76 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 77 "Natural width accessor invalid for 64-bit field"); 78 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 79 "Natural width accessor invalid for 64-bit high field"); 80 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 81 "Natural width accessor invalid for 32-bit field"); 82 } 83 84 static __always_inline unsigned long __vmcs_readl(unsigned long field) 85 { 86 unsigned long value; 87 88 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 89 90 asm_volatile_goto("1: vmread %[field], %[output]\n\t" 91 "jna %l[do_fail]\n\t" 92 93 _ASM_EXTABLE(1b, %l[do_exception]) 94 95 : [output] "=r" (value) 96 : [field] "r" (field) 97 : "cc" 98 : do_fail, do_exception); 99 100 return value; 101 102 do_fail: 103 WARN_ONCE(1, "kvm: vmread failed: field=%lx\n", field); 104 pr_warn_ratelimited("kvm: vmread failed: field=%lx\n", field); 105 return 0; 106 107 do_exception: 108 kvm_spurious_fault(); 109 return 0; 110 111 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 112 113 asm volatile("1: vmread %2, %1\n\t" 114 ".byte 0x3e\n\t" /* branch taken hint */ 115 "ja 3f\n\t" 116 117 /* 118 * VMREAD failed. Push '0' for @fault, push the failing 119 * @field, and bounce through the trampoline to preserve 120 * volatile registers. 121 */ 122 "xorl %k1, %k1\n\t" 123 "2:\n\t" 124 "push %1\n\t" 125 "push %2\n\t" 126 "call vmread_error_trampoline\n\t" 127 128 /* 129 * Unwind the stack. Note, the trampoline zeros out the 130 * memory for @fault so that the result is '0' on error. 131 */ 132 "pop %2\n\t" 133 "pop %1\n\t" 134 "3:\n\t" 135 136 /* VMREAD faulted. As above, except push '1' for @fault. */ 137 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %1) 138 139 : ASM_CALL_CONSTRAINT, "=&r"(value) : "r"(field) : "cc"); 140 return value; 141 142 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 143 } 144 145 static __always_inline u16 vmcs_read16(unsigned long field) 146 { 147 vmcs_check16(field); 148 if (static_branch_unlikely(&enable_evmcs)) 149 return evmcs_read16(field); 150 return __vmcs_readl(field); 151 } 152 153 static __always_inline u32 vmcs_read32(unsigned long field) 154 { 155 vmcs_check32(field); 156 if (static_branch_unlikely(&enable_evmcs)) 157 return evmcs_read32(field); 158 return __vmcs_readl(field); 159 } 160 161 static __always_inline u64 vmcs_read64(unsigned long field) 162 { 163 vmcs_check64(field); 164 if (static_branch_unlikely(&enable_evmcs)) 165 return evmcs_read64(field); 166 #ifdef CONFIG_X86_64 167 return __vmcs_readl(field); 168 #else 169 return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32); 170 #endif 171 } 172 173 static __always_inline unsigned long vmcs_readl(unsigned long field) 174 { 175 vmcs_checkl(field); 176 if (static_branch_unlikely(&enable_evmcs)) 177 return evmcs_read64(field); 178 return __vmcs_readl(field); 179 } 180 181 #define vmx_asm1(insn, op1, error_args...) \ 182 do { \ 183 asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \ 184 ".byte 0x2e\n\t" /* branch not taken hint */ \ 185 "jna %l[error]\n\t" \ 186 _ASM_EXTABLE(1b, %l[fault]) \ 187 : : op1 : "cc" : error, fault); \ 188 return; \ 189 error: \ 190 instrumentation_begin(); \ 191 insn##_error(error_args); \ 192 instrumentation_end(); \ 193 return; \ 194 fault: \ 195 kvm_spurious_fault(); \ 196 } while (0) 197 198 #define vmx_asm2(insn, op1, op2, error_args...) \ 199 do { \ 200 asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \ 201 ".byte 0x2e\n\t" /* branch not taken hint */ \ 202 "jna %l[error]\n\t" \ 203 _ASM_EXTABLE(1b, %l[fault]) \ 204 : : op1, op2 : "cc" : error, fault); \ 205 return; \ 206 error: \ 207 instrumentation_begin(); \ 208 insn##_error(error_args); \ 209 instrumentation_end(); \ 210 return; \ 211 fault: \ 212 kvm_spurious_fault(); \ 213 } while (0) 214 215 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value) 216 { 217 vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value); 218 } 219 220 static __always_inline void vmcs_write16(unsigned long field, u16 value) 221 { 222 vmcs_check16(field); 223 if (static_branch_unlikely(&enable_evmcs)) 224 return evmcs_write16(field, value); 225 226 __vmcs_writel(field, value); 227 } 228 229 static __always_inline void vmcs_write32(unsigned long field, u32 value) 230 { 231 vmcs_check32(field); 232 if (static_branch_unlikely(&enable_evmcs)) 233 return evmcs_write32(field, value); 234 235 __vmcs_writel(field, value); 236 } 237 238 static __always_inline void vmcs_write64(unsigned long field, u64 value) 239 { 240 vmcs_check64(field); 241 if (static_branch_unlikely(&enable_evmcs)) 242 return evmcs_write64(field, value); 243 244 __vmcs_writel(field, value); 245 #ifndef CONFIG_X86_64 246 __vmcs_writel(field+1, value >> 32); 247 #endif 248 } 249 250 static __always_inline void vmcs_writel(unsigned long field, unsigned long value) 251 { 252 vmcs_checkl(field); 253 if (static_branch_unlikely(&enable_evmcs)) 254 return evmcs_write64(field, value); 255 256 __vmcs_writel(field, value); 257 } 258 259 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask) 260 { 261 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 262 "vmcs_clear_bits does not support 64-bit fields"); 263 if (static_branch_unlikely(&enable_evmcs)) 264 return evmcs_write32(field, evmcs_read32(field) & ~mask); 265 266 __vmcs_writel(field, __vmcs_readl(field) & ~mask); 267 } 268 269 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask) 270 { 271 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 272 "vmcs_set_bits does not support 64-bit fields"); 273 if (static_branch_unlikely(&enable_evmcs)) 274 return evmcs_write32(field, evmcs_read32(field) | mask); 275 276 __vmcs_writel(field, __vmcs_readl(field) | mask); 277 } 278 279 static inline void vmcs_clear(struct vmcs *vmcs) 280 { 281 u64 phys_addr = __pa(vmcs); 282 283 vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr); 284 } 285 286 static inline void vmcs_load(struct vmcs *vmcs) 287 { 288 u64 phys_addr = __pa(vmcs); 289 290 if (static_branch_unlikely(&enable_evmcs)) 291 return evmcs_load(phys_addr); 292 293 vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr); 294 } 295 296 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva) 297 { 298 struct { 299 u64 vpid : 16; 300 u64 rsvd : 48; 301 u64 gva; 302 } operand = { vpid, 0, gva }; 303 304 vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva); 305 } 306 307 static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa) 308 { 309 struct { 310 u64 eptp, gpa; 311 } operand = {eptp, gpa}; 312 313 vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa); 314 } 315 316 static inline void vpid_sync_vcpu_single(int vpid) 317 { 318 if (vpid == 0) 319 return; 320 321 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); 322 } 323 324 static inline void vpid_sync_vcpu_global(void) 325 { 326 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); 327 } 328 329 static inline void vpid_sync_context(int vpid) 330 { 331 if (cpu_has_vmx_invvpid_single()) 332 vpid_sync_vcpu_single(vpid); 333 else if (vpid != 0) 334 vpid_sync_vcpu_global(); 335 } 336 337 static inline void vpid_sync_vcpu_addr(int vpid, gva_t addr) 338 { 339 if (vpid == 0) 340 return; 341 342 if (cpu_has_vmx_invvpid_individual_addr()) 343 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); 344 else 345 vpid_sync_context(vpid); 346 } 347 348 static inline void ept_sync_global(void) 349 { 350 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0); 351 } 352 353 static inline void ept_sync_context(u64 eptp) 354 { 355 if (cpu_has_vmx_invept_context()) 356 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0); 357 else 358 ept_sync_global(); 359 } 360 361 #endif /* __KVM_X86_VMX_INSN_H */ 362