1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __KVM_X86_VMX_INSN_H 3 #define __KVM_X86_VMX_INSN_H 4 5 #include <linux/nospec.h> 6 7 #include <asm/vmx.h> 8 9 #include "vmx_onhyperv.h" 10 #include "vmcs.h" 11 #include "../x86.h" 12 13 void vmread_error(unsigned long field); 14 void vmwrite_error(unsigned long field, unsigned long value); 15 void vmclear_error(struct vmcs *vmcs, u64 phys_addr); 16 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr); 17 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva); 18 void invept_error(unsigned long ext, u64 eptp, gpa_t gpa); 19 20 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 21 /* 22 * The VMREAD error trampoline _always_ uses the stack to pass parameters, even 23 * for 64-bit targets. Preserving all registers allows the VMREAD inline asm 24 * blob to avoid clobbering GPRs, which in turn allows the compiler to better 25 * optimize sequences of VMREADs. 26 * 27 * Declare the trampoline as an opaque label as it's not safe to call from C 28 * code; there is no way to tell the compiler to pass params on the stack for 29 * 64-bit targets. 30 * 31 * void vmread_error_trampoline(unsigned long field, bool fault); 32 */ 33 extern unsigned long vmread_error_trampoline; 34 35 /* 36 * The second VMREAD error trampoline, called from the assembly trampoline, 37 * exists primarily to enable instrumentation for the VM-Fail path. 38 */ 39 void vmread_error_trampoline2(unsigned long field, bool fault); 40 41 #endif 42 43 static __always_inline void vmcs_check16(unsigned long field) 44 { 45 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 46 "16-bit accessor invalid for 64-bit field"); 47 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 48 "16-bit accessor invalid for 64-bit high field"); 49 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 50 "16-bit accessor invalid for 32-bit high field"); 51 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 52 "16-bit accessor invalid for natural width field"); 53 } 54 55 static __always_inline void vmcs_check32(unsigned long field) 56 { 57 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 58 "32-bit accessor invalid for 16-bit field"); 59 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 60 "32-bit accessor invalid for 64-bit field"); 61 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 62 "32-bit accessor invalid for 64-bit high field"); 63 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 64 "32-bit accessor invalid for natural width field"); 65 } 66 67 static __always_inline void vmcs_check64(unsigned long field) 68 { 69 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 70 "64-bit accessor invalid for 16-bit field"); 71 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 72 "64-bit accessor invalid for 64-bit high field"); 73 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 74 "64-bit accessor invalid for 32-bit field"); 75 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 76 "64-bit accessor invalid for natural width field"); 77 } 78 79 static __always_inline void vmcs_checkl(unsigned long field) 80 { 81 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 82 "Natural width accessor invalid for 16-bit field"); 83 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 84 "Natural width accessor invalid for 64-bit field"); 85 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 86 "Natural width accessor invalid for 64-bit high field"); 87 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 88 "Natural width accessor invalid for 32-bit field"); 89 } 90 91 static __always_inline unsigned long __vmcs_readl(unsigned long field) 92 { 93 unsigned long value; 94 95 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 96 97 asm_goto_output("1: vmread %[field], %[output]\n\t" 98 "jna %l[do_fail]\n\t" 99 100 _ASM_EXTABLE(1b, %l[do_exception]) 101 102 : [output] "=r" (value) 103 : [field] "r" (field) 104 : "cc" 105 : do_fail, do_exception); 106 107 return value; 108 109 do_fail: 110 instrumentation_begin(); 111 vmread_error(field); 112 instrumentation_end(); 113 return 0; 114 115 do_exception: 116 kvm_spurious_fault(); 117 return 0; 118 119 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 120 121 asm volatile("1: vmread %2, %1\n\t" 122 ".byte 0x3e\n\t" /* branch taken hint */ 123 "ja 3f\n\t" 124 125 /* 126 * VMREAD failed. Push '0' for @fault, push the failing 127 * @field, and bounce through the trampoline to preserve 128 * volatile registers. 129 */ 130 "xorl %k1, %k1\n\t" 131 "2:\n\t" 132 "push %1\n\t" 133 "push %2\n\t" 134 "call vmread_error_trampoline\n\t" 135 136 /* 137 * Unwind the stack. Note, the trampoline zeros out the 138 * memory for @fault so that the result is '0' on error. 139 */ 140 "pop %2\n\t" 141 "pop %1\n\t" 142 "3:\n\t" 143 144 /* VMREAD faulted. As above, except push '1' for @fault. */ 145 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %1) 146 147 : ASM_CALL_CONSTRAINT, "=&r"(value) : "r"(field) : "cc"); 148 return value; 149 150 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 151 } 152 153 static __always_inline u16 vmcs_read16(unsigned long field) 154 { 155 vmcs_check16(field); 156 if (kvm_is_using_evmcs()) 157 return evmcs_read16(field); 158 return __vmcs_readl(field); 159 } 160 161 static __always_inline u32 vmcs_read32(unsigned long field) 162 { 163 vmcs_check32(field); 164 if (kvm_is_using_evmcs()) 165 return evmcs_read32(field); 166 return __vmcs_readl(field); 167 } 168 169 static __always_inline u64 vmcs_read64(unsigned long field) 170 { 171 vmcs_check64(field); 172 if (kvm_is_using_evmcs()) 173 return evmcs_read64(field); 174 #ifdef CONFIG_X86_64 175 return __vmcs_readl(field); 176 #else 177 return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32); 178 #endif 179 } 180 181 static __always_inline unsigned long vmcs_readl(unsigned long field) 182 { 183 vmcs_checkl(field); 184 if (kvm_is_using_evmcs()) 185 return evmcs_read64(field); 186 return __vmcs_readl(field); 187 } 188 189 #define vmx_asm1(insn, op1, error_args...) \ 190 do { \ 191 asm goto("1: " __stringify(insn) " %0\n\t" \ 192 ".byte 0x2e\n\t" /* branch not taken hint */ \ 193 "jna %l[error]\n\t" \ 194 _ASM_EXTABLE(1b, %l[fault]) \ 195 : : op1 : "cc" : error, fault); \ 196 return; \ 197 error: \ 198 instrumentation_begin(); \ 199 insn##_error(error_args); \ 200 instrumentation_end(); \ 201 return; \ 202 fault: \ 203 kvm_spurious_fault(); \ 204 } while (0) 205 206 #define vmx_asm2(insn, op1, op2, error_args...) \ 207 do { \ 208 asm goto("1: " __stringify(insn) " %1, %0\n\t" \ 209 ".byte 0x2e\n\t" /* branch not taken hint */ \ 210 "jna %l[error]\n\t" \ 211 _ASM_EXTABLE(1b, %l[fault]) \ 212 : : op1, op2 : "cc" : error, fault); \ 213 return; \ 214 error: \ 215 instrumentation_begin(); \ 216 insn##_error(error_args); \ 217 instrumentation_end(); \ 218 return; \ 219 fault: \ 220 kvm_spurious_fault(); \ 221 } while (0) 222 223 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value) 224 { 225 vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value); 226 } 227 228 static __always_inline void vmcs_write16(unsigned long field, u16 value) 229 { 230 vmcs_check16(field); 231 if (kvm_is_using_evmcs()) 232 return evmcs_write16(field, value); 233 234 __vmcs_writel(field, value); 235 } 236 237 static __always_inline void vmcs_write32(unsigned long field, u32 value) 238 { 239 vmcs_check32(field); 240 if (kvm_is_using_evmcs()) 241 return evmcs_write32(field, value); 242 243 __vmcs_writel(field, value); 244 } 245 246 static __always_inline void vmcs_write64(unsigned long field, u64 value) 247 { 248 vmcs_check64(field); 249 if (kvm_is_using_evmcs()) 250 return evmcs_write64(field, value); 251 252 __vmcs_writel(field, value); 253 #ifndef CONFIG_X86_64 254 __vmcs_writel(field+1, value >> 32); 255 #endif 256 } 257 258 static __always_inline void vmcs_writel(unsigned long field, unsigned long value) 259 { 260 vmcs_checkl(field); 261 if (kvm_is_using_evmcs()) 262 return evmcs_write64(field, value); 263 264 __vmcs_writel(field, value); 265 } 266 267 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask) 268 { 269 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 270 "vmcs_clear_bits does not support 64-bit fields"); 271 if (kvm_is_using_evmcs()) 272 return evmcs_write32(field, evmcs_read32(field) & ~mask); 273 274 __vmcs_writel(field, __vmcs_readl(field) & ~mask); 275 } 276 277 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask) 278 { 279 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 280 "vmcs_set_bits does not support 64-bit fields"); 281 if (kvm_is_using_evmcs()) 282 return evmcs_write32(field, evmcs_read32(field) | mask); 283 284 __vmcs_writel(field, __vmcs_readl(field) | mask); 285 } 286 287 static inline void vmcs_clear(struct vmcs *vmcs) 288 { 289 u64 phys_addr = __pa(vmcs); 290 291 vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr); 292 } 293 294 static inline void vmcs_load(struct vmcs *vmcs) 295 { 296 u64 phys_addr = __pa(vmcs); 297 298 if (kvm_is_using_evmcs()) 299 return evmcs_load(phys_addr); 300 301 vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr); 302 } 303 304 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva) 305 { 306 struct { 307 u64 vpid : 16; 308 u64 rsvd : 48; 309 u64 gva; 310 } operand = { vpid, 0, gva }; 311 312 vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva); 313 } 314 315 static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa) 316 { 317 struct { 318 u64 eptp, gpa; 319 } operand = {eptp, gpa}; 320 321 vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa); 322 } 323 324 static inline void vpid_sync_vcpu_single(int vpid) 325 { 326 if (vpid == 0) 327 return; 328 329 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); 330 } 331 332 static inline void vpid_sync_vcpu_global(void) 333 { 334 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); 335 } 336 337 static inline void vpid_sync_context(int vpid) 338 { 339 if (cpu_has_vmx_invvpid_single()) 340 vpid_sync_vcpu_single(vpid); 341 else if (vpid != 0) 342 vpid_sync_vcpu_global(); 343 } 344 345 static inline void vpid_sync_vcpu_addr(int vpid, gva_t addr) 346 { 347 if (vpid == 0) 348 return; 349 350 if (cpu_has_vmx_invvpid_individual_addr()) 351 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); 352 else 353 vpid_sync_context(vpid); 354 } 355 356 static inline void ept_sync_global(void) 357 { 358 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0); 359 } 360 361 static inline void ept_sync_context(u64 eptp) 362 { 363 if (cpu_has_vmx_invept_context()) 364 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0); 365 else 366 ept_sync_global(); 367 } 368 369 #endif /* __KVM_X86_VMX_INSN_H */ 370