1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __KVM_X86_VMX_INSN_H 3 #define __KVM_X86_VMX_INSN_H 4 5 #include <linux/nospec.h> 6 7 #include <asm/vmx.h> 8 9 #include "evmcs.h" 10 #include "vmcs.h" 11 #include "x86.h" 12 13 asmlinkage void vmread_error(unsigned long field, bool fault); 14 __attribute__((regparm(0))) void vmread_error_trampoline(unsigned long field, 15 bool fault); 16 void vmwrite_error(unsigned long field, unsigned long value); 17 void vmclear_error(struct vmcs *vmcs, u64 phys_addr); 18 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr); 19 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva); 20 void invept_error(unsigned long ext, u64 eptp, gpa_t gpa); 21 22 static __always_inline void vmcs_check16(unsigned long field) 23 { 24 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 25 "16-bit accessor invalid for 64-bit field"); 26 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 27 "16-bit accessor invalid for 64-bit high field"); 28 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 29 "16-bit accessor invalid for 32-bit high field"); 30 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 31 "16-bit accessor invalid for natural width field"); 32 } 33 34 static __always_inline void vmcs_check32(unsigned long field) 35 { 36 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 37 "32-bit accessor invalid for 16-bit field"); 38 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 39 "32-bit accessor invalid for 64-bit field"); 40 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 41 "32-bit accessor invalid for 64-bit high field"); 42 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 43 "32-bit accessor invalid for natural width field"); 44 } 45 46 static __always_inline void vmcs_check64(unsigned long field) 47 { 48 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 49 "64-bit accessor invalid for 16-bit field"); 50 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 51 "64-bit accessor invalid for 64-bit high field"); 52 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 53 "64-bit accessor invalid for 32-bit field"); 54 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 55 "64-bit accessor invalid for natural width field"); 56 } 57 58 static __always_inline void vmcs_checkl(unsigned long field) 59 { 60 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 61 "Natural width accessor invalid for 16-bit field"); 62 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 63 "Natural width accessor invalid for 64-bit field"); 64 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 65 "Natural width accessor invalid for 64-bit high field"); 66 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 67 "Natural width accessor invalid for 32-bit field"); 68 } 69 70 static __always_inline unsigned long __vmcs_readl(unsigned long field) 71 { 72 unsigned long value; 73 74 asm volatile("1: vmread %2, %1\n\t" 75 ".byte 0x3e\n\t" /* branch taken hint */ 76 "ja 3f\n\t" 77 78 /* 79 * VMREAD failed. Push '0' for @fault, push the failing 80 * @field, and bounce through the trampoline to preserve 81 * volatile registers. 82 */ 83 "push $0\n\t" 84 "push %2\n\t" 85 "2:call vmread_error_trampoline\n\t" 86 87 /* 88 * Unwind the stack. Note, the trampoline zeros out the 89 * memory for @fault so that the result is '0' on error. 90 */ 91 "pop %2\n\t" 92 "pop %1\n\t" 93 "3:\n\t" 94 95 /* VMREAD faulted. As above, except push '1' for @fault. */ 96 ".pushsection .fixup, \"ax\"\n\t" 97 "4: push $1\n\t" 98 "push %2\n\t" 99 "jmp 2b\n\t" 100 ".popsection\n\t" 101 _ASM_EXTABLE(1b, 4b) 102 : ASM_CALL_CONSTRAINT, "=r"(value) : "r"(field) : "cc"); 103 return value; 104 } 105 106 static __always_inline u16 vmcs_read16(unsigned long field) 107 { 108 vmcs_check16(field); 109 if (static_branch_unlikely(&enable_evmcs)) 110 return evmcs_read16(field); 111 return __vmcs_readl(field); 112 } 113 114 static __always_inline u32 vmcs_read32(unsigned long field) 115 { 116 vmcs_check32(field); 117 if (static_branch_unlikely(&enable_evmcs)) 118 return evmcs_read32(field); 119 return __vmcs_readl(field); 120 } 121 122 static __always_inline u64 vmcs_read64(unsigned long field) 123 { 124 vmcs_check64(field); 125 if (static_branch_unlikely(&enable_evmcs)) 126 return evmcs_read64(field); 127 #ifdef CONFIG_X86_64 128 return __vmcs_readl(field); 129 #else 130 return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32); 131 #endif 132 } 133 134 static __always_inline unsigned long vmcs_readl(unsigned long field) 135 { 136 vmcs_checkl(field); 137 if (static_branch_unlikely(&enable_evmcs)) 138 return evmcs_read64(field); 139 return __vmcs_readl(field); 140 } 141 142 #define vmx_asm1(insn, op1, error_args...) \ 143 do { \ 144 asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \ 145 ".byte 0x2e\n\t" /* branch not taken hint */ \ 146 "jna %l[error]\n\t" \ 147 _ASM_EXTABLE(1b, %l[fault]) \ 148 : : op1 : "cc" : error, fault); \ 149 return; \ 150 error: \ 151 instrumentation_begin(); \ 152 insn##_error(error_args); \ 153 instrumentation_end(); \ 154 return; \ 155 fault: \ 156 kvm_spurious_fault(); \ 157 } while (0) 158 159 #define vmx_asm2(insn, op1, op2, error_args...) \ 160 do { \ 161 asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \ 162 ".byte 0x2e\n\t" /* branch not taken hint */ \ 163 "jna %l[error]\n\t" \ 164 _ASM_EXTABLE(1b, %l[fault]) \ 165 : : op1, op2 : "cc" : error, fault); \ 166 return; \ 167 error: \ 168 instrumentation_begin(); \ 169 insn##_error(error_args); \ 170 instrumentation_end(); \ 171 return; \ 172 fault: \ 173 kvm_spurious_fault(); \ 174 } while (0) 175 176 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value) 177 { 178 vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value); 179 } 180 181 static __always_inline void vmcs_write16(unsigned long field, u16 value) 182 { 183 vmcs_check16(field); 184 if (static_branch_unlikely(&enable_evmcs)) 185 return evmcs_write16(field, value); 186 187 __vmcs_writel(field, value); 188 } 189 190 static __always_inline void vmcs_write32(unsigned long field, u32 value) 191 { 192 vmcs_check32(field); 193 if (static_branch_unlikely(&enable_evmcs)) 194 return evmcs_write32(field, value); 195 196 __vmcs_writel(field, value); 197 } 198 199 static __always_inline void vmcs_write64(unsigned long field, u64 value) 200 { 201 vmcs_check64(field); 202 if (static_branch_unlikely(&enable_evmcs)) 203 return evmcs_write64(field, value); 204 205 __vmcs_writel(field, value); 206 #ifndef CONFIG_X86_64 207 __vmcs_writel(field+1, value >> 32); 208 #endif 209 } 210 211 static __always_inline void vmcs_writel(unsigned long field, unsigned long value) 212 { 213 vmcs_checkl(field); 214 if (static_branch_unlikely(&enable_evmcs)) 215 return evmcs_write64(field, value); 216 217 __vmcs_writel(field, value); 218 } 219 220 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask) 221 { 222 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 223 "vmcs_clear_bits does not support 64-bit fields"); 224 if (static_branch_unlikely(&enable_evmcs)) 225 return evmcs_write32(field, evmcs_read32(field) & ~mask); 226 227 __vmcs_writel(field, __vmcs_readl(field) & ~mask); 228 } 229 230 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask) 231 { 232 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 233 "vmcs_set_bits does not support 64-bit fields"); 234 if (static_branch_unlikely(&enable_evmcs)) 235 return evmcs_write32(field, evmcs_read32(field) | mask); 236 237 __vmcs_writel(field, __vmcs_readl(field) | mask); 238 } 239 240 static inline void vmcs_clear(struct vmcs *vmcs) 241 { 242 u64 phys_addr = __pa(vmcs); 243 244 vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr); 245 } 246 247 static inline void vmcs_load(struct vmcs *vmcs) 248 { 249 u64 phys_addr = __pa(vmcs); 250 251 if (static_branch_unlikely(&enable_evmcs)) 252 return evmcs_load(phys_addr); 253 254 vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr); 255 } 256 257 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva) 258 { 259 struct { 260 u64 vpid : 16; 261 u64 rsvd : 48; 262 u64 gva; 263 } operand = { vpid, 0, gva }; 264 265 vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva); 266 } 267 268 static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa) 269 { 270 struct { 271 u64 eptp, gpa; 272 } operand = {eptp, gpa}; 273 274 vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa); 275 } 276 277 static inline void vpid_sync_vcpu_single(int vpid) 278 { 279 if (vpid == 0) 280 return; 281 282 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); 283 } 284 285 static inline void vpid_sync_vcpu_global(void) 286 { 287 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); 288 } 289 290 static inline void vpid_sync_context(int vpid) 291 { 292 if (cpu_has_vmx_invvpid_single()) 293 vpid_sync_vcpu_single(vpid); 294 else if (vpid != 0) 295 vpid_sync_vcpu_global(); 296 } 297 298 static inline void vpid_sync_vcpu_addr(int vpid, gva_t addr) 299 { 300 if (vpid == 0) 301 return; 302 303 if (cpu_has_vmx_invvpid_individual_addr()) 304 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); 305 else 306 vpid_sync_context(vpid); 307 } 308 309 static inline void ept_sync_global(void) 310 { 311 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0); 312 } 313 314 static inline void ept_sync_context(u64 eptp) 315 { 316 if (cpu_has_vmx_invept_context()) 317 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0); 318 else 319 ept_sync_global(); 320 } 321 322 #endif /* __KVM_X86_VMX_INSN_H */ 323