1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * tools/testing/selftests/kvm/include/x86_64/processor.h 4 * 5 * Copyright (C) 2018, Google LLC. 6 */ 7 8 #ifndef SELFTEST_KVM_PROCESSOR_H 9 #define SELFTEST_KVM_PROCESSOR_H 10 11 #include <assert.h> 12 #include <stdint.h> 13 #include <syscall.h> 14 15 #include <asm/msr-index.h> 16 #include <asm/prctl.h> 17 18 #include <linux/stringify.h> 19 20 #include "../kvm_util.h" 21 22 extern bool host_cpu_is_intel; 23 extern bool host_cpu_is_amd; 24 25 #define NMI_VECTOR 0x02 26 27 #define X86_EFLAGS_FIXED (1u << 1) 28 29 #define X86_CR4_VME (1ul << 0) 30 #define X86_CR4_PVI (1ul << 1) 31 #define X86_CR4_TSD (1ul << 2) 32 #define X86_CR4_DE (1ul << 3) 33 #define X86_CR4_PSE (1ul << 4) 34 #define X86_CR4_PAE (1ul << 5) 35 #define X86_CR4_MCE (1ul << 6) 36 #define X86_CR4_PGE (1ul << 7) 37 #define X86_CR4_PCE (1ul << 8) 38 #define X86_CR4_OSFXSR (1ul << 9) 39 #define X86_CR4_OSXMMEXCPT (1ul << 10) 40 #define X86_CR4_UMIP (1ul << 11) 41 #define X86_CR4_LA57 (1ul << 12) 42 #define X86_CR4_VMXE (1ul << 13) 43 #define X86_CR4_SMXE (1ul << 14) 44 #define X86_CR4_FSGSBASE (1ul << 16) 45 #define X86_CR4_PCIDE (1ul << 17) 46 #define X86_CR4_OSXSAVE (1ul << 18) 47 #define X86_CR4_SMEP (1ul << 20) 48 #define X86_CR4_SMAP (1ul << 21) 49 #define X86_CR4_PKE (1ul << 22) 50 51 /* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */ 52 enum cpuid_output_regs { 53 KVM_CPUID_EAX, 54 KVM_CPUID_EBX, 55 KVM_CPUID_ECX, 56 KVM_CPUID_EDX 57 }; 58 59 /* 60 * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be 61 * passed by value with no overhead. 62 */ 63 struct kvm_x86_cpu_feature { 64 u32 function; 65 u16 index; 66 u8 reg; 67 u8 bit; 68 }; 69 #define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \ 70 ({ \ 71 struct kvm_x86_cpu_feature feature = { \ 72 .function = fn, \ 73 .index = idx, \ 74 .reg = KVM_CPUID_##gpr, \ 75 .bit = __bit, \ 76 }; \ 77 \ 78 kvm_static_assert((fn & 0xc0000000) == 0 || \ 79 (fn & 0xc0000000) == 0x40000000 || \ 80 (fn & 0xc0000000) == 0x80000000 || \ 81 (fn & 0xc0000000) == 0xc0000000); \ 82 kvm_static_assert(idx < BIT(sizeof(feature.index) * BITS_PER_BYTE)); \ 83 feature; \ 84 }) 85 86 /* 87 * Basic Leafs, a.k.a. Intel defined 88 */ 89 #define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3) 90 #define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5) 91 #define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6) 92 #define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15) 93 #define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17) 94 #define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21) 95 #define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22) 96 #define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24) 97 #define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26) 98 #define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27) 99 #define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30) 100 #define X86_FEATURE_HYPERVISOR KVM_X86_CPU_FEATURE(0x1, 0, ECX, 31) 101 #define X86_FEATURE_PAE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 6) 102 #define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7) 103 #define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9) 104 #define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19) 105 #define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25) 106 #define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26) 107 #define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0) 108 #define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1) 109 #define X86_FEATURE_SGX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 2) 110 #define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4) 111 #define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7) 112 #define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10) 113 #define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11) 114 #define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14) 115 #define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20) 116 #define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22) 117 #define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23) 118 #define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24) 119 #define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2) 120 #define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3) 121 #define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16) 122 #define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22) 123 #define X86_FEATURE_SGX_LC KVM_X86_CPU_FEATURE(0x7, 0, ECX, 30) 124 #define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7) 125 #define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20) 126 #define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24) 127 #define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26) 128 #define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29) 129 #define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31) 130 #define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17) 131 #define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18) 132 #define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3) 133 #define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4) 134 135 /* 136 * Extended Leafs, a.k.a. AMD defined 137 */ 138 #define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2) 139 #define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20) 140 #define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26) 141 #define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27) 142 #define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29) 143 #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) 144 #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) 145 #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) 146 #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) 147 #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) 148 #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) 149 #define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4) 150 #define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10) 151 #define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12) 152 #define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16) 153 #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) 154 #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) 155 156 /* 157 * KVM defined paravirt features. 158 */ 159 #define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0) 160 #define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1) 161 #define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2) 162 #define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3) 163 #define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4) 164 #define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5) 165 #define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6) 166 #define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7) 167 /* Bit 8 apparently isn't used?!?! */ 168 #define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9) 169 #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10) 170 #define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11) 171 #define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12) 172 #define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13) 173 #define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14) 174 #define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15) 175 #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16) 176 #define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17) 177 178 /* 179 * Same idea as X86_FEATURE_XXX, but X86_PROPERTY_XXX retrieves a multi-bit 180 * value/property as opposed to a single-bit feature. Again, pack the info 181 * into a 64-bit value to pass by value with no overhead. 182 */ 183 struct kvm_x86_cpu_property { 184 u32 function; 185 u8 index; 186 u8 reg; 187 u8 lo_bit; 188 u8 hi_bit; 189 }; 190 #define KVM_X86_CPU_PROPERTY(fn, idx, gpr, low_bit, high_bit) \ 191 ({ \ 192 struct kvm_x86_cpu_property property = { \ 193 .function = fn, \ 194 .index = idx, \ 195 .reg = KVM_CPUID_##gpr, \ 196 .lo_bit = low_bit, \ 197 .hi_bit = high_bit, \ 198 }; \ 199 \ 200 kvm_static_assert(low_bit < high_bit); \ 201 kvm_static_assert((fn & 0xc0000000) == 0 || \ 202 (fn & 0xc0000000) == 0x40000000 || \ 203 (fn & 0xc0000000) == 0x80000000 || \ 204 (fn & 0xc0000000) == 0xc0000000); \ 205 kvm_static_assert(idx < BIT(sizeof(property.index) * BITS_PER_BYTE)); \ 206 property; \ 207 }) 208 209 #define X86_PROPERTY_MAX_BASIC_LEAF KVM_X86_CPU_PROPERTY(0, 0, EAX, 0, 31) 210 #define X86_PROPERTY_PMU_VERSION KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 0, 7) 211 #define X86_PROPERTY_PMU_NR_GP_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15) 212 #define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31) 213 214 #define X86_PROPERTY_XSTATE_MAX_SIZE_XCR0 KVM_X86_CPU_PROPERTY(0xd, 0, EBX, 0, 31) 215 #define X86_PROPERTY_XSTATE_MAX_SIZE KVM_X86_CPU_PROPERTY(0xd, 0, ECX, 0, 31) 216 #define X86_PROPERTY_XSTATE_TILE_SIZE KVM_X86_CPU_PROPERTY(0xd, 18, EAX, 0, 31) 217 #define X86_PROPERTY_XSTATE_TILE_OFFSET KVM_X86_CPU_PROPERTY(0xd, 18, EBX, 0, 31) 218 #define X86_PROPERTY_AMX_TOTAL_TILE_BYTES KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 0, 15) 219 #define X86_PROPERTY_AMX_BYTES_PER_TILE KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 16, 31) 220 #define X86_PROPERTY_AMX_BYTES_PER_ROW KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 0, 15) 221 #define X86_PROPERTY_AMX_NR_TILE_REGS KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 16, 31) 222 #define X86_PROPERTY_AMX_MAX_ROWS KVM_X86_CPU_PROPERTY(0x1d, 1, ECX, 0, 15) 223 224 #define X86_PROPERTY_MAX_KVM_LEAF KVM_X86_CPU_PROPERTY(0x40000000, 0, EAX, 0, 31) 225 226 #define X86_PROPERTY_MAX_EXT_LEAF KVM_X86_CPU_PROPERTY(0x80000000, 0, EAX, 0, 31) 227 #define X86_PROPERTY_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 0, 7) 228 #define X86_PROPERTY_MAX_VIRT_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 8, 15) 229 #define X86_PROPERTY_PHYS_ADDR_REDUCTION KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 6, 11) 230 231 #define X86_PROPERTY_MAX_CENTAUR_LEAF KVM_X86_CPU_PROPERTY(0xC0000000, 0, EAX, 0, 31) 232 233 /* 234 * Intel's architectural PMU events are bizarre. They have a "feature" bit 235 * that indicates the feature is _not_ supported, and a property that states 236 * the length of the bit mask of unsupported features. A feature is supported 237 * if the size of the bit mask is larger than the "unavailable" bit, and said 238 * bit is not set. 239 * 240 * Wrap the "unavailable" feature to simplify checking whether or not a given 241 * architectural event is supported. 242 */ 243 struct kvm_x86_pmu_feature { 244 struct kvm_x86_cpu_feature anti_feature; 245 }; 246 #define KVM_X86_PMU_FEATURE(name, __bit) \ 247 ({ \ 248 struct kvm_x86_pmu_feature feature = { \ 249 .anti_feature = KVM_X86_CPU_FEATURE(0xa, 0, EBX, __bit), \ 250 }; \ 251 \ 252 feature; \ 253 }) 254 255 #define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED KVM_X86_PMU_FEATURE(BRANCH_INSNS_RETIRED, 5) 256 257 static inline unsigned int x86_family(unsigned int eax) 258 { 259 unsigned int x86; 260 261 x86 = (eax >> 8) & 0xf; 262 263 if (x86 == 0xf) 264 x86 += (eax >> 20) & 0xff; 265 266 return x86; 267 } 268 269 static inline unsigned int x86_model(unsigned int eax) 270 { 271 return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f); 272 } 273 274 /* Page table bitfield declarations */ 275 #define PTE_PRESENT_MASK BIT_ULL(0) 276 #define PTE_WRITABLE_MASK BIT_ULL(1) 277 #define PTE_USER_MASK BIT_ULL(2) 278 #define PTE_ACCESSED_MASK BIT_ULL(5) 279 #define PTE_DIRTY_MASK BIT_ULL(6) 280 #define PTE_LARGE_MASK BIT_ULL(7) 281 #define PTE_GLOBAL_MASK BIT_ULL(8) 282 #define PTE_NX_MASK BIT_ULL(63) 283 284 #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12) 285 286 #define PAGE_SHIFT 12 287 #define PAGE_SIZE (1ULL << PAGE_SHIFT) 288 #define PAGE_MASK (~(PAGE_SIZE-1) & PHYSICAL_PAGE_MASK) 289 290 #define HUGEPAGE_SHIFT(x) (PAGE_SHIFT + (((x) - 1) * 9)) 291 #define HUGEPAGE_SIZE(x) (1UL << HUGEPAGE_SHIFT(x)) 292 #define HUGEPAGE_MASK(x) (~(HUGEPAGE_SIZE(x) - 1) & PHYSICAL_PAGE_MASK) 293 294 #define PTE_GET_PA(pte) ((pte) & PHYSICAL_PAGE_MASK) 295 #define PTE_GET_PFN(pte) (PTE_GET_PA(pte) >> PAGE_SHIFT) 296 297 /* General Registers in 64-Bit Mode */ 298 struct gpr64_regs { 299 u64 rax; 300 u64 rcx; 301 u64 rdx; 302 u64 rbx; 303 u64 rsp; 304 u64 rbp; 305 u64 rsi; 306 u64 rdi; 307 u64 r8; 308 u64 r9; 309 u64 r10; 310 u64 r11; 311 u64 r12; 312 u64 r13; 313 u64 r14; 314 u64 r15; 315 }; 316 317 struct desc64 { 318 uint16_t limit0; 319 uint16_t base0; 320 unsigned base1:8, type:4, s:1, dpl:2, p:1; 321 unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8; 322 uint32_t base3; 323 uint32_t zero1; 324 } __attribute__((packed)); 325 326 struct desc_ptr { 327 uint16_t size; 328 uint64_t address; 329 } __attribute__((packed)); 330 331 struct kvm_x86_state { 332 struct kvm_xsave *xsave; 333 struct kvm_vcpu_events events; 334 struct kvm_mp_state mp_state; 335 struct kvm_regs regs; 336 struct kvm_xcrs xcrs; 337 struct kvm_sregs sregs; 338 struct kvm_debugregs debugregs; 339 union { 340 struct kvm_nested_state nested; 341 char nested_[16384]; 342 }; 343 struct kvm_msrs msrs; 344 }; 345 346 static inline uint64_t get_desc64_base(const struct desc64 *desc) 347 { 348 return ((uint64_t)desc->base3 << 32) | 349 (desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24)); 350 } 351 352 static inline uint64_t rdtsc(void) 353 { 354 uint32_t eax, edx; 355 uint64_t tsc_val; 356 /* 357 * The lfence is to wait (on Intel CPUs) until all previous 358 * instructions have been executed. If software requires RDTSC to be 359 * executed prior to execution of any subsequent instruction, it can 360 * execute LFENCE immediately after RDTSC 361 */ 362 __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 363 tsc_val = ((uint64_t)edx) << 32 | eax; 364 return tsc_val; 365 } 366 367 static inline uint64_t rdtscp(uint32_t *aux) 368 { 369 uint32_t eax, edx; 370 371 __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux)); 372 return ((uint64_t)edx) << 32 | eax; 373 } 374 375 static inline uint64_t rdmsr(uint32_t msr) 376 { 377 uint32_t a, d; 378 379 __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory"); 380 381 return a | ((uint64_t) d << 32); 382 } 383 384 static inline void wrmsr(uint32_t msr, uint64_t value) 385 { 386 uint32_t a = value; 387 uint32_t d = value >> 32; 388 389 __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory"); 390 } 391 392 393 static inline uint16_t inw(uint16_t port) 394 { 395 uint16_t tmp; 396 397 __asm__ __volatile__("in %%dx, %%ax" 398 : /* output */ "=a" (tmp) 399 : /* input */ "d" (port)); 400 401 return tmp; 402 } 403 404 static inline uint16_t get_es(void) 405 { 406 uint16_t es; 407 408 __asm__ __volatile__("mov %%es, %[es]" 409 : /* output */ [es]"=rm"(es)); 410 return es; 411 } 412 413 static inline uint16_t get_cs(void) 414 { 415 uint16_t cs; 416 417 __asm__ __volatile__("mov %%cs, %[cs]" 418 : /* output */ [cs]"=rm"(cs)); 419 return cs; 420 } 421 422 static inline uint16_t get_ss(void) 423 { 424 uint16_t ss; 425 426 __asm__ __volatile__("mov %%ss, %[ss]" 427 : /* output */ [ss]"=rm"(ss)); 428 return ss; 429 } 430 431 static inline uint16_t get_ds(void) 432 { 433 uint16_t ds; 434 435 __asm__ __volatile__("mov %%ds, %[ds]" 436 : /* output */ [ds]"=rm"(ds)); 437 return ds; 438 } 439 440 static inline uint16_t get_fs(void) 441 { 442 uint16_t fs; 443 444 __asm__ __volatile__("mov %%fs, %[fs]" 445 : /* output */ [fs]"=rm"(fs)); 446 return fs; 447 } 448 449 static inline uint16_t get_gs(void) 450 { 451 uint16_t gs; 452 453 __asm__ __volatile__("mov %%gs, %[gs]" 454 : /* output */ [gs]"=rm"(gs)); 455 return gs; 456 } 457 458 static inline uint16_t get_tr(void) 459 { 460 uint16_t tr; 461 462 __asm__ __volatile__("str %[tr]" 463 : /* output */ [tr]"=rm"(tr)); 464 return tr; 465 } 466 467 static inline uint64_t get_cr0(void) 468 { 469 uint64_t cr0; 470 471 __asm__ __volatile__("mov %%cr0, %[cr0]" 472 : /* output */ [cr0]"=r"(cr0)); 473 return cr0; 474 } 475 476 static inline uint64_t get_cr3(void) 477 { 478 uint64_t cr3; 479 480 __asm__ __volatile__("mov %%cr3, %[cr3]" 481 : /* output */ [cr3]"=r"(cr3)); 482 return cr3; 483 } 484 485 static inline uint64_t get_cr4(void) 486 { 487 uint64_t cr4; 488 489 __asm__ __volatile__("mov %%cr4, %[cr4]" 490 : /* output */ [cr4]"=r"(cr4)); 491 return cr4; 492 } 493 494 static inline void set_cr4(uint64_t val) 495 { 496 __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory"); 497 } 498 499 static inline struct desc_ptr get_gdt(void) 500 { 501 struct desc_ptr gdt; 502 __asm__ __volatile__("sgdt %[gdt]" 503 : /* output */ [gdt]"=m"(gdt)); 504 return gdt; 505 } 506 507 static inline struct desc_ptr get_idt(void) 508 { 509 struct desc_ptr idt; 510 __asm__ __volatile__("sidt %[idt]" 511 : /* output */ [idt]"=m"(idt)); 512 return idt; 513 } 514 515 static inline void outl(uint16_t port, uint32_t value) 516 { 517 __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value)); 518 } 519 520 static inline void __cpuid(uint32_t function, uint32_t index, 521 uint32_t *eax, uint32_t *ebx, 522 uint32_t *ecx, uint32_t *edx) 523 { 524 *eax = function; 525 *ecx = index; 526 527 asm volatile("cpuid" 528 : "=a" (*eax), 529 "=b" (*ebx), 530 "=c" (*ecx), 531 "=d" (*edx) 532 : "0" (*eax), "2" (*ecx) 533 : "memory"); 534 } 535 536 static inline void cpuid(uint32_t function, 537 uint32_t *eax, uint32_t *ebx, 538 uint32_t *ecx, uint32_t *edx) 539 { 540 return __cpuid(function, 0, eax, ebx, ecx, edx); 541 } 542 543 static inline uint32_t this_cpu_fms(void) 544 { 545 uint32_t eax, ebx, ecx, edx; 546 547 cpuid(1, &eax, &ebx, &ecx, &edx); 548 return eax; 549 } 550 551 static inline uint32_t this_cpu_family(void) 552 { 553 return x86_family(this_cpu_fms()); 554 } 555 556 static inline uint32_t this_cpu_model(void) 557 { 558 return x86_model(this_cpu_fms()); 559 } 560 561 static inline bool this_cpu_vendor_string_is(const char *vendor) 562 { 563 const uint32_t *chunk = (const uint32_t *)vendor; 564 uint32_t eax, ebx, ecx, edx; 565 566 cpuid(0, &eax, &ebx, &ecx, &edx); 567 return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]); 568 } 569 570 static inline bool this_cpu_is_intel(void) 571 { 572 return this_cpu_vendor_string_is("GenuineIntel"); 573 } 574 575 /* 576 * Exclude early K5 samples with a vendor string of "AMDisbetter!" 577 */ 578 static inline bool this_cpu_is_amd(void) 579 { 580 return this_cpu_vendor_string_is("AuthenticAMD"); 581 } 582 583 static inline uint32_t __this_cpu_has(uint32_t function, uint32_t index, 584 uint8_t reg, uint8_t lo, uint8_t hi) 585 { 586 uint32_t gprs[4]; 587 588 __cpuid(function, index, 589 &gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX], 590 &gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]); 591 592 return (gprs[reg] & GENMASK(hi, lo)) >> lo; 593 } 594 595 static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature) 596 { 597 return __this_cpu_has(feature.function, feature.index, 598 feature.reg, feature.bit, feature.bit); 599 } 600 601 static inline uint32_t this_cpu_property(struct kvm_x86_cpu_property property) 602 { 603 return __this_cpu_has(property.function, property.index, 604 property.reg, property.lo_bit, property.hi_bit); 605 } 606 607 static __always_inline bool this_cpu_has_p(struct kvm_x86_cpu_property property) 608 { 609 uint32_t max_leaf; 610 611 switch (property.function & 0xc0000000) { 612 case 0: 613 max_leaf = this_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF); 614 break; 615 case 0x40000000: 616 max_leaf = this_cpu_property(X86_PROPERTY_MAX_KVM_LEAF); 617 break; 618 case 0x80000000: 619 max_leaf = this_cpu_property(X86_PROPERTY_MAX_EXT_LEAF); 620 break; 621 case 0xc0000000: 622 max_leaf = this_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF); 623 } 624 return max_leaf >= property.function; 625 } 626 627 static inline bool this_pmu_has(struct kvm_x86_pmu_feature feature) 628 { 629 uint32_t nr_bits = this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH); 630 631 return nr_bits > feature.anti_feature.bit && 632 !this_cpu_has(feature.anti_feature); 633 } 634 635 typedef u32 __attribute__((vector_size(16))) sse128_t; 636 #define __sse128_u union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; } 637 #define sse128_lo(x) ({ __sse128_u t; t.vec = x; t.as_u64[0]; }) 638 #define sse128_hi(x) ({ __sse128_u t; t.vec = x; t.as_u64[1]; }) 639 640 static inline void read_sse_reg(int reg, sse128_t *data) 641 { 642 switch (reg) { 643 case 0: 644 asm("movdqa %%xmm0, %0" : "=m"(*data)); 645 break; 646 case 1: 647 asm("movdqa %%xmm1, %0" : "=m"(*data)); 648 break; 649 case 2: 650 asm("movdqa %%xmm2, %0" : "=m"(*data)); 651 break; 652 case 3: 653 asm("movdqa %%xmm3, %0" : "=m"(*data)); 654 break; 655 case 4: 656 asm("movdqa %%xmm4, %0" : "=m"(*data)); 657 break; 658 case 5: 659 asm("movdqa %%xmm5, %0" : "=m"(*data)); 660 break; 661 case 6: 662 asm("movdqa %%xmm6, %0" : "=m"(*data)); 663 break; 664 case 7: 665 asm("movdqa %%xmm7, %0" : "=m"(*data)); 666 break; 667 default: 668 BUG(); 669 } 670 } 671 672 static inline void write_sse_reg(int reg, const sse128_t *data) 673 { 674 switch (reg) { 675 case 0: 676 asm("movdqa %0, %%xmm0" : : "m"(*data)); 677 break; 678 case 1: 679 asm("movdqa %0, %%xmm1" : : "m"(*data)); 680 break; 681 case 2: 682 asm("movdqa %0, %%xmm2" : : "m"(*data)); 683 break; 684 case 3: 685 asm("movdqa %0, %%xmm3" : : "m"(*data)); 686 break; 687 case 4: 688 asm("movdqa %0, %%xmm4" : : "m"(*data)); 689 break; 690 case 5: 691 asm("movdqa %0, %%xmm5" : : "m"(*data)); 692 break; 693 case 6: 694 asm("movdqa %0, %%xmm6" : : "m"(*data)); 695 break; 696 case 7: 697 asm("movdqa %0, %%xmm7" : : "m"(*data)); 698 break; 699 default: 700 BUG(); 701 } 702 } 703 704 static inline void cpu_relax(void) 705 { 706 asm volatile("rep; nop" ::: "memory"); 707 } 708 709 #define ud2() \ 710 __asm__ __volatile__( \ 711 "ud2\n" \ 712 ) 713 714 #define hlt() \ 715 __asm__ __volatile__( \ 716 "hlt\n" \ 717 ) 718 719 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu); 720 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state); 721 void kvm_x86_state_cleanup(struct kvm_x86_state *state); 722 723 const struct kvm_msr_list *kvm_get_msr_index_list(void); 724 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void); 725 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index); 726 uint64_t kvm_get_feature_msr(uint64_t msr_index); 727 728 static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu, 729 struct kvm_msrs *msrs) 730 { 731 int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs); 732 733 TEST_ASSERT(r == msrs->nmsrs, 734 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)", 735 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index); 736 } 737 static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs) 738 { 739 int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs); 740 741 TEST_ASSERT(r == msrs->nmsrs, 742 "KVM_SET_MSRS failed, r: %i (failed on MSR %x)", 743 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index); 744 } 745 static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu, 746 struct kvm_debugregs *debugregs) 747 { 748 vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs); 749 } 750 static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu, 751 struct kvm_debugregs *debugregs) 752 { 753 vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs); 754 } 755 static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu, 756 struct kvm_xsave *xsave) 757 { 758 vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave); 759 } 760 static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu, 761 struct kvm_xsave *xsave) 762 { 763 vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave); 764 } 765 static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu, 766 struct kvm_xsave *xsave) 767 { 768 vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave); 769 } 770 static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu, 771 struct kvm_xcrs *xcrs) 772 { 773 vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs); 774 } 775 static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs) 776 { 777 vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs); 778 } 779 780 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid, 781 uint32_t function, uint32_t index); 782 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void); 783 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void); 784 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu); 785 786 static inline uint32_t kvm_cpu_fms(void) 787 { 788 return get_cpuid_entry(kvm_get_supported_cpuid(), 0x1, 0)->eax; 789 } 790 791 static inline uint32_t kvm_cpu_family(void) 792 { 793 return x86_family(kvm_cpu_fms()); 794 } 795 796 static inline uint32_t kvm_cpu_model(void) 797 { 798 return x86_model(kvm_cpu_fms()); 799 } 800 801 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid, 802 struct kvm_x86_cpu_feature feature); 803 804 static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature) 805 { 806 return kvm_cpuid_has(kvm_get_supported_cpuid(), feature); 807 } 808 809 uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid, 810 struct kvm_x86_cpu_property property); 811 812 static inline uint32_t kvm_cpu_property(struct kvm_x86_cpu_property property) 813 { 814 return kvm_cpuid_property(kvm_get_supported_cpuid(), property); 815 } 816 817 static __always_inline bool kvm_cpu_has_p(struct kvm_x86_cpu_property property) 818 { 819 uint32_t max_leaf; 820 821 switch (property.function & 0xc0000000) { 822 case 0: 823 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF); 824 break; 825 case 0x40000000: 826 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_KVM_LEAF); 827 break; 828 case 0x80000000: 829 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_EXT_LEAF); 830 break; 831 case 0xc0000000: 832 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF); 833 } 834 return max_leaf >= property.function; 835 } 836 837 static inline bool kvm_pmu_has(struct kvm_x86_pmu_feature feature) 838 { 839 uint32_t nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH); 840 841 return nr_bits > feature.anti_feature.bit && 842 !kvm_cpu_has(feature.anti_feature); 843 } 844 845 static inline size_t kvm_cpuid2_size(int nr_entries) 846 { 847 return sizeof(struct kvm_cpuid2) + 848 sizeof(struct kvm_cpuid_entry2) * nr_entries; 849 } 850 851 /* 852 * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of 853 * entries sized to hold @nr_entries. The caller is responsible for freeing 854 * the struct. 855 */ 856 static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries) 857 { 858 struct kvm_cpuid2 *cpuid; 859 860 cpuid = malloc(kvm_cpuid2_size(nr_entries)); 861 TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2"); 862 863 cpuid->nent = nr_entries; 864 865 return cpuid; 866 } 867 868 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid); 869 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu); 870 871 static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu, 872 uint32_t function, 873 uint32_t index) 874 { 875 return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid, 876 function, index); 877 } 878 879 static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu, 880 uint32_t function) 881 { 882 return __vcpu_get_cpuid_entry(vcpu, function, 0); 883 } 884 885 static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu) 886 { 887 int r; 888 889 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first"); 890 r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid); 891 if (r) 892 return r; 893 894 /* On success, refresh the cache to pick up adjustments made by KVM. */ 895 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid); 896 return 0; 897 } 898 899 static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) 900 { 901 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first"); 902 vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid); 903 904 /* Refresh the cache to pick up adjustments made by KVM. */ 905 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid); 906 } 907 908 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr); 909 910 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function); 911 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu, 912 struct kvm_x86_cpu_feature feature, 913 bool set); 914 915 static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu, 916 struct kvm_x86_cpu_feature feature) 917 { 918 vcpu_set_or_clear_cpuid_feature(vcpu, feature, true); 919 920 } 921 922 static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu, 923 struct kvm_x86_cpu_feature feature) 924 { 925 vcpu_set_or_clear_cpuid_feature(vcpu, feature, false); 926 } 927 928 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index); 929 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value); 930 931 static inline void vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, 932 uint64_t msr_value) 933 { 934 int r = _vcpu_set_msr(vcpu, msr_index, msr_value); 935 936 TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_SET_MSRS, r)); 937 } 938 939 940 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits); 941 bool vm_is_unrestricted_guest(struct kvm_vm *vm); 942 943 struct ex_regs { 944 uint64_t rax, rcx, rdx, rbx; 945 uint64_t rbp, rsi, rdi; 946 uint64_t r8, r9, r10, r11; 947 uint64_t r12, r13, r14, r15; 948 uint64_t vector; 949 uint64_t error_code; 950 uint64_t rip; 951 uint64_t cs; 952 uint64_t rflags; 953 }; 954 955 struct idt_entry { 956 uint16_t offset0; 957 uint16_t selector; 958 uint16_t ist : 3; 959 uint16_t : 5; 960 uint16_t type : 4; 961 uint16_t : 1; 962 uint16_t dpl : 2; 963 uint16_t p : 1; 964 uint16_t offset1; 965 uint32_t offset2; uint32_t reserved; 966 }; 967 968 void vm_init_descriptor_tables(struct kvm_vm *vm); 969 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu); 970 void vm_install_exception_handler(struct kvm_vm *vm, int vector, 971 void (*handler)(struct ex_regs *)); 972 973 /* If a toddler were to say "abracadabra". */ 974 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL 975 976 /* 977 * KVM selftest exception fixup uses registers to coordinate with the exception 978 * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory 979 * per-CPU data. Using only registers avoids having to map memory into the 980 * guest, doesn't require a valid, stable GS.base, and reduces the risk of 981 * for recursive faults when accessing memory in the handler. The downside to 982 * using registers is that it restricts what registers can be used by the actual 983 * instruction. But, selftests are 64-bit only, making register* pressure a 984 * minor concern. Use r9-r11 as they are volatile, i.e. don't need to be saved 985 * by the callee, and except for r11 are not implicit parameters to any 986 * instructions. Ideally, fixup would use r8-r10 and thus avoid implicit 987 * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V 988 * is higher priority than testing non-faulting SYSCALL/SYSRET. 989 * 990 * Note, the fixup handler deliberately does not handle #DE, i.e. the vector 991 * is guaranteed to be non-zero on fault. 992 * 993 * REGISTER INPUTS: 994 * r9 = MAGIC 995 * r10 = RIP 996 * r11 = new RIP on fault 997 * 998 * REGISTER OUTPUTS: 999 * r9 = exception vector (non-zero) 1000 * r10 = error code 1001 */ 1002 #define KVM_ASM_SAFE(insn) \ 1003 "mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t" \ 1004 "lea 1f(%%rip), %%r10\n\t" \ 1005 "lea 2f(%%rip), %%r11\n\t" \ 1006 "1: " insn "\n\t" \ 1007 "xor %%r9, %%r9\n\t" \ 1008 "2:\n\t" \ 1009 "mov %%r9b, %[vector]\n\t" \ 1010 "mov %%r10, %[error_code]\n\t" 1011 1012 #define KVM_ASM_SAFE_OUTPUTS(v, ec) [vector] "=qm"(v), [error_code] "=rm"(ec) 1013 #define KVM_ASM_SAFE_CLOBBERS "r9", "r10", "r11" 1014 1015 #define kvm_asm_safe(insn, inputs...) \ 1016 ({ \ 1017 uint64_t ign_error_code; \ 1018 uint8_t vector; \ 1019 \ 1020 asm volatile(KVM_ASM_SAFE(insn) \ 1021 : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \ 1022 : inputs \ 1023 : KVM_ASM_SAFE_CLOBBERS); \ 1024 vector; \ 1025 }) 1026 1027 #define kvm_asm_safe_ec(insn, error_code, inputs...) \ 1028 ({ \ 1029 uint8_t vector; \ 1030 \ 1031 asm volatile(KVM_ASM_SAFE(insn) \ 1032 : KVM_ASM_SAFE_OUTPUTS(vector, error_code) \ 1033 : inputs \ 1034 : KVM_ASM_SAFE_CLOBBERS); \ 1035 vector; \ 1036 }) 1037 1038 static inline uint8_t rdmsr_safe(uint32_t msr, uint64_t *val) 1039 { 1040 uint64_t error_code; 1041 uint8_t vector; 1042 uint32_t a, d; 1043 1044 asm volatile(KVM_ASM_SAFE("rdmsr") 1045 : "=a"(a), "=d"(d), KVM_ASM_SAFE_OUTPUTS(vector, error_code) 1046 : "c"(msr) 1047 : KVM_ASM_SAFE_CLOBBERS); 1048 1049 *val = (uint64_t)a | ((uint64_t)d << 32); 1050 return vector; 1051 } 1052 1053 static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val) 1054 { 1055 return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr)); 1056 } 1057 1058 bool kvm_is_tdp_enabled(void); 1059 1060 uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr, 1061 int *level); 1062 uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr); 1063 1064 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, 1065 uint64_t a3); 1066 1067 void __vm_xsave_require_permission(int bit, const char *name); 1068 1069 #define vm_xsave_require_permission(perm) \ 1070 __vm_xsave_require_permission(perm, #perm) 1071 1072 enum pg_level { 1073 PG_LEVEL_NONE, 1074 PG_LEVEL_4K, 1075 PG_LEVEL_2M, 1076 PG_LEVEL_1G, 1077 PG_LEVEL_512G, 1078 PG_LEVEL_NUM 1079 }; 1080 1081 #define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12) 1082 #define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level)) 1083 1084 #define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K) 1085 #define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M) 1086 #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G) 1087 1088 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level); 1089 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 1090 uint64_t nr_bytes, int level); 1091 1092 /* 1093 * Basic CPU control in CR0 1094 */ 1095 #define X86_CR0_PE (1UL<<0) /* Protection Enable */ 1096 #define X86_CR0_MP (1UL<<1) /* Monitor Coprocessor */ 1097 #define X86_CR0_EM (1UL<<2) /* Emulation */ 1098 #define X86_CR0_TS (1UL<<3) /* Task Switched */ 1099 #define X86_CR0_ET (1UL<<4) /* Extension Type */ 1100 #define X86_CR0_NE (1UL<<5) /* Numeric Error */ 1101 #define X86_CR0_WP (1UL<<16) /* Write Protect */ 1102 #define X86_CR0_AM (1UL<<18) /* Alignment Mask */ 1103 #define X86_CR0_NW (1UL<<29) /* Not Write-through */ 1104 #define X86_CR0_CD (1UL<<30) /* Cache Disable */ 1105 #define X86_CR0_PG (1UL<<31) /* Paging */ 1106 1107 #define XSTATE_XTILE_CFG_BIT 17 1108 #define XSTATE_XTILE_DATA_BIT 18 1109 1110 #define XSTATE_XTILE_CFG_MASK (1ULL << XSTATE_XTILE_CFG_BIT) 1111 #define XSTATE_XTILE_DATA_MASK (1ULL << XSTATE_XTILE_DATA_BIT) 1112 #define XFEATURE_XTILE_MASK (XSTATE_XTILE_CFG_MASK | \ 1113 XSTATE_XTILE_DATA_MASK) 1114 1115 #define PFERR_PRESENT_BIT 0 1116 #define PFERR_WRITE_BIT 1 1117 #define PFERR_USER_BIT 2 1118 #define PFERR_RSVD_BIT 3 1119 #define PFERR_FETCH_BIT 4 1120 #define PFERR_PK_BIT 5 1121 #define PFERR_SGX_BIT 15 1122 #define PFERR_GUEST_FINAL_BIT 32 1123 #define PFERR_GUEST_PAGE_BIT 33 1124 #define PFERR_IMPLICIT_ACCESS_BIT 48 1125 1126 #define PFERR_PRESENT_MASK BIT(PFERR_PRESENT_BIT) 1127 #define PFERR_WRITE_MASK BIT(PFERR_WRITE_BIT) 1128 #define PFERR_USER_MASK BIT(PFERR_USER_BIT) 1129 #define PFERR_RSVD_MASK BIT(PFERR_RSVD_BIT) 1130 #define PFERR_FETCH_MASK BIT(PFERR_FETCH_BIT) 1131 #define PFERR_PK_MASK BIT(PFERR_PK_BIT) 1132 #define PFERR_SGX_MASK BIT(PFERR_SGX_BIT) 1133 #define PFERR_GUEST_FINAL_MASK BIT_ULL(PFERR_GUEST_FINAL_BIT) 1134 #define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT) 1135 #define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT) 1136 1137 #endif /* SELFTEST_KVM_PROCESSOR_H */ 1138