1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * tools/testing/selftests/kvm/include/x86_64/processor.h 4 * 5 * Copyright (C) 2018, Google LLC. 6 */ 7 8 #ifndef SELFTEST_KVM_PROCESSOR_H 9 #define SELFTEST_KVM_PROCESSOR_H 10 11 #include <assert.h> 12 #include <stdint.h> 13 #include <syscall.h> 14 15 #include <asm/msr-index.h> 16 #include <asm/prctl.h> 17 18 #include <linux/stringify.h> 19 20 #include "../kvm_util.h" 21 22 #define NMI_VECTOR 0x02 23 24 #define X86_EFLAGS_FIXED (1u << 1) 25 26 #define X86_CR4_VME (1ul << 0) 27 #define X86_CR4_PVI (1ul << 1) 28 #define X86_CR4_TSD (1ul << 2) 29 #define X86_CR4_DE (1ul << 3) 30 #define X86_CR4_PSE (1ul << 4) 31 #define X86_CR4_PAE (1ul << 5) 32 #define X86_CR4_MCE (1ul << 6) 33 #define X86_CR4_PGE (1ul << 7) 34 #define X86_CR4_PCE (1ul << 8) 35 #define X86_CR4_OSFXSR (1ul << 9) 36 #define X86_CR4_OSXMMEXCPT (1ul << 10) 37 #define X86_CR4_UMIP (1ul << 11) 38 #define X86_CR4_LA57 (1ul << 12) 39 #define X86_CR4_VMXE (1ul << 13) 40 #define X86_CR4_SMXE (1ul << 14) 41 #define X86_CR4_FSGSBASE (1ul << 16) 42 #define X86_CR4_PCIDE (1ul << 17) 43 #define X86_CR4_OSXSAVE (1ul << 18) 44 #define X86_CR4_SMEP (1ul << 20) 45 #define X86_CR4_SMAP (1ul << 21) 46 #define X86_CR4_PKE (1ul << 22) 47 48 /* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */ 49 enum cpuid_output_regs { 50 KVM_CPUID_EAX, 51 KVM_CPUID_EBX, 52 KVM_CPUID_ECX, 53 KVM_CPUID_EDX 54 }; 55 56 /* 57 * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be 58 * passed by value with no overhead. 59 */ 60 struct kvm_x86_cpu_feature { 61 u32 function; 62 u16 index; 63 u8 reg; 64 u8 bit; 65 }; 66 #define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \ 67 ({ \ 68 struct kvm_x86_cpu_feature feature = { \ 69 .function = fn, \ 70 .index = idx, \ 71 .reg = KVM_CPUID_##gpr, \ 72 .bit = __bit, \ 73 }; \ 74 \ 75 feature; \ 76 }) 77 78 /* 79 * Basic Leafs, a.k.a. Intel defined 80 */ 81 #define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3) 82 #define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5) 83 #define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6) 84 #define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15) 85 #define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17) 86 #define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21) 87 #define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22) 88 #define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24) 89 #define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26) 90 #define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27) 91 #define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30) 92 #define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7) 93 #define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9) 94 #define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19) 95 #define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25) 96 #define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26) 97 #define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0) 98 #define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1) 99 #define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4) 100 #define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7) 101 #define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10) 102 #define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11) 103 #define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14) 104 #define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20) 105 #define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22) 106 #define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23) 107 #define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24) 108 #define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2) 109 #define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3) 110 #define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16) 111 #define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22) 112 #define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7) 113 #define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20) 114 #define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24) 115 #define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26) 116 #define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29) 117 #define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31) 118 #define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17) 119 #define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18) 120 #define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3) 121 #define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4) 122 123 /* 124 * Extended Leafs, a.k.a. AMD defined 125 */ 126 #define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2) 127 #define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20) 128 #define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26) 129 #define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27) 130 #define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29) 131 #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) 132 #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) 133 #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) 134 #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) 135 #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) 136 #define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4) 137 #define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10) 138 #define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12) 139 #define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16) 140 #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) 141 #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) 142 143 /* 144 * KVM defined paravirt features. 145 */ 146 #define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0) 147 #define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1) 148 #define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2) 149 #define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3) 150 #define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4) 151 #define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5) 152 #define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6) 153 #define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7) 154 /* Bit 8 apparently isn't used?!?! */ 155 #define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9) 156 #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10) 157 #define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11) 158 #define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12) 159 #define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13) 160 #define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14) 161 #define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15) 162 #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16) 163 #define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17) 164 165 /* Page table bitfield declarations */ 166 #define PTE_PRESENT_MASK BIT_ULL(0) 167 #define PTE_WRITABLE_MASK BIT_ULL(1) 168 #define PTE_USER_MASK BIT_ULL(2) 169 #define PTE_ACCESSED_MASK BIT_ULL(5) 170 #define PTE_DIRTY_MASK BIT_ULL(6) 171 #define PTE_LARGE_MASK BIT_ULL(7) 172 #define PTE_GLOBAL_MASK BIT_ULL(8) 173 #define PTE_NX_MASK BIT_ULL(63) 174 175 #define PAGE_SHIFT 12 176 #define PAGE_SIZE (1ULL << PAGE_SHIFT) 177 #define PAGE_MASK (~(PAGE_SIZE-1)) 178 179 #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12) 180 #define PTE_GET_PFN(pte) (((pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) 181 182 /* General Registers in 64-Bit Mode */ 183 struct gpr64_regs { 184 u64 rax; 185 u64 rcx; 186 u64 rdx; 187 u64 rbx; 188 u64 rsp; 189 u64 rbp; 190 u64 rsi; 191 u64 rdi; 192 u64 r8; 193 u64 r9; 194 u64 r10; 195 u64 r11; 196 u64 r12; 197 u64 r13; 198 u64 r14; 199 u64 r15; 200 }; 201 202 struct desc64 { 203 uint16_t limit0; 204 uint16_t base0; 205 unsigned base1:8, type:4, s:1, dpl:2, p:1; 206 unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8; 207 uint32_t base3; 208 uint32_t zero1; 209 } __attribute__((packed)); 210 211 struct desc_ptr { 212 uint16_t size; 213 uint64_t address; 214 } __attribute__((packed)); 215 216 struct kvm_x86_state { 217 struct kvm_xsave *xsave; 218 struct kvm_vcpu_events events; 219 struct kvm_mp_state mp_state; 220 struct kvm_regs regs; 221 struct kvm_xcrs xcrs; 222 struct kvm_sregs sregs; 223 struct kvm_debugregs debugregs; 224 union { 225 struct kvm_nested_state nested; 226 char nested_[16384]; 227 }; 228 struct kvm_msrs msrs; 229 }; 230 231 static inline uint64_t get_desc64_base(const struct desc64 *desc) 232 { 233 return ((uint64_t)desc->base3 << 32) | 234 (desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24)); 235 } 236 237 static inline uint64_t rdtsc(void) 238 { 239 uint32_t eax, edx; 240 uint64_t tsc_val; 241 /* 242 * The lfence is to wait (on Intel CPUs) until all previous 243 * instructions have been executed. If software requires RDTSC to be 244 * executed prior to execution of any subsequent instruction, it can 245 * execute LFENCE immediately after RDTSC 246 */ 247 __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 248 tsc_val = ((uint64_t)edx) << 32 | eax; 249 return tsc_val; 250 } 251 252 static inline uint64_t rdtscp(uint32_t *aux) 253 { 254 uint32_t eax, edx; 255 256 __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux)); 257 return ((uint64_t)edx) << 32 | eax; 258 } 259 260 static inline uint64_t rdmsr(uint32_t msr) 261 { 262 uint32_t a, d; 263 264 __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory"); 265 266 return a | ((uint64_t) d << 32); 267 } 268 269 static inline void wrmsr(uint32_t msr, uint64_t value) 270 { 271 uint32_t a = value; 272 uint32_t d = value >> 32; 273 274 __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory"); 275 } 276 277 278 static inline uint16_t inw(uint16_t port) 279 { 280 uint16_t tmp; 281 282 __asm__ __volatile__("in %%dx, %%ax" 283 : /* output */ "=a" (tmp) 284 : /* input */ "d" (port)); 285 286 return tmp; 287 } 288 289 static inline uint16_t get_es(void) 290 { 291 uint16_t es; 292 293 __asm__ __volatile__("mov %%es, %[es]" 294 : /* output */ [es]"=rm"(es)); 295 return es; 296 } 297 298 static inline uint16_t get_cs(void) 299 { 300 uint16_t cs; 301 302 __asm__ __volatile__("mov %%cs, %[cs]" 303 : /* output */ [cs]"=rm"(cs)); 304 return cs; 305 } 306 307 static inline uint16_t get_ss(void) 308 { 309 uint16_t ss; 310 311 __asm__ __volatile__("mov %%ss, %[ss]" 312 : /* output */ [ss]"=rm"(ss)); 313 return ss; 314 } 315 316 static inline uint16_t get_ds(void) 317 { 318 uint16_t ds; 319 320 __asm__ __volatile__("mov %%ds, %[ds]" 321 : /* output */ [ds]"=rm"(ds)); 322 return ds; 323 } 324 325 static inline uint16_t get_fs(void) 326 { 327 uint16_t fs; 328 329 __asm__ __volatile__("mov %%fs, %[fs]" 330 : /* output */ [fs]"=rm"(fs)); 331 return fs; 332 } 333 334 static inline uint16_t get_gs(void) 335 { 336 uint16_t gs; 337 338 __asm__ __volatile__("mov %%gs, %[gs]" 339 : /* output */ [gs]"=rm"(gs)); 340 return gs; 341 } 342 343 static inline uint16_t get_tr(void) 344 { 345 uint16_t tr; 346 347 __asm__ __volatile__("str %[tr]" 348 : /* output */ [tr]"=rm"(tr)); 349 return tr; 350 } 351 352 static inline uint64_t get_cr0(void) 353 { 354 uint64_t cr0; 355 356 __asm__ __volatile__("mov %%cr0, %[cr0]" 357 : /* output */ [cr0]"=r"(cr0)); 358 return cr0; 359 } 360 361 static inline uint64_t get_cr3(void) 362 { 363 uint64_t cr3; 364 365 __asm__ __volatile__("mov %%cr3, %[cr3]" 366 : /* output */ [cr3]"=r"(cr3)); 367 return cr3; 368 } 369 370 static inline uint64_t get_cr4(void) 371 { 372 uint64_t cr4; 373 374 __asm__ __volatile__("mov %%cr4, %[cr4]" 375 : /* output */ [cr4]"=r"(cr4)); 376 return cr4; 377 } 378 379 static inline void set_cr4(uint64_t val) 380 { 381 __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory"); 382 } 383 384 static inline struct desc_ptr get_gdt(void) 385 { 386 struct desc_ptr gdt; 387 __asm__ __volatile__("sgdt %[gdt]" 388 : /* output */ [gdt]"=m"(gdt)); 389 return gdt; 390 } 391 392 static inline struct desc_ptr get_idt(void) 393 { 394 struct desc_ptr idt; 395 __asm__ __volatile__("sidt %[idt]" 396 : /* output */ [idt]"=m"(idt)); 397 return idt; 398 } 399 400 static inline void outl(uint16_t port, uint32_t value) 401 { 402 __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value)); 403 } 404 405 static inline void __cpuid(uint32_t function, uint32_t index, 406 uint32_t *eax, uint32_t *ebx, 407 uint32_t *ecx, uint32_t *edx) 408 { 409 *eax = function; 410 *ecx = index; 411 412 asm volatile("cpuid" 413 : "=a" (*eax), 414 "=b" (*ebx), 415 "=c" (*ecx), 416 "=d" (*edx) 417 : "0" (*eax), "2" (*ecx) 418 : "memory"); 419 } 420 421 static inline void cpuid(uint32_t function, 422 uint32_t *eax, uint32_t *ebx, 423 uint32_t *ecx, uint32_t *edx) 424 { 425 return __cpuid(function, 0, eax, ebx, ecx, edx); 426 } 427 428 static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature) 429 { 430 uint32_t gprs[4]; 431 432 __cpuid(feature.function, feature.index, 433 &gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX], 434 &gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]); 435 436 return gprs[feature.reg] & BIT(feature.bit); 437 } 438 439 #define SET_XMM(__var, __xmm) \ 440 asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm) 441 442 static inline void set_xmm(int n, unsigned long val) 443 { 444 switch (n) { 445 case 0: 446 SET_XMM(val, xmm0); 447 break; 448 case 1: 449 SET_XMM(val, xmm1); 450 break; 451 case 2: 452 SET_XMM(val, xmm2); 453 break; 454 case 3: 455 SET_XMM(val, xmm3); 456 break; 457 case 4: 458 SET_XMM(val, xmm4); 459 break; 460 case 5: 461 SET_XMM(val, xmm5); 462 break; 463 case 6: 464 SET_XMM(val, xmm6); 465 break; 466 case 7: 467 SET_XMM(val, xmm7); 468 break; 469 } 470 } 471 472 #define GET_XMM(__xmm) \ 473 ({ \ 474 unsigned long __val; \ 475 asm volatile("movq %%"#__xmm", %0" : "=r"(__val)); \ 476 __val; \ 477 }) 478 479 static inline unsigned long get_xmm(int n) 480 { 481 assert(n >= 0 && n <= 7); 482 483 switch (n) { 484 case 0: 485 return GET_XMM(xmm0); 486 case 1: 487 return GET_XMM(xmm1); 488 case 2: 489 return GET_XMM(xmm2); 490 case 3: 491 return GET_XMM(xmm3); 492 case 4: 493 return GET_XMM(xmm4); 494 case 5: 495 return GET_XMM(xmm5); 496 case 6: 497 return GET_XMM(xmm6); 498 case 7: 499 return GET_XMM(xmm7); 500 } 501 502 /* never reached */ 503 return 0; 504 } 505 506 static inline void cpu_relax(void) 507 { 508 asm volatile("rep; nop" ::: "memory"); 509 } 510 511 #define vmmcall() \ 512 __asm__ __volatile__( \ 513 "vmmcall\n" \ 514 ) 515 516 #define ud2() \ 517 __asm__ __volatile__( \ 518 "ud2\n" \ 519 ) 520 521 #define hlt() \ 522 __asm__ __volatile__( \ 523 "hlt\n" \ 524 ) 525 526 bool is_intel_cpu(void); 527 bool is_amd_cpu(void); 528 529 static inline unsigned int x86_family(unsigned int eax) 530 { 531 unsigned int x86; 532 533 x86 = (eax >> 8) & 0xf; 534 535 if (x86 == 0xf) 536 x86 += (eax >> 20) & 0xff; 537 538 return x86; 539 } 540 541 static inline unsigned int x86_model(unsigned int eax) 542 { 543 return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f); 544 } 545 546 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu); 547 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state); 548 void kvm_x86_state_cleanup(struct kvm_x86_state *state); 549 550 const struct kvm_msr_list *kvm_get_msr_index_list(void); 551 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void); 552 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index); 553 uint64_t kvm_get_feature_msr(uint64_t msr_index); 554 555 static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu, 556 struct kvm_msrs *msrs) 557 { 558 int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs); 559 560 TEST_ASSERT(r == msrs->nmsrs, 561 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)", 562 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index); 563 } 564 static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs) 565 { 566 int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs); 567 568 TEST_ASSERT(r == msrs->nmsrs, 569 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)", 570 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index); 571 } 572 static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu, 573 struct kvm_debugregs *debugregs) 574 { 575 vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs); 576 } 577 static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu, 578 struct kvm_debugregs *debugregs) 579 { 580 vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs); 581 } 582 static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu, 583 struct kvm_xsave *xsave) 584 { 585 vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave); 586 } 587 static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu, 588 struct kvm_xsave *xsave) 589 { 590 vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave); 591 } 592 static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu, 593 struct kvm_xsave *xsave) 594 { 595 vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave); 596 } 597 static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu, 598 struct kvm_xcrs *xcrs) 599 { 600 vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs); 601 } 602 static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs) 603 { 604 vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs); 605 } 606 607 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void); 608 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void); 609 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu); 610 611 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid, 612 struct kvm_x86_cpu_feature feature); 613 614 static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature) 615 { 616 return kvm_cpuid_has(kvm_get_supported_cpuid(), feature); 617 } 618 619 static inline size_t kvm_cpuid2_size(int nr_entries) 620 { 621 return sizeof(struct kvm_cpuid2) + 622 sizeof(struct kvm_cpuid_entry2) * nr_entries; 623 } 624 625 /* 626 * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of 627 * entries sized to hold @nr_entries. The caller is responsible for freeing 628 * the struct. 629 */ 630 static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries) 631 { 632 struct kvm_cpuid2 *cpuid; 633 634 cpuid = malloc(kvm_cpuid2_size(nr_entries)); 635 TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2"); 636 637 cpuid->nent = nr_entries; 638 639 return cpuid; 640 } 641 642 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid, 643 uint32_t function, uint32_t index); 644 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid); 645 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu); 646 647 static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu, 648 uint32_t function, 649 uint32_t index) 650 { 651 return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid, 652 function, index); 653 } 654 655 static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu, 656 uint32_t function) 657 { 658 return __vcpu_get_cpuid_entry(vcpu, function, 0); 659 } 660 661 static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu) 662 { 663 int r; 664 665 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first"); 666 r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid); 667 if (r) 668 return r; 669 670 /* On success, refresh the cache to pick up adjustments made by KVM. */ 671 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid); 672 return 0; 673 } 674 675 static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) 676 { 677 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first"); 678 vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid); 679 680 /* Refresh the cache to pick up adjustments made by KVM. */ 681 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid); 682 } 683 684 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr); 685 686 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function); 687 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu, 688 struct kvm_x86_cpu_feature feature, 689 bool set); 690 691 static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu, 692 struct kvm_x86_cpu_feature feature) 693 { 694 vcpu_set_or_clear_cpuid_feature(vcpu, feature, true); 695 696 } 697 698 static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu, 699 struct kvm_x86_cpu_feature feature) 700 { 701 vcpu_set_or_clear_cpuid_feature(vcpu, feature, false); 702 } 703 704 static inline const struct kvm_cpuid_entry2 *__kvm_get_supported_cpuid_entry(uint32_t function, 705 uint32_t index) 706 { 707 return get_cpuid_entry(kvm_get_supported_cpuid(), function, index); 708 } 709 710 static inline const struct kvm_cpuid_entry2 *kvm_get_supported_cpuid_entry(uint32_t function) 711 { 712 return __kvm_get_supported_cpuid_entry(function, 0); 713 } 714 715 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index); 716 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value); 717 718 static inline void vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, 719 uint64_t msr_value) 720 { 721 int r = _vcpu_set_msr(vcpu, msr_index, msr_value); 722 723 TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_SET_MSRS, r)); 724 } 725 726 static inline uint32_t kvm_get_cpuid_max_basic(void) 727 { 728 return kvm_get_supported_cpuid_entry(0)->eax; 729 } 730 731 static inline uint32_t kvm_get_cpuid_max_extended(void) 732 { 733 return kvm_get_supported_cpuid_entry(0x80000000)->eax; 734 } 735 736 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits); 737 bool vm_is_unrestricted_guest(struct kvm_vm *vm); 738 739 struct ex_regs { 740 uint64_t rax, rcx, rdx, rbx; 741 uint64_t rbp, rsi, rdi; 742 uint64_t r8, r9, r10, r11; 743 uint64_t r12, r13, r14, r15; 744 uint64_t vector; 745 uint64_t error_code; 746 uint64_t rip; 747 uint64_t cs; 748 uint64_t rflags; 749 }; 750 751 void vm_init_descriptor_tables(struct kvm_vm *vm); 752 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu); 753 void vm_install_exception_handler(struct kvm_vm *vm, int vector, 754 void (*handler)(struct ex_regs *)); 755 756 /* If a toddler were to say "abracadabra". */ 757 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL 758 759 /* 760 * KVM selftest exception fixup uses registers to coordinate with the exception 761 * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory 762 * per-CPU data. Using only registers avoids having to map memory into the 763 * guest, doesn't require a valid, stable GS.base, and reduces the risk of 764 * for recursive faults when accessing memory in the handler. The downside to 765 * using registers is that it restricts what registers can be used by the actual 766 * instruction. But, selftests are 64-bit only, making register* pressure a 767 * minor concern. Use r9-r11 as they are volatile, i.e. don't need* to be saved 768 * by the callee, and except for r11 are not implicit parameters to any 769 * instructions. Ideally, fixup would use r8-r10 and thus avoid implicit 770 * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V 771 * is higher priority than testing non-faulting SYSCALL/SYSRET. 772 * 773 * Note, the fixup handler deliberately does not handle #DE, i.e. the vector 774 * is guaranteed to be non-zero on fault. 775 * 776 * REGISTER INPUTS: 777 * r9 = MAGIC 778 * r10 = RIP 779 * r11 = new RIP on fault 780 * 781 * REGISTER OUTPUTS: 782 * r9 = exception vector (non-zero) 783 */ 784 #define KVM_ASM_SAFE(insn) \ 785 "mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t" \ 786 "lea 1f(%%rip), %%r10\n\t" \ 787 "lea 2f(%%rip), %%r11\n\t" \ 788 "1: " insn "\n\t" \ 789 "movb $0, %[vector]\n\t" \ 790 "jmp 3f\n\t" \ 791 "2:\n\t" \ 792 "mov %%r9b, %[vector]\n\t" \ 793 "3:\n\t" 794 795 #define KVM_ASM_SAFE_OUTPUTS(v) [vector] "=qm"(v) 796 #define KVM_ASM_SAFE_CLOBBERS "r9", "r10", "r11" 797 798 #define kvm_asm_safe(insn, inputs...) \ 799 ({ \ 800 uint8_t vector; \ 801 \ 802 asm volatile(KVM_ASM_SAFE(insn) \ 803 : KVM_ASM_SAFE_OUTPUTS(vector) \ 804 : inputs \ 805 : KVM_ASM_SAFE_CLOBBERS); \ 806 vector; \ 807 }) 808 809 static inline uint8_t rdmsr_safe(uint32_t msr, uint64_t *val) 810 { 811 uint8_t vector; 812 uint32_t a, d; 813 814 asm volatile(KVM_ASM_SAFE("rdmsr") 815 : "=a"(a), "=d"(d), KVM_ASM_SAFE_OUTPUTS(vector) 816 : "c"(msr) 817 : KVM_ASM_SAFE_CLOBBERS); 818 819 *val = (uint64_t)a | ((uint64_t)d << 32); 820 return vector; 821 } 822 823 static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val) 824 { 825 return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr)); 826 } 827 828 bool kvm_is_tdp_enabled(void); 829 830 uint64_t vm_get_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu, 831 uint64_t vaddr); 832 void vm_set_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu, 833 uint64_t vaddr, uint64_t pte); 834 835 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, 836 uint64_t a3); 837 838 void __vm_xsave_require_permission(int bit, const char *name); 839 840 #define vm_xsave_require_permission(perm) \ 841 __vm_xsave_require_permission(perm, #perm) 842 843 enum pg_level { 844 PG_LEVEL_NONE, 845 PG_LEVEL_4K, 846 PG_LEVEL_2M, 847 PG_LEVEL_1G, 848 PG_LEVEL_512G, 849 PG_LEVEL_NUM 850 }; 851 852 #define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12) 853 #define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level)) 854 855 #define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K) 856 #define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M) 857 #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G) 858 859 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level); 860 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 861 uint64_t nr_bytes, int level); 862 863 /* 864 * Basic CPU control in CR0 865 */ 866 #define X86_CR0_PE (1UL<<0) /* Protection Enable */ 867 #define X86_CR0_MP (1UL<<1) /* Monitor Coprocessor */ 868 #define X86_CR0_EM (1UL<<2) /* Emulation */ 869 #define X86_CR0_TS (1UL<<3) /* Task Switched */ 870 #define X86_CR0_ET (1UL<<4) /* Extension Type */ 871 #define X86_CR0_NE (1UL<<5) /* Numeric Error */ 872 #define X86_CR0_WP (1UL<<16) /* Write Protect */ 873 #define X86_CR0_AM (1UL<<18) /* Alignment Mask */ 874 #define X86_CR0_NW (1UL<<29) /* Not Write-through */ 875 #define X86_CR0_CD (1UL<<30) /* Cache Disable */ 876 #define X86_CR0_PG (1UL<<31) /* Paging */ 877 878 #define XSTATE_XTILE_CFG_BIT 17 879 #define XSTATE_XTILE_DATA_BIT 18 880 881 #define XSTATE_XTILE_CFG_MASK (1ULL << XSTATE_XTILE_CFG_BIT) 882 #define XSTATE_XTILE_DATA_MASK (1ULL << XSTATE_XTILE_DATA_BIT) 883 #define XFEATURE_XTILE_MASK (XSTATE_XTILE_CFG_MASK | \ 884 XSTATE_XTILE_DATA_MASK) 885 #endif /* SELFTEST_KVM_PROCESSOR_H */ 886