1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/errno.h> 4 #include <linux/smp.h> 5 6 #include "../cpuid.h" 7 #include "hyperv.h" 8 #include "nested.h" 9 #include "vmcs.h" 10 #include "vmx.h" 11 #include "trace.h" 12 13 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 14 15 DEFINE_STATIC_KEY_FALSE(enable_evmcs); 16 17 #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) 18 #define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \ 19 {EVMCS1_OFFSET(name), clean_field} 20 21 const struct evmcs_field vmcs_field_to_evmcs_1[] = { 22 /* 64 bit rw */ 23 EVMCS1_FIELD(GUEST_RIP, guest_rip, 24 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 25 EVMCS1_FIELD(GUEST_RSP, guest_rsp, 26 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 27 EVMCS1_FIELD(GUEST_RFLAGS, guest_rflags, 28 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 29 EVMCS1_FIELD(HOST_IA32_PAT, host_ia32_pat, 30 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 31 EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer, 32 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 33 EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl, 34 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 35 EVMCS1_FIELD(HOST_CR0, host_cr0, 36 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 37 EVMCS1_FIELD(HOST_CR3, host_cr3, 38 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 39 EVMCS1_FIELD(HOST_CR4, host_cr4, 40 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 41 EVMCS1_FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp, 42 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 43 EVMCS1_FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip, 44 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 45 EVMCS1_FIELD(HOST_RIP, host_rip, 46 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 47 EVMCS1_FIELD(IO_BITMAP_A, io_bitmap_a, 48 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), 49 EVMCS1_FIELD(IO_BITMAP_B, io_bitmap_b, 50 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), 51 EVMCS1_FIELD(MSR_BITMAP, msr_bitmap, 52 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP), 53 EVMCS1_FIELD(GUEST_ES_BASE, guest_es_base, 54 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 55 EVMCS1_FIELD(GUEST_CS_BASE, guest_cs_base, 56 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 57 EVMCS1_FIELD(GUEST_SS_BASE, guest_ss_base, 58 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 59 EVMCS1_FIELD(GUEST_DS_BASE, guest_ds_base, 60 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 61 EVMCS1_FIELD(GUEST_FS_BASE, guest_fs_base, 62 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 63 EVMCS1_FIELD(GUEST_GS_BASE, guest_gs_base, 64 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 65 EVMCS1_FIELD(GUEST_LDTR_BASE, guest_ldtr_base, 66 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 67 EVMCS1_FIELD(GUEST_TR_BASE, guest_tr_base, 68 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 69 EVMCS1_FIELD(GUEST_GDTR_BASE, guest_gdtr_base, 70 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 71 EVMCS1_FIELD(GUEST_IDTR_BASE, guest_idtr_base, 72 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 73 EVMCS1_FIELD(TSC_OFFSET, tsc_offset, 74 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 75 EVMCS1_FIELD(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr, 76 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 77 EVMCS1_FIELD(VMCS_LINK_POINTER, vmcs_link_pointer, 78 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 79 EVMCS1_FIELD(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl, 80 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 81 EVMCS1_FIELD(GUEST_IA32_PAT, guest_ia32_pat, 82 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 83 EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer, 84 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 85 EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl, 86 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 87 EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0, 88 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 89 EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1, 90 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 91 EVMCS1_FIELD(GUEST_PDPTR2, guest_pdptr2, 92 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 93 EVMCS1_FIELD(GUEST_PDPTR3, guest_pdptr3, 94 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 95 EVMCS1_FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions, 96 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 97 EVMCS1_FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp, 98 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 99 EVMCS1_FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip, 100 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 101 EVMCS1_FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask, 102 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 103 EVMCS1_FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask, 104 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 105 EVMCS1_FIELD(CR0_READ_SHADOW, cr0_read_shadow, 106 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 107 EVMCS1_FIELD(CR4_READ_SHADOW, cr4_read_shadow, 108 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 109 EVMCS1_FIELD(GUEST_CR0, guest_cr0, 110 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 111 EVMCS1_FIELD(GUEST_CR3, guest_cr3, 112 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 113 EVMCS1_FIELD(GUEST_CR4, guest_cr4, 114 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 115 EVMCS1_FIELD(GUEST_DR7, guest_dr7, 116 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 117 EVMCS1_FIELD(HOST_FS_BASE, host_fs_base, 118 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 119 EVMCS1_FIELD(HOST_GS_BASE, host_gs_base, 120 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 121 EVMCS1_FIELD(HOST_TR_BASE, host_tr_base, 122 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 123 EVMCS1_FIELD(HOST_GDTR_BASE, host_gdtr_base, 124 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 125 EVMCS1_FIELD(HOST_IDTR_BASE, host_idtr_base, 126 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 127 EVMCS1_FIELD(HOST_RSP, host_rsp, 128 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 129 EVMCS1_FIELD(EPT_POINTER, ept_pointer, 130 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), 131 EVMCS1_FIELD(GUEST_BNDCFGS, guest_bndcfgs, 132 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 133 EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap, 134 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 135 EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap, 136 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 137 EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier, 138 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 139 /* 140 * Not used by KVM: 141 * 142 * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet, 143 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 144 * EVMCS1_FIELD(0x0000682A, guest_ssp, 145 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 146 * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr, 147 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 148 * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl, 149 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 150 * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet, 151 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 152 * EVMCS1_FIELD(0x00006C1A, host_ssp, 153 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 154 * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr, 155 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 156 */ 157 158 /* 64 bit read only */ 159 EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address, 160 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 161 EVMCS1_FIELD(EXIT_QUALIFICATION, exit_qualification, 162 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 163 /* 164 * Not defined in KVM: 165 * 166 * EVMCS1_FIELD(0x00006402, exit_io_instruction_ecx, 167 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 168 * EVMCS1_FIELD(0x00006404, exit_io_instruction_esi, 169 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 170 * EVMCS1_FIELD(0x00006406, exit_io_instruction_esi, 171 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 172 * EVMCS1_FIELD(0x00006408, exit_io_instruction_eip, 173 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 174 */ 175 EVMCS1_FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address, 176 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 177 178 /* 179 * No mask defined in the spec as Hyper-V doesn't currently support 180 * these. Future proof by resetting the whole clean field mask on 181 * access. 182 */ 183 EVMCS1_FIELD(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr, 184 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 185 EVMCS1_FIELD(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr, 186 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 187 EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr, 188 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 189 190 /* 32 bit rw */ 191 EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold, 192 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 193 EVMCS1_FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info, 194 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 195 EVMCS1_FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control, 196 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC), 197 EVMCS1_FIELD(EXCEPTION_BITMAP, exception_bitmap, 198 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN), 199 EVMCS1_FIELD(VM_ENTRY_CONTROLS, vm_entry_controls, 200 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY), 201 EVMCS1_FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field, 202 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), 203 EVMCS1_FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, 204 vm_entry_exception_error_code, 205 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), 206 EVMCS1_FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len, 207 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), 208 EVMCS1_FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs, 209 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 210 EVMCS1_FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control, 211 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), 212 EVMCS1_FIELD(VM_EXIT_CONTROLS, vm_exit_controls, 213 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), 214 EVMCS1_FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control, 215 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), 216 EVMCS1_FIELD(GUEST_ES_LIMIT, guest_es_limit, 217 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 218 EVMCS1_FIELD(GUEST_CS_LIMIT, guest_cs_limit, 219 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 220 EVMCS1_FIELD(GUEST_SS_LIMIT, guest_ss_limit, 221 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 222 EVMCS1_FIELD(GUEST_DS_LIMIT, guest_ds_limit, 223 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 224 EVMCS1_FIELD(GUEST_FS_LIMIT, guest_fs_limit, 225 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 226 EVMCS1_FIELD(GUEST_GS_LIMIT, guest_gs_limit, 227 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 228 EVMCS1_FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit, 229 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 230 EVMCS1_FIELD(GUEST_TR_LIMIT, guest_tr_limit, 231 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 232 EVMCS1_FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit, 233 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 234 EVMCS1_FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit, 235 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 236 EVMCS1_FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes, 237 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 238 EVMCS1_FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes, 239 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 240 EVMCS1_FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes, 241 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 242 EVMCS1_FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes, 243 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 244 EVMCS1_FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes, 245 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 246 EVMCS1_FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes, 247 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 248 EVMCS1_FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes, 249 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 250 EVMCS1_FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes, 251 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 252 EVMCS1_FIELD(GUEST_ACTIVITY_STATE, guest_activity_state, 253 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 254 EVMCS1_FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs, 255 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 256 257 /* 32 bit read only */ 258 EVMCS1_FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error, 259 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 260 EVMCS1_FIELD(VM_EXIT_REASON, vm_exit_reason, 261 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 262 EVMCS1_FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info, 263 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 264 EVMCS1_FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code, 265 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 266 EVMCS1_FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field, 267 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 268 EVMCS1_FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code, 269 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 270 EVMCS1_FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len, 271 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 272 EVMCS1_FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info, 273 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 274 275 /* No mask defined in the spec (not used) */ 276 EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask, 277 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 278 EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match, 279 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 280 EVMCS1_FIELD(CR3_TARGET_COUNT, cr3_target_count, 281 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 282 EVMCS1_FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count, 283 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 284 EVMCS1_FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count, 285 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 286 EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count, 287 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 288 289 /* 16 bit rw */ 290 EVMCS1_FIELD(HOST_ES_SELECTOR, host_es_selector, 291 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 292 EVMCS1_FIELD(HOST_CS_SELECTOR, host_cs_selector, 293 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 294 EVMCS1_FIELD(HOST_SS_SELECTOR, host_ss_selector, 295 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 296 EVMCS1_FIELD(HOST_DS_SELECTOR, host_ds_selector, 297 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 298 EVMCS1_FIELD(HOST_FS_SELECTOR, host_fs_selector, 299 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 300 EVMCS1_FIELD(HOST_GS_SELECTOR, host_gs_selector, 301 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 302 EVMCS1_FIELD(HOST_TR_SELECTOR, host_tr_selector, 303 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 304 EVMCS1_FIELD(GUEST_ES_SELECTOR, guest_es_selector, 305 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 306 EVMCS1_FIELD(GUEST_CS_SELECTOR, guest_cs_selector, 307 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 308 EVMCS1_FIELD(GUEST_SS_SELECTOR, guest_ss_selector, 309 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 310 EVMCS1_FIELD(GUEST_DS_SELECTOR, guest_ds_selector, 311 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 312 EVMCS1_FIELD(GUEST_FS_SELECTOR, guest_fs_selector, 313 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 314 EVMCS1_FIELD(GUEST_GS_SELECTOR, guest_gs_selector, 315 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 316 EVMCS1_FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector, 317 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 318 EVMCS1_FIELD(GUEST_TR_SELECTOR, guest_tr_selector, 319 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 320 EVMCS1_FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id, 321 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), 322 }; 323 const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1); 324 325 u64 nested_get_evmptr(struct kvm_vcpu *vcpu) 326 { 327 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 328 329 if (unlikely(kvm_hv_get_assist_page(vcpu))) 330 return EVMPTR_INVALID; 331 332 if (unlikely(!hv_vcpu->vp_assist_page.enlighten_vmentry)) 333 return EVMPTR_INVALID; 334 335 return hv_vcpu->vp_assist_page.current_nested_vmcs; 336 } 337 338 uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu) 339 { 340 /* 341 * vmcs_version represents the range of supported Enlightened VMCS 342 * versions: lower 8 bits is the minimal version, higher 8 bits is the 343 * maximum supported version. KVM supports versions from 1 to 344 * KVM_EVMCS_VERSION. 345 * 346 * Note, do not check the Hyper-V is fully enabled in guest CPUID, this 347 * helper is used to _get_ the vCPU's supported CPUID. 348 */ 349 if (kvm_cpu_cap_get(X86_FEATURE_VMX) && 350 (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled)) 351 return (KVM_EVMCS_VERSION << 8) | 1; 352 353 return 0; 354 } 355 356 enum evmcs_revision { 357 EVMCSv1_LEGACY, 358 NR_EVMCS_REVISIONS, 359 }; 360 361 enum evmcs_ctrl_type { 362 EVMCS_EXIT_CTRLS, 363 EVMCS_ENTRY_CTRLS, 364 EVMCS_2NDEXEC, 365 EVMCS_PINCTRL, 366 EVMCS_VMFUNC, 367 NR_EVMCS_CTRLS, 368 }; 369 370 static const u32 evmcs_unsupported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = { 371 [EVMCS_EXIT_CTRLS] = { 372 [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_VMEXIT_CTRL, 373 }, 374 [EVMCS_ENTRY_CTRLS] = { 375 [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_VMENTRY_CTRL, 376 }, 377 [EVMCS_2NDEXEC] = { 378 [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_2NDEXEC, 379 }, 380 [EVMCS_PINCTRL] = { 381 [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_PINCTRL, 382 }, 383 [EVMCS_VMFUNC] = { 384 [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_VMFUNC, 385 }, 386 }; 387 388 static u32 evmcs_get_unsupported_ctls(enum evmcs_ctrl_type ctrl_type) 389 { 390 enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY; 391 392 return evmcs_unsupported_ctrls[ctrl_type][evmcs_rev]; 393 } 394 395 static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu) 396 { 397 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 398 399 /* 400 * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to 401 * boot if a PV CPUID feature flag is not also set. Treat the fields 402 * as unsupported if the flag is not set in guest CPUID. This should 403 * be called only for guest accesses, and all guest accesses should be 404 * gated on Hyper-V being enabled and initialized. 405 */ 406 if (WARN_ON_ONCE(!hv_vcpu)) 407 return false; 408 409 return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; 410 } 411 412 void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) 413 { 414 u32 ctl_low = (u32)*pdata; 415 u32 ctl_high = (u32)(*pdata >> 32); 416 u32 unsupported_ctrls; 417 418 /* 419 * Hyper-V 2016 and 2019 try using these features even when eVMCS 420 * is enabled but there are no corresponding fields. 421 */ 422 switch (msr_index) { 423 case MSR_IA32_VMX_EXIT_CTLS: 424 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 425 unsupported_ctrls = evmcs_get_unsupported_ctls(EVMCS_EXIT_CTRLS); 426 if (!evmcs_has_perf_global_ctrl(vcpu)) 427 unsupported_ctrls |= VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 428 ctl_high &= ~unsupported_ctrls; 429 break; 430 case MSR_IA32_VMX_ENTRY_CTLS: 431 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 432 unsupported_ctrls = evmcs_get_unsupported_ctls(EVMCS_ENTRY_CTRLS); 433 if (!evmcs_has_perf_global_ctrl(vcpu)) 434 unsupported_ctrls |= VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 435 ctl_high &= ~unsupported_ctrls; 436 break; 437 case MSR_IA32_VMX_PROCBASED_CTLS2: 438 ctl_high &= ~evmcs_get_unsupported_ctls(EVMCS_2NDEXEC); 439 break; 440 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 441 case MSR_IA32_VMX_PINBASED_CTLS: 442 ctl_high &= ~evmcs_get_unsupported_ctls(EVMCS_PINCTRL); 443 break; 444 case MSR_IA32_VMX_VMFUNC: 445 ctl_low &= ~evmcs_get_unsupported_ctls(EVMCS_VMFUNC); 446 break; 447 } 448 449 *pdata = ctl_low | ((u64)ctl_high << 32); 450 } 451 452 static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type, 453 u32 val) 454 { 455 return !(val & evmcs_get_unsupported_ctls(ctrl_type)); 456 } 457 458 int nested_evmcs_check_controls(struct vmcs12 *vmcs12) 459 { 460 if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL, 461 vmcs12->pin_based_vm_exec_control))) 462 return -EINVAL; 463 464 if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC, 465 vmcs12->secondary_vm_exec_control))) 466 return -EINVAL; 467 468 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS, 469 vmcs12->vm_exit_controls))) 470 return -EINVAL; 471 472 if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS, 473 vmcs12->vm_entry_controls))) 474 return -EINVAL; 475 476 /* 477 * VM-Func controls are 64-bit, but KVM currently doesn't support any 478 * controls in bits 63:32, i.e. dropping those bits on the consistency 479 * check is intentional. 480 */ 481 if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32)) 482 return -EINVAL; 483 484 if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC, 485 vmcs12->vm_function_control))) 486 return -EINVAL; 487 488 return 0; 489 } 490 491 int nested_enable_evmcs(struct kvm_vcpu *vcpu, 492 uint16_t *vmcs_version) 493 { 494 struct vcpu_vmx *vmx = to_vmx(vcpu); 495 496 vmx->nested.enlightened_vmcs_enabled = true; 497 498 if (vmcs_version) 499 *vmcs_version = nested_get_evmcs_version(vcpu); 500 501 return 0; 502 } 503 504 bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu) 505 { 506 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 507 struct vcpu_vmx *vmx = to_vmx(vcpu); 508 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 509 510 if (!hv_vcpu || !evmcs) 511 return false; 512 513 if (!evmcs->hv_enlightenments_control.nested_flush_hypercall) 514 return false; 515 516 return hv_vcpu->vp_assist_page.nested_control.features.directhypercall; 517 } 518 519 void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu) 520 { 521 nested_vmx_vmexit(vcpu, HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH, 0, 0); 522 } 523