1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/errno.h> 5 #include <linux/smp.h> 6 7 #include "../cpuid.h" 8 #include "hyperv.h" 9 #include "nested.h" 10 #include "vmcs.h" 11 #include "vmx.h" 12 #include "trace.h" 13 14 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 15 16 /* 17 * Enlightened VMCSv1 doesn't support these: 18 * 19 * POSTED_INTR_NV = 0x00000002, 20 * GUEST_INTR_STATUS = 0x00000810, 21 * APIC_ACCESS_ADDR = 0x00002014, 22 * POSTED_INTR_DESC_ADDR = 0x00002016, 23 * EOI_EXIT_BITMAP0 = 0x0000201c, 24 * EOI_EXIT_BITMAP1 = 0x0000201e, 25 * EOI_EXIT_BITMAP2 = 0x00002020, 26 * EOI_EXIT_BITMAP3 = 0x00002022, 27 * GUEST_PML_INDEX = 0x00000812, 28 * PML_ADDRESS = 0x0000200e, 29 * VM_FUNCTION_CONTROL = 0x00002018, 30 * EPTP_LIST_ADDRESS = 0x00002024, 31 * VMREAD_BITMAP = 0x00002026, 32 * VMWRITE_BITMAP = 0x00002028, 33 * 34 * TSC_MULTIPLIER = 0x00002032, 35 * PLE_GAP = 0x00004020, 36 * PLE_WINDOW = 0x00004022, 37 * VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, 38 * 39 * Currently unsupported in KVM: 40 * GUEST_IA32_RTIT_CTL = 0x00002814, 41 */ 42 #define EVMCS1_SUPPORTED_PINCTRL \ 43 (PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ 44 PIN_BASED_EXT_INTR_MASK | \ 45 PIN_BASED_NMI_EXITING | \ 46 PIN_BASED_VIRTUAL_NMIS) 47 48 #define EVMCS1_SUPPORTED_EXEC_CTRL \ 49 (CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ 50 CPU_BASED_HLT_EXITING | \ 51 CPU_BASED_CR3_LOAD_EXITING | \ 52 CPU_BASED_CR3_STORE_EXITING | \ 53 CPU_BASED_UNCOND_IO_EXITING | \ 54 CPU_BASED_MOV_DR_EXITING | \ 55 CPU_BASED_USE_TSC_OFFSETTING | \ 56 CPU_BASED_MWAIT_EXITING | \ 57 CPU_BASED_MONITOR_EXITING | \ 58 CPU_BASED_INVLPG_EXITING | \ 59 CPU_BASED_RDPMC_EXITING | \ 60 CPU_BASED_INTR_WINDOW_EXITING | \ 61 CPU_BASED_CR8_LOAD_EXITING | \ 62 CPU_BASED_CR8_STORE_EXITING | \ 63 CPU_BASED_RDTSC_EXITING | \ 64 CPU_BASED_TPR_SHADOW | \ 65 CPU_BASED_USE_IO_BITMAPS | \ 66 CPU_BASED_MONITOR_TRAP_FLAG | \ 67 CPU_BASED_USE_MSR_BITMAPS | \ 68 CPU_BASED_NMI_WINDOW_EXITING | \ 69 CPU_BASED_PAUSE_EXITING | \ 70 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 71 72 #define EVMCS1_SUPPORTED_2NDEXEC \ 73 (SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \ 74 SECONDARY_EXEC_WBINVD_EXITING | \ 75 SECONDARY_EXEC_ENABLE_VPID | \ 76 SECONDARY_EXEC_ENABLE_EPT | \ 77 SECONDARY_EXEC_UNRESTRICTED_GUEST | \ 78 SECONDARY_EXEC_DESC | \ 79 SECONDARY_EXEC_ENABLE_RDTSCP | \ 80 SECONDARY_EXEC_ENABLE_INVPCID | \ 81 SECONDARY_EXEC_XSAVES | \ 82 SECONDARY_EXEC_RDSEED_EXITING | \ 83 SECONDARY_EXEC_RDRAND_EXITING | \ 84 SECONDARY_EXEC_TSC_SCALING | \ 85 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \ 86 SECONDARY_EXEC_PT_USE_GPA | \ 87 SECONDARY_EXEC_PT_CONCEAL_VMX | \ 88 SECONDARY_EXEC_BUS_LOCK_DETECTION | \ 89 SECONDARY_EXEC_NOTIFY_VM_EXITING | \ 90 SECONDARY_EXEC_ENCLS_EXITING) 91 92 #define EVMCS1_SUPPORTED_3RDEXEC (0ULL) 93 94 #define EVMCS1_SUPPORTED_VMEXIT_CTRL \ 95 (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | \ 96 VM_EXIT_SAVE_DEBUG_CONTROLS | \ 97 VM_EXIT_ACK_INTR_ON_EXIT | \ 98 VM_EXIT_HOST_ADDR_SPACE_SIZE | \ 99 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ 100 VM_EXIT_SAVE_IA32_PAT | \ 101 VM_EXIT_LOAD_IA32_PAT | \ 102 VM_EXIT_SAVE_IA32_EFER | \ 103 VM_EXIT_LOAD_IA32_EFER | \ 104 VM_EXIT_CLEAR_BNDCFGS | \ 105 VM_EXIT_PT_CONCEAL_PIP | \ 106 VM_EXIT_CLEAR_IA32_RTIT_CTL) 107 108 #define EVMCS1_SUPPORTED_VMENTRY_CTRL \ 109 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | \ 110 VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 111 VM_ENTRY_IA32E_MODE | \ 112 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \ 113 VM_ENTRY_LOAD_IA32_PAT | \ 114 VM_ENTRY_LOAD_IA32_EFER | \ 115 VM_ENTRY_LOAD_BNDCFGS | \ 116 VM_ENTRY_PT_CONCEAL_PIP | \ 117 VM_ENTRY_LOAD_IA32_RTIT_CTL) 118 119 #define EVMCS1_SUPPORTED_VMFUNC (0) 120 121 #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) 122 #define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \ 123 {EVMCS1_OFFSET(name), clean_field} 124 125 const struct evmcs_field vmcs_field_to_evmcs_1[] = { 126 /* 64 bit rw */ 127 EVMCS1_FIELD(GUEST_RIP, guest_rip, 128 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 129 EVMCS1_FIELD(GUEST_RSP, guest_rsp, 130 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 131 EVMCS1_FIELD(GUEST_RFLAGS, guest_rflags, 132 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 133 EVMCS1_FIELD(HOST_IA32_PAT, host_ia32_pat, 134 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 135 EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer, 136 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 137 EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl, 138 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 139 EVMCS1_FIELD(HOST_CR0, host_cr0, 140 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 141 EVMCS1_FIELD(HOST_CR3, host_cr3, 142 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 143 EVMCS1_FIELD(HOST_CR4, host_cr4, 144 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 145 EVMCS1_FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp, 146 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 147 EVMCS1_FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip, 148 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 149 EVMCS1_FIELD(HOST_RIP, host_rip, 150 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 151 EVMCS1_FIELD(IO_BITMAP_A, io_bitmap_a, 152 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), 153 EVMCS1_FIELD(IO_BITMAP_B, io_bitmap_b, 154 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), 155 EVMCS1_FIELD(MSR_BITMAP, msr_bitmap, 156 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP), 157 EVMCS1_FIELD(GUEST_ES_BASE, guest_es_base, 158 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 159 EVMCS1_FIELD(GUEST_CS_BASE, guest_cs_base, 160 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 161 EVMCS1_FIELD(GUEST_SS_BASE, guest_ss_base, 162 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 163 EVMCS1_FIELD(GUEST_DS_BASE, guest_ds_base, 164 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 165 EVMCS1_FIELD(GUEST_FS_BASE, guest_fs_base, 166 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 167 EVMCS1_FIELD(GUEST_GS_BASE, guest_gs_base, 168 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 169 EVMCS1_FIELD(GUEST_LDTR_BASE, guest_ldtr_base, 170 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 171 EVMCS1_FIELD(GUEST_TR_BASE, guest_tr_base, 172 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 173 EVMCS1_FIELD(GUEST_GDTR_BASE, guest_gdtr_base, 174 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 175 EVMCS1_FIELD(GUEST_IDTR_BASE, guest_idtr_base, 176 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 177 EVMCS1_FIELD(TSC_OFFSET, tsc_offset, 178 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 179 EVMCS1_FIELD(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr, 180 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 181 EVMCS1_FIELD(VMCS_LINK_POINTER, vmcs_link_pointer, 182 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 183 EVMCS1_FIELD(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl, 184 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 185 EVMCS1_FIELD(GUEST_IA32_PAT, guest_ia32_pat, 186 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 187 EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer, 188 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 189 EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl, 190 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 191 EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0, 192 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 193 EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1, 194 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 195 EVMCS1_FIELD(GUEST_PDPTR2, guest_pdptr2, 196 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 197 EVMCS1_FIELD(GUEST_PDPTR3, guest_pdptr3, 198 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 199 EVMCS1_FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions, 200 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 201 EVMCS1_FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp, 202 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 203 EVMCS1_FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip, 204 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 205 EVMCS1_FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask, 206 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 207 EVMCS1_FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask, 208 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 209 EVMCS1_FIELD(CR0_READ_SHADOW, cr0_read_shadow, 210 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 211 EVMCS1_FIELD(CR4_READ_SHADOW, cr4_read_shadow, 212 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 213 EVMCS1_FIELD(GUEST_CR0, guest_cr0, 214 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 215 EVMCS1_FIELD(GUEST_CR3, guest_cr3, 216 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 217 EVMCS1_FIELD(GUEST_CR4, guest_cr4, 218 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 219 EVMCS1_FIELD(GUEST_DR7, guest_dr7, 220 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), 221 EVMCS1_FIELD(HOST_FS_BASE, host_fs_base, 222 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 223 EVMCS1_FIELD(HOST_GS_BASE, host_gs_base, 224 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 225 EVMCS1_FIELD(HOST_TR_BASE, host_tr_base, 226 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 227 EVMCS1_FIELD(HOST_GDTR_BASE, host_gdtr_base, 228 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 229 EVMCS1_FIELD(HOST_IDTR_BASE, host_idtr_base, 230 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 231 EVMCS1_FIELD(HOST_RSP, host_rsp, 232 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), 233 EVMCS1_FIELD(EPT_POINTER, ept_pointer, 234 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), 235 EVMCS1_FIELD(GUEST_BNDCFGS, guest_bndcfgs, 236 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 237 EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap, 238 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 239 EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap, 240 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 241 EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier, 242 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), 243 /* 244 * Not used by KVM: 245 * 246 * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet, 247 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 248 * EVMCS1_FIELD(0x0000682A, guest_ssp, 249 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 250 * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr, 251 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 252 * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl, 253 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 254 * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet, 255 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 256 * EVMCS1_FIELD(0x00006C1A, host_ssp, 257 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 258 * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr, 259 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 260 */ 261 262 /* 64 bit read only */ 263 EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address, 264 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 265 EVMCS1_FIELD(EXIT_QUALIFICATION, exit_qualification, 266 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 267 /* 268 * Not defined in KVM: 269 * 270 * EVMCS1_FIELD(0x00006402, exit_io_instruction_ecx, 271 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 272 * EVMCS1_FIELD(0x00006404, exit_io_instruction_esi, 273 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 274 * EVMCS1_FIELD(0x00006406, exit_io_instruction_esi, 275 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 276 * EVMCS1_FIELD(0x00006408, exit_io_instruction_eip, 277 * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); 278 */ 279 EVMCS1_FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address, 280 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 281 282 /* 283 * No mask defined in the spec as Hyper-V doesn't currently support 284 * these. Future proof by resetting the whole clean field mask on 285 * access. 286 */ 287 EVMCS1_FIELD(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr, 288 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 289 EVMCS1_FIELD(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr, 290 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 291 EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr, 292 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 293 294 /* 32 bit rw */ 295 EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold, 296 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 297 EVMCS1_FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info, 298 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), 299 EVMCS1_FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control, 300 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC), 301 EVMCS1_FIELD(EXCEPTION_BITMAP, exception_bitmap, 302 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN), 303 EVMCS1_FIELD(VM_ENTRY_CONTROLS, vm_entry_controls, 304 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY), 305 EVMCS1_FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field, 306 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), 307 EVMCS1_FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, 308 vm_entry_exception_error_code, 309 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), 310 EVMCS1_FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len, 311 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), 312 EVMCS1_FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs, 313 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 314 EVMCS1_FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control, 315 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), 316 EVMCS1_FIELD(VM_EXIT_CONTROLS, vm_exit_controls, 317 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), 318 EVMCS1_FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control, 319 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), 320 EVMCS1_FIELD(GUEST_ES_LIMIT, guest_es_limit, 321 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 322 EVMCS1_FIELD(GUEST_CS_LIMIT, guest_cs_limit, 323 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 324 EVMCS1_FIELD(GUEST_SS_LIMIT, guest_ss_limit, 325 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 326 EVMCS1_FIELD(GUEST_DS_LIMIT, guest_ds_limit, 327 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 328 EVMCS1_FIELD(GUEST_FS_LIMIT, guest_fs_limit, 329 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 330 EVMCS1_FIELD(GUEST_GS_LIMIT, guest_gs_limit, 331 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 332 EVMCS1_FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit, 333 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 334 EVMCS1_FIELD(GUEST_TR_LIMIT, guest_tr_limit, 335 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 336 EVMCS1_FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit, 337 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 338 EVMCS1_FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit, 339 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 340 EVMCS1_FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes, 341 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 342 EVMCS1_FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes, 343 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 344 EVMCS1_FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes, 345 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 346 EVMCS1_FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes, 347 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 348 EVMCS1_FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes, 349 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 350 EVMCS1_FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes, 351 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 352 EVMCS1_FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes, 353 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 354 EVMCS1_FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes, 355 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 356 EVMCS1_FIELD(GUEST_ACTIVITY_STATE, guest_activity_state, 357 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 358 EVMCS1_FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs, 359 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), 360 361 /* 32 bit read only */ 362 EVMCS1_FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error, 363 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 364 EVMCS1_FIELD(VM_EXIT_REASON, vm_exit_reason, 365 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 366 EVMCS1_FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info, 367 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 368 EVMCS1_FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code, 369 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 370 EVMCS1_FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field, 371 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 372 EVMCS1_FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code, 373 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 374 EVMCS1_FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len, 375 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 376 EVMCS1_FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info, 377 HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), 378 379 /* No mask defined in the spec (not used) */ 380 EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask, 381 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 382 EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match, 383 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 384 EVMCS1_FIELD(CR3_TARGET_COUNT, cr3_target_count, 385 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 386 EVMCS1_FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count, 387 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 388 EVMCS1_FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count, 389 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 390 EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count, 391 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), 392 393 /* 16 bit rw */ 394 EVMCS1_FIELD(HOST_ES_SELECTOR, host_es_selector, 395 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 396 EVMCS1_FIELD(HOST_CS_SELECTOR, host_cs_selector, 397 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 398 EVMCS1_FIELD(HOST_SS_SELECTOR, host_ss_selector, 399 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 400 EVMCS1_FIELD(HOST_DS_SELECTOR, host_ds_selector, 401 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 402 EVMCS1_FIELD(HOST_FS_SELECTOR, host_fs_selector, 403 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 404 EVMCS1_FIELD(HOST_GS_SELECTOR, host_gs_selector, 405 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 406 EVMCS1_FIELD(HOST_TR_SELECTOR, host_tr_selector, 407 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), 408 EVMCS1_FIELD(GUEST_ES_SELECTOR, guest_es_selector, 409 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 410 EVMCS1_FIELD(GUEST_CS_SELECTOR, guest_cs_selector, 411 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 412 EVMCS1_FIELD(GUEST_SS_SELECTOR, guest_ss_selector, 413 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 414 EVMCS1_FIELD(GUEST_DS_SELECTOR, guest_ds_selector, 415 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 416 EVMCS1_FIELD(GUEST_FS_SELECTOR, guest_fs_selector, 417 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 418 EVMCS1_FIELD(GUEST_GS_SELECTOR, guest_gs_selector, 419 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 420 EVMCS1_FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector, 421 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 422 EVMCS1_FIELD(GUEST_TR_SELECTOR, guest_tr_selector, 423 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), 424 EVMCS1_FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id, 425 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), 426 }; 427 const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1); 428 429 u64 nested_get_evmptr(struct kvm_vcpu *vcpu) 430 { 431 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 432 433 if (unlikely(kvm_hv_get_assist_page(vcpu))) 434 return EVMPTR_INVALID; 435 436 if (unlikely(!hv_vcpu->vp_assist_page.enlighten_vmentry)) 437 return EVMPTR_INVALID; 438 439 return hv_vcpu->vp_assist_page.current_nested_vmcs; 440 } 441 442 uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu) 443 { 444 /* 445 * vmcs_version represents the range of supported Enlightened VMCS 446 * versions: lower 8 bits is the minimal version, higher 8 bits is the 447 * maximum supported version. KVM supports versions from 1 to 448 * KVM_EVMCS_VERSION. 449 * 450 * Note, do not check the Hyper-V is fully enabled in guest CPUID, this 451 * helper is used to _get_ the vCPU's supported CPUID. 452 */ 453 if (kvm_cpu_cap_get(X86_FEATURE_VMX) && 454 (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled)) 455 return (KVM_EVMCS_VERSION << 8) | 1; 456 457 return 0; 458 } 459 460 enum evmcs_revision { 461 EVMCSv1_LEGACY, 462 NR_EVMCS_REVISIONS, 463 }; 464 465 enum evmcs_ctrl_type { 466 EVMCS_EXIT_CTRLS, 467 EVMCS_ENTRY_CTRLS, 468 EVMCS_EXEC_CTRL, 469 EVMCS_2NDEXEC, 470 EVMCS_3RDEXEC, 471 EVMCS_PINCTRL, 472 EVMCS_VMFUNC, 473 NR_EVMCS_CTRLS, 474 }; 475 476 static const u32 evmcs_supported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = { 477 [EVMCS_EXIT_CTRLS] = { 478 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMEXIT_CTRL, 479 }, 480 [EVMCS_ENTRY_CTRLS] = { 481 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMENTRY_CTRL, 482 }, 483 [EVMCS_EXEC_CTRL] = { 484 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_EXEC_CTRL, 485 }, 486 [EVMCS_2NDEXEC] = { 487 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_2NDEXEC & ~SECONDARY_EXEC_TSC_SCALING, 488 }, 489 [EVMCS_3RDEXEC] = { 490 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_3RDEXEC, 491 }, 492 [EVMCS_PINCTRL] = { 493 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_PINCTRL, 494 }, 495 [EVMCS_VMFUNC] = { 496 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMFUNC, 497 }, 498 }; 499 500 static u32 evmcs_get_supported_ctls(enum evmcs_ctrl_type ctrl_type) 501 { 502 enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY; 503 504 return evmcs_supported_ctrls[ctrl_type][evmcs_rev]; 505 } 506 507 static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu) 508 { 509 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 510 511 /* 512 * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to 513 * boot if a PV CPUID feature flag is not also set. Treat the fields 514 * as unsupported if the flag is not set in guest CPUID. This should 515 * be called only for guest accesses, and all guest accesses should be 516 * gated on Hyper-V being enabled and initialized. 517 */ 518 if (WARN_ON_ONCE(!hv_vcpu)) 519 return false; 520 521 return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; 522 } 523 524 void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) 525 { 526 u32 ctl_low = (u32)*pdata; 527 u32 ctl_high = (u32)(*pdata >> 32); 528 u32 supported_ctrls; 529 530 /* 531 * Hyper-V 2016 and 2019 try using these features even when eVMCS 532 * is enabled but there are no corresponding fields. 533 */ 534 switch (msr_index) { 535 case MSR_IA32_VMX_EXIT_CTLS: 536 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 537 supported_ctrls = evmcs_get_supported_ctls(EVMCS_EXIT_CTRLS); 538 if (!evmcs_has_perf_global_ctrl(vcpu)) 539 supported_ctrls &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 540 ctl_high &= supported_ctrls; 541 break; 542 case MSR_IA32_VMX_ENTRY_CTLS: 543 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 544 supported_ctrls = evmcs_get_supported_ctls(EVMCS_ENTRY_CTRLS); 545 if (!evmcs_has_perf_global_ctrl(vcpu)) 546 supported_ctrls &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 547 ctl_high &= supported_ctrls; 548 break; 549 case MSR_IA32_VMX_PROCBASED_CTLS: 550 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 551 ctl_high &= evmcs_get_supported_ctls(EVMCS_EXEC_CTRL); 552 break; 553 case MSR_IA32_VMX_PROCBASED_CTLS2: 554 ctl_high &= evmcs_get_supported_ctls(EVMCS_2NDEXEC); 555 break; 556 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 557 case MSR_IA32_VMX_PINBASED_CTLS: 558 ctl_high &= evmcs_get_supported_ctls(EVMCS_PINCTRL); 559 break; 560 case MSR_IA32_VMX_VMFUNC: 561 ctl_low &= evmcs_get_supported_ctls(EVMCS_VMFUNC); 562 break; 563 } 564 565 *pdata = ctl_low | ((u64)ctl_high << 32); 566 } 567 568 static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type, 569 u32 val) 570 { 571 return !(val & ~evmcs_get_supported_ctls(ctrl_type)); 572 } 573 574 int nested_evmcs_check_controls(struct vmcs12 *vmcs12) 575 { 576 if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL, 577 vmcs12->pin_based_vm_exec_control))) 578 return -EINVAL; 579 580 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXEC_CTRL, 581 vmcs12->cpu_based_vm_exec_control))) 582 return -EINVAL; 583 584 if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC, 585 vmcs12->secondary_vm_exec_control))) 586 return -EINVAL; 587 588 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS, 589 vmcs12->vm_exit_controls))) 590 return -EINVAL; 591 592 if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS, 593 vmcs12->vm_entry_controls))) 594 return -EINVAL; 595 596 /* 597 * VM-Func controls are 64-bit, but KVM currently doesn't support any 598 * controls in bits 63:32, i.e. dropping those bits on the consistency 599 * check is intentional. 600 */ 601 if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32)) 602 return -EINVAL; 603 604 if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC, 605 vmcs12->vm_function_control))) 606 return -EINVAL; 607 608 return 0; 609 } 610 611 #if IS_ENABLED(CONFIG_HYPERV) 612 DEFINE_STATIC_KEY_FALSE(__kvm_is_using_evmcs); 613 614 /* 615 * KVM on Hyper-V always uses the latest known eVMCSv1 revision, the assumption 616 * is: in case a feature has corresponding fields in eVMCS described and it was 617 * exposed in VMX feature MSRs, KVM is free to use it. Warn if KVM meets a 618 * feature which has no corresponding eVMCS field, this likely means that KVM 619 * needs to be updated. 620 */ 621 #define evmcs_check_vmcs_conf(field, ctrl) \ 622 do { \ 623 typeof(vmcs_conf->field) unsupported; \ 624 \ 625 unsupported = vmcs_conf->field & ~EVMCS1_SUPPORTED_ ## ctrl; \ 626 if (unsupported) { \ 627 pr_warn_once(#field " unsupported with eVMCS: 0x%llx\n",\ 628 (u64)unsupported); \ 629 vmcs_conf->field &= EVMCS1_SUPPORTED_ ## ctrl; \ 630 } \ 631 } \ 632 while (0) 633 634 void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) 635 { 636 evmcs_check_vmcs_conf(cpu_based_exec_ctrl, EXEC_CTRL); 637 evmcs_check_vmcs_conf(pin_based_exec_ctrl, PINCTRL); 638 evmcs_check_vmcs_conf(cpu_based_2nd_exec_ctrl, 2NDEXEC); 639 evmcs_check_vmcs_conf(cpu_based_3rd_exec_ctrl, 3RDEXEC); 640 evmcs_check_vmcs_conf(vmentry_ctrl, VMENTRY_CTRL); 641 evmcs_check_vmcs_conf(vmexit_ctrl, VMEXIT_CTRL); 642 } 643 #endif 644 645 int nested_enable_evmcs(struct kvm_vcpu *vcpu, 646 uint16_t *vmcs_version) 647 { 648 struct vcpu_vmx *vmx = to_vmx(vcpu); 649 650 vmx->nested.enlightened_vmcs_enabled = true; 651 652 if (vmcs_version) 653 *vmcs_version = nested_get_evmcs_version(vcpu); 654 655 return 0; 656 } 657 658 bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu) 659 { 660 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 661 struct vcpu_vmx *vmx = to_vmx(vcpu); 662 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 663 664 if (!hv_vcpu || !evmcs) 665 return false; 666 667 if (!evmcs->hv_enlightenments_control.nested_flush_hypercall) 668 return false; 669 670 return hv_vcpu->vp_assist_page.nested_control.features.directhypercall; 671 } 672 673 void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu) 674 { 675 nested_vmx_vmexit(vcpu, HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH, 0, 0); 676 } 677