103 	((struct kvm_vcpu *)(ctxt)->vcpu)
125 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
126 static void process_nmi(struct kvm_vcpu *vcpu);
127 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
128 static void store_regs(struct kvm_vcpu *vcpu);
129 static int sync_regs(struct kvm_vcpu *vcpu);
130 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
132 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
133 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
272 	STATS_DESC_COUNTER(VCPU, pf_taken),
273 	STATS_DESC_COUNTER(VCPU, pf_fixed),
274 	STATS_DESC_COUNTER(VCPU, pf_emulate),
275 	STATS_DESC_COUNTER(VCPU, pf_spurious),
276 	STATS_DESC_COUNTER(VCPU, pf_fast),
277 	STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
278 	STATS_DESC_COUNTER(VCPU, pf_guest),
279 	STATS_DESC_COUNTER(VCPU, tlb_flush),
280 	STATS_DESC_COUNTER(VCPU, invlpg),
281 	STATS_DESC_COUNTER(VCPU, exits),
282 	STATS_DESC_COUNTER(VCPU, io_exits),
283 	STATS_DESC_COUNTER(VCPU, mmio_exits),
284 	STATS_DESC_COUNTER(VCPU, signal_exits),
285 	STATS_DESC_COUNTER(VCPU, irq_window_exits),
286 	STATS_DESC_COUNTER(VCPU, nmi_window_exits),
287 	STATS_DESC_COUNTER(VCPU, l1d_flush),
288 	STATS_DESC_COUNTER(VCPU, halt_exits),
289 	STATS_DESC_COUNTER(VCPU, request_irq_exits),
290 	STATS_DESC_COUNTER(VCPU, irq_exits),
291 	STATS_DESC_COUNTER(VCPU, host_state_reload),
292 	STATS_DESC_COUNTER(VCPU, fpu_reload),
293 	STATS_DESC_COUNTER(VCPU, insn_emulation),
294 	STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
295 	STATS_DESC_COUNTER(VCPU, hypercalls),
296 	STATS_DESC_COUNTER(VCPU, irq_injections),
297 	STATS_DESC_COUNTER(VCPU, nmi_injections),
298 	STATS_DESC_COUNTER(VCPU, req_event),
299 	STATS_DESC_COUNTER(VCPU, nested_run),
300 	STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
301 	STATS_DESC_COUNTER(VCPU, directed_yield_successful),
302 	STATS_DESC_COUNTER(VCPU, preemption_reported),
303 	STATS_DESC_COUNTER(VCPU, preemption_other),
304 	STATS_DESC_IBOOLEAN(VCPU, guest_mode),
305 	STATS_DESC_COUNTER(VCPU, notify_window_exits),
355 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)  in kvm_async_pf_hash_reset()  argument
359 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_async_pf_hash_reset()
474 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)  in kvm_get_apic_base()  argument
476 	return vcpu->arch.apic_base;  in kvm_get_apic_base()
479 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)  in kvm_get_apic_mode()  argument
481 	return kvm_apic_mode(kvm_get_apic_base(vcpu));  in kvm_get_apic_mode()
485 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_apic_base()  argument
487 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);  in kvm_set_apic_base()
489 	u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |  in kvm_set_apic_base()
490 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);  in kvm_set_apic_base()
501 	kvm_lapic_set_base(vcpu, msr_info->data);  in kvm_set_apic_base()
502 	kvm_recalculate_apic_map(vcpu->kvm);  in kvm_set_apic_base()
573 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,  in kvm_deliver_exception_payload()  argument
586 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;  in kvm_deliver_exception_payload()
603 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
604 		vcpu->arch.dr6 |= ex->payload;  in kvm_deliver_exception_payload()
605 		vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
613 		vcpu->arch.dr6 &= ~BIT(12);  in kvm_deliver_exception_payload()
616 		vcpu->arch.cr2 = ex->payload;  in kvm_deliver_exception_payload()
625 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,  in kvm_queue_exception_vmexit()  argument
629 	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;  in kvm_queue_exception_vmexit()
640 /* Forcibly leave the nested mode in cases like a vCPU reset */
641 static void kvm_leave_nested(struct kvm_vcpu *vcpu)  in kvm_leave_nested()  argument
643 	kvm_x86_ops.nested_ops->leave_nested(vcpu);  in kvm_leave_nested()
646 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,  in kvm_multiple_exception()  argument
653 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_multiple_exception()
662 	if (!reinject && is_guest_mode(vcpu) &&  in kvm_multiple_exception()
663 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {  in kvm_multiple_exception()
664 		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,  in kvm_multiple_exception()
669 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {  in kvm_multiple_exception()
679 			WARN_ON_ONCE(kvm_is_exception_pending(vcpu));  in kvm_multiple_exception()
680 			vcpu->arch.exception.injected = true;  in kvm_multiple_exception()
690 			vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
691 			vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
693 		vcpu->arch.exception.has_error_code = has_error;  in kvm_multiple_exception()
694 		vcpu->arch.exception.vector = nr;  in kvm_multiple_exception()
695 		vcpu->arch.exception.error_code = error_code;  in kvm_multiple_exception()
696 		vcpu->arch.exception.has_payload = has_payload;  in kvm_multiple_exception()
697 		vcpu->arch.exception.payload = payload;  in kvm_multiple_exception()
698 		if (!is_guest_mode(vcpu))  in kvm_multiple_exception()
699 			kvm_deliver_exception_payload(vcpu,  in kvm_multiple_exception()
700 						      &vcpu->arch.exception);  in kvm_multiple_exception()
705 	prev_nr = vcpu->arch.exception.vector;  in kvm_multiple_exception()
708 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_multiple_exception()
719 		vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
720 		vcpu->arch.exception.pending = false;  in kvm_multiple_exception()
722 		kvm_queue_exception_e(vcpu, DF_VECTOR, 0);  in kvm_multiple_exception()
731 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_queue_exception()  argument
733 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);  in kvm_queue_exception()
737 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_requeue_exception()  argument
739 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);  in kvm_requeue_exception()
743 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_p()  argument
746 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);  in kvm_queue_exception_p()
750 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_e_p()  argument
753 	kvm_multiple_exception(vcpu, nr, true, error_code,  in kvm_queue_exception_e_p()
757 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)  in kvm_complete_insn_gp()  argument
760 		kvm_inject_gp(vcpu, 0);  in kvm_complete_insn_gp()
762 		return kvm_skip_emulated_instruction(vcpu);  in kvm_complete_insn_gp()
768 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)  in complete_emulated_insn_gp()  argument
771 		kvm_inject_gp(vcpu, 0);  in complete_emulated_insn_gp()
775 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |  in complete_emulated_insn_gp()
779 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)  in kvm_inject_page_fault()  argument
781 	++vcpu->stat.pf_guest;  in kvm_inject_page_fault()
787 	if (is_guest_mode(vcpu) && fault->async_page_fault)  in kvm_inject_page_fault()
788 		kvm_queue_exception_vmexit(vcpu, PF_VECTOR,  in kvm_inject_page_fault()
792 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,  in kvm_inject_page_fault()
796 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,  in kvm_inject_emulated_page_fault()  argument
802 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :  in kvm_inject_emulated_page_fault()
803 					       vcpu->arch.walk_mmu;  in kvm_inject_emulated_page_fault()
811 		kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,  in kvm_inject_emulated_page_fault()
814 	fault_mmu->inject_page_fault(vcpu, fault);  in kvm_inject_emulated_page_fault()
818 void kvm_inject_nmi(struct kvm_vcpu *vcpu)  in kvm_inject_nmi()  argument
820 	atomic_inc(&vcpu->arch.nmi_queued);  in kvm_inject_nmi()
821 	kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_inject_nmi()
824 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_queue_exception_e()  argument
826 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);  in kvm_queue_exception_e()
830 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_requeue_exception_e()  argument
832 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);  in kvm_requeue_exception_e()
840 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)  in kvm_require_cpl()  argument
842 	if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)  in kvm_require_cpl()
844 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in kvm_require_cpl()
848 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)  in kvm_require_dr()  argument
850 	if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))  in kvm_require_dr()
853 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_require_dr()
858 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)  in pdptr_rsvd_bits()  argument
860 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);  in pdptr_rsvd_bits()
866 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)  in load_pdptrs()  argument
868 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in load_pdptrs()
879 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),  in load_pdptrs()
885 	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,  in load_pdptrs()
892 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {  in load_pdptrs()
902 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);  in load_pdptrs()
905 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in load_pdptrs()
906 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);  in load_pdptrs()
907 	vcpu->arch.pdptrs_from_userspace = false;  in load_pdptrs()
913 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_is_valid_cr0()  argument
926 	return static_call(kvm_x86_is_valid_cr0)(vcpu, cr0);  in kvm_is_valid_cr0()
929 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)  in kvm_post_set_cr0()  argument
944 			kvm_init_mmu(vcpu);  in kvm_post_set_cr0()
950 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_post_set_cr0()
951 		kvm_async_pf_hash_reset(vcpu);  in kvm_post_set_cr0()
958 			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr0()
962 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr0()
965 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&  in kvm_post_set_cr0()
966 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))  in kvm_post_set_cr0()
967 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);  in kvm_post_set_cr0()
971 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_set_cr0()  argument
973 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_set_cr0()
975 	if (!kvm_is_valid_cr0(vcpu, cr0))  in kvm_set_cr0()
984 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&  in kvm_set_cr0()
988 		if (!is_pae(vcpu))  in kvm_set_cr0()
990 		static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in kvm_set_cr0()
995 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&  in kvm_set_cr0()
996 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&  in kvm_set_cr0()
997 	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr0()
1001 	    (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))  in kvm_set_cr0()
1004 	static_call(kvm_x86_set_cr0)(vcpu, cr0);  in kvm_set_cr0()
1006 	kvm_post_set_cr0(vcpu, old_cr0, cr0);  in kvm_set_cr0()
1012 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)  in kvm_lmsw()  argument
1014 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));  in kvm_lmsw()
1018 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_guest_xsave_state()  argument
1020 	if (vcpu->arch.guest_state_protected)  in kvm_load_guest_xsave_state()
1023 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_guest_xsave_state()
1025 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_guest_xsave_state()
1026 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);  in kvm_load_guest_xsave_state()
1028 		if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_guest_xsave_state()
1029 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_guest_xsave_state()
1030 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);  in kvm_load_guest_xsave_state()
1034 	    vcpu->arch.pkru != vcpu->arch.host_pkru &&  in kvm_load_guest_xsave_state()
1035 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_guest_xsave_state()
1036 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))  in kvm_load_guest_xsave_state()
1037 		write_pkru(vcpu->arch.pkru);  in kvm_load_guest_xsave_state()
1041 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_host_xsave_state()  argument
1043 	if (vcpu->arch.guest_state_protected)  in kvm_load_host_xsave_state()
1047 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_host_xsave_state()
1048 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {  in kvm_load_host_xsave_state()
1049 		vcpu->arch.pkru = rdpkru();  in kvm_load_host_xsave_state()
1050 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_host_xsave_state()
1051 			write_pkru(vcpu->arch.host_pkru);  in kvm_load_host_xsave_state()
1054 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_host_xsave_state()
1056 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_host_xsave_state()
1059 		if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_host_xsave_state()
1060 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_host_xsave_state()
1068 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)  in kvm_guest_supported_xfd()  argument
1070 	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;  in kvm_guest_supported_xfd()
1074 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)  in __kvm_set_xcr()  argument
1077 	u64 old_xcr0 = vcpu->arch.xcr0;  in __kvm_set_xcr()
1093 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;  in __kvm_set_xcr()
1112 	vcpu->arch.xcr0 = xcr0;  in __kvm_set_xcr()
1115 		kvm_update_cpuid_runtime(vcpu);  in __kvm_set_xcr()
1119 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)  in kvm_emulate_xsetbv()  argument
1122 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||  in kvm_emulate_xsetbv()
1123 	    __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {  in kvm_emulate_xsetbv()
1124 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_xsetbv()
1128 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_xsetbv()
1132 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in __kvm_is_valid_cr4()  argument
1137 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)  in __kvm_is_valid_cr4()
1144 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_is_valid_cr4()  argument
1146 	return __kvm_is_valid_cr4(vcpu, cr4) &&  in kvm_is_valid_cr4()
1147 	       static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);  in kvm_is_valid_cr4()
1150 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)  in kvm_post_set_cr4()  argument
1153 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr4()
1165 		kvm_mmu_unload(vcpu);  in kvm_post_set_cr4()
1177 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr4()
1187 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_post_set_cr4()
1192 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_set_cr4()  argument
1194 	unsigned long old_cr4 = kvm_read_cr4(vcpu);  in kvm_set_cr4()
1196 	if (!kvm_is_valid_cr4(vcpu, cr4))  in kvm_set_cr4()
1199 	if (is_long_mode(vcpu)) {  in kvm_set_cr4()
1204 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)  in kvm_set_cr4()
1206 		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr4()
1211 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))  in kvm_set_cr4()
1215 	static_call(kvm_x86_set_cr4)(vcpu, cr4);  in kvm_set_cr4()
1217 	kvm_post_set_cr4(vcpu, old_cr4, cr4);  in kvm_set_cr4()
1223 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)  in kvm_invalidate_pcid()  argument
1225 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_invalidate_pcid()
1237 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_invalidate_pcid()
1246 	if (kvm_get_active_pcid(vcpu) == pcid) {  in kvm_invalidate_pcid()
1247 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);  in kvm_invalidate_pcid()
1248 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_invalidate_pcid()
1256 	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))  in kvm_invalidate_pcid()
1260 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)  in kvm_invalidate_pcid()
1263 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);  in kvm_invalidate_pcid()
1266 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)  in kvm_set_cr3()  argument
1271 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {  in kvm_set_cr3()
1279 	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))  in kvm_set_cr3()
1285 	 * the current vCPU mode is accurate.  in kvm_set_cr3()
1287 	if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))  in kvm_set_cr3()
1290 	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))  in kvm_set_cr3()
1293 	if (cr3 != kvm_read_cr3(vcpu))  in kvm_set_cr3()
1294 		kvm_mmu_new_pgd(vcpu, cr3);  in kvm_set_cr3()
1296 	vcpu->arch.cr3 = cr3;  in kvm_set_cr3()
1297 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_set_cr3()
1309 		kvm_invalidate_pcid(vcpu, pcid);  in kvm_set_cr3()
1315 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)  in kvm_set_cr8()  argument
1319 	if (lapic_in_kernel(vcpu))  in kvm_set_cr8()
1320 		kvm_lapic_set_tpr(vcpu, cr8);  in kvm_set_cr8()
1322 		vcpu->arch.cr8 = cr8;  in kvm_set_cr8()
1327 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)  in kvm_get_cr8()  argument
1329 	if (lapic_in_kernel(vcpu))  in kvm_get_cr8()
1330 		return kvm_lapic_get_cr8(vcpu);  in kvm_get_cr8()
1332 		return vcpu->arch.cr8;  in kvm_get_cr8()
1336 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)  in kvm_update_dr0123()  argument
1340 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {  in kvm_update_dr0123()
1342 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_update_dr0123()
1346 void kvm_update_dr7(struct kvm_vcpu *vcpu)  in kvm_update_dr7()  argument
1350 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)  in kvm_update_dr7()
1351 		dr7 = vcpu->arch.guest_debug_dr7;  in kvm_update_dr7()
1353 		dr7 = vcpu->arch.dr7;  in kvm_update_dr7()
1354 	static_call(kvm_x86_set_dr7)(vcpu, dr7);  in kvm_update_dr7()
1355 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1357 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1361 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)  in kvm_dr6_fixed()  argument
1365 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))  in kvm_dr6_fixed()
1368 	if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))  in kvm_dr6_fixed()
1373 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)  in kvm_set_dr()  argument
1375 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_set_dr()
1379 		vcpu->arch.db[array_index_nospec(dr, size)] = val;  in kvm_set_dr()
1380 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))  in kvm_set_dr()
1381 			vcpu->arch.eff_db[dr] = val;  in kvm_set_dr()
1387 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);  in kvm_set_dr()
1393 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;  in kvm_set_dr()
1394 		kvm_update_dr7(vcpu);  in kvm_set_dr()
1402 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)  in kvm_get_dr()  argument
1404 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_get_dr()
1408 		*val = vcpu->arch.db[array_index_nospec(dr, size)];  in kvm_get_dr()
1412 		*val = vcpu->arch.dr6;  in kvm_get_dr()
1416 		*val = vcpu->arch.dr7;  in kvm_get_dr()
1422 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)  in kvm_emulate_rdpmc()  argument
1424 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdpmc()
1427 	if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {  in kvm_emulate_rdpmc()
1428 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_rdpmc()
1432 	kvm_rax_write(vcpu, (u32)data);  in kvm_emulate_rdpmc()
1433 	kvm_rdx_write(vcpu, data >> 32);  in kvm_emulate_rdpmc()
1434 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_rdpmc()
1590  * patch, are immutable once the vCPU model is defined.
1704 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr_feature()  argument
1727 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in __kvm_valid_efer()  argument
1729 	if (efer & EFER_AUTOIBRS && !guest_cpuid_has(vcpu, X86_FEATURE_AUTOIBRS))  in __kvm_valid_efer()
1732 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))  in __kvm_valid_efer()
1735 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))  in __kvm_valid_efer()
1739 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))  in __kvm_valid_efer()
1742 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))  in __kvm_valid_efer()
1748 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in kvm_valid_efer()  argument
1753 	return __kvm_valid_efer(vcpu, efer);  in kvm_valid_efer()
1757 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_efer()  argument
1759 	u64 old_efer = vcpu->arch.efer;  in set_efer()
1767 		if (!__kvm_valid_efer(vcpu, efer))  in set_efer()
1770 		if (is_paging(vcpu) &&  in set_efer()
1771 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))  in set_efer()
1776 	efer |= vcpu->arch.efer & EFER_LMA;  in set_efer()
1778 	r = static_call(kvm_x86_set_efer)(vcpu, efer);  in set_efer()
1785 		kvm_mmu_reset_context(vcpu);  in set_efer()
1796 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)  in kvm_msr_allowed()  argument
1800 	struct kvm *kvm = vcpu->kvm;  in kvm_msr_allowed()
1845 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,  in __kvm_set_msr()  argument
1856 		if (is_noncanonical_address(data, vcpu))  in __kvm_set_msr()
1873 		data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));  in __kvm_set_msr()
1880 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_set_msr()
1881 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))  in __kvm_set_msr()
1893 		if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)  in __kvm_set_msr()
1904 	return static_call(kvm_x86_set_msr)(vcpu, &msr);  in __kvm_set_msr()
1907 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_set_msr_ignored_check()  argument
1910 	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);  in kvm_set_msr_ignored_check()
1925 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in __kvm_get_msr()  argument
1937 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_get_msr()
1938 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))  in __kvm_get_msr()
1946 	ret = static_call(kvm_x86_get_msr)(vcpu, &msr);  in __kvm_get_msr()
1952 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_get_msr_ignored_check()  argument
1955 	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);  in kvm_get_msr_ignored_check()
1967 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr_with_filter()  argument
1969 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))  in kvm_get_msr_with_filter()
1971 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr_with_filter()
1974 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr_with_filter()  argument
1976 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))  in kvm_set_msr_with_filter()
1978 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr_with_filter()
1981 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr()  argument
1983 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr()
1987 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr()  argument
1989 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr()
1993 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)  in complete_userspace_rdmsr()  argument
1995 	if (!vcpu->run->msr.error) {  in complete_userspace_rdmsr()
1996 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);  in complete_userspace_rdmsr()
1997 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);  in complete_userspace_rdmsr()
2001 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)  in complete_emulated_msr_access()  argument
2003 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);  in complete_emulated_msr_access()
2006 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)  in complete_emulated_rdmsr()  argument
2008 	complete_userspace_rdmsr(vcpu);  in complete_emulated_rdmsr()
2009 	return complete_emulated_msr_access(vcpu);  in complete_emulated_rdmsr()
2012 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)  in complete_fast_msr_access()  argument
2014 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);  in complete_fast_msr_access()
2017 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)  in complete_fast_rdmsr()  argument
2019 	complete_userspace_rdmsr(vcpu);  in complete_fast_rdmsr()
2020 	return complete_fast_msr_access(vcpu);  in complete_fast_rdmsr()
2035 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,  in kvm_msr_user_space()  argument
2037 			      int (*completion)(struct kvm_vcpu *vcpu),  in kvm_msr_user_space()  argument
2043 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))  in kvm_msr_user_space()
2046 	vcpu->run->exit_reason = exit_reason;  in kvm_msr_user_space()
2047 	vcpu->run->msr.error = 0;  in kvm_msr_user_space()
2048 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));  in kvm_msr_user_space()
2049 	vcpu->run->msr.reason = msr_reason;  in kvm_msr_user_space()
2050 	vcpu->run->msr.index = index;  in kvm_msr_user_space()
2051 	vcpu->run->msr.data = data;  in kvm_msr_user_space()
2052 	vcpu->arch.complete_userspace_io = completion;  in kvm_msr_user_space()
2057 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_rdmsr()  argument
2059 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdmsr()
2063 	r = kvm_get_msr_with_filter(vcpu, ecx, &data);  in kvm_emulate_rdmsr()
2068 		kvm_rax_write(vcpu, data & -1u);  in kvm_emulate_rdmsr()
2069 		kvm_rdx_write(vcpu, (data >> 32) & -1u);  in kvm_emulate_rdmsr()
2072 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,  in kvm_emulate_rdmsr()
2078 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);  in kvm_emulate_rdmsr()
2082 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_wrmsr()  argument
2084 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_wrmsr()
2085 	u64 data = kvm_read_edx_eax(vcpu);  in kvm_emulate_wrmsr()
2088 	r = kvm_set_msr_with_filter(vcpu, ecx, data);  in kvm_emulate_wrmsr()
2094 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,  in kvm_emulate_wrmsr()
2103 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);  in kvm_emulate_wrmsr()
2107 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)  in kvm_emulate_as_nop()  argument
2109 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_as_nop()
2112 int kvm_emulate_invd(struct kvm_vcpu *vcpu)  in kvm_emulate_invd()  argument
2115 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_invd()
2119 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)  in kvm_handle_invalid_op()  argument
2121 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_handle_invalid_op()
2127 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)  in kvm_emulate_monitor_mwait()  argument
2129 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&  in kvm_emulate_monitor_mwait()
2130 	    !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))  in kvm_emulate_monitor_mwait()
2131 		return kvm_handle_invalid_op(vcpu);  in kvm_emulate_monitor_mwait()
2134 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_monitor_mwait()
2136 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)  in kvm_emulate_mwait()  argument
2138 	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");  in kvm_emulate_mwait()
2142 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)  in kvm_emulate_monitor()  argument
2144 	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");  in kvm_emulate_monitor()
2148 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)  in kvm_vcpu_exit_request()  argument
2151 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||  in kvm_vcpu_exit_request()
2162 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_x2apic_icr_irqoff()  argument
2164 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))  in handle_fastpath_set_x2apic_icr_irqoff()
2171 		return kvm_x2apic_icr_write(vcpu->arch.apic, data);  in handle_fastpath_set_x2apic_icr_irqoff()
2176 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_tscdeadline()  argument
2178 	if (!kvm_can_use_hv_timer(vcpu))  in handle_fastpath_set_tscdeadline()
2181 	kvm_set_lapic_tscdeadline_msr(vcpu, data);  in handle_fastpath_set_tscdeadline()
2185 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)  in handle_fastpath_set_msr_irqoff()  argument
2187 	u32 msr = kvm_rcx_read(vcpu);  in handle_fastpath_set_msr_irqoff()
2191 	kvm_vcpu_srcu_read_lock(vcpu);  in handle_fastpath_set_msr_irqoff()
2195 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2196 		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
2197 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
2202 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2203 		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
2204 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
2215 	kvm_vcpu_srcu_read_unlock(vcpu);  in handle_fastpath_set_msr_irqoff()
2224 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr()  argument
2226 	return kvm_get_msr_ignored_check(vcpu, index, data, true);  in do_get_msr()
2229 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_set_msr()  argument
2235 	 * not support modifying the guest vCPU model on the fly, e.g. changing  in do_set_msr()
2240 	if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index)) {  in do_set_msr()
2241 		if (do_get_msr(vcpu, index, &val) || *data != val)  in do_set_msr()
2247 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);  in do_set_msr()
2362 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,  in kvm_write_system_time()  argument
2365 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_write_system_time()
2367 	if (vcpu->vcpu_id == 0 && !host_initiated) {  in kvm_write_system_time()
2369 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2374 	vcpu->arch.time = system_time;  in kvm_write_system_time()
2375 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2379 		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,  in kvm_write_system_time()
2382 		kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvm_write_system_time()
2435 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2437 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)  in set_tsc_khz()  argument
2443 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in set_tsc_khz()
2450 			vcpu->arch.tsc_catchup = 1;  in set_tsc_khz()
2451 			vcpu->arch.tsc_always_catchup = 1;  in set_tsc_khz()
2469 	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);  in set_tsc_khz()
2473 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)  in kvm_set_tsc_khz()  argument
2481 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in kvm_set_tsc_khz()
2487 			   &vcpu->arch.virtual_tsc_shift,  in kvm_set_tsc_khz()
2488 			   &vcpu->arch.virtual_tsc_mult);  in kvm_set_tsc_khz()
2489 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;  in kvm_set_tsc_khz()
2504 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);  in kvm_set_tsc_khz()
2507 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)  in compute_guest_tsc()  argument
2509 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,  in compute_guest_tsc()
2510 				      vcpu->arch.virtual_tsc_mult,  in compute_guest_tsc()
2511 				      vcpu->arch.virtual_tsc_shift);  in compute_guest_tsc()
2512 	tsc += vcpu->arch.this_tsc_write;  in compute_guest_tsc()
2523 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)  in kvm_track_tsc_matching()  argument
2527 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_track_tsc_matching()
2531 			 atomic_read(&vcpu->kvm->online_vcpus));  in kvm_track_tsc_matching()
2543 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_track_tsc_matching()
2545 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,  in kvm_track_tsc_matching()
2546 			    atomic_read(&vcpu->kvm->online_vcpus),  in kvm_track_tsc_matching()
2576 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)  in kvm_compute_l1_tsc_offset()  argument
2580 	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);  in kvm_compute_l1_tsc_offset()
2585 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)  in kvm_read_l1_tsc()  argument
2587 	return vcpu->arch.l1_tsc_offset +  in kvm_read_l1_tsc()
2588 		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);  in kvm_read_l1_tsc()
2617 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)  in kvm_vcpu_write_tsc_offset()  argument
2619 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,  in kvm_vcpu_write_tsc_offset()
2620 				   vcpu->arch.l1_tsc_offset,  in kvm_vcpu_write_tsc_offset()
2623 	vcpu->arch.l1_tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2630 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_offset()
2631 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(  in kvm_vcpu_write_tsc_offset()
2633 			static_call(kvm_x86_get_l2_tsc_offset)(vcpu),  in kvm_vcpu_write_tsc_offset()
2634 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_offset()
2636 		vcpu->arch.tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2638 	static_call(kvm_x86_write_tsc_offset)(vcpu);  in kvm_vcpu_write_tsc_offset()
2641 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)  in kvm_vcpu_write_tsc_multiplier()  argument
2643 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2646 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_multiplier()
2647 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(  in kvm_vcpu_write_tsc_multiplier()
2649 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_multiplier()
2651 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2654 		static_call(kvm_x86_write_tsc_multiplier)(vcpu);  in kvm_vcpu_write_tsc_multiplier()
2672  * offset for the vcpu and tracks the TSC matching generation that the vcpu
2675 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,  in __kvm_synchronize_tsc()  argument
2678 	struct kvm *kvm = vcpu->kvm;  in __kvm_synchronize_tsc()
2688 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;  in __kvm_synchronize_tsc()
2691 	vcpu->arch.last_guest_tsc = tsc;  in __kvm_synchronize_tsc()
2693 	kvm_vcpu_write_tsc_offset(vcpu, offset);  in __kvm_synchronize_tsc()
2710 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {  in __kvm_synchronize_tsc()
2714 	/* Keep track of which generation this VCPU has synchronized to */  in __kvm_synchronize_tsc()
2715 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;  in __kvm_synchronize_tsc()
2716 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;  in __kvm_synchronize_tsc()
2717 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;  in __kvm_synchronize_tsc()
2719 	kvm_track_tsc_matching(vcpu);  in __kvm_synchronize_tsc()
2722 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)  in kvm_synchronize_tsc()  argument
2724 	struct kvm *kvm = vcpu->kvm;  in kvm_synchronize_tsc()
2731 	offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2735 	if (vcpu->arch.virtual_tsc_khz) {  in kvm_synchronize_tsc()
2738 			 * detection of vcpu initialization -- need to sync  in kvm_synchronize_tsc()
2745 						nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2746 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;  in kvm_synchronize_tsc()
2764 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {  in kvm_synchronize_tsc()
2768 			u64 delta = nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2770 			offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2775 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched);  in kvm_synchronize_tsc()
2779 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,  in adjust_tsc_offset_guest()  argument
2782 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;  in adjust_tsc_offset_guest()
2783 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);  in adjust_tsc_offset_guest()
2786 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)  in adjust_tsc_offset_host()  argument
2788 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)  in adjust_tsc_offset_host()
2791 				   vcpu->arch.l1_tsc_scaling_ratio);  in adjust_tsc_offset_host()
2792 	adjust_tsc_offset_guest(vcpu, adjustment);  in adjust_tsc_offset_host()
3010 	struct kvm_vcpu *vcpu;  in kvm_end_pvclock_update()  local
3015 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3016 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_end_pvclock_update()
3019 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3020 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);  in kvm_end_pvclock_update()
3106 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_setup_guest_pvclock()  local
3123 	 * This VCPU is paused, but it's legal for a guest to read another  in kvm_setup_guest_pvclock()
3124 	 * VCPU's kvmclock, so we really have to follow the specification where  in kvm_setup_guest_pvclock()
3129 	guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;  in kvm_setup_guest_pvclock()
3133 	vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);  in kvm_setup_guest_pvclock()
3135 	if (vcpu->pvclock_set_guest_stopped_request) {  in kvm_setup_guest_pvclock()
3136 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;  in kvm_setup_guest_pvclock()
3137 		vcpu->pvclock_set_guest_stopped_request = false;  in kvm_setup_guest_pvclock()
3140 	memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));  in kvm_setup_guest_pvclock()
3143 	guest_hv_clock->version = ++vcpu->hv_clock.version;  in kvm_setup_guest_pvclock()
3148 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);  in kvm_setup_guest_pvclock()
3155 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_guest_time_update()  local
3197 	 *   2) Broken TSC compensation resets the base at each VCPU  in kvm_guest_time_update()
3203 	if (vcpu->tsc_catchup) {  in kvm_guest_time_update()
3219 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {  in kvm_guest_time_update()
3221 				   &vcpu->hv_clock.tsc_shift,  in kvm_guest_time_update()
3222 				   &vcpu->hv_clock.tsc_to_system_mul);  in kvm_guest_time_update()
3223 		vcpu->hw_tsc_khz = tgt_tsc_khz;  in kvm_guest_time_update()
3227 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;  in kvm_guest_time_update()
3228 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;  in kvm_guest_time_update()
3229 	vcpu->last_guest_tsc = tsc_timestamp;  in kvm_guest_time_update()
3236 	vcpu->hv_clock.flags = pvclock_flags;  in kvm_guest_time_update()
3238 	if (vcpu->pv_time.active)  in kvm_guest_time_update()
3239 		kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0);  in kvm_guest_time_update()
3240 	if (vcpu->xen.vcpu_info_cache.active)  in kvm_guest_time_update()
3241 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,  in kvm_guest_time_update()
3243 	if (vcpu->xen.vcpu_time_info_cache.active)  in kvm_guest_time_update()
3244 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0);  in kvm_guest_time_update()
3245 	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);  in kvm_guest_time_update()
3250  * kvmclock updates which are isolated to a given vcpu, such as
3251  * vcpu->cpu migration, should not allow system_timestamp from
3253  * correction applies to one vcpu's system_timestamp but not
3259  * The time for a remote vcpu to update its kvmclock is bound
3272 	struct kvm_vcpu *vcpu;  in kvmclock_update_fn()  local
3274 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmclock_update_fn()
3275 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvmclock_update_fn()
3276 		kvm_vcpu_kick(vcpu);  in kvmclock_update_fn()
3319 static bool can_set_mci_status(struct kvm_vcpu *vcpu)  in can_set_mci_status()  argument
3322 	if (guest_cpuid_is_amd_compatible(vcpu))  in can_set_mci_status()
3323 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));  in can_set_mci_status()
3328 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_msr_mce()  argument
3330 	u64 mcg_cap = vcpu->arch.mcg_cap;  in set_msr_mce()
3338 		vcpu->arch.mcg_status = data;  in set_msr_mce()
3346 		vcpu->arch.mcg_ctl = data;  in set_msr_mce()
3360 		vcpu->arch.mci_ctl2_banks[offset] = data;  in set_msr_mce()
3387 		    data != 0 && !can_set_mci_status(vcpu))  in set_msr_mce()
3392 		vcpu->arch.mce_banks[offset] = data;  in set_msr_mce()
3400 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)  in kvm_pv_async_pf_enabled()  argument
3404 	return (vcpu->arch.apf.msr_en_val & mask) == mask;  in kvm_pv_async_pf_enabled()
3407 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf()  argument
3415 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&  in kvm_pv_enable_async_pf()
3419 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&  in kvm_pv_enable_async_pf()
3423 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf()
3426 	vcpu->arch.apf.msr_en_val = data;  in kvm_pv_enable_async_pf()
3428 	if (!kvm_pv_async_pf_enabled(vcpu)) {  in kvm_pv_enable_async_pf()
3429 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_pv_enable_async_pf()
3430 		kvm_async_pf_hash_reset(vcpu);  in kvm_pv_enable_async_pf()
3434 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,  in kvm_pv_enable_async_pf()
3438 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);  in kvm_pv_enable_async_pf()
3439 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;  in kvm_pv_enable_async_pf()
3441 	kvm_async_pf_wakeup_all(vcpu);  in kvm_pv_enable_async_pf()
3446 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf_int()  argument
3452 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf_int()
3455 	vcpu->arch.apf.msr_int_val = data;  in kvm_pv_enable_async_pf_int()
3457 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;  in kvm_pv_enable_async_pf_int()
3462 static void kvmclock_reset(struct kvm_vcpu *vcpu)  in kvmclock_reset()  argument
3464 	kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvmclock_reset()
3465 	vcpu->arch.time = 0;  in kvmclock_reset()
3468 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_all()  argument
3470 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_all()
3471 	static_call(kvm_x86_flush_tlb_all)(vcpu);  in kvm_vcpu_flush_tlb_all()
3474 	kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_vcpu_flush_tlb_all()
3477 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_guest()  argument
3479 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_guest()
3488 		kvm_mmu_sync_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3489 		kvm_mmu_sync_prev_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3492 	static_call(kvm_x86_flush_tlb_guest)(vcpu);  in kvm_vcpu_flush_tlb_guest()
3498 	kvm_hv_vcpu_purge_flush_tlb(vcpu);  in kvm_vcpu_flush_tlb_guest()
3502 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_current()  argument
3504 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_current()
3505 	static_call(kvm_x86_flush_tlb_current)(vcpu);  in kvm_vcpu_flush_tlb_current()
3514 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)  in kvm_service_local_tlb_flush_requests()  argument
3516 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))  in kvm_service_local_tlb_flush_requests()
3517 		kvm_vcpu_flush_tlb_current(vcpu);  in kvm_service_local_tlb_flush_requests()
3519 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))  in kvm_service_local_tlb_flush_requests()
3520 		kvm_vcpu_flush_tlb_guest(vcpu);  in kvm_service_local_tlb_flush_requests()
3524 static void record_steal_time(struct kvm_vcpu *vcpu)  in record_steal_time()  argument
3526 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in record_steal_time()
3529 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in record_steal_time()
3533 	if (kvm_xen_msr_enabled(vcpu->kvm)) {  in record_steal_time()
3534 		kvm_xen_runstate_set_running(vcpu);  in record_steal_time()
3538 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in record_steal_time()
3541 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))  in record_steal_time()
3544 	slots = kvm_memslots(vcpu->kvm);  in record_steal_time()
3552 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||  in record_steal_time()
3562 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {  in record_steal_time()
3581 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3583 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,  in record_steal_time()
3586 			kvm_vcpu_flush_tlb_guest(vcpu);  in record_steal_time()
3595 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3609 		vcpu->arch.st.last_steal;  in record_steal_time()
3610 	vcpu->arch.st.last_steal = current->sched_info.run_delay;  in record_steal_time()
3619 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in record_steal_time()
3634 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_msr_common()  argument
3639 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)  in kvm_set_msr_common()
3640 		return kvm_xen_write_hypercall_page(vcpu, data);  in kvm_set_msr_common()
3655 			vcpu->arch.microcode_version = data;  in kvm_set_msr_common()
3660 		vcpu->arch.arch_capabilities = data;  in kvm_set_msr_common()
3670 		 * disallows changing feature MSRs after the vCPU has run; PMU  in kvm_set_msr_common()
3671 		 * refresh will bug the VM if called after the vCPU has run.  in kvm_set_msr_common()
3673 		if (vcpu->arch.perf_capabilities == data)  in kvm_set_msr_common()
3676 		vcpu->arch.perf_capabilities = data;  in kvm_set_msr_common()
3677 		kvm_pmu_refresh(vcpu);  in kvm_set_msr_common()
3680 		if (!msr_info->host_initiated && !guest_has_pred_cmd_msr(vcpu))  in kvm_set_msr_common()
3692 		    !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D))  in kvm_set_msr_common()
3703 		return set_efer(vcpu, msr_info);  in kvm_set_msr_common()
3711 			vcpu->arch.msr_hwcr = data;  in kvm_set_msr_common()
3713 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3719 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3727 		vcpu->arch.pat = data;  in kvm_set_msr_common()
3731 		return kvm_mtrr_set_msr(vcpu, msr, data);  in kvm_set_msr_common()
3733 		return kvm_set_apic_base(vcpu, msr_info);  in kvm_set_msr_common()
3735 		return kvm_x2apic_msr_write(vcpu, msr, data);  in kvm_set_msr_common()
3737 		kvm_set_lapic_tscdeadline_msr(vcpu, data);  in kvm_set_msr_common()
3740 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {  in kvm_set_msr_common()
3742 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;  in kvm_set_msr_common()
3743 				adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3747 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_msr_common()
3749 			vcpu->arch.ia32_tsc_adjust_msr = data;  in kvm_set_msr_common()
3753 		u64 old_val = vcpu->arch.ia32_misc_enable_msr;  in kvm_set_msr_common()
3765 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&  in kvm_set_msr_common()
3767 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))  in kvm_set_msr_common()
3769 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3770 			kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3772 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3779 		vcpu->arch.smbase = data;  in kvm_set_msr_common()
3782 		vcpu->arch.msr_ia32_power_ctl = data;  in kvm_set_msr_common()
3786 			kvm_synchronize_tsc(vcpu, data);  in kvm_set_msr_common()
3788 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;  in kvm_set_msr_common()
3789 			adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3790 			vcpu->arch.ia32_tsc_adjust_msr += adj;  in kvm_set_msr_common()
3795 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_set_msr_common()
3804 		vcpu->arch.ia32_xss = data;  in kvm_set_msr_common()
3805 		kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3810 		vcpu->arch.smi_count = data;  in kvm_set_msr_common()
3813 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3816 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3817 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3820 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3823 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3824 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3827 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3830 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);  in kvm_set_msr_common()
3833 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3836 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);  in kvm_set_msr_common()
3839 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_set_msr_common()
3842 		if (kvm_pv_enable_async_pf(vcpu, data))  in kvm_set_msr_common()
3846 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3849 		if (kvm_pv_enable_async_pf_int(vcpu, data))  in kvm_set_msr_common()
3853 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3856 			vcpu->arch.apf.pageready_pending = false;  in kvm_set_msr_common()
3857 			kvm_check_async_pf_completion(vcpu);  in kvm_set_msr_common()
3861 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_set_msr_common()
3870 		vcpu->arch.st.msr_val = data;  in kvm_set_msr_common()
3875 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_set_msr_common()
3879 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_set_msr_common()
3882 		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))  in kvm_set_msr_common()
3887 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_set_msr_common()
3894 		vcpu->arch.msr_kvm_poll_control = data;  in kvm_set_msr_common()
3901 		return set_msr_mce(vcpu, msr_info);  in kvm_set_msr_common()
3907 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
3908 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
3911 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3933 		return kvm_hv_set_msr_common(vcpu, msr, data,  in kvm_set_msr_common()
3939 		kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3942 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
3944 		vcpu->arch.osvw.length = data;  in kvm_set_msr_common()
3947 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
3949 		vcpu->arch.osvw.status = data;  in kvm_set_msr_common()
3954 		     cpuid_fault_enabled(vcpu)))  in kvm_set_msr_common()
3956 		vcpu->arch.msr_platform_info = data;  in kvm_set_msr_common()
3961 		     !supports_cpuid_fault(vcpu)))  in kvm_set_msr_common()
3963 		vcpu->arch.msr_misc_features_enables = data;  in kvm_set_msr_common()
3968 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
3971 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
3974 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);  in kvm_set_msr_common()
3978 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
3981 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
3984 		vcpu->arch.guest_fpu.xfd_err = data;  in kvm_set_msr_common()
3988 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
3989 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
4005 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)  in get_msr_mce()  argument
4008 	u64 mcg_cap = vcpu->arch.mcg_cap;  in get_msr_mce()
4018 		data = vcpu->arch.mcg_cap;  in get_msr_mce()
4023 		data = vcpu->arch.mcg_ctl;  in get_msr_mce()
4026 		data = vcpu->arch.mcg_status;  in get_msr_mce()
4037 		data = vcpu->arch.mci_ctl2_banks[offset];  in get_msr_mce()
4046 		data = vcpu->arch.mce_banks[offset];  in get_msr_mce()
4055 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_get_msr_common()  argument
4093 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4094 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4098 		msr_info->data = vcpu->arch.microcode_version;  in kvm_get_msr_common()
4102 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))  in kvm_get_msr_common()
4104 		msr_info->data = vcpu->arch.arch_capabilities;  in kvm_get_msr_common()
4108 		    !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))  in kvm_get_msr_common()
4110 		msr_info->data = vcpu->arch.perf_capabilities;  in kvm_get_msr_common()
4113 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;  in kvm_get_msr_common()
4128 			offset = vcpu->arch.l1_tsc_offset;  in kvm_get_msr_common()
4129 			ratio = vcpu->arch.l1_tsc_scaling_ratio;  in kvm_get_msr_common()
4131 			offset = vcpu->arch.tsc_offset;  in kvm_get_msr_common()
4132 			ratio = vcpu->arch.tsc_scaling_ratio;  in kvm_get_msr_common()
4139 		msr_info->data = vcpu->arch.pat;  in kvm_get_msr_common()
4144 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4163 		msr_info->data = kvm_get_apic_base(vcpu);  in kvm_get_msr_common()
4166 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4168 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);  in kvm_get_msr_common()
4171 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;  in kvm_get_msr_common()
4174 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;  in kvm_get_msr_common()
4179 		msr_info->data = vcpu->arch.smbase;  in kvm_get_msr_common()
4182 		msr_info->data = vcpu->arch.smi_count;  in kvm_get_msr_common()
4191 		msr_info->data = vcpu->arch.efer;  in kvm_get_msr_common()
4194 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4197 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4200 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4203 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4206 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4209 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4212 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4215 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4218 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_get_msr_common()
4221 		msr_info->data = vcpu->arch.apf.msr_en_val;  in kvm_get_msr_common()
4224 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4227 		msr_info->data = vcpu->arch.apf.msr_int_val;  in kvm_get_msr_common()
4230 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4236 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_get_msr_common()
4239 		msr_info->data = vcpu->arch.st.msr_val;  in kvm_get_msr_common()
4242 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_get_msr_common()
4245 		msr_info->data = vcpu->arch.pv_eoi.msr_val;  in kvm_get_msr_common()
4248 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_get_msr_common()
4251 		msr_info->data = vcpu->arch.msr_kvm_poll_control;  in kvm_get_msr_common()
4260 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,  in kvm_get_msr_common()
4264 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_get_msr_common()
4266 		msr_info->data = vcpu->arch.ia32_xss;  in kvm_get_msr_common()
4290 		return kvm_hv_get_msr_common(vcpu,  in kvm_get_msr_common()
4307 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4309 		msr_info->data = vcpu->arch.osvw.length;  in kvm_get_msr_common()
4312 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4314 		msr_info->data = vcpu->arch.osvw.status;  in kvm_get_msr_common()
4318 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)  in kvm_get_msr_common()
4320 		msr_info->data = vcpu->arch.msr_platform_info;  in kvm_get_msr_common()
4323 		msr_info->data = vcpu->arch.msr_misc_features_enables;  in kvm_get_msr_common()
4326 		msr_info->data = vcpu->arch.msr_hwcr;  in kvm_get_msr_common()
4331 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4334 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;  in kvm_get_msr_common()
4338 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4341 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;  in kvm_get_msr_common()
4345 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4346 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4369 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,  in __msr_io()  argument
4371 		    int (*do_msr)(struct kvm_vcpu *vcpu,  in __msr_io()  argument
4377 		if (do_msr(vcpu, entries[i].index, &entries[i].data))  in __msr_io()
4388 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,  in msr_io()  argument
4389 		  int (*do_msr)(struct kvm_vcpu *vcpu,  in msr_io()  argument
4413 	r = __msr_io(vcpu, &msrs, entries, do_msr);  in msr_io()
4430 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,  in kvm_ioctl_get_supported_hv_cpuid()  argument
4440 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_ioctl_get_supported_hv_cpuid()
4806 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)  in need_emulate_wbinvd()  argument
4808 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);  in need_emulate_wbinvd()
4811 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_vcpu_load()  argument
4814 	if (need_emulate_wbinvd(vcpu)) {  in kvm_arch_vcpu_load()
4816 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_load()
4817 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4818 			smp_call_function_single(vcpu->cpu,  in kvm_arch_vcpu_load()
4822 	static_call(kvm_x86_vcpu_load)(vcpu, cpu);  in kvm_arch_vcpu_load()
4825 	vcpu->arch.host_pkru = read_pkru();  in kvm_arch_vcpu_load()
4828 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {  in kvm_arch_vcpu_load()
4829 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);  in kvm_arch_vcpu_load()
4830 		vcpu->arch.tsc_offset_adjustment = 0;  in kvm_arch_vcpu_load()
4831 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4834 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {  in kvm_arch_vcpu_load()
4835 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :  in kvm_arch_vcpu_load()
4836 				rdtsc() - vcpu->arch.last_host_tsc;  in kvm_arch_vcpu_load()
4841 			u64 offset = kvm_compute_l1_tsc_offset(vcpu,  in kvm_arch_vcpu_load()
4842 						vcpu->arch.last_guest_tsc);  in kvm_arch_vcpu_load()
4843 			kvm_vcpu_write_tsc_offset(vcpu, offset);  in kvm_arch_vcpu_load()
4844 			vcpu->arch.tsc_catchup = 1;  in kvm_arch_vcpu_load()
4847 		if (kvm_lapic_hv_timer_in_use(vcpu))  in kvm_arch_vcpu_load()
4848 			kvm_lapic_restart_hv_timer(vcpu);  in kvm_arch_vcpu_load()
4852 		 * kvmclock on vcpu->cpu migration  in kvm_arch_vcpu_load()
4854 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)  in kvm_arch_vcpu_load()
4855 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4856 		if (vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4857 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);  in kvm_arch_vcpu_load()
4858 		vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
4861 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4864 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)  in kvm_steal_time_set_preempted()  argument
4866 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in kvm_steal_time_set_preempted()
4870 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in kvm_steal_time_set_preempted()
4873 	 * The vCPU can be marked preempted if and only if the VM-Exit was on  in kvm_steal_time_set_preempted()
4876 	 * when this is true, for example allowing the vCPU to be marked  in kvm_steal_time_set_preempted()
4879 	if (!vcpu->arch.at_instruction_boundary) {  in kvm_steal_time_set_preempted()
4880 		vcpu->stat.preemption_other++;  in kvm_steal_time_set_preempted()
4884 	vcpu->stat.preemption_reported++;  in kvm_steal_time_set_preempted()
4885 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in kvm_steal_time_set_preempted()
4888 	if (vcpu->arch.st.preempted)  in kvm_steal_time_set_preempted()
4892 	if (unlikely(current->mm != vcpu->kvm->mm))  in kvm_steal_time_set_preempted()
4895 	slots = kvm_memslots(vcpu->kvm);  in kvm_steal_time_set_preempted()
4906 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;  in kvm_steal_time_set_preempted()
4908 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in kvm_steal_time_set_preempted()
4911 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_put()  argument
4915 	if (vcpu->preempted) {  in kvm_arch_vcpu_put()
4916 		if (!vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_put()
4917 			vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);  in kvm_arch_vcpu_put()
4923 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_put()
4924 		if (kvm_xen_msr_enabled(vcpu->kvm))  in kvm_arch_vcpu_put()
4925 			kvm_xen_runstate_set_preempted(vcpu);  in kvm_arch_vcpu_put()
4927 			kvm_steal_time_set_preempted(vcpu);  in kvm_arch_vcpu_put()
4928 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_put()
4931 	static_call(kvm_x86_vcpu_put)(vcpu);  in kvm_arch_vcpu_put()
4932 	vcpu->arch.last_host_tsc = rdtsc();  in kvm_arch_vcpu_put()
4935 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_get_lapic()  argument
4938 	static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in kvm_vcpu_ioctl_get_lapic()
4940 	return kvm_apic_get_state(vcpu, s);  in kvm_vcpu_ioctl_get_lapic()
4943 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_set_lapic()  argument
4948 	r = kvm_apic_set_state(vcpu, s);  in kvm_vcpu_ioctl_set_lapic()
4951 	update_cr8_intercept(vcpu);  in kvm_vcpu_ioctl_set_lapic()
4956 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)  in kvm_cpu_accept_dm_intr()  argument
4964 	if (kvm_cpu_has_extint(vcpu))  in kvm_cpu_accept_dm_intr()
4968 	return (!lapic_in_kernel(vcpu) ||  in kvm_cpu_accept_dm_intr()
4969 		kvm_apic_accept_pic_intr(vcpu));  in kvm_cpu_accept_dm_intr()
4972 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)  in kvm_vcpu_ready_for_interrupt_injection()  argument
4981 	return (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4982 		kvm_cpu_accept_dm_intr(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4983 		!kvm_event_needs_reinjection(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4984 		!kvm_is_exception_pending(vcpu));  in kvm_vcpu_ready_for_interrupt_injection()
4987 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_interrupt()  argument
4993 	if (!irqchip_in_kernel(vcpu->kvm)) {  in kvm_vcpu_ioctl_interrupt()
4994 		kvm_queue_interrupt(vcpu, irq->irq, false);  in kvm_vcpu_ioctl_interrupt()
4995 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5003 	if (pic_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_interrupt()
5006 	if (vcpu->arch.pending_external_vector != -1)  in kvm_vcpu_ioctl_interrupt()
5009 	vcpu->arch.pending_external_vector = irq->irq;  in kvm_vcpu_ioctl_interrupt()
5010 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5014 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)  in kvm_vcpu_ioctl_nmi()  argument
5016 	kvm_inject_nmi(vcpu);  in kvm_vcpu_ioctl_nmi()
5021 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,  in vcpu_ioctl_tpr_access_reporting()  argument
5026 	vcpu->arch.tpr_access_reporting = !!tac->enabled;  in vcpu_ioctl_tpr_access_reporting()
5030 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_setup_mce()  argument
5042 	vcpu->arch.mcg_cap = mcg_cap;  in kvm_vcpu_ioctl_x86_setup_mce()
5045 		vcpu->arch.mcg_ctl = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5048 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5050 			vcpu->arch.mci_ctl2_banks[bank] = 0;  in kvm_vcpu_ioctl_x86_setup_mce()
5053 	kvm_apic_after_set_mcg_cap(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5055 	static_call(kvm_x86_setup_mce)(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5077 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)  in kvm_vcpu_x86_set_ucna()  argument
5079 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_x86_set_ucna()
5084 	vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_x86_set_ucna()
5087 	    !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))  in kvm_vcpu_x86_set_ucna()
5090 	if (lapic_in_kernel(vcpu))  in kvm_vcpu_x86_set_ucna()
5091 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);  in kvm_vcpu_x86_set_ucna()
5096 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_mce()  argument
5099 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_ioctl_x86_set_mce()
5101 	u64 *banks = vcpu->arch.mce_banks;  in kvm_vcpu_ioctl_x86_set_mce()
5109 		return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);  in kvm_vcpu_ioctl_x86_set_mce()
5116 	    vcpu->arch.mcg_ctl != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
5125 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||  in kvm_vcpu_ioctl_x86_set_mce()
5126 		    !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) {  in kvm_vcpu_ioctl_x86_set_mce()
5127 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_mce()
5134 		vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_ioctl_x86_set_mce()
5136 		kvm_queue_exception(vcpu, MC_VECTOR);  in kvm_vcpu_ioctl_x86_set_mce()
5149 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_vcpu_events()  argument
5154 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5157 	if (kvm_check_request(KVM_REQ_SMI, vcpu))  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5158 		process_smi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5168 	if (vcpu->arch.exception_vmexit.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5169 	    !vcpu->arch.exception.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5170 	    !vcpu->arch.exception.injected)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5171 		ex = &vcpu->arch.exception_vmexit;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5173 		ex = &vcpu->arch.exception;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5183 	if (!vcpu->kvm->arch.exception_payload_enabled &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5185 		kvm_deliver_exception_payload(vcpu, ex);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5203 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5213 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5214 	events->interrupt.nr = vcpu->arch.interrupt.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5215 	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5217 	events->nmi.injected = vcpu->arch.nmi_injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5218 	events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5219 	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5224 	events->smi.smm = is_smm(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5225 	events->smi.pending = vcpu->arch.smi_pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5227 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5229 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5234 	if (vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5236 	if (vcpu->kvm->arch.triple_fault_event) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5237 		events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5242 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()  argument
5254 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5272 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5275 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5285 	vcpu->arch.exception_from_userspace = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5287 	vcpu->arch.exception_vmexit.pending = false;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5289 	vcpu->arch.exception.injected = events->exception.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5290 	vcpu->arch.exception.pending = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5291 	vcpu->arch.exception.vector = events->exception.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5292 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5293 	vcpu->arch.exception.error_code = events->exception.error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5294 	vcpu->arch.exception.has_payload = events->exception_has_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5295 	vcpu->arch.exception.payload = events->exception_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5297 	vcpu->arch.interrupt.injected = events->interrupt.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5298 	vcpu->arch.interrupt.nr = events->interrupt.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5299 	vcpu->arch.interrupt.soft = events->interrupt.soft;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5301 		static_call(kvm_x86_set_interrupt_shadow)(vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5304 	vcpu->arch.nmi_injected = events->nmi.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5306 		vcpu->arch.nmi_pending = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5307 		atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5309 			kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5311 	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5314 	    lapic_in_kernel(vcpu))  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5315 		vcpu->arch.apic->sipi_vector = events->sipi_vector;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5319 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5320 			kvm_leave_nested(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5321 			kvm_smm_changed(vcpu, events->smi.smm);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5324 		vcpu->arch.smi_pending = events->smi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5328 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5330 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5339 		if (lapic_in_kernel(vcpu)) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5341 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5343 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5348 		if (!vcpu->kvm->arch.triple_fault_event)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5351 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5353 			kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5356 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5361 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_debugregs()  argument
5367 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_get_debugregs()
5368 	kvm_get_dr(vcpu, 6, &val);  in kvm_vcpu_ioctl_x86_get_debugregs()
5370 	dbgregs->dr7 = vcpu->arch.dr7;  in kvm_vcpu_ioctl_x86_get_debugregs()
5373 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_debugregs()  argument
5384 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_set_debugregs()
5385 	kvm_update_dr0123(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5386 	vcpu->arch.dr6 = dbgregs->dr6;  in kvm_vcpu_ioctl_x86_set_debugregs()
5387 	vcpu->arch.dr7 = dbgregs->dr7;  in kvm_vcpu_ioctl_x86_set_debugregs()
5388 	kvm_update_dr7(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5394 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave2()  argument
5409 	u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |  in kvm_vcpu_ioctl_x86_get_xsave2()
5412 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_get_xsave2()
5415 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,  in kvm_vcpu_ioctl_x86_get_xsave2()
5416 				       supported_xcr0, vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_get_xsave2()
5419 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave()  argument
5422 	return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,  in kvm_vcpu_ioctl_x86_get_xsave()
5426 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xsave()  argument
5429 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_set_xsave()
5432 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,  in kvm_vcpu_ioctl_x86_set_xsave()
5435 					      &vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_set_xsave()
5438 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xcrs()  argument
5449 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5452 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xcrs()  argument
5466 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,  in kvm_vcpu_ioctl_x86_set_xcrs()
5481 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)  in kvm_set_guest_paused()  argument
5483 	if (!vcpu->arch.pv_time.active)  in kvm_set_guest_paused()
5485 	vcpu->arch.pvclock_set_guest_stopped_request = true;  in kvm_set_guest_paused()
5486 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_guest_paused()
5490 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_has_attr()  argument
5506 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_get_attr()  argument
5518 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))  in kvm_arch_tsc_get_attr()
5529 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_set_attr()  argument
5533 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_tsc_set_attr()
5551 		matched = (vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5552 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5555 		tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;  in kvm_arch_tsc_set_attr()
5558 		__kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);  in kvm_arch_tsc_set_attr()
5571 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_device_attr()  argument
5586 		r = kvm_arch_tsc_has_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5589 		r = kvm_arch_tsc_get_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5592 		r = kvm_arch_tsc_set_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5599 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_enable_cap()  argument
5616 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_enable_cap()
5618 		return kvm_hv_activate_synic(vcpu, cap->cap ==  in kvm_vcpu_ioctl_enable_cap()
5623 		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);  in kvm_vcpu_ioctl_enable_cap()
5635 		return static_call(kvm_x86_enable_l2_tlb_flush)(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5638 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);  in kvm_vcpu_ioctl_enable_cap()
5641 		vcpu->arch.pv_cpuid.enforce = cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
5642 		if (vcpu->arch.pv_cpuid.enforce)  in kvm_vcpu_ioctl_enable_cap()
5643 			kvm_update_pv_runtime(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5654 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()  local
5665 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl()
5671 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5679 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5690 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5698 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5707 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);  in kvm_arch_vcpu_ioctl()
5711 		r = kvm_vcpu_ioctl_nmi(vcpu);  in kvm_arch_vcpu_ioctl()
5715 		r = kvm_inject_smi(vcpu);  in kvm_arch_vcpu_ioctl()
5725 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5735 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5746 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5757 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5758 		r = msr_io(vcpu, argp, do_get_msr, 1);  in kvm_arch_vcpu_ioctl()
5759 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5763 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5764 		r = msr_io(vcpu, argp, do_set_msr, 0);  in kvm_arch_vcpu_ioctl()
5765 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5774 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);  in kvm_arch_vcpu_ioctl()
5788 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5793 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5794 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);  in kvm_arch_vcpu_ioctl()
5795 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5804 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);  in kvm_arch_vcpu_ioctl()
5813 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);  in kvm_arch_vcpu_ioctl()
5819 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
5834 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl()
5835 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
5836 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl()
5842 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
5859 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
5864 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))  in kvm_arch_vcpu_ioctl()
5872 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
5881 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
5889 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
5894 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
5901 		kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);  in kvm_arch_vcpu_ioctl()
5917 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
5933 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
5949 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))  in kvm_arch_vcpu_ioctl()
5955 		r = vcpu->arch.virtual_tsc_khz;  in kvm_arch_vcpu_ioctl()
5959 		r = kvm_set_guest_paused(vcpu);  in kvm_arch_vcpu_ioctl()
5968 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);  in kvm_arch_vcpu_ioctl()
5984 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,  in kvm_arch_vcpu_ioctl()
6028 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
6029 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);  in kvm_arch_vcpu_ioctl()
6030 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
6034 		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);  in kvm_arch_vcpu_ioctl()
6043 		r = kvm_xen_vcpu_get_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6054 		r = kvm_xen_vcpu_set_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6063 		__get_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6077 		r = __set_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6083 		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);  in kvm_arch_vcpu_ioctl()
6091 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl()
6095 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)  in kvm_arch_vcpu_fault()  argument
6269 	struct kvm_vcpu *vcpu;  in kvm_arch_sync_dirty_log()  local
6272 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_sync_dirty_log()
6273 		kvm_vcpu_kick(vcpu);  in kvm_arch_sync_dirty_log()
6687 	struct kvm_vcpu *vcpu;  in kvm_arch_suspend_notifier()  local
6692 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_suspend_notifier()
6693 		if (!vcpu->arch.pv_time.active)  in kvm_arch_suspend_notifier()
6696 		ret = kvm_set_guest_paused(vcpu);  in kvm_arch_suspend_notifier()
6698 			kvm_err("Failed to pause guest VCPU%d: %d\n",  in kvm_arch_suspend_notifier()
6699 				vcpu->vcpu_id, ret);  in kvm_arch_suspend_notifier()
7249 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,  in vcpu_mmio_write()  argument
7257 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_write()
7258 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))  in vcpu_mmio_write()
7259 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_write()
7270 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)  in vcpu_mmio_read()  argument
7277 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_read()
7278 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,  in vcpu_mmio_read()
7280 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_read()
7292 void kvm_set_segment(struct kvm_vcpu *vcpu,  in kvm_set_segment()  argument
7295 	static_call(kvm_x86_set_segment)(vcpu, var, seg);  in kvm_set_segment()
7298 void kvm_get_segment(struct kvm_vcpu *vcpu,  in kvm_get_segment()  argument
7301 	static_call(kvm_x86_get_segment)(vcpu, var, seg);  in kvm_get_segment()
7304 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,  in translate_nested_gpa()  argument
7307 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in translate_nested_gpa()
7310 	BUG_ON(!mmu_is_nested(vcpu));  in translate_nested_gpa()
7314 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);  in translate_nested_gpa()
7319 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_read()  argument
7322 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_read()
7324 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_read()
7325 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_read()
7329 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_write()  argument
7332 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_write()
7334 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_write()
7336 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_write()
7341 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_system()  argument
7344 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_system()
7346 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);  in kvm_mmu_gva_to_gpa_system()
7350 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_read_guest_virt_helper()  argument
7353 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_read_guest_virt_helper()
7358 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_read_guest_virt_helper()
7365 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,  in kvm_read_guest_virt_helper()
7385 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_fetch_guest_virt()  local
7386 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fetch_guest_virt()
7387 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_fetch_guest_virt()
7392 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,  in kvm_fetch_guest_virt()
7400 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,  in kvm_fetch_guest_virt()
7408 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,  in kvm_read_guest_virt()  argument
7412 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_read_guest_virt()
7421 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,  in kvm_read_guest_virt()
7430 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_std()  local
7435 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)  in emulator_read_std()
7438 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);  in emulator_read_std()
7442 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_write_guest_virt_helper()  argument
7445 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_write_guest_virt_helper()
7450 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_write_guest_virt_helper()
7457 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);  in kvm_write_guest_virt_helper()
7475 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_write_std()  local
7480 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)  in emulator_write_std()
7483 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in emulator_write_std()
7487 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,  in kvm_write_guest_virt_system()  argument
7491 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_write_guest_virt_system()
7493 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in kvm_write_guest_virt_system()
7498 static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,  in kvm_can_emulate_insn()  argument
7501 	return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,  in kvm_can_emulate_insn()
7505 int handle_ud(struct kvm_vcpu *vcpu)  in handle_ud()  argument
7513 	if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))  in handle_ud()
7517 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),  in handle_ud()
7521 			kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);  in handle_ud()
7522 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));  in handle_ud()
7526 	return kvm_emulate_instruction(vcpu, emul_type);  in handle_ud()
7530 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_is_mmio_gpa()  argument
7537 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {  in vcpu_is_mmio_gpa()
7545 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_mmio_gva_to_gpa()  argument
7549 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in vcpu_mmio_gva_to_gpa()
7550 	u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)  in vcpu_mmio_gva_to_gpa()
7558 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||  in vcpu_mmio_gva_to_gpa()
7559 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,  in vcpu_mmio_gva_to_gpa()
7560 			      vcpu->arch.mmio_access, 0, access))) {  in vcpu_mmio_gva_to_gpa()
7561 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |  in vcpu_mmio_gva_to_gpa()
7567 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in vcpu_mmio_gva_to_gpa()
7572 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);  in vcpu_mmio_gva_to_gpa()
7575 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,  in emulator_write_phys()  argument
7580 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);  in emulator_write_phys()
7583 	kvm_page_track_write(vcpu, gpa, val, bytes);  in emulator_write_phys()
7588 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7590 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7592 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7594 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7599 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)  in read_prepare()  argument
7601 	if (vcpu->mmio_read_completed) {  in read_prepare()
7603 			       vcpu->mmio_fragments[0].gpa, val);  in read_prepare()
7604 		vcpu->mmio_read_completed = 0;  in read_prepare()
7611 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_emulate()  argument
7614 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);  in read_emulate()
7617 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_emulate()  argument
7620 	return emulator_write_phys(vcpu, gpa, val, bytes);  in write_emulate()
7623 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)  in write_mmio()  argument
7626 	return vcpu_mmio_write(vcpu, gpa, bytes, val);  in write_mmio()
7629 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_exit_mmio()  argument
7636 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_exit_mmio()  argument
7639 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];  in write_exit_mmio()
7641 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in write_exit_mmio()
7662 				       struct kvm_vcpu *vcpu,  in emulator_read_write_onepage()  argument
7669 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()
7681 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);  in emulator_read_write_onepage()
7683 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);  in emulator_read_write_onepage()
7688 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))  in emulator_read_write_onepage()
7694 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);  in emulator_read_write_onepage()
7702 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);  in emulator_read_write_onepage()
7703 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];  in emulator_read_write_onepage()
7716 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_write()  local
7721 		  ops->read_write_prepare(vcpu, val, bytes))  in emulator_read_write()
7724 	vcpu->mmio_nr_fragments = 0;  in emulator_read_write()
7732 						 vcpu, ops);  in emulator_read_write()
7744 					 vcpu, ops);  in emulator_read_write()
7748 	if (!vcpu->mmio_nr_fragments)  in emulator_read_write()
7751 	gpa = vcpu->mmio_fragments[0].gpa;  in emulator_read_write()
7753 	vcpu->mmio_needed = 1;  in emulator_read_write()
7754 	vcpu->mmio_cur_fragment = 0;  in emulator_read_write()
7756 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);  in emulator_read_write()
7757 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;  in emulator_read_write()
7758 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in emulator_read_write()
7759 	vcpu->run->mmio.phys_addr = gpa;  in emulator_read_write()
7761 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);  in emulator_read_write()
7794 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_cmpxchg_emulated()  local
7804 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);  in emulator_cmpxchg_emulated()
7822 	hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
7855 	kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
7860 	kvm_page_track_write(vcpu, gpa, new, bytes);  in emulator_cmpxchg_emulated()
7870 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in_out()  argument
7877 	WARN_ON_ONCE(vcpu->arch.pio.count);  in emulator_pio_in_out()
7880 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
7882 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
7902 	vcpu->arch.pio.port = port;  in emulator_pio_in_out()
7903 	vcpu->arch.pio.in = in;  in emulator_pio_in_out()
7904 	vcpu->arch.pio.count = count;  in emulator_pio_in_out()
7905 	vcpu->arch.pio.size = size;  in emulator_pio_in_out()
7908 		memset(vcpu->arch.pio_data, 0, size * count);  in emulator_pio_in_out()
7910 		memcpy(vcpu->arch.pio_data, data, size * count);  in emulator_pio_in_out()
7912 	vcpu->run->exit_reason = KVM_EXIT_IO;  in emulator_pio_in_out()
7913 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;  in emulator_pio_in_out()
7914 	vcpu->run->io.size = size;  in emulator_pio_in_out()
7915 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;  in emulator_pio_in_out()
7916 	vcpu->run->io.count = count;  in emulator_pio_in_out()
7917 	vcpu->run->io.port = port;  in emulator_pio_in_out()
7921 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in()  argument
7924 	int r = emulator_pio_in_out(vcpu, size, port, val, count, true);  in emulator_pio_in()
7931 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)  in complete_emulator_pio_in()  argument
7933 	int size = vcpu->arch.pio.size;  in complete_emulator_pio_in()
7934 	unsigned int count = vcpu->arch.pio.count;  in complete_emulator_pio_in()
7935 	memcpy(val, vcpu->arch.pio_data, size * count);  in complete_emulator_pio_in()
7936 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);  in complete_emulator_pio_in()
7937 	vcpu->arch.pio.count = 0;  in complete_emulator_pio_in()
7944 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_pio_in_emulated()  local
7945 	if (vcpu->arch.pio.count) {  in emulator_pio_in_emulated()
7949 		 * can modify ECX before rerunning the vCPU.  Ignore any such  in emulator_pio_in_emulated()
7953 		complete_emulator_pio_in(vcpu, val);  in emulator_pio_in_emulated()
7957 	return emulator_pio_in(vcpu, size, port, val, count);  in emulator_pio_in_emulated()
7960 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_out()  argument
7965 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);  in emulator_pio_out()
7975 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)  in get_segment_base()  argument
7977 	return static_call(kvm_x86_get_segment_base)(vcpu, seg);  in get_segment_base()
7985 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd_noskip()  argument
7987 	if (!need_emulate_wbinvd(vcpu))  in kvm_emulate_wbinvd_noskip()
7993 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
7994 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,  in kvm_emulate_wbinvd_noskip()
7997 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
8003 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd()  argument
8005 	kvm_emulate_wbinvd_noskip(vcpu);  in kvm_emulate_wbinvd()
8006 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_wbinvd()
8037 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_cr()  local
8042 		value = kvm_read_cr0(vcpu);  in emulator_get_cr()
8045 		value = vcpu->arch.cr2;  in emulator_get_cr()
8048 		value = kvm_read_cr3(vcpu);  in emulator_get_cr()
8051 		value = kvm_read_cr4(vcpu);  in emulator_get_cr()
8054 		value = kvm_get_cr8(vcpu);  in emulator_get_cr()
8066 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_cr()  local
8071 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));  in emulator_set_cr()
8074 		vcpu->arch.cr2 = val;  in emulator_set_cr()
8077 		res = kvm_set_cr3(vcpu, val);  in emulator_set_cr()
8080 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));  in emulator_set_cr()
8083 		res = kvm_set_cr8(vcpu, val);  in emulator_set_cr()
8164 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_segment()  local
8186 	kvm_set_segment(vcpu, &var, seg);  in emulator_set_segment()
8193 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_msr_with_filter()  local
8196 	r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);  in emulator_get_msr_with_filter()
8201 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,  in emulator_get_msr_with_filter()
8216 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_msr_with_filter()  local
8219 	r = kvm_set_msr_with_filter(vcpu, msr_index, data);  in emulator_set_msr_with_filter()
8224 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,  in emulator_set_msr_with_filter()
8388 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)  in toggle_interruptibility()  argument
8390 	u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in toggle_interruptibility()
8401 		static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);  in toggle_interruptibility()
8403 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in toggle_interruptibility()
8407 static void inject_emulated_exception(struct kvm_vcpu *vcpu)  in inject_emulated_exception()  argument
8409 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()
8412 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
8414 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
8417 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
8420 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)  in alloc_emulate_ctxt()  argument
8426 		pr_err("failed to allocate vcpu's emulator\n");  in alloc_emulate_ctxt()
8430 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
8432 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
8437 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)  in init_emulate_ctxt()  argument
8439 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()
8442 	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in init_emulate_ctxt()
8445 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
8448 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
8449 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
8451 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :  in init_emulate_ctxt()
8460 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in init_emulate_ctxt()
8463 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)  in kvm_inject_realmode_interrupt()  argument
8465 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()
8468 	init_emulate_ctxt(vcpu);  in kvm_inject_realmode_interrupt()
8476 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_inject_realmode_interrupt()
8479 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
8480 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
8485 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in prepare_emulation_failure_exit()  argument
8488 	struct kvm_run *run = vcpu->run;  in prepare_emulation_failure_exit()
8498 	static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],  in prepare_emulation_failure_exit()
8536 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)  in prepare_emulation_ctxt_failure_exit()  argument
8538 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in prepare_emulation_ctxt_failure_exit()
8540 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,  in prepare_emulation_ctxt_failure_exit()
8544 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in __kvm_prepare_emulation_failure_exit()  argument
8547 	prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);  in __kvm_prepare_emulation_failure_exit()
8551 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)  in kvm_prepare_emulation_failure_exit()  argument
8553 	__kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);  in kvm_prepare_emulation_failure_exit()
8557 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)  in handle_emulation_failure()  argument
8559 	struct kvm *kvm = vcpu->kvm;  in handle_emulation_failure()
8561 	++vcpu->stat.insn_emulation_fail;  in handle_emulation_failure()
8562 	trace_kvm_emulate_insn_failed(vcpu);  in handle_emulation_failure()
8565 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in handle_emulation_failure()
8571 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8575 	kvm_queue_exception(vcpu, UD_VECTOR);  in handle_emulation_failure()
8577 	if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {  in handle_emulation_failure()
8578 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8585 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in reexecute_instruction()  argument
8594 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in reexecute_instruction()
8598 	if (!vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8603 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in reexecute_instruction()
8619 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8631 	if (vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8634 		write_lock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8635 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;  in reexecute_instruction()
8636 		write_unlock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8639 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8649 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8662 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in retry_instruction()  local
8665 	last_retry_eip = vcpu->arch.last_retry_eip;  in retry_instruction()
8666 	last_retry_addr = vcpu->arch.last_retry_addr;  in retry_instruction()
8681 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;  in retry_instruction()
8686 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in retry_instruction()
8696 	vcpu->arch.last_retry_eip = ctxt->eip;  in retry_instruction()
8697 	vcpu->arch.last_retry_addr = cr2_or_gpa;  in retry_instruction()
8699 	if (!vcpu->arch.mmu->root_role.direct)  in retry_instruction()
8700 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in retry_instruction()
8702 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in retry_instruction()
8707 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8708 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8725 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)  in kvm_vcpu_do_singlestep()  argument
8727 	struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_do_singlestep()
8729 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {  in kvm_vcpu_do_singlestep()
8731 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);  in kvm_vcpu_do_singlestep()
8736 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);  in kvm_vcpu_do_singlestep()
8740 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)  in kvm_skip_emulated_instruction()  argument
8742 	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);  in kvm_skip_emulated_instruction()
8745 	r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);  in kvm_skip_emulated_instruction()
8749 	kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);  in kvm_skip_emulated_instruction()
8760 		r = kvm_vcpu_do_singlestep(vcpu);  in kvm_skip_emulated_instruction()
8765 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)  in kvm_is_code_breakpoint_inhibited()  argument
8769 	if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)  in kvm_is_code_breakpoint_inhibited()
8777 	shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_is_code_breakpoint_inhibited()
8779 	       guest_cpuid_is_intel(vcpu);  in kvm_is_code_breakpoint_inhibited()
8782 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,  in kvm_vcpu_check_code_breakpoint()  argument
8805 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&  in kvm_vcpu_check_code_breakpoint()
8806 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {  in kvm_vcpu_check_code_breakpoint()
8807 		struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_check_code_breakpoint()
8808 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
8810 					   vcpu->arch.guest_debug_dr7,  in kvm_vcpu_check_code_breakpoint()
8811 					   vcpu->arch.eff_db);  in kvm_vcpu_check_code_breakpoint()
8823 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&  in kvm_vcpu_check_code_breakpoint()
8824 	    !kvm_is_code_breakpoint_inhibited(vcpu)) {  in kvm_vcpu_check_code_breakpoint()
8825 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
8827 					   vcpu->arch.dr7,  in kvm_vcpu_check_code_breakpoint()
8828 					   vcpu->arch.db);  in kvm_vcpu_check_code_breakpoint()
8831 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);  in kvm_vcpu_check_code_breakpoint()
8881 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,  in x86_decode_emulated_instruction()  argument
8884 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_decode_emulated_instruction()
8887 	init_emulate_ctxt(vcpu);  in x86_decode_emulated_instruction()
8891 	trace_kvm_emulate_insn_start(vcpu);  in x86_decode_emulated_instruction()
8892 	++vcpu->stat.insn_emulation;  in x86_decode_emulated_instruction()
8898 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in x86_emulate_instruction()  argument
8902 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()
8905 	if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))  in x86_emulate_instruction()
8908 	vcpu->arch.l1tf_flush_l1d = true;  in x86_emulate_instruction()
8911 		kvm_clear_exception_queue(vcpu);  in x86_emulate_instruction()
8918 		if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))  in x86_emulate_instruction()
8921 		r = x86_decode_emulated_instruction(vcpu, emulation_type,  in x86_emulate_instruction()
8926 				kvm_queue_exception(vcpu, UD_VECTOR);  in x86_emulate_instruction()
8929 			if (reexecute_instruction(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
8941 				inject_emulated_exception(vcpu);  in x86_emulate_instruction()
8944 			return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
8950 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in x86_emulate_instruction()
8971 		kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
8973 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
8982 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {  in x86_emulate_instruction()
8983 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in x86_emulate_instruction()
8993 		if (vcpu->arch.mmu->root_role.direct) {  in x86_emulate_instruction()
9008 		if (reexecute_instruction(vcpu, cr2_or_gpa, emulation_type))  in x86_emulate_instruction()
9011 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9015 		WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);  in x86_emulate_instruction()
9016 		vcpu->mmio_needed = false;  in x86_emulate_instruction()
9018 		inject_emulated_exception(vcpu);  in x86_emulate_instruction()
9019 	} else if (vcpu->arch.pio.count) {  in x86_emulate_instruction()
9020 		if (!vcpu->arch.pio.in) {  in x86_emulate_instruction()
9022 			vcpu->arch.pio.count = 0;  in x86_emulate_instruction()
9025 			vcpu->arch.complete_userspace_io = complete_emulated_pio;  in x86_emulate_instruction()
9028 	} else if (vcpu->mmio_needed) {  in x86_emulate_instruction()
9029 		++vcpu->stat.mmio_exits;  in x86_emulate_instruction()
9031 		if (!vcpu->mmio_is_write)  in x86_emulate_instruction()
9034 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in x86_emulate_instruction()
9035 	} else if (vcpu->arch.complete_userspace_io) {  in x86_emulate_instruction()
9045 		unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);  in x86_emulate_instruction()
9046 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
9047 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in x86_emulate_instruction()
9056 			kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);  in x86_emulate_instruction()
9058 				kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);  in x86_emulate_instruction()
9059 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9060 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
9061 				r = kvm_vcpu_do_singlestep(vcpu);  in x86_emulate_instruction()
9062 			static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);  in x86_emulate_instruction()
9063 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
9073 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in x86_emulate_instruction()
9075 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;  in x86_emulate_instruction()
9080 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)  in kvm_emulate_instruction()  argument
9082 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);  in kvm_emulate_instruction()
9086 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,  in kvm_emulate_instruction_from_buffer()  argument
9089 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);  in kvm_emulate_instruction_from_buffer()
9093 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)  in complete_fast_pio_out_port_0x7e()  argument
9095 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out_port_0x7e()
9099 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)  in complete_fast_pio_out()  argument
9101 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out()
9103 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))  in complete_fast_pio_out()
9106 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_out()
9109 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_out()  argument
9112 	unsigned long val = kvm_rax_read(vcpu);  in kvm_fast_pio_out()
9113 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);  in kvm_fast_pio_out()
9123 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {  in kvm_fast_pio_out()
9124 		vcpu->arch.complete_userspace_io =  in kvm_fast_pio_out()
9126 		kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio_out()
9128 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_out()
9129 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;  in kvm_fast_pio_out()
9134 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)  in complete_fast_pio_in()  argument
9139 	BUG_ON(vcpu->arch.pio.count != 1);  in complete_fast_pio_in()
9141 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {  in complete_fast_pio_in()
9142 		vcpu->arch.pio.count = 0;  in complete_fast_pio_in()
9147 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;  in complete_fast_pio_in()
9149 	complete_emulator_pio_in(vcpu, &val);  in complete_fast_pio_in()
9150 	kvm_rax_write(vcpu, val);  in complete_fast_pio_in()
9152 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_in()
9155 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_in()  argument
9162 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;  in kvm_fast_pio_in()
9164 	ret = emulator_pio_in(vcpu, size, port, &val, 1);  in kvm_fast_pio_in()
9166 		kvm_rax_write(vcpu, val);  in kvm_fast_pio_in()
9170 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_in()
9171 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;  in kvm_fast_pio_in()
9176 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)  in kvm_fast_pio()  argument
9181 		ret = kvm_fast_pio_in(vcpu, size, port);  in kvm_fast_pio()
9183 		ret = kvm_fast_pio_out(vcpu, size, port);  in kvm_fast_pio()
9184 	return ret && kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio()
9243 	struct kvm_vcpu *vcpu;  in __kvmclock_cpufreq_notifier()  local
9258 	 * the TSC for each VCPU.  We must flag these local variables  in __kvmclock_cpufreq_notifier()
9277 	 * anytime after the setting of the VCPU's request bit, the  in __kvmclock_cpufreq_notifier()
9290 		kvm_for_each_vcpu(i, vcpu, kvm) {  in __kvmclock_cpufreq_notifier()
9291 			if (vcpu->cpu != cpu)  in __kvmclock_cpufreq_notifier()
9293 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in __kvmclock_cpufreq_notifier()
9294 			if (vcpu->cpu != raw_smp_processor_id())  in __kvmclock_cpufreq_notifier()
9374 	struct kvm_vcpu *vcpu;  in pvclock_gtod_update_fn()  local
9379 		kvm_for_each_vcpu(i, vcpu, kvm)  in pvclock_gtod_update_fn()
9380 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in pvclock_gtod_update_fn()
9482 	 * vCPU's FPU state as a fxregs_state struct.  in __kvm_x86_vendor_init()
9650 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)  in __kvm_emulate_halt()  argument
9653 	 * The vCPU has halted, e.g. executed HLT.  Update the run state if the  in __kvm_emulate_halt()
9655 	 * state and halt the vCPU.  Exit to userspace if the local APIC is  in __kvm_emulate_halt()
9659 	++vcpu->stat.halt_exits;  in __kvm_emulate_halt()
9660 	if (lapic_in_kernel(vcpu)) {  in __kvm_emulate_halt()
9661 		vcpu->arch.mp_state = state;  in __kvm_emulate_halt()
9664 		vcpu->run->exit_reason = reason;  in __kvm_emulate_halt()
9669 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_halt_noskip()  argument
9671 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);  in kvm_emulate_halt_noskip()
9675 int kvm_emulate_halt(struct kvm_vcpu *vcpu)  in kvm_emulate_halt()  argument
9677 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_halt()
9682 	return kvm_emulate_halt_noskip(vcpu) && ret;  in kvm_emulate_halt()
9686 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)  in kvm_emulate_ap_reset_hold()  argument
9688 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_ap_reset_hold()
9690 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,  in kvm_emulate_ap_reset_hold()
9696 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,  in kvm_pv_clock_pairing()  argument
9711 	if (vcpu->arch.tsc_always_catchup)  in kvm_pv_clock_pairing()
9719 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);  in kvm_pv_clock_pairing()
9724 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,  in kvm_pv_clock_pairing()
9733  * kvm_pv_kick_cpu_op:  Kick a vcpu.
9735  * @apicid - apicid of vcpu to be kicked.
9759 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)  in kvm_vcpu_apicv_activated()  argument
9761 	ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);  in kvm_vcpu_apicv_activated()
9762 	ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);  in kvm_vcpu_apicv_activated()
9792 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)  in kvm_sched_yield()  argument
9797 	vcpu->stat.directed_yield_attempted++;  in kvm_sched_yield()
9803 	map = rcu_dereference(vcpu->kvm->arch.apic_map);  in kvm_sched_yield()
9806 		target = map->phys_map[dest_id]->vcpu;  in kvm_sched_yield()
9814 	if (vcpu == target)  in kvm_sched_yield()
9820 	vcpu->stat.directed_yield_successful++;  in kvm_sched_yield()
9826 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)  in complete_hypercall_exit()  argument
9828 	u64 ret = vcpu->run->hypercall.ret;  in complete_hypercall_exit()
9830 	if (!is_64_bit_hypercall(vcpu))  in complete_hypercall_exit()
9832 	kvm_rax_write(vcpu, ret);  in complete_hypercall_exit()
9833 	++vcpu->stat.hypercalls;  in complete_hypercall_exit()
9834 	return kvm_skip_emulated_instruction(vcpu);  in complete_hypercall_exit()
9837 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)  in kvm_emulate_hypercall()  argument
9842 	if (kvm_xen_hypercall_enabled(vcpu->kvm))  in kvm_emulate_hypercall()
9843 		return kvm_xen_hypercall(vcpu);  in kvm_emulate_hypercall()
9845 	if (kvm_hv_hypercall_enabled(vcpu))  in kvm_emulate_hypercall()
9846 		return kvm_hv_hypercall(vcpu);  in kvm_emulate_hypercall()
9848 	nr = kvm_rax_read(vcpu);  in kvm_emulate_hypercall()
9849 	a0 = kvm_rbx_read(vcpu);  in kvm_emulate_hypercall()
9850 	a1 = kvm_rcx_read(vcpu);  in kvm_emulate_hypercall()
9851 	a2 = kvm_rdx_read(vcpu);  in kvm_emulate_hypercall()
9852 	a3 = kvm_rsi_read(vcpu);  in kvm_emulate_hypercall()
9856 	op_64_bit = is_64_bit_hypercall(vcpu);  in kvm_emulate_hypercall()
9865 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {  in kvm_emulate_hypercall()
9877 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))  in kvm_emulate_hypercall()
9880 		kvm_pv_kick_cpu_op(vcpu->kvm, a1);  in kvm_emulate_hypercall()
9881 		kvm_sched_yield(vcpu, a1);  in kvm_emulate_hypercall()
9886 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);  in kvm_emulate_hypercall()
9890 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))  in kvm_emulate_hypercall()
9893 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);  in kvm_emulate_hypercall()
9896 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))  in kvm_emulate_hypercall()
9899 		kvm_sched_yield(vcpu, a0);  in kvm_emulate_hypercall()
9906 		if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))  in kvm_emulate_hypercall()
9915 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;  in kvm_emulate_hypercall()
9916 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;  in kvm_emulate_hypercall()
9917 		vcpu->run->hypercall.args[0]  = gpa;  in kvm_emulate_hypercall()
9918 		vcpu->run->hypercall.args[1]  = npages;  in kvm_emulate_hypercall()
9919 		vcpu->run->hypercall.args[2]  = attrs;  in kvm_emulate_hypercall()
9920 		vcpu->run->hypercall.flags    = 0;  in kvm_emulate_hypercall()
9922 			vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;  in kvm_emulate_hypercall()
9924 		WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);  in kvm_emulate_hypercall()
9925 		vcpu->arch.complete_userspace_io = complete_hypercall_exit;  in kvm_emulate_hypercall()
9935 	kvm_rax_write(vcpu, ret);  in kvm_emulate_hypercall()
9937 	++vcpu->stat.hypercalls;  in kvm_emulate_hypercall()
9938 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_hypercall()
9944 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_fix_hypercall()  local
9946 	unsigned long rip = kvm_rip_read(vcpu);  in emulator_fix_hypercall()
9952 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {  in emulator_fix_hypercall()
9959 	static_call(kvm_x86_patch_hypercall)(vcpu, instruction);  in emulator_fix_hypercall()
9965 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)  in dm_request_for_irq_injection()  argument
9967 	return vcpu->run->request_interrupt_window &&  in dm_request_for_irq_injection()
9968 		likely(!pic_in_kernel(vcpu->kvm));  in dm_request_for_irq_injection()
9972 static void post_kvm_run_save(struct kvm_vcpu *vcpu)  in post_kvm_run_save()  argument
9974 	struct kvm_run *kvm_run = vcpu->run;  in post_kvm_run_save()
9976 	kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);  in post_kvm_run_save()
9977 	kvm_run->cr8 = kvm_get_cr8(vcpu);  in post_kvm_run_save()
9978 	kvm_run->apic_base = kvm_get_apic_base(vcpu);  in post_kvm_run_save()
9981 		pic_in_kernel(vcpu->kvm) ||  in post_kvm_run_save()
9982 		kvm_vcpu_ready_for_interrupt_injection(vcpu);  in post_kvm_run_save()
9984 	if (is_smm(vcpu))  in post_kvm_run_save()
9988 static void update_cr8_intercept(struct kvm_vcpu *vcpu)  in update_cr8_intercept()  argument
9995 	if (!lapic_in_kernel(vcpu))  in update_cr8_intercept()
9998 	if (vcpu->arch.apic->apicv_active)  in update_cr8_intercept()
10001 	if (!vcpu->arch.apic->vapic_addr)  in update_cr8_intercept()
10002 		max_irr = kvm_lapic_find_highest_irr(vcpu);  in update_cr8_intercept()
10009 	tpr = kvm_lapic_get_cr8(vcpu);  in update_cr8_intercept()
10011 	static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);  in update_cr8_intercept()
10015 int kvm_check_nested_events(struct kvm_vcpu *vcpu)  in kvm_check_nested_events()  argument
10017 	if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in kvm_check_nested_events()
10018 		kvm_x86_ops.nested_ops->triple_fault(vcpu);  in kvm_check_nested_events()
10022 	return kvm_x86_ops.nested_ops->check_events(vcpu);  in kvm_check_nested_events()
10025 static void kvm_inject_exception(struct kvm_vcpu *vcpu)  in kvm_inject_exception()  argument
10028 	 * Suppress the error code if the vCPU is in Real Mode, as Real Mode  in kvm_inject_exception()
10034 	vcpu->arch.exception.has_error_code &= is_protmode(vcpu);  in kvm_inject_exception()
10036 	trace_kvm_inj_exception(vcpu->arch.exception.vector,  in kvm_inject_exception()
10037 				vcpu->arch.exception.has_error_code,  in kvm_inject_exception()
10038 				vcpu->arch.exception.error_code,  in kvm_inject_exception()
10039 				vcpu->arch.exception.injected);  in kvm_inject_exception()
10041 	static_call(kvm_x86_inject_exception)(vcpu);  in kvm_inject_exception()
10083 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,  in kvm_check_and_inject_events()  argument
10094 	if (is_guest_mode(vcpu))  in kvm_check_and_inject_events()
10095 		r = kvm_check_nested_events(vcpu);  in kvm_check_and_inject_events()
10122 	if (vcpu->arch.exception.injected)  in kvm_check_and_inject_events()
10123 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10124 	else if (kvm_is_exception_pending(vcpu))  in kvm_check_and_inject_events()
10126 	else if (vcpu->arch.nmi_injected)  in kvm_check_and_inject_events()
10127 		static_call(kvm_x86_inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10128 	else if (vcpu->arch.interrupt.injected)  in kvm_check_and_inject_events()
10129 		static_call(kvm_x86_inject_irq)(vcpu, true);  in kvm_check_and_inject_events()
10136 	WARN_ON_ONCE(vcpu->arch.exception.injected &&  in kvm_check_and_inject_events()
10137 		     vcpu->arch.exception.pending);  in kvm_check_and_inject_events()
10156 	WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||  in kvm_check_and_inject_events()
10157 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10164 	can_inject = !kvm_event_needs_reinjection(vcpu);  in kvm_check_and_inject_events()
10166 	if (vcpu->arch.exception.pending) {  in kvm_check_and_inject_events()
10177 		if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)  in kvm_check_and_inject_events()
10178 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |  in kvm_check_and_inject_events()
10181 		if (vcpu->arch.exception.vector == DB_VECTOR) {  in kvm_check_and_inject_events()
10182 			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);  in kvm_check_and_inject_events()
10183 			if (vcpu->arch.dr7 & DR7_GD) {  in kvm_check_and_inject_events()
10184 				vcpu->arch.dr7 &= ~DR7_GD;  in kvm_check_and_inject_events()
10185 				kvm_update_dr7(vcpu);  in kvm_check_and_inject_events()
10189 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10191 		vcpu->arch.exception.pending = false;  in kvm_check_and_inject_events()
10192 		vcpu->arch.exception.injected = true;  in kvm_check_and_inject_events()
10198 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)  in kvm_check_and_inject_events()
10213 	if (vcpu->arch.smi_pending) {  in kvm_check_and_inject_events()
10214 		r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10218 			vcpu->arch.smi_pending = false;  in kvm_check_and_inject_events()
10219 			++vcpu->arch.smi_count;  in kvm_check_and_inject_events()
10220 			enter_smm(vcpu);  in kvm_check_and_inject_events()
10223 			static_call(kvm_x86_enable_smi_window)(vcpu);  in kvm_check_and_inject_events()
10227 	if (vcpu->arch.nmi_pending) {  in kvm_check_and_inject_events()
10228 		r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10232 			--vcpu->arch.nmi_pending;  in kvm_check_and_inject_events()
10233 			vcpu->arch.nmi_injected = true;  in kvm_check_and_inject_events()
10234 			static_call(kvm_x86_inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10236 			WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10238 		if (vcpu->arch.nmi_pending)  in kvm_check_and_inject_events()
10239 			static_call(kvm_x86_enable_nmi_window)(vcpu);  in kvm_check_and_inject_events()
10242 	if (kvm_cpu_has_injectable_intr(vcpu)) {  in kvm_check_and_inject_events()
10243 		r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10247 			int irq = kvm_cpu_get_interrupt(vcpu);  in kvm_check_and_inject_events()
10250 				kvm_queue_interrupt(vcpu, irq, false);  in kvm_check_and_inject_events()
10251 				static_call(kvm_x86_inject_irq)(vcpu, false);  in kvm_check_and_inject_events()
10252 				WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10255 		if (kvm_cpu_has_injectable_intr(vcpu))  in kvm_check_and_inject_events()
10256 			static_call(kvm_x86_enable_irq_window)(vcpu);  in kvm_check_and_inject_events()
10259 	if (is_guest_mode(vcpu) &&  in kvm_check_and_inject_events()
10261 	    kvm_x86_ops.nested_ops->has_events(vcpu, true))  in kvm_check_and_inject_events()
10267 	 * to the VMCS/VMCB.  Queueing a new exception can put the vCPU into an  in kvm_check_and_inject_events()
10272 	 * vCPU into an infinite loop.  Triple fault can be queued when running  in kvm_check_and_inject_events()
10276 	WARN_ON_ONCE(vcpu->arch.exception.pending ||  in kvm_check_and_inject_events()
10277 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10288 static void process_nmi(struct kvm_vcpu *vcpu)  in process_nmi()  argument
10294 	 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is  in process_nmi()
10302 	if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)  in process_nmi()
10309 	 * tracked in vcpu->arch.nmi_pending.  in process_nmi()
10311 	if (static_call(kvm_x86_is_vnmi_pending)(vcpu))  in process_nmi()
10314 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);  in process_nmi()
10315 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);  in process_nmi()
10317 	if (vcpu->arch.nmi_pending &&  in process_nmi()
10318 	    (static_call(kvm_x86_set_vnmi_pending)(vcpu)))  in process_nmi()
10319 		vcpu->arch.nmi_pending--;  in process_nmi()
10321 	if (vcpu->arch.nmi_pending)  in process_nmi()
10322 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in process_nmi()
10326 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu)  in kvm_get_nr_pending_nmis()  argument
10328 	return vcpu->arch.nmi_pending +  in kvm_get_nr_pending_nmis()
10329 	       static_call(kvm_x86_is_vnmi_pending)(vcpu);  in kvm_get_nr_pending_nmis()
10343 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in __kvm_vcpu_update_apicv()  argument
10345 	struct kvm_lapic *apic = vcpu->arch.apic;  in __kvm_vcpu_update_apicv()
10348 	if (!lapic_in_kernel(vcpu))  in __kvm_vcpu_update_apicv()
10351 	down_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10355 	activate = kvm_vcpu_apicv_activated(vcpu) &&  in __kvm_vcpu_update_apicv()
10356 		   (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);  in __kvm_vcpu_update_apicv()
10362 	kvm_apic_update_apicv(vcpu);  in __kvm_vcpu_update_apicv()
10363 	static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);  in __kvm_vcpu_update_apicv()
10372 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __kvm_vcpu_update_apicv()
10376 	up_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10380 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in kvm_vcpu_update_apicv()  argument
10382 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_update_apicv()
10387 	 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but  in kvm_vcpu_update_apicv()
10392 	 * the vCPU would incorrectly be able to access the vAPIC page via MMIO  in kvm_vcpu_update_apicv()
10396 	if (apic_x2apic_mode(vcpu->arch.apic) &&  in kvm_vcpu_update_apicv()
10398 		kvm_inhibit_apic_access_page(vcpu);  in kvm_vcpu_update_apicv()
10400 	__kvm_vcpu_update_apicv(vcpu);  in kvm_vcpu_update_apicv()
10456 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)  in vcpu_scan_ioapic()  argument
10458 	if (!kvm_apic_present(vcpu))  in vcpu_scan_ioapic()
10461 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);  in vcpu_scan_ioapic()
10463 	static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_scan_ioapic()
10465 	if (irqchip_split(vcpu->kvm))  in vcpu_scan_ioapic()
10466 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10467 	else if (ioapic_in_kernel(vcpu->kvm))  in vcpu_scan_ioapic()
10468 		kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10470 	if (is_guest_mode(vcpu))  in vcpu_scan_ioapic()
10471 		vcpu->arch.load_eoi_exitmap_pending = true;  in vcpu_scan_ioapic()
10473 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);  in vcpu_scan_ioapic()
10476 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)  in vcpu_load_eoi_exitmap()  argument
10480 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))  in vcpu_load_eoi_exitmap()
10483 	if (to_hv_vcpu(vcpu)) {  in vcpu_load_eoi_exitmap()
10485 			  vcpu->arch.ioapic_handled_vectors,  in vcpu_load_eoi_exitmap()
10486 			  to_hv_synic(vcpu)->vec_bitmap, 256);  in vcpu_load_eoi_exitmap()
10487 		static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);  in vcpu_load_eoi_exitmap()
10492 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);  in vcpu_load_eoi_exitmap()
10500 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)  in kvm_vcpu_reload_apic_access_page()  argument
10502 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_reload_apic_access_page()
10505 	static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);  in kvm_vcpu_reload_apic_access_page()
10508 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)  in __kvm_request_immediate_exit()  argument
10510 	smp_send_reschedule(vcpu->cpu);  in __kvm_request_immediate_exit()
10520 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)  in vcpu_enter_guest()  argument
10524 		dm_request_for_irq_injection(vcpu) &&  in vcpu_enter_guest()
10525 		kvm_cpu_accept_dm_intr(vcpu);  in vcpu_enter_guest()
10530 	if (kvm_request_pending(vcpu)) {  in vcpu_enter_guest()
10531 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {  in vcpu_enter_guest()
10536 		if (kvm_dirty_ring_check_request(vcpu)) {  in vcpu_enter_guest()
10541 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {  in vcpu_enter_guest()
10542 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {  in vcpu_enter_guest()
10547 		if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))  in vcpu_enter_guest()
10548 			kvm_mmu_free_obsolete_roots(vcpu);  in vcpu_enter_guest()
10549 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))  in vcpu_enter_guest()
10550 			__kvm_migrate_timers(vcpu);  in vcpu_enter_guest()
10551 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10552 			kvm_update_masterclock(vcpu->kvm);  in vcpu_enter_guest()
10553 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10554 			kvm_gen_kvmclock_update(vcpu);  in vcpu_enter_guest()
10555 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {  in vcpu_enter_guest()
10556 			r = kvm_guest_time_update(vcpu);  in vcpu_enter_guest()
10560 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))  in vcpu_enter_guest()
10561 			kvm_mmu_sync_roots(vcpu);  in vcpu_enter_guest()
10562 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))  in vcpu_enter_guest()
10563 			kvm_mmu_load_pgd(vcpu);  in vcpu_enter_guest()
10570 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))  in vcpu_enter_guest()
10571 			kvm_vcpu_flush_tlb_all(vcpu);  in vcpu_enter_guest()
10573 		kvm_service_local_tlb_flush_requests(vcpu);  in vcpu_enter_guest()
10577 		 * flushing fails.  Note, Hyper-V's flushing is per-vCPU, but  in vcpu_enter_guest()
10581 		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) &&  in vcpu_enter_guest()
10582 		    kvm_hv_vcpu_flush_tlb(vcpu))  in vcpu_enter_guest()
10583 			kvm_vcpu_flush_tlb_guest(vcpu);  in vcpu_enter_guest()
10585 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {  in vcpu_enter_guest()
10586 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;  in vcpu_enter_guest()
10590 		if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10591 			if (is_guest_mode(vcpu))  in vcpu_enter_guest()
10592 				kvm_x86_ops.nested_ops->triple_fault(vcpu);  in vcpu_enter_guest()
10594 			if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10595 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;  in vcpu_enter_guest()
10596 				vcpu->mmio_needed = 0;  in vcpu_enter_guest()
10601 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {  in vcpu_enter_guest()
10603 			vcpu->arch.apf.halted = true;  in vcpu_enter_guest()
10607 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))  in vcpu_enter_guest()
10608 			record_steal_time(vcpu);  in vcpu_enter_guest()
10610 		if (kvm_check_request(KVM_REQ_SMI, vcpu))  in vcpu_enter_guest()
10611 			process_smi(vcpu);  in vcpu_enter_guest()
10613 		if (kvm_check_request(KVM_REQ_NMI, vcpu))  in vcpu_enter_guest()
10614 			process_nmi(vcpu);  in vcpu_enter_guest()
10615 		if (kvm_check_request(KVM_REQ_PMU, vcpu))  in vcpu_enter_guest()
10616 			kvm_pmu_handle_event(vcpu);  in vcpu_enter_guest()
10617 		if (kvm_check_request(KVM_REQ_PMI, vcpu))  in vcpu_enter_guest()
10618 			kvm_pmu_deliver_pmi(vcpu);  in vcpu_enter_guest()
10619 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {  in vcpu_enter_guest()
10620 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);  in vcpu_enter_guest()
10621 			if (test_bit(vcpu->arch.pending_ioapic_eoi,  in vcpu_enter_guest()
10622 				     vcpu->arch.ioapic_handled_vectors)) {  in vcpu_enter_guest()
10623 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;  in vcpu_enter_guest()
10624 				vcpu->run->eoi.vector =  in vcpu_enter_guest()
10625 						vcpu->arch.pending_ioapic_eoi;  in vcpu_enter_guest()
10630 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))  in vcpu_enter_guest()
10631 			vcpu_scan_ioapic(vcpu);  in vcpu_enter_guest()
10632 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))  in vcpu_enter_guest()
10633 			vcpu_load_eoi_exitmap(vcpu);  in vcpu_enter_guest()
10634 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))  in vcpu_enter_guest()
10635 			kvm_vcpu_reload_apic_access_page(vcpu);  in vcpu_enter_guest()
10636 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {  in vcpu_enter_guest()
10637 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10638 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;  in vcpu_enter_guest()
10639 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10643 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {  in vcpu_enter_guest()
10644 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10645 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;  in vcpu_enter_guest()
10646 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10650 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {  in vcpu_enter_guest()
10651 			struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);  in vcpu_enter_guest()
10653 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;  in vcpu_enter_guest()
10654 			vcpu->run->hyperv = hv_vcpu->exit;  in vcpu_enter_guest()
10664 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))  in vcpu_enter_guest()
10665 			kvm_hv_process_stimers(vcpu);  in vcpu_enter_guest()
10666 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))  in vcpu_enter_guest()
10667 			kvm_vcpu_update_apicv(vcpu);  in vcpu_enter_guest()
10668 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))  in vcpu_enter_guest()
10669 			kvm_check_async_pf_completion(vcpu);  in vcpu_enter_guest()
10670 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))  in vcpu_enter_guest()
10671 			static_call(kvm_x86_msr_filter_changed)(vcpu);  in vcpu_enter_guest()
10673 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))  in vcpu_enter_guest()
10674 			static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);  in vcpu_enter_guest()
10677 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||  in vcpu_enter_guest()
10678 	    kvm_xen_has_interrupt(vcpu)) {  in vcpu_enter_guest()
10679 		++vcpu->stat.req_event;  in vcpu_enter_guest()
10680 		r = kvm_apic_accept_events(vcpu);  in vcpu_enter_guest()
10685 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {  in vcpu_enter_guest()
10690 		r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);  in vcpu_enter_guest()
10696 			static_call(kvm_x86_enable_irq_window)(vcpu);  in vcpu_enter_guest()
10698 		if (kvm_lapic_enabled(vcpu)) {  in vcpu_enter_guest()
10699 			update_cr8_intercept(vcpu);  in vcpu_enter_guest()
10700 			kvm_lapic_sync_to_vapic(vcpu);  in vcpu_enter_guest()
10704 	r = kvm_mmu_reload(vcpu);  in vcpu_enter_guest()
10711 	static_call(kvm_x86_prepare_switch_to_guest)(vcpu);  in vcpu_enter_guest()
10720 	/* Store vcpu->apicv_active before vcpu->mode.  */  in vcpu_enter_guest()
10721 	smp_store_release(&vcpu->mode, IN_GUEST_MODE);  in vcpu_enter_guest()
10723 	kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_enter_guest()
10734 	 * tables done while the VCPU is running.  Please see the comment  in vcpu_enter_guest()
10742 	 * target vCPU wasn't running).  Do this regardless of the vCPU's APICv  in vcpu_enter_guest()
10746 	if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10747 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10749 	if (kvm_vcpu_exit_request(vcpu)) {  in vcpu_enter_guest()
10750 		vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10754 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
10760 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
10761 		static_call(kvm_x86_request_immediate_exit)(vcpu);  in vcpu_enter_guest()
10768 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10769 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);  in vcpu_enter_guest()
10771 	if (unlikely(vcpu->arch.switch_db_regs)) {  in vcpu_enter_guest()
10773 		set_debugreg(vcpu->arch.eff_db[0], 0);  in vcpu_enter_guest()
10774 		set_debugreg(vcpu->arch.eff_db[1], 1);  in vcpu_enter_guest()
10775 		set_debugreg(vcpu->arch.eff_db[2], 2);  in vcpu_enter_guest()
10776 		set_debugreg(vcpu->arch.eff_db[3], 3);  in vcpu_enter_guest()
10778 		if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))  in vcpu_enter_guest()
10779 			static_call(kvm_x86_set_dr6)(vcpu, vcpu->arch.dr6);  in vcpu_enter_guest()
10788 		 * Assert that vCPU vs. VM APICv state is consistent.  An APICv  in vcpu_enter_guest()
10793 		WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&  in vcpu_enter_guest()
10794 			     (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));  in vcpu_enter_guest()
10796 		exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu);  in vcpu_enter_guest()
10800 		if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10801 			static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10803 		if (unlikely(kvm_vcpu_exit_request(vcpu))) {  in vcpu_enter_guest()
10809 		++vcpu->stat.exits;  in vcpu_enter_guest()
10818 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {  in vcpu_enter_guest()
10819 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);  in vcpu_enter_guest()
10820 		static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);  in vcpu_enter_guest()
10821 		kvm_update_dr0123(vcpu);  in vcpu_enter_guest()
10822 		kvm_update_dr7(vcpu);  in vcpu_enter_guest()
10835 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;  in vcpu_enter_guest()
10836 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());  in vcpu_enter_guest()
10838 	vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10846 	if (vcpu->arch.xfd_no_write_intercept)  in vcpu_enter_guest()
10849 	static_call(kvm_x86_handle_exit_irqoff)(vcpu);  in vcpu_enter_guest()
10851 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10861 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);  in vcpu_enter_guest()
10863 	++vcpu->stat.exits;  in vcpu_enter_guest()
10865 	kvm_after_interrupt(vcpu);  in vcpu_enter_guest()
10879 	kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
10885 		unsigned long rip = kvm_rip_read(vcpu);  in vcpu_enter_guest()
10889 	if (unlikely(vcpu->arch.tsc_always_catchup))  in vcpu_enter_guest()
10890 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in vcpu_enter_guest()
10892 	if (vcpu->arch.apic_attention)  in vcpu_enter_guest()
10893 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
10895 	r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);  in vcpu_enter_guest()
10900 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
10901 	static_call(kvm_x86_cancel_injection)(vcpu);  in vcpu_enter_guest()
10902 	if (unlikely(vcpu->arch.apic_attention))  in vcpu_enter_guest()
10903 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
10909 static inline int vcpu_block(struct kvm_vcpu *vcpu)  in vcpu_block()  argument
10913 	if (!kvm_arch_vcpu_runnable(vcpu)) {  in vcpu_block()
10916 		 * the guest's timer may be a break event for the vCPU, and the  in vcpu_block()
10921 		hv_timer = kvm_lapic_hv_timer_in_use(vcpu);  in vcpu_block()
10923 			kvm_lapic_switch_to_sw_timer(vcpu);  in vcpu_block()
10925 		kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_block()
10926 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)  in vcpu_block()
10927 			kvm_vcpu_halt(vcpu);  in vcpu_block()
10929 			kvm_vcpu_block(vcpu);  in vcpu_block()
10930 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_block()
10933 			kvm_lapic_switch_to_hv_timer(vcpu);  in vcpu_block()
10936 		 * If the vCPU is not runnable, a signal or another host event  in vcpu_block()
10938 		 * vCPU's activity state.  in vcpu_block()
10940 		if (!kvm_arch_vcpu_runnable(vcpu))  in vcpu_block()
10950 	if (is_guest_mode(vcpu)) {  in vcpu_block()
10951 		if (kvm_check_nested_events(vcpu) < 0)  in vcpu_block()
10955 	if (kvm_apic_accept_events(vcpu) < 0)  in vcpu_block()
10957 	switch(vcpu->arch.mp_state) {  in vcpu_block()
10960 		vcpu->arch.pv.pv_unhalted = false;  in vcpu_block()
10961 		vcpu->arch.mp_state =  in vcpu_block()
10965 		vcpu->arch.apf.halted = false;  in vcpu_block()
10976 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)  in kvm_vcpu_running()  argument
10978 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&  in kvm_vcpu_running()
10979 		!vcpu->arch.apf.halted);  in kvm_vcpu_running()
10983 static int vcpu_run(struct kvm_vcpu *vcpu)  in vcpu_run()  argument
10987 	vcpu->arch.l1tf_flush_l1d = true;  in vcpu_run()
10991 		 * If another guest vCPU requests a PV TLB flush in the middle  in vcpu_run()
10996 		vcpu->arch.at_instruction_boundary = false;  in vcpu_run()
10997 		if (kvm_vcpu_running(vcpu)) {  in vcpu_run()
10998 			r = vcpu_enter_guest(vcpu);  in vcpu_run()
11000 			r = vcpu_block(vcpu);  in vcpu_run()
11006 		kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);  in vcpu_run()
11007 		if (kvm_xen_has_pending_events(vcpu))  in vcpu_run()
11008 			kvm_xen_inject_pending_events(vcpu);  in vcpu_run()
11010 		if (kvm_cpu_has_pending_timer(vcpu))  in vcpu_run()
11011 			kvm_inject_pending_timer_irqs(vcpu);  in vcpu_run()
11013 		if (dm_request_for_irq_injection(vcpu) &&  in vcpu_run()
11014 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {  in vcpu_run()
11016 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;  in vcpu_run()
11017 			++vcpu->stat.request_irq_exits;  in vcpu_run()
11022 			kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_run()
11023 			r = xfer_to_guest_mode_handle_work(vcpu);  in vcpu_run()
11024 			kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_run()
11033 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)  in complete_emulated_io()  argument
11035 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);  in complete_emulated_io()
11038 static int complete_emulated_pio(struct kvm_vcpu *vcpu)  in complete_emulated_pio()  argument
11040 	BUG_ON(!vcpu->arch.pio.count);  in complete_emulated_pio()
11042 	return complete_emulated_io(vcpu);  in complete_emulated_pio()
11063 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_emulated_mmio()  argument
11065 	struct kvm_run *run = vcpu->run;  in complete_emulated_mmio()
11069 	BUG_ON(!vcpu->mmio_needed);  in complete_emulated_mmio()
11072 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_emulated_mmio()
11074 	if (!vcpu->mmio_is_write)  in complete_emulated_mmio()
11080 		vcpu->mmio_cur_fragment++;  in complete_emulated_mmio()
11088 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_emulated_mmio()
11089 		vcpu->mmio_needed = 0;  in complete_emulated_mmio()
11092 		if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11094 		vcpu->mmio_read_completed = 1;  in complete_emulated_mmio()
11095 		return complete_emulated_io(vcpu);  in complete_emulated_mmio()
11100 	if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11103 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_emulated_mmio()
11104 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in complete_emulated_mmio()
11109 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_load_guest_fpu()  argument
11112 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);  in kvm_load_guest_fpu()
11117 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_put_guest_fpu()  argument
11119 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);  in kvm_put_guest_fpu()
11120 	++vcpu->stat.fpu_reload;  in kvm_put_guest_fpu()
11124 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_ioctl_run()  argument
11126 	struct kvm_queued_exception *ex = &vcpu->arch.exception;  in kvm_arch_vcpu_ioctl_run()
11127 	struct kvm_run *kvm_run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
11130 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_run()
11131 	kvm_sigset_activate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11133 	kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11135 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11136 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {  in kvm_arch_vcpu_ioctl_run()
11145 		 * APIC timer to be active is if userspace stuffed vCPU state,  in kvm_arch_vcpu_ioctl_run()
11146 		 * i.e. put the vCPU into a nonsensical state.  Only an INIT  in kvm_arch_vcpu_ioctl_run()
11147 		 * will transition the vCPU out of UNINITIALIZED (without more  in kvm_arch_vcpu_ioctl_run()
11152 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11153 		kvm_vcpu_block(vcpu);  in kvm_arch_vcpu_ioctl_run()
11154 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11156 		if (kvm_apic_accept_events(vcpu) < 0) {  in kvm_arch_vcpu_ioctl_run()
11164 			++vcpu->stat.signal_exits;  in kvm_arch_vcpu_ioctl_run()
11176 		r = sync_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11182 	if (!lapic_in_kernel(vcpu)) {  in kvm_arch_vcpu_ioctl_run()
11183 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {  in kvm_arch_vcpu_ioctl_run()
11193 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&  in kvm_arch_vcpu_ioctl_run()
11194 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11196 		kvm_queue_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11202 	vcpu->arch.exception_from_userspace = false;  in kvm_arch_vcpu_ioctl_run()
11204 	if (unlikely(vcpu->arch.complete_userspace_io)) {  in kvm_arch_vcpu_ioctl_run()
11205 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;  in kvm_arch_vcpu_ioctl_run()
11206 		vcpu->arch.complete_userspace_io = NULL;  in kvm_arch_vcpu_ioctl_run()
11207 		r = cui(vcpu);  in kvm_arch_vcpu_ioctl_run()
11211 		WARN_ON_ONCE(vcpu->arch.pio.count);  in kvm_arch_vcpu_ioctl_run()
11212 		WARN_ON_ONCE(vcpu->mmio_needed);  in kvm_arch_vcpu_ioctl_run()
11220 	r = static_call(kvm_x86_vcpu_pre_run)(vcpu);  in kvm_arch_vcpu_ioctl_run()
11224 	r = vcpu_run(vcpu);  in kvm_arch_vcpu_ioctl_run()
11227 	kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11229 		store_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11230 	post_kvm_run_save(vcpu);  in kvm_arch_vcpu_ioctl_run()
11231 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11233 	kvm_sigset_deactivate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11234 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_run()
11238 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __get_regs()  argument
11240 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {  in __get_regs()
11244 		 * back from emulation context to vcpu. Userspace shouldn't do  in __get_regs()
11248 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);  in __get_regs()
11249 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __get_regs()
11251 	regs->rax = kvm_rax_read(vcpu);  in __get_regs()
11252 	regs->rbx = kvm_rbx_read(vcpu);  in __get_regs()
11253 	regs->rcx = kvm_rcx_read(vcpu);  in __get_regs()
11254 	regs->rdx = kvm_rdx_read(vcpu);  in __get_regs()
11255 	regs->rsi = kvm_rsi_read(vcpu);  in __get_regs()
11256 	regs->rdi = kvm_rdi_read(vcpu);  in __get_regs()
11257 	regs->rsp = kvm_rsp_read(vcpu);  in __get_regs()
11258 	regs->rbp = kvm_rbp_read(vcpu);  in __get_regs()
11260 	regs->r8 = kvm_r8_read(vcpu);  in __get_regs()
11261 	regs->r9 = kvm_r9_read(vcpu);  in __get_regs()
11262 	regs->r10 = kvm_r10_read(vcpu);  in __get_regs()
11263 	regs->r11 = kvm_r11_read(vcpu);  in __get_regs()
11264 	regs->r12 = kvm_r12_read(vcpu);  in __get_regs()
11265 	regs->r13 = kvm_r13_read(vcpu);  in __get_regs()
11266 	regs->r14 = kvm_r14_read(vcpu);  in __get_regs()
11267 	regs->r15 = kvm_r15_read(vcpu);  in __get_regs()
11270 	regs->rip = kvm_rip_read(vcpu);  in __get_regs()
11271 	regs->rflags = kvm_get_rflags(vcpu);  in __get_regs()
11274 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_get_regs()  argument
11276 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11277 	__get_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_get_regs()
11278 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11282 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __set_regs()  argument
11284 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;  in __set_regs()
11285 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __set_regs()
11287 	kvm_rax_write(vcpu, regs->rax);  in __set_regs()
11288 	kvm_rbx_write(vcpu, regs->rbx);  in __set_regs()
11289 	kvm_rcx_write(vcpu, regs->rcx);  in __set_regs()
11290 	kvm_rdx_write(vcpu, regs->rdx);  in __set_regs()
11291 	kvm_rsi_write(vcpu, regs->rsi);  in __set_regs()
11292 	kvm_rdi_write(vcpu, regs->rdi);  in __set_regs()
11293 	kvm_rsp_write(vcpu, regs->rsp);  in __set_regs()
11294 	kvm_rbp_write(vcpu, regs->rbp);  in __set_regs()
11296 	kvm_r8_write(vcpu, regs->r8);  in __set_regs()
11297 	kvm_r9_write(vcpu, regs->r9);  in __set_regs()
11298 	kvm_r10_write(vcpu, regs->r10);  in __set_regs()
11299 	kvm_r11_write(vcpu, regs->r11);  in __set_regs()
11300 	kvm_r12_write(vcpu, regs->r12);  in __set_regs()
11301 	kvm_r13_write(vcpu, regs->r13);  in __set_regs()
11302 	kvm_r14_write(vcpu, regs->r14);  in __set_regs()
11303 	kvm_r15_write(vcpu, regs->r15);  in __set_regs()
11306 	kvm_rip_write(vcpu, regs->rip);  in __set_regs()
11307 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);  in __set_regs()
11309 	vcpu->arch.exception.pending = false;  in __set_regs()
11310 	vcpu->arch.exception_vmexit.pending = false;  in __set_regs()
11312 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_regs()
11315 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_set_regs()  argument
11317 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11318 	__set_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_set_regs()
11319 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11323 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs_common()  argument
11327 	if (vcpu->arch.guest_state_protected)  in __get_sregs_common()
11330 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __get_sregs_common()
11331 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __get_sregs_common()
11332 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __get_sregs_common()
11333 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __get_sregs_common()
11334 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __get_sregs_common()
11335 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __get_sregs_common()
11337 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __get_sregs_common()
11338 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __get_sregs_common()
11340 	static_call(kvm_x86_get_idt)(vcpu, &dt);  in __get_sregs_common()
11343 	static_call(kvm_x86_get_gdt)(vcpu, &dt);  in __get_sregs_common()
11347 	sregs->cr2 = vcpu->arch.cr2;  in __get_sregs_common()
11348 	sregs->cr3 = kvm_read_cr3(vcpu);  in __get_sregs_common()
11351 	sregs->cr0 = kvm_read_cr0(vcpu);  in __get_sregs_common()
11352 	sregs->cr4 = kvm_read_cr4(vcpu);  in __get_sregs_common()
11353 	sregs->cr8 = kvm_get_cr8(vcpu);  in __get_sregs_common()
11354 	sregs->efer = vcpu->arch.efer;  in __get_sregs_common()
11355 	sregs->apic_base = kvm_get_apic_base(vcpu);  in __get_sregs_common()
11358 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs()  argument
11360 	__get_sregs_common(vcpu, sregs);  in __get_sregs()
11362 	if (vcpu->arch.guest_state_protected)  in __get_sregs()
11365 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)  in __get_sregs()
11366 		set_bit(vcpu->arch.interrupt.nr,  in __get_sregs()
11370 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __get_sregs2()  argument
11374 	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);  in __get_sregs2()
11376 	if (vcpu->arch.guest_state_protected)  in __get_sregs2()
11379 	if (is_pae_paging(vcpu)) {  in __get_sregs2()
11381 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);  in __get_sregs2()
11386 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_sregs()  argument
11389 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11390 	__get_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_get_sregs()
11391 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11395 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_mpstate()  argument
11400 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11402 		kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11404 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11406 	r = kvm_apic_accept_events(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11411 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||  in kvm_arch_vcpu_ioctl_get_mpstate()
11412 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&  in kvm_arch_vcpu_ioctl_get_mpstate()
11413 	    vcpu->arch.pv.pv_unhalted)  in kvm_arch_vcpu_ioctl_get_mpstate()
11416 		mp_state->mp_state = vcpu->arch.mp_state;  in kvm_arch_vcpu_ioctl_get_mpstate()
11419 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11422 		kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11423 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11427 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_mpstate()  argument
11432 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11440 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl_set_mpstate()
11457 	if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
11463 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;  in kvm_arch_vcpu_ioctl_set_mpstate()
11464 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);  in kvm_arch_vcpu_ioctl_set_mpstate()
11466 		vcpu->arch.mp_state = mp_state->mp_state;  in kvm_arch_vcpu_ioctl_set_mpstate()
11467 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11471 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11475 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,  in kvm_task_switch()  argument
11478 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()
11481 	init_emulate_ctxt(vcpu);  in kvm_task_switch()
11486 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_task_switch()
11487 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_task_switch()
11488 		vcpu->run->internal.ndata = 0;  in kvm_task_switch()
11492 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
11493 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()
11498 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvm_is_valid_sregs()  argument
11508 		if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))  in kvm_is_valid_sregs()
11519 	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&  in kvm_is_valid_sregs()
11520 	       kvm_is_valid_cr0(vcpu, sregs->cr0);  in kvm_is_valid_sregs()
11523 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,  in __set_sregs_common()  argument
11530 	if (!kvm_is_valid_sregs(vcpu, sregs))  in __set_sregs_common()
11535 	if (kvm_set_apic_base(vcpu, &apic_base_msr))  in __set_sregs_common()
11538 	if (vcpu->arch.guest_state_protected)  in __set_sregs_common()
11543 	static_call(kvm_x86_set_idt)(vcpu, &dt);  in __set_sregs_common()
11546 	static_call(kvm_x86_set_gdt)(vcpu, &dt);  in __set_sregs_common()
11548 	vcpu->arch.cr2 = sregs->cr2;  in __set_sregs_common()
11549 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;  in __set_sregs_common()
11550 	vcpu->arch.cr3 = sregs->cr3;  in __set_sregs_common()
11551 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in __set_sregs_common()
11552 	static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);  in __set_sregs_common()
11554 	kvm_set_cr8(vcpu, sregs->cr8);  in __set_sregs_common()
11556 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;  in __set_sregs_common()
11557 	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);  in __set_sregs_common()
11559 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;  in __set_sregs_common()
11560 	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);  in __set_sregs_common()
11561 	vcpu->arch.cr0 = sregs->cr0;  in __set_sregs_common()
11563 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;  in __set_sregs_common()
11564 	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);  in __set_sregs_common()
11567 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in __set_sregs_common()
11568 		if (is_pae_paging(vcpu)) {  in __set_sregs_common()
11569 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));  in __set_sregs_common()
11572 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in __set_sregs_common()
11575 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __set_sregs_common()
11576 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __set_sregs_common()
11577 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __set_sregs_common()
11578 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __set_sregs_common()
11579 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __set_sregs_common()
11580 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __set_sregs_common()
11582 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __set_sregs_common()
11583 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __set_sregs_common()
11585 	update_cr8_intercept(vcpu);  in __set_sregs_common()
11587 	/* Older userspace won't unhalt the vcpu on reset. */  in __set_sregs_common()
11588 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&  in __set_sregs_common()
11590 	    !is_protmode(vcpu))  in __set_sregs_common()
11591 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in __set_sregs_common()
11596 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __set_sregs()  argument
11600 	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);  in __set_sregs()
11606 		kvm_mmu_reset_context(vcpu);  in __set_sregs()
11613 		kvm_queue_interrupt(vcpu, pending_vec, false);  in __set_sregs()
11615 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_sregs()
11620 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __set_sregs2()  argument
11631 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))  in __set_sregs2()
11634 	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,  in __set_sregs2()
11641 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);  in __set_sregs2()
11643 		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in __set_sregs2()
11645 		vcpu->arch.pdptrs_from_userspace = true;  in __set_sregs2()
11648 		kvm_mmu_reset_context(vcpu);  in __set_sregs2()
11652 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_sregs()  argument
11657 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
11658 	ret = __set_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_set_sregs()
11659 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
11666 	struct kvm_vcpu *vcpu;  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()  local
11674 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11675 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11684 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_guest_debug()  argument
11690 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11693 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11697 		if (kvm_is_exception_pending(vcpu))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11700 			kvm_queue_exception(vcpu, DB_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11702 			kvm_queue_exception(vcpu, BP_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11709 	rflags = kvm_get_rflags(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11711 	vcpu->guest_debug = dbg->control;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11712 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11713 		vcpu->guest_debug = 0;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11715 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
11717 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11718 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11721 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11723 	kvm_update_dr7(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11725 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11726 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11732 	kvm_set_rflags(vcpu, rflags);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11734 	static_call(kvm_x86_update_exception_bitmap)(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11736 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11741 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11748 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_translate()  argument
11755 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_translate()
11757 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl_translate()
11758 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);  in kvm_arch_vcpu_ioctl_translate()
11759 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl_translate()
11765 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_translate()
11769 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_get_fpu()  argument
11773 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_get_fpu()
11776 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
11778 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_get_fpu()
11788 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
11792 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_set_fpu()  argument
11796 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_set_fpu()
11799 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
11801 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_set_fpu()
11812 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
11816 static void store_regs(struct kvm_vcpu *vcpu)  in store_regs()  argument
11820 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)  in store_regs()
11821 		__get_regs(vcpu, &vcpu->run->s.regs.regs);  in store_regs()
11823 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)  in store_regs()
11824 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);  in store_regs()
11826 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)  in store_regs()
11828 				vcpu, &vcpu->run->s.regs.events);  in store_regs()
11831 static int sync_regs(struct kvm_vcpu *vcpu)  in sync_regs()  argument
11833 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {  in sync_regs()
11834 		__set_regs(vcpu, &vcpu->run->s.regs.regs);  in sync_regs()
11835 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;  in sync_regs()
11838 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {  in sync_regs()
11839 		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;  in sync_regs()
11841 		if (__set_sregs(vcpu, &sregs))  in sync_regs()
11844 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;  in sync_regs()
11847 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {  in sync_regs()
11848 		struct kvm_vcpu_events events = vcpu->run->s.regs.events;  in sync_regs()
11850 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events))  in sync_regs()
11853 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;  in sync_regs()
11874 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_create()  argument
11879 	vcpu->arch.last_vmentry_cpu = -1;  in kvm_arch_vcpu_create()
11880 	vcpu->arch.regs_avail = ~0;  in kvm_arch_vcpu_create()
11881 	vcpu->arch.regs_dirty = ~0;  in kvm_arch_vcpu_create()
11883 	kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN);  in kvm_arch_vcpu_create()
11885 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))  in kvm_arch_vcpu_create()
11886 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_create()
11888 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;  in kvm_arch_vcpu_create()
11890 	r = kvm_mmu_create(vcpu);  in kvm_arch_vcpu_create()
11894 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arch_vcpu_create()
11895 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);  in kvm_arch_vcpu_create()
11900 		 * Defer evaluating inhibits until the vCPU is first run, as  in kvm_arch_vcpu_create()
11901 		 * this vCPU will not get notified of any changes until this  in kvm_arch_vcpu_create()
11902 		 * vCPU is visible to other vCPUs (marked online and added to  in kvm_arch_vcpu_create()
11905 		 * Ignore the current per-VM APICv state so that vCPU creation  in kvm_arch_vcpu_create()
11907 		 * will ensure the vCPU gets the correct state before VM-Entry.  in kvm_arch_vcpu_create()
11910 			vcpu->arch.apic->apicv_active = true;  in kvm_arch_vcpu_create()
11911 			kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);  in kvm_arch_vcpu_create()
11921 	vcpu->arch.pio_data = page_address(page);  in kvm_arch_vcpu_create()
11923 	vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),  in kvm_arch_vcpu_create()
11925 	vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),  in kvm_arch_vcpu_create()
11927 	if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)  in kvm_arch_vcpu_create()
11929 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;  in kvm_arch_vcpu_create()
11931 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,  in kvm_arch_vcpu_create()
11935 	if (!alloc_emulate_ctxt(vcpu))  in kvm_arch_vcpu_create()
11938 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {  in kvm_arch_vcpu_create()
11939 		pr_err("failed to allocate vcpu's fpu\n");  in kvm_arch_vcpu_create()
11943 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);  in kvm_arch_vcpu_create()
11944 	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);  in kvm_arch_vcpu_create()
11946 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;  in kvm_arch_vcpu_create()
11948 	kvm_async_pf_hash_reset(vcpu);  in kvm_arch_vcpu_create()
11950 	vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;  in kvm_arch_vcpu_create()
11951 	kvm_pmu_init(vcpu);  in kvm_arch_vcpu_create()
11953 	vcpu->arch.pending_external_vector = -1;  in kvm_arch_vcpu_create()
11954 	vcpu->arch.preempted_in_kernel = false;  in kvm_arch_vcpu_create()
11957 	vcpu->arch.hv_root_tdp = INVALID_PAGE;  in kvm_arch_vcpu_create()
11960 	r = static_call(kvm_x86_vcpu_create)(vcpu);  in kvm_arch_vcpu_create()
11964 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();  in kvm_arch_vcpu_create()
11965 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;  in kvm_arch_vcpu_create()
11966 	kvm_xen_init_vcpu(vcpu);  in kvm_arch_vcpu_create()
11967 	kvm_vcpu_mtrr_init(vcpu);  in kvm_arch_vcpu_create()
11968 	vcpu_load(vcpu);  in kvm_arch_vcpu_create()
11969 	kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);  in kvm_arch_vcpu_create()
11970 	kvm_vcpu_reset(vcpu, false);  in kvm_arch_vcpu_create()
11971 	kvm_init_mmu(vcpu);  in kvm_arch_vcpu_create()
11972 	vcpu_put(vcpu);  in kvm_arch_vcpu_create()
11976 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_create()
11978 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_create()
11980 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_create()
11982 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_create()
11983 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_create()
11984 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_create()
11986 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_create()
11988 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_create()
11992 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_postcreate()  argument
11994 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_vcpu_postcreate()
11996 	if (mutex_lock_killable(&vcpu->mutex))  in kvm_arch_vcpu_postcreate()
11998 	vcpu_load(vcpu);  in kvm_arch_vcpu_postcreate()
11999 	kvm_synchronize_tsc(vcpu, 0);  in kvm_arch_vcpu_postcreate()
12000 	vcpu_put(vcpu);  in kvm_arch_vcpu_postcreate()
12003 	vcpu->arch.msr_kvm_poll_control = 1;  in kvm_arch_vcpu_postcreate()
12005 	mutex_unlock(&vcpu->mutex);  in kvm_arch_vcpu_postcreate()
12007 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)  in kvm_arch_vcpu_postcreate()
12012 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_destroy()  argument
12016 	kvmclock_reset(vcpu);  in kvm_arch_vcpu_destroy()
12018 	static_call(kvm_x86_vcpu_free)(vcpu);  in kvm_arch_vcpu_destroy()
12020 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_destroy()
12021 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_destroy()
12022 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_destroy()
12024 	kvm_xen_destroy_vcpu(vcpu);  in kvm_arch_vcpu_destroy()
12025 	kvm_hv_vcpu_uninit(vcpu);  in kvm_arch_vcpu_destroy()
12026 	kvm_pmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12027 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_destroy()
12028 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_destroy()
12029 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_destroy()
12030 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_destroy()
12031 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12032 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_destroy()
12033 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_destroy()
12034 	kvfree(vcpu->arch.cpuid_entries);  in kvm_arch_vcpu_destroy()
12035 	if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_destroy()
12039 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)  in kvm_vcpu_reset()  argument
12042 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_vcpu_reset()
12049 	 * into hardware, to be zeroed at vCPU creation.  Use CRs as a sentinel  in kvm_vcpu_reset()
12053 		     (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));  in kvm_vcpu_reset()
12057 	 * possible to INIT the vCPU while L2 is active.  Force the vCPU back  in kvm_vcpu_reset()
12061 	if (is_guest_mode(vcpu))  in kvm_vcpu_reset()
12062 		kvm_leave_nested(vcpu);  in kvm_vcpu_reset()
12064 	kvm_lapic_reset(vcpu, init_event);  in kvm_vcpu_reset()
12066 	WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));  in kvm_vcpu_reset()
12067 	vcpu->arch.hflags = 0;  in kvm_vcpu_reset()
12069 	vcpu->arch.smi_pending = 0;  in kvm_vcpu_reset()
12070 	vcpu->arch.smi_count = 0;  in kvm_vcpu_reset()
12071 	atomic_set(&vcpu->arch.nmi_queued, 0);  in kvm_vcpu_reset()
12072 	vcpu->arch.nmi_pending = 0;  in kvm_vcpu_reset()
12073 	vcpu->arch.nmi_injected = false;  in kvm_vcpu_reset()
12074 	kvm_clear_interrupt_queue(vcpu);  in kvm_vcpu_reset()
12075 	kvm_clear_exception_queue(vcpu);  in kvm_vcpu_reset()
12077 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));  in kvm_vcpu_reset()
12078 	kvm_update_dr0123(vcpu);  in kvm_vcpu_reset()
12079 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;  in kvm_vcpu_reset()
12080 	vcpu->arch.dr7 = DR7_FIXED_1;  in kvm_vcpu_reset()
12081 	kvm_update_dr7(vcpu);  in kvm_vcpu_reset()
12083 	vcpu->arch.cr2 = 0;  in kvm_vcpu_reset()
12085 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_reset()
12086 	vcpu->arch.apf.msr_en_val = 0;  in kvm_vcpu_reset()
12087 	vcpu->arch.apf.msr_int_val = 0;  in kvm_vcpu_reset()
12088 	vcpu->arch.st.msr_val = 0;  in kvm_vcpu_reset()
12090 	kvmclock_reset(vcpu);  in kvm_vcpu_reset()
12092 	kvm_clear_async_pf_completion_queue(vcpu);  in kvm_vcpu_reset()
12093 	kvm_async_pf_hash_reset(vcpu);  in kvm_vcpu_reset()
12094 	vcpu->arch.apf.halted = false;  in kvm_vcpu_reset()
12096 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {  in kvm_vcpu_reset()
12097 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;  in kvm_vcpu_reset()
12104 			kvm_put_guest_fpu(vcpu);  in kvm_vcpu_reset()
12110 			kvm_load_guest_fpu(vcpu);  in kvm_vcpu_reset()
12114 		kvm_pmu_reset(vcpu);  in kvm_vcpu_reset()
12115 		vcpu->arch.smbase = 0x30000;  in kvm_vcpu_reset()
12117 		vcpu->arch.msr_misc_features_enables = 0;  in kvm_vcpu_reset()
12118 		vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |  in kvm_vcpu_reset()
12121 		__kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);  in kvm_vcpu_reset()
12122 		__kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);  in kvm_vcpu_reset()
12126 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));  in kvm_vcpu_reset()
12127 	kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);  in kvm_vcpu_reset()
12132 	 * RESET since KVM emulates RESET before exposing the vCPU to userspace,  in kvm_vcpu_reset()
12136 	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);  in kvm_vcpu_reset()
12137 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);  in kvm_vcpu_reset()
12139 	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);  in kvm_vcpu_reset()
12141 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);  in kvm_vcpu_reset()
12142 	kvm_rip_write(vcpu, 0xfff0);  in kvm_vcpu_reset()
12144 	vcpu->arch.cr3 = 0;  in kvm_vcpu_reset()
12145 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_vcpu_reset()
12158 	static_call(kvm_x86_set_cr0)(vcpu, new_cr0);  in kvm_vcpu_reset()
12159 	static_call(kvm_x86_set_cr4)(vcpu, 0);  in kvm_vcpu_reset()
12160 	static_call(kvm_x86_set_efer)(vcpu, 0);  in kvm_vcpu_reset()
12161 	static_call(kvm_x86_update_exception_bitmap)(vcpu);  in kvm_vcpu_reset()
12172 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12173 		kvm_mmu_reset_context(vcpu);  in kvm_vcpu_reset()
12186 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12190 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)  in kvm_vcpu_deliver_sipi_vector()  argument
12194 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12197 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12198 	kvm_rip_write(vcpu, 0);  in kvm_vcpu_deliver_sipi_vector()
12205 	struct kvm_vcpu *vcpu;  in kvm_arch_hardware_enable()  local
12225 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
12226 			if (!stable && vcpu->cpu == smp_processor_id())  in kvm_arch_hardware_enable()
12227 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
12228 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {  in kvm_arch_hardware_enable()
12230 				if (vcpu->arch.last_host_tsc > max_tsc)  in kvm_arch_hardware_enable()
12231 					max_tsc = vcpu->arch.last_host_tsc;  in kvm_arch_hardware_enable()
12248 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,  in kvm_arch_hardware_enable()
12250 	 * adjustments, in case multiple suspend cycles happen before some VCPU  in kvm_arch_hardware_enable()
12278 			kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
12279 				vcpu->arch.tsc_offset_adjustment += delta_cyc;  in kvm_arch_hardware_enable()
12280 				vcpu->arch.last_host_tsc = local_tsc;  in kvm_arch_hardware_enable()
12281 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
12304 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_reset_bsp()  argument
12306 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;  in kvm_vcpu_is_reset_bsp()
12309 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_bsp()  argument
12311 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;  in kvm_vcpu_is_bsp()
12317 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_sched_in()  argument
12319 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);  in kvm_arch_sched_in()
12321 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_arch_sched_in()
12324 		kvm_make_request(KVM_REQ_PMU, vcpu);  in kvm_arch_sched_in()
12326 	static_call(kvm_x86_sched_in)(vcpu, cpu);  in kvm_arch_sched_in()
12403 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)  in kvm_unload_vcpu_mmu()  argument
12405 	vcpu_load(vcpu);  in kvm_unload_vcpu_mmu()
12406 	kvm_mmu_unload(vcpu);  in kvm_unload_vcpu_mmu()
12407 	vcpu_put(vcpu);  in kvm_unload_vcpu_mmu()
12413 	struct kvm_vcpu *vcpu;  in kvm_unload_vcpu_mmus()  local
12415 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_unload_vcpu_mmus()
12416 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_unload_vcpu_mmus()
12417 		kvm_unload_vcpu_mmu(vcpu);  in kvm_unload_vcpu_mmus()
12651 	struct kvm_vcpu *vcpu;  in kvm_arch_memslots_updated()  local
12661 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_memslots_updated()
12662 		kvm_vcpu_kick(vcpu);  in kvm_arch_memslots_updated()
12850 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)  in kvm_guest_apic_has_interrupt()  argument
12852 	return (is_guest_mode(vcpu) &&  in kvm_guest_apic_has_interrupt()
12853 		static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));  in kvm_guest_apic_has_interrupt()
12856 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)  in kvm_vcpu_has_events()  argument
12858 	if (!list_empty_careful(&vcpu->async_pf.done))  in kvm_vcpu_has_events()
12861 	if (kvm_apic_has_pending_init_or_sipi(vcpu) &&  in kvm_vcpu_has_events()
12862 	    kvm_apic_init_sipi_allowed(vcpu))  in kvm_vcpu_has_events()
12865 	if (vcpu->arch.pv.pv_unhalted)  in kvm_vcpu_has_events()
12868 	if (kvm_is_exception_pending(vcpu))  in kvm_vcpu_has_events()
12871 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_vcpu_has_events()
12872 	    (vcpu->arch.nmi_pending &&  in kvm_vcpu_has_events()
12873 	     static_call(kvm_x86_nmi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
12877 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_vcpu_has_events()
12878 	    (vcpu->arch.smi_pending &&  in kvm_vcpu_has_events()
12879 	     static_call(kvm_x86_smi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
12883 	if (kvm_test_request(KVM_REQ_PMI, vcpu))  in kvm_vcpu_has_events()
12886 	if (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_has_events()
12887 	    (kvm_cpu_has_interrupt(vcpu) ||  in kvm_vcpu_has_events()
12888 	    kvm_guest_apic_has_interrupt(vcpu)))  in kvm_vcpu_has_events()
12891 	if (kvm_hv_has_stimer_pending(vcpu))  in kvm_vcpu_has_events()
12894 	if (is_guest_mode(vcpu) &&  in kvm_vcpu_has_events()
12896 	    kvm_x86_ops.nested_ops->has_events(vcpu, false))  in kvm_vcpu_has_events()
12899 	if (kvm_xen_has_pending_events(vcpu))  in kvm_vcpu_has_events()
12905 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_runnable()  argument
12907 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);  in kvm_arch_vcpu_runnable()
12910 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)  in kvm_arch_dy_has_pending_interrupt()  argument
12912 	if (kvm_vcpu_apicv_active(vcpu) &&  in kvm_arch_dy_has_pending_interrupt()
12913 	    static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))  in kvm_arch_dy_has_pending_interrupt()
12919 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_dy_runnable()  argument
12921 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))  in kvm_arch_dy_runnable()
12924 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_arch_dy_runnable()
12926 		kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_arch_dy_runnable()
12928 		 kvm_test_request(KVM_REQ_EVENT, vcpu))  in kvm_arch_dy_runnable()
12931 	return kvm_arch_dy_has_pending_interrupt(vcpu);  in kvm_arch_dy_runnable()
12934 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_in_kernel()  argument
12936 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_in_kernel()
12939 	return vcpu->arch.preempted_in_kernel;  in kvm_arch_vcpu_in_kernel()
12942 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_get_ip()  argument
12944 	return kvm_rip_read(vcpu);  in kvm_arch_vcpu_get_ip()
12947 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_should_kick()  argument
12949 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;  in kvm_arch_vcpu_should_kick()
12952 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)  in kvm_arch_interrupt_allowed()  argument
12954 	return static_call(kvm_x86_interrupt_allowed)(vcpu, false);  in kvm_arch_interrupt_allowed()
12957 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)  in kvm_get_linear_rip()  argument
12960 	if (vcpu->arch.guest_state_protected)  in kvm_get_linear_rip()
12963 	if (is_64_bit_mode(vcpu))  in kvm_get_linear_rip()
12964 		return kvm_rip_read(vcpu);  in kvm_get_linear_rip()
12965 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +  in kvm_get_linear_rip()
12966 		     kvm_rip_read(vcpu));  in kvm_get_linear_rip()
12970 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)  in kvm_is_linear_rip()  argument
12972 	return kvm_get_linear_rip(vcpu) == linear_rip;  in kvm_is_linear_rip()
12976 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)  in kvm_get_rflags()  argument
12980 	rflags = static_call(kvm_x86_get_rflags)(vcpu);  in kvm_get_rflags()
12981 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_get_rflags()
12987 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in __kvm_set_rflags()  argument
12989 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&  in __kvm_set_rflags()
12990 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))  in __kvm_set_rflags()
12992 	static_call(kvm_x86_set_rflags)(vcpu, rflags);  in __kvm_set_rflags()
12995 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in kvm_set_rflags()  argument
12997 	__kvm_set_rflags(vcpu, rflags);  in kvm_set_rflags()
12998 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_set_rflags()
13014 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_add_async_pf_gfn()  argument
13018 	while (vcpu->arch.apf.gfns[key] != ~0)  in kvm_add_async_pf_gfn()
13021 	vcpu->arch.apf.gfns[key] = gfn;  in kvm_add_async_pf_gfn()
13024 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_async_pf_gfn_slot()  argument
13030 		     (vcpu->arch.apf.gfns[key] != gfn &&  in kvm_async_pf_gfn_slot()
13031 		      vcpu->arch.apf.gfns[key] != ~0); i++)  in kvm_async_pf_gfn_slot()
13037 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_find_async_pf_gfn()  argument
13039 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;  in kvm_find_async_pf_gfn()
13042 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_del_async_pf_gfn()  argument
13046 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);  in kvm_del_async_pf_gfn()
13048 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))  in kvm_del_async_pf_gfn()
13052 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_del_async_pf_gfn()
13055 			if (vcpu->arch.apf.gfns[j] == ~0)  in kvm_del_async_pf_gfn()
13057 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);  in kvm_del_async_pf_gfn()
13064 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];  in kvm_del_async_pf_gfn()
13069 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)  in apf_put_user_notpresent()  argument
13073 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,  in apf_put_user_notpresent()
13077 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)  in apf_put_user_ready()  argument
13081 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_put_user_ready()
13085 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)  in apf_pageready_slot_free()  argument
13090 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_pageready_slot_free()
13097 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_deliver_async_pf()  argument
13100 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_can_deliver_async_pf()
13103 	if (vcpu->arch.apf.send_user_only &&  in kvm_can_deliver_async_pf()
13104 	    static_call(kvm_x86_get_cpl)(vcpu) == 0)  in kvm_can_deliver_async_pf()
13107 	if (is_guest_mode(vcpu)) {  in kvm_can_deliver_async_pf()
13112 		return vcpu->arch.apf.delivery_as_pf_vmexit;  in kvm_can_deliver_async_pf()
13119 		return is_paging(vcpu);  in kvm_can_deliver_async_pf()
13123 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_do_async_pf()  argument
13125 	if (unlikely(!lapic_in_kernel(vcpu) ||  in kvm_can_do_async_pf()
13126 		     kvm_event_needs_reinjection(vcpu) ||  in kvm_can_do_async_pf()
13127 		     kvm_is_exception_pending(vcpu)))  in kvm_can_do_async_pf()
13130 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))  in kvm_can_do_async_pf()
13137 	return kvm_arch_interrupt_allowed(vcpu);  in kvm_can_do_async_pf()
13140 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_not_present()  argument
13146 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_not_present()
13148 	if (kvm_can_deliver_async_pf(vcpu) &&  in kvm_arch_async_page_not_present()
13149 	    !apf_put_user_notpresent(vcpu)) {  in kvm_arch_async_page_not_present()
13156 		kvm_inject_page_fault(vcpu, &fault);  in kvm_arch_async_page_not_present()
13167 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);  in kvm_arch_async_page_not_present()
13172 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_present()  argument
13177 		.vector = vcpu->arch.apf.vec  in kvm_arch_async_page_present()
13183 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_present()
13187 	    kvm_pv_async_pf_enabled(vcpu) &&  in kvm_arch_async_page_present()
13188 	    !apf_put_user_ready(vcpu, work->arch.token)) {  in kvm_arch_async_page_present()
13189 		vcpu->arch.apf.pageready_pending = true;  in kvm_arch_async_page_present()
13190 		kvm_apic_set_irq(vcpu, &irq, NULL);  in kvm_arch_async_page_present()
13193 	vcpu->arch.apf.halted = false;  in kvm_arch_async_page_present()
13194 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_async_page_present()
13197 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)  in kvm_arch_async_page_present_queued()  argument
13199 	kvm_make_request(KVM_REQ_APF_READY, vcpu);  in kvm_arch_async_page_present_queued()
13200 	if (!vcpu->arch.apf.pageready_pending)  in kvm_arch_async_page_present_queued()
13201 		kvm_vcpu_kick(vcpu);  in kvm_arch_async_page_present_queued()
13204 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)  in kvm_arch_can_dequeue_async_page_present()  argument
13206 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_arch_can_dequeue_async_page_present()
13209 		return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);  in kvm_arch_can_dequeue_async_page_present()
13317 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)  in kvm_arch_no_poll()  argument
13319 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;  in kvm_arch_no_poll()
13350 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)  in kvm_fixup_and_inject_pf_error()  argument
13352 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fixup_and_inject_pf_error()
13358 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {  in kvm_fixup_and_inject_pf_error()
13360 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page  in kvm_fixup_and_inject_pf_error()
13371 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);  in kvm_fixup_and_inject_pf_error()
13380 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,  in kvm_handle_memory_failure()  argument
13384 		if (KVM_BUG_ON(!e, vcpu->kvm))  in kvm_handle_memory_failure()
13387 		kvm_inject_emulated_page_fault(vcpu, e);  in kvm_handle_memory_failure()
13398 	kvm_prepare_emulation_failure_exit(vcpu);  in kvm_handle_memory_failure()
13404 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)  in kvm_handle_invpcid()  argument
13414 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);  in kvm_handle_invpcid()
13416 		return kvm_handle_memory_failure(vcpu, r, &e);  in kvm_handle_invpcid()
13419 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13423 	pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);  in kvm_handle_invpcid()
13428 		    is_noncanonical_address(operand.gla, vcpu)) {  in kvm_handle_invpcid()
13429 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13432 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);  in kvm_handle_invpcid()
13433 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13437 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13441 		kvm_invalidate_pcid(vcpu, operand.pcid);  in kvm_handle_invpcid()
13442 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13454 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_handle_invpcid()
13455 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13458 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13464 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_mmio()  argument
13466 	struct kvm_run *run = vcpu->run;  in complete_sev_es_emulated_mmio()
13470 	BUG_ON(!vcpu->mmio_needed);  in complete_sev_es_emulated_mmio()
13473 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_sev_es_emulated_mmio()
13475 	if (!vcpu->mmio_is_write)  in complete_sev_es_emulated_mmio()
13481 		vcpu->mmio_cur_fragment++;  in complete_sev_es_emulated_mmio()
13489 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_sev_es_emulated_mmio()
13490 		vcpu->mmio_needed = 0;  in complete_sev_es_emulated_mmio()
13500 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_sev_es_emulated_mmio()
13505 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in complete_sev_es_emulated_mmio()
13510 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_write()  argument
13519 	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_write()
13528 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_write()
13529 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_write()
13534 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_write()
13535 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_write()
13537 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_write()
13538 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_write()
13539 	vcpu->run->mmio.is_write = 1;  in kvm_sev_es_mmio_write()
13540 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in kvm_sev_es_mmio_write()
13541 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_write()
13543 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_write()
13549 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_read()  argument
13558 	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_read()
13567 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_read()
13568 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_read()
13573 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_read()
13574 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_read()
13576 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_read()
13577 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_read()
13578 	vcpu->run->mmio.is_write = 0;  in kvm_sev_es_mmio_read()
13579 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_read()
13581 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_read()
13587 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)  in advance_sev_es_emulated_pio()  argument
13589 	vcpu->arch.sev_pio_count -= count;  in advance_sev_es_emulated_pio()
13590 	vcpu->arch.sev_pio_data += count * size;  in advance_sev_es_emulated_pio()
13593 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13596 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_outs()  argument
13598 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_outs()
13599 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_outs()
13601 	vcpu->arch.pio.count = 0;  in complete_sev_es_emulated_outs()
13602 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_outs()
13603 		return kvm_sev_es_outs(vcpu, size, port);  in complete_sev_es_emulated_outs()
13607 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_outs()  argument
13612 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_outs()
13613 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);  in kvm_sev_es_outs()
13616 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_outs()
13621 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_outs()
13625 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;  in kvm_sev_es_outs()
13629 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13632 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_ins()  argument
13634 	unsigned count = vcpu->arch.pio.count;  in complete_sev_es_emulated_ins()
13635 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_ins()
13636 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_ins()
13638 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);  in complete_sev_es_emulated_ins()
13639 	advance_sev_es_emulated_pio(vcpu, count, size);  in complete_sev_es_emulated_ins()
13640 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_ins()
13641 		return kvm_sev_es_ins(vcpu, size, port);  in complete_sev_es_emulated_ins()
13645 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_ins()  argument
13650 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_ins()
13651 		if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))  in kvm_sev_es_ins()
13655 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_ins()
13656 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_ins()
13660 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;  in kvm_sev_es_ins()
13664 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_string_io()  argument
13668 	vcpu->arch.sev_pio_data = data;  in kvm_sev_es_string_io()
13669 	vcpu->arch.sev_pio_count = count;  in kvm_sev_es_string_io()
13670 	return in ? kvm_sev_es_ins(vcpu, size, port)  in kvm_sev_es_string_io()
13671 		  : kvm_sev_es_outs(vcpu, size, port);  in kvm_sev_es_string_io()