xref: /openbmc/linux/arch/x86/kvm/x86.c (revision 4a0a1436)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine driver for Linux
4  *
5  * derived from drivers/kvm/kvm_main.c
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright (C) 2008 Qumranet, Inc.
9  * Copyright IBM Corporation, 2008
10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11  *
12  * Authors:
13  *   Avi Kivity   <avi@qumranet.com>
14  *   Yaniv Kamay  <yaniv@qumranet.com>
15  *   Amit Shah    <amit.shah@qumranet.com>
16  *   Ben-Ami Yassour <benami@il.ibm.com>
17  */
18 
19 #include <linux/kvm_host.h>
20 #include "irq.h"
21 #include "ioapic.h"
22 #include "mmu.h"
23 #include "i8254.h"
24 #include "tss.h"
25 #include "kvm_cache_regs.h"
26 #include "kvm_emulate.h"
27 #include "x86.h"
28 #include "cpuid.h"
29 #include "pmu.h"
30 #include "hyperv.h"
31 #include "lapic.h"
32 #include "xen.h"
33 
34 #include <linux/clocksource.h>
35 #include <linux/interrupt.h>
36 #include <linux/kvm.h>
37 #include <linux/fs.h>
38 #include <linux/vmalloc.h>
39 #include <linux/export.h>
40 #include <linux/moduleparam.h>
41 #include <linux/mman.h>
42 #include <linux/highmem.h>
43 #include <linux/iommu.h>
44 #include <linux/intel-iommu.h>
45 #include <linux/cpufreq.h>
46 #include <linux/user-return-notifier.h>
47 #include <linux/srcu.h>
48 #include <linux/slab.h>
49 #include <linux/perf_event.h>
50 #include <linux/uaccess.h>
51 #include <linux/hash.h>
52 #include <linux/pci.h>
53 #include <linux/timekeeper_internal.h>
54 #include <linux/pvclock_gtod.h>
55 #include <linux/kvm_irqfd.h>
56 #include <linux/irqbypass.h>
57 #include <linux/sched/stat.h>
58 #include <linux/sched/isolation.h>
59 #include <linux/mem_encrypt.h>
60 #include <linux/entry-kvm.h>
61 #include <linux/suspend.h>
62 
63 #include <trace/events/kvm.h>
64 
65 #include <asm/debugreg.h>
66 #include <asm/msr.h>
67 #include <asm/desc.h>
68 #include <asm/mce.h>
69 #include <asm/pkru.h>
70 #include <linux/kernel_stat.h>
71 #include <asm/fpu/api.h>
72 #include <asm/fpu/xcr.h>
73 #include <asm/fpu/xstate.h>
74 #include <asm/pvclock.h>
75 #include <asm/div64.h>
76 #include <asm/irq_remapping.h>
77 #include <asm/mshyperv.h>
78 #include <asm/hypervisor.h>
79 #include <asm/tlbflush.h>
80 #include <asm/intel_pt.h>
81 #include <asm/emulate_prefix.h>
82 #include <asm/sgx.h>
83 #include <clocksource/hyperv_timer.h>
84 
85 #define CREATE_TRACE_POINTS
86 #include "trace.h"
87 
88 #define MAX_IO_MSRS 256
89 #define KVM_MAX_MCE_BANKS 32
90 u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
91 EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
92 
93 #define  ERR_PTR_USR(e)  ((void __user *)ERR_PTR(e))
94 
95 #define emul_to_vcpu(ctxt) \
96 	((struct kvm_vcpu *)(ctxt)->vcpu)
97 
98 /* EFER defaults:
99  * - enable syscall per default because its emulated by KVM
100  * - enable LME and LMA per default on 64 bit KVM
101  */
102 #ifdef CONFIG_X86_64
103 static
104 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
105 #else
106 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
107 #endif
108 
109 static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;
110 
111 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)
112 
113 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
114                                     KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
115 
116 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
117 static void process_nmi(struct kvm_vcpu *vcpu);
118 static void process_smi(struct kvm_vcpu *vcpu);
119 static void enter_smm(struct kvm_vcpu *vcpu);
120 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
121 static void store_regs(struct kvm_vcpu *vcpu);
122 static int sync_regs(struct kvm_vcpu *vcpu);
123 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
124 
125 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
126 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
127 
128 struct kvm_x86_ops kvm_x86_ops __read_mostly;
129 EXPORT_SYMBOL_GPL(kvm_x86_ops);
130 
131 #define KVM_X86_OP(func)					     \
132 	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,			     \
133 				*(((struct kvm_x86_ops *)0)->func));
134 #define KVM_X86_OP_NULL KVM_X86_OP
135 #include <asm/kvm-x86-ops.h>
136 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
137 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
138 EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);
139 
140 static bool __read_mostly ignore_msrs = 0;
141 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
142 
143 bool __read_mostly report_ignored_msrs = true;
144 module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);
145 EXPORT_SYMBOL_GPL(report_ignored_msrs);
146 
147 unsigned int min_timer_period_us = 200;
148 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
149 
150 static bool __read_mostly kvmclock_periodic_sync = true;
151 module_param(kvmclock_periodic_sync, bool, S_IRUGO);
152 
153 bool __read_mostly kvm_has_tsc_control;
154 EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
155 u32  __read_mostly kvm_max_guest_tsc_khz;
156 EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
157 u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
158 EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
159 u64  __read_mostly kvm_max_tsc_scaling_ratio;
160 EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
161 u64 __read_mostly kvm_default_tsc_scaling_ratio;
162 EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
163 bool __read_mostly kvm_has_bus_lock_exit;
164 EXPORT_SYMBOL_GPL(kvm_has_bus_lock_exit);
165 
166 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
167 static u32 __read_mostly tsc_tolerance_ppm = 250;
168 module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
169 
170 /*
171  * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
172  * adaptive tuning starting from default advancement of 1000ns.  '0' disables
173  * advancement entirely.  Any other value is used as-is and disables adaptive
174  * tuning, i.e. allows privileged userspace to set an exact advancement time.
175  */
176 static int __read_mostly lapic_timer_advance_ns = -1;
177 module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
178 
179 static bool __read_mostly vector_hashing = true;
180 module_param(vector_hashing, bool, S_IRUGO);
181 
182 bool __read_mostly enable_vmware_backdoor = false;
183 module_param(enable_vmware_backdoor, bool, S_IRUGO);
184 EXPORT_SYMBOL_GPL(enable_vmware_backdoor);
185 
186 static bool __read_mostly force_emulation_prefix = false;
187 module_param(force_emulation_prefix, bool, S_IRUGO);
188 
189 int __read_mostly pi_inject_timer = -1;
190 module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);
191 
192 /* Enable/disable PMU virtualization */
193 bool __read_mostly enable_pmu = true;
194 EXPORT_SYMBOL_GPL(enable_pmu);
195 module_param(enable_pmu, bool, 0444);
196 
197 /*
198  * Restoring the host value for MSRs that are only consumed when running in
199  * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
200  * returns to userspace, i.e. the kernel can run with the guest's value.
201  */
202 #define KVM_MAX_NR_USER_RETURN_MSRS 16
203 
204 struct kvm_user_return_msrs {
205 	struct user_return_notifier urn;
206 	bool registered;
207 	struct kvm_user_return_msr_values {
208 		u64 host;
209 		u64 curr;
210 	} values[KVM_MAX_NR_USER_RETURN_MSRS];
211 };
212 
213 u32 __read_mostly kvm_nr_uret_msrs;
214 EXPORT_SYMBOL_GPL(kvm_nr_uret_msrs);
215 static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
216 static struct kvm_user_return_msrs __percpu *user_return_msrs;
217 
218 #define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
219 				| XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
220 				| XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
221 				| XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE)
222 
223 u64 __read_mostly host_efer;
224 EXPORT_SYMBOL_GPL(host_efer);
225 
226 bool __read_mostly allow_smaller_maxphyaddr = 0;
227 EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
228 
229 bool __read_mostly enable_apicv = true;
230 EXPORT_SYMBOL_GPL(enable_apicv);
231 
232 u64 __read_mostly host_xss;
233 EXPORT_SYMBOL_GPL(host_xss);
234 u64 __read_mostly supported_xss;
235 EXPORT_SYMBOL_GPL(supported_xss);
236 
237 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
238 	KVM_GENERIC_VM_STATS(),
239 	STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
240 	STATS_DESC_COUNTER(VM, mmu_pte_write),
241 	STATS_DESC_COUNTER(VM, mmu_pde_zapped),
242 	STATS_DESC_COUNTER(VM, mmu_flooded),
243 	STATS_DESC_COUNTER(VM, mmu_recycled),
244 	STATS_DESC_COUNTER(VM, mmu_cache_miss),
245 	STATS_DESC_ICOUNTER(VM, mmu_unsync),
246 	STATS_DESC_ICOUNTER(VM, pages_4k),
247 	STATS_DESC_ICOUNTER(VM, pages_2m),
248 	STATS_DESC_ICOUNTER(VM, pages_1g),
249 	STATS_DESC_ICOUNTER(VM, nx_lpage_splits),
250 	STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size),
251 	STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions)
252 };
253 
254 const struct kvm_stats_header kvm_vm_stats_header = {
255 	.name_size = KVM_STATS_NAME_SIZE,
256 	.num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
257 	.id_offset = sizeof(struct kvm_stats_header),
258 	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
259 	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
260 		       sizeof(kvm_vm_stats_desc),
261 };
262 
263 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
264 	KVM_GENERIC_VCPU_STATS(),
265 	STATS_DESC_COUNTER(VCPU, pf_fixed),
266 	STATS_DESC_COUNTER(VCPU, pf_guest),
267 	STATS_DESC_COUNTER(VCPU, tlb_flush),
268 	STATS_DESC_COUNTER(VCPU, invlpg),
269 	STATS_DESC_COUNTER(VCPU, exits),
270 	STATS_DESC_COUNTER(VCPU, io_exits),
271 	STATS_DESC_COUNTER(VCPU, mmio_exits),
272 	STATS_DESC_COUNTER(VCPU, signal_exits),
273 	STATS_DESC_COUNTER(VCPU, irq_window_exits),
274 	STATS_DESC_COUNTER(VCPU, nmi_window_exits),
275 	STATS_DESC_COUNTER(VCPU, l1d_flush),
276 	STATS_DESC_COUNTER(VCPU, halt_exits),
277 	STATS_DESC_COUNTER(VCPU, request_irq_exits),
278 	STATS_DESC_COUNTER(VCPU, irq_exits),
279 	STATS_DESC_COUNTER(VCPU, host_state_reload),
280 	STATS_DESC_COUNTER(VCPU, fpu_reload),
281 	STATS_DESC_COUNTER(VCPU, insn_emulation),
282 	STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
283 	STATS_DESC_COUNTER(VCPU, hypercalls),
284 	STATS_DESC_COUNTER(VCPU, irq_injections),
285 	STATS_DESC_COUNTER(VCPU, nmi_injections),
286 	STATS_DESC_COUNTER(VCPU, req_event),
287 	STATS_DESC_COUNTER(VCPU, nested_run),
288 	STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
289 	STATS_DESC_COUNTER(VCPU, directed_yield_successful),
290 	STATS_DESC_ICOUNTER(VCPU, guest_mode)
291 };
292 
293 const struct kvm_stats_header kvm_vcpu_stats_header = {
294 	.name_size = KVM_STATS_NAME_SIZE,
295 	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
296 	.id_offset = sizeof(struct kvm_stats_header),
297 	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
298 	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
299 		       sizeof(kvm_vcpu_stats_desc),
300 };
301 
302 u64 __read_mostly host_xcr0;
303 u64 __read_mostly supported_xcr0;
304 EXPORT_SYMBOL_GPL(supported_xcr0);
305 
306 static struct kmem_cache *x86_emulator_cache;
307 
308 /*
309  * When called, it means the previous get/set msr reached an invalid msr.
310  * Return true if we want to ignore/silent this failed msr access.
311  */
312 static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)
313 {
314 	const char *op = write ? "wrmsr" : "rdmsr";
315 
316 	if (ignore_msrs) {
317 		if (report_ignored_msrs)
318 			kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n",
319 				      op, msr, data);
320 		/* Mask the error */
321 		return true;
322 	} else {
323 		kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
324 				      op, msr, data);
325 		return false;
326 	}
327 }
328 
329 static struct kmem_cache *kvm_alloc_emulator_cache(void)
330 {
331 	unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
332 	unsigned int size = sizeof(struct x86_emulate_ctxt);
333 
334 	return kmem_cache_create_usercopy("x86_emulator", size,
335 					  __alignof__(struct x86_emulate_ctxt),
336 					  SLAB_ACCOUNT, useroffset,
337 					  size - useroffset, NULL);
338 }
339 
340 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
341 
342 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
343 {
344 	int i;
345 	for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
346 		vcpu->arch.apf.gfns[i] = ~0;
347 }
348 
349 static void kvm_on_user_return(struct user_return_notifier *urn)
350 {
351 	unsigned slot;
352 	struct kvm_user_return_msrs *msrs
353 		= container_of(urn, struct kvm_user_return_msrs, urn);
354 	struct kvm_user_return_msr_values *values;
355 	unsigned long flags;
356 
357 	/*
358 	 * Disabling irqs at this point since the following code could be
359 	 * interrupted and executed through kvm_arch_hardware_disable()
360 	 */
361 	local_irq_save(flags);
362 	if (msrs->registered) {
363 		msrs->registered = false;
364 		user_return_notifier_unregister(urn);
365 	}
366 	local_irq_restore(flags);
367 	for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
368 		values = &msrs->values[slot];
369 		if (values->host != values->curr) {
370 			wrmsrl(kvm_uret_msrs_list[slot], values->host);
371 			values->curr = values->host;
372 		}
373 	}
374 }
375 
376 static int kvm_probe_user_return_msr(u32 msr)
377 {
378 	u64 val;
379 	int ret;
380 
381 	preempt_disable();
382 	ret = rdmsrl_safe(msr, &val);
383 	if (ret)
384 		goto out;
385 	ret = wrmsrl_safe(msr, val);
386 out:
387 	preempt_enable();
388 	return ret;
389 }
390 
391 int kvm_add_user_return_msr(u32 msr)
392 {
393 	BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);
394 
395 	if (kvm_probe_user_return_msr(msr))
396 		return -1;
397 
398 	kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
399 	return kvm_nr_uret_msrs++;
400 }
401 EXPORT_SYMBOL_GPL(kvm_add_user_return_msr);
402 
403 int kvm_find_user_return_msr(u32 msr)
404 {
405 	int i;
406 
407 	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
408 		if (kvm_uret_msrs_list[i] == msr)
409 			return i;
410 	}
411 	return -1;
412 }
413 EXPORT_SYMBOL_GPL(kvm_find_user_return_msr);
414 
415 static void kvm_user_return_msr_cpu_online(void)
416 {
417 	unsigned int cpu = smp_processor_id();
418 	struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
419 	u64 value;
420 	int i;
421 
422 	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
423 		rdmsrl_safe(kvm_uret_msrs_list[i], &value);
424 		msrs->values[i].host = value;
425 		msrs->values[i].curr = value;
426 	}
427 }
428 
429 int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
430 {
431 	unsigned int cpu = smp_processor_id();
432 	struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
433 	int err;
434 
435 	value = (value & mask) | (msrs->values[slot].host & ~mask);
436 	if (value == msrs->values[slot].curr)
437 		return 0;
438 	err = wrmsrl_safe(kvm_uret_msrs_list[slot], value);
439 	if (err)
440 		return 1;
441 
442 	msrs->values[slot].curr = value;
443 	if (!msrs->registered) {
444 		msrs->urn.on_user_return = kvm_on_user_return;
445 		user_return_notifier_register(&msrs->urn);
446 		msrs->registered = true;
447 	}
448 	return 0;
449 }
450 EXPORT_SYMBOL_GPL(kvm_set_user_return_msr);
451 
452 static void drop_user_return_notifiers(void)
453 {
454 	unsigned int cpu = smp_processor_id();
455 	struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
456 
457 	if (msrs->registered)
458 		kvm_on_user_return(&msrs->urn);
459 }
460 
461 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
462 {
463 	return vcpu->arch.apic_base;
464 }
465 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
466 
467 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
468 {
469 	return kvm_apic_mode(kvm_get_apic_base(vcpu));
470 }
471 EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
472 
473 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
474 {
475 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
476 	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
477 	u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |
478 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
479 
480 	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
481 		return 1;
482 	if (!msr_info->host_initiated) {
483 		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
484 			return 1;
485 		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
486 			return 1;
487 	}
488 
489 	kvm_lapic_set_base(vcpu, msr_info->data);
490 	kvm_recalculate_apic_map(vcpu->kvm);
491 	return 0;
492 }
493 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
494 
495 /*
496  * Handle a fault on a hardware virtualization (VMX or SVM) instruction.
497  *
498  * Hardware virtualization extension instructions may fault if a reboot turns
499  * off virtualization while processes are running.  Usually after catching the
500  * fault we just panic; during reboot instead the instruction is ignored.
501  */
502 noinstr void kvm_spurious_fault(void)
503 {
504 	/* Fault while not rebooting.  We want the trace. */
505 	BUG_ON(!kvm_rebooting);
506 }
507 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
508 
509 #define EXCPT_BENIGN		0
510 #define EXCPT_CONTRIBUTORY	1
511 #define EXCPT_PF		2
512 
513 static int exception_class(int vector)
514 {
515 	switch (vector) {
516 	case PF_VECTOR:
517 		return EXCPT_PF;
518 	case DE_VECTOR:
519 	case TS_VECTOR:
520 	case NP_VECTOR:
521 	case SS_VECTOR:
522 	case GP_VECTOR:
523 		return EXCPT_CONTRIBUTORY;
524 	default:
525 		break;
526 	}
527 	return EXCPT_BENIGN;
528 }
529 
530 #define EXCPT_FAULT		0
531 #define EXCPT_TRAP		1
532 #define EXCPT_ABORT		2
533 #define EXCPT_INTERRUPT		3
534 
535 static int exception_type(int vector)
536 {
537 	unsigned int mask;
538 
539 	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
540 		return EXCPT_INTERRUPT;
541 
542 	mask = 1 << vector;
543 
544 	/* #DB is trap, as instruction watchpoints are handled elsewhere */
545 	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
546 		return EXCPT_TRAP;
547 
548 	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
549 		return EXCPT_ABORT;
550 
551 	/* Reserved exceptions will result in fault */
552 	return EXCPT_FAULT;
553 }
554 
555 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
556 {
557 	unsigned nr = vcpu->arch.exception.nr;
558 	bool has_payload = vcpu->arch.exception.has_payload;
559 	unsigned long payload = vcpu->arch.exception.payload;
560 
561 	if (!has_payload)
562 		return;
563 
564 	switch (nr) {
565 	case DB_VECTOR:
566 		/*
567 		 * "Certain debug exceptions may clear bit 0-3.  The
568 		 * remaining contents of the DR6 register are never
569 		 * cleared by the processor".
570 		 */
571 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
572 		/*
573 		 * In order to reflect the #DB exception payload in guest
574 		 * dr6, three components need to be considered: active low
575 		 * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
576 		 * DR6_BS and DR6_BT)
577 		 * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
578 		 * In the target guest dr6:
579 		 * FIXED_1 bits should always be set.
580 		 * Active low bits should be cleared if 1-setting in payload.
581 		 * Active high bits should be set if 1-setting in payload.
582 		 *
583 		 * Note, the payload is compatible with the pending debug
584 		 * exceptions/exit qualification under VMX, that active_low bits
585 		 * are active high in payload.
586 		 * So they need to be flipped for DR6.
587 		 */
588 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
589 		vcpu->arch.dr6 |= payload;
590 		vcpu->arch.dr6 ^= payload & DR6_ACTIVE_LOW;
591 
592 		/*
593 		 * The #DB payload is defined as compatible with the 'pending
594 		 * debug exceptions' field under VMX, not DR6. While bit 12 is
595 		 * defined in the 'pending debug exceptions' field (enabled
596 		 * breakpoint), it is reserved and must be zero in DR6.
597 		 */
598 		vcpu->arch.dr6 &= ~BIT(12);
599 		break;
600 	case PF_VECTOR:
601 		vcpu->arch.cr2 = payload;
602 		break;
603 	}
604 
605 	vcpu->arch.exception.has_payload = false;
606 	vcpu->arch.exception.payload = 0;
607 }
608 EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);
609 
610 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
611 		unsigned nr, bool has_error, u32 error_code,
612 	        bool has_payload, unsigned long payload, bool reinject)
613 {
614 	u32 prev_nr;
615 	int class1, class2;
616 
617 	kvm_make_request(KVM_REQ_EVENT, vcpu);
618 
619 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
620 	queue:
621 		if (reinject) {
622 			/*
623 			 * On vmentry, vcpu->arch.exception.pending is only
624 			 * true if an event injection was blocked by
625 			 * nested_run_pending.  In that case, however,
626 			 * vcpu_enter_guest requests an immediate exit,
627 			 * and the guest shouldn't proceed far enough to
628 			 * need reinjection.
629 			 */
630 			WARN_ON_ONCE(vcpu->arch.exception.pending);
631 			vcpu->arch.exception.injected = true;
632 			if (WARN_ON_ONCE(has_payload)) {
633 				/*
634 				 * A reinjected event has already
635 				 * delivered its payload.
636 				 */
637 				has_payload = false;
638 				payload = 0;
639 			}
640 		} else {
641 			vcpu->arch.exception.pending = true;
642 			vcpu->arch.exception.injected = false;
643 		}
644 		vcpu->arch.exception.has_error_code = has_error;
645 		vcpu->arch.exception.nr = nr;
646 		vcpu->arch.exception.error_code = error_code;
647 		vcpu->arch.exception.has_payload = has_payload;
648 		vcpu->arch.exception.payload = payload;
649 		if (!is_guest_mode(vcpu))
650 			kvm_deliver_exception_payload(vcpu);
651 		return;
652 	}
653 
654 	/* to check exception */
655 	prev_nr = vcpu->arch.exception.nr;
656 	if (prev_nr == DF_VECTOR) {
657 		/* triple fault -> shutdown */
658 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
659 		return;
660 	}
661 	class1 = exception_class(prev_nr);
662 	class2 = exception_class(nr);
663 	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
664 		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
665 		/*
666 		 * Generate double fault per SDM Table 5-5.  Set
667 		 * exception.pending = true so that the double fault
668 		 * can trigger a nested vmexit.
669 		 */
670 		vcpu->arch.exception.pending = true;
671 		vcpu->arch.exception.injected = false;
672 		vcpu->arch.exception.has_error_code = true;
673 		vcpu->arch.exception.nr = DF_VECTOR;
674 		vcpu->arch.exception.error_code = 0;
675 		vcpu->arch.exception.has_payload = false;
676 		vcpu->arch.exception.payload = 0;
677 	} else
678 		/* replace previous exception with a new one in a hope
679 		   that instruction re-execution will regenerate lost
680 		   exception */
681 		goto queue;
682 }
683 
684 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
685 {
686 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
687 }
688 EXPORT_SYMBOL_GPL(kvm_queue_exception);
689 
690 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
691 {
692 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
693 }
694 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
695 
696 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
697 			   unsigned long payload)
698 {
699 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
700 }
701 EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
702 
703 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
704 				    u32 error_code, unsigned long payload)
705 {
706 	kvm_multiple_exception(vcpu, nr, true, error_code,
707 			       true, payload, false);
708 }
709 
710 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
711 {
712 	if (err)
713 		kvm_inject_gp(vcpu, 0);
714 	else
715 		return kvm_skip_emulated_instruction(vcpu);
716 
717 	return 1;
718 }
719 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
720 
721 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
722 {
723 	if (err) {
724 		kvm_inject_gp(vcpu, 0);
725 		return 1;
726 	}
727 
728 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
729 				       EMULTYPE_COMPLETE_USER_EXIT);
730 }
731 
732 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
733 {
734 	++vcpu->stat.pf_guest;
735 	vcpu->arch.exception.nested_apf =
736 		is_guest_mode(vcpu) && fault->async_page_fault;
737 	if (vcpu->arch.exception.nested_apf) {
738 		vcpu->arch.apf.nested_apf_token = fault->address;
739 		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
740 	} else {
741 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
742 					fault->address);
743 	}
744 }
745 EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
746 
747 bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
748 				    struct x86_exception *fault)
749 {
750 	struct kvm_mmu *fault_mmu;
751 	WARN_ON_ONCE(fault->vector != PF_VECTOR);
752 
753 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
754 					       vcpu->arch.walk_mmu;
755 
756 	/*
757 	 * Invalidate the TLB entry for the faulting address, if it exists,
758 	 * else the access will fault indefinitely (and to emulate hardware).
759 	 */
760 	if ((fault->error_code & PFERR_PRESENT_MASK) &&
761 	    !(fault->error_code & PFERR_RSVD_MASK))
762 		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
763 				       fault_mmu->root_hpa);
764 
765 	fault_mmu->inject_page_fault(vcpu, fault);
766 	return fault->nested_page_fault;
767 }
768 EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);
769 
770 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
771 {
772 	atomic_inc(&vcpu->arch.nmi_queued);
773 	kvm_make_request(KVM_REQ_NMI, vcpu);
774 }
775 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
776 
777 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
778 {
779 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
780 }
781 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
782 
783 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
784 {
785 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
786 }
787 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
788 
789 /*
790  * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
791  * a #GP and return false.
792  */
793 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
794 {
795 	if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
796 		return true;
797 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
798 	return false;
799 }
800 EXPORT_SYMBOL_GPL(kvm_require_cpl);
801 
802 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
803 {
804 	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
805 		return true;
806 
807 	kvm_queue_exception(vcpu, UD_VECTOR);
808 	return false;
809 }
810 EXPORT_SYMBOL_GPL(kvm_require_dr);
811 
812 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
813 {
814 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
815 }
816 
817 /*
818  * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
819  */
820 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
821 {
822 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
823 	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
824 	gpa_t real_gpa;
825 	int i;
826 	int ret;
827 	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
828 
829 	/*
830 	 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
831 	 * to an L1 GPA.
832 	 */
833 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
834 				     PFERR_USER_MASK | PFERR_WRITE_MASK, NULL);
835 	if (real_gpa == UNMAPPED_GVA)
836 		return 0;
837 
838 	/* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */
839 	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
840 				       cr3 & GENMASK(11, 5), sizeof(pdpte));
841 	if (ret < 0)
842 		return 0;
843 
844 	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
845 		if ((pdpte[i] & PT_PRESENT_MASK) &&
846 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
847 			return 0;
848 		}
849 	}
850 
851 	/*
852 	 * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled.
853 	 * Shadow page roots need to be reconstructed instead.
854 	 */
855 	if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)))
856 		kvm_mmu_free_roots(vcpu, mmu, KVM_MMU_ROOT_CURRENT);
857 
858 	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
859 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
860 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
861 	vcpu->arch.pdptrs_from_userspace = false;
862 
863 	return 1;
864 }
865 EXPORT_SYMBOL_GPL(load_pdptrs);
866 
867 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
868 {
869 	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
870 		kvm_clear_async_pf_completion_queue(vcpu);
871 		kvm_async_pf_hash_reset(vcpu);
872 	}
873 
874 	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
875 		kvm_mmu_reset_context(vcpu);
876 
877 	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
878 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
879 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
880 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
881 }
882 EXPORT_SYMBOL_GPL(kvm_post_set_cr0);
883 
884 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
885 {
886 	unsigned long old_cr0 = kvm_read_cr0(vcpu);
887 
888 	cr0 |= X86_CR0_ET;
889 
890 #ifdef CONFIG_X86_64
891 	if (cr0 & 0xffffffff00000000UL)
892 		return 1;
893 #endif
894 
895 	cr0 &= ~CR0_RESERVED_BITS;
896 
897 	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
898 		return 1;
899 
900 	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
901 		return 1;
902 
903 #ifdef CONFIG_X86_64
904 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
905 	    (cr0 & X86_CR0_PG)) {
906 		int cs_db, cs_l;
907 
908 		if (!is_pae(vcpu))
909 			return 1;
910 		static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
911 		if (cs_l)
912 			return 1;
913 	}
914 #endif
915 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
916 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
917 	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
918 		return 1;
919 
920 	if (!(cr0 & X86_CR0_PG) &&
921 	    (is_64_bit_mode(vcpu) || kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)))
922 		return 1;
923 
924 	static_call(kvm_x86_set_cr0)(vcpu, cr0);
925 
926 	kvm_post_set_cr0(vcpu, old_cr0, cr0);
927 
928 	return 0;
929 }
930 EXPORT_SYMBOL_GPL(kvm_set_cr0);
931 
932 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
933 {
934 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
935 }
936 EXPORT_SYMBOL_GPL(kvm_lmsw);
937 
938 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
939 {
940 	if (vcpu->arch.guest_state_protected)
941 		return;
942 
943 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
944 
945 		if (vcpu->arch.xcr0 != host_xcr0)
946 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
947 
948 		if (vcpu->arch.xsaves_enabled &&
949 		    vcpu->arch.ia32_xss != host_xss)
950 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
951 	}
952 
953 	if (static_cpu_has(X86_FEATURE_PKU) &&
954 	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
955 	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&
956 	    vcpu->arch.pkru != vcpu->arch.host_pkru)
957 		write_pkru(vcpu->arch.pkru);
958 }
959 EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
960 
961 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
962 {
963 	if (vcpu->arch.guest_state_protected)
964 		return;
965 
966 	if (static_cpu_has(X86_FEATURE_PKU) &&
967 	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
968 	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {
969 		vcpu->arch.pkru = rdpkru();
970 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)
971 			write_pkru(vcpu->arch.host_pkru);
972 	}
973 
974 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
975 
976 		if (vcpu->arch.xcr0 != host_xcr0)
977 			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
978 
979 		if (vcpu->arch.xsaves_enabled &&
980 		    vcpu->arch.ia32_xss != host_xss)
981 			wrmsrl(MSR_IA32_XSS, host_xss);
982 	}
983 
984 }
985 EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
986 
987 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
988 {
989 	u64 xcr0 = xcr;
990 	u64 old_xcr0 = vcpu->arch.xcr0;
991 	u64 valid_bits;
992 
993 	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
994 	if (index != XCR_XFEATURE_ENABLED_MASK)
995 		return 1;
996 	if (!(xcr0 & XFEATURE_MASK_FP))
997 		return 1;
998 	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
999 		return 1;
1000 
1001 	/*
1002 	 * Do not allow the guest to set bits that we do not support
1003 	 * saving.  However, xcr0 bit 0 is always set, even if the
1004 	 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()).
1005 	 */
1006 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
1007 	if (xcr0 & ~valid_bits)
1008 		return 1;
1009 
1010 	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
1011 	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
1012 		return 1;
1013 
1014 	if (xcr0 & XFEATURE_MASK_AVX512) {
1015 		if (!(xcr0 & XFEATURE_MASK_YMM))
1016 			return 1;
1017 		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
1018 			return 1;
1019 	}
1020 
1021 	if ((xcr0 & XFEATURE_MASK_XTILE) &&
1022 	    ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE))
1023 		return 1;
1024 
1025 	vcpu->arch.xcr0 = xcr0;
1026 
1027 	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
1028 		kvm_update_cpuid_runtime(vcpu);
1029 	return 0;
1030 }
1031 
1032 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
1033 {
1034 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||
1035 	    __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
1036 		kvm_inject_gp(vcpu, 0);
1037 		return 1;
1038 	}
1039 
1040 	return kvm_skip_emulated_instruction(vcpu);
1041 }
1042 EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv);
1043 
1044 bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1045 {
1046 	if (cr4 & cr4_reserved_bits)
1047 		return false;
1048 
1049 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
1050 		return false;
1051 
1052 	return static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
1053 }
1054 EXPORT_SYMBOL_GPL(kvm_is_valid_cr4);
1055 
1056 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
1057 {
1058 	/*
1059 	 * If any role bit is changed, the MMU needs to be reset.
1060 	 *
1061 	 * If CR4.PCIDE is changed 1 -> 0, the guest TLB must be flushed.
1062 	 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB
1063 	 * according to the SDM; however, stale prev_roots could be reused
1064 	 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we
1065 	 * free them all.  KVM_REQ_MMU_RELOAD is fit for the both cases; it
1066 	 * is slow, but changing CR4.PCIDE is a rare case.
1067 	 *
1068 	 * If CR4.PGE is changed, the guest TLB must be flushed.
1069 	 *
1070 	 * Note: resetting MMU is a superset of KVM_REQ_MMU_RELOAD and
1071 	 * KVM_REQ_MMU_RELOAD is a superset of KVM_REQ_TLB_FLUSH_GUEST, hence
1072 	 * the usage of "else if".
1073 	 */
1074 	if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS)
1075 		kvm_mmu_reset_context(vcpu);
1076 	else if ((cr4 ^ old_cr4) & X86_CR4_PCIDE)
1077 		kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1078 	else if ((cr4 ^ old_cr4) & X86_CR4_PGE)
1079 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1080 }
1081 EXPORT_SYMBOL_GPL(kvm_post_set_cr4);
1082 
1083 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1084 {
1085 	unsigned long old_cr4 = kvm_read_cr4(vcpu);
1086 
1087 	if (!kvm_is_valid_cr4(vcpu, cr4))
1088 		return 1;
1089 
1090 	if (is_long_mode(vcpu)) {
1091 		if (!(cr4 & X86_CR4_PAE))
1092 			return 1;
1093 		if ((cr4 ^ old_cr4) & X86_CR4_LA57)
1094 			return 1;
1095 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
1096 		   && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS)
1097 		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
1098 		return 1;
1099 
1100 	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
1101 		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
1102 			return 1;
1103 
1104 		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
1105 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
1106 			return 1;
1107 	}
1108 
1109 	static_call(kvm_x86_set_cr4)(vcpu, cr4);
1110 
1111 	kvm_post_set_cr4(vcpu, old_cr4, cr4);
1112 
1113 	return 0;
1114 }
1115 EXPORT_SYMBOL_GPL(kvm_set_cr4);
1116 
1117 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
1118 {
1119 	struct kvm_mmu *mmu = vcpu->arch.mmu;
1120 	unsigned long roots_to_free = 0;
1121 	int i;
1122 
1123 	/*
1124 	 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
1125 	 * this is reachable when running EPT=1 and unrestricted_guest=0,  and
1126 	 * also via the emulator.  KVM's TDP page tables are not in the scope of
1127 	 * the invalidation, but the guest's TLB entries need to be flushed as
1128 	 * the CPU may have cached entries in its TLB for the target PCID.
1129 	 */
1130 	if (unlikely(tdp_enabled)) {
1131 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1132 		return;
1133 	}
1134 
1135 	/*
1136 	 * If neither the current CR3 nor any of the prev_roots use the given
1137 	 * PCID, then nothing needs to be done here because a resync will
1138 	 * happen anyway before switching to any other CR3.
1139 	 */
1140 	if (kvm_get_active_pcid(vcpu) == pcid) {
1141 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1142 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1143 	}
1144 
1145 	/*
1146 	 * If PCID is disabled, there is no need to free prev_roots even if the
1147 	 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB
1148 	 * with PCIDE=0.
1149 	 */
1150 	if (!kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
1151 		return;
1152 
1153 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
1154 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)
1155 			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
1156 
1157 	kvm_mmu_free_roots(vcpu, mmu, roots_to_free);
1158 }
1159 
1160 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1161 {
1162 	bool skip_tlb_flush = false;
1163 	unsigned long pcid = 0;
1164 #ifdef CONFIG_X86_64
1165 	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
1166 
1167 	if (pcid_enabled) {
1168 		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
1169 		cr3 &= ~X86_CR3_PCID_NOFLUSH;
1170 		pcid = cr3 & X86_CR3_PCID_MASK;
1171 	}
1172 #endif
1173 
1174 	/* PDPTRs are always reloaded for PAE paging. */
1175 	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
1176 		goto handle_tlb_flush;
1177 
1178 	/*
1179 	 * Do not condition the GPA check on long mode, this helper is used to
1180 	 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
1181 	 * the current vCPU mode is accurate.
1182 	 */
1183 	if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
1184 		return 1;
1185 
1186 	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
1187 		return 1;
1188 
1189 	if (cr3 != kvm_read_cr3(vcpu))
1190 		kvm_mmu_new_pgd(vcpu, cr3);
1191 
1192 	vcpu->arch.cr3 = cr3;
1193 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
1194 	/* Do not call post_set_cr3, we do not get here for confidential guests.  */
1195 
1196 handle_tlb_flush:
1197 	/*
1198 	 * A load of CR3 that flushes the TLB flushes only the current PCID,
1199 	 * even if PCID is disabled, in which case PCID=0 is flushed.  It's a
1200 	 * moot point in the end because _disabling_ PCID will flush all PCIDs,
1201 	 * and it's impossible to use a non-zero PCID when PCID is disabled,
1202 	 * i.e. only PCID=0 can be relevant.
1203 	 */
1204 	if (!skip_tlb_flush)
1205 		kvm_invalidate_pcid(vcpu, pcid);
1206 
1207 	return 0;
1208 }
1209 EXPORT_SYMBOL_GPL(kvm_set_cr3);
1210 
1211 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1212 {
1213 	if (cr8 & CR8_RESERVED_BITS)
1214 		return 1;
1215 	if (lapic_in_kernel(vcpu))
1216 		kvm_lapic_set_tpr(vcpu, cr8);
1217 	else
1218 		vcpu->arch.cr8 = cr8;
1219 	return 0;
1220 }
1221 EXPORT_SYMBOL_GPL(kvm_set_cr8);
1222 
1223 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1224 {
1225 	if (lapic_in_kernel(vcpu))
1226 		return kvm_lapic_get_cr8(vcpu);
1227 	else
1228 		return vcpu->arch.cr8;
1229 }
1230 EXPORT_SYMBOL_GPL(kvm_get_cr8);
1231 
1232 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1233 {
1234 	int i;
1235 
1236 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1237 		for (i = 0; i < KVM_NR_DB_REGS; i++)
1238 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1239 	}
1240 }
1241 
1242 void kvm_update_dr7(struct kvm_vcpu *vcpu)
1243 {
1244 	unsigned long dr7;
1245 
1246 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1247 		dr7 = vcpu->arch.guest_debug_dr7;
1248 	else
1249 		dr7 = vcpu->arch.dr7;
1250 	static_call(kvm_x86_set_dr7)(vcpu, dr7);
1251 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1252 	if (dr7 & DR7_BP_EN_MASK)
1253 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1254 }
1255 EXPORT_SYMBOL_GPL(kvm_update_dr7);
1256 
1257 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1258 {
1259 	u64 fixed = DR6_FIXED_1;
1260 
1261 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1262 		fixed |= DR6_RTM;
1263 
1264 	if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
1265 		fixed |= DR6_BUS_LOCK;
1266 	return fixed;
1267 }
1268 
1269 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1270 {
1271 	size_t size = ARRAY_SIZE(vcpu->arch.db);
1272 
1273 	switch (dr) {
1274 	case 0 ... 3:
1275 		vcpu->arch.db[array_index_nospec(dr, size)] = val;
1276 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1277 			vcpu->arch.eff_db[dr] = val;
1278 		break;
1279 	case 4:
1280 	case 6:
1281 		if (!kvm_dr6_valid(val))
1282 			return 1; /* #GP */
1283 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1284 		break;
1285 	case 5:
1286 	default: /* 7 */
1287 		if (!kvm_dr7_valid(val))
1288 			return 1; /* #GP */
1289 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1290 		kvm_update_dr7(vcpu);
1291 		break;
1292 	}
1293 
1294 	return 0;
1295 }
1296 EXPORT_SYMBOL_GPL(kvm_set_dr);
1297 
1298 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1299 {
1300 	size_t size = ARRAY_SIZE(vcpu->arch.db);
1301 
1302 	switch (dr) {
1303 	case 0 ... 3:
1304 		*val = vcpu->arch.db[array_index_nospec(dr, size)];
1305 		break;
1306 	case 4:
1307 	case 6:
1308 		*val = vcpu->arch.dr6;
1309 		break;
1310 	case 5:
1311 	default: /* 7 */
1312 		*val = vcpu->arch.dr7;
1313 		break;
1314 	}
1315 }
1316 EXPORT_SYMBOL_GPL(kvm_get_dr);
1317 
1318 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
1319 {
1320 	u32 ecx = kvm_rcx_read(vcpu);
1321 	u64 data;
1322 
1323 	if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {
1324 		kvm_inject_gp(vcpu, 0);
1325 		return 1;
1326 	}
1327 
1328 	kvm_rax_write(vcpu, (u32)data);
1329 	kvm_rdx_write(vcpu, data >> 32);
1330 	return kvm_skip_emulated_instruction(vcpu);
1331 }
1332 EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc);
1333 
1334 /*
1335  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1336  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1337  *
1338  * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
1339  * extract the supported MSRs from the related const lists.
1340  * msrs_to_save is selected from the msrs_to_save_all to reflect the
1341  * capabilities of the host cpu. This capabilities test skips MSRs that are
1342  * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1343  * may depend on host virtualization features rather than host cpu features.
1344  */
1345 
1346 static const u32 msrs_to_save_all[] = {
1347 	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1348 	MSR_STAR,
1349 #ifdef CONFIG_X86_64
1350 	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1351 #endif
1352 	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1353 	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1354 	MSR_IA32_SPEC_CTRL,
1355 	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
1356 	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
1357 	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
1358 	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
1359 	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
1360 	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1361 	MSR_IA32_UMWAIT_CONTROL,
1362 
1363 	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
1364 	MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
1365 	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
1366 	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
1367 	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
1368 	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
1369 	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
1370 	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
1371 	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
1372 	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
1373 	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
1374 	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
1375 	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
1376 	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
1377 	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1378 	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1379 	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1380 	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
1381 	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
1382 	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
1383 	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
1384 	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1385 
1386 	MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
1387 	MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
1388 	MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
1389 	MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
1390 	MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
1391 	MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
1392 	MSR_IA32_XFD, MSR_IA32_XFD_ERR,
1393 };
1394 
1395 static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1396 static unsigned num_msrs_to_save;
1397 
1398 static const u32 emulated_msrs_all[] = {
1399 	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
1400 	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
1401 	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
1402 	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1403 	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1404 	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
1405 	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1406 	HV_X64_MSR_RESET,
1407 	HV_X64_MSR_VP_INDEX,
1408 	HV_X64_MSR_VP_RUNTIME,
1409 	HV_X64_MSR_SCONTROL,
1410 	HV_X64_MSR_STIMER0_CONFIG,
1411 	HV_X64_MSR_VP_ASSIST_PAGE,
1412 	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
1413 	HV_X64_MSR_TSC_EMULATION_STATUS,
1414 	HV_X64_MSR_SYNDBG_OPTIONS,
1415 	HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
1416 	HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
1417 	HV_X64_MSR_SYNDBG_PENDING_BUFFER,
1418 
1419 	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1420 	MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
1421 
1422 	MSR_IA32_TSC_ADJUST,
1423 	MSR_IA32_TSC_DEADLINE,
1424 	MSR_IA32_ARCH_CAPABILITIES,
1425 	MSR_IA32_PERF_CAPABILITIES,
1426 	MSR_IA32_MISC_ENABLE,
1427 	MSR_IA32_MCG_STATUS,
1428 	MSR_IA32_MCG_CTL,
1429 	MSR_IA32_MCG_EXT_CTL,
1430 	MSR_IA32_SMBASE,
1431 	MSR_SMI_COUNT,
1432 	MSR_PLATFORM_INFO,
1433 	MSR_MISC_FEATURES_ENABLES,
1434 	MSR_AMD64_VIRT_SPEC_CTRL,
1435 	MSR_AMD64_TSC_RATIO,
1436 	MSR_IA32_POWER_CTL,
1437 	MSR_IA32_UCODE_REV,
1438 
1439 	/*
1440 	 * The following list leaves out MSRs whose values are determined
1441 	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
1442 	 * We always support the "true" VMX control MSRs, even if the host
1443 	 * processor does not, so I am putting these registers here rather
1444 	 * than in msrs_to_save_all.
1445 	 */
1446 	MSR_IA32_VMX_BASIC,
1447 	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1448 	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1449 	MSR_IA32_VMX_TRUE_EXIT_CTLS,
1450 	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1451 	MSR_IA32_VMX_MISC,
1452 	MSR_IA32_VMX_CR0_FIXED0,
1453 	MSR_IA32_VMX_CR4_FIXED0,
1454 	MSR_IA32_VMX_VMCS_ENUM,
1455 	MSR_IA32_VMX_PROCBASED_CTLS2,
1456 	MSR_IA32_VMX_EPT_VPID_CAP,
1457 	MSR_IA32_VMX_VMFUNC,
1458 
1459 	MSR_K7_HWCR,
1460 	MSR_KVM_POLL_CONTROL,
1461 };
1462 
1463 static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1464 static unsigned num_emulated_msrs;
1465 
1466 /*
1467  * List of msr numbers which are used to expose MSR-based features that
1468  * can be used by a hypervisor to validate requested CPU features.
1469  */
1470 static const u32 msr_based_features_all[] = {
1471 	MSR_IA32_VMX_BASIC,
1472 	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1473 	MSR_IA32_VMX_PINBASED_CTLS,
1474 	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1475 	MSR_IA32_VMX_PROCBASED_CTLS,
1476 	MSR_IA32_VMX_TRUE_EXIT_CTLS,
1477 	MSR_IA32_VMX_EXIT_CTLS,
1478 	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1479 	MSR_IA32_VMX_ENTRY_CTLS,
1480 	MSR_IA32_VMX_MISC,
1481 	MSR_IA32_VMX_CR0_FIXED0,
1482 	MSR_IA32_VMX_CR0_FIXED1,
1483 	MSR_IA32_VMX_CR4_FIXED0,
1484 	MSR_IA32_VMX_CR4_FIXED1,
1485 	MSR_IA32_VMX_VMCS_ENUM,
1486 	MSR_IA32_VMX_PROCBASED_CTLS2,
1487 	MSR_IA32_VMX_EPT_VPID_CAP,
1488 	MSR_IA32_VMX_VMFUNC,
1489 
1490 	MSR_F10H_DECFG,
1491 	MSR_IA32_UCODE_REV,
1492 	MSR_IA32_ARCH_CAPABILITIES,
1493 	MSR_IA32_PERF_CAPABILITIES,
1494 };
1495 
1496 static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1497 static unsigned int num_msr_based_features;
1498 
1499 static u64 kvm_get_arch_capabilities(void)
1500 {
1501 	u64 data = 0;
1502 
1503 	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1504 		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1505 
1506 	/*
1507 	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
1508 	 * the nested hypervisor runs with NX huge pages.  If it is not,
1509 	 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
1510 	 * L1 guests, so it need not worry about its own (L2) guests.
1511 	 */
1512 	data |= ARCH_CAP_PSCHANGE_MC_NO;
1513 
1514 	/*
1515 	 * If we're doing cache flushes (either "always" or "cond")
1516 	 * we will do one whenever the guest does a vmlaunch/vmresume.
1517 	 * If an outer hypervisor is doing the cache flush for us
1518 	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
1519 	 * capability to the guest too, and if EPT is disabled we're not
1520 	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
1521 	 * require a nested hypervisor to do a flush of its own.
1522 	 */
1523 	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1524 		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1525 
1526 	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
1527 		data |= ARCH_CAP_RDCL_NO;
1528 	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1529 		data |= ARCH_CAP_SSB_NO;
1530 	if (!boot_cpu_has_bug(X86_BUG_MDS))
1531 		data |= ARCH_CAP_MDS_NO;
1532 
1533 	if (!boot_cpu_has(X86_FEATURE_RTM)) {
1534 		/*
1535 		 * If RTM=0 because the kernel has disabled TSX, the host might
1536 		 * have TAA_NO or TSX_CTRL.  Clear TAA_NO (the guest sees RTM=0
1537 		 * and therefore knows that there cannot be TAA) but keep
1538 		 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
1539 		 * and we want to allow migrating those guests to tsx=off hosts.
1540 		 */
1541 		data &= ~ARCH_CAP_TAA_NO;
1542 	} else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
1543 		data |= ARCH_CAP_TAA_NO;
1544 	} else {
1545 		/*
1546 		 * Nothing to do here; we emulate TSX_CTRL if present on the
1547 		 * host so the guest can choose between disabling TSX or
1548 		 * using VERW to clear CPU buffers.
1549 		 */
1550 	}
1551 
1552 	return data;
1553 }
1554 
1555 static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
1556 {
1557 	switch (msr->index) {
1558 	case MSR_IA32_ARCH_CAPABILITIES:
1559 		msr->data = kvm_get_arch_capabilities();
1560 		break;
1561 	case MSR_IA32_UCODE_REV:
1562 		rdmsrl_safe(msr->index, &msr->data);
1563 		break;
1564 	default:
1565 		return static_call(kvm_x86_get_msr_feature)(msr);
1566 	}
1567 	return 0;
1568 }
1569 
1570 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1571 {
1572 	struct kvm_msr_entry msr;
1573 	int r;
1574 
1575 	msr.index = index;
1576 	r = kvm_get_msr_feature(&msr);
1577 
1578 	if (r == KVM_MSR_RET_INVALID) {
1579 		/* Unconditionally clear the output for simplicity */
1580 		*data = 0;
1581 		if (kvm_msr_ignored_check(index, 0, false))
1582 			r = 0;
1583 	}
1584 
1585 	if (r)
1586 		return r;
1587 
1588 	*data = msr.data;
1589 
1590 	return 0;
1591 }
1592 
1593 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1594 {
1595 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1596 		return false;
1597 
1598 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1599 		return false;
1600 
1601 	if (efer & (EFER_LME | EFER_LMA) &&
1602 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1603 		return false;
1604 
1605 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1606 		return false;
1607 
1608 	return true;
1609 
1610 }
1611 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1612 {
1613 	if (efer & efer_reserved_bits)
1614 		return false;
1615 
1616 	return __kvm_valid_efer(vcpu, efer);
1617 }
1618 EXPORT_SYMBOL_GPL(kvm_valid_efer);
1619 
1620 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1621 {
1622 	u64 old_efer = vcpu->arch.efer;
1623 	u64 efer = msr_info->data;
1624 	int r;
1625 
1626 	if (efer & efer_reserved_bits)
1627 		return 1;
1628 
1629 	if (!msr_info->host_initiated) {
1630 		if (!__kvm_valid_efer(vcpu, efer))
1631 			return 1;
1632 
1633 		if (is_paging(vcpu) &&
1634 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1635 			return 1;
1636 	}
1637 
1638 	efer &= ~EFER_LMA;
1639 	efer |= vcpu->arch.efer & EFER_LMA;
1640 
1641 	r = static_call(kvm_x86_set_efer)(vcpu, efer);
1642 	if (r) {
1643 		WARN_ON(r > 0);
1644 		return r;
1645 	}
1646 
1647 	/* Update reserved bits */
1648 	if ((efer ^ old_efer) & EFER_NX)
1649 		kvm_mmu_reset_context(vcpu);
1650 
1651 	return 0;
1652 }
1653 
1654 void kvm_enable_efer_bits(u64 mask)
1655 {
1656        efer_reserved_bits &= ~mask;
1657 }
1658 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1659 
1660 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
1661 {
1662 	struct kvm_x86_msr_filter *msr_filter;
1663 	struct msr_bitmap_range *ranges;
1664 	struct kvm *kvm = vcpu->kvm;
1665 	bool allowed;
1666 	int idx;
1667 	u32 i;
1668 
1669 	/* x2APIC MSRs do not support filtering. */
1670 	if (index >= 0x800 && index <= 0x8ff)
1671 		return true;
1672 
1673 	idx = srcu_read_lock(&kvm->srcu);
1674 
1675 	msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
1676 	if (!msr_filter) {
1677 		allowed = true;
1678 		goto out;
1679 	}
1680 
1681 	allowed = msr_filter->default_allow;
1682 	ranges = msr_filter->ranges;
1683 
1684 	for (i = 0; i < msr_filter->count; i++) {
1685 		u32 start = ranges[i].base;
1686 		u32 end = start + ranges[i].nmsrs;
1687 		u32 flags = ranges[i].flags;
1688 		unsigned long *bitmap = ranges[i].bitmap;
1689 
1690 		if ((index >= start) && (index < end) && (flags & type)) {
1691 			allowed = !!test_bit(index - start, bitmap);
1692 			break;
1693 		}
1694 	}
1695 
1696 out:
1697 	srcu_read_unlock(&kvm->srcu, idx);
1698 
1699 	return allowed;
1700 }
1701 EXPORT_SYMBOL_GPL(kvm_msr_allowed);
1702 
1703 /*
1704  * Write @data into the MSR specified by @index.  Select MSR specific fault
1705  * checks are bypassed if @host_initiated is %true.
1706  * Returns 0 on success, non-0 otherwise.
1707  * Assumes vcpu_load() was already called.
1708  */
1709 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1710 			 bool host_initiated)
1711 {
1712 	struct msr_data msr;
1713 
1714 	if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
1715 		return KVM_MSR_RET_FILTERED;
1716 
1717 	switch (index) {
1718 	case MSR_FS_BASE:
1719 	case MSR_GS_BASE:
1720 	case MSR_KERNEL_GS_BASE:
1721 	case MSR_CSTAR:
1722 	case MSR_LSTAR:
1723 		if (is_noncanonical_address(data, vcpu))
1724 			return 1;
1725 		break;
1726 	case MSR_IA32_SYSENTER_EIP:
1727 	case MSR_IA32_SYSENTER_ESP:
1728 		/*
1729 		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
1730 		 * non-canonical address is written on Intel but not on
1731 		 * AMD (which ignores the top 32-bits, because it does
1732 		 * not implement 64-bit SYSENTER).
1733 		 *
1734 		 * 64-bit code should hence be able to write a non-canonical
1735 		 * value on AMD.  Making the address canonical ensures that
1736 		 * vmentry does not fail on Intel after writing a non-canonical
1737 		 * value, and that something deterministic happens if the guest
1738 		 * invokes 64-bit SYSENTER.
1739 		 */
1740 		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1741 		break;
1742 	case MSR_TSC_AUX:
1743 		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1744 			return 1;
1745 
1746 		if (!host_initiated &&
1747 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1748 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1749 			return 1;
1750 
1751 		/*
1752 		 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
1753 		 * incomplete and conflicting architectural behavior.  Current
1754 		 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
1755 		 * reserved and always read as zeros.  Enforce Intel's reserved
1756 		 * bits check if and only if the guest CPU is Intel, and clear
1757 		 * the bits in all other cases.  This ensures cross-vendor
1758 		 * migration will provide consistent behavior for the guest.
1759 		 */
1760 		if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)
1761 			return 1;
1762 
1763 		data = (u32)data;
1764 		break;
1765 	}
1766 
1767 	msr.data = data;
1768 	msr.index = index;
1769 	msr.host_initiated = host_initiated;
1770 
1771 	return static_call(kvm_x86_set_msr)(vcpu, &msr);
1772 }
1773 
1774 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
1775 				     u32 index, u64 data, bool host_initiated)
1776 {
1777 	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);
1778 
1779 	if (ret == KVM_MSR_RET_INVALID)
1780 		if (kvm_msr_ignored_check(index, data, true))
1781 			ret = 0;
1782 
1783 	return ret;
1784 }
1785 
1786 /*
1787  * Read the MSR specified by @index into @data.  Select MSR specific fault
1788  * checks are bypassed if @host_initiated is %true.
1789  * Returns 0 on success, non-0 otherwise.
1790  * Assumes vcpu_load() was already called.
1791  */
1792 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1793 		  bool host_initiated)
1794 {
1795 	struct msr_data msr;
1796 	int ret;
1797 
1798 	if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
1799 		return KVM_MSR_RET_FILTERED;
1800 
1801 	switch (index) {
1802 	case MSR_TSC_AUX:
1803 		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1804 			return 1;
1805 
1806 		if (!host_initiated &&
1807 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1808 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1809 			return 1;
1810 		break;
1811 	}
1812 
1813 	msr.index = index;
1814 	msr.host_initiated = host_initiated;
1815 
1816 	ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
1817 	if (!ret)
1818 		*data = msr.data;
1819 	return ret;
1820 }
1821 
1822 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
1823 				     u32 index, u64 *data, bool host_initiated)
1824 {
1825 	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);
1826 
1827 	if (ret == KVM_MSR_RET_INVALID) {
1828 		/* Unconditionally clear *data for simplicity */
1829 		*data = 0;
1830 		if (kvm_msr_ignored_check(index, 0, false))
1831 			ret = 0;
1832 	}
1833 
1834 	return ret;
1835 }
1836 
1837 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1838 {
1839 	return kvm_get_msr_ignored_check(vcpu, index, data, false);
1840 }
1841 EXPORT_SYMBOL_GPL(kvm_get_msr);
1842 
1843 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1844 {
1845 	return kvm_set_msr_ignored_check(vcpu, index, data, false);
1846 }
1847 EXPORT_SYMBOL_GPL(kvm_set_msr);
1848 
1849 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
1850 {
1851 	if (!vcpu->run->msr.error) {
1852 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
1853 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1854 	}
1855 }
1856 
1857 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
1858 {
1859 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
1860 }
1861 
1862 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
1863 {
1864 	complete_userspace_rdmsr(vcpu);
1865 	return complete_emulated_msr_access(vcpu);
1866 }
1867 
1868 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
1869 {
1870 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
1871 }
1872 
1873 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
1874 {
1875 	complete_userspace_rdmsr(vcpu);
1876 	return complete_fast_msr_access(vcpu);
1877 }
1878 
1879 static u64 kvm_msr_reason(int r)
1880 {
1881 	switch (r) {
1882 	case KVM_MSR_RET_INVALID:
1883 		return KVM_MSR_EXIT_REASON_UNKNOWN;
1884 	case KVM_MSR_RET_FILTERED:
1885 		return KVM_MSR_EXIT_REASON_FILTER;
1886 	default:
1887 		return KVM_MSR_EXIT_REASON_INVAL;
1888 	}
1889 }
1890 
1891 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
1892 			      u32 exit_reason, u64 data,
1893 			      int (*completion)(struct kvm_vcpu *vcpu),
1894 			      int r)
1895 {
1896 	u64 msr_reason = kvm_msr_reason(r);
1897 
1898 	/* Check if the user wanted to know about this MSR fault */
1899 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
1900 		return 0;
1901 
1902 	vcpu->run->exit_reason = exit_reason;
1903 	vcpu->run->msr.error = 0;
1904 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
1905 	vcpu->run->msr.reason = msr_reason;
1906 	vcpu->run->msr.index = index;
1907 	vcpu->run->msr.data = data;
1908 	vcpu->arch.complete_userspace_io = completion;
1909 
1910 	return 1;
1911 }
1912 
1913 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
1914 {
1915 	u32 ecx = kvm_rcx_read(vcpu);
1916 	u64 data;
1917 	int r;
1918 
1919 	r = kvm_get_msr(vcpu, ecx, &data);
1920 
1921 	if (!r) {
1922 		trace_kvm_msr_read(ecx, data);
1923 
1924 		kvm_rax_write(vcpu, data & -1u);
1925 		kvm_rdx_write(vcpu, (data >> 32) & -1u);
1926 	} else {
1927 		/* MSR read failed? See if we should ask user space */
1928 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,
1929 				       complete_fast_rdmsr, r))
1930 			return 0;
1931 		trace_kvm_msr_read_ex(ecx);
1932 	}
1933 
1934 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
1935 }
1936 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
1937 
1938 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
1939 {
1940 	u32 ecx = kvm_rcx_read(vcpu);
1941 	u64 data = kvm_read_edx_eax(vcpu);
1942 	int r;
1943 
1944 	r = kvm_set_msr(vcpu, ecx, data);
1945 
1946 	if (!r) {
1947 		trace_kvm_msr_write(ecx, data);
1948 	} else {
1949 		/* MSR write failed? See if we should ask user space */
1950 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,
1951 				       complete_fast_msr_access, r))
1952 			return 0;
1953 		/* Signal all other negative errors to userspace */
1954 		if (r < 0)
1955 			return r;
1956 		trace_kvm_msr_write_ex(ecx, data);
1957 	}
1958 
1959 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
1960 }
1961 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
1962 
1963 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
1964 {
1965 	return kvm_skip_emulated_instruction(vcpu);
1966 }
1967 EXPORT_SYMBOL_GPL(kvm_emulate_as_nop);
1968 
1969 int kvm_emulate_invd(struct kvm_vcpu *vcpu)
1970 {
1971 	/* Treat an INVD instruction as a NOP and just skip it. */
1972 	return kvm_emulate_as_nop(vcpu);
1973 }
1974 EXPORT_SYMBOL_GPL(kvm_emulate_invd);
1975 
1976 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
1977 {
1978 	pr_warn_once("kvm: MWAIT instruction emulated as NOP!\n");
1979 	return kvm_emulate_as_nop(vcpu);
1980 }
1981 EXPORT_SYMBOL_GPL(kvm_emulate_mwait);
1982 
1983 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
1984 {
1985 	kvm_queue_exception(vcpu, UD_VECTOR);
1986 	return 1;
1987 }
1988 EXPORT_SYMBOL_GPL(kvm_handle_invalid_op);
1989 
1990 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
1991 {
1992 	pr_warn_once("kvm: MONITOR instruction emulated as NOP!\n");
1993 	return kvm_emulate_as_nop(vcpu);
1994 }
1995 EXPORT_SYMBOL_GPL(kvm_emulate_monitor);
1996 
1997 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
1998 {
1999 	xfer_to_guest_mode_prepare();
2000 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
2001 		xfer_to_guest_mode_work_pending();
2002 }
2003 
2004 /*
2005  * The fast path for frequent and performance sensitive wrmsr emulation,
2006  * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
2007  * the latency of virtual IPI by avoiding the expensive bits of transitioning
2008  * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
2009  * other cases which must be called after interrupts are enabled on the host.
2010  */
2011 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
2012 {
2013 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
2014 		return 1;
2015 
2016 	if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
2017 		((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
2018 		((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
2019 		((u32)(data >> 32) != X2APIC_BROADCAST)) {
2020 
2021 		data &= ~(1 << 12);
2022 		kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));
2023 		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
2024 		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);
2025 		trace_kvm_apic_write(APIC_ICR, (u32)data);
2026 		return 0;
2027 	}
2028 
2029 	return 1;
2030 }
2031 
2032 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
2033 {
2034 	if (!kvm_can_use_hv_timer(vcpu))
2035 		return 1;
2036 
2037 	kvm_set_lapic_tscdeadline_msr(vcpu, data);
2038 	return 0;
2039 }
2040 
2041 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
2042 {
2043 	u32 msr = kvm_rcx_read(vcpu);
2044 	u64 data;
2045 	fastpath_t ret = EXIT_FASTPATH_NONE;
2046 
2047 	switch (msr) {
2048 	case APIC_BASE_MSR + (APIC_ICR >> 4):
2049 		data = kvm_read_edx_eax(vcpu);
2050 		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
2051 			kvm_skip_emulated_instruction(vcpu);
2052 			ret = EXIT_FASTPATH_EXIT_HANDLED;
2053 		}
2054 		break;
2055 	case MSR_IA32_TSC_DEADLINE:
2056 		data = kvm_read_edx_eax(vcpu);
2057 		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
2058 			kvm_skip_emulated_instruction(vcpu);
2059 			ret = EXIT_FASTPATH_REENTER_GUEST;
2060 		}
2061 		break;
2062 	default:
2063 		break;
2064 	}
2065 
2066 	if (ret != EXIT_FASTPATH_NONE)
2067 		trace_kvm_msr_write(msr, data);
2068 
2069 	return ret;
2070 }
2071 EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);
2072 
2073 /*
2074  * Adapt set_msr() to msr_io()'s calling convention
2075  */
2076 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2077 {
2078 	return kvm_get_msr_ignored_check(vcpu, index, data, true);
2079 }
2080 
2081 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2082 {
2083 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);
2084 }
2085 
2086 #ifdef CONFIG_X86_64
2087 struct pvclock_clock {
2088 	int vclock_mode;
2089 	u64 cycle_last;
2090 	u64 mask;
2091 	u32 mult;
2092 	u32 shift;
2093 	u64 base_cycles;
2094 	u64 offset;
2095 };
2096 
2097 struct pvclock_gtod_data {
2098 	seqcount_t	seq;
2099 
2100 	struct pvclock_clock clock; /* extract of a clocksource struct */
2101 	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
2102 
2103 	ktime_t		offs_boot;
2104 	u64		wall_time_sec;
2105 };
2106 
2107 static struct pvclock_gtod_data pvclock_gtod_data;
2108 
2109 static void update_pvclock_gtod(struct timekeeper *tk)
2110 {
2111 	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
2112 
2113 	write_seqcount_begin(&vdata->seq);
2114 
2115 	/* copy pvclock gtod data */
2116 	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;
2117 	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
2118 	vdata->clock.mask		= tk->tkr_mono.mask;
2119 	vdata->clock.mult		= tk->tkr_mono.mult;
2120 	vdata->clock.shift		= tk->tkr_mono.shift;
2121 	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
2122 	vdata->clock.offset		= tk->tkr_mono.base;
2123 
2124 	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;
2125 	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
2126 	vdata->raw_clock.mask		= tk->tkr_raw.mask;
2127 	vdata->raw_clock.mult		= tk->tkr_raw.mult;
2128 	vdata->raw_clock.shift		= tk->tkr_raw.shift;
2129 	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
2130 	vdata->raw_clock.offset		= tk->tkr_raw.base;
2131 
2132 	vdata->wall_time_sec            = tk->xtime_sec;
2133 
2134 	vdata->offs_boot		= tk->offs_boot;
2135 
2136 	write_seqcount_end(&vdata->seq);
2137 }
2138 
2139 static s64 get_kvmclock_base_ns(void)
2140 {
2141 	/* Count up from boot time, but with the frequency of the raw clock.  */
2142 	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
2143 }
2144 #else
2145 static s64 get_kvmclock_base_ns(void)
2146 {
2147 	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
2148 	return ktime_get_boottime_ns();
2149 }
2150 #endif
2151 
2152 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
2153 {
2154 	int version;
2155 	int r;
2156 	struct pvclock_wall_clock wc;
2157 	u32 wc_sec_hi;
2158 	u64 wall_nsec;
2159 
2160 	if (!wall_clock)
2161 		return;
2162 
2163 	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
2164 	if (r)
2165 		return;
2166 
2167 	if (version & 1)
2168 		++version;  /* first time write, random junk */
2169 
2170 	++version;
2171 
2172 	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
2173 		return;
2174 
2175 	/*
2176 	 * The guest calculates current wall clock time by adding
2177 	 * system time (updated by kvm_guest_time_update below) to the
2178 	 * wall clock specified here.  We do the reverse here.
2179 	 */
2180 	wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
2181 
2182 	wc.nsec = do_div(wall_nsec, 1000000000);
2183 	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
2184 	wc.version = version;
2185 
2186 	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
2187 
2188 	if (sec_hi_ofs) {
2189 		wc_sec_hi = wall_nsec >> 32;
2190 		kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
2191 				&wc_sec_hi, sizeof(wc_sec_hi));
2192 	}
2193 
2194 	version++;
2195 	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
2196 }
2197 
2198 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
2199 				  bool old_msr, bool host_initiated)
2200 {
2201 	struct kvm_arch *ka = &vcpu->kvm->arch;
2202 
2203 	if (vcpu->vcpu_id == 0 && !host_initiated) {
2204 		if (ka->boot_vcpu_runs_old_kvmclock != old_msr)
2205 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2206 
2207 		ka->boot_vcpu_runs_old_kvmclock = old_msr;
2208 	}
2209 
2210 	vcpu->arch.time = system_time;
2211 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2212 
2213 	/* we verify if the enable bit is set... */
2214 	vcpu->arch.pv_time_enabled = false;
2215 	if (!(system_time & 1))
2216 		return;
2217 
2218 	if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2219 				       &vcpu->arch.pv_time, system_time & ~1ULL,
2220 				       sizeof(struct pvclock_vcpu_time_info)))
2221 		vcpu->arch.pv_time_enabled = true;
2222 
2223 	return;
2224 }
2225 
2226 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
2227 {
2228 	do_shl32_div32(dividend, divisor);
2229 	return dividend;
2230 }
2231 
2232 static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
2233 			       s8 *pshift, u32 *pmultiplier)
2234 {
2235 	uint64_t scaled64;
2236 	int32_t  shift = 0;
2237 	uint64_t tps64;
2238 	uint32_t tps32;
2239 
2240 	tps64 = base_hz;
2241 	scaled64 = scaled_hz;
2242 	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
2243 		tps64 >>= 1;
2244 		shift--;
2245 	}
2246 
2247 	tps32 = (uint32_t)tps64;
2248 	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
2249 		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
2250 			scaled64 >>= 1;
2251 		else
2252 			tps32 <<= 1;
2253 		shift++;
2254 	}
2255 
2256 	*pshift = shift;
2257 	*pmultiplier = div_frac(scaled64, tps32);
2258 }
2259 
2260 #ifdef CONFIG_X86_64
2261 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
2262 #endif
2263 
2264 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
2265 static unsigned long max_tsc_khz;
2266 
2267 static u32 adjust_tsc_khz(u32 khz, s32 ppm)
2268 {
2269 	u64 v = (u64)khz * (1000000 + ppm);
2270 	do_div(v, 1000000);
2271 	return v;
2272 }
2273 
2274 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2275 
2276 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2277 {
2278 	u64 ratio;
2279 
2280 	/* Guest TSC same frequency as host TSC? */
2281 	if (!scale) {
2282 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_default_tsc_scaling_ratio);
2283 		return 0;
2284 	}
2285 
2286 	/* TSC scaling supported? */
2287 	if (!kvm_has_tsc_control) {
2288 		if (user_tsc_khz > tsc_khz) {
2289 			vcpu->arch.tsc_catchup = 1;
2290 			vcpu->arch.tsc_always_catchup = 1;
2291 			return 0;
2292 		} else {
2293 			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
2294 			return -1;
2295 		}
2296 	}
2297 
2298 	/* TSC scaling required  - calculate ratio */
2299 	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
2300 				user_tsc_khz, tsc_khz);
2301 
2302 	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
2303 		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
2304 			            user_tsc_khz);
2305 		return -1;
2306 	}
2307 
2308 	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
2309 	return 0;
2310 }
2311 
2312 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2313 {
2314 	u32 thresh_lo, thresh_hi;
2315 	int use_scaling = 0;
2316 
2317 	/* tsc_khz can be zero if TSC calibration fails */
2318 	if (user_tsc_khz == 0) {
2319 		/* set tsc_scaling_ratio to a safe value */
2320 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_default_tsc_scaling_ratio);
2321 		return -1;
2322 	}
2323 
2324 	/* Compute a scale to convert nanoseconds in TSC cycles */
2325 	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
2326 			   &vcpu->arch.virtual_tsc_shift,
2327 			   &vcpu->arch.virtual_tsc_mult);
2328 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
2329 
2330 	/*
2331 	 * Compute the variation in TSC rate which is acceptable
2332 	 * within the range of tolerance and decide if the
2333 	 * rate being applied is within that bounds of the hardware
2334 	 * rate.  If so, no scaling or compensation need be done.
2335 	 */
2336 	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
2337 	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
2338 	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
2339 		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
2340 		use_scaling = 1;
2341 	}
2342 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
2343 }
2344 
2345 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
2346 {
2347 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
2348 				      vcpu->arch.virtual_tsc_mult,
2349 				      vcpu->arch.virtual_tsc_shift);
2350 	tsc += vcpu->arch.this_tsc_write;
2351 	return tsc;
2352 }
2353 
2354 static inline int gtod_is_based_on_tsc(int mode)
2355 {
2356 	return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
2357 }
2358 
2359 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
2360 {
2361 #ifdef CONFIG_X86_64
2362 	bool vcpus_matched;
2363 	struct kvm_arch *ka = &vcpu->kvm->arch;
2364 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2365 
2366 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2367 			 atomic_read(&vcpu->kvm->online_vcpus));
2368 
2369 	/*
2370 	 * Once the masterclock is enabled, always perform request in
2371 	 * order to update it.
2372 	 *
2373 	 * In order to enable masterclock, the host clocksource must be TSC
2374 	 * and the vcpus need to have matched TSCs.  When that happens,
2375 	 * perform request to enable masterclock.
2376 	 */
2377 	if (ka->use_master_clock ||
2378 	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
2379 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2380 
2381 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
2382 			    atomic_read(&vcpu->kvm->online_vcpus),
2383 		            ka->use_master_clock, gtod->clock.vclock_mode);
2384 #endif
2385 }
2386 
2387 /*
2388  * Multiply tsc by a fixed point number represented by ratio.
2389  *
2390  * The most significant 64-N bits (mult) of ratio represent the
2391  * integral part of the fixed point number; the remaining N bits
2392  * (frac) represent the fractional part, ie. ratio represents a fixed
2393  * point number (mult + frac * 2^(-N)).
2394  *
2395  * N equals to kvm_tsc_scaling_ratio_frac_bits.
2396  */
2397 static inline u64 __scale_tsc(u64 ratio, u64 tsc)
2398 {
2399 	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
2400 }
2401 
2402 u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc, u64 ratio)
2403 {
2404 	u64 _tsc = tsc;
2405 
2406 	if (ratio != kvm_default_tsc_scaling_ratio)
2407 		_tsc = __scale_tsc(ratio, tsc);
2408 
2409 	return _tsc;
2410 }
2411 EXPORT_SYMBOL_GPL(kvm_scale_tsc);
2412 
2413 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2414 {
2415 	u64 tsc;
2416 
2417 	tsc = kvm_scale_tsc(vcpu, rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
2418 
2419 	return target_tsc - tsc;
2420 }
2421 
2422 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2423 {
2424 	return vcpu->arch.l1_tsc_offset +
2425 		kvm_scale_tsc(vcpu, host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
2426 }
2427 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
2428 
2429 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
2430 {
2431 	u64 nested_offset;
2432 
2433 	if (l2_multiplier == kvm_default_tsc_scaling_ratio)
2434 		nested_offset = l1_offset;
2435 	else
2436 		nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
2437 						kvm_tsc_scaling_ratio_frac_bits);
2438 
2439 	nested_offset += l2_offset;
2440 	return nested_offset;
2441 }
2442 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset);
2443 
2444 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
2445 {
2446 	if (l2_multiplier != kvm_default_tsc_scaling_ratio)
2447 		return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
2448 				       kvm_tsc_scaling_ratio_frac_bits);
2449 
2450 	return l1_multiplier;
2451 }
2452 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier);
2453 
2454 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
2455 {
2456 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2457 				   vcpu->arch.l1_tsc_offset,
2458 				   l1_offset);
2459 
2460 	vcpu->arch.l1_tsc_offset = l1_offset;
2461 
2462 	/*
2463 	 * If we are here because L1 chose not to trap WRMSR to TSC then
2464 	 * according to the spec this should set L1's TSC (as opposed to
2465 	 * setting L1's offset for L2).
2466 	 */
2467 	if (is_guest_mode(vcpu))
2468 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
2469 			l1_offset,
2470 			static_call(kvm_x86_get_l2_tsc_offset)(vcpu),
2471 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2472 	else
2473 		vcpu->arch.tsc_offset = l1_offset;
2474 
2475 	static_call(kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset);
2476 }
2477 
2478 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
2479 {
2480 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
2481 
2482 	/* Userspace is changing the multiplier while L2 is active */
2483 	if (is_guest_mode(vcpu))
2484 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2485 			l1_multiplier,
2486 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2487 	else
2488 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2489 
2490 	if (kvm_has_tsc_control)
2491 		static_call(kvm_x86_write_tsc_multiplier)(
2492 			vcpu, vcpu->arch.tsc_scaling_ratio);
2493 }
2494 
2495 static inline bool kvm_check_tsc_unstable(void)
2496 {
2497 #ifdef CONFIG_X86_64
2498 	/*
2499 	 * TSC is marked unstable when we're running on Hyper-V,
2500 	 * 'TSC page' clocksource is good.
2501 	 */
2502 	if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
2503 		return false;
2504 #endif
2505 	return check_tsc_unstable();
2506 }
2507 
2508 /*
2509  * Infers attempts to synchronize the guest's tsc from host writes. Sets the
2510  * offset for the vcpu and tracks the TSC matching generation that the vcpu
2511  * participates in.
2512  */
2513 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
2514 				  u64 ns, bool matched)
2515 {
2516 	struct kvm *kvm = vcpu->kvm;
2517 
2518 	lockdep_assert_held(&kvm->arch.tsc_write_lock);
2519 
2520 	/*
2521 	 * We also track th most recent recorded KHZ, write and time to
2522 	 * allow the matching interval to be extended at each write.
2523 	 */
2524 	kvm->arch.last_tsc_nsec = ns;
2525 	kvm->arch.last_tsc_write = tsc;
2526 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2527 	kvm->arch.last_tsc_offset = offset;
2528 
2529 	vcpu->arch.last_guest_tsc = tsc;
2530 
2531 	kvm_vcpu_write_tsc_offset(vcpu, offset);
2532 
2533 	if (!matched) {
2534 		/*
2535 		 * We split periods of matched TSC writes into generations.
2536 		 * For each generation, we track the original measured
2537 		 * nanosecond time, offset, and write, so if TSCs are in
2538 		 * sync, we can match exact offset, and if not, we can match
2539 		 * exact software computation in compute_guest_tsc()
2540 		 *
2541 		 * These values are tracked in kvm->arch.cur_xxx variables.
2542 		 */
2543 		kvm->arch.cur_tsc_generation++;
2544 		kvm->arch.cur_tsc_nsec = ns;
2545 		kvm->arch.cur_tsc_write = tsc;
2546 		kvm->arch.cur_tsc_offset = offset;
2547 		kvm->arch.nr_vcpus_matched_tsc = 0;
2548 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
2549 		kvm->arch.nr_vcpus_matched_tsc++;
2550 	}
2551 
2552 	/* Keep track of which generation this VCPU has synchronized to */
2553 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2554 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2555 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2556 
2557 	kvm_track_tsc_matching(vcpu);
2558 }
2559 
2560 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)
2561 {
2562 	struct kvm *kvm = vcpu->kvm;
2563 	u64 offset, ns, elapsed;
2564 	unsigned long flags;
2565 	bool matched = false;
2566 	bool synchronizing = false;
2567 
2568 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
2569 	offset = kvm_compute_l1_tsc_offset(vcpu, data);
2570 	ns = get_kvmclock_base_ns();
2571 	elapsed = ns - kvm->arch.last_tsc_nsec;
2572 
2573 	if (vcpu->arch.virtual_tsc_khz) {
2574 		if (data == 0) {
2575 			/*
2576 			 * detection of vcpu initialization -- need to sync
2577 			 * with other vCPUs. This particularly helps to keep
2578 			 * kvm_clock stable after CPU hotplug
2579 			 */
2580 			synchronizing = true;
2581 		} else {
2582 			u64 tsc_exp = kvm->arch.last_tsc_write +
2583 						nsec_to_cycles(vcpu, elapsed);
2584 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2585 			/*
2586 			 * Special case: TSC write with a small delta (1 second)
2587 			 * of virtual cycle time against real time is
2588 			 * interpreted as an attempt to synchronize the CPU.
2589 			 */
2590 			synchronizing = data < tsc_exp + tsc_hz &&
2591 					data + tsc_hz > tsc_exp;
2592 		}
2593 	}
2594 
2595 	/*
2596 	 * For a reliable TSC, we can match TSC offsets, and for an unstable
2597 	 * TSC, we add elapsed time in this computation.  We could let the
2598 	 * compensation code attempt to catch up if we fall behind, but
2599 	 * it's better to try to match offsets from the beginning.
2600          */
2601 	if (synchronizing &&
2602 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2603 		if (!kvm_check_tsc_unstable()) {
2604 			offset = kvm->arch.cur_tsc_offset;
2605 		} else {
2606 			u64 delta = nsec_to_cycles(vcpu, elapsed);
2607 			data += delta;
2608 			offset = kvm_compute_l1_tsc_offset(vcpu, data);
2609 		}
2610 		matched = true;
2611 	}
2612 
2613 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched);
2614 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2615 }
2616 
2617 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2618 					   s64 adjustment)
2619 {
2620 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2621 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2622 }
2623 
2624 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2625 {
2626 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
2627 		WARN_ON(adjustment < 0);
2628 	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment,
2629 				   vcpu->arch.l1_tsc_scaling_ratio);
2630 	adjust_tsc_offset_guest(vcpu, adjustment);
2631 }
2632 
2633 #ifdef CONFIG_X86_64
2634 
2635 static u64 read_tsc(void)
2636 {
2637 	u64 ret = (u64)rdtsc_ordered();
2638 	u64 last = pvclock_gtod_data.clock.cycle_last;
2639 
2640 	if (likely(ret >= last))
2641 		return ret;
2642 
2643 	/*
2644 	 * GCC likes to generate cmov here, but this branch is extremely
2645 	 * predictable (it's just a function of time and the likely is
2646 	 * very likely) and there's a data dependence, so force GCC
2647 	 * to generate a branch instead.  I don't barrier() because
2648 	 * we don't actually need a barrier, and if this function
2649 	 * ever gets inlined it will generate worse code.
2650 	 */
2651 	asm volatile ("");
2652 	return last;
2653 }
2654 
2655 static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
2656 			  int *mode)
2657 {
2658 	long v;
2659 	u64 tsc_pg_val;
2660 
2661 	switch (clock->vclock_mode) {
2662 	case VDSO_CLOCKMODE_HVCLOCK:
2663 		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
2664 						  tsc_timestamp);
2665 		if (tsc_pg_val != U64_MAX) {
2666 			/* TSC page valid */
2667 			*mode = VDSO_CLOCKMODE_HVCLOCK;
2668 			v = (tsc_pg_val - clock->cycle_last) &
2669 				clock->mask;
2670 		} else {
2671 			/* TSC page invalid */
2672 			*mode = VDSO_CLOCKMODE_NONE;
2673 		}
2674 		break;
2675 	case VDSO_CLOCKMODE_TSC:
2676 		*mode = VDSO_CLOCKMODE_TSC;
2677 		*tsc_timestamp = read_tsc();
2678 		v = (*tsc_timestamp - clock->cycle_last) &
2679 			clock->mask;
2680 		break;
2681 	default:
2682 		*mode = VDSO_CLOCKMODE_NONE;
2683 	}
2684 
2685 	if (*mode == VDSO_CLOCKMODE_NONE)
2686 		*tsc_timestamp = v = 0;
2687 
2688 	return v * clock->mult;
2689 }
2690 
2691 static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
2692 {
2693 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2694 	unsigned long seq;
2695 	int mode;
2696 	u64 ns;
2697 
2698 	do {
2699 		seq = read_seqcount_begin(&gtod->seq);
2700 		ns = gtod->raw_clock.base_cycles;
2701 		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
2702 		ns >>= gtod->raw_clock.shift;
2703 		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
2704 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2705 	*t = ns;
2706 
2707 	return mode;
2708 }
2709 
2710 static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2711 {
2712 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2713 	unsigned long seq;
2714 	int mode;
2715 	u64 ns;
2716 
2717 	do {
2718 		seq = read_seqcount_begin(&gtod->seq);
2719 		ts->tv_sec = gtod->wall_time_sec;
2720 		ns = gtod->clock.base_cycles;
2721 		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
2722 		ns >>= gtod->clock.shift;
2723 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2724 
2725 	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
2726 	ts->tv_nsec = ns;
2727 
2728 	return mode;
2729 }
2730 
2731 /* returns true if host is using TSC based clocksource */
2732 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2733 {
2734 	/* checked again under seqlock below */
2735 	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2736 		return false;
2737 
2738 	return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
2739 						      tsc_timestamp));
2740 }
2741 
2742 /* returns true if host is using TSC based clocksource */
2743 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2744 					   u64 *tsc_timestamp)
2745 {
2746 	/* checked again under seqlock below */
2747 	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2748 		return false;
2749 
2750 	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2751 }
2752 #endif
2753 
2754 /*
2755  *
2756  * Assuming a stable TSC across physical CPUS, and a stable TSC
2757  * across virtual CPUs, the following condition is possible.
2758  * Each numbered line represents an event visible to both
2759  * CPUs at the next numbered event.
2760  *
2761  * "timespecX" represents host monotonic time. "tscX" represents
2762  * RDTSC value.
2763  *
2764  * 		VCPU0 on CPU0		|	VCPU1 on CPU1
2765  *
2766  * 1.  read timespec0,tsc0
2767  * 2.					| timespec1 = timespec0 + N
2768  * 					| tsc1 = tsc0 + M
2769  * 3. transition to guest		| transition to guest
2770  * 4. ret0 = timespec0 + (rdtsc - tsc0) |
2771  * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
2772  * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
2773  *
2774  * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
2775  *
2776  * 	- ret0 < ret1
2777  *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
2778  *		...
2779  *	- 0 < N - M => M < N
2780  *
2781  * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
2782  * always the case (the difference between two distinct xtime instances
2783  * might be smaller then the difference between corresponding TSC reads,
2784  * when updating guest vcpus pvclock areas).
2785  *
2786  * To avoid that problem, do not allow visibility of distinct
2787  * system_timestamp/tsc_timestamp values simultaneously: use a master
2788  * copy of host monotonic time values. Update that master copy
2789  * in lockstep.
2790  *
2791  * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2792  *
2793  */
2794 
2795 static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
2796 {
2797 #ifdef CONFIG_X86_64
2798 	struct kvm_arch *ka = &kvm->arch;
2799 	int vclock_mode;
2800 	bool host_tsc_clocksource, vcpus_matched;
2801 
2802 	lockdep_assert_held(&kvm->arch.tsc_write_lock);
2803 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2804 			atomic_read(&kvm->online_vcpus));
2805 
2806 	/*
2807 	 * If the host uses TSC clock, then passthrough TSC as stable
2808 	 * to the guest.
2809 	 */
2810 	host_tsc_clocksource = kvm_get_time_and_clockread(
2811 					&ka->master_kernel_ns,
2812 					&ka->master_cycle_now);
2813 
2814 	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2815 				&& !ka->backwards_tsc_observed
2816 				&& !ka->boot_vcpu_runs_old_kvmclock;
2817 
2818 	if (ka->use_master_clock)
2819 		atomic_set(&kvm_guest_has_master_clock, 1);
2820 
2821 	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2822 	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
2823 					vcpus_matched);
2824 #endif
2825 }
2826 
2827 static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
2828 {
2829 	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
2830 }
2831 
2832 static void __kvm_start_pvclock_update(struct kvm *kvm)
2833 {
2834 	raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
2835 	write_seqcount_begin(&kvm->arch.pvclock_sc);
2836 }
2837 
2838 static void kvm_start_pvclock_update(struct kvm *kvm)
2839 {
2840 	kvm_make_mclock_inprogress_request(kvm);
2841 
2842 	/* no guest entries from this point */
2843 	__kvm_start_pvclock_update(kvm);
2844 }
2845 
2846 static void kvm_end_pvclock_update(struct kvm *kvm)
2847 {
2848 	struct kvm_arch *ka = &kvm->arch;
2849 	struct kvm_vcpu *vcpu;
2850 	unsigned long i;
2851 
2852 	write_seqcount_end(&ka->pvclock_sc);
2853 	raw_spin_unlock_irq(&ka->tsc_write_lock);
2854 	kvm_for_each_vcpu(i, vcpu, kvm)
2855 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2856 
2857 	/* guest entries allowed */
2858 	kvm_for_each_vcpu(i, vcpu, kvm)
2859 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2860 }
2861 
2862 static void kvm_update_masterclock(struct kvm *kvm)
2863 {
2864 	kvm_hv_invalidate_tsc_page(kvm);
2865 	kvm_start_pvclock_update(kvm);
2866 	pvclock_update_vm_gtod_copy(kvm);
2867 	kvm_end_pvclock_update(kvm);
2868 }
2869 
2870 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc.  */
2871 static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
2872 {
2873 	struct kvm_arch *ka = &kvm->arch;
2874 	struct pvclock_vcpu_time_info hv_clock;
2875 
2876 	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
2877 	get_cpu();
2878 
2879 	data->flags = 0;
2880 	if (ka->use_master_clock && __this_cpu_read(cpu_tsc_khz)) {
2881 #ifdef CONFIG_X86_64
2882 		struct timespec64 ts;
2883 
2884 		if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
2885 			data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
2886 			data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
2887 		} else
2888 #endif
2889 		data->host_tsc = rdtsc();
2890 
2891 		data->flags |= KVM_CLOCK_TSC_STABLE;
2892 		hv_clock.tsc_timestamp = ka->master_cycle_now;
2893 		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
2894 		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
2895 				   &hv_clock.tsc_shift,
2896 				   &hv_clock.tsc_to_system_mul);
2897 		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
2898 	} else {
2899 		data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
2900 	}
2901 
2902 	put_cpu();
2903 }
2904 
2905 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
2906 {
2907 	struct kvm_arch *ka = &kvm->arch;
2908 	unsigned seq;
2909 
2910 	do {
2911 		seq = read_seqcount_begin(&ka->pvclock_sc);
2912 		__get_kvmclock(kvm, data);
2913 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
2914 }
2915 
2916 u64 get_kvmclock_ns(struct kvm *kvm)
2917 {
2918 	struct kvm_clock_data data;
2919 
2920 	get_kvmclock(kvm, &data);
2921 	return data.clock;
2922 }
2923 
2924 static void kvm_setup_pvclock_page(struct kvm_vcpu *v,
2925 				   struct gfn_to_hva_cache *cache,
2926 				   unsigned int offset)
2927 {
2928 	struct kvm_vcpu_arch *vcpu = &v->arch;
2929 	struct pvclock_vcpu_time_info guest_hv_clock;
2930 
2931 	if (unlikely(kvm_read_guest_offset_cached(v->kvm, cache,
2932 		&guest_hv_clock, offset, sizeof(guest_hv_clock))))
2933 		return;
2934 
2935 	/* This VCPU is paused, but it's legal for a guest to read another
2936 	 * VCPU's kvmclock, so we really have to follow the specification where
2937 	 * it says that version is odd if data is being modified, and even after
2938 	 * it is consistent.
2939 	 *
2940 	 * Version field updates must be kept separate.  This is because
2941 	 * kvm_write_guest_cached might use a "rep movs" instruction, and
2942 	 * writes within a string instruction are weakly ordered.  So there
2943 	 * are three writes overall.
2944 	 *
2945 	 * As a small optimization, only write the version field in the first
2946 	 * and third write.  The vcpu->pv_time cache is still valid, because the
2947 	 * version field is the first in the struct.
2948 	 */
2949 	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);
2950 
2951 	if (guest_hv_clock.version & 1)
2952 		++guest_hv_clock.version;  /* first time write, random junk */
2953 
2954 	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2955 	kvm_write_guest_offset_cached(v->kvm, cache,
2956 				      &vcpu->hv_clock, offset,
2957 				      sizeof(vcpu->hv_clock.version));
2958 
2959 	smp_wmb();
2960 
2961 	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
2962 	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
2963 
2964 	if (vcpu->pvclock_set_guest_stopped_request) {
2965 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
2966 		vcpu->pvclock_set_guest_stopped_request = false;
2967 	}
2968 
2969 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
2970 
2971 	kvm_write_guest_offset_cached(v->kvm, cache,
2972 				      &vcpu->hv_clock, offset,
2973 				      sizeof(vcpu->hv_clock));
2974 
2975 	smp_wmb();
2976 
2977 	vcpu->hv_clock.version++;
2978 	kvm_write_guest_offset_cached(v->kvm, cache,
2979 				     &vcpu->hv_clock, offset,
2980 				     sizeof(vcpu->hv_clock.version));
2981 }
2982 
2983 static int kvm_guest_time_update(struct kvm_vcpu *v)
2984 {
2985 	unsigned long flags, tgt_tsc_khz;
2986 	unsigned seq;
2987 	struct kvm_vcpu_arch *vcpu = &v->arch;
2988 	struct kvm_arch *ka = &v->kvm->arch;
2989 	s64 kernel_ns;
2990 	u64 tsc_timestamp, host_tsc;
2991 	u8 pvclock_flags;
2992 	bool use_master_clock;
2993 
2994 	kernel_ns = 0;
2995 	host_tsc = 0;
2996 
2997 	/*
2998 	 * If the host uses TSC clock, then passthrough TSC as stable
2999 	 * to the guest.
3000 	 */
3001 	do {
3002 		seq = read_seqcount_begin(&ka->pvclock_sc);
3003 		use_master_clock = ka->use_master_clock;
3004 		if (use_master_clock) {
3005 			host_tsc = ka->master_cycle_now;
3006 			kernel_ns = ka->master_kernel_ns;
3007 		}
3008 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
3009 
3010 	/* Keep irq disabled to prevent changes to the clock */
3011 	local_irq_save(flags);
3012 	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
3013 	if (unlikely(tgt_tsc_khz == 0)) {
3014 		local_irq_restore(flags);
3015 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3016 		return 1;
3017 	}
3018 	if (!use_master_clock) {
3019 		host_tsc = rdtsc();
3020 		kernel_ns = get_kvmclock_base_ns();
3021 	}
3022 
3023 	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
3024 
3025 	/*
3026 	 * We may have to catch up the TSC to match elapsed wall clock
3027 	 * time for two reasons, even if kvmclock is used.
3028 	 *   1) CPU could have been running below the maximum TSC rate
3029 	 *   2) Broken TSC compensation resets the base at each VCPU
3030 	 *      entry to avoid unknown leaps of TSC even when running
3031 	 *      again on the same CPU.  This may cause apparent elapsed
3032 	 *      time to disappear, and the guest to stand still or run
3033 	 *	very slowly.
3034 	 */
3035 	if (vcpu->tsc_catchup) {
3036 		u64 tsc = compute_guest_tsc(v, kernel_ns);
3037 		if (tsc > tsc_timestamp) {
3038 			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
3039 			tsc_timestamp = tsc;
3040 		}
3041 	}
3042 
3043 	local_irq_restore(flags);
3044 
3045 	/* With all the info we got, fill in the values */
3046 
3047 	if (kvm_has_tsc_control)
3048 		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz,
3049 					    v->arch.l1_tsc_scaling_ratio);
3050 
3051 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
3052 		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
3053 				   &vcpu->hv_clock.tsc_shift,
3054 				   &vcpu->hv_clock.tsc_to_system_mul);
3055 		vcpu->hw_tsc_khz = tgt_tsc_khz;
3056 	}
3057 
3058 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
3059 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
3060 	vcpu->last_guest_tsc = tsc_timestamp;
3061 
3062 	/* If the host uses TSC clocksource, then it is stable */
3063 	pvclock_flags = 0;
3064 	if (use_master_clock)
3065 		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
3066 
3067 	vcpu->hv_clock.flags = pvclock_flags;
3068 
3069 	if (vcpu->pv_time_enabled)
3070 		kvm_setup_pvclock_page(v, &vcpu->pv_time, 0);
3071 	if (vcpu->xen.vcpu_info_set)
3072 		kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_info_cache,
3073 				       offsetof(struct compat_vcpu_info, time));
3074 	if (vcpu->xen.vcpu_time_info_set)
3075 		kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_time_info_cache, 0);
3076 	if (!v->vcpu_idx)
3077 		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
3078 	return 0;
3079 }
3080 
3081 /*
3082  * kvmclock updates which are isolated to a given vcpu, such as
3083  * vcpu->cpu migration, should not allow system_timestamp from
3084  * the rest of the vcpus to remain static. Otherwise ntp frequency
3085  * correction applies to one vcpu's system_timestamp but not
3086  * the others.
3087  *
3088  * So in those cases, request a kvmclock update for all vcpus.
3089  * We need to rate-limit these requests though, as they can
3090  * considerably slow guests that have a large number of vcpus.
3091  * The time for a remote vcpu to update its kvmclock is bound
3092  * by the delay we use to rate-limit the updates.
3093  */
3094 
3095 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
3096 
3097 static void kvmclock_update_fn(struct work_struct *work)
3098 {
3099 	unsigned long i;
3100 	struct delayed_work *dwork = to_delayed_work(work);
3101 	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3102 					   kvmclock_update_work);
3103 	struct kvm *kvm = container_of(ka, struct kvm, arch);
3104 	struct kvm_vcpu *vcpu;
3105 
3106 	kvm_for_each_vcpu(i, vcpu, kvm) {
3107 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3108 		kvm_vcpu_kick(vcpu);
3109 	}
3110 }
3111 
3112 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
3113 {
3114 	struct kvm *kvm = v->kvm;
3115 
3116 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3117 	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
3118 					KVMCLOCK_UPDATE_DELAY);
3119 }
3120 
3121 #define KVMCLOCK_SYNC_PERIOD (300 * HZ)
3122 
3123 static void kvmclock_sync_fn(struct work_struct *work)
3124 {
3125 	struct delayed_work *dwork = to_delayed_work(work);
3126 	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3127 					   kvmclock_sync_work);
3128 	struct kvm *kvm = container_of(ka, struct kvm, arch);
3129 
3130 	if (!kvmclock_periodic_sync)
3131 		return;
3132 
3133 	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
3134 	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
3135 					KVMCLOCK_SYNC_PERIOD);
3136 }
3137 
3138 /*
3139  * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
3140  */
3141 static bool can_set_mci_status(struct kvm_vcpu *vcpu)
3142 {
3143 	/* McStatusWrEn enabled? */
3144 	if (guest_cpuid_is_amd_or_hygon(vcpu))
3145 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
3146 
3147 	return false;
3148 }
3149 
3150 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3151 {
3152 	u64 mcg_cap = vcpu->arch.mcg_cap;
3153 	unsigned bank_num = mcg_cap & 0xff;
3154 	u32 msr = msr_info->index;
3155 	u64 data = msr_info->data;
3156 
3157 	switch (msr) {
3158 	case MSR_IA32_MCG_STATUS:
3159 		vcpu->arch.mcg_status = data;
3160 		break;
3161 	case MSR_IA32_MCG_CTL:
3162 		if (!(mcg_cap & MCG_CTL_P) &&
3163 		    (data || !msr_info->host_initiated))
3164 			return 1;
3165 		if (data != 0 && data != ~(u64)0)
3166 			return 1;
3167 		vcpu->arch.mcg_ctl = data;
3168 		break;
3169 	default:
3170 		if (msr >= MSR_IA32_MC0_CTL &&
3171 		    msr < MSR_IA32_MCx_CTL(bank_num)) {
3172 			u32 offset = array_index_nospec(
3173 				msr - MSR_IA32_MC0_CTL,
3174 				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);
3175 
3176 			/* only 0 or all 1s can be written to IA32_MCi_CTL
3177 			 * some Linux kernels though clear bit 10 in bank 4 to
3178 			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
3179 			 * this to avoid an uncatched #GP in the guest
3180 			 */
3181 			if ((offset & 0x3) == 0 &&
3182 			    data != 0 && (data | (1 << 10)) != ~(u64)0)
3183 				return -1;
3184 
3185 			/* MCi_STATUS */
3186 			if (!msr_info->host_initiated &&
3187 			    (offset & 0x3) == 1 && data != 0) {
3188 				if (!can_set_mci_status(vcpu))
3189 					return -1;
3190 			}
3191 
3192 			vcpu->arch.mce_banks[offset] = data;
3193 			break;
3194 		}
3195 		return 1;
3196 	}
3197 	return 0;
3198 }
3199 
3200 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
3201 {
3202 	u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
3203 
3204 	return (vcpu->arch.apf.msr_en_val & mask) == mask;
3205 }
3206 
3207 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
3208 {
3209 	gpa_t gpa = data & ~0x3f;
3210 
3211 	/* Bits 4:5 are reserved, Should be zero */
3212 	if (data & 0x30)
3213 		return 1;
3214 
3215 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
3216 	    (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT))
3217 		return 1;
3218 
3219 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
3220 	    (data & KVM_ASYNC_PF_DELIVERY_AS_INT))
3221 		return 1;
3222 
3223 	if (!lapic_in_kernel(vcpu))
3224 		return data ? 1 : 0;
3225 
3226 	vcpu->arch.apf.msr_en_val = data;
3227 
3228 	if (!kvm_pv_async_pf_enabled(vcpu)) {
3229 		kvm_clear_async_pf_completion_queue(vcpu);
3230 		kvm_async_pf_hash_reset(vcpu);
3231 		return 0;
3232 	}
3233 
3234 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
3235 					sizeof(u64)))
3236 		return 1;
3237 
3238 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
3239 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
3240 
3241 	kvm_async_pf_wakeup_all(vcpu);
3242 
3243 	return 0;
3244 }
3245 
3246 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
3247 {
3248 	/* Bits 8-63 are reserved */
3249 	if (data >> 8)
3250 		return 1;
3251 
3252 	if (!lapic_in_kernel(vcpu))
3253 		return 1;
3254 
3255 	vcpu->arch.apf.msr_int_val = data;
3256 
3257 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3258 
3259 	return 0;
3260 }
3261 
3262 static void kvmclock_reset(struct kvm_vcpu *vcpu)
3263 {
3264 	vcpu->arch.pv_time_enabled = false;
3265 	vcpu->arch.time = 0;
3266 }
3267 
3268 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
3269 {
3270 	++vcpu->stat.tlb_flush;
3271 	static_call(kvm_x86_tlb_flush_all)(vcpu);
3272 }
3273 
3274 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
3275 {
3276 	++vcpu->stat.tlb_flush;
3277 
3278 	if (!tdp_enabled) {
3279 		/*
3280 		 * A TLB flush on behalf of the guest is equivalent to
3281 		 * INVPCID(all), toggling CR4.PGE, etc., which requires
3282 		 * a forced sync of the shadow page tables.  Ensure all the
3283 		 * roots are synced and the guest TLB in hardware is clean.
3284 		 */
3285 		kvm_mmu_sync_roots(vcpu);
3286 		kvm_mmu_sync_prev_roots(vcpu);
3287 	}
3288 
3289 	static_call(kvm_x86_tlb_flush_guest)(vcpu);
3290 }
3291 
3292 
3293 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
3294 {
3295 	++vcpu->stat.tlb_flush;
3296 	static_call(kvm_x86_tlb_flush_current)(vcpu);
3297 }
3298 
3299 /*
3300  * Service "local" TLB flush requests, which are specific to the current MMU
3301  * context.  In addition to the generic event handling in vcpu_enter_guest(),
3302  * TLB flushes that are targeted at an MMU context also need to be serviced
3303  * prior before nested VM-Enter/VM-Exit.
3304  */
3305 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
3306 {
3307 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3308 		kvm_vcpu_flush_tlb_current(vcpu);
3309 
3310 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
3311 		kvm_vcpu_flush_tlb_guest(vcpu);
3312 }
3313 EXPORT_SYMBOL_GPL(kvm_service_local_tlb_flush_requests);
3314 
3315 static void record_steal_time(struct kvm_vcpu *vcpu)
3316 {
3317 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3318 	struct kvm_steal_time __user *st;
3319 	struct kvm_memslots *slots;
3320 	u64 steal;
3321 	u32 version;
3322 
3323 	if (kvm_xen_msr_enabled(vcpu->kvm)) {
3324 		kvm_xen_runstate_set_running(vcpu);
3325 		return;
3326 	}
3327 
3328 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3329 		return;
3330 
3331 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
3332 		return;
3333 
3334 	slots = kvm_memslots(vcpu->kvm);
3335 
3336 	if (unlikely(slots->generation != ghc->generation ||
3337 		     kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
3338 		gfn_t gfn = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
3339 
3340 		/* We rely on the fact that it fits in a single page. */
3341 		BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
3342 
3343 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gfn, sizeof(*st)) ||
3344 		    kvm_is_error_hva(ghc->hva) || !ghc->memslot)
3345 			return;
3346 	}
3347 
3348 	st = (struct kvm_steal_time __user *)ghc->hva;
3349 	/*
3350 	 * Doing a TLB flush here, on the guest's behalf, can avoid
3351 	 * expensive IPIs.
3352 	 */
3353 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
3354 		u8 st_preempted = 0;
3355 		int err = -EFAULT;
3356 
3357 		if (!user_access_begin(st, sizeof(*st)))
3358 			return;
3359 
3360 		asm volatile("1: xchgb %0, %2\n"
3361 			     "xor %1, %1\n"
3362 			     "2:\n"
3363 			     _ASM_EXTABLE_UA(1b, 2b)
3364 			     : "+q" (st_preempted),
3365 			       "+&r" (err),
3366 			       "+m" (st->preempted));
3367 		if (err)
3368 			goto out;
3369 
3370 		user_access_end();
3371 
3372 		vcpu->arch.st.preempted = 0;
3373 
3374 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
3375 				       st_preempted & KVM_VCPU_FLUSH_TLB);
3376 		if (st_preempted & KVM_VCPU_FLUSH_TLB)
3377 			kvm_vcpu_flush_tlb_guest(vcpu);
3378 
3379 		if (!user_access_begin(st, sizeof(*st)))
3380 			goto dirty;
3381 	} else {
3382 		if (!user_access_begin(st, sizeof(*st)))
3383 			return;
3384 
3385 		unsafe_put_user(0, &st->preempted, out);
3386 		vcpu->arch.st.preempted = 0;
3387 	}
3388 
3389 	unsafe_get_user(version, &st->version, out);
3390 	if (version & 1)
3391 		version += 1;  /* first time write, random junk */
3392 
3393 	version += 1;
3394 	unsafe_put_user(version, &st->version, out);
3395 
3396 	smp_wmb();
3397 
3398 	unsafe_get_user(steal, &st->steal, out);
3399 	steal += current->sched_info.run_delay -
3400 		vcpu->arch.st.last_steal;
3401 	vcpu->arch.st.last_steal = current->sched_info.run_delay;
3402 	unsafe_put_user(steal, &st->steal, out);
3403 
3404 	version += 1;
3405 	unsafe_put_user(version, &st->version, out);
3406 
3407  out:
3408 	user_access_end();
3409  dirty:
3410 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
3411 }
3412 
3413 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3414 {
3415 	bool pr = false;
3416 	u32 msr = msr_info->index;
3417 	u64 data = msr_info->data;
3418 
3419 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
3420 		return kvm_xen_write_hypercall_page(vcpu, data);
3421 
3422 	switch (msr) {
3423 	case MSR_AMD64_NB_CFG:
3424 	case MSR_IA32_UCODE_WRITE:
3425 	case MSR_VM_HSAVE_PA:
3426 	case MSR_AMD64_PATCH_LOADER:
3427 	case MSR_AMD64_BU_CFG2:
3428 	case MSR_AMD64_DC_CFG:
3429 	case MSR_F15H_EX_CFG:
3430 		break;
3431 
3432 	case MSR_IA32_UCODE_REV:
3433 		if (msr_info->host_initiated)
3434 			vcpu->arch.microcode_version = data;
3435 		break;
3436 	case MSR_IA32_ARCH_CAPABILITIES:
3437 		if (!msr_info->host_initiated)
3438 			return 1;
3439 		vcpu->arch.arch_capabilities = data;
3440 		break;
3441 	case MSR_IA32_PERF_CAPABILITIES: {
3442 		struct kvm_msr_entry msr_ent = {.index = msr, .data = 0};
3443 
3444 		if (!msr_info->host_initiated)
3445 			return 1;
3446 		if (kvm_get_msr_feature(&msr_ent))
3447 			return 1;
3448 		if (data & ~msr_ent.data)
3449 			return 1;
3450 
3451 		vcpu->arch.perf_capabilities = data;
3452 
3453 		return 0;
3454 		}
3455 	case MSR_EFER:
3456 		return set_efer(vcpu, msr_info);
3457 	case MSR_K7_HWCR:
3458 		data &= ~(u64)0x40;	/* ignore flush filter disable */
3459 		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
3460 		data &= ~(u64)0x8;	/* ignore TLB cache disable */
3461 
3462 		/* Handle McStatusWrEn */
3463 		if (data == BIT_ULL(18)) {
3464 			vcpu->arch.msr_hwcr = data;
3465 		} else if (data != 0) {
3466 			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
3467 				    data);
3468 			return 1;
3469 		}
3470 		break;
3471 	case MSR_FAM10H_MMIO_CONF_BASE:
3472 		if (data != 0) {
3473 			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
3474 				    "0x%llx\n", data);
3475 			return 1;
3476 		}
3477 		break;
3478 	case 0x200 ... 0x2ff:
3479 		return kvm_mtrr_set_msr(vcpu, msr, data);
3480 	case MSR_IA32_APICBASE:
3481 		return kvm_set_apic_base(vcpu, msr_info);
3482 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
3483 		return kvm_x2apic_msr_write(vcpu, msr, data);
3484 	case MSR_IA32_TSC_DEADLINE:
3485 		kvm_set_lapic_tscdeadline_msr(vcpu, data);
3486 		break;
3487 	case MSR_IA32_TSC_ADJUST:
3488 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
3489 			if (!msr_info->host_initiated) {
3490 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
3491 				adjust_tsc_offset_guest(vcpu, adj);
3492 				/* Before back to guest, tsc_timestamp must be adjusted
3493 				 * as well, otherwise guest's percpu pvclock time could jump.
3494 				 */
3495 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3496 			}
3497 			vcpu->arch.ia32_tsc_adjust_msr = data;
3498 		}
3499 		break;
3500 	case MSR_IA32_MISC_ENABLE:
3501 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3502 		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
3503 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3504 				return 1;
3505 			vcpu->arch.ia32_misc_enable_msr = data;
3506 			kvm_update_cpuid_runtime(vcpu);
3507 		} else {
3508 			vcpu->arch.ia32_misc_enable_msr = data;
3509 		}
3510 		break;
3511 	case MSR_IA32_SMBASE:
3512 		if (!msr_info->host_initiated)
3513 			return 1;
3514 		vcpu->arch.smbase = data;
3515 		break;
3516 	case MSR_IA32_POWER_CTL:
3517 		vcpu->arch.msr_ia32_power_ctl = data;
3518 		break;
3519 	case MSR_IA32_TSC:
3520 		if (msr_info->host_initiated) {
3521 			kvm_synchronize_tsc(vcpu, data);
3522 		} else {
3523 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
3524 			adjust_tsc_offset_guest(vcpu, adj);
3525 			vcpu->arch.ia32_tsc_adjust_msr += adj;
3526 		}
3527 		break;
3528 	case MSR_IA32_XSS:
3529 		if (!msr_info->host_initiated &&
3530 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3531 			return 1;
3532 		/*
3533 		 * KVM supports exposing PT to the guest, but does not support
3534 		 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
3535 		 * XSAVES/XRSTORS to save/restore PT MSRs.
3536 		 */
3537 		if (data & ~supported_xss)
3538 			return 1;
3539 		vcpu->arch.ia32_xss = data;
3540 		kvm_update_cpuid_runtime(vcpu);
3541 		break;
3542 	case MSR_SMI_COUNT:
3543 		if (!msr_info->host_initiated)
3544 			return 1;
3545 		vcpu->arch.smi_count = data;
3546 		break;
3547 	case MSR_KVM_WALL_CLOCK_NEW:
3548 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3549 			return 1;
3550 
3551 		vcpu->kvm->arch.wall_clock = data;
3552 		kvm_write_wall_clock(vcpu->kvm, data, 0);
3553 		break;
3554 	case MSR_KVM_WALL_CLOCK:
3555 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3556 			return 1;
3557 
3558 		vcpu->kvm->arch.wall_clock = data;
3559 		kvm_write_wall_clock(vcpu->kvm, data, 0);
3560 		break;
3561 	case MSR_KVM_SYSTEM_TIME_NEW:
3562 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3563 			return 1;
3564 
3565 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
3566 		break;
3567 	case MSR_KVM_SYSTEM_TIME:
3568 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3569 			return 1;
3570 
3571 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);
3572 		break;
3573 	case MSR_KVM_ASYNC_PF_EN:
3574 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3575 			return 1;
3576 
3577 		if (kvm_pv_enable_async_pf(vcpu, data))
3578 			return 1;
3579 		break;
3580 	case MSR_KVM_ASYNC_PF_INT:
3581 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3582 			return 1;
3583 
3584 		if (kvm_pv_enable_async_pf_int(vcpu, data))
3585 			return 1;
3586 		break;
3587 	case MSR_KVM_ASYNC_PF_ACK:
3588 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3589 			return 1;
3590 		if (data & 0x1) {
3591 			vcpu->arch.apf.pageready_pending = false;
3592 			kvm_check_async_pf_completion(vcpu);
3593 		}
3594 		break;
3595 	case MSR_KVM_STEAL_TIME:
3596 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3597 			return 1;
3598 
3599 		if (unlikely(!sched_info_on()))
3600 			return 1;
3601 
3602 		if (data & KVM_STEAL_RESERVED_MASK)
3603 			return 1;
3604 
3605 		vcpu->arch.st.msr_val = data;
3606 
3607 		if (!(data & KVM_MSR_ENABLED))
3608 			break;
3609 
3610 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3611 
3612 		break;
3613 	case MSR_KVM_PV_EOI_EN:
3614 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3615 			return 1;
3616 
3617 		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
3618 			return 1;
3619 		break;
3620 
3621 	case MSR_KVM_POLL_CONTROL:
3622 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3623 			return 1;
3624 
3625 		/* only enable bit supported */
3626 		if (data & (-1ULL << 1))
3627 			return 1;
3628 
3629 		vcpu->arch.msr_kvm_poll_control = data;
3630 		break;
3631 
3632 	case MSR_IA32_MCG_CTL:
3633 	case MSR_IA32_MCG_STATUS:
3634 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3635 		return set_msr_mce(vcpu, msr_info);
3636 
3637 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3638 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3639 		pr = true;
3640 		fallthrough;
3641 	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3642 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3643 		if (kvm_pmu_is_valid_msr(vcpu, msr))
3644 			return kvm_pmu_set_msr(vcpu, msr_info);
3645 
3646 		if (pr || data != 0)
3647 			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
3648 				    "0x%x data 0x%llx\n", msr, data);
3649 		break;
3650 	case MSR_K7_CLK_CTL:
3651 		/*
3652 		 * Ignore all writes to this no longer documented MSR.
3653 		 * Writes are only relevant for old K7 processors,
3654 		 * all pre-dating SVM, but a recommended workaround from
3655 		 * AMD for these chips. It is possible to specify the
3656 		 * affected processor models on the command line, hence
3657 		 * the need to ignore the workaround.
3658 		 */
3659 		break;
3660 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3661 	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
3662 	case HV_X64_MSR_SYNDBG_OPTIONS:
3663 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
3664 	case HV_X64_MSR_CRASH_CTL:
3665 	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3666 	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
3667 	case HV_X64_MSR_TSC_EMULATION_CONTROL:
3668 	case HV_X64_MSR_TSC_EMULATION_STATUS:
3669 		return kvm_hv_set_msr_common(vcpu, msr, data,
3670 					     msr_info->host_initiated);
3671 	case MSR_IA32_BBL_CR_CTL3:
3672 		/* Drop writes to this legacy MSR -- see rdmsr
3673 		 * counterpart for further detail.
3674 		 */
3675 		if (report_ignored_msrs)
3676 			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
3677 				msr, data);
3678 		break;
3679 	case MSR_AMD64_OSVW_ID_LENGTH:
3680 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3681 			return 1;
3682 		vcpu->arch.osvw.length = data;
3683 		break;
3684 	case MSR_AMD64_OSVW_STATUS:
3685 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3686 			return 1;
3687 		vcpu->arch.osvw.status = data;
3688 		break;
3689 	case MSR_PLATFORM_INFO:
3690 		if (!msr_info->host_initiated ||
3691 		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
3692 		     cpuid_fault_enabled(vcpu)))
3693 			return 1;
3694 		vcpu->arch.msr_platform_info = data;
3695 		break;
3696 	case MSR_MISC_FEATURES_ENABLES:
3697 		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
3698 		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3699 		     !supports_cpuid_fault(vcpu)))
3700 			return 1;
3701 		vcpu->arch.msr_misc_features_enables = data;
3702 		break;
3703 #ifdef CONFIG_X86_64
3704 	case MSR_IA32_XFD:
3705 		if (!msr_info->host_initiated &&
3706 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3707 			return 1;
3708 
3709 		if (data & ~(XFEATURE_MASK_USER_DYNAMIC &
3710 			     vcpu->arch.guest_supported_xcr0))
3711 			return 1;
3712 
3713 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
3714 		break;
3715 	case MSR_IA32_XFD_ERR:
3716 		if (!msr_info->host_initiated &&
3717 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3718 			return 1;
3719 
3720 		if (data & ~(XFEATURE_MASK_USER_DYNAMIC &
3721 			     vcpu->arch.guest_supported_xcr0))
3722 			return 1;
3723 
3724 		vcpu->arch.guest_fpu.xfd_err = data;
3725 		break;
3726 #endif
3727 	default:
3728 		if (kvm_pmu_is_valid_msr(vcpu, msr))
3729 			return kvm_pmu_set_msr(vcpu, msr_info);
3730 		return KVM_MSR_RET_INVALID;
3731 	}
3732 	return 0;
3733 }
3734 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
3735 
3736 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
3737 {
3738 	u64 data;
3739 	u64 mcg_cap = vcpu->arch.mcg_cap;
3740 	unsigned bank_num = mcg_cap & 0xff;
3741 
3742 	switch (msr) {
3743 	case MSR_IA32_P5_MC_ADDR:
3744 	case MSR_IA32_P5_MC_TYPE:
3745 		data = 0;
3746 		break;
3747 	case MSR_IA32_MCG_CAP:
3748 		data = vcpu->arch.mcg_cap;
3749 		break;
3750 	case MSR_IA32_MCG_CTL:
3751 		if (!(mcg_cap & MCG_CTL_P) && !host)
3752 			return 1;
3753 		data = vcpu->arch.mcg_ctl;
3754 		break;
3755 	case MSR_IA32_MCG_STATUS:
3756 		data = vcpu->arch.mcg_status;
3757 		break;
3758 	default:
3759 		if (msr >= MSR_IA32_MC0_CTL &&
3760 		    msr < MSR_IA32_MCx_CTL(bank_num)) {
3761 			u32 offset = array_index_nospec(
3762 				msr - MSR_IA32_MC0_CTL,
3763 				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);
3764 
3765 			data = vcpu->arch.mce_banks[offset];
3766 			break;
3767 		}
3768 		return 1;
3769 	}
3770 	*pdata = data;
3771 	return 0;
3772 }
3773 
3774 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3775 {
3776 	switch (msr_info->index) {
3777 	case MSR_IA32_PLATFORM_ID:
3778 	case MSR_IA32_EBL_CR_POWERON:
3779 	case MSR_IA32_LASTBRANCHFROMIP:
3780 	case MSR_IA32_LASTBRANCHTOIP:
3781 	case MSR_IA32_LASTINTFROMIP:
3782 	case MSR_IA32_LASTINTTOIP:
3783 	case MSR_AMD64_SYSCFG:
3784 	case MSR_K8_TSEG_ADDR:
3785 	case MSR_K8_TSEG_MASK:
3786 	case MSR_VM_HSAVE_PA:
3787 	case MSR_K8_INT_PENDING_MSG:
3788 	case MSR_AMD64_NB_CFG:
3789 	case MSR_FAM10H_MMIO_CONF_BASE:
3790 	case MSR_AMD64_BU_CFG2:
3791 	case MSR_IA32_PERF_CTL:
3792 	case MSR_AMD64_DC_CFG:
3793 	case MSR_F15H_EX_CFG:
3794 	/*
3795 	 * Intel Sandy Bridge CPUs must support the RAPL (running average power
3796 	 * limit) MSRs. Just return 0, as we do not want to expose the host
3797 	 * data here. Do not conditionalize this on CPUID, as KVM does not do
3798 	 * so for existing CPU-specific MSRs.
3799 	 */
3800 	case MSR_RAPL_POWER_UNIT:
3801 	case MSR_PP0_ENERGY_STATUS:	/* Power plane 0 (core) */
3802 	case MSR_PP1_ENERGY_STATUS:	/* Power plane 1 (graphics uncore) */
3803 	case MSR_PKG_ENERGY_STATUS:	/* Total package */
3804 	case MSR_DRAM_ENERGY_STATUS:	/* DRAM controller */
3805 		msr_info->data = 0;
3806 		break;
3807 	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3808 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3809 			return kvm_pmu_get_msr(vcpu, msr_info);
3810 		if (!msr_info->host_initiated)
3811 			return 1;
3812 		msr_info->data = 0;
3813 		break;
3814 	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3815 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3816 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3817 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3818 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3819 			return kvm_pmu_get_msr(vcpu, msr_info);
3820 		msr_info->data = 0;
3821 		break;
3822 	case MSR_IA32_UCODE_REV:
3823 		msr_info->data = vcpu->arch.microcode_version;
3824 		break;
3825 	case MSR_IA32_ARCH_CAPABILITIES:
3826 		if (!msr_info->host_initiated &&
3827 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
3828 			return 1;
3829 		msr_info->data = vcpu->arch.arch_capabilities;
3830 		break;
3831 	case MSR_IA32_PERF_CAPABILITIES:
3832 		if (!msr_info->host_initiated &&
3833 		    !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
3834 			return 1;
3835 		msr_info->data = vcpu->arch.perf_capabilities;
3836 		break;
3837 	case MSR_IA32_POWER_CTL:
3838 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
3839 		break;
3840 	case MSR_IA32_TSC: {
3841 		/*
3842 		 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
3843 		 * even when not intercepted. AMD manual doesn't explicitly
3844 		 * state this but appears to behave the same.
3845 		 *
3846 		 * On userspace reads and writes, however, we unconditionally
3847 		 * return L1's TSC value to ensure backwards-compatible
3848 		 * behavior for migration.
3849 		 */
3850 		u64 offset, ratio;
3851 
3852 		if (msr_info->host_initiated) {
3853 			offset = vcpu->arch.l1_tsc_offset;
3854 			ratio = vcpu->arch.l1_tsc_scaling_ratio;
3855 		} else {
3856 			offset = vcpu->arch.tsc_offset;
3857 			ratio = vcpu->arch.tsc_scaling_ratio;
3858 		}
3859 
3860 		msr_info->data = kvm_scale_tsc(vcpu, rdtsc(), ratio) + offset;
3861 		break;
3862 	}
3863 	case MSR_MTRRcap:
3864 	case 0x200 ... 0x2ff:
3865 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
3866 	case 0xcd: /* fsb frequency */
3867 		msr_info->data = 3;
3868 		break;
3869 		/*
3870 		 * MSR_EBC_FREQUENCY_ID
3871 		 * Conservative value valid for even the basic CPU models.
3872 		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
3873 		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
3874 		 * and 266MHz for model 3, or 4. Set Core Clock
3875 		 * Frequency to System Bus Frequency Ratio to 1 (bits
3876 		 * 31:24) even though these are only valid for CPU
3877 		 * models > 2, however guests may end up dividing or
3878 		 * multiplying by zero otherwise.
3879 		 */
3880 	case MSR_EBC_FREQUENCY_ID:
3881 		msr_info->data = 1 << 24;
3882 		break;
3883 	case MSR_IA32_APICBASE:
3884 		msr_info->data = kvm_get_apic_base(vcpu);
3885 		break;
3886 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
3887 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
3888 	case MSR_IA32_TSC_DEADLINE:
3889 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
3890 		break;
3891 	case MSR_IA32_TSC_ADJUST:
3892 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
3893 		break;
3894 	case MSR_IA32_MISC_ENABLE:
3895 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3896 		break;
3897 	case MSR_IA32_SMBASE:
3898 		if (!msr_info->host_initiated)
3899 			return 1;
3900 		msr_info->data = vcpu->arch.smbase;
3901 		break;
3902 	case MSR_SMI_COUNT:
3903 		msr_info->data = vcpu->arch.smi_count;
3904 		break;
3905 	case MSR_IA32_PERF_STATUS:
3906 		/* TSC increment by tick */
3907 		msr_info->data = 1000ULL;
3908 		/* CPU multiplier */
3909 		msr_info->data |= (((uint64_t)4ULL) << 40);
3910 		break;
3911 	case MSR_EFER:
3912 		msr_info->data = vcpu->arch.efer;
3913 		break;
3914 	case MSR_KVM_WALL_CLOCK:
3915 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3916 			return 1;
3917 
3918 		msr_info->data = vcpu->kvm->arch.wall_clock;
3919 		break;
3920 	case MSR_KVM_WALL_CLOCK_NEW:
3921 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3922 			return 1;
3923 
3924 		msr_info->data = vcpu->kvm->arch.wall_clock;
3925 		break;
3926 	case MSR_KVM_SYSTEM_TIME:
3927 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3928 			return 1;
3929 
3930 		msr_info->data = vcpu->arch.time;
3931 		break;
3932 	case MSR_KVM_SYSTEM_TIME_NEW:
3933 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3934 			return 1;
3935 
3936 		msr_info->data = vcpu->arch.time;
3937 		break;
3938 	case MSR_KVM_ASYNC_PF_EN:
3939 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3940 			return 1;
3941 
3942 		msr_info->data = vcpu->arch.apf.msr_en_val;
3943 		break;
3944 	case MSR_KVM_ASYNC_PF_INT:
3945 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3946 			return 1;
3947 
3948 		msr_info->data = vcpu->arch.apf.msr_int_val;
3949 		break;
3950 	case MSR_KVM_ASYNC_PF_ACK:
3951 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3952 			return 1;
3953 
3954 		msr_info->data = 0;
3955 		break;
3956 	case MSR_KVM_STEAL_TIME:
3957 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3958 			return 1;
3959 
3960 		msr_info->data = vcpu->arch.st.msr_val;
3961 		break;
3962 	case MSR_KVM_PV_EOI_EN:
3963 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3964 			return 1;
3965 
3966 		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3967 		break;
3968 	case MSR_KVM_POLL_CONTROL:
3969 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3970 			return 1;
3971 
3972 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
3973 		break;
3974 	case MSR_IA32_P5_MC_ADDR:
3975 	case MSR_IA32_P5_MC_TYPE:
3976 	case MSR_IA32_MCG_CAP:
3977 	case MSR_IA32_MCG_CTL:
3978 	case MSR_IA32_MCG_STATUS:
3979 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3980 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
3981 				   msr_info->host_initiated);
3982 	case MSR_IA32_XSS:
3983 		if (!msr_info->host_initiated &&
3984 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3985 			return 1;
3986 		msr_info->data = vcpu->arch.ia32_xss;
3987 		break;
3988 	case MSR_K7_CLK_CTL:
3989 		/*
3990 		 * Provide expected ramp-up count for K7. All other
3991 		 * are set to zero, indicating minimum divisors for
3992 		 * every field.
3993 		 *
3994 		 * This prevents guest kernels on AMD host with CPU
3995 		 * type 6, model 8 and higher from exploding due to
3996 		 * the rdmsr failing.
3997 		 */
3998 		msr_info->data = 0x20000000;
3999 		break;
4000 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
4001 	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
4002 	case HV_X64_MSR_SYNDBG_OPTIONS:
4003 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
4004 	case HV_X64_MSR_CRASH_CTL:
4005 	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
4006 	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
4007 	case HV_X64_MSR_TSC_EMULATION_CONTROL:
4008 	case HV_X64_MSR_TSC_EMULATION_STATUS:
4009 		return kvm_hv_get_msr_common(vcpu,
4010 					     msr_info->index, &msr_info->data,
4011 					     msr_info->host_initiated);
4012 	case MSR_IA32_BBL_CR_CTL3:
4013 		/* This legacy MSR exists but isn't fully documented in current
4014 		 * silicon.  It is however accessed by winxp in very narrow
4015 		 * scenarios where it sets bit #19, itself documented as
4016 		 * a "reserved" bit.  Best effort attempt to source coherent
4017 		 * read data here should the balance of the register be
4018 		 * interpreted by the guest:
4019 		 *
4020 		 * L2 cache control register 3: 64GB range, 256KB size,
4021 		 * enabled, latency 0x1, configured
4022 		 */
4023 		msr_info->data = 0xbe702111;
4024 		break;
4025 	case MSR_AMD64_OSVW_ID_LENGTH:
4026 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4027 			return 1;
4028 		msr_info->data = vcpu->arch.osvw.length;
4029 		break;
4030 	case MSR_AMD64_OSVW_STATUS:
4031 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4032 			return 1;
4033 		msr_info->data = vcpu->arch.osvw.status;
4034 		break;
4035 	case MSR_PLATFORM_INFO:
4036 		if (!msr_info->host_initiated &&
4037 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
4038 			return 1;
4039 		msr_info->data = vcpu->arch.msr_platform_info;
4040 		break;
4041 	case MSR_MISC_FEATURES_ENABLES:
4042 		msr_info->data = vcpu->arch.msr_misc_features_enables;
4043 		break;
4044 	case MSR_K7_HWCR:
4045 		msr_info->data = vcpu->arch.msr_hwcr;
4046 		break;
4047 #ifdef CONFIG_X86_64
4048 	case MSR_IA32_XFD:
4049 		if (!msr_info->host_initiated &&
4050 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4051 			return 1;
4052 
4053 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
4054 		break;
4055 	case MSR_IA32_XFD_ERR:
4056 		if (!msr_info->host_initiated &&
4057 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4058 			return 1;
4059 
4060 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4061 		break;
4062 #endif
4063 	default:
4064 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4065 			return kvm_pmu_get_msr(vcpu, msr_info);
4066 		return KVM_MSR_RET_INVALID;
4067 	}
4068 	return 0;
4069 }
4070 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
4071 
4072 /*
4073  * Read or write a bunch of msrs. All parameters are kernel addresses.
4074  *
4075  * @return number of msrs set successfully.
4076  */
4077 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
4078 		    struct kvm_msr_entry *entries,
4079 		    int (*do_msr)(struct kvm_vcpu *vcpu,
4080 				  unsigned index, u64 *data))
4081 {
4082 	int i;
4083 
4084 	for (i = 0; i < msrs->nmsrs; ++i)
4085 		if (do_msr(vcpu, entries[i].index, &entries[i].data))
4086 			break;
4087 
4088 	return i;
4089 }
4090 
4091 /*
4092  * Read or write a bunch of msrs. Parameters are user addresses.
4093  *
4094  * @return number of msrs set successfully.
4095  */
4096 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
4097 		  int (*do_msr)(struct kvm_vcpu *vcpu,
4098 				unsigned index, u64 *data),
4099 		  int writeback)
4100 {
4101 	struct kvm_msrs msrs;
4102 	struct kvm_msr_entry *entries;
4103 	int r, n;
4104 	unsigned size;
4105 
4106 	r = -EFAULT;
4107 	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
4108 		goto out;
4109 
4110 	r = -E2BIG;
4111 	if (msrs.nmsrs >= MAX_IO_MSRS)
4112 		goto out;
4113 
4114 	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
4115 	entries = memdup_user(user_msrs->entries, size);
4116 	if (IS_ERR(entries)) {
4117 		r = PTR_ERR(entries);
4118 		goto out;
4119 	}
4120 
4121 	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
4122 	if (r < 0)
4123 		goto out_free;
4124 
4125 	r = -EFAULT;
4126 	if (writeback && copy_to_user(user_msrs->entries, entries, size))
4127 		goto out_free;
4128 
4129 	r = n;
4130 
4131 out_free:
4132 	kfree(entries);
4133 out:
4134 	return r;
4135 }
4136 
4137 static inline bool kvm_can_mwait_in_guest(void)
4138 {
4139 	return boot_cpu_has(X86_FEATURE_MWAIT) &&
4140 		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
4141 		boot_cpu_has(X86_FEATURE_ARAT);
4142 }
4143 
4144 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
4145 					    struct kvm_cpuid2 __user *cpuid_arg)
4146 {
4147 	struct kvm_cpuid2 cpuid;
4148 	int r;
4149 
4150 	r = -EFAULT;
4151 	if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4152 		return r;
4153 
4154 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4155 	if (r)
4156 		return r;
4157 
4158 	r = -EFAULT;
4159 	if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4160 		return r;
4161 
4162 	return 0;
4163 }
4164 
4165 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
4166 {
4167 	int r = 0;
4168 
4169 	switch (ext) {
4170 	case KVM_CAP_IRQCHIP:
4171 	case KVM_CAP_HLT:
4172 	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
4173 	case KVM_CAP_SET_TSS_ADDR:
4174 	case KVM_CAP_EXT_CPUID:
4175 	case KVM_CAP_EXT_EMUL_CPUID:
4176 	case KVM_CAP_CLOCKSOURCE:
4177 	case KVM_CAP_PIT:
4178 	case KVM_CAP_NOP_IO_DELAY:
4179 	case KVM_CAP_MP_STATE:
4180 	case KVM_CAP_SYNC_MMU:
4181 	case KVM_CAP_USER_NMI:
4182 	case KVM_CAP_REINJECT_CONTROL:
4183 	case KVM_CAP_IRQ_INJECT_STATUS:
4184 	case KVM_CAP_IOEVENTFD:
4185 	case KVM_CAP_IOEVENTFD_NO_LENGTH:
4186 	case KVM_CAP_PIT2:
4187 	case KVM_CAP_PIT_STATE2:
4188 	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
4189 	case KVM_CAP_VCPU_EVENTS:
4190 	case KVM_CAP_HYPERV:
4191 	case KVM_CAP_HYPERV_VAPIC:
4192 	case KVM_CAP_HYPERV_SPIN:
4193 	case KVM_CAP_HYPERV_SYNIC:
4194 	case KVM_CAP_HYPERV_SYNIC2:
4195 	case KVM_CAP_HYPERV_VP_INDEX:
4196 	case KVM_CAP_HYPERV_EVENTFD:
4197 	case KVM_CAP_HYPERV_TLBFLUSH:
4198 	case KVM_CAP_HYPERV_SEND_IPI:
4199 	case KVM_CAP_HYPERV_CPUID:
4200 	case KVM_CAP_HYPERV_ENFORCE_CPUID:
4201 	case KVM_CAP_SYS_HYPERV_CPUID:
4202 	case KVM_CAP_PCI_SEGMENT:
4203 	case KVM_CAP_DEBUGREGS:
4204 	case KVM_CAP_X86_ROBUST_SINGLESTEP:
4205 	case KVM_CAP_XSAVE:
4206 	case KVM_CAP_ASYNC_PF:
4207 	case KVM_CAP_ASYNC_PF_INT:
4208 	case KVM_CAP_GET_TSC_KHZ:
4209 	case KVM_CAP_KVMCLOCK_CTRL:
4210 	case KVM_CAP_READONLY_MEM:
4211 	case KVM_CAP_HYPERV_TIME:
4212 	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
4213 	case KVM_CAP_TSC_DEADLINE_TIMER:
4214 	case KVM_CAP_DISABLE_QUIRKS:
4215 	case KVM_CAP_SET_BOOT_CPU_ID:
4216  	case KVM_CAP_SPLIT_IRQCHIP:
4217 	case KVM_CAP_IMMEDIATE_EXIT:
4218 	case KVM_CAP_PMU_EVENT_FILTER:
4219 	case KVM_CAP_GET_MSR_FEATURES:
4220 	case KVM_CAP_MSR_PLATFORM_INFO:
4221 	case KVM_CAP_EXCEPTION_PAYLOAD:
4222 	case KVM_CAP_SET_GUEST_DEBUG:
4223 	case KVM_CAP_LAST_CPU:
4224 	case KVM_CAP_X86_USER_SPACE_MSR:
4225 	case KVM_CAP_X86_MSR_FILTER:
4226 	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4227 #ifdef CONFIG_X86_SGX_KVM
4228 	case KVM_CAP_SGX_ATTRIBUTE:
4229 #endif
4230 	case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
4231 	case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
4232 	case KVM_CAP_SREGS2:
4233 	case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
4234 	case KVM_CAP_VCPU_ATTRIBUTES:
4235 	case KVM_CAP_SYS_ATTRIBUTES:
4236 		r = 1;
4237 		break;
4238 	case KVM_CAP_EXIT_HYPERCALL:
4239 		r = KVM_EXIT_HYPERCALL_VALID_MASK;
4240 		break;
4241 	case KVM_CAP_SET_GUEST_DEBUG2:
4242 		return KVM_GUESTDBG_VALID_MASK;
4243 #ifdef CONFIG_KVM_XEN
4244 	case KVM_CAP_XEN_HVM:
4245 		r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR |
4246 		    KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL |
4247 		    KVM_XEN_HVM_CONFIG_SHARED_INFO |
4248 		    KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL;
4249 		if (sched_info_on())
4250 			r |= KVM_XEN_HVM_CONFIG_RUNSTATE;
4251 		break;
4252 #endif
4253 	case KVM_CAP_SYNC_REGS:
4254 		r = KVM_SYNC_X86_VALID_FIELDS;
4255 		break;
4256 	case KVM_CAP_ADJUST_CLOCK:
4257 		r = KVM_CLOCK_VALID_FLAGS;
4258 		break;
4259 	case KVM_CAP_X86_DISABLE_EXITS:
4260 		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
4261 		      KVM_X86_DISABLE_EXITS_CSTATE;
4262 		if(kvm_can_mwait_in_guest())
4263 			r |= KVM_X86_DISABLE_EXITS_MWAIT;
4264 		break;
4265 	case KVM_CAP_X86_SMM:
4266 		/* SMBASE is usually relocated above 1M on modern chipsets,
4267 		 * and SMM handlers might indeed rely on 4G segment limits,
4268 		 * so do not report SMM to be available if real mode is
4269 		 * emulated via vm86 mode.  Still, do not go to great lengths
4270 		 * to avoid userspace's usage of the feature, because it is a
4271 		 * fringe case that is not enabled except via specific settings
4272 		 * of the module parameters.
4273 		 */
4274 		r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE);
4275 		break;
4276 	case KVM_CAP_VAPIC:
4277 		r = !static_call(kvm_x86_cpu_has_accelerated_tpr)();
4278 		break;
4279 	case KVM_CAP_NR_VCPUS:
4280 		r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
4281 		break;
4282 	case KVM_CAP_MAX_VCPUS:
4283 		r = KVM_MAX_VCPUS;
4284 		break;
4285 	case KVM_CAP_MAX_VCPU_ID:
4286 		r = KVM_MAX_VCPU_IDS;
4287 		break;
4288 	case KVM_CAP_PV_MMU:	/* obsolete */
4289 		r = 0;
4290 		break;
4291 	case KVM_CAP_MCE:
4292 		r = KVM_MAX_MCE_BANKS;
4293 		break;
4294 	case KVM_CAP_XCRS:
4295 		r = boot_cpu_has(X86_FEATURE_XSAVE);
4296 		break;
4297 	case KVM_CAP_TSC_CONTROL:
4298 		r = kvm_has_tsc_control;
4299 		break;
4300 	case KVM_CAP_X2APIC_API:
4301 		r = KVM_X2APIC_API_VALID_FLAGS;
4302 		break;
4303 	case KVM_CAP_NESTED_STATE:
4304 		r = kvm_x86_ops.nested_ops->get_state ?
4305 			kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
4306 		break;
4307 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
4308 		r = kvm_x86_ops.enable_direct_tlbflush != NULL;
4309 		break;
4310 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4311 		r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
4312 		break;
4313 	case KVM_CAP_SMALLER_MAXPHYADDR:
4314 		r = (int) allow_smaller_maxphyaddr;
4315 		break;
4316 	case KVM_CAP_STEAL_TIME:
4317 		r = sched_info_on();
4318 		break;
4319 	case KVM_CAP_X86_BUS_LOCK_EXIT:
4320 		if (kvm_has_bus_lock_exit)
4321 			r = KVM_BUS_LOCK_DETECTION_OFF |
4322 			    KVM_BUS_LOCK_DETECTION_EXIT;
4323 		else
4324 			r = 0;
4325 		break;
4326 	case KVM_CAP_XSAVE2: {
4327 		u64 guest_perm = xstate_get_guest_group_perm();
4328 
4329 		r = xstate_required_size(supported_xcr0 & guest_perm, false);
4330 		if (r < sizeof(struct kvm_xsave))
4331 			r = sizeof(struct kvm_xsave);
4332 		break;
4333 	}
4334 	default:
4335 		break;
4336 	}
4337 	return r;
4338 }
4339 
4340 static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr)
4341 {
4342 	void __user *uaddr = (void __user*)(unsigned long)attr->addr;
4343 
4344 	if ((u64)(unsigned long)uaddr != attr->addr)
4345 		return ERR_PTR_USR(-EFAULT);
4346 	return uaddr;
4347 }
4348 
4349 static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr)
4350 {
4351 	u64 __user *uaddr = kvm_get_attr_addr(attr);
4352 
4353 	if (attr->group)
4354 		return -ENXIO;
4355 
4356 	if (IS_ERR(uaddr))
4357 		return PTR_ERR(uaddr);
4358 
4359 	switch (attr->attr) {
4360 	case KVM_X86_XCOMP_GUEST_SUPP:
4361 		if (put_user(supported_xcr0, uaddr))
4362 			return -EFAULT;
4363 		return 0;
4364 	default:
4365 		return -ENXIO;
4366 		break;
4367 	}
4368 }
4369 
4370 static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr)
4371 {
4372 	if (attr->group)
4373 		return -ENXIO;
4374 
4375 	switch (attr->attr) {
4376 	case KVM_X86_XCOMP_GUEST_SUPP:
4377 		return 0;
4378 	default:
4379 		return -ENXIO;
4380 	}
4381 }
4382 
4383 long kvm_arch_dev_ioctl(struct file *filp,
4384 			unsigned int ioctl, unsigned long arg)
4385 {
4386 	void __user *argp = (void __user *)arg;
4387 	long r;
4388 
4389 	switch (ioctl) {
4390 	case KVM_GET_MSR_INDEX_LIST: {
4391 		struct kvm_msr_list __user *user_msr_list = argp;
4392 		struct kvm_msr_list msr_list;
4393 		unsigned n;
4394 
4395 		r = -EFAULT;
4396 		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4397 			goto out;
4398 		n = msr_list.nmsrs;
4399 		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
4400 		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4401 			goto out;
4402 		r = -E2BIG;
4403 		if (n < msr_list.nmsrs)
4404 			goto out;
4405 		r = -EFAULT;
4406 		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
4407 				 num_msrs_to_save * sizeof(u32)))
4408 			goto out;
4409 		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
4410 				 &emulated_msrs,
4411 				 num_emulated_msrs * sizeof(u32)))
4412 			goto out;
4413 		r = 0;
4414 		break;
4415 	}
4416 	case KVM_GET_SUPPORTED_CPUID:
4417 	case KVM_GET_EMULATED_CPUID: {
4418 		struct kvm_cpuid2 __user *cpuid_arg = argp;
4419 		struct kvm_cpuid2 cpuid;
4420 
4421 		r = -EFAULT;
4422 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4423 			goto out;
4424 
4425 		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
4426 					    ioctl);
4427 		if (r)
4428 			goto out;
4429 
4430 		r = -EFAULT;
4431 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4432 			goto out;
4433 		r = 0;
4434 		break;
4435 	}
4436 	case KVM_X86_GET_MCE_CAP_SUPPORTED:
4437 		r = -EFAULT;
4438 		if (copy_to_user(argp, &kvm_mce_cap_supported,
4439 				 sizeof(kvm_mce_cap_supported)))
4440 			goto out;
4441 		r = 0;
4442 		break;
4443 	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
4444 		struct kvm_msr_list __user *user_msr_list = argp;
4445 		struct kvm_msr_list msr_list;
4446 		unsigned int n;
4447 
4448 		r = -EFAULT;
4449 		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4450 			goto out;
4451 		n = msr_list.nmsrs;
4452 		msr_list.nmsrs = num_msr_based_features;
4453 		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4454 			goto out;
4455 		r = -E2BIG;
4456 		if (n < msr_list.nmsrs)
4457 			goto out;
4458 		r = -EFAULT;
4459 		if (copy_to_user(user_msr_list->indices, &msr_based_features,
4460 				 num_msr_based_features * sizeof(u32)))
4461 			goto out;
4462 		r = 0;
4463 		break;
4464 	}
4465 	case KVM_GET_MSRS:
4466 		r = msr_io(NULL, argp, do_get_msr_feature, 1);
4467 		break;
4468 	case KVM_GET_SUPPORTED_HV_CPUID:
4469 		r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp);
4470 		break;
4471 	case KVM_GET_DEVICE_ATTR: {
4472 		struct kvm_device_attr attr;
4473 		r = -EFAULT;
4474 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4475 			break;
4476 		r = kvm_x86_dev_get_attr(&attr);
4477 		break;
4478 	}
4479 	case KVM_HAS_DEVICE_ATTR: {
4480 		struct kvm_device_attr attr;
4481 		r = -EFAULT;
4482 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4483 			break;
4484 		r = kvm_x86_dev_has_attr(&attr);
4485 		break;
4486 	}
4487 	default:
4488 		r = -EINVAL;
4489 		break;
4490 	}
4491 out:
4492 	return r;
4493 }
4494 
4495 static void wbinvd_ipi(void *garbage)
4496 {
4497 	wbinvd();
4498 }
4499 
4500 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
4501 {
4502 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
4503 }
4504 
4505 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
4506 {
4507 	/* Address WBINVD may be executed by guest */
4508 	if (need_emulate_wbinvd(vcpu)) {
4509 		if (static_call(kvm_x86_has_wbinvd_exit)())
4510 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4511 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
4512 			smp_call_function_single(vcpu->cpu,
4513 					wbinvd_ipi, NULL, 1);
4514 	}
4515 
4516 	static_call(kvm_x86_vcpu_load)(vcpu, cpu);
4517 
4518 	/* Save host pkru register if supported */
4519 	vcpu->arch.host_pkru = read_pkru();
4520 
4521 	/* Apply any externally detected TSC adjustments (due to suspend) */
4522 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
4523 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
4524 		vcpu->arch.tsc_offset_adjustment = 0;
4525 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4526 	}
4527 
4528 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
4529 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
4530 				rdtsc() - vcpu->arch.last_host_tsc;
4531 		if (tsc_delta < 0)
4532 			mark_tsc_unstable("KVM discovered backwards TSC");
4533 
4534 		if (kvm_check_tsc_unstable()) {
4535 			u64 offset = kvm_compute_l1_tsc_offset(vcpu,
4536 						vcpu->arch.last_guest_tsc);
4537 			kvm_vcpu_write_tsc_offset(vcpu, offset);
4538 			vcpu->arch.tsc_catchup = 1;
4539 		}
4540 
4541 		if (kvm_lapic_hv_timer_in_use(vcpu))
4542 			kvm_lapic_restart_hv_timer(vcpu);
4543 
4544 		/*
4545 		 * On a host with synchronized TSC, there is no need to update
4546 		 * kvmclock on vcpu->cpu migration
4547 		 */
4548 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
4549 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
4550 		if (vcpu->cpu != cpu)
4551 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
4552 		vcpu->cpu = cpu;
4553 	}
4554 
4555 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
4556 }
4557 
4558 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4559 {
4560 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4561 	struct kvm_steal_time __user *st;
4562 	struct kvm_memslots *slots;
4563 	static const u8 preempted = KVM_VCPU_PREEMPTED;
4564 
4565 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4566 		return;
4567 
4568 	if (vcpu->arch.st.preempted)
4569 		return;
4570 
4571 	/* This happens on process exit */
4572 	if (unlikely(current->mm != vcpu->kvm->mm))
4573 		return;
4574 
4575 	slots = kvm_memslots(vcpu->kvm);
4576 
4577 	if (unlikely(slots->generation != ghc->generation ||
4578 		     kvm_is_error_hva(ghc->hva) || !ghc->memslot))
4579 		return;
4580 
4581 	st = (struct kvm_steal_time __user *)ghc->hva;
4582 	BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted));
4583 
4584 	if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted)))
4585 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4586 
4587 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
4588 }
4589 
4590 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4591 {
4592 	int idx;
4593 
4594 	if (vcpu->preempted && !vcpu->arch.guest_state_protected)
4595 		vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4596 
4597 	/*
4598 	 * Take the srcu lock as memslots will be accessed to check the gfn
4599 	 * cache generation against the memslots generation.
4600 	 */
4601 	idx = srcu_read_lock(&vcpu->kvm->srcu);
4602 	if (kvm_xen_msr_enabled(vcpu->kvm))
4603 		kvm_xen_runstate_set_preempted(vcpu);
4604 	else
4605 		kvm_steal_time_set_preempted(vcpu);
4606 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
4607 
4608 	static_call(kvm_x86_vcpu_put)(vcpu);
4609 	vcpu->arch.last_host_tsc = rdtsc();
4610 }
4611 
4612 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4613 				    struct kvm_lapic_state *s)
4614 {
4615 	static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
4616 
4617 	return kvm_apic_get_state(vcpu, s);
4618 }
4619 
4620 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
4621 				    struct kvm_lapic_state *s)
4622 {
4623 	int r;
4624 
4625 	r = kvm_apic_set_state(vcpu, s);
4626 	if (r)
4627 		return r;
4628 	update_cr8_intercept(vcpu);
4629 
4630 	return 0;
4631 }
4632 
4633 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
4634 {
4635 	/*
4636 	 * We can accept userspace's request for interrupt injection
4637 	 * as long as we have a place to store the interrupt number.
4638 	 * The actual injection will happen when the CPU is able to
4639 	 * deliver the interrupt.
4640 	 */
4641 	if (kvm_cpu_has_extint(vcpu))
4642 		return false;
4643 
4644 	/* Acknowledging ExtINT does not happen if LINT0 is masked.  */
4645 	return (!lapic_in_kernel(vcpu) ||
4646 		kvm_apic_accept_pic_intr(vcpu));
4647 }
4648 
4649 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
4650 {
4651 	/*
4652 	 * Do not cause an interrupt window exit if an exception
4653 	 * is pending or an event needs reinjection; userspace
4654 	 * might want to inject the interrupt manually using KVM_SET_REGS
4655 	 * or KVM_SET_SREGS.  For that to work, we must be at an
4656 	 * instruction boundary and with no events half-injected.
4657 	 */
4658 	return (kvm_arch_interrupt_allowed(vcpu) &&
4659 		kvm_cpu_accept_dm_intr(vcpu) &&
4660 		!kvm_event_needs_reinjection(vcpu) &&
4661 		!vcpu->arch.exception.pending);
4662 }
4663 
4664 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
4665 				    struct kvm_interrupt *irq)
4666 {
4667 	if (irq->irq >= KVM_NR_INTERRUPTS)
4668 		return -EINVAL;
4669 
4670 	if (!irqchip_in_kernel(vcpu->kvm)) {
4671 		kvm_queue_interrupt(vcpu, irq->irq, false);
4672 		kvm_make_request(KVM_REQ_EVENT, vcpu);
4673 		return 0;
4674 	}
4675 
4676 	/*
4677 	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
4678 	 * fail for in-kernel 8259.
4679 	 */
4680 	if (pic_in_kernel(vcpu->kvm))
4681 		return -ENXIO;
4682 
4683 	if (vcpu->arch.pending_external_vector != -1)
4684 		return -EEXIST;
4685 
4686 	vcpu->arch.pending_external_vector = irq->irq;
4687 	kvm_make_request(KVM_REQ_EVENT, vcpu);
4688 	return 0;
4689 }
4690 
4691 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
4692 {
4693 	kvm_inject_nmi(vcpu);
4694 
4695 	return 0;
4696 }
4697 
4698 static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
4699 {
4700 	kvm_make_request(KVM_REQ_SMI, vcpu);
4701 
4702 	return 0;
4703 }
4704 
4705 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
4706 					   struct kvm_tpr_access_ctl *tac)
4707 {
4708 	if (tac->flags)
4709 		return -EINVAL;
4710 	vcpu->arch.tpr_access_reporting = !!tac->enabled;
4711 	return 0;
4712 }
4713 
4714 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
4715 					u64 mcg_cap)
4716 {
4717 	int r;
4718 	unsigned bank_num = mcg_cap & 0xff, bank;
4719 
4720 	r = -EINVAL;
4721 	if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
4722 		goto out;
4723 	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
4724 		goto out;
4725 	r = 0;
4726 	vcpu->arch.mcg_cap = mcg_cap;
4727 	/* Init IA32_MCG_CTL to all 1s */
4728 	if (mcg_cap & MCG_CTL_P)
4729 		vcpu->arch.mcg_ctl = ~(u64)0;
4730 	/* Init IA32_MCi_CTL to all 1s */
4731 	for (bank = 0; bank < bank_num; bank++)
4732 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
4733 
4734 	static_call(kvm_x86_setup_mce)(vcpu);
4735 out:
4736 	return r;
4737 }
4738 
4739 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
4740 				      struct kvm_x86_mce *mce)
4741 {
4742 	u64 mcg_cap = vcpu->arch.mcg_cap;
4743 	unsigned bank_num = mcg_cap & 0xff;
4744 	u64 *banks = vcpu->arch.mce_banks;
4745 
4746 	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
4747 		return -EINVAL;
4748 	/*
4749 	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
4750 	 * reporting is disabled
4751 	 */
4752 	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
4753 	    vcpu->arch.mcg_ctl != ~(u64)0)
4754 		return 0;
4755 	banks += 4 * mce->bank;
4756 	/*
4757 	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
4758 	 * reporting is disabled for the bank
4759 	 */
4760 	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
4761 		return 0;
4762 	if (mce->status & MCI_STATUS_UC) {
4763 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
4764 		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
4765 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4766 			return 0;
4767 		}
4768 		if (banks[1] & MCI_STATUS_VAL)
4769 			mce->status |= MCI_STATUS_OVER;
4770 		banks[2] = mce->addr;
4771 		banks[3] = mce->misc;
4772 		vcpu->arch.mcg_status = mce->mcg_status;
4773 		banks[1] = mce->status;
4774 		kvm_queue_exception(vcpu, MC_VECTOR);
4775 	} else if (!(banks[1] & MCI_STATUS_VAL)
4776 		   || !(banks[1] & MCI_STATUS_UC)) {
4777 		if (banks[1] & MCI_STATUS_VAL)
4778 			mce->status |= MCI_STATUS_OVER;
4779 		banks[2] = mce->addr;
4780 		banks[3] = mce->misc;
4781 		banks[1] = mce->status;
4782 	} else
4783 		banks[1] |= MCI_STATUS_OVER;
4784 	return 0;
4785 }
4786 
4787 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
4788 					       struct kvm_vcpu_events *events)
4789 {
4790 	process_nmi(vcpu);
4791 
4792 	if (kvm_check_request(KVM_REQ_SMI, vcpu))
4793 		process_smi(vcpu);
4794 
4795 	/*
4796 	 * In guest mode, payload delivery should be deferred,
4797 	 * so that the L1 hypervisor can intercept #PF before
4798 	 * CR2 is modified (or intercept #DB before DR6 is
4799 	 * modified under nVMX). Unless the per-VM capability,
4800 	 * KVM_CAP_EXCEPTION_PAYLOAD, is set, we may not defer the delivery of
4801 	 * an exception payload and handle after a KVM_GET_VCPU_EVENTS. Since we
4802 	 * opportunistically defer the exception payload, deliver it if the
4803 	 * capability hasn't been requested before processing a
4804 	 * KVM_GET_VCPU_EVENTS.
4805 	 */
4806 	if (!vcpu->kvm->arch.exception_payload_enabled &&
4807 	    vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)
4808 		kvm_deliver_exception_payload(vcpu);
4809 
4810 	/*
4811 	 * The API doesn't provide the instruction length for software
4812 	 * exceptions, so don't report them. As long as the guest RIP
4813 	 * isn't advanced, we should expect to encounter the exception
4814 	 * again.
4815 	 */
4816 	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
4817 		events->exception.injected = 0;
4818 		events->exception.pending = 0;
4819 	} else {
4820 		events->exception.injected = vcpu->arch.exception.injected;
4821 		events->exception.pending = vcpu->arch.exception.pending;
4822 		/*
4823 		 * For ABI compatibility, deliberately conflate
4824 		 * pending and injected exceptions when
4825 		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
4826 		 */
4827 		if (!vcpu->kvm->arch.exception_payload_enabled)
4828 			events->exception.injected |=
4829 				vcpu->arch.exception.pending;
4830 	}
4831 	events->exception.nr = vcpu->arch.exception.nr;
4832 	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
4833 	events->exception.error_code = vcpu->arch.exception.error_code;
4834 	events->exception_has_payload = vcpu->arch.exception.has_payload;
4835 	events->exception_payload = vcpu->arch.exception.payload;
4836 
4837 	events->interrupt.injected =
4838 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
4839 	events->interrupt.nr = vcpu->arch.interrupt.nr;
4840 	events->interrupt.soft = 0;
4841 	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
4842 
4843 	events->nmi.injected = vcpu->arch.nmi_injected;
4844 	events->nmi.pending = vcpu->arch.nmi_pending != 0;
4845 	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
4846 	events->nmi.pad = 0;
4847 
4848 	events->sipi_vector = 0; /* never valid when reporting to user space */
4849 
4850 	events->smi.smm = is_smm(vcpu);
4851 	events->smi.pending = vcpu->arch.smi_pending;
4852 	events->smi.smm_inside_nmi =
4853 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
4854 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);
4855 
4856 	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
4857 			 | KVM_VCPUEVENT_VALID_SHADOW
4858 			 | KVM_VCPUEVENT_VALID_SMM);
4859 	if (vcpu->kvm->arch.exception_payload_enabled)
4860 		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
4861 
4862 	memset(&events->reserved, 0, sizeof(events->reserved));
4863 }
4864 
4865 static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm);
4866 
4867 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
4868 					      struct kvm_vcpu_events *events)
4869 {
4870 	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
4871 			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
4872 			      | KVM_VCPUEVENT_VALID_SHADOW
4873 			      | KVM_VCPUEVENT_VALID_SMM
4874 			      | KVM_VCPUEVENT_VALID_PAYLOAD))
4875 		return -EINVAL;
4876 
4877 	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
4878 		if (!vcpu->kvm->arch.exception_payload_enabled)
4879 			return -EINVAL;
4880 		if (events->exception.pending)
4881 			events->exception.injected = 0;
4882 		else
4883 			events->exception_has_payload = 0;
4884 	} else {
4885 		events->exception.pending = 0;
4886 		events->exception_has_payload = 0;
4887 	}
4888 
4889 	if ((events->exception.injected || events->exception.pending) &&
4890 	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
4891 		return -EINVAL;
4892 
4893 	/* INITs are latched while in SMM */
4894 	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
4895 	    (events->smi.smm || events->smi.pending) &&
4896 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
4897 		return -EINVAL;
4898 
4899 	process_nmi(vcpu);
4900 	vcpu->arch.exception.injected = events->exception.injected;
4901 	vcpu->arch.exception.pending = events->exception.pending;
4902 	vcpu->arch.exception.nr = events->exception.nr;
4903 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
4904 	vcpu->arch.exception.error_code = events->exception.error_code;
4905 	vcpu->arch.exception.has_payload = events->exception_has_payload;
4906 	vcpu->arch.exception.payload = events->exception_payload;
4907 
4908 	vcpu->arch.interrupt.injected = events->interrupt.injected;
4909 	vcpu->arch.interrupt.nr = events->interrupt.nr;
4910 	vcpu->arch.interrupt.soft = events->interrupt.soft;
4911 	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
4912 		static_call(kvm_x86_set_interrupt_shadow)(vcpu,
4913 						events->interrupt.shadow);
4914 
4915 	vcpu->arch.nmi_injected = events->nmi.injected;
4916 	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
4917 		vcpu->arch.nmi_pending = events->nmi.pending;
4918 	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
4919 
4920 	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
4921 	    lapic_in_kernel(vcpu))
4922 		vcpu->arch.apic->sipi_vector = events->sipi_vector;
4923 
4924 	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
4925 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
4926 			kvm_x86_ops.nested_ops->leave_nested(vcpu);
4927 			kvm_smm_changed(vcpu, events->smi.smm);
4928 		}
4929 
4930 		vcpu->arch.smi_pending = events->smi.pending;
4931 
4932 		if (events->smi.smm) {
4933 			if (events->smi.smm_inside_nmi)
4934 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
4935 			else
4936 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
4937 		}
4938 
4939 		if (lapic_in_kernel(vcpu)) {
4940 			if (events->smi.latched_init)
4941 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4942 			else
4943 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4944 		}
4945 	}
4946 
4947 	kvm_make_request(KVM_REQ_EVENT, vcpu);
4948 
4949 	return 0;
4950 }
4951 
4952 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
4953 					     struct kvm_debugregs *dbgregs)
4954 {
4955 	unsigned long val;
4956 
4957 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
4958 	kvm_get_dr(vcpu, 6, &val);
4959 	dbgregs->dr6 = val;
4960 	dbgregs->dr7 = vcpu->arch.dr7;
4961 	dbgregs->flags = 0;
4962 	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
4963 }
4964 
4965 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
4966 					    struct kvm_debugregs *dbgregs)
4967 {
4968 	if (dbgregs->flags)
4969 		return -EINVAL;
4970 
4971 	if (!kvm_dr6_valid(dbgregs->dr6))
4972 		return -EINVAL;
4973 	if (!kvm_dr7_valid(dbgregs->dr7))
4974 		return -EINVAL;
4975 
4976 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
4977 	kvm_update_dr0123(vcpu);
4978 	vcpu->arch.dr6 = dbgregs->dr6;
4979 	vcpu->arch.dr7 = dbgregs->dr7;
4980 	kvm_update_dr7(vcpu);
4981 
4982 	return 0;
4983 }
4984 
4985 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
4986 					 struct kvm_xsave *guest_xsave)
4987 {
4988 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
4989 		return;
4990 
4991 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
4992 				       guest_xsave->region,
4993 				       sizeof(guest_xsave->region),
4994 				       vcpu->arch.pkru);
4995 }
4996 
4997 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,
4998 					  u8 *state, unsigned int size)
4999 {
5000 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5001 		return;
5002 
5003 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
5004 				       state, size, vcpu->arch.pkru);
5005 }
5006 
5007 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5008 					struct kvm_xsave *guest_xsave)
5009 {
5010 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5011 		return 0;
5012 
5013 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5014 					      guest_xsave->region,
5015 					      supported_xcr0, &vcpu->arch.pkru);
5016 }
5017 
5018 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5019 					struct kvm_xcrs *guest_xcrs)
5020 {
5021 	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
5022 		guest_xcrs->nr_xcrs = 0;
5023 		return;
5024 	}
5025 
5026 	guest_xcrs->nr_xcrs = 1;
5027 	guest_xcrs->flags = 0;
5028 	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
5029 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
5030 }
5031 
5032 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
5033 				       struct kvm_xcrs *guest_xcrs)
5034 {
5035 	int i, r = 0;
5036 
5037 	if (!boot_cpu_has(X86_FEATURE_XSAVE))
5038 		return -EINVAL;
5039 
5040 	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
5041 		return -EINVAL;
5042 
5043 	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
5044 		/* Only support XCR0 currently */
5045 		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
5046 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
5047 				guest_xcrs->xcrs[i].value);
5048 			break;
5049 		}
5050 	if (r)
5051 		r = -EINVAL;
5052 	return r;
5053 }
5054 
5055 /*
5056  * kvm_set_guest_paused() indicates to the guest kernel that it has been
5057  * stopped by the hypervisor.  This function will be called from the host only.
5058  * EINVAL is returned when the host attempts to set the flag for a guest that
5059  * does not support pv clocks.
5060  */
5061 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
5062 {
5063 	if (!vcpu->arch.pv_time_enabled)
5064 		return -EINVAL;
5065 	vcpu->arch.pvclock_set_guest_stopped_request = true;
5066 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5067 	return 0;
5068 }
5069 
5070 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,
5071 				 struct kvm_device_attr *attr)
5072 {
5073 	int r;
5074 
5075 	switch (attr->attr) {
5076 	case KVM_VCPU_TSC_OFFSET:
5077 		r = 0;
5078 		break;
5079 	default:
5080 		r = -ENXIO;
5081 	}
5082 
5083 	return r;
5084 }
5085 
5086 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
5087 				 struct kvm_device_attr *attr)
5088 {
5089 	u64 __user *uaddr = kvm_get_attr_addr(attr);
5090 	int r;
5091 
5092 	if (IS_ERR(uaddr))
5093 		return PTR_ERR(uaddr);
5094 
5095 	switch (attr->attr) {
5096 	case KVM_VCPU_TSC_OFFSET:
5097 		r = -EFAULT;
5098 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))
5099 			break;
5100 		r = 0;
5101 		break;
5102 	default:
5103 		r = -ENXIO;
5104 	}
5105 
5106 	return r;
5107 }
5108 
5109 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
5110 				 struct kvm_device_attr *attr)
5111 {
5112 	u64 __user *uaddr = kvm_get_attr_addr(attr);
5113 	struct kvm *kvm = vcpu->kvm;
5114 	int r;
5115 
5116 	if (IS_ERR(uaddr))
5117 		return PTR_ERR(uaddr);
5118 
5119 	switch (attr->attr) {
5120 	case KVM_VCPU_TSC_OFFSET: {
5121 		u64 offset, tsc, ns;
5122 		unsigned long flags;
5123 		bool matched;
5124 
5125 		r = -EFAULT;
5126 		if (get_user(offset, uaddr))
5127 			break;
5128 
5129 		raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
5130 
5131 		matched = (vcpu->arch.virtual_tsc_khz &&
5132 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
5133 			   kvm->arch.last_tsc_offset == offset);
5134 
5135 		tsc = kvm_scale_tsc(vcpu, rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;
5136 		ns = get_kvmclock_base_ns();
5137 
5138 		__kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);
5139 		raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
5140 
5141 		r = 0;
5142 		break;
5143 	}
5144 	default:
5145 		r = -ENXIO;
5146 	}
5147 
5148 	return r;
5149 }
5150 
5151 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,
5152 				      unsigned int ioctl,
5153 				      void __user *argp)
5154 {
5155 	struct kvm_device_attr attr;
5156 	int r;
5157 
5158 	if (copy_from_user(&attr, argp, sizeof(attr)))
5159 		return -EFAULT;
5160 
5161 	if (attr.group != KVM_VCPU_TSC_CTRL)
5162 		return -ENXIO;
5163 
5164 	switch (ioctl) {
5165 	case KVM_HAS_DEVICE_ATTR:
5166 		r = kvm_arch_tsc_has_attr(vcpu, &attr);
5167 		break;
5168 	case KVM_GET_DEVICE_ATTR:
5169 		r = kvm_arch_tsc_get_attr(vcpu, &attr);
5170 		break;
5171 	case KVM_SET_DEVICE_ATTR:
5172 		r = kvm_arch_tsc_set_attr(vcpu, &attr);
5173 		break;
5174 	}
5175 
5176 	return r;
5177 }
5178 
5179 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5180 				     struct kvm_enable_cap *cap)
5181 {
5182 	int r;
5183 	uint16_t vmcs_version;
5184 	void __user *user_ptr;
5185 
5186 	if (cap->flags)
5187 		return -EINVAL;
5188 
5189 	switch (cap->cap) {
5190 	case KVM_CAP_HYPERV_SYNIC2:
5191 		if (cap->args[0])
5192 			return -EINVAL;
5193 		fallthrough;
5194 
5195 	case KVM_CAP_HYPERV_SYNIC:
5196 		if (!irqchip_in_kernel(vcpu->kvm))
5197 			return -EINVAL;
5198 		return kvm_hv_activate_synic(vcpu, cap->cap ==
5199 					     KVM_CAP_HYPERV_SYNIC2);
5200 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
5201 		if (!kvm_x86_ops.nested_ops->enable_evmcs)
5202 			return -ENOTTY;
5203 		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
5204 		if (!r) {
5205 			user_ptr = (void __user *)(uintptr_t)cap->args[0];
5206 			if (copy_to_user(user_ptr, &vmcs_version,
5207 					 sizeof(vmcs_version)))
5208 				r = -EFAULT;
5209 		}
5210 		return r;
5211 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
5212 		if (!kvm_x86_ops.enable_direct_tlbflush)
5213 			return -ENOTTY;
5214 
5215 		return static_call(kvm_x86_enable_direct_tlbflush)(vcpu);
5216 
5217 	case KVM_CAP_HYPERV_ENFORCE_CPUID:
5218 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);
5219 
5220 	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
5221 		vcpu->arch.pv_cpuid.enforce = cap->args[0];
5222 		if (vcpu->arch.pv_cpuid.enforce)
5223 			kvm_update_pv_runtime(vcpu);
5224 
5225 		return 0;
5226 	default:
5227 		return -EINVAL;
5228 	}
5229 }
5230 
5231 long kvm_arch_vcpu_ioctl(struct file *filp,
5232 			 unsigned int ioctl, unsigned long arg)
5233 {
5234 	struct kvm_vcpu *vcpu = filp->private_data;
5235 	void __user *argp = (void __user *)arg;
5236 	int r;
5237 	union {
5238 		struct kvm_sregs2 *sregs2;
5239 		struct kvm_lapic_state *lapic;
5240 		struct kvm_xsave *xsave;
5241 		struct kvm_xcrs *xcrs;
5242 		void *buffer;
5243 	} u;
5244 
5245 	vcpu_load(vcpu);
5246 
5247 	u.buffer = NULL;
5248 	switch (ioctl) {
5249 	case KVM_GET_LAPIC: {
5250 		r = -EINVAL;
5251 		if (!lapic_in_kernel(vcpu))
5252 			goto out;
5253 		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
5254 				GFP_KERNEL_ACCOUNT);
5255 
5256 		r = -ENOMEM;
5257 		if (!u.lapic)
5258 			goto out;
5259 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
5260 		if (r)
5261 			goto out;
5262 		r = -EFAULT;
5263 		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
5264 			goto out;
5265 		r = 0;
5266 		break;
5267 	}
5268 	case KVM_SET_LAPIC: {
5269 		r = -EINVAL;
5270 		if (!lapic_in_kernel(vcpu))
5271 			goto out;
5272 		u.lapic = memdup_user(argp, sizeof(*u.lapic));
5273 		if (IS_ERR(u.lapic)) {
5274 			r = PTR_ERR(u.lapic);
5275 			goto out_nofree;
5276 		}
5277 
5278 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
5279 		break;
5280 	}
5281 	case KVM_INTERRUPT: {
5282 		struct kvm_interrupt irq;
5283 
5284 		r = -EFAULT;
5285 		if (copy_from_user(&irq, argp, sizeof(irq)))
5286 			goto out;
5287 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
5288 		break;
5289 	}
5290 	case KVM_NMI: {
5291 		r = kvm_vcpu_ioctl_nmi(vcpu);
5292 		break;
5293 	}
5294 	case KVM_SMI: {
5295 		r = kvm_vcpu_ioctl_smi(vcpu);
5296 		break;
5297 	}
5298 	case KVM_SET_CPUID: {
5299 		struct kvm_cpuid __user *cpuid_arg = argp;
5300 		struct kvm_cpuid cpuid;
5301 
5302 		r = -EFAULT;
5303 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5304 			goto out;
5305 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
5306 		break;
5307 	}
5308 	case KVM_SET_CPUID2: {
5309 		struct kvm_cpuid2 __user *cpuid_arg = argp;
5310 		struct kvm_cpuid2 cpuid;
5311 
5312 		r = -EFAULT;
5313 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5314 			goto out;
5315 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
5316 					      cpuid_arg->entries);
5317 		break;
5318 	}
5319 	case KVM_GET_CPUID2: {
5320 		struct kvm_cpuid2 __user *cpuid_arg = argp;
5321 		struct kvm_cpuid2 cpuid;
5322 
5323 		r = -EFAULT;
5324 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5325 			goto out;
5326 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
5327 					      cpuid_arg->entries);
5328 		if (r)
5329 			goto out;
5330 		r = -EFAULT;
5331 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
5332 			goto out;
5333 		r = 0;
5334 		break;
5335 	}
5336 	case KVM_GET_MSRS: {
5337 		int idx = srcu_read_lock(&vcpu->kvm->srcu);
5338 		r = msr_io(vcpu, argp, do_get_msr, 1);
5339 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5340 		break;
5341 	}
5342 	case KVM_SET_MSRS: {
5343 		int idx = srcu_read_lock(&vcpu->kvm->srcu);
5344 		r = msr_io(vcpu, argp, do_set_msr, 0);
5345 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5346 		break;
5347 	}
5348 	case KVM_TPR_ACCESS_REPORTING: {
5349 		struct kvm_tpr_access_ctl tac;
5350 
5351 		r = -EFAULT;
5352 		if (copy_from_user(&tac, argp, sizeof(tac)))
5353 			goto out;
5354 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
5355 		if (r)
5356 			goto out;
5357 		r = -EFAULT;
5358 		if (copy_to_user(argp, &tac, sizeof(tac)))
5359 			goto out;
5360 		r = 0;
5361 		break;
5362 	};
5363 	case KVM_SET_VAPIC_ADDR: {
5364 		struct kvm_vapic_addr va;
5365 		int idx;
5366 
5367 		r = -EINVAL;
5368 		if (!lapic_in_kernel(vcpu))
5369 			goto out;
5370 		r = -EFAULT;
5371 		if (copy_from_user(&va, argp, sizeof(va)))
5372 			goto out;
5373 		idx = srcu_read_lock(&vcpu->kvm->srcu);
5374 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
5375 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5376 		break;
5377 	}
5378 	case KVM_X86_SETUP_MCE: {
5379 		u64 mcg_cap;
5380 
5381 		r = -EFAULT;
5382 		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
5383 			goto out;
5384 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
5385 		break;
5386 	}
5387 	case KVM_X86_SET_MCE: {
5388 		struct kvm_x86_mce mce;
5389 
5390 		r = -EFAULT;
5391 		if (copy_from_user(&mce, argp, sizeof(mce)))
5392 			goto out;
5393 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
5394 		break;
5395 	}
5396 	case KVM_GET_VCPU_EVENTS: {
5397 		struct kvm_vcpu_events events;
5398 
5399 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
5400 
5401 		r = -EFAULT;
5402 		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
5403 			break;
5404 		r = 0;
5405 		break;
5406 	}
5407 	case KVM_SET_VCPU_EVENTS: {
5408 		struct kvm_vcpu_events events;
5409 
5410 		r = -EFAULT;
5411 		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
5412 			break;
5413 
5414 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
5415 		break;
5416 	}
5417 	case KVM_GET_DEBUGREGS: {
5418 		struct kvm_debugregs dbgregs;
5419 
5420 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
5421 
5422 		r = -EFAULT;
5423 		if (copy_to_user(argp, &dbgregs,
5424 				 sizeof(struct kvm_debugregs)))
5425 			break;
5426 		r = 0;
5427 		break;
5428 	}
5429 	case KVM_SET_DEBUGREGS: {
5430 		struct kvm_debugregs dbgregs;
5431 
5432 		r = -EFAULT;
5433 		if (copy_from_user(&dbgregs, argp,
5434 				   sizeof(struct kvm_debugregs)))
5435 			break;
5436 
5437 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
5438 		break;
5439 	}
5440 	case KVM_GET_XSAVE: {
5441 		r = -EINVAL;
5442 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))
5443 			break;
5444 
5445 		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
5446 		r = -ENOMEM;
5447 		if (!u.xsave)
5448 			break;
5449 
5450 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
5451 
5452 		r = -EFAULT;
5453 		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
5454 			break;
5455 		r = 0;
5456 		break;
5457 	}
5458 	case KVM_SET_XSAVE: {
5459 		int size = vcpu->arch.guest_fpu.uabi_size;
5460 
5461 		u.xsave = memdup_user(argp, size);
5462 		if (IS_ERR(u.xsave)) {
5463 			r = PTR_ERR(u.xsave);
5464 			goto out_nofree;
5465 		}
5466 
5467 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
5468 		break;
5469 	}
5470 
5471 	case KVM_GET_XSAVE2: {
5472 		int size = vcpu->arch.guest_fpu.uabi_size;
5473 
5474 		u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT);
5475 		r = -ENOMEM;
5476 		if (!u.xsave)
5477 			break;
5478 
5479 		kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);
5480 
5481 		r = -EFAULT;
5482 		if (copy_to_user(argp, u.xsave, size))
5483 			break;
5484 
5485 		r = 0;
5486 		break;
5487 	}
5488 
5489 	case KVM_GET_XCRS: {
5490 		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
5491 		r = -ENOMEM;
5492 		if (!u.xcrs)
5493 			break;
5494 
5495 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
5496 
5497 		r = -EFAULT;
5498 		if (copy_to_user(argp, u.xcrs,
5499 				 sizeof(struct kvm_xcrs)))
5500 			break;
5501 		r = 0;
5502 		break;
5503 	}
5504 	case KVM_SET_XCRS: {
5505 		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
5506 		if (IS_ERR(u.xcrs)) {
5507 			r = PTR_ERR(u.xcrs);
5508 			goto out_nofree;
5509 		}
5510 
5511 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
5512 		break;
5513 	}
5514 	case KVM_SET_TSC_KHZ: {
5515 		u32 user_tsc_khz;
5516 
5517 		r = -EINVAL;
5518 		user_tsc_khz = (u32)arg;
5519 
5520 		if (kvm_has_tsc_control &&
5521 		    user_tsc_khz >= kvm_max_guest_tsc_khz)
5522 			goto out;
5523 
5524 		if (user_tsc_khz == 0)
5525 			user_tsc_khz = tsc_khz;
5526 
5527 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5528 			r = 0;
5529 
5530 		goto out;
5531 	}
5532 	case KVM_GET_TSC_KHZ: {
5533 		r = vcpu->arch.virtual_tsc_khz;
5534 		goto out;
5535 	}
5536 	case KVM_KVMCLOCK_CTRL: {
5537 		r = kvm_set_guest_paused(vcpu);
5538 		goto out;
5539 	}
5540 	case KVM_ENABLE_CAP: {
5541 		struct kvm_enable_cap cap;
5542 
5543 		r = -EFAULT;
5544 		if (copy_from_user(&cap, argp, sizeof(cap)))
5545 			goto out;
5546 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5547 		break;
5548 	}
5549 	case KVM_GET_NESTED_STATE: {
5550 		struct kvm_nested_state __user *user_kvm_nested_state = argp;
5551 		u32 user_data_size;
5552 
5553 		r = -EINVAL;
5554 		if (!kvm_x86_ops.nested_ops->get_state)
5555 			break;
5556 
5557 		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
5558 		r = -EFAULT;
5559 		if (get_user(user_data_size, &user_kvm_nested_state->size))
5560 			break;
5561 
5562 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
5563 						     user_data_size);
5564 		if (r < 0)
5565 			break;
5566 
5567 		if (r > user_data_size) {
5568 			if (put_user(r, &user_kvm_nested_state->size))
5569 				r = -EFAULT;
5570 			else
5571 				r = -E2BIG;
5572 			break;
5573 		}
5574 
5575 		r = 0;
5576 		break;
5577 	}
5578 	case KVM_SET_NESTED_STATE: {
5579 		struct kvm_nested_state __user *user_kvm_nested_state = argp;
5580 		struct kvm_nested_state kvm_state;
5581 		int idx;
5582 
5583 		r = -EINVAL;
5584 		if (!kvm_x86_ops.nested_ops->set_state)
5585 			break;
5586 
5587 		r = -EFAULT;
5588 		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
5589 			break;
5590 
5591 		r = -EINVAL;
5592 		if (kvm_state.size < sizeof(kvm_state))
5593 			break;
5594 
5595 		if (kvm_state.flags &
5596 		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
5597 		      | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING
5598 		      | KVM_STATE_NESTED_GIF_SET))
5599 			break;
5600 
5601 		/* nested_run_pending implies guest_mode.  */
5602 		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
5603 		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
5604 			break;
5605 
5606 		idx = srcu_read_lock(&vcpu->kvm->srcu);
5607 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
5608 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5609 		break;
5610 	}
5611 	case KVM_GET_SUPPORTED_HV_CPUID:
5612 		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
5613 		break;
5614 #ifdef CONFIG_KVM_XEN
5615 	case KVM_XEN_VCPU_GET_ATTR: {
5616 		struct kvm_xen_vcpu_attr xva;
5617 
5618 		r = -EFAULT;
5619 		if (copy_from_user(&xva, argp, sizeof(xva)))
5620 			goto out;
5621 		r = kvm_xen_vcpu_get_attr(vcpu, &xva);
5622 		if (!r && copy_to_user(argp, &xva, sizeof(xva)))
5623 			r = -EFAULT;
5624 		break;
5625 	}
5626 	case KVM_XEN_VCPU_SET_ATTR: {
5627 		struct kvm_xen_vcpu_attr xva;
5628 
5629 		r = -EFAULT;
5630 		if (copy_from_user(&xva, argp, sizeof(xva)))
5631 			goto out;
5632 		r = kvm_xen_vcpu_set_attr(vcpu, &xva);
5633 		break;
5634 	}
5635 #endif
5636 	case KVM_GET_SREGS2: {
5637 		u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL);
5638 		r = -ENOMEM;
5639 		if (!u.sregs2)
5640 			goto out;
5641 		__get_sregs2(vcpu, u.sregs2);
5642 		r = -EFAULT;
5643 		if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2)))
5644 			goto out;
5645 		r = 0;
5646 		break;
5647 	}
5648 	case KVM_SET_SREGS2: {
5649 		u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2));
5650 		if (IS_ERR(u.sregs2)) {
5651 			r = PTR_ERR(u.sregs2);
5652 			u.sregs2 = NULL;
5653 			goto out;
5654 		}
5655 		r = __set_sregs2(vcpu, u.sregs2);
5656 		break;
5657 	}
5658 	case KVM_HAS_DEVICE_ATTR:
5659 	case KVM_GET_DEVICE_ATTR:
5660 	case KVM_SET_DEVICE_ATTR:
5661 		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
5662 		break;
5663 	default:
5664 		r = -EINVAL;
5665 	}
5666 out:
5667 	kfree(u.buffer);
5668 out_nofree:
5669 	vcpu_put(vcpu);
5670 	return r;
5671 }
5672 
5673 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5674 {
5675 	return VM_FAULT_SIGBUS;
5676 }
5677 
5678 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
5679 {
5680 	int ret;
5681 
5682 	if (addr > (unsigned int)(-3 * PAGE_SIZE))
5683 		return -EINVAL;
5684 	ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
5685 	return ret;
5686 }
5687 
5688 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
5689 					      u64 ident_addr)
5690 {
5691 	return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr);
5692 }
5693 
5694 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
5695 					 unsigned long kvm_nr_mmu_pages)
5696 {
5697 	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
5698 		return -EINVAL;
5699 
5700 	mutex_lock(&kvm->slots_lock);
5701 
5702 	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
5703 	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
5704 
5705 	mutex_unlock(&kvm->slots_lock);
5706 	return 0;
5707 }
5708 
5709 static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
5710 {
5711 	return kvm->arch.n_max_mmu_pages;
5712 }
5713 
5714 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
5715 {
5716 	struct kvm_pic *pic = kvm->arch.vpic;
5717 	int r;
5718 
5719 	r = 0;
5720 	switch (chip->chip_id) {
5721 	case KVM_IRQCHIP_PIC_MASTER:
5722 		memcpy(&chip->chip.pic, &pic->pics[0],
5723 			sizeof(struct kvm_pic_state));
5724 		break;
5725 	case KVM_IRQCHIP_PIC_SLAVE:
5726 		memcpy(&chip->chip.pic, &pic->pics[1],
5727 			sizeof(struct kvm_pic_state));
5728 		break;
5729 	case KVM_IRQCHIP_IOAPIC:
5730 		kvm_get_ioapic(kvm, &chip->chip.ioapic);
5731 		break;
5732 	default:
5733 		r = -EINVAL;
5734 		break;
5735 	}
5736 	return r;
5737 }
5738 
5739 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
5740 {
5741 	struct kvm_pic *pic = kvm->arch.vpic;
5742 	int r;
5743 
5744 	r = 0;
5745 	switch (chip->chip_id) {
5746 	case KVM_IRQCHIP_PIC_MASTER:
5747 		spin_lock(&pic->lock);
5748 		memcpy(&pic->pics[0], &chip->chip.pic,
5749 			sizeof(struct kvm_pic_state));
5750 		spin_unlock(&pic->lock);
5751 		break;
5752 	case KVM_IRQCHIP_PIC_SLAVE:
5753 		spin_lock(&pic->lock);
5754 		memcpy(&pic->pics[1], &chip->chip.pic,
5755 			sizeof(struct kvm_pic_state));
5756 		spin_unlock(&pic->lock);
5757 		break;
5758 	case KVM_IRQCHIP_IOAPIC:
5759 		kvm_set_ioapic(kvm, &chip->chip.ioapic);
5760 		break;
5761 	default:
5762 		r = -EINVAL;
5763 		break;
5764 	}
5765 	kvm_pic_update_irq(pic);
5766 	return r;
5767 }
5768 
5769 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
5770 {
5771 	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
5772 
5773 	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));
5774 
5775 	mutex_lock(&kps->lock);
5776 	memcpy(ps, &kps->channels, sizeof(*ps));
5777 	mutex_unlock(&kps->lock);
5778 	return 0;
5779 }
5780 
5781 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
5782 {
5783 	int i;
5784 	struct kvm_pit *pit = kvm->arch.vpit;
5785 
5786 	mutex_lock(&pit->pit_state.lock);
5787 	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
5788 	for (i = 0; i < 3; i++)
5789 		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
5790 	mutex_unlock(&pit->pit_state.lock);
5791 	return 0;
5792 }
5793 
5794 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
5795 {
5796 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
5797 	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
5798 		sizeof(ps->channels));
5799 	ps->flags = kvm->arch.vpit->pit_state.flags;
5800 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
5801 	memset(&ps->reserved, 0, sizeof(ps->reserved));
5802 	return 0;
5803 }
5804 
5805 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
5806 {
5807 	int start = 0;
5808 	int i;
5809 	u32 prev_legacy, cur_legacy;
5810 	struct kvm_pit *pit = kvm->arch.vpit;
5811 
5812 	mutex_lock(&pit->pit_state.lock);
5813 	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
5814 	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
5815 	if (!prev_legacy && cur_legacy)
5816 		start = 1;
5817 	memcpy(&pit->pit_state.channels, &ps->channels,
5818 	       sizeof(pit->pit_state.channels));
5819 	pit->pit_state.flags = ps->flags;
5820 	for (i = 0; i < 3; i++)
5821 		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
5822 				   start && i == 0);
5823 	mutex_unlock(&pit->pit_state.lock);
5824 	return 0;
5825 }
5826 
5827 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
5828 				 struct kvm_reinject_control *control)
5829 {
5830 	struct kvm_pit *pit = kvm->arch.vpit;
5831 
5832 	/* pit->pit_state.lock was overloaded to prevent userspace from getting
5833 	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
5834 	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
5835 	 */
5836 	mutex_lock(&pit->pit_state.lock);
5837 	kvm_pit_set_reinject(pit, control->pit_reinject);
5838 	mutex_unlock(&pit->pit_state.lock);
5839 
5840 	return 0;
5841 }
5842 
5843 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
5844 {
5845 
5846 	/*
5847 	 * Flush all CPUs' dirty log buffers to the  dirty_bitmap.  Called
5848 	 * before reporting dirty_bitmap to userspace.  KVM flushes the buffers
5849 	 * on all VM-Exits, thus we only need to kick running vCPUs to force a
5850 	 * VM-Exit.
5851 	 */
5852 	struct kvm_vcpu *vcpu;
5853 	unsigned long i;
5854 
5855 	kvm_for_each_vcpu(i, vcpu, kvm)
5856 		kvm_vcpu_kick(vcpu);
5857 }
5858 
5859 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
5860 			bool line_status)
5861 {
5862 	if (!irqchip_in_kernel(kvm))
5863 		return -ENXIO;
5864 
5865 	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
5866 					irq_event->irq, irq_event->level,
5867 					line_status);
5868 	return 0;
5869 }
5870 
5871 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
5872 			    struct kvm_enable_cap *cap)
5873 {
5874 	int r;
5875 
5876 	if (cap->flags)
5877 		return -EINVAL;
5878 
5879 	switch (cap->cap) {
5880 	case KVM_CAP_DISABLE_QUIRKS:
5881 		kvm->arch.disabled_quirks = cap->args[0];
5882 		r = 0;
5883 		break;
5884 	case KVM_CAP_SPLIT_IRQCHIP: {
5885 		mutex_lock(&kvm->lock);
5886 		r = -EINVAL;
5887 		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
5888 			goto split_irqchip_unlock;
5889 		r = -EEXIST;
5890 		if (irqchip_in_kernel(kvm))
5891 			goto split_irqchip_unlock;
5892 		if (kvm->created_vcpus)
5893 			goto split_irqchip_unlock;
5894 		r = kvm_setup_empty_irq_routing(kvm);
5895 		if (r)
5896 			goto split_irqchip_unlock;
5897 		/* Pairs with irqchip_in_kernel. */
5898 		smp_wmb();
5899 		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
5900 		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
5901 		kvm_request_apicv_update(kvm, true, APICV_INHIBIT_REASON_ABSENT);
5902 		r = 0;
5903 split_irqchip_unlock:
5904 		mutex_unlock(&kvm->lock);
5905 		break;
5906 	}
5907 	case KVM_CAP_X2APIC_API:
5908 		r = -EINVAL;
5909 		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
5910 			break;
5911 
5912 		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
5913 			kvm->arch.x2apic_format = true;
5914 		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
5915 			kvm->arch.x2apic_broadcast_quirk_disabled = true;
5916 
5917 		r = 0;
5918 		break;
5919 	case KVM_CAP_X86_DISABLE_EXITS:
5920 		r = -EINVAL;
5921 		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
5922 			break;
5923 
5924 		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
5925 			kvm_can_mwait_in_guest())
5926 			kvm->arch.mwait_in_guest = true;
5927 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
5928 			kvm->arch.hlt_in_guest = true;
5929 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
5930 			kvm->arch.pause_in_guest = true;
5931 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
5932 			kvm->arch.cstate_in_guest = true;
5933 		r = 0;
5934 		break;
5935 	case KVM_CAP_MSR_PLATFORM_INFO:
5936 		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
5937 		r = 0;
5938 		break;
5939 	case KVM_CAP_EXCEPTION_PAYLOAD:
5940 		kvm->arch.exception_payload_enabled = cap->args[0];
5941 		r = 0;
5942 		break;
5943 	case KVM_CAP_X86_USER_SPACE_MSR:
5944 		kvm->arch.user_space_msr_mask = cap->args[0];
5945 		r = 0;
5946 		break;
5947 	case KVM_CAP_X86_BUS_LOCK_EXIT:
5948 		r = -EINVAL;
5949 		if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
5950 			break;
5951 
5952 		if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
5953 		    (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
5954 			break;
5955 
5956 		if (kvm_has_bus_lock_exit &&
5957 		    cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
5958 			kvm->arch.bus_lock_detection_enabled = true;
5959 		r = 0;
5960 		break;
5961 #ifdef CONFIG_X86_SGX_KVM
5962 	case KVM_CAP_SGX_ATTRIBUTE: {
5963 		unsigned long allowed_attributes = 0;
5964 
5965 		r = sgx_set_attribute(&allowed_attributes, cap->args[0]);
5966 		if (r)
5967 			break;
5968 
5969 		/* KVM only supports the PROVISIONKEY privileged attribute. */
5970 		if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) &&
5971 		    !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY))
5972 			kvm->arch.sgx_provisioning_allowed = true;
5973 		else
5974 			r = -EINVAL;
5975 		break;
5976 	}
5977 #endif
5978 	case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
5979 		r = -EINVAL;
5980 		if (kvm_x86_ops.vm_copy_enc_context_from)
5981 			r = kvm_x86_ops.vm_copy_enc_context_from(kvm, cap->args[0]);
5982 		return r;
5983 	case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
5984 		r = -EINVAL;
5985 		if (kvm_x86_ops.vm_move_enc_context_from)
5986 			r = kvm_x86_ops.vm_move_enc_context_from(
5987 				kvm, cap->args[0]);
5988 		return r;
5989 	case KVM_CAP_EXIT_HYPERCALL:
5990 		if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {
5991 			r = -EINVAL;
5992 			break;
5993 		}
5994 		kvm->arch.hypercall_exit_enabled = cap->args[0];
5995 		r = 0;
5996 		break;
5997 	case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
5998 		r = -EINVAL;
5999 		if (cap->args[0] & ~1)
6000 			break;
6001 		kvm->arch.exit_on_emulation_error = cap->args[0];
6002 		r = 0;
6003 		break;
6004 	default:
6005 		r = -EINVAL;
6006 		break;
6007 	}
6008 	return r;
6009 }
6010 
6011 static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)
6012 {
6013 	struct kvm_x86_msr_filter *msr_filter;
6014 
6015 	msr_filter = kzalloc(sizeof(*msr_filter), GFP_KERNEL_ACCOUNT);
6016 	if (!msr_filter)
6017 		return NULL;
6018 
6019 	msr_filter->default_allow = default_allow;
6020 	return msr_filter;
6021 }
6022 
6023 static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
6024 {
6025 	u32 i;
6026 
6027 	if (!msr_filter)
6028 		return;
6029 
6030 	for (i = 0; i < msr_filter->count; i++)
6031 		kfree(msr_filter->ranges[i].bitmap);
6032 
6033 	kfree(msr_filter);
6034 }
6035 
6036 static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter,
6037 			      struct kvm_msr_filter_range *user_range)
6038 {
6039 	unsigned long *bitmap = NULL;
6040 	size_t bitmap_size;
6041 
6042 	if (!user_range->nmsrs)
6043 		return 0;
6044 
6045 	if (user_range->flags & ~(KVM_MSR_FILTER_READ | KVM_MSR_FILTER_WRITE))
6046 		return -EINVAL;
6047 
6048 	if (!user_range->flags)
6049 		return -EINVAL;
6050 
6051 	bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);
6052 	if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE)
6053 		return -EINVAL;
6054 
6055 	bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);
6056 	if (IS_ERR(bitmap))
6057 		return PTR_ERR(bitmap);
6058 
6059 	msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {
6060 		.flags = user_range->flags,
6061 		.base = user_range->base,
6062 		.nmsrs = user_range->nmsrs,
6063 		.bitmap = bitmap,
6064 	};
6065 
6066 	msr_filter->count++;
6067 	return 0;
6068 }
6069 
6070 static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, void __user *argp)
6071 {
6072 	struct kvm_msr_filter __user *user_msr_filter = argp;
6073 	struct kvm_x86_msr_filter *new_filter, *old_filter;
6074 	struct kvm_msr_filter filter;
6075 	bool default_allow;
6076 	bool empty = true;
6077 	int r = 0;
6078 	u32 i;
6079 
6080 	if (copy_from_user(&filter, user_msr_filter, sizeof(filter)))
6081 		return -EFAULT;
6082 
6083 	for (i = 0; i < ARRAY_SIZE(filter.ranges); i++)
6084 		empty &= !filter.ranges[i].nmsrs;
6085 
6086 	default_allow = !(filter.flags & KVM_MSR_FILTER_DEFAULT_DENY);
6087 	if (empty && !default_allow)
6088 		return -EINVAL;
6089 
6090 	new_filter = kvm_alloc_msr_filter(default_allow);
6091 	if (!new_filter)
6092 		return -ENOMEM;
6093 
6094 	for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) {
6095 		r = kvm_add_msr_filter(new_filter, &filter.ranges[i]);
6096 		if (r) {
6097 			kvm_free_msr_filter(new_filter);
6098 			return r;
6099 		}
6100 	}
6101 
6102 	mutex_lock(&kvm->lock);
6103 
6104 	/* The per-VM filter is protected by kvm->lock... */
6105 	old_filter = srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1);
6106 
6107 	rcu_assign_pointer(kvm->arch.msr_filter, new_filter);
6108 	synchronize_srcu(&kvm->srcu);
6109 
6110 	kvm_free_msr_filter(old_filter);
6111 
6112 	kvm_make_all_cpus_request(kvm, KVM_REQ_MSR_FILTER_CHANGED);
6113 	mutex_unlock(&kvm->lock);
6114 
6115 	return 0;
6116 }
6117 
6118 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
6119 static int kvm_arch_suspend_notifier(struct kvm *kvm)
6120 {
6121 	struct kvm_vcpu *vcpu;
6122 	unsigned long i;
6123 	int ret = 0;
6124 
6125 	mutex_lock(&kvm->lock);
6126 	kvm_for_each_vcpu(i, vcpu, kvm) {
6127 		if (!vcpu->arch.pv_time_enabled)
6128 			continue;
6129 
6130 		ret = kvm_set_guest_paused(vcpu);
6131 		if (ret) {
6132 			kvm_err("Failed to pause guest VCPU%d: %d\n",
6133 				vcpu->vcpu_id, ret);
6134 			break;
6135 		}
6136 	}
6137 	mutex_unlock(&kvm->lock);
6138 
6139 	return ret ? NOTIFY_BAD : NOTIFY_DONE;
6140 }
6141 
6142 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)
6143 {
6144 	switch (state) {
6145 	case PM_HIBERNATION_PREPARE:
6146 	case PM_SUSPEND_PREPARE:
6147 		return kvm_arch_suspend_notifier(kvm);
6148 	}
6149 
6150 	return NOTIFY_DONE;
6151 }
6152 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
6153 
6154 static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp)
6155 {
6156 	struct kvm_clock_data data = { 0 };
6157 
6158 	get_kvmclock(kvm, &data);
6159 	if (copy_to_user(argp, &data, sizeof(data)))
6160 		return -EFAULT;
6161 
6162 	return 0;
6163 }
6164 
6165 static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp)
6166 {
6167 	struct kvm_arch *ka = &kvm->arch;
6168 	struct kvm_clock_data data;
6169 	u64 now_raw_ns;
6170 
6171 	if (copy_from_user(&data, argp, sizeof(data)))
6172 		return -EFAULT;
6173 
6174 	/*
6175 	 * Only KVM_CLOCK_REALTIME is used, but allow passing the
6176 	 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK.
6177 	 */
6178 	if (data.flags & ~KVM_CLOCK_VALID_FLAGS)
6179 		return -EINVAL;
6180 
6181 	kvm_hv_invalidate_tsc_page(kvm);
6182 	kvm_start_pvclock_update(kvm);
6183 	pvclock_update_vm_gtod_copy(kvm);
6184 
6185 	/*
6186 	 * This pairs with kvm_guest_time_update(): when masterclock is
6187 	 * in use, we use master_kernel_ns + kvmclock_offset to set
6188 	 * unsigned 'system_time' so if we use get_kvmclock_ns() (which
6189 	 * is slightly ahead) here we risk going negative on unsigned
6190 	 * 'system_time' when 'data.clock' is very small.
6191 	 */
6192 	if (data.flags & KVM_CLOCK_REALTIME) {
6193 		u64 now_real_ns = ktime_get_real_ns();
6194 
6195 		/*
6196 		 * Avoid stepping the kvmclock backwards.
6197 		 */
6198 		if (now_real_ns > data.realtime)
6199 			data.clock += now_real_ns - data.realtime;
6200 	}
6201 
6202 	if (ka->use_master_clock)
6203 		now_raw_ns = ka->master_kernel_ns;
6204 	else
6205 		now_raw_ns = get_kvmclock_base_ns();
6206 	ka->kvmclock_offset = data.clock - now_raw_ns;
6207 	kvm_end_pvclock_update(kvm);
6208 	return 0;
6209 }
6210 
6211 long kvm_arch_vm_ioctl(struct file *filp,
6212 		       unsigned int ioctl, unsigned long arg)
6213 {
6214 	struct kvm *kvm = filp->private_data;
6215 	void __user *argp = (void __user *)arg;
6216 	int r = -ENOTTY;
6217 	/*
6218 	 * This union makes it completely explicit to gcc-3.x
6219 	 * that these two variables' stack usage should be
6220 	 * combined, not added together.
6221 	 */
6222 	union {
6223 		struct kvm_pit_state ps;
6224 		struct kvm_pit_state2 ps2;
6225 		struct kvm_pit_config pit_config;
6226 	} u;
6227 
6228 	switch (ioctl) {
6229 	case KVM_SET_TSS_ADDR:
6230 		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
6231 		break;
6232 	case KVM_SET_IDENTITY_MAP_ADDR: {
6233 		u64 ident_addr;
6234 
6235 		mutex_lock(&kvm->lock);
6236 		r = -EINVAL;
6237 		if (kvm->created_vcpus)
6238 			goto set_identity_unlock;
6239 		r = -EFAULT;
6240 		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
6241 			goto set_identity_unlock;
6242 		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
6243 set_identity_unlock:
6244 		mutex_unlock(&kvm->lock);
6245 		break;
6246 	}
6247 	case KVM_SET_NR_MMU_PAGES:
6248 		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
6249 		break;
6250 	case KVM_GET_NR_MMU_PAGES:
6251 		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
6252 		break;
6253 	case KVM_CREATE_IRQCHIP: {
6254 		mutex_lock(&kvm->lock);
6255 
6256 		r = -EEXIST;
6257 		if (irqchip_in_kernel(kvm))
6258 			goto create_irqchip_unlock;
6259 
6260 		r = -EINVAL;
6261 		if (kvm->created_vcpus)
6262 			goto create_irqchip_unlock;
6263 
6264 		r = kvm_pic_init(kvm);
6265 		if (r)
6266 			goto create_irqchip_unlock;
6267 
6268 		r = kvm_ioapic_init(kvm);
6269 		if (r) {
6270 			kvm_pic_destroy(kvm);
6271 			goto create_irqchip_unlock;
6272 		}
6273 
6274 		r = kvm_setup_default_irq_routing(kvm);
6275 		if (r) {
6276 			kvm_ioapic_destroy(kvm);
6277 			kvm_pic_destroy(kvm);
6278 			goto create_irqchip_unlock;
6279 		}
6280 		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
6281 		smp_wmb();
6282 		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
6283 		kvm_request_apicv_update(kvm, true, APICV_INHIBIT_REASON_ABSENT);
6284 	create_irqchip_unlock:
6285 		mutex_unlock(&kvm->lock);
6286 		break;
6287 	}
6288 	case KVM_CREATE_PIT:
6289 		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
6290 		goto create_pit;
6291 	case KVM_CREATE_PIT2:
6292 		r = -EFAULT;
6293 		if (copy_from_user(&u.pit_config, argp,
6294 				   sizeof(struct kvm_pit_config)))
6295 			goto out;
6296 	create_pit:
6297 		mutex_lock(&kvm->lock);
6298 		r = -EEXIST;
6299 		if (kvm->arch.vpit)
6300 			goto create_pit_unlock;
6301 		r = -ENOMEM;
6302 		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
6303 		if (kvm->arch.vpit)
6304 			r = 0;
6305 	create_pit_unlock:
6306 		mutex_unlock(&kvm->lock);
6307 		break;
6308 	case KVM_GET_IRQCHIP: {
6309 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
6310 		struct kvm_irqchip *chip;
6311 
6312 		chip = memdup_user(argp, sizeof(*chip));
6313 		if (IS_ERR(chip)) {
6314 			r = PTR_ERR(chip);
6315 			goto out;
6316 		}
6317 
6318 		r = -ENXIO;
6319 		if (!irqchip_kernel(kvm))
6320 			goto get_irqchip_out;
6321 		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
6322 		if (r)
6323 			goto get_irqchip_out;
6324 		r = -EFAULT;
6325 		if (copy_to_user(argp, chip, sizeof(*chip)))
6326 			goto get_irqchip_out;
6327 		r = 0;
6328 	get_irqchip_out:
6329 		kfree(chip);
6330 		break;
6331 	}
6332 	case KVM_SET_IRQCHIP: {
6333 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
6334 		struct kvm_irqchip *chip;
6335 
6336 		chip = memdup_user(argp, sizeof(*chip));
6337 		if (IS_ERR(chip)) {
6338 			r = PTR_ERR(chip);
6339 			goto out;
6340 		}
6341 
6342 		r = -ENXIO;
6343 		if (!irqchip_kernel(kvm))
6344 			goto set_irqchip_out;
6345 		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
6346 	set_irqchip_out:
6347 		kfree(chip);
6348 		break;
6349 	}
6350 	case KVM_GET_PIT: {
6351 		r = -EFAULT;
6352 		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
6353 			goto out;
6354 		r = -ENXIO;
6355 		if (!kvm->arch.vpit)
6356 			goto out;
6357 		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
6358 		if (r)
6359 			goto out;
6360 		r = -EFAULT;
6361 		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
6362 			goto out;
6363 		r = 0;
6364 		break;
6365 	}
6366 	case KVM_SET_PIT: {
6367 		r = -EFAULT;
6368 		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
6369 			goto out;
6370 		mutex_lock(&kvm->lock);
6371 		r = -ENXIO;
6372 		if (!kvm->arch.vpit)
6373 			goto set_pit_out;
6374 		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
6375 set_pit_out:
6376 		mutex_unlock(&kvm->lock);
6377 		break;
6378 	}
6379 	case KVM_GET_PIT2: {
6380 		r = -ENXIO;
6381 		if (!kvm->arch.vpit)
6382 			goto out;
6383 		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
6384 		if (r)
6385 			goto out;
6386 		r = -EFAULT;
6387 		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
6388 			goto out;
6389 		r = 0;
6390 		break;
6391 	}
6392 	case KVM_SET_PIT2: {
6393 		r = -EFAULT;
6394 		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
6395 			goto out;
6396 		mutex_lock(&kvm->lock);
6397 		r = -ENXIO;
6398 		if (!kvm->arch.vpit)
6399 			goto set_pit2_out;
6400 		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
6401 set_pit2_out:
6402 		mutex_unlock(&kvm->lock);
6403 		break;
6404 	}
6405 	case KVM_REINJECT_CONTROL: {
6406 		struct kvm_reinject_control control;
6407 		r =  -EFAULT;
6408 		if (copy_from_user(&control, argp, sizeof(control)))
6409 			goto out;
6410 		r = -ENXIO;
6411 		if (!kvm->arch.vpit)
6412 			goto out;
6413 		r = kvm_vm_ioctl_reinject(kvm, &control);
6414 		break;
6415 	}
6416 	case KVM_SET_BOOT_CPU_ID:
6417 		r = 0;
6418 		mutex_lock(&kvm->lock);
6419 		if (kvm->created_vcpus)
6420 			r = -EBUSY;
6421 		else
6422 			kvm->arch.bsp_vcpu_id = arg;
6423 		mutex_unlock(&kvm->lock);
6424 		break;
6425 #ifdef CONFIG_KVM_XEN
6426 	case KVM_XEN_HVM_CONFIG: {
6427 		struct kvm_xen_hvm_config xhc;
6428 		r = -EFAULT;
6429 		if (copy_from_user(&xhc, argp, sizeof(xhc)))
6430 			goto out;
6431 		r = kvm_xen_hvm_config(kvm, &xhc);
6432 		break;
6433 	}
6434 	case KVM_XEN_HVM_GET_ATTR: {
6435 		struct kvm_xen_hvm_attr xha;
6436 
6437 		r = -EFAULT;
6438 		if (copy_from_user(&xha, argp, sizeof(xha)))
6439 			goto out;
6440 		r = kvm_xen_hvm_get_attr(kvm, &xha);
6441 		if (!r && copy_to_user(argp, &xha, sizeof(xha)))
6442 			r = -EFAULT;
6443 		break;
6444 	}
6445 	case KVM_XEN_HVM_SET_ATTR: {
6446 		struct kvm_xen_hvm_attr xha;
6447 
6448 		r = -EFAULT;
6449 		if (copy_from_user(&xha, argp, sizeof(xha)))
6450 			goto out;
6451 		r = kvm_xen_hvm_set_attr(kvm, &xha);
6452 		break;
6453 	}
6454 #endif
6455 	case KVM_SET_CLOCK:
6456 		r = kvm_vm_ioctl_set_clock(kvm, argp);
6457 		break;
6458 	case KVM_GET_CLOCK:
6459 		r = kvm_vm_ioctl_get_clock(kvm, argp);
6460 		break;
6461 	case KVM_MEMORY_ENCRYPT_OP: {
6462 		r = -ENOTTY;
6463 		if (kvm_x86_ops.mem_enc_op)
6464 			r = static_call(kvm_x86_mem_enc_op)(kvm, argp);
6465 		break;
6466 	}
6467 	case KVM_MEMORY_ENCRYPT_REG_REGION: {
6468 		struct kvm_enc_region region;
6469 
6470 		r = -EFAULT;
6471 		if (copy_from_user(&region, argp, sizeof(region)))
6472 			goto out;
6473 
6474 		r = -ENOTTY;
6475 		if (kvm_x86_ops.mem_enc_reg_region)
6476 			r = static_call(kvm_x86_mem_enc_reg_region)(kvm, &region);
6477 		break;
6478 	}
6479 	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
6480 		struct kvm_enc_region region;
6481 
6482 		r = -EFAULT;
6483 		if (copy_from_user(&region, argp, sizeof(region)))
6484 			goto out;
6485 
6486 		r = -ENOTTY;
6487 		if (kvm_x86_ops.mem_enc_unreg_region)
6488 			r = static_call(kvm_x86_mem_enc_unreg_region)(kvm, &region);
6489 		break;
6490 	}
6491 	case KVM_HYPERV_EVENTFD: {
6492 		struct kvm_hyperv_eventfd hvevfd;
6493 
6494 		r = -EFAULT;
6495 		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
6496 			goto out;
6497 		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
6498 		break;
6499 	}
6500 	case KVM_SET_PMU_EVENT_FILTER:
6501 		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
6502 		break;
6503 	case KVM_X86_SET_MSR_FILTER:
6504 		r = kvm_vm_ioctl_set_msr_filter(kvm, argp);
6505 		break;
6506 	default:
6507 		r = -ENOTTY;
6508 	}
6509 out:
6510 	return r;
6511 }
6512 
6513 static void kvm_init_msr_list(void)
6514 {
6515 	struct x86_pmu_capability x86_pmu;
6516 	u32 dummy[2];
6517 	unsigned i;
6518 
6519 	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
6520 			 "Please update the fixed PMCs in msrs_to_saved_all[]");
6521 
6522 	perf_get_x86_pmu_capability(&x86_pmu);
6523 
6524 	num_msrs_to_save = 0;
6525 	num_emulated_msrs = 0;
6526 	num_msr_based_features = 0;
6527 
6528 	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
6529 		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
6530 			continue;
6531 
6532 		/*
6533 		 * Even MSRs that are valid in the host may not be exposed
6534 		 * to the guests in some cases.
6535 		 */
6536 		switch (msrs_to_save_all[i]) {
6537 		case MSR_IA32_BNDCFGS:
6538 			if (!kvm_mpx_supported())
6539 				continue;
6540 			break;
6541 		case MSR_TSC_AUX:
6542 			if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) &&
6543 			    !kvm_cpu_cap_has(X86_FEATURE_RDPID))
6544 				continue;
6545 			break;
6546 		case MSR_IA32_UMWAIT_CONTROL:
6547 			if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
6548 				continue;
6549 			break;
6550 		case MSR_IA32_RTIT_CTL:
6551 		case MSR_IA32_RTIT_STATUS:
6552 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
6553 				continue;
6554 			break;
6555 		case MSR_IA32_RTIT_CR3_MATCH:
6556 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
6557 			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
6558 				continue;
6559 			break;
6560 		case MSR_IA32_RTIT_OUTPUT_BASE:
6561 		case MSR_IA32_RTIT_OUTPUT_MASK:
6562 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
6563 				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
6564 				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
6565 				continue;
6566 			break;
6567 		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
6568 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
6569 				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
6570 				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
6571 				continue;
6572 			break;
6573 		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
6574 			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
6575 			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
6576 				continue;
6577 			break;
6578 		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
6579 			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
6580 			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
6581 				continue;
6582 			break;
6583 		case MSR_IA32_XFD:
6584 		case MSR_IA32_XFD_ERR:
6585 			if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
6586 				continue;
6587 			break;
6588 		default:
6589 			break;
6590 		}
6591 
6592 		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
6593 	}
6594 
6595 	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
6596 		if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i]))
6597 			continue;
6598 
6599 		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
6600 	}
6601 
6602 	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
6603 		struct kvm_msr_entry msr;
6604 
6605 		msr.index = msr_based_features_all[i];
6606 		if (kvm_get_msr_feature(&msr))
6607 			continue;
6608 
6609 		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
6610 	}
6611 }
6612 
6613 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
6614 			   const void *v)
6615 {
6616 	int handled = 0;
6617 	int n;
6618 
6619 	do {
6620 		n = min(len, 8);
6621 		if (!(lapic_in_kernel(vcpu) &&
6622 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
6623 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
6624 			break;
6625 		handled += n;
6626 		addr += n;
6627 		len -= n;
6628 		v += n;
6629 	} while (len);
6630 
6631 	return handled;
6632 }
6633 
6634 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
6635 {
6636 	int handled = 0;
6637 	int n;
6638 
6639 	do {
6640 		n = min(len, 8);
6641 		if (!(lapic_in_kernel(vcpu) &&
6642 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
6643 					 addr, n, v))
6644 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
6645 			break;
6646 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
6647 		handled += n;
6648 		addr += n;
6649 		len -= n;
6650 		v += n;
6651 	} while (len);
6652 
6653 	return handled;
6654 }
6655 
6656 static void kvm_set_segment(struct kvm_vcpu *vcpu,
6657 			struct kvm_segment *var, int seg)
6658 {
6659 	static_call(kvm_x86_set_segment)(vcpu, var, seg);
6660 }
6661 
6662 void kvm_get_segment(struct kvm_vcpu *vcpu,
6663 		     struct kvm_segment *var, int seg)
6664 {
6665 	static_call(kvm_x86_get_segment)(vcpu, var, seg);
6666 }
6667 
6668 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
6669 			   struct x86_exception *exception)
6670 {
6671 	struct kvm_mmu *mmu = vcpu->arch.mmu;
6672 	gpa_t t_gpa;
6673 
6674 	BUG_ON(!mmu_is_nested(vcpu));
6675 
6676 	/* NPT walks are always user-walks */
6677 	access |= PFERR_USER_MASK;
6678 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
6679 
6680 	return t_gpa;
6681 }
6682 
6683 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
6684 			      struct x86_exception *exception)
6685 {
6686 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6687 
6688 	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
6689 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
6690 }
6691 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read);
6692 
6693  gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
6694 				struct x86_exception *exception)
6695 {
6696 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6697 
6698 	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
6699 	access |= PFERR_FETCH_MASK;
6700 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
6701 }
6702 
6703 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
6704 			       struct x86_exception *exception)
6705 {
6706 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6707 
6708 	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
6709 	access |= PFERR_WRITE_MASK;
6710 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
6711 }
6712 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write);
6713 
6714 /* uses this to access any guest's mapped memory without checking CPL */
6715 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
6716 				struct x86_exception *exception)
6717 {
6718 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6719 
6720 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
6721 }
6722 
6723 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
6724 				      struct kvm_vcpu *vcpu, u32 access,
6725 				      struct x86_exception *exception)
6726 {
6727 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6728 	void *data = val;
6729 	int r = X86EMUL_CONTINUE;
6730 
6731 	while (bytes) {
6732 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
6733 		unsigned offset = addr & (PAGE_SIZE-1);
6734 		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
6735 		int ret;
6736 
6737 		if (gpa == UNMAPPED_GVA)
6738 			return X86EMUL_PROPAGATE_FAULT;
6739 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
6740 					       offset, toread);
6741 		if (ret < 0) {
6742 			r = X86EMUL_IO_NEEDED;
6743 			goto out;
6744 		}
6745 
6746 		bytes -= toread;
6747 		data += toread;
6748 		addr += toread;
6749 	}
6750 out:
6751 	return r;
6752 }
6753 
6754 /* used for instruction fetching */
6755 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
6756 				gva_t addr, void *val, unsigned int bytes,
6757 				struct x86_exception *exception)
6758 {
6759 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6760 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6761 	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
6762 	unsigned offset;
6763 	int ret;
6764 
6765 	/* Inline kvm_read_guest_virt_helper for speed.  */
6766 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
6767 				    exception);
6768 	if (unlikely(gpa == UNMAPPED_GVA))
6769 		return X86EMUL_PROPAGATE_FAULT;
6770 
6771 	offset = addr & (PAGE_SIZE-1);
6772 	if (WARN_ON(offset + bytes > PAGE_SIZE))
6773 		bytes = (unsigned)PAGE_SIZE - offset;
6774 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
6775 				       offset, bytes);
6776 	if (unlikely(ret < 0))
6777 		return X86EMUL_IO_NEEDED;
6778 
6779 	return X86EMUL_CONTINUE;
6780 }
6781 
6782 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
6783 			       gva_t addr, void *val, unsigned int bytes,
6784 			       struct x86_exception *exception)
6785 {
6786 	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
6787 
6788 	/*
6789 	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
6790 	 * is returned, but our callers are not ready for that and they blindly
6791 	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
6792 	 * uninitialized kernel stack memory into cr2 and error code.
6793 	 */
6794 	memset(exception, 0, sizeof(*exception));
6795 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
6796 					  exception);
6797 }
6798 EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
6799 
6800 static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
6801 			     gva_t addr, void *val, unsigned int bytes,
6802 			     struct x86_exception *exception, bool system)
6803 {
6804 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6805 	u32 access = 0;
6806 
6807 	if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3)
6808 		access |= PFERR_USER_MASK;
6809 
6810 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
6811 }
6812 
6813 static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
6814 		unsigned long addr, void *val, unsigned int bytes)
6815 {
6816 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6817 	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);
6818 
6819 	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
6820 }
6821 
6822 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
6823 				      struct kvm_vcpu *vcpu, u32 access,
6824 				      struct x86_exception *exception)
6825 {
6826 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6827 	void *data = val;
6828 	int r = X86EMUL_CONTINUE;
6829 
6830 	while (bytes) {
6831 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
6832 		unsigned offset = addr & (PAGE_SIZE-1);
6833 		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
6834 		int ret;
6835 
6836 		if (gpa == UNMAPPED_GVA)
6837 			return X86EMUL_PROPAGATE_FAULT;
6838 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
6839 		if (ret < 0) {
6840 			r = X86EMUL_IO_NEEDED;
6841 			goto out;
6842 		}
6843 
6844 		bytes -= towrite;
6845 		data += towrite;
6846 		addr += towrite;
6847 	}
6848 out:
6849 	return r;
6850 }
6851 
6852 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
6853 			      unsigned int bytes, struct x86_exception *exception,
6854 			      bool system)
6855 {
6856 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6857 	u32 access = PFERR_WRITE_MASK;
6858 
6859 	if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3)
6860 		access |= PFERR_USER_MASK;
6861 
6862 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
6863 					   access, exception);
6864 }
6865 
6866 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
6867 				unsigned int bytes, struct x86_exception *exception)
6868 {
6869 	/* kvm_write_guest_virt_system can pull in tons of pages. */
6870 	vcpu->arch.l1tf_flush_l1d = true;
6871 
6872 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
6873 					   PFERR_WRITE_MASK, exception);
6874 }
6875 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
6876 
6877 static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,
6878 				void *insn, int insn_len)
6879 {
6880 	return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,
6881 							    insn, insn_len);
6882 }
6883 
6884 int handle_ud(struct kvm_vcpu *vcpu)
6885 {
6886 	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
6887 	int emul_type = EMULTYPE_TRAP_UD;
6888 	char sig[5]; /* ud2; .ascii "kvm" */
6889 	struct x86_exception e;
6890 
6891 	if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))
6892 		return 1;
6893 
6894 	if (force_emulation_prefix &&
6895 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
6896 				sig, sizeof(sig), &e) == 0 &&
6897 	    memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
6898 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
6899 		emul_type = EMULTYPE_TRAP_UD_FORCED;
6900 	}
6901 
6902 	return kvm_emulate_instruction(vcpu, emul_type);
6903 }
6904 EXPORT_SYMBOL_GPL(handle_ud);
6905 
6906 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
6907 			    gpa_t gpa, bool write)
6908 {
6909 	/* For APIC access vmexit */
6910 	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
6911 		return 1;
6912 
6913 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
6914 		trace_vcpu_match_mmio(gva, gpa, write, true);
6915 		return 1;
6916 	}
6917 
6918 	return 0;
6919 }
6920 
6921 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
6922 				gpa_t *gpa, struct x86_exception *exception,
6923 				bool write)
6924 {
6925 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
6926 	u32 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
6927 		| (write ? PFERR_WRITE_MASK : 0);
6928 
6929 	/*
6930 	 * currently PKRU is only applied to ept enabled guest so
6931 	 * there is no pkey in EPT page table for L1 guest or EPT
6932 	 * shadow page table for L2 guest.
6933 	 */
6934 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
6935 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,
6936 			      vcpu->arch.mmio_access, 0, access))) {
6937 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
6938 					(gva & (PAGE_SIZE - 1));
6939 		trace_vcpu_match_mmio(gva, *gpa, write, false);
6940 		return 1;
6941 	}
6942 
6943 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
6944 
6945 	if (*gpa == UNMAPPED_GVA)
6946 		return -1;
6947 
6948 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
6949 }
6950 
6951 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
6952 			const void *val, int bytes)
6953 {
6954 	int ret;
6955 
6956 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
6957 	if (ret < 0)
6958 		return 0;
6959 	kvm_page_track_write(vcpu, gpa, val, bytes);
6960 	return 1;
6961 }
6962 
6963 struct read_write_emulator_ops {
6964 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
6965 				  int bytes);
6966 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
6967 				  void *val, int bytes);
6968 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
6969 			       int bytes, void *val);
6970 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
6971 				    void *val, int bytes);
6972 	bool write;
6973 };
6974 
6975 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
6976 {
6977 	if (vcpu->mmio_read_completed) {
6978 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
6979 			       vcpu->mmio_fragments[0].gpa, val);
6980 		vcpu->mmio_read_completed = 0;
6981 		return 1;
6982 	}
6983 
6984 	return 0;
6985 }
6986 
6987 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
6988 			void *val, int bytes)
6989 {
6990 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
6991 }
6992 
6993 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
6994 			 void *val, int bytes)
6995 {
6996 	return emulator_write_phys(vcpu, gpa, val, bytes);
6997 }
6998 
6999 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
7000 {
7001 	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
7002 	return vcpu_mmio_write(vcpu, gpa, bytes, val);
7003 }
7004 
7005 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7006 			  void *val, int bytes)
7007 {
7008 	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
7009 	return X86EMUL_IO_NEEDED;
7010 }
7011 
7012 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7013 			   void *val, int bytes)
7014 {
7015 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
7016 
7017 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
7018 	return X86EMUL_CONTINUE;
7019 }
7020 
7021 static const struct read_write_emulator_ops read_emultor = {
7022 	.read_write_prepare = read_prepare,
7023 	.read_write_emulate = read_emulate,
7024 	.read_write_mmio = vcpu_mmio_read,
7025 	.read_write_exit_mmio = read_exit_mmio,
7026 };
7027 
7028 static const struct read_write_emulator_ops write_emultor = {
7029 	.read_write_emulate = write_emulate,
7030 	.read_write_mmio = write_mmio,
7031 	.read_write_exit_mmio = write_exit_mmio,
7032 	.write = true,
7033 };
7034 
7035 static int emulator_read_write_onepage(unsigned long addr, void *val,
7036 				       unsigned int bytes,
7037 				       struct x86_exception *exception,
7038 				       struct kvm_vcpu *vcpu,
7039 				       const struct read_write_emulator_ops *ops)
7040 {
7041 	gpa_t gpa;
7042 	int handled, ret;
7043 	bool write = ops->write;
7044 	struct kvm_mmio_fragment *frag;
7045 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7046 
7047 	/*
7048 	 * If the exit was due to a NPF we may already have a GPA.
7049 	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
7050 	 * Note, this cannot be used on string operations since string
7051 	 * operation using rep will only have the initial GPA from the NPF
7052 	 * occurred.
7053 	 */
7054 	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
7055 	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
7056 		gpa = ctxt->gpa_val;
7057 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
7058 	} else {
7059 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
7060 		if (ret < 0)
7061 			return X86EMUL_PROPAGATE_FAULT;
7062 	}
7063 
7064 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
7065 		return X86EMUL_CONTINUE;
7066 
7067 	/*
7068 	 * Is this MMIO handled locally?
7069 	 */
7070 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
7071 	if (handled == bytes)
7072 		return X86EMUL_CONTINUE;
7073 
7074 	gpa += handled;
7075 	bytes -= handled;
7076 	val += handled;
7077 
7078 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
7079 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
7080 	frag->gpa = gpa;
7081 	frag->data = val;
7082 	frag->len = bytes;
7083 	return X86EMUL_CONTINUE;
7084 }
7085 
7086 static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
7087 			unsigned long addr,
7088 			void *val, unsigned int bytes,
7089 			struct x86_exception *exception,
7090 			const struct read_write_emulator_ops *ops)
7091 {
7092 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7093 	gpa_t gpa;
7094 	int rc;
7095 
7096 	if (ops->read_write_prepare &&
7097 		  ops->read_write_prepare(vcpu, val, bytes))
7098 		return X86EMUL_CONTINUE;
7099 
7100 	vcpu->mmio_nr_fragments = 0;
7101 
7102 	/* Crossing a page boundary? */
7103 	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
7104 		int now;
7105 
7106 		now = -addr & ~PAGE_MASK;
7107 		rc = emulator_read_write_onepage(addr, val, now, exception,
7108 						 vcpu, ops);
7109 
7110 		if (rc != X86EMUL_CONTINUE)
7111 			return rc;
7112 		addr += now;
7113 		if (ctxt->mode != X86EMUL_MODE_PROT64)
7114 			addr = (u32)addr;
7115 		val += now;
7116 		bytes -= now;
7117 	}
7118 
7119 	rc = emulator_read_write_onepage(addr, val, bytes, exception,
7120 					 vcpu, ops);
7121 	if (rc != X86EMUL_CONTINUE)
7122 		return rc;
7123 
7124 	if (!vcpu->mmio_nr_fragments)
7125 		return rc;
7126 
7127 	gpa = vcpu->mmio_fragments[0].gpa;
7128 
7129 	vcpu->mmio_needed = 1;
7130 	vcpu->mmio_cur_fragment = 0;
7131 
7132 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
7133 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
7134 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
7135 	vcpu->run->mmio.phys_addr = gpa;
7136 
7137 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
7138 }
7139 
7140 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
7141 				  unsigned long addr,
7142 				  void *val,
7143 				  unsigned int bytes,
7144 				  struct x86_exception *exception)
7145 {
7146 	return emulator_read_write(ctxt, addr, val, bytes,
7147 				   exception, &read_emultor);
7148 }
7149 
7150 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
7151 			    unsigned long addr,
7152 			    const void *val,
7153 			    unsigned int bytes,
7154 			    struct x86_exception *exception)
7155 {
7156 	return emulator_read_write(ctxt, addr, (void *)val, bytes,
7157 				   exception, &write_emultor);
7158 }
7159 
7160 #define CMPXCHG_TYPE(t, ptr, old, new) \
7161 	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))
7162 
7163 #ifdef CONFIG_X86_64
7164 #  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
7165 #else
7166 #  define CMPXCHG64(ptr, old, new) \
7167 	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
7168 #endif
7169 
7170 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
7171 				     unsigned long addr,
7172 				     const void *old,
7173 				     const void *new,
7174 				     unsigned int bytes,
7175 				     struct x86_exception *exception)
7176 {
7177 	struct kvm_host_map map;
7178 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7179 	u64 page_line_mask;
7180 	gpa_t gpa;
7181 	char *kaddr;
7182 	bool exchanged;
7183 
7184 	/* guests cmpxchg8b have to be emulated atomically */
7185 	if (bytes > 8 || (bytes & (bytes - 1)))
7186 		goto emul_write;
7187 
7188 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
7189 
7190 	if (gpa == UNMAPPED_GVA ||
7191 	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
7192 		goto emul_write;
7193 
7194 	/*
7195 	 * Emulate the atomic as a straight write to avoid #AC if SLD is
7196 	 * enabled in the host and the access splits a cache line.
7197 	 */
7198 	if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
7199 		page_line_mask = ~(cache_line_size() - 1);
7200 	else
7201 		page_line_mask = PAGE_MASK;
7202 
7203 	if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
7204 		goto emul_write;
7205 
7206 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
7207 		goto emul_write;
7208 
7209 	kaddr = map.hva + offset_in_page(gpa);
7210 
7211 	switch (bytes) {
7212 	case 1:
7213 		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
7214 		break;
7215 	case 2:
7216 		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
7217 		break;
7218 	case 4:
7219 		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
7220 		break;
7221 	case 8:
7222 		exchanged = CMPXCHG64(kaddr, old, new);
7223 		break;
7224 	default:
7225 		BUG();
7226 	}
7227 
7228 	kvm_vcpu_unmap(vcpu, &map, true);
7229 
7230 	if (!exchanged)
7231 		return X86EMUL_CMPXCHG_FAILED;
7232 
7233 	kvm_page_track_write(vcpu, gpa, new, bytes);
7234 
7235 	return X86EMUL_CONTINUE;
7236 
7237 emul_write:
7238 	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
7239 
7240 	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
7241 }
7242 
7243 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
7244 {
7245 	int r = 0, i;
7246 
7247 	for (i = 0; i < vcpu->arch.pio.count; i++) {
7248 		if (vcpu->arch.pio.in)
7249 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
7250 					    vcpu->arch.pio.size, pd);
7251 		else
7252 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
7253 					     vcpu->arch.pio.port, vcpu->arch.pio.size,
7254 					     pd);
7255 		if (r)
7256 			break;
7257 		pd += vcpu->arch.pio.size;
7258 	}
7259 	return r;
7260 }
7261 
7262 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
7263 			       unsigned short port,
7264 			       unsigned int count, bool in)
7265 {
7266 	vcpu->arch.pio.port = port;
7267 	vcpu->arch.pio.in = in;
7268 	vcpu->arch.pio.count  = count;
7269 	vcpu->arch.pio.size = size;
7270 
7271 	if (!kernel_pio(vcpu, vcpu->arch.pio_data))
7272 		return 1;
7273 
7274 	vcpu->run->exit_reason = KVM_EXIT_IO;
7275 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
7276 	vcpu->run->io.size = size;
7277 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
7278 	vcpu->run->io.count = count;
7279 	vcpu->run->io.port = port;
7280 
7281 	return 0;
7282 }
7283 
7284 static int __emulator_pio_in(struct kvm_vcpu *vcpu, int size,
7285 			     unsigned short port, unsigned int count)
7286 {
7287 	WARN_ON(vcpu->arch.pio.count);
7288 	memset(vcpu->arch.pio_data, 0, size * count);
7289 	return emulator_pio_in_out(vcpu, size, port, count, true);
7290 }
7291 
7292 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)
7293 {
7294 	int size = vcpu->arch.pio.size;
7295 	unsigned count = vcpu->arch.pio.count;
7296 	memcpy(val, vcpu->arch.pio_data, size * count);
7297 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);
7298 	vcpu->arch.pio.count = 0;
7299 }
7300 
7301 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
7302 			   unsigned short port, void *val, unsigned int count)
7303 {
7304 	if (vcpu->arch.pio.count) {
7305 		/*
7306 		 * Complete a previous iteration that required userspace I/O.
7307 		 * Note, @count isn't guaranteed to match pio.count as userspace
7308 		 * can modify ECX before rerunning the vCPU.  Ignore any such
7309 		 * shenanigans as KVM doesn't support modifying the rep count,
7310 		 * and the emulator ensures @count doesn't overflow the buffer.
7311 		 */
7312 	} else {
7313 		int r = __emulator_pio_in(vcpu, size, port, count);
7314 		if (!r)
7315 			return r;
7316 
7317 		/* Results already available, fall through.  */
7318 	}
7319 
7320 	complete_emulator_pio_in(vcpu, val);
7321 	return 1;
7322 }
7323 
7324 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
7325 				    int size, unsigned short port, void *val,
7326 				    unsigned int count)
7327 {
7328 	return emulator_pio_in(emul_to_vcpu(ctxt), size, port, val, count);
7329 
7330 }
7331 
7332 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
7333 			    unsigned short port, const void *val,
7334 			    unsigned int count)
7335 {
7336 	int ret;
7337 
7338 	memcpy(vcpu->arch.pio_data, val, size * count);
7339 	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
7340 	ret = emulator_pio_in_out(vcpu, size, port, count, false);
7341 	if (ret)
7342                 vcpu->arch.pio.count = 0;
7343 
7344         return ret;
7345 }
7346 
7347 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
7348 				     int size, unsigned short port,
7349 				     const void *val, unsigned int count)
7350 {
7351 	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
7352 }
7353 
7354 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
7355 {
7356 	return static_call(kvm_x86_get_segment_base)(vcpu, seg);
7357 }
7358 
7359 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
7360 {
7361 	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
7362 }
7363 
7364 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
7365 {
7366 	if (!need_emulate_wbinvd(vcpu))
7367 		return X86EMUL_CONTINUE;
7368 
7369 	if (static_call(kvm_x86_has_wbinvd_exit)()) {
7370 		int cpu = get_cpu();
7371 
7372 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
7373 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,
7374 				wbinvd_ipi, NULL, 1);
7375 		put_cpu();
7376 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
7377 	} else
7378 		wbinvd();
7379 	return X86EMUL_CONTINUE;
7380 }
7381 
7382 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
7383 {
7384 	kvm_emulate_wbinvd_noskip(vcpu);
7385 	return kvm_skip_emulated_instruction(vcpu);
7386 }
7387 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
7388 
7389 
7390 
7391 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
7392 {
7393 	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
7394 }
7395 
7396 static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
7397 			    unsigned long *dest)
7398 {
7399 	kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
7400 }
7401 
7402 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
7403 			   unsigned long value)
7404 {
7405 
7406 	return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
7407 }
7408 
7409 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
7410 {
7411 	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
7412 }
7413 
7414 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
7415 {
7416 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7417 	unsigned long value;
7418 
7419 	switch (cr) {
7420 	case 0:
7421 		value = kvm_read_cr0(vcpu);
7422 		break;
7423 	case 2:
7424 		value = vcpu->arch.cr2;
7425 		break;
7426 	case 3:
7427 		value = kvm_read_cr3(vcpu);
7428 		break;
7429 	case 4:
7430 		value = kvm_read_cr4(vcpu);
7431 		break;
7432 	case 8:
7433 		value = kvm_get_cr8(vcpu);
7434 		break;
7435 	default:
7436 		kvm_err("%s: unexpected cr %u\n", __func__, cr);
7437 		return 0;
7438 	}
7439 
7440 	return value;
7441 }
7442 
7443 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
7444 {
7445 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7446 	int res = 0;
7447 
7448 	switch (cr) {
7449 	case 0:
7450 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
7451 		break;
7452 	case 2:
7453 		vcpu->arch.cr2 = val;
7454 		break;
7455 	case 3:
7456 		res = kvm_set_cr3(vcpu, val);
7457 		break;
7458 	case 4:
7459 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
7460 		break;
7461 	case 8:
7462 		res = kvm_set_cr8(vcpu, val);
7463 		break;
7464 	default:
7465 		kvm_err("%s: unexpected cr %u\n", __func__, cr);
7466 		res = -1;
7467 	}
7468 
7469 	return res;
7470 }
7471 
7472 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
7473 {
7474 	return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));
7475 }
7476 
7477 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7478 {
7479 	static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
7480 }
7481 
7482 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7483 {
7484 	static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
7485 }
7486 
7487 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7488 {
7489 	static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);
7490 }
7491 
7492 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7493 {
7494 	static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);
7495 }
7496 
7497 static unsigned long emulator_get_cached_segment_base(
7498 	struct x86_emulate_ctxt *ctxt, int seg)
7499 {
7500 	return get_segment_base(emul_to_vcpu(ctxt), seg);
7501 }
7502 
7503 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
7504 				 struct desc_struct *desc, u32 *base3,
7505 				 int seg)
7506 {
7507 	struct kvm_segment var;
7508 
7509 	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
7510 	*selector = var.selector;
7511 
7512 	if (var.unusable) {
7513 		memset(desc, 0, sizeof(*desc));
7514 		if (base3)
7515 			*base3 = 0;
7516 		return false;
7517 	}
7518 
7519 	if (var.g)
7520 		var.limit >>= 12;
7521 	set_desc_limit(desc, var.limit);
7522 	set_desc_base(desc, (unsigned long)var.base);
7523 #ifdef CONFIG_X86_64
7524 	if (base3)
7525 		*base3 = var.base >> 32;
7526 #endif
7527 	desc->type = var.type;
7528 	desc->s = var.s;
7529 	desc->dpl = var.dpl;
7530 	desc->p = var.present;
7531 	desc->avl = var.avl;
7532 	desc->l = var.l;
7533 	desc->d = var.db;
7534 	desc->g = var.g;
7535 
7536 	return true;
7537 }
7538 
7539 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
7540 				 struct desc_struct *desc, u32 base3,
7541 				 int seg)
7542 {
7543 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7544 	struct kvm_segment var;
7545 
7546 	var.selector = selector;
7547 	var.base = get_desc_base(desc);
7548 #ifdef CONFIG_X86_64
7549 	var.base |= ((u64)base3) << 32;
7550 #endif
7551 	var.limit = get_desc_limit(desc);
7552 	if (desc->g)
7553 		var.limit = (var.limit << 12) | 0xfff;
7554 	var.type = desc->type;
7555 	var.dpl = desc->dpl;
7556 	var.db = desc->d;
7557 	var.s = desc->s;
7558 	var.l = desc->l;
7559 	var.g = desc->g;
7560 	var.avl = desc->avl;
7561 	var.present = desc->p;
7562 	var.unusable = !var.present;
7563 	var.padding = 0;
7564 
7565 	kvm_set_segment(vcpu, &var, seg);
7566 	return;
7567 }
7568 
7569 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
7570 			    u32 msr_index, u64 *pdata)
7571 {
7572 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7573 	int r;
7574 
7575 	r = kvm_get_msr(vcpu, msr_index, pdata);
7576 
7577 	if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
7578 				    complete_emulated_rdmsr, r)) {
7579 		/* Bounce to user space */
7580 		return X86EMUL_IO_NEEDED;
7581 	}
7582 
7583 	return r;
7584 }
7585 
7586 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
7587 			    u32 msr_index, u64 data)
7588 {
7589 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7590 	int r;
7591 
7592 	r = kvm_set_msr(vcpu, msr_index, data);
7593 
7594 	if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
7595 				    complete_emulated_msr_access, r)) {
7596 		/* Bounce to user space */
7597 		return X86EMUL_IO_NEEDED;
7598 	}
7599 
7600 	return r;
7601 }
7602 
7603 static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
7604 {
7605 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7606 
7607 	return vcpu->arch.smbase;
7608 }
7609 
7610 static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
7611 {
7612 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7613 
7614 	vcpu->arch.smbase = smbase;
7615 }
7616 
7617 static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
7618 			      u32 pmc)
7619 {
7620 	if (kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc))
7621 		return 0;
7622 	return -EINVAL;
7623 }
7624 
7625 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
7626 			     u32 pmc, u64 *pdata)
7627 {
7628 	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
7629 }
7630 
7631 static void emulator_halt(struct x86_emulate_ctxt *ctxt)
7632 {
7633 	emul_to_vcpu(ctxt)->arch.halt_request = 1;
7634 }
7635 
7636 static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
7637 			      struct x86_instruction_info *info,
7638 			      enum x86_intercept_stage stage)
7639 {
7640 	return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,
7641 					    &ctxt->exception);
7642 }
7643 
7644 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
7645 			      u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
7646 			      bool exact_only)
7647 {
7648 	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);
7649 }
7650 
7651 static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
7652 {
7653 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
7654 }
7655 
7656 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
7657 {
7658 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
7659 }
7660 
7661 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
7662 {
7663 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
7664 }
7665 
7666 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
7667 {
7668 	return kvm_register_read_raw(emul_to_vcpu(ctxt), reg);
7669 }
7670 
7671 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
7672 {
7673 	kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val);
7674 }
7675 
7676 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
7677 {
7678 	static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);
7679 }
7680 
7681 static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
7682 {
7683 	return emul_to_vcpu(ctxt)->arch.hflags;
7684 }
7685 
7686 static void emulator_exiting_smm(struct x86_emulate_ctxt *ctxt)
7687 {
7688 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7689 
7690 	kvm_smm_changed(vcpu, false);
7691 }
7692 
7693 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt,
7694 				  const char *smstate)
7695 {
7696 	return static_call(kvm_x86_leave_smm)(emul_to_vcpu(ctxt), smstate);
7697 }
7698 
7699 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt)
7700 {
7701 	kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));
7702 }
7703 
7704 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
7705 {
7706 	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
7707 }
7708 
7709 static const struct x86_emulate_ops emulate_ops = {
7710 	.read_gpr            = emulator_read_gpr,
7711 	.write_gpr           = emulator_write_gpr,
7712 	.read_std            = emulator_read_std,
7713 	.write_std           = emulator_write_std,
7714 	.read_phys           = kvm_read_guest_phys_system,
7715 	.fetch               = kvm_fetch_guest_virt,
7716 	.read_emulated       = emulator_read_emulated,
7717 	.write_emulated      = emulator_write_emulated,
7718 	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
7719 	.invlpg              = emulator_invlpg,
7720 	.pio_in_emulated     = emulator_pio_in_emulated,
7721 	.pio_out_emulated    = emulator_pio_out_emulated,
7722 	.get_segment         = emulator_get_segment,
7723 	.set_segment         = emulator_set_segment,
7724 	.get_cached_segment_base = emulator_get_cached_segment_base,
7725 	.get_gdt             = emulator_get_gdt,
7726 	.get_idt	     = emulator_get_idt,
7727 	.set_gdt             = emulator_set_gdt,
7728 	.set_idt	     = emulator_set_idt,
7729 	.get_cr              = emulator_get_cr,
7730 	.set_cr              = emulator_set_cr,
7731 	.cpl                 = emulator_get_cpl,
7732 	.get_dr              = emulator_get_dr,
7733 	.set_dr              = emulator_set_dr,
7734 	.get_smbase          = emulator_get_smbase,
7735 	.set_smbase          = emulator_set_smbase,
7736 	.set_msr             = emulator_set_msr,
7737 	.get_msr             = emulator_get_msr,
7738 	.check_pmc	     = emulator_check_pmc,
7739 	.read_pmc            = emulator_read_pmc,
7740 	.halt                = emulator_halt,
7741 	.wbinvd              = emulator_wbinvd,
7742 	.fix_hypercall       = emulator_fix_hypercall,
7743 	.intercept           = emulator_intercept,
7744 	.get_cpuid           = emulator_get_cpuid,
7745 	.guest_has_long_mode = emulator_guest_has_long_mode,
7746 	.guest_has_movbe     = emulator_guest_has_movbe,
7747 	.guest_has_fxsr      = emulator_guest_has_fxsr,
7748 	.set_nmi_mask        = emulator_set_nmi_mask,
7749 	.get_hflags          = emulator_get_hflags,
7750 	.exiting_smm         = emulator_exiting_smm,
7751 	.leave_smm           = emulator_leave_smm,
7752 	.triple_fault        = emulator_triple_fault,
7753 	.set_xcr             = emulator_set_xcr,
7754 };
7755 
7756 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
7757 {
7758 	u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
7759 	/*
7760 	 * an sti; sti; sequence only disable interrupts for the first
7761 	 * instruction. So, if the last instruction, be it emulated or
7762 	 * not, left the system with the INT_STI flag enabled, it
7763 	 * means that the last instruction is an sti. We should not
7764 	 * leave the flag on in this case. The same goes for mov ss
7765 	 */
7766 	if (int_shadow & mask)
7767 		mask = 0;
7768 	if (unlikely(int_shadow || mask)) {
7769 		static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
7770 		if (!mask)
7771 			kvm_make_request(KVM_REQ_EVENT, vcpu);
7772 	}
7773 }
7774 
7775 static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
7776 {
7777 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7778 	if (ctxt->exception.vector == PF_VECTOR)
7779 		return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
7780 
7781 	if (ctxt->exception.error_code_valid)
7782 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
7783 				      ctxt->exception.error_code);
7784 	else
7785 		kvm_queue_exception(vcpu, ctxt->exception.vector);
7786 	return false;
7787 }
7788 
7789 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
7790 {
7791 	struct x86_emulate_ctxt *ctxt;
7792 
7793 	ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);
7794 	if (!ctxt) {
7795 		pr_err("kvm: failed to allocate vcpu's emulator\n");
7796 		return NULL;
7797 	}
7798 
7799 	ctxt->vcpu = vcpu;
7800 	ctxt->ops = &emulate_ops;
7801 	vcpu->arch.emulate_ctxt = ctxt;
7802 
7803 	return ctxt;
7804 }
7805 
7806 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
7807 {
7808 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7809 	int cs_db, cs_l;
7810 
7811 	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
7812 
7813 	ctxt->gpa_available = false;
7814 	ctxt->eflags = kvm_get_rflags(vcpu);
7815 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
7816 
7817 	ctxt->eip = kvm_rip_read(vcpu);
7818 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
7819 		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
7820 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
7821 		     cs_db				? X86EMUL_MODE_PROT32 :
7822 							  X86EMUL_MODE_PROT16;
7823 	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
7824 	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
7825 	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
7826 
7827 	ctxt->interruptibility = 0;
7828 	ctxt->have_exception = false;
7829 	ctxt->exception.vector = -1;
7830 	ctxt->perm_ok = false;
7831 
7832 	init_decode_cache(ctxt);
7833 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
7834 }
7835 
7836 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
7837 {
7838 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7839 	int ret;
7840 
7841 	init_emulate_ctxt(vcpu);
7842 
7843 	ctxt->op_bytes = 2;
7844 	ctxt->ad_bytes = 2;
7845 	ctxt->_eip = ctxt->eip + inc_eip;
7846 	ret = emulate_int_real(ctxt, irq);
7847 
7848 	if (ret != X86EMUL_CONTINUE) {
7849 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7850 	} else {
7851 		ctxt->eip = ctxt->_eip;
7852 		kvm_rip_write(vcpu, ctxt->eip);
7853 		kvm_set_rflags(vcpu, ctxt->eflags);
7854 	}
7855 }
7856 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
7857 
7858 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
7859 					   u8 ndata, u8 *insn_bytes, u8 insn_size)
7860 {
7861 	struct kvm_run *run = vcpu->run;
7862 	u64 info[5];
7863 	u8 info_start;
7864 
7865 	/*
7866 	 * Zero the whole array used to retrieve the exit info, as casting to
7867 	 * u32 for select entries will leave some chunks uninitialized.
7868 	 */
7869 	memset(&info, 0, sizeof(info));
7870 
7871 	static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],
7872 					   &info[2], (u32 *)&info[3],
7873 					   (u32 *)&info[4]);
7874 
7875 	run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7876 	run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION;
7877 
7878 	/*
7879 	 * There's currently space for 13 entries, but 5 are used for the exit
7880 	 * reason and info.  Restrict to 4 to reduce the maintenance burden
7881 	 * when expanding kvm_run.emulation_failure in the future.
7882 	 */
7883 	if (WARN_ON_ONCE(ndata > 4))
7884 		ndata = 4;
7885 
7886 	/* Always include the flags as a 'data' entry. */
7887 	info_start = 1;
7888 	run->emulation_failure.flags = 0;
7889 
7890 	if (insn_size) {
7891 		BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) +
7892 			      sizeof(run->emulation_failure.insn_bytes) != 16));
7893 		info_start += 2;
7894 		run->emulation_failure.flags |=
7895 			KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES;
7896 		run->emulation_failure.insn_size = insn_size;
7897 		memset(run->emulation_failure.insn_bytes, 0x90,
7898 		       sizeof(run->emulation_failure.insn_bytes));
7899 		memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size);
7900 	}
7901 
7902 	memcpy(&run->internal.data[info_start], info, sizeof(info));
7903 	memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data,
7904 	       ndata * sizeof(data[0]));
7905 
7906 	run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata;
7907 }
7908 
7909 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)
7910 {
7911 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7912 
7913 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,
7914 				       ctxt->fetch.end - ctxt->fetch.data);
7915 }
7916 
7917 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
7918 					  u8 ndata)
7919 {
7920 	prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);
7921 }
7922 EXPORT_SYMBOL_GPL(__kvm_prepare_emulation_failure_exit);
7923 
7924 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)
7925 {
7926 	__kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);
7927 }
7928 EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit);
7929 
7930 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
7931 {
7932 	struct kvm *kvm = vcpu->kvm;
7933 
7934 	++vcpu->stat.insn_emulation_fail;
7935 	trace_kvm_emulate_insn_failed(vcpu);
7936 
7937 	if (emulation_type & EMULTYPE_VMWARE_GP) {
7938 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
7939 		return 1;
7940 	}
7941 
7942 	if (kvm->arch.exit_on_emulation_error ||
7943 	    (emulation_type & EMULTYPE_SKIP)) {
7944 		prepare_emulation_ctxt_failure_exit(vcpu);
7945 		return 0;
7946 	}
7947 
7948 	kvm_queue_exception(vcpu, UD_VECTOR);
7949 
7950 	if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
7951 		prepare_emulation_ctxt_failure_exit(vcpu);
7952 		return 0;
7953 	}
7954 
7955 	return 1;
7956 }
7957 
7958 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
7959 				  bool write_fault_to_shadow_pgtable,
7960 				  int emulation_type)
7961 {
7962 	gpa_t gpa = cr2_or_gpa;
7963 	kvm_pfn_t pfn;
7964 
7965 	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
7966 		return false;
7967 
7968 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
7969 	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
7970 		return false;
7971 
7972 	if (!vcpu->arch.mmu->direct_map) {
7973 		/*
7974 		 * Write permission should be allowed since only
7975 		 * write access need to be emulated.
7976 		 */
7977 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
7978 
7979 		/*
7980 		 * If the mapping is invalid in guest, let cpu retry
7981 		 * it to generate fault.
7982 		 */
7983 		if (gpa == UNMAPPED_GVA)
7984 			return true;
7985 	}
7986 
7987 	/*
7988 	 * Do not retry the unhandleable instruction if it faults on the
7989 	 * readonly host memory, otherwise it will goto a infinite loop:
7990 	 * retry instruction -> write #PF -> emulation fail -> retry
7991 	 * instruction -> ...
7992 	 */
7993 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
7994 
7995 	/*
7996 	 * If the instruction failed on the error pfn, it can not be fixed,
7997 	 * report the error to userspace.
7998 	 */
7999 	if (is_error_noslot_pfn(pfn))
8000 		return false;
8001 
8002 	kvm_release_pfn_clean(pfn);
8003 
8004 	/* The instructions are well-emulated on direct mmu. */
8005 	if (vcpu->arch.mmu->direct_map) {
8006 		unsigned int indirect_shadow_pages;
8007 
8008 		write_lock(&vcpu->kvm->mmu_lock);
8009 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
8010 		write_unlock(&vcpu->kvm->mmu_lock);
8011 
8012 		if (indirect_shadow_pages)
8013 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8014 
8015 		return true;
8016 	}
8017 
8018 	/*
8019 	 * if emulation was due to access to shadowed page table
8020 	 * and it failed try to unshadow page and re-enter the
8021 	 * guest to let CPU execute the instruction.
8022 	 */
8023 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8024 
8025 	/*
8026 	 * If the access faults on its page table, it can not
8027 	 * be fixed by unprotecting shadow page and it should
8028 	 * be reported to userspace.
8029 	 */
8030 	return !write_fault_to_shadow_pgtable;
8031 }
8032 
8033 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
8034 			      gpa_t cr2_or_gpa,  int emulation_type)
8035 {
8036 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8037 	unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
8038 
8039 	last_retry_eip = vcpu->arch.last_retry_eip;
8040 	last_retry_addr = vcpu->arch.last_retry_addr;
8041 
8042 	/*
8043 	 * If the emulation is caused by #PF and it is non-page_table
8044 	 * writing instruction, it means the VM-EXIT is caused by shadow
8045 	 * page protected, we can zap the shadow page and retry this
8046 	 * instruction directly.
8047 	 *
8048 	 * Note: if the guest uses a non-page-table modifying instruction
8049 	 * on the PDE that points to the instruction, then we will unmap
8050 	 * the instruction and go to an infinite loop. So, we cache the
8051 	 * last retried eip and the last fault address, if we meet the eip
8052 	 * and the address again, we can break out of the potential infinite
8053 	 * loop.
8054 	 */
8055 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
8056 
8057 	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
8058 		return false;
8059 
8060 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8061 	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
8062 		return false;
8063 
8064 	if (x86_page_table_writing_insn(ctxt))
8065 		return false;
8066 
8067 	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
8068 		return false;
8069 
8070 	vcpu->arch.last_retry_eip = ctxt->eip;
8071 	vcpu->arch.last_retry_addr = cr2_or_gpa;
8072 
8073 	if (!vcpu->arch.mmu->direct_map)
8074 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8075 
8076 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8077 
8078 	return true;
8079 }
8080 
8081 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8082 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8083 
8084 static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm)
8085 {
8086 	trace_kvm_smm_transition(vcpu->vcpu_id, vcpu->arch.smbase, entering_smm);
8087 
8088 	if (entering_smm) {
8089 		vcpu->arch.hflags |= HF_SMM_MASK;
8090 	} else {
8091 		vcpu->arch.hflags &= ~(HF_SMM_MASK | HF_SMM_INSIDE_NMI_MASK);
8092 
8093 		/* Process a latched INIT or SMI, if any.  */
8094 		kvm_make_request(KVM_REQ_EVENT, vcpu);
8095 
8096 		/*
8097 		 * Even if KVM_SET_SREGS2 loaded PDPTRs out of band,
8098 		 * on SMM exit we still need to reload them from
8099 		 * guest memory
8100 		 */
8101 		vcpu->arch.pdptrs_from_userspace = false;
8102 	}
8103 
8104 	kvm_mmu_reset_context(vcpu);
8105 }
8106 
8107 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
8108 				unsigned long *db)
8109 {
8110 	u32 dr6 = 0;
8111 	int i;
8112 	u32 enable, rwlen;
8113 
8114 	enable = dr7;
8115 	rwlen = dr7 >> 16;
8116 	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
8117 		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
8118 			dr6 |= (1 << i);
8119 	return dr6;
8120 }
8121 
8122 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
8123 {
8124 	struct kvm_run *kvm_run = vcpu->run;
8125 
8126 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
8127 		kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW;
8128 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
8129 		kvm_run->debug.arch.exception = DB_VECTOR;
8130 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
8131 		return 0;
8132 	}
8133 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
8134 	return 1;
8135 }
8136 
8137 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
8138 {
8139 	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
8140 	int r;
8141 
8142 	r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
8143 	if (unlikely(!r))
8144 		return 0;
8145 
8146 	kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
8147 
8148 	/*
8149 	 * rflags is the old, "raw" value of the flags.  The new value has
8150 	 * not been saved yet.
8151 	 *
8152 	 * This is correct even for TF set by the guest, because "the
8153 	 * processor will not generate this exception after the instruction
8154 	 * that sets the TF flag".
8155 	 */
8156 	if (unlikely(rflags & X86_EFLAGS_TF))
8157 		r = kvm_vcpu_do_singlestep(vcpu);
8158 	return r;
8159 }
8160 EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);
8161 
8162 static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
8163 {
8164 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
8165 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
8166 		struct kvm_run *kvm_run = vcpu->run;
8167 		unsigned long eip = kvm_get_linear_rip(vcpu);
8168 		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
8169 					   vcpu->arch.guest_debug_dr7,
8170 					   vcpu->arch.eff_db);
8171 
8172 		if (dr6 != 0) {
8173 			kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
8174 			kvm_run->debug.arch.pc = eip;
8175 			kvm_run->debug.arch.exception = DB_VECTOR;
8176 			kvm_run->exit_reason = KVM_EXIT_DEBUG;
8177 			*r = 0;
8178 			return true;
8179 		}
8180 	}
8181 
8182 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
8183 	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
8184 		unsigned long eip = kvm_get_linear_rip(vcpu);
8185 		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
8186 					   vcpu->arch.dr7,
8187 					   vcpu->arch.db);
8188 
8189 		if (dr6 != 0) {
8190 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
8191 			*r = 1;
8192 			return true;
8193 		}
8194 	}
8195 
8196 	return false;
8197 }
8198 
8199 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
8200 {
8201 	switch (ctxt->opcode_len) {
8202 	case 1:
8203 		switch (ctxt->b) {
8204 		case 0xe4:	/* IN */
8205 		case 0xe5:
8206 		case 0xec:
8207 		case 0xed:
8208 		case 0xe6:	/* OUT */
8209 		case 0xe7:
8210 		case 0xee:
8211 		case 0xef:
8212 		case 0x6c:	/* INS */
8213 		case 0x6d:
8214 		case 0x6e:	/* OUTS */
8215 		case 0x6f:
8216 			return true;
8217 		}
8218 		break;
8219 	case 2:
8220 		switch (ctxt->b) {
8221 		case 0x33:	/* RDPMC */
8222 			return true;
8223 		}
8224 		break;
8225 	}
8226 
8227 	return false;
8228 }
8229 
8230 /*
8231  * Decode to be emulated instruction. Return EMULATION_OK if success.
8232  */
8233 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
8234 				    void *insn, int insn_len)
8235 {
8236 	int r = EMULATION_OK;
8237 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8238 
8239 	init_emulate_ctxt(vcpu);
8240 
8241 	/*
8242 	 * We will reenter on the same instruction since we do not set
8243 	 * complete_userspace_io. This does not handle watchpoints yet,
8244 	 * those would be handled in the emulate_ops.
8245 	 */
8246 	if (!(emulation_type & EMULTYPE_SKIP) &&
8247 	    kvm_vcpu_check_breakpoint(vcpu, &r))
8248 		return r;
8249 
8250 	r = x86_decode_insn(ctxt, insn, insn_len, emulation_type);
8251 
8252 	trace_kvm_emulate_insn_start(vcpu);
8253 	++vcpu->stat.insn_emulation;
8254 
8255 	return r;
8256 }
8257 EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction);
8258 
8259 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8260 			    int emulation_type, void *insn, int insn_len)
8261 {
8262 	int r;
8263 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8264 	bool writeback = true;
8265 	bool write_fault_to_spt;
8266 
8267 	if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))
8268 		return 1;
8269 
8270 	vcpu->arch.l1tf_flush_l1d = true;
8271 
8272 	/*
8273 	 * Clear write_fault_to_shadow_pgtable here to ensure it is
8274 	 * never reused.
8275 	 */
8276 	write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
8277 	vcpu->arch.write_fault_to_shadow_pgtable = false;
8278 
8279 	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
8280 		kvm_clear_exception_queue(vcpu);
8281 
8282 		r = x86_decode_emulated_instruction(vcpu, emulation_type,
8283 						    insn, insn_len);
8284 		if (r != EMULATION_OK)  {
8285 			if ((emulation_type & EMULTYPE_TRAP_UD) ||
8286 			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
8287 				kvm_queue_exception(vcpu, UD_VECTOR);
8288 				return 1;
8289 			}
8290 			if (reexecute_instruction(vcpu, cr2_or_gpa,
8291 						  write_fault_to_spt,
8292 						  emulation_type))
8293 				return 1;
8294 			if (ctxt->have_exception) {
8295 				/*
8296 				 * #UD should result in just EMULATION_FAILED, and trap-like
8297 				 * exception should not be encountered during decode.
8298 				 */
8299 				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
8300 					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
8301 				inject_emulated_exception(vcpu);
8302 				return 1;
8303 			}
8304 			return handle_emulation_failure(vcpu, emulation_type);
8305 		}
8306 	}
8307 
8308 	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
8309 	    !is_vmware_backdoor_opcode(ctxt)) {
8310 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8311 		return 1;
8312 	}
8313 
8314 	/*
8315 	 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for
8316 	 * use *only* by vendor callbacks for kvm_skip_emulated_instruction().
8317 	 * The caller is responsible for updating interruptibility state and
8318 	 * injecting single-step #DBs.
8319 	 */
8320 	if (emulation_type & EMULTYPE_SKIP) {
8321 		if (ctxt->mode != X86EMUL_MODE_PROT64)
8322 			ctxt->eip = (u32)ctxt->_eip;
8323 		else
8324 			ctxt->eip = ctxt->_eip;
8325 
8326 		if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) {
8327 			r = 1;
8328 			goto writeback;
8329 		}
8330 
8331 		kvm_rip_write(vcpu, ctxt->eip);
8332 		if (ctxt->eflags & X86_EFLAGS_RF)
8333 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
8334 		return 1;
8335 	}
8336 
8337 	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
8338 		return 1;
8339 
8340 	/* this is needed for vmware backdoor interface to work since it
8341 	   changes registers values  during IO operation */
8342 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
8343 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
8344 		emulator_invalidate_register_cache(ctxt);
8345 	}
8346 
8347 restart:
8348 	if (emulation_type & EMULTYPE_PF) {
8349 		/* Save the faulting GPA (cr2) in the address field */
8350 		ctxt->exception.address = cr2_or_gpa;
8351 
8352 		/* With shadow page tables, cr2 contains a GVA or nGPA. */
8353 		if (vcpu->arch.mmu->direct_map) {
8354 			ctxt->gpa_available = true;
8355 			ctxt->gpa_val = cr2_or_gpa;
8356 		}
8357 	} else {
8358 		/* Sanitize the address out of an abundance of paranoia. */
8359 		ctxt->exception.address = 0;
8360 	}
8361 
8362 	r = x86_emulate_insn(ctxt);
8363 
8364 	if (r == EMULATION_INTERCEPTED)
8365 		return 1;
8366 
8367 	if (r == EMULATION_FAILED) {
8368 		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
8369 					emulation_type))
8370 			return 1;
8371 
8372 		return handle_emulation_failure(vcpu, emulation_type);
8373 	}
8374 
8375 	if (ctxt->have_exception) {
8376 		r = 1;
8377 		if (inject_emulated_exception(vcpu))
8378 			return r;
8379 	} else if (vcpu->arch.pio.count) {
8380 		if (!vcpu->arch.pio.in) {
8381 			/* FIXME: return into emulator if single-stepping.  */
8382 			vcpu->arch.pio.count = 0;
8383 		} else {
8384 			writeback = false;
8385 			vcpu->arch.complete_userspace_io = complete_emulated_pio;
8386 		}
8387 		r = 0;
8388 	} else if (vcpu->mmio_needed) {
8389 		++vcpu->stat.mmio_exits;
8390 
8391 		if (!vcpu->mmio_is_write)
8392 			writeback = false;
8393 		r = 0;
8394 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
8395 	} else if (vcpu->arch.complete_userspace_io) {
8396 		writeback = false;
8397 		r = 0;
8398 	} else if (r == EMULATION_RESTART)
8399 		goto restart;
8400 	else
8401 		r = 1;
8402 
8403 writeback:
8404 	if (writeback) {
8405 		unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
8406 		toggle_interruptibility(vcpu, ctxt->interruptibility);
8407 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
8408 		if (!ctxt->have_exception ||
8409 		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
8410 			kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
8411 			if (ctxt->is_branch)
8412 				kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
8413 			kvm_rip_write(vcpu, ctxt->eip);
8414 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
8415 				r = kvm_vcpu_do_singlestep(vcpu);
8416 			if (kvm_x86_ops.update_emulated_instruction)
8417 				static_call(kvm_x86_update_emulated_instruction)(vcpu);
8418 			__kvm_set_rflags(vcpu, ctxt->eflags);
8419 		}
8420 
8421 		/*
8422 		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
8423 		 * do nothing, and it will be requested again as soon as
8424 		 * the shadow expires.  But we still need to check here,
8425 		 * because POPF has no interrupt shadow.
8426 		 */
8427 		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
8428 			kvm_make_request(KVM_REQ_EVENT, vcpu);
8429 	} else
8430 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
8431 
8432 	return r;
8433 }
8434 
8435 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
8436 {
8437 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
8438 }
8439 EXPORT_SYMBOL_GPL(kvm_emulate_instruction);
8440 
8441 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
8442 					void *insn, int insn_len)
8443 {
8444 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
8445 }
8446 EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
8447 
8448 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
8449 {
8450 	vcpu->arch.pio.count = 0;
8451 	return 1;
8452 }
8453 
8454 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
8455 {
8456 	vcpu->arch.pio.count = 0;
8457 
8458 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
8459 		return 1;
8460 
8461 	return kvm_skip_emulated_instruction(vcpu);
8462 }
8463 
8464 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
8465 			    unsigned short port)
8466 {
8467 	unsigned long val = kvm_rax_read(vcpu);
8468 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);
8469 
8470 	if (ret)
8471 		return ret;
8472 
8473 	/*
8474 	 * Workaround userspace that relies on old KVM behavior of %rip being
8475 	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
8476 	 */
8477 	if (port == 0x7e &&
8478 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
8479 		vcpu->arch.complete_userspace_io =
8480 			complete_fast_pio_out_port_0x7e;
8481 		kvm_skip_emulated_instruction(vcpu);
8482 	} else {
8483 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
8484 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
8485 	}
8486 	return 0;
8487 }
8488 
8489 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
8490 {
8491 	unsigned long val;
8492 
8493 	/* We should only ever be called with arch.pio.count equal to 1 */
8494 	BUG_ON(vcpu->arch.pio.count != 1);
8495 
8496 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
8497 		vcpu->arch.pio.count = 0;
8498 		return 1;
8499 	}
8500 
8501 	/* For size less than 4 we merge, else we zero extend */
8502 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
8503 
8504 	/*
8505 	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in perform
8506 	 * the copy and tracing
8507 	 */
8508 	emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1);
8509 	kvm_rax_write(vcpu, val);
8510 
8511 	return kvm_skip_emulated_instruction(vcpu);
8512 }
8513 
8514 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
8515 			   unsigned short port)
8516 {
8517 	unsigned long val;
8518 	int ret;
8519 
8520 	/* For size less than 4 we merge, else we zero extend */
8521 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
8522 
8523 	ret = emulator_pio_in(vcpu, size, port, &val, 1);
8524 	if (ret) {
8525 		kvm_rax_write(vcpu, val);
8526 		return ret;
8527 	}
8528 
8529 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
8530 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;
8531 
8532 	return 0;
8533 }
8534 
8535 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
8536 {
8537 	int ret;
8538 
8539 	if (in)
8540 		ret = kvm_fast_pio_in(vcpu, size, port);
8541 	else
8542 		ret = kvm_fast_pio_out(vcpu, size, port);
8543 	return ret && kvm_skip_emulated_instruction(vcpu);
8544 }
8545 EXPORT_SYMBOL_GPL(kvm_fast_pio);
8546 
8547 static int kvmclock_cpu_down_prep(unsigned int cpu)
8548 {
8549 	__this_cpu_write(cpu_tsc_khz, 0);
8550 	return 0;
8551 }
8552 
8553 static void tsc_khz_changed(void *data)
8554 {
8555 	struct cpufreq_freqs *freq = data;
8556 	unsigned long khz = 0;
8557 
8558 	if (data)
8559 		khz = freq->new;
8560 	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
8561 		khz = cpufreq_quick_get(raw_smp_processor_id());
8562 	if (!khz)
8563 		khz = tsc_khz;
8564 	__this_cpu_write(cpu_tsc_khz, khz);
8565 }
8566 
8567 #ifdef CONFIG_X86_64
8568 static void kvm_hyperv_tsc_notifier(void)
8569 {
8570 	struct kvm *kvm;
8571 	int cpu;
8572 
8573 	mutex_lock(&kvm_lock);
8574 	list_for_each_entry(kvm, &vm_list, vm_list)
8575 		kvm_make_mclock_inprogress_request(kvm);
8576 
8577 	/* no guest entries from this point */
8578 	hyperv_stop_tsc_emulation();
8579 
8580 	/* TSC frequency always matches when on Hyper-V */
8581 	for_each_present_cpu(cpu)
8582 		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
8583 	kvm_max_guest_tsc_khz = tsc_khz;
8584 
8585 	list_for_each_entry(kvm, &vm_list, vm_list) {
8586 		__kvm_start_pvclock_update(kvm);
8587 		pvclock_update_vm_gtod_copy(kvm);
8588 		kvm_end_pvclock_update(kvm);
8589 	}
8590 
8591 	mutex_unlock(&kvm_lock);
8592 }
8593 #endif
8594 
8595 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
8596 {
8597 	struct kvm *kvm;
8598 	struct kvm_vcpu *vcpu;
8599 	int send_ipi = 0;
8600 	unsigned long i;
8601 
8602 	/*
8603 	 * We allow guests to temporarily run on slowing clocks,
8604 	 * provided we notify them after, or to run on accelerating
8605 	 * clocks, provided we notify them before.  Thus time never
8606 	 * goes backwards.
8607 	 *
8608 	 * However, we have a problem.  We can't atomically update
8609 	 * the frequency of a given CPU from this function; it is
8610 	 * merely a notifier, which can be called from any CPU.
8611 	 * Changing the TSC frequency at arbitrary points in time
8612 	 * requires a recomputation of local variables related to
8613 	 * the TSC for each VCPU.  We must flag these local variables
8614 	 * to be updated and be sure the update takes place with the
8615 	 * new frequency before any guests proceed.
8616 	 *
8617 	 * Unfortunately, the combination of hotplug CPU and frequency
8618 	 * change creates an intractable locking scenario; the order
8619 	 * of when these callouts happen is undefined with respect to
8620 	 * CPU hotplug, and they can race with each other.  As such,
8621 	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
8622 	 * undefined; you can actually have a CPU frequency change take
8623 	 * place in between the computation of X and the setting of the
8624 	 * variable.  To protect against this problem, all updates of
8625 	 * the per_cpu tsc_khz variable are done in an interrupt
8626 	 * protected IPI, and all callers wishing to update the value
8627 	 * must wait for a synchronous IPI to complete (which is trivial
8628 	 * if the caller is on the CPU already).  This establishes the
8629 	 * necessary total order on variable updates.
8630 	 *
8631 	 * Note that because a guest time update may take place
8632 	 * anytime after the setting of the VCPU's request bit, the
8633 	 * correct TSC value must be set before the request.  However,
8634 	 * to ensure the update actually makes it to any guest which
8635 	 * starts running in hardware virtualization between the set
8636 	 * and the acquisition of the spinlock, we must also ping the
8637 	 * CPU after setting the request bit.
8638 	 *
8639 	 */
8640 
8641 	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
8642 
8643 	mutex_lock(&kvm_lock);
8644 	list_for_each_entry(kvm, &vm_list, vm_list) {
8645 		kvm_for_each_vcpu(i, vcpu, kvm) {
8646 			if (vcpu->cpu != cpu)
8647 				continue;
8648 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8649 			if (vcpu->cpu != raw_smp_processor_id())
8650 				send_ipi = 1;
8651 		}
8652 	}
8653 	mutex_unlock(&kvm_lock);
8654 
8655 	if (freq->old < freq->new && send_ipi) {
8656 		/*
8657 		 * We upscale the frequency.  Must make the guest
8658 		 * doesn't see old kvmclock values while running with
8659 		 * the new frequency, otherwise we risk the guest sees
8660 		 * time go backwards.
8661 		 *
8662 		 * In case we update the frequency for another cpu
8663 		 * (which might be in guest context) send an interrupt
8664 		 * to kick the cpu out of guest context.  Next time
8665 		 * guest context is entered kvmclock will be updated,
8666 		 * so the guest will not see stale values.
8667 		 */
8668 		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
8669 	}
8670 }
8671 
8672 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
8673 				     void *data)
8674 {
8675 	struct cpufreq_freqs *freq = data;
8676 	int cpu;
8677 
8678 	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
8679 		return 0;
8680 	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
8681 		return 0;
8682 
8683 	for_each_cpu(cpu, freq->policy->cpus)
8684 		__kvmclock_cpufreq_notifier(freq, cpu);
8685 
8686 	return 0;
8687 }
8688 
8689 static struct notifier_block kvmclock_cpufreq_notifier_block = {
8690 	.notifier_call  = kvmclock_cpufreq_notifier
8691 };
8692 
8693 static int kvmclock_cpu_online(unsigned int cpu)
8694 {
8695 	tsc_khz_changed(NULL);
8696 	return 0;
8697 }
8698 
8699 static void kvm_timer_init(void)
8700 {
8701 	max_tsc_khz = tsc_khz;
8702 
8703 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
8704 #ifdef CONFIG_CPU_FREQ
8705 		struct cpufreq_policy *policy;
8706 		int cpu;
8707 
8708 		cpu = get_cpu();
8709 		policy = cpufreq_cpu_get(cpu);
8710 		if (policy) {
8711 			if (policy->cpuinfo.max_freq)
8712 				max_tsc_khz = policy->cpuinfo.max_freq;
8713 			cpufreq_cpu_put(policy);
8714 		}
8715 		put_cpu();
8716 #endif
8717 		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
8718 					  CPUFREQ_TRANSITION_NOTIFIER);
8719 	}
8720 
8721 	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
8722 			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
8723 }
8724 
8725 #ifdef CONFIG_X86_64
8726 static void pvclock_gtod_update_fn(struct work_struct *work)
8727 {
8728 	struct kvm *kvm;
8729 	struct kvm_vcpu *vcpu;
8730 	unsigned long i;
8731 
8732 	mutex_lock(&kvm_lock);
8733 	list_for_each_entry(kvm, &vm_list, vm_list)
8734 		kvm_for_each_vcpu(i, vcpu, kvm)
8735 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
8736 	atomic_set(&kvm_guest_has_master_clock, 0);
8737 	mutex_unlock(&kvm_lock);
8738 }
8739 
8740 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
8741 
8742 /*
8743  * Indirection to move queue_work() out of the tk_core.seq write held
8744  * region to prevent possible deadlocks against time accessors which
8745  * are invoked with work related locks held.
8746  */
8747 static void pvclock_irq_work_fn(struct irq_work *w)
8748 {
8749 	queue_work(system_long_wq, &pvclock_gtod_work);
8750 }
8751 
8752 static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn);
8753 
8754 /*
8755  * Notification about pvclock gtod data update.
8756  */
8757 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
8758 			       void *priv)
8759 {
8760 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
8761 	struct timekeeper *tk = priv;
8762 
8763 	update_pvclock_gtod(tk);
8764 
8765 	/*
8766 	 * Disable master clock if host does not trust, or does not use,
8767 	 * TSC based clocksource. Delegate queue_work() to irq_work as
8768 	 * this is invoked with tk_core.seq write held.
8769 	 */
8770 	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
8771 	    atomic_read(&kvm_guest_has_master_clock) != 0)
8772 		irq_work_queue(&pvclock_irq_work);
8773 	return 0;
8774 }
8775 
8776 static struct notifier_block pvclock_gtod_notifier = {
8777 	.notifier_call = pvclock_gtod_notify,
8778 };
8779 #endif
8780 
8781 int kvm_arch_init(void *opaque)
8782 {
8783 	struct kvm_x86_init_ops *ops = opaque;
8784 	int r;
8785 
8786 	if (kvm_x86_ops.hardware_enable) {
8787 		pr_err("kvm: already loaded vendor module '%s'\n", kvm_x86_ops.name);
8788 		r = -EEXIST;
8789 		goto out;
8790 	}
8791 
8792 	if (!ops->cpu_has_kvm_support()) {
8793 		pr_err_ratelimited("kvm: no hardware support for '%s'\n",
8794 				   ops->runtime_ops->name);
8795 		r = -EOPNOTSUPP;
8796 		goto out;
8797 	}
8798 	if (ops->disabled_by_bios()) {
8799 		pr_err_ratelimited("kvm: support for '%s' disabled by bios\n",
8800 				   ops->runtime_ops->name);
8801 		r = -EOPNOTSUPP;
8802 		goto out;
8803 	}
8804 
8805 	/*
8806 	 * KVM explicitly assumes that the guest has an FPU and
8807 	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
8808 	 * vCPU's FPU state as a fxregs_state struct.
8809 	 */
8810 	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
8811 		printk(KERN_ERR "kvm: inadequate fpu\n");
8812 		r = -EOPNOTSUPP;
8813 		goto out;
8814 	}
8815 
8816 	r = -ENOMEM;
8817 
8818 	x86_emulator_cache = kvm_alloc_emulator_cache();
8819 	if (!x86_emulator_cache) {
8820 		pr_err("kvm: failed to allocate cache for x86 emulator\n");
8821 		goto out;
8822 	}
8823 
8824 	user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
8825 	if (!user_return_msrs) {
8826 		printk(KERN_ERR "kvm: failed to allocate percpu kvm_user_return_msrs\n");
8827 		goto out_free_x86_emulator_cache;
8828 	}
8829 	kvm_nr_uret_msrs = 0;
8830 
8831 	r = kvm_mmu_module_init();
8832 	if (r)
8833 		goto out_free_percpu;
8834 
8835 	kvm_timer_init();
8836 
8837 	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
8838 		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
8839 		supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
8840 	}
8841 
8842 	if (pi_inject_timer == -1)
8843 		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
8844 #ifdef CONFIG_X86_64
8845 	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
8846 
8847 	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
8848 		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
8849 #endif
8850 
8851 	return 0;
8852 
8853 out_free_percpu:
8854 	free_percpu(user_return_msrs);
8855 out_free_x86_emulator_cache:
8856 	kmem_cache_destroy(x86_emulator_cache);
8857 out:
8858 	return r;
8859 }
8860 
8861 void kvm_arch_exit(void)
8862 {
8863 #ifdef CONFIG_X86_64
8864 	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
8865 		clear_hv_tscchange_cb();
8866 #endif
8867 	kvm_lapic_exit();
8868 
8869 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
8870 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
8871 					    CPUFREQ_TRANSITION_NOTIFIER);
8872 	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
8873 #ifdef CONFIG_X86_64
8874 	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
8875 	irq_work_sync(&pvclock_irq_work);
8876 	cancel_work_sync(&pvclock_gtod_work);
8877 #endif
8878 	kvm_x86_ops.hardware_enable = NULL;
8879 	kvm_mmu_module_exit();
8880 	free_percpu(user_return_msrs);
8881 	kmem_cache_destroy(x86_emulator_cache);
8882 #ifdef CONFIG_KVM_XEN
8883 	static_key_deferred_flush(&kvm_xen_enabled);
8884 	WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
8885 #endif
8886 }
8887 
8888 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)
8889 {
8890 	/*
8891 	 * The vCPU has halted, e.g. executed HLT.  Update the run state if the
8892 	 * local APIC is in-kernel, the run loop will detect the non-runnable
8893 	 * state and halt the vCPU.  Exit to userspace if the local APIC is
8894 	 * managed by userspace, in which case userspace is responsible for
8895 	 * handling wake events.
8896 	 */
8897 	++vcpu->stat.halt_exits;
8898 	if (lapic_in_kernel(vcpu)) {
8899 		vcpu->arch.mp_state = state;
8900 		return 1;
8901 	} else {
8902 		vcpu->run->exit_reason = reason;
8903 		return 0;
8904 	}
8905 }
8906 
8907 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)
8908 {
8909 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);
8910 }
8911 EXPORT_SYMBOL_GPL(kvm_emulate_halt_noskip);
8912 
8913 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
8914 {
8915 	int ret = kvm_skip_emulated_instruction(vcpu);
8916 	/*
8917 	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
8918 	 * KVM_EXIT_DEBUG here.
8919 	 */
8920 	return kvm_emulate_halt_noskip(vcpu) && ret;
8921 }
8922 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
8923 
8924 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)
8925 {
8926 	int ret = kvm_skip_emulated_instruction(vcpu);
8927 
8928 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,
8929 					KVM_EXIT_AP_RESET_HOLD) && ret;
8930 }
8931 EXPORT_SYMBOL_GPL(kvm_emulate_ap_reset_hold);
8932 
8933 #ifdef CONFIG_X86_64
8934 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
8935 			        unsigned long clock_type)
8936 {
8937 	struct kvm_clock_pairing clock_pairing;
8938 	struct timespec64 ts;
8939 	u64 cycle;
8940 	int ret;
8941 
8942 	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
8943 		return -KVM_EOPNOTSUPP;
8944 
8945 	if (!kvm_get_walltime_and_clockread(&ts, &cycle))
8946 		return -KVM_EOPNOTSUPP;
8947 
8948 	clock_pairing.sec = ts.tv_sec;
8949 	clock_pairing.nsec = ts.tv_nsec;
8950 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
8951 	clock_pairing.flags = 0;
8952 	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
8953 
8954 	ret = 0;
8955 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
8956 			    sizeof(struct kvm_clock_pairing)))
8957 		ret = -KVM_EFAULT;
8958 
8959 	return ret;
8960 }
8961 #endif
8962 
8963 /*
8964  * kvm_pv_kick_cpu_op:  Kick a vcpu.
8965  *
8966  * @apicid - apicid of vcpu to be kicked.
8967  */
8968 static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
8969 {
8970 	struct kvm_lapic_irq lapic_irq;
8971 
8972 	lapic_irq.shorthand = APIC_DEST_NOSHORT;
8973 	lapic_irq.dest_mode = APIC_DEST_PHYSICAL;
8974 	lapic_irq.level = 0;
8975 	lapic_irq.dest_id = apicid;
8976 	lapic_irq.msi_redir_hint = false;
8977 
8978 	lapic_irq.delivery_mode = APIC_DM_REMRD;
8979 	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
8980 }
8981 
8982 bool kvm_apicv_activated(struct kvm *kvm)
8983 {
8984 	return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
8985 }
8986 EXPORT_SYMBOL_GPL(kvm_apicv_activated);
8987 
8988 static void kvm_apicv_init(struct kvm *kvm)
8989 {
8990 	init_rwsem(&kvm->arch.apicv_update_lock);
8991 
8992 	set_bit(APICV_INHIBIT_REASON_ABSENT,
8993 		&kvm->arch.apicv_inhibit_reasons);
8994 	if (!enable_apicv)
8995 		set_bit(APICV_INHIBIT_REASON_DISABLE,
8996 			&kvm->arch.apicv_inhibit_reasons);
8997 }
8998 
8999 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)
9000 {
9001 	struct kvm_vcpu *target = NULL;
9002 	struct kvm_apic_map *map;
9003 
9004 	vcpu->stat.directed_yield_attempted++;
9005 
9006 	if (single_task_running())
9007 		goto no_yield;
9008 
9009 	rcu_read_lock();
9010 	map = rcu_dereference(vcpu->kvm->arch.apic_map);
9011 
9012 	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
9013 		target = map->phys_map[dest_id]->vcpu;
9014 
9015 	rcu_read_unlock();
9016 
9017 	if (!target || !READ_ONCE(target->ready))
9018 		goto no_yield;
9019 
9020 	/* Ignore requests to yield to self */
9021 	if (vcpu == target)
9022 		goto no_yield;
9023 
9024 	if (kvm_vcpu_yield_to(target) <= 0)
9025 		goto no_yield;
9026 
9027 	vcpu->stat.directed_yield_successful++;
9028 
9029 no_yield:
9030 	return;
9031 }
9032 
9033 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
9034 {
9035 	u64 ret = vcpu->run->hypercall.ret;
9036 
9037 	if (!is_64_bit_mode(vcpu))
9038 		ret = (u32)ret;
9039 	kvm_rax_write(vcpu, ret);
9040 	++vcpu->stat.hypercalls;
9041 	return kvm_skip_emulated_instruction(vcpu);
9042 }
9043 
9044 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
9045 {
9046 	unsigned long nr, a0, a1, a2, a3, ret;
9047 	int op_64_bit;
9048 
9049 	if (kvm_xen_hypercall_enabled(vcpu->kvm))
9050 		return kvm_xen_hypercall(vcpu);
9051 
9052 	if (kvm_hv_hypercall_enabled(vcpu))
9053 		return kvm_hv_hypercall(vcpu);
9054 
9055 	nr = kvm_rax_read(vcpu);
9056 	a0 = kvm_rbx_read(vcpu);
9057 	a1 = kvm_rcx_read(vcpu);
9058 	a2 = kvm_rdx_read(vcpu);
9059 	a3 = kvm_rsi_read(vcpu);
9060 
9061 	trace_kvm_hypercall(nr, a0, a1, a2, a3);
9062 
9063 	op_64_bit = is_64_bit_hypercall(vcpu);
9064 	if (!op_64_bit) {
9065 		nr &= 0xFFFFFFFF;
9066 		a0 &= 0xFFFFFFFF;
9067 		a1 &= 0xFFFFFFFF;
9068 		a2 &= 0xFFFFFFFF;
9069 		a3 &= 0xFFFFFFFF;
9070 	}
9071 
9072 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
9073 		ret = -KVM_EPERM;
9074 		goto out;
9075 	}
9076 
9077 	ret = -KVM_ENOSYS;
9078 
9079 	switch (nr) {
9080 	case KVM_HC_VAPIC_POLL_IRQ:
9081 		ret = 0;
9082 		break;
9083 	case KVM_HC_KICK_CPU:
9084 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))
9085 			break;
9086 
9087 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
9088 		kvm_sched_yield(vcpu, a1);
9089 		ret = 0;
9090 		break;
9091 #ifdef CONFIG_X86_64
9092 	case KVM_HC_CLOCK_PAIRING:
9093 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
9094 		break;
9095 #endif
9096 	case KVM_HC_SEND_IPI:
9097 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))
9098 			break;
9099 
9100 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
9101 		break;
9102 	case KVM_HC_SCHED_YIELD:
9103 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))
9104 			break;
9105 
9106 		kvm_sched_yield(vcpu, a0);
9107 		ret = 0;
9108 		break;
9109 	case KVM_HC_MAP_GPA_RANGE: {
9110 		u64 gpa = a0, npages = a1, attrs = a2;
9111 
9112 		ret = -KVM_ENOSYS;
9113 		if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))
9114 			break;
9115 
9116 		if (!PAGE_ALIGNED(gpa) || !npages ||
9117 		    gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) {
9118 			ret = -KVM_EINVAL;
9119 			break;
9120 		}
9121 
9122 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;
9123 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;
9124 		vcpu->run->hypercall.args[0]  = gpa;
9125 		vcpu->run->hypercall.args[1]  = npages;
9126 		vcpu->run->hypercall.args[2]  = attrs;
9127 		vcpu->run->hypercall.longmode = op_64_bit;
9128 		vcpu->arch.complete_userspace_io = complete_hypercall_exit;
9129 		return 0;
9130 	}
9131 	default:
9132 		ret = -KVM_ENOSYS;
9133 		break;
9134 	}
9135 out:
9136 	if (!op_64_bit)
9137 		ret = (u32)ret;
9138 	kvm_rax_write(vcpu, ret);
9139 
9140 	++vcpu->stat.hypercalls;
9141 	return kvm_skip_emulated_instruction(vcpu);
9142 }
9143 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
9144 
9145 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
9146 {
9147 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
9148 	char instruction[3];
9149 	unsigned long rip = kvm_rip_read(vcpu);
9150 
9151 	static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
9152 
9153 	return emulator_write_emulated(ctxt, rip, instruction, 3,
9154 		&ctxt->exception);
9155 }
9156 
9157 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
9158 {
9159 	return vcpu->run->request_interrupt_window &&
9160 		likely(!pic_in_kernel(vcpu->kvm));
9161 }
9162 
9163 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
9164 {
9165 	struct kvm_run *kvm_run = vcpu->run;
9166 
9167 	kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);
9168 	kvm_run->cr8 = kvm_get_cr8(vcpu);
9169 	kvm_run->apic_base = kvm_get_apic_base(vcpu);
9170 
9171 	/*
9172 	 * The call to kvm_ready_for_interrupt_injection() may end up in
9173 	 * kvm_xen_has_interrupt() which may require the srcu lock to be
9174 	 * held, to protect against changes in the vcpu_info address.
9175 	 */
9176 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
9177 	kvm_run->ready_for_interrupt_injection =
9178 		pic_in_kernel(vcpu->kvm) ||
9179 		kvm_vcpu_ready_for_interrupt_injection(vcpu);
9180 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
9181 
9182 	if (is_smm(vcpu))
9183 		kvm_run->flags |= KVM_RUN_X86_SMM;
9184 }
9185 
9186 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
9187 {
9188 	int max_irr, tpr;
9189 
9190 	if (!kvm_x86_ops.update_cr8_intercept)
9191 		return;
9192 
9193 	if (!lapic_in_kernel(vcpu))
9194 		return;
9195 
9196 	if (vcpu->arch.apicv_active)
9197 		return;
9198 
9199 	if (!vcpu->arch.apic->vapic_addr)
9200 		max_irr = kvm_lapic_find_highest_irr(vcpu);
9201 	else
9202 		max_irr = -1;
9203 
9204 	if (max_irr != -1)
9205 		max_irr >>= 4;
9206 
9207 	tpr = kvm_lapic_get_cr8(vcpu);
9208 
9209 	static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
9210 }
9211 
9212 
9213 int kvm_check_nested_events(struct kvm_vcpu *vcpu)
9214 {
9215 	if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
9216 		kvm_x86_ops.nested_ops->triple_fault(vcpu);
9217 		return 1;
9218 	}
9219 
9220 	return kvm_x86_ops.nested_ops->check_events(vcpu);
9221 }
9222 
9223 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
9224 {
9225 	if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
9226 		vcpu->arch.exception.error_code = false;
9227 	static_call(kvm_x86_queue_exception)(vcpu);
9228 }
9229 
9230 static int inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit)
9231 {
9232 	int r;
9233 	bool can_inject = true;
9234 
9235 	/* try to reinject previous events if any */
9236 
9237 	if (vcpu->arch.exception.injected) {
9238 		kvm_inject_exception(vcpu);
9239 		can_inject = false;
9240 	}
9241 	/*
9242 	 * Do not inject an NMI or interrupt if there is a pending
9243 	 * exception.  Exceptions and interrupts are recognized at
9244 	 * instruction boundaries, i.e. the start of an instruction.
9245 	 * Trap-like exceptions, e.g. #DB, have higher priority than
9246 	 * NMIs and interrupts, i.e. traps are recognized before an
9247 	 * NMI/interrupt that's pending on the same instruction.
9248 	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
9249 	 * priority, but are only generated (pended) during instruction
9250 	 * execution, i.e. a pending fault-like exception means the
9251 	 * fault occurred on the *previous* instruction and must be
9252 	 * serviced prior to recognizing any new events in order to
9253 	 * fully complete the previous instruction.
9254 	 */
9255 	else if (!vcpu->arch.exception.pending) {
9256 		if (vcpu->arch.nmi_injected) {
9257 			static_call(kvm_x86_set_nmi)(vcpu);
9258 			can_inject = false;
9259 		} else if (vcpu->arch.interrupt.injected) {
9260 			static_call(kvm_x86_set_irq)(vcpu);
9261 			can_inject = false;
9262 		}
9263 	}
9264 
9265 	WARN_ON_ONCE(vcpu->arch.exception.injected &&
9266 		     vcpu->arch.exception.pending);
9267 
9268 	/*
9269 	 * Call check_nested_events() even if we reinjected a previous event
9270 	 * in order for caller to determine if it should require immediate-exit
9271 	 * from L2 to L1 due to pending L1 events which require exit
9272 	 * from L2 to L1.
9273 	 */
9274 	if (is_guest_mode(vcpu)) {
9275 		r = kvm_check_nested_events(vcpu);
9276 		if (r < 0)
9277 			goto out;
9278 	}
9279 
9280 	/* try to inject new event if pending */
9281 	if (vcpu->arch.exception.pending) {
9282 		trace_kvm_inj_exception(vcpu->arch.exception.nr,
9283 					vcpu->arch.exception.has_error_code,
9284 					vcpu->arch.exception.error_code);
9285 
9286 		vcpu->arch.exception.pending = false;
9287 		vcpu->arch.exception.injected = true;
9288 
9289 		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
9290 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
9291 					     X86_EFLAGS_RF);
9292 
9293 		if (vcpu->arch.exception.nr == DB_VECTOR) {
9294 			kvm_deliver_exception_payload(vcpu);
9295 			if (vcpu->arch.dr7 & DR7_GD) {
9296 				vcpu->arch.dr7 &= ~DR7_GD;
9297 				kvm_update_dr7(vcpu);
9298 			}
9299 		}
9300 
9301 		kvm_inject_exception(vcpu);
9302 		can_inject = false;
9303 	}
9304 
9305 	/* Don't inject interrupts if the user asked to avoid doing so */
9306 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
9307 		return 0;
9308 
9309 	/*
9310 	 * Finally, inject interrupt events.  If an event cannot be injected
9311 	 * due to architectural conditions (e.g. IF=0) a window-open exit
9312 	 * will re-request KVM_REQ_EVENT.  Sometimes however an event is pending
9313 	 * and can architecturally be injected, but we cannot do it right now:
9314 	 * an interrupt could have arrived just now and we have to inject it
9315 	 * as a vmexit, or there could already an event in the queue, which is
9316 	 * indicated by can_inject.  In that case we request an immediate exit
9317 	 * in order to make progress and get back here for another iteration.
9318 	 * The kvm_x86_ops hooks communicate this by returning -EBUSY.
9319 	 */
9320 	if (vcpu->arch.smi_pending) {
9321 		r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
9322 		if (r < 0)
9323 			goto out;
9324 		if (r) {
9325 			vcpu->arch.smi_pending = false;
9326 			++vcpu->arch.smi_count;
9327 			enter_smm(vcpu);
9328 			can_inject = false;
9329 		} else
9330 			static_call(kvm_x86_enable_smi_window)(vcpu);
9331 	}
9332 
9333 	if (vcpu->arch.nmi_pending) {
9334 		r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
9335 		if (r < 0)
9336 			goto out;
9337 		if (r) {
9338 			--vcpu->arch.nmi_pending;
9339 			vcpu->arch.nmi_injected = true;
9340 			static_call(kvm_x86_set_nmi)(vcpu);
9341 			can_inject = false;
9342 			WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
9343 		}
9344 		if (vcpu->arch.nmi_pending)
9345 			static_call(kvm_x86_enable_nmi_window)(vcpu);
9346 	}
9347 
9348 	if (kvm_cpu_has_injectable_intr(vcpu)) {
9349 		r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
9350 		if (r < 0)
9351 			goto out;
9352 		if (r) {
9353 			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
9354 			static_call(kvm_x86_set_irq)(vcpu);
9355 			WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
9356 		}
9357 		if (kvm_cpu_has_injectable_intr(vcpu))
9358 			static_call(kvm_x86_enable_irq_window)(vcpu);
9359 	}
9360 
9361 	if (is_guest_mode(vcpu) &&
9362 	    kvm_x86_ops.nested_ops->hv_timer_pending &&
9363 	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
9364 		*req_immediate_exit = true;
9365 
9366 	WARN_ON(vcpu->arch.exception.pending);
9367 	return 0;
9368 
9369 out:
9370 	if (r == -EBUSY) {
9371 		*req_immediate_exit = true;
9372 		r = 0;
9373 	}
9374 	return r;
9375 }
9376 
9377 static void process_nmi(struct kvm_vcpu *vcpu)
9378 {
9379 	unsigned limit = 2;
9380 
9381 	/*
9382 	 * x86 is limited to one NMI running, and one NMI pending after it.
9383 	 * If an NMI is already in progress, limit further NMIs to just one.
9384 	 * Otherwise, allow two (and we'll inject the first one immediately).
9385 	 */
9386 	if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)
9387 		limit = 1;
9388 
9389 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
9390 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
9391 	kvm_make_request(KVM_REQ_EVENT, vcpu);
9392 }
9393 
9394 static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
9395 {
9396 	u32 flags = 0;
9397 	flags |= seg->g       << 23;
9398 	flags |= seg->db      << 22;
9399 	flags |= seg->l       << 21;
9400 	flags |= seg->avl     << 20;
9401 	flags |= seg->present << 15;
9402 	flags |= seg->dpl     << 13;
9403 	flags |= seg->s       << 12;
9404 	flags |= seg->type    << 8;
9405 	return flags;
9406 }
9407 
9408 static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
9409 {
9410 	struct kvm_segment seg;
9411 	int offset;
9412 
9413 	kvm_get_segment(vcpu, &seg, n);
9414 	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);
9415 
9416 	if (n < 3)
9417 		offset = 0x7f84 + n * 12;
9418 	else
9419 		offset = 0x7f2c + (n - 3) * 12;
9420 
9421 	put_smstate(u32, buf, offset + 8, seg.base);
9422 	put_smstate(u32, buf, offset + 4, seg.limit);
9423 	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
9424 }
9425 
9426 #ifdef CONFIG_X86_64
9427 static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
9428 {
9429 	struct kvm_segment seg;
9430 	int offset;
9431 	u16 flags;
9432 
9433 	kvm_get_segment(vcpu, &seg, n);
9434 	offset = 0x7e00 + n * 16;
9435 
9436 	flags = enter_smm_get_segment_flags(&seg) >> 8;
9437 	put_smstate(u16, buf, offset, seg.selector);
9438 	put_smstate(u16, buf, offset + 2, flags);
9439 	put_smstate(u32, buf, offset + 4, seg.limit);
9440 	put_smstate(u64, buf, offset + 8, seg.base);
9441 }
9442 #endif
9443 
9444 static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
9445 {
9446 	struct desc_ptr dt;
9447 	struct kvm_segment seg;
9448 	unsigned long val;
9449 	int i;
9450 
9451 	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
9452 	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
9453 	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
9454 	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));
9455 
9456 	for (i = 0; i < 8; i++)
9457 		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read_raw(vcpu, i));
9458 
9459 	kvm_get_dr(vcpu, 6, &val);
9460 	put_smstate(u32, buf, 0x7fcc, (u32)val);
9461 	kvm_get_dr(vcpu, 7, &val);
9462 	put_smstate(u32, buf, 0x7fc8, (u32)val);
9463 
9464 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
9465 	put_smstate(u32, buf, 0x7fc4, seg.selector);
9466 	put_smstate(u32, buf, 0x7f64, seg.base);
9467 	put_smstate(u32, buf, 0x7f60, seg.limit);
9468 	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
9469 
9470 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
9471 	put_smstate(u32, buf, 0x7fc0, seg.selector);
9472 	put_smstate(u32, buf, 0x7f80, seg.base);
9473 	put_smstate(u32, buf, 0x7f7c, seg.limit);
9474 	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
9475 
9476 	static_call(kvm_x86_get_gdt)(vcpu, &dt);
9477 	put_smstate(u32, buf, 0x7f74, dt.address);
9478 	put_smstate(u32, buf, 0x7f70, dt.size);
9479 
9480 	static_call(kvm_x86_get_idt)(vcpu, &dt);
9481 	put_smstate(u32, buf, 0x7f58, dt.address);
9482 	put_smstate(u32, buf, 0x7f54, dt.size);
9483 
9484 	for (i = 0; i < 6; i++)
9485 		enter_smm_save_seg_32(vcpu, buf, i);
9486 
9487 	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));
9488 
9489 	/* revision id */
9490 	put_smstate(u32, buf, 0x7efc, 0x00020000);
9491 	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
9492 }
9493 
9494 #ifdef CONFIG_X86_64
9495 static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
9496 {
9497 	struct desc_ptr dt;
9498 	struct kvm_segment seg;
9499 	unsigned long val;
9500 	int i;
9501 
9502 	for (i = 0; i < 16; i++)
9503 		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read_raw(vcpu, i));
9504 
9505 	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
9506 	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));
9507 
9508 	kvm_get_dr(vcpu, 6, &val);
9509 	put_smstate(u64, buf, 0x7f68, val);
9510 	kvm_get_dr(vcpu, 7, &val);
9511 	put_smstate(u64, buf, 0x7f60, val);
9512 
9513 	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
9514 	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
9515 	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));
9516 
9517 	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);
9518 
9519 	/* revision id */
9520 	put_smstate(u32, buf, 0x7efc, 0x00020064);
9521 
9522 	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);
9523 
9524 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
9525 	put_smstate(u16, buf, 0x7e90, seg.selector);
9526 	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
9527 	put_smstate(u32, buf, 0x7e94, seg.limit);
9528 	put_smstate(u64, buf, 0x7e98, seg.base);
9529 
9530 	static_call(kvm_x86_get_idt)(vcpu, &dt);
9531 	put_smstate(u32, buf, 0x7e84, dt.size);
9532 	put_smstate(u64, buf, 0x7e88, dt.address);
9533 
9534 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
9535 	put_smstate(u16, buf, 0x7e70, seg.selector);
9536 	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
9537 	put_smstate(u32, buf, 0x7e74, seg.limit);
9538 	put_smstate(u64, buf, 0x7e78, seg.base);
9539 
9540 	static_call(kvm_x86_get_gdt)(vcpu, &dt);
9541 	put_smstate(u32, buf, 0x7e64, dt.size);
9542 	put_smstate(u64, buf, 0x7e68, dt.address);
9543 
9544 	for (i = 0; i < 6; i++)
9545 		enter_smm_save_seg_64(vcpu, buf, i);
9546 }
9547 #endif
9548 
9549 static void enter_smm(struct kvm_vcpu *vcpu)
9550 {
9551 	struct kvm_segment cs, ds;
9552 	struct desc_ptr dt;
9553 	unsigned long cr0;
9554 	char buf[512];
9555 
9556 	memset(buf, 0, 512);
9557 #ifdef CONFIG_X86_64
9558 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
9559 		enter_smm_save_state_64(vcpu, buf);
9560 	else
9561 #endif
9562 		enter_smm_save_state_32(vcpu, buf);
9563 
9564 	/*
9565 	 * Give enter_smm() a chance to make ISA-specific changes to the vCPU
9566 	 * state (e.g. leave guest mode) after we've saved the state into the
9567 	 * SMM state-save area.
9568 	 */
9569 	static_call(kvm_x86_enter_smm)(vcpu, buf);
9570 
9571 	kvm_smm_changed(vcpu, true);
9572 	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
9573 
9574 	if (static_call(kvm_x86_get_nmi_mask)(vcpu))
9575 		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
9576 	else
9577 		static_call(kvm_x86_set_nmi_mask)(vcpu, true);
9578 
9579 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
9580 	kvm_rip_write(vcpu, 0x8000);
9581 
9582 	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
9583 	static_call(kvm_x86_set_cr0)(vcpu, cr0);
9584 	vcpu->arch.cr0 = cr0;
9585 
9586 	static_call(kvm_x86_set_cr4)(vcpu, 0);
9587 
9588 	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
9589 	dt.address = dt.size = 0;
9590 	static_call(kvm_x86_set_idt)(vcpu, &dt);
9591 
9592 	kvm_set_dr(vcpu, 7, DR7_FIXED_1);
9593 
9594 	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
9595 	cs.base = vcpu->arch.smbase;
9596 
9597 	ds.selector = 0;
9598 	ds.base = 0;
9599 
9600 	cs.limit    = ds.limit = 0xffffffff;
9601 	cs.type     = ds.type = 0x3;
9602 	cs.dpl      = ds.dpl = 0;
9603 	cs.db       = ds.db = 0;
9604 	cs.s        = ds.s = 1;
9605 	cs.l        = ds.l = 0;
9606 	cs.g        = ds.g = 1;
9607 	cs.avl      = ds.avl = 0;
9608 	cs.present  = ds.present = 1;
9609 	cs.unusable = ds.unusable = 0;
9610 	cs.padding  = ds.padding = 0;
9611 
9612 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
9613 	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
9614 	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
9615 	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
9616 	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
9617 	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);
9618 
9619 #ifdef CONFIG_X86_64
9620 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
9621 		static_call(kvm_x86_set_efer)(vcpu, 0);
9622 #endif
9623 
9624 	kvm_update_cpuid_runtime(vcpu);
9625 	kvm_mmu_reset_context(vcpu);
9626 }
9627 
9628 static void process_smi(struct kvm_vcpu *vcpu)
9629 {
9630 	vcpu->arch.smi_pending = true;
9631 	kvm_make_request(KVM_REQ_EVENT, vcpu);
9632 }
9633 
9634 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
9635 				       unsigned long *vcpu_bitmap)
9636 {
9637 	kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap);
9638 }
9639 
9640 void kvm_make_scan_ioapic_request(struct kvm *kvm)
9641 {
9642 	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
9643 }
9644 
9645 void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
9646 {
9647 	bool activate;
9648 
9649 	if (!lapic_in_kernel(vcpu))
9650 		return;
9651 
9652 	down_read(&vcpu->kvm->arch.apicv_update_lock);
9653 
9654 	activate = kvm_apicv_activated(vcpu->kvm);
9655 	if (vcpu->arch.apicv_active == activate)
9656 		goto out;
9657 
9658 	vcpu->arch.apicv_active = activate;
9659 	kvm_apic_update_apicv(vcpu);
9660 	static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
9661 
9662 	/*
9663 	 * When APICv gets disabled, we may still have injected interrupts
9664 	 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was
9665 	 * still active when the interrupt got accepted. Make sure
9666 	 * inject_pending_event() is called to check for that.
9667 	 */
9668 	if (!vcpu->arch.apicv_active)
9669 		kvm_make_request(KVM_REQ_EVENT, vcpu);
9670 
9671 out:
9672 	up_read(&vcpu->kvm->arch.apicv_update_lock);
9673 }
9674 EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);
9675 
9676 void __kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit)
9677 {
9678 	unsigned long old, new;
9679 
9680 	lockdep_assert_held_write(&kvm->arch.apicv_update_lock);
9681 
9682 	if (!kvm_x86_ops.check_apicv_inhibit_reasons ||
9683 	    !static_call(kvm_x86_check_apicv_inhibit_reasons)(bit))
9684 		return;
9685 
9686 	old = new = kvm->arch.apicv_inhibit_reasons;
9687 
9688 	if (activate)
9689 		__clear_bit(bit, &new);
9690 	else
9691 		__set_bit(bit, &new);
9692 
9693 	if (!!old != !!new) {
9694 		trace_kvm_apicv_update_request(activate, bit);
9695 		/*
9696 		 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid
9697 		 * false positives in the sanity check WARN in svm_vcpu_run().
9698 		 * This task will wait for all vCPUs to ack the kick IRQ before
9699 		 * updating apicv_inhibit_reasons, and all other vCPUs will
9700 		 * block on acquiring apicv_update_lock so that vCPUs can't
9701 		 * redo svm_vcpu_run() without seeing the new inhibit state.
9702 		 *
9703 		 * Note, holding apicv_update_lock and taking it in the read
9704 		 * side (handling the request) also prevents other vCPUs from
9705 		 * servicing the request with a stale apicv_inhibit_reasons.
9706 		 */
9707 		kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE);
9708 		kvm->arch.apicv_inhibit_reasons = new;
9709 		if (new) {
9710 			unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE);
9711 			kvm_zap_gfn_range(kvm, gfn, gfn+1);
9712 		}
9713 	} else
9714 		kvm->arch.apicv_inhibit_reasons = new;
9715 }
9716 EXPORT_SYMBOL_GPL(__kvm_request_apicv_update);
9717 
9718 void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit)
9719 {
9720 	down_write(&kvm->arch.apicv_update_lock);
9721 	__kvm_request_apicv_update(kvm, activate, bit);
9722 	up_write(&kvm->arch.apicv_update_lock);
9723 }
9724 EXPORT_SYMBOL_GPL(kvm_request_apicv_update);
9725 
9726 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
9727 {
9728 	if (!kvm_apic_present(vcpu))
9729 		return;
9730 
9731 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
9732 
9733 	if (irqchip_split(vcpu->kvm))
9734 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
9735 	else {
9736 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
9737 		if (ioapic_in_kernel(vcpu->kvm))
9738 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
9739 	}
9740 
9741 	if (is_guest_mode(vcpu))
9742 		vcpu->arch.load_eoi_exitmap_pending = true;
9743 	else
9744 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
9745 }
9746 
9747 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
9748 {
9749 	u64 eoi_exit_bitmap[4];
9750 
9751 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
9752 		return;
9753 
9754 	if (to_hv_vcpu(vcpu)) {
9755 		bitmap_or((ulong *)eoi_exit_bitmap,
9756 			  vcpu->arch.ioapic_handled_vectors,
9757 			  to_hv_synic(vcpu)->vec_bitmap, 256);
9758 		static_call(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
9759 		return;
9760 	}
9761 
9762 	static_call(kvm_x86_load_eoi_exitmap)(
9763 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);
9764 }
9765 
9766 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
9767 					    unsigned long start, unsigned long end)
9768 {
9769 	unsigned long apic_address;
9770 
9771 	/*
9772 	 * The physical address of apic access page is stored in the VMCS.
9773 	 * Update it when it becomes invalid.
9774 	 */
9775 	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
9776 	if (start <= apic_address && apic_address < end)
9777 		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
9778 }
9779 
9780 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
9781 {
9782 	if (!lapic_in_kernel(vcpu))
9783 		return;
9784 
9785 	if (!kvm_x86_ops.set_apic_access_page_addr)
9786 		return;
9787 
9788 	static_call(kvm_x86_set_apic_access_page_addr)(vcpu);
9789 }
9790 
9791 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
9792 {
9793 	smp_send_reschedule(vcpu->cpu);
9794 }
9795 EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);
9796 
9797 /*
9798  * Returns 1 to let vcpu_run() continue the guest execution loop without
9799  * exiting to the userspace.  Otherwise, the value will be returned to the
9800  * userspace.
9801  */
9802 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
9803 {
9804 	int r;
9805 	bool req_int_win =
9806 		dm_request_for_irq_injection(vcpu) &&
9807 		kvm_cpu_accept_dm_intr(vcpu);
9808 	fastpath_t exit_fastpath;
9809 
9810 	bool req_immediate_exit = false;
9811 
9812 	/* Forbid vmenter if vcpu dirty ring is soft-full */
9813 	if (unlikely(vcpu->kvm->dirty_ring_size &&
9814 		     kvm_dirty_ring_soft_full(&vcpu->dirty_ring))) {
9815 		vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
9816 		trace_kvm_dirty_ring_exit(vcpu);
9817 		r = 0;
9818 		goto out;
9819 	}
9820 
9821 	if (kvm_request_pending(vcpu)) {
9822 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
9823 			r = -EIO;
9824 			goto out;
9825 		}
9826 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
9827 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
9828 				r = 0;
9829 				goto out;
9830 			}
9831 		}
9832 		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
9833 			kvm_mmu_unload(vcpu);
9834 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
9835 			__kvm_migrate_timers(vcpu);
9836 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
9837 			kvm_update_masterclock(vcpu->kvm);
9838 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
9839 			kvm_gen_kvmclock_update(vcpu);
9840 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
9841 			r = kvm_guest_time_update(vcpu);
9842 			if (unlikely(r))
9843 				goto out;
9844 		}
9845 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
9846 			kvm_mmu_sync_roots(vcpu);
9847 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
9848 			kvm_mmu_load_pgd(vcpu);
9849 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
9850 			kvm_vcpu_flush_tlb_all(vcpu);
9851 
9852 			/* Flushing all ASIDs flushes the current ASID... */
9853 			kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
9854 		}
9855 		kvm_service_local_tlb_flush_requests(vcpu);
9856 
9857 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
9858 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
9859 			r = 0;
9860 			goto out;
9861 		}
9862 		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
9863 			if (is_guest_mode(vcpu)) {
9864 				kvm_x86_ops.nested_ops->triple_fault(vcpu);
9865 			} else {
9866 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
9867 				vcpu->mmio_needed = 0;
9868 				r = 0;
9869 				goto out;
9870 			}
9871 		}
9872 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
9873 			/* Page is swapped out. Do synthetic halt */
9874 			vcpu->arch.apf.halted = true;
9875 			r = 1;
9876 			goto out;
9877 		}
9878 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
9879 			record_steal_time(vcpu);
9880 		if (kvm_check_request(KVM_REQ_SMI, vcpu))
9881 			process_smi(vcpu);
9882 		if (kvm_check_request(KVM_REQ_NMI, vcpu))
9883 			process_nmi(vcpu);
9884 		if (kvm_check_request(KVM_REQ_PMU, vcpu))
9885 			kvm_pmu_handle_event(vcpu);
9886 		if (kvm_check_request(KVM_REQ_PMI, vcpu))
9887 			kvm_pmu_deliver_pmi(vcpu);
9888 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
9889 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
9890 			if (test_bit(vcpu->arch.pending_ioapic_eoi,
9891 				     vcpu->arch.ioapic_handled_vectors)) {
9892 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
9893 				vcpu->run->eoi.vector =
9894 						vcpu->arch.pending_ioapic_eoi;
9895 				r = 0;
9896 				goto out;
9897 			}
9898 		}
9899 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
9900 			vcpu_scan_ioapic(vcpu);
9901 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
9902 			vcpu_load_eoi_exitmap(vcpu);
9903 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
9904 			kvm_vcpu_reload_apic_access_page(vcpu);
9905 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
9906 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
9907 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
9908 			r = 0;
9909 			goto out;
9910 		}
9911 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
9912 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
9913 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
9914 			r = 0;
9915 			goto out;
9916 		}
9917 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
9918 			struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
9919 
9920 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
9921 			vcpu->run->hyperv = hv_vcpu->exit;
9922 			r = 0;
9923 			goto out;
9924 		}
9925 
9926 		/*
9927 		 * KVM_REQ_HV_STIMER has to be processed after
9928 		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
9929 		 * depend on the guest clock being up-to-date
9930 		 */
9931 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
9932 			kvm_hv_process_stimers(vcpu);
9933 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
9934 			kvm_vcpu_update_apicv(vcpu);
9935 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
9936 			kvm_check_async_pf_completion(vcpu);
9937 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
9938 			static_call(kvm_x86_msr_filter_changed)(vcpu);
9939 
9940 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
9941 			static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
9942 	}
9943 
9944 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
9945 	    kvm_xen_has_interrupt(vcpu)) {
9946 		++vcpu->stat.req_event;
9947 		r = kvm_apic_accept_events(vcpu);
9948 		if (r < 0) {
9949 			r = 0;
9950 			goto out;
9951 		}
9952 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
9953 			r = 1;
9954 			goto out;
9955 		}
9956 
9957 		r = inject_pending_event(vcpu, &req_immediate_exit);
9958 		if (r < 0) {
9959 			r = 0;
9960 			goto out;
9961 		}
9962 		if (req_int_win)
9963 			static_call(kvm_x86_enable_irq_window)(vcpu);
9964 
9965 		if (kvm_lapic_enabled(vcpu)) {
9966 			update_cr8_intercept(vcpu);
9967 			kvm_lapic_sync_to_vapic(vcpu);
9968 		}
9969 	}
9970 
9971 	r = kvm_mmu_reload(vcpu);
9972 	if (unlikely(r)) {
9973 		goto cancel_injection;
9974 	}
9975 
9976 	preempt_disable();
9977 
9978 	static_call(kvm_x86_prepare_guest_switch)(vcpu);
9979 
9980 	/*
9981 	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
9982 	 * IPI are then delayed after guest entry, which ensures that they
9983 	 * result in virtual interrupt delivery.
9984 	 */
9985 	local_irq_disable();
9986 	vcpu->mode = IN_GUEST_MODE;
9987 
9988 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
9989 
9990 	/*
9991 	 * 1) We should set ->mode before checking ->requests.  Please see
9992 	 * the comment in kvm_vcpu_exiting_guest_mode().
9993 	 *
9994 	 * 2) For APICv, we should set ->mode before checking PID.ON. This
9995 	 * pairs with the memory barrier implicit in pi_test_and_set_on
9996 	 * (see vmx_deliver_posted_interrupt).
9997 	 *
9998 	 * 3) This also orders the write to mode from any reads to the page
9999 	 * tables done while the VCPU is running.  Please see the comment
10000 	 * in kvm_flush_remote_tlbs.
10001 	 */
10002 	smp_mb__after_srcu_read_unlock();
10003 
10004 	/*
10005 	 * Process pending posted interrupts to handle the case where the
10006 	 * notification IRQ arrived in the host, or was never sent (because the
10007 	 * target vCPU wasn't running).  Do this regardless of the vCPU's APICv
10008 	 * status, KVM doesn't update assigned devices when APICv is inhibited,
10009 	 * i.e. they can post interrupts even if APICv is temporarily disabled.
10010 	 */
10011 	if (kvm_lapic_enabled(vcpu))
10012 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10013 
10014 	if (kvm_vcpu_exit_request(vcpu)) {
10015 		vcpu->mode = OUTSIDE_GUEST_MODE;
10016 		smp_wmb();
10017 		local_irq_enable();
10018 		preempt_enable();
10019 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
10020 		r = 1;
10021 		goto cancel_injection;
10022 	}
10023 
10024 	if (req_immediate_exit) {
10025 		kvm_make_request(KVM_REQ_EVENT, vcpu);
10026 		static_call(kvm_x86_request_immediate_exit)(vcpu);
10027 	}
10028 
10029 	fpregs_assert_state_consistent();
10030 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
10031 		switch_fpu_return();
10032 
10033 	if (vcpu->arch.guest_fpu.xfd_err)
10034 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
10035 
10036 	if (unlikely(vcpu->arch.switch_db_regs)) {
10037 		set_debugreg(0, 7);
10038 		set_debugreg(vcpu->arch.eff_db[0], 0);
10039 		set_debugreg(vcpu->arch.eff_db[1], 1);
10040 		set_debugreg(vcpu->arch.eff_db[2], 2);
10041 		set_debugreg(vcpu->arch.eff_db[3], 3);
10042 	} else if (unlikely(hw_breakpoint_active())) {
10043 		set_debugreg(0, 7);
10044 	}
10045 
10046 	guest_timing_enter_irqoff();
10047 
10048 	for (;;) {
10049 		/*
10050 		 * Assert that vCPU vs. VM APICv state is consistent.  An APICv
10051 		 * update must kick and wait for all vCPUs before toggling the
10052 		 * per-VM state, and responsing vCPUs must wait for the update
10053 		 * to complete before servicing KVM_REQ_APICV_UPDATE.
10054 		 */
10055 		WARN_ON_ONCE(kvm_apicv_activated(vcpu->kvm) != kvm_vcpu_apicv_active(vcpu));
10056 
10057 		exit_fastpath = static_call(kvm_x86_run)(vcpu);
10058 		if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
10059 			break;
10060 
10061 		if (kvm_lapic_enabled(vcpu))
10062 			static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10063 
10064 		if (unlikely(kvm_vcpu_exit_request(vcpu))) {
10065 			exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
10066 			break;
10067 		}
10068 	}
10069 
10070 	/*
10071 	 * Do this here before restoring debug registers on the host.  And
10072 	 * since we do this before handling the vmexit, a DR access vmexit
10073 	 * can (a) read the correct value of the debug registers, (b) set
10074 	 * KVM_DEBUGREG_WONT_EXIT again.
10075 	 */
10076 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
10077 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
10078 		static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
10079 		kvm_update_dr0123(vcpu);
10080 		kvm_update_dr7(vcpu);
10081 	}
10082 
10083 	/*
10084 	 * If the guest has used debug registers, at least dr7
10085 	 * will be disabled while returning to the host.
10086 	 * If we don't have active breakpoints in the host, we don't
10087 	 * care about the messed up debug address registers. But if
10088 	 * we have some of them active, restore the old state.
10089 	 */
10090 	if (hw_breakpoint_active())
10091 		hw_breakpoint_restore();
10092 
10093 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;
10094 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
10095 
10096 	vcpu->mode = OUTSIDE_GUEST_MODE;
10097 	smp_wmb();
10098 
10099 	/*
10100 	 * Sync xfd before calling handle_exit_irqoff() which may
10101 	 * rely on the fact that guest_fpu::xfd is up-to-date (e.g.
10102 	 * in #NM irqoff handler).
10103 	 */
10104 	if (vcpu->arch.xfd_no_write_intercept)
10105 		fpu_sync_guest_vmexit_xfd_state();
10106 
10107 	static_call(kvm_x86_handle_exit_irqoff)(vcpu);
10108 
10109 	if (vcpu->arch.guest_fpu.xfd_err)
10110 		wrmsrl(MSR_IA32_XFD_ERR, 0);
10111 
10112 	/*
10113 	 * Consume any pending interrupts, including the possible source of
10114 	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
10115 	 * An instruction is required after local_irq_enable() to fully unblock
10116 	 * interrupts on processors that implement an interrupt shadow, the
10117 	 * stat.exits increment will do nicely.
10118 	 */
10119 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
10120 	local_irq_enable();
10121 	++vcpu->stat.exits;
10122 	local_irq_disable();
10123 	kvm_after_interrupt(vcpu);
10124 
10125 	/*
10126 	 * Wait until after servicing IRQs to account guest time so that any
10127 	 * ticks that occurred while running the guest are properly accounted
10128 	 * to the guest.  Waiting until IRQs are enabled degrades the accuracy
10129 	 * of accounting via context tracking, but the loss of accuracy is
10130 	 * acceptable for all known use cases.
10131 	 */
10132 	guest_timing_exit_irqoff();
10133 
10134 	if (lapic_in_kernel(vcpu)) {
10135 		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
10136 		if (delta != S64_MIN) {
10137 			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
10138 			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
10139 		}
10140 	}
10141 
10142 	local_irq_enable();
10143 	preempt_enable();
10144 
10145 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
10146 
10147 	/*
10148 	 * Profile KVM exit RIPs:
10149 	 */
10150 	if (unlikely(prof_on == KVM_PROFILING)) {
10151 		unsigned long rip = kvm_rip_read(vcpu);
10152 		profile_hit(KVM_PROFILING, (void *)rip);
10153 	}
10154 
10155 	if (unlikely(vcpu->arch.tsc_always_catchup))
10156 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
10157 
10158 	if (vcpu->arch.apic_attention)
10159 		kvm_lapic_sync_from_vapic(vcpu);
10160 
10161 	r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
10162 	return r;
10163 
10164 cancel_injection:
10165 	if (req_immediate_exit)
10166 		kvm_make_request(KVM_REQ_EVENT, vcpu);
10167 	static_call(kvm_x86_cancel_injection)(vcpu);
10168 	if (unlikely(vcpu->arch.apic_attention))
10169 		kvm_lapic_sync_from_vapic(vcpu);
10170 out:
10171 	return r;
10172 }
10173 
10174 static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
10175 {
10176 	bool hv_timer;
10177 
10178 	if (!kvm_arch_vcpu_runnable(vcpu)) {
10179 		/*
10180 		 * Switch to the software timer before halt-polling/blocking as
10181 		 * the guest's timer may be a break event for the vCPU, and the
10182 		 * hypervisor timer runs only when the CPU is in guest mode.
10183 		 * Switch before halt-polling so that KVM recognizes an expired
10184 		 * timer before blocking.
10185 		 */
10186 		hv_timer = kvm_lapic_hv_timer_in_use(vcpu);
10187 		if (hv_timer)
10188 			kvm_lapic_switch_to_sw_timer(vcpu);
10189 
10190 		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
10191 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
10192 			kvm_vcpu_halt(vcpu);
10193 		else
10194 			kvm_vcpu_block(vcpu);
10195 		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
10196 
10197 		if (hv_timer)
10198 			kvm_lapic_switch_to_hv_timer(vcpu);
10199 
10200 		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
10201 			return 1;
10202 	}
10203 
10204 	if (kvm_apic_accept_events(vcpu) < 0)
10205 		return 0;
10206 	switch(vcpu->arch.mp_state) {
10207 	case KVM_MP_STATE_HALTED:
10208 	case KVM_MP_STATE_AP_RESET_HOLD:
10209 		vcpu->arch.pv.pv_unhalted = false;
10210 		vcpu->arch.mp_state =
10211 			KVM_MP_STATE_RUNNABLE;
10212 		fallthrough;
10213 	case KVM_MP_STATE_RUNNABLE:
10214 		vcpu->arch.apf.halted = false;
10215 		break;
10216 	case KVM_MP_STATE_INIT_RECEIVED:
10217 		break;
10218 	default:
10219 		return -EINTR;
10220 	}
10221 	return 1;
10222 }
10223 
10224 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
10225 {
10226 	if (is_guest_mode(vcpu))
10227 		kvm_check_nested_events(vcpu);
10228 
10229 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
10230 		!vcpu->arch.apf.halted);
10231 }
10232 
10233 static int vcpu_run(struct kvm_vcpu *vcpu)
10234 {
10235 	int r;
10236 	struct kvm *kvm = vcpu->kvm;
10237 
10238 	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
10239 	vcpu->arch.l1tf_flush_l1d = true;
10240 
10241 	for (;;) {
10242 		if (kvm_vcpu_running(vcpu)) {
10243 			r = vcpu_enter_guest(vcpu);
10244 		} else {
10245 			r = vcpu_block(kvm, vcpu);
10246 		}
10247 
10248 		if (r <= 0)
10249 			break;
10250 
10251 		kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);
10252 		if (kvm_cpu_has_pending_timer(vcpu))
10253 			kvm_inject_pending_timer_irqs(vcpu);
10254 
10255 		if (dm_request_for_irq_injection(vcpu) &&
10256 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
10257 			r = 0;
10258 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
10259 			++vcpu->stat.request_irq_exits;
10260 			break;
10261 		}
10262 
10263 		if (__xfer_to_guest_mode_work_pending()) {
10264 			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
10265 			r = xfer_to_guest_mode_handle_work(vcpu);
10266 			if (r)
10267 				return r;
10268 			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
10269 		}
10270 	}
10271 
10272 	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
10273 
10274 	return r;
10275 }
10276 
10277 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
10278 {
10279 	int r;
10280 
10281 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
10282 	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
10283 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
10284 	return r;
10285 }
10286 
10287 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
10288 {
10289 	BUG_ON(!vcpu->arch.pio.count);
10290 
10291 	return complete_emulated_io(vcpu);
10292 }
10293 
10294 /*
10295  * Implements the following, as a state machine:
10296  *
10297  * read:
10298  *   for each fragment
10299  *     for each mmio piece in the fragment
10300  *       write gpa, len
10301  *       exit
10302  *       copy data
10303  *   execute insn
10304  *
10305  * write:
10306  *   for each fragment
10307  *     for each mmio piece in the fragment
10308  *       write gpa, len
10309  *       copy data
10310  *       exit
10311  */
10312 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
10313 {
10314 	struct kvm_run *run = vcpu->run;
10315 	struct kvm_mmio_fragment *frag;
10316 	unsigned len;
10317 
10318 	BUG_ON(!vcpu->mmio_needed);
10319 
10320 	/* Complete previous fragment */
10321 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
10322 	len = min(8u, frag->len);
10323 	if (!vcpu->mmio_is_write)
10324 		memcpy(frag->data, run->mmio.data, len);
10325 
10326 	if (frag->len <= 8) {
10327 		/* Switch to the next fragment. */
10328 		frag++;
10329 		vcpu->mmio_cur_fragment++;
10330 	} else {
10331 		/* Go forward to the next mmio piece. */
10332 		frag->data += len;
10333 		frag->gpa += len;
10334 		frag->len -= len;
10335 	}
10336 
10337 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
10338 		vcpu->mmio_needed = 0;
10339 
10340 		/* FIXME: return into emulator if single-stepping.  */
10341 		if (vcpu->mmio_is_write)
10342 			return 1;
10343 		vcpu->mmio_read_completed = 1;
10344 		return complete_emulated_io(vcpu);
10345 	}
10346 
10347 	run->exit_reason = KVM_EXIT_MMIO;
10348 	run->mmio.phys_addr = frag->gpa;
10349 	if (vcpu->mmio_is_write)
10350 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
10351 	run->mmio.len = min(8u, frag->len);
10352 	run->mmio.is_write = vcpu->mmio_is_write;
10353 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
10354 	return 0;
10355 }
10356 
10357 /* Swap (qemu) user FPU context for the guest FPU context. */
10358 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
10359 {
10360 	/*
10361 	 * Exclude PKRU from restore as restored separately in
10362 	 * kvm_x86_ops.run().
10363 	 */
10364 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
10365 	trace_kvm_fpu(1);
10366 }
10367 
10368 /* When vcpu_run ends, restore user space FPU context. */
10369 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
10370 {
10371 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
10372 	++vcpu->stat.fpu_reload;
10373 	trace_kvm_fpu(0);
10374 }
10375 
10376 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
10377 {
10378 	struct kvm_run *kvm_run = vcpu->run;
10379 	int r;
10380 
10381 	vcpu_load(vcpu);
10382 	kvm_sigset_activate(vcpu);
10383 	kvm_run->flags = 0;
10384 	kvm_load_guest_fpu(vcpu);
10385 
10386 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
10387 		if (kvm_run->immediate_exit) {
10388 			r = -EINTR;
10389 			goto out;
10390 		}
10391 		/*
10392 		 * It should be impossible for the hypervisor timer to be in
10393 		 * use before KVM has ever run the vCPU.
10394 		 */
10395 		WARN_ON_ONCE(kvm_lapic_hv_timer_in_use(vcpu));
10396 		kvm_vcpu_block(vcpu);
10397 		if (kvm_apic_accept_events(vcpu) < 0) {
10398 			r = 0;
10399 			goto out;
10400 		}
10401 		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
10402 		r = -EAGAIN;
10403 		if (signal_pending(current)) {
10404 			r = -EINTR;
10405 			kvm_run->exit_reason = KVM_EXIT_INTR;
10406 			++vcpu->stat.signal_exits;
10407 		}
10408 		goto out;
10409 	}
10410 
10411 	if ((kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) ||
10412 	    (kvm_run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)) {
10413 		r = -EINVAL;
10414 		goto out;
10415 	}
10416 
10417 	if (kvm_run->kvm_dirty_regs) {
10418 		r = sync_regs(vcpu);
10419 		if (r != 0)
10420 			goto out;
10421 	}
10422 
10423 	/* re-sync apic's tpr */
10424 	if (!lapic_in_kernel(vcpu)) {
10425 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
10426 			r = -EINVAL;
10427 			goto out;
10428 		}
10429 	}
10430 
10431 	if (unlikely(vcpu->arch.complete_userspace_io)) {
10432 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
10433 		vcpu->arch.complete_userspace_io = NULL;
10434 		r = cui(vcpu);
10435 		if (r <= 0)
10436 			goto out;
10437 	} else
10438 		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
10439 
10440 	if (kvm_run->immediate_exit) {
10441 		r = -EINTR;
10442 		goto out;
10443 	}
10444 
10445 	r = static_call(kvm_x86_vcpu_pre_run)(vcpu);
10446 	if (r <= 0)
10447 		goto out;
10448 
10449 	r = vcpu_run(vcpu);
10450 
10451 out:
10452 	kvm_put_guest_fpu(vcpu);
10453 	if (kvm_run->kvm_valid_regs)
10454 		store_regs(vcpu);
10455 	post_kvm_run_save(vcpu);
10456 	kvm_sigset_deactivate(vcpu);
10457 
10458 	vcpu_put(vcpu);
10459 	return r;
10460 }
10461 
10462 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
10463 {
10464 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
10465 		/*
10466 		 * We are here if userspace calls get_regs() in the middle of
10467 		 * instruction emulation. Registers state needs to be copied
10468 		 * back from emulation context to vcpu. Userspace shouldn't do
10469 		 * that usually, but some bad designed PV devices (vmware
10470 		 * backdoor interface) need this to work
10471 		 */
10472 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
10473 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
10474 	}
10475 	regs->rax = kvm_rax_read(vcpu);
10476 	regs->rbx = kvm_rbx_read(vcpu);
10477 	regs->rcx = kvm_rcx_read(vcpu);
10478 	regs->rdx = kvm_rdx_read(vcpu);
10479 	regs->rsi = kvm_rsi_read(vcpu);
10480 	regs->rdi = kvm_rdi_read(vcpu);
10481 	regs->rsp = kvm_rsp_read(vcpu);
10482 	regs->rbp = kvm_rbp_read(vcpu);
10483 #ifdef CONFIG_X86_64
10484 	regs->r8 = kvm_r8_read(vcpu);
10485 	regs->r9 = kvm_r9_read(vcpu);
10486 	regs->r10 = kvm_r10_read(vcpu);
10487 	regs->r11 = kvm_r11_read(vcpu);
10488 	regs->r12 = kvm_r12_read(vcpu);
10489 	regs->r13 = kvm_r13_read(vcpu);
10490 	regs->r14 = kvm_r14_read(vcpu);
10491 	regs->r15 = kvm_r15_read(vcpu);
10492 #endif
10493 
10494 	regs->rip = kvm_rip_read(vcpu);
10495 	regs->rflags = kvm_get_rflags(vcpu);
10496 }
10497 
10498 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
10499 {
10500 	vcpu_load(vcpu);
10501 	__get_regs(vcpu, regs);
10502 	vcpu_put(vcpu);
10503 	return 0;
10504 }
10505 
10506 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
10507 {
10508 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
10509 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
10510 
10511 	kvm_rax_write(vcpu, regs->rax);
10512 	kvm_rbx_write(vcpu, regs->rbx);
10513 	kvm_rcx_write(vcpu, regs->rcx);
10514 	kvm_rdx_write(vcpu, regs->rdx);
10515 	kvm_rsi_write(vcpu, regs->rsi);
10516 	kvm_rdi_write(vcpu, regs->rdi);
10517 	kvm_rsp_write(vcpu, regs->rsp);
10518 	kvm_rbp_write(vcpu, regs->rbp);
10519 #ifdef CONFIG_X86_64
10520 	kvm_r8_write(vcpu, regs->r8);
10521 	kvm_r9_write(vcpu, regs->r9);
10522 	kvm_r10_write(vcpu, regs->r10);
10523 	kvm_r11_write(vcpu, regs->r11);
10524 	kvm_r12_write(vcpu, regs->r12);
10525 	kvm_r13_write(vcpu, regs->r13);
10526 	kvm_r14_write(vcpu, regs->r14);
10527 	kvm_r15_write(vcpu, regs->r15);
10528 #endif
10529 
10530 	kvm_rip_write(vcpu, regs->rip);
10531 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
10532 
10533 	vcpu->arch.exception.pending = false;
10534 
10535 	kvm_make_request(KVM_REQ_EVENT, vcpu);
10536 }
10537 
10538 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
10539 {
10540 	vcpu_load(vcpu);
10541 	__set_regs(vcpu, regs);
10542 	vcpu_put(vcpu);
10543 	return 0;
10544 }
10545 
10546 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
10547 {
10548 	struct kvm_segment cs;
10549 
10550 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
10551 	*db = cs.db;
10552 	*l = cs.l;
10553 }
10554 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
10555 
10556 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
10557 {
10558 	struct desc_ptr dt;
10559 
10560 	if (vcpu->arch.guest_state_protected)
10561 		goto skip_protected_regs;
10562 
10563 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
10564 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
10565 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
10566 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
10567 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
10568 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
10569 
10570 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
10571 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
10572 
10573 	static_call(kvm_x86_get_idt)(vcpu, &dt);
10574 	sregs->idt.limit = dt.size;
10575 	sregs->idt.base = dt.address;
10576 	static_call(kvm_x86_get_gdt)(vcpu, &dt);
10577 	sregs->gdt.limit = dt.size;
10578 	sregs->gdt.base = dt.address;
10579 
10580 	sregs->cr2 = vcpu->arch.cr2;
10581 	sregs->cr3 = kvm_read_cr3(vcpu);
10582 
10583 skip_protected_regs:
10584 	sregs->cr0 = kvm_read_cr0(vcpu);
10585 	sregs->cr4 = kvm_read_cr4(vcpu);
10586 	sregs->cr8 = kvm_get_cr8(vcpu);
10587 	sregs->efer = vcpu->arch.efer;
10588 	sregs->apic_base = kvm_get_apic_base(vcpu);
10589 }
10590 
10591 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
10592 {
10593 	__get_sregs_common(vcpu, sregs);
10594 
10595 	if (vcpu->arch.guest_state_protected)
10596 		return;
10597 
10598 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
10599 		set_bit(vcpu->arch.interrupt.nr,
10600 			(unsigned long *)sregs->interrupt_bitmap);
10601 }
10602 
10603 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
10604 {
10605 	int i;
10606 
10607 	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
10608 
10609 	if (vcpu->arch.guest_state_protected)
10610 		return;
10611 
10612 	if (is_pae_paging(vcpu)) {
10613 		for (i = 0 ; i < 4 ; i++)
10614 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
10615 		sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
10616 	}
10617 }
10618 
10619 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
10620 				  struct kvm_sregs *sregs)
10621 {
10622 	vcpu_load(vcpu);
10623 	__get_sregs(vcpu, sregs);
10624 	vcpu_put(vcpu);
10625 	return 0;
10626 }
10627 
10628 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
10629 				    struct kvm_mp_state *mp_state)
10630 {
10631 	int r;
10632 
10633 	vcpu_load(vcpu);
10634 	if (kvm_mpx_supported())
10635 		kvm_load_guest_fpu(vcpu);
10636 
10637 	r = kvm_apic_accept_events(vcpu);
10638 	if (r < 0)
10639 		goto out;
10640 	r = 0;
10641 
10642 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||
10643 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&
10644 	    vcpu->arch.pv.pv_unhalted)
10645 		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
10646 	else
10647 		mp_state->mp_state = vcpu->arch.mp_state;
10648 
10649 out:
10650 	if (kvm_mpx_supported())
10651 		kvm_put_guest_fpu(vcpu);
10652 	vcpu_put(vcpu);
10653 	return r;
10654 }
10655 
10656 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
10657 				    struct kvm_mp_state *mp_state)
10658 {
10659 	int ret = -EINVAL;
10660 
10661 	vcpu_load(vcpu);
10662 
10663 	if (!lapic_in_kernel(vcpu) &&
10664 	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
10665 		goto out;
10666 
10667 	/*
10668 	 * KVM_MP_STATE_INIT_RECEIVED means the processor is in
10669 	 * INIT state; latched init should be reported using
10670 	 * KVM_SET_VCPU_EVENTS, so reject it here.
10671 	 */
10672 	if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
10673 	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
10674 	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
10675 		goto out;
10676 
10677 	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
10678 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
10679 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
10680 	} else
10681 		vcpu->arch.mp_state = mp_state->mp_state;
10682 	kvm_make_request(KVM_REQ_EVENT, vcpu);
10683 
10684 	ret = 0;
10685 out:
10686 	vcpu_put(vcpu);
10687 	return ret;
10688 }
10689 
10690 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
10691 		    int reason, bool has_error_code, u32 error_code)
10692 {
10693 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
10694 	int ret;
10695 
10696 	init_emulate_ctxt(vcpu);
10697 
10698 	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
10699 				   has_error_code, error_code);
10700 	if (ret) {
10701 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
10702 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
10703 		vcpu->run->internal.ndata = 0;
10704 		return 0;
10705 	}
10706 
10707 	kvm_rip_write(vcpu, ctxt->eip);
10708 	kvm_set_rflags(vcpu, ctxt->eflags);
10709 	return 1;
10710 }
10711 EXPORT_SYMBOL_GPL(kvm_task_switch);
10712 
10713 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
10714 {
10715 	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
10716 		/*
10717 		 * When EFER.LME and CR0.PG are set, the processor is in
10718 		 * 64-bit mode (though maybe in a 32-bit code segment).
10719 		 * CR4.PAE and EFER.LMA must be set.
10720 		 */
10721 		if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
10722 			return false;
10723 		if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
10724 			return false;
10725 	} else {
10726 		/*
10727 		 * Not in 64-bit mode: EFER.LMA is clear and the code
10728 		 * segment cannot be 64-bit.
10729 		 */
10730 		if (sregs->efer & EFER_LMA || sregs->cs.l)
10731 			return false;
10732 	}
10733 
10734 	return kvm_is_valid_cr4(vcpu, sregs->cr4);
10735 }
10736 
10737 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
10738 		int *mmu_reset_needed, bool update_pdptrs)
10739 {
10740 	struct msr_data apic_base_msr;
10741 	int idx;
10742 	struct desc_ptr dt;
10743 
10744 	if (!kvm_is_valid_sregs(vcpu, sregs))
10745 		return -EINVAL;
10746 
10747 	apic_base_msr.data = sregs->apic_base;
10748 	apic_base_msr.host_initiated = true;
10749 	if (kvm_set_apic_base(vcpu, &apic_base_msr))
10750 		return -EINVAL;
10751 
10752 	if (vcpu->arch.guest_state_protected)
10753 		return 0;
10754 
10755 	dt.size = sregs->idt.limit;
10756 	dt.address = sregs->idt.base;
10757 	static_call(kvm_x86_set_idt)(vcpu, &dt);
10758 	dt.size = sregs->gdt.limit;
10759 	dt.address = sregs->gdt.base;
10760 	static_call(kvm_x86_set_gdt)(vcpu, &dt);
10761 
10762 	vcpu->arch.cr2 = sregs->cr2;
10763 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
10764 	vcpu->arch.cr3 = sregs->cr3;
10765 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
10766 	static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);
10767 
10768 	kvm_set_cr8(vcpu, sregs->cr8);
10769 
10770 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
10771 	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
10772 
10773 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
10774 	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
10775 	vcpu->arch.cr0 = sregs->cr0;
10776 
10777 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
10778 	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
10779 
10780 	if (update_pdptrs) {
10781 		idx = srcu_read_lock(&vcpu->kvm->srcu);
10782 		if (is_pae_paging(vcpu)) {
10783 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
10784 			*mmu_reset_needed = 1;
10785 		}
10786 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
10787 	}
10788 
10789 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
10790 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
10791 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
10792 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
10793 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
10794 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
10795 
10796 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
10797 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
10798 
10799 	update_cr8_intercept(vcpu);
10800 
10801 	/* Older userspace won't unhalt the vcpu on reset. */
10802 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
10803 	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
10804 	    !is_protmode(vcpu))
10805 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10806 
10807 	return 0;
10808 }
10809 
10810 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
10811 {
10812 	int pending_vec, max_bits;
10813 	int mmu_reset_needed = 0;
10814 	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
10815 
10816 	if (ret)
10817 		return ret;
10818 
10819 	if (mmu_reset_needed)
10820 		kvm_mmu_reset_context(vcpu);
10821 
10822 	max_bits = KVM_NR_INTERRUPTS;
10823 	pending_vec = find_first_bit(
10824 		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
10825 
10826 	if (pending_vec < max_bits) {
10827 		kvm_queue_interrupt(vcpu, pending_vec, false);
10828 		pr_debug("Set back pending irq %d\n", pending_vec);
10829 		kvm_make_request(KVM_REQ_EVENT, vcpu);
10830 	}
10831 	return 0;
10832 }
10833 
10834 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
10835 {
10836 	int mmu_reset_needed = 0;
10837 	bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
10838 	bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&
10839 		!(sregs2->efer & EFER_LMA);
10840 	int i, ret;
10841 
10842 	if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)
10843 		return -EINVAL;
10844 
10845 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
10846 		return -EINVAL;
10847 
10848 	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
10849 				 &mmu_reset_needed, !valid_pdptrs);
10850 	if (ret)
10851 		return ret;
10852 
10853 	if (valid_pdptrs) {
10854 		for (i = 0; i < 4 ; i++)
10855 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
10856 
10857 		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
10858 		mmu_reset_needed = 1;
10859 		vcpu->arch.pdptrs_from_userspace = true;
10860 	}
10861 	if (mmu_reset_needed)
10862 		kvm_mmu_reset_context(vcpu);
10863 	return 0;
10864 }
10865 
10866 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
10867 				  struct kvm_sregs *sregs)
10868 {
10869 	int ret;
10870 
10871 	vcpu_load(vcpu);
10872 	ret = __set_sregs(vcpu, sregs);
10873 	vcpu_put(vcpu);
10874 	return ret;
10875 }
10876 
10877 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm)
10878 {
10879 	bool inhibit = false;
10880 	struct kvm_vcpu *vcpu;
10881 	unsigned long i;
10882 
10883 	down_write(&kvm->arch.apicv_update_lock);
10884 
10885 	kvm_for_each_vcpu(i, vcpu, kvm) {
10886 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {
10887 			inhibit = true;
10888 			break;
10889 		}
10890 	}
10891 	__kvm_request_apicv_update(kvm, !inhibit, APICV_INHIBIT_REASON_BLOCKIRQ);
10892 	up_write(&kvm->arch.apicv_update_lock);
10893 }
10894 
10895 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
10896 					struct kvm_guest_debug *dbg)
10897 {
10898 	unsigned long rflags;
10899 	int i, r;
10900 
10901 	if (vcpu->arch.guest_state_protected)
10902 		return -EINVAL;
10903 
10904 	vcpu_load(vcpu);
10905 
10906 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
10907 		r = -EBUSY;
10908 		if (vcpu->arch.exception.pending)
10909 			goto out;
10910 		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
10911 			kvm_queue_exception(vcpu, DB_VECTOR);
10912 		else
10913 			kvm_queue_exception(vcpu, BP_VECTOR);
10914 	}
10915 
10916 	/*
10917 	 * Read rflags as long as potentially injected trace flags are still
10918 	 * filtered out.
10919 	 */
10920 	rflags = kvm_get_rflags(vcpu);
10921 
10922 	vcpu->guest_debug = dbg->control;
10923 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
10924 		vcpu->guest_debug = 0;
10925 
10926 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
10927 		for (i = 0; i < KVM_NR_DB_REGS; ++i)
10928 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
10929 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
10930 	} else {
10931 		for (i = 0; i < KVM_NR_DB_REGS; i++)
10932 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
10933 	}
10934 	kvm_update_dr7(vcpu);
10935 
10936 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10937 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);
10938 
10939 	/*
10940 	 * Trigger an rflags update that will inject or remove the trace
10941 	 * flags.
10942 	 */
10943 	kvm_set_rflags(vcpu, rflags);
10944 
10945 	static_call(kvm_x86_update_exception_bitmap)(vcpu);
10946 
10947 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);
10948 
10949 	r = 0;
10950 
10951 out:
10952 	vcpu_put(vcpu);
10953 	return r;
10954 }
10955 
10956 /*
10957  * Translate a guest virtual address to a guest physical address.
10958  */
10959 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
10960 				    struct kvm_translation *tr)
10961 {
10962 	unsigned long vaddr = tr->linear_address;
10963 	gpa_t gpa;
10964 	int idx;
10965 
10966 	vcpu_load(vcpu);
10967 
10968 	idx = srcu_read_lock(&vcpu->kvm->srcu);
10969 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
10970 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
10971 	tr->physical_address = gpa;
10972 	tr->valid = gpa != UNMAPPED_GVA;
10973 	tr->writeable = 1;
10974 	tr->usermode = 0;
10975 
10976 	vcpu_put(vcpu);
10977 	return 0;
10978 }
10979 
10980 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
10981 {
10982 	struct fxregs_state *fxsave;
10983 
10984 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
10985 		return 0;
10986 
10987 	vcpu_load(vcpu);
10988 
10989 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
10990 	memcpy(fpu->fpr, fxsave->st_space, 128);
10991 	fpu->fcw = fxsave->cwd;
10992 	fpu->fsw = fxsave->swd;
10993 	fpu->ftwx = fxsave->twd;
10994 	fpu->last_opcode = fxsave->fop;
10995 	fpu->last_ip = fxsave->rip;
10996 	fpu->last_dp = fxsave->rdp;
10997 	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
10998 
10999 	vcpu_put(vcpu);
11000 	return 0;
11001 }
11002 
11003 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11004 {
11005 	struct fxregs_state *fxsave;
11006 
11007 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
11008 		return 0;
11009 
11010 	vcpu_load(vcpu);
11011 
11012 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
11013 
11014 	memcpy(fxsave->st_space, fpu->fpr, 128);
11015 	fxsave->cwd = fpu->fcw;
11016 	fxsave->swd = fpu->fsw;
11017 	fxsave->twd = fpu->ftwx;
11018 	fxsave->fop = fpu->last_opcode;
11019 	fxsave->rip = fpu->last_ip;
11020 	fxsave->rdp = fpu->last_dp;
11021 	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
11022 
11023 	vcpu_put(vcpu);
11024 	return 0;
11025 }
11026 
11027 static void store_regs(struct kvm_vcpu *vcpu)
11028 {
11029 	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
11030 
11031 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
11032 		__get_regs(vcpu, &vcpu->run->s.regs.regs);
11033 
11034 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
11035 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);
11036 
11037 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
11038 		kvm_vcpu_ioctl_x86_get_vcpu_events(
11039 				vcpu, &vcpu->run->s.regs.events);
11040 }
11041 
11042 static int sync_regs(struct kvm_vcpu *vcpu)
11043 {
11044 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
11045 		__set_regs(vcpu, &vcpu->run->s.regs.regs);
11046 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
11047 	}
11048 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
11049 		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
11050 			return -EINVAL;
11051 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
11052 	}
11053 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
11054 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
11055 				vcpu, &vcpu->run->s.regs.events))
11056 			return -EINVAL;
11057 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
11058 	}
11059 
11060 	return 0;
11061 }
11062 
11063 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
11064 {
11065 	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
11066 		pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
11067 			     "guest TSC will not be reliable\n");
11068 
11069 	return 0;
11070 }
11071 
11072 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
11073 {
11074 	struct page *page;
11075 	int r;
11076 
11077 	vcpu->arch.last_vmentry_cpu = -1;
11078 	vcpu->arch.regs_avail = ~0;
11079 	vcpu->arch.regs_dirty = ~0;
11080 
11081 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
11082 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11083 	else
11084 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
11085 
11086 	r = kvm_mmu_create(vcpu);
11087 	if (r < 0)
11088 		return r;
11089 
11090 	if (irqchip_in_kernel(vcpu->kvm)) {
11091 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
11092 		if (r < 0)
11093 			goto fail_mmu_destroy;
11094 		if (kvm_apicv_activated(vcpu->kvm))
11095 			vcpu->arch.apicv_active = true;
11096 	} else
11097 		static_branch_inc(&kvm_has_noapic_vcpu);
11098 
11099 	r = -ENOMEM;
11100 
11101 	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
11102 	if (!page)
11103 		goto fail_free_lapic;
11104 	vcpu->arch.pio_data = page_address(page);
11105 
11106 	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
11107 				       GFP_KERNEL_ACCOUNT);
11108 	if (!vcpu->arch.mce_banks)
11109 		goto fail_free_pio_data;
11110 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
11111 
11112 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
11113 				GFP_KERNEL_ACCOUNT))
11114 		goto fail_free_mce_banks;
11115 
11116 	if (!alloc_emulate_ctxt(vcpu))
11117 		goto free_wbinvd_dirty_mask;
11118 
11119 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {
11120 		pr_err("kvm: failed to allocate vcpu's fpu\n");
11121 		goto free_emulate_ctxt;
11122 	}
11123 
11124 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
11125 	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
11126 
11127 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
11128 
11129 	kvm_async_pf_hash_reset(vcpu);
11130 	kvm_pmu_init(vcpu);
11131 
11132 	vcpu->arch.pending_external_vector = -1;
11133 	vcpu->arch.preempted_in_kernel = false;
11134 
11135 #if IS_ENABLED(CONFIG_HYPERV)
11136 	vcpu->arch.hv_root_tdp = INVALID_PAGE;
11137 #endif
11138 
11139 	r = static_call(kvm_x86_vcpu_create)(vcpu);
11140 	if (r)
11141 		goto free_guest_fpu;
11142 
11143 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
11144 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
11145 	kvm_vcpu_mtrr_init(vcpu);
11146 	vcpu_load(vcpu);
11147 	kvm_set_tsc_khz(vcpu, max_tsc_khz);
11148 	kvm_vcpu_reset(vcpu, false);
11149 	kvm_init_mmu(vcpu);
11150 	vcpu_put(vcpu);
11151 	return 0;
11152 
11153 free_guest_fpu:
11154 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
11155 free_emulate_ctxt:
11156 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
11157 free_wbinvd_dirty_mask:
11158 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
11159 fail_free_mce_banks:
11160 	kfree(vcpu->arch.mce_banks);
11161 fail_free_pio_data:
11162 	free_page((unsigned long)vcpu->arch.pio_data);
11163 fail_free_lapic:
11164 	kvm_free_lapic(vcpu);
11165 fail_mmu_destroy:
11166 	kvm_mmu_destroy(vcpu);
11167 	return r;
11168 }
11169 
11170 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
11171 {
11172 	struct kvm *kvm = vcpu->kvm;
11173 
11174 	if (mutex_lock_killable(&vcpu->mutex))
11175 		return;
11176 	vcpu_load(vcpu);
11177 	kvm_synchronize_tsc(vcpu, 0);
11178 	vcpu_put(vcpu);
11179 
11180 	/* poll control enabled by default */
11181 	vcpu->arch.msr_kvm_poll_control = 1;
11182 
11183 	mutex_unlock(&vcpu->mutex);
11184 
11185 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
11186 		schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
11187 						KVMCLOCK_SYNC_PERIOD);
11188 }
11189 
11190 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
11191 {
11192 	int idx;
11193 
11194 	kvmclock_reset(vcpu);
11195 
11196 	static_call(kvm_x86_vcpu_free)(vcpu);
11197 
11198 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
11199 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
11200 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
11201 
11202 	kvm_hv_vcpu_uninit(vcpu);
11203 	kvm_pmu_destroy(vcpu);
11204 	kfree(vcpu->arch.mce_banks);
11205 	kvm_free_lapic(vcpu);
11206 	idx = srcu_read_lock(&vcpu->kvm->srcu);
11207 	kvm_mmu_destroy(vcpu);
11208 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
11209 	free_page((unsigned long)vcpu->arch.pio_data);
11210 	kvfree(vcpu->arch.cpuid_entries);
11211 	if (!lapic_in_kernel(vcpu))
11212 		static_branch_dec(&kvm_has_noapic_vcpu);
11213 }
11214 
11215 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
11216 {
11217 	struct kvm_cpuid_entry2 *cpuid_0x1;
11218 	unsigned long old_cr0 = kvm_read_cr0(vcpu);
11219 	unsigned long new_cr0;
11220 
11221 	/*
11222 	 * Several of the "set" flows, e.g. ->set_cr0(), read other registers
11223 	 * to handle side effects.  RESET emulation hits those flows and relies
11224 	 * on emulated/virtualized registers, including those that are loaded
11225 	 * into hardware, to be zeroed at vCPU creation.  Use CRs as a sentinel
11226 	 * to detect improper or missing initialization.
11227 	 */
11228 	WARN_ON_ONCE(!init_event &&
11229 		     (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));
11230 
11231 	kvm_lapic_reset(vcpu, init_event);
11232 
11233 	vcpu->arch.hflags = 0;
11234 
11235 	vcpu->arch.smi_pending = 0;
11236 	vcpu->arch.smi_count = 0;
11237 	atomic_set(&vcpu->arch.nmi_queued, 0);
11238 	vcpu->arch.nmi_pending = 0;
11239 	vcpu->arch.nmi_injected = false;
11240 	kvm_clear_interrupt_queue(vcpu);
11241 	kvm_clear_exception_queue(vcpu);
11242 
11243 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
11244 	kvm_update_dr0123(vcpu);
11245 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;
11246 	vcpu->arch.dr7 = DR7_FIXED_1;
11247 	kvm_update_dr7(vcpu);
11248 
11249 	vcpu->arch.cr2 = 0;
11250 
11251 	kvm_make_request(KVM_REQ_EVENT, vcpu);
11252 	vcpu->arch.apf.msr_en_val = 0;
11253 	vcpu->arch.apf.msr_int_val = 0;
11254 	vcpu->arch.st.msr_val = 0;
11255 
11256 	kvmclock_reset(vcpu);
11257 
11258 	kvm_clear_async_pf_completion_queue(vcpu);
11259 	kvm_async_pf_hash_reset(vcpu);
11260 	vcpu->arch.apf.halted = false;
11261 
11262 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {
11263 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;
11264 
11265 		/*
11266 		 * To avoid have the INIT path from kvm_apic_has_events() that be
11267 		 * called with loaded FPU and does not let userspace fix the state.
11268 		 */
11269 		if (init_event)
11270 			kvm_put_guest_fpu(vcpu);
11271 
11272 		fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS);
11273 		fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR);
11274 
11275 		if (init_event)
11276 			kvm_load_guest_fpu(vcpu);
11277 	}
11278 
11279 	if (!init_event) {
11280 		kvm_pmu_reset(vcpu);
11281 		vcpu->arch.smbase = 0x30000;
11282 
11283 		vcpu->arch.msr_misc_features_enables = 0;
11284 
11285 		__kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
11286 		__kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
11287 	}
11288 
11289 	/* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */
11290 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
11291 	kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);
11292 
11293 	/*
11294 	 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon)
11295 	 * if no CPUID match is found.  Note, it's impossible to get a match at
11296 	 * RESET since KVM emulates RESET before exposing the vCPU to userspace,
11297 	 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry
11298 	 * on RESET.  But, go through the motions in case that's ever remedied.
11299 	 */
11300 	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1, 0);
11301 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
11302 
11303 	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
11304 
11305 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
11306 	kvm_rip_write(vcpu, 0xfff0);
11307 
11308 	vcpu->arch.cr3 = 0;
11309 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
11310 
11311 	/*
11312 	 * CR0.CD/NW are set on RESET, preserved on INIT.  Note, some versions
11313 	 * of Intel's SDM list CD/NW as being set on INIT, but they contradict
11314 	 * (or qualify) that with a footnote stating that CD/NW are preserved.
11315 	 */
11316 	new_cr0 = X86_CR0_ET;
11317 	if (init_event)
11318 		new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD));
11319 	else
11320 		new_cr0 |= X86_CR0_NW | X86_CR0_CD;
11321 
11322 	static_call(kvm_x86_set_cr0)(vcpu, new_cr0);
11323 	static_call(kvm_x86_set_cr4)(vcpu, 0);
11324 	static_call(kvm_x86_set_efer)(vcpu, 0);
11325 	static_call(kvm_x86_update_exception_bitmap)(vcpu);
11326 
11327 	/*
11328 	 * Reset the MMU context if paging was enabled prior to INIT (which is
11329 	 * implied if CR0.PG=1 as CR0 will be '0' prior to RESET).  Unlike the
11330 	 * standard CR0/CR4/EFER modification paths, only CR0.PG needs to be
11331 	 * checked because it is unconditionally cleared on INIT and all other
11332 	 * paging related bits are ignored if paging is disabled, i.e. CR0.WP,
11333 	 * CR4, and EFER changes are all irrelevant if CR0.PG was '0'.
11334 	 */
11335 	if (old_cr0 & X86_CR0_PG)
11336 		kvm_mmu_reset_context(vcpu);
11337 
11338 	/*
11339 	 * Intel's SDM states that all TLB entries are flushed on INIT.  AMD's
11340 	 * APM states the TLBs are untouched by INIT, but it also states that
11341 	 * the TLBs are flushed on "External initialization of the processor."
11342 	 * Flush the guest TLB regardless of vendor, there is no meaningful
11343 	 * benefit in relying on the guest to flush the TLB immediately after
11344 	 * INIT.  A spurious TLB flush is benign and likely negligible from a
11345 	 * performance perspective.
11346 	 */
11347 	if (init_event)
11348 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
11349 }
11350 EXPORT_SYMBOL_GPL(kvm_vcpu_reset);
11351 
11352 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
11353 {
11354 	struct kvm_segment cs;
11355 
11356 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
11357 	cs.selector = vector << 8;
11358 	cs.base = vector << 12;
11359 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
11360 	kvm_rip_write(vcpu, 0);
11361 }
11362 EXPORT_SYMBOL_GPL(kvm_vcpu_deliver_sipi_vector);
11363 
11364 int kvm_arch_hardware_enable(void)
11365 {
11366 	struct kvm *kvm;
11367 	struct kvm_vcpu *vcpu;
11368 	unsigned long i;
11369 	int ret;
11370 	u64 local_tsc;
11371 	u64 max_tsc = 0;
11372 	bool stable, backwards_tsc = false;
11373 
11374 	kvm_user_return_msr_cpu_online();
11375 	ret = static_call(kvm_x86_hardware_enable)();
11376 	if (ret != 0)
11377 		return ret;
11378 
11379 	local_tsc = rdtsc();
11380 	stable = !kvm_check_tsc_unstable();
11381 	list_for_each_entry(kvm, &vm_list, vm_list) {
11382 		kvm_for_each_vcpu(i, vcpu, kvm) {
11383 			if (!stable && vcpu->cpu == smp_processor_id())
11384 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
11385 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
11386 				backwards_tsc = true;
11387 				if (vcpu->arch.last_host_tsc > max_tsc)
11388 					max_tsc = vcpu->arch.last_host_tsc;
11389 			}
11390 		}
11391 	}
11392 
11393 	/*
11394 	 * Sometimes, even reliable TSCs go backwards.  This happens on
11395 	 * platforms that reset TSC during suspend or hibernate actions, but
11396 	 * maintain synchronization.  We must compensate.  Fortunately, we can
11397 	 * detect that condition here, which happens early in CPU bringup,
11398 	 * before any KVM threads can be running.  Unfortunately, we can't
11399 	 * bring the TSCs fully up to date with real time, as we aren't yet far
11400 	 * enough into CPU bringup that we know how much real time has actually
11401 	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
11402 	 * variables that haven't been updated yet.
11403 	 *
11404 	 * So we simply find the maximum observed TSC above, then record the
11405 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
11406 	 * the adjustment will be applied.  Note that we accumulate
11407 	 * adjustments, in case multiple suspend cycles happen before some VCPU
11408 	 * gets a chance to run again.  In the event that no KVM threads get a
11409 	 * chance to run, we will miss the entire elapsed period, as we'll have
11410 	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
11411 	 * loose cycle time.  This isn't too big a deal, since the loss will be
11412 	 * uniform across all VCPUs (not to mention the scenario is extremely
11413 	 * unlikely). It is possible that a second hibernate recovery happens
11414 	 * much faster than a first, causing the observed TSC here to be
11415 	 * smaller; this would require additional padding adjustment, which is
11416 	 * why we set last_host_tsc to the local tsc observed here.
11417 	 *
11418 	 * N.B. - this code below runs only on platforms with reliable TSC,
11419 	 * as that is the only way backwards_tsc is set above.  Also note
11420 	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
11421 	 * have the same delta_cyc adjustment applied if backwards_tsc
11422 	 * is detected.  Note further, this adjustment is only done once,
11423 	 * as we reset last_host_tsc on all VCPUs to stop this from being
11424 	 * called multiple times (one for each physical CPU bringup).
11425 	 *
11426 	 * Platforms with unreliable TSCs don't have to deal with this, they
11427 	 * will be compensated by the logic in vcpu_load, which sets the TSC to
11428 	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
11429 	 * guarantee that they stay in perfect synchronization.
11430 	 */
11431 	if (backwards_tsc) {
11432 		u64 delta_cyc = max_tsc - local_tsc;
11433 		list_for_each_entry(kvm, &vm_list, vm_list) {
11434 			kvm->arch.backwards_tsc_observed = true;
11435 			kvm_for_each_vcpu(i, vcpu, kvm) {
11436 				vcpu->arch.tsc_offset_adjustment += delta_cyc;
11437 				vcpu->arch.last_host_tsc = local_tsc;
11438 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
11439 			}
11440 
11441 			/*
11442 			 * We have to disable TSC offset matching.. if you were
11443 			 * booting a VM while issuing an S4 host suspend....
11444 			 * you may have some problem.  Solving this issue is
11445 			 * left as an exercise to the reader.
11446 			 */
11447 			kvm->arch.last_tsc_nsec = 0;
11448 			kvm->arch.last_tsc_write = 0;
11449 		}
11450 
11451 	}
11452 	return 0;
11453 }
11454 
11455 void kvm_arch_hardware_disable(void)
11456 {
11457 	static_call(kvm_x86_hardware_disable)();
11458 	drop_user_return_notifiers();
11459 }
11460 
11461 int kvm_arch_hardware_setup(void *opaque)
11462 {
11463 	struct kvm_x86_init_ops *ops = opaque;
11464 	int r;
11465 
11466 	rdmsrl_safe(MSR_EFER, &host_efer);
11467 
11468 	if (boot_cpu_has(X86_FEATURE_XSAVES))
11469 		rdmsrl(MSR_IA32_XSS, host_xss);
11470 
11471 	r = ops->hardware_setup();
11472 	if (r != 0)
11473 		return r;
11474 
11475 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
11476 	kvm_ops_static_call_update();
11477 
11478 	kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
11479 
11480 	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
11481 		supported_xss = 0;
11482 
11483 #define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
11484 	cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
11485 #undef __kvm_cpu_cap_has
11486 
11487 	if (kvm_has_tsc_control) {
11488 		/*
11489 		 * Make sure the user can only configure tsc_khz values that
11490 		 * fit into a signed integer.
11491 		 * A min value is not calculated because it will always
11492 		 * be 1 on all machines.
11493 		 */
11494 		u64 max = min(0x7fffffffULL,
11495 			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
11496 		kvm_max_guest_tsc_khz = max;
11497 
11498 		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
11499 	}
11500 
11501 	kvm_init_msr_list();
11502 	return 0;
11503 }
11504 
11505 void kvm_arch_hardware_unsetup(void)
11506 {
11507 	kvm_unregister_perf_callbacks();
11508 
11509 	static_call(kvm_x86_hardware_unsetup)();
11510 }
11511 
11512 int kvm_arch_check_processor_compat(void *opaque)
11513 {
11514 	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
11515 	struct kvm_x86_init_ops *ops = opaque;
11516 
11517 	WARN_ON(!irqs_disabled());
11518 
11519 	if (__cr4_reserved_bits(cpu_has, c) !=
11520 	    __cr4_reserved_bits(cpu_has, &boot_cpu_data))
11521 		return -EIO;
11522 
11523 	return ops->check_processor_compatibility();
11524 }
11525 
11526 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
11527 {
11528 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
11529 }
11530 EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);
11531 
11532 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
11533 {
11534 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
11535 }
11536 
11537 __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
11538 EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);
11539 
11540 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
11541 {
11542 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
11543 
11544 	vcpu->arch.l1tf_flush_l1d = true;
11545 	if (pmu->version && unlikely(pmu->event_count)) {
11546 		pmu->need_cleanup = true;
11547 		kvm_make_request(KVM_REQ_PMU, vcpu);
11548 	}
11549 	static_call(kvm_x86_sched_in)(vcpu, cpu);
11550 }
11551 
11552 void kvm_arch_free_vm(struct kvm *kvm)
11553 {
11554 	kfree(to_kvm_hv(kvm)->hv_pa_pg);
11555 	__kvm_arch_free_vm(kvm);
11556 }
11557 
11558 
11559 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
11560 {
11561 	int ret;
11562 	unsigned long flags;
11563 
11564 	if (type)
11565 		return -EINVAL;
11566 
11567 	ret = kvm_page_track_init(kvm);
11568 	if (ret)
11569 		return ret;
11570 
11571 	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
11572 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
11573 	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
11574 	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
11575 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
11576 	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
11577 
11578 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
11579 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
11580 	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
11581 	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
11582 		&kvm->arch.irq_sources_bitmap);
11583 
11584 	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
11585 	mutex_init(&kvm->arch.apic_map_lock);
11586 	seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
11587 	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
11588 
11589 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
11590 	pvclock_update_vm_gtod_copy(kvm);
11591 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
11592 
11593 	kvm->arch.guest_can_read_msr_platform_info = true;
11594 
11595 #if IS_ENABLED(CONFIG_HYPERV)
11596 	spin_lock_init(&kvm->arch.hv_root_tdp_lock);
11597 	kvm->arch.hv_root_tdp = INVALID_PAGE;
11598 #endif
11599 
11600 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
11601 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
11602 
11603 	kvm_apicv_init(kvm);
11604 	kvm_hv_init_vm(kvm);
11605 	kvm_mmu_init_vm(kvm);
11606 	kvm_xen_init_vm(kvm);
11607 
11608 	return static_call(kvm_x86_vm_init)(kvm);
11609 }
11610 
11611 int kvm_arch_post_init_vm(struct kvm *kvm)
11612 {
11613 	return kvm_mmu_post_init_vm(kvm);
11614 }
11615 
11616 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
11617 {
11618 	vcpu_load(vcpu);
11619 	kvm_mmu_unload(vcpu);
11620 	vcpu_put(vcpu);
11621 }
11622 
11623 static void kvm_free_vcpus(struct kvm *kvm)
11624 {
11625 	unsigned long i;
11626 	struct kvm_vcpu *vcpu;
11627 
11628 	/*
11629 	 * Unpin any mmu pages first.
11630 	 */
11631 	kvm_for_each_vcpu(i, vcpu, kvm) {
11632 		kvm_clear_async_pf_completion_queue(vcpu);
11633 		kvm_unload_vcpu_mmu(vcpu);
11634 	}
11635 
11636 	kvm_destroy_vcpus(kvm);
11637 }
11638 
11639 void kvm_arch_sync_events(struct kvm *kvm)
11640 {
11641 	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
11642 	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
11643 	kvm_free_pit(kvm);
11644 }
11645 
11646 /**
11647  * __x86_set_memory_region: Setup KVM internal memory slot
11648  *
11649  * @kvm: the kvm pointer to the VM.
11650  * @id: the slot ID to setup.
11651  * @gpa: the GPA to install the slot (unused when @size == 0).
11652  * @size: the size of the slot. Set to zero to uninstall a slot.
11653  *
11654  * This function helps to setup a KVM internal memory slot.  Specify
11655  * @size > 0 to install a new slot, while @size == 0 to uninstall a
11656  * slot.  The return code can be one of the following:
11657  *
11658  *   HVA:           on success (uninstall will return a bogus HVA)
11659  *   -errno:        on error
11660  *
11661  * The caller should always use IS_ERR() to check the return value
11662  * before use.  Note, the KVM internal memory slots are guaranteed to
11663  * remain valid and unchanged until the VM is destroyed, i.e., the
11664  * GPA->HVA translation will not change.  However, the HVA is a user
11665  * address, i.e. its accessibility is not guaranteed, and must be
11666  * accessed via __copy_{to,from}_user().
11667  */
11668 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
11669 				      u32 size)
11670 {
11671 	int i, r;
11672 	unsigned long hva, old_npages;
11673 	struct kvm_memslots *slots = kvm_memslots(kvm);
11674 	struct kvm_memory_slot *slot;
11675 
11676 	/* Called with kvm->slots_lock held.  */
11677 	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
11678 		return ERR_PTR_USR(-EINVAL);
11679 
11680 	slot = id_to_memslot(slots, id);
11681 	if (size) {
11682 		if (slot && slot->npages)
11683 			return ERR_PTR_USR(-EEXIST);
11684 
11685 		/*
11686 		 * MAP_SHARED to prevent internal slot pages from being moved
11687 		 * by fork()/COW.
11688 		 */
11689 		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
11690 			      MAP_SHARED | MAP_ANONYMOUS, 0);
11691 		if (IS_ERR((void *)hva))
11692 			return (void __user *)hva;
11693 	} else {
11694 		if (!slot || !slot->npages)
11695 			return NULL;
11696 
11697 		old_npages = slot->npages;
11698 		hva = slot->userspace_addr;
11699 	}
11700 
11701 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
11702 		struct kvm_userspace_memory_region m;
11703 
11704 		m.slot = id | (i << 16);
11705 		m.flags = 0;
11706 		m.guest_phys_addr = gpa;
11707 		m.userspace_addr = hva;
11708 		m.memory_size = size;
11709 		r = __kvm_set_memory_region(kvm, &m);
11710 		if (r < 0)
11711 			return ERR_PTR_USR(r);
11712 	}
11713 
11714 	if (!size)
11715 		vm_munmap(hva, old_npages * PAGE_SIZE);
11716 
11717 	return (void __user *)hva;
11718 }
11719 EXPORT_SYMBOL_GPL(__x86_set_memory_region);
11720 
11721 void kvm_arch_pre_destroy_vm(struct kvm *kvm)
11722 {
11723 	kvm_mmu_pre_destroy_vm(kvm);
11724 }
11725 
11726 void kvm_arch_destroy_vm(struct kvm *kvm)
11727 {
11728 	if (current->mm == kvm->mm) {
11729 		/*
11730 		 * Free memory regions allocated on behalf of userspace,
11731 		 * unless the the memory map has changed due to process exit
11732 		 * or fd copying.
11733 		 */
11734 		mutex_lock(&kvm->slots_lock);
11735 		__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
11736 					0, 0);
11737 		__x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
11738 					0, 0);
11739 		__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
11740 		mutex_unlock(&kvm->slots_lock);
11741 	}
11742 	static_call_cond(kvm_x86_vm_destroy)(kvm);
11743 	kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
11744 	kvm_pic_destroy(kvm);
11745 	kvm_ioapic_destroy(kvm);
11746 	kvm_free_vcpus(kvm);
11747 	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
11748 	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
11749 	kvm_mmu_uninit_vm(kvm);
11750 	kvm_page_track_cleanup(kvm);
11751 	kvm_xen_destroy_vm(kvm);
11752 	kvm_hv_destroy_vm(kvm);
11753 }
11754 
11755 static void memslot_rmap_free(struct kvm_memory_slot *slot)
11756 {
11757 	int i;
11758 
11759 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
11760 		kvfree(slot->arch.rmap[i]);
11761 		slot->arch.rmap[i] = NULL;
11762 	}
11763 }
11764 
11765 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
11766 {
11767 	int i;
11768 
11769 	memslot_rmap_free(slot);
11770 
11771 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
11772 		kvfree(slot->arch.lpage_info[i - 1]);
11773 		slot->arch.lpage_info[i - 1] = NULL;
11774 	}
11775 
11776 	kvm_page_track_free_memslot(slot);
11777 }
11778 
11779 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages)
11780 {
11781 	const int sz = sizeof(*slot->arch.rmap[0]);
11782 	int i;
11783 
11784 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
11785 		int level = i + 1;
11786 		int lpages = __kvm_mmu_slot_lpages(slot, npages, level);
11787 
11788 		if (slot->arch.rmap[i])
11789 			continue;
11790 
11791 		slot->arch.rmap[i] = kvcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);
11792 		if (!slot->arch.rmap[i]) {
11793 			memslot_rmap_free(slot);
11794 			return -ENOMEM;
11795 		}
11796 	}
11797 
11798 	return 0;
11799 }
11800 
11801 static int kvm_alloc_memslot_metadata(struct kvm *kvm,
11802 				      struct kvm_memory_slot *slot)
11803 {
11804 	unsigned long npages = slot->npages;
11805 	int i, r;
11806 
11807 	/*
11808 	 * Clear out the previous array pointers for the KVM_MR_MOVE case.  The
11809 	 * old arrays will be freed by __kvm_set_memory_region() if installing
11810 	 * the new memslot is successful.
11811 	 */
11812 	memset(&slot->arch, 0, sizeof(slot->arch));
11813 
11814 	if (kvm_memslots_have_rmaps(kvm)) {
11815 		r = memslot_rmap_alloc(slot, npages);
11816 		if (r)
11817 			return r;
11818 	}
11819 
11820 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
11821 		struct kvm_lpage_info *linfo;
11822 		unsigned long ugfn;
11823 		int lpages;
11824 		int level = i + 1;
11825 
11826 		lpages = __kvm_mmu_slot_lpages(slot, npages, level);
11827 
11828 		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
11829 		if (!linfo)
11830 			goto out_free;
11831 
11832 		slot->arch.lpage_info[i - 1] = linfo;
11833 
11834 		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
11835 			linfo[0].disallow_lpage = 1;
11836 		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
11837 			linfo[lpages - 1].disallow_lpage = 1;
11838 		ugfn = slot->userspace_addr >> PAGE_SHIFT;
11839 		/*
11840 		 * If the gfn and userspace address are not aligned wrt each
11841 		 * other, disable large page support for this slot.
11842 		 */
11843 		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {
11844 			unsigned long j;
11845 
11846 			for (j = 0; j < lpages; ++j)
11847 				linfo[j].disallow_lpage = 1;
11848 		}
11849 	}
11850 
11851 	if (kvm_page_track_create_memslot(kvm, slot, npages))
11852 		goto out_free;
11853 
11854 	return 0;
11855 
11856 out_free:
11857 	memslot_rmap_free(slot);
11858 
11859 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
11860 		kvfree(slot->arch.lpage_info[i - 1]);
11861 		slot->arch.lpage_info[i - 1] = NULL;
11862 	}
11863 	return -ENOMEM;
11864 }
11865 
11866 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
11867 {
11868 	struct kvm_vcpu *vcpu;
11869 	unsigned long i;
11870 
11871 	/*
11872 	 * memslots->generation has been incremented.
11873 	 * mmio generation may have reached its maximum value.
11874 	 */
11875 	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
11876 
11877 	/* Force re-initialization of steal_time cache */
11878 	kvm_for_each_vcpu(i, vcpu, kvm)
11879 		kvm_vcpu_kick(vcpu);
11880 }
11881 
11882 int kvm_arch_prepare_memory_region(struct kvm *kvm,
11883 				   const struct kvm_memory_slot *old,
11884 				   struct kvm_memory_slot *new,
11885 				   enum kvm_mr_change change)
11886 {
11887 	if (change == KVM_MR_CREATE || change == KVM_MR_MOVE)
11888 		return kvm_alloc_memslot_metadata(kvm, new);
11889 
11890 	if (change == KVM_MR_FLAGS_ONLY)
11891 		memcpy(&new->arch, &old->arch, sizeof(old->arch));
11892 	else if (WARN_ON_ONCE(change != KVM_MR_DELETE))
11893 		return -EIO;
11894 
11895 	return 0;
11896 }
11897 
11898 
11899 static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
11900 {
11901 	struct kvm_arch *ka = &kvm->arch;
11902 
11903 	if (!kvm_x86_ops.cpu_dirty_log_size)
11904 		return;
11905 
11906 	if ((enable && ++ka->cpu_dirty_logging_count == 1) ||
11907 	    (!enable && --ka->cpu_dirty_logging_count == 0))
11908 		kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
11909 
11910 	WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0);
11911 }
11912 
11913 static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
11914 				     struct kvm_memory_slot *old,
11915 				     const struct kvm_memory_slot *new,
11916 				     enum kvm_mr_change change)
11917 {
11918 	u32 old_flags = old ? old->flags : 0;
11919 	u32 new_flags = new ? new->flags : 0;
11920 	bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES;
11921 
11922 	/*
11923 	 * Update CPU dirty logging if dirty logging is being toggled.  This
11924 	 * applies to all operations.
11925 	 */
11926 	if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)
11927 		kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
11928 
11929 	/*
11930 	 * Nothing more to do for RO slots (which can't be dirtied and can't be
11931 	 * made writable) or CREATE/MOVE/DELETE of a slot.
11932 	 *
11933 	 * For a memslot with dirty logging disabled:
11934 	 * CREATE:      No dirty mappings will already exist.
11935 	 * MOVE/DELETE: The old mappings will already have been cleaned up by
11936 	 *		kvm_arch_flush_shadow_memslot()
11937 	 *
11938 	 * For a memslot with dirty logging enabled:
11939 	 * CREATE:      No shadow pages exist, thus nothing to write-protect
11940 	 *		and no dirty bits to clear.
11941 	 * MOVE/DELETE: The old mappings will already have been cleaned up by
11942 	 *		kvm_arch_flush_shadow_memslot().
11943 	 */
11944 	if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY))
11945 		return;
11946 
11947 	/*
11948 	 * READONLY and non-flags changes were filtered out above, and the only
11949 	 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty
11950 	 * logging isn't being toggled on or off.
11951 	 */
11952 	if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)))
11953 		return;
11954 
11955 	if (!log_dirty_pages) {
11956 		/*
11957 		 * Dirty logging tracks sptes in 4k granularity, meaning that
11958 		 * large sptes have to be split.  If live migration succeeds,
11959 		 * the guest in the source machine will be destroyed and large
11960 		 * sptes will be created in the destination.  However, if the
11961 		 * guest continues to run in the source machine (for example if
11962 		 * live migration fails), small sptes will remain around and
11963 		 * cause bad performance.
11964 		 *
11965 		 * Scan sptes if dirty logging has been stopped, dropping those
11966 		 * which can be collapsed into a single large-page spte.  Later
11967 		 * page faults will create the large-page sptes.
11968 		 */
11969 		kvm_mmu_zap_collapsible_sptes(kvm, new);
11970 	} else {
11971 		/*
11972 		 * Initially-all-set does not require write protecting any page,
11973 		 * because they're all assumed to be dirty.
11974 		 */
11975 		if (kvm_dirty_log_manual_protect_and_init_set(kvm))
11976 			return;
11977 
11978 		if (kvm_x86_ops.cpu_dirty_log_size) {
11979 			kvm_mmu_slot_leaf_clear_dirty(kvm, new);
11980 			kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M);
11981 		} else {
11982 			kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K);
11983 		}
11984 	}
11985 }
11986 
11987 void kvm_arch_commit_memory_region(struct kvm *kvm,
11988 				struct kvm_memory_slot *old,
11989 				const struct kvm_memory_slot *new,
11990 				enum kvm_mr_change change)
11991 {
11992 	if (!kvm->arch.n_requested_mmu_pages &&
11993 	    (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) {
11994 		unsigned long nr_mmu_pages;
11995 
11996 		nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO;
11997 		nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES);
11998 		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
11999 	}
12000 
12001 	kvm_mmu_slot_apply_flags(kvm, old, new, change);
12002 
12003 	/* Free the arrays associated with the old memslot. */
12004 	if (change == KVM_MR_MOVE)
12005 		kvm_arch_free_memslot(kvm, old);
12006 }
12007 
12008 void kvm_arch_flush_shadow_all(struct kvm *kvm)
12009 {
12010 	kvm_mmu_zap_all(kvm);
12011 }
12012 
12013 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
12014 				   struct kvm_memory_slot *slot)
12015 {
12016 	kvm_page_track_flush_slot(kvm, slot);
12017 }
12018 
12019 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
12020 {
12021 	return (is_guest_mode(vcpu) &&
12022 			kvm_x86_ops.guest_apic_has_interrupt &&
12023 			static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
12024 }
12025 
12026 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
12027 {
12028 	if (!list_empty_careful(&vcpu->async_pf.done))
12029 		return true;
12030 
12031 	if (kvm_apic_has_events(vcpu))
12032 		return true;
12033 
12034 	if (vcpu->arch.pv.pv_unhalted)
12035 		return true;
12036 
12037 	if (vcpu->arch.exception.pending)
12038 		return true;
12039 
12040 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12041 	    (vcpu->arch.nmi_pending &&
12042 	     static_call(kvm_x86_nmi_allowed)(vcpu, false)))
12043 		return true;
12044 
12045 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
12046 	    (vcpu->arch.smi_pending &&
12047 	     static_call(kvm_x86_smi_allowed)(vcpu, false)))
12048 		return true;
12049 
12050 	if (kvm_arch_interrupt_allowed(vcpu) &&
12051 	    (kvm_cpu_has_interrupt(vcpu) ||
12052 	    kvm_guest_apic_has_interrupt(vcpu)))
12053 		return true;
12054 
12055 	if (kvm_hv_has_stimer_pending(vcpu))
12056 		return true;
12057 
12058 	if (is_guest_mode(vcpu) &&
12059 	    kvm_x86_ops.nested_ops->hv_timer_pending &&
12060 	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
12061 		return true;
12062 
12063 	return false;
12064 }
12065 
12066 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
12067 {
12068 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
12069 }
12070 
12071 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
12072 {
12073 	if (vcpu->arch.apicv_active && static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))
12074 		return true;
12075 
12076 	return false;
12077 }
12078 
12079 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
12080 {
12081 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
12082 		return true;
12083 
12084 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12085 		kvm_test_request(KVM_REQ_SMI, vcpu) ||
12086 		 kvm_test_request(KVM_REQ_EVENT, vcpu))
12087 		return true;
12088 
12089 	return kvm_arch_dy_has_pending_interrupt(vcpu);
12090 }
12091 
12092 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
12093 {
12094 	if (vcpu->arch.guest_state_protected)
12095 		return true;
12096 
12097 	return vcpu->arch.preempted_in_kernel;
12098 }
12099 
12100 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
12101 {
12102 	return kvm_rip_read(vcpu);
12103 }
12104 
12105 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
12106 {
12107 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
12108 }
12109 
12110 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
12111 {
12112 	return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
12113 }
12114 
12115 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
12116 {
12117 	/* Can't read the RIP when guest state is protected, just return 0 */
12118 	if (vcpu->arch.guest_state_protected)
12119 		return 0;
12120 
12121 	if (is_64_bit_mode(vcpu))
12122 		return kvm_rip_read(vcpu);
12123 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
12124 		     kvm_rip_read(vcpu));
12125 }
12126 EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
12127 
12128 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
12129 {
12130 	return kvm_get_linear_rip(vcpu) == linear_rip;
12131 }
12132 EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
12133 
12134 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
12135 {
12136 	unsigned long rflags;
12137 
12138 	rflags = static_call(kvm_x86_get_rflags)(vcpu);
12139 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
12140 		rflags &= ~X86_EFLAGS_TF;
12141 	return rflags;
12142 }
12143 EXPORT_SYMBOL_GPL(kvm_get_rflags);
12144 
12145 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
12146 {
12147 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
12148 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
12149 		rflags |= X86_EFLAGS_TF;
12150 	static_call(kvm_x86_set_rflags)(vcpu, rflags);
12151 }
12152 
12153 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
12154 {
12155 	__kvm_set_rflags(vcpu, rflags);
12156 	kvm_make_request(KVM_REQ_EVENT, vcpu);
12157 }
12158 EXPORT_SYMBOL_GPL(kvm_set_rflags);
12159 
12160 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
12161 {
12162 	int r;
12163 
12164 	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
12165 	      work->wakeup_all)
12166 		return;
12167 
12168 	r = kvm_mmu_reload(vcpu);
12169 	if (unlikely(r))
12170 		return;
12171 
12172 	if (!vcpu->arch.mmu->direct_map &&
12173 	      work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
12174 		return;
12175 
12176 	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
12177 }
12178 
12179 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
12180 {
12181 	BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU));
12182 
12183 	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
12184 }
12185 
12186 static inline u32 kvm_async_pf_next_probe(u32 key)
12187 {
12188 	return (key + 1) & (ASYNC_PF_PER_VCPU - 1);
12189 }
12190 
12191 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
12192 {
12193 	u32 key = kvm_async_pf_hash_fn(gfn);
12194 
12195 	while (vcpu->arch.apf.gfns[key] != ~0)
12196 		key = kvm_async_pf_next_probe(key);
12197 
12198 	vcpu->arch.apf.gfns[key] = gfn;
12199 }
12200 
12201 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
12202 {
12203 	int i;
12204 	u32 key = kvm_async_pf_hash_fn(gfn);
12205 
12206 	for (i = 0; i < ASYNC_PF_PER_VCPU &&
12207 		     (vcpu->arch.apf.gfns[key] != gfn &&
12208 		      vcpu->arch.apf.gfns[key] != ~0); i++)
12209 		key = kvm_async_pf_next_probe(key);
12210 
12211 	return key;
12212 }
12213 
12214 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
12215 {
12216 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
12217 }
12218 
12219 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
12220 {
12221 	u32 i, j, k;
12222 
12223 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
12224 
12225 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
12226 		return;
12227 
12228 	while (true) {
12229 		vcpu->arch.apf.gfns[i] = ~0;
12230 		do {
12231 			j = kvm_async_pf_next_probe(j);
12232 			if (vcpu->arch.apf.gfns[j] == ~0)
12233 				return;
12234 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
12235 			/*
12236 			 * k lies cyclically in ]i,j]
12237 			 * |    i.k.j |
12238 			 * |....j i.k.| or  |.k..j i...|
12239 			 */
12240 		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
12241 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
12242 		i = j;
12243 	}
12244 }
12245 
12246 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
12247 {
12248 	u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT;
12249 
12250 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
12251 				      sizeof(reason));
12252 }
12253 
12254 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
12255 {
12256 	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
12257 
12258 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
12259 					     &token, offset, sizeof(token));
12260 }
12261 
12262 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
12263 {
12264 	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
12265 	u32 val;
12266 
12267 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
12268 					 &val, offset, sizeof(val)))
12269 		return false;
12270 
12271 	return !val;
12272 }
12273 
12274 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
12275 {
12276 	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
12277 		return false;
12278 
12279 	if (!kvm_pv_async_pf_enabled(vcpu) ||
12280 	    (vcpu->arch.apf.send_user_only && static_call(kvm_x86_get_cpl)(vcpu) == 0))
12281 		return false;
12282 
12283 	return true;
12284 }
12285 
12286 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
12287 {
12288 	if (unlikely(!lapic_in_kernel(vcpu) ||
12289 		     kvm_event_needs_reinjection(vcpu) ||
12290 		     vcpu->arch.exception.pending))
12291 		return false;
12292 
12293 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
12294 		return false;
12295 
12296 	/*
12297 	 * If interrupts are off we cannot even use an artificial
12298 	 * halt state.
12299 	 */
12300 	return kvm_arch_interrupt_allowed(vcpu);
12301 }
12302 
12303 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
12304 				     struct kvm_async_pf *work)
12305 {
12306 	struct x86_exception fault;
12307 
12308 	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
12309 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
12310 
12311 	if (kvm_can_deliver_async_pf(vcpu) &&
12312 	    !apf_put_user_notpresent(vcpu)) {
12313 		fault.vector = PF_VECTOR;
12314 		fault.error_code_valid = true;
12315 		fault.error_code = 0;
12316 		fault.nested_page_fault = false;
12317 		fault.address = work->arch.token;
12318 		fault.async_page_fault = true;
12319 		kvm_inject_page_fault(vcpu, &fault);
12320 		return true;
12321 	} else {
12322 		/*
12323 		 * It is not possible to deliver a paravirtualized asynchronous
12324 		 * page fault, but putting the guest in an artificial halt state
12325 		 * can be beneficial nevertheless: if an interrupt arrives, we
12326 		 * can deliver it timely and perhaps the guest will schedule
12327 		 * another process.  When the instruction that triggered a page
12328 		 * fault is retried, hopefully the page will be ready in the host.
12329 		 */
12330 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
12331 		return false;
12332 	}
12333 }
12334 
12335 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
12336 				 struct kvm_async_pf *work)
12337 {
12338 	struct kvm_lapic_irq irq = {
12339 		.delivery_mode = APIC_DM_FIXED,
12340 		.vector = vcpu->arch.apf.vec
12341 	};
12342 
12343 	if (work->wakeup_all)
12344 		work->arch.token = ~0; /* broadcast wakeup */
12345 	else
12346 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
12347 	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
12348 
12349 	if ((work->wakeup_all || work->notpresent_injected) &&
12350 	    kvm_pv_async_pf_enabled(vcpu) &&
12351 	    !apf_put_user_ready(vcpu, work->arch.token)) {
12352 		vcpu->arch.apf.pageready_pending = true;
12353 		kvm_apic_set_irq(vcpu, &irq, NULL);
12354 	}
12355 
12356 	vcpu->arch.apf.halted = false;
12357 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
12358 }
12359 
12360 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
12361 {
12362 	kvm_make_request(KVM_REQ_APF_READY, vcpu);
12363 	if (!vcpu->arch.apf.pageready_pending)
12364 		kvm_vcpu_kick(vcpu);
12365 }
12366 
12367 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
12368 {
12369 	if (!kvm_pv_async_pf_enabled(vcpu))
12370 		return true;
12371 	else
12372 		return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);
12373 }
12374 
12375 void kvm_arch_start_assignment(struct kvm *kvm)
12376 {
12377 	if (atomic_inc_return(&kvm->arch.assigned_device_count) == 1)
12378 		static_call_cond(kvm_x86_start_assignment)(kvm);
12379 }
12380 EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);
12381 
12382 void kvm_arch_end_assignment(struct kvm *kvm)
12383 {
12384 	atomic_dec(&kvm->arch.assigned_device_count);
12385 }
12386 EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);
12387 
12388 bool kvm_arch_has_assigned_device(struct kvm *kvm)
12389 {
12390 	return atomic_read(&kvm->arch.assigned_device_count);
12391 }
12392 EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
12393 
12394 void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
12395 {
12396 	atomic_inc(&kvm->arch.noncoherent_dma_count);
12397 }
12398 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
12399 
12400 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
12401 {
12402 	atomic_dec(&kvm->arch.noncoherent_dma_count);
12403 }
12404 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
12405 
12406 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
12407 {
12408 	return atomic_read(&kvm->arch.noncoherent_dma_count);
12409 }
12410 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
12411 
12412 bool kvm_arch_has_irq_bypass(void)
12413 {
12414 	return true;
12415 }
12416 
12417 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
12418 				      struct irq_bypass_producer *prod)
12419 {
12420 	struct kvm_kernel_irqfd *irqfd =
12421 		container_of(cons, struct kvm_kernel_irqfd, consumer);
12422 	int ret;
12423 
12424 	irqfd->producer = prod;
12425 	kvm_arch_start_assignment(irqfd->kvm);
12426 	ret = static_call(kvm_x86_update_pi_irte)(irqfd->kvm,
12427 					 prod->irq, irqfd->gsi, 1);
12428 
12429 	if (ret)
12430 		kvm_arch_end_assignment(irqfd->kvm);
12431 
12432 	return ret;
12433 }
12434 
12435 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
12436 				      struct irq_bypass_producer *prod)
12437 {
12438 	int ret;
12439 	struct kvm_kernel_irqfd *irqfd =
12440 		container_of(cons, struct kvm_kernel_irqfd, consumer);
12441 
12442 	WARN_ON(irqfd->producer != prod);
12443 	irqfd->producer = NULL;
12444 
12445 	/*
12446 	 * When producer of consumer is unregistered, we change back to
12447 	 * remapped mode, so we can re-use the current implementation
12448 	 * when the irq is masked/disabled or the consumer side (KVM
12449 	 * int this case doesn't want to receive the interrupts.
12450 	*/
12451 	ret = static_call(kvm_x86_update_pi_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);
12452 	if (ret)
12453 		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
12454 		       " fails: %d\n", irqfd->consumer.token, ret);
12455 
12456 	kvm_arch_end_assignment(irqfd->kvm);
12457 }
12458 
12459 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
12460 				   uint32_t guest_irq, bool set)
12461 {
12462 	return static_call(kvm_x86_update_pi_irte)(kvm, host_irq, guest_irq, set);
12463 }
12464 
12465 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old,
12466 				  struct kvm_kernel_irq_routing_entry *new)
12467 {
12468 	if (new->type != KVM_IRQ_ROUTING_MSI)
12469 		return true;
12470 
12471 	return !!memcmp(&old->msi, &new->msi, sizeof(new->msi));
12472 }
12473 
12474 bool kvm_vector_hashing_enabled(void)
12475 {
12476 	return vector_hashing;
12477 }
12478 
12479 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
12480 {
12481 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
12482 }
12483 EXPORT_SYMBOL_GPL(kvm_arch_no_poll);
12484 
12485 
12486 int kvm_spec_ctrl_test_value(u64 value)
12487 {
12488 	/*
12489 	 * test that setting IA32_SPEC_CTRL to given value
12490 	 * is allowed by the host processor
12491 	 */
12492 
12493 	u64 saved_value;
12494 	unsigned long flags;
12495 	int ret = 0;
12496 
12497 	local_irq_save(flags);
12498 
12499 	if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value))
12500 		ret = 1;
12501 	else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value))
12502 		ret = 1;
12503 	else
12504 		wrmsrl(MSR_IA32_SPEC_CTRL, saved_value);
12505 
12506 	local_irq_restore(flags);
12507 
12508 	return ret;
12509 }
12510 EXPORT_SYMBOL_GPL(kvm_spec_ctrl_test_value);
12511 
12512 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
12513 {
12514 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
12515 	struct x86_exception fault;
12516 	u32 access = error_code &
12517 		(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
12518 
12519 	if (!(error_code & PFERR_PRESENT_MASK) ||
12520 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != UNMAPPED_GVA) {
12521 		/*
12522 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
12523 		 * tables probably do not match the TLB.  Just proceed
12524 		 * with the error code that the processor gave.
12525 		 */
12526 		fault.vector = PF_VECTOR;
12527 		fault.error_code_valid = true;
12528 		fault.error_code = error_code;
12529 		fault.nested_page_fault = false;
12530 		fault.address = gva;
12531 	}
12532 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
12533 }
12534 EXPORT_SYMBOL_GPL(kvm_fixup_and_inject_pf_error);
12535 
12536 /*
12537  * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns
12538  * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value
12539  * indicates whether exit to userspace is needed.
12540  */
12541 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
12542 			      struct x86_exception *e)
12543 {
12544 	if (r == X86EMUL_PROPAGATE_FAULT) {
12545 		kvm_inject_emulated_page_fault(vcpu, e);
12546 		return 1;
12547 	}
12548 
12549 	/*
12550 	 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED
12551 	 * while handling a VMX instruction KVM could've handled the request
12552 	 * correctly by exiting to userspace and performing I/O but there
12553 	 * doesn't seem to be a real use-case behind such requests, just return
12554 	 * KVM_EXIT_INTERNAL_ERROR for now.
12555 	 */
12556 	kvm_prepare_emulation_failure_exit(vcpu);
12557 
12558 	return 0;
12559 }
12560 EXPORT_SYMBOL_GPL(kvm_handle_memory_failure);
12561 
12562 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
12563 {
12564 	bool pcid_enabled;
12565 	struct x86_exception e;
12566 	struct {
12567 		u64 pcid;
12568 		u64 gla;
12569 	} operand;
12570 	int r;
12571 
12572 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
12573 	if (r != X86EMUL_CONTINUE)
12574 		return kvm_handle_memory_failure(vcpu, r, &e);
12575 
12576 	if (operand.pcid >> 12 != 0) {
12577 		kvm_inject_gp(vcpu, 0);
12578 		return 1;
12579 	}
12580 
12581 	pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
12582 
12583 	switch (type) {
12584 	case INVPCID_TYPE_INDIV_ADDR:
12585 		if ((!pcid_enabled && (operand.pcid != 0)) ||
12586 		    is_noncanonical_address(operand.gla, vcpu)) {
12587 			kvm_inject_gp(vcpu, 0);
12588 			return 1;
12589 		}
12590 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
12591 		return kvm_skip_emulated_instruction(vcpu);
12592 
12593 	case INVPCID_TYPE_SINGLE_CTXT:
12594 		if (!pcid_enabled && (operand.pcid != 0)) {
12595 			kvm_inject_gp(vcpu, 0);
12596 			return 1;
12597 		}
12598 
12599 		kvm_invalidate_pcid(vcpu, operand.pcid);
12600 		return kvm_skip_emulated_instruction(vcpu);
12601 
12602 	case INVPCID_TYPE_ALL_NON_GLOBAL:
12603 		/*
12604 		 * Currently, KVM doesn't mark global entries in the shadow
12605 		 * page tables, so a non-global flush just degenerates to a
12606 		 * global flush. If needed, we could optimize this later by
12607 		 * keeping track of global entries in shadow page tables.
12608 		 */
12609 
12610 		fallthrough;
12611 	case INVPCID_TYPE_ALL_INCL_GLOBAL:
12612 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12613 		return kvm_skip_emulated_instruction(vcpu);
12614 
12615 	default:
12616 		kvm_inject_gp(vcpu, 0);
12617 		return 1;
12618 	}
12619 }
12620 EXPORT_SYMBOL_GPL(kvm_handle_invpcid);
12621 
12622 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
12623 {
12624 	struct kvm_run *run = vcpu->run;
12625 	struct kvm_mmio_fragment *frag;
12626 	unsigned int len;
12627 
12628 	BUG_ON(!vcpu->mmio_needed);
12629 
12630 	/* Complete previous fragment */
12631 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
12632 	len = min(8u, frag->len);
12633 	if (!vcpu->mmio_is_write)
12634 		memcpy(frag->data, run->mmio.data, len);
12635 
12636 	if (frag->len <= 8) {
12637 		/* Switch to the next fragment. */
12638 		frag++;
12639 		vcpu->mmio_cur_fragment++;
12640 	} else {
12641 		/* Go forward to the next mmio piece. */
12642 		frag->data += len;
12643 		frag->gpa += len;
12644 		frag->len -= len;
12645 	}
12646 
12647 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
12648 		vcpu->mmio_needed = 0;
12649 
12650 		// VMG change, at this point, we're always done
12651 		// RIP has already been advanced
12652 		return 1;
12653 	}
12654 
12655 	// More MMIO is needed
12656 	run->mmio.phys_addr = frag->gpa;
12657 	run->mmio.len = min(8u, frag->len);
12658 	run->mmio.is_write = vcpu->mmio_is_write;
12659 	if (run->mmio.is_write)
12660 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
12661 	run->exit_reason = KVM_EXIT_MMIO;
12662 
12663 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
12664 
12665 	return 0;
12666 }
12667 
12668 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
12669 			  void *data)
12670 {
12671 	int handled;
12672 	struct kvm_mmio_fragment *frag;
12673 
12674 	if (!data)
12675 		return -EINVAL;
12676 
12677 	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
12678 	if (handled == bytes)
12679 		return 1;
12680 
12681 	bytes -= handled;
12682 	gpa += handled;
12683 	data += handled;
12684 
12685 	/*TODO: Check if need to increment number of frags */
12686 	frag = vcpu->mmio_fragments;
12687 	vcpu->mmio_nr_fragments = 1;
12688 	frag->len = bytes;
12689 	frag->gpa = gpa;
12690 	frag->data = data;
12691 
12692 	vcpu->mmio_needed = 1;
12693 	vcpu->mmio_cur_fragment = 0;
12694 
12695 	vcpu->run->mmio.phys_addr = gpa;
12696 	vcpu->run->mmio.len = min(8u, frag->len);
12697 	vcpu->run->mmio.is_write = 1;
12698 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
12699 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
12700 
12701 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
12702 
12703 	return 0;
12704 }
12705 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_write);
12706 
12707 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
12708 			 void *data)
12709 {
12710 	int handled;
12711 	struct kvm_mmio_fragment *frag;
12712 
12713 	if (!data)
12714 		return -EINVAL;
12715 
12716 	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
12717 	if (handled == bytes)
12718 		return 1;
12719 
12720 	bytes -= handled;
12721 	gpa += handled;
12722 	data += handled;
12723 
12724 	/*TODO: Check if need to increment number of frags */
12725 	frag = vcpu->mmio_fragments;
12726 	vcpu->mmio_nr_fragments = 1;
12727 	frag->len = bytes;
12728 	frag->gpa = gpa;
12729 	frag->data = data;
12730 
12731 	vcpu->mmio_needed = 1;
12732 	vcpu->mmio_cur_fragment = 0;
12733 
12734 	vcpu->run->mmio.phys_addr = gpa;
12735 	vcpu->run->mmio.len = min(8u, frag->len);
12736 	vcpu->run->mmio.is_write = 0;
12737 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
12738 
12739 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
12740 
12741 	return 0;
12742 }
12743 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_read);
12744 
12745 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
12746 			   unsigned int port);
12747 
12748 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)
12749 {
12750 	int size = vcpu->arch.pio.size;
12751 	int port = vcpu->arch.pio.port;
12752 
12753 	vcpu->arch.pio.count = 0;
12754 	if (vcpu->arch.sev_pio_count)
12755 		return kvm_sev_es_outs(vcpu, size, port);
12756 	return 1;
12757 }
12758 
12759 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
12760 			   unsigned int port)
12761 {
12762 	for (;;) {
12763 		unsigned int count =
12764 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
12765 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);
12766 
12767 		/* memcpy done already by emulator_pio_out.  */
12768 		vcpu->arch.sev_pio_count -= count;
12769 		vcpu->arch.sev_pio_data += count * vcpu->arch.pio.size;
12770 		if (!ret)
12771 			break;
12772 
12773 		/* Emulation done by the kernel.  */
12774 		if (!vcpu->arch.sev_pio_count)
12775 			return 1;
12776 	}
12777 
12778 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;
12779 	return 0;
12780 }
12781 
12782 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
12783 			  unsigned int port);
12784 
12785 static void advance_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
12786 {
12787 	unsigned count = vcpu->arch.pio.count;
12788 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);
12789 	vcpu->arch.sev_pio_count -= count;
12790 	vcpu->arch.sev_pio_data += count * vcpu->arch.pio.size;
12791 }
12792 
12793 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
12794 {
12795 	int size = vcpu->arch.pio.size;
12796 	int port = vcpu->arch.pio.port;
12797 
12798 	advance_sev_es_emulated_ins(vcpu);
12799 	if (vcpu->arch.sev_pio_count)
12800 		return kvm_sev_es_ins(vcpu, size, port);
12801 	return 1;
12802 }
12803 
12804 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
12805 			  unsigned int port)
12806 {
12807 	for (;;) {
12808 		unsigned int count =
12809 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
12810 		if (!__emulator_pio_in(vcpu, size, port, count))
12811 			break;
12812 
12813 		/* Emulation done by the kernel.  */
12814 		advance_sev_es_emulated_ins(vcpu);
12815 		if (!vcpu->arch.sev_pio_count)
12816 			return 1;
12817 	}
12818 
12819 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
12820 	return 0;
12821 }
12822 
12823 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
12824 			 unsigned int port, void *data,  unsigned int count,
12825 			 int in)
12826 {
12827 	vcpu->arch.sev_pio_data = data;
12828 	vcpu->arch.sev_pio_count = count;
12829 	return in ? kvm_sev_es_ins(vcpu, size, port)
12830 		  : kvm_sev_es_outs(vcpu, size, port);
12831 }
12832 EXPORT_SYMBOL_GPL(kvm_sev_es_string_io);
12833 
12834 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry);
12835 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
12836 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
12837 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
12838 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
12839 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
12840 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
12841 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
12842 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
12843 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
12844 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
12845 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
12846 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
12847 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
12848 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
12849 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
12850 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
12851 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
12852 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
12853 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
12854 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
12855 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log);
12856 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request);
12857 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq);
12858 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter);
12859 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit);
12860 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter);
12861 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit);
12862