xref: /openbmc/linux/arch/x86/kvm/x86.c (revision 5edb7691)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine driver for Linux
4  *
5  * derived from drivers/kvm/kvm_main.c
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright (C) 2008 Qumranet, Inc.
9  * Copyright IBM Corporation, 2008
10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11  *
12  * Authors:
13  *   Avi Kivity   <avi@qumranet.com>
14  *   Yaniv Kamay  <yaniv@qumranet.com>
15  *   Amit Shah    <amit.shah@qumranet.com>
16  *   Ben-Ami Yassour <benami@il.ibm.com>
17  */
18 
19 #include <linux/kvm_host.h>
20 #include "irq.h"
21 #include "mmu.h"
22 #include "i8254.h"
23 #include "tss.h"
24 #include "kvm_cache_regs.h"
25 #include "x86.h"
26 #include "cpuid.h"
27 #include "pmu.h"
28 #include "hyperv.h"
29 #include "lapic.h"
30 
31 #include <linux/clocksource.h>
32 #include <linux/interrupt.h>
33 #include <linux/kvm.h>
34 #include <linux/fs.h>
35 #include <linux/vmalloc.h>
36 #include <linux/export.h>
37 #include <linux/moduleparam.h>
38 #include <linux/mman.h>
39 #include <linux/highmem.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/cpufreq.h>
43 #include <linux/user-return-notifier.h>
44 #include <linux/srcu.h>
45 #include <linux/slab.h>
46 #include <linux/perf_event.h>
47 #include <linux/uaccess.h>
48 #include <linux/hash.h>
49 #include <linux/pci.h>
50 #include <linux/timekeeper_internal.h>
51 #include <linux/pvclock_gtod.h>
52 #include <linux/kvm_irqfd.h>
53 #include <linux/irqbypass.h>
54 #include <linux/sched/stat.h>
55 #include <linux/sched/isolation.h>
56 #include <linux/mem_encrypt.h>
57 
58 #include <trace/events/kvm.h>
59 
60 #include <asm/debugreg.h>
61 #include <asm/msr.h>
62 #include <asm/desc.h>
63 #include <asm/mce.h>
64 #include <linux/kernel_stat.h>
65 #include <asm/fpu/internal.h> /* Ugh! */
66 #include <asm/pvclock.h>
67 #include <asm/div64.h>
68 #include <asm/irq_remapping.h>
69 #include <asm/mshyperv.h>
70 #include <asm/hypervisor.h>
71 #include <asm/intel_pt.h>
72 #include <asm/emulate_prefix.h>
73 #include <clocksource/hyperv_timer.h>
74 
75 #define CREATE_TRACE_POINTS
76 #include "trace.h"
77 
78 #define MAX_IO_MSRS 256
79 #define KVM_MAX_MCE_BANKS 32
80 u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
81 EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
82 
83 #define emul_to_vcpu(ctxt) \
84 	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)
85 
86 /* EFER defaults:
87  * - enable syscall per default because its emulated by KVM
88  * - enable LME and LMA per default on 64 bit KVM
89  */
90 #ifdef CONFIG_X86_64
91 static
92 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
93 #else
94 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
95 #endif
96 
97 static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;
98 
99 #define VM_STAT(x, ...) offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__
100 #define VCPU_STAT(x, ...) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__
101 
102 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
103                                     KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
104 
105 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
106 static void process_nmi(struct kvm_vcpu *vcpu);
107 static void enter_smm(struct kvm_vcpu *vcpu);
108 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
109 static void store_regs(struct kvm_vcpu *vcpu);
110 static int sync_regs(struct kvm_vcpu *vcpu);
111 
112 struct kvm_x86_ops *kvm_x86_ops __read_mostly;
113 EXPORT_SYMBOL_GPL(kvm_x86_ops);
114 
115 static bool __read_mostly ignore_msrs = 0;
116 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
117 
118 static bool __read_mostly report_ignored_msrs = true;
119 module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);
120 
121 unsigned int min_timer_period_us = 200;
122 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
123 
124 static bool __read_mostly kvmclock_periodic_sync = true;
125 module_param(kvmclock_periodic_sync, bool, S_IRUGO);
126 
127 bool __read_mostly kvm_has_tsc_control;
128 EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
129 u32  __read_mostly kvm_max_guest_tsc_khz;
130 EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
131 u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
132 EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
133 u64  __read_mostly kvm_max_tsc_scaling_ratio;
134 EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
135 u64 __read_mostly kvm_default_tsc_scaling_ratio;
136 EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
137 
138 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
139 static u32 __read_mostly tsc_tolerance_ppm = 250;
140 module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
141 
142 /*
143  * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
144  * adaptive tuning starting from default advancment of 1000ns.  '0' disables
145  * advancement entirely.  Any other value is used as-is and disables adaptive
146  * tuning, i.e. allows priveleged userspace to set an exact advancement time.
147  */
148 static int __read_mostly lapic_timer_advance_ns = -1;
149 module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
150 
151 static bool __read_mostly vector_hashing = true;
152 module_param(vector_hashing, bool, S_IRUGO);
153 
154 bool __read_mostly enable_vmware_backdoor = false;
155 module_param(enable_vmware_backdoor, bool, S_IRUGO);
156 EXPORT_SYMBOL_GPL(enable_vmware_backdoor);
157 
158 static bool __read_mostly force_emulation_prefix = false;
159 module_param(force_emulation_prefix, bool, S_IRUGO);
160 
161 int __read_mostly pi_inject_timer = -1;
162 module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);
163 
164 #define KVM_NR_SHARED_MSRS 16
165 
166 struct kvm_shared_msrs_global {
167 	int nr;
168 	u32 msrs[KVM_NR_SHARED_MSRS];
169 };
170 
171 struct kvm_shared_msrs {
172 	struct user_return_notifier urn;
173 	bool registered;
174 	struct kvm_shared_msr_values {
175 		u64 host;
176 		u64 curr;
177 	} values[KVM_NR_SHARED_MSRS];
178 };
179 
180 static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
181 static struct kvm_shared_msrs __percpu *shared_msrs;
182 
183 static u64 __read_mostly host_xss;
184 
185 struct kvm_stats_debugfs_item debugfs_entries[] = {
186 	{ "pf_fixed", VCPU_STAT(pf_fixed) },
187 	{ "pf_guest", VCPU_STAT(pf_guest) },
188 	{ "tlb_flush", VCPU_STAT(tlb_flush) },
189 	{ "invlpg", VCPU_STAT(invlpg) },
190 	{ "exits", VCPU_STAT(exits) },
191 	{ "io_exits", VCPU_STAT(io_exits) },
192 	{ "mmio_exits", VCPU_STAT(mmio_exits) },
193 	{ "signal_exits", VCPU_STAT(signal_exits) },
194 	{ "irq_window", VCPU_STAT(irq_window_exits) },
195 	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
196 	{ "halt_exits", VCPU_STAT(halt_exits) },
197 	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
198 	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
199 	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
200 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
201 	{ "hypercalls", VCPU_STAT(hypercalls) },
202 	{ "request_irq", VCPU_STAT(request_irq_exits) },
203 	{ "irq_exits", VCPU_STAT(irq_exits) },
204 	{ "host_state_reload", VCPU_STAT(host_state_reload) },
205 	{ "fpu_reload", VCPU_STAT(fpu_reload) },
206 	{ "insn_emulation", VCPU_STAT(insn_emulation) },
207 	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
208 	{ "irq_injections", VCPU_STAT(irq_injections) },
209 	{ "nmi_injections", VCPU_STAT(nmi_injections) },
210 	{ "req_event", VCPU_STAT(req_event) },
211 	{ "l1d_flush", VCPU_STAT(l1d_flush) },
212 	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
213 	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
214 	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
215 	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
216 	{ "mmu_flooded", VM_STAT(mmu_flooded) },
217 	{ "mmu_recycled", VM_STAT(mmu_recycled) },
218 	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
219 	{ "mmu_unsync", VM_STAT(mmu_unsync) },
220 	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
221 	{ "largepages", VM_STAT(lpages, .mode = 0444) },
222 	{ "nx_largepages_splitted", VM_STAT(nx_lpage_splits, .mode = 0444) },
223 	{ "max_mmu_page_hash_collisions",
224 		VM_STAT(max_mmu_page_hash_collisions) },
225 	{ NULL }
226 };
227 
228 u64 __read_mostly host_xcr0;
229 
230 struct kmem_cache *x86_fpu_cache;
231 EXPORT_SYMBOL_GPL(x86_fpu_cache);
232 
233 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
234 
235 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
236 {
237 	int i;
238 	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
239 		vcpu->arch.apf.gfns[i] = ~0;
240 }
241 
242 static void kvm_on_user_return(struct user_return_notifier *urn)
243 {
244 	unsigned slot;
245 	struct kvm_shared_msrs *locals
246 		= container_of(urn, struct kvm_shared_msrs, urn);
247 	struct kvm_shared_msr_values *values;
248 	unsigned long flags;
249 
250 	/*
251 	 * Disabling irqs at this point since the following code could be
252 	 * interrupted and executed through kvm_arch_hardware_disable()
253 	 */
254 	local_irq_save(flags);
255 	if (locals->registered) {
256 		locals->registered = false;
257 		user_return_notifier_unregister(urn);
258 	}
259 	local_irq_restore(flags);
260 	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
261 		values = &locals->values[slot];
262 		if (values->host != values->curr) {
263 			wrmsrl(shared_msrs_global.msrs[slot], values->host);
264 			values->curr = values->host;
265 		}
266 	}
267 }
268 
269 void kvm_define_shared_msr(unsigned slot, u32 msr)
270 {
271 	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
272 	shared_msrs_global.msrs[slot] = msr;
273 	if (slot >= shared_msrs_global.nr)
274 		shared_msrs_global.nr = slot + 1;
275 }
276 EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
277 
278 static void kvm_shared_msr_cpu_online(void)
279 {
280 	unsigned int cpu = smp_processor_id();
281 	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
282 	u64 value;
283 	int i;
284 
285 	for (i = 0; i < shared_msrs_global.nr; ++i) {
286 		rdmsrl_safe(shared_msrs_global.msrs[i], &value);
287 		smsr->values[i].host = value;
288 		smsr->values[i].curr = value;
289 	}
290 }
291 
292 int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
293 {
294 	unsigned int cpu = smp_processor_id();
295 	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
296 	int err;
297 
298 	value = (value & mask) | (smsr->values[slot].host & ~mask);
299 	if (value == smsr->values[slot].curr)
300 		return 0;
301 	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
302 	if (err)
303 		return 1;
304 
305 	smsr->values[slot].curr = value;
306 	if (!smsr->registered) {
307 		smsr->urn.on_user_return = kvm_on_user_return;
308 		user_return_notifier_register(&smsr->urn);
309 		smsr->registered = true;
310 	}
311 	return 0;
312 }
313 EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
314 
315 static void drop_user_return_notifiers(void)
316 {
317 	unsigned int cpu = smp_processor_id();
318 	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
319 
320 	if (smsr->registered)
321 		kvm_on_user_return(&smsr->urn);
322 }
323 
324 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
325 {
326 	return vcpu->arch.apic_base;
327 }
328 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
329 
330 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
331 {
332 	return kvm_apic_mode(kvm_get_apic_base(vcpu));
333 }
334 EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
335 
336 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
337 {
338 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
339 	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
340 	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
341 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
342 
343 	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
344 		return 1;
345 	if (!msr_info->host_initiated) {
346 		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
347 			return 1;
348 		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
349 			return 1;
350 	}
351 
352 	kvm_lapic_set_base(vcpu, msr_info->data);
353 	return 0;
354 }
355 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
356 
357 asmlinkage __visible void kvm_spurious_fault(void)
358 {
359 	/* Fault while not rebooting.  We want the trace. */
360 	BUG_ON(!kvm_rebooting);
361 }
362 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
363 
364 #define EXCPT_BENIGN		0
365 #define EXCPT_CONTRIBUTORY	1
366 #define EXCPT_PF		2
367 
368 static int exception_class(int vector)
369 {
370 	switch (vector) {
371 	case PF_VECTOR:
372 		return EXCPT_PF;
373 	case DE_VECTOR:
374 	case TS_VECTOR:
375 	case NP_VECTOR:
376 	case SS_VECTOR:
377 	case GP_VECTOR:
378 		return EXCPT_CONTRIBUTORY;
379 	default:
380 		break;
381 	}
382 	return EXCPT_BENIGN;
383 }
384 
385 #define EXCPT_FAULT		0
386 #define EXCPT_TRAP		1
387 #define EXCPT_ABORT		2
388 #define EXCPT_INTERRUPT		3
389 
390 static int exception_type(int vector)
391 {
392 	unsigned int mask;
393 
394 	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
395 		return EXCPT_INTERRUPT;
396 
397 	mask = 1 << vector;
398 
399 	/* #DB is trap, as instruction watchpoints are handled elsewhere */
400 	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
401 		return EXCPT_TRAP;
402 
403 	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
404 		return EXCPT_ABORT;
405 
406 	/* Reserved exceptions will result in fault */
407 	return EXCPT_FAULT;
408 }
409 
410 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
411 {
412 	unsigned nr = vcpu->arch.exception.nr;
413 	bool has_payload = vcpu->arch.exception.has_payload;
414 	unsigned long payload = vcpu->arch.exception.payload;
415 
416 	if (!has_payload)
417 		return;
418 
419 	switch (nr) {
420 	case DB_VECTOR:
421 		/*
422 		 * "Certain debug exceptions may clear bit 0-3.  The
423 		 * remaining contents of the DR6 register are never
424 		 * cleared by the processor".
425 		 */
426 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
427 		/*
428 		 * DR6.RTM is set by all #DB exceptions that don't clear it.
429 		 */
430 		vcpu->arch.dr6 |= DR6_RTM;
431 		vcpu->arch.dr6 |= payload;
432 		/*
433 		 * Bit 16 should be set in the payload whenever the #DB
434 		 * exception should clear DR6.RTM. This makes the payload
435 		 * compatible with the pending debug exceptions under VMX.
436 		 * Though not currently documented in the SDM, this also
437 		 * makes the payload compatible with the exit qualification
438 		 * for #DB exceptions under VMX.
439 		 */
440 		vcpu->arch.dr6 ^= payload & DR6_RTM;
441 
442 		/*
443 		 * The #DB payload is defined as compatible with the 'pending
444 		 * debug exceptions' field under VMX, not DR6. While bit 12 is
445 		 * defined in the 'pending debug exceptions' field (enabled
446 		 * breakpoint), it is reserved and must be zero in DR6.
447 		 */
448 		vcpu->arch.dr6 &= ~BIT(12);
449 		break;
450 	case PF_VECTOR:
451 		vcpu->arch.cr2 = payload;
452 		break;
453 	}
454 
455 	vcpu->arch.exception.has_payload = false;
456 	vcpu->arch.exception.payload = 0;
457 }
458 EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);
459 
460 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
461 		unsigned nr, bool has_error, u32 error_code,
462 	        bool has_payload, unsigned long payload, bool reinject)
463 {
464 	u32 prev_nr;
465 	int class1, class2;
466 
467 	kvm_make_request(KVM_REQ_EVENT, vcpu);
468 
469 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
470 	queue:
471 		if (has_error && !is_protmode(vcpu))
472 			has_error = false;
473 		if (reinject) {
474 			/*
475 			 * On vmentry, vcpu->arch.exception.pending is only
476 			 * true if an event injection was blocked by
477 			 * nested_run_pending.  In that case, however,
478 			 * vcpu_enter_guest requests an immediate exit,
479 			 * and the guest shouldn't proceed far enough to
480 			 * need reinjection.
481 			 */
482 			WARN_ON_ONCE(vcpu->arch.exception.pending);
483 			vcpu->arch.exception.injected = true;
484 			if (WARN_ON_ONCE(has_payload)) {
485 				/*
486 				 * A reinjected event has already
487 				 * delivered its payload.
488 				 */
489 				has_payload = false;
490 				payload = 0;
491 			}
492 		} else {
493 			vcpu->arch.exception.pending = true;
494 			vcpu->arch.exception.injected = false;
495 		}
496 		vcpu->arch.exception.has_error_code = has_error;
497 		vcpu->arch.exception.nr = nr;
498 		vcpu->arch.exception.error_code = error_code;
499 		vcpu->arch.exception.has_payload = has_payload;
500 		vcpu->arch.exception.payload = payload;
501 		if (!is_guest_mode(vcpu))
502 			kvm_deliver_exception_payload(vcpu);
503 		return;
504 	}
505 
506 	/* to check exception */
507 	prev_nr = vcpu->arch.exception.nr;
508 	if (prev_nr == DF_VECTOR) {
509 		/* triple fault -> shutdown */
510 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
511 		return;
512 	}
513 	class1 = exception_class(prev_nr);
514 	class2 = exception_class(nr);
515 	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
516 		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
517 		/*
518 		 * Generate double fault per SDM Table 5-5.  Set
519 		 * exception.pending = true so that the double fault
520 		 * can trigger a nested vmexit.
521 		 */
522 		vcpu->arch.exception.pending = true;
523 		vcpu->arch.exception.injected = false;
524 		vcpu->arch.exception.has_error_code = true;
525 		vcpu->arch.exception.nr = DF_VECTOR;
526 		vcpu->arch.exception.error_code = 0;
527 		vcpu->arch.exception.has_payload = false;
528 		vcpu->arch.exception.payload = 0;
529 	} else
530 		/* replace previous exception with a new one in a hope
531 		   that instruction re-execution will regenerate lost
532 		   exception */
533 		goto queue;
534 }
535 
536 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
537 {
538 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
539 }
540 EXPORT_SYMBOL_GPL(kvm_queue_exception);
541 
542 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
543 {
544 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
545 }
546 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
547 
548 static void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
549 				  unsigned long payload)
550 {
551 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
552 }
553 
554 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
555 				    u32 error_code, unsigned long payload)
556 {
557 	kvm_multiple_exception(vcpu, nr, true, error_code,
558 			       true, payload, false);
559 }
560 
561 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
562 {
563 	if (err)
564 		kvm_inject_gp(vcpu, 0);
565 	else
566 		return kvm_skip_emulated_instruction(vcpu);
567 
568 	return 1;
569 }
570 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
571 
572 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
573 {
574 	++vcpu->stat.pf_guest;
575 	vcpu->arch.exception.nested_apf =
576 		is_guest_mode(vcpu) && fault->async_page_fault;
577 	if (vcpu->arch.exception.nested_apf) {
578 		vcpu->arch.apf.nested_apf_token = fault->address;
579 		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
580 	} else {
581 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
582 					fault->address);
583 	}
584 }
585 EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
586 
587 static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
588 {
589 	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
590 		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
591 	else
592 		vcpu->arch.mmu->inject_page_fault(vcpu, fault);
593 
594 	return fault->nested_page_fault;
595 }
596 
597 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
598 {
599 	atomic_inc(&vcpu->arch.nmi_queued);
600 	kvm_make_request(KVM_REQ_NMI, vcpu);
601 }
602 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
603 
604 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
605 {
606 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
607 }
608 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
609 
610 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
611 {
612 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
613 }
614 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
615 
616 /*
617  * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
618  * a #GP and return false.
619  */
620 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
621 {
622 	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
623 		return true;
624 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
625 	return false;
626 }
627 EXPORT_SYMBOL_GPL(kvm_require_cpl);
628 
629 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
630 {
631 	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
632 		return true;
633 
634 	kvm_queue_exception(vcpu, UD_VECTOR);
635 	return false;
636 }
637 EXPORT_SYMBOL_GPL(kvm_require_dr);
638 
639 /*
640  * This function will be used to read from the physical memory of the currently
641  * running guest. The difference to kvm_vcpu_read_guest_page is that this function
642  * can read from guest physical or from the guest's guest physical memory.
643  */
644 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
645 			    gfn_t ngfn, void *data, int offset, int len,
646 			    u32 access)
647 {
648 	struct x86_exception exception;
649 	gfn_t real_gfn;
650 	gpa_t ngpa;
651 
652 	ngpa     = gfn_to_gpa(ngfn);
653 	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
654 	if (real_gfn == UNMAPPED_GVA)
655 		return -EFAULT;
656 
657 	real_gfn = gpa_to_gfn(real_gfn);
658 
659 	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
660 }
661 EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);
662 
663 static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
664 			       void *data, int offset, int len, u32 access)
665 {
666 	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
667 				       data, offset, len, access);
668 }
669 
670 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
671 {
672 	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
673 	       rsvd_bits(1, 2);
674 }
675 
676 /*
677  * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
678  */
679 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
680 {
681 	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
682 	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
683 	int i;
684 	int ret;
685 	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
686 
687 	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
688 				      offset * sizeof(u64), sizeof(pdpte),
689 				      PFERR_USER_MASK|PFERR_WRITE_MASK);
690 	if (ret < 0) {
691 		ret = 0;
692 		goto out;
693 	}
694 	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
695 		if ((pdpte[i] & PT_PRESENT_MASK) &&
696 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
697 			ret = 0;
698 			goto out;
699 		}
700 	}
701 	ret = 1;
702 
703 	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
704 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
705 
706 out:
707 
708 	return ret;
709 }
710 EXPORT_SYMBOL_GPL(load_pdptrs);
711 
712 bool pdptrs_changed(struct kvm_vcpu *vcpu)
713 {
714 	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
715 	int offset;
716 	gfn_t gfn;
717 	int r;
718 
719 	if (!is_pae_paging(vcpu))
720 		return false;
721 
722 	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
723 		return true;
724 
725 	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
726 	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
727 	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
728 				       PFERR_USER_MASK | PFERR_WRITE_MASK);
729 	if (r < 0)
730 		return true;
731 
732 	return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
733 }
734 EXPORT_SYMBOL_GPL(pdptrs_changed);
735 
736 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
737 {
738 	unsigned long old_cr0 = kvm_read_cr0(vcpu);
739 	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
740 
741 	cr0 |= X86_CR0_ET;
742 
743 #ifdef CONFIG_X86_64
744 	if (cr0 & 0xffffffff00000000UL)
745 		return 1;
746 #endif
747 
748 	cr0 &= ~CR0_RESERVED_BITS;
749 
750 	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
751 		return 1;
752 
753 	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
754 		return 1;
755 
756 	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
757 #ifdef CONFIG_X86_64
758 		if ((vcpu->arch.efer & EFER_LME)) {
759 			int cs_db, cs_l;
760 
761 			if (!is_pae(vcpu))
762 				return 1;
763 			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
764 			if (cs_l)
765 				return 1;
766 		} else
767 #endif
768 		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
769 						 kvm_read_cr3(vcpu)))
770 			return 1;
771 	}
772 
773 	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
774 		return 1;
775 
776 	kvm_x86_ops->set_cr0(vcpu, cr0);
777 
778 	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
779 		kvm_clear_async_pf_completion_queue(vcpu);
780 		kvm_async_pf_hash_reset(vcpu);
781 	}
782 
783 	if ((cr0 ^ old_cr0) & update_bits)
784 		kvm_mmu_reset_context(vcpu);
785 
786 	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
787 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
788 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
789 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
790 
791 	return 0;
792 }
793 EXPORT_SYMBOL_GPL(kvm_set_cr0);
794 
795 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
796 {
797 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
798 }
799 EXPORT_SYMBOL_GPL(kvm_lmsw);
800 
801 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
802 {
803 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
804 
805 		if (vcpu->arch.xcr0 != host_xcr0)
806 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
807 
808 		if (vcpu->arch.xsaves_enabled &&
809 		    vcpu->arch.ia32_xss != host_xss)
810 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
811 	}
812 }
813 EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
814 
815 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
816 {
817 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
818 
819 		if (vcpu->arch.xcr0 != host_xcr0)
820 			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
821 
822 		if (vcpu->arch.xsaves_enabled &&
823 		    vcpu->arch.ia32_xss != host_xss)
824 			wrmsrl(MSR_IA32_XSS, host_xss);
825 	}
826 
827 }
828 EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
829 
830 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
831 {
832 	u64 xcr0 = xcr;
833 	u64 old_xcr0 = vcpu->arch.xcr0;
834 	u64 valid_bits;
835 
836 	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
837 	if (index != XCR_XFEATURE_ENABLED_MASK)
838 		return 1;
839 	if (!(xcr0 & XFEATURE_MASK_FP))
840 		return 1;
841 	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
842 		return 1;
843 
844 	/*
845 	 * Do not allow the guest to set bits that we do not support
846 	 * saving.  However, xcr0 bit 0 is always set, even if the
847 	 * emulated CPU does not support XSAVE (see fx_init).
848 	 */
849 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
850 	if (xcr0 & ~valid_bits)
851 		return 1;
852 
853 	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
854 	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
855 		return 1;
856 
857 	if (xcr0 & XFEATURE_MASK_AVX512) {
858 		if (!(xcr0 & XFEATURE_MASK_YMM))
859 			return 1;
860 		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
861 			return 1;
862 	}
863 	vcpu->arch.xcr0 = xcr0;
864 
865 	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
866 		kvm_update_cpuid(vcpu);
867 	return 0;
868 }
869 
870 int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
871 {
872 	if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
873 	    __kvm_set_xcr(vcpu, index, xcr)) {
874 		kvm_inject_gp(vcpu, 0);
875 		return 1;
876 	}
877 	return 0;
878 }
879 EXPORT_SYMBOL_GPL(kvm_set_xcr);
880 
881 #define __cr4_reserved_bits(__cpu_has, __c)		\
882 ({							\
883 	u64 __reserved_bits = CR4_RESERVED_BITS;	\
884 							\
885 	if (!__cpu_has(__c, X86_FEATURE_XSAVE))		\
886 		__reserved_bits |= X86_CR4_OSXSAVE;	\
887 	if (!__cpu_has(__c, X86_FEATURE_SMEP))		\
888 		__reserved_bits |= X86_CR4_SMEP;	\
889 	if (!__cpu_has(__c, X86_FEATURE_SMAP))		\
890 		__reserved_bits |= X86_CR4_SMAP;	\
891 	if (!__cpu_has(__c, X86_FEATURE_FSGSBASE))	\
892 		__reserved_bits |= X86_CR4_FSGSBASE;	\
893 	if (!__cpu_has(__c, X86_FEATURE_PKU))		\
894 		__reserved_bits |= X86_CR4_PKE;		\
895 	if (!__cpu_has(__c, X86_FEATURE_LA57))		\
896 		__reserved_bits |= X86_CR4_LA57;	\
897 	if (!__cpu_has(__c, X86_FEATURE_UMIP))		\
898 		__reserved_bits |= X86_CR4_UMIP;	\
899 	__reserved_bits;				\
900 })
901 
902 static u64 kvm_host_cr4_reserved_bits(struct cpuinfo_x86 *c)
903 {
904 	u64 reserved_bits = __cr4_reserved_bits(cpu_has, c);
905 
906 	if (cpuid_ecx(0x7) & feature_bit(LA57))
907 		reserved_bits &= ~X86_CR4_LA57;
908 
909 	if (kvm_x86_ops->umip_emulated())
910 		reserved_bits &= ~X86_CR4_UMIP;
911 
912 	return reserved_bits;
913 }
914 
915 static int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
916 {
917 	if (cr4 & cr4_reserved_bits)
918 		return -EINVAL;
919 
920 	if (cr4 & __cr4_reserved_bits(guest_cpuid_has, vcpu))
921 		return -EINVAL;
922 
923 	return 0;
924 }
925 
926 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
927 {
928 	unsigned long old_cr4 = kvm_read_cr4(vcpu);
929 	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
930 				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;
931 
932 	if (kvm_valid_cr4(vcpu, cr4))
933 		return 1;
934 
935 	if (is_long_mode(vcpu)) {
936 		if (!(cr4 & X86_CR4_PAE))
937 			return 1;
938 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
939 		   && ((cr4 ^ old_cr4) & pdptr_bits)
940 		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
941 				   kvm_read_cr3(vcpu)))
942 		return 1;
943 
944 	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
945 		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
946 			return 1;
947 
948 		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
949 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
950 			return 1;
951 	}
952 
953 	if (kvm_x86_ops->set_cr4(vcpu, cr4))
954 		return 1;
955 
956 	if (((cr4 ^ old_cr4) & pdptr_bits) ||
957 	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
958 		kvm_mmu_reset_context(vcpu);
959 
960 	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
961 		kvm_update_cpuid(vcpu);
962 
963 	return 0;
964 }
965 EXPORT_SYMBOL_GPL(kvm_set_cr4);
966 
967 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
968 {
969 	bool skip_tlb_flush = false;
970 #ifdef CONFIG_X86_64
971 	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
972 
973 	if (pcid_enabled) {
974 		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
975 		cr3 &= ~X86_CR3_PCID_NOFLUSH;
976 	}
977 #endif
978 
979 	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
980 		if (!skip_tlb_flush) {
981 			kvm_mmu_sync_roots(vcpu);
982 			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
983 		}
984 		return 0;
985 	}
986 
987 	if (is_long_mode(vcpu) &&
988 	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
989 		return 1;
990 	else if (is_pae_paging(vcpu) &&
991 		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
992 		return 1;
993 
994 	kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush);
995 	vcpu->arch.cr3 = cr3;
996 	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
997 
998 	return 0;
999 }
1000 EXPORT_SYMBOL_GPL(kvm_set_cr3);
1001 
1002 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1003 {
1004 	if (cr8 & CR8_RESERVED_BITS)
1005 		return 1;
1006 	if (lapic_in_kernel(vcpu))
1007 		kvm_lapic_set_tpr(vcpu, cr8);
1008 	else
1009 		vcpu->arch.cr8 = cr8;
1010 	return 0;
1011 }
1012 EXPORT_SYMBOL_GPL(kvm_set_cr8);
1013 
1014 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1015 {
1016 	if (lapic_in_kernel(vcpu))
1017 		return kvm_lapic_get_cr8(vcpu);
1018 	else
1019 		return vcpu->arch.cr8;
1020 }
1021 EXPORT_SYMBOL_GPL(kvm_get_cr8);
1022 
1023 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1024 {
1025 	int i;
1026 
1027 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1028 		for (i = 0; i < KVM_NR_DB_REGS; i++)
1029 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1030 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
1031 	}
1032 }
1033 
1034 static void kvm_update_dr6(struct kvm_vcpu *vcpu)
1035 {
1036 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1037 		kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
1038 }
1039 
1040 static void kvm_update_dr7(struct kvm_vcpu *vcpu)
1041 {
1042 	unsigned long dr7;
1043 
1044 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1045 		dr7 = vcpu->arch.guest_debug_dr7;
1046 	else
1047 		dr7 = vcpu->arch.dr7;
1048 	kvm_x86_ops->set_dr7(vcpu, dr7);
1049 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1050 	if (dr7 & DR7_BP_EN_MASK)
1051 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1052 }
1053 
1054 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1055 {
1056 	u64 fixed = DR6_FIXED_1;
1057 
1058 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1059 		fixed |= DR6_RTM;
1060 	return fixed;
1061 }
1062 
1063 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1064 {
1065 	size_t size = ARRAY_SIZE(vcpu->arch.db);
1066 
1067 	switch (dr) {
1068 	case 0 ... 3:
1069 		vcpu->arch.db[array_index_nospec(dr, size)] = val;
1070 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1071 			vcpu->arch.eff_db[dr] = val;
1072 		break;
1073 	case 4:
1074 		/* fall through */
1075 	case 6:
1076 		if (val & 0xffffffff00000000ULL)
1077 			return -1; /* #GP */
1078 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1079 		kvm_update_dr6(vcpu);
1080 		break;
1081 	case 5:
1082 		/* fall through */
1083 	default: /* 7 */
1084 		if (!kvm_dr7_valid(val))
1085 			return -1; /* #GP */
1086 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1087 		kvm_update_dr7(vcpu);
1088 		break;
1089 	}
1090 
1091 	return 0;
1092 }
1093 
1094 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1095 {
1096 	if (__kvm_set_dr(vcpu, dr, val)) {
1097 		kvm_inject_gp(vcpu, 0);
1098 		return 1;
1099 	}
1100 	return 0;
1101 }
1102 EXPORT_SYMBOL_GPL(kvm_set_dr);
1103 
1104 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1105 {
1106 	size_t size = ARRAY_SIZE(vcpu->arch.db);
1107 
1108 	switch (dr) {
1109 	case 0 ... 3:
1110 		*val = vcpu->arch.db[array_index_nospec(dr, size)];
1111 		break;
1112 	case 4:
1113 		/* fall through */
1114 	case 6:
1115 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1116 			*val = vcpu->arch.dr6;
1117 		else
1118 			*val = kvm_x86_ops->get_dr6(vcpu);
1119 		break;
1120 	case 5:
1121 		/* fall through */
1122 	default: /* 7 */
1123 		*val = vcpu->arch.dr7;
1124 		break;
1125 	}
1126 	return 0;
1127 }
1128 EXPORT_SYMBOL_GPL(kvm_get_dr);
1129 
1130 bool kvm_rdpmc(struct kvm_vcpu *vcpu)
1131 {
1132 	u32 ecx = kvm_rcx_read(vcpu);
1133 	u64 data;
1134 	int err;
1135 
1136 	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
1137 	if (err)
1138 		return err;
1139 	kvm_rax_write(vcpu, (u32)data);
1140 	kvm_rdx_write(vcpu, data >> 32);
1141 	return err;
1142 }
1143 EXPORT_SYMBOL_GPL(kvm_rdpmc);
1144 
1145 /*
1146  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1147  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1148  *
1149  * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
1150  * extract the supported MSRs from the related const lists.
1151  * msrs_to_save is selected from the msrs_to_save_all to reflect the
1152  * capabilities of the host cpu. This capabilities test skips MSRs that are
1153  * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1154  * may depend on host virtualization features rather than host cpu features.
1155  */
1156 
1157 static const u32 msrs_to_save_all[] = {
1158 	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1159 	MSR_STAR,
1160 #ifdef CONFIG_X86_64
1161 	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1162 #endif
1163 	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1164 	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1165 	MSR_IA32_SPEC_CTRL,
1166 	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
1167 	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
1168 	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
1169 	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
1170 	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
1171 	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1172 	MSR_IA32_UMWAIT_CONTROL,
1173 
1174 	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
1175 	MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
1176 	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
1177 	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
1178 	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
1179 	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
1180 	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
1181 	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
1182 	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
1183 	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
1184 	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
1185 	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
1186 	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
1187 	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
1188 	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1189 	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1190 	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1191 	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
1192 	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
1193 	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
1194 	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
1195 	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1196 };
1197 
1198 static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1199 static unsigned num_msrs_to_save;
1200 
1201 static const u32 emulated_msrs_all[] = {
1202 	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
1203 	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
1204 	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
1205 	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1206 	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1207 	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
1208 	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1209 	HV_X64_MSR_RESET,
1210 	HV_X64_MSR_VP_INDEX,
1211 	HV_X64_MSR_VP_RUNTIME,
1212 	HV_X64_MSR_SCONTROL,
1213 	HV_X64_MSR_STIMER0_CONFIG,
1214 	HV_X64_MSR_VP_ASSIST_PAGE,
1215 	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
1216 	HV_X64_MSR_TSC_EMULATION_STATUS,
1217 
1218 	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1219 	MSR_KVM_PV_EOI_EN,
1220 
1221 	MSR_IA32_TSC_ADJUST,
1222 	MSR_IA32_TSCDEADLINE,
1223 	MSR_IA32_ARCH_CAPABILITIES,
1224 	MSR_IA32_MISC_ENABLE,
1225 	MSR_IA32_MCG_STATUS,
1226 	MSR_IA32_MCG_CTL,
1227 	MSR_IA32_MCG_EXT_CTL,
1228 	MSR_IA32_SMBASE,
1229 	MSR_SMI_COUNT,
1230 	MSR_PLATFORM_INFO,
1231 	MSR_MISC_FEATURES_ENABLES,
1232 	MSR_AMD64_VIRT_SPEC_CTRL,
1233 	MSR_IA32_POWER_CTL,
1234 	MSR_IA32_UCODE_REV,
1235 
1236 	/*
1237 	 * The following list leaves out MSRs whose values are determined
1238 	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
1239 	 * We always support the "true" VMX control MSRs, even if the host
1240 	 * processor does not, so I am putting these registers here rather
1241 	 * than in msrs_to_save_all.
1242 	 */
1243 	MSR_IA32_VMX_BASIC,
1244 	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1245 	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1246 	MSR_IA32_VMX_TRUE_EXIT_CTLS,
1247 	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1248 	MSR_IA32_VMX_MISC,
1249 	MSR_IA32_VMX_CR0_FIXED0,
1250 	MSR_IA32_VMX_CR4_FIXED0,
1251 	MSR_IA32_VMX_VMCS_ENUM,
1252 	MSR_IA32_VMX_PROCBASED_CTLS2,
1253 	MSR_IA32_VMX_EPT_VPID_CAP,
1254 	MSR_IA32_VMX_VMFUNC,
1255 
1256 	MSR_K7_HWCR,
1257 	MSR_KVM_POLL_CONTROL,
1258 };
1259 
1260 static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1261 static unsigned num_emulated_msrs;
1262 
1263 /*
1264  * List of msr numbers which are used to expose MSR-based features that
1265  * can be used by a hypervisor to validate requested CPU features.
1266  */
1267 static const u32 msr_based_features_all[] = {
1268 	MSR_IA32_VMX_BASIC,
1269 	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1270 	MSR_IA32_VMX_PINBASED_CTLS,
1271 	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1272 	MSR_IA32_VMX_PROCBASED_CTLS,
1273 	MSR_IA32_VMX_TRUE_EXIT_CTLS,
1274 	MSR_IA32_VMX_EXIT_CTLS,
1275 	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1276 	MSR_IA32_VMX_ENTRY_CTLS,
1277 	MSR_IA32_VMX_MISC,
1278 	MSR_IA32_VMX_CR0_FIXED0,
1279 	MSR_IA32_VMX_CR0_FIXED1,
1280 	MSR_IA32_VMX_CR4_FIXED0,
1281 	MSR_IA32_VMX_CR4_FIXED1,
1282 	MSR_IA32_VMX_VMCS_ENUM,
1283 	MSR_IA32_VMX_PROCBASED_CTLS2,
1284 	MSR_IA32_VMX_EPT_VPID_CAP,
1285 	MSR_IA32_VMX_VMFUNC,
1286 
1287 	MSR_F10H_DECFG,
1288 	MSR_IA32_UCODE_REV,
1289 	MSR_IA32_ARCH_CAPABILITIES,
1290 };
1291 
1292 static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1293 static unsigned int num_msr_based_features;
1294 
1295 static u64 kvm_get_arch_capabilities(void)
1296 {
1297 	u64 data = 0;
1298 
1299 	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1300 		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1301 
1302 	/*
1303 	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
1304 	 * the nested hypervisor runs with NX huge pages.  If it is not,
1305 	 * L1 is anyway vulnerable to ITLB_MULTIHIT explots from other
1306 	 * L1 guests, so it need not worry about its own (L2) guests.
1307 	 */
1308 	data |= ARCH_CAP_PSCHANGE_MC_NO;
1309 
1310 	/*
1311 	 * If we're doing cache flushes (either "always" or "cond")
1312 	 * we will do one whenever the guest does a vmlaunch/vmresume.
1313 	 * If an outer hypervisor is doing the cache flush for us
1314 	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
1315 	 * capability to the guest too, and if EPT is disabled we're not
1316 	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
1317 	 * require a nested hypervisor to do a flush of its own.
1318 	 */
1319 	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1320 		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1321 
1322 	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
1323 		data |= ARCH_CAP_RDCL_NO;
1324 	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1325 		data |= ARCH_CAP_SSB_NO;
1326 	if (!boot_cpu_has_bug(X86_BUG_MDS))
1327 		data |= ARCH_CAP_MDS_NO;
1328 
1329 	/*
1330 	 * On TAA affected systems:
1331 	 *      - nothing to do if TSX is disabled on the host.
1332 	 *      - we emulate TSX_CTRL if present on the host.
1333 	 *	  This lets the guest use VERW to clear CPU buffers.
1334 	 */
1335 	if (!boot_cpu_has(X86_FEATURE_RTM))
1336 		data &= ~(ARCH_CAP_TAA_NO | ARCH_CAP_TSX_CTRL_MSR);
1337 	else if (!boot_cpu_has_bug(X86_BUG_TAA))
1338 		data |= ARCH_CAP_TAA_NO;
1339 
1340 	return data;
1341 }
1342 
1343 static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
1344 {
1345 	switch (msr->index) {
1346 	case MSR_IA32_ARCH_CAPABILITIES:
1347 		msr->data = kvm_get_arch_capabilities();
1348 		break;
1349 	case MSR_IA32_UCODE_REV:
1350 		rdmsrl_safe(msr->index, &msr->data);
1351 		break;
1352 	default:
1353 		if (kvm_x86_ops->get_msr_feature(msr))
1354 			return 1;
1355 	}
1356 	return 0;
1357 }
1358 
1359 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1360 {
1361 	struct kvm_msr_entry msr;
1362 	int r;
1363 
1364 	msr.index = index;
1365 	r = kvm_get_msr_feature(&msr);
1366 	if (r)
1367 		return r;
1368 
1369 	*data = msr.data;
1370 
1371 	return 0;
1372 }
1373 
1374 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1375 {
1376 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1377 		return false;
1378 
1379 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1380 		return false;
1381 
1382 	if (efer & (EFER_LME | EFER_LMA) &&
1383 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1384 		return false;
1385 
1386 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1387 		return false;
1388 
1389 	return true;
1390 
1391 }
1392 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1393 {
1394 	if (efer & efer_reserved_bits)
1395 		return false;
1396 
1397 	return __kvm_valid_efer(vcpu, efer);
1398 }
1399 EXPORT_SYMBOL_GPL(kvm_valid_efer);
1400 
1401 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1402 {
1403 	u64 old_efer = vcpu->arch.efer;
1404 	u64 efer = msr_info->data;
1405 
1406 	if (efer & efer_reserved_bits)
1407 		return 1;
1408 
1409 	if (!msr_info->host_initiated) {
1410 		if (!__kvm_valid_efer(vcpu, efer))
1411 			return 1;
1412 
1413 		if (is_paging(vcpu) &&
1414 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1415 			return 1;
1416 	}
1417 
1418 	efer &= ~EFER_LMA;
1419 	efer |= vcpu->arch.efer & EFER_LMA;
1420 
1421 	kvm_x86_ops->set_efer(vcpu, efer);
1422 
1423 	/* Update reserved bits */
1424 	if ((efer ^ old_efer) & EFER_NX)
1425 		kvm_mmu_reset_context(vcpu);
1426 
1427 	return 0;
1428 }
1429 
1430 void kvm_enable_efer_bits(u64 mask)
1431 {
1432        efer_reserved_bits &= ~mask;
1433 }
1434 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1435 
1436 /*
1437  * Write @data into the MSR specified by @index.  Select MSR specific fault
1438  * checks are bypassed if @host_initiated is %true.
1439  * Returns 0 on success, non-0 otherwise.
1440  * Assumes vcpu_load() was already called.
1441  */
1442 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1443 			 bool host_initiated)
1444 {
1445 	struct msr_data msr;
1446 
1447 	switch (index) {
1448 	case MSR_FS_BASE:
1449 	case MSR_GS_BASE:
1450 	case MSR_KERNEL_GS_BASE:
1451 	case MSR_CSTAR:
1452 	case MSR_LSTAR:
1453 		if (is_noncanonical_address(data, vcpu))
1454 			return 1;
1455 		break;
1456 	case MSR_IA32_SYSENTER_EIP:
1457 	case MSR_IA32_SYSENTER_ESP:
1458 		/*
1459 		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
1460 		 * non-canonical address is written on Intel but not on
1461 		 * AMD (which ignores the top 32-bits, because it does
1462 		 * not implement 64-bit SYSENTER).
1463 		 *
1464 		 * 64-bit code should hence be able to write a non-canonical
1465 		 * value on AMD.  Making the address canonical ensures that
1466 		 * vmentry does not fail on Intel after writing a non-canonical
1467 		 * value, and that something deterministic happens if the guest
1468 		 * invokes 64-bit SYSENTER.
1469 		 */
1470 		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1471 	}
1472 
1473 	msr.data = data;
1474 	msr.index = index;
1475 	msr.host_initiated = host_initiated;
1476 
1477 	return kvm_x86_ops->set_msr(vcpu, &msr);
1478 }
1479 
1480 /*
1481  * Read the MSR specified by @index into @data.  Select MSR specific fault
1482  * checks are bypassed if @host_initiated is %true.
1483  * Returns 0 on success, non-0 otherwise.
1484  * Assumes vcpu_load() was already called.
1485  */
1486 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1487 		  bool host_initiated)
1488 {
1489 	struct msr_data msr;
1490 	int ret;
1491 
1492 	msr.index = index;
1493 	msr.host_initiated = host_initiated;
1494 
1495 	ret = kvm_x86_ops->get_msr(vcpu, &msr);
1496 	if (!ret)
1497 		*data = msr.data;
1498 	return ret;
1499 }
1500 
1501 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1502 {
1503 	return __kvm_get_msr(vcpu, index, data, false);
1504 }
1505 EXPORT_SYMBOL_GPL(kvm_get_msr);
1506 
1507 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1508 {
1509 	return __kvm_set_msr(vcpu, index, data, false);
1510 }
1511 EXPORT_SYMBOL_GPL(kvm_set_msr);
1512 
1513 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
1514 {
1515 	u32 ecx = kvm_rcx_read(vcpu);
1516 	u64 data;
1517 
1518 	if (kvm_get_msr(vcpu, ecx, &data)) {
1519 		trace_kvm_msr_read_ex(ecx);
1520 		kvm_inject_gp(vcpu, 0);
1521 		return 1;
1522 	}
1523 
1524 	trace_kvm_msr_read(ecx, data);
1525 
1526 	kvm_rax_write(vcpu, data & -1u);
1527 	kvm_rdx_write(vcpu, (data >> 32) & -1u);
1528 	return kvm_skip_emulated_instruction(vcpu);
1529 }
1530 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
1531 
1532 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
1533 {
1534 	u32 ecx = kvm_rcx_read(vcpu);
1535 	u64 data = kvm_read_edx_eax(vcpu);
1536 
1537 	if (kvm_set_msr(vcpu, ecx, data)) {
1538 		trace_kvm_msr_write_ex(ecx, data);
1539 		kvm_inject_gp(vcpu, 0);
1540 		return 1;
1541 	}
1542 
1543 	trace_kvm_msr_write(ecx, data);
1544 	return kvm_skip_emulated_instruction(vcpu);
1545 }
1546 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
1547 
1548 /*
1549  * The fast path for frequent and performance sensitive wrmsr emulation,
1550  * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
1551  * the latency of virtual IPI by avoiding the expensive bits of transitioning
1552  * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
1553  * other cases which must be called after interrupts are enabled on the host.
1554  */
1555 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
1556 {
1557 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
1558 		return 1;
1559 
1560 	if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
1561 		((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
1562 		((data & APIC_MODE_MASK) == APIC_DM_FIXED)) {
1563 
1564 		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
1565 		return kvm_lapic_reg_write(vcpu->arch.apic, APIC_ICR, (u32)data);
1566 	}
1567 
1568 	return 1;
1569 }
1570 
1571 enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
1572 {
1573 	u32 msr = kvm_rcx_read(vcpu);
1574 	u64 data = kvm_read_edx_eax(vcpu);
1575 	int ret = 0;
1576 
1577 	switch (msr) {
1578 	case APIC_BASE_MSR + (APIC_ICR >> 4):
1579 		ret = handle_fastpath_set_x2apic_icr_irqoff(vcpu, data);
1580 		break;
1581 	default:
1582 		return EXIT_FASTPATH_NONE;
1583 	}
1584 
1585 	if (!ret) {
1586 		trace_kvm_msr_write(msr, data);
1587 		return EXIT_FASTPATH_SKIP_EMUL_INS;
1588 	}
1589 
1590 	return EXIT_FASTPATH_NONE;
1591 }
1592 EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);
1593 
1594 /*
1595  * Adapt set_msr() to msr_io()'s calling convention
1596  */
1597 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1598 {
1599 	return __kvm_get_msr(vcpu, index, data, true);
1600 }
1601 
1602 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1603 {
1604 	return __kvm_set_msr(vcpu, index, *data, true);
1605 }
1606 
1607 #ifdef CONFIG_X86_64
1608 struct pvclock_clock {
1609 	int vclock_mode;
1610 	u64 cycle_last;
1611 	u64 mask;
1612 	u32 mult;
1613 	u32 shift;
1614 	u64 base_cycles;
1615 	u64 offset;
1616 };
1617 
1618 struct pvclock_gtod_data {
1619 	seqcount_t	seq;
1620 
1621 	struct pvclock_clock clock; /* extract of a clocksource struct */
1622 	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
1623 
1624 	ktime_t		offs_boot;
1625 	u64		wall_time_sec;
1626 };
1627 
1628 static struct pvclock_gtod_data pvclock_gtod_data;
1629 
1630 static void update_pvclock_gtod(struct timekeeper *tk)
1631 {
1632 	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
1633 
1634 	write_seqcount_begin(&vdata->seq);
1635 
1636 	/* copy pvclock gtod data */
1637 	vdata->clock.vclock_mode	= tk->tkr_mono.clock->archdata.vclock_mode;
1638 	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
1639 	vdata->clock.mask		= tk->tkr_mono.mask;
1640 	vdata->clock.mult		= tk->tkr_mono.mult;
1641 	vdata->clock.shift		= tk->tkr_mono.shift;
1642 	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
1643 	vdata->clock.offset		= tk->tkr_mono.base;
1644 
1645 	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->archdata.vclock_mode;
1646 	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
1647 	vdata->raw_clock.mask		= tk->tkr_raw.mask;
1648 	vdata->raw_clock.mult		= tk->tkr_raw.mult;
1649 	vdata->raw_clock.shift		= tk->tkr_raw.shift;
1650 	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
1651 	vdata->raw_clock.offset		= tk->tkr_raw.base;
1652 
1653 	vdata->wall_time_sec            = tk->xtime_sec;
1654 
1655 	vdata->offs_boot		= tk->offs_boot;
1656 
1657 	write_seqcount_end(&vdata->seq);
1658 }
1659 
1660 static s64 get_kvmclock_base_ns(void)
1661 {
1662 	/* Count up from boot time, but with the frequency of the raw clock.  */
1663 	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
1664 }
1665 #else
1666 static s64 get_kvmclock_base_ns(void)
1667 {
1668 	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
1669 	return ktime_get_boottime_ns();
1670 }
1671 #endif
1672 
1673 void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
1674 {
1675 	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1676 	kvm_vcpu_kick(vcpu);
1677 }
1678 
1679 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
1680 {
1681 	int version;
1682 	int r;
1683 	struct pvclock_wall_clock wc;
1684 	u64 wall_nsec;
1685 
1686 	if (!wall_clock)
1687 		return;
1688 
1689 	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
1690 	if (r)
1691 		return;
1692 
1693 	if (version & 1)
1694 		++version;  /* first time write, random junk */
1695 
1696 	++version;
1697 
1698 	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
1699 		return;
1700 
1701 	/*
1702 	 * The guest calculates current wall clock time by adding
1703 	 * system time (updated by kvm_guest_time_update below) to the
1704 	 * wall clock specified here.  We do the reverse here.
1705 	 */
1706 	wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
1707 
1708 	wc.nsec = do_div(wall_nsec, 1000000000);
1709 	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
1710 	wc.version = version;
1711 
1712 	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
1713 
1714 	version++;
1715 	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
1716 }
1717 
1718 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
1719 {
1720 	do_shl32_div32(dividend, divisor);
1721 	return dividend;
1722 }
1723 
1724 static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1725 			       s8 *pshift, u32 *pmultiplier)
1726 {
1727 	uint64_t scaled64;
1728 	int32_t  shift = 0;
1729 	uint64_t tps64;
1730 	uint32_t tps32;
1731 
1732 	tps64 = base_hz;
1733 	scaled64 = scaled_hz;
1734 	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1735 		tps64 >>= 1;
1736 		shift--;
1737 	}
1738 
1739 	tps32 = (uint32_t)tps64;
1740 	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
1741 		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1742 			scaled64 >>= 1;
1743 		else
1744 			tps32 <<= 1;
1745 		shift++;
1746 	}
1747 
1748 	*pshift = shift;
1749 	*pmultiplier = div_frac(scaled64, tps32);
1750 }
1751 
1752 #ifdef CONFIG_X86_64
1753 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1754 #endif
1755 
1756 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1757 static unsigned long max_tsc_khz;
1758 
1759 static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1760 {
1761 	u64 v = (u64)khz * (1000000 + ppm);
1762 	do_div(v, 1000000);
1763 	return v;
1764 }
1765 
1766 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
1767 {
1768 	u64 ratio;
1769 
1770 	/* Guest TSC same frequency as host TSC? */
1771 	if (!scale) {
1772 		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1773 		return 0;
1774 	}
1775 
1776 	/* TSC scaling supported? */
1777 	if (!kvm_has_tsc_control) {
1778 		if (user_tsc_khz > tsc_khz) {
1779 			vcpu->arch.tsc_catchup = 1;
1780 			vcpu->arch.tsc_always_catchup = 1;
1781 			return 0;
1782 		} else {
1783 			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
1784 			return -1;
1785 		}
1786 	}
1787 
1788 	/* TSC scaling required  - calculate ratio */
1789 	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
1790 				user_tsc_khz, tsc_khz);
1791 
1792 	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
1793 		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
1794 			            user_tsc_khz);
1795 		return -1;
1796 	}
1797 
1798 	vcpu->arch.tsc_scaling_ratio = ratio;
1799 	return 0;
1800 }
1801 
1802 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1803 {
1804 	u32 thresh_lo, thresh_hi;
1805 	int use_scaling = 0;
1806 
1807 	/* tsc_khz can be zero if TSC calibration fails */
1808 	if (user_tsc_khz == 0) {
1809 		/* set tsc_scaling_ratio to a safe value */
1810 		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1811 		return -1;
1812 	}
1813 
1814 	/* Compute a scale to convert nanoseconds in TSC cycles */
1815 	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1816 			   &vcpu->arch.virtual_tsc_shift,
1817 			   &vcpu->arch.virtual_tsc_mult);
1818 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1819 
1820 	/*
1821 	 * Compute the variation in TSC rate which is acceptable
1822 	 * within the range of tolerance and decide if the
1823 	 * rate being applied is within that bounds of the hardware
1824 	 * rate.  If so, no scaling or compensation need be done.
1825 	 */
1826 	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
1827 	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1828 	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
1829 		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1830 		use_scaling = 1;
1831 	}
1832 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
1833 }
1834 
1835 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
1836 {
1837 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1838 				      vcpu->arch.virtual_tsc_mult,
1839 				      vcpu->arch.virtual_tsc_shift);
1840 	tsc += vcpu->arch.this_tsc_write;
1841 	return tsc;
1842 }
1843 
1844 static inline int gtod_is_based_on_tsc(int mode)
1845 {
1846 	return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK;
1847 }
1848 
1849 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1850 {
1851 #ifdef CONFIG_X86_64
1852 	bool vcpus_matched;
1853 	struct kvm_arch *ka = &vcpu->kvm->arch;
1854 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1855 
1856 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
1857 			 atomic_read(&vcpu->kvm->online_vcpus));
1858 
1859 	/*
1860 	 * Once the masterclock is enabled, always perform request in
1861 	 * order to update it.
1862 	 *
1863 	 * In order to enable masterclock, the host clocksource must be TSC
1864 	 * and the vcpus need to have matched TSCs.  When that happens,
1865 	 * perform request to enable masterclock.
1866 	 */
1867 	if (ka->use_master_clock ||
1868 	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1869 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
1870 
1871 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
1872 			    atomic_read(&vcpu->kvm->online_vcpus),
1873 		            ka->use_master_clock, gtod->clock.vclock_mode);
1874 #endif
1875 }
1876 
1877 static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
1878 {
1879 	u64 curr_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
1880 	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
1881 }
1882 
1883 /*
1884  * Multiply tsc by a fixed point number represented by ratio.
1885  *
1886  * The most significant 64-N bits (mult) of ratio represent the
1887  * integral part of the fixed point number; the remaining N bits
1888  * (frac) represent the fractional part, ie. ratio represents a fixed
1889  * point number (mult + frac * 2^(-N)).
1890  *
1891  * N equals to kvm_tsc_scaling_ratio_frac_bits.
1892  */
1893 static inline u64 __scale_tsc(u64 ratio, u64 tsc)
1894 {
1895 	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
1896 }
1897 
1898 u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
1899 {
1900 	u64 _tsc = tsc;
1901 	u64 ratio = vcpu->arch.tsc_scaling_ratio;
1902 
1903 	if (ratio != kvm_default_tsc_scaling_ratio)
1904 		_tsc = __scale_tsc(ratio, tsc);
1905 
1906 	return _tsc;
1907 }
1908 EXPORT_SYMBOL_GPL(kvm_scale_tsc);
1909 
1910 static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
1911 {
1912 	u64 tsc;
1913 
1914 	tsc = kvm_scale_tsc(vcpu, rdtsc());
1915 
1916 	return target_tsc - tsc;
1917 }
1918 
1919 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
1920 {
1921 	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
1922 
1923 	return tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1924 }
1925 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
1926 
1927 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1928 {
1929 	vcpu->arch.tsc_offset = kvm_x86_ops->write_l1_tsc_offset(vcpu, offset);
1930 }
1931 
1932 static inline bool kvm_check_tsc_unstable(void)
1933 {
1934 #ifdef CONFIG_X86_64
1935 	/*
1936 	 * TSC is marked unstable when we're running on Hyper-V,
1937 	 * 'TSC page' clocksource is good.
1938 	 */
1939 	if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK)
1940 		return false;
1941 #endif
1942 	return check_tsc_unstable();
1943 }
1944 
1945 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1946 {
1947 	struct kvm *kvm = vcpu->kvm;
1948 	u64 offset, ns, elapsed;
1949 	unsigned long flags;
1950 	bool matched;
1951 	bool already_matched;
1952 	u64 data = msr->data;
1953 	bool synchronizing = false;
1954 
1955 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1956 	offset = kvm_compute_tsc_offset(vcpu, data);
1957 	ns = get_kvmclock_base_ns();
1958 	elapsed = ns - kvm->arch.last_tsc_nsec;
1959 
1960 	if (vcpu->arch.virtual_tsc_khz) {
1961 		if (data == 0 && msr->host_initiated) {
1962 			/*
1963 			 * detection of vcpu initialization -- need to sync
1964 			 * with other vCPUs. This particularly helps to keep
1965 			 * kvm_clock stable after CPU hotplug
1966 			 */
1967 			synchronizing = true;
1968 		} else {
1969 			u64 tsc_exp = kvm->arch.last_tsc_write +
1970 						nsec_to_cycles(vcpu, elapsed);
1971 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
1972 			/*
1973 			 * Special case: TSC write with a small delta (1 second)
1974 			 * of virtual cycle time against real time is
1975 			 * interpreted as an attempt to synchronize the CPU.
1976 			 */
1977 			synchronizing = data < tsc_exp + tsc_hz &&
1978 					data + tsc_hz > tsc_exp;
1979 		}
1980 	}
1981 
1982 	/*
1983 	 * For a reliable TSC, we can match TSC offsets, and for an unstable
1984 	 * TSC, we add elapsed time in this computation.  We could let the
1985 	 * compensation code attempt to catch up if we fall behind, but
1986 	 * it's better to try to match offsets from the beginning.
1987          */
1988 	if (synchronizing &&
1989 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
1990 		if (!kvm_check_tsc_unstable()) {
1991 			offset = kvm->arch.cur_tsc_offset;
1992 		} else {
1993 			u64 delta = nsec_to_cycles(vcpu, elapsed);
1994 			data += delta;
1995 			offset = kvm_compute_tsc_offset(vcpu, data);
1996 		}
1997 		matched = true;
1998 		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
1999 	} else {
2000 		/*
2001 		 * We split periods of matched TSC writes into generations.
2002 		 * For each generation, we track the original measured
2003 		 * nanosecond time, offset, and write, so if TSCs are in
2004 		 * sync, we can match exact offset, and if not, we can match
2005 		 * exact software computation in compute_guest_tsc()
2006 		 *
2007 		 * These values are tracked in kvm->arch.cur_xxx variables.
2008 		 */
2009 		kvm->arch.cur_tsc_generation++;
2010 		kvm->arch.cur_tsc_nsec = ns;
2011 		kvm->arch.cur_tsc_write = data;
2012 		kvm->arch.cur_tsc_offset = offset;
2013 		matched = false;
2014 	}
2015 
2016 	/*
2017 	 * We also track th most recent recorded KHZ, write and time to
2018 	 * allow the matching interval to be extended at each write.
2019 	 */
2020 	kvm->arch.last_tsc_nsec = ns;
2021 	kvm->arch.last_tsc_write = data;
2022 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2023 
2024 	vcpu->arch.last_guest_tsc = data;
2025 
2026 	/* Keep track of which generation this VCPU has synchronized to */
2027 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2028 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2029 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2030 
2031 	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
2032 		update_ia32_tsc_adjust_msr(vcpu, offset);
2033 
2034 	kvm_vcpu_write_tsc_offset(vcpu, offset);
2035 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2036 
2037 	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
2038 	if (!matched) {
2039 		kvm->arch.nr_vcpus_matched_tsc = 0;
2040 	} else if (!already_matched) {
2041 		kvm->arch.nr_vcpus_matched_tsc++;
2042 	}
2043 
2044 	kvm_track_tsc_matching(vcpu);
2045 	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
2046 }
2047 
2048 EXPORT_SYMBOL_GPL(kvm_write_tsc);
2049 
2050 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2051 					   s64 adjustment)
2052 {
2053 	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
2054 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2055 }
2056 
2057 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2058 {
2059 	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
2060 		WARN_ON(adjustment < 0);
2061 	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
2062 	adjust_tsc_offset_guest(vcpu, adjustment);
2063 }
2064 
2065 #ifdef CONFIG_X86_64
2066 
2067 static u64 read_tsc(void)
2068 {
2069 	u64 ret = (u64)rdtsc_ordered();
2070 	u64 last = pvclock_gtod_data.clock.cycle_last;
2071 
2072 	if (likely(ret >= last))
2073 		return ret;
2074 
2075 	/*
2076 	 * GCC likes to generate cmov here, but this branch is extremely
2077 	 * predictable (it's just a function of time and the likely is
2078 	 * very likely) and there's a data dependence, so force GCC
2079 	 * to generate a branch instead.  I don't barrier() because
2080 	 * we don't actually need a barrier, and if this function
2081 	 * ever gets inlined it will generate worse code.
2082 	 */
2083 	asm volatile ("");
2084 	return last;
2085 }
2086 
2087 static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
2088 			  int *mode)
2089 {
2090 	long v;
2091 	u64 tsc_pg_val;
2092 
2093 	switch (clock->vclock_mode) {
2094 	case VCLOCK_HVCLOCK:
2095 		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
2096 						  tsc_timestamp);
2097 		if (tsc_pg_val != U64_MAX) {
2098 			/* TSC page valid */
2099 			*mode = VCLOCK_HVCLOCK;
2100 			v = (tsc_pg_val - clock->cycle_last) &
2101 				clock->mask;
2102 		} else {
2103 			/* TSC page invalid */
2104 			*mode = VCLOCK_NONE;
2105 		}
2106 		break;
2107 	case VCLOCK_TSC:
2108 		*mode = VCLOCK_TSC;
2109 		*tsc_timestamp = read_tsc();
2110 		v = (*tsc_timestamp - clock->cycle_last) &
2111 			clock->mask;
2112 		break;
2113 	default:
2114 		*mode = VCLOCK_NONE;
2115 	}
2116 
2117 	if (*mode == VCLOCK_NONE)
2118 		*tsc_timestamp = v = 0;
2119 
2120 	return v * clock->mult;
2121 }
2122 
2123 static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
2124 {
2125 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2126 	unsigned long seq;
2127 	int mode;
2128 	u64 ns;
2129 
2130 	do {
2131 		seq = read_seqcount_begin(&gtod->seq);
2132 		ns = gtod->raw_clock.base_cycles;
2133 		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
2134 		ns >>= gtod->raw_clock.shift;
2135 		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
2136 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2137 	*t = ns;
2138 
2139 	return mode;
2140 }
2141 
2142 static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2143 {
2144 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2145 	unsigned long seq;
2146 	int mode;
2147 	u64 ns;
2148 
2149 	do {
2150 		seq = read_seqcount_begin(&gtod->seq);
2151 		ts->tv_sec = gtod->wall_time_sec;
2152 		ns = gtod->clock.base_cycles;
2153 		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
2154 		ns >>= gtod->clock.shift;
2155 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2156 
2157 	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
2158 	ts->tv_nsec = ns;
2159 
2160 	return mode;
2161 }
2162 
2163 /* returns true if host is using TSC based clocksource */
2164 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2165 {
2166 	/* checked again under seqlock below */
2167 	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2168 		return false;
2169 
2170 	return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
2171 						      tsc_timestamp));
2172 }
2173 
2174 /* returns true if host is using TSC based clocksource */
2175 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2176 					   u64 *tsc_timestamp)
2177 {
2178 	/* checked again under seqlock below */
2179 	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2180 		return false;
2181 
2182 	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2183 }
2184 #endif
2185 
2186 /*
2187  *
2188  * Assuming a stable TSC across physical CPUS, and a stable TSC
2189  * across virtual CPUs, the following condition is possible.
2190  * Each numbered line represents an event visible to both
2191  * CPUs at the next numbered event.
2192  *
2193  * "timespecX" represents host monotonic time. "tscX" represents
2194  * RDTSC value.
2195  *
2196  * 		VCPU0 on CPU0		|	VCPU1 on CPU1
2197  *
2198  * 1.  read timespec0,tsc0
2199  * 2.					| timespec1 = timespec0 + N
2200  * 					| tsc1 = tsc0 + M
2201  * 3. transition to guest		| transition to guest
2202  * 4. ret0 = timespec0 + (rdtsc - tsc0) |
2203  * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
2204  * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
2205  *
2206  * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
2207  *
2208  * 	- ret0 < ret1
2209  *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
2210  *		...
2211  *	- 0 < N - M => M < N
2212  *
2213  * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
2214  * always the case (the difference between two distinct xtime instances
2215  * might be smaller then the difference between corresponding TSC reads,
2216  * when updating guest vcpus pvclock areas).
2217  *
2218  * To avoid that problem, do not allow visibility of distinct
2219  * system_timestamp/tsc_timestamp values simultaneously: use a master
2220  * copy of host monotonic time values. Update that master copy
2221  * in lockstep.
2222  *
2223  * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2224  *
2225  */
2226 
2227 static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
2228 {
2229 #ifdef CONFIG_X86_64
2230 	struct kvm_arch *ka = &kvm->arch;
2231 	int vclock_mode;
2232 	bool host_tsc_clocksource, vcpus_matched;
2233 
2234 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2235 			atomic_read(&kvm->online_vcpus));
2236 
2237 	/*
2238 	 * If the host uses TSC clock, then passthrough TSC as stable
2239 	 * to the guest.
2240 	 */
2241 	host_tsc_clocksource = kvm_get_time_and_clockread(
2242 					&ka->master_kernel_ns,
2243 					&ka->master_cycle_now);
2244 
2245 	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2246 				&& !ka->backwards_tsc_observed
2247 				&& !ka->boot_vcpu_runs_old_kvmclock;
2248 
2249 	if (ka->use_master_clock)
2250 		atomic_set(&kvm_guest_has_master_clock, 1);
2251 
2252 	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2253 	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
2254 					vcpus_matched);
2255 #endif
2256 }
2257 
2258 void kvm_make_mclock_inprogress_request(struct kvm *kvm)
2259 {
2260 	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
2261 }
2262 
2263 static void kvm_gen_update_masterclock(struct kvm *kvm)
2264 {
2265 #ifdef CONFIG_X86_64
2266 	int i;
2267 	struct kvm_vcpu *vcpu;
2268 	struct kvm_arch *ka = &kvm->arch;
2269 
2270 	spin_lock(&ka->pvclock_gtod_sync_lock);
2271 	kvm_make_mclock_inprogress_request(kvm);
2272 	/* no guest entries from this point */
2273 	pvclock_update_vm_gtod_copy(kvm);
2274 
2275 	kvm_for_each_vcpu(i, vcpu, kvm)
2276 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2277 
2278 	/* guest entries allowed */
2279 	kvm_for_each_vcpu(i, vcpu, kvm)
2280 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2281 
2282 	spin_unlock(&ka->pvclock_gtod_sync_lock);
2283 #endif
2284 }
2285 
2286 u64 get_kvmclock_ns(struct kvm *kvm)
2287 {
2288 	struct kvm_arch *ka = &kvm->arch;
2289 	struct pvclock_vcpu_time_info hv_clock;
2290 	u64 ret;
2291 
2292 	spin_lock(&ka->pvclock_gtod_sync_lock);
2293 	if (!ka->use_master_clock) {
2294 		spin_unlock(&ka->pvclock_gtod_sync_lock);
2295 		return get_kvmclock_base_ns() + ka->kvmclock_offset;
2296 	}
2297 
2298 	hv_clock.tsc_timestamp = ka->master_cycle_now;
2299 	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
2300 	spin_unlock(&ka->pvclock_gtod_sync_lock);
2301 
2302 	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
2303 	get_cpu();
2304 
2305 	if (__this_cpu_read(cpu_tsc_khz)) {
2306 		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
2307 				   &hv_clock.tsc_shift,
2308 				   &hv_clock.tsc_to_system_mul);
2309 		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
2310 	} else
2311 		ret = get_kvmclock_base_ns() + ka->kvmclock_offset;
2312 
2313 	put_cpu();
2314 
2315 	return ret;
2316 }
2317 
2318 static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
2319 {
2320 	struct kvm_vcpu_arch *vcpu = &v->arch;
2321 	struct pvclock_vcpu_time_info guest_hv_clock;
2322 
2323 	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2324 		&guest_hv_clock, sizeof(guest_hv_clock))))
2325 		return;
2326 
2327 	/* This VCPU is paused, but it's legal for a guest to read another
2328 	 * VCPU's kvmclock, so we really have to follow the specification where
2329 	 * it says that version is odd if data is being modified, and even after
2330 	 * it is consistent.
2331 	 *
2332 	 * Version field updates must be kept separate.  This is because
2333 	 * kvm_write_guest_cached might use a "rep movs" instruction, and
2334 	 * writes within a string instruction are weakly ordered.  So there
2335 	 * are three writes overall.
2336 	 *
2337 	 * As a small optimization, only write the version field in the first
2338 	 * and third write.  The vcpu->pv_time cache is still valid, because the
2339 	 * version field is the first in the struct.
2340 	 */
2341 	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);
2342 
2343 	if (guest_hv_clock.version & 1)
2344 		++guest_hv_clock.version;  /* first time write, random junk */
2345 
2346 	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2347 	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2348 				&vcpu->hv_clock,
2349 				sizeof(vcpu->hv_clock.version));
2350 
2351 	smp_wmb();
2352 
2353 	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
2354 	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
2355 
2356 	if (vcpu->pvclock_set_guest_stopped_request) {
2357 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
2358 		vcpu->pvclock_set_guest_stopped_request = false;
2359 	}
2360 
2361 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
2362 
2363 	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2364 				&vcpu->hv_clock,
2365 				sizeof(vcpu->hv_clock));
2366 
2367 	smp_wmb();
2368 
2369 	vcpu->hv_clock.version++;
2370 	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2371 				&vcpu->hv_clock,
2372 				sizeof(vcpu->hv_clock.version));
2373 }
2374 
2375 static int kvm_guest_time_update(struct kvm_vcpu *v)
2376 {
2377 	unsigned long flags, tgt_tsc_khz;
2378 	struct kvm_vcpu_arch *vcpu = &v->arch;
2379 	struct kvm_arch *ka = &v->kvm->arch;
2380 	s64 kernel_ns;
2381 	u64 tsc_timestamp, host_tsc;
2382 	u8 pvclock_flags;
2383 	bool use_master_clock;
2384 
2385 	kernel_ns = 0;
2386 	host_tsc = 0;
2387 
2388 	/*
2389 	 * If the host uses TSC clock, then passthrough TSC as stable
2390 	 * to the guest.
2391 	 */
2392 	spin_lock(&ka->pvclock_gtod_sync_lock);
2393 	use_master_clock = ka->use_master_clock;
2394 	if (use_master_clock) {
2395 		host_tsc = ka->master_cycle_now;
2396 		kernel_ns = ka->master_kernel_ns;
2397 	}
2398 	spin_unlock(&ka->pvclock_gtod_sync_lock);
2399 
2400 	/* Keep irq disabled to prevent changes to the clock */
2401 	local_irq_save(flags);
2402 	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
2403 	if (unlikely(tgt_tsc_khz == 0)) {
2404 		local_irq_restore(flags);
2405 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2406 		return 1;
2407 	}
2408 	if (!use_master_clock) {
2409 		host_tsc = rdtsc();
2410 		kernel_ns = get_kvmclock_base_ns();
2411 	}
2412 
2413 	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2414 
2415 	/*
2416 	 * We may have to catch up the TSC to match elapsed wall clock
2417 	 * time for two reasons, even if kvmclock is used.
2418 	 *   1) CPU could have been running below the maximum TSC rate
2419 	 *   2) Broken TSC compensation resets the base at each VCPU
2420 	 *      entry to avoid unknown leaps of TSC even when running
2421 	 *      again on the same CPU.  This may cause apparent elapsed
2422 	 *      time to disappear, and the guest to stand still or run
2423 	 *	very slowly.
2424 	 */
2425 	if (vcpu->tsc_catchup) {
2426 		u64 tsc = compute_guest_tsc(v, kernel_ns);
2427 		if (tsc > tsc_timestamp) {
2428 			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
2429 			tsc_timestamp = tsc;
2430 		}
2431 	}
2432 
2433 	local_irq_restore(flags);
2434 
2435 	/* With all the info we got, fill in the values */
2436 
2437 	if (kvm_has_tsc_control)
2438 		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);
2439 
2440 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2441 		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2442 				   &vcpu->hv_clock.tsc_shift,
2443 				   &vcpu->hv_clock.tsc_to_system_mul);
2444 		vcpu->hw_tsc_khz = tgt_tsc_khz;
2445 	}
2446 
2447 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2448 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
2449 	vcpu->last_guest_tsc = tsc_timestamp;
2450 
2451 	/* If the host uses TSC clocksource, then it is stable */
2452 	pvclock_flags = 0;
2453 	if (use_master_clock)
2454 		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
2455 
2456 	vcpu->hv_clock.flags = pvclock_flags;
2457 
2458 	if (vcpu->pv_time_enabled)
2459 		kvm_setup_pvclock_page(v);
2460 	if (v == kvm_get_vcpu(v->kvm, 0))
2461 		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2462 	return 0;
2463 }
2464 
2465 /*
2466  * kvmclock updates which are isolated to a given vcpu, such as
2467  * vcpu->cpu migration, should not allow system_timestamp from
2468  * the rest of the vcpus to remain static. Otherwise ntp frequency
2469  * correction applies to one vcpu's system_timestamp but not
2470  * the others.
2471  *
2472  * So in those cases, request a kvmclock update for all vcpus.
2473  * We need to rate-limit these requests though, as they can
2474  * considerably slow guests that have a large number of vcpus.
2475  * The time for a remote vcpu to update its kvmclock is bound
2476  * by the delay we use to rate-limit the updates.
2477  */
2478 
2479 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
2480 
2481 static void kvmclock_update_fn(struct work_struct *work)
2482 {
2483 	int i;
2484 	struct delayed_work *dwork = to_delayed_work(work);
2485 	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
2486 					   kvmclock_update_work);
2487 	struct kvm *kvm = container_of(ka, struct kvm, arch);
2488 	struct kvm_vcpu *vcpu;
2489 
2490 	kvm_for_each_vcpu(i, vcpu, kvm) {
2491 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2492 		kvm_vcpu_kick(vcpu);
2493 	}
2494 }
2495 
2496 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
2497 {
2498 	struct kvm *kvm = v->kvm;
2499 
2500 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2501 	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
2502 					KVMCLOCK_UPDATE_DELAY);
2503 }
2504 
2505 #define KVMCLOCK_SYNC_PERIOD (300 * HZ)
2506 
2507 static void kvmclock_sync_fn(struct work_struct *work)
2508 {
2509 	struct delayed_work *dwork = to_delayed_work(work);
2510 	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
2511 					   kvmclock_sync_work);
2512 	struct kvm *kvm = container_of(ka, struct kvm, arch);
2513 
2514 	if (!kvmclock_periodic_sync)
2515 		return;
2516 
2517 	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
2518 	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
2519 					KVMCLOCK_SYNC_PERIOD);
2520 }
2521 
2522 /*
2523  * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
2524  */
2525 static bool can_set_mci_status(struct kvm_vcpu *vcpu)
2526 {
2527 	/* McStatusWrEn enabled? */
2528 	if (guest_cpuid_is_amd(vcpu))
2529 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
2530 
2531 	return false;
2532 }
2533 
2534 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2535 {
2536 	u64 mcg_cap = vcpu->arch.mcg_cap;
2537 	unsigned bank_num = mcg_cap & 0xff;
2538 	u32 msr = msr_info->index;
2539 	u64 data = msr_info->data;
2540 
2541 	switch (msr) {
2542 	case MSR_IA32_MCG_STATUS:
2543 		vcpu->arch.mcg_status = data;
2544 		break;
2545 	case MSR_IA32_MCG_CTL:
2546 		if (!(mcg_cap & MCG_CTL_P) &&
2547 		    (data || !msr_info->host_initiated))
2548 			return 1;
2549 		if (data != 0 && data != ~(u64)0)
2550 			return 1;
2551 		vcpu->arch.mcg_ctl = data;
2552 		break;
2553 	default:
2554 		if (msr >= MSR_IA32_MC0_CTL &&
2555 		    msr < MSR_IA32_MCx_CTL(bank_num)) {
2556 			u32 offset = array_index_nospec(
2557 				msr - MSR_IA32_MC0_CTL,
2558 				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);
2559 
2560 			/* only 0 or all 1s can be written to IA32_MCi_CTL
2561 			 * some Linux kernels though clear bit 10 in bank 4 to
2562 			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
2563 			 * this to avoid an uncatched #GP in the guest
2564 			 */
2565 			if ((offset & 0x3) == 0 &&
2566 			    data != 0 && (data | (1 << 10)) != ~(u64)0)
2567 				return -1;
2568 
2569 			/* MCi_STATUS */
2570 			if (!msr_info->host_initiated &&
2571 			    (offset & 0x3) == 1 && data != 0) {
2572 				if (!can_set_mci_status(vcpu))
2573 					return -1;
2574 			}
2575 
2576 			vcpu->arch.mce_banks[offset] = data;
2577 			break;
2578 		}
2579 		return 1;
2580 	}
2581 	return 0;
2582 }
2583 
2584 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
2585 {
2586 	struct kvm *kvm = vcpu->kvm;
2587 	int lm = is_long_mode(vcpu);
2588 	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
2589 		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
2590 	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
2591 		: kvm->arch.xen_hvm_config.blob_size_32;
2592 	u32 page_num = data & ~PAGE_MASK;
2593 	u64 page_addr = data & PAGE_MASK;
2594 	u8 *page;
2595 	int r;
2596 
2597 	r = -E2BIG;
2598 	if (page_num >= blob_size)
2599 		goto out;
2600 	r = -ENOMEM;
2601 	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
2602 	if (IS_ERR(page)) {
2603 		r = PTR_ERR(page);
2604 		goto out;
2605 	}
2606 	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
2607 		goto out_free;
2608 	r = 0;
2609 out_free:
2610 	kfree(page);
2611 out:
2612 	return r;
2613 }
2614 
2615 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
2616 {
2617 	gpa_t gpa = data & ~0x3f;
2618 
2619 	/* Bits 3:5 are reserved, Should be zero */
2620 	if (data & 0x38)
2621 		return 1;
2622 
2623 	vcpu->arch.apf.msr_val = data;
2624 
2625 	if (!(data & KVM_ASYNC_PF_ENABLED)) {
2626 		kvm_clear_async_pf_completion_queue(vcpu);
2627 		kvm_async_pf_hash_reset(vcpu);
2628 		return 0;
2629 	}
2630 
2631 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2632 					sizeof(u32)))
2633 		return 1;
2634 
2635 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2636 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2637 	kvm_async_pf_wakeup_all(vcpu);
2638 	return 0;
2639 }
2640 
2641 static void kvmclock_reset(struct kvm_vcpu *vcpu)
2642 {
2643 	vcpu->arch.pv_time_enabled = false;
2644 	vcpu->arch.time = 0;
2645 }
2646 
2647 static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
2648 {
2649 	++vcpu->stat.tlb_flush;
2650 	kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa);
2651 }
2652 
2653 static void record_steal_time(struct kvm_vcpu *vcpu)
2654 {
2655 	struct kvm_host_map map;
2656 	struct kvm_steal_time *st;
2657 
2658 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
2659 		return;
2660 
2661 	/* -EAGAIN is returned in atomic context so we can just return. */
2662 	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT,
2663 			&map, &vcpu->arch.st.cache, false))
2664 		return;
2665 
2666 	st = map.hva +
2667 		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
2668 
2669 	/*
2670 	 * Doing a TLB flush here, on the guest's behalf, can avoid
2671 	 * expensive IPIs.
2672 	 */
2673 	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
2674 		st->preempted & KVM_VCPU_FLUSH_TLB);
2675 	if (xchg(&st->preempted, 0) & KVM_VCPU_FLUSH_TLB)
2676 		kvm_vcpu_flush_tlb(vcpu, false);
2677 
2678 	vcpu->arch.st.preempted = 0;
2679 
2680 	if (st->version & 1)
2681 		st->version += 1;  /* first time write, random junk */
2682 
2683 	st->version += 1;
2684 
2685 	smp_wmb();
2686 
2687 	st->steal += current->sched_info.run_delay -
2688 		vcpu->arch.st.last_steal;
2689 	vcpu->arch.st.last_steal = current->sched_info.run_delay;
2690 
2691 	smp_wmb();
2692 
2693 	st->version += 1;
2694 
2695 	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false);
2696 }
2697 
2698 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2699 {
2700 	bool pr = false;
2701 	u32 msr = msr_info->index;
2702 	u64 data = msr_info->data;
2703 
2704 	switch (msr) {
2705 	case MSR_AMD64_NB_CFG:
2706 	case MSR_IA32_UCODE_WRITE:
2707 	case MSR_VM_HSAVE_PA:
2708 	case MSR_AMD64_PATCH_LOADER:
2709 	case MSR_AMD64_BU_CFG2:
2710 	case MSR_AMD64_DC_CFG:
2711 	case MSR_F15H_EX_CFG:
2712 		break;
2713 
2714 	case MSR_IA32_UCODE_REV:
2715 		if (msr_info->host_initiated)
2716 			vcpu->arch.microcode_version = data;
2717 		break;
2718 	case MSR_IA32_ARCH_CAPABILITIES:
2719 		if (!msr_info->host_initiated)
2720 			return 1;
2721 		vcpu->arch.arch_capabilities = data;
2722 		break;
2723 	case MSR_EFER:
2724 		return set_efer(vcpu, msr_info);
2725 	case MSR_K7_HWCR:
2726 		data &= ~(u64)0x40;	/* ignore flush filter disable */
2727 		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2728 		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2729 
2730 		/* Handle McStatusWrEn */
2731 		if (data == BIT_ULL(18)) {
2732 			vcpu->arch.msr_hwcr = data;
2733 		} else if (data != 0) {
2734 			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
2735 				    data);
2736 			return 1;
2737 		}
2738 		break;
2739 	case MSR_FAM10H_MMIO_CONF_BASE:
2740 		if (data != 0) {
2741 			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
2742 				    "0x%llx\n", data);
2743 			return 1;
2744 		}
2745 		break;
2746 	case MSR_IA32_DEBUGCTLMSR:
2747 		if (!data) {
2748 			/* We support the non-activated case already */
2749 			break;
2750 		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
2751 			/* Values other than LBR and BTF are vendor-specific,
2752 			   thus reserved and should throw a #GP */
2753 			return 1;
2754 		}
2755 		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
2756 			    __func__, data);
2757 		break;
2758 	case 0x200 ... 0x2ff:
2759 		return kvm_mtrr_set_msr(vcpu, msr, data);
2760 	case MSR_IA32_APICBASE:
2761 		return kvm_set_apic_base(vcpu, msr_info);
2762 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
2763 		return kvm_x2apic_msr_write(vcpu, msr, data);
2764 	case MSR_IA32_TSCDEADLINE:
2765 		kvm_set_lapic_tscdeadline_msr(vcpu, data);
2766 		break;
2767 	case MSR_IA32_TSC_ADJUST:
2768 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
2769 			if (!msr_info->host_initiated) {
2770 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2771 				adjust_tsc_offset_guest(vcpu, adj);
2772 			}
2773 			vcpu->arch.ia32_tsc_adjust_msr = data;
2774 		}
2775 		break;
2776 	case MSR_IA32_MISC_ENABLE:
2777 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
2778 		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
2779 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
2780 				return 1;
2781 			vcpu->arch.ia32_misc_enable_msr = data;
2782 			kvm_update_cpuid(vcpu);
2783 		} else {
2784 			vcpu->arch.ia32_misc_enable_msr = data;
2785 		}
2786 		break;
2787 	case MSR_IA32_SMBASE:
2788 		if (!msr_info->host_initiated)
2789 			return 1;
2790 		vcpu->arch.smbase = data;
2791 		break;
2792 	case MSR_IA32_POWER_CTL:
2793 		vcpu->arch.msr_ia32_power_ctl = data;
2794 		break;
2795 	case MSR_IA32_TSC:
2796 		kvm_write_tsc(vcpu, msr_info);
2797 		break;
2798 	case MSR_IA32_XSS:
2799 		if (!msr_info->host_initiated &&
2800 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
2801 			return 1;
2802 		/*
2803 		 * We do support PT if kvm_x86_ops->pt_supported(), but we do
2804 		 * not support IA32_XSS[bit 8]. Guests will have to use
2805 		 * RDMSR/WRMSR rather than XSAVES/XRSTORS to save/restore PT
2806 		 * MSRs.
2807 		 */
2808 		if (data != 0)
2809 			return 1;
2810 		vcpu->arch.ia32_xss = data;
2811 		break;
2812 	case MSR_SMI_COUNT:
2813 		if (!msr_info->host_initiated)
2814 			return 1;
2815 		vcpu->arch.smi_count = data;
2816 		break;
2817 	case MSR_KVM_WALL_CLOCK_NEW:
2818 	case MSR_KVM_WALL_CLOCK:
2819 		vcpu->kvm->arch.wall_clock = data;
2820 		kvm_write_wall_clock(vcpu->kvm, data);
2821 		break;
2822 	case MSR_KVM_SYSTEM_TIME_NEW:
2823 	case MSR_KVM_SYSTEM_TIME: {
2824 		struct kvm_arch *ka = &vcpu->kvm->arch;
2825 
2826 		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
2827 			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);
2828 
2829 			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2830 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2831 
2832 			ka->boot_vcpu_runs_old_kvmclock = tmp;
2833 		}
2834 
2835 		vcpu->arch.time = data;
2836 		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2837 
2838 		/* we verify if the enable bit is set... */
2839 		vcpu->arch.pv_time_enabled = false;
2840 		if (!(data & 1))
2841 			break;
2842 
2843 		if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2844 		     &vcpu->arch.pv_time, data & ~1ULL,
2845 		     sizeof(struct pvclock_vcpu_time_info)))
2846 			vcpu->arch.pv_time_enabled = true;
2847 
2848 		break;
2849 	}
2850 	case MSR_KVM_ASYNC_PF_EN:
2851 		if (kvm_pv_enable_async_pf(vcpu, data))
2852 			return 1;
2853 		break;
2854 	case MSR_KVM_STEAL_TIME:
2855 
2856 		if (unlikely(!sched_info_on()))
2857 			return 1;
2858 
2859 		if (data & KVM_STEAL_RESERVED_MASK)
2860 			return 1;
2861 
2862 		vcpu->arch.st.msr_val = data;
2863 
2864 		if (!(data & KVM_MSR_ENABLED))
2865 			break;
2866 
2867 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
2868 
2869 		break;
2870 	case MSR_KVM_PV_EOI_EN:
2871 		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2872 			return 1;
2873 		break;
2874 
2875 	case MSR_KVM_POLL_CONTROL:
2876 		/* only enable bit supported */
2877 		if (data & (-1ULL << 1))
2878 			return 1;
2879 
2880 		vcpu->arch.msr_kvm_poll_control = data;
2881 		break;
2882 
2883 	case MSR_IA32_MCG_CTL:
2884 	case MSR_IA32_MCG_STATUS:
2885 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2886 		return set_msr_mce(vcpu, msr_info);
2887 
2888 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
2889 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
2890 		pr = true; /* fall through */
2891 	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
2892 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2893 		if (kvm_pmu_is_valid_msr(vcpu, msr))
2894 			return kvm_pmu_set_msr(vcpu, msr_info);
2895 
2896 		if (pr || data != 0)
2897 			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
2898 				    "0x%x data 0x%llx\n", msr, data);
2899 		break;
2900 	case MSR_K7_CLK_CTL:
2901 		/*
2902 		 * Ignore all writes to this no longer documented MSR.
2903 		 * Writes are only relevant for old K7 processors,
2904 		 * all pre-dating SVM, but a recommended workaround from
2905 		 * AMD for these chips. It is possible to specify the
2906 		 * affected processor models on the command line, hence
2907 		 * the need to ignore the workaround.
2908 		 */
2909 		break;
2910 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2911 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
2912 	case HV_X64_MSR_CRASH_CTL:
2913 	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2914 	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
2915 	case HV_X64_MSR_TSC_EMULATION_CONTROL:
2916 	case HV_X64_MSR_TSC_EMULATION_STATUS:
2917 		return kvm_hv_set_msr_common(vcpu, msr, data,
2918 					     msr_info->host_initiated);
2919 	case MSR_IA32_BBL_CR_CTL3:
2920 		/* Drop writes to this legacy MSR -- see rdmsr
2921 		 * counterpart for further detail.
2922 		 */
2923 		if (report_ignored_msrs)
2924 			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
2925 				msr, data);
2926 		break;
2927 	case MSR_AMD64_OSVW_ID_LENGTH:
2928 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2929 			return 1;
2930 		vcpu->arch.osvw.length = data;
2931 		break;
2932 	case MSR_AMD64_OSVW_STATUS:
2933 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2934 			return 1;
2935 		vcpu->arch.osvw.status = data;
2936 		break;
2937 	case MSR_PLATFORM_INFO:
2938 		if (!msr_info->host_initiated ||
2939 		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
2940 		     cpuid_fault_enabled(vcpu)))
2941 			return 1;
2942 		vcpu->arch.msr_platform_info = data;
2943 		break;
2944 	case MSR_MISC_FEATURES_ENABLES:
2945 		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
2946 		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
2947 		     !supports_cpuid_fault(vcpu)))
2948 			return 1;
2949 		vcpu->arch.msr_misc_features_enables = data;
2950 		break;
2951 	default:
2952 		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
2953 			return xen_hvm_config(vcpu, data);
2954 		if (kvm_pmu_is_valid_msr(vcpu, msr))
2955 			return kvm_pmu_set_msr(vcpu, msr_info);
2956 		if (!ignore_msrs) {
2957 			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
2958 				    msr, data);
2959 			return 1;
2960 		} else {
2961 			if (report_ignored_msrs)
2962 				vcpu_unimpl(vcpu,
2963 					"ignored wrmsr: 0x%x data 0x%llx\n",
2964 					msr, data);
2965 			break;
2966 		}
2967 	}
2968 	return 0;
2969 }
2970 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
2971 
2972 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
2973 {
2974 	u64 data;
2975 	u64 mcg_cap = vcpu->arch.mcg_cap;
2976 	unsigned bank_num = mcg_cap & 0xff;
2977 
2978 	switch (msr) {
2979 	case MSR_IA32_P5_MC_ADDR:
2980 	case MSR_IA32_P5_MC_TYPE:
2981 		data = 0;
2982 		break;
2983 	case MSR_IA32_MCG_CAP:
2984 		data = vcpu->arch.mcg_cap;
2985 		break;
2986 	case MSR_IA32_MCG_CTL:
2987 		if (!(mcg_cap & MCG_CTL_P) && !host)
2988 			return 1;
2989 		data = vcpu->arch.mcg_ctl;
2990 		break;
2991 	case MSR_IA32_MCG_STATUS:
2992 		data = vcpu->arch.mcg_status;
2993 		break;
2994 	default:
2995 		if (msr >= MSR_IA32_MC0_CTL &&
2996 		    msr < MSR_IA32_MCx_CTL(bank_num)) {
2997 			u32 offset = array_index_nospec(
2998 				msr - MSR_IA32_MC0_CTL,
2999 				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);
3000 
3001 			data = vcpu->arch.mce_banks[offset];
3002 			break;
3003 		}
3004 		return 1;
3005 	}
3006 	*pdata = data;
3007 	return 0;
3008 }
3009 
3010 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3011 {
3012 	switch (msr_info->index) {
3013 	case MSR_IA32_PLATFORM_ID:
3014 	case MSR_IA32_EBL_CR_POWERON:
3015 	case MSR_IA32_DEBUGCTLMSR:
3016 	case MSR_IA32_LASTBRANCHFROMIP:
3017 	case MSR_IA32_LASTBRANCHTOIP:
3018 	case MSR_IA32_LASTINTFROMIP:
3019 	case MSR_IA32_LASTINTTOIP:
3020 	case MSR_K8_SYSCFG:
3021 	case MSR_K8_TSEG_ADDR:
3022 	case MSR_K8_TSEG_MASK:
3023 	case MSR_VM_HSAVE_PA:
3024 	case MSR_K8_INT_PENDING_MSG:
3025 	case MSR_AMD64_NB_CFG:
3026 	case MSR_FAM10H_MMIO_CONF_BASE:
3027 	case MSR_AMD64_BU_CFG2:
3028 	case MSR_IA32_PERF_CTL:
3029 	case MSR_AMD64_DC_CFG:
3030 	case MSR_F15H_EX_CFG:
3031 		msr_info->data = 0;
3032 		break;
3033 	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3034 	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3035 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3036 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3037 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3038 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3039 			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
3040 		msr_info->data = 0;
3041 		break;
3042 	case MSR_IA32_UCODE_REV:
3043 		msr_info->data = vcpu->arch.microcode_version;
3044 		break;
3045 	case MSR_IA32_ARCH_CAPABILITIES:
3046 		if (!msr_info->host_initiated &&
3047 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
3048 			return 1;
3049 		msr_info->data = vcpu->arch.arch_capabilities;
3050 		break;
3051 	case MSR_IA32_POWER_CTL:
3052 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
3053 		break;
3054 	case MSR_IA32_TSC:
3055 		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
3056 		break;
3057 	case MSR_MTRRcap:
3058 	case 0x200 ... 0x2ff:
3059 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
3060 	case 0xcd: /* fsb frequency */
3061 		msr_info->data = 3;
3062 		break;
3063 		/*
3064 		 * MSR_EBC_FREQUENCY_ID
3065 		 * Conservative value valid for even the basic CPU models.
3066 		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
3067 		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
3068 		 * and 266MHz for model 3, or 4. Set Core Clock
3069 		 * Frequency to System Bus Frequency Ratio to 1 (bits
3070 		 * 31:24) even though these are only valid for CPU
3071 		 * models > 2, however guests may end up dividing or
3072 		 * multiplying by zero otherwise.
3073 		 */
3074 	case MSR_EBC_FREQUENCY_ID:
3075 		msr_info->data = 1 << 24;
3076 		break;
3077 	case MSR_IA32_APICBASE:
3078 		msr_info->data = kvm_get_apic_base(vcpu);
3079 		break;
3080 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
3081 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
3082 		break;
3083 	case MSR_IA32_TSCDEADLINE:
3084 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
3085 		break;
3086 	case MSR_IA32_TSC_ADJUST:
3087 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
3088 		break;
3089 	case MSR_IA32_MISC_ENABLE:
3090 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3091 		break;
3092 	case MSR_IA32_SMBASE:
3093 		if (!msr_info->host_initiated)
3094 			return 1;
3095 		msr_info->data = vcpu->arch.smbase;
3096 		break;
3097 	case MSR_SMI_COUNT:
3098 		msr_info->data = vcpu->arch.smi_count;
3099 		break;
3100 	case MSR_IA32_PERF_STATUS:
3101 		/* TSC increment by tick */
3102 		msr_info->data = 1000ULL;
3103 		/* CPU multiplier */
3104 		msr_info->data |= (((uint64_t)4ULL) << 40);
3105 		break;
3106 	case MSR_EFER:
3107 		msr_info->data = vcpu->arch.efer;
3108 		break;
3109 	case MSR_KVM_WALL_CLOCK:
3110 	case MSR_KVM_WALL_CLOCK_NEW:
3111 		msr_info->data = vcpu->kvm->arch.wall_clock;
3112 		break;
3113 	case MSR_KVM_SYSTEM_TIME:
3114 	case MSR_KVM_SYSTEM_TIME_NEW:
3115 		msr_info->data = vcpu->arch.time;
3116 		break;
3117 	case MSR_KVM_ASYNC_PF_EN:
3118 		msr_info->data = vcpu->arch.apf.msr_val;
3119 		break;
3120 	case MSR_KVM_STEAL_TIME:
3121 		msr_info->data = vcpu->arch.st.msr_val;
3122 		break;
3123 	case MSR_KVM_PV_EOI_EN:
3124 		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3125 		break;
3126 	case MSR_KVM_POLL_CONTROL:
3127 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
3128 		break;
3129 	case MSR_IA32_P5_MC_ADDR:
3130 	case MSR_IA32_P5_MC_TYPE:
3131 	case MSR_IA32_MCG_CAP:
3132 	case MSR_IA32_MCG_CTL:
3133 	case MSR_IA32_MCG_STATUS:
3134 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3135 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
3136 				   msr_info->host_initiated);
3137 	case MSR_IA32_XSS:
3138 		if (!msr_info->host_initiated &&
3139 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3140 			return 1;
3141 		msr_info->data = vcpu->arch.ia32_xss;
3142 		break;
3143 	case MSR_K7_CLK_CTL:
3144 		/*
3145 		 * Provide expected ramp-up count for K7. All other
3146 		 * are set to zero, indicating minimum divisors for
3147 		 * every field.
3148 		 *
3149 		 * This prevents guest kernels on AMD host with CPU
3150 		 * type 6, model 8 and higher from exploding due to
3151 		 * the rdmsr failing.
3152 		 */
3153 		msr_info->data = 0x20000000;
3154 		break;
3155 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3156 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
3157 	case HV_X64_MSR_CRASH_CTL:
3158 	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3159 	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
3160 	case HV_X64_MSR_TSC_EMULATION_CONTROL:
3161 	case HV_X64_MSR_TSC_EMULATION_STATUS:
3162 		return kvm_hv_get_msr_common(vcpu,
3163 					     msr_info->index, &msr_info->data,
3164 					     msr_info->host_initiated);
3165 		break;
3166 	case MSR_IA32_BBL_CR_CTL3:
3167 		/* This legacy MSR exists but isn't fully documented in current
3168 		 * silicon.  It is however accessed by winxp in very narrow
3169 		 * scenarios where it sets bit #19, itself documented as
3170 		 * a "reserved" bit.  Best effort attempt to source coherent
3171 		 * read data here should the balance of the register be
3172 		 * interpreted by the guest:
3173 		 *
3174 		 * L2 cache control register 3: 64GB range, 256KB size,
3175 		 * enabled, latency 0x1, configured
3176 		 */
3177 		msr_info->data = 0xbe702111;
3178 		break;
3179 	case MSR_AMD64_OSVW_ID_LENGTH:
3180 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3181 			return 1;
3182 		msr_info->data = vcpu->arch.osvw.length;
3183 		break;
3184 	case MSR_AMD64_OSVW_STATUS:
3185 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3186 			return 1;
3187 		msr_info->data = vcpu->arch.osvw.status;
3188 		break;
3189 	case MSR_PLATFORM_INFO:
3190 		if (!msr_info->host_initiated &&
3191 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
3192 			return 1;
3193 		msr_info->data = vcpu->arch.msr_platform_info;
3194 		break;
3195 	case MSR_MISC_FEATURES_ENABLES:
3196 		msr_info->data = vcpu->arch.msr_misc_features_enables;
3197 		break;
3198 	case MSR_K7_HWCR:
3199 		msr_info->data = vcpu->arch.msr_hwcr;
3200 		break;
3201 	default:
3202 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3203 			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
3204 		if (!ignore_msrs) {
3205 			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
3206 					       msr_info->index);
3207 			return 1;
3208 		} else {
3209 			if (report_ignored_msrs)
3210 				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
3211 					msr_info->index);
3212 			msr_info->data = 0;
3213 		}
3214 		break;
3215 	}
3216 	return 0;
3217 }
3218 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
3219 
3220 /*
3221  * Read or write a bunch of msrs. All parameters are kernel addresses.
3222  *
3223  * @return number of msrs set successfully.
3224  */
3225 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
3226 		    struct kvm_msr_entry *entries,
3227 		    int (*do_msr)(struct kvm_vcpu *vcpu,
3228 				  unsigned index, u64 *data))
3229 {
3230 	int i;
3231 
3232 	for (i = 0; i < msrs->nmsrs; ++i)
3233 		if (do_msr(vcpu, entries[i].index, &entries[i].data))
3234 			break;
3235 
3236 	return i;
3237 }
3238 
3239 /*
3240  * Read or write a bunch of msrs. Parameters are user addresses.
3241  *
3242  * @return number of msrs set successfully.
3243  */
3244 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
3245 		  int (*do_msr)(struct kvm_vcpu *vcpu,
3246 				unsigned index, u64 *data),
3247 		  int writeback)
3248 {
3249 	struct kvm_msrs msrs;
3250 	struct kvm_msr_entry *entries;
3251 	int r, n;
3252 	unsigned size;
3253 
3254 	r = -EFAULT;
3255 	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3256 		goto out;
3257 
3258 	r = -E2BIG;
3259 	if (msrs.nmsrs >= MAX_IO_MSRS)
3260 		goto out;
3261 
3262 	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3263 	entries = memdup_user(user_msrs->entries, size);
3264 	if (IS_ERR(entries)) {
3265 		r = PTR_ERR(entries);
3266 		goto out;
3267 	}
3268 
3269 	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
3270 	if (r < 0)
3271 		goto out_free;
3272 
3273 	r = -EFAULT;
3274 	if (writeback && copy_to_user(user_msrs->entries, entries, size))
3275 		goto out_free;
3276 
3277 	r = n;
3278 
3279 out_free:
3280 	kfree(entries);
3281 out:
3282 	return r;
3283 }
3284 
3285 static inline bool kvm_can_mwait_in_guest(void)
3286 {
3287 	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3288 		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
3289 		boot_cpu_has(X86_FEATURE_ARAT);
3290 }
3291 
3292 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3293 {
3294 	int r = 0;
3295 
3296 	switch (ext) {
3297 	case KVM_CAP_IRQCHIP:
3298 	case KVM_CAP_HLT:
3299 	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
3300 	case KVM_CAP_SET_TSS_ADDR:
3301 	case KVM_CAP_EXT_CPUID:
3302 	case KVM_CAP_EXT_EMUL_CPUID:
3303 	case KVM_CAP_CLOCKSOURCE:
3304 	case KVM_CAP_PIT:
3305 	case KVM_CAP_NOP_IO_DELAY:
3306 	case KVM_CAP_MP_STATE:
3307 	case KVM_CAP_SYNC_MMU:
3308 	case KVM_CAP_USER_NMI:
3309 	case KVM_CAP_REINJECT_CONTROL:
3310 	case KVM_CAP_IRQ_INJECT_STATUS:
3311 	case KVM_CAP_IOEVENTFD:
3312 	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3313 	case KVM_CAP_PIT2:
3314 	case KVM_CAP_PIT_STATE2:
3315 	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
3316 	case KVM_CAP_XEN_HVM:
3317 	case KVM_CAP_VCPU_EVENTS:
3318 	case KVM_CAP_HYPERV:
3319 	case KVM_CAP_HYPERV_VAPIC:
3320 	case KVM_CAP_HYPERV_SPIN:
3321 	case KVM_CAP_HYPERV_SYNIC:
3322 	case KVM_CAP_HYPERV_SYNIC2:
3323 	case KVM_CAP_HYPERV_VP_INDEX:
3324 	case KVM_CAP_HYPERV_EVENTFD:
3325 	case KVM_CAP_HYPERV_TLBFLUSH:
3326 	case KVM_CAP_HYPERV_SEND_IPI:
3327 	case KVM_CAP_HYPERV_CPUID:
3328 	case KVM_CAP_PCI_SEGMENT:
3329 	case KVM_CAP_DEBUGREGS:
3330 	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3331 	case KVM_CAP_XSAVE:
3332 	case KVM_CAP_ASYNC_PF:
3333 	case KVM_CAP_GET_TSC_KHZ:
3334 	case KVM_CAP_KVMCLOCK_CTRL:
3335 	case KVM_CAP_READONLY_MEM:
3336 	case KVM_CAP_HYPERV_TIME:
3337 	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3338 	case KVM_CAP_TSC_DEADLINE_TIMER:
3339 	case KVM_CAP_DISABLE_QUIRKS:
3340 	case KVM_CAP_SET_BOOT_CPU_ID:
3341  	case KVM_CAP_SPLIT_IRQCHIP:
3342 	case KVM_CAP_IMMEDIATE_EXIT:
3343 	case KVM_CAP_PMU_EVENT_FILTER:
3344 	case KVM_CAP_GET_MSR_FEATURES:
3345 	case KVM_CAP_MSR_PLATFORM_INFO:
3346 	case KVM_CAP_EXCEPTION_PAYLOAD:
3347 		r = 1;
3348 		break;
3349 	case KVM_CAP_SYNC_REGS:
3350 		r = KVM_SYNC_X86_VALID_FIELDS;
3351 		break;
3352 	case KVM_CAP_ADJUST_CLOCK:
3353 		r = KVM_CLOCK_TSC_STABLE;
3354 		break;
3355 	case KVM_CAP_X86_DISABLE_EXITS:
3356 		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
3357 		      KVM_X86_DISABLE_EXITS_CSTATE;
3358 		if(kvm_can_mwait_in_guest())
3359 			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3360 		break;
3361 	case KVM_CAP_X86_SMM:
3362 		/* SMBASE is usually relocated above 1M on modern chipsets,
3363 		 * and SMM handlers might indeed rely on 4G segment limits,
3364 		 * so do not report SMM to be available if real mode is
3365 		 * emulated via vm86 mode.  Still, do not go to great lengths
3366 		 * to avoid userspace's usage of the feature, because it is a
3367 		 * fringe case that is not enabled except via specific settings
3368 		 * of the module parameters.
3369 		 */
3370 		r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE);
3371 		break;
3372 	case KVM_CAP_VAPIC:
3373 		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
3374 		break;
3375 	case KVM_CAP_NR_VCPUS:
3376 		r = KVM_SOFT_MAX_VCPUS;
3377 		break;
3378 	case KVM_CAP_MAX_VCPUS:
3379 		r = KVM_MAX_VCPUS;
3380 		break;
3381 	case KVM_CAP_MAX_VCPU_ID:
3382 		r = KVM_MAX_VCPU_ID;
3383 		break;
3384 	case KVM_CAP_PV_MMU:	/* obsolete */
3385 		r = 0;
3386 		break;
3387 	case KVM_CAP_MCE:
3388 		r = KVM_MAX_MCE_BANKS;
3389 		break;
3390 	case KVM_CAP_XCRS:
3391 		r = boot_cpu_has(X86_FEATURE_XSAVE);
3392 		break;
3393 	case KVM_CAP_TSC_CONTROL:
3394 		r = kvm_has_tsc_control;
3395 		break;
3396 	case KVM_CAP_X2APIC_API:
3397 		r = KVM_X2APIC_API_VALID_FLAGS;
3398 		break;
3399 	case KVM_CAP_NESTED_STATE:
3400 		r = kvm_x86_ops->get_nested_state ?
3401 			kvm_x86_ops->get_nested_state(NULL, NULL, 0) : 0;
3402 		break;
3403 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
3404 		r = kvm_x86_ops->enable_direct_tlbflush != NULL;
3405 		break;
3406 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
3407 		r = kvm_x86_ops->nested_enable_evmcs != NULL;
3408 		break;
3409 	default:
3410 		break;
3411 	}
3412 	return r;
3413 
3414 }
3415 
3416 long kvm_arch_dev_ioctl(struct file *filp,
3417 			unsigned int ioctl, unsigned long arg)
3418 {
3419 	void __user *argp = (void __user *)arg;
3420 	long r;
3421 
3422 	switch (ioctl) {
3423 	case KVM_GET_MSR_INDEX_LIST: {
3424 		struct kvm_msr_list __user *user_msr_list = argp;
3425 		struct kvm_msr_list msr_list;
3426 		unsigned n;
3427 
3428 		r = -EFAULT;
3429 		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3430 			goto out;
3431 		n = msr_list.nmsrs;
3432 		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3433 		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3434 			goto out;
3435 		r = -E2BIG;
3436 		if (n < msr_list.nmsrs)
3437 			goto out;
3438 		r = -EFAULT;
3439 		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
3440 				 num_msrs_to_save * sizeof(u32)))
3441 			goto out;
3442 		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3443 				 &emulated_msrs,
3444 				 num_emulated_msrs * sizeof(u32)))
3445 			goto out;
3446 		r = 0;
3447 		break;
3448 	}
3449 	case KVM_GET_SUPPORTED_CPUID:
3450 	case KVM_GET_EMULATED_CPUID: {
3451 		struct kvm_cpuid2 __user *cpuid_arg = argp;
3452 		struct kvm_cpuid2 cpuid;
3453 
3454 		r = -EFAULT;
3455 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3456 			goto out;
3457 
3458 		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
3459 					    ioctl);
3460 		if (r)
3461 			goto out;
3462 
3463 		r = -EFAULT;
3464 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3465 			goto out;
3466 		r = 0;
3467 		break;
3468 	}
3469 	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
3470 		r = -EFAULT;
3471 		if (copy_to_user(argp, &kvm_mce_cap_supported,
3472 				 sizeof(kvm_mce_cap_supported)))
3473 			goto out;
3474 		r = 0;
3475 		break;
3476 	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
3477 		struct kvm_msr_list __user *user_msr_list = argp;
3478 		struct kvm_msr_list msr_list;
3479 		unsigned int n;
3480 
3481 		r = -EFAULT;
3482 		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3483 			goto out;
3484 		n = msr_list.nmsrs;
3485 		msr_list.nmsrs = num_msr_based_features;
3486 		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3487 			goto out;
3488 		r = -E2BIG;
3489 		if (n < msr_list.nmsrs)
3490 			goto out;
3491 		r = -EFAULT;
3492 		if (copy_to_user(user_msr_list->indices, &msr_based_features,
3493 				 num_msr_based_features * sizeof(u32)))
3494 			goto out;
3495 		r = 0;
3496 		break;
3497 	}
3498 	case KVM_GET_MSRS:
3499 		r = msr_io(NULL, argp, do_get_msr_feature, 1);
3500 		break;
3501 	}
3502 	default:
3503 		r = -EINVAL;
3504 	}
3505 out:
3506 	return r;
3507 }
3508 
3509 static void wbinvd_ipi(void *garbage)
3510 {
3511 	wbinvd();
3512 }
3513 
3514 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
3515 {
3516 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3517 }
3518 
3519 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3520 {
3521 	/* Address WBINVD may be executed by guest */
3522 	if (need_emulate_wbinvd(vcpu)) {
3523 		if (kvm_x86_ops->has_wbinvd_exit())
3524 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
3525 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
3526 			smp_call_function_single(vcpu->cpu,
3527 					wbinvd_ipi, NULL, 1);
3528 	}
3529 
3530 	kvm_x86_ops->vcpu_load(vcpu, cpu);
3531 
3532 	/* Apply any externally detected TSC adjustments (due to suspend) */
3533 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
3534 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
3535 		vcpu->arch.tsc_offset_adjustment = 0;
3536 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3537 	}
3538 
3539 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3540 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3541 				rdtsc() - vcpu->arch.last_host_tsc;
3542 		if (tsc_delta < 0)
3543 			mark_tsc_unstable("KVM discovered backwards TSC");
3544 
3545 		if (kvm_check_tsc_unstable()) {
3546 			u64 offset = kvm_compute_tsc_offset(vcpu,
3547 						vcpu->arch.last_guest_tsc);
3548 			kvm_vcpu_write_tsc_offset(vcpu, offset);
3549 			vcpu->arch.tsc_catchup = 1;
3550 		}
3551 
3552 		if (kvm_lapic_hv_timer_in_use(vcpu))
3553 			kvm_lapic_restart_hv_timer(vcpu);
3554 
3555 		/*
3556 		 * On a host with synchronized TSC, there is no need to update
3557 		 * kvmclock on vcpu->cpu migration
3558 		 */
3559 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3560 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
3561 		if (vcpu->cpu != cpu)
3562 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
3563 		vcpu->cpu = cpu;
3564 	}
3565 
3566 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3567 }
3568 
3569 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
3570 {
3571 	struct kvm_host_map map;
3572 	struct kvm_steal_time *st;
3573 
3574 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3575 		return;
3576 
3577 	if (vcpu->arch.st.preempted)
3578 		return;
3579 
3580 	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,
3581 			&vcpu->arch.st.cache, true))
3582 		return;
3583 
3584 	st = map.hva +
3585 		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
3586 
3587 	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
3588 
3589 	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);
3590 }
3591 
3592 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3593 {
3594 	int idx;
3595 
3596 	if (vcpu->preempted)
3597 		vcpu->arch.preempted_in_kernel = !kvm_x86_ops->get_cpl(vcpu);
3598 
3599 	/*
3600 	 * Disable page faults because we're in atomic context here.
3601 	 * kvm_write_guest_offset_cached() would call might_fault()
3602 	 * that relies on pagefault_disable() to tell if there's a
3603 	 * bug. NOTE: the write to guest memory may not go through if
3604 	 * during postcopy live migration or if there's heavy guest
3605 	 * paging.
3606 	 */
3607 	pagefault_disable();
3608 	/*
3609 	 * kvm_memslots() will be called by
3610 	 * kvm_write_guest_offset_cached() so take the srcu lock.
3611 	 */
3612 	idx = srcu_read_lock(&vcpu->kvm->srcu);
3613 	kvm_steal_time_set_preempted(vcpu);
3614 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3615 	pagefault_enable();
3616 	kvm_x86_ops->vcpu_put(vcpu);
3617 	vcpu->arch.last_host_tsc = rdtsc();
3618 	/*
3619 	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
3620 	 * on every vmexit, but if not, we might have a stale dr6 from the
3621 	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3622 	 */
3623 	set_debugreg(0, 6);
3624 }
3625 
3626 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
3627 				    struct kvm_lapic_state *s)
3628 {
3629 	if (vcpu->arch.apicv_active)
3630 		kvm_x86_ops->sync_pir_to_irr(vcpu);
3631 
3632 	return kvm_apic_get_state(vcpu, s);
3633 }
3634 
3635 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
3636 				    struct kvm_lapic_state *s)
3637 {
3638 	int r;
3639 
3640 	r = kvm_apic_set_state(vcpu, s);
3641 	if (r)
3642 		return r;
3643 	update_cr8_intercept(vcpu);
3644 
3645 	return 0;
3646 }
3647 
3648 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
3649 {
3650 	return (!lapic_in_kernel(vcpu) ||
3651 		kvm_apic_accept_pic_intr(vcpu));
3652 }
3653 
3654 /*
3655  * if userspace requested an interrupt window, check that the
3656  * interrupt window is open.
3657  *
3658  * No need to exit to userspace if we already have an interrupt queued.
3659  */
3660 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
3661 {
3662 	return kvm_arch_interrupt_allowed(vcpu) &&
3663 		!kvm_cpu_has_interrupt(vcpu) &&
3664 		!kvm_event_needs_reinjection(vcpu) &&
3665 		kvm_cpu_accept_dm_intr(vcpu);
3666 }
3667 
3668 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
3669 				    struct kvm_interrupt *irq)
3670 {
3671 	if (irq->irq >= KVM_NR_INTERRUPTS)
3672 		return -EINVAL;
3673 
3674 	if (!irqchip_in_kernel(vcpu->kvm)) {
3675 		kvm_queue_interrupt(vcpu, irq->irq, false);
3676 		kvm_make_request(KVM_REQ_EVENT, vcpu);
3677 		return 0;
3678 	}
3679 
3680 	/*
3681 	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
3682 	 * fail for in-kernel 8259.
3683 	 */
3684 	if (pic_in_kernel(vcpu->kvm))
3685 		return -ENXIO;
3686 
3687 	if (vcpu->arch.pending_external_vector != -1)
3688 		return -EEXIST;
3689 
3690 	vcpu->arch.pending_external_vector = irq->irq;
3691 	kvm_make_request(KVM_REQ_EVENT, vcpu);
3692 	return 0;
3693 }
3694 
3695 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
3696 {
3697 	kvm_inject_nmi(vcpu);
3698 
3699 	return 0;
3700 }
3701 
3702 static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
3703 {
3704 	kvm_make_request(KVM_REQ_SMI, vcpu);
3705 
3706 	return 0;
3707 }
3708 
3709 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
3710 					   struct kvm_tpr_access_ctl *tac)
3711 {
3712 	if (tac->flags)
3713 		return -EINVAL;
3714 	vcpu->arch.tpr_access_reporting = !!tac->enabled;
3715 	return 0;
3716 }
3717 
3718 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
3719 					u64 mcg_cap)
3720 {
3721 	int r;
3722 	unsigned bank_num = mcg_cap & 0xff, bank;
3723 
3724 	r = -EINVAL;
3725 	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
3726 		goto out;
3727 	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
3728 		goto out;
3729 	r = 0;
3730 	vcpu->arch.mcg_cap = mcg_cap;
3731 	/* Init IA32_MCG_CTL to all 1s */
3732 	if (mcg_cap & MCG_CTL_P)
3733 		vcpu->arch.mcg_ctl = ~(u64)0;
3734 	/* Init IA32_MCi_CTL to all 1s */
3735 	for (bank = 0; bank < bank_num; bank++)
3736 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3737 
3738 	kvm_x86_ops->setup_mce(vcpu);
3739 out:
3740 	return r;
3741 }
3742 
3743 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
3744 				      struct kvm_x86_mce *mce)
3745 {
3746 	u64 mcg_cap = vcpu->arch.mcg_cap;
3747 	unsigned bank_num = mcg_cap & 0xff;
3748 	u64 *banks = vcpu->arch.mce_banks;
3749 
3750 	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
3751 		return -EINVAL;
3752 	/*
3753 	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
3754 	 * reporting is disabled
3755 	 */
3756 	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
3757 	    vcpu->arch.mcg_ctl != ~(u64)0)
3758 		return 0;
3759 	banks += 4 * mce->bank;
3760 	/*
3761 	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
3762 	 * reporting is disabled for the bank
3763 	 */
3764 	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
3765 		return 0;
3766 	if (mce->status & MCI_STATUS_UC) {
3767 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3768 		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3769 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3770 			return 0;
3771 		}
3772 		if (banks[1] & MCI_STATUS_VAL)
3773 			mce->status |= MCI_STATUS_OVER;
3774 		banks[2] = mce->addr;
3775 		banks[3] = mce->misc;
3776 		vcpu->arch.mcg_status = mce->mcg_status;
3777 		banks[1] = mce->status;
3778 		kvm_queue_exception(vcpu, MC_VECTOR);
3779 	} else if (!(banks[1] & MCI_STATUS_VAL)
3780 		   || !(banks[1] & MCI_STATUS_UC)) {
3781 		if (banks[1] & MCI_STATUS_VAL)
3782 			mce->status |= MCI_STATUS_OVER;
3783 		banks[2] = mce->addr;
3784 		banks[3] = mce->misc;
3785 		banks[1] = mce->status;
3786 	} else
3787 		banks[1] |= MCI_STATUS_OVER;
3788 	return 0;
3789 }
3790 
3791 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
3792 					       struct kvm_vcpu_events *events)
3793 {
3794 	process_nmi(vcpu);
3795 
3796 	/*
3797 	 * In guest mode, payload delivery should be deferred,
3798 	 * so that the L1 hypervisor can intercept #PF before
3799 	 * CR2 is modified (or intercept #DB before DR6 is
3800 	 * modified under nVMX). Unless the per-VM capability,
3801 	 * KVM_CAP_EXCEPTION_PAYLOAD, is set, we may not defer the delivery of
3802 	 * an exception payload and handle after a KVM_GET_VCPU_EVENTS. Since we
3803 	 * opportunistically defer the exception payload, deliver it if the
3804 	 * capability hasn't been requested before processing a
3805 	 * KVM_GET_VCPU_EVENTS.
3806 	 */
3807 	if (!vcpu->kvm->arch.exception_payload_enabled &&
3808 	    vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)
3809 		kvm_deliver_exception_payload(vcpu);
3810 
3811 	/*
3812 	 * The API doesn't provide the instruction length for software
3813 	 * exceptions, so don't report them. As long as the guest RIP
3814 	 * isn't advanced, we should expect to encounter the exception
3815 	 * again.
3816 	 */
3817 	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
3818 		events->exception.injected = 0;
3819 		events->exception.pending = 0;
3820 	} else {
3821 		events->exception.injected = vcpu->arch.exception.injected;
3822 		events->exception.pending = vcpu->arch.exception.pending;
3823 		/*
3824 		 * For ABI compatibility, deliberately conflate
3825 		 * pending and injected exceptions when
3826 		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
3827 		 */
3828 		if (!vcpu->kvm->arch.exception_payload_enabled)
3829 			events->exception.injected |=
3830 				vcpu->arch.exception.pending;
3831 	}
3832 	events->exception.nr = vcpu->arch.exception.nr;
3833 	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
3834 	events->exception.error_code = vcpu->arch.exception.error_code;
3835 	events->exception_has_payload = vcpu->arch.exception.has_payload;
3836 	events->exception_payload = vcpu->arch.exception.payload;
3837 
3838 	events->interrupt.injected =
3839 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
3840 	events->interrupt.nr = vcpu->arch.interrupt.nr;
3841 	events->interrupt.soft = 0;
3842 	events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
3843 
3844 	events->nmi.injected = vcpu->arch.nmi_injected;
3845 	events->nmi.pending = vcpu->arch.nmi_pending != 0;
3846 	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
3847 	events->nmi.pad = 0;
3848 
3849 	events->sipi_vector = 0; /* never valid when reporting to user space */
3850 
3851 	events->smi.smm = is_smm(vcpu);
3852 	events->smi.pending = vcpu->arch.smi_pending;
3853 	events->smi.smm_inside_nmi =
3854 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
3855 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);
3856 
3857 	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3858 			 | KVM_VCPUEVENT_VALID_SHADOW
3859 			 | KVM_VCPUEVENT_VALID_SMM);
3860 	if (vcpu->kvm->arch.exception_payload_enabled)
3861 		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
3862 
3863 	memset(&events->reserved, 0, sizeof(events->reserved));
3864 }
3865 
3866 static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3867 
3868 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
3869 					      struct kvm_vcpu_events *events)
3870 {
3871 	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3872 			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3873 			      | KVM_VCPUEVENT_VALID_SHADOW
3874 			      | KVM_VCPUEVENT_VALID_SMM
3875 			      | KVM_VCPUEVENT_VALID_PAYLOAD))
3876 		return -EINVAL;
3877 
3878 	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
3879 		if (!vcpu->kvm->arch.exception_payload_enabled)
3880 			return -EINVAL;
3881 		if (events->exception.pending)
3882 			events->exception.injected = 0;
3883 		else
3884 			events->exception_has_payload = 0;
3885 	} else {
3886 		events->exception.pending = 0;
3887 		events->exception_has_payload = 0;
3888 	}
3889 
3890 	if ((events->exception.injected || events->exception.pending) &&
3891 	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
3892 		return -EINVAL;
3893 
3894 	/* INITs are latched while in SMM */
3895 	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
3896 	    (events->smi.smm || events->smi.pending) &&
3897 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
3898 		return -EINVAL;
3899 
3900 	process_nmi(vcpu);
3901 	vcpu->arch.exception.injected = events->exception.injected;
3902 	vcpu->arch.exception.pending = events->exception.pending;
3903 	vcpu->arch.exception.nr = events->exception.nr;
3904 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
3905 	vcpu->arch.exception.error_code = events->exception.error_code;
3906 	vcpu->arch.exception.has_payload = events->exception_has_payload;
3907 	vcpu->arch.exception.payload = events->exception_payload;
3908 
3909 	vcpu->arch.interrupt.injected = events->interrupt.injected;
3910 	vcpu->arch.interrupt.nr = events->interrupt.nr;
3911 	vcpu->arch.interrupt.soft = events->interrupt.soft;
3912 	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
3913 		kvm_x86_ops->set_interrupt_shadow(vcpu,
3914 						  events->interrupt.shadow);
3915 
3916 	vcpu->arch.nmi_injected = events->nmi.injected;
3917 	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
3918 		vcpu->arch.nmi_pending = events->nmi.pending;
3919 	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
3920 
3921 	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3922 	    lapic_in_kernel(vcpu))
3923 		vcpu->arch.apic->sipi_vector = events->sipi_vector;
3924 
3925 	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
3926 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
3927 			if (events->smi.smm)
3928 				vcpu->arch.hflags |= HF_SMM_MASK;
3929 			else
3930 				vcpu->arch.hflags &= ~HF_SMM_MASK;
3931 			kvm_smm_changed(vcpu);
3932 		}
3933 
3934 		vcpu->arch.smi_pending = events->smi.pending;
3935 
3936 		if (events->smi.smm) {
3937 			if (events->smi.smm_inside_nmi)
3938 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
3939 			else
3940 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
3941 		}
3942 
3943 		if (lapic_in_kernel(vcpu)) {
3944 			if (events->smi.latched_init)
3945 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
3946 			else
3947 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
3948 		}
3949 	}
3950 
3951 	kvm_make_request(KVM_REQ_EVENT, vcpu);
3952 
3953 	return 0;
3954 }
3955 
3956 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
3957 					     struct kvm_debugregs *dbgregs)
3958 {
3959 	unsigned long val;
3960 
3961 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3962 	kvm_get_dr(vcpu, 6, &val);
3963 	dbgregs->dr6 = val;
3964 	dbgregs->dr7 = vcpu->arch.dr7;
3965 	dbgregs->flags = 0;
3966 	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3967 }
3968 
3969 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
3970 					    struct kvm_debugregs *dbgregs)
3971 {
3972 	if (dbgregs->flags)
3973 		return -EINVAL;
3974 
3975 	if (dbgregs->dr6 & ~0xffffffffull)
3976 		return -EINVAL;
3977 	if (dbgregs->dr7 & ~0xffffffffull)
3978 		return -EINVAL;
3979 
3980 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3981 	kvm_update_dr0123(vcpu);
3982 	vcpu->arch.dr6 = dbgregs->dr6;
3983 	kvm_update_dr6(vcpu);
3984 	vcpu->arch.dr7 = dbgregs->dr7;
3985 	kvm_update_dr7(vcpu);
3986 
3987 	return 0;
3988 }
3989 
3990 #define XSTATE_COMPACTION_ENABLED (1ULL << 63)
3991 
3992 static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
3993 {
3994 	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3995 	u64 xstate_bv = xsave->header.xfeatures;
3996 	u64 valid;
3997 
3998 	/*
3999 	 * Copy legacy XSAVE area, to avoid complications with CPUID
4000 	 * leaves 0 and 1 in the loop below.
4001 	 */
4002 	memcpy(dest, xsave, XSAVE_HDR_OFFSET);
4003 
4004 	/* Set XSTATE_BV */
4005 	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
4006 	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;
4007 
4008 	/*
4009 	 * Copy each region from the possibly compacted offset to the
4010 	 * non-compacted offset.
4011 	 */
4012 	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4013 	while (valid) {
4014 		u64 xfeature_mask = valid & -valid;
4015 		int xfeature_nr = fls64(xfeature_mask) - 1;
4016 		void *src = get_xsave_addr(xsave, xfeature_nr);
4017 
4018 		if (src) {
4019 			u32 size, offset, ecx, edx;
4020 			cpuid_count(XSTATE_CPUID, xfeature_nr,
4021 				    &size, &offset, &ecx, &edx);
4022 			if (xfeature_nr == XFEATURE_PKRU)
4023 				memcpy(dest + offset, &vcpu->arch.pkru,
4024 				       sizeof(vcpu->arch.pkru));
4025 			else
4026 				memcpy(dest + offset, src, size);
4027 
4028 		}
4029 
4030 		valid -= xfeature_mask;
4031 	}
4032 }
4033 
4034 static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
4035 {
4036 	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4037 	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
4038 	u64 valid;
4039 
4040 	/*
4041 	 * Copy legacy XSAVE area, to avoid complications with CPUID
4042 	 * leaves 0 and 1 in the loop below.
4043 	 */
4044 	memcpy(xsave, src, XSAVE_HDR_OFFSET);
4045 
4046 	/* Set XSTATE_BV and possibly XCOMP_BV.  */
4047 	xsave->header.xfeatures = xstate_bv;
4048 	if (boot_cpu_has(X86_FEATURE_XSAVES))
4049 		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
4050 
4051 	/*
4052 	 * Copy each region from the non-compacted offset to the
4053 	 * possibly compacted offset.
4054 	 */
4055 	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4056 	while (valid) {
4057 		u64 xfeature_mask = valid & -valid;
4058 		int xfeature_nr = fls64(xfeature_mask) - 1;
4059 		void *dest = get_xsave_addr(xsave, xfeature_nr);
4060 
4061 		if (dest) {
4062 			u32 size, offset, ecx, edx;
4063 			cpuid_count(XSTATE_CPUID, xfeature_nr,
4064 				    &size, &offset, &ecx, &edx);
4065 			if (xfeature_nr == XFEATURE_PKRU)
4066 				memcpy(&vcpu->arch.pkru, src + offset,
4067 				       sizeof(vcpu->arch.pkru));
4068 			else
4069 				memcpy(dest, src + offset, size);
4070 		}
4071 
4072 		valid -= xfeature_mask;
4073 	}
4074 }
4075 
4076 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
4077 					 struct kvm_xsave *guest_xsave)
4078 {
4079 	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4080 		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
4081 		fill_xsave((u8 *) guest_xsave->region, vcpu);
4082 	} else {
4083 		memcpy(guest_xsave->region,
4084 			&vcpu->arch.guest_fpu->state.fxsave,
4085 			sizeof(struct fxregs_state));
4086 		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
4087 			XFEATURE_MASK_FPSSE;
4088 	}
4089 }
4090 
4091 #define XSAVE_MXCSR_OFFSET 24
4092 
4093 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
4094 					struct kvm_xsave *guest_xsave)
4095 {
4096 	u64 xstate_bv =
4097 		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
4098 	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
4099 
4100 	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4101 		/*
4102 		 * Here we allow setting states that are not present in
4103 		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
4104 		 * with old userspace.
4105 		 */
4106 		if (xstate_bv & ~kvm_supported_xcr0() ||
4107 			mxcsr & ~mxcsr_feature_mask)
4108 			return -EINVAL;
4109 		load_xsave(vcpu, (u8 *)guest_xsave->region);
4110 	} else {
4111 		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
4112 			mxcsr & ~mxcsr_feature_mask)
4113 			return -EINVAL;
4114 		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4115 			guest_xsave->region, sizeof(struct fxregs_state));
4116 	}
4117 	return 0;
4118 }
4119 
4120 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
4121 					struct kvm_xcrs *guest_xcrs)
4122 {
4123 	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
4124 		guest_xcrs->nr_xcrs = 0;
4125 		return;
4126 	}
4127 
4128 	guest_xcrs->nr_xcrs = 1;
4129 	guest_xcrs->flags = 0;
4130 	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
4131 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
4132 }
4133 
4134 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
4135 				       struct kvm_xcrs *guest_xcrs)
4136 {
4137 	int i, r = 0;
4138 
4139 	if (!boot_cpu_has(X86_FEATURE_XSAVE))
4140 		return -EINVAL;
4141 
4142 	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
4143 		return -EINVAL;
4144 
4145 	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
4146 		/* Only support XCR0 currently */
4147 		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
4148 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
4149 				guest_xcrs->xcrs[i].value);
4150 			break;
4151 		}
4152 	if (r)
4153 		r = -EINVAL;
4154 	return r;
4155 }
4156 
4157 /*
4158  * kvm_set_guest_paused() indicates to the guest kernel that it has been
4159  * stopped by the hypervisor.  This function will be called from the host only.
4160  * EINVAL is returned when the host attempts to set the flag for a guest that
4161  * does not support pv clocks.
4162  */
4163 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
4164 {
4165 	if (!vcpu->arch.pv_time_enabled)
4166 		return -EINVAL;
4167 	vcpu->arch.pvclock_set_guest_stopped_request = true;
4168 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4169 	return 0;
4170 }
4171 
4172 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4173 				     struct kvm_enable_cap *cap)
4174 {
4175 	int r;
4176 	uint16_t vmcs_version;
4177 	void __user *user_ptr;
4178 
4179 	if (cap->flags)
4180 		return -EINVAL;
4181 
4182 	switch (cap->cap) {
4183 	case KVM_CAP_HYPERV_SYNIC2:
4184 		if (cap->args[0])
4185 			return -EINVAL;
4186 		/* fall through */
4187 
4188 	case KVM_CAP_HYPERV_SYNIC:
4189 		if (!irqchip_in_kernel(vcpu->kvm))
4190 			return -EINVAL;
4191 		return kvm_hv_activate_synic(vcpu, cap->cap ==
4192 					     KVM_CAP_HYPERV_SYNIC2);
4193 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4194 		if (!kvm_x86_ops->nested_enable_evmcs)
4195 			return -ENOTTY;
4196 		r = kvm_x86_ops->nested_enable_evmcs(vcpu, &vmcs_version);
4197 		if (!r) {
4198 			user_ptr = (void __user *)(uintptr_t)cap->args[0];
4199 			if (copy_to_user(user_ptr, &vmcs_version,
4200 					 sizeof(vmcs_version)))
4201 				r = -EFAULT;
4202 		}
4203 		return r;
4204 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
4205 		if (!kvm_x86_ops->enable_direct_tlbflush)
4206 			return -ENOTTY;
4207 
4208 		return kvm_x86_ops->enable_direct_tlbflush(vcpu);
4209 
4210 	default:
4211 		return -EINVAL;
4212 	}
4213 }
4214 
4215 long kvm_arch_vcpu_ioctl(struct file *filp,
4216 			 unsigned int ioctl, unsigned long arg)
4217 {
4218 	struct kvm_vcpu *vcpu = filp->private_data;
4219 	void __user *argp = (void __user *)arg;
4220 	int r;
4221 	union {
4222 		struct kvm_lapic_state *lapic;
4223 		struct kvm_xsave *xsave;
4224 		struct kvm_xcrs *xcrs;
4225 		void *buffer;
4226 	} u;
4227 
4228 	vcpu_load(vcpu);
4229 
4230 	u.buffer = NULL;
4231 	switch (ioctl) {
4232 	case KVM_GET_LAPIC: {
4233 		r = -EINVAL;
4234 		if (!lapic_in_kernel(vcpu))
4235 			goto out;
4236 		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
4237 				GFP_KERNEL_ACCOUNT);
4238 
4239 		r = -ENOMEM;
4240 		if (!u.lapic)
4241 			goto out;
4242 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4243 		if (r)
4244 			goto out;
4245 		r = -EFAULT;
4246 		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
4247 			goto out;
4248 		r = 0;
4249 		break;
4250 	}
4251 	case KVM_SET_LAPIC: {
4252 		r = -EINVAL;
4253 		if (!lapic_in_kernel(vcpu))
4254 			goto out;
4255 		u.lapic = memdup_user(argp, sizeof(*u.lapic));
4256 		if (IS_ERR(u.lapic)) {
4257 			r = PTR_ERR(u.lapic);
4258 			goto out_nofree;
4259 		}
4260 
4261 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4262 		break;
4263 	}
4264 	case KVM_INTERRUPT: {
4265 		struct kvm_interrupt irq;
4266 
4267 		r = -EFAULT;
4268 		if (copy_from_user(&irq, argp, sizeof(irq)))
4269 			goto out;
4270 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
4271 		break;
4272 	}
4273 	case KVM_NMI: {
4274 		r = kvm_vcpu_ioctl_nmi(vcpu);
4275 		break;
4276 	}
4277 	case KVM_SMI: {
4278 		r = kvm_vcpu_ioctl_smi(vcpu);
4279 		break;
4280 	}
4281 	case KVM_SET_CPUID: {
4282 		struct kvm_cpuid __user *cpuid_arg = argp;
4283 		struct kvm_cpuid cpuid;
4284 
4285 		r = -EFAULT;
4286 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4287 			goto out;
4288 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4289 		break;
4290 	}
4291 	case KVM_SET_CPUID2: {
4292 		struct kvm_cpuid2 __user *cpuid_arg = argp;
4293 		struct kvm_cpuid2 cpuid;
4294 
4295 		r = -EFAULT;
4296 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4297 			goto out;
4298 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4299 					      cpuid_arg->entries);
4300 		break;
4301 	}
4302 	case KVM_GET_CPUID2: {
4303 		struct kvm_cpuid2 __user *cpuid_arg = argp;
4304 		struct kvm_cpuid2 cpuid;
4305 
4306 		r = -EFAULT;
4307 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4308 			goto out;
4309 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4310 					      cpuid_arg->entries);
4311 		if (r)
4312 			goto out;
4313 		r = -EFAULT;
4314 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4315 			goto out;
4316 		r = 0;
4317 		break;
4318 	}
4319 	case KVM_GET_MSRS: {
4320 		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4321 		r = msr_io(vcpu, argp, do_get_msr, 1);
4322 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4323 		break;
4324 	}
4325 	case KVM_SET_MSRS: {
4326 		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4327 		r = msr_io(vcpu, argp, do_set_msr, 0);
4328 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4329 		break;
4330 	}
4331 	case KVM_TPR_ACCESS_REPORTING: {
4332 		struct kvm_tpr_access_ctl tac;
4333 
4334 		r = -EFAULT;
4335 		if (copy_from_user(&tac, argp, sizeof(tac)))
4336 			goto out;
4337 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
4338 		if (r)
4339 			goto out;
4340 		r = -EFAULT;
4341 		if (copy_to_user(argp, &tac, sizeof(tac)))
4342 			goto out;
4343 		r = 0;
4344 		break;
4345 	};
4346 	case KVM_SET_VAPIC_ADDR: {
4347 		struct kvm_vapic_addr va;
4348 		int idx;
4349 
4350 		r = -EINVAL;
4351 		if (!lapic_in_kernel(vcpu))
4352 			goto out;
4353 		r = -EFAULT;
4354 		if (copy_from_user(&va, argp, sizeof(va)))
4355 			goto out;
4356 		idx = srcu_read_lock(&vcpu->kvm->srcu);
4357 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4358 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4359 		break;
4360 	}
4361 	case KVM_X86_SETUP_MCE: {
4362 		u64 mcg_cap;
4363 
4364 		r = -EFAULT;
4365 		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
4366 			goto out;
4367 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
4368 		break;
4369 	}
4370 	case KVM_X86_SET_MCE: {
4371 		struct kvm_x86_mce mce;
4372 
4373 		r = -EFAULT;
4374 		if (copy_from_user(&mce, argp, sizeof(mce)))
4375 			goto out;
4376 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
4377 		break;
4378 	}
4379 	case KVM_GET_VCPU_EVENTS: {
4380 		struct kvm_vcpu_events events;
4381 
4382 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
4383 
4384 		r = -EFAULT;
4385 		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
4386 			break;
4387 		r = 0;
4388 		break;
4389 	}
4390 	case KVM_SET_VCPU_EVENTS: {
4391 		struct kvm_vcpu_events events;
4392 
4393 		r = -EFAULT;
4394 		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
4395 			break;
4396 
4397 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
4398 		break;
4399 	}
4400 	case KVM_GET_DEBUGREGS: {
4401 		struct kvm_debugregs dbgregs;
4402 
4403 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
4404 
4405 		r = -EFAULT;
4406 		if (copy_to_user(argp, &dbgregs,
4407 				 sizeof(struct kvm_debugregs)))
4408 			break;
4409 		r = 0;
4410 		break;
4411 	}
4412 	case KVM_SET_DEBUGREGS: {
4413 		struct kvm_debugregs dbgregs;
4414 
4415 		r = -EFAULT;
4416 		if (copy_from_user(&dbgregs, argp,
4417 				   sizeof(struct kvm_debugregs)))
4418 			break;
4419 
4420 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
4421 		break;
4422 	}
4423 	case KVM_GET_XSAVE: {
4424 		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4425 		r = -ENOMEM;
4426 		if (!u.xsave)
4427 			break;
4428 
4429 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4430 
4431 		r = -EFAULT;
4432 		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4433 			break;
4434 		r = 0;
4435 		break;
4436 	}
4437 	case KVM_SET_XSAVE: {
4438 		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4439 		if (IS_ERR(u.xsave)) {
4440 			r = PTR_ERR(u.xsave);
4441 			goto out_nofree;
4442 		}
4443 
4444 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4445 		break;
4446 	}
4447 	case KVM_GET_XCRS: {
4448 		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4449 		r = -ENOMEM;
4450 		if (!u.xcrs)
4451 			break;
4452 
4453 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4454 
4455 		r = -EFAULT;
4456 		if (copy_to_user(argp, u.xcrs,
4457 				 sizeof(struct kvm_xcrs)))
4458 			break;
4459 		r = 0;
4460 		break;
4461 	}
4462 	case KVM_SET_XCRS: {
4463 		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4464 		if (IS_ERR(u.xcrs)) {
4465 			r = PTR_ERR(u.xcrs);
4466 			goto out_nofree;
4467 		}
4468 
4469 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4470 		break;
4471 	}
4472 	case KVM_SET_TSC_KHZ: {
4473 		u32 user_tsc_khz;
4474 
4475 		r = -EINVAL;
4476 		user_tsc_khz = (u32)arg;
4477 
4478 		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
4479 			goto out;
4480 
4481 		if (user_tsc_khz == 0)
4482 			user_tsc_khz = tsc_khz;
4483 
4484 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
4485 			r = 0;
4486 
4487 		goto out;
4488 	}
4489 	case KVM_GET_TSC_KHZ: {
4490 		r = vcpu->arch.virtual_tsc_khz;
4491 		goto out;
4492 	}
4493 	case KVM_KVMCLOCK_CTRL: {
4494 		r = kvm_set_guest_paused(vcpu);
4495 		goto out;
4496 	}
4497 	case KVM_ENABLE_CAP: {
4498 		struct kvm_enable_cap cap;
4499 
4500 		r = -EFAULT;
4501 		if (copy_from_user(&cap, argp, sizeof(cap)))
4502 			goto out;
4503 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4504 		break;
4505 	}
4506 	case KVM_GET_NESTED_STATE: {
4507 		struct kvm_nested_state __user *user_kvm_nested_state = argp;
4508 		u32 user_data_size;
4509 
4510 		r = -EINVAL;
4511 		if (!kvm_x86_ops->get_nested_state)
4512 			break;
4513 
4514 		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4515 		r = -EFAULT;
4516 		if (get_user(user_data_size, &user_kvm_nested_state->size))
4517 			break;
4518 
4519 		r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state,
4520 						  user_data_size);
4521 		if (r < 0)
4522 			break;
4523 
4524 		if (r > user_data_size) {
4525 			if (put_user(r, &user_kvm_nested_state->size))
4526 				r = -EFAULT;
4527 			else
4528 				r = -E2BIG;
4529 			break;
4530 		}
4531 
4532 		r = 0;
4533 		break;
4534 	}
4535 	case KVM_SET_NESTED_STATE: {
4536 		struct kvm_nested_state __user *user_kvm_nested_state = argp;
4537 		struct kvm_nested_state kvm_state;
4538 		int idx;
4539 
4540 		r = -EINVAL;
4541 		if (!kvm_x86_ops->set_nested_state)
4542 			break;
4543 
4544 		r = -EFAULT;
4545 		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4546 			break;
4547 
4548 		r = -EINVAL;
4549 		if (kvm_state.size < sizeof(kvm_state))
4550 			break;
4551 
4552 		if (kvm_state.flags &
4553 		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
4554 		      | KVM_STATE_NESTED_EVMCS))
4555 			break;
4556 
4557 		/* nested_run_pending implies guest_mode.  */
4558 		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
4559 		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4560 			break;
4561 
4562 		idx = srcu_read_lock(&vcpu->kvm->srcu);
4563 		r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state);
4564 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4565 		break;
4566 	}
4567 	case KVM_GET_SUPPORTED_HV_CPUID: {
4568 		struct kvm_cpuid2 __user *cpuid_arg = argp;
4569 		struct kvm_cpuid2 cpuid;
4570 
4571 		r = -EFAULT;
4572 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4573 			goto out;
4574 
4575 		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
4576 						cpuid_arg->entries);
4577 		if (r)
4578 			goto out;
4579 
4580 		r = -EFAULT;
4581 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4582 			goto out;
4583 		r = 0;
4584 		break;
4585 	}
4586 	default:
4587 		r = -EINVAL;
4588 	}
4589 out:
4590 	kfree(u.buffer);
4591 out_nofree:
4592 	vcpu_put(vcpu);
4593 	return r;
4594 }
4595 
4596 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4597 {
4598 	return VM_FAULT_SIGBUS;
4599 }
4600 
4601 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
4602 {
4603 	int ret;
4604 
4605 	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4606 		return -EINVAL;
4607 	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
4608 	return ret;
4609 }
4610 
4611 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
4612 					      u64 ident_addr)
4613 {
4614 	return kvm_x86_ops->set_identity_map_addr(kvm, ident_addr);
4615 }
4616 
4617 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4618 					 unsigned long kvm_nr_mmu_pages)
4619 {
4620 	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
4621 		return -EINVAL;
4622 
4623 	mutex_lock(&kvm->slots_lock);
4624 
4625 	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4626 	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4627 
4628 	mutex_unlock(&kvm->slots_lock);
4629 	return 0;
4630 }
4631 
4632 static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4633 {
4634 	return kvm->arch.n_max_mmu_pages;
4635 }
4636 
4637 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
4638 {
4639 	struct kvm_pic *pic = kvm->arch.vpic;
4640 	int r;
4641 
4642 	r = 0;
4643 	switch (chip->chip_id) {
4644 	case KVM_IRQCHIP_PIC_MASTER:
4645 		memcpy(&chip->chip.pic, &pic->pics[0],
4646 			sizeof(struct kvm_pic_state));
4647 		break;
4648 	case KVM_IRQCHIP_PIC_SLAVE:
4649 		memcpy(&chip->chip.pic, &pic->pics[1],
4650 			sizeof(struct kvm_pic_state));
4651 		break;
4652 	case KVM_IRQCHIP_IOAPIC:
4653 		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4654 		break;
4655 	default:
4656 		r = -EINVAL;
4657 		break;
4658 	}
4659 	return r;
4660 }
4661 
4662 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
4663 {
4664 	struct kvm_pic *pic = kvm->arch.vpic;
4665 	int r;
4666 
4667 	r = 0;
4668 	switch (chip->chip_id) {
4669 	case KVM_IRQCHIP_PIC_MASTER:
4670 		spin_lock(&pic->lock);
4671 		memcpy(&pic->pics[0], &chip->chip.pic,
4672 			sizeof(struct kvm_pic_state));
4673 		spin_unlock(&pic->lock);
4674 		break;
4675 	case KVM_IRQCHIP_PIC_SLAVE:
4676 		spin_lock(&pic->lock);
4677 		memcpy(&pic->pics[1], &chip->chip.pic,
4678 			sizeof(struct kvm_pic_state));
4679 		spin_unlock(&pic->lock);
4680 		break;
4681 	case KVM_IRQCHIP_IOAPIC:
4682 		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4683 		break;
4684 	default:
4685 		r = -EINVAL;
4686 		break;
4687 	}
4688 	kvm_pic_update_irq(pic);
4689 	return r;
4690 }
4691 
4692 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
4693 {
4694 	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
4695 
4696 	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));
4697 
4698 	mutex_lock(&kps->lock);
4699 	memcpy(ps, &kps->channels, sizeof(*ps));
4700 	mutex_unlock(&kps->lock);
4701 	return 0;
4702 }
4703 
4704 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
4705 {
4706 	int i;
4707 	struct kvm_pit *pit = kvm->arch.vpit;
4708 
4709 	mutex_lock(&pit->pit_state.lock);
4710 	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4711 	for (i = 0; i < 3; i++)
4712 		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
4713 	mutex_unlock(&pit->pit_state.lock);
4714 	return 0;
4715 }
4716 
4717 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
4718 {
4719 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
4720 	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
4721 		sizeof(ps->channels));
4722 	ps->flags = kvm->arch.vpit->pit_state.flags;
4723 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4724 	memset(&ps->reserved, 0, sizeof(ps->reserved));
4725 	return 0;
4726 }
4727 
4728 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
4729 {
4730 	int start = 0;
4731 	int i;
4732 	u32 prev_legacy, cur_legacy;
4733 	struct kvm_pit *pit = kvm->arch.vpit;
4734 
4735 	mutex_lock(&pit->pit_state.lock);
4736 	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
4737 	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
4738 	if (!prev_legacy && cur_legacy)
4739 		start = 1;
4740 	memcpy(&pit->pit_state.channels, &ps->channels,
4741 	       sizeof(pit->pit_state.channels));
4742 	pit->pit_state.flags = ps->flags;
4743 	for (i = 0; i < 3; i++)
4744 		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4745 				   start && i == 0);
4746 	mutex_unlock(&pit->pit_state.lock);
4747 	return 0;
4748 }
4749 
4750 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
4751 				 struct kvm_reinject_control *control)
4752 {
4753 	struct kvm_pit *pit = kvm->arch.vpit;
4754 
4755 	/* pit->pit_state.lock was overloaded to prevent userspace from getting
4756 	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
4757 	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
4758 	 */
4759 	mutex_lock(&pit->pit_state.lock);
4760 	kvm_pit_set_reinject(pit, control->pit_reinject);
4761 	mutex_unlock(&pit->pit_state.lock);
4762 
4763 	return 0;
4764 }
4765 
4766 /**
4767  * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
4768  * @kvm: kvm instance
4769  * @log: slot id and address to which we copy the log
4770  *
4771  * Steps 1-4 below provide general overview of dirty page logging. See
4772  * kvm_get_dirty_log_protect() function description for additional details.
4773  *
4774  * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
4775  * always flush the TLB (step 4) even if previous step failed  and the dirty
4776  * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
4777  * does not preclude user space subsequent dirty log read. Flushing TLB ensures
4778  * writes will be marked dirty for next log read.
4779  *
4780  *   1. Take a snapshot of the bit and clear it if needed.
4781  *   2. Write protect the corresponding page.
4782  *   3. Copy the snapshot to the userspace.
4783  *   4. Flush TLB's if needed.
4784  */
4785 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
4786 {
4787 	bool flush = false;
4788 	int r;
4789 
4790 	mutex_lock(&kvm->slots_lock);
4791 
4792 	/*
4793 	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
4794 	 */
4795 	if (kvm_x86_ops->flush_log_dirty)
4796 		kvm_x86_ops->flush_log_dirty(kvm);
4797 
4798 	r = kvm_get_dirty_log_protect(kvm, log, &flush);
4799 
4800 	/*
4801 	 * All the TLBs can be flushed out of mmu lock, see the comments in
4802 	 * kvm_mmu_slot_remove_write_access().
4803 	 */
4804 	lockdep_assert_held(&kvm->slots_lock);
4805 	if (flush)
4806 		kvm_flush_remote_tlbs(kvm);
4807 
4808 	mutex_unlock(&kvm->slots_lock);
4809 	return r;
4810 }
4811 
4812 int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
4813 {
4814 	bool flush = false;
4815 	int r;
4816 
4817 	mutex_lock(&kvm->slots_lock);
4818 
4819 	/*
4820 	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
4821 	 */
4822 	if (kvm_x86_ops->flush_log_dirty)
4823 		kvm_x86_ops->flush_log_dirty(kvm);
4824 
4825 	r = kvm_clear_dirty_log_protect(kvm, log, &flush);
4826 
4827 	/*
4828 	 * All the TLBs can be flushed out of mmu lock, see the comments in
4829 	 * kvm_mmu_slot_remove_write_access().
4830 	 */
4831 	lockdep_assert_held(&kvm->slots_lock);
4832 	if (flush)
4833 		kvm_flush_remote_tlbs(kvm);
4834 
4835 	mutex_unlock(&kvm->slots_lock);
4836 	return r;
4837 }
4838 
4839 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
4840 			bool line_status)
4841 {
4842 	if (!irqchip_in_kernel(kvm))
4843 		return -ENXIO;
4844 
4845 	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4846 					irq_event->irq, irq_event->level,
4847 					line_status);
4848 	return 0;
4849 }
4850 
4851 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
4852 			    struct kvm_enable_cap *cap)
4853 {
4854 	int r;
4855 
4856 	if (cap->flags)
4857 		return -EINVAL;
4858 
4859 	switch (cap->cap) {
4860 	case KVM_CAP_DISABLE_QUIRKS:
4861 		kvm->arch.disabled_quirks = cap->args[0];
4862 		r = 0;
4863 		break;
4864 	case KVM_CAP_SPLIT_IRQCHIP: {
4865 		mutex_lock(&kvm->lock);
4866 		r = -EINVAL;
4867 		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
4868 			goto split_irqchip_unlock;
4869 		r = -EEXIST;
4870 		if (irqchip_in_kernel(kvm))
4871 			goto split_irqchip_unlock;
4872 		if (kvm->created_vcpus)
4873 			goto split_irqchip_unlock;
4874 		r = kvm_setup_empty_irq_routing(kvm);
4875 		if (r)
4876 			goto split_irqchip_unlock;
4877 		/* Pairs with irqchip_in_kernel. */
4878 		smp_wmb();
4879 		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4880 		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4881 		r = 0;
4882 split_irqchip_unlock:
4883 		mutex_unlock(&kvm->lock);
4884 		break;
4885 	}
4886 	case KVM_CAP_X2APIC_API:
4887 		r = -EINVAL;
4888 		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
4889 			break;
4890 
4891 		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
4892 			kvm->arch.x2apic_format = true;
4893 		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
4894 			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4895 
4896 		r = 0;
4897 		break;
4898 	case KVM_CAP_X86_DISABLE_EXITS:
4899 		r = -EINVAL;
4900 		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
4901 			break;
4902 
4903 		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
4904 			kvm_can_mwait_in_guest())
4905 			kvm->arch.mwait_in_guest = true;
4906 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4907 			kvm->arch.hlt_in_guest = true;
4908 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
4909 			kvm->arch.pause_in_guest = true;
4910 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
4911 			kvm->arch.cstate_in_guest = true;
4912 		r = 0;
4913 		break;
4914 	case KVM_CAP_MSR_PLATFORM_INFO:
4915 		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
4916 		r = 0;
4917 		break;
4918 	case KVM_CAP_EXCEPTION_PAYLOAD:
4919 		kvm->arch.exception_payload_enabled = cap->args[0];
4920 		r = 0;
4921 		break;
4922 	default:
4923 		r = -EINVAL;
4924 		break;
4925 	}
4926 	return r;
4927 }
4928 
4929 long kvm_arch_vm_ioctl(struct file *filp,
4930 		       unsigned int ioctl, unsigned long arg)
4931 {
4932 	struct kvm *kvm = filp->private_data;
4933 	void __user *argp = (void __user *)arg;
4934 	int r = -ENOTTY;
4935 	/*
4936 	 * This union makes it completely explicit to gcc-3.x
4937 	 * that these two variables' stack usage should be
4938 	 * combined, not added together.
4939 	 */
4940 	union {
4941 		struct kvm_pit_state ps;
4942 		struct kvm_pit_state2 ps2;
4943 		struct kvm_pit_config pit_config;
4944 	} u;
4945 
4946 	switch (ioctl) {
4947 	case KVM_SET_TSS_ADDR:
4948 		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
4949 		break;
4950 	case KVM_SET_IDENTITY_MAP_ADDR: {
4951 		u64 ident_addr;
4952 
4953 		mutex_lock(&kvm->lock);
4954 		r = -EINVAL;
4955 		if (kvm->created_vcpus)
4956 			goto set_identity_unlock;
4957 		r = -EFAULT;
4958 		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
4959 			goto set_identity_unlock;
4960 		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
4961 set_identity_unlock:
4962 		mutex_unlock(&kvm->lock);
4963 		break;
4964 	}
4965 	case KVM_SET_NR_MMU_PAGES:
4966 		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
4967 		break;
4968 	case KVM_GET_NR_MMU_PAGES:
4969 		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
4970 		break;
4971 	case KVM_CREATE_IRQCHIP: {
4972 		mutex_lock(&kvm->lock);
4973 
4974 		r = -EEXIST;
4975 		if (irqchip_in_kernel(kvm))
4976 			goto create_irqchip_unlock;
4977 
4978 		r = -EINVAL;
4979 		if (kvm->created_vcpus)
4980 			goto create_irqchip_unlock;
4981 
4982 		r = kvm_pic_init(kvm);
4983 		if (r)
4984 			goto create_irqchip_unlock;
4985 
4986 		r = kvm_ioapic_init(kvm);
4987 		if (r) {
4988 			kvm_pic_destroy(kvm);
4989 			goto create_irqchip_unlock;
4990 		}
4991 
4992 		r = kvm_setup_default_irq_routing(kvm);
4993 		if (r) {
4994 			kvm_ioapic_destroy(kvm);
4995 			kvm_pic_destroy(kvm);
4996 			goto create_irqchip_unlock;
4997 		}
4998 		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
4999 		smp_wmb();
5000 		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
5001 	create_irqchip_unlock:
5002 		mutex_unlock(&kvm->lock);
5003 		break;
5004 	}
5005 	case KVM_CREATE_PIT:
5006 		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
5007 		goto create_pit;
5008 	case KVM_CREATE_PIT2:
5009 		r = -EFAULT;
5010 		if (copy_from_user(&u.pit_config, argp,
5011 				   sizeof(struct kvm_pit_config)))
5012 			goto out;
5013 	create_pit:
5014 		mutex_lock(&kvm->lock);
5015 		r = -EEXIST;
5016 		if (kvm->arch.vpit)
5017 			goto create_pit_unlock;
5018 		r = -ENOMEM;
5019 		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
5020 		if (kvm->arch.vpit)
5021 			r = 0;
5022 	create_pit_unlock:
5023 		mutex_unlock(&kvm->lock);
5024 		break;
5025 	case KVM_GET_IRQCHIP: {
5026 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5027 		struct kvm_irqchip *chip;
5028 
5029 		chip = memdup_user(argp, sizeof(*chip));
5030 		if (IS_ERR(chip)) {
5031 			r = PTR_ERR(chip);
5032 			goto out;
5033 		}
5034 
5035 		r = -ENXIO;
5036 		if (!irqchip_kernel(kvm))
5037 			goto get_irqchip_out;
5038 		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
5039 		if (r)
5040 			goto get_irqchip_out;
5041 		r = -EFAULT;
5042 		if (copy_to_user(argp, chip, sizeof(*chip)))
5043 			goto get_irqchip_out;
5044 		r = 0;
5045 	get_irqchip_out:
5046 		kfree(chip);
5047 		break;
5048 	}
5049 	case KVM_SET_IRQCHIP: {
5050 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5051 		struct kvm_irqchip *chip;
5052 
5053 		chip = memdup_user(argp, sizeof(*chip));
5054 		if (IS_ERR(chip)) {
5055 			r = PTR_ERR(chip);
5056 			goto out;
5057 		}
5058 
5059 		r = -ENXIO;
5060 		if (!irqchip_kernel(kvm))
5061 			goto set_irqchip_out;
5062 		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
5063 	set_irqchip_out:
5064 		kfree(chip);
5065 		break;
5066 	}
5067 	case KVM_GET_PIT: {
5068 		r = -EFAULT;
5069 		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
5070 			goto out;
5071 		r = -ENXIO;
5072 		if (!kvm->arch.vpit)
5073 			goto out;
5074 		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
5075 		if (r)
5076 			goto out;
5077 		r = -EFAULT;
5078 		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
5079 			goto out;
5080 		r = 0;
5081 		break;
5082 	}
5083 	case KVM_SET_PIT: {
5084 		r = -EFAULT;
5085 		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
5086 			goto out;
5087 		r = -ENXIO;
5088 		if (!kvm->arch.vpit)
5089 			goto out;
5090 		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
5091 		break;
5092 	}
5093 	case KVM_GET_PIT2: {
5094 		r = -ENXIO;
5095 		if (!kvm->arch.vpit)
5096 			goto out;
5097 		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
5098 		if (r)
5099 			goto out;
5100 		r = -EFAULT;
5101 		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
5102 			goto out;
5103 		r = 0;
5104 		break;
5105 	}
5106 	case KVM_SET_PIT2: {
5107 		r = -EFAULT;
5108 		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
5109 			goto out;
5110 		r = -ENXIO;
5111 		if (!kvm->arch.vpit)
5112 			goto out;
5113 		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
5114 		break;
5115 	}
5116 	case KVM_REINJECT_CONTROL: {
5117 		struct kvm_reinject_control control;
5118 		r =  -EFAULT;
5119 		if (copy_from_user(&control, argp, sizeof(control)))
5120 			goto out;
5121 		r = -ENXIO;
5122 		if (!kvm->arch.vpit)
5123 			goto out;
5124 		r = kvm_vm_ioctl_reinject(kvm, &control);
5125 		break;
5126 	}
5127 	case KVM_SET_BOOT_CPU_ID:
5128 		r = 0;
5129 		mutex_lock(&kvm->lock);
5130 		if (kvm->created_vcpus)
5131 			r = -EBUSY;
5132 		else
5133 			kvm->arch.bsp_vcpu_id = arg;
5134 		mutex_unlock(&kvm->lock);
5135 		break;
5136 	case KVM_XEN_HVM_CONFIG: {
5137 		struct kvm_xen_hvm_config xhc;
5138 		r = -EFAULT;
5139 		if (copy_from_user(&xhc, argp, sizeof(xhc)))
5140 			goto out;
5141 		r = -EINVAL;
5142 		if (xhc.flags)
5143 			goto out;
5144 		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
5145 		r = 0;
5146 		break;
5147 	}
5148 	case KVM_SET_CLOCK: {
5149 		struct kvm_clock_data user_ns;
5150 		u64 now_ns;
5151 
5152 		r = -EFAULT;
5153 		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
5154 			goto out;
5155 
5156 		r = -EINVAL;
5157 		if (user_ns.flags)
5158 			goto out;
5159 
5160 		r = 0;
5161 		/*
5162 		 * TODO: userspace has to take care of races with VCPU_RUN, so
5163 		 * kvm_gen_update_masterclock() can be cut down to locked
5164 		 * pvclock_update_vm_gtod_copy().
5165 		 */
5166 		kvm_gen_update_masterclock(kvm);
5167 		now_ns = get_kvmclock_ns(kvm);
5168 		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5169 		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
5170 		break;
5171 	}
5172 	case KVM_GET_CLOCK: {
5173 		struct kvm_clock_data user_ns;
5174 		u64 now_ns;
5175 
5176 		now_ns = get_kvmclock_ns(kvm);
5177 		user_ns.clock = now_ns;
5178 		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
5179 		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
5180 
5181 		r = -EFAULT;
5182 		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
5183 			goto out;
5184 		r = 0;
5185 		break;
5186 	}
5187 	case KVM_MEMORY_ENCRYPT_OP: {
5188 		r = -ENOTTY;
5189 		if (kvm_x86_ops->mem_enc_op)
5190 			r = kvm_x86_ops->mem_enc_op(kvm, argp);
5191 		break;
5192 	}
5193 	case KVM_MEMORY_ENCRYPT_REG_REGION: {
5194 		struct kvm_enc_region region;
5195 
5196 		r = -EFAULT;
5197 		if (copy_from_user(&region, argp, sizeof(region)))
5198 			goto out;
5199 
5200 		r = -ENOTTY;
5201 		if (kvm_x86_ops->mem_enc_reg_region)
5202 			r = kvm_x86_ops->mem_enc_reg_region(kvm, &region);
5203 		break;
5204 	}
5205 	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
5206 		struct kvm_enc_region region;
5207 
5208 		r = -EFAULT;
5209 		if (copy_from_user(&region, argp, sizeof(region)))
5210 			goto out;
5211 
5212 		r = -ENOTTY;
5213 		if (kvm_x86_ops->mem_enc_unreg_region)
5214 			r = kvm_x86_ops->mem_enc_unreg_region(kvm, &region);
5215 		break;
5216 	}
5217 	case KVM_HYPERV_EVENTFD: {
5218 		struct kvm_hyperv_eventfd hvevfd;
5219 
5220 		r = -EFAULT;
5221 		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
5222 			goto out;
5223 		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
5224 		break;
5225 	}
5226 	case KVM_SET_PMU_EVENT_FILTER:
5227 		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
5228 		break;
5229 	default:
5230 		r = -ENOTTY;
5231 	}
5232 out:
5233 	return r;
5234 }
5235 
5236 static void kvm_init_msr_list(void)
5237 {
5238 	struct x86_pmu_capability x86_pmu;
5239 	u32 dummy[2];
5240 	unsigned i;
5241 
5242 	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
5243 			 "Please update the fixed PMCs in msrs_to_saved_all[]");
5244 
5245 	perf_get_x86_pmu_capability(&x86_pmu);
5246 
5247 	num_msrs_to_save = 0;
5248 	num_emulated_msrs = 0;
5249 	num_msr_based_features = 0;
5250 
5251 	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
5252 		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
5253 			continue;
5254 
5255 		/*
5256 		 * Even MSRs that are valid in the host may not be exposed
5257 		 * to the guests in some cases.
5258 		 */
5259 		switch (msrs_to_save_all[i]) {
5260 		case MSR_IA32_BNDCFGS:
5261 			if (!kvm_mpx_supported())
5262 				continue;
5263 			break;
5264 		case MSR_TSC_AUX:
5265 			if (!kvm_x86_ops->rdtscp_supported())
5266 				continue;
5267 			break;
5268 		case MSR_IA32_RTIT_CTL:
5269 		case MSR_IA32_RTIT_STATUS:
5270 			if (!kvm_x86_ops->pt_supported())
5271 				continue;
5272 			break;
5273 		case MSR_IA32_RTIT_CR3_MATCH:
5274 			if (!kvm_x86_ops->pt_supported() ||
5275 			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
5276 				continue;
5277 			break;
5278 		case MSR_IA32_RTIT_OUTPUT_BASE:
5279 		case MSR_IA32_RTIT_OUTPUT_MASK:
5280 			if (!kvm_x86_ops->pt_supported() ||
5281 				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
5282 				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
5283 				continue;
5284 			break;
5285 		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: {
5286 			if (!kvm_x86_ops->pt_supported() ||
5287 				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
5288 				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
5289 				continue;
5290 			break;
5291 		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
5292 			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
5293 			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
5294 				continue;
5295 			break;
5296 		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
5297 			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
5298 			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
5299 				continue;
5300 		}
5301 		default:
5302 			break;
5303 		}
5304 
5305 		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
5306 	}
5307 
5308 	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
5309 		if (!kvm_x86_ops->has_emulated_msr(emulated_msrs_all[i]))
5310 			continue;
5311 
5312 		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
5313 	}
5314 
5315 	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
5316 		struct kvm_msr_entry msr;
5317 
5318 		msr.index = msr_based_features_all[i];
5319 		if (kvm_get_msr_feature(&msr))
5320 			continue;
5321 
5322 		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
5323 	}
5324 }
5325 
5326 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
5327 			   const void *v)
5328 {
5329 	int handled = 0;
5330 	int n;
5331 
5332 	do {
5333 		n = min(len, 8);
5334 		if (!(lapic_in_kernel(vcpu) &&
5335 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
5336 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5337 			break;
5338 		handled += n;
5339 		addr += n;
5340 		len -= n;
5341 		v += n;
5342 	} while (len);
5343 
5344 	return handled;
5345 }
5346 
5347 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5348 {
5349 	int handled = 0;
5350 	int n;
5351 
5352 	do {
5353 		n = min(len, 8);
5354 		if (!(lapic_in_kernel(vcpu) &&
5355 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
5356 					 addr, n, v))
5357 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5358 			break;
5359 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5360 		handled += n;
5361 		addr += n;
5362 		len -= n;
5363 		v += n;
5364 	} while (len);
5365 
5366 	return handled;
5367 }
5368 
5369 static void kvm_set_segment(struct kvm_vcpu *vcpu,
5370 			struct kvm_segment *var, int seg)
5371 {
5372 	kvm_x86_ops->set_segment(vcpu, var, seg);
5373 }
5374 
5375 void kvm_get_segment(struct kvm_vcpu *vcpu,
5376 		     struct kvm_segment *var, int seg)
5377 {
5378 	kvm_x86_ops->get_segment(vcpu, var, seg);
5379 }
5380 
5381 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
5382 			   struct x86_exception *exception)
5383 {
5384 	gpa_t t_gpa;
5385 
5386 	BUG_ON(!mmu_is_nested(vcpu));
5387 
5388 	/* NPT walks are always user-walks */
5389 	access |= PFERR_USER_MASK;
5390 	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5391 
5392 	return t_gpa;
5393 }
5394 
5395 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
5396 			      struct x86_exception *exception)
5397 {
5398 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5399 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5400 }
5401 
5402  gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
5403 				struct x86_exception *exception)
5404 {
5405 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5406 	access |= PFERR_FETCH_MASK;
5407 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5408 }
5409 
5410 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
5411 			       struct x86_exception *exception)
5412 {
5413 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5414 	access |= PFERR_WRITE_MASK;
5415 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5416 }
5417 
5418 /* uses this to access any guest's mapped memory without checking CPL */
5419 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
5420 				struct x86_exception *exception)
5421 {
5422 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5423 }
5424 
5425 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
5426 				      struct kvm_vcpu *vcpu, u32 access,
5427 				      struct x86_exception *exception)
5428 {
5429 	void *data = val;
5430 	int r = X86EMUL_CONTINUE;
5431 
5432 	while (bytes) {
5433 		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5434 							    exception);
5435 		unsigned offset = addr & (PAGE_SIZE-1);
5436 		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5437 		int ret;
5438 
5439 		if (gpa == UNMAPPED_GVA)
5440 			return X86EMUL_PROPAGATE_FAULT;
5441 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
5442 					       offset, toread);
5443 		if (ret < 0) {
5444 			r = X86EMUL_IO_NEEDED;
5445 			goto out;
5446 		}
5447 
5448 		bytes -= toread;
5449 		data += toread;
5450 		addr += toread;
5451 	}
5452 out:
5453 	return r;
5454 }
5455 
5456 /* used for instruction fetching */
5457 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
5458 				gva_t addr, void *val, unsigned int bytes,
5459 				struct x86_exception *exception)
5460 {
5461 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5462 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5463 	unsigned offset;
5464 	int ret;
5465 
5466 	/* Inline kvm_read_guest_virt_helper for speed.  */
5467 	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
5468 						    exception);
5469 	if (unlikely(gpa == UNMAPPED_GVA))
5470 		return X86EMUL_PROPAGATE_FAULT;
5471 
5472 	offset = addr & (PAGE_SIZE-1);
5473 	if (WARN_ON(offset + bytes > PAGE_SIZE))
5474 		bytes = (unsigned)PAGE_SIZE - offset;
5475 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
5476 				       offset, bytes);
5477 	if (unlikely(ret < 0))
5478 		return X86EMUL_IO_NEEDED;
5479 
5480 	return X86EMUL_CONTINUE;
5481 }
5482 
5483 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5484 			       gva_t addr, void *val, unsigned int bytes,
5485 			       struct x86_exception *exception)
5486 {
5487 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5488 
5489 	/*
5490 	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
5491 	 * is returned, but our callers are not ready for that and they blindly
5492 	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
5493 	 * uninitialized kernel stack memory into cr2 and error code.
5494 	 */
5495 	memset(exception, 0, sizeof(*exception));
5496 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5497 					  exception);
5498 }
5499 EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5500 
5501 static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
5502 			     gva_t addr, void *val, unsigned int bytes,
5503 			     struct x86_exception *exception, bool system)
5504 {
5505 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5506 	u32 access = 0;
5507 
5508 	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
5509 		access |= PFERR_USER_MASK;
5510 
5511 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5512 }
5513 
5514 static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
5515 		unsigned long addr, void *val, unsigned int bytes)
5516 {
5517 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5518 	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);
5519 
5520 	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
5521 }
5522 
5523 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
5524 				      struct kvm_vcpu *vcpu, u32 access,
5525 				      struct x86_exception *exception)
5526 {
5527 	void *data = val;
5528 	int r = X86EMUL_CONTINUE;
5529 
5530 	while (bytes) {
5531 		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5532 							     access,
5533 							     exception);
5534 		unsigned offset = addr & (PAGE_SIZE-1);
5535 		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
5536 		int ret;
5537 
5538 		if (gpa == UNMAPPED_GVA)
5539 			return X86EMUL_PROPAGATE_FAULT;
5540 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5541 		if (ret < 0) {
5542 			r = X86EMUL_IO_NEEDED;
5543 			goto out;
5544 		}
5545 
5546 		bytes -= towrite;
5547 		data += towrite;
5548 		addr += towrite;
5549 	}
5550 out:
5551 	return r;
5552 }
5553 
5554 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5555 			      unsigned int bytes, struct x86_exception *exception,
5556 			      bool system)
5557 {
5558 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5559 	u32 access = PFERR_WRITE_MASK;
5560 
5561 	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
5562 		access |= PFERR_USER_MASK;
5563 
5564 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5565 					   access, exception);
5566 }
5567 
5568 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
5569 				unsigned int bytes, struct x86_exception *exception)
5570 {
5571 	/* kvm_write_guest_virt_system can pull in tons of pages. */
5572 	vcpu->arch.l1tf_flush_l1d = true;
5573 
5574 	/*
5575 	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
5576 	 * is returned, but our callers are not ready for that and they blindly
5577 	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
5578 	 * uninitialized kernel stack memory into cr2 and error code.
5579 	 */
5580 	memset(exception, 0, sizeof(*exception));
5581 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5582 					   PFERR_WRITE_MASK, exception);
5583 }
5584 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5585 
5586 int handle_ud(struct kvm_vcpu *vcpu)
5587 {
5588 	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
5589 	int emul_type = EMULTYPE_TRAP_UD;
5590 	char sig[5]; /* ud2; .ascii "kvm" */
5591 	struct x86_exception e;
5592 
5593 	if (force_emulation_prefix &&
5594 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
5595 				sig, sizeof(sig), &e) == 0 &&
5596 	    memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
5597 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
5598 		emul_type = EMULTYPE_TRAP_UD_FORCED;
5599 	}
5600 
5601 	return kvm_emulate_instruction(vcpu, emul_type);
5602 }
5603 EXPORT_SYMBOL_GPL(handle_ud);
5604 
5605 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
5606 			    gpa_t gpa, bool write)
5607 {
5608 	/* For APIC access vmexit */
5609 	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
5610 		return 1;
5611 
5612 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
5613 		trace_vcpu_match_mmio(gva, gpa, write, true);
5614 		return 1;
5615 	}
5616 
5617 	return 0;
5618 }
5619 
5620 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
5621 				gpa_t *gpa, struct x86_exception *exception,
5622 				bool write)
5623 {
5624 	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
5625 		| (write ? PFERR_WRITE_MASK : 0);
5626 
5627 	/*
5628 	 * currently PKRU is only applied to ept enabled guest so
5629 	 * there is no pkey in EPT page table for L1 guest or EPT
5630 	 * shadow page table for L2 guest.
5631 	 */
5632 	if (vcpu_match_mmio_gva(vcpu, gva)
5633 	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5634 				 vcpu->arch.mmio_access, 0, access)) {
5635 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
5636 					(gva & (PAGE_SIZE - 1));
5637 		trace_vcpu_match_mmio(gva, *gpa, write, false);
5638 		return 1;
5639 	}
5640 
5641 	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5642 
5643 	if (*gpa == UNMAPPED_GVA)
5644 		return -1;
5645 
5646 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5647 }
5648 
5649 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5650 			const void *val, int bytes)
5651 {
5652 	int ret;
5653 
5654 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5655 	if (ret < 0)
5656 		return 0;
5657 	kvm_page_track_write(vcpu, gpa, val, bytes);
5658 	return 1;
5659 }
5660 
5661 struct read_write_emulator_ops {
5662 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
5663 				  int bytes);
5664 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
5665 				  void *val, int bytes);
5666 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
5667 			       int bytes, void *val);
5668 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
5669 				    void *val, int bytes);
5670 	bool write;
5671 };
5672 
5673 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
5674 {
5675 	if (vcpu->mmio_read_completed) {
5676 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5677 			       vcpu->mmio_fragments[0].gpa, val);
5678 		vcpu->mmio_read_completed = 0;
5679 		return 1;
5680 	}
5681 
5682 	return 0;
5683 }
5684 
5685 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
5686 			void *val, int bytes)
5687 {
5688 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5689 }
5690 
5691 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
5692 			 void *val, int bytes)
5693 {
5694 	return emulator_write_phys(vcpu, gpa, val, bytes);
5695 }
5696 
5697 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
5698 {
5699 	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5700 	return vcpu_mmio_write(vcpu, gpa, bytes, val);
5701 }
5702 
5703 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
5704 			  void *val, int bytes)
5705 {
5706 	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5707 	return X86EMUL_IO_NEEDED;
5708 }
5709 
5710 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
5711 			   void *val, int bytes)
5712 {
5713 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
5714 
5715 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5716 	return X86EMUL_CONTINUE;
5717 }
5718 
5719 static const struct read_write_emulator_ops read_emultor = {
5720 	.read_write_prepare = read_prepare,
5721 	.read_write_emulate = read_emulate,
5722 	.read_write_mmio = vcpu_mmio_read,
5723 	.read_write_exit_mmio = read_exit_mmio,
5724 };
5725 
5726 static const struct read_write_emulator_ops write_emultor = {
5727 	.read_write_emulate = write_emulate,
5728 	.read_write_mmio = write_mmio,
5729 	.read_write_exit_mmio = write_exit_mmio,
5730 	.write = true,
5731 };
5732 
5733 static int emulator_read_write_onepage(unsigned long addr, void *val,
5734 				       unsigned int bytes,
5735 				       struct x86_exception *exception,
5736 				       struct kvm_vcpu *vcpu,
5737 				       const struct read_write_emulator_ops *ops)
5738 {
5739 	gpa_t gpa;
5740 	int handled, ret;
5741 	bool write = ops->write;
5742 	struct kvm_mmio_fragment *frag;
5743 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
5744 
5745 	/*
5746 	 * If the exit was due to a NPF we may already have a GPA.
5747 	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
5748 	 * Note, this cannot be used on string operations since string
5749 	 * operation using rep will only have the initial GPA from the NPF
5750 	 * occurred.
5751 	 */
5752 	if (vcpu->arch.gpa_available &&
5753 	    emulator_can_use_gpa(ctxt) &&
5754 	    (addr & ~PAGE_MASK) == (vcpu->arch.gpa_val & ~PAGE_MASK)) {
5755 		gpa = vcpu->arch.gpa_val;
5756 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
5757 	} else {
5758 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
5759 		if (ret < 0)
5760 			return X86EMUL_PROPAGATE_FAULT;
5761 	}
5762 
5763 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5764 		return X86EMUL_CONTINUE;
5765 
5766 	/*
5767 	 * Is this MMIO handled locally?
5768 	 */
5769 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5770 	if (handled == bytes)
5771 		return X86EMUL_CONTINUE;
5772 
5773 	gpa += handled;
5774 	bytes -= handled;
5775 	val += handled;
5776 
5777 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
5778 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
5779 	frag->gpa = gpa;
5780 	frag->data = val;
5781 	frag->len = bytes;
5782 	return X86EMUL_CONTINUE;
5783 }
5784 
5785 static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
5786 			unsigned long addr,
5787 			void *val, unsigned int bytes,
5788 			struct x86_exception *exception,
5789 			const struct read_write_emulator_ops *ops)
5790 {
5791 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5792 	gpa_t gpa;
5793 	int rc;
5794 
5795 	if (ops->read_write_prepare &&
5796 		  ops->read_write_prepare(vcpu, val, bytes))
5797 		return X86EMUL_CONTINUE;
5798 
5799 	vcpu->mmio_nr_fragments = 0;
5800 
5801 	/* Crossing a page boundary? */
5802 	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
5803 		int now;
5804 
5805 		now = -addr & ~PAGE_MASK;
5806 		rc = emulator_read_write_onepage(addr, val, now, exception,
5807 						 vcpu, ops);
5808 
5809 		if (rc != X86EMUL_CONTINUE)
5810 			return rc;
5811 		addr += now;
5812 		if (ctxt->mode != X86EMUL_MODE_PROT64)
5813 			addr = (u32)addr;
5814 		val += now;
5815 		bytes -= now;
5816 	}
5817 
5818 	rc = emulator_read_write_onepage(addr, val, bytes, exception,
5819 					 vcpu, ops);
5820 	if (rc != X86EMUL_CONTINUE)
5821 		return rc;
5822 
5823 	if (!vcpu->mmio_nr_fragments)
5824 		return rc;
5825 
5826 	gpa = vcpu->mmio_fragments[0].gpa;
5827 
5828 	vcpu->mmio_needed = 1;
5829 	vcpu->mmio_cur_fragment = 0;
5830 
5831 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
5832 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
5833 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
5834 	vcpu->run->mmio.phys_addr = gpa;
5835 
5836 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5837 }
5838 
5839 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
5840 				  unsigned long addr,
5841 				  void *val,
5842 				  unsigned int bytes,
5843 				  struct x86_exception *exception)
5844 {
5845 	return emulator_read_write(ctxt, addr, val, bytes,
5846 				   exception, &read_emultor);
5847 }
5848 
5849 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5850 			    unsigned long addr,
5851 			    const void *val,
5852 			    unsigned int bytes,
5853 			    struct x86_exception *exception)
5854 {
5855 	return emulator_read_write(ctxt, addr, (void *)val, bytes,
5856 				   exception, &write_emultor);
5857 }
5858 
5859 #define CMPXCHG_TYPE(t, ptr, old, new) \
5860 	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))
5861 
5862 #ifdef CONFIG_X86_64
5863 #  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
5864 #else
5865 #  define CMPXCHG64(ptr, old, new) \
5866 	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5867 #endif
5868 
5869 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
5870 				     unsigned long addr,
5871 				     const void *old,
5872 				     const void *new,
5873 				     unsigned int bytes,
5874 				     struct x86_exception *exception)
5875 {
5876 	struct kvm_host_map map;
5877 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5878 	gpa_t gpa;
5879 	char *kaddr;
5880 	bool exchanged;
5881 
5882 	/* guests cmpxchg8b have to be emulated atomically */
5883 	if (bytes > 8 || (bytes & (bytes - 1)))
5884 		goto emul_write;
5885 
5886 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5887 
5888 	if (gpa == UNMAPPED_GVA ||
5889 	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
5890 		goto emul_write;
5891 
5892 	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
5893 		goto emul_write;
5894 
5895 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5896 		goto emul_write;
5897 
5898 	kaddr = map.hva + offset_in_page(gpa);
5899 
5900 	switch (bytes) {
5901 	case 1:
5902 		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
5903 		break;
5904 	case 2:
5905 		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
5906 		break;
5907 	case 4:
5908 		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
5909 		break;
5910 	case 8:
5911 		exchanged = CMPXCHG64(kaddr, old, new);
5912 		break;
5913 	default:
5914 		BUG();
5915 	}
5916 
5917 	kvm_vcpu_unmap(vcpu, &map, true);
5918 
5919 	if (!exchanged)
5920 		return X86EMUL_CMPXCHG_FAILED;
5921 
5922 	kvm_page_track_write(vcpu, gpa, new, bytes);
5923 
5924 	return X86EMUL_CONTINUE;
5925 
5926 emul_write:
5927 	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
5928 
5929 	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
5930 }
5931 
5932 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
5933 {
5934 	int r = 0, i;
5935 
5936 	for (i = 0; i < vcpu->arch.pio.count; i++) {
5937 		if (vcpu->arch.pio.in)
5938 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
5939 					    vcpu->arch.pio.size, pd);
5940 		else
5941 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
5942 					     vcpu->arch.pio.port, vcpu->arch.pio.size,
5943 					     pd);
5944 		if (r)
5945 			break;
5946 		pd += vcpu->arch.pio.size;
5947 	}
5948 	return r;
5949 }
5950 
5951 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
5952 			       unsigned short port, void *val,
5953 			       unsigned int count, bool in)
5954 {
5955 	vcpu->arch.pio.port = port;
5956 	vcpu->arch.pio.in = in;
5957 	vcpu->arch.pio.count  = count;
5958 	vcpu->arch.pio.size = size;
5959 
5960 	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
5961 		vcpu->arch.pio.count = 0;
5962 		return 1;
5963 	}
5964 
5965 	vcpu->run->exit_reason = KVM_EXIT_IO;
5966 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
5967 	vcpu->run->io.size = size;
5968 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
5969 	vcpu->run->io.count = count;
5970 	vcpu->run->io.port = port;
5971 
5972 	return 0;
5973 }
5974 
5975 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
5976 				    int size, unsigned short port, void *val,
5977 				    unsigned int count)
5978 {
5979 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5980 	int ret;
5981 
5982 	if (vcpu->arch.pio.count)
5983 		goto data_avail;
5984 
5985 	memset(vcpu->arch.pio_data, 0, size * count);
5986 
5987 	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
5988 	if (ret) {
5989 data_avail:
5990 		memcpy(val, vcpu->arch.pio_data, size * count);
5991 		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
5992 		vcpu->arch.pio.count = 0;
5993 		return 1;
5994 	}
5995 
5996 	return 0;
5997 }
5998 
5999 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
6000 				     int size, unsigned short port,
6001 				     const void *val, unsigned int count)
6002 {
6003 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6004 
6005 	memcpy(vcpu->arch.pio_data, val, size * count);
6006 	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
6007 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
6008 }
6009 
6010 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
6011 {
6012 	return kvm_x86_ops->get_segment_base(vcpu, seg);
6013 }
6014 
6015 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
6016 {
6017 	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
6018 }
6019 
6020 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
6021 {
6022 	if (!need_emulate_wbinvd(vcpu))
6023 		return X86EMUL_CONTINUE;
6024 
6025 	if (kvm_x86_ops->has_wbinvd_exit()) {
6026 		int cpu = get_cpu();
6027 
6028 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
6029 		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
6030 				wbinvd_ipi, NULL, 1);
6031 		put_cpu();
6032 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
6033 	} else
6034 		wbinvd();
6035 	return X86EMUL_CONTINUE;
6036 }
6037 
6038 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
6039 {
6040 	kvm_emulate_wbinvd_noskip(vcpu);
6041 	return kvm_skip_emulated_instruction(vcpu);
6042 }
6043 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
6044 
6045 
6046 
6047 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
6048 {
6049 	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
6050 }
6051 
6052 static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
6053 			   unsigned long *dest)
6054 {
6055 	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
6056 }
6057 
6058 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
6059 			   unsigned long value)
6060 {
6061 
6062 	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
6063 }
6064 
6065 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
6066 {
6067 	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
6068 }
6069 
6070 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
6071 {
6072 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6073 	unsigned long value;
6074 
6075 	switch (cr) {
6076 	case 0:
6077 		value = kvm_read_cr0(vcpu);
6078 		break;
6079 	case 2:
6080 		value = vcpu->arch.cr2;
6081 		break;
6082 	case 3:
6083 		value = kvm_read_cr3(vcpu);
6084 		break;
6085 	case 4:
6086 		value = kvm_read_cr4(vcpu);
6087 		break;
6088 	case 8:
6089 		value = kvm_get_cr8(vcpu);
6090 		break;
6091 	default:
6092 		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6093 		return 0;
6094 	}
6095 
6096 	return value;
6097 }
6098 
6099 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
6100 {
6101 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6102 	int res = 0;
6103 
6104 	switch (cr) {
6105 	case 0:
6106 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
6107 		break;
6108 	case 2:
6109 		vcpu->arch.cr2 = val;
6110 		break;
6111 	case 3:
6112 		res = kvm_set_cr3(vcpu, val);
6113 		break;
6114 	case 4:
6115 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
6116 		break;
6117 	case 8:
6118 		res = kvm_set_cr8(vcpu, val);
6119 		break;
6120 	default:
6121 		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6122 		res = -1;
6123 	}
6124 
6125 	return res;
6126 }
6127 
6128 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6129 {
6130 	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
6131 }
6132 
6133 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6134 {
6135 	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
6136 }
6137 
6138 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6139 {
6140 	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
6141 }
6142 
6143 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6144 {
6145 	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
6146 }
6147 
6148 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6149 {
6150 	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
6151 }
6152 
6153 static unsigned long emulator_get_cached_segment_base(
6154 	struct x86_emulate_ctxt *ctxt, int seg)
6155 {
6156 	return get_segment_base(emul_to_vcpu(ctxt), seg);
6157 }
6158 
6159 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
6160 				 struct desc_struct *desc, u32 *base3,
6161 				 int seg)
6162 {
6163 	struct kvm_segment var;
6164 
6165 	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
6166 	*selector = var.selector;
6167 
6168 	if (var.unusable) {
6169 		memset(desc, 0, sizeof(*desc));
6170 		if (base3)
6171 			*base3 = 0;
6172 		return false;
6173 	}
6174 
6175 	if (var.g)
6176 		var.limit >>= 12;
6177 	set_desc_limit(desc, var.limit);
6178 	set_desc_base(desc, (unsigned long)var.base);
6179 #ifdef CONFIG_X86_64
6180 	if (base3)
6181 		*base3 = var.base >> 32;
6182 #endif
6183 	desc->type = var.type;
6184 	desc->s = var.s;
6185 	desc->dpl = var.dpl;
6186 	desc->p = var.present;
6187 	desc->avl = var.avl;
6188 	desc->l = var.l;
6189 	desc->d = var.db;
6190 	desc->g = var.g;
6191 
6192 	return true;
6193 }
6194 
6195 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
6196 				 struct desc_struct *desc, u32 base3,
6197 				 int seg)
6198 {
6199 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6200 	struct kvm_segment var;
6201 
6202 	var.selector = selector;
6203 	var.base = get_desc_base(desc);
6204 #ifdef CONFIG_X86_64
6205 	var.base |= ((u64)base3) << 32;
6206 #endif
6207 	var.limit = get_desc_limit(desc);
6208 	if (desc->g)
6209 		var.limit = (var.limit << 12) | 0xfff;
6210 	var.type = desc->type;
6211 	var.dpl = desc->dpl;
6212 	var.db = desc->d;
6213 	var.s = desc->s;
6214 	var.l = desc->l;
6215 	var.g = desc->g;
6216 	var.avl = desc->avl;
6217 	var.present = desc->p;
6218 	var.unusable = !var.present;
6219 	var.padding = 0;
6220 
6221 	kvm_set_segment(vcpu, &var, seg);
6222 	return;
6223 }
6224 
6225 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
6226 			    u32 msr_index, u64 *pdata)
6227 {
6228 	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
6229 }
6230 
6231 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
6232 			    u32 msr_index, u64 data)
6233 {
6234 	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
6235 }
6236 
6237 static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
6238 {
6239 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6240 
6241 	return vcpu->arch.smbase;
6242 }
6243 
6244 static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
6245 {
6246 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6247 
6248 	vcpu->arch.smbase = smbase;
6249 }
6250 
6251 static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
6252 			      u32 pmc)
6253 {
6254 	return kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc);
6255 }
6256 
6257 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
6258 			     u32 pmc, u64 *pdata)
6259 {
6260 	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
6261 }
6262 
6263 static void emulator_halt(struct x86_emulate_ctxt *ctxt)
6264 {
6265 	emul_to_vcpu(ctxt)->arch.halt_request = 1;
6266 }
6267 
6268 static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
6269 			      struct x86_instruction_info *info,
6270 			      enum x86_intercept_stage stage)
6271 {
6272 	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
6273 }
6274 
6275 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
6276 			u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool check_limit)
6277 {
6278 	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, check_limit);
6279 }
6280 
6281 static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
6282 {
6283 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
6284 }
6285 
6286 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
6287 {
6288 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
6289 }
6290 
6291 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
6292 {
6293 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
6294 }
6295 
6296 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
6297 {
6298 	return kvm_register_read(emul_to_vcpu(ctxt), reg);
6299 }
6300 
6301 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
6302 {
6303 	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
6304 }
6305 
6306 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
6307 {
6308 	kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked);
6309 }
6310 
6311 static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
6312 {
6313 	return emul_to_vcpu(ctxt)->arch.hflags;
6314 }
6315 
6316 static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
6317 {
6318 	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6319 }
6320 
6321 static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
6322 				  const char *smstate)
6323 {
6324 	return kvm_x86_ops->pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6325 }
6326 
6327 static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
6328 {
6329 	kvm_smm_changed(emul_to_vcpu(ctxt));
6330 }
6331 
6332 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
6333 {
6334 	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
6335 }
6336 
6337 static const struct x86_emulate_ops emulate_ops = {
6338 	.read_gpr            = emulator_read_gpr,
6339 	.write_gpr           = emulator_write_gpr,
6340 	.read_std            = emulator_read_std,
6341 	.write_std           = emulator_write_std,
6342 	.read_phys           = kvm_read_guest_phys_system,
6343 	.fetch               = kvm_fetch_guest_virt,
6344 	.read_emulated       = emulator_read_emulated,
6345 	.write_emulated      = emulator_write_emulated,
6346 	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6347 	.invlpg              = emulator_invlpg,
6348 	.pio_in_emulated     = emulator_pio_in_emulated,
6349 	.pio_out_emulated    = emulator_pio_out_emulated,
6350 	.get_segment         = emulator_get_segment,
6351 	.set_segment         = emulator_set_segment,
6352 	.get_cached_segment_base = emulator_get_cached_segment_base,
6353 	.get_gdt             = emulator_get_gdt,
6354 	.get_idt	     = emulator_get_idt,
6355 	.set_gdt             = emulator_set_gdt,
6356 	.set_idt	     = emulator_set_idt,
6357 	.get_cr              = emulator_get_cr,
6358 	.set_cr              = emulator_set_cr,
6359 	.cpl                 = emulator_get_cpl,
6360 	.get_dr              = emulator_get_dr,
6361 	.set_dr              = emulator_set_dr,
6362 	.get_smbase          = emulator_get_smbase,
6363 	.set_smbase          = emulator_set_smbase,
6364 	.set_msr             = emulator_set_msr,
6365 	.get_msr             = emulator_get_msr,
6366 	.check_pmc	     = emulator_check_pmc,
6367 	.read_pmc            = emulator_read_pmc,
6368 	.halt                = emulator_halt,
6369 	.wbinvd              = emulator_wbinvd,
6370 	.fix_hypercall       = emulator_fix_hypercall,
6371 	.intercept           = emulator_intercept,
6372 	.get_cpuid           = emulator_get_cpuid,
6373 	.guest_has_long_mode = emulator_guest_has_long_mode,
6374 	.guest_has_movbe     = emulator_guest_has_movbe,
6375 	.guest_has_fxsr      = emulator_guest_has_fxsr,
6376 	.set_nmi_mask        = emulator_set_nmi_mask,
6377 	.get_hflags          = emulator_get_hflags,
6378 	.set_hflags          = emulator_set_hflags,
6379 	.pre_leave_smm       = emulator_pre_leave_smm,
6380 	.post_leave_smm      = emulator_post_leave_smm,
6381 	.set_xcr             = emulator_set_xcr,
6382 };
6383 
6384 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
6385 {
6386 	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
6387 	/*
6388 	 * an sti; sti; sequence only disable interrupts for the first
6389 	 * instruction. So, if the last instruction, be it emulated or
6390 	 * not, left the system with the INT_STI flag enabled, it
6391 	 * means that the last instruction is an sti. We should not
6392 	 * leave the flag on in this case. The same goes for mov ss
6393 	 */
6394 	if (int_shadow & mask)
6395 		mask = 0;
6396 	if (unlikely(int_shadow || mask)) {
6397 		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
6398 		if (!mask)
6399 			kvm_make_request(KVM_REQ_EVENT, vcpu);
6400 	}
6401 }
6402 
6403 static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6404 {
6405 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6406 	if (ctxt->exception.vector == PF_VECTOR)
6407 		return kvm_propagate_fault(vcpu, &ctxt->exception);
6408 
6409 	if (ctxt->exception.error_code_valid)
6410 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
6411 				      ctxt->exception.error_code);
6412 	else
6413 		kvm_queue_exception(vcpu, ctxt->exception.vector);
6414 	return false;
6415 }
6416 
6417 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
6418 {
6419 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6420 	int cs_db, cs_l;
6421 
6422 	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
6423 
6424 	ctxt->eflags = kvm_get_rflags(vcpu);
6425 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
6426 
6427 	ctxt->eip = kvm_rip_read(vcpu);
6428 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
6429 		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6430 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6431 		     cs_db				? X86EMUL_MODE_PROT32 :
6432 							  X86EMUL_MODE_PROT16;
6433 	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
6434 	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
6435 	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6436 
6437 	init_decode_cache(ctxt);
6438 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6439 }
6440 
6441 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6442 {
6443 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6444 	int ret;
6445 
6446 	init_emulate_ctxt(vcpu);
6447 
6448 	ctxt->op_bytes = 2;
6449 	ctxt->ad_bytes = 2;
6450 	ctxt->_eip = ctxt->eip + inc_eip;
6451 	ret = emulate_int_real(ctxt, irq);
6452 
6453 	if (ret != X86EMUL_CONTINUE) {
6454 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
6455 	} else {
6456 		ctxt->eip = ctxt->_eip;
6457 		kvm_rip_write(vcpu, ctxt->eip);
6458 		kvm_set_rflags(vcpu, ctxt->eflags);
6459 	}
6460 }
6461 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
6462 
6463 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6464 {
6465 	++vcpu->stat.insn_emulation_fail;
6466 	trace_kvm_emulate_insn_failed(vcpu);
6467 
6468 	if (emulation_type & EMULTYPE_VMWARE_GP) {
6469 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6470 		return 1;
6471 	}
6472 
6473 	if (emulation_type & EMULTYPE_SKIP) {
6474 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6475 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
6476 		vcpu->run->internal.ndata = 0;
6477 		return 0;
6478 	}
6479 
6480 	kvm_queue_exception(vcpu, UD_VECTOR);
6481 
6482 	if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) {
6483 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6484 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
6485 		vcpu->run->internal.ndata = 0;
6486 		return 0;
6487 	}
6488 
6489 	return 1;
6490 }
6491 
6492 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
6493 				  bool write_fault_to_shadow_pgtable,
6494 				  int emulation_type)
6495 {
6496 	gpa_t gpa = cr2_or_gpa;
6497 	kvm_pfn_t pfn;
6498 
6499 	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6500 		return false;
6501 
6502 	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
6503 		return false;
6504 
6505 	if (!vcpu->arch.mmu->direct_map) {
6506 		/*
6507 		 * Write permission should be allowed since only
6508 		 * write access need to be emulated.
6509 		 */
6510 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6511 
6512 		/*
6513 		 * If the mapping is invalid in guest, let cpu retry
6514 		 * it to generate fault.
6515 		 */
6516 		if (gpa == UNMAPPED_GVA)
6517 			return true;
6518 	}
6519 
6520 	/*
6521 	 * Do not retry the unhandleable instruction if it faults on the
6522 	 * readonly host memory, otherwise it will goto a infinite loop:
6523 	 * retry instruction -> write #PF -> emulation fail -> retry
6524 	 * instruction -> ...
6525 	 */
6526 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6527 
6528 	/*
6529 	 * If the instruction failed on the error pfn, it can not be fixed,
6530 	 * report the error to userspace.
6531 	 */
6532 	if (is_error_noslot_pfn(pfn))
6533 		return false;
6534 
6535 	kvm_release_pfn_clean(pfn);
6536 
6537 	/* The instructions are well-emulated on direct mmu. */
6538 	if (vcpu->arch.mmu->direct_map) {
6539 		unsigned int indirect_shadow_pages;
6540 
6541 		spin_lock(&vcpu->kvm->mmu_lock);
6542 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
6543 		spin_unlock(&vcpu->kvm->mmu_lock);
6544 
6545 		if (indirect_shadow_pages)
6546 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6547 
6548 		return true;
6549 	}
6550 
6551 	/*
6552 	 * if emulation was due to access to shadowed page table
6553 	 * and it failed try to unshadow page and re-enter the
6554 	 * guest to let CPU execute the instruction.
6555 	 */
6556 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6557 
6558 	/*
6559 	 * If the access faults on its page table, it can not
6560 	 * be fixed by unprotecting shadow page and it should
6561 	 * be reported to userspace.
6562 	 */
6563 	return !write_fault_to_shadow_pgtable;
6564 }
6565 
6566 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
6567 			      gpa_t cr2_or_gpa,  int emulation_type)
6568 {
6569 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6570 	unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
6571 
6572 	last_retry_eip = vcpu->arch.last_retry_eip;
6573 	last_retry_addr = vcpu->arch.last_retry_addr;
6574 
6575 	/*
6576 	 * If the emulation is caused by #PF and it is non-page_table
6577 	 * writing instruction, it means the VM-EXIT is caused by shadow
6578 	 * page protected, we can zap the shadow page and retry this
6579 	 * instruction directly.
6580 	 *
6581 	 * Note: if the guest uses a non-page-table modifying instruction
6582 	 * on the PDE that points to the instruction, then we will unmap
6583 	 * the instruction and go to an infinite loop. So, we cache the
6584 	 * last retried eip and the last fault address, if we meet the eip
6585 	 * and the address again, we can break out of the potential infinite
6586 	 * loop.
6587 	 */
6588 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
6589 
6590 	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6591 		return false;
6592 
6593 	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
6594 		return false;
6595 
6596 	if (x86_page_table_writing_insn(ctxt))
6597 		return false;
6598 
6599 	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
6600 		return false;
6601 
6602 	vcpu->arch.last_retry_eip = ctxt->eip;
6603 	vcpu->arch.last_retry_addr = cr2_or_gpa;
6604 
6605 	if (!vcpu->arch.mmu->direct_map)
6606 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6607 
6608 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6609 
6610 	return true;
6611 }
6612 
6613 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
6614 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
6615 
6616 static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6617 {
6618 	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6619 		/* This is a good place to trace that we are exiting SMM.  */
6620 		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);
6621 
6622 		/* Process a latched INIT or SMI, if any.  */
6623 		kvm_make_request(KVM_REQ_EVENT, vcpu);
6624 	}
6625 
6626 	kvm_mmu_reset_context(vcpu);
6627 }
6628 
6629 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
6630 				unsigned long *db)
6631 {
6632 	u32 dr6 = 0;
6633 	int i;
6634 	u32 enable, rwlen;
6635 
6636 	enable = dr7;
6637 	rwlen = dr7 >> 16;
6638 	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
6639 		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
6640 			dr6 |= (1 << i);
6641 	return dr6;
6642 }
6643 
6644 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
6645 {
6646 	struct kvm_run *kvm_run = vcpu->run;
6647 
6648 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
6649 		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
6650 		kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
6651 		kvm_run->debug.arch.exception = DB_VECTOR;
6652 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
6653 		return 0;
6654 	}
6655 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6656 	return 1;
6657 }
6658 
6659 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
6660 {
6661 	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6662 	int r;
6663 
6664 	r = kvm_x86_ops->skip_emulated_instruction(vcpu);
6665 	if (unlikely(!r))
6666 		return 0;
6667 
6668 	/*
6669 	 * rflags is the old, "raw" value of the flags.  The new value has
6670 	 * not been saved yet.
6671 	 *
6672 	 * This is correct even for TF set by the guest, because "the
6673 	 * processor will not generate this exception after the instruction
6674 	 * that sets the TF flag".
6675 	 */
6676 	if (unlikely(rflags & X86_EFLAGS_TF))
6677 		r = kvm_vcpu_do_singlestep(vcpu);
6678 	return r;
6679 }
6680 EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);
6681 
6682 static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
6683 {
6684 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
6685 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6686 		struct kvm_run *kvm_run = vcpu->run;
6687 		unsigned long eip = kvm_get_linear_rip(vcpu);
6688 		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6689 					   vcpu->arch.guest_debug_dr7,
6690 					   vcpu->arch.eff_db);
6691 
6692 		if (dr6 != 0) {
6693 			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6694 			kvm_run->debug.arch.pc = eip;
6695 			kvm_run->debug.arch.exception = DB_VECTOR;
6696 			kvm_run->exit_reason = KVM_EXIT_DEBUG;
6697 			*r = 0;
6698 			return true;
6699 		}
6700 	}
6701 
6702 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
6703 	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6704 		unsigned long eip = kvm_get_linear_rip(vcpu);
6705 		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6706 					   vcpu->arch.dr7,
6707 					   vcpu->arch.db);
6708 
6709 		if (dr6 != 0) {
6710 			vcpu->arch.dr6 &= ~DR_TRAP_BITS;
6711 			vcpu->arch.dr6 |= dr6 | DR6_RTM;
6712 			kvm_queue_exception(vcpu, DB_VECTOR);
6713 			*r = 1;
6714 			return true;
6715 		}
6716 	}
6717 
6718 	return false;
6719 }
6720 
6721 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
6722 {
6723 	switch (ctxt->opcode_len) {
6724 	case 1:
6725 		switch (ctxt->b) {
6726 		case 0xe4:	/* IN */
6727 		case 0xe5:
6728 		case 0xec:
6729 		case 0xed:
6730 		case 0xe6:	/* OUT */
6731 		case 0xe7:
6732 		case 0xee:
6733 		case 0xef:
6734 		case 0x6c:	/* INS */
6735 		case 0x6d:
6736 		case 0x6e:	/* OUTS */
6737 		case 0x6f:
6738 			return true;
6739 		}
6740 		break;
6741 	case 2:
6742 		switch (ctxt->b) {
6743 		case 0x33:	/* RDPMC */
6744 			return true;
6745 		}
6746 		break;
6747 	}
6748 
6749 	return false;
6750 }
6751 
6752 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
6753 			    int emulation_type, void *insn, int insn_len)
6754 {
6755 	int r;
6756 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6757 	bool writeback = true;
6758 	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6759 
6760 	vcpu->arch.l1tf_flush_l1d = true;
6761 
6762 	/*
6763 	 * Clear write_fault_to_shadow_pgtable here to ensure it is
6764 	 * never reused.
6765 	 */
6766 	vcpu->arch.write_fault_to_shadow_pgtable = false;
6767 	kvm_clear_exception_queue(vcpu);
6768 
6769 	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6770 		init_emulate_ctxt(vcpu);
6771 
6772 		/*
6773 		 * We will reenter on the same instruction since
6774 		 * we do not set complete_userspace_io.  This does not
6775 		 * handle watchpoints yet, those would be handled in
6776 		 * the emulate_ops.
6777 		 */
6778 		if (!(emulation_type & EMULTYPE_SKIP) &&
6779 		    kvm_vcpu_check_breakpoint(vcpu, &r))
6780 			return r;
6781 
6782 		ctxt->interruptibility = 0;
6783 		ctxt->have_exception = false;
6784 		ctxt->exception.vector = -1;
6785 		ctxt->perm_ok = false;
6786 
6787 		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6788 
6789 		r = x86_decode_insn(ctxt, insn, insn_len);
6790 
6791 		trace_kvm_emulate_insn_start(vcpu);
6792 		++vcpu->stat.insn_emulation;
6793 		if (r != EMULATION_OK)  {
6794 			if ((emulation_type & EMULTYPE_TRAP_UD) ||
6795 			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
6796 				kvm_queue_exception(vcpu, UD_VECTOR);
6797 				return 1;
6798 			}
6799 			if (reexecute_instruction(vcpu, cr2_or_gpa,
6800 						  write_fault_to_spt,
6801 						  emulation_type))
6802 				return 1;
6803 			if (ctxt->have_exception) {
6804 				/*
6805 				 * #UD should result in just EMULATION_FAILED, and trap-like
6806 				 * exception should not be encountered during decode.
6807 				 */
6808 				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
6809 					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
6810 				inject_emulated_exception(vcpu);
6811 				return 1;
6812 			}
6813 			return handle_emulation_failure(vcpu, emulation_type);
6814 		}
6815 	}
6816 
6817 	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
6818 	    !is_vmware_backdoor_opcode(ctxt)) {
6819 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6820 		return 1;
6821 	}
6822 
6823 	/*
6824 	 * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks
6825 	 * for kvm_skip_emulated_instruction().  The caller is responsible for
6826 	 * updating interruptibility state and injecting single-step #DBs.
6827 	 */
6828 	if (emulation_type & EMULTYPE_SKIP) {
6829 		kvm_rip_write(vcpu, ctxt->_eip);
6830 		if (ctxt->eflags & X86_EFLAGS_RF)
6831 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6832 		return 1;
6833 	}
6834 
6835 	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
6836 		return 1;
6837 
6838 	/* this is needed for vmware backdoor interface to work since it
6839 	   changes registers values  during IO operation */
6840 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
6841 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6842 		emulator_invalidate_register_cache(ctxt);
6843 	}
6844 
6845 restart:
6846 	/* Save the faulting GPA (cr2) in the address field */
6847 	ctxt->exception.address = cr2_or_gpa;
6848 
6849 	r = x86_emulate_insn(ctxt);
6850 
6851 	if (r == EMULATION_INTERCEPTED)
6852 		return 1;
6853 
6854 	if (r == EMULATION_FAILED) {
6855 		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
6856 					emulation_type))
6857 			return 1;
6858 
6859 		return handle_emulation_failure(vcpu, emulation_type);
6860 	}
6861 
6862 	if (ctxt->have_exception) {
6863 		r = 1;
6864 		if (inject_emulated_exception(vcpu))
6865 			return r;
6866 	} else if (vcpu->arch.pio.count) {
6867 		if (!vcpu->arch.pio.in) {
6868 			/* FIXME: return into emulator if single-stepping.  */
6869 			vcpu->arch.pio.count = 0;
6870 		} else {
6871 			writeback = false;
6872 			vcpu->arch.complete_userspace_io = complete_emulated_pio;
6873 		}
6874 		r = 0;
6875 	} else if (vcpu->mmio_needed) {
6876 		++vcpu->stat.mmio_exits;
6877 
6878 		if (!vcpu->mmio_is_write)
6879 			writeback = false;
6880 		r = 0;
6881 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6882 	} else if (r == EMULATION_RESTART)
6883 		goto restart;
6884 	else
6885 		r = 1;
6886 
6887 	if (writeback) {
6888 		unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6889 		toggle_interruptibility(vcpu, ctxt->interruptibility);
6890 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6891 		if (!ctxt->have_exception ||
6892 		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
6893 			kvm_rip_write(vcpu, ctxt->eip);
6894 			if (r && ctxt->tf)
6895 				r = kvm_vcpu_do_singlestep(vcpu);
6896 			if (kvm_x86_ops->update_emulated_instruction)
6897 				kvm_x86_ops->update_emulated_instruction(vcpu);
6898 			__kvm_set_rflags(vcpu, ctxt->eflags);
6899 		}
6900 
6901 		/*
6902 		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
6903 		 * do nothing, and it will be requested again as soon as
6904 		 * the shadow expires.  But we still need to check here,
6905 		 * because POPF has no interrupt shadow.
6906 		 */
6907 		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
6908 			kvm_make_request(KVM_REQ_EVENT, vcpu);
6909 	} else
6910 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
6911 
6912 	return r;
6913 }
6914 
6915 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
6916 {
6917 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
6918 }
6919 EXPORT_SYMBOL_GPL(kvm_emulate_instruction);
6920 
6921 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
6922 					void *insn, int insn_len)
6923 {
6924 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
6925 }
6926 EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6927 
6928 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
6929 {
6930 	vcpu->arch.pio.count = 0;
6931 	return 1;
6932 }
6933 
6934 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
6935 {
6936 	vcpu->arch.pio.count = 0;
6937 
6938 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
6939 		return 1;
6940 
6941 	return kvm_skip_emulated_instruction(vcpu);
6942 }
6943 
6944 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
6945 			    unsigned short port)
6946 {
6947 	unsigned long val = kvm_rax_read(vcpu);
6948 	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
6949 					    size, port, &val, 1);
6950 	if (ret)
6951 		return ret;
6952 
6953 	/*
6954 	 * Workaround userspace that relies on old KVM behavior of %rip being
6955 	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
6956 	 */
6957 	if (port == 0x7e &&
6958 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
6959 		vcpu->arch.complete_userspace_io =
6960 			complete_fast_pio_out_port_0x7e;
6961 		kvm_skip_emulated_instruction(vcpu);
6962 	} else {
6963 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
6964 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
6965 	}
6966 	return 0;
6967 }
6968 
6969 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
6970 {
6971 	unsigned long val;
6972 
6973 	/* We should only ever be called with arch.pio.count equal to 1 */
6974 	BUG_ON(vcpu->arch.pio.count != 1);
6975 
6976 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
6977 		vcpu->arch.pio.count = 0;
6978 		return 1;
6979 	}
6980 
6981 	/* For size less than 4 we merge, else we zero extend */
6982 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
6983 
6984 	/*
6985 	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform
6986 	 * the copy and tracing
6987 	 */
6988 	emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size,
6989 				 vcpu->arch.pio.port, &val, 1);
6990 	kvm_rax_write(vcpu, val);
6991 
6992 	return kvm_skip_emulated_instruction(vcpu);
6993 }
6994 
6995 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
6996 			   unsigned short port)
6997 {
6998 	unsigned long val;
6999 	int ret;
7000 
7001 	/* For size less than 4 we merge, else we zero extend */
7002 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
7003 
7004 	ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
7005 				       &val, 1);
7006 	if (ret) {
7007 		kvm_rax_write(vcpu, val);
7008 		return ret;
7009 	}
7010 
7011 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7012 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;
7013 
7014 	return 0;
7015 }
7016 
7017 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
7018 {
7019 	int ret;
7020 
7021 	if (in)
7022 		ret = kvm_fast_pio_in(vcpu, size, port);
7023 	else
7024 		ret = kvm_fast_pio_out(vcpu, size, port);
7025 	return ret && kvm_skip_emulated_instruction(vcpu);
7026 }
7027 EXPORT_SYMBOL_GPL(kvm_fast_pio);
7028 
7029 static int kvmclock_cpu_down_prep(unsigned int cpu)
7030 {
7031 	__this_cpu_write(cpu_tsc_khz, 0);
7032 	return 0;
7033 }
7034 
7035 static void tsc_khz_changed(void *data)
7036 {
7037 	struct cpufreq_freqs *freq = data;
7038 	unsigned long khz = 0;
7039 
7040 	if (data)
7041 		khz = freq->new;
7042 	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
7043 		khz = cpufreq_quick_get(raw_smp_processor_id());
7044 	if (!khz)
7045 		khz = tsc_khz;
7046 	__this_cpu_write(cpu_tsc_khz, khz);
7047 }
7048 
7049 #ifdef CONFIG_X86_64
7050 static void kvm_hyperv_tsc_notifier(void)
7051 {
7052 	struct kvm *kvm;
7053 	struct kvm_vcpu *vcpu;
7054 	int cpu;
7055 
7056 	mutex_lock(&kvm_lock);
7057 	list_for_each_entry(kvm, &vm_list, vm_list)
7058 		kvm_make_mclock_inprogress_request(kvm);
7059 
7060 	hyperv_stop_tsc_emulation();
7061 
7062 	/* TSC frequency always matches when on Hyper-V */
7063 	for_each_present_cpu(cpu)
7064 		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
7065 	kvm_max_guest_tsc_khz = tsc_khz;
7066 
7067 	list_for_each_entry(kvm, &vm_list, vm_list) {
7068 		struct kvm_arch *ka = &kvm->arch;
7069 
7070 		spin_lock(&ka->pvclock_gtod_sync_lock);
7071 
7072 		pvclock_update_vm_gtod_copy(kvm);
7073 
7074 		kvm_for_each_vcpu(cpu, vcpu, kvm)
7075 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7076 
7077 		kvm_for_each_vcpu(cpu, vcpu, kvm)
7078 			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
7079 
7080 		spin_unlock(&ka->pvclock_gtod_sync_lock);
7081 	}
7082 	mutex_unlock(&kvm_lock);
7083 }
7084 #endif
7085 
7086 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
7087 {
7088 	struct kvm *kvm;
7089 	struct kvm_vcpu *vcpu;
7090 	int i, send_ipi = 0;
7091 
7092 	/*
7093 	 * We allow guests to temporarily run on slowing clocks,
7094 	 * provided we notify them after, or to run on accelerating
7095 	 * clocks, provided we notify them before.  Thus time never
7096 	 * goes backwards.
7097 	 *
7098 	 * However, we have a problem.  We can't atomically update
7099 	 * the frequency of a given CPU from this function; it is
7100 	 * merely a notifier, which can be called from any CPU.
7101 	 * Changing the TSC frequency at arbitrary points in time
7102 	 * requires a recomputation of local variables related to
7103 	 * the TSC for each VCPU.  We must flag these local variables
7104 	 * to be updated and be sure the update takes place with the
7105 	 * new frequency before any guests proceed.
7106 	 *
7107 	 * Unfortunately, the combination of hotplug CPU and frequency
7108 	 * change creates an intractable locking scenario; the order
7109 	 * of when these callouts happen is undefined with respect to
7110 	 * CPU hotplug, and they can race with each other.  As such,
7111 	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
7112 	 * undefined; you can actually have a CPU frequency change take
7113 	 * place in between the computation of X and the setting of the
7114 	 * variable.  To protect against this problem, all updates of
7115 	 * the per_cpu tsc_khz variable are done in an interrupt
7116 	 * protected IPI, and all callers wishing to update the value
7117 	 * must wait for a synchronous IPI to complete (which is trivial
7118 	 * if the caller is on the CPU already).  This establishes the
7119 	 * necessary total order on variable updates.
7120 	 *
7121 	 * Note that because a guest time update may take place
7122 	 * anytime after the setting of the VCPU's request bit, the
7123 	 * correct TSC value must be set before the request.  However,
7124 	 * to ensure the update actually makes it to any guest which
7125 	 * starts running in hardware virtualization between the set
7126 	 * and the acquisition of the spinlock, we must also ping the
7127 	 * CPU after setting the request bit.
7128 	 *
7129 	 */
7130 
7131 	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7132 
7133 	mutex_lock(&kvm_lock);
7134 	list_for_each_entry(kvm, &vm_list, vm_list) {
7135 		kvm_for_each_vcpu(i, vcpu, kvm) {
7136 			if (vcpu->cpu != cpu)
7137 				continue;
7138 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7139 			if (vcpu->cpu != raw_smp_processor_id())
7140 				send_ipi = 1;
7141 		}
7142 	}
7143 	mutex_unlock(&kvm_lock);
7144 
7145 	if (freq->old < freq->new && send_ipi) {
7146 		/*
7147 		 * We upscale the frequency.  Must make the guest
7148 		 * doesn't see old kvmclock values while running with
7149 		 * the new frequency, otherwise we risk the guest sees
7150 		 * time go backwards.
7151 		 *
7152 		 * In case we update the frequency for another cpu
7153 		 * (which might be in guest context) send an interrupt
7154 		 * to kick the cpu out of guest context.  Next time
7155 		 * guest context is entered kvmclock will be updated,
7156 		 * so the guest will not see stale values.
7157 		 */
7158 		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7159 	}
7160 }
7161 
7162 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
7163 				     void *data)
7164 {
7165 	struct cpufreq_freqs *freq = data;
7166 	int cpu;
7167 
7168 	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
7169 		return 0;
7170 	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
7171 		return 0;
7172 
7173 	for_each_cpu(cpu, freq->policy->cpus)
7174 		__kvmclock_cpufreq_notifier(freq, cpu);
7175 
7176 	return 0;
7177 }
7178 
7179 static struct notifier_block kvmclock_cpufreq_notifier_block = {
7180 	.notifier_call  = kvmclock_cpufreq_notifier
7181 };
7182 
7183 static int kvmclock_cpu_online(unsigned int cpu)
7184 {
7185 	tsc_khz_changed(NULL);
7186 	return 0;
7187 }
7188 
7189 static void kvm_timer_init(void)
7190 {
7191 	max_tsc_khz = tsc_khz;
7192 
7193 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
7194 #ifdef CONFIG_CPU_FREQ
7195 		struct cpufreq_policy *policy;
7196 		int cpu;
7197 
7198 		cpu = get_cpu();
7199 		policy = cpufreq_cpu_get(cpu);
7200 		if (policy) {
7201 			if (policy->cpuinfo.max_freq)
7202 				max_tsc_khz = policy->cpuinfo.max_freq;
7203 			cpufreq_cpu_put(policy);
7204 		}
7205 		put_cpu();
7206 #endif
7207 		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
7208 					  CPUFREQ_TRANSITION_NOTIFIER);
7209 	}
7210 
7211 	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
7212 			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
7213 }
7214 
7215 DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
7216 EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
7217 
7218 int kvm_is_in_guest(void)
7219 {
7220 	return __this_cpu_read(current_vcpu) != NULL;
7221 }
7222 
7223 static int kvm_is_user_mode(void)
7224 {
7225 	int user_mode = 3;
7226 
7227 	if (__this_cpu_read(current_vcpu))
7228 		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
7229 
7230 	return user_mode != 0;
7231 }
7232 
7233 static unsigned long kvm_get_guest_ip(void)
7234 {
7235 	unsigned long ip = 0;
7236 
7237 	if (__this_cpu_read(current_vcpu))
7238 		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
7239 
7240 	return ip;
7241 }
7242 
7243 static void kvm_handle_intel_pt_intr(void)
7244 {
7245 	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
7246 
7247 	kvm_make_request(KVM_REQ_PMI, vcpu);
7248 	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
7249 			(unsigned long *)&vcpu->arch.pmu.global_status);
7250 }
7251 
7252 static struct perf_guest_info_callbacks kvm_guest_cbs = {
7253 	.is_in_guest		= kvm_is_in_guest,
7254 	.is_user_mode		= kvm_is_user_mode,
7255 	.get_guest_ip		= kvm_get_guest_ip,
7256 	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
7257 };
7258 
7259 #ifdef CONFIG_X86_64
7260 static void pvclock_gtod_update_fn(struct work_struct *work)
7261 {
7262 	struct kvm *kvm;
7263 
7264 	struct kvm_vcpu *vcpu;
7265 	int i;
7266 
7267 	mutex_lock(&kvm_lock);
7268 	list_for_each_entry(kvm, &vm_list, vm_list)
7269 		kvm_for_each_vcpu(i, vcpu, kvm)
7270 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7271 	atomic_set(&kvm_guest_has_master_clock, 0);
7272 	mutex_unlock(&kvm_lock);
7273 }
7274 
7275 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
7276 
7277 /*
7278  * Notification about pvclock gtod data update.
7279  */
7280 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
7281 			       void *priv)
7282 {
7283 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
7284 	struct timekeeper *tk = priv;
7285 
7286 	update_pvclock_gtod(tk);
7287 
7288 	/* disable master clock if host does not trust, or does not
7289 	 * use, TSC based clocksource.
7290 	 */
7291 	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
7292 	    atomic_read(&kvm_guest_has_master_clock) != 0)
7293 		queue_work(system_long_wq, &pvclock_gtod_work);
7294 
7295 	return 0;
7296 }
7297 
7298 static struct notifier_block pvclock_gtod_notifier = {
7299 	.notifier_call = pvclock_gtod_notify,
7300 };
7301 #endif
7302 
7303 int kvm_arch_init(void *opaque)
7304 {
7305 	int r;
7306 	struct kvm_x86_ops *ops = opaque;
7307 
7308 	if (kvm_x86_ops) {
7309 		printk(KERN_ERR "kvm: already loaded the other module\n");
7310 		r = -EEXIST;
7311 		goto out;
7312 	}
7313 
7314 	if (!ops->cpu_has_kvm_support()) {
7315 		pr_err_ratelimited("kvm: no hardware support\n");
7316 		r = -EOPNOTSUPP;
7317 		goto out;
7318 	}
7319 	if (ops->disabled_by_bios()) {
7320 		pr_err_ratelimited("kvm: disabled by bios\n");
7321 		r = -EOPNOTSUPP;
7322 		goto out;
7323 	}
7324 
7325 	/*
7326 	 * KVM explicitly assumes that the guest has an FPU and
7327 	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
7328 	 * vCPU's FPU state as a fxregs_state struct.
7329 	 */
7330 	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
7331 		printk(KERN_ERR "kvm: inadequate fpu\n");
7332 		r = -EOPNOTSUPP;
7333 		goto out;
7334 	}
7335 
7336 	r = -ENOMEM;
7337 	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7338 					  __alignof__(struct fpu), SLAB_ACCOUNT,
7339 					  NULL);
7340 	if (!x86_fpu_cache) {
7341 		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
7342 		goto out;
7343 	}
7344 
7345 	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
7346 	if (!shared_msrs) {
7347 		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7348 		goto out_free_x86_fpu_cache;
7349 	}
7350 
7351 	r = kvm_mmu_module_init();
7352 	if (r)
7353 		goto out_free_percpu;
7354 
7355 	kvm_x86_ops = ops;
7356 
7357 	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7358 			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7359 			PT_PRESENT_MASK, 0, sme_me_mask);
7360 	kvm_timer_init();
7361 
7362 	perf_register_guest_info_callbacks(&kvm_guest_cbs);
7363 
7364 	if (boot_cpu_has(X86_FEATURE_XSAVE))
7365 		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
7366 
7367 	kvm_lapic_init();
7368 	if (pi_inject_timer == -1)
7369 		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7370 #ifdef CONFIG_X86_64
7371 	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7372 
7373 	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7374 		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7375 #endif
7376 
7377 	return 0;
7378 
7379 out_free_percpu:
7380 	free_percpu(shared_msrs);
7381 out_free_x86_fpu_cache:
7382 	kmem_cache_destroy(x86_fpu_cache);
7383 out:
7384 	return r;
7385 }
7386 
7387 void kvm_arch_exit(void)
7388 {
7389 #ifdef CONFIG_X86_64
7390 	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7391 		clear_hv_tscchange_cb();
7392 #endif
7393 	kvm_lapic_exit();
7394 	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
7395 
7396 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
7397 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
7398 					    CPUFREQ_TRANSITION_NOTIFIER);
7399 	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7400 #ifdef CONFIG_X86_64
7401 	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
7402 #endif
7403 	kvm_x86_ops = NULL;
7404 	kvm_mmu_module_exit();
7405 	free_percpu(shared_msrs);
7406 	kmem_cache_destroy(x86_fpu_cache);
7407 }
7408 
7409 int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7410 {
7411 	++vcpu->stat.halt_exits;
7412 	if (lapic_in_kernel(vcpu)) {
7413 		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7414 		return 1;
7415 	} else {
7416 		vcpu->run->exit_reason = KVM_EXIT_HLT;
7417 		return 0;
7418 	}
7419 }
7420 EXPORT_SYMBOL_GPL(kvm_vcpu_halt);
7421 
7422 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
7423 {
7424 	int ret = kvm_skip_emulated_instruction(vcpu);
7425 	/*
7426 	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
7427 	 * KVM_EXIT_DEBUG here.
7428 	 */
7429 	return kvm_vcpu_halt(vcpu) && ret;
7430 }
7431 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
7432 
7433 #ifdef CONFIG_X86_64
7434 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
7435 			        unsigned long clock_type)
7436 {
7437 	struct kvm_clock_pairing clock_pairing;
7438 	struct timespec64 ts;
7439 	u64 cycle;
7440 	int ret;
7441 
7442 	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
7443 		return -KVM_EOPNOTSUPP;
7444 
7445 	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
7446 		return -KVM_EOPNOTSUPP;
7447 
7448 	clock_pairing.sec = ts.tv_sec;
7449 	clock_pairing.nsec = ts.tv_nsec;
7450 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
7451 	clock_pairing.flags = 0;
7452 	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7453 
7454 	ret = 0;
7455 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
7456 			    sizeof(struct kvm_clock_pairing)))
7457 		ret = -KVM_EFAULT;
7458 
7459 	return ret;
7460 }
7461 #endif
7462 
7463 /*
7464  * kvm_pv_kick_cpu_op:  Kick a vcpu.
7465  *
7466  * @apicid - apicid of vcpu to be kicked.
7467  */
7468 static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
7469 {
7470 	struct kvm_lapic_irq lapic_irq;
7471 
7472 	lapic_irq.shorthand = APIC_DEST_NOSHORT;
7473 	lapic_irq.dest_mode = APIC_DEST_PHYSICAL;
7474 	lapic_irq.level = 0;
7475 	lapic_irq.dest_id = apicid;
7476 	lapic_irq.msi_redir_hint = false;
7477 
7478 	lapic_irq.delivery_mode = APIC_DM_REMRD;
7479 	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7480 }
7481 
7482 bool kvm_apicv_activated(struct kvm *kvm)
7483 {
7484 	return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
7485 }
7486 EXPORT_SYMBOL_GPL(kvm_apicv_activated);
7487 
7488 void kvm_apicv_init(struct kvm *kvm, bool enable)
7489 {
7490 	if (enable)
7491 		clear_bit(APICV_INHIBIT_REASON_DISABLE,
7492 			  &kvm->arch.apicv_inhibit_reasons);
7493 	else
7494 		set_bit(APICV_INHIBIT_REASON_DISABLE,
7495 			&kvm->arch.apicv_inhibit_reasons);
7496 }
7497 EXPORT_SYMBOL_GPL(kvm_apicv_init);
7498 
7499 static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
7500 {
7501 	struct kvm_vcpu *target = NULL;
7502 	struct kvm_apic_map *map;
7503 
7504 	rcu_read_lock();
7505 	map = rcu_dereference(kvm->arch.apic_map);
7506 
7507 	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
7508 		target = map->phys_map[dest_id]->vcpu;
7509 
7510 	rcu_read_unlock();
7511 
7512 	if (target && READ_ONCE(target->ready))
7513 		kvm_vcpu_yield_to(target);
7514 }
7515 
7516 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
7517 {
7518 	unsigned long nr, a0, a1, a2, a3, ret;
7519 	int op_64_bit;
7520 
7521 	if (kvm_hv_hypercall_enabled(vcpu->kvm))
7522 		return kvm_hv_hypercall(vcpu);
7523 
7524 	nr = kvm_rax_read(vcpu);
7525 	a0 = kvm_rbx_read(vcpu);
7526 	a1 = kvm_rcx_read(vcpu);
7527 	a2 = kvm_rdx_read(vcpu);
7528 	a3 = kvm_rsi_read(vcpu);
7529 
7530 	trace_kvm_hypercall(nr, a0, a1, a2, a3);
7531 
7532 	op_64_bit = is_64_bit_mode(vcpu);
7533 	if (!op_64_bit) {
7534 		nr &= 0xFFFFFFFF;
7535 		a0 &= 0xFFFFFFFF;
7536 		a1 &= 0xFFFFFFFF;
7537 		a2 &= 0xFFFFFFFF;
7538 		a3 &= 0xFFFFFFFF;
7539 	}
7540 
7541 	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
7542 		ret = -KVM_EPERM;
7543 		goto out;
7544 	}
7545 
7546 	switch (nr) {
7547 	case KVM_HC_VAPIC_POLL_IRQ:
7548 		ret = 0;
7549 		break;
7550 	case KVM_HC_KICK_CPU:
7551 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
7552 		kvm_sched_yield(vcpu->kvm, a1);
7553 		ret = 0;
7554 		break;
7555 #ifdef CONFIG_X86_64
7556 	case KVM_HC_CLOCK_PAIRING:
7557 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
7558 		break;
7559 #endif
7560 	case KVM_HC_SEND_IPI:
7561 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
7562 		break;
7563 	case KVM_HC_SCHED_YIELD:
7564 		kvm_sched_yield(vcpu->kvm, a0);
7565 		ret = 0;
7566 		break;
7567 	default:
7568 		ret = -KVM_ENOSYS;
7569 		break;
7570 	}
7571 out:
7572 	if (!op_64_bit)
7573 		ret = (u32)ret;
7574 	kvm_rax_write(vcpu, ret);
7575 
7576 	++vcpu->stat.hypercalls;
7577 	return kvm_skip_emulated_instruction(vcpu);
7578 }
7579 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
7580 
7581 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7582 {
7583 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7584 	char instruction[3];
7585 	unsigned long rip = kvm_rip_read(vcpu);
7586 
7587 	kvm_x86_ops->patch_hypercall(vcpu, instruction);
7588 
7589 	return emulator_write_emulated(ctxt, rip, instruction, 3,
7590 		&ctxt->exception);
7591 }
7592 
7593 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7594 {
7595 	return vcpu->run->request_interrupt_window &&
7596 		likely(!pic_in_kernel(vcpu->kvm));
7597 }
7598 
7599 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7600 {
7601 	struct kvm_run *kvm_run = vcpu->run;
7602 
7603 	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7604 	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7605 	kvm_run->cr8 = kvm_get_cr8(vcpu);
7606 	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7607 	kvm_run->ready_for_interrupt_injection =
7608 		pic_in_kernel(vcpu->kvm) ||
7609 		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7610 }
7611 
7612 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
7613 {
7614 	int max_irr, tpr;
7615 
7616 	if (!kvm_x86_ops->update_cr8_intercept)
7617 		return;
7618 
7619 	if (!lapic_in_kernel(vcpu))
7620 		return;
7621 
7622 	if (vcpu->arch.apicv_active)
7623 		return;
7624 
7625 	if (!vcpu->arch.apic->vapic_addr)
7626 		max_irr = kvm_lapic_find_highest_irr(vcpu);
7627 	else
7628 		max_irr = -1;
7629 
7630 	if (max_irr != -1)
7631 		max_irr >>= 4;
7632 
7633 	tpr = kvm_lapic_get_cr8(vcpu);
7634 
7635 	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
7636 }
7637 
7638 static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
7639 {
7640 	int r;
7641 
7642 	/* try to reinject previous events if any */
7643 
7644 	if (vcpu->arch.exception.injected)
7645 		kvm_x86_ops->queue_exception(vcpu);
7646 	/*
7647 	 * Do not inject an NMI or interrupt if there is a pending
7648 	 * exception.  Exceptions and interrupts are recognized at
7649 	 * instruction boundaries, i.e. the start of an instruction.
7650 	 * Trap-like exceptions, e.g. #DB, have higher priority than
7651 	 * NMIs and interrupts, i.e. traps are recognized before an
7652 	 * NMI/interrupt that's pending on the same instruction.
7653 	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
7654 	 * priority, but are only generated (pended) during instruction
7655 	 * execution, i.e. a pending fault-like exception means the
7656 	 * fault occurred on the *previous* instruction and must be
7657 	 * serviced prior to recognizing any new events in order to
7658 	 * fully complete the previous instruction.
7659 	 */
7660 	else if (!vcpu->arch.exception.pending) {
7661 		if (vcpu->arch.nmi_injected)
7662 			kvm_x86_ops->set_nmi(vcpu);
7663 		else if (vcpu->arch.interrupt.injected)
7664 			kvm_x86_ops->set_irq(vcpu);
7665 	}
7666 
7667 	/*
7668 	 * Call check_nested_events() even if we reinjected a previous event
7669 	 * in order for caller to determine if it should require immediate-exit
7670 	 * from L2 to L1 due to pending L1 events which require exit
7671 	 * from L2 to L1.
7672 	 */
7673 	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
7674 		r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
7675 		if (r != 0)
7676 			return r;
7677 	}
7678 
7679 	/* try to inject new event if pending */
7680 	if (vcpu->arch.exception.pending) {
7681 		trace_kvm_inj_exception(vcpu->arch.exception.nr,
7682 					vcpu->arch.exception.has_error_code,
7683 					vcpu->arch.exception.error_code);
7684 
7685 		WARN_ON_ONCE(vcpu->arch.exception.injected);
7686 		vcpu->arch.exception.pending = false;
7687 		vcpu->arch.exception.injected = true;
7688 
7689 		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
7690 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
7691 					     X86_EFLAGS_RF);
7692 
7693 		if (vcpu->arch.exception.nr == DB_VECTOR) {
7694 			/*
7695 			 * This code assumes that nSVM doesn't use
7696 			 * check_nested_events(). If it does, the
7697 			 * DR6/DR7 changes should happen before L1
7698 			 * gets a #VMEXIT for an intercepted #DB in
7699 			 * L2.  (Under VMX, on the other hand, the
7700 			 * DR6/DR7 changes should not happen in the
7701 			 * event of a VM-exit to L1 for an intercepted
7702 			 * #DB in L2.)
7703 			 */
7704 			kvm_deliver_exception_payload(vcpu);
7705 			if (vcpu->arch.dr7 & DR7_GD) {
7706 				vcpu->arch.dr7 &= ~DR7_GD;
7707 				kvm_update_dr7(vcpu);
7708 			}
7709 		}
7710 
7711 		kvm_x86_ops->queue_exception(vcpu);
7712 	}
7713 
7714 	/* Don't consider new event if we re-injected an event */
7715 	if (kvm_event_needs_reinjection(vcpu))
7716 		return 0;
7717 
7718 	if (vcpu->arch.smi_pending && !is_smm(vcpu) &&
7719 	    kvm_x86_ops->smi_allowed(vcpu)) {
7720 		vcpu->arch.smi_pending = false;
7721 		++vcpu->arch.smi_count;
7722 		enter_smm(vcpu);
7723 	} else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) {
7724 		--vcpu->arch.nmi_pending;
7725 		vcpu->arch.nmi_injected = true;
7726 		kvm_x86_ops->set_nmi(vcpu);
7727 	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
7728 		/*
7729 		 * Because interrupts can be injected asynchronously, we are
7730 		 * calling check_nested_events again here to avoid a race condition.
7731 		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
7732 		 * proposal and current concerns.  Perhaps we should be setting
7733 		 * KVM_REQ_EVENT only on certain events and not unconditionally?
7734 		 */
7735 		if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
7736 			r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
7737 			if (r != 0)
7738 				return r;
7739 		}
7740 		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
7741 			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
7742 					    false);
7743 			kvm_x86_ops->set_irq(vcpu);
7744 		}
7745 	}
7746 
7747 	return 0;
7748 }
7749 
7750 static void process_nmi(struct kvm_vcpu *vcpu)
7751 {
7752 	unsigned limit = 2;
7753 
7754 	/*
7755 	 * x86 is limited to one NMI running, and one NMI pending after it.
7756 	 * If an NMI is already in progress, limit further NMIs to just one.
7757 	 * Otherwise, allow two (and we'll inject the first one immediately).
7758 	 */
7759 	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
7760 		limit = 1;
7761 
7762 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
7763 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
7764 	kvm_make_request(KVM_REQ_EVENT, vcpu);
7765 }
7766 
7767 static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7768 {
7769 	u32 flags = 0;
7770 	flags |= seg->g       << 23;
7771 	flags |= seg->db      << 22;
7772 	flags |= seg->l       << 21;
7773 	flags |= seg->avl     << 20;
7774 	flags |= seg->present << 15;
7775 	flags |= seg->dpl     << 13;
7776 	flags |= seg->s       << 12;
7777 	flags |= seg->type    << 8;
7778 	return flags;
7779 }
7780 
7781 static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7782 {
7783 	struct kvm_segment seg;
7784 	int offset;
7785 
7786 	kvm_get_segment(vcpu, &seg, n);
7787 	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);
7788 
7789 	if (n < 3)
7790 		offset = 0x7f84 + n * 12;
7791 	else
7792 		offset = 0x7f2c + (n - 3) * 12;
7793 
7794 	put_smstate(u32, buf, offset + 8, seg.base);
7795 	put_smstate(u32, buf, offset + 4, seg.limit);
7796 	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7797 }
7798 
7799 #ifdef CONFIG_X86_64
7800 static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7801 {
7802 	struct kvm_segment seg;
7803 	int offset;
7804 	u16 flags;
7805 
7806 	kvm_get_segment(vcpu, &seg, n);
7807 	offset = 0x7e00 + n * 16;
7808 
7809 	flags = enter_smm_get_segment_flags(&seg) >> 8;
7810 	put_smstate(u16, buf, offset, seg.selector);
7811 	put_smstate(u16, buf, offset + 2, flags);
7812 	put_smstate(u32, buf, offset + 4, seg.limit);
7813 	put_smstate(u64, buf, offset + 8, seg.base);
7814 }
7815 #endif
7816 
7817 static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7818 {
7819 	struct desc_ptr dt;
7820 	struct kvm_segment seg;
7821 	unsigned long val;
7822 	int i;
7823 
7824 	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
7825 	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
7826 	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
7827 	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));
7828 
7829 	for (i = 0; i < 8; i++)
7830 		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));
7831 
7832 	kvm_get_dr(vcpu, 6, &val);
7833 	put_smstate(u32, buf, 0x7fcc, (u32)val);
7834 	kvm_get_dr(vcpu, 7, &val);
7835 	put_smstate(u32, buf, 0x7fc8, (u32)val);
7836 
7837 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
7838 	put_smstate(u32, buf, 0x7fc4, seg.selector);
7839 	put_smstate(u32, buf, 0x7f64, seg.base);
7840 	put_smstate(u32, buf, 0x7f60, seg.limit);
7841 	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7842 
7843 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
7844 	put_smstate(u32, buf, 0x7fc0, seg.selector);
7845 	put_smstate(u32, buf, 0x7f80, seg.base);
7846 	put_smstate(u32, buf, 0x7f7c, seg.limit);
7847 	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7848 
7849 	kvm_x86_ops->get_gdt(vcpu, &dt);
7850 	put_smstate(u32, buf, 0x7f74, dt.address);
7851 	put_smstate(u32, buf, 0x7f70, dt.size);
7852 
7853 	kvm_x86_ops->get_idt(vcpu, &dt);
7854 	put_smstate(u32, buf, 0x7f58, dt.address);
7855 	put_smstate(u32, buf, 0x7f54, dt.size);
7856 
7857 	for (i = 0; i < 6; i++)
7858 		enter_smm_save_seg_32(vcpu, buf, i);
7859 
7860 	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));
7861 
7862 	/* revision id */
7863 	put_smstate(u32, buf, 0x7efc, 0x00020000);
7864 	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
7865 }
7866 
7867 #ifdef CONFIG_X86_64
7868 static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
7869 {
7870 	struct desc_ptr dt;
7871 	struct kvm_segment seg;
7872 	unsigned long val;
7873 	int i;
7874 
7875 	for (i = 0; i < 16; i++)
7876 		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));
7877 
7878 	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
7879 	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));
7880 
7881 	kvm_get_dr(vcpu, 6, &val);
7882 	put_smstate(u64, buf, 0x7f68, val);
7883 	kvm_get_dr(vcpu, 7, &val);
7884 	put_smstate(u64, buf, 0x7f60, val);
7885 
7886 	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
7887 	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
7888 	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));
7889 
7890 	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);
7891 
7892 	/* revision id */
7893 	put_smstate(u32, buf, 0x7efc, 0x00020064);
7894 
7895 	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);
7896 
7897 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
7898 	put_smstate(u16, buf, 0x7e90, seg.selector);
7899 	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
7900 	put_smstate(u32, buf, 0x7e94, seg.limit);
7901 	put_smstate(u64, buf, 0x7e98, seg.base);
7902 
7903 	kvm_x86_ops->get_idt(vcpu, &dt);
7904 	put_smstate(u32, buf, 0x7e84, dt.size);
7905 	put_smstate(u64, buf, 0x7e88, dt.address);
7906 
7907 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
7908 	put_smstate(u16, buf, 0x7e70, seg.selector);
7909 	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
7910 	put_smstate(u32, buf, 0x7e74, seg.limit);
7911 	put_smstate(u64, buf, 0x7e78, seg.base);
7912 
7913 	kvm_x86_ops->get_gdt(vcpu, &dt);
7914 	put_smstate(u32, buf, 0x7e64, dt.size);
7915 	put_smstate(u64, buf, 0x7e68, dt.address);
7916 
7917 	for (i = 0; i < 6; i++)
7918 		enter_smm_save_seg_64(vcpu, buf, i);
7919 }
7920 #endif
7921 
7922 static void enter_smm(struct kvm_vcpu *vcpu)
7923 {
7924 	struct kvm_segment cs, ds;
7925 	struct desc_ptr dt;
7926 	char buf[512];
7927 	u32 cr0;
7928 
7929 	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
7930 	memset(buf, 0, 512);
7931 #ifdef CONFIG_X86_64
7932 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7933 		enter_smm_save_state_64(vcpu, buf);
7934 	else
7935 #endif
7936 		enter_smm_save_state_32(vcpu, buf);
7937 
7938 	/*
7939 	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
7940 	 * vCPU state (e.g. leave guest mode) after we've saved the state into
7941 	 * the SMM state-save area.
7942 	 */
7943 	kvm_x86_ops->pre_enter_smm(vcpu, buf);
7944 
7945 	vcpu->arch.hflags |= HF_SMM_MASK;
7946 	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
7947 
7948 	if (kvm_x86_ops->get_nmi_mask(vcpu))
7949 		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
7950 	else
7951 		kvm_x86_ops->set_nmi_mask(vcpu, true);
7952 
7953 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
7954 	kvm_rip_write(vcpu, 0x8000);
7955 
7956 	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
7957 	kvm_x86_ops->set_cr0(vcpu, cr0);
7958 	vcpu->arch.cr0 = cr0;
7959 
7960 	kvm_x86_ops->set_cr4(vcpu, 0);
7961 
7962 	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
7963 	dt.address = dt.size = 0;
7964 	kvm_x86_ops->set_idt(vcpu, &dt);
7965 
7966 	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);
7967 
7968 	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
7969 	cs.base = vcpu->arch.smbase;
7970 
7971 	ds.selector = 0;
7972 	ds.base = 0;
7973 
7974 	cs.limit    = ds.limit = 0xffffffff;
7975 	cs.type     = ds.type = 0x3;
7976 	cs.dpl      = ds.dpl = 0;
7977 	cs.db       = ds.db = 0;
7978 	cs.s        = ds.s = 1;
7979 	cs.l        = ds.l = 0;
7980 	cs.g        = ds.g = 1;
7981 	cs.avl      = ds.avl = 0;
7982 	cs.present  = ds.present = 1;
7983 	cs.unusable = ds.unusable = 0;
7984 	cs.padding  = ds.padding = 0;
7985 
7986 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
7987 	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
7988 	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
7989 	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
7990 	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
7991 	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);
7992 
7993 #ifdef CONFIG_X86_64
7994 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7995 		kvm_x86_ops->set_efer(vcpu, 0);
7996 #endif
7997 
7998 	kvm_update_cpuid(vcpu);
7999 	kvm_mmu_reset_context(vcpu);
8000 }
8001 
8002 static void process_smi(struct kvm_vcpu *vcpu)
8003 {
8004 	vcpu->arch.smi_pending = true;
8005 	kvm_make_request(KVM_REQ_EVENT, vcpu);
8006 }
8007 
8008 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
8009 				       unsigned long *vcpu_bitmap)
8010 {
8011 	cpumask_var_t cpus;
8012 
8013 	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
8014 
8015 	kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC,
8016 				    vcpu_bitmap, cpus);
8017 
8018 	free_cpumask_var(cpus);
8019 }
8020 
8021 void kvm_make_scan_ioapic_request(struct kvm *kvm)
8022 {
8023 	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
8024 }
8025 
8026 void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
8027 {
8028 	if (!lapic_in_kernel(vcpu))
8029 		return;
8030 
8031 	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
8032 	kvm_apic_update_apicv(vcpu);
8033 	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
8034 }
8035 EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);
8036 
8037 /*
8038  * NOTE: Do not hold any lock prior to calling this.
8039  *
8040  * In particular, kvm_request_apicv_update() expects kvm->srcu not to be
8041  * locked, because it calls __x86_set_memory_region() which does
8042  * synchronize_srcu(&kvm->srcu).
8043  */
8044 void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit)
8045 {
8046 	if (!kvm_x86_ops->check_apicv_inhibit_reasons ||
8047 	    !kvm_x86_ops->check_apicv_inhibit_reasons(bit))
8048 		return;
8049 
8050 	if (activate) {
8051 		if (!test_and_clear_bit(bit, &kvm->arch.apicv_inhibit_reasons) ||
8052 		    !kvm_apicv_activated(kvm))
8053 			return;
8054 	} else {
8055 		if (test_and_set_bit(bit, &kvm->arch.apicv_inhibit_reasons) ||
8056 		    kvm_apicv_activated(kvm))
8057 			return;
8058 	}
8059 
8060 	trace_kvm_apicv_update_request(activate, bit);
8061 	if (kvm_x86_ops->pre_update_apicv_exec_ctrl)
8062 		kvm_x86_ops->pre_update_apicv_exec_ctrl(kvm, activate);
8063 	kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE);
8064 }
8065 EXPORT_SYMBOL_GPL(kvm_request_apicv_update);
8066 
8067 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
8068 {
8069 	if (!kvm_apic_present(vcpu))
8070 		return;
8071 
8072 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
8073 
8074 	if (irqchip_split(vcpu->kvm))
8075 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
8076 	else {
8077 		if (vcpu->arch.apicv_active)
8078 			kvm_x86_ops->sync_pir_to_irr(vcpu);
8079 		if (ioapic_in_kernel(vcpu->kvm))
8080 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
8081 	}
8082 
8083 	if (is_guest_mode(vcpu))
8084 		vcpu->arch.load_eoi_exitmap_pending = true;
8085 	else
8086 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
8087 }
8088 
8089 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
8090 {
8091 	u64 eoi_exit_bitmap[4];
8092 
8093 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
8094 		return;
8095 
8096 	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
8097 		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
8098 	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
8099 }
8100 
8101 int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
8102 		unsigned long start, unsigned long end,
8103 		bool blockable)
8104 {
8105 	unsigned long apic_address;
8106 
8107 	/*
8108 	 * The physical address of apic access page is stored in the VMCS.
8109 	 * Update it when it becomes invalid.
8110 	 */
8111 	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
8112 	if (start <= apic_address && apic_address < end)
8113 		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
8114 
8115 	return 0;
8116 }
8117 
8118 void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
8119 {
8120 	struct page *page = NULL;
8121 
8122 	if (!lapic_in_kernel(vcpu))
8123 		return;
8124 
8125 	if (!kvm_x86_ops->set_apic_access_page_addr)
8126 		return;
8127 
8128 	page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
8129 	if (is_error_page(page))
8130 		return;
8131 	kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page));
8132 
8133 	/*
8134 	 * Do not pin apic access page in memory, the MMU notifier
8135 	 * will call us again if it is migrated or swapped out.
8136 	 */
8137 	put_page(page);
8138 }
8139 
8140 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
8141 {
8142 	smp_send_reschedule(vcpu->cpu);
8143 }
8144 EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);
8145 
8146 /*
8147  * Returns 1 to let vcpu_run() continue the guest execution loop without
8148  * exiting to the userspace.  Otherwise, the value will be returned to the
8149  * userspace.
8150  */
8151 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
8152 {
8153 	int r;
8154 	bool req_int_win =
8155 		dm_request_for_irq_injection(vcpu) &&
8156 		kvm_cpu_accept_dm_intr(vcpu);
8157 	enum exit_fastpath_completion exit_fastpath = EXIT_FASTPATH_NONE;
8158 
8159 	bool req_immediate_exit = false;
8160 
8161 	if (kvm_request_pending(vcpu)) {
8162 		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) {
8163 			if (unlikely(!kvm_x86_ops->get_vmcs12_pages(vcpu))) {
8164 				r = 0;
8165 				goto out;
8166 			}
8167 		}
8168 		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
8169 			kvm_mmu_unload(vcpu);
8170 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
8171 			__kvm_migrate_timers(vcpu);
8172 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
8173 			kvm_gen_update_masterclock(vcpu->kvm);
8174 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
8175 			kvm_gen_kvmclock_update(vcpu);
8176 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
8177 			r = kvm_guest_time_update(vcpu);
8178 			if (unlikely(r))
8179 				goto out;
8180 		}
8181 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
8182 			kvm_mmu_sync_roots(vcpu);
8183 		if (kvm_check_request(KVM_REQ_LOAD_CR3, vcpu))
8184 			kvm_mmu_load_cr3(vcpu);
8185 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
8186 			kvm_vcpu_flush_tlb(vcpu, true);
8187 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
8188 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
8189 			r = 0;
8190 			goto out;
8191 		}
8192 		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
8193 			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
8194 			vcpu->mmio_needed = 0;
8195 			r = 0;
8196 			goto out;
8197 		}
8198 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
8199 			/* Page is swapped out. Do synthetic halt */
8200 			vcpu->arch.apf.halted = true;
8201 			r = 1;
8202 			goto out;
8203 		}
8204 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
8205 			record_steal_time(vcpu);
8206 		if (kvm_check_request(KVM_REQ_SMI, vcpu))
8207 			process_smi(vcpu);
8208 		if (kvm_check_request(KVM_REQ_NMI, vcpu))
8209 			process_nmi(vcpu);
8210 		if (kvm_check_request(KVM_REQ_PMU, vcpu))
8211 			kvm_pmu_handle_event(vcpu);
8212 		if (kvm_check_request(KVM_REQ_PMI, vcpu))
8213 			kvm_pmu_deliver_pmi(vcpu);
8214 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
8215 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
8216 			if (test_bit(vcpu->arch.pending_ioapic_eoi,
8217 				     vcpu->arch.ioapic_handled_vectors)) {
8218 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
8219 				vcpu->run->eoi.vector =
8220 						vcpu->arch.pending_ioapic_eoi;
8221 				r = 0;
8222 				goto out;
8223 			}
8224 		}
8225 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
8226 			vcpu_scan_ioapic(vcpu);
8227 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
8228 			vcpu_load_eoi_exitmap(vcpu);
8229 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
8230 			kvm_vcpu_reload_apic_access_page(vcpu);
8231 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
8232 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
8233 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
8234 			r = 0;
8235 			goto out;
8236 		}
8237 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
8238 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
8239 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
8240 			r = 0;
8241 			goto out;
8242 		}
8243 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
8244 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
8245 			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
8246 			r = 0;
8247 			goto out;
8248 		}
8249 
8250 		/*
8251 		 * KVM_REQ_HV_STIMER has to be processed after
8252 		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
8253 		 * depend on the guest clock being up-to-date
8254 		 */
8255 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
8256 			kvm_hv_process_stimers(vcpu);
8257 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
8258 			kvm_vcpu_update_apicv(vcpu);
8259 	}
8260 
8261 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
8262 		++vcpu->stat.req_event;
8263 		kvm_apic_accept_events(vcpu);
8264 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
8265 			r = 1;
8266 			goto out;
8267 		}
8268 
8269 		if (inject_pending_event(vcpu, req_int_win) != 0)
8270 			req_immediate_exit = true;
8271 		else {
8272 			/* Enable SMI/NMI/IRQ window open exits if needed.
8273 			 *
8274 			 * SMIs have three cases:
8275 			 * 1) They can be nested, and then there is nothing to
8276 			 *    do here because RSM will cause a vmexit anyway.
8277 			 * 2) There is an ISA-specific reason why SMI cannot be
8278 			 *    injected, and the moment when this changes can be
8279 			 *    intercepted.
8280 			 * 3) Or the SMI can be pending because
8281 			 *    inject_pending_event has completed the injection
8282 			 *    of an IRQ or NMI from the previous vmexit, and
8283 			 *    then we request an immediate exit to inject the
8284 			 *    SMI.
8285 			 */
8286 			if (vcpu->arch.smi_pending && !is_smm(vcpu))
8287 				if (!kvm_x86_ops->enable_smi_window(vcpu))
8288 					req_immediate_exit = true;
8289 			if (vcpu->arch.nmi_pending)
8290 				kvm_x86_ops->enable_nmi_window(vcpu);
8291 			if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
8292 				kvm_x86_ops->enable_irq_window(vcpu);
8293 			WARN_ON(vcpu->arch.exception.pending);
8294 		}
8295 
8296 		if (kvm_lapic_enabled(vcpu)) {
8297 			update_cr8_intercept(vcpu);
8298 			kvm_lapic_sync_to_vapic(vcpu);
8299 		}
8300 	}
8301 
8302 	r = kvm_mmu_reload(vcpu);
8303 	if (unlikely(r)) {
8304 		goto cancel_injection;
8305 	}
8306 
8307 	preempt_disable();
8308 
8309 	kvm_x86_ops->prepare_guest_switch(vcpu);
8310 
8311 	/*
8312 	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
8313 	 * IPI are then delayed after guest entry, which ensures that they
8314 	 * result in virtual interrupt delivery.
8315 	 */
8316 	local_irq_disable();
8317 	vcpu->mode = IN_GUEST_MODE;
8318 
8319 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8320 
8321 	/*
8322 	 * 1) We should set ->mode before checking ->requests.  Please see
8323 	 * the comment in kvm_vcpu_exiting_guest_mode().
8324 	 *
8325 	 * 2) For APICv, we should set ->mode before checking PID.ON. This
8326 	 * pairs with the memory barrier implicit in pi_test_and_set_on
8327 	 * (see vmx_deliver_posted_interrupt).
8328 	 *
8329 	 * 3) This also orders the write to mode from any reads to the page
8330 	 * tables done while the VCPU is running.  Please see the comment
8331 	 * in kvm_flush_remote_tlbs.
8332 	 */
8333 	smp_mb__after_srcu_read_unlock();
8334 
8335 	/*
8336 	 * This handles the case where a posted interrupt was
8337 	 * notified with kvm_vcpu_kick.
8338 	 */
8339 	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
8340 		kvm_x86_ops->sync_pir_to_irr(vcpu);
8341 
8342 	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
8343 	    || need_resched() || signal_pending(current)) {
8344 		vcpu->mode = OUTSIDE_GUEST_MODE;
8345 		smp_wmb();
8346 		local_irq_enable();
8347 		preempt_enable();
8348 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8349 		r = 1;
8350 		goto cancel_injection;
8351 	}
8352 
8353 	if (req_immediate_exit) {
8354 		kvm_make_request(KVM_REQ_EVENT, vcpu);
8355 		kvm_x86_ops->request_immediate_exit(vcpu);
8356 	}
8357 
8358 	trace_kvm_entry(vcpu->vcpu_id);
8359 	guest_enter_irqoff();
8360 
8361 	fpregs_assert_state_consistent();
8362 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
8363 		switch_fpu_return();
8364 
8365 	if (unlikely(vcpu->arch.switch_db_regs)) {
8366 		set_debugreg(0, 7);
8367 		set_debugreg(vcpu->arch.eff_db[0], 0);
8368 		set_debugreg(vcpu->arch.eff_db[1], 1);
8369 		set_debugreg(vcpu->arch.eff_db[2], 2);
8370 		set_debugreg(vcpu->arch.eff_db[3], 3);
8371 		set_debugreg(vcpu->arch.dr6, 6);
8372 		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8373 	}
8374 
8375 	kvm_x86_ops->run(vcpu);
8376 
8377 	/*
8378 	 * Do this here before restoring debug registers on the host.  And
8379 	 * since we do this before handling the vmexit, a DR access vmexit
8380 	 * can (a) read the correct value of the debug registers, (b) set
8381 	 * KVM_DEBUGREG_WONT_EXIT again.
8382 	 */
8383 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
8384 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
8385 		kvm_x86_ops->sync_dirty_debug_regs(vcpu);
8386 		kvm_update_dr0123(vcpu);
8387 		kvm_update_dr6(vcpu);
8388 		kvm_update_dr7(vcpu);
8389 		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8390 	}
8391 
8392 	/*
8393 	 * If the guest has used debug registers, at least dr7
8394 	 * will be disabled while returning to the host.
8395 	 * If we don't have active breakpoints in the host, we don't
8396 	 * care about the messed up debug address registers. But if
8397 	 * we have some of them active, restore the old state.
8398 	 */
8399 	if (hw_breakpoint_active())
8400 		hw_breakpoint_restore();
8401 
8402 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
8403 
8404 	vcpu->mode = OUTSIDE_GUEST_MODE;
8405 	smp_wmb();
8406 
8407 	kvm_x86_ops->handle_exit_irqoff(vcpu, &exit_fastpath);
8408 
8409 	/*
8410 	 * Consume any pending interrupts, including the possible source of
8411 	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
8412 	 * An instruction is required after local_irq_enable() to fully unblock
8413 	 * interrupts on processors that implement an interrupt shadow, the
8414 	 * stat.exits increment will do nicely.
8415 	 */
8416 	kvm_before_interrupt(vcpu);
8417 	local_irq_enable();
8418 	++vcpu->stat.exits;
8419 	local_irq_disable();
8420 	kvm_after_interrupt(vcpu);
8421 
8422 	guest_exit_irqoff();
8423 	if (lapic_in_kernel(vcpu)) {
8424 		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
8425 		if (delta != S64_MIN) {
8426 			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
8427 			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
8428 		}
8429 	}
8430 
8431 	local_irq_enable();
8432 	preempt_enable();
8433 
8434 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8435 
8436 	/*
8437 	 * Profile KVM exit RIPs:
8438 	 */
8439 	if (unlikely(prof_on == KVM_PROFILING)) {
8440 		unsigned long rip = kvm_rip_read(vcpu);
8441 		profile_hit(KVM_PROFILING, (void *)rip);
8442 	}
8443 
8444 	if (unlikely(vcpu->arch.tsc_always_catchup))
8445 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8446 
8447 	if (vcpu->arch.apic_attention)
8448 		kvm_lapic_sync_from_vapic(vcpu);
8449 
8450 	vcpu->arch.gpa_available = false;
8451 	r = kvm_x86_ops->handle_exit(vcpu, exit_fastpath);
8452 	return r;
8453 
8454 cancel_injection:
8455 	kvm_x86_ops->cancel_injection(vcpu);
8456 	if (unlikely(vcpu->arch.apic_attention))
8457 		kvm_lapic_sync_from_vapic(vcpu);
8458 out:
8459 	return r;
8460 }
8461 
8462 static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
8463 {
8464 	if (!kvm_arch_vcpu_runnable(vcpu) &&
8465 	    (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) {
8466 		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8467 		kvm_vcpu_block(vcpu);
8468 		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8469 
8470 		if (kvm_x86_ops->post_block)
8471 			kvm_x86_ops->post_block(vcpu);
8472 
8473 		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
8474 			return 1;
8475 	}
8476 
8477 	kvm_apic_accept_events(vcpu);
8478 	switch(vcpu->arch.mp_state) {
8479 	case KVM_MP_STATE_HALTED:
8480 		vcpu->arch.pv.pv_unhalted = false;
8481 		vcpu->arch.mp_state =
8482 			KVM_MP_STATE_RUNNABLE;
8483 		/* fall through */
8484 	case KVM_MP_STATE_RUNNABLE:
8485 		vcpu->arch.apf.halted = false;
8486 		break;
8487 	case KVM_MP_STATE_INIT_RECEIVED:
8488 		break;
8489 	default:
8490 		return -EINTR;
8491 		break;
8492 	}
8493 	return 1;
8494 }
8495 
8496 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
8497 {
8498 	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
8499 		kvm_x86_ops->check_nested_events(vcpu, false);
8500 
8501 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
8502 		!vcpu->arch.apf.halted);
8503 }
8504 
8505 static int vcpu_run(struct kvm_vcpu *vcpu)
8506 {
8507 	int r;
8508 	struct kvm *kvm = vcpu->kvm;
8509 
8510 	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8511 	vcpu->arch.l1tf_flush_l1d = true;
8512 
8513 	for (;;) {
8514 		if (kvm_vcpu_running(vcpu)) {
8515 			r = vcpu_enter_guest(vcpu);
8516 		} else {
8517 			r = vcpu_block(kvm, vcpu);
8518 		}
8519 
8520 		if (r <= 0)
8521 			break;
8522 
8523 		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8524 		if (kvm_cpu_has_pending_timer(vcpu))
8525 			kvm_inject_pending_timer_irqs(vcpu);
8526 
8527 		if (dm_request_for_irq_injection(vcpu) &&
8528 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8529 			r = 0;
8530 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8531 			++vcpu->stat.request_irq_exits;
8532 			break;
8533 		}
8534 
8535 		kvm_check_async_pf_completion(vcpu);
8536 
8537 		if (signal_pending(current)) {
8538 			r = -EINTR;
8539 			vcpu->run->exit_reason = KVM_EXIT_INTR;
8540 			++vcpu->stat.signal_exits;
8541 			break;
8542 		}
8543 		if (need_resched()) {
8544 			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8545 			cond_resched();
8546 			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8547 		}
8548 	}
8549 
8550 	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8551 
8552 	return r;
8553 }
8554 
8555 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
8556 {
8557 	int r;
8558 
8559 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8560 	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8561 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8562 	return r;
8563 }
8564 
8565 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
8566 {
8567 	BUG_ON(!vcpu->arch.pio.count);
8568 
8569 	return complete_emulated_io(vcpu);
8570 }
8571 
8572 /*
8573  * Implements the following, as a state machine:
8574  *
8575  * read:
8576  *   for each fragment
8577  *     for each mmio piece in the fragment
8578  *       write gpa, len
8579  *       exit
8580  *       copy data
8581  *   execute insn
8582  *
8583  * write:
8584  *   for each fragment
8585  *     for each mmio piece in the fragment
8586  *       write gpa, len
8587  *       copy data
8588  *       exit
8589  */
8590 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8591 {
8592 	struct kvm_run *run = vcpu->run;
8593 	struct kvm_mmio_fragment *frag;
8594 	unsigned len;
8595 
8596 	BUG_ON(!vcpu->mmio_needed);
8597 
8598 	/* Complete previous fragment */
8599 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
8600 	len = min(8u, frag->len);
8601 	if (!vcpu->mmio_is_write)
8602 		memcpy(frag->data, run->mmio.data, len);
8603 
8604 	if (frag->len <= 8) {
8605 		/* Switch to the next fragment. */
8606 		frag++;
8607 		vcpu->mmio_cur_fragment++;
8608 	} else {
8609 		/* Go forward to the next mmio piece. */
8610 		frag->data += len;
8611 		frag->gpa += len;
8612 		frag->len -= len;
8613 	}
8614 
8615 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8616 		vcpu->mmio_needed = 0;
8617 
8618 		/* FIXME: return into emulator if single-stepping.  */
8619 		if (vcpu->mmio_is_write)
8620 			return 1;
8621 		vcpu->mmio_read_completed = 1;
8622 		return complete_emulated_io(vcpu);
8623 	}
8624 
8625 	run->exit_reason = KVM_EXIT_MMIO;
8626 	run->mmio.phys_addr = frag->gpa;
8627 	if (vcpu->mmio_is_write)
8628 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
8629 	run->mmio.len = min(8u, frag->len);
8630 	run->mmio.is_write = vcpu->mmio_is_write;
8631 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
8632 	return 0;
8633 }
8634 
8635 static void kvm_save_current_fpu(struct fpu *fpu)
8636 {
8637 	/*
8638 	 * If the target FPU state is not resident in the CPU registers, just
8639 	 * memcpy() from current, else save CPU state directly to the target.
8640 	 */
8641 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
8642 		memcpy(&fpu->state, &current->thread.fpu.state,
8643 		       fpu_kernel_xstate_size);
8644 	else
8645 		copy_fpregs_to_fpstate(fpu);
8646 }
8647 
8648 /* Swap (qemu) user FPU context for the guest FPU context. */
8649 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
8650 {
8651 	fpregs_lock();
8652 
8653 	kvm_save_current_fpu(vcpu->arch.user_fpu);
8654 
8655 	/* PKRU is separately restored in kvm_x86_ops->run.  */
8656 	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8657 				~XFEATURE_MASK_PKRU);
8658 
8659 	fpregs_mark_activate();
8660 	fpregs_unlock();
8661 
8662 	trace_kvm_fpu(1);
8663 }
8664 
8665 /* When vcpu_run ends, restore user space FPU context. */
8666 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
8667 {
8668 	fpregs_lock();
8669 
8670 	kvm_save_current_fpu(vcpu->arch.guest_fpu);
8671 
8672 	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
8673 
8674 	fpregs_mark_activate();
8675 	fpregs_unlock();
8676 
8677 	++vcpu->stat.fpu_reload;
8678 	trace_kvm_fpu(0);
8679 }
8680 
8681 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
8682 {
8683 	int r;
8684 
8685 	vcpu_load(vcpu);
8686 	kvm_sigset_activate(vcpu);
8687 	kvm_load_guest_fpu(vcpu);
8688 
8689 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8690 		if (kvm_run->immediate_exit) {
8691 			r = -EINTR;
8692 			goto out;
8693 		}
8694 		kvm_vcpu_block(vcpu);
8695 		kvm_apic_accept_events(vcpu);
8696 		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8697 		r = -EAGAIN;
8698 		if (signal_pending(current)) {
8699 			r = -EINTR;
8700 			vcpu->run->exit_reason = KVM_EXIT_INTR;
8701 			++vcpu->stat.signal_exits;
8702 		}
8703 		goto out;
8704 	}
8705 
8706 	if (vcpu->run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
8707 		r = -EINVAL;
8708 		goto out;
8709 	}
8710 
8711 	if (vcpu->run->kvm_dirty_regs) {
8712 		r = sync_regs(vcpu);
8713 		if (r != 0)
8714 			goto out;
8715 	}
8716 
8717 	/* re-sync apic's tpr */
8718 	if (!lapic_in_kernel(vcpu)) {
8719 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
8720 			r = -EINVAL;
8721 			goto out;
8722 		}
8723 	}
8724 
8725 	if (unlikely(vcpu->arch.complete_userspace_io)) {
8726 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
8727 		vcpu->arch.complete_userspace_io = NULL;
8728 		r = cui(vcpu);
8729 		if (r <= 0)
8730 			goto out;
8731 	} else
8732 		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8733 
8734 	if (kvm_run->immediate_exit)
8735 		r = -EINTR;
8736 	else
8737 		r = vcpu_run(vcpu);
8738 
8739 out:
8740 	kvm_put_guest_fpu(vcpu);
8741 	if (vcpu->run->kvm_valid_regs)
8742 		store_regs(vcpu);
8743 	post_kvm_run_save(vcpu);
8744 	kvm_sigset_deactivate(vcpu);
8745 
8746 	vcpu_put(vcpu);
8747 	return r;
8748 }
8749 
8750 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8751 {
8752 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
8753 		/*
8754 		 * We are here if userspace calls get_regs() in the middle of
8755 		 * instruction emulation. Registers state needs to be copied
8756 		 * back from emulation context to vcpu. Userspace shouldn't do
8757 		 * that usually, but some bad designed PV devices (vmware
8758 		 * backdoor interface) need this to work
8759 		 */
8760 		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
8761 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
8762 	}
8763 	regs->rax = kvm_rax_read(vcpu);
8764 	regs->rbx = kvm_rbx_read(vcpu);
8765 	regs->rcx = kvm_rcx_read(vcpu);
8766 	regs->rdx = kvm_rdx_read(vcpu);
8767 	regs->rsi = kvm_rsi_read(vcpu);
8768 	regs->rdi = kvm_rdi_read(vcpu);
8769 	regs->rsp = kvm_rsp_read(vcpu);
8770 	regs->rbp = kvm_rbp_read(vcpu);
8771 #ifdef CONFIG_X86_64
8772 	regs->r8 = kvm_r8_read(vcpu);
8773 	regs->r9 = kvm_r9_read(vcpu);
8774 	regs->r10 = kvm_r10_read(vcpu);
8775 	regs->r11 = kvm_r11_read(vcpu);
8776 	regs->r12 = kvm_r12_read(vcpu);
8777 	regs->r13 = kvm_r13_read(vcpu);
8778 	regs->r14 = kvm_r14_read(vcpu);
8779 	regs->r15 = kvm_r15_read(vcpu);
8780 #endif
8781 
8782 	regs->rip = kvm_rip_read(vcpu);
8783 	regs->rflags = kvm_get_rflags(vcpu);
8784 }
8785 
8786 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8787 {
8788 	vcpu_load(vcpu);
8789 	__get_regs(vcpu, regs);
8790 	vcpu_put(vcpu);
8791 	return 0;
8792 }
8793 
8794 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8795 {
8796 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
8797 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
8798 
8799 	kvm_rax_write(vcpu, regs->rax);
8800 	kvm_rbx_write(vcpu, regs->rbx);
8801 	kvm_rcx_write(vcpu, regs->rcx);
8802 	kvm_rdx_write(vcpu, regs->rdx);
8803 	kvm_rsi_write(vcpu, regs->rsi);
8804 	kvm_rdi_write(vcpu, regs->rdi);
8805 	kvm_rsp_write(vcpu, regs->rsp);
8806 	kvm_rbp_write(vcpu, regs->rbp);
8807 #ifdef CONFIG_X86_64
8808 	kvm_r8_write(vcpu, regs->r8);
8809 	kvm_r9_write(vcpu, regs->r9);
8810 	kvm_r10_write(vcpu, regs->r10);
8811 	kvm_r11_write(vcpu, regs->r11);
8812 	kvm_r12_write(vcpu, regs->r12);
8813 	kvm_r13_write(vcpu, regs->r13);
8814 	kvm_r14_write(vcpu, regs->r14);
8815 	kvm_r15_write(vcpu, regs->r15);
8816 #endif
8817 
8818 	kvm_rip_write(vcpu, regs->rip);
8819 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8820 
8821 	vcpu->arch.exception.pending = false;
8822 
8823 	kvm_make_request(KVM_REQ_EVENT, vcpu);
8824 }
8825 
8826 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8827 {
8828 	vcpu_load(vcpu);
8829 	__set_regs(vcpu, regs);
8830 	vcpu_put(vcpu);
8831 	return 0;
8832 }
8833 
8834 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
8835 {
8836 	struct kvm_segment cs;
8837 
8838 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8839 	*db = cs.db;
8840 	*l = cs.l;
8841 }
8842 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
8843 
8844 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8845 {
8846 	struct desc_ptr dt;
8847 
8848 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
8849 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
8850 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
8851 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
8852 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
8853 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8854 
8855 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
8856 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8857 
8858 	kvm_x86_ops->get_idt(vcpu, &dt);
8859 	sregs->idt.limit = dt.size;
8860 	sregs->idt.base = dt.address;
8861 	kvm_x86_ops->get_gdt(vcpu, &dt);
8862 	sregs->gdt.limit = dt.size;
8863 	sregs->gdt.base = dt.address;
8864 
8865 	sregs->cr0 = kvm_read_cr0(vcpu);
8866 	sregs->cr2 = vcpu->arch.cr2;
8867 	sregs->cr3 = kvm_read_cr3(vcpu);
8868 	sregs->cr4 = kvm_read_cr4(vcpu);
8869 	sregs->cr8 = kvm_get_cr8(vcpu);
8870 	sregs->efer = vcpu->arch.efer;
8871 	sregs->apic_base = kvm_get_apic_base(vcpu);
8872 
8873 	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
8874 
8875 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
8876 		set_bit(vcpu->arch.interrupt.nr,
8877 			(unsigned long *)sregs->interrupt_bitmap);
8878 }
8879 
8880 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
8881 				  struct kvm_sregs *sregs)
8882 {
8883 	vcpu_load(vcpu);
8884 	__get_sregs(vcpu, sregs);
8885 	vcpu_put(vcpu);
8886 	return 0;
8887 }
8888 
8889 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
8890 				    struct kvm_mp_state *mp_state)
8891 {
8892 	vcpu_load(vcpu);
8893 	if (kvm_mpx_supported())
8894 		kvm_load_guest_fpu(vcpu);
8895 
8896 	kvm_apic_accept_events(vcpu);
8897 	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
8898 					vcpu->arch.pv.pv_unhalted)
8899 		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
8900 	else
8901 		mp_state->mp_state = vcpu->arch.mp_state;
8902 
8903 	if (kvm_mpx_supported())
8904 		kvm_put_guest_fpu(vcpu);
8905 	vcpu_put(vcpu);
8906 	return 0;
8907 }
8908 
8909 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
8910 				    struct kvm_mp_state *mp_state)
8911 {
8912 	int ret = -EINVAL;
8913 
8914 	vcpu_load(vcpu);
8915 
8916 	if (!lapic_in_kernel(vcpu) &&
8917 	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
8918 		goto out;
8919 
8920 	/*
8921 	 * KVM_MP_STATE_INIT_RECEIVED means the processor is in
8922 	 * INIT state; latched init should be reported using
8923 	 * KVM_SET_VCPU_EVENTS, so reject it here.
8924 	 */
8925 	if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
8926 	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
8927 	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
8928 		goto out;
8929 
8930 	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
8931 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
8932 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
8933 	} else
8934 		vcpu->arch.mp_state = mp_state->mp_state;
8935 	kvm_make_request(KVM_REQ_EVENT, vcpu);
8936 
8937 	ret = 0;
8938 out:
8939 	vcpu_put(vcpu);
8940 	return ret;
8941 }
8942 
8943 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
8944 		    int reason, bool has_error_code, u32 error_code)
8945 {
8946 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
8947 	int ret;
8948 
8949 	init_emulate_ctxt(vcpu);
8950 
8951 	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
8952 				   has_error_code, error_code);
8953 	if (ret) {
8954 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
8955 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
8956 		vcpu->run->internal.ndata = 0;
8957 		return 0;
8958 	}
8959 
8960 	kvm_rip_write(vcpu, ctxt->eip);
8961 	kvm_set_rflags(vcpu, ctxt->eflags);
8962 	return 1;
8963 }
8964 EXPORT_SYMBOL_GPL(kvm_task_switch);
8965 
8966 static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8967 {
8968 	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
8969 		/*
8970 		 * When EFER.LME and CR0.PG are set, the processor is in
8971 		 * 64-bit mode (though maybe in a 32-bit code segment).
8972 		 * CR4.PAE and EFER.LMA must be set.
8973 		 */
8974 		if (!(sregs->cr4 & X86_CR4_PAE)
8975 		    || !(sregs->efer & EFER_LMA))
8976 			return -EINVAL;
8977 	} else {
8978 		/*
8979 		 * Not in 64-bit mode: EFER.LMA is clear and the code
8980 		 * segment cannot be 64-bit.
8981 		 */
8982 		if (sregs->efer & EFER_LMA || sregs->cs.l)
8983 			return -EINVAL;
8984 	}
8985 
8986 	return kvm_valid_cr4(vcpu, sregs->cr4);
8987 }
8988 
8989 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8990 {
8991 	struct msr_data apic_base_msr;
8992 	int mmu_reset_needed = 0;
8993 	int cpuid_update_needed = 0;
8994 	int pending_vec, max_bits, idx;
8995 	struct desc_ptr dt;
8996 	int ret = -EINVAL;
8997 
8998 	if (kvm_valid_sregs(vcpu, sregs))
8999 		goto out;
9000 
9001 	apic_base_msr.data = sregs->apic_base;
9002 	apic_base_msr.host_initiated = true;
9003 	if (kvm_set_apic_base(vcpu, &apic_base_msr))
9004 		goto out;
9005 
9006 	dt.size = sregs->idt.limit;
9007 	dt.address = sregs->idt.base;
9008 	kvm_x86_ops->set_idt(vcpu, &dt);
9009 	dt.size = sregs->gdt.limit;
9010 	dt.address = sregs->gdt.base;
9011 	kvm_x86_ops->set_gdt(vcpu, &dt);
9012 
9013 	vcpu->arch.cr2 = sregs->cr2;
9014 	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
9015 	vcpu->arch.cr3 = sregs->cr3;
9016 	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
9017 
9018 	kvm_set_cr8(vcpu, sregs->cr8);
9019 
9020 	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
9021 	kvm_x86_ops->set_efer(vcpu, sregs->efer);
9022 
9023 	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
9024 	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
9025 	vcpu->arch.cr0 = sregs->cr0;
9026 
9027 	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
9028 	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
9029 				(X86_CR4_OSXSAVE | X86_CR4_PKE));
9030 	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
9031 	if (cpuid_update_needed)
9032 		kvm_update_cpuid(vcpu);
9033 
9034 	idx = srcu_read_lock(&vcpu->kvm->srcu);
9035 	if (is_pae_paging(vcpu)) {
9036 		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
9037 		mmu_reset_needed = 1;
9038 	}
9039 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9040 
9041 	if (mmu_reset_needed)
9042 		kvm_mmu_reset_context(vcpu);
9043 
9044 	max_bits = KVM_NR_INTERRUPTS;
9045 	pending_vec = find_first_bit(
9046 		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
9047 	if (pending_vec < max_bits) {
9048 		kvm_queue_interrupt(vcpu, pending_vec, false);
9049 		pr_debug("Set back pending irq %d\n", pending_vec);
9050 	}
9051 
9052 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
9053 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
9054 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
9055 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
9056 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
9057 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
9058 
9059 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
9060 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9061 
9062 	update_cr8_intercept(vcpu);
9063 
9064 	/* Older userspace won't unhalt the vcpu on reset. */
9065 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
9066 	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
9067 	    !is_protmode(vcpu))
9068 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9069 
9070 	kvm_make_request(KVM_REQ_EVENT, vcpu);
9071 
9072 	ret = 0;
9073 out:
9074 	return ret;
9075 }
9076 
9077 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
9078 				  struct kvm_sregs *sregs)
9079 {
9080 	int ret;
9081 
9082 	vcpu_load(vcpu);
9083 	ret = __set_sregs(vcpu, sregs);
9084 	vcpu_put(vcpu);
9085 	return ret;
9086 }
9087 
9088 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
9089 					struct kvm_guest_debug *dbg)
9090 {
9091 	unsigned long rflags;
9092 	int i, r;
9093 
9094 	vcpu_load(vcpu);
9095 
9096 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
9097 		r = -EBUSY;
9098 		if (vcpu->arch.exception.pending)
9099 			goto out;
9100 		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
9101 			kvm_queue_exception(vcpu, DB_VECTOR);
9102 		else
9103 			kvm_queue_exception(vcpu, BP_VECTOR);
9104 	}
9105 
9106 	/*
9107 	 * Read rflags as long as potentially injected trace flags are still
9108 	 * filtered out.
9109 	 */
9110 	rflags = kvm_get_rflags(vcpu);
9111 
9112 	vcpu->guest_debug = dbg->control;
9113 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
9114 		vcpu->guest_debug = 0;
9115 
9116 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
9117 		for (i = 0; i < KVM_NR_DB_REGS; ++i)
9118 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
9119 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
9120 	} else {
9121 		for (i = 0; i < KVM_NR_DB_REGS; i++)
9122 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
9123 	}
9124 	kvm_update_dr7(vcpu);
9125 
9126 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
9127 		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
9128 			get_segment_base(vcpu, VCPU_SREG_CS);
9129 
9130 	/*
9131 	 * Trigger an rflags update that will inject or remove the trace
9132 	 * flags.
9133 	 */
9134 	kvm_set_rflags(vcpu, rflags);
9135 
9136 	kvm_x86_ops->update_bp_intercept(vcpu);
9137 
9138 	r = 0;
9139 
9140 out:
9141 	vcpu_put(vcpu);
9142 	return r;
9143 }
9144 
9145 /*
9146  * Translate a guest virtual address to a guest physical address.
9147  */
9148 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
9149 				    struct kvm_translation *tr)
9150 {
9151 	unsigned long vaddr = tr->linear_address;
9152 	gpa_t gpa;
9153 	int idx;
9154 
9155 	vcpu_load(vcpu);
9156 
9157 	idx = srcu_read_lock(&vcpu->kvm->srcu);
9158 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
9159 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9160 	tr->physical_address = gpa;
9161 	tr->valid = gpa != UNMAPPED_GVA;
9162 	tr->writeable = 1;
9163 	tr->usermode = 0;
9164 
9165 	vcpu_put(vcpu);
9166 	return 0;
9167 }
9168 
9169 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
9170 {
9171 	struct fxregs_state *fxsave;
9172 
9173 	vcpu_load(vcpu);
9174 
9175 	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9176 	memcpy(fpu->fpr, fxsave->st_space, 128);
9177 	fpu->fcw = fxsave->cwd;
9178 	fpu->fsw = fxsave->swd;
9179 	fpu->ftwx = fxsave->twd;
9180 	fpu->last_opcode = fxsave->fop;
9181 	fpu->last_ip = fxsave->rip;
9182 	fpu->last_dp = fxsave->rdp;
9183 	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
9184 
9185 	vcpu_put(vcpu);
9186 	return 0;
9187 }
9188 
9189 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
9190 {
9191 	struct fxregs_state *fxsave;
9192 
9193 	vcpu_load(vcpu);
9194 
9195 	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9196 
9197 	memcpy(fxsave->st_space, fpu->fpr, 128);
9198 	fxsave->cwd = fpu->fcw;
9199 	fxsave->swd = fpu->fsw;
9200 	fxsave->twd = fpu->ftwx;
9201 	fxsave->fop = fpu->last_opcode;
9202 	fxsave->rip = fpu->last_ip;
9203 	fxsave->rdp = fpu->last_dp;
9204 	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
9205 
9206 	vcpu_put(vcpu);
9207 	return 0;
9208 }
9209 
9210 static void store_regs(struct kvm_vcpu *vcpu)
9211 {
9212 	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
9213 
9214 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
9215 		__get_regs(vcpu, &vcpu->run->s.regs.regs);
9216 
9217 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
9218 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);
9219 
9220 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
9221 		kvm_vcpu_ioctl_x86_get_vcpu_events(
9222 				vcpu, &vcpu->run->s.regs.events);
9223 }
9224 
9225 static int sync_regs(struct kvm_vcpu *vcpu)
9226 {
9227 	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
9228 		return -EINVAL;
9229 
9230 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
9231 		__set_regs(vcpu, &vcpu->run->s.regs.regs);
9232 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
9233 	}
9234 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
9235 		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
9236 			return -EINVAL;
9237 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
9238 	}
9239 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
9240 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
9241 				vcpu, &vcpu->run->s.regs.events))
9242 			return -EINVAL;
9243 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
9244 	}
9245 
9246 	return 0;
9247 }
9248 
9249 static void fx_init(struct kvm_vcpu *vcpu)
9250 {
9251 	fpstate_init(&vcpu->arch.guest_fpu->state);
9252 	if (boot_cpu_has(X86_FEATURE_XSAVES))
9253 		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
9254 			host_xcr0 | XSTATE_COMPACTION_ENABLED;
9255 
9256 	/*
9257 	 * Ensure guest xcr0 is valid for loading
9258 	 */
9259 	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9260 
9261 	vcpu->arch.cr0 |= X86_CR0_ET;
9262 }
9263 
9264 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
9265 {
9266 	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
9267 		pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
9268 			     "guest TSC will not be reliable\n");
9269 
9270 	return 0;
9271 }
9272 
9273 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
9274 {
9275 	struct page *page;
9276 	int r;
9277 
9278 	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
9279 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
9280 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9281 	else
9282 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9283 
9284 	kvm_set_tsc_khz(vcpu, max_tsc_khz);
9285 
9286 	r = kvm_mmu_create(vcpu);
9287 	if (r < 0)
9288 		return r;
9289 
9290 	if (irqchip_in_kernel(vcpu->kvm)) {
9291 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
9292 		if (r < 0)
9293 			goto fail_mmu_destroy;
9294 		if (kvm_apicv_activated(vcpu->kvm))
9295 			vcpu->arch.apicv_active = true;
9296 	} else
9297 		static_key_slow_inc(&kvm_no_apic_vcpu);
9298 
9299 	r = -ENOMEM;
9300 
9301 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
9302 	if (!page)
9303 		goto fail_free_lapic;
9304 	vcpu->arch.pio_data = page_address(page);
9305 
9306 	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
9307 				       GFP_KERNEL_ACCOUNT);
9308 	if (!vcpu->arch.mce_banks)
9309 		goto fail_free_pio_data;
9310 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
9311 
9312 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
9313 				GFP_KERNEL_ACCOUNT))
9314 		goto fail_free_mce_banks;
9315 
9316 	vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
9317 						GFP_KERNEL_ACCOUNT);
9318 	if (!vcpu->arch.user_fpu) {
9319 		pr_err("kvm: failed to allocate userspace's fpu\n");
9320 		goto free_wbinvd_dirty_mask;
9321 	}
9322 
9323 	vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
9324 						 GFP_KERNEL_ACCOUNT);
9325 	if (!vcpu->arch.guest_fpu) {
9326 		pr_err("kvm: failed to allocate vcpu's fpu\n");
9327 		goto free_user_fpu;
9328 	}
9329 	fx_init(vcpu);
9330 
9331 	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
9332 
9333 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
9334 
9335 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
9336 
9337 	kvm_async_pf_hash_reset(vcpu);
9338 	kvm_pmu_init(vcpu);
9339 
9340 	vcpu->arch.pending_external_vector = -1;
9341 	vcpu->arch.preempted_in_kernel = false;
9342 
9343 	kvm_hv_vcpu_init(vcpu);
9344 
9345 	r = kvm_x86_ops->vcpu_create(vcpu);
9346 	if (r)
9347 		goto free_guest_fpu;
9348 
9349 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
9350 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
9351 	kvm_vcpu_mtrr_init(vcpu);
9352 	vcpu_load(vcpu);
9353 	kvm_vcpu_reset(vcpu, false);
9354 	kvm_init_mmu(vcpu, false);
9355 	vcpu_put(vcpu);
9356 	return 0;
9357 
9358 free_guest_fpu:
9359 	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
9360 free_user_fpu:
9361 	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
9362 free_wbinvd_dirty_mask:
9363 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
9364 fail_free_mce_banks:
9365 	kfree(vcpu->arch.mce_banks);
9366 fail_free_pio_data:
9367 	free_page((unsigned long)vcpu->arch.pio_data);
9368 fail_free_lapic:
9369 	kvm_free_lapic(vcpu);
9370 fail_mmu_destroy:
9371 	kvm_mmu_destroy(vcpu);
9372 	return r;
9373 }
9374 
9375 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
9376 {
9377 	struct msr_data msr;
9378 	struct kvm *kvm = vcpu->kvm;
9379 
9380 	kvm_hv_vcpu_postcreate(vcpu);
9381 
9382 	if (mutex_lock_killable(&vcpu->mutex))
9383 		return;
9384 	vcpu_load(vcpu);
9385 	msr.data = 0x0;
9386 	msr.index = MSR_IA32_TSC;
9387 	msr.host_initiated = true;
9388 	kvm_write_tsc(vcpu, &msr);
9389 	vcpu_put(vcpu);
9390 
9391 	/* poll control enabled by default */
9392 	vcpu->arch.msr_kvm_poll_control = 1;
9393 
9394 	mutex_unlock(&vcpu->mutex);
9395 
9396 	if (!kvmclock_periodic_sync)
9397 		return;
9398 
9399 	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
9400 					KVMCLOCK_SYNC_PERIOD);
9401 }
9402 
9403 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
9404 {
9405 	struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache;
9406 	int idx;
9407 
9408 	kvm_release_pfn(cache->pfn, cache->dirty, cache);
9409 
9410 	kvmclock_reset(vcpu);
9411 
9412 	kvm_x86_ops->vcpu_free(vcpu);
9413 
9414 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
9415 	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
9416 	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
9417 
9418 	kvm_hv_vcpu_uninit(vcpu);
9419 	kvm_pmu_destroy(vcpu);
9420 	kfree(vcpu->arch.mce_banks);
9421 	kvm_free_lapic(vcpu);
9422 	idx = srcu_read_lock(&vcpu->kvm->srcu);
9423 	kvm_mmu_destroy(vcpu);
9424 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9425 	free_page((unsigned long)vcpu->arch.pio_data);
9426 	if (!lapic_in_kernel(vcpu))
9427 		static_key_slow_dec(&kvm_no_apic_vcpu);
9428 }
9429 
9430 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
9431 {
9432 	kvm_lapic_reset(vcpu, init_event);
9433 
9434 	vcpu->arch.hflags = 0;
9435 
9436 	vcpu->arch.smi_pending = 0;
9437 	vcpu->arch.smi_count = 0;
9438 	atomic_set(&vcpu->arch.nmi_queued, 0);
9439 	vcpu->arch.nmi_pending = 0;
9440 	vcpu->arch.nmi_injected = false;
9441 	kvm_clear_interrupt_queue(vcpu);
9442 	kvm_clear_exception_queue(vcpu);
9443 
9444 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
9445 	kvm_update_dr0123(vcpu);
9446 	vcpu->arch.dr6 = DR6_INIT;
9447 	kvm_update_dr6(vcpu);
9448 	vcpu->arch.dr7 = DR7_FIXED_1;
9449 	kvm_update_dr7(vcpu);
9450 
9451 	vcpu->arch.cr2 = 0;
9452 
9453 	kvm_make_request(KVM_REQ_EVENT, vcpu);
9454 	vcpu->arch.apf.msr_val = 0;
9455 	vcpu->arch.st.msr_val = 0;
9456 
9457 	kvmclock_reset(vcpu);
9458 
9459 	kvm_clear_async_pf_completion_queue(vcpu);
9460 	kvm_async_pf_hash_reset(vcpu);
9461 	vcpu->arch.apf.halted = false;
9462 
9463 	if (kvm_mpx_supported()) {
9464 		void *mpx_state_buffer;
9465 
9466 		/*
9467 		 * To avoid have the INIT path from kvm_apic_has_events() that be
9468 		 * called with loaded FPU and does not let userspace fix the state.
9469 		 */
9470 		if (init_event)
9471 			kvm_put_guest_fpu(vcpu);
9472 		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9473 					XFEATURE_BNDREGS);
9474 		if (mpx_state_buffer)
9475 			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
9476 		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9477 					XFEATURE_BNDCSR);
9478 		if (mpx_state_buffer)
9479 			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
9480 		if (init_event)
9481 			kvm_load_guest_fpu(vcpu);
9482 	}
9483 
9484 	if (!init_event) {
9485 		kvm_pmu_reset(vcpu);
9486 		vcpu->arch.smbase = 0x30000;
9487 
9488 		vcpu->arch.msr_misc_features_enables = 0;
9489 
9490 		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9491 	}
9492 
9493 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
9494 	vcpu->arch.regs_avail = ~0;
9495 	vcpu->arch.regs_dirty = ~0;
9496 
9497 	vcpu->arch.ia32_xss = 0;
9498 
9499 	kvm_x86_ops->vcpu_reset(vcpu, init_event);
9500 }
9501 
9502 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
9503 {
9504 	struct kvm_segment cs;
9505 
9506 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
9507 	cs.selector = vector << 8;
9508 	cs.base = vector << 12;
9509 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
9510 	kvm_rip_write(vcpu, 0);
9511 }
9512 
9513 int kvm_arch_hardware_enable(void)
9514 {
9515 	struct kvm *kvm;
9516 	struct kvm_vcpu *vcpu;
9517 	int i;
9518 	int ret;
9519 	u64 local_tsc;
9520 	u64 max_tsc = 0;
9521 	bool stable, backwards_tsc = false;
9522 
9523 	kvm_shared_msr_cpu_online();
9524 	ret = kvm_x86_ops->hardware_enable();
9525 	if (ret != 0)
9526 		return ret;
9527 
9528 	local_tsc = rdtsc();
9529 	stable = !kvm_check_tsc_unstable();
9530 	list_for_each_entry(kvm, &vm_list, vm_list) {
9531 		kvm_for_each_vcpu(i, vcpu, kvm) {
9532 			if (!stable && vcpu->cpu == smp_processor_id())
9533 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9534 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
9535 				backwards_tsc = true;
9536 				if (vcpu->arch.last_host_tsc > max_tsc)
9537 					max_tsc = vcpu->arch.last_host_tsc;
9538 			}
9539 		}
9540 	}
9541 
9542 	/*
9543 	 * Sometimes, even reliable TSCs go backwards.  This happens on
9544 	 * platforms that reset TSC during suspend or hibernate actions, but
9545 	 * maintain synchronization.  We must compensate.  Fortunately, we can
9546 	 * detect that condition here, which happens early in CPU bringup,
9547 	 * before any KVM threads can be running.  Unfortunately, we can't
9548 	 * bring the TSCs fully up to date with real time, as we aren't yet far
9549 	 * enough into CPU bringup that we know how much real time has actually
9550 	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
9551 	 * variables that haven't been updated yet.
9552 	 *
9553 	 * So we simply find the maximum observed TSC above, then record the
9554 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
9555 	 * the adjustment will be applied.  Note that we accumulate
9556 	 * adjustments, in case multiple suspend cycles happen before some VCPU
9557 	 * gets a chance to run again.  In the event that no KVM threads get a
9558 	 * chance to run, we will miss the entire elapsed period, as we'll have
9559 	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
9560 	 * loose cycle time.  This isn't too big a deal, since the loss will be
9561 	 * uniform across all VCPUs (not to mention the scenario is extremely
9562 	 * unlikely). It is possible that a second hibernate recovery happens
9563 	 * much faster than a first, causing the observed TSC here to be
9564 	 * smaller; this would require additional padding adjustment, which is
9565 	 * why we set last_host_tsc to the local tsc observed here.
9566 	 *
9567 	 * N.B. - this code below runs only on platforms with reliable TSC,
9568 	 * as that is the only way backwards_tsc is set above.  Also note
9569 	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
9570 	 * have the same delta_cyc adjustment applied if backwards_tsc
9571 	 * is detected.  Note further, this adjustment is only done once,
9572 	 * as we reset last_host_tsc on all VCPUs to stop this from being
9573 	 * called multiple times (one for each physical CPU bringup).
9574 	 *
9575 	 * Platforms with unreliable TSCs don't have to deal with this, they
9576 	 * will be compensated by the logic in vcpu_load, which sets the TSC to
9577 	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
9578 	 * guarantee that they stay in perfect synchronization.
9579 	 */
9580 	if (backwards_tsc) {
9581 		u64 delta_cyc = max_tsc - local_tsc;
9582 		list_for_each_entry(kvm, &vm_list, vm_list) {
9583 			kvm->arch.backwards_tsc_observed = true;
9584 			kvm_for_each_vcpu(i, vcpu, kvm) {
9585 				vcpu->arch.tsc_offset_adjustment += delta_cyc;
9586 				vcpu->arch.last_host_tsc = local_tsc;
9587 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9588 			}
9589 
9590 			/*
9591 			 * We have to disable TSC offset matching.. if you were
9592 			 * booting a VM while issuing an S4 host suspend....
9593 			 * you may have some problem.  Solving this issue is
9594 			 * left as an exercise to the reader.
9595 			 */
9596 			kvm->arch.last_tsc_nsec = 0;
9597 			kvm->arch.last_tsc_write = 0;
9598 		}
9599 
9600 	}
9601 	return 0;
9602 }
9603 
9604 void kvm_arch_hardware_disable(void)
9605 {
9606 	kvm_x86_ops->hardware_disable();
9607 	drop_user_return_notifiers();
9608 }
9609 
9610 int kvm_arch_hardware_setup(void)
9611 {
9612 	int r;
9613 
9614 	r = kvm_x86_ops->hardware_setup();
9615 	if (r != 0)
9616 		return r;
9617 
9618 	cr4_reserved_bits = kvm_host_cr4_reserved_bits(&boot_cpu_data);
9619 
9620 	if (kvm_has_tsc_control) {
9621 		/*
9622 		 * Make sure the user can only configure tsc_khz values that
9623 		 * fit into a signed integer.
9624 		 * A min value is not calculated because it will always
9625 		 * be 1 on all machines.
9626 		 */
9627 		u64 max = min(0x7fffffffULL,
9628 			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
9629 		kvm_max_guest_tsc_khz = max;
9630 
9631 		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9632 	}
9633 
9634 	if (boot_cpu_has(X86_FEATURE_XSAVES))
9635 		rdmsrl(MSR_IA32_XSS, host_xss);
9636 
9637 	kvm_init_msr_list();
9638 	return 0;
9639 }
9640 
9641 void kvm_arch_hardware_unsetup(void)
9642 {
9643 	kvm_x86_ops->hardware_unsetup();
9644 }
9645 
9646 int kvm_arch_check_processor_compat(void)
9647 {
9648 	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
9649 
9650 	WARN_ON(!irqs_disabled());
9651 
9652 	if (kvm_host_cr4_reserved_bits(c) != cr4_reserved_bits)
9653 		return -EIO;
9654 
9655 	return kvm_x86_ops->check_processor_compatibility();
9656 }
9657 
9658 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
9659 {
9660 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
9661 }
9662 EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);
9663 
9664 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
9665 {
9666 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9667 }
9668 
9669 struct static_key kvm_no_apic_vcpu __read_mostly;
9670 EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9671 
9672 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
9673 {
9674 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
9675 
9676 	vcpu->arch.l1tf_flush_l1d = true;
9677 	if (pmu->version && unlikely(pmu->event_count)) {
9678 		pmu->need_cleanup = true;
9679 		kvm_make_request(KVM_REQ_PMU, vcpu);
9680 	}
9681 	kvm_x86_ops->sched_in(vcpu, cpu);
9682 }
9683 
9684 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9685 {
9686 	if (type)
9687 		return -EINVAL;
9688 
9689 	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9690 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
9691 	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
9692 	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
9693 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9694 	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9695 
9696 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
9697 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9698 	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
9699 	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
9700 		&kvm->arch.irq_sources_bitmap);
9701 
9702 	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9703 	mutex_init(&kvm->arch.apic_map_lock);
9704 	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);
9705 
9706 	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
9707 	pvclock_update_vm_gtod_copy(kvm);
9708 
9709 	kvm->arch.guest_can_read_msr_platform_info = true;
9710 
9711 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9712 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9713 
9714 	kvm_hv_init_vm(kvm);
9715 	kvm_page_track_init(kvm);
9716 	kvm_mmu_init_vm(kvm);
9717 
9718 	return kvm_x86_ops->vm_init(kvm);
9719 }
9720 
9721 int kvm_arch_post_init_vm(struct kvm *kvm)
9722 {
9723 	return kvm_mmu_post_init_vm(kvm);
9724 }
9725 
9726 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
9727 {
9728 	vcpu_load(vcpu);
9729 	kvm_mmu_unload(vcpu);
9730 	vcpu_put(vcpu);
9731 }
9732 
9733 static void kvm_free_vcpus(struct kvm *kvm)
9734 {
9735 	unsigned int i;
9736 	struct kvm_vcpu *vcpu;
9737 
9738 	/*
9739 	 * Unpin any mmu pages first.
9740 	 */
9741 	kvm_for_each_vcpu(i, vcpu, kvm) {
9742 		kvm_clear_async_pf_completion_queue(vcpu);
9743 		kvm_unload_vcpu_mmu(vcpu);
9744 	}
9745 	kvm_for_each_vcpu(i, vcpu, kvm)
9746 		kvm_vcpu_destroy(vcpu);
9747 
9748 	mutex_lock(&kvm->lock);
9749 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
9750 		kvm->vcpus[i] = NULL;
9751 
9752 	atomic_set(&kvm->online_vcpus, 0);
9753 	mutex_unlock(&kvm->lock);
9754 }
9755 
9756 void kvm_arch_sync_events(struct kvm *kvm)
9757 {
9758 	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9759 	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9760 	kvm_free_pit(kvm);
9761 }
9762 
9763 int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9764 {
9765 	int i, r;
9766 	unsigned long hva;
9767 	struct kvm_memslots *slots = kvm_memslots(kvm);
9768 	struct kvm_memory_slot *slot, old;
9769 
9770 	/* Called with kvm->slots_lock held.  */
9771 	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
9772 		return -EINVAL;
9773 
9774 	slot = id_to_memslot(slots, id);
9775 	if (size) {
9776 		if (slot->npages)
9777 			return -EEXIST;
9778 
9779 		/*
9780 		 * MAP_SHARED to prevent internal slot pages from being moved
9781 		 * by fork()/COW.
9782 		 */
9783 		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
9784 			      MAP_SHARED | MAP_ANONYMOUS, 0);
9785 		if (IS_ERR((void *)hva))
9786 			return PTR_ERR((void *)hva);
9787 	} else {
9788 		if (!slot->npages)
9789 			return 0;
9790 
9791 		hva = 0;
9792 	}
9793 
9794 	old = *slot;
9795 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9796 		struct kvm_userspace_memory_region m;
9797 
9798 		m.slot = id | (i << 16);
9799 		m.flags = 0;
9800 		m.guest_phys_addr = gpa;
9801 		m.userspace_addr = hva;
9802 		m.memory_size = size;
9803 		r = __kvm_set_memory_region(kvm, &m);
9804 		if (r < 0)
9805 			return r;
9806 	}
9807 
9808 	if (!size)
9809 		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
9810 
9811 	return 0;
9812 }
9813 EXPORT_SYMBOL_GPL(__x86_set_memory_region);
9814 
9815 void kvm_arch_pre_destroy_vm(struct kvm *kvm)
9816 {
9817 	kvm_mmu_pre_destroy_vm(kvm);
9818 }
9819 
9820 void kvm_arch_destroy_vm(struct kvm *kvm)
9821 {
9822 	if (current->mm == kvm->mm) {
9823 		/*
9824 		 * Free memory regions allocated on behalf of userspace,
9825 		 * unless the the memory map has changed due to process exit
9826 		 * or fd copying.
9827 		 */
9828 		mutex_lock(&kvm->slots_lock);
9829 		__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
9830 					0, 0);
9831 		__x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
9832 					0, 0);
9833 		__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
9834 		mutex_unlock(&kvm->slots_lock);
9835 	}
9836 	if (kvm_x86_ops->vm_destroy)
9837 		kvm_x86_ops->vm_destroy(kvm);
9838 	kvm_pic_destroy(kvm);
9839 	kvm_ioapic_destroy(kvm);
9840 	kvm_free_vcpus(kvm);
9841 	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
9842 	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
9843 	kvm_mmu_uninit_vm(kvm);
9844 	kvm_page_track_cleanup(kvm);
9845 	kvm_hv_destroy_vm(kvm);
9846 }
9847 
9848 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
9849 			   struct kvm_memory_slot *dont)
9850 {
9851 	int i;
9852 
9853 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9854 		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
9855 			kvfree(free->arch.rmap[i]);
9856 			free->arch.rmap[i] = NULL;
9857 		}
9858 		if (i == 0)
9859 			continue;
9860 
9861 		if (!dont || free->arch.lpage_info[i - 1] !=
9862 			     dont->arch.lpage_info[i - 1]) {
9863 			kvfree(free->arch.lpage_info[i - 1]);
9864 			free->arch.lpage_info[i - 1] = NULL;
9865 		}
9866 	}
9867 
9868 	kvm_page_track_free_memslot(free, dont);
9869 }
9870 
9871 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
9872 			    unsigned long npages)
9873 {
9874 	int i;
9875 
9876 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9877 		struct kvm_lpage_info *linfo;
9878 		unsigned long ugfn;
9879 		int lpages;
9880 		int level = i + 1;
9881 
9882 		lpages = gfn_to_index(slot->base_gfn + npages - 1,
9883 				      slot->base_gfn, level) + 1;
9884 
9885 		slot->arch.rmap[i] =
9886 			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
9887 				 GFP_KERNEL_ACCOUNT);
9888 		if (!slot->arch.rmap[i])
9889 			goto out_free;
9890 		if (i == 0)
9891 			continue;
9892 
9893 		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
9894 		if (!linfo)
9895 			goto out_free;
9896 
9897 		slot->arch.lpage_info[i - 1] = linfo;
9898 
9899 		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
9900 			linfo[0].disallow_lpage = 1;
9901 		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
9902 			linfo[lpages - 1].disallow_lpage = 1;
9903 		ugfn = slot->userspace_addr >> PAGE_SHIFT;
9904 		/*
9905 		 * If the gfn and userspace address are not aligned wrt each
9906 		 * other, or if explicitly asked to, disable large page
9907 		 * support for this slot
9908 		 */
9909 		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
9910 		    !kvm_largepages_enabled()) {
9911 			unsigned long j;
9912 
9913 			for (j = 0; j < lpages; ++j)
9914 				linfo[j].disallow_lpage = 1;
9915 		}
9916 	}
9917 
9918 	if (kvm_page_track_create_memslot(slot, npages))
9919 		goto out_free;
9920 
9921 	return 0;
9922 
9923 out_free:
9924 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9925 		kvfree(slot->arch.rmap[i]);
9926 		slot->arch.rmap[i] = NULL;
9927 		if (i == 0)
9928 			continue;
9929 
9930 		kvfree(slot->arch.lpage_info[i - 1]);
9931 		slot->arch.lpage_info[i - 1] = NULL;
9932 	}
9933 	return -ENOMEM;
9934 }
9935 
9936 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
9937 {
9938 	struct kvm_vcpu *vcpu;
9939 	int i;
9940 
9941 	/*
9942 	 * memslots->generation has been incremented.
9943 	 * mmio generation may have reached its maximum value.
9944 	 */
9945 	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
9946 
9947 	/* Force re-initialization of steal_time cache */
9948 	kvm_for_each_vcpu(i, vcpu, kvm)
9949 		kvm_vcpu_kick(vcpu);
9950 }
9951 
9952 int kvm_arch_prepare_memory_region(struct kvm *kvm,
9953 				struct kvm_memory_slot *memslot,
9954 				const struct kvm_userspace_memory_region *mem,
9955 				enum kvm_mr_change change)
9956 {
9957 	return 0;
9958 }
9959 
9960 static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
9961 				     struct kvm_memory_slot *new)
9962 {
9963 	/* Still write protect RO slot */
9964 	if (new->flags & KVM_MEM_READONLY) {
9965 		kvm_mmu_slot_remove_write_access(kvm, new);
9966 		return;
9967 	}
9968 
9969 	/*
9970 	 * Call kvm_x86_ops dirty logging hooks when they are valid.
9971 	 *
9972 	 * kvm_x86_ops->slot_disable_log_dirty is called when:
9973 	 *
9974 	 *  - KVM_MR_CREATE with dirty logging is disabled
9975 	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
9976 	 *
9977 	 * The reason is, in case of PML, we need to set D-bit for any slots
9978 	 * with dirty logging disabled in order to eliminate unnecessary GPA
9979 	 * logging in PML buffer (and potential PML buffer full VMEXIT). This
9980 	 * guarantees leaving PML enabled during guest's lifetime won't have
9981 	 * any additional overhead from PML when guest is running with dirty
9982 	 * logging disabled for memory slots.
9983 	 *
9984 	 * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot
9985 	 * to dirty logging mode.
9986 	 *
9987 	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
9988 	 *
9989 	 * In case of write protect:
9990 	 *
9991 	 * Write protect all pages for dirty logging.
9992 	 *
9993 	 * All the sptes including the large sptes which point to this
9994 	 * slot are set to readonly. We can not create any new large
9995 	 * spte on this slot until the end of the logging.
9996 	 *
9997 	 * See the comments in fast_page_fault().
9998 	 */
9999 	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
10000 		if (kvm_x86_ops->slot_enable_log_dirty)
10001 			kvm_x86_ops->slot_enable_log_dirty(kvm, new);
10002 		else
10003 			kvm_mmu_slot_remove_write_access(kvm, new);
10004 	} else {
10005 		if (kvm_x86_ops->slot_disable_log_dirty)
10006 			kvm_x86_ops->slot_disable_log_dirty(kvm, new);
10007 	}
10008 }
10009 
10010 void kvm_arch_commit_memory_region(struct kvm *kvm,
10011 				const struct kvm_userspace_memory_region *mem,
10012 				const struct kvm_memory_slot *old,
10013 				const struct kvm_memory_slot *new,
10014 				enum kvm_mr_change change)
10015 {
10016 	if (!kvm->arch.n_requested_mmu_pages)
10017 		kvm_mmu_change_mmu_pages(kvm,
10018 				kvm_mmu_calculate_default_mmu_pages(kvm));
10019 
10020 	/*
10021 	 * Dirty logging tracks sptes in 4k granularity, meaning that large
10022 	 * sptes have to be split.  If live migration is successful, the guest
10023 	 * in the source machine will be destroyed and large sptes will be
10024 	 * created in the destination. However, if the guest continues to run
10025 	 * in the source machine (for example if live migration fails), small
10026 	 * sptes will remain around and cause bad performance.
10027 	 *
10028 	 * Scan sptes if dirty logging has been stopped, dropping those
10029 	 * which can be collapsed into a single large-page spte.  Later
10030 	 * page faults will create the large-page sptes.
10031 	 *
10032 	 * There is no need to do this in any of the following cases:
10033 	 * CREATE:	No dirty mappings will already exist.
10034 	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
10035 	 *		kvm_arch_flush_shadow_memslot()
10036 	 */
10037 	if (change == KVM_MR_FLAGS_ONLY &&
10038 		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
10039 		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
10040 		kvm_mmu_zap_collapsible_sptes(kvm, new);
10041 
10042 	/*
10043 	 * Set up write protection and/or dirty logging for the new slot.
10044 	 *
10045 	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
10046 	 * been zapped so no dirty logging staff is needed for old slot. For
10047 	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
10048 	 * new and it's also covered when dealing with the new slot.
10049 	 *
10050 	 * FIXME: const-ify all uses of struct kvm_memory_slot.
10051 	 */
10052 	if (change != KVM_MR_DELETE)
10053 		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
10054 }
10055 
10056 void kvm_arch_flush_shadow_all(struct kvm *kvm)
10057 {
10058 	kvm_mmu_zap_all(kvm);
10059 }
10060 
10061 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
10062 				   struct kvm_memory_slot *slot)
10063 {
10064 	kvm_page_track_flush_slot(kvm, slot);
10065 }
10066 
10067 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
10068 {
10069 	return (is_guest_mode(vcpu) &&
10070 			kvm_x86_ops->guest_apic_has_interrupt &&
10071 			kvm_x86_ops->guest_apic_has_interrupt(vcpu));
10072 }
10073 
10074 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
10075 {
10076 	if (!list_empty_careful(&vcpu->async_pf.done))
10077 		return true;
10078 
10079 	if (kvm_apic_has_events(vcpu))
10080 		return true;
10081 
10082 	if (vcpu->arch.pv.pv_unhalted)
10083 		return true;
10084 
10085 	if (vcpu->arch.exception.pending)
10086 		return true;
10087 
10088 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
10089 	    (vcpu->arch.nmi_pending &&
10090 	     kvm_x86_ops->nmi_allowed(vcpu)))
10091 		return true;
10092 
10093 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
10094 	    (vcpu->arch.smi_pending && !is_smm(vcpu)))
10095 		return true;
10096 
10097 	if (kvm_arch_interrupt_allowed(vcpu) &&
10098 	    (kvm_cpu_has_interrupt(vcpu) ||
10099 	    kvm_guest_apic_has_interrupt(vcpu)))
10100 		return true;
10101 
10102 	if (kvm_hv_has_stimer_pending(vcpu))
10103 		return true;
10104 
10105 	return false;
10106 }
10107 
10108 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
10109 {
10110 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
10111 }
10112 
10113 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
10114 {
10115 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
10116 		return true;
10117 
10118 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
10119 		kvm_test_request(KVM_REQ_SMI, vcpu) ||
10120 		 kvm_test_request(KVM_REQ_EVENT, vcpu))
10121 		return true;
10122 
10123 	if (vcpu->arch.apicv_active && kvm_x86_ops->dy_apicv_has_pending_interrupt(vcpu))
10124 		return true;
10125 
10126 	return false;
10127 }
10128 
10129 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
10130 {
10131 	return vcpu->arch.preempted_in_kernel;
10132 }
10133 
10134 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
10135 {
10136 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
10137 }
10138 
10139 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
10140 {
10141 	return kvm_x86_ops->interrupt_allowed(vcpu);
10142 }
10143 
10144 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
10145 {
10146 	if (is_64_bit_mode(vcpu))
10147 		return kvm_rip_read(vcpu);
10148 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
10149 		     kvm_rip_read(vcpu));
10150 }
10151 EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
10152 
10153 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
10154 {
10155 	return kvm_get_linear_rip(vcpu) == linear_rip;
10156 }
10157 EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
10158 
10159 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
10160 {
10161 	unsigned long rflags;
10162 
10163 	rflags = kvm_x86_ops->get_rflags(vcpu);
10164 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10165 		rflags &= ~X86_EFLAGS_TF;
10166 	return rflags;
10167 }
10168 EXPORT_SYMBOL_GPL(kvm_get_rflags);
10169 
10170 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
10171 {
10172 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
10173 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
10174 		rflags |= X86_EFLAGS_TF;
10175 	kvm_x86_ops->set_rflags(vcpu, rflags);
10176 }
10177 
10178 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
10179 {
10180 	__kvm_set_rflags(vcpu, rflags);
10181 	kvm_make_request(KVM_REQ_EVENT, vcpu);
10182 }
10183 EXPORT_SYMBOL_GPL(kvm_set_rflags);
10184 
10185 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
10186 {
10187 	int r;
10188 
10189 	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
10190 	      work->wakeup_all)
10191 		return;
10192 
10193 	r = kvm_mmu_reload(vcpu);
10194 	if (unlikely(r))
10195 		return;
10196 
10197 	if (!vcpu->arch.mmu->direct_map &&
10198 	      work->arch.cr3 != vcpu->arch.mmu->get_cr3(vcpu))
10199 		return;
10200 
10201 	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
10202 }
10203 
10204 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
10205 {
10206 	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
10207 }
10208 
10209 static inline u32 kvm_async_pf_next_probe(u32 key)
10210 {
10211 	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
10212 }
10213 
10214 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
10215 {
10216 	u32 key = kvm_async_pf_hash_fn(gfn);
10217 
10218 	while (vcpu->arch.apf.gfns[key] != ~0)
10219 		key = kvm_async_pf_next_probe(key);
10220 
10221 	vcpu->arch.apf.gfns[key] = gfn;
10222 }
10223 
10224 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
10225 {
10226 	int i;
10227 	u32 key = kvm_async_pf_hash_fn(gfn);
10228 
10229 	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
10230 		     (vcpu->arch.apf.gfns[key] != gfn &&
10231 		      vcpu->arch.apf.gfns[key] != ~0); i++)
10232 		key = kvm_async_pf_next_probe(key);
10233 
10234 	return key;
10235 }
10236 
10237 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
10238 {
10239 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
10240 }
10241 
10242 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
10243 {
10244 	u32 i, j, k;
10245 
10246 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
10247 	while (true) {
10248 		vcpu->arch.apf.gfns[i] = ~0;
10249 		do {
10250 			j = kvm_async_pf_next_probe(j);
10251 			if (vcpu->arch.apf.gfns[j] == ~0)
10252 				return;
10253 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
10254 			/*
10255 			 * k lies cyclically in ]i,j]
10256 			 * |    i.k.j |
10257 			 * |....j i.k.| or  |.k..j i...|
10258 			 */
10259 		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
10260 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
10261 		i = j;
10262 	}
10263 }
10264 
10265 static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
10266 {
10267 
10268 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
10269 				      sizeof(val));
10270 }
10271 
10272 static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val)
10273 {
10274 
10275 	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val,
10276 				      sizeof(u32));
10277 }
10278 
10279 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
10280 {
10281 	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
10282 		return false;
10283 
10284 	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
10285 	    (vcpu->arch.apf.send_user_only &&
10286 	     kvm_x86_ops->get_cpl(vcpu) == 0))
10287 		return false;
10288 
10289 	return true;
10290 }
10291 
10292 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
10293 {
10294 	if (unlikely(!lapic_in_kernel(vcpu) ||
10295 		     kvm_event_needs_reinjection(vcpu) ||
10296 		     vcpu->arch.exception.pending))
10297 		return false;
10298 
10299 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
10300 		return false;
10301 
10302 	/*
10303 	 * If interrupts are off we cannot even use an artificial
10304 	 * halt state.
10305 	 */
10306 	return kvm_x86_ops->interrupt_allowed(vcpu);
10307 }
10308 
10309 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
10310 				     struct kvm_async_pf *work)
10311 {
10312 	struct x86_exception fault;
10313 
10314 	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
10315 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
10316 
10317 	if (kvm_can_deliver_async_pf(vcpu) &&
10318 	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
10319 		fault.vector = PF_VECTOR;
10320 		fault.error_code_valid = true;
10321 		fault.error_code = 0;
10322 		fault.nested_page_fault = false;
10323 		fault.address = work->arch.token;
10324 		fault.async_page_fault = true;
10325 		kvm_inject_page_fault(vcpu, &fault);
10326 	} else {
10327 		/*
10328 		 * It is not possible to deliver a paravirtualized asynchronous
10329 		 * page fault, but putting the guest in an artificial halt state
10330 		 * can be beneficial nevertheless: if an interrupt arrives, we
10331 		 * can deliver it timely and perhaps the guest will schedule
10332 		 * another process.  When the instruction that triggered a page
10333 		 * fault is retried, hopefully the page will be ready in the host.
10334 		 */
10335 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
10336 	}
10337 }
10338 
10339 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
10340 				 struct kvm_async_pf *work)
10341 {
10342 	struct x86_exception fault;
10343 	u32 val;
10344 
10345 	if (work->wakeup_all)
10346 		work->arch.token = ~0; /* broadcast wakeup */
10347 	else
10348 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
10349 	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
10350 
10351 	if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED &&
10352 	    !apf_get_user(vcpu, &val)) {
10353 		if (val == KVM_PV_REASON_PAGE_NOT_PRESENT &&
10354 		    vcpu->arch.exception.pending &&
10355 		    vcpu->arch.exception.nr == PF_VECTOR &&
10356 		    !apf_put_user(vcpu, 0)) {
10357 			vcpu->arch.exception.injected = false;
10358 			vcpu->arch.exception.pending = false;
10359 			vcpu->arch.exception.nr = 0;
10360 			vcpu->arch.exception.has_error_code = false;
10361 			vcpu->arch.exception.error_code = 0;
10362 			vcpu->arch.exception.has_payload = false;
10363 			vcpu->arch.exception.payload = 0;
10364 		} else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
10365 			fault.vector = PF_VECTOR;
10366 			fault.error_code_valid = true;
10367 			fault.error_code = 0;
10368 			fault.nested_page_fault = false;
10369 			fault.address = work->arch.token;
10370 			fault.async_page_fault = true;
10371 			kvm_inject_page_fault(vcpu, &fault);
10372 		}
10373 	}
10374 	vcpu->arch.apf.halted = false;
10375 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10376 }
10377 
10378 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
10379 {
10380 	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
10381 		return true;
10382 	else
10383 		return kvm_can_do_async_pf(vcpu);
10384 }
10385 
10386 void kvm_arch_start_assignment(struct kvm *kvm)
10387 {
10388 	atomic_inc(&kvm->arch.assigned_device_count);
10389 }
10390 EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);
10391 
10392 void kvm_arch_end_assignment(struct kvm *kvm)
10393 {
10394 	atomic_dec(&kvm->arch.assigned_device_count);
10395 }
10396 EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);
10397 
10398 bool kvm_arch_has_assigned_device(struct kvm *kvm)
10399 {
10400 	return atomic_read(&kvm->arch.assigned_device_count);
10401 }
10402 EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
10403 
10404 void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
10405 {
10406 	atomic_inc(&kvm->arch.noncoherent_dma_count);
10407 }
10408 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
10409 
10410 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
10411 {
10412 	atomic_dec(&kvm->arch.noncoherent_dma_count);
10413 }
10414 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
10415 
10416 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
10417 {
10418 	return atomic_read(&kvm->arch.noncoherent_dma_count);
10419 }
10420 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
10421 
10422 bool kvm_arch_has_irq_bypass(void)
10423 {
10424 	return true;
10425 }
10426 
10427 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
10428 				      struct irq_bypass_producer *prod)
10429 {
10430 	struct kvm_kernel_irqfd *irqfd =
10431 		container_of(cons, struct kvm_kernel_irqfd, consumer);
10432 
10433 	irqfd->producer = prod;
10434 
10435 	return kvm_x86_ops->update_pi_irte(irqfd->kvm,
10436 					   prod->irq, irqfd->gsi, 1);
10437 }
10438 
10439 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
10440 				      struct irq_bypass_producer *prod)
10441 {
10442 	int ret;
10443 	struct kvm_kernel_irqfd *irqfd =
10444 		container_of(cons, struct kvm_kernel_irqfd, consumer);
10445 
10446 	WARN_ON(irqfd->producer != prod);
10447 	irqfd->producer = NULL;
10448 
10449 	/*
10450 	 * When producer of consumer is unregistered, we change back to
10451 	 * remapped mode, so we can re-use the current implementation
10452 	 * when the irq is masked/disabled or the consumer side (KVM
10453 	 * int this case doesn't want to receive the interrupts.
10454 	*/
10455 	ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
10456 	if (ret)
10457 		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
10458 		       " fails: %d\n", irqfd->consumer.token, ret);
10459 }
10460 
10461 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
10462 				   uint32_t guest_irq, bool set)
10463 {
10464 	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
10465 }
10466 
10467 bool kvm_vector_hashing_enabled(void)
10468 {
10469 	return vector_hashing;
10470 }
10471 
10472 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
10473 {
10474 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
10475 }
10476 EXPORT_SYMBOL_GPL(kvm_arch_no_poll);
10477 
10478 u64 kvm_spec_ctrl_valid_bits(struct kvm_vcpu *vcpu)
10479 {
10480 	uint64_t bits = SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD;
10481 
10482 	/* The STIBP bit doesn't fault even if it's not advertised */
10483 	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
10484 	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS))
10485 		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);
10486 	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
10487 	    !boot_cpu_has(X86_FEATURE_AMD_IBRS))
10488 		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);
10489 
10490 	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL_SSBD) &&
10491 	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
10492 		bits &= ~SPEC_CTRL_SSBD;
10493 	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
10494 	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
10495 		bits &= ~SPEC_CTRL_SSBD;
10496 
10497 	return bits;
10498 }
10499 EXPORT_SYMBOL_GPL(kvm_spec_ctrl_valid_bits);
10500 
10501 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
10502 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
10503 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
10504 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
10505 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
10506 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
10507 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
10508 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10509 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10510 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
10511 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10512 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10513 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10514 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10515 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
10516 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
10517 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10518 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10519 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
10520 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
10521 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request);
10522