xref: /openbmc/linux/arch/x86/kvm/svm/svm.c (revision ca2478a7d974f38d29d27acb42a952c7f168916e)
1  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2  
3  #include <linux/kvm_host.h>
4  
5  #include "irq.h"
6  #include "mmu.h"
7  #include "kvm_cache_regs.h"
8  #include "x86.h"
9  #include "smm.h"
10  #include "cpuid.h"
11  #include "pmu.h"
12  
13  #include <linux/module.h>
14  #include <linux/mod_devicetable.h>
15  #include <linux/kernel.h>
16  #include <linux/vmalloc.h>
17  #include <linux/highmem.h>
18  #include <linux/amd-iommu.h>
19  #include <linux/sched.h>
20  #include <linux/trace_events.h>
21  #include <linux/slab.h>
22  #include <linux/hashtable.h>
23  #include <linux/objtool.h>
24  #include <linux/psp-sev.h>
25  #include <linux/file.h>
26  #include <linux/pagemap.h>
27  #include <linux/swap.h>
28  #include <linux/rwsem.h>
29  #include <linux/cc_platform.h>
30  #include <linux/smp.h>
31  
32  #include <asm/apic.h>
33  #include <asm/perf_event.h>
34  #include <asm/tlbflush.h>
35  #include <asm/desc.h>
36  #include <asm/debugreg.h>
37  #include <asm/kvm_para.h>
38  #include <asm/irq_remapping.h>
39  #include <asm/spec-ctrl.h>
40  #include <asm/cpu_device_id.h>
41  #include <asm/traps.h>
42  #include <asm/reboot.h>
43  #include <asm/fpu/api.h>
44  
45  #include <trace/events/ipi.h>
46  
47  #include "trace.h"
48  
49  #include "svm.h"
50  #include "svm_ops.h"
51  
52  #include "kvm_onhyperv.h"
53  #include "svm_onhyperv.h"
54  
55  MODULE_AUTHOR("Qumranet");
56  MODULE_LICENSE("GPL");
57  
58  #ifdef MODULE
59  static const struct x86_cpu_id svm_cpu_id[] = {
60  	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
61  	{}
62  };
63  MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
64  #endif
65  
66  #define SEG_TYPE_LDT 2
67  #define SEG_TYPE_BUSY_TSS16 3
68  
69  static bool erratum_383_found __read_mostly;
70  
71  u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
72  
73  /*
74   * Set osvw_len to higher value when updated Revision Guides
75   * are published and we know what the new status bits are
76   */
77  static uint64_t osvw_len = 4, osvw_status;
78  
79  static DEFINE_PER_CPU(u64, current_tsc_ratio);
80  
81  #define X2APIC_MSR(x)	(APIC_BASE_MSR + (x >> 4))
82  
83  static const struct svm_direct_access_msrs {
84  	u32 index;   /* Index of the MSR */
85  	bool always; /* True if intercept is initially cleared */
86  } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
87  	{ .index = MSR_STAR,				.always = true  },
88  	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
89  	{ .index = MSR_IA32_SYSENTER_EIP,		.always = false },
90  	{ .index = MSR_IA32_SYSENTER_ESP,		.always = false },
91  #ifdef CONFIG_X86_64
92  	{ .index = MSR_GS_BASE,				.always = true  },
93  	{ .index = MSR_FS_BASE,				.always = true  },
94  	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
95  	{ .index = MSR_LSTAR,				.always = true  },
96  	{ .index = MSR_CSTAR,				.always = true  },
97  	{ .index = MSR_SYSCALL_MASK,			.always = true  },
98  #endif
99  	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
100  	{ .index = MSR_IA32_PRED_CMD,			.always = false },
101  	{ .index = MSR_IA32_FLUSH_CMD,			.always = false },
102  	{ .index = MSR_IA32_DEBUGCTLMSR,		.always = false },
103  	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
104  	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
105  	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
106  	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
107  	{ .index = MSR_IA32_XSS,			.always = false },
108  	{ .index = MSR_EFER,				.always = false },
109  	{ .index = MSR_IA32_CR_PAT,			.always = false },
110  	{ .index = MSR_AMD64_SEV_ES_GHCB,		.always = true  },
111  	{ .index = MSR_TSC_AUX,				.always = false },
112  	{ .index = X2APIC_MSR(APIC_ID),			.always = false },
113  	{ .index = X2APIC_MSR(APIC_LVR),		.always = false },
114  	{ .index = X2APIC_MSR(APIC_TASKPRI),		.always = false },
115  	{ .index = X2APIC_MSR(APIC_ARBPRI),		.always = false },
116  	{ .index = X2APIC_MSR(APIC_PROCPRI),		.always = false },
117  	{ .index = X2APIC_MSR(APIC_EOI),		.always = false },
118  	{ .index = X2APIC_MSR(APIC_RRR),		.always = false },
119  	{ .index = X2APIC_MSR(APIC_LDR),		.always = false },
120  	{ .index = X2APIC_MSR(APIC_DFR),		.always = false },
121  	{ .index = X2APIC_MSR(APIC_SPIV),		.always = false },
122  	{ .index = X2APIC_MSR(APIC_ISR),		.always = false },
123  	{ .index = X2APIC_MSR(APIC_TMR),		.always = false },
124  	{ .index = X2APIC_MSR(APIC_IRR),		.always = false },
125  	{ .index = X2APIC_MSR(APIC_ESR),		.always = false },
126  	{ .index = X2APIC_MSR(APIC_ICR),		.always = false },
127  	{ .index = X2APIC_MSR(APIC_ICR2),		.always = false },
128  
129  	/*
130  	 * Note:
131  	 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
132  	 * emulated by KVM. When setting APIC LVTT (0x832) register bit 18,
133  	 * the AVIC hardware would generate GP fault. Therefore, always
134  	 * intercept the MSR 0x832, and do not setup direct_access_msr.
135  	 */
136  	{ .index = X2APIC_MSR(APIC_LVTTHMR),		.always = false },
137  	{ .index = X2APIC_MSR(APIC_LVTPC),		.always = false },
138  	{ .index = X2APIC_MSR(APIC_LVT0),		.always = false },
139  	{ .index = X2APIC_MSR(APIC_LVT1),		.always = false },
140  	{ .index = X2APIC_MSR(APIC_LVTERR),		.always = false },
141  	{ .index = X2APIC_MSR(APIC_TMICT),		.always = false },
142  	{ .index = X2APIC_MSR(APIC_TMCCT),		.always = false },
143  	{ .index = X2APIC_MSR(APIC_TDCR),		.always = false },
144  	{ .index = MSR_INVALID,				.always = false },
145  };
146  
147  /*
148   * These 2 parameters are used to config the controls for Pause-Loop Exiting:
149   * pause_filter_count: On processors that support Pause filtering(indicated
150   *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
151   *	count value. On VMRUN this value is loaded into an internal counter.
152   *	Each time a pause instruction is executed, this counter is decremented
153   *	until it reaches zero at which time a #VMEXIT is generated if pause
154   *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
155   *	Intercept Filtering for more details.
156   *	This also indicate if ple logic enabled.
157   *
158   * pause_filter_thresh: In addition, some processor families support advanced
159   *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
160   *	the amount of time a guest is allowed to execute in a pause loop.
161   *	In this mode, a 16-bit pause filter threshold field is added in the
162   *	VMCB. The threshold value is a cycle count that is used to reset the
163   *	pause counter. As with simple pause filtering, VMRUN loads the pause
164   *	count value from VMCB into an internal counter. Then, on each pause
165   *	instruction the hardware checks the elapsed number of cycles since
166   *	the most recent pause instruction against the pause filter threshold.
167   *	If the elapsed cycle count is greater than the pause filter threshold,
168   *	then the internal pause count is reloaded from the VMCB and execution
169   *	continues. If the elapsed cycle count is less than the pause filter
170   *	threshold, then the internal pause count is decremented. If the count
171   *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
172   *	triggered. If advanced pause filtering is supported and pause filter
173   *	threshold field is set to zero, the filter will operate in the simpler,
174   *	count only mode.
175   */
176  
177  static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
178  module_param(pause_filter_thresh, ushort, 0444);
179  
180  static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
181  module_param(pause_filter_count, ushort, 0444);
182  
183  /* Default doubles per-vcpu window every exit. */
184  static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
185  module_param(pause_filter_count_grow, ushort, 0444);
186  
187  /* Default resets per-vcpu window every exit to pause_filter_count. */
188  static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
189  module_param(pause_filter_count_shrink, ushort, 0444);
190  
191  /* Default is to compute the maximum so we can never overflow. */
192  static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
193  module_param(pause_filter_count_max, ushort, 0444);
194  
195  /*
196   * Use nested page tables by default.  Note, NPT may get forced off by
197   * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
198   */
199  bool npt_enabled = true;
200  module_param_named(npt, npt_enabled, bool, 0444);
201  
202  /* allow nested virtualization in KVM/SVM */
203  static int nested = true;
204  module_param(nested, int, S_IRUGO);
205  
206  /* enable/disable Next RIP Save */
207  int nrips = true;
208  module_param(nrips, int, 0444);
209  
210  /* enable/disable Virtual VMLOAD VMSAVE */
211  static int vls = true;
212  module_param(vls, int, 0444);
213  
214  /* enable/disable Virtual GIF */
215  int vgif = true;
216  module_param(vgif, int, 0444);
217  
218  /* enable/disable LBR virtualization */
219  int lbrv = true;
220  module_param(lbrv, int, 0444);
221  
222  static int tsc_scaling = true;
223  module_param(tsc_scaling, int, 0444);
224  
225  /*
226   * enable / disable AVIC.  Because the defaults differ for APICv
227   * support between VMX and SVM we cannot use module_param_named.
228   */
229  static bool avic;
230  module_param(avic, bool, 0444);
231  
232  bool __read_mostly dump_invalid_vmcb;
233  module_param(dump_invalid_vmcb, bool, 0644);
234  
235  
236  bool intercept_smi = true;
237  module_param(intercept_smi, bool, 0444);
238  
239  bool vnmi = true;
240  module_param(vnmi, bool, 0444);
241  
242  static bool svm_gp_erratum_intercept = true;
243  
244  static u8 rsm_ins_bytes[] = "\x0f\xaa";
245  
246  static unsigned long iopm_base;
247  
248  DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
249  
250  /*
251   * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
252   * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
253   *
254   * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
255   * defer the restoration of TSC_AUX until the CPU returns to userspace.
256   */
257  static int tsc_aux_uret_slot __read_mostly = -1;
258  
259  static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
260  
261  #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
262  #define MSRS_RANGE_SIZE 2048
263  #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
264  
265  u32 svm_msrpm_offset(u32 msr)
266  {
267  	u32 offset;
268  	int i;
269  
270  	for (i = 0; i < NUM_MSR_MAPS; i++) {
271  		if (msr < msrpm_ranges[i] ||
272  		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
273  			continue;
274  
275  		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
276  		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
277  
278  		/* Now we have the u8 offset - but need the u32 offset */
279  		return offset / 4;
280  	}
281  
282  	/* MSR not in any range */
283  	return MSR_INVALID;
284  }
285  
286  static void svm_flush_tlb_current(struct kvm_vcpu *vcpu);
287  
288  static int get_npt_level(void)
289  {
290  #ifdef CONFIG_X86_64
291  	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
292  #else
293  	return PT32E_ROOT_LEVEL;
294  #endif
295  }
296  
297  int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
298  {
299  	struct vcpu_svm *svm = to_svm(vcpu);
300  	u64 old_efer = vcpu->arch.efer;
301  	vcpu->arch.efer = efer;
302  
303  	if (!npt_enabled) {
304  		/* Shadow paging assumes NX to be available.  */
305  		efer |= EFER_NX;
306  
307  		if (!(efer & EFER_LMA))
308  			efer &= ~EFER_LME;
309  	}
310  
311  	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
312  		if (!(efer & EFER_SVME)) {
313  			svm_leave_nested(vcpu);
314  			svm_set_gif(svm, true);
315  			/* #GP intercept is still needed for vmware backdoor */
316  			if (!enable_vmware_backdoor)
317  				clr_exception_intercept(svm, GP_VECTOR);
318  
319  			/*
320  			 * Free the nested guest state, unless we are in SMM.
321  			 * In this case we will return to the nested guest
322  			 * as soon as we leave SMM.
323  			 */
324  			if (!is_smm(vcpu))
325  				svm_free_nested(svm);
326  
327  		} else {
328  			int ret = svm_allocate_nested(svm);
329  
330  			if (ret) {
331  				vcpu->arch.efer = old_efer;
332  				return ret;
333  			}
334  
335  			/*
336  			 * Never intercept #GP for SEV guests, KVM can't
337  			 * decrypt guest memory to workaround the erratum.
338  			 */
339  			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
340  				set_exception_intercept(svm, GP_VECTOR);
341  		}
342  	}
343  
344  	svm->vmcb->save.efer = efer | EFER_SVME;
345  	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
346  	return 0;
347  }
348  
349  static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
350  {
351  	struct vcpu_svm *svm = to_svm(vcpu);
352  	u32 ret = 0;
353  
354  	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
355  		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
356  	return ret;
357  }
358  
359  static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
360  {
361  	struct vcpu_svm *svm = to_svm(vcpu);
362  
363  	if (mask == 0)
364  		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
365  	else
366  		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
367  
368  }
369  static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
370  					void *insn, int insn_len);
371  
372  static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
373  					   bool commit_side_effects)
374  {
375  	struct vcpu_svm *svm = to_svm(vcpu);
376  	unsigned long old_rflags;
377  
378  	/*
379  	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
380  	 * the type of exit and the #VC handler in the guest.
381  	 */
382  	if (sev_es_guest(vcpu->kvm))
383  		goto done;
384  
385  	if (nrips && svm->vmcb->control.next_rip != 0) {
386  		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
387  		svm->next_rip = svm->vmcb->control.next_rip;
388  	}
389  
390  	if (!svm->next_rip) {
391  		/*
392  		 * FIXME: Drop this when kvm_emulate_instruction() does the
393  		 * right thing and treats "can't emulate" as outright failure
394  		 * for EMULTYPE_SKIP.
395  		 */
396  		if (!svm_can_emulate_instruction(vcpu, EMULTYPE_SKIP, NULL, 0))
397  			return 0;
398  
399  		if (unlikely(!commit_side_effects))
400  			old_rflags = svm->vmcb->save.rflags;
401  
402  		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
403  			return 0;
404  
405  		if (unlikely(!commit_side_effects))
406  			svm->vmcb->save.rflags = old_rflags;
407  	} else {
408  		kvm_rip_write(vcpu, svm->next_rip);
409  	}
410  
411  done:
412  	if (likely(commit_side_effects))
413  		svm_set_interrupt_shadow(vcpu, 0);
414  
415  	return 1;
416  }
417  
418  static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
419  {
420  	return __svm_skip_emulated_instruction(vcpu, true);
421  }
422  
423  static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
424  {
425  	unsigned long rip, old_rip = kvm_rip_read(vcpu);
426  	struct vcpu_svm *svm = to_svm(vcpu);
427  
428  	/*
429  	 * Due to architectural shortcomings, the CPU doesn't always provide
430  	 * NextRIP, e.g. if KVM intercepted an exception that occurred while
431  	 * the CPU was vectoring an INTO/INT3 in the guest.  Temporarily skip
432  	 * the instruction even if NextRIP is supported to acquire the next
433  	 * RIP so that it can be shoved into the NextRIP field, otherwise
434  	 * hardware will fail to advance guest RIP during event injection.
435  	 * Drop the exception/interrupt if emulation fails and effectively
436  	 * retry the instruction, it's the least awful option.  If NRIPS is
437  	 * in use, the skip must not commit any side effects such as clearing
438  	 * the interrupt shadow or RFLAGS.RF.
439  	 */
440  	if (!__svm_skip_emulated_instruction(vcpu, !nrips))
441  		return -EIO;
442  
443  	rip = kvm_rip_read(vcpu);
444  
445  	/*
446  	 * Save the injection information, even when using next_rip, as the
447  	 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection
448  	 * doesn't complete due to a VM-Exit occurring while the CPU is
449  	 * vectoring the event.   Decoding the instruction isn't guaranteed to
450  	 * work as there may be no backing instruction, e.g. if the event is
451  	 * being injected by L1 for L2, or if the guest is patching INT3 into
452  	 * a different instruction.
453  	 */
454  	svm->soft_int_injected = true;
455  	svm->soft_int_csbase = svm->vmcb->save.cs.base;
456  	svm->soft_int_old_rip = old_rip;
457  	svm->soft_int_next_rip = rip;
458  
459  	if (nrips)
460  		kvm_rip_write(vcpu, old_rip);
461  
462  	if (static_cpu_has(X86_FEATURE_NRIPS))
463  		svm->vmcb->control.next_rip = rip;
464  
465  	return 0;
466  }
467  
468  static void svm_inject_exception(struct kvm_vcpu *vcpu)
469  {
470  	struct kvm_queued_exception *ex = &vcpu->arch.exception;
471  	struct vcpu_svm *svm = to_svm(vcpu);
472  
473  	kvm_deliver_exception_payload(vcpu, ex);
474  
475  	if (kvm_exception_is_soft(ex->vector) &&
476  	    svm_update_soft_interrupt_rip(vcpu))
477  		return;
478  
479  	svm->vmcb->control.event_inj = ex->vector
480  		| SVM_EVTINJ_VALID
481  		| (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
482  		| SVM_EVTINJ_TYPE_EXEPT;
483  	svm->vmcb->control.event_inj_err = ex->error_code;
484  }
485  
486  static void svm_init_erratum_383(void)
487  {
488  	u32 low, high;
489  	int err;
490  	u64 val;
491  
492  	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
493  		return;
494  
495  	/* Use _safe variants to not break nested virtualization */
496  	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
497  	if (err)
498  		return;
499  
500  	val |= (1ULL << 47);
501  
502  	low  = lower_32_bits(val);
503  	high = upper_32_bits(val);
504  
505  	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
506  
507  	erratum_383_found = true;
508  }
509  
510  static void svm_init_osvw(struct kvm_vcpu *vcpu)
511  {
512  	/*
513  	 * Guests should see errata 400 and 415 as fixed (assuming that
514  	 * HLT and IO instructions are intercepted).
515  	 */
516  	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
517  	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
518  
519  	/*
520  	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
521  	 * all osvw.status bits inside that length, including bit 0 (which is
522  	 * reserved for erratum 298), are valid. However, if host processor's
523  	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
524  	 * be conservative here and therefore we tell the guest that erratum 298
525  	 * is present (because we really don't know).
526  	 */
527  	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
528  		vcpu->arch.osvw.status |= 1;
529  }
530  
531  static bool __kvm_is_svm_supported(void)
532  {
533  	int cpu = smp_processor_id();
534  	struct cpuinfo_x86 *c = &cpu_data(cpu);
535  
536  	u64 vm_cr;
537  
538  	if (c->x86_vendor != X86_VENDOR_AMD &&
539  	    c->x86_vendor != X86_VENDOR_HYGON) {
540  		pr_err("CPU %d isn't AMD or Hygon\n", cpu);
541  		return false;
542  	}
543  
544  	if (!cpu_has(c, X86_FEATURE_SVM)) {
545  		pr_err("SVM not supported by CPU %d\n", cpu);
546  		return false;
547  	}
548  
549  	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
550  		pr_info("KVM is unsupported when running as an SEV guest\n");
551  		return false;
552  	}
553  
554  	rdmsrl(MSR_VM_CR, vm_cr);
555  	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) {
556  		pr_err("SVM disabled (by BIOS) in MSR_VM_CR on CPU %d\n", cpu);
557  		return false;
558  	}
559  
560  	return true;
561  }
562  
563  static bool kvm_is_svm_supported(void)
564  {
565  	bool supported;
566  
567  	migrate_disable();
568  	supported = __kvm_is_svm_supported();
569  	migrate_enable();
570  
571  	return supported;
572  }
573  
574  static int svm_check_processor_compat(void)
575  {
576  	if (!__kvm_is_svm_supported())
577  		return -EIO;
578  
579  	return 0;
580  }
581  
582  static void __svm_write_tsc_multiplier(u64 multiplier)
583  {
584  	if (multiplier == __this_cpu_read(current_tsc_ratio))
585  		return;
586  
587  	wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
588  	__this_cpu_write(current_tsc_ratio, multiplier);
589  }
590  
591  static inline void kvm_cpu_svm_disable(void)
592  {
593  	uint64_t efer;
594  
595  	wrmsrl(MSR_VM_HSAVE_PA, 0);
596  	rdmsrl(MSR_EFER, efer);
597  	if (efer & EFER_SVME) {
598  		/*
599  		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
600  		 * NMI aren't blocked.
601  		 */
602  		stgi();
603  		wrmsrl(MSR_EFER, efer & ~EFER_SVME);
604  	}
605  }
606  
607  static void svm_emergency_disable(void)
608  {
609  	kvm_rebooting = true;
610  
611  	kvm_cpu_svm_disable();
612  }
613  
614  static void svm_hardware_disable(void)
615  {
616  	/* Make sure we clean up behind us */
617  	if (tsc_scaling)
618  		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
619  
620  	kvm_cpu_svm_disable();
621  
622  	amd_pmu_disable_virt();
623  }
624  
625  static int svm_hardware_enable(void)
626  {
627  
628  	struct svm_cpu_data *sd;
629  	uint64_t efer;
630  	int me = raw_smp_processor_id();
631  
632  	rdmsrl(MSR_EFER, efer);
633  	if (efer & EFER_SVME)
634  		return -EBUSY;
635  
636  	sd = per_cpu_ptr(&svm_data, me);
637  	sd->asid_generation = 1;
638  	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
639  	sd->next_asid = sd->max_asid + 1;
640  	sd->min_asid = max_sev_asid + 1;
641  
642  	wrmsrl(MSR_EFER, efer | EFER_SVME);
643  
644  	wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa);
645  
646  	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
647  		/*
648  		 * Set the default value, even if we don't use TSC scaling
649  		 * to avoid having stale value in the msr
650  		 */
651  		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
652  	}
653  
654  
655  	/*
656  	 * Get OSVW bits.
657  	 *
658  	 * Note that it is possible to have a system with mixed processor
659  	 * revisions and therefore different OSVW bits. If bits are not the same
660  	 * on different processors then choose the worst case (i.e. if erratum
661  	 * is present on one processor and not on another then assume that the
662  	 * erratum is present everywhere).
663  	 */
664  	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
665  		uint64_t len, status = 0;
666  		int err;
667  
668  		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
669  		if (!err)
670  			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
671  						      &err);
672  
673  		if (err)
674  			osvw_status = osvw_len = 0;
675  		else {
676  			if (len < osvw_len)
677  				osvw_len = len;
678  			osvw_status |= status;
679  			osvw_status &= (1ULL << osvw_len) - 1;
680  		}
681  	} else
682  		osvw_status = osvw_len = 0;
683  
684  	svm_init_erratum_383();
685  
686  	amd_pmu_enable_virt();
687  
688  	/*
689  	 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
690  	 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
691  	 * Since Linux does not change the value of TSC_AUX once set, prime the
692  	 * TSC_AUX field now to avoid a RDMSR on every vCPU run.
693  	 */
694  	if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
695  		struct sev_es_save_area *hostsa;
696  		u32 __maybe_unused msr_hi;
697  
698  		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
699  
700  		rdmsr(MSR_TSC_AUX, hostsa->tsc_aux, msr_hi);
701  	}
702  
703  	return 0;
704  }
705  
706  static void svm_cpu_uninit(int cpu)
707  {
708  	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
709  
710  	if (!sd->save_area)
711  		return;
712  
713  	kfree(sd->sev_vmcbs);
714  	__free_page(sd->save_area);
715  	sd->save_area_pa = 0;
716  	sd->save_area = NULL;
717  }
718  
719  static int svm_cpu_init(int cpu)
720  {
721  	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
722  	int ret = -ENOMEM;
723  
724  	memset(sd, 0, sizeof(struct svm_cpu_data));
725  	sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO);
726  	if (!sd->save_area)
727  		return ret;
728  
729  	ret = sev_cpu_init(sd);
730  	if (ret)
731  		goto free_save_area;
732  
733  	sd->save_area_pa = __sme_page_pa(sd->save_area);
734  	return 0;
735  
736  free_save_area:
737  	__free_page(sd->save_area);
738  	sd->save_area = NULL;
739  	return ret;
740  
741  }
742  
743  static void set_dr_intercepts(struct vcpu_svm *svm)
744  {
745  	struct vmcb *vmcb = svm->vmcb01.ptr;
746  
747  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ);
748  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ);
749  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ);
750  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ);
751  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ);
752  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ);
753  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ);
754  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE);
755  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE);
756  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE);
757  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE);
758  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE);
759  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE);
760  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE);
761  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
762  	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
763  
764  	recalc_intercepts(svm);
765  }
766  
767  static void clr_dr_intercepts(struct vcpu_svm *svm)
768  {
769  	struct vmcb *vmcb = svm->vmcb01.ptr;
770  
771  	vmcb->control.intercepts[INTERCEPT_DR] = 0;
772  
773  	recalc_intercepts(svm);
774  }
775  
776  static int direct_access_msr_slot(u32 msr)
777  {
778  	u32 i;
779  
780  	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
781  		if (direct_access_msrs[i].index == msr)
782  			return i;
783  
784  	return -ENOENT;
785  }
786  
787  static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
788  				     int write)
789  {
790  	struct vcpu_svm *svm = to_svm(vcpu);
791  	int slot = direct_access_msr_slot(msr);
792  
793  	if (slot == -ENOENT)
794  		return;
795  
796  	/* Set the shadow bitmaps to the desired intercept states */
797  	if (read)
798  		set_bit(slot, svm->shadow_msr_intercept.read);
799  	else
800  		clear_bit(slot, svm->shadow_msr_intercept.read);
801  
802  	if (write)
803  		set_bit(slot, svm->shadow_msr_intercept.write);
804  	else
805  		clear_bit(slot, svm->shadow_msr_intercept.write);
806  }
807  
808  static bool valid_msr_intercept(u32 index)
809  {
810  	return direct_access_msr_slot(index) != -ENOENT;
811  }
812  
813  static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
814  {
815  	u8 bit_write;
816  	unsigned long tmp;
817  	u32 offset;
818  	u32 *msrpm;
819  
820  	/*
821  	 * For non-nested case:
822  	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
823  	 * save it.
824  	 *
825  	 * For nested case:
826  	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
827  	 * save it.
828  	 */
829  	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
830  				      to_svm(vcpu)->msrpm;
831  
832  	offset    = svm_msrpm_offset(msr);
833  	bit_write = 2 * (msr & 0x0f) + 1;
834  	tmp       = msrpm[offset];
835  
836  	BUG_ON(offset == MSR_INVALID);
837  
838  	return test_bit(bit_write, &tmp);
839  }
840  
841  static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
842  					u32 msr, int read, int write)
843  {
844  	struct vcpu_svm *svm = to_svm(vcpu);
845  	u8 bit_read, bit_write;
846  	unsigned long tmp;
847  	u32 offset;
848  
849  	/*
850  	 * If this warning triggers extend the direct_access_msrs list at the
851  	 * beginning of the file
852  	 */
853  	WARN_ON(!valid_msr_intercept(msr));
854  
855  	/* Enforce non allowed MSRs to trap */
856  	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
857  		read = 0;
858  
859  	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
860  		write = 0;
861  
862  	offset    = svm_msrpm_offset(msr);
863  	bit_read  = 2 * (msr & 0x0f);
864  	bit_write = 2 * (msr & 0x0f) + 1;
865  	tmp       = msrpm[offset];
866  
867  	BUG_ON(offset == MSR_INVALID);
868  
869  	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
870  	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
871  
872  	msrpm[offset] = tmp;
873  
874  	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
875  	svm->nested.force_msr_bitmap_recalc = true;
876  }
877  
878  void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
879  			  int read, int write)
880  {
881  	set_shadow_msr_intercept(vcpu, msr, read, write);
882  	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
883  }
884  
885  u32 *svm_vcpu_alloc_msrpm(void)
886  {
887  	unsigned int order = get_order(MSRPM_SIZE);
888  	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
889  	u32 *msrpm;
890  
891  	if (!pages)
892  		return NULL;
893  
894  	msrpm = page_address(pages);
895  	memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
896  
897  	return msrpm;
898  }
899  
900  void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
901  {
902  	int i;
903  
904  	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
905  		if (!direct_access_msrs[i].always)
906  			continue;
907  		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
908  	}
909  }
910  
911  void svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept)
912  {
913  	int i;
914  
915  	if (intercept == svm->x2avic_msrs_intercepted)
916  		return;
917  
918  	if (!x2avic_enabled)
919  		return;
920  
921  	for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) {
922  		int index = direct_access_msrs[i].index;
923  
924  		if ((index < APIC_BASE_MSR) ||
925  		    (index > APIC_BASE_MSR + 0xff))
926  			continue;
927  		set_msr_interception(&svm->vcpu, svm->msrpm, index,
928  				     !intercept, !intercept);
929  	}
930  
931  	svm->x2avic_msrs_intercepted = intercept;
932  }
933  
934  void svm_vcpu_free_msrpm(u32 *msrpm)
935  {
936  	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
937  }
938  
939  static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
940  {
941  	struct vcpu_svm *svm = to_svm(vcpu);
942  	u32 i;
943  
944  	/*
945  	 * Set intercept permissions for all direct access MSRs again. They
946  	 * will automatically get filtered through the MSR filter, so we are
947  	 * back in sync after this.
948  	 */
949  	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
950  		u32 msr = direct_access_msrs[i].index;
951  		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
952  		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
953  
954  		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
955  	}
956  }
957  
958  static void add_msr_offset(u32 offset)
959  {
960  	int i;
961  
962  	for (i = 0; i < MSRPM_OFFSETS; ++i) {
963  
964  		/* Offset already in list? */
965  		if (msrpm_offsets[i] == offset)
966  			return;
967  
968  		/* Slot used by another offset? */
969  		if (msrpm_offsets[i] != MSR_INVALID)
970  			continue;
971  
972  		/* Add offset to list */
973  		msrpm_offsets[i] = offset;
974  
975  		return;
976  	}
977  
978  	/*
979  	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
980  	 * increase MSRPM_OFFSETS in this case.
981  	 */
982  	BUG();
983  }
984  
985  static void init_msrpm_offsets(void)
986  {
987  	int i;
988  
989  	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
990  
991  	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
992  		u32 offset;
993  
994  		offset = svm_msrpm_offset(direct_access_msrs[i].index);
995  		BUG_ON(offset == MSR_INVALID);
996  
997  		add_msr_offset(offset);
998  	}
999  }
1000  
1001  void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
1002  {
1003  	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
1004  	to_vmcb->save.br_from		= from_vmcb->save.br_from;
1005  	to_vmcb->save.br_to		= from_vmcb->save.br_to;
1006  	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
1007  	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
1008  
1009  	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
1010  }
1011  
1012  void svm_enable_lbrv(struct kvm_vcpu *vcpu)
1013  {
1014  	struct vcpu_svm *svm = to_svm(vcpu);
1015  
1016  	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1017  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1018  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1019  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1020  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1021  
1022  	if (sev_es_guest(vcpu->kvm))
1023  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1);
1024  
1025  	/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
1026  	if (is_guest_mode(vcpu))
1027  		svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
1028  }
1029  
1030  static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
1031  {
1032  	struct vcpu_svm *svm = to_svm(vcpu);
1033  
1034  	KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
1035  
1036  	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1037  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1038  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1039  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1040  	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1041  
1042  	/*
1043  	 * Move the LBR msrs back to the vmcb01 to avoid copying them
1044  	 * on nested guest entries.
1045  	 */
1046  	if (is_guest_mode(vcpu))
1047  		svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb);
1048  }
1049  
1050  static struct vmcb *svm_get_lbr_vmcb(struct vcpu_svm *svm)
1051  {
1052  	/*
1053  	 * If LBR virtualization is disabled, the LBR MSRs are always kept in
1054  	 * vmcb01.  If LBR virtualization is enabled and L1 is running VMs of
1055  	 * its own, the MSRs are moved between vmcb01 and vmcb02 as needed.
1056  	 */
1057  	return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb :
1058  								   svm->vmcb01.ptr;
1059  }
1060  
1061  void svm_update_lbrv(struct kvm_vcpu *vcpu)
1062  {
1063  	struct vcpu_svm *svm = to_svm(vcpu);
1064  	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
1065  	bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) ||
1066  			    (is_guest_mode(vcpu) && guest_can_use(vcpu, X86_FEATURE_LBRV) &&
1067  			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
1068  
1069  	if (enable_lbrv == current_enable_lbrv)
1070  		return;
1071  
1072  	if (enable_lbrv)
1073  		svm_enable_lbrv(vcpu);
1074  	else
1075  		svm_disable_lbrv(vcpu);
1076  }
1077  
1078  void disable_nmi_singlestep(struct vcpu_svm *svm)
1079  {
1080  	svm->nmi_singlestep = false;
1081  
1082  	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1083  		/* Clear our flags if they were not set by the guest */
1084  		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1085  			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1086  		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1087  			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1088  	}
1089  }
1090  
1091  static void grow_ple_window(struct kvm_vcpu *vcpu)
1092  {
1093  	struct vcpu_svm *svm = to_svm(vcpu);
1094  	struct vmcb_control_area *control = &svm->vmcb->control;
1095  	int old = control->pause_filter_count;
1096  
1097  	if (kvm_pause_in_guest(vcpu->kvm))
1098  		return;
1099  
1100  	control->pause_filter_count = __grow_ple_window(old,
1101  							pause_filter_count,
1102  							pause_filter_count_grow,
1103  							pause_filter_count_max);
1104  
1105  	if (control->pause_filter_count != old) {
1106  		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1107  		trace_kvm_ple_window_update(vcpu->vcpu_id,
1108  					    control->pause_filter_count, old);
1109  	}
1110  }
1111  
1112  static void shrink_ple_window(struct kvm_vcpu *vcpu)
1113  {
1114  	struct vcpu_svm *svm = to_svm(vcpu);
1115  	struct vmcb_control_area *control = &svm->vmcb->control;
1116  	int old = control->pause_filter_count;
1117  
1118  	if (kvm_pause_in_guest(vcpu->kvm))
1119  		return;
1120  
1121  	control->pause_filter_count =
1122  				__shrink_ple_window(old,
1123  						    pause_filter_count,
1124  						    pause_filter_count_shrink,
1125  						    pause_filter_count);
1126  	if (control->pause_filter_count != old) {
1127  		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1128  		trace_kvm_ple_window_update(vcpu->vcpu_id,
1129  					    control->pause_filter_count, old);
1130  	}
1131  }
1132  
1133  static void svm_hardware_unsetup(void)
1134  {
1135  	int cpu;
1136  
1137  	sev_hardware_unsetup();
1138  
1139  	for_each_possible_cpu(cpu)
1140  		svm_cpu_uninit(cpu);
1141  
1142  	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT),
1143  	get_order(IOPM_SIZE));
1144  	iopm_base = 0;
1145  }
1146  
1147  static void init_seg(struct vmcb_seg *seg)
1148  {
1149  	seg->selector = 0;
1150  	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1151  		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1152  	seg->limit = 0xffff;
1153  	seg->base = 0;
1154  }
1155  
1156  static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1157  {
1158  	seg->selector = 0;
1159  	seg->attrib = SVM_SELECTOR_P_MASK | type;
1160  	seg->limit = 0xffff;
1161  	seg->base = 0;
1162  }
1163  
1164  static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1165  {
1166  	struct vcpu_svm *svm = to_svm(vcpu);
1167  
1168  	return svm->nested.ctl.tsc_offset;
1169  }
1170  
1171  static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1172  {
1173  	struct vcpu_svm *svm = to_svm(vcpu);
1174  
1175  	return svm->tsc_ratio_msr;
1176  }
1177  
1178  static void svm_write_tsc_offset(struct kvm_vcpu *vcpu)
1179  {
1180  	struct vcpu_svm *svm = to_svm(vcpu);
1181  
1182  	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1183  	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset;
1184  	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1185  }
1186  
1187  void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
1188  {
1189  	preempt_disable();
1190  	if (to_svm(vcpu)->guest_state_loaded)
1191  		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1192  	preempt_enable();
1193  }
1194  
1195  /* Evaluate instruction intercepts that depend on guest CPUID features. */
1196  static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1197  					      struct vcpu_svm *svm)
1198  {
1199  	/*
1200  	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1201  	 * roots, or if INVPCID is disabled in the guest to inject #UD.
1202  	 */
1203  	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1204  		if (!npt_enabled ||
1205  		    !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1206  			svm_set_intercept(svm, INTERCEPT_INVPCID);
1207  		else
1208  			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1209  	}
1210  
1211  	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1212  		if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1213  			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1214  		else
1215  			svm_set_intercept(svm, INTERCEPT_RDTSCP);
1216  	}
1217  }
1218  
1219  static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1220  {
1221  	struct vcpu_svm *svm = to_svm(vcpu);
1222  
1223  	if (guest_cpuid_is_intel(vcpu)) {
1224  		/*
1225  		 * We must intercept SYSENTER_EIP and SYSENTER_ESP
1226  		 * accesses because the processor only stores 32 bits.
1227  		 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1228  		 */
1229  		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1230  		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1231  		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1232  
1233  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1234  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1235  	} else {
1236  		/*
1237  		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1238  		 * in VMCB and clear intercepts to avoid #VMEXIT.
1239  		 */
1240  		if (vls) {
1241  			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1242  			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1243  			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1244  		}
1245  		/* No need to intercept these MSRs */
1246  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1247  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1248  	}
1249  }
1250  
1251  static void init_vmcb(struct kvm_vcpu *vcpu)
1252  {
1253  	struct vcpu_svm *svm = to_svm(vcpu);
1254  	struct vmcb *vmcb = svm->vmcb01.ptr;
1255  	struct vmcb_control_area *control = &vmcb->control;
1256  	struct vmcb_save_area *save = &vmcb->save;
1257  
1258  	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1259  	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1260  	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1261  	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1262  	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1263  	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1264  	if (!kvm_vcpu_apicv_active(vcpu))
1265  		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1266  
1267  	set_dr_intercepts(svm);
1268  
1269  	set_exception_intercept(svm, PF_VECTOR);
1270  	set_exception_intercept(svm, UD_VECTOR);
1271  	set_exception_intercept(svm, MC_VECTOR);
1272  	set_exception_intercept(svm, AC_VECTOR);
1273  	set_exception_intercept(svm, DB_VECTOR);
1274  	/*
1275  	 * Guest access to VMware backdoor ports could legitimately
1276  	 * trigger #GP because of TSS I/O permission bitmap.
1277  	 * We intercept those #GP and allow access to them anyway
1278  	 * as VMware does.
1279  	 */
1280  	if (enable_vmware_backdoor)
1281  		set_exception_intercept(svm, GP_VECTOR);
1282  
1283  	svm_set_intercept(svm, INTERCEPT_INTR);
1284  	svm_set_intercept(svm, INTERCEPT_NMI);
1285  
1286  	if (intercept_smi)
1287  		svm_set_intercept(svm, INTERCEPT_SMI);
1288  
1289  	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1290  	svm_set_intercept(svm, INTERCEPT_RDPMC);
1291  	svm_set_intercept(svm, INTERCEPT_CPUID);
1292  	svm_set_intercept(svm, INTERCEPT_INVD);
1293  	svm_set_intercept(svm, INTERCEPT_INVLPG);
1294  	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1295  	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1296  	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1297  	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1298  	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1299  	svm_set_intercept(svm, INTERCEPT_VMRUN);
1300  	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1301  	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1302  	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1303  	svm_set_intercept(svm, INTERCEPT_STGI);
1304  	svm_set_intercept(svm, INTERCEPT_CLGI);
1305  	svm_set_intercept(svm, INTERCEPT_SKINIT);
1306  	svm_set_intercept(svm, INTERCEPT_WBINVD);
1307  	svm_set_intercept(svm, INTERCEPT_XSETBV);
1308  	svm_set_intercept(svm, INTERCEPT_RDPRU);
1309  	svm_set_intercept(svm, INTERCEPT_RSM);
1310  
1311  	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1312  		svm_set_intercept(svm, INTERCEPT_MONITOR);
1313  		svm_set_intercept(svm, INTERCEPT_MWAIT);
1314  	}
1315  
1316  	if (!kvm_hlt_in_guest(vcpu->kvm))
1317  		svm_set_intercept(svm, INTERCEPT_HLT);
1318  
1319  	control->iopm_base_pa = __sme_set(iopm_base);
1320  	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1321  	control->int_ctl = V_INTR_MASKING_MASK;
1322  
1323  	init_seg(&save->es);
1324  	init_seg(&save->ss);
1325  	init_seg(&save->ds);
1326  	init_seg(&save->fs);
1327  	init_seg(&save->gs);
1328  
1329  	save->cs.selector = 0xf000;
1330  	save->cs.base = 0xffff0000;
1331  	/* Executable/Readable Code Segment */
1332  	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1333  		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1334  	save->cs.limit = 0xffff;
1335  
1336  	save->gdtr.base = 0;
1337  	save->gdtr.limit = 0xffff;
1338  	save->idtr.base = 0;
1339  	save->idtr.limit = 0xffff;
1340  
1341  	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1342  	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1343  
1344  	if (npt_enabled) {
1345  		/* Setup VMCB for Nested Paging */
1346  		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1347  		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1348  		clr_exception_intercept(svm, PF_VECTOR);
1349  		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1350  		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1351  		save->g_pat = vcpu->arch.pat;
1352  		save->cr3 = 0;
1353  	}
1354  	svm->current_vmcb->asid_generation = 0;
1355  	svm->asid = 0;
1356  
1357  	svm->nested.vmcb12_gpa = INVALID_GPA;
1358  	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1359  
1360  	if (!kvm_pause_in_guest(vcpu->kvm)) {
1361  		control->pause_filter_count = pause_filter_count;
1362  		if (pause_filter_thresh)
1363  			control->pause_filter_thresh = pause_filter_thresh;
1364  		svm_set_intercept(svm, INTERCEPT_PAUSE);
1365  	} else {
1366  		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1367  	}
1368  
1369  	svm_recalc_instruction_intercepts(vcpu, svm);
1370  
1371  	/*
1372  	 * If the host supports V_SPEC_CTRL then disable the interception
1373  	 * of MSR_IA32_SPEC_CTRL.
1374  	 */
1375  	if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1376  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1377  
1378  	if (kvm_vcpu_apicv_active(vcpu))
1379  		avic_init_vmcb(svm, vmcb);
1380  
1381  	if (vnmi)
1382  		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
1383  
1384  	if (vgif) {
1385  		svm_clr_intercept(svm, INTERCEPT_STGI);
1386  		svm_clr_intercept(svm, INTERCEPT_CLGI);
1387  		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1388  	}
1389  
1390  	if (sev_guest(vcpu->kvm))
1391  		sev_init_vmcb(svm);
1392  
1393  	svm_hv_init_vmcb(vmcb);
1394  	init_vmcb_after_set_cpuid(vcpu);
1395  
1396  	vmcb_mark_all_dirty(vmcb);
1397  
1398  	enable_gif(svm);
1399  }
1400  
1401  static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1402  {
1403  	struct vcpu_svm *svm = to_svm(vcpu);
1404  
1405  	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1406  
1407  	svm_init_osvw(vcpu);
1408  	vcpu->arch.microcode_version = 0x01000065;
1409  	svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio;
1410  
1411  	svm->nmi_masked = false;
1412  	svm->awaiting_iret_completion = false;
1413  
1414  	if (sev_es_guest(vcpu->kvm))
1415  		sev_es_vcpu_reset(svm);
1416  }
1417  
1418  static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1419  {
1420  	struct vcpu_svm *svm = to_svm(vcpu);
1421  
1422  	svm->spec_ctrl = 0;
1423  	svm->virt_spec_ctrl = 0;
1424  
1425  	init_vmcb(vcpu);
1426  
1427  	if (!init_event)
1428  		__svm_vcpu_reset(vcpu);
1429  }
1430  
1431  void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1432  {
1433  	svm->current_vmcb = target_vmcb;
1434  	svm->vmcb = target_vmcb->ptr;
1435  }
1436  
1437  static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1438  {
1439  	struct vcpu_svm *svm;
1440  	struct page *vmcb01_page;
1441  	struct page *vmsa_page = NULL;
1442  	int err;
1443  
1444  	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1445  	svm = to_svm(vcpu);
1446  
1447  	err = -ENOMEM;
1448  	vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1449  	if (!vmcb01_page)
1450  		goto out;
1451  
1452  	if (sev_es_guest(vcpu->kvm)) {
1453  		/*
1454  		 * SEV-ES guests require a separate VMSA page used to contain
1455  		 * the encrypted register state of the guest.
1456  		 */
1457  		vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1458  		if (!vmsa_page)
1459  			goto error_free_vmcb_page;
1460  
1461  		/*
1462  		 * SEV-ES guests maintain an encrypted version of their FPU
1463  		 * state which is restored and saved on VMRUN and VMEXIT.
1464  		 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't
1465  		 * do xsave/xrstor on it.
1466  		 */
1467  		fpstate_set_confidential(&vcpu->arch.guest_fpu);
1468  	}
1469  
1470  	err = avic_init_vcpu(svm);
1471  	if (err)
1472  		goto error_free_vmsa_page;
1473  
1474  	svm->msrpm = svm_vcpu_alloc_msrpm();
1475  	if (!svm->msrpm) {
1476  		err = -ENOMEM;
1477  		goto error_free_vmsa_page;
1478  	}
1479  
1480  	svm->x2avic_msrs_intercepted = true;
1481  
1482  	svm->vmcb01.ptr = page_address(vmcb01_page);
1483  	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1484  	svm_switch_vmcb(svm, &svm->vmcb01);
1485  
1486  	if (vmsa_page)
1487  		svm->sev_es.vmsa = page_address(vmsa_page);
1488  
1489  	svm->guest_state_loaded = false;
1490  
1491  	return 0;
1492  
1493  error_free_vmsa_page:
1494  	if (vmsa_page)
1495  		__free_page(vmsa_page);
1496  error_free_vmcb_page:
1497  	__free_page(vmcb01_page);
1498  out:
1499  	return err;
1500  }
1501  
1502  static void svm_clear_current_vmcb(struct vmcb *vmcb)
1503  {
1504  	int i;
1505  
1506  	for_each_online_cpu(i)
1507  		cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL);
1508  }
1509  
1510  static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1511  {
1512  	struct vcpu_svm *svm = to_svm(vcpu);
1513  
1514  	/*
1515  	 * The vmcb page can be recycled, causing a false negative in
1516  	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1517  	 * vmcb page recorded as its current vmcb.
1518  	 */
1519  	svm_clear_current_vmcb(svm->vmcb);
1520  
1521  	svm_leave_nested(vcpu);
1522  	svm_free_nested(svm);
1523  
1524  	sev_free_vcpu(vcpu);
1525  
1526  	__free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT));
1527  	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1528  }
1529  
1530  static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1531  {
1532  	struct vcpu_svm *svm = to_svm(vcpu);
1533  	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
1534  
1535  	if (sev_es_guest(vcpu->kvm))
1536  		sev_es_unmap_ghcb(svm);
1537  
1538  	if (svm->guest_state_loaded)
1539  		return;
1540  
1541  	/*
1542  	 * Save additional host state that will be restored on VMEXIT (sev-es)
1543  	 * or subsequent vmload of host save area.
1544  	 */
1545  	vmsave(sd->save_area_pa);
1546  	if (sev_es_guest(vcpu->kvm)) {
1547  		struct sev_es_save_area *hostsa;
1548  		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
1549  
1550  		sev_es_prepare_switch_to_guest(hostsa);
1551  	}
1552  
1553  	if (tsc_scaling)
1554  		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1555  
1556  	/*
1557  	 * TSC_AUX is always virtualized for SEV-ES guests when the feature is
1558  	 * available. The user return MSR support is not required in this case
1559  	 * because TSC_AUX is restored on #VMEXIT from the host save area
1560  	 * (which has been initialized in svm_hardware_enable()).
1561  	 */
1562  	if (likely(tsc_aux_uret_slot >= 0) &&
1563  	    (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1564  		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1565  
1566  	svm->guest_state_loaded = true;
1567  }
1568  
1569  static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1570  {
1571  	to_svm(vcpu)->guest_state_loaded = false;
1572  }
1573  
1574  static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1575  {
1576  	struct vcpu_svm *svm = to_svm(vcpu);
1577  	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
1578  
1579  	if (sd->current_vmcb != svm->vmcb) {
1580  		sd->current_vmcb = svm->vmcb;
1581  
1582  		if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT))
1583  			indirect_branch_prediction_barrier();
1584  	}
1585  	if (kvm_vcpu_apicv_active(vcpu))
1586  		avic_vcpu_load(vcpu, cpu);
1587  }
1588  
1589  static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1590  {
1591  	if (kvm_vcpu_apicv_active(vcpu))
1592  		avic_vcpu_put(vcpu);
1593  
1594  	svm_prepare_host_switch(vcpu);
1595  
1596  	++vcpu->stat.host_state_reload;
1597  }
1598  
1599  static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1600  {
1601  	struct vcpu_svm *svm = to_svm(vcpu);
1602  	unsigned long rflags = svm->vmcb->save.rflags;
1603  
1604  	if (svm->nmi_singlestep) {
1605  		/* Hide our flags if they were not set by the guest */
1606  		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1607  			rflags &= ~X86_EFLAGS_TF;
1608  		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1609  			rflags &= ~X86_EFLAGS_RF;
1610  	}
1611  	return rflags;
1612  }
1613  
1614  static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1615  {
1616  	if (to_svm(vcpu)->nmi_singlestep)
1617  		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1618  
1619         /*
1620          * Any change of EFLAGS.VM is accompanied by a reload of SS
1621          * (caused by either a task switch or an inter-privilege IRET),
1622          * so we do not need to update the CPL here.
1623          */
1624  	to_svm(vcpu)->vmcb->save.rflags = rflags;
1625  }
1626  
1627  static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1628  {
1629  	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1630  
1631  	return sev_es_guest(vcpu->kvm)
1632  		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1633  		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1634  }
1635  
1636  static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1637  {
1638  	kvm_register_mark_available(vcpu, reg);
1639  
1640  	switch (reg) {
1641  	case VCPU_EXREG_PDPTR:
1642  		/*
1643  		 * When !npt_enabled, mmu->pdptrs[] is already available since
1644  		 * it is always updated per SDM when moving to CRs.
1645  		 */
1646  		if (npt_enabled)
1647  			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1648  		break;
1649  	default:
1650  		KVM_BUG_ON(1, vcpu->kvm);
1651  	}
1652  }
1653  
1654  static void svm_set_vintr(struct vcpu_svm *svm)
1655  {
1656  	struct vmcb_control_area *control;
1657  
1658  	/*
1659  	 * The following fields are ignored when AVIC is enabled
1660  	 */
1661  	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1662  
1663  	svm_set_intercept(svm, INTERCEPT_VINTR);
1664  
1665  	/*
1666  	 * Recalculating intercepts may have cleared the VINTR intercept.  If
1667  	 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF
1668  	 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN.
1669  	 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as
1670  	 * interrupts will never be unblocked while L2 is running.
1671  	 */
1672  	if (!svm_is_intercept(svm, INTERCEPT_VINTR))
1673  		return;
1674  
1675  	/*
1676  	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1677  	 * Actual injection of virtual interrupts happens through EVENTINJ.
1678  	 */
1679  	control = &svm->vmcb->control;
1680  	control->int_vector = 0x0;
1681  	control->int_ctl &= ~V_INTR_PRIO_MASK;
1682  	control->int_ctl |= V_IRQ_MASK |
1683  		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1684  	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1685  }
1686  
1687  static void svm_clear_vintr(struct vcpu_svm *svm)
1688  {
1689  	svm_clr_intercept(svm, INTERCEPT_VINTR);
1690  
1691  	/* Drop int_ctl fields related to VINTR injection.  */
1692  	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1693  	if (is_guest_mode(&svm->vcpu)) {
1694  		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1695  
1696  		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1697  			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1698  
1699  		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1700  			V_IRQ_INJECTION_BITS_MASK;
1701  
1702  		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1703  	}
1704  
1705  	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1706  }
1707  
1708  static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1709  {
1710  	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1711  	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1712  
1713  	switch (seg) {
1714  	case VCPU_SREG_CS: return &save->cs;
1715  	case VCPU_SREG_DS: return &save->ds;
1716  	case VCPU_SREG_ES: return &save->es;
1717  	case VCPU_SREG_FS: return &save01->fs;
1718  	case VCPU_SREG_GS: return &save01->gs;
1719  	case VCPU_SREG_SS: return &save->ss;
1720  	case VCPU_SREG_TR: return &save01->tr;
1721  	case VCPU_SREG_LDTR: return &save01->ldtr;
1722  	}
1723  	BUG();
1724  	return NULL;
1725  }
1726  
1727  static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1728  {
1729  	struct vmcb_seg *s = svm_seg(vcpu, seg);
1730  
1731  	return s->base;
1732  }
1733  
1734  static void svm_get_segment(struct kvm_vcpu *vcpu,
1735  			    struct kvm_segment *var, int seg)
1736  {
1737  	struct vmcb_seg *s = svm_seg(vcpu, seg);
1738  
1739  	var->base = s->base;
1740  	var->limit = s->limit;
1741  	var->selector = s->selector;
1742  	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1743  	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1744  	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1745  	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1746  	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1747  	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1748  	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1749  
1750  	/*
1751  	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1752  	 * However, the SVM spec states that the G bit is not observed by the
1753  	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1754  	 * So let's synthesize a legal G bit for all segments, this helps
1755  	 * running KVM nested. It also helps cross-vendor migration, because
1756  	 * Intel's vmentry has a check on the 'G' bit.
1757  	 */
1758  	var->g = s->limit > 0xfffff;
1759  
1760  	/*
1761  	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1762  	 * for cross vendor migration purposes by "not present"
1763  	 */
1764  	var->unusable = !var->present;
1765  
1766  	switch (seg) {
1767  	case VCPU_SREG_TR:
1768  		/*
1769  		 * Work around a bug where the busy flag in the tr selector
1770  		 * isn't exposed
1771  		 */
1772  		var->type |= 0x2;
1773  		break;
1774  	case VCPU_SREG_DS:
1775  	case VCPU_SREG_ES:
1776  	case VCPU_SREG_FS:
1777  	case VCPU_SREG_GS:
1778  		/*
1779  		 * The accessed bit must always be set in the segment
1780  		 * descriptor cache, although it can be cleared in the
1781  		 * descriptor, the cached bit always remains at 1. Since
1782  		 * Intel has a check on this, set it here to support
1783  		 * cross-vendor migration.
1784  		 */
1785  		if (!var->unusable)
1786  			var->type |= 0x1;
1787  		break;
1788  	case VCPU_SREG_SS:
1789  		/*
1790  		 * On AMD CPUs sometimes the DB bit in the segment
1791  		 * descriptor is left as 1, although the whole segment has
1792  		 * been made unusable. Clear it here to pass an Intel VMX
1793  		 * entry check when cross vendor migrating.
1794  		 */
1795  		if (var->unusable)
1796  			var->db = 0;
1797  		/* This is symmetric with svm_set_segment() */
1798  		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1799  		break;
1800  	}
1801  }
1802  
1803  static int svm_get_cpl(struct kvm_vcpu *vcpu)
1804  {
1805  	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1806  
1807  	return save->cpl;
1808  }
1809  
1810  static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1811  {
1812  	struct kvm_segment cs;
1813  
1814  	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1815  	*db = cs.db;
1816  	*l = cs.l;
1817  }
1818  
1819  static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1820  {
1821  	struct vcpu_svm *svm = to_svm(vcpu);
1822  
1823  	dt->size = svm->vmcb->save.idtr.limit;
1824  	dt->address = svm->vmcb->save.idtr.base;
1825  }
1826  
1827  static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1828  {
1829  	struct vcpu_svm *svm = to_svm(vcpu);
1830  
1831  	svm->vmcb->save.idtr.limit = dt->size;
1832  	svm->vmcb->save.idtr.base = dt->address ;
1833  	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1834  }
1835  
1836  static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1837  {
1838  	struct vcpu_svm *svm = to_svm(vcpu);
1839  
1840  	dt->size = svm->vmcb->save.gdtr.limit;
1841  	dt->address = svm->vmcb->save.gdtr.base;
1842  }
1843  
1844  static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1845  {
1846  	struct vcpu_svm *svm = to_svm(vcpu);
1847  
1848  	svm->vmcb->save.gdtr.limit = dt->size;
1849  	svm->vmcb->save.gdtr.base = dt->address ;
1850  	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1851  }
1852  
1853  static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1854  {
1855  	struct vcpu_svm *svm = to_svm(vcpu);
1856  
1857  	/*
1858  	 * For guests that don't set guest_state_protected, the cr3 update is
1859  	 * handled via kvm_mmu_load() while entering the guest. For guests
1860  	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1861  	 * VMCB save area now, since the save area will become the initial
1862  	 * contents of the VMSA, and future VMCB save area updates won't be
1863  	 * seen.
1864  	 */
1865  	if (sev_es_guest(vcpu->kvm)) {
1866  		svm->vmcb->save.cr3 = cr3;
1867  		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1868  	}
1869  }
1870  
1871  static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1872  {
1873  	return true;
1874  }
1875  
1876  void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1877  {
1878  	struct vcpu_svm *svm = to_svm(vcpu);
1879  	u64 hcr0 = cr0;
1880  	bool old_paging = is_paging(vcpu);
1881  
1882  #ifdef CONFIG_X86_64
1883  	if (vcpu->arch.efer & EFER_LME) {
1884  		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1885  			vcpu->arch.efer |= EFER_LMA;
1886  			if (!vcpu->arch.guest_state_protected)
1887  				svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1888  		}
1889  
1890  		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1891  			vcpu->arch.efer &= ~EFER_LMA;
1892  			if (!vcpu->arch.guest_state_protected)
1893  				svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1894  		}
1895  	}
1896  #endif
1897  	vcpu->arch.cr0 = cr0;
1898  
1899  	if (!npt_enabled) {
1900  		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1901  		if (old_paging != is_paging(vcpu))
1902  			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1903  	}
1904  
1905  	/*
1906  	 * re-enable caching here because the QEMU bios
1907  	 * does not do it - this results in some delay at
1908  	 * reboot
1909  	 */
1910  	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1911  		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1912  
1913  	svm->vmcb->save.cr0 = hcr0;
1914  	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1915  
1916  	/*
1917  	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1918  	 * tracking is done using the CR write traps.
1919  	 */
1920  	if (sev_es_guest(vcpu->kvm))
1921  		return;
1922  
1923  	if (hcr0 == cr0) {
1924  		/* Selective CR0 write remains on.  */
1925  		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1926  		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1927  	} else {
1928  		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1929  		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1930  	}
1931  }
1932  
1933  static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1934  {
1935  	return true;
1936  }
1937  
1938  void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1939  {
1940  	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1941  	unsigned long old_cr4 = vcpu->arch.cr4;
1942  
1943  	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1944  		svm_flush_tlb_current(vcpu);
1945  
1946  	vcpu->arch.cr4 = cr4;
1947  	if (!npt_enabled) {
1948  		cr4 |= X86_CR4_PAE;
1949  
1950  		if (!is_paging(vcpu))
1951  			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1952  	}
1953  	cr4 |= host_cr4_mce;
1954  	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1955  	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1956  
1957  	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1958  		kvm_update_cpuid_runtime(vcpu);
1959  }
1960  
1961  static void svm_set_segment(struct kvm_vcpu *vcpu,
1962  			    struct kvm_segment *var, int seg)
1963  {
1964  	struct vcpu_svm *svm = to_svm(vcpu);
1965  	struct vmcb_seg *s = svm_seg(vcpu, seg);
1966  
1967  	s->base = var->base;
1968  	s->limit = var->limit;
1969  	s->selector = var->selector;
1970  	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1971  	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1972  	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1973  	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1974  	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1975  	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1976  	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1977  	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1978  
1979  	/*
1980  	 * This is always accurate, except if SYSRET returned to a segment
1981  	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1982  	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1983  	 * would entail passing the CPL to userspace and back.
1984  	 */
1985  	if (seg == VCPU_SREG_SS)
1986  		/* This is symmetric with svm_get_segment() */
1987  		svm->vmcb->save.cpl = (var->dpl & 3);
1988  
1989  	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1990  }
1991  
1992  static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1993  {
1994  	struct vcpu_svm *svm = to_svm(vcpu);
1995  
1996  	clr_exception_intercept(svm, BP_VECTOR);
1997  
1998  	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1999  		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2000  			set_exception_intercept(svm, BP_VECTOR);
2001  	}
2002  }
2003  
2004  static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2005  {
2006  	if (sd->next_asid > sd->max_asid) {
2007  		++sd->asid_generation;
2008  		sd->next_asid = sd->min_asid;
2009  		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2010  		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
2011  	}
2012  
2013  	svm->current_vmcb->asid_generation = sd->asid_generation;
2014  	svm->asid = sd->next_asid++;
2015  }
2016  
2017  static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
2018  {
2019  	struct vmcb *vmcb = svm->vmcb;
2020  
2021  	if (svm->vcpu.arch.guest_state_protected)
2022  		return;
2023  
2024  	if (unlikely(value != vmcb->save.dr6)) {
2025  		vmcb->save.dr6 = value;
2026  		vmcb_mark_dirty(vmcb, VMCB_DR);
2027  	}
2028  }
2029  
2030  static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2031  {
2032  	struct vcpu_svm *svm = to_svm(vcpu);
2033  
2034  	if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm)))
2035  		return;
2036  
2037  	get_debugreg(vcpu->arch.db[0], 0);
2038  	get_debugreg(vcpu->arch.db[1], 1);
2039  	get_debugreg(vcpu->arch.db[2], 2);
2040  	get_debugreg(vcpu->arch.db[3], 3);
2041  	/*
2042  	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
2043  	 * because db_interception might need it.  We can do it before vmentry.
2044  	 */
2045  	vcpu->arch.dr6 = svm->vmcb->save.dr6;
2046  	vcpu->arch.dr7 = svm->vmcb->save.dr7;
2047  	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2048  	set_dr_intercepts(svm);
2049  }
2050  
2051  static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2052  {
2053  	struct vcpu_svm *svm = to_svm(vcpu);
2054  
2055  	if (vcpu->arch.guest_state_protected)
2056  		return;
2057  
2058  	svm->vmcb->save.dr7 = value;
2059  	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
2060  }
2061  
2062  static int pf_interception(struct kvm_vcpu *vcpu)
2063  {
2064  	struct vcpu_svm *svm = to_svm(vcpu);
2065  
2066  	u64 fault_address = svm->vmcb->control.exit_info_2;
2067  	u64 error_code = svm->vmcb->control.exit_info_1;
2068  
2069  	return kvm_handle_page_fault(vcpu, error_code, fault_address,
2070  			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2071  			svm->vmcb->control.insn_bytes : NULL,
2072  			svm->vmcb->control.insn_len);
2073  }
2074  
2075  static int npf_interception(struct kvm_vcpu *vcpu)
2076  {
2077  	struct vcpu_svm *svm = to_svm(vcpu);
2078  
2079  	u64 fault_address = svm->vmcb->control.exit_info_2;
2080  	u64 error_code = svm->vmcb->control.exit_info_1;
2081  
2082  	trace_kvm_page_fault(vcpu, fault_address, error_code);
2083  	return kvm_mmu_page_fault(vcpu, fault_address, error_code,
2084  			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2085  			svm->vmcb->control.insn_bytes : NULL,
2086  			svm->vmcb->control.insn_len);
2087  }
2088  
2089  static int db_interception(struct kvm_vcpu *vcpu)
2090  {
2091  	struct kvm_run *kvm_run = vcpu->run;
2092  	struct vcpu_svm *svm = to_svm(vcpu);
2093  
2094  	if (!(vcpu->guest_debug &
2095  	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2096  		!svm->nmi_singlestep) {
2097  		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
2098  		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
2099  		return 1;
2100  	}
2101  
2102  	if (svm->nmi_singlestep) {
2103  		disable_nmi_singlestep(svm);
2104  		/* Make sure we check for pending NMIs upon entry */
2105  		kvm_make_request(KVM_REQ_EVENT, vcpu);
2106  	}
2107  
2108  	if (vcpu->guest_debug &
2109  	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2110  		kvm_run->exit_reason = KVM_EXIT_DEBUG;
2111  		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
2112  		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
2113  		kvm_run->debug.arch.pc =
2114  			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2115  		kvm_run->debug.arch.exception = DB_VECTOR;
2116  		return 0;
2117  	}
2118  
2119  	return 1;
2120  }
2121  
2122  static int bp_interception(struct kvm_vcpu *vcpu)
2123  {
2124  	struct vcpu_svm *svm = to_svm(vcpu);
2125  	struct kvm_run *kvm_run = vcpu->run;
2126  
2127  	kvm_run->exit_reason = KVM_EXIT_DEBUG;
2128  	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2129  	kvm_run->debug.arch.exception = BP_VECTOR;
2130  	return 0;
2131  }
2132  
2133  static int ud_interception(struct kvm_vcpu *vcpu)
2134  {
2135  	return handle_ud(vcpu);
2136  }
2137  
2138  static int ac_interception(struct kvm_vcpu *vcpu)
2139  {
2140  	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
2141  	return 1;
2142  }
2143  
2144  static bool is_erratum_383(void)
2145  {
2146  	int err, i;
2147  	u64 value;
2148  
2149  	if (!erratum_383_found)
2150  		return false;
2151  
2152  	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2153  	if (err)
2154  		return false;
2155  
2156  	/* Bit 62 may or may not be set for this mce */
2157  	value &= ~(1ULL << 62);
2158  
2159  	if (value != 0xb600000000010015ULL)
2160  		return false;
2161  
2162  	/* Clear MCi_STATUS registers */
2163  	for (i = 0; i < 6; ++i)
2164  		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2165  
2166  	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2167  	if (!err) {
2168  		u32 low, high;
2169  
2170  		value &= ~(1ULL << 2);
2171  		low    = lower_32_bits(value);
2172  		high   = upper_32_bits(value);
2173  
2174  		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2175  	}
2176  
2177  	/* Flush tlb to evict multi-match entries */
2178  	__flush_tlb_all();
2179  
2180  	return true;
2181  }
2182  
2183  static void svm_handle_mce(struct kvm_vcpu *vcpu)
2184  {
2185  	if (is_erratum_383()) {
2186  		/*
2187  		 * Erratum 383 triggered. Guest state is corrupt so kill the
2188  		 * guest.
2189  		 */
2190  		pr_err("Guest triggered AMD Erratum 383\n");
2191  
2192  		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2193  
2194  		return;
2195  	}
2196  
2197  	/*
2198  	 * On an #MC intercept the MCE handler is not called automatically in
2199  	 * the host. So do it by hand here.
2200  	 */
2201  	kvm_machine_check();
2202  }
2203  
2204  static int mc_interception(struct kvm_vcpu *vcpu)
2205  {
2206  	return 1;
2207  }
2208  
2209  static int shutdown_interception(struct kvm_vcpu *vcpu)
2210  {
2211  	struct kvm_run *kvm_run = vcpu->run;
2212  	struct vcpu_svm *svm = to_svm(vcpu);
2213  
2214  	/*
2215  	 * The VM save area has already been encrypted so it
2216  	 * cannot be reinitialized - just terminate.
2217  	 */
2218  	if (sev_es_guest(vcpu->kvm))
2219  		return -EINVAL;
2220  
2221  	/*
2222  	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2223  	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2224  	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2225  	 * userspace.  At a platform view, INIT is acceptable behavior as
2226  	 * there exist bare metal platforms that automatically INIT the CPU
2227  	 * in response to shutdown.
2228  	 */
2229  	clear_page(svm->vmcb);
2230  	kvm_vcpu_reset(vcpu, true);
2231  
2232  	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2233  	return 0;
2234  }
2235  
2236  static int io_interception(struct kvm_vcpu *vcpu)
2237  {
2238  	struct vcpu_svm *svm = to_svm(vcpu);
2239  	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2240  	int size, in, string;
2241  	unsigned port;
2242  
2243  	++vcpu->stat.io_exits;
2244  	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2245  	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2246  	port = io_info >> 16;
2247  	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2248  
2249  	if (string) {
2250  		if (sev_es_guest(vcpu->kvm))
2251  			return sev_es_string_io(svm, size, port, in);
2252  		else
2253  			return kvm_emulate_instruction(vcpu, 0);
2254  	}
2255  
2256  	svm->next_rip = svm->vmcb->control.exit_info_2;
2257  
2258  	return kvm_fast_pio(vcpu, size, port, in);
2259  }
2260  
2261  static int nmi_interception(struct kvm_vcpu *vcpu)
2262  {
2263  	return 1;
2264  }
2265  
2266  static int smi_interception(struct kvm_vcpu *vcpu)
2267  {
2268  	return 1;
2269  }
2270  
2271  static int intr_interception(struct kvm_vcpu *vcpu)
2272  {
2273  	++vcpu->stat.irq_exits;
2274  	return 1;
2275  }
2276  
2277  static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2278  {
2279  	struct vcpu_svm *svm = to_svm(vcpu);
2280  	struct vmcb *vmcb12;
2281  	struct kvm_host_map map;
2282  	int ret;
2283  
2284  	if (nested_svm_check_permissions(vcpu))
2285  		return 1;
2286  
2287  	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2288  	if (ret) {
2289  		if (ret == -EINVAL)
2290  			kvm_inject_gp(vcpu, 0);
2291  		return 1;
2292  	}
2293  
2294  	vmcb12 = map.hva;
2295  
2296  	ret = kvm_skip_emulated_instruction(vcpu);
2297  
2298  	if (vmload) {
2299  		svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2300  		svm->sysenter_eip_hi = 0;
2301  		svm->sysenter_esp_hi = 0;
2302  	} else {
2303  		svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2304  	}
2305  
2306  	kvm_vcpu_unmap(vcpu, &map, true);
2307  
2308  	return ret;
2309  }
2310  
2311  static int vmload_interception(struct kvm_vcpu *vcpu)
2312  {
2313  	return vmload_vmsave_interception(vcpu, true);
2314  }
2315  
2316  static int vmsave_interception(struct kvm_vcpu *vcpu)
2317  {
2318  	return vmload_vmsave_interception(vcpu, false);
2319  }
2320  
2321  static int vmrun_interception(struct kvm_vcpu *vcpu)
2322  {
2323  	if (nested_svm_check_permissions(vcpu))
2324  		return 1;
2325  
2326  	return nested_svm_vmrun(vcpu);
2327  }
2328  
2329  enum {
2330  	NONE_SVM_INSTR,
2331  	SVM_INSTR_VMRUN,
2332  	SVM_INSTR_VMLOAD,
2333  	SVM_INSTR_VMSAVE,
2334  };
2335  
2336  /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2337  static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2338  {
2339  	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2340  
2341  	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2342  		return NONE_SVM_INSTR;
2343  
2344  	switch (ctxt->modrm) {
2345  	case 0xd8: /* VMRUN */
2346  		return SVM_INSTR_VMRUN;
2347  	case 0xda: /* VMLOAD */
2348  		return SVM_INSTR_VMLOAD;
2349  	case 0xdb: /* VMSAVE */
2350  		return SVM_INSTR_VMSAVE;
2351  	default:
2352  		break;
2353  	}
2354  
2355  	return NONE_SVM_INSTR;
2356  }
2357  
2358  static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2359  {
2360  	const int guest_mode_exit_codes[] = {
2361  		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2362  		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2363  		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2364  	};
2365  	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2366  		[SVM_INSTR_VMRUN] = vmrun_interception,
2367  		[SVM_INSTR_VMLOAD] = vmload_interception,
2368  		[SVM_INSTR_VMSAVE] = vmsave_interception,
2369  	};
2370  	struct vcpu_svm *svm = to_svm(vcpu);
2371  	int ret;
2372  
2373  	if (is_guest_mode(vcpu)) {
2374  		/* Returns '1' or -errno on failure, '0' on success. */
2375  		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2376  		if (ret)
2377  			return ret;
2378  		return 1;
2379  	}
2380  	return svm_instr_handlers[opcode](vcpu);
2381  }
2382  
2383  /*
2384   * #GP handling code. Note that #GP can be triggered under the following two
2385   * cases:
2386   *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2387   *      some AMD CPUs when EAX of these instructions are in the reserved memory
2388   *      regions (e.g. SMM memory on host).
2389   *   2) VMware backdoor
2390   */
2391  static int gp_interception(struct kvm_vcpu *vcpu)
2392  {
2393  	struct vcpu_svm *svm = to_svm(vcpu);
2394  	u32 error_code = svm->vmcb->control.exit_info_1;
2395  	int opcode;
2396  
2397  	/* Both #GP cases have zero error_code */
2398  	if (error_code)
2399  		goto reinject;
2400  
2401  	/* Decode the instruction for usage later */
2402  	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2403  		goto reinject;
2404  
2405  	opcode = svm_instr_opcode(vcpu);
2406  
2407  	if (opcode == NONE_SVM_INSTR) {
2408  		if (!enable_vmware_backdoor)
2409  			goto reinject;
2410  
2411  		/*
2412  		 * VMware backdoor emulation on #GP interception only handles
2413  		 * IN{S}, OUT{S}, and RDPMC.
2414  		 */
2415  		if (!is_guest_mode(vcpu))
2416  			return kvm_emulate_instruction(vcpu,
2417  				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2418  	} else {
2419  		/* All SVM instructions expect page aligned RAX */
2420  		if (svm->vmcb->save.rax & ~PAGE_MASK)
2421  			goto reinject;
2422  
2423  		return emulate_svm_instr(vcpu, opcode);
2424  	}
2425  
2426  reinject:
2427  	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2428  	return 1;
2429  }
2430  
2431  void svm_set_gif(struct vcpu_svm *svm, bool value)
2432  {
2433  	if (value) {
2434  		/*
2435  		 * If VGIF is enabled, the STGI intercept is only added to
2436  		 * detect the opening of the SMI/NMI window; remove it now.
2437  		 * Likewise, clear the VINTR intercept, we will set it
2438  		 * again while processing KVM_REQ_EVENT if needed.
2439  		 */
2440  		if (vgif)
2441  			svm_clr_intercept(svm, INTERCEPT_STGI);
2442  		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2443  			svm_clear_vintr(svm);
2444  
2445  		enable_gif(svm);
2446  		if (svm->vcpu.arch.smi_pending ||
2447  		    svm->vcpu.arch.nmi_pending ||
2448  		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
2449  		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
2450  			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2451  	} else {
2452  		disable_gif(svm);
2453  
2454  		/*
2455  		 * After a CLGI no interrupts should come.  But if vGIF is
2456  		 * in use, we still rely on the VINTR intercept (rather than
2457  		 * STGI) to detect an open interrupt window.
2458  		*/
2459  		if (!vgif)
2460  			svm_clear_vintr(svm);
2461  	}
2462  }
2463  
2464  static int stgi_interception(struct kvm_vcpu *vcpu)
2465  {
2466  	int ret;
2467  
2468  	if (nested_svm_check_permissions(vcpu))
2469  		return 1;
2470  
2471  	ret = kvm_skip_emulated_instruction(vcpu);
2472  	svm_set_gif(to_svm(vcpu), true);
2473  	return ret;
2474  }
2475  
2476  static int clgi_interception(struct kvm_vcpu *vcpu)
2477  {
2478  	int ret;
2479  
2480  	if (nested_svm_check_permissions(vcpu))
2481  		return 1;
2482  
2483  	ret = kvm_skip_emulated_instruction(vcpu);
2484  	svm_set_gif(to_svm(vcpu), false);
2485  	return ret;
2486  }
2487  
2488  static int invlpga_interception(struct kvm_vcpu *vcpu)
2489  {
2490  	gva_t gva = kvm_rax_read(vcpu);
2491  	u32 asid = kvm_rcx_read(vcpu);
2492  
2493  	/* FIXME: Handle an address size prefix. */
2494  	if (!is_long_mode(vcpu))
2495  		gva = (u32)gva;
2496  
2497  	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2498  
2499  	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2500  	kvm_mmu_invlpg(vcpu, gva);
2501  
2502  	return kvm_skip_emulated_instruction(vcpu);
2503  }
2504  
2505  static int skinit_interception(struct kvm_vcpu *vcpu)
2506  {
2507  	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2508  
2509  	kvm_queue_exception(vcpu, UD_VECTOR);
2510  	return 1;
2511  }
2512  
2513  static int task_switch_interception(struct kvm_vcpu *vcpu)
2514  {
2515  	struct vcpu_svm *svm = to_svm(vcpu);
2516  	u16 tss_selector;
2517  	int reason;
2518  	int int_type = svm->vmcb->control.exit_int_info &
2519  		SVM_EXITINTINFO_TYPE_MASK;
2520  	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2521  	uint32_t type =
2522  		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2523  	uint32_t idt_v =
2524  		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2525  	bool has_error_code = false;
2526  	u32 error_code = 0;
2527  
2528  	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2529  
2530  	if (svm->vmcb->control.exit_info_2 &
2531  	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2532  		reason = TASK_SWITCH_IRET;
2533  	else if (svm->vmcb->control.exit_info_2 &
2534  		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2535  		reason = TASK_SWITCH_JMP;
2536  	else if (idt_v)
2537  		reason = TASK_SWITCH_GATE;
2538  	else
2539  		reason = TASK_SWITCH_CALL;
2540  
2541  	if (reason == TASK_SWITCH_GATE) {
2542  		switch (type) {
2543  		case SVM_EXITINTINFO_TYPE_NMI:
2544  			vcpu->arch.nmi_injected = false;
2545  			break;
2546  		case SVM_EXITINTINFO_TYPE_EXEPT:
2547  			if (svm->vmcb->control.exit_info_2 &
2548  			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2549  				has_error_code = true;
2550  				error_code =
2551  					(u32)svm->vmcb->control.exit_info_2;
2552  			}
2553  			kvm_clear_exception_queue(vcpu);
2554  			break;
2555  		case SVM_EXITINTINFO_TYPE_INTR:
2556  		case SVM_EXITINTINFO_TYPE_SOFT:
2557  			kvm_clear_interrupt_queue(vcpu);
2558  			break;
2559  		default:
2560  			break;
2561  		}
2562  	}
2563  
2564  	if (reason != TASK_SWITCH_GATE ||
2565  	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2566  	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2567  	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2568  		if (!svm_skip_emulated_instruction(vcpu))
2569  			return 0;
2570  	}
2571  
2572  	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2573  		int_vec = -1;
2574  
2575  	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2576  			       has_error_code, error_code);
2577  }
2578  
2579  static void svm_clr_iret_intercept(struct vcpu_svm *svm)
2580  {
2581  	if (!sev_es_guest(svm->vcpu.kvm))
2582  		svm_clr_intercept(svm, INTERCEPT_IRET);
2583  }
2584  
2585  static void svm_set_iret_intercept(struct vcpu_svm *svm)
2586  {
2587  	if (!sev_es_guest(svm->vcpu.kvm))
2588  		svm_set_intercept(svm, INTERCEPT_IRET);
2589  }
2590  
2591  static int iret_interception(struct kvm_vcpu *vcpu)
2592  {
2593  	struct vcpu_svm *svm = to_svm(vcpu);
2594  
2595  	WARN_ON_ONCE(sev_es_guest(vcpu->kvm));
2596  
2597  	++vcpu->stat.nmi_window_exits;
2598  	svm->awaiting_iret_completion = true;
2599  
2600  	svm_clr_iret_intercept(svm);
2601  	svm->nmi_iret_rip = kvm_rip_read(vcpu);
2602  
2603  	kvm_make_request(KVM_REQ_EVENT, vcpu);
2604  	return 1;
2605  }
2606  
2607  static int invlpg_interception(struct kvm_vcpu *vcpu)
2608  {
2609  	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2610  		return kvm_emulate_instruction(vcpu, 0);
2611  
2612  	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2613  	return kvm_skip_emulated_instruction(vcpu);
2614  }
2615  
2616  static int emulate_on_interception(struct kvm_vcpu *vcpu)
2617  {
2618  	return kvm_emulate_instruction(vcpu, 0);
2619  }
2620  
2621  static int rsm_interception(struct kvm_vcpu *vcpu)
2622  {
2623  	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2624  }
2625  
2626  static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2627  					    unsigned long val)
2628  {
2629  	struct vcpu_svm *svm = to_svm(vcpu);
2630  	unsigned long cr0 = vcpu->arch.cr0;
2631  	bool ret = false;
2632  
2633  	if (!is_guest_mode(vcpu) ||
2634  	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2635  		return false;
2636  
2637  	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2638  	val &= ~SVM_CR0_SELECTIVE_MASK;
2639  
2640  	if (cr0 ^ val) {
2641  		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2642  		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2643  	}
2644  
2645  	return ret;
2646  }
2647  
2648  #define CR_VALID (1ULL << 63)
2649  
2650  static int cr_interception(struct kvm_vcpu *vcpu)
2651  {
2652  	struct vcpu_svm *svm = to_svm(vcpu);
2653  	int reg, cr;
2654  	unsigned long val;
2655  	int err;
2656  
2657  	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2658  		return emulate_on_interception(vcpu);
2659  
2660  	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2661  		return emulate_on_interception(vcpu);
2662  
2663  	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2664  	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2665  		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2666  	else
2667  		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2668  
2669  	err = 0;
2670  	if (cr >= 16) { /* mov to cr */
2671  		cr -= 16;
2672  		val = kvm_register_read(vcpu, reg);
2673  		trace_kvm_cr_write(cr, val);
2674  		switch (cr) {
2675  		case 0:
2676  			if (!check_selective_cr0_intercepted(vcpu, val))
2677  				err = kvm_set_cr0(vcpu, val);
2678  			else
2679  				return 1;
2680  
2681  			break;
2682  		case 3:
2683  			err = kvm_set_cr3(vcpu, val);
2684  			break;
2685  		case 4:
2686  			err = kvm_set_cr4(vcpu, val);
2687  			break;
2688  		case 8:
2689  			err = kvm_set_cr8(vcpu, val);
2690  			break;
2691  		default:
2692  			WARN(1, "unhandled write to CR%d", cr);
2693  			kvm_queue_exception(vcpu, UD_VECTOR);
2694  			return 1;
2695  		}
2696  	} else { /* mov from cr */
2697  		switch (cr) {
2698  		case 0:
2699  			val = kvm_read_cr0(vcpu);
2700  			break;
2701  		case 2:
2702  			val = vcpu->arch.cr2;
2703  			break;
2704  		case 3:
2705  			val = kvm_read_cr3(vcpu);
2706  			break;
2707  		case 4:
2708  			val = kvm_read_cr4(vcpu);
2709  			break;
2710  		case 8:
2711  			val = kvm_get_cr8(vcpu);
2712  			break;
2713  		default:
2714  			WARN(1, "unhandled read from CR%d", cr);
2715  			kvm_queue_exception(vcpu, UD_VECTOR);
2716  			return 1;
2717  		}
2718  		kvm_register_write(vcpu, reg, val);
2719  		trace_kvm_cr_read(cr, val);
2720  	}
2721  	return kvm_complete_insn_gp(vcpu, err);
2722  }
2723  
2724  static int cr_trap(struct kvm_vcpu *vcpu)
2725  {
2726  	struct vcpu_svm *svm = to_svm(vcpu);
2727  	unsigned long old_value, new_value;
2728  	unsigned int cr;
2729  	int ret = 0;
2730  
2731  	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2732  
2733  	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2734  	switch (cr) {
2735  	case 0:
2736  		old_value = kvm_read_cr0(vcpu);
2737  		svm_set_cr0(vcpu, new_value);
2738  
2739  		kvm_post_set_cr0(vcpu, old_value, new_value);
2740  		break;
2741  	case 4:
2742  		old_value = kvm_read_cr4(vcpu);
2743  		svm_set_cr4(vcpu, new_value);
2744  
2745  		kvm_post_set_cr4(vcpu, old_value, new_value);
2746  		break;
2747  	case 8:
2748  		ret = kvm_set_cr8(vcpu, new_value);
2749  		break;
2750  	default:
2751  		WARN(1, "unhandled CR%d write trap", cr);
2752  		kvm_queue_exception(vcpu, UD_VECTOR);
2753  		return 1;
2754  	}
2755  
2756  	return kvm_complete_insn_gp(vcpu, ret);
2757  }
2758  
2759  static int dr_interception(struct kvm_vcpu *vcpu)
2760  {
2761  	struct vcpu_svm *svm = to_svm(vcpu);
2762  	int reg, dr;
2763  	unsigned long val;
2764  	int err = 0;
2765  
2766  	/*
2767  	 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT
2768  	 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early.
2769  	 */
2770  	if (sev_es_guest(vcpu->kvm))
2771  		return 1;
2772  
2773  	if (vcpu->guest_debug == 0) {
2774  		/*
2775  		 * No more DR vmexits; force a reload of the debug registers
2776  		 * and reenter on this instruction.  The next vmexit will
2777  		 * retrieve the full state of the debug registers.
2778  		 */
2779  		clr_dr_intercepts(svm);
2780  		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2781  		return 1;
2782  	}
2783  
2784  	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2785  		return emulate_on_interception(vcpu);
2786  
2787  	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2788  	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2789  	if (dr >= 16) { /* mov to DRn  */
2790  		dr -= 16;
2791  		val = kvm_register_read(vcpu, reg);
2792  		err = kvm_set_dr(vcpu, dr, val);
2793  	} else {
2794  		kvm_get_dr(vcpu, dr, &val);
2795  		kvm_register_write(vcpu, reg, val);
2796  	}
2797  
2798  	return kvm_complete_insn_gp(vcpu, err);
2799  }
2800  
2801  static int cr8_write_interception(struct kvm_vcpu *vcpu)
2802  {
2803  	int r;
2804  
2805  	u8 cr8_prev = kvm_get_cr8(vcpu);
2806  	/* instruction emulation calls kvm_set_cr8() */
2807  	r = cr_interception(vcpu);
2808  	if (lapic_in_kernel(vcpu))
2809  		return r;
2810  	if (cr8_prev <= kvm_get_cr8(vcpu))
2811  		return r;
2812  	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2813  	return 0;
2814  }
2815  
2816  static int efer_trap(struct kvm_vcpu *vcpu)
2817  {
2818  	struct msr_data msr_info;
2819  	int ret;
2820  
2821  	/*
2822  	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2823  	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2824  	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2825  	 * the guest doesn't have X86_FEATURE_SVM.
2826  	 */
2827  	msr_info.host_initiated = false;
2828  	msr_info.index = MSR_EFER;
2829  	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2830  	ret = kvm_set_msr_common(vcpu, &msr_info);
2831  
2832  	return kvm_complete_insn_gp(vcpu, ret);
2833  }
2834  
2835  static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2836  {
2837  	msr->data = 0;
2838  
2839  	switch (msr->index) {
2840  	case MSR_AMD64_DE_CFG:
2841  		if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
2842  			msr->data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
2843  		break;
2844  	default:
2845  		return KVM_MSR_RET_INVALID;
2846  	}
2847  
2848  	return 0;
2849  }
2850  
2851  static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2852  {
2853  	struct vcpu_svm *svm = to_svm(vcpu);
2854  
2855  	switch (msr_info->index) {
2856  	case MSR_AMD64_TSC_RATIO:
2857  		if (!msr_info->host_initiated &&
2858  		    !guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR))
2859  			return 1;
2860  		msr_info->data = svm->tsc_ratio_msr;
2861  		break;
2862  	case MSR_STAR:
2863  		msr_info->data = svm->vmcb01.ptr->save.star;
2864  		break;
2865  #ifdef CONFIG_X86_64
2866  	case MSR_LSTAR:
2867  		msr_info->data = svm->vmcb01.ptr->save.lstar;
2868  		break;
2869  	case MSR_CSTAR:
2870  		msr_info->data = svm->vmcb01.ptr->save.cstar;
2871  		break;
2872  	case MSR_GS_BASE:
2873  		msr_info->data = svm->vmcb01.ptr->save.gs.base;
2874  		break;
2875  	case MSR_FS_BASE:
2876  		msr_info->data = svm->vmcb01.ptr->save.fs.base;
2877  		break;
2878  	case MSR_KERNEL_GS_BASE:
2879  		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2880  		break;
2881  	case MSR_SYSCALL_MASK:
2882  		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2883  		break;
2884  #endif
2885  	case MSR_IA32_SYSENTER_CS:
2886  		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2887  		break;
2888  	case MSR_IA32_SYSENTER_EIP:
2889  		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2890  		if (guest_cpuid_is_intel(vcpu))
2891  			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2892  		break;
2893  	case MSR_IA32_SYSENTER_ESP:
2894  		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2895  		if (guest_cpuid_is_intel(vcpu))
2896  			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2897  		break;
2898  	case MSR_TSC_AUX:
2899  		msr_info->data = svm->tsc_aux;
2900  		break;
2901  	case MSR_IA32_DEBUGCTLMSR:
2902  		msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl;
2903  		break;
2904  	case MSR_IA32_LASTBRANCHFROMIP:
2905  		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from;
2906  		break;
2907  	case MSR_IA32_LASTBRANCHTOIP:
2908  		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to;
2909  		break;
2910  	case MSR_IA32_LASTINTFROMIP:
2911  		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from;
2912  		break;
2913  	case MSR_IA32_LASTINTTOIP:
2914  		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to;
2915  		break;
2916  	case MSR_VM_HSAVE_PA:
2917  		msr_info->data = svm->nested.hsave_msr;
2918  		break;
2919  	case MSR_VM_CR:
2920  		msr_info->data = svm->nested.vm_cr_msr;
2921  		break;
2922  	case MSR_IA32_SPEC_CTRL:
2923  		if (!msr_info->host_initiated &&
2924  		    !guest_has_spec_ctrl_msr(vcpu))
2925  			return 1;
2926  
2927  		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2928  			msr_info->data = svm->vmcb->save.spec_ctrl;
2929  		else
2930  			msr_info->data = svm->spec_ctrl;
2931  		break;
2932  	case MSR_AMD64_VIRT_SPEC_CTRL:
2933  		if (!msr_info->host_initiated &&
2934  		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2935  			return 1;
2936  
2937  		msr_info->data = svm->virt_spec_ctrl;
2938  		break;
2939  	case MSR_F15H_IC_CFG: {
2940  
2941  		int family, model;
2942  
2943  		family = guest_cpuid_family(vcpu);
2944  		model  = guest_cpuid_model(vcpu);
2945  
2946  		if (family < 0 || model < 0)
2947  			return kvm_get_msr_common(vcpu, msr_info);
2948  
2949  		msr_info->data = 0;
2950  
2951  		if (family == 0x15 &&
2952  		    (model >= 0x2 && model < 0x20))
2953  			msr_info->data = 0x1E;
2954  		}
2955  		break;
2956  	case MSR_AMD64_DE_CFG:
2957  		msr_info->data = svm->msr_decfg;
2958  		break;
2959  	default:
2960  		return kvm_get_msr_common(vcpu, msr_info);
2961  	}
2962  	return 0;
2963  }
2964  
2965  static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2966  {
2967  	struct vcpu_svm *svm = to_svm(vcpu);
2968  	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2969  		return kvm_complete_insn_gp(vcpu, err);
2970  
2971  	ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1);
2972  	ghcb_set_sw_exit_info_2(svm->sev_es.ghcb,
2973  				X86_TRAP_GP |
2974  				SVM_EVTINJ_TYPE_EXEPT |
2975  				SVM_EVTINJ_VALID);
2976  	return 1;
2977  }
2978  
2979  static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2980  {
2981  	struct vcpu_svm *svm = to_svm(vcpu);
2982  	int svm_dis, chg_mask;
2983  
2984  	if (data & ~SVM_VM_CR_VALID_MASK)
2985  		return 1;
2986  
2987  	chg_mask = SVM_VM_CR_VALID_MASK;
2988  
2989  	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2990  		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2991  
2992  	svm->nested.vm_cr_msr &= ~chg_mask;
2993  	svm->nested.vm_cr_msr |= (data & chg_mask);
2994  
2995  	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2996  
2997  	/* check for svm_disable while efer.svme is set */
2998  	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2999  		return 1;
3000  
3001  	return 0;
3002  }
3003  
3004  static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3005  {
3006  	struct vcpu_svm *svm = to_svm(vcpu);
3007  	int ret = 0;
3008  
3009  	u32 ecx = msr->index;
3010  	u64 data = msr->data;
3011  	switch (ecx) {
3012  	case MSR_AMD64_TSC_RATIO:
3013  
3014  		if (!guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR)) {
3015  
3016  			if (!msr->host_initiated)
3017  				return 1;
3018  			/*
3019  			 * In case TSC scaling is not enabled, always
3020  			 * leave this MSR at the default value.
3021  			 *
3022  			 * Due to bug in qemu 6.2.0, it would try to set
3023  			 * this msr to 0 if tsc scaling is not enabled.
3024  			 * Ignore this value as well.
3025  			 */
3026  			if (data != 0 && data != svm->tsc_ratio_msr)
3027  				return 1;
3028  			break;
3029  		}
3030  
3031  		if (data & SVM_TSC_RATIO_RSVD)
3032  			return 1;
3033  
3034  		svm->tsc_ratio_msr = data;
3035  
3036  		if (guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR) &&
3037  		    is_guest_mode(vcpu))
3038  			nested_svm_update_tsc_ratio_msr(vcpu);
3039  
3040  		break;
3041  	case MSR_IA32_CR_PAT:
3042  		ret = kvm_set_msr_common(vcpu, msr);
3043  		if (ret)
3044  			break;
3045  
3046  		svm->vmcb01.ptr->save.g_pat = data;
3047  		if (is_guest_mode(vcpu))
3048  			nested_vmcb02_compute_g_pat(svm);
3049  		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3050  		break;
3051  	case MSR_IA32_SPEC_CTRL:
3052  		if (!msr->host_initiated &&
3053  		    !guest_has_spec_ctrl_msr(vcpu))
3054  			return 1;
3055  
3056  		if (kvm_spec_ctrl_test_value(data))
3057  			return 1;
3058  
3059  		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3060  			svm->vmcb->save.spec_ctrl = data;
3061  		else
3062  			svm->spec_ctrl = data;
3063  		if (!data)
3064  			break;
3065  
3066  		/*
3067  		 * For non-nested:
3068  		 * When it's written (to non-zero) for the first time, pass
3069  		 * it through.
3070  		 *
3071  		 * For nested:
3072  		 * The handling of the MSR bitmap for L2 guests is done in
3073  		 * nested_svm_vmrun_msrpm.
3074  		 * We update the L1 MSR bit as well since it will end up
3075  		 * touching the MSR anyway now.
3076  		 */
3077  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
3078  		break;
3079  	case MSR_AMD64_VIRT_SPEC_CTRL:
3080  		if (!msr->host_initiated &&
3081  		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
3082  			return 1;
3083  
3084  		if (data & ~SPEC_CTRL_SSBD)
3085  			return 1;
3086  
3087  		svm->virt_spec_ctrl = data;
3088  		break;
3089  	case MSR_STAR:
3090  		svm->vmcb01.ptr->save.star = data;
3091  		break;
3092  #ifdef CONFIG_X86_64
3093  	case MSR_LSTAR:
3094  		svm->vmcb01.ptr->save.lstar = data;
3095  		break;
3096  	case MSR_CSTAR:
3097  		svm->vmcb01.ptr->save.cstar = data;
3098  		break;
3099  	case MSR_GS_BASE:
3100  		svm->vmcb01.ptr->save.gs.base = data;
3101  		break;
3102  	case MSR_FS_BASE:
3103  		svm->vmcb01.ptr->save.fs.base = data;
3104  		break;
3105  	case MSR_KERNEL_GS_BASE:
3106  		svm->vmcb01.ptr->save.kernel_gs_base = data;
3107  		break;
3108  	case MSR_SYSCALL_MASK:
3109  		svm->vmcb01.ptr->save.sfmask = data;
3110  		break;
3111  #endif
3112  	case MSR_IA32_SYSENTER_CS:
3113  		svm->vmcb01.ptr->save.sysenter_cs = data;
3114  		break;
3115  	case MSR_IA32_SYSENTER_EIP:
3116  		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
3117  		/*
3118  		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
3119  		 * when we spoof an Intel vendor ID (for cross vendor migration).
3120  		 * In this case we use this intercept to track the high
3121  		 * 32 bit part of these msrs to support Intel's
3122  		 * implementation of SYSENTER/SYSEXIT.
3123  		 */
3124  		svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3125  		break;
3126  	case MSR_IA32_SYSENTER_ESP:
3127  		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
3128  		svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3129  		break;
3130  	case MSR_TSC_AUX:
3131  		/*
3132  		 * TSC_AUX is always virtualized for SEV-ES guests when the
3133  		 * feature is available. The user return MSR support is not
3134  		 * required in this case because TSC_AUX is restored on #VMEXIT
3135  		 * from the host save area (which has been initialized in
3136  		 * svm_hardware_enable()).
3137  		 */
3138  		if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
3139  			break;
3140  
3141  		/*
3142  		 * TSC_AUX is usually changed only during boot and never read
3143  		 * directly.  Intercept TSC_AUX instead of exposing it to the
3144  		 * guest via direct_access_msrs, and switch it via user return.
3145  		 */
3146  		preempt_disable();
3147  		ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
3148  		preempt_enable();
3149  		if (ret)
3150  			break;
3151  
3152  		svm->tsc_aux = data;
3153  		break;
3154  	case MSR_IA32_DEBUGCTLMSR:
3155  		if (!lbrv) {
3156  			kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3157  			break;
3158  		}
3159  		if (data & DEBUGCTL_RESERVED_BITS)
3160  			return 1;
3161  
3162  		svm_get_lbr_vmcb(svm)->save.dbgctl = data;
3163  		svm_update_lbrv(vcpu);
3164  		break;
3165  	case MSR_VM_HSAVE_PA:
3166  		/*
3167  		 * Old kernels did not validate the value written to
3168  		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3169  		 * value to allow live migrating buggy or malicious guests
3170  		 * originating from those kernels.
3171  		 */
3172  		if (!msr->host_initiated && !page_address_valid(vcpu, data))
3173  			return 1;
3174  
3175  		svm->nested.hsave_msr = data & PAGE_MASK;
3176  		break;
3177  	case MSR_VM_CR:
3178  		return svm_set_vm_cr(vcpu, data);
3179  	case MSR_VM_IGNNE:
3180  		kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3181  		break;
3182  	case MSR_AMD64_DE_CFG: {
3183  		struct kvm_msr_entry msr_entry;
3184  
3185  		msr_entry.index = msr->index;
3186  		if (svm_get_msr_feature(&msr_entry))
3187  			return 1;
3188  
3189  		/* Check the supported bits */
3190  		if (data & ~msr_entry.data)
3191  			return 1;
3192  
3193  		/* Don't allow the guest to change a bit, #GP */
3194  		if (!msr->host_initiated && (data ^ msr_entry.data))
3195  			return 1;
3196  
3197  		svm->msr_decfg = data;
3198  		break;
3199  	}
3200  	default:
3201  		return kvm_set_msr_common(vcpu, msr);
3202  	}
3203  	return ret;
3204  }
3205  
3206  static int msr_interception(struct kvm_vcpu *vcpu)
3207  {
3208  	if (to_svm(vcpu)->vmcb->control.exit_info_1)
3209  		return kvm_emulate_wrmsr(vcpu);
3210  	else
3211  		return kvm_emulate_rdmsr(vcpu);
3212  }
3213  
3214  static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3215  {
3216  	kvm_make_request(KVM_REQ_EVENT, vcpu);
3217  	svm_clear_vintr(to_svm(vcpu));
3218  
3219  	/*
3220  	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
3221  	 * In this case AVIC was temporarily disabled for
3222  	 * requesting the IRQ window and we have to re-enable it.
3223  	 *
3224  	 * If running nested, still remove the VM wide AVIC inhibit to
3225  	 * support case in which the interrupt window was requested when the
3226  	 * vCPU was not running nested.
3227  
3228  	 * All vCPUs which run still run nested, will remain to have their
3229  	 * AVIC still inhibited due to per-cpu AVIC inhibition.
3230  	 */
3231  	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3232  
3233  	++vcpu->stat.irq_window_exits;
3234  	return 1;
3235  }
3236  
3237  static int pause_interception(struct kvm_vcpu *vcpu)
3238  {
3239  	bool in_kernel;
3240  	/*
3241  	 * CPL is not made available for an SEV-ES guest, therefore
3242  	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3243  	 * set in_kernel to false as well.
3244  	 */
3245  	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3246  
3247  	grow_ple_window(vcpu);
3248  
3249  	kvm_vcpu_on_spin(vcpu, in_kernel);
3250  	return kvm_skip_emulated_instruction(vcpu);
3251  }
3252  
3253  static int invpcid_interception(struct kvm_vcpu *vcpu)
3254  {
3255  	struct vcpu_svm *svm = to_svm(vcpu);
3256  	unsigned long type;
3257  	gva_t gva;
3258  
3259  	if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3260  		kvm_queue_exception(vcpu, UD_VECTOR);
3261  		return 1;
3262  	}
3263  
3264  	/*
3265  	 * For an INVPCID intercept:
3266  	 * EXITINFO1 provides the linear address of the memory operand.
3267  	 * EXITINFO2 provides the contents of the register operand.
3268  	 */
3269  	type = svm->vmcb->control.exit_info_2;
3270  	gva = svm->vmcb->control.exit_info_1;
3271  
3272  	return kvm_handle_invpcid(vcpu, type, gva);
3273  }
3274  
3275  static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3276  	[SVM_EXIT_READ_CR0]			= cr_interception,
3277  	[SVM_EXIT_READ_CR3]			= cr_interception,
3278  	[SVM_EXIT_READ_CR4]			= cr_interception,
3279  	[SVM_EXIT_READ_CR8]			= cr_interception,
3280  	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3281  	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3282  	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3283  	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3284  	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3285  	[SVM_EXIT_READ_DR0]			= dr_interception,
3286  	[SVM_EXIT_READ_DR1]			= dr_interception,
3287  	[SVM_EXIT_READ_DR2]			= dr_interception,
3288  	[SVM_EXIT_READ_DR3]			= dr_interception,
3289  	[SVM_EXIT_READ_DR4]			= dr_interception,
3290  	[SVM_EXIT_READ_DR5]			= dr_interception,
3291  	[SVM_EXIT_READ_DR6]			= dr_interception,
3292  	[SVM_EXIT_READ_DR7]			= dr_interception,
3293  	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3294  	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3295  	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3296  	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3297  	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3298  	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3299  	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3300  	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3301  	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3302  	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3303  	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3304  	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3305  	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3306  	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3307  	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3308  	[SVM_EXIT_INTR]				= intr_interception,
3309  	[SVM_EXIT_NMI]				= nmi_interception,
3310  	[SVM_EXIT_SMI]				= smi_interception,
3311  	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3312  	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3313  	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3314  	[SVM_EXIT_IRET]                         = iret_interception,
3315  	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3316  	[SVM_EXIT_PAUSE]			= pause_interception,
3317  	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3318  	[SVM_EXIT_INVLPG]			= invlpg_interception,
3319  	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3320  	[SVM_EXIT_IOIO]				= io_interception,
3321  	[SVM_EXIT_MSR]				= msr_interception,
3322  	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3323  	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3324  	[SVM_EXIT_VMRUN]			= vmrun_interception,
3325  	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3326  	[SVM_EXIT_VMLOAD]			= vmload_interception,
3327  	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3328  	[SVM_EXIT_STGI]				= stgi_interception,
3329  	[SVM_EXIT_CLGI]				= clgi_interception,
3330  	[SVM_EXIT_SKINIT]			= skinit_interception,
3331  	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3332  	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3333  	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3334  	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3335  	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3336  	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3337  	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3338  	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3339  	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3340  	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3341  	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3342  	[SVM_EXIT_NPF]				= npf_interception,
3343  	[SVM_EXIT_RSM]                          = rsm_interception,
3344  	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3345  	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3346  	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3347  };
3348  
3349  static void dump_vmcb(struct kvm_vcpu *vcpu)
3350  {
3351  	struct vcpu_svm *svm = to_svm(vcpu);
3352  	struct vmcb_control_area *control = &svm->vmcb->control;
3353  	struct vmcb_save_area *save = &svm->vmcb->save;
3354  	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3355  
3356  	if (!dump_invalid_vmcb) {
3357  		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3358  		return;
3359  	}
3360  
3361  	pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3362  	       svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3363  	pr_err("VMCB Control Area:\n");
3364  	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3365  	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3366  	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3367  	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3368  	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3369  	pr_err("%-20s%08x %08x\n", "intercepts:",
3370                control->intercepts[INTERCEPT_WORD3],
3371  	       control->intercepts[INTERCEPT_WORD4]);
3372  	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3373  	pr_err("%-20s%d\n", "pause filter threshold:",
3374  	       control->pause_filter_thresh);
3375  	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3376  	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3377  	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3378  	pr_err("%-20s%d\n", "asid:", control->asid);
3379  	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3380  	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3381  	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3382  	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3383  	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3384  	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3385  	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3386  	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3387  	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3388  	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3389  	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3390  	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3391  	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3392  	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3393  	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3394  	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3395  	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3396  	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3397  	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3398  	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3399  	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3400  	pr_err("VMCB State Save Area:\n");
3401  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3402  	       "es:",
3403  	       save->es.selector, save->es.attrib,
3404  	       save->es.limit, save->es.base);
3405  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3406  	       "cs:",
3407  	       save->cs.selector, save->cs.attrib,
3408  	       save->cs.limit, save->cs.base);
3409  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3410  	       "ss:",
3411  	       save->ss.selector, save->ss.attrib,
3412  	       save->ss.limit, save->ss.base);
3413  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3414  	       "ds:",
3415  	       save->ds.selector, save->ds.attrib,
3416  	       save->ds.limit, save->ds.base);
3417  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3418  	       "fs:",
3419  	       save01->fs.selector, save01->fs.attrib,
3420  	       save01->fs.limit, save01->fs.base);
3421  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3422  	       "gs:",
3423  	       save01->gs.selector, save01->gs.attrib,
3424  	       save01->gs.limit, save01->gs.base);
3425  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3426  	       "gdtr:",
3427  	       save->gdtr.selector, save->gdtr.attrib,
3428  	       save->gdtr.limit, save->gdtr.base);
3429  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3430  	       "ldtr:",
3431  	       save01->ldtr.selector, save01->ldtr.attrib,
3432  	       save01->ldtr.limit, save01->ldtr.base);
3433  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3434  	       "idtr:",
3435  	       save->idtr.selector, save->idtr.attrib,
3436  	       save->idtr.limit, save->idtr.base);
3437  	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3438  	       "tr:",
3439  	       save01->tr.selector, save01->tr.attrib,
3440  	       save01->tr.limit, save01->tr.base);
3441  	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3442  	       save->vmpl, save->cpl, save->efer);
3443  	pr_err("%-15s %016llx %-13s %016llx\n",
3444  	       "cr0:", save->cr0, "cr2:", save->cr2);
3445  	pr_err("%-15s %016llx %-13s %016llx\n",
3446  	       "cr3:", save->cr3, "cr4:", save->cr4);
3447  	pr_err("%-15s %016llx %-13s %016llx\n",
3448  	       "dr6:", save->dr6, "dr7:", save->dr7);
3449  	pr_err("%-15s %016llx %-13s %016llx\n",
3450  	       "rip:", save->rip, "rflags:", save->rflags);
3451  	pr_err("%-15s %016llx %-13s %016llx\n",
3452  	       "rsp:", save->rsp, "rax:", save->rax);
3453  	pr_err("%-15s %016llx %-13s %016llx\n",
3454  	       "star:", save01->star, "lstar:", save01->lstar);
3455  	pr_err("%-15s %016llx %-13s %016llx\n",
3456  	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3457  	pr_err("%-15s %016llx %-13s %016llx\n",
3458  	       "kernel_gs_base:", save01->kernel_gs_base,
3459  	       "sysenter_cs:", save01->sysenter_cs);
3460  	pr_err("%-15s %016llx %-13s %016llx\n",
3461  	       "sysenter_esp:", save01->sysenter_esp,
3462  	       "sysenter_eip:", save01->sysenter_eip);
3463  	pr_err("%-15s %016llx %-13s %016llx\n",
3464  	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3465  	pr_err("%-15s %016llx %-13s %016llx\n",
3466  	       "br_from:", save->br_from, "br_to:", save->br_to);
3467  	pr_err("%-15s %016llx %-13s %016llx\n",
3468  	       "excp_from:", save->last_excp_from,
3469  	       "excp_to:", save->last_excp_to);
3470  }
3471  
3472  static bool svm_check_exit_valid(u64 exit_code)
3473  {
3474  	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3475  		svm_exit_handlers[exit_code]);
3476  }
3477  
3478  static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3479  {
3480  	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3481  	dump_vmcb(vcpu);
3482  	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3483  	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3484  	vcpu->run->internal.ndata = 2;
3485  	vcpu->run->internal.data[0] = exit_code;
3486  	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3487  	return 0;
3488  }
3489  
3490  int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3491  {
3492  	if (!svm_check_exit_valid(exit_code))
3493  		return svm_handle_invalid_exit(vcpu, exit_code);
3494  
3495  #ifdef CONFIG_RETPOLINE
3496  	if (exit_code == SVM_EXIT_MSR)
3497  		return msr_interception(vcpu);
3498  	else if (exit_code == SVM_EXIT_VINTR)
3499  		return interrupt_window_interception(vcpu);
3500  	else if (exit_code == SVM_EXIT_INTR)
3501  		return intr_interception(vcpu);
3502  	else if (exit_code == SVM_EXIT_HLT)
3503  		return kvm_emulate_halt(vcpu);
3504  	else if (exit_code == SVM_EXIT_NPF)
3505  		return npf_interception(vcpu);
3506  #endif
3507  	return svm_exit_handlers[exit_code](vcpu);
3508  }
3509  
3510  static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3511  			      u64 *info1, u64 *info2,
3512  			      u32 *intr_info, u32 *error_code)
3513  {
3514  	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3515  
3516  	*reason = control->exit_code;
3517  	*info1 = control->exit_info_1;
3518  	*info2 = control->exit_info_2;
3519  	*intr_info = control->exit_int_info;
3520  	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3521  	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3522  		*error_code = control->exit_int_info_err;
3523  	else
3524  		*error_code = 0;
3525  }
3526  
3527  static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3528  {
3529  	struct vcpu_svm *svm = to_svm(vcpu);
3530  	struct kvm_run *kvm_run = vcpu->run;
3531  	u32 exit_code = svm->vmcb->control.exit_code;
3532  
3533  	/* SEV-ES guests must use the CR write traps to track CR registers. */
3534  	if (!sev_es_guest(vcpu->kvm)) {
3535  		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3536  			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3537  		if (npt_enabled)
3538  			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3539  	}
3540  
3541  	if (is_guest_mode(vcpu)) {
3542  		int vmexit;
3543  
3544  		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3545  
3546  		vmexit = nested_svm_exit_special(svm);
3547  
3548  		if (vmexit == NESTED_EXIT_CONTINUE)
3549  			vmexit = nested_svm_exit_handled(svm);
3550  
3551  		if (vmexit == NESTED_EXIT_DONE)
3552  			return 1;
3553  	}
3554  
3555  	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3556  		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3557  		kvm_run->fail_entry.hardware_entry_failure_reason
3558  			= svm->vmcb->control.exit_code;
3559  		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3560  		dump_vmcb(vcpu);
3561  		return 0;
3562  	}
3563  
3564  	if (exit_fastpath != EXIT_FASTPATH_NONE)
3565  		return 1;
3566  
3567  	return svm_invoke_exit_handler(vcpu, exit_code);
3568  }
3569  
3570  static void pre_svm_run(struct kvm_vcpu *vcpu)
3571  {
3572  	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
3573  	struct vcpu_svm *svm = to_svm(vcpu);
3574  
3575  	/*
3576  	 * If the previous vmrun of the vmcb occurred on a different physical
3577  	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3578  	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3579  	 */
3580  	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3581  		svm->current_vmcb->asid_generation = 0;
3582  		vmcb_mark_all_dirty(svm->vmcb);
3583  		svm->current_vmcb->cpu = vcpu->cpu;
3584          }
3585  
3586  	if (sev_guest(vcpu->kvm))
3587  		return pre_sev_run(svm, vcpu->cpu);
3588  
3589  	/* FIXME: handle wraparound of asid_generation */
3590  	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3591  		new_asid(svm, sd);
3592  }
3593  
3594  static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3595  {
3596  	struct vcpu_svm *svm = to_svm(vcpu);
3597  
3598  	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3599  
3600  	if (svm->nmi_l1_to_l2)
3601  		return;
3602  
3603  	svm->nmi_masked = true;
3604  	svm_set_iret_intercept(svm);
3605  	++vcpu->stat.nmi_injections;
3606  }
3607  
3608  static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu)
3609  {
3610  	struct vcpu_svm *svm = to_svm(vcpu);
3611  
3612  	if (!is_vnmi_enabled(svm))
3613  		return false;
3614  
3615  	return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK);
3616  }
3617  
3618  static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
3619  {
3620  	struct vcpu_svm *svm = to_svm(vcpu);
3621  
3622  	if (!is_vnmi_enabled(svm))
3623  		return false;
3624  
3625  	if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK)
3626  		return false;
3627  
3628  	svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
3629  	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
3630  
3631  	/*
3632  	 * Because the pending NMI is serviced by hardware, KVM can't know when
3633  	 * the NMI is "injected", but for all intents and purposes, passing the
3634  	 * NMI off to hardware counts as injection.
3635  	 */
3636  	++vcpu->stat.nmi_injections;
3637  
3638  	return true;
3639  }
3640  
3641  static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
3642  {
3643  	struct vcpu_svm *svm = to_svm(vcpu);
3644  	u32 type;
3645  
3646  	if (vcpu->arch.interrupt.soft) {
3647  		if (svm_update_soft_interrupt_rip(vcpu))
3648  			return;
3649  
3650  		type = SVM_EVTINJ_TYPE_SOFT;
3651  	} else {
3652  		type = SVM_EVTINJ_TYPE_INTR;
3653  	}
3654  
3655  	trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
3656  			   vcpu->arch.interrupt.soft, reinjected);
3657  	++vcpu->stat.irq_injections;
3658  
3659  	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3660  				       SVM_EVTINJ_VALID | type;
3661  }
3662  
3663  void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3664  				     int trig_mode, int vector)
3665  {
3666  	/*
3667  	 * apic->apicv_active must be read after vcpu->mode.
3668  	 * Pairs with smp_store_release in vcpu_enter_guest.
3669  	 */
3670  	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3671  
3672  	/* Note, this is called iff the local APIC is in-kernel. */
3673  	if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
3674  		/* Process the interrupt via kvm_check_and_inject_events(). */
3675  		kvm_make_request(KVM_REQ_EVENT, vcpu);
3676  		kvm_vcpu_kick(vcpu);
3677  		return;
3678  	}
3679  
3680  	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3681  	if (in_guest_mode) {
3682  		/*
3683  		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3684  		 * the vCPU exits the guest before the doorbell chimes, hardware
3685  		 * will automatically process AVIC interrupts at the next VMRUN.
3686  		 */
3687  		avic_ring_doorbell(vcpu);
3688  	} else {
3689  		/*
3690  		 * Wake the vCPU if it was blocking.  KVM will then detect the
3691  		 * pending IRQ when checking if the vCPU has a wake event.
3692  		 */
3693  		kvm_vcpu_wake_up(vcpu);
3694  	}
3695  }
3696  
3697  static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3698  				  int trig_mode, int vector)
3699  {
3700  	kvm_lapic_set_irr(vector, apic);
3701  
3702  	/*
3703  	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3704  	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3705  	 * the read of guest_mode.  This guarantees that either VMRUN will see
3706  	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3707  	 * will signal the doorbell if the CPU has already entered the guest.
3708  	 */
3709  	smp_mb__after_atomic();
3710  	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3711  }
3712  
3713  static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3714  {
3715  	struct vcpu_svm *svm = to_svm(vcpu);
3716  
3717  	/*
3718  	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3719  	 * tracking is done using the CR write traps.
3720  	 */
3721  	if (sev_es_guest(vcpu->kvm))
3722  		return;
3723  
3724  	if (nested_svm_virtualize_tpr(vcpu))
3725  		return;
3726  
3727  	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3728  
3729  	if (irr == -1)
3730  		return;
3731  
3732  	if (tpr >= irr)
3733  		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3734  }
3735  
3736  static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3737  {
3738  	struct vcpu_svm *svm = to_svm(vcpu);
3739  
3740  	if (is_vnmi_enabled(svm))
3741  		return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK;
3742  	else
3743  		return svm->nmi_masked;
3744  }
3745  
3746  static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3747  {
3748  	struct vcpu_svm *svm = to_svm(vcpu);
3749  
3750  	if (is_vnmi_enabled(svm)) {
3751  		if (masked)
3752  			svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK;
3753  		else
3754  			svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
3755  
3756  	} else {
3757  		svm->nmi_masked = masked;
3758  		if (masked)
3759  			svm_set_iret_intercept(svm);
3760  		else
3761  			svm_clr_iret_intercept(svm);
3762  	}
3763  }
3764  
3765  bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3766  {
3767  	struct vcpu_svm *svm = to_svm(vcpu);
3768  	struct vmcb *vmcb = svm->vmcb;
3769  
3770  	if (!gif_set(svm))
3771  		return true;
3772  
3773  	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3774  		return false;
3775  
3776  	if (svm_get_nmi_mask(vcpu))
3777  		return true;
3778  
3779  	return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
3780  }
3781  
3782  static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3783  {
3784  	struct vcpu_svm *svm = to_svm(vcpu);
3785  	if (svm->nested.nested_run_pending)
3786  		return -EBUSY;
3787  
3788  	if (svm_nmi_blocked(vcpu))
3789  		return 0;
3790  
3791  	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3792  	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3793  		return -EBUSY;
3794  	return 1;
3795  }
3796  
3797  bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3798  {
3799  	struct vcpu_svm *svm = to_svm(vcpu);
3800  	struct vmcb *vmcb = svm->vmcb;
3801  
3802  	if (!gif_set(svm))
3803  		return true;
3804  
3805  	if (is_guest_mode(vcpu)) {
3806  		/* As long as interrupts are being delivered...  */
3807  		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3808  		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3809  		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3810  			return true;
3811  
3812  		/* ... vmexits aren't blocked by the interrupt shadow  */
3813  		if (nested_exit_on_intr(svm))
3814  			return false;
3815  	} else {
3816  		if (!svm_get_if_flag(vcpu))
3817  			return true;
3818  	}
3819  
3820  	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3821  }
3822  
3823  static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3824  {
3825  	struct vcpu_svm *svm = to_svm(vcpu);
3826  
3827  	if (svm->nested.nested_run_pending)
3828  		return -EBUSY;
3829  
3830  	if (svm_interrupt_blocked(vcpu))
3831  		return 0;
3832  
3833  	/*
3834  	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3835  	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3836  	 */
3837  	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3838  		return -EBUSY;
3839  
3840  	return 1;
3841  }
3842  
3843  static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3844  {
3845  	struct vcpu_svm *svm = to_svm(vcpu);
3846  
3847  	/*
3848  	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3849  	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3850  	 * get that intercept, this function will be called again though and
3851  	 * we'll get the vintr intercept. However, if the vGIF feature is
3852  	 * enabled, the STGI interception will not occur. Enable the irq
3853  	 * window under the assumption that the hardware will set the GIF.
3854  	 */
3855  	if (vgif || gif_set(svm)) {
3856  		/*
3857  		 * IRQ window is not needed when AVIC is enabled,
3858  		 * unless we have pending ExtINT since it cannot be injected
3859  		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3860  		 * and fallback to injecting IRQ via V_IRQ.
3861  		 *
3862  		 * If running nested, AVIC is already locally inhibited
3863  		 * on this vCPU, therefore there is no need to request
3864  		 * the VM wide AVIC inhibition.
3865  		 */
3866  		if (!is_guest_mode(vcpu))
3867  			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3868  
3869  		svm_set_vintr(svm);
3870  	}
3871  }
3872  
3873  static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3874  {
3875  	struct vcpu_svm *svm = to_svm(vcpu);
3876  
3877  	/*
3878  	 * If NMIs are outright masked, i.e. the vCPU is already handling an
3879  	 * NMI, and KVM has not yet intercepted an IRET, then there is nothing
3880  	 * more to do at this time as KVM has already enabled IRET intercepts.
3881  	 * If KVM has already intercepted IRET, then single-step over the IRET,
3882  	 * as NMIs aren't architecturally unmasked until the IRET completes.
3883  	 *
3884  	 * If vNMI is enabled, KVM should never request an NMI window if NMIs
3885  	 * are masked, as KVM allows at most one to-be-injected NMI and one
3886  	 * pending NMI.  If two NMIs arrive simultaneously, KVM will inject one
3887  	 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are
3888  	 * unmasked.  KVM _will_ request an NMI window in some situations, e.g.
3889  	 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately
3890  	 * inject the NMI.  In those situations, KVM needs to single-step over
3891  	 * the STI shadow or intercept STGI.
3892  	 */
3893  	if (svm_get_nmi_mask(vcpu)) {
3894  		WARN_ON_ONCE(is_vnmi_enabled(svm));
3895  
3896  		if (!svm->awaiting_iret_completion)
3897  			return; /* IRET will cause a vm exit */
3898  	}
3899  
3900  	/*
3901  	 * SEV-ES guests are responsible for signaling when a vCPU is ready to
3902  	 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e.
3903  	 * KVM can't intercept and single-step IRET to detect when NMIs are
3904  	 * unblocked (architecturally speaking).  See SVM_VMGEXIT_NMI_COMPLETE.
3905  	 *
3906  	 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware
3907  	 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not
3908  	 * supported NAEs in the GHCB protocol.
3909  	 */
3910  	if (sev_es_guest(vcpu->kvm))
3911  		return;
3912  
3913  	if (!gif_set(svm)) {
3914  		if (vgif)
3915  			svm_set_intercept(svm, INTERCEPT_STGI);
3916  		return; /* STGI will cause a vm exit */
3917  	}
3918  
3919  	/*
3920  	 * Something prevents NMI from been injected. Single step over possible
3921  	 * problem (IRET or exception injection or interrupt shadow)
3922  	 */
3923  	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3924  	svm->nmi_singlestep = true;
3925  	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3926  }
3927  
3928  static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
3929  {
3930  	struct vcpu_svm *svm = to_svm(vcpu);
3931  
3932  	/*
3933  	 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries.
3934  	 * A TLB flush for the current ASID flushes both "host" and "guest" TLB
3935  	 * entries, and thus is a superset of Hyper-V's fine grained flushing.
3936  	 */
3937  	kvm_hv_vcpu_purge_flush_tlb(vcpu);
3938  
3939  	/*
3940  	 * Flush only the current ASID even if the TLB flush was invoked via
3941  	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3942  	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3943  	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3944  	 * VM-Exit (via kvm_mmu_reset_context()).
3945  	 */
3946  	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3947  		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3948  	else
3949  		svm->current_vmcb->asid_generation--;
3950  }
3951  
3952  static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
3953  {
3954  	hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
3955  
3956  	/*
3957  	 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
3958  	 * flush the NPT mappings via hypercall as flushing the ASID only
3959  	 * affects virtual to physical mappings, it does not invalidate guest
3960  	 * physical to host physical mappings.
3961  	 */
3962  	if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
3963  		hyperv_flush_guest_mapping(root_tdp);
3964  
3965  	svm_flush_tlb_asid(vcpu);
3966  }
3967  
3968  static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
3969  {
3970  	/*
3971  	 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
3972  	 * flushes should be routed to hv_flush_remote_tlbs() without requesting
3973  	 * a "regular" remote flush.  Reaching this point means either there's
3974  	 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of
3975  	 * which might be fatal to the guest.  Yell, but try to recover.
3976  	 */
3977  	if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
3978  		hv_flush_remote_tlbs(vcpu->kvm);
3979  
3980  	svm_flush_tlb_asid(vcpu);
3981  }
3982  
3983  static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3984  {
3985  	struct vcpu_svm *svm = to_svm(vcpu);
3986  
3987  	invlpga(gva, svm->vmcb->control.asid);
3988  }
3989  
3990  static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3991  {
3992  	struct vcpu_svm *svm = to_svm(vcpu);
3993  
3994  	if (nested_svm_virtualize_tpr(vcpu))
3995  		return;
3996  
3997  	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3998  		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3999  		kvm_set_cr8(vcpu, cr8);
4000  	}
4001  }
4002  
4003  static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
4004  {
4005  	struct vcpu_svm *svm = to_svm(vcpu);
4006  	u64 cr8;
4007  
4008  	if (nested_svm_virtualize_tpr(vcpu) ||
4009  	    kvm_vcpu_apicv_active(vcpu))
4010  		return;
4011  
4012  	cr8 = kvm_get_cr8(vcpu);
4013  	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
4014  	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
4015  }
4016  
4017  static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
4018  					int type)
4019  {
4020  	bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT);
4021  	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
4022  	struct vcpu_svm *svm = to_svm(vcpu);
4023  
4024  	/*
4025  	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
4026  	 * associated with the original soft exception/interrupt.  next_rip is
4027  	 * cleared on all exits that can occur while vectoring an event, so KVM
4028  	 * needs to manually set next_rip for re-injection.  Unlike the !nrips
4029  	 * case below, this needs to be done if and only if KVM is re-injecting
4030  	 * the same event, i.e. if the event is a soft exception/interrupt,
4031  	 * otherwise next_rip is unused on VMRUN.
4032  	 */
4033  	if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) &&
4034  	    kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase))
4035  		svm->vmcb->control.next_rip = svm->soft_int_next_rip;
4036  	/*
4037  	 * If NRIPS isn't enabled, KVM must manually advance RIP prior to
4038  	 * injecting the soft exception/interrupt.  That advancement needs to
4039  	 * be unwound if vectoring didn't complete.  Note, the new event may
4040  	 * not be the injected event, e.g. if KVM injected an INTn, the INTn
4041  	 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will
4042  	 * be the reported vectored event, but RIP still needs to be unwound.
4043  	 */
4044  	else if (!nrips && (is_soft || is_exception) &&
4045  		 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase))
4046  		kvm_rip_write(vcpu, svm->soft_int_old_rip);
4047  }
4048  
4049  static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
4050  {
4051  	struct vcpu_svm *svm = to_svm(vcpu);
4052  	u8 vector;
4053  	int type;
4054  	u32 exitintinfo = svm->vmcb->control.exit_int_info;
4055  	bool nmi_l1_to_l2 = svm->nmi_l1_to_l2;
4056  	bool soft_int_injected = svm->soft_int_injected;
4057  
4058  	svm->nmi_l1_to_l2 = false;
4059  	svm->soft_int_injected = false;
4060  
4061  	/*
4062  	 * If we've made progress since setting awaiting_iret_completion, we've
4063  	 * executed an IRET and can allow NMI injection.
4064  	 */
4065  	if (svm->awaiting_iret_completion &&
4066  	    kvm_rip_read(vcpu) != svm->nmi_iret_rip) {
4067  		svm->awaiting_iret_completion = false;
4068  		svm->nmi_masked = false;
4069  		kvm_make_request(KVM_REQ_EVENT, vcpu);
4070  	}
4071  
4072  	vcpu->arch.nmi_injected = false;
4073  	kvm_clear_exception_queue(vcpu);
4074  	kvm_clear_interrupt_queue(vcpu);
4075  
4076  	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4077  		return;
4078  
4079  	kvm_make_request(KVM_REQ_EVENT, vcpu);
4080  
4081  	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4082  	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4083  
4084  	if (soft_int_injected)
4085  		svm_complete_soft_interrupt(vcpu, vector, type);
4086  
4087  	switch (type) {
4088  	case SVM_EXITINTINFO_TYPE_NMI:
4089  		vcpu->arch.nmi_injected = true;
4090  		svm->nmi_l1_to_l2 = nmi_l1_to_l2;
4091  		break;
4092  	case SVM_EXITINTINFO_TYPE_EXEPT:
4093  		/*
4094  		 * Never re-inject a #VC exception.
4095  		 */
4096  		if (vector == X86_TRAP_VC)
4097  			break;
4098  
4099  		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
4100  			u32 err = svm->vmcb->control.exit_int_info_err;
4101  			kvm_requeue_exception_e(vcpu, vector, err);
4102  
4103  		} else
4104  			kvm_requeue_exception(vcpu, vector);
4105  		break;
4106  	case SVM_EXITINTINFO_TYPE_INTR:
4107  		kvm_queue_interrupt(vcpu, vector, false);
4108  		break;
4109  	case SVM_EXITINTINFO_TYPE_SOFT:
4110  		kvm_queue_interrupt(vcpu, vector, true);
4111  		break;
4112  	default:
4113  		break;
4114  	}
4115  
4116  }
4117  
4118  static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4119  {
4120  	struct vcpu_svm *svm = to_svm(vcpu);
4121  	struct vmcb_control_area *control = &svm->vmcb->control;
4122  
4123  	control->exit_int_info = control->event_inj;
4124  	control->exit_int_info_err = control->event_inj_err;
4125  	control->event_inj = 0;
4126  	svm_complete_interrupts(vcpu);
4127  }
4128  
4129  static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
4130  {
4131  	return 1;
4132  }
4133  
4134  static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
4135  {
4136  	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
4137  	    to_svm(vcpu)->vmcb->control.exit_info_1)
4138  		return handle_fastpath_set_msr_irqoff(vcpu);
4139  
4140  	return EXIT_FASTPATH_NONE;
4141  }
4142  
4143  static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
4144  {
4145  	struct vcpu_svm *svm = to_svm(vcpu);
4146  
4147  	guest_state_enter_irqoff();
4148  
4149  	amd_clear_divider();
4150  
4151  	if (sev_es_guest(vcpu->kvm))
4152  		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted);
4153  	else
4154  		__svm_vcpu_run(svm, spec_ctrl_intercepted);
4155  
4156  	guest_state_exit_irqoff();
4157  }
4158  
4159  static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
4160  {
4161  	struct vcpu_svm *svm = to_svm(vcpu);
4162  	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
4163  
4164  	trace_kvm_entry(vcpu);
4165  
4166  	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4167  	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4168  	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4169  
4170  	/*
4171  	 * Disable singlestep if we're injecting an interrupt/exception.
4172  	 * We don't want our modified rflags to be pushed on the stack where
4173  	 * we might not be able to easily reset them if we disabled NMI
4174  	 * singlestep later.
4175  	 */
4176  	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4177  		/*
4178  		 * Event injection happens before external interrupts cause a
4179  		 * vmexit and interrupts are disabled here, so smp_send_reschedule
4180  		 * is enough to force an immediate vmexit.
4181  		 */
4182  		disable_nmi_singlestep(svm);
4183  		smp_send_reschedule(vcpu->cpu);
4184  	}
4185  
4186  	pre_svm_run(vcpu);
4187  
4188  	sync_lapic_to_cr8(vcpu);
4189  
4190  	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
4191  		svm->vmcb->control.asid = svm->asid;
4192  		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
4193  	}
4194  	svm->vmcb->save.cr2 = vcpu->arch.cr2;
4195  
4196  	svm_hv_update_vp_id(svm->vmcb, vcpu);
4197  
4198  	/*
4199  	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
4200  	 * of a #DB.
4201  	 */
4202  	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
4203  		svm_set_dr6(svm, vcpu->arch.dr6);
4204  	else
4205  		svm_set_dr6(svm, DR6_ACTIVE_LOW);
4206  
4207  	clgi();
4208  	kvm_load_guest_xsave_state(vcpu);
4209  
4210  	kvm_wait_lapic_expire(vcpu);
4211  
4212  	/*
4213  	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
4214  	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
4215  	 * is no need to worry about the conditional branch over the wrmsr
4216  	 * being speculatively taken.
4217  	 */
4218  	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4219  		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
4220  
4221  	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
4222  
4223  	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4224  		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
4225  
4226  	if (!sev_es_guest(vcpu->kvm)) {
4227  		vcpu->arch.cr2 = svm->vmcb->save.cr2;
4228  		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4229  		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4230  		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4231  	}
4232  	vcpu->arch.regs_dirty = 0;
4233  
4234  	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4235  		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
4236  
4237  	kvm_load_host_xsave_state(vcpu);
4238  	stgi();
4239  
4240  	/* Any pending NMI will happen here */
4241  
4242  	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4243  		kvm_after_interrupt(vcpu);
4244  
4245  	sync_cr8_to_lapic(vcpu);
4246  
4247  	svm->next_rip = 0;
4248  	if (is_guest_mode(vcpu)) {
4249  		nested_sync_control_from_vmcb02(svm);
4250  
4251  		/* Track VMRUNs that have made past consistency checking */
4252  		if (svm->nested.nested_run_pending &&
4253  		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
4254                          ++vcpu->stat.nested_run;
4255  
4256  		svm->nested.nested_run_pending = 0;
4257  	}
4258  
4259  	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4260  	vmcb_mark_all_clean(svm->vmcb);
4261  
4262  	/* if exit due to PF check for async PF */
4263  	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4264  		vcpu->arch.apf.host_apf_flags =
4265  			kvm_read_and_reset_apf_flags();
4266  
4267  	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
4268  
4269  	/*
4270  	 * We need to handle MC intercepts here before the vcpu has a chance to
4271  	 * change the physical cpu
4272  	 */
4273  	if (unlikely(svm->vmcb->control.exit_code ==
4274  		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
4275  		svm_handle_mce(vcpu);
4276  
4277  	trace_kvm_exit(vcpu, KVM_ISA_SVM);
4278  
4279  	svm_complete_interrupts(vcpu);
4280  
4281  	if (is_guest_mode(vcpu))
4282  		return EXIT_FASTPATH_NONE;
4283  
4284  	return svm_exit_handlers_fastpath(vcpu);
4285  }
4286  
4287  static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
4288  			     int root_level)
4289  {
4290  	struct vcpu_svm *svm = to_svm(vcpu);
4291  	unsigned long cr3;
4292  
4293  	if (npt_enabled) {
4294  		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
4295  		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
4296  
4297  		hv_track_root_tdp(vcpu, root_hpa);
4298  
4299  		cr3 = vcpu->arch.cr3;
4300  	} else if (root_level >= PT64_ROOT_4LEVEL) {
4301  		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4302  	} else {
4303  		/* PCID in the guest should be impossible with a 32-bit MMU. */
4304  		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4305  		cr3 = root_hpa;
4306  	}
4307  
4308  	svm->vmcb->save.cr3 = cr3;
4309  	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4310  }
4311  
4312  static void
4313  svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4314  {
4315  	/*
4316  	 * Patch in the VMMCALL instruction:
4317  	 */
4318  	hypercall[0] = 0x0f;
4319  	hypercall[1] = 0x01;
4320  	hypercall[2] = 0xd9;
4321  }
4322  
4323  /*
4324   * The kvm parameter can be NULL (module initialization, or invocation before
4325   * VM creation). Be sure to check the kvm parameter before using it.
4326   */
4327  static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4328  {
4329  	switch (index) {
4330  	case MSR_IA32_MCG_EXT_CTL:
4331  	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
4332  		return false;
4333  	case MSR_IA32_SMBASE:
4334  		if (!IS_ENABLED(CONFIG_KVM_SMM))
4335  			return false;
4336  		/* SEV-ES guests do not support SMM, so report false */
4337  		if (kvm && sev_es_guest(kvm))
4338  			return false;
4339  		break;
4340  	default:
4341  		break;
4342  	}
4343  
4344  	return true;
4345  }
4346  
4347  static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4348  {
4349  	struct vcpu_svm *svm = to_svm(vcpu);
4350  
4351  	/*
4352  	 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
4353  	 * can only disable all variants of by disallowing CR4.OSXSAVE from
4354  	 * being set.  As a result, if the host has XSAVE and XSAVES, and the
4355  	 * guest has XSAVE enabled, the guest can execute XSAVES without
4356  	 * faulting.  Treat XSAVES as enabled in this case regardless of
4357  	 * whether it's advertised to the guest so that KVM context switches
4358  	 * XSS on VM-Enter/VM-Exit.  Failure to do so would effectively give
4359  	 * the guest read/write access to the host's XSS.
4360  	 */
4361  	if (boot_cpu_has(X86_FEATURE_XSAVE) &&
4362  	    boot_cpu_has(X86_FEATURE_XSAVES) &&
4363  	    guest_cpuid_has(vcpu, X86_FEATURE_XSAVE))
4364  		kvm_governed_feature_set(vcpu, X86_FEATURE_XSAVES);
4365  
4366  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_NRIPS);
4367  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_TSCRATEMSR);
4368  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LBRV);
4369  
4370  	/*
4371  	 * Intercept VMLOAD if the vCPU mode is Intel in order to emulate that
4372  	 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing
4373  	 * SVM on Intel is bonkers and extremely unlikely to work).
4374  	 */
4375  	if (!guest_cpuid_is_intel(vcpu))
4376  		kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4377  
4378  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PAUSEFILTER);
4379  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PFTHRESHOLD);
4380  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VGIF);
4381  	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VNMI);
4382  
4383  	svm_recalc_instruction_intercepts(vcpu, svm);
4384  
4385  	if (boot_cpu_has(X86_FEATURE_IBPB))
4386  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0,
4387  				     !!guest_has_pred_cmd_msr(vcpu));
4388  
4389  	if (boot_cpu_has(X86_FEATURE_FLUSH_L1D))
4390  		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4391  				     !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));
4392  
4393  	if (sev_guest(vcpu->kvm))
4394  		sev_vcpu_after_set_cpuid(svm);
4395  
4396  	init_vmcb_after_set_cpuid(vcpu);
4397  }
4398  
4399  static bool svm_has_wbinvd_exit(void)
4400  {
4401  	return true;
4402  }
4403  
4404  #define PRE_EX(exit)  { .exit_code = (exit), \
4405  			.stage = X86_ICPT_PRE_EXCEPT, }
4406  #define POST_EX(exit) { .exit_code = (exit), \
4407  			.stage = X86_ICPT_POST_EXCEPT, }
4408  #define POST_MEM(exit) { .exit_code = (exit), \
4409  			.stage = X86_ICPT_POST_MEMACCESS, }
4410  
4411  static const struct __x86_intercept {
4412  	u32 exit_code;
4413  	enum x86_intercept_stage stage;
4414  } x86_intercept_map[] = {
4415  	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4416  	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4417  	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4418  	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4419  	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4420  	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4421  	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4422  	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4423  	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4424  	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4425  	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4426  	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4427  	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4428  	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4429  	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4430  	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4431  	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4432  	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4433  	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4434  	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4435  	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4436  	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4437  	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4438  	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4439  	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4440  	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4441  	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4442  	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4443  	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4444  	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4445  	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4446  	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4447  	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4448  	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4449  	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4450  	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4451  	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4452  	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4453  	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4454  	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4455  	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4456  	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4457  	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4458  	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4459  	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4460  	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4461  	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4462  };
4463  
4464  #undef PRE_EX
4465  #undef POST_EX
4466  #undef POST_MEM
4467  
4468  static int svm_check_intercept(struct kvm_vcpu *vcpu,
4469  			       struct x86_instruction_info *info,
4470  			       enum x86_intercept_stage stage,
4471  			       struct x86_exception *exception)
4472  {
4473  	struct vcpu_svm *svm = to_svm(vcpu);
4474  	int vmexit, ret = X86EMUL_CONTINUE;
4475  	struct __x86_intercept icpt_info;
4476  	struct vmcb *vmcb = svm->vmcb;
4477  
4478  	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4479  		goto out;
4480  
4481  	icpt_info = x86_intercept_map[info->intercept];
4482  
4483  	if (stage != icpt_info.stage)
4484  		goto out;
4485  
4486  	switch (icpt_info.exit_code) {
4487  	case SVM_EXIT_READ_CR0:
4488  		if (info->intercept == x86_intercept_cr_read)
4489  			icpt_info.exit_code += info->modrm_reg;
4490  		break;
4491  	case SVM_EXIT_WRITE_CR0: {
4492  		unsigned long cr0, val;
4493  
4494  		if (info->intercept == x86_intercept_cr_write)
4495  			icpt_info.exit_code += info->modrm_reg;
4496  
4497  		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4498  		    info->intercept == x86_intercept_clts)
4499  			break;
4500  
4501  		if (!(vmcb12_is_intercept(&svm->nested.ctl,
4502  					INTERCEPT_SELECTIVE_CR0)))
4503  			break;
4504  
4505  		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4506  		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4507  
4508  		if (info->intercept == x86_intercept_lmsw) {
4509  			cr0 &= 0xfUL;
4510  			val &= 0xfUL;
4511  			/* lmsw can't clear PE - catch this here */
4512  			if (cr0 & X86_CR0_PE)
4513  				val |= X86_CR0_PE;
4514  		}
4515  
4516  		if (cr0 ^ val)
4517  			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4518  
4519  		break;
4520  	}
4521  	case SVM_EXIT_READ_DR0:
4522  	case SVM_EXIT_WRITE_DR0:
4523  		icpt_info.exit_code += info->modrm_reg;
4524  		break;
4525  	case SVM_EXIT_MSR:
4526  		if (info->intercept == x86_intercept_wrmsr)
4527  			vmcb->control.exit_info_1 = 1;
4528  		else
4529  			vmcb->control.exit_info_1 = 0;
4530  		break;
4531  	case SVM_EXIT_PAUSE:
4532  		/*
4533  		 * We get this for NOP only, but pause
4534  		 * is rep not, check this here
4535  		 */
4536  		if (info->rep_prefix != REPE_PREFIX)
4537  			goto out;
4538  		break;
4539  	case SVM_EXIT_IOIO: {
4540  		u64 exit_info;
4541  		u32 bytes;
4542  
4543  		if (info->intercept == x86_intercept_in ||
4544  		    info->intercept == x86_intercept_ins) {
4545  			exit_info = ((info->src_val & 0xffff) << 16) |
4546  				SVM_IOIO_TYPE_MASK;
4547  			bytes = info->dst_bytes;
4548  		} else {
4549  			exit_info = (info->dst_val & 0xffff) << 16;
4550  			bytes = info->src_bytes;
4551  		}
4552  
4553  		if (info->intercept == x86_intercept_outs ||
4554  		    info->intercept == x86_intercept_ins)
4555  			exit_info |= SVM_IOIO_STR_MASK;
4556  
4557  		if (info->rep_prefix)
4558  			exit_info |= SVM_IOIO_REP_MASK;
4559  
4560  		bytes = min(bytes, 4u);
4561  
4562  		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4563  
4564  		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4565  
4566  		vmcb->control.exit_info_1 = exit_info;
4567  		vmcb->control.exit_info_2 = info->next_rip;
4568  
4569  		break;
4570  	}
4571  	default:
4572  		break;
4573  	}
4574  
4575  	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4576  	if (static_cpu_has(X86_FEATURE_NRIPS))
4577  		vmcb->control.next_rip  = info->next_rip;
4578  	vmcb->control.exit_code = icpt_info.exit_code;
4579  	vmexit = nested_svm_exit_handled(svm);
4580  
4581  	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4582  					   : X86EMUL_CONTINUE;
4583  
4584  out:
4585  	return ret;
4586  }
4587  
4588  static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4589  {
4590  	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
4591  		vcpu->arch.at_instruction_boundary = true;
4592  }
4593  
4594  static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4595  {
4596  	if (!kvm_pause_in_guest(vcpu->kvm))
4597  		shrink_ple_window(vcpu);
4598  }
4599  
4600  static void svm_setup_mce(struct kvm_vcpu *vcpu)
4601  {
4602  	/* [63:9] are reserved. */
4603  	vcpu->arch.mcg_cap &= 0x1ff;
4604  }
4605  
4606  #ifdef CONFIG_KVM_SMM
4607  bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4608  {
4609  	struct vcpu_svm *svm = to_svm(vcpu);
4610  
4611  	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4612  	if (!gif_set(svm))
4613  		return true;
4614  
4615  	return is_smm(vcpu);
4616  }
4617  
4618  static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4619  {
4620  	struct vcpu_svm *svm = to_svm(vcpu);
4621  	if (svm->nested.nested_run_pending)
4622  		return -EBUSY;
4623  
4624  	if (svm_smi_blocked(vcpu))
4625  		return 0;
4626  
4627  	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4628  	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4629  		return -EBUSY;
4630  
4631  	return 1;
4632  }
4633  
4634  static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
4635  {
4636  	struct vcpu_svm *svm = to_svm(vcpu);
4637  	struct kvm_host_map map_save;
4638  	int ret;
4639  
4640  	if (!is_guest_mode(vcpu))
4641  		return 0;
4642  
4643  	/*
4644  	 * 32-bit SMRAM format doesn't preserve EFER and SVM state.  Userspace is
4645  	 * responsible for ensuring nested SVM and SMIs are mutually exclusive.
4646  	 */
4647  
4648  	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4649  		return 1;
4650  
4651  	smram->smram64.svm_guest_flag = 1;
4652  	smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa;
4653  
4654  	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4655  	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4656  	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4657  
4658  	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4659  	if (ret)
4660  		return ret;
4661  
4662  	/*
4663  	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4664  	 * VMCB01 is going to be used during SMM and thus the state will
4665  	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4666  	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4667  	 * format of the area is identical to guest save area offsetted
4668  	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4669  	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4670  	 * L1 hypervisor to save additional host context (e.g. KVM does
4671  	 * that, see svm_prepare_switch_to_guest()) which must be
4672  	 * preserved.
4673  	 */
4674  	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4675  		return 1;
4676  
4677  	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4678  
4679  	svm_copy_vmrun_state(map_save.hva + 0x400,
4680  			     &svm->vmcb01.ptr->save);
4681  
4682  	kvm_vcpu_unmap(vcpu, &map_save, true);
4683  	return 0;
4684  }
4685  
4686  static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
4687  {
4688  	struct vcpu_svm *svm = to_svm(vcpu);
4689  	struct kvm_host_map map, map_save;
4690  	struct vmcb *vmcb12;
4691  	int ret;
4692  
4693  	const struct kvm_smram_state_64 *smram64 = &smram->smram64;
4694  
4695  	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4696  		return 0;
4697  
4698  	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4699  	if (!smram64->svm_guest_flag)
4700  		return 0;
4701  
4702  	if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4703  		return 1;
4704  
4705  	if (!(smram64->efer & EFER_SVME))
4706  		return 1;
4707  
4708  	if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map))
4709  		return 1;
4710  
4711  	ret = 1;
4712  	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4713  		goto unmap_map;
4714  
4715  	if (svm_allocate_nested(svm))
4716  		goto unmap_save;
4717  
4718  	/*
4719  	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4720  	 * used during SMM (see svm_enter_smm())
4721  	 */
4722  
4723  	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4724  
4725  	/*
4726  	 * Enter the nested guest now
4727  	 */
4728  
4729  	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4730  
4731  	vmcb12 = map.hva;
4732  	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4733  	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4734  	ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false);
4735  
4736  	if (ret)
4737  		goto unmap_save;
4738  
4739  	svm->nested.nested_run_pending = 1;
4740  
4741  unmap_save:
4742  	kvm_vcpu_unmap(vcpu, &map_save, true);
4743  unmap_map:
4744  	kvm_vcpu_unmap(vcpu, &map, true);
4745  	return ret;
4746  }
4747  
4748  static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4749  {
4750  	struct vcpu_svm *svm = to_svm(vcpu);
4751  
4752  	if (!gif_set(svm)) {
4753  		if (vgif)
4754  			svm_set_intercept(svm, INTERCEPT_STGI);
4755  		/* STGI will cause a vm exit */
4756  	} else {
4757  		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4758  	}
4759  }
4760  #endif
4761  
4762  static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4763  					void *insn, int insn_len)
4764  {
4765  	bool smep, smap, is_user;
4766  	u64 error_code;
4767  
4768  	/* Emulation is always possible when KVM has access to all guest state. */
4769  	if (!sev_guest(vcpu->kvm))
4770  		return true;
4771  
4772  	/* #UD and #GP should never be intercepted for SEV guests. */
4773  	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4774  				  EMULTYPE_TRAP_UD_FORCED |
4775  				  EMULTYPE_VMWARE_GP));
4776  
4777  	/*
4778  	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4779  	 * to guest register state.
4780  	 */
4781  	if (sev_es_guest(vcpu->kvm))
4782  		return false;
4783  
4784  	/*
4785  	 * Emulation is possible if the instruction is already decoded, e.g.
4786  	 * when completing I/O after returning from userspace.
4787  	 */
4788  	if (emul_type & EMULTYPE_NO_DECODE)
4789  		return true;
4790  
4791  	/*
4792  	 * Emulation is possible for SEV guests if and only if a prefilled
4793  	 * buffer containing the bytes of the intercepted instruction is
4794  	 * available. SEV guest memory is encrypted with a guest specific key
4795  	 * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and
4796  	 * decode garbage.
4797  	 *
4798  	 * If KVM is NOT trying to simply skip an instruction, inject #UD if
4799  	 * KVM reached this point without an instruction buffer.  In practice,
4800  	 * this path should never be hit by a well-behaved guest, e.g. KVM
4801  	 * doesn't intercept #UD or #GP for SEV guests, but this path is still
4802  	 * theoretically reachable, e.g. via unaccelerated fault-like AVIC
4803  	 * access, and needs to be handled by KVM to avoid putting the guest
4804  	 * into an infinite loop.   Injecting #UD is somewhat arbitrary, but
4805  	 * its the least awful option given lack of insight into the guest.
4806  	 *
4807  	 * If KVM is trying to skip an instruction, simply resume the guest.
4808  	 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM
4809  	 * will attempt to re-inject the INT3/INTO and skip the instruction.
4810  	 * In that scenario, retrying the INT3/INTO and hoping the guest will
4811  	 * make forward progress is the only option that has a chance of
4812  	 * success (and in practice it will work the vast majority of the time).
4813  	 */
4814  	if (unlikely(!insn)) {
4815  		if (!(emul_type & EMULTYPE_SKIP))
4816  			kvm_queue_exception(vcpu, UD_VECTOR);
4817  		return false;
4818  	}
4819  
4820  	/*
4821  	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4822  	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4823  	 * the faulting instruction because the code fetch itself faulted, e.g.
4824  	 * the guest attempted to fetch from emulated MMIO or a guest page
4825  	 * table used to translate CS:RIP resides in emulated MMIO.
4826  	 */
4827  	if (likely(insn_len))
4828  		return true;
4829  
4830  	/*
4831  	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4832  	 *
4833  	 * Errata:
4834  	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4835  	 * possible that CPU microcode implementing DecodeAssist will fail to
4836  	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4837  	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
4838  	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
4839  	 * gives up and does not fill the instruction bytes buffer.
4840  	 *
4841  	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4842  	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4843  	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4844  	 * GuestIntrBytes field of the VMCB.
4845  	 *
4846  	 * This does _not_ mean that the erratum has been encountered, as the
4847  	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4848  	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4849  	 * encountered a reserved/not-present #PF.
4850  	 *
4851  	 * To hit the erratum, the following conditions must be true:
4852  	 *    1. CR4.SMAP=1 (obviously).
4853  	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
4854  	 *       have been hit as the guest would have encountered a SMEP
4855  	 *       violation #PF, not a #NPF.
4856  	 *    3. The #NPF is not due to a code fetch, in which case failure to
4857  	 *       retrieve the instruction bytes is legitimate (see abvoe).
4858  	 *
4859  	 * In addition, don't apply the erratum workaround if the #NPF occurred
4860  	 * while translating guest page tables (see below).
4861  	 */
4862  	error_code = to_svm(vcpu)->vmcb->control.exit_info_1;
4863  	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
4864  		goto resume_guest;
4865  
4866  	smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP);
4867  	smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP);
4868  	is_user = svm_get_cpl(vcpu) == 3;
4869  	if (smap && (!smep || is_user)) {
4870  		pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n");
4871  
4872  		/*
4873  		 * If the fault occurred in userspace, arbitrarily inject #GP
4874  		 * to avoid killing the guest and to hopefully avoid confusing
4875  		 * the guest kernel too much, e.g. injecting #PF would not be
4876  		 * coherent with respect to the guest's page tables.  Request
4877  		 * triple fault if the fault occurred in the kernel as there's
4878  		 * no fault that KVM can inject without confusing the guest.
4879  		 * In practice, the triple fault is moot as no sane SEV kernel
4880  		 * will execute from user memory while also running with SMAP=1.
4881  		 */
4882  		if (is_user)
4883  			kvm_inject_gp(vcpu, 0);
4884  		else
4885  			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4886  	}
4887  
4888  resume_guest:
4889  	/*
4890  	 * If the erratum was not hit, simply resume the guest and let it fault
4891  	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
4892  	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
4893  	 * userspace will kill the guest, and letting the emulator read garbage
4894  	 * will yield random behavior and potentially corrupt the guest.
4895  	 *
4896  	 * Simply resuming the guest is technically not a violation of the SEV
4897  	 * architecture.  AMD's APM states that all code fetches and page table
4898  	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
4899  	 * APM also states that encrypted accesses to MMIO are "ignored", but
4900  	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4901  	 * the guest spin is technically "ignoring" the access.
4902  	 */
4903  	return false;
4904  }
4905  
4906  static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4907  {
4908  	struct vcpu_svm *svm = to_svm(vcpu);
4909  
4910  	return !gif_set(svm);
4911  }
4912  
4913  static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4914  {
4915  	if (!sev_es_guest(vcpu->kvm))
4916  		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4917  
4918  	sev_vcpu_deliver_sipi_vector(vcpu, vector);
4919  }
4920  
4921  static void svm_vm_destroy(struct kvm *kvm)
4922  {
4923  	avic_vm_destroy(kvm);
4924  	sev_vm_destroy(kvm);
4925  }
4926  
4927  static int svm_vm_init(struct kvm *kvm)
4928  {
4929  	if (!pause_filter_count || !pause_filter_thresh)
4930  		kvm->arch.pause_in_guest = true;
4931  
4932  	if (enable_apicv) {
4933  		int ret = avic_vm_init(kvm);
4934  		if (ret)
4935  			return ret;
4936  	}
4937  
4938  	return 0;
4939  }
4940  
4941  static struct kvm_x86_ops svm_x86_ops __initdata = {
4942  	.name = KBUILD_MODNAME,
4943  
4944  	.check_processor_compatibility = svm_check_processor_compat,
4945  
4946  	.hardware_unsetup = svm_hardware_unsetup,
4947  	.hardware_enable = svm_hardware_enable,
4948  	.hardware_disable = svm_hardware_disable,
4949  	.has_emulated_msr = svm_has_emulated_msr,
4950  
4951  	.vcpu_create = svm_vcpu_create,
4952  	.vcpu_free = svm_vcpu_free,
4953  	.vcpu_reset = svm_vcpu_reset,
4954  
4955  	.vm_size = sizeof(struct kvm_svm),
4956  	.vm_init = svm_vm_init,
4957  	.vm_destroy = svm_vm_destroy,
4958  
4959  	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
4960  	.vcpu_load = svm_vcpu_load,
4961  	.vcpu_put = svm_vcpu_put,
4962  	.vcpu_blocking = avic_vcpu_blocking,
4963  	.vcpu_unblocking = avic_vcpu_unblocking,
4964  
4965  	.update_exception_bitmap = svm_update_exception_bitmap,
4966  	.get_msr_feature = svm_get_msr_feature,
4967  	.get_msr = svm_get_msr,
4968  	.set_msr = svm_set_msr,
4969  	.get_segment_base = svm_get_segment_base,
4970  	.get_segment = svm_get_segment,
4971  	.set_segment = svm_set_segment,
4972  	.get_cpl = svm_get_cpl,
4973  	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
4974  	.is_valid_cr0 = svm_is_valid_cr0,
4975  	.set_cr0 = svm_set_cr0,
4976  	.post_set_cr3 = sev_post_set_cr3,
4977  	.is_valid_cr4 = svm_is_valid_cr4,
4978  	.set_cr4 = svm_set_cr4,
4979  	.set_efer = svm_set_efer,
4980  	.get_idt = svm_get_idt,
4981  	.set_idt = svm_set_idt,
4982  	.get_gdt = svm_get_gdt,
4983  	.set_gdt = svm_set_gdt,
4984  	.set_dr7 = svm_set_dr7,
4985  	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4986  	.cache_reg = svm_cache_reg,
4987  	.get_rflags = svm_get_rflags,
4988  	.set_rflags = svm_set_rflags,
4989  	.get_if_flag = svm_get_if_flag,
4990  
4991  	.flush_tlb_all = svm_flush_tlb_all,
4992  	.flush_tlb_current = svm_flush_tlb_current,
4993  	.flush_tlb_gva = svm_flush_tlb_gva,
4994  	.flush_tlb_guest = svm_flush_tlb_asid,
4995  
4996  	.vcpu_pre_run = svm_vcpu_pre_run,
4997  	.vcpu_run = svm_vcpu_run,
4998  	.handle_exit = svm_handle_exit,
4999  	.skip_emulated_instruction = svm_skip_emulated_instruction,
5000  	.update_emulated_instruction = NULL,
5001  	.set_interrupt_shadow = svm_set_interrupt_shadow,
5002  	.get_interrupt_shadow = svm_get_interrupt_shadow,
5003  	.patch_hypercall = svm_patch_hypercall,
5004  	.inject_irq = svm_inject_irq,
5005  	.inject_nmi = svm_inject_nmi,
5006  	.is_vnmi_pending = svm_is_vnmi_pending,
5007  	.set_vnmi_pending = svm_set_vnmi_pending,
5008  	.inject_exception = svm_inject_exception,
5009  	.cancel_injection = svm_cancel_injection,
5010  	.interrupt_allowed = svm_interrupt_allowed,
5011  	.nmi_allowed = svm_nmi_allowed,
5012  	.get_nmi_mask = svm_get_nmi_mask,
5013  	.set_nmi_mask = svm_set_nmi_mask,
5014  	.enable_nmi_window = svm_enable_nmi_window,
5015  	.enable_irq_window = svm_enable_irq_window,
5016  	.update_cr8_intercept = svm_update_cr8_intercept,
5017  	.set_virtual_apic_mode = avic_refresh_virtual_apic_mode,
5018  	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
5019  	.apicv_post_state_restore = avic_apicv_post_state_restore,
5020  	.required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS,
5021  
5022  	.get_exit_info = svm_get_exit_info,
5023  
5024  	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
5025  
5026  	.has_wbinvd_exit = svm_has_wbinvd_exit,
5027  
5028  	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
5029  	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
5030  	.write_tsc_offset = svm_write_tsc_offset,
5031  	.write_tsc_multiplier = svm_write_tsc_multiplier,
5032  
5033  	.load_mmu_pgd = svm_load_mmu_pgd,
5034  
5035  	.check_intercept = svm_check_intercept,
5036  	.handle_exit_irqoff = svm_handle_exit_irqoff,
5037  
5038  	.request_immediate_exit = __kvm_request_immediate_exit,
5039  
5040  	.sched_in = svm_sched_in,
5041  
5042  	.nested_ops = &svm_nested_ops,
5043  
5044  	.deliver_interrupt = svm_deliver_interrupt,
5045  	.pi_update_irte = avic_pi_update_irte,
5046  	.setup_mce = svm_setup_mce,
5047  
5048  #ifdef CONFIG_KVM_SMM
5049  	.smi_allowed = svm_smi_allowed,
5050  	.enter_smm = svm_enter_smm,
5051  	.leave_smm = svm_leave_smm,
5052  	.enable_smi_window = svm_enable_smi_window,
5053  #endif
5054  
5055  	.mem_enc_ioctl = sev_mem_enc_ioctl,
5056  	.mem_enc_register_region = sev_mem_enc_register_region,
5057  	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
5058  	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
5059  
5060  	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
5061  	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
5062  
5063  	.can_emulate_instruction = svm_can_emulate_instruction,
5064  
5065  	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
5066  
5067  	.msr_filter_changed = svm_msr_filter_changed,
5068  	.complete_emulated_msr = svm_complete_emulated_msr,
5069  
5070  	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
5071  	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
5072  };
5073  
5074  /*
5075   * The default MMIO mask is a single bit (excluding the present bit),
5076   * which could conflict with the memory encryption bit. Check for
5077   * memory encryption support and override the default MMIO mask if
5078   * memory encryption is enabled.
5079   */
5080  static __init void svm_adjust_mmio_mask(void)
5081  {
5082  	unsigned int enc_bit, mask_bit;
5083  	u64 msr, mask;
5084  
5085  	/* If there is no memory encryption support, use existing mask */
5086  	if (cpuid_eax(0x80000000) < 0x8000001f)
5087  		return;
5088  
5089  	/* If memory encryption is not enabled, use existing mask */
5090  	rdmsrl(MSR_AMD64_SYSCFG, msr);
5091  	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
5092  		return;
5093  
5094  	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
5095  	mask_bit = boot_cpu_data.x86_phys_bits;
5096  
5097  	/* Increment the mask bit if it is the same as the encryption bit */
5098  	if (enc_bit == mask_bit)
5099  		mask_bit++;
5100  
5101  	/*
5102  	 * If the mask bit location is below 52, then some bits above the
5103  	 * physical addressing limit will always be reserved, so use the
5104  	 * rsvd_bits() function to generate the mask. This mask, along with
5105  	 * the present bit, will be used to generate a page fault with
5106  	 * PFER.RSV = 1.
5107  	 *
5108  	 * If the mask bit location is 52 (or above), then clear the mask.
5109  	 */
5110  	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
5111  
5112  	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
5113  }
5114  
5115  static __init void svm_set_cpu_caps(void)
5116  {
5117  	kvm_set_cpu_caps();
5118  
5119  	kvm_caps.supported_perf_cap = 0;
5120  	kvm_caps.supported_xss = 0;
5121  
5122  	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
5123  	if (nested) {
5124  		kvm_cpu_cap_set(X86_FEATURE_SVM);
5125  		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
5126  
5127  		if (nrips)
5128  			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
5129  
5130  		if (npt_enabled)
5131  			kvm_cpu_cap_set(X86_FEATURE_NPT);
5132  
5133  		if (tsc_scaling)
5134  			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
5135  
5136  		if (vls)
5137  			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
5138  		if (lbrv)
5139  			kvm_cpu_cap_set(X86_FEATURE_LBRV);
5140  
5141  		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
5142  			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
5143  
5144  		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
5145  			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
5146  
5147  		if (vgif)
5148  			kvm_cpu_cap_set(X86_FEATURE_VGIF);
5149  
5150  		if (vnmi)
5151  			kvm_cpu_cap_set(X86_FEATURE_VNMI);
5152  
5153  		/* Nested VM can receive #VMEXIT instead of triggering #GP */
5154  		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
5155  	}
5156  
5157  	/* CPUID 0x80000008 */
5158  	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5159  	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
5160  		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
5161  
5162  	if (enable_pmu) {
5163  		/*
5164  		 * Enumerate support for PERFCTR_CORE if and only if KVM has
5165  		 * access to enough counters to virtualize "core" support,
5166  		 * otherwise limit vPMU support to the legacy number of counters.
5167  		 */
5168  		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
5169  			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
5170  							  kvm_pmu_cap.num_counters_gp);
5171  		else
5172  			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
5173  
5174  		if (kvm_pmu_cap.version != 2 ||
5175  		    !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
5176  			kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2);
5177  	}
5178  
5179  	/* CPUID 0x8000001F (SME/SEV features) */
5180  	sev_set_cpu_caps();
5181  
5182  	/* Don't advertise Bus Lock Detect to guest if SVM support is absent */
5183  	kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT);
5184  }
5185  
5186  static __init int svm_hardware_setup(void)
5187  {
5188  	int cpu;
5189  	struct page *iopm_pages;
5190  	void *iopm_va;
5191  	int r;
5192  	unsigned int order = get_order(IOPM_SIZE);
5193  
5194  	/*
5195  	 * NX is required for shadow paging and for NPT if the NX huge pages
5196  	 * mitigation is enabled.
5197  	 */
5198  	if (!boot_cpu_has(X86_FEATURE_NX)) {
5199  		pr_err_ratelimited("NX (Execute Disable) not supported\n");
5200  		return -EOPNOTSUPP;
5201  	}
5202  	kvm_enable_efer_bits(EFER_NX);
5203  
5204  	iopm_pages = alloc_pages(GFP_KERNEL, order);
5205  
5206  	if (!iopm_pages)
5207  		return -ENOMEM;
5208  
5209  	iopm_va = page_address(iopm_pages);
5210  	memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
5211  	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
5212  
5213  	init_msrpm_offsets();
5214  
5215  	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
5216  				     XFEATURE_MASK_BNDCSR);
5217  
5218  	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
5219  		kvm_enable_efer_bits(EFER_FFXSR);
5220  
5221  	if (tsc_scaling) {
5222  		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
5223  			tsc_scaling = false;
5224  		} else {
5225  			pr_info("TSC scaling supported\n");
5226  			kvm_caps.has_tsc_control = true;
5227  		}
5228  	}
5229  	kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
5230  	kvm_caps.tsc_scaling_ratio_frac_bits = 32;
5231  
5232  	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
5233  
5234  	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
5235  		kvm_enable_efer_bits(EFER_AUTOIBRS);
5236  
5237  	/* Check for pause filtering support */
5238  	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
5239  		pause_filter_count = 0;
5240  		pause_filter_thresh = 0;
5241  	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
5242  		pause_filter_thresh = 0;
5243  	}
5244  
5245  	if (nested) {
5246  		pr_info("Nested Virtualization enabled\n");
5247  		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
5248  	}
5249  
5250  	/*
5251  	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
5252  	 * NPT isn't supported if the host is using 2-level paging since host
5253  	 * CR4 is unchanged on VMRUN.
5254  	 */
5255  	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
5256  		npt_enabled = false;
5257  
5258  	if (!boot_cpu_has(X86_FEATURE_NPT))
5259  		npt_enabled = false;
5260  
5261  	/* Force VM NPT level equal to the host's paging level */
5262  	kvm_configure_mmu(npt_enabled, get_npt_level(),
5263  			  get_npt_level(), PG_LEVEL_1G);
5264  	pr_info("Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
5265  
5266  	/* Setup shadow_me_value and shadow_me_mask */
5267  	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
5268  
5269  	svm_adjust_mmio_mask();
5270  
5271  	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5272  
5273  	if (lbrv) {
5274  		if (!boot_cpu_has(X86_FEATURE_LBRV))
5275  			lbrv = false;
5276  		else
5277  			pr_info("LBR virtualization supported\n");
5278  	}
5279  	/*
5280  	 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5281  	 * may be modified by svm_adjust_mmio_mask()), as well as nrips.
5282  	 */
5283  	sev_hardware_setup();
5284  
5285  	svm_hv_hardware_setup();
5286  
5287  	for_each_possible_cpu(cpu) {
5288  		r = svm_cpu_init(cpu);
5289  		if (r)
5290  			goto err;
5291  	}
5292  
5293  	enable_apicv = avic = avic && avic_hardware_setup();
5294  
5295  	if (!enable_apicv) {
5296  		svm_x86_ops.vcpu_blocking = NULL;
5297  		svm_x86_ops.vcpu_unblocking = NULL;
5298  		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
5299  	} else if (!x2avic_enabled) {
5300  		svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;
5301  	}
5302  
5303  	if (vls) {
5304  		if (!npt_enabled ||
5305  		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
5306  		    !IS_ENABLED(CONFIG_X86_64)) {
5307  			vls = false;
5308  		} else {
5309  			pr_info("Virtual VMLOAD VMSAVE supported\n");
5310  		}
5311  	}
5312  
5313  	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
5314  		svm_gp_erratum_intercept = false;
5315  
5316  	if (vgif) {
5317  		if (!boot_cpu_has(X86_FEATURE_VGIF))
5318  			vgif = false;
5319  		else
5320  			pr_info("Virtual GIF supported\n");
5321  	}
5322  
5323  	vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI);
5324  	if (vnmi)
5325  		pr_info("Virtual NMI enabled\n");
5326  
5327  	if (!vnmi) {
5328  		svm_x86_ops.is_vnmi_pending = NULL;
5329  		svm_x86_ops.set_vnmi_pending = NULL;
5330  	}
5331  
5332  	if (!enable_pmu)
5333  		pr_info("PMU virtualization is disabled\n");
5334  
5335  	svm_set_cpu_caps();
5336  
5337  	/*
5338  	 * It seems that on AMD processors PTE's accessed bit is
5339  	 * being set by the CPU hardware before the NPF vmexit.
5340  	 * This is not expected behaviour and our tests fail because
5341  	 * of it.
5342  	 * A workaround here is to disable support for
5343  	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
5344  	 * In this case userspace can know if there is support using
5345  	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
5346  	 * it
5347  	 * If future AMD CPU models change the behaviour described above,
5348  	 * this variable can be changed accordingly
5349  	 */
5350  	allow_smaller_maxphyaddr = !npt_enabled;
5351  
5352  	return 0;
5353  
5354  err:
5355  	svm_hardware_unsetup();
5356  	return r;
5357  }
5358  
5359  
5360  static struct kvm_x86_init_ops svm_init_ops __initdata = {
5361  	.hardware_setup = svm_hardware_setup,
5362  
5363  	.runtime_ops = &svm_x86_ops,
5364  	.pmu_ops = &amd_pmu_ops,
5365  };
5366  
5367  static void __svm_exit(void)
5368  {
5369  	kvm_x86_vendor_exit();
5370  
5371  	cpu_emergency_unregister_virt_callback(svm_emergency_disable);
5372  }
5373  
5374  static int __init svm_init(void)
5375  {
5376  	int r;
5377  
5378  	__unused_size_checks();
5379  
5380  	if (!kvm_is_svm_supported())
5381  		return -EOPNOTSUPP;
5382  
5383  	r = kvm_x86_vendor_init(&svm_init_ops);
5384  	if (r)
5385  		return r;
5386  
5387  	cpu_emergency_register_virt_callback(svm_emergency_disable);
5388  
5389  	/*
5390  	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
5391  	 * exposed to userspace!
5392  	 */
5393  	r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm),
5394  		     THIS_MODULE);
5395  	if (r)
5396  		goto err_kvm_init;
5397  
5398  	return 0;
5399  
5400  err_kvm_init:
5401  	__svm_exit();
5402  	return r;
5403  }
5404  
5405  static void __exit svm_exit(void)
5406  {
5407  	kvm_exit();
5408  	__svm_exit();
5409  }
5410  
5411  module_init(svm_init)
5412  module_exit(svm_exit)
5413