xref: /openbmc/linux/arch/x86/kvm/svm/svm.c (revision 173940b3)
1 #define pr_fmt(fmt) "SVM: " fmt
2 
3 #include <linux/kvm_host.h>
4 
5 #include "irq.h"
6 #include "mmu.h"
7 #include "kvm_cache_regs.h"
8 #include "x86.h"
9 #include "cpuid.h"
10 #include "pmu.h"
11 
12 #include <linux/module.h>
13 #include <linux/mod_devicetable.h>
14 #include <linux/kernel.h>
15 #include <linux/vmalloc.h>
16 #include <linux/highmem.h>
17 #include <linux/amd-iommu.h>
18 #include <linux/sched.h>
19 #include <linux/trace_events.h>
20 #include <linux/slab.h>
21 #include <linux/hashtable.h>
22 #include <linux/objtool.h>
23 #include <linux/psp-sev.h>
24 #include <linux/file.h>
25 #include <linux/pagemap.h>
26 #include <linux/swap.h>
27 #include <linux/rwsem.h>
28 #include <linux/cc_platform.h>
29 
30 #include <asm/apic.h>
31 #include <asm/perf_event.h>
32 #include <asm/tlbflush.h>
33 #include <asm/desc.h>
34 #include <asm/debugreg.h>
35 #include <asm/kvm_para.h>
36 #include <asm/irq_remapping.h>
37 #include <asm/spec-ctrl.h>
38 #include <asm/cpu_device_id.h>
39 #include <asm/traps.h>
40 #include <asm/fpu/api.h>
41 
42 #include <asm/virtext.h>
43 #include "trace.h"
44 
45 #include "svm.h"
46 #include "svm_ops.h"
47 
48 #include "kvm_onhyperv.h"
49 #include "svm_onhyperv.h"
50 
51 MODULE_AUTHOR("Qumranet");
52 MODULE_LICENSE("GPL");
53 
54 #ifdef MODULE
55 static const struct x86_cpu_id svm_cpu_id[] = {
56 	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
57 	{}
58 };
59 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
60 #endif
61 
62 #define SEG_TYPE_LDT 2
63 #define SEG_TYPE_BUSY_TSS16 3
64 
65 static bool erratum_383_found __read_mostly;
66 
67 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
68 
69 /*
70  * Set osvw_len to higher value when updated Revision Guides
71  * are published and we know what the new status bits are
72  */
73 static uint64_t osvw_len = 4, osvw_status;
74 
75 static DEFINE_PER_CPU(u64, current_tsc_ratio);
76 
77 static const struct svm_direct_access_msrs {
78 	u32 index;   /* Index of the MSR */
79 	bool always; /* True if intercept is initially cleared */
80 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
81 	{ .index = MSR_STAR,				.always = true  },
82 	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
83 	{ .index = MSR_IA32_SYSENTER_EIP,		.always = false },
84 	{ .index = MSR_IA32_SYSENTER_ESP,		.always = false },
85 #ifdef CONFIG_X86_64
86 	{ .index = MSR_GS_BASE,				.always = true  },
87 	{ .index = MSR_FS_BASE,				.always = true  },
88 	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
89 	{ .index = MSR_LSTAR,				.always = true  },
90 	{ .index = MSR_CSTAR,				.always = true  },
91 	{ .index = MSR_SYSCALL_MASK,			.always = true  },
92 #endif
93 	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
94 	{ .index = MSR_IA32_PRED_CMD,			.always = false },
95 	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
96 	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
97 	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
98 	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
99 	{ .index = MSR_EFER,				.always = false },
100 	{ .index = MSR_IA32_CR_PAT,			.always = false },
101 	{ .index = MSR_AMD64_SEV_ES_GHCB,		.always = true  },
102 	{ .index = MSR_TSC_AUX,				.always = false },
103 	{ .index = MSR_INVALID,				.always = false },
104 };
105 
106 /*
107  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
108  * pause_filter_count: On processors that support Pause filtering(indicated
109  *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
110  *	count value. On VMRUN this value is loaded into an internal counter.
111  *	Each time a pause instruction is executed, this counter is decremented
112  *	until it reaches zero at which time a #VMEXIT is generated if pause
113  *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
114  *	Intercept Filtering for more details.
115  *	This also indicate if ple logic enabled.
116  *
117  * pause_filter_thresh: In addition, some processor families support advanced
118  *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
119  *	the amount of time a guest is allowed to execute in a pause loop.
120  *	In this mode, a 16-bit pause filter threshold field is added in the
121  *	VMCB. The threshold value is a cycle count that is used to reset the
122  *	pause counter. As with simple pause filtering, VMRUN loads the pause
123  *	count value from VMCB into an internal counter. Then, on each pause
124  *	instruction the hardware checks the elapsed number of cycles since
125  *	the most recent pause instruction against the pause filter threshold.
126  *	If the elapsed cycle count is greater than the pause filter threshold,
127  *	then the internal pause count is reloaded from the VMCB and execution
128  *	continues. If the elapsed cycle count is less than the pause filter
129  *	threshold, then the internal pause count is decremented. If the count
130  *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
131  *	triggered. If advanced pause filtering is supported and pause filter
132  *	threshold field is set to zero, the filter will operate in the simpler,
133  *	count only mode.
134  */
135 
136 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
137 module_param(pause_filter_thresh, ushort, 0444);
138 
139 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
140 module_param(pause_filter_count, ushort, 0444);
141 
142 /* Default doubles per-vcpu window every exit. */
143 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
144 module_param(pause_filter_count_grow, ushort, 0444);
145 
146 /* Default resets per-vcpu window every exit to pause_filter_count. */
147 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
148 module_param(pause_filter_count_shrink, ushort, 0444);
149 
150 /* Default is to compute the maximum so we can never overflow. */
151 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
152 module_param(pause_filter_count_max, ushort, 0444);
153 
154 /*
155  * Use nested page tables by default.  Note, NPT may get forced off by
156  * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
157  */
158 bool npt_enabled = true;
159 module_param_named(npt, npt_enabled, bool, 0444);
160 
161 /* allow nested virtualization in KVM/SVM */
162 static int nested = true;
163 module_param(nested, int, S_IRUGO);
164 
165 /* enable/disable Next RIP Save */
166 static int nrips = true;
167 module_param(nrips, int, 0444);
168 
169 /* enable/disable Virtual VMLOAD VMSAVE */
170 static int vls = true;
171 module_param(vls, int, 0444);
172 
173 /* enable/disable Virtual GIF */
174 int vgif = true;
175 module_param(vgif, int, 0444);
176 
177 /* enable/disable LBR virtualization */
178 static int lbrv = true;
179 module_param(lbrv, int, 0444);
180 
181 static int tsc_scaling = true;
182 module_param(tsc_scaling, int, 0444);
183 
184 /*
185  * enable / disable AVIC.  Because the defaults differ for APICv
186  * support between VMX and SVM we cannot use module_param_named.
187  */
188 static bool avic;
189 module_param(avic, bool, 0444);
190 
191 static bool force_avic;
192 module_param_unsafe(force_avic, bool, 0444);
193 
194 bool __read_mostly dump_invalid_vmcb;
195 module_param(dump_invalid_vmcb, bool, 0644);
196 
197 
198 bool intercept_smi = true;
199 module_param(intercept_smi, bool, 0444);
200 
201 
202 static bool svm_gp_erratum_intercept = true;
203 
204 static u8 rsm_ins_bytes[] = "\x0f\xaa";
205 
206 static unsigned long iopm_base;
207 
208 struct kvm_ldttss_desc {
209 	u16 limit0;
210 	u16 base0;
211 	unsigned base1:8, type:5, dpl:2, p:1;
212 	unsigned limit1:4, zero0:3, g:1, base2:8;
213 	u32 base3;
214 	u32 zero1;
215 } __attribute__((packed));
216 
217 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
218 
219 /*
220  * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
221  * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
222  *
223  * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
224  * defer the restoration of TSC_AUX until the CPU returns to userspace.
225  */
226 static int tsc_aux_uret_slot __read_mostly = -1;
227 
228 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
229 
230 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
231 #define MSRS_RANGE_SIZE 2048
232 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
233 
234 u32 svm_msrpm_offset(u32 msr)
235 {
236 	u32 offset;
237 	int i;
238 
239 	for (i = 0; i < NUM_MSR_MAPS; i++) {
240 		if (msr < msrpm_ranges[i] ||
241 		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
242 			continue;
243 
244 		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
245 		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
246 
247 		/* Now we have the u8 offset - but need the u32 offset */
248 		return offset / 4;
249 	}
250 
251 	/* MSR not in any range */
252 	return MSR_INVALID;
253 }
254 
255 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu);
256 
257 static int get_npt_level(void)
258 {
259 #ifdef CONFIG_X86_64
260 	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
261 #else
262 	return PT32E_ROOT_LEVEL;
263 #endif
264 }
265 
266 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
267 {
268 	struct vcpu_svm *svm = to_svm(vcpu);
269 	u64 old_efer = vcpu->arch.efer;
270 	vcpu->arch.efer = efer;
271 
272 	if (!npt_enabled) {
273 		/* Shadow paging assumes NX to be available.  */
274 		efer |= EFER_NX;
275 
276 		if (!(efer & EFER_LMA))
277 			efer &= ~EFER_LME;
278 	}
279 
280 	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
281 		if (!(efer & EFER_SVME)) {
282 			svm_leave_nested(vcpu);
283 			svm_set_gif(svm, true);
284 			/* #GP intercept is still needed for vmware backdoor */
285 			if (!enable_vmware_backdoor)
286 				clr_exception_intercept(svm, GP_VECTOR);
287 
288 			/*
289 			 * Free the nested guest state, unless we are in SMM.
290 			 * In this case we will return to the nested guest
291 			 * as soon as we leave SMM.
292 			 */
293 			if (!is_smm(vcpu))
294 				svm_free_nested(svm);
295 
296 		} else {
297 			int ret = svm_allocate_nested(svm);
298 
299 			if (ret) {
300 				vcpu->arch.efer = old_efer;
301 				return ret;
302 			}
303 
304 			/*
305 			 * Never intercept #GP for SEV guests, KVM can't
306 			 * decrypt guest memory to workaround the erratum.
307 			 */
308 			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
309 				set_exception_intercept(svm, GP_VECTOR);
310 		}
311 	}
312 
313 	svm->vmcb->save.efer = efer | EFER_SVME;
314 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
315 	return 0;
316 }
317 
318 static int is_external_interrupt(u32 info)
319 {
320 	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
321 	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
322 }
323 
324 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
325 {
326 	struct vcpu_svm *svm = to_svm(vcpu);
327 	u32 ret = 0;
328 
329 	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
330 		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
331 	return ret;
332 }
333 
334 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
335 {
336 	struct vcpu_svm *svm = to_svm(vcpu);
337 
338 	if (mask == 0)
339 		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
340 	else
341 		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
342 
343 }
344 
345 static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
346 {
347 	struct vcpu_svm *svm = to_svm(vcpu);
348 
349 	/*
350 	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
351 	 * the type of exit and the #VC handler in the guest.
352 	 */
353 	if (sev_es_guest(vcpu->kvm))
354 		goto done;
355 
356 	if (nrips && svm->vmcb->control.next_rip != 0) {
357 		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
358 		svm->next_rip = svm->vmcb->control.next_rip;
359 	}
360 
361 	if (!svm->next_rip) {
362 		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
363 			return 0;
364 	} else {
365 		kvm_rip_write(vcpu, svm->next_rip);
366 	}
367 
368 done:
369 	svm_set_interrupt_shadow(vcpu, 0);
370 
371 	return 1;
372 }
373 
374 static void svm_queue_exception(struct kvm_vcpu *vcpu)
375 {
376 	struct vcpu_svm *svm = to_svm(vcpu);
377 	unsigned nr = vcpu->arch.exception.nr;
378 	bool has_error_code = vcpu->arch.exception.has_error_code;
379 	u32 error_code = vcpu->arch.exception.error_code;
380 
381 	kvm_deliver_exception_payload(vcpu);
382 
383 	if (nr == BP_VECTOR && !nrips) {
384 		unsigned long rip, old_rip = kvm_rip_read(vcpu);
385 
386 		/*
387 		 * For guest debugging where we have to reinject #BP if some
388 		 * INT3 is guest-owned:
389 		 * Emulate nRIP by moving RIP forward. Will fail if injection
390 		 * raises a fault that is not intercepted. Still better than
391 		 * failing in all cases.
392 		 */
393 		(void)svm_skip_emulated_instruction(vcpu);
394 		rip = kvm_rip_read(vcpu);
395 		svm->int3_rip = rip + svm->vmcb->save.cs.base;
396 		svm->int3_injected = rip - old_rip;
397 	}
398 
399 	svm->vmcb->control.event_inj = nr
400 		| SVM_EVTINJ_VALID
401 		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
402 		| SVM_EVTINJ_TYPE_EXEPT;
403 	svm->vmcb->control.event_inj_err = error_code;
404 }
405 
406 static void svm_init_erratum_383(void)
407 {
408 	u32 low, high;
409 	int err;
410 	u64 val;
411 
412 	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
413 		return;
414 
415 	/* Use _safe variants to not break nested virtualization */
416 	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
417 	if (err)
418 		return;
419 
420 	val |= (1ULL << 47);
421 
422 	low  = lower_32_bits(val);
423 	high = upper_32_bits(val);
424 
425 	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
426 
427 	erratum_383_found = true;
428 }
429 
430 static void svm_init_osvw(struct kvm_vcpu *vcpu)
431 {
432 	/*
433 	 * Guests should see errata 400 and 415 as fixed (assuming that
434 	 * HLT and IO instructions are intercepted).
435 	 */
436 	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
437 	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
438 
439 	/*
440 	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
441 	 * all osvw.status bits inside that length, including bit 0 (which is
442 	 * reserved for erratum 298), are valid. However, if host processor's
443 	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
444 	 * be conservative here and therefore we tell the guest that erratum 298
445 	 * is present (because we really don't know).
446 	 */
447 	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
448 		vcpu->arch.osvw.status |= 1;
449 }
450 
451 static int has_svm(void)
452 {
453 	const char *msg;
454 
455 	if (!cpu_has_svm(&msg)) {
456 		printk(KERN_INFO "has_svm: %s\n", msg);
457 		return 0;
458 	}
459 
460 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
461 		pr_info("KVM is unsupported when running as an SEV guest\n");
462 		return 0;
463 	}
464 
465 	return 1;
466 }
467 
468 void __svm_write_tsc_multiplier(u64 multiplier)
469 {
470 	preempt_disable();
471 
472 	if (multiplier == __this_cpu_read(current_tsc_ratio))
473 		goto out;
474 
475 	wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
476 	__this_cpu_write(current_tsc_ratio, multiplier);
477 out:
478 	preempt_enable();
479 }
480 
481 static void svm_hardware_disable(void)
482 {
483 	/* Make sure we clean up behind us */
484 	if (tsc_scaling)
485 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
486 
487 	cpu_svm_disable();
488 
489 	amd_pmu_disable_virt();
490 }
491 
492 static int svm_hardware_enable(void)
493 {
494 
495 	struct svm_cpu_data *sd;
496 	uint64_t efer;
497 	struct desc_struct *gdt;
498 	int me = raw_smp_processor_id();
499 
500 	rdmsrl(MSR_EFER, efer);
501 	if (efer & EFER_SVME)
502 		return -EBUSY;
503 
504 	if (!has_svm()) {
505 		pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
506 		return -EINVAL;
507 	}
508 	sd = per_cpu(svm_data, me);
509 	if (!sd) {
510 		pr_err("%s: svm_data is NULL on %d\n", __func__, me);
511 		return -EINVAL;
512 	}
513 
514 	sd->asid_generation = 1;
515 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
516 	sd->next_asid = sd->max_asid + 1;
517 	sd->min_asid = max_sev_asid + 1;
518 
519 	gdt = get_current_gdt_rw();
520 	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
521 
522 	wrmsrl(MSR_EFER, efer | EFER_SVME);
523 
524 	wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area));
525 
526 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
527 		/*
528 		 * Set the default value, even if we don't use TSC scaling
529 		 * to avoid having stale value in the msr
530 		 */
531 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
532 	}
533 
534 
535 	/*
536 	 * Get OSVW bits.
537 	 *
538 	 * Note that it is possible to have a system with mixed processor
539 	 * revisions and therefore different OSVW bits. If bits are not the same
540 	 * on different processors then choose the worst case (i.e. if erratum
541 	 * is present on one processor and not on another then assume that the
542 	 * erratum is present everywhere).
543 	 */
544 	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
545 		uint64_t len, status = 0;
546 		int err;
547 
548 		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
549 		if (!err)
550 			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
551 						      &err);
552 
553 		if (err)
554 			osvw_status = osvw_len = 0;
555 		else {
556 			if (len < osvw_len)
557 				osvw_len = len;
558 			osvw_status |= status;
559 			osvw_status &= (1ULL << osvw_len) - 1;
560 		}
561 	} else
562 		osvw_status = osvw_len = 0;
563 
564 	svm_init_erratum_383();
565 
566 	amd_pmu_enable_virt();
567 
568 	return 0;
569 }
570 
571 static void svm_cpu_uninit(int cpu)
572 {
573 	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
574 
575 	if (!sd)
576 		return;
577 
578 	per_cpu(svm_data, cpu) = NULL;
579 	kfree(sd->sev_vmcbs);
580 	__free_page(sd->save_area);
581 	kfree(sd);
582 }
583 
584 static int svm_cpu_init(int cpu)
585 {
586 	struct svm_cpu_data *sd;
587 	int ret = -ENOMEM;
588 
589 	sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
590 	if (!sd)
591 		return ret;
592 	sd->cpu = cpu;
593 	sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO);
594 	if (!sd->save_area)
595 		goto free_cpu_data;
596 
597 	ret = sev_cpu_init(sd);
598 	if (ret)
599 		goto free_save_area;
600 
601 	per_cpu(svm_data, cpu) = sd;
602 
603 	return 0;
604 
605 free_save_area:
606 	__free_page(sd->save_area);
607 free_cpu_data:
608 	kfree(sd);
609 	return ret;
610 
611 }
612 
613 static int direct_access_msr_slot(u32 msr)
614 {
615 	u32 i;
616 
617 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
618 		if (direct_access_msrs[i].index == msr)
619 			return i;
620 
621 	return -ENOENT;
622 }
623 
624 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
625 				     int write)
626 {
627 	struct vcpu_svm *svm = to_svm(vcpu);
628 	int slot = direct_access_msr_slot(msr);
629 
630 	if (slot == -ENOENT)
631 		return;
632 
633 	/* Set the shadow bitmaps to the desired intercept states */
634 	if (read)
635 		set_bit(slot, svm->shadow_msr_intercept.read);
636 	else
637 		clear_bit(slot, svm->shadow_msr_intercept.read);
638 
639 	if (write)
640 		set_bit(slot, svm->shadow_msr_intercept.write);
641 	else
642 		clear_bit(slot, svm->shadow_msr_intercept.write);
643 }
644 
645 static bool valid_msr_intercept(u32 index)
646 {
647 	return direct_access_msr_slot(index) != -ENOENT;
648 }
649 
650 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
651 {
652 	u8 bit_write;
653 	unsigned long tmp;
654 	u32 offset;
655 	u32 *msrpm;
656 
657 	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
658 				      to_svm(vcpu)->msrpm;
659 
660 	offset    = svm_msrpm_offset(msr);
661 	bit_write = 2 * (msr & 0x0f) + 1;
662 	tmp       = msrpm[offset];
663 
664 	BUG_ON(offset == MSR_INVALID);
665 
666 	return !!test_bit(bit_write,  &tmp);
667 }
668 
669 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
670 					u32 msr, int read, int write)
671 {
672 	struct vcpu_svm *svm = to_svm(vcpu);
673 	u8 bit_read, bit_write;
674 	unsigned long tmp;
675 	u32 offset;
676 
677 	/*
678 	 * If this warning triggers extend the direct_access_msrs list at the
679 	 * beginning of the file
680 	 */
681 	WARN_ON(!valid_msr_intercept(msr));
682 
683 	/* Enforce non allowed MSRs to trap */
684 	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
685 		read = 0;
686 
687 	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
688 		write = 0;
689 
690 	offset    = svm_msrpm_offset(msr);
691 	bit_read  = 2 * (msr & 0x0f);
692 	bit_write = 2 * (msr & 0x0f) + 1;
693 	tmp       = msrpm[offset];
694 
695 	BUG_ON(offset == MSR_INVALID);
696 
697 	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
698 	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
699 
700 	msrpm[offset] = tmp;
701 
702 	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
703 	svm->nested.force_msr_bitmap_recalc = true;
704 }
705 
706 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
707 			  int read, int write)
708 {
709 	set_shadow_msr_intercept(vcpu, msr, read, write);
710 	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
711 }
712 
713 u32 *svm_vcpu_alloc_msrpm(void)
714 {
715 	unsigned int order = get_order(MSRPM_SIZE);
716 	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
717 	u32 *msrpm;
718 
719 	if (!pages)
720 		return NULL;
721 
722 	msrpm = page_address(pages);
723 	memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
724 
725 	return msrpm;
726 }
727 
728 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
729 {
730 	int i;
731 
732 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
733 		if (!direct_access_msrs[i].always)
734 			continue;
735 		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
736 	}
737 }
738 
739 
740 void svm_vcpu_free_msrpm(u32 *msrpm)
741 {
742 	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
743 }
744 
745 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
746 {
747 	struct vcpu_svm *svm = to_svm(vcpu);
748 	u32 i;
749 
750 	/*
751 	 * Set intercept permissions for all direct access MSRs again. They
752 	 * will automatically get filtered through the MSR filter, so we are
753 	 * back in sync after this.
754 	 */
755 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
756 		u32 msr = direct_access_msrs[i].index;
757 		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
758 		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
759 
760 		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
761 	}
762 }
763 
764 static void add_msr_offset(u32 offset)
765 {
766 	int i;
767 
768 	for (i = 0; i < MSRPM_OFFSETS; ++i) {
769 
770 		/* Offset already in list? */
771 		if (msrpm_offsets[i] == offset)
772 			return;
773 
774 		/* Slot used by another offset? */
775 		if (msrpm_offsets[i] != MSR_INVALID)
776 			continue;
777 
778 		/* Add offset to list */
779 		msrpm_offsets[i] = offset;
780 
781 		return;
782 	}
783 
784 	/*
785 	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
786 	 * increase MSRPM_OFFSETS in this case.
787 	 */
788 	BUG();
789 }
790 
791 static void init_msrpm_offsets(void)
792 {
793 	int i;
794 
795 	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
796 
797 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
798 		u32 offset;
799 
800 		offset = svm_msrpm_offset(direct_access_msrs[i].index);
801 		BUG_ON(offset == MSR_INVALID);
802 
803 		add_msr_offset(offset);
804 	}
805 }
806 
807 void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
808 {
809 	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
810 	to_vmcb->save.br_from		= from_vmcb->save.br_from;
811 	to_vmcb->save.br_to		= from_vmcb->save.br_to;
812 	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
813 	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
814 
815 	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
816 }
817 
818 static void svm_enable_lbrv(struct kvm_vcpu *vcpu)
819 {
820 	struct vcpu_svm *svm = to_svm(vcpu);
821 
822 	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
823 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
824 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
825 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
826 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
827 
828 	/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
829 	if (is_guest_mode(vcpu))
830 		svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
831 }
832 
833 static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
834 {
835 	struct vcpu_svm *svm = to_svm(vcpu);
836 
837 	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
838 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
839 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
840 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
841 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
842 
843 	/*
844 	 * Move the LBR msrs back to the vmcb01 to avoid copying them
845 	 * on nested guest entries.
846 	 */
847 	if (is_guest_mode(vcpu))
848 		svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb);
849 }
850 
851 static int svm_get_lbr_msr(struct vcpu_svm *svm, u32 index)
852 {
853 	/*
854 	 * If the LBR virtualization is disabled, the LBR msrs are always
855 	 * kept in the vmcb01 to avoid copying them on nested guest entries.
856 	 *
857 	 * If nested, and the LBR virtualization is enabled/disabled, the msrs
858 	 * are moved between the vmcb01 and vmcb02 as needed.
859 	 */
860 	struct vmcb *vmcb =
861 		(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) ?
862 			svm->vmcb : svm->vmcb01.ptr;
863 
864 	switch (index) {
865 	case MSR_IA32_DEBUGCTLMSR:
866 		return vmcb->save.dbgctl;
867 	case MSR_IA32_LASTBRANCHFROMIP:
868 		return vmcb->save.br_from;
869 	case MSR_IA32_LASTBRANCHTOIP:
870 		return vmcb->save.br_to;
871 	case MSR_IA32_LASTINTFROMIP:
872 		return vmcb->save.last_excp_from;
873 	case MSR_IA32_LASTINTTOIP:
874 		return vmcb->save.last_excp_to;
875 	default:
876 		KVM_BUG(false, svm->vcpu.kvm,
877 			"%s: Unknown MSR 0x%x", __func__, index);
878 		return 0;
879 	}
880 }
881 
882 void svm_update_lbrv(struct kvm_vcpu *vcpu)
883 {
884 	struct vcpu_svm *svm = to_svm(vcpu);
885 
886 	bool enable_lbrv = svm_get_lbr_msr(svm, MSR_IA32_DEBUGCTLMSR) &
887 					   DEBUGCTLMSR_LBR;
888 
889 	bool current_enable_lbrv = !!(svm->vmcb->control.virt_ext &
890 				      LBR_CTL_ENABLE_MASK);
891 
892 	if (unlikely(is_guest_mode(vcpu) && svm->lbrv_enabled))
893 		if (unlikely(svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))
894 			enable_lbrv = true;
895 
896 	if (enable_lbrv == current_enable_lbrv)
897 		return;
898 
899 	if (enable_lbrv)
900 		svm_enable_lbrv(vcpu);
901 	else
902 		svm_disable_lbrv(vcpu);
903 }
904 
905 void disable_nmi_singlestep(struct vcpu_svm *svm)
906 {
907 	svm->nmi_singlestep = false;
908 
909 	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
910 		/* Clear our flags if they were not set by the guest */
911 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
912 			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
913 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
914 			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
915 	}
916 }
917 
918 static void grow_ple_window(struct kvm_vcpu *vcpu)
919 {
920 	struct vcpu_svm *svm = to_svm(vcpu);
921 	struct vmcb_control_area *control = &svm->vmcb->control;
922 	int old = control->pause_filter_count;
923 
924 	if (kvm_pause_in_guest(vcpu->kvm) || !old)
925 		return;
926 
927 	control->pause_filter_count = __grow_ple_window(old,
928 							pause_filter_count,
929 							pause_filter_count_grow,
930 							pause_filter_count_max);
931 
932 	if (control->pause_filter_count != old) {
933 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
934 		trace_kvm_ple_window_update(vcpu->vcpu_id,
935 					    control->pause_filter_count, old);
936 	}
937 }
938 
939 static void shrink_ple_window(struct kvm_vcpu *vcpu)
940 {
941 	struct vcpu_svm *svm = to_svm(vcpu);
942 	struct vmcb_control_area *control = &svm->vmcb->control;
943 	int old = control->pause_filter_count;
944 
945 	if (kvm_pause_in_guest(vcpu->kvm) || !old)
946 		return;
947 
948 	control->pause_filter_count =
949 				__shrink_ple_window(old,
950 						    pause_filter_count,
951 						    pause_filter_count_shrink,
952 						    pause_filter_count);
953 	if (control->pause_filter_count != old) {
954 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
955 		trace_kvm_ple_window_update(vcpu->vcpu_id,
956 					    control->pause_filter_count, old);
957 	}
958 }
959 
960 static void svm_hardware_unsetup(void)
961 {
962 	int cpu;
963 
964 	sev_hardware_unsetup();
965 
966 	for_each_possible_cpu(cpu)
967 		svm_cpu_uninit(cpu);
968 
969 	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT),
970 	get_order(IOPM_SIZE));
971 	iopm_base = 0;
972 }
973 
974 static void init_seg(struct vmcb_seg *seg)
975 {
976 	seg->selector = 0;
977 	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
978 		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
979 	seg->limit = 0xffff;
980 	seg->base = 0;
981 }
982 
983 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
984 {
985 	seg->selector = 0;
986 	seg->attrib = SVM_SELECTOR_P_MASK | type;
987 	seg->limit = 0xffff;
988 	seg->base = 0;
989 }
990 
991 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
992 {
993 	struct vcpu_svm *svm = to_svm(vcpu);
994 
995 	return svm->nested.ctl.tsc_offset;
996 }
997 
998 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
999 {
1000 	struct vcpu_svm *svm = to_svm(vcpu);
1001 
1002 	return svm->tsc_ratio_msr;
1003 }
1004 
1005 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1006 {
1007 	struct vcpu_svm *svm = to_svm(vcpu);
1008 
1009 	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1010 	svm->vmcb->control.tsc_offset = offset;
1011 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1012 }
1013 
1014 static void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier)
1015 {
1016 	__svm_write_tsc_multiplier(multiplier);
1017 }
1018 
1019 
1020 /* Evaluate instruction intercepts that depend on guest CPUID features. */
1021 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1022 					      struct vcpu_svm *svm)
1023 {
1024 	/*
1025 	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1026 	 * roots, or if INVPCID is disabled in the guest to inject #UD.
1027 	 */
1028 	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1029 		if (!npt_enabled ||
1030 		    !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1031 			svm_set_intercept(svm, INTERCEPT_INVPCID);
1032 		else
1033 			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1034 	}
1035 
1036 	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1037 		if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1038 			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1039 		else
1040 			svm_set_intercept(svm, INTERCEPT_RDTSCP);
1041 	}
1042 }
1043 
1044 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1045 {
1046 	struct vcpu_svm *svm = to_svm(vcpu);
1047 
1048 	if (guest_cpuid_is_intel(vcpu)) {
1049 		/*
1050 		 * We must intercept SYSENTER_EIP and SYSENTER_ESP
1051 		 * accesses because the processor only stores 32 bits.
1052 		 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1053 		 */
1054 		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1055 		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1056 		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1057 
1058 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1059 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1060 
1061 		svm->v_vmload_vmsave_enabled = false;
1062 	} else {
1063 		/*
1064 		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1065 		 * in VMCB and clear intercepts to avoid #VMEXIT.
1066 		 */
1067 		if (vls) {
1068 			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1069 			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1070 			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1071 		}
1072 		/* No need to intercept these MSRs */
1073 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1074 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1075 	}
1076 }
1077 
1078 static void init_vmcb(struct kvm_vcpu *vcpu)
1079 {
1080 	struct vcpu_svm *svm = to_svm(vcpu);
1081 	struct vmcb *vmcb = svm->vmcb01.ptr;
1082 	struct vmcb_control_area *control = &vmcb->control;
1083 	struct vmcb_save_area *save = &vmcb->save;
1084 
1085 	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1086 	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1087 	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1088 	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1089 	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1090 	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1091 	if (!kvm_vcpu_apicv_active(vcpu))
1092 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1093 
1094 	set_dr_intercepts(svm);
1095 
1096 	set_exception_intercept(svm, PF_VECTOR);
1097 	set_exception_intercept(svm, UD_VECTOR);
1098 	set_exception_intercept(svm, MC_VECTOR);
1099 	set_exception_intercept(svm, AC_VECTOR);
1100 	set_exception_intercept(svm, DB_VECTOR);
1101 	/*
1102 	 * Guest access to VMware backdoor ports could legitimately
1103 	 * trigger #GP because of TSS I/O permission bitmap.
1104 	 * We intercept those #GP and allow access to them anyway
1105 	 * as VMware does.  Don't intercept #GP for SEV guests as KVM can't
1106 	 * decrypt guest memory to decode the faulting instruction.
1107 	 */
1108 	if (enable_vmware_backdoor && !sev_guest(vcpu->kvm))
1109 		set_exception_intercept(svm, GP_VECTOR);
1110 
1111 	svm_set_intercept(svm, INTERCEPT_INTR);
1112 	svm_set_intercept(svm, INTERCEPT_NMI);
1113 
1114 	if (intercept_smi)
1115 		svm_set_intercept(svm, INTERCEPT_SMI);
1116 
1117 	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1118 	svm_set_intercept(svm, INTERCEPT_RDPMC);
1119 	svm_set_intercept(svm, INTERCEPT_CPUID);
1120 	svm_set_intercept(svm, INTERCEPT_INVD);
1121 	svm_set_intercept(svm, INTERCEPT_INVLPG);
1122 	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1123 	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1124 	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1125 	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1126 	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1127 	svm_set_intercept(svm, INTERCEPT_VMRUN);
1128 	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1129 	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1130 	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1131 	svm_set_intercept(svm, INTERCEPT_STGI);
1132 	svm_set_intercept(svm, INTERCEPT_CLGI);
1133 	svm_set_intercept(svm, INTERCEPT_SKINIT);
1134 	svm_set_intercept(svm, INTERCEPT_WBINVD);
1135 	svm_set_intercept(svm, INTERCEPT_XSETBV);
1136 	svm_set_intercept(svm, INTERCEPT_RDPRU);
1137 	svm_set_intercept(svm, INTERCEPT_RSM);
1138 
1139 	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1140 		svm_set_intercept(svm, INTERCEPT_MONITOR);
1141 		svm_set_intercept(svm, INTERCEPT_MWAIT);
1142 	}
1143 
1144 	if (!kvm_hlt_in_guest(vcpu->kvm))
1145 		svm_set_intercept(svm, INTERCEPT_HLT);
1146 
1147 	control->iopm_base_pa = __sme_set(iopm_base);
1148 	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1149 	control->int_ctl = V_INTR_MASKING_MASK;
1150 
1151 	init_seg(&save->es);
1152 	init_seg(&save->ss);
1153 	init_seg(&save->ds);
1154 	init_seg(&save->fs);
1155 	init_seg(&save->gs);
1156 
1157 	save->cs.selector = 0xf000;
1158 	save->cs.base = 0xffff0000;
1159 	/* Executable/Readable Code Segment */
1160 	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1161 		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1162 	save->cs.limit = 0xffff;
1163 
1164 	save->gdtr.base = 0;
1165 	save->gdtr.limit = 0xffff;
1166 	save->idtr.base = 0;
1167 	save->idtr.limit = 0xffff;
1168 
1169 	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1170 	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1171 
1172 	if (npt_enabled) {
1173 		/* Setup VMCB for Nested Paging */
1174 		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1175 		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1176 		clr_exception_intercept(svm, PF_VECTOR);
1177 		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1178 		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1179 		save->g_pat = vcpu->arch.pat;
1180 		save->cr3 = 0;
1181 	}
1182 	svm->current_vmcb->asid_generation = 0;
1183 	svm->asid = 0;
1184 
1185 	svm->nested.vmcb12_gpa = INVALID_GPA;
1186 	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1187 
1188 	if (!kvm_pause_in_guest(vcpu->kvm)) {
1189 		control->pause_filter_count = pause_filter_count;
1190 		if (pause_filter_thresh)
1191 			control->pause_filter_thresh = pause_filter_thresh;
1192 		svm_set_intercept(svm, INTERCEPT_PAUSE);
1193 	} else {
1194 		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1195 	}
1196 
1197 	svm_recalc_instruction_intercepts(vcpu, svm);
1198 
1199 	/*
1200 	 * If the host supports V_SPEC_CTRL then disable the interception
1201 	 * of MSR_IA32_SPEC_CTRL.
1202 	 */
1203 	if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1204 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1205 
1206 	if (kvm_vcpu_apicv_active(vcpu))
1207 		avic_init_vmcb(svm, vmcb);
1208 
1209 	if (vgif) {
1210 		svm_clr_intercept(svm, INTERCEPT_STGI);
1211 		svm_clr_intercept(svm, INTERCEPT_CLGI);
1212 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1213 	}
1214 
1215 	if (sev_guest(vcpu->kvm)) {
1216 		svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1217 		clr_exception_intercept(svm, UD_VECTOR);
1218 
1219 		if (sev_es_guest(vcpu->kvm)) {
1220 			/* Perform SEV-ES specific VMCB updates */
1221 			sev_es_init_vmcb(svm);
1222 		}
1223 	}
1224 
1225 	svm_hv_init_vmcb(vmcb);
1226 	init_vmcb_after_set_cpuid(vcpu);
1227 
1228 	vmcb_mark_all_dirty(vmcb);
1229 
1230 	enable_gif(svm);
1231 }
1232 
1233 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1234 {
1235 	struct vcpu_svm *svm = to_svm(vcpu);
1236 
1237 	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1238 
1239 	svm_init_osvw(vcpu);
1240 	vcpu->arch.microcode_version = 0x01000065;
1241 	svm->tsc_ratio_msr = kvm_default_tsc_scaling_ratio;
1242 
1243 	if (sev_es_guest(vcpu->kvm))
1244 		sev_es_vcpu_reset(svm);
1245 }
1246 
1247 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1248 {
1249 	struct vcpu_svm *svm = to_svm(vcpu);
1250 
1251 	svm->spec_ctrl = 0;
1252 	svm->virt_spec_ctrl = 0;
1253 
1254 	init_vmcb(vcpu);
1255 
1256 	if (!init_event)
1257 		__svm_vcpu_reset(vcpu);
1258 }
1259 
1260 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1261 {
1262 	svm->current_vmcb = target_vmcb;
1263 	svm->vmcb = target_vmcb->ptr;
1264 }
1265 
1266 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1267 {
1268 	struct vcpu_svm *svm;
1269 	struct page *vmcb01_page;
1270 	struct page *vmsa_page = NULL;
1271 	int err;
1272 
1273 	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1274 	svm = to_svm(vcpu);
1275 
1276 	err = -ENOMEM;
1277 	vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1278 	if (!vmcb01_page)
1279 		goto out;
1280 
1281 	if (sev_es_guest(vcpu->kvm)) {
1282 		/*
1283 		 * SEV-ES guests require a separate VMSA page used to contain
1284 		 * the encrypted register state of the guest.
1285 		 */
1286 		vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1287 		if (!vmsa_page)
1288 			goto error_free_vmcb_page;
1289 
1290 		/*
1291 		 * SEV-ES guests maintain an encrypted version of their FPU
1292 		 * state which is restored and saved on VMRUN and VMEXIT.
1293 		 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't
1294 		 * do xsave/xrstor on it.
1295 		 */
1296 		fpstate_set_confidential(&vcpu->arch.guest_fpu);
1297 	}
1298 
1299 	err = avic_init_vcpu(svm);
1300 	if (err)
1301 		goto error_free_vmsa_page;
1302 
1303 	svm->msrpm = svm_vcpu_alloc_msrpm();
1304 	if (!svm->msrpm) {
1305 		err = -ENOMEM;
1306 		goto error_free_vmsa_page;
1307 	}
1308 
1309 	svm->vmcb01.ptr = page_address(vmcb01_page);
1310 	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1311 	svm_switch_vmcb(svm, &svm->vmcb01);
1312 
1313 	if (vmsa_page)
1314 		svm->sev_es.vmsa = page_address(vmsa_page);
1315 
1316 	svm->guest_state_loaded = false;
1317 
1318 	return 0;
1319 
1320 error_free_vmsa_page:
1321 	if (vmsa_page)
1322 		__free_page(vmsa_page);
1323 error_free_vmcb_page:
1324 	__free_page(vmcb01_page);
1325 out:
1326 	return err;
1327 }
1328 
1329 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1330 {
1331 	int i;
1332 
1333 	for_each_online_cpu(i)
1334 		cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
1335 }
1336 
1337 static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1338 {
1339 	struct vcpu_svm *svm = to_svm(vcpu);
1340 
1341 	/*
1342 	 * The vmcb page can be recycled, causing a false negative in
1343 	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1344 	 * vmcb page recorded as its current vmcb.
1345 	 */
1346 	svm_clear_current_vmcb(svm->vmcb);
1347 
1348 	svm_free_nested(svm);
1349 
1350 	sev_free_vcpu(vcpu);
1351 
1352 	__free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT));
1353 	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1354 }
1355 
1356 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1357 {
1358 	struct vcpu_svm *svm = to_svm(vcpu);
1359 	struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
1360 
1361 	if (sev_es_guest(vcpu->kvm))
1362 		sev_es_unmap_ghcb(svm);
1363 
1364 	if (svm->guest_state_loaded)
1365 		return;
1366 
1367 	/*
1368 	 * Save additional host state that will be restored on VMEXIT (sev-es)
1369 	 * or subsequent vmload of host save area.
1370 	 */
1371 	vmsave(__sme_page_pa(sd->save_area));
1372 	if (sev_es_guest(vcpu->kvm)) {
1373 		struct sev_es_save_area *hostsa;
1374 		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
1375 
1376 		sev_es_prepare_switch_to_guest(hostsa);
1377 	}
1378 
1379 	if (tsc_scaling)
1380 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1381 
1382 	if (likely(tsc_aux_uret_slot >= 0))
1383 		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1384 
1385 	svm->guest_state_loaded = true;
1386 }
1387 
1388 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1389 {
1390 	to_svm(vcpu)->guest_state_loaded = false;
1391 }
1392 
1393 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1394 {
1395 	struct vcpu_svm *svm = to_svm(vcpu);
1396 	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1397 
1398 	if (sd->current_vmcb != svm->vmcb) {
1399 		sd->current_vmcb = svm->vmcb;
1400 		indirect_branch_prediction_barrier();
1401 	}
1402 	if (kvm_vcpu_apicv_active(vcpu))
1403 		__avic_vcpu_load(vcpu, cpu);
1404 }
1405 
1406 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1407 {
1408 	if (kvm_vcpu_apicv_active(vcpu))
1409 		__avic_vcpu_put(vcpu);
1410 
1411 	svm_prepare_host_switch(vcpu);
1412 
1413 	++vcpu->stat.host_state_reload;
1414 }
1415 
1416 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1417 {
1418 	struct vcpu_svm *svm = to_svm(vcpu);
1419 	unsigned long rflags = svm->vmcb->save.rflags;
1420 
1421 	if (svm->nmi_singlestep) {
1422 		/* Hide our flags if they were not set by the guest */
1423 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1424 			rflags &= ~X86_EFLAGS_TF;
1425 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1426 			rflags &= ~X86_EFLAGS_RF;
1427 	}
1428 	return rflags;
1429 }
1430 
1431 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1432 {
1433 	if (to_svm(vcpu)->nmi_singlestep)
1434 		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1435 
1436        /*
1437         * Any change of EFLAGS.VM is accompanied by a reload of SS
1438         * (caused by either a task switch or an inter-privilege IRET),
1439         * so we do not need to update the CPL here.
1440         */
1441 	to_svm(vcpu)->vmcb->save.rflags = rflags;
1442 }
1443 
1444 static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1445 {
1446 	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1447 
1448 	return sev_es_guest(vcpu->kvm)
1449 		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1450 		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1451 }
1452 
1453 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1454 {
1455 	kvm_register_mark_available(vcpu, reg);
1456 
1457 	switch (reg) {
1458 	case VCPU_EXREG_PDPTR:
1459 		/*
1460 		 * When !npt_enabled, mmu->pdptrs[] is already available since
1461 		 * it is always updated per SDM when moving to CRs.
1462 		 */
1463 		if (npt_enabled)
1464 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1465 		break;
1466 	default:
1467 		KVM_BUG_ON(1, vcpu->kvm);
1468 	}
1469 }
1470 
1471 static void svm_set_vintr(struct vcpu_svm *svm)
1472 {
1473 	struct vmcb_control_area *control;
1474 
1475 	/*
1476 	 * The following fields are ignored when AVIC is enabled
1477 	 */
1478 	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1479 
1480 	svm_set_intercept(svm, INTERCEPT_VINTR);
1481 
1482 	/*
1483 	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1484 	 * Actual injection of virtual interrupts happens through EVENTINJ.
1485 	 */
1486 	control = &svm->vmcb->control;
1487 	control->int_vector = 0x0;
1488 	control->int_ctl &= ~V_INTR_PRIO_MASK;
1489 	control->int_ctl |= V_IRQ_MASK |
1490 		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1491 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1492 }
1493 
1494 static void svm_clear_vintr(struct vcpu_svm *svm)
1495 {
1496 	svm_clr_intercept(svm, INTERCEPT_VINTR);
1497 
1498 	/* Drop int_ctl fields related to VINTR injection.  */
1499 	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1500 	if (is_guest_mode(&svm->vcpu)) {
1501 		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1502 
1503 		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1504 			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1505 
1506 		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1507 			V_IRQ_INJECTION_BITS_MASK;
1508 
1509 		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1510 	}
1511 
1512 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1513 }
1514 
1515 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1516 {
1517 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1518 	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1519 
1520 	switch (seg) {
1521 	case VCPU_SREG_CS: return &save->cs;
1522 	case VCPU_SREG_DS: return &save->ds;
1523 	case VCPU_SREG_ES: return &save->es;
1524 	case VCPU_SREG_FS: return &save01->fs;
1525 	case VCPU_SREG_GS: return &save01->gs;
1526 	case VCPU_SREG_SS: return &save->ss;
1527 	case VCPU_SREG_TR: return &save01->tr;
1528 	case VCPU_SREG_LDTR: return &save01->ldtr;
1529 	}
1530 	BUG();
1531 	return NULL;
1532 }
1533 
1534 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1535 {
1536 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1537 
1538 	return s->base;
1539 }
1540 
1541 static void svm_get_segment(struct kvm_vcpu *vcpu,
1542 			    struct kvm_segment *var, int seg)
1543 {
1544 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1545 
1546 	var->base = s->base;
1547 	var->limit = s->limit;
1548 	var->selector = s->selector;
1549 	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1550 	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1551 	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1552 	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1553 	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1554 	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1555 	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1556 
1557 	/*
1558 	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1559 	 * However, the SVM spec states that the G bit is not observed by the
1560 	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1561 	 * So let's synthesize a legal G bit for all segments, this helps
1562 	 * running KVM nested. It also helps cross-vendor migration, because
1563 	 * Intel's vmentry has a check on the 'G' bit.
1564 	 */
1565 	var->g = s->limit > 0xfffff;
1566 
1567 	/*
1568 	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1569 	 * for cross vendor migration purposes by "not present"
1570 	 */
1571 	var->unusable = !var->present;
1572 
1573 	switch (seg) {
1574 	case VCPU_SREG_TR:
1575 		/*
1576 		 * Work around a bug where the busy flag in the tr selector
1577 		 * isn't exposed
1578 		 */
1579 		var->type |= 0x2;
1580 		break;
1581 	case VCPU_SREG_DS:
1582 	case VCPU_SREG_ES:
1583 	case VCPU_SREG_FS:
1584 	case VCPU_SREG_GS:
1585 		/*
1586 		 * The accessed bit must always be set in the segment
1587 		 * descriptor cache, although it can be cleared in the
1588 		 * descriptor, the cached bit always remains at 1. Since
1589 		 * Intel has a check on this, set it here to support
1590 		 * cross-vendor migration.
1591 		 */
1592 		if (!var->unusable)
1593 			var->type |= 0x1;
1594 		break;
1595 	case VCPU_SREG_SS:
1596 		/*
1597 		 * On AMD CPUs sometimes the DB bit in the segment
1598 		 * descriptor is left as 1, although the whole segment has
1599 		 * been made unusable. Clear it here to pass an Intel VMX
1600 		 * entry check when cross vendor migrating.
1601 		 */
1602 		if (var->unusable)
1603 			var->db = 0;
1604 		/* This is symmetric with svm_set_segment() */
1605 		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1606 		break;
1607 	}
1608 }
1609 
1610 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1611 {
1612 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1613 
1614 	return save->cpl;
1615 }
1616 
1617 static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1618 {
1619 	struct kvm_segment cs;
1620 
1621 	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1622 	*db = cs.db;
1623 	*l = cs.l;
1624 }
1625 
1626 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1627 {
1628 	struct vcpu_svm *svm = to_svm(vcpu);
1629 
1630 	dt->size = svm->vmcb->save.idtr.limit;
1631 	dt->address = svm->vmcb->save.idtr.base;
1632 }
1633 
1634 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1635 {
1636 	struct vcpu_svm *svm = to_svm(vcpu);
1637 
1638 	svm->vmcb->save.idtr.limit = dt->size;
1639 	svm->vmcb->save.idtr.base = dt->address ;
1640 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1641 }
1642 
1643 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1644 {
1645 	struct vcpu_svm *svm = to_svm(vcpu);
1646 
1647 	dt->size = svm->vmcb->save.gdtr.limit;
1648 	dt->address = svm->vmcb->save.gdtr.base;
1649 }
1650 
1651 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1652 {
1653 	struct vcpu_svm *svm = to_svm(vcpu);
1654 
1655 	svm->vmcb->save.gdtr.limit = dt->size;
1656 	svm->vmcb->save.gdtr.base = dt->address ;
1657 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1658 }
1659 
1660 static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1661 {
1662 	struct vcpu_svm *svm = to_svm(vcpu);
1663 
1664 	/*
1665 	 * For guests that don't set guest_state_protected, the cr3 update is
1666 	 * handled via kvm_mmu_load() while entering the guest. For guests
1667 	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1668 	 * VMCB save area now, since the save area will become the initial
1669 	 * contents of the VMSA, and future VMCB save area updates won't be
1670 	 * seen.
1671 	 */
1672 	if (sev_es_guest(vcpu->kvm)) {
1673 		svm->vmcb->save.cr3 = cr3;
1674 		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1675 	}
1676 }
1677 
1678 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1679 {
1680 	struct vcpu_svm *svm = to_svm(vcpu);
1681 	u64 hcr0 = cr0;
1682 	bool old_paging = is_paging(vcpu);
1683 
1684 #ifdef CONFIG_X86_64
1685 	if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) {
1686 		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1687 			vcpu->arch.efer |= EFER_LMA;
1688 			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1689 		}
1690 
1691 		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1692 			vcpu->arch.efer &= ~EFER_LMA;
1693 			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1694 		}
1695 	}
1696 #endif
1697 	vcpu->arch.cr0 = cr0;
1698 
1699 	if (!npt_enabled) {
1700 		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1701 		if (old_paging != is_paging(vcpu))
1702 			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1703 	}
1704 
1705 	/*
1706 	 * re-enable caching here because the QEMU bios
1707 	 * does not do it - this results in some delay at
1708 	 * reboot
1709 	 */
1710 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1711 		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1712 
1713 	svm->vmcb->save.cr0 = hcr0;
1714 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1715 
1716 	/*
1717 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1718 	 * tracking is done using the CR write traps.
1719 	 */
1720 	if (sev_es_guest(vcpu->kvm))
1721 		return;
1722 
1723 	if (hcr0 == cr0) {
1724 		/* Selective CR0 write remains on.  */
1725 		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1726 		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1727 	} else {
1728 		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1729 		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1730 	}
1731 }
1732 
1733 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1734 {
1735 	return true;
1736 }
1737 
1738 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1739 {
1740 	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1741 	unsigned long old_cr4 = vcpu->arch.cr4;
1742 
1743 	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1744 		svm_flush_tlb_current(vcpu);
1745 
1746 	vcpu->arch.cr4 = cr4;
1747 	if (!npt_enabled) {
1748 		cr4 |= X86_CR4_PAE;
1749 
1750 		if (!is_paging(vcpu))
1751 			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1752 	}
1753 	cr4 |= host_cr4_mce;
1754 	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1755 	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1756 
1757 	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1758 		kvm_update_cpuid_runtime(vcpu);
1759 }
1760 
1761 static void svm_set_segment(struct kvm_vcpu *vcpu,
1762 			    struct kvm_segment *var, int seg)
1763 {
1764 	struct vcpu_svm *svm = to_svm(vcpu);
1765 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1766 
1767 	s->base = var->base;
1768 	s->limit = var->limit;
1769 	s->selector = var->selector;
1770 	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1771 	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1772 	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1773 	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1774 	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1775 	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1776 	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1777 	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1778 
1779 	/*
1780 	 * This is always accurate, except if SYSRET returned to a segment
1781 	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1782 	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1783 	 * would entail passing the CPL to userspace and back.
1784 	 */
1785 	if (seg == VCPU_SREG_SS)
1786 		/* This is symmetric with svm_get_segment() */
1787 		svm->vmcb->save.cpl = (var->dpl & 3);
1788 
1789 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1790 }
1791 
1792 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1793 {
1794 	struct vcpu_svm *svm = to_svm(vcpu);
1795 
1796 	clr_exception_intercept(svm, BP_VECTOR);
1797 
1798 	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1799 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1800 			set_exception_intercept(svm, BP_VECTOR);
1801 	}
1802 }
1803 
1804 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1805 {
1806 	if (sd->next_asid > sd->max_asid) {
1807 		++sd->asid_generation;
1808 		sd->next_asid = sd->min_asid;
1809 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1810 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1811 	}
1812 
1813 	svm->current_vmcb->asid_generation = sd->asid_generation;
1814 	svm->asid = sd->next_asid++;
1815 }
1816 
1817 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
1818 {
1819 	struct vmcb *vmcb = svm->vmcb;
1820 
1821 	if (svm->vcpu.arch.guest_state_protected)
1822 		return;
1823 
1824 	if (unlikely(value != vmcb->save.dr6)) {
1825 		vmcb->save.dr6 = value;
1826 		vmcb_mark_dirty(vmcb, VMCB_DR);
1827 	}
1828 }
1829 
1830 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1831 {
1832 	struct vcpu_svm *svm = to_svm(vcpu);
1833 
1834 	if (vcpu->arch.guest_state_protected)
1835 		return;
1836 
1837 	get_debugreg(vcpu->arch.db[0], 0);
1838 	get_debugreg(vcpu->arch.db[1], 1);
1839 	get_debugreg(vcpu->arch.db[2], 2);
1840 	get_debugreg(vcpu->arch.db[3], 3);
1841 	/*
1842 	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
1843 	 * because db_interception might need it.  We can do it before vmentry.
1844 	 */
1845 	vcpu->arch.dr6 = svm->vmcb->save.dr6;
1846 	vcpu->arch.dr7 = svm->vmcb->save.dr7;
1847 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1848 	set_dr_intercepts(svm);
1849 }
1850 
1851 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1852 {
1853 	struct vcpu_svm *svm = to_svm(vcpu);
1854 
1855 	if (vcpu->arch.guest_state_protected)
1856 		return;
1857 
1858 	svm->vmcb->save.dr7 = value;
1859 	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
1860 }
1861 
1862 static int pf_interception(struct kvm_vcpu *vcpu)
1863 {
1864 	struct vcpu_svm *svm = to_svm(vcpu);
1865 
1866 	u64 fault_address = svm->vmcb->control.exit_info_2;
1867 	u64 error_code = svm->vmcb->control.exit_info_1;
1868 
1869 	return kvm_handle_page_fault(vcpu, error_code, fault_address,
1870 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1871 			svm->vmcb->control.insn_bytes : NULL,
1872 			svm->vmcb->control.insn_len);
1873 }
1874 
1875 static int npf_interception(struct kvm_vcpu *vcpu)
1876 {
1877 	struct vcpu_svm *svm = to_svm(vcpu);
1878 
1879 	u64 fault_address = svm->vmcb->control.exit_info_2;
1880 	u64 error_code = svm->vmcb->control.exit_info_1;
1881 
1882 	trace_kvm_page_fault(fault_address, error_code);
1883 	return kvm_mmu_page_fault(vcpu, fault_address, error_code,
1884 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1885 			svm->vmcb->control.insn_bytes : NULL,
1886 			svm->vmcb->control.insn_len);
1887 }
1888 
1889 static int db_interception(struct kvm_vcpu *vcpu)
1890 {
1891 	struct kvm_run *kvm_run = vcpu->run;
1892 	struct vcpu_svm *svm = to_svm(vcpu);
1893 
1894 	if (!(vcpu->guest_debug &
1895 	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1896 		!svm->nmi_singlestep) {
1897 		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
1898 		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
1899 		return 1;
1900 	}
1901 
1902 	if (svm->nmi_singlestep) {
1903 		disable_nmi_singlestep(svm);
1904 		/* Make sure we check for pending NMIs upon entry */
1905 		kvm_make_request(KVM_REQ_EVENT, vcpu);
1906 	}
1907 
1908 	if (vcpu->guest_debug &
1909 	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1910 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
1911 		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
1912 		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
1913 		kvm_run->debug.arch.pc =
1914 			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1915 		kvm_run->debug.arch.exception = DB_VECTOR;
1916 		return 0;
1917 	}
1918 
1919 	return 1;
1920 }
1921 
1922 static int bp_interception(struct kvm_vcpu *vcpu)
1923 {
1924 	struct vcpu_svm *svm = to_svm(vcpu);
1925 	struct kvm_run *kvm_run = vcpu->run;
1926 
1927 	kvm_run->exit_reason = KVM_EXIT_DEBUG;
1928 	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1929 	kvm_run->debug.arch.exception = BP_VECTOR;
1930 	return 0;
1931 }
1932 
1933 static int ud_interception(struct kvm_vcpu *vcpu)
1934 {
1935 	return handle_ud(vcpu);
1936 }
1937 
1938 static int ac_interception(struct kvm_vcpu *vcpu)
1939 {
1940 	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
1941 	return 1;
1942 }
1943 
1944 static bool is_erratum_383(void)
1945 {
1946 	int err, i;
1947 	u64 value;
1948 
1949 	if (!erratum_383_found)
1950 		return false;
1951 
1952 	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1953 	if (err)
1954 		return false;
1955 
1956 	/* Bit 62 may or may not be set for this mce */
1957 	value &= ~(1ULL << 62);
1958 
1959 	if (value != 0xb600000000010015ULL)
1960 		return false;
1961 
1962 	/* Clear MCi_STATUS registers */
1963 	for (i = 0; i < 6; ++i)
1964 		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1965 
1966 	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1967 	if (!err) {
1968 		u32 low, high;
1969 
1970 		value &= ~(1ULL << 2);
1971 		low    = lower_32_bits(value);
1972 		high   = upper_32_bits(value);
1973 
1974 		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1975 	}
1976 
1977 	/* Flush tlb to evict multi-match entries */
1978 	__flush_tlb_all();
1979 
1980 	return true;
1981 }
1982 
1983 static void svm_handle_mce(struct kvm_vcpu *vcpu)
1984 {
1985 	if (is_erratum_383()) {
1986 		/*
1987 		 * Erratum 383 triggered. Guest state is corrupt so kill the
1988 		 * guest.
1989 		 */
1990 		pr_err("KVM: Guest triggered AMD Erratum 383\n");
1991 
1992 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1993 
1994 		return;
1995 	}
1996 
1997 	/*
1998 	 * On an #MC intercept the MCE handler is not called automatically in
1999 	 * the host. So do it by hand here.
2000 	 */
2001 	kvm_machine_check();
2002 }
2003 
2004 static int mc_interception(struct kvm_vcpu *vcpu)
2005 {
2006 	return 1;
2007 }
2008 
2009 static int shutdown_interception(struct kvm_vcpu *vcpu)
2010 {
2011 	struct kvm_run *kvm_run = vcpu->run;
2012 	struct vcpu_svm *svm = to_svm(vcpu);
2013 
2014 	/*
2015 	 * The VM save area has already been encrypted so it
2016 	 * cannot be reinitialized - just terminate.
2017 	 */
2018 	if (sev_es_guest(vcpu->kvm))
2019 		return -EINVAL;
2020 
2021 	/*
2022 	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2023 	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2024 	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2025 	 * userspace.  At a platform view, INIT is acceptable behavior as
2026 	 * there exist bare metal platforms that automatically INIT the CPU
2027 	 * in response to shutdown.
2028 	 */
2029 	clear_page(svm->vmcb);
2030 	kvm_vcpu_reset(vcpu, true);
2031 
2032 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2033 	return 0;
2034 }
2035 
2036 static int io_interception(struct kvm_vcpu *vcpu)
2037 {
2038 	struct vcpu_svm *svm = to_svm(vcpu);
2039 	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2040 	int size, in, string;
2041 	unsigned port;
2042 
2043 	++vcpu->stat.io_exits;
2044 	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2045 	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2046 	port = io_info >> 16;
2047 	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2048 
2049 	if (string) {
2050 		if (sev_es_guest(vcpu->kvm))
2051 			return sev_es_string_io(svm, size, port, in);
2052 		else
2053 			return kvm_emulate_instruction(vcpu, 0);
2054 	}
2055 
2056 	svm->next_rip = svm->vmcb->control.exit_info_2;
2057 
2058 	return kvm_fast_pio(vcpu, size, port, in);
2059 }
2060 
2061 static int nmi_interception(struct kvm_vcpu *vcpu)
2062 {
2063 	return 1;
2064 }
2065 
2066 static int smi_interception(struct kvm_vcpu *vcpu)
2067 {
2068 	return 1;
2069 }
2070 
2071 static int intr_interception(struct kvm_vcpu *vcpu)
2072 {
2073 	++vcpu->stat.irq_exits;
2074 	return 1;
2075 }
2076 
2077 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2078 {
2079 	struct vcpu_svm *svm = to_svm(vcpu);
2080 	struct vmcb *vmcb12;
2081 	struct kvm_host_map map;
2082 	int ret;
2083 
2084 	if (nested_svm_check_permissions(vcpu))
2085 		return 1;
2086 
2087 	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2088 	if (ret) {
2089 		if (ret == -EINVAL)
2090 			kvm_inject_gp(vcpu, 0);
2091 		return 1;
2092 	}
2093 
2094 	vmcb12 = map.hva;
2095 
2096 	ret = kvm_skip_emulated_instruction(vcpu);
2097 
2098 	if (vmload) {
2099 		svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2100 		svm->sysenter_eip_hi = 0;
2101 		svm->sysenter_esp_hi = 0;
2102 	} else {
2103 		svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2104 	}
2105 
2106 	kvm_vcpu_unmap(vcpu, &map, true);
2107 
2108 	return ret;
2109 }
2110 
2111 static int vmload_interception(struct kvm_vcpu *vcpu)
2112 {
2113 	return vmload_vmsave_interception(vcpu, true);
2114 }
2115 
2116 static int vmsave_interception(struct kvm_vcpu *vcpu)
2117 {
2118 	return vmload_vmsave_interception(vcpu, false);
2119 }
2120 
2121 static int vmrun_interception(struct kvm_vcpu *vcpu)
2122 {
2123 	if (nested_svm_check_permissions(vcpu))
2124 		return 1;
2125 
2126 	return nested_svm_vmrun(vcpu);
2127 }
2128 
2129 enum {
2130 	NONE_SVM_INSTR,
2131 	SVM_INSTR_VMRUN,
2132 	SVM_INSTR_VMLOAD,
2133 	SVM_INSTR_VMSAVE,
2134 };
2135 
2136 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2137 static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2138 {
2139 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2140 
2141 	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2142 		return NONE_SVM_INSTR;
2143 
2144 	switch (ctxt->modrm) {
2145 	case 0xd8: /* VMRUN */
2146 		return SVM_INSTR_VMRUN;
2147 	case 0xda: /* VMLOAD */
2148 		return SVM_INSTR_VMLOAD;
2149 	case 0xdb: /* VMSAVE */
2150 		return SVM_INSTR_VMSAVE;
2151 	default:
2152 		break;
2153 	}
2154 
2155 	return NONE_SVM_INSTR;
2156 }
2157 
2158 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2159 {
2160 	const int guest_mode_exit_codes[] = {
2161 		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2162 		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2163 		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2164 	};
2165 	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2166 		[SVM_INSTR_VMRUN] = vmrun_interception,
2167 		[SVM_INSTR_VMLOAD] = vmload_interception,
2168 		[SVM_INSTR_VMSAVE] = vmsave_interception,
2169 	};
2170 	struct vcpu_svm *svm = to_svm(vcpu);
2171 	int ret;
2172 
2173 	if (is_guest_mode(vcpu)) {
2174 		/* Returns '1' or -errno on failure, '0' on success. */
2175 		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2176 		if (ret)
2177 			return ret;
2178 		return 1;
2179 	}
2180 	return svm_instr_handlers[opcode](vcpu);
2181 }
2182 
2183 /*
2184  * #GP handling code. Note that #GP can be triggered under the following two
2185  * cases:
2186  *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2187  *      some AMD CPUs when EAX of these instructions are in the reserved memory
2188  *      regions (e.g. SMM memory on host).
2189  *   2) VMware backdoor
2190  */
2191 static int gp_interception(struct kvm_vcpu *vcpu)
2192 {
2193 	struct vcpu_svm *svm = to_svm(vcpu);
2194 	u32 error_code = svm->vmcb->control.exit_info_1;
2195 	int opcode;
2196 
2197 	/* Both #GP cases have zero error_code */
2198 	if (error_code)
2199 		goto reinject;
2200 
2201 	/* Decode the instruction for usage later */
2202 	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2203 		goto reinject;
2204 
2205 	opcode = svm_instr_opcode(vcpu);
2206 
2207 	if (opcode == NONE_SVM_INSTR) {
2208 		if (!enable_vmware_backdoor)
2209 			goto reinject;
2210 
2211 		/*
2212 		 * VMware backdoor emulation on #GP interception only handles
2213 		 * IN{S}, OUT{S}, and RDPMC.
2214 		 */
2215 		if (!is_guest_mode(vcpu))
2216 			return kvm_emulate_instruction(vcpu,
2217 				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2218 	} else {
2219 		/* All SVM instructions expect page aligned RAX */
2220 		if (svm->vmcb->save.rax & ~PAGE_MASK)
2221 			goto reinject;
2222 
2223 		return emulate_svm_instr(vcpu, opcode);
2224 	}
2225 
2226 reinject:
2227 	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2228 	return 1;
2229 }
2230 
2231 void svm_set_gif(struct vcpu_svm *svm, bool value)
2232 {
2233 	if (value) {
2234 		/*
2235 		 * If VGIF is enabled, the STGI intercept is only added to
2236 		 * detect the opening of the SMI/NMI window; remove it now.
2237 		 * Likewise, clear the VINTR intercept, we will set it
2238 		 * again while processing KVM_REQ_EVENT if needed.
2239 		 */
2240 		if (vgif)
2241 			svm_clr_intercept(svm, INTERCEPT_STGI);
2242 		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2243 			svm_clear_vintr(svm);
2244 
2245 		enable_gif(svm);
2246 		if (svm->vcpu.arch.smi_pending ||
2247 		    svm->vcpu.arch.nmi_pending ||
2248 		    kvm_cpu_has_injectable_intr(&svm->vcpu))
2249 			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2250 	} else {
2251 		disable_gif(svm);
2252 
2253 		/*
2254 		 * After a CLGI no interrupts should come.  But if vGIF is
2255 		 * in use, we still rely on the VINTR intercept (rather than
2256 		 * STGI) to detect an open interrupt window.
2257 		*/
2258 		if (!vgif)
2259 			svm_clear_vintr(svm);
2260 	}
2261 }
2262 
2263 static int stgi_interception(struct kvm_vcpu *vcpu)
2264 {
2265 	int ret;
2266 
2267 	if (nested_svm_check_permissions(vcpu))
2268 		return 1;
2269 
2270 	ret = kvm_skip_emulated_instruction(vcpu);
2271 	svm_set_gif(to_svm(vcpu), true);
2272 	return ret;
2273 }
2274 
2275 static int clgi_interception(struct kvm_vcpu *vcpu)
2276 {
2277 	int ret;
2278 
2279 	if (nested_svm_check_permissions(vcpu))
2280 		return 1;
2281 
2282 	ret = kvm_skip_emulated_instruction(vcpu);
2283 	svm_set_gif(to_svm(vcpu), false);
2284 	return ret;
2285 }
2286 
2287 static int invlpga_interception(struct kvm_vcpu *vcpu)
2288 {
2289 	gva_t gva = kvm_rax_read(vcpu);
2290 	u32 asid = kvm_rcx_read(vcpu);
2291 
2292 	/* FIXME: Handle an address size prefix. */
2293 	if (!is_long_mode(vcpu))
2294 		gva = (u32)gva;
2295 
2296 	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2297 
2298 	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2299 	kvm_mmu_invlpg(vcpu, gva);
2300 
2301 	return kvm_skip_emulated_instruction(vcpu);
2302 }
2303 
2304 static int skinit_interception(struct kvm_vcpu *vcpu)
2305 {
2306 	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2307 
2308 	kvm_queue_exception(vcpu, UD_VECTOR);
2309 	return 1;
2310 }
2311 
2312 static int task_switch_interception(struct kvm_vcpu *vcpu)
2313 {
2314 	struct vcpu_svm *svm = to_svm(vcpu);
2315 	u16 tss_selector;
2316 	int reason;
2317 	int int_type = svm->vmcb->control.exit_int_info &
2318 		SVM_EXITINTINFO_TYPE_MASK;
2319 	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2320 	uint32_t type =
2321 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2322 	uint32_t idt_v =
2323 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2324 	bool has_error_code = false;
2325 	u32 error_code = 0;
2326 
2327 	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2328 
2329 	if (svm->vmcb->control.exit_info_2 &
2330 	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2331 		reason = TASK_SWITCH_IRET;
2332 	else if (svm->vmcb->control.exit_info_2 &
2333 		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2334 		reason = TASK_SWITCH_JMP;
2335 	else if (idt_v)
2336 		reason = TASK_SWITCH_GATE;
2337 	else
2338 		reason = TASK_SWITCH_CALL;
2339 
2340 	if (reason == TASK_SWITCH_GATE) {
2341 		switch (type) {
2342 		case SVM_EXITINTINFO_TYPE_NMI:
2343 			vcpu->arch.nmi_injected = false;
2344 			break;
2345 		case SVM_EXITINTINFO_TYPE_EXEPT:
2346 			if (svm->vmcb->control.exit_info_2 &
2347 			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2348 				has_error_code = true;
2349 				error_code =
2350 					(u32)svm->vmcb->control.exit_info_2;
2351 			}
2352 			kvm_clear_exception_queue(vcpu);
2353 			break;
2354 		case SVM_EXITINTINFO_TYPE_INTR:
2355 			kvm_clear_interrupt_queue(vcpu);
2356 			break;
2357 		default:
2358 			break;
2359 		}
2360 	}
2361 
2362 	if (reason != TASK_SWITCH_GATE ||
2363 	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2364 	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2365 	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2366 		if (!svm_skip_emulated_instruction(vcpu))
2367 			return 0;
2368 	}
2369 
2370 	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2371 		int_vec = -1;
2372 
2373 	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2374 			       has_error_code, error_code);
2375 }
2376 
2377 static int iret_interception(struct kvm_vcpu *vcpu)
2378 {
2379 	struct vcpu_svm *svm = to_svm(vcpu);
2380 
2381 	++vcpu->stat.nmi_window_exits;
2382 	vcpu->arch.hflags |= HF_IRET_MASK;
2383 	if (!sev_es_guest(vcpu->kvm)) {
2384 		svm_clr_intercept(svm, INTERCEPT_IRET);
2385 		svm->nmi_iret_rip = kvm_rip_read(vcpu);
2386 	}
2387 	kvm_make_request(KVM_REQ_EVENT, vcpu);
2388 	return 1;
2389 }
2390 
2391 static int invlpg_interception(struct kvm_vcpu *vcpu)
2392 {
2393 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2394 		return kvm_emulate_instruction(vcpu, 0);
2395 
2396 	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2397 	return kvm_skip_emulated_instruction(vcpu);
2398 }
2399 
2400 static int emulate_on_interception(struct kvm_vcpu *vcpu)
2401 {
2402 	return kvm_emulate_instruction(vcpu, 0);
2403 }
2404 
2405 static int rsm_interception(struct kvm_vcpu *vcpu)
2406 {
2407 	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2408 }
2409 
2410 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2411 					    unsigned long val)
2412 {
2413 	struct vcpu_svm *svm = to_svm(vcpu);
2414 	unsigned long cr0 = vcpu->arch.cr0;
2415 	bool ret = false;
2416 
2417 	if (!is_guest_mode(vcpu) ||
2418 	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2419 		return false;
2420 
2421 	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2422 	val &= ~SVM_CR0_SELECTIVE_MASK;
2423 
2424 	if (cr0 ^ val) {
2425 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2426 		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2427 	}
2428 
2429 	return ret;
2430 }
2431 
2432 #define CR_VALID (1ULL << 63)
2433 
2434 static int cr_interception(struct kvm_vcpu *vcpu)
2435 {
2436 	struct vcpu_svm *svm = to_svm(vcpu);
2437 	int reg, cr;
2438 	unsigned long val;
2439 	int err;
2440 
2441 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2442 		return emulate_on_interception(vcpu);
2443 
2444 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2445 		return emulate_on_interception(vcpu);
2446 
2447 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2448 	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2449 		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2450 	else
2451 		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2452 
2453 	err = 0;
2454 	if (cr >= 16) { /* mov to cr */
2455 		cr -= 16;
2456 		val = kvm_register_read(vcpu, reg);
2457 		trace_kvm_cr_write(cr, val);
2458 		switch (cr) {
2459 		case 0:
2460 			if (!check_selective_cr0_intercepted(vcpu, val))
2461 				err = kvm_set_cr0(vcpu, val);
2462 			else
2463 				return 1;
2464 
2465 			break;
2466 		case 3:
2467 			err = kvm_set_cr3(vcpu, val);
2468 			break;
2469 		case 4:
2470 			err = kvm_set_cr4(vcpu, val);
2471 			break;
2472 		case 8:
2473 			err = kvm_set_cr8(vcpu, val);
2474 			break;
2475 		default:
2476 			WARN(1, "unhandled write to CR%d", cr);
2477 			kvm_queue_exception(vcpu, UD_VECTOR);
2478 			return 1;
2479 		}
2480 	} else { /* mov from cr */
2481 		switch (cr) {
2482 		case 0:
2483 			val = kvm_read_cr0(vcpu);
2484 			break;
2485 		case 2:
2486 			val = vcpu->arch.cr2;
2487 			break;
2488 		case 3:
2489 			val = kvm_read_cr3(vcpu);
2490 			break;
2491 		case 4:
2492 			val = kvm_read_cr4(vcpu);
2493 			break;
2494 		case 8:
2495 			val = kvm_get_cr8(vcpu);
2496 			break;
2497 		default:
2498 			WARN(1, "unhandled read from CR%d", cr);
2499 			kvm_queue_exception(vcpu, UD_VECTOR);
2500 			return 1;
2501 		}
2502 		kvm_register_write(vcpu, reg, val);
2503 		trace_kvm_cr_read(cr, val);
2504 	}
2505 	return kvm_complete_insn_gp(vcpu, err);
2506 }
2507 
2508 static int cr_trap(struct kvm_vcpu *vcpu)
2509 {
2510 	struct vcpu_svm *svm = to_svm(vcpu);
2511 	unsigned long old_value, new_value;
2512 	unsigned int cr;
2513 	int ret = 0;
2514 
2515 	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2516 
2517 	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2518 	switch (cr) {
2519 	case 0:
2520 		old_value = kvm_read_cr0(vcpu);
2521 		svm_set_cr0(vcpu, new_value);
2522 
2523 		kvm_post_set_cr0(vcpu, old_value, new_value);
2524 		break;
2525 	case 4:
2526 		old_value = kvm_read_cr4(vcpu);
2527 		svm_set_cr4(vcpu, new_value);
2528 
2529 		kvm_post_set_cr4(vcpu, old_value, new_value);
2530 		break;
2531 	case 8:
2532 		ret = kvm_set_cr8(vcpu, new_value);
2533 		break;
2534 	default:
2535 		WARN(1, "unhandled CR%d write trap", cr);
2536 		kvm_queue_exception(vcpu, UD_VECTOR);
2537 		return 1;
2538 	}
2539 
2540 	return kvm_complete_insn_gp(vcpu, ret);
2541 }
2542 
2543 static int dr_interception(struct kvm_vcpu *vcpu)
2544 {
2545 	struct vcpu_svm *svm = to_svm(vcpu);
2546 	int reg, dr;
2547 	unsigned long val;
2548 	int err = 0;
2549 
2550 	if (vcpu->guest_debug == 0) {
2551 		/*
2552 		 * No more DR vmexits; force a reload of the debug registers
2553 		 * and reenter on this instruction.  The next vmexit will
2554 		 * retrieve the full state of the debug registers.
2555 		 */
2556 		clr_dr_intercepts(svm);
2557 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2558 		return 1;
2559 	}
2560 
2561 	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2562 		return emulate_on_interception(vcpu);
2563 
2564 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2565 	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2566 	if (dr >= 16) { /* mov to DRn  */
2567 		dr -= 16;
2568 		val = kvm_register_read(vcpu, reg);
2569 		err = kvm_set_dr(vcpu, dr, val);
2570 	} else {
2571 		kvm_get_dr(vcpu, dr, &val);
2572 		kvm_register_write(vcpu, reg, val);
2573 	}
2574 
2575 	return kvm_complete_insn_gp(vcpu, err);
2576 }
2577 
2578 static int cr8_write_interception(struct kvm_vcpu *vcpu)
2579 {
2580 	int r;
2581 
2582 	u8 cr8_prev = kvm_get_cr8(vcpu);
2583 	/* instruction emulation calls kvm_set_cr8() */
2584 	r = cr_interception(vcpu);
2585 	if (lapic_in_kernel(vcpu))
2586 		return r;
2587 	if (cr8_prev <= kvm_get_cr8(vcpu))
2588 		return r;
2589 	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2590 	return 0;
2591 }
2592 
2593 static int efer_trap(struct kvm_vcpu *vcpu)
2594 {
2595 	struct msr_data msr_info;
2596 	int ret;
2597 
2598 	/*
2599 	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2600 	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2601 	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2602 	 * the guest doesn't have X86_FEATURE_SVM.
2603 	 */
2604 	msr_info.host_initiated = false;
2605 	msr_info.index = MSR_EFER;
2606 	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2607 	ret = kvm_set_msr_common(vcpu, &msr_info);
2608 
2609 	return kvm_complete_insn_gp(vcpu, ret);
2610 }
2611 
2612 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2613 {
2614 	msr->data = 0;
2615 
2616 	switch (msr->index) {
2617 	case MSR_F10H_DECFG:
2618 		if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
2619 			msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
2620 		break;
2621 	case MSR_IA32_PERF_CAPABILITIES:
2622 		return 0;
2623 	default:
2624 		return KVM_MSR_RET_INVALID;
2625 	}
2626 
2627 	return 0;
2628 }
2629 
2630 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2631 {
2632 	struct vcpu_svm *svm = to_svm(vcpu);
2633 
2634 	switch (msr_info->index) {
2635 	case MSR_AMD64_TSC_RATIO:
2636 		if (!msr_info->host_initiated && !svm->tsc_scaling_enabled)
2637 			return 1;
2638 		msr_info->data = svm->tsc_ratio_msr;
2639 		break;
2640 	case MSR_STAR:
2641 		msr_info->data = svm->vmcb01.ptr->save.star;
2642 		break;
2643 #ifdef CONFIG_X86_64
2644 	case MSR_LSTAR:
2645 		msr_info->data = svm->vmcb01.ptr->save.lstar;
2646 		break;
2647 	case MSR_CSTAR:
2648 		msr_info->data = svm->vmcb01.ptr->save.cstar;
2649 		break;
2650 	case MSR_KERNEL_GS_BASE:
2651 		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2652 		break;
2653 	case MSR_SYSCALL_MASK:
2654 		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2655 		break;
2656 #endif
2657 	case MSR_IA32_SYSENTER_CS:
2658 		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2659 		break;
2660 	case MSR_IA32_SYSENTER_EIP:
2661 		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2662 		if (guest_cpuid_is_intel(vcpu))
2663 			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2664 		break;
2665 	case MSR_IA32_SYSENTER_ESP:
2666 		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2667 		if (guest_cpuid_is_intel(vcpu))
2668 			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2669 		break;
2670 	case MSR_TSC_AUX:
2671 		msr_info->data = svm->tsc_aux;
2672 		break;
2673 	case MSR_IA32_DEBUGCTLMSR:
2674 	case MSR_IA32_LASTBRANCHFROMIP:
2675 	case MSR_IA32_LASTBRANCHTOIP:
2676 	case MSR_IA32_LASTINTFROMIP:
2677 	case MSR_IA32_LASTINTTOIP:
2678 		msr_info->data = svm_get_lbr_msr(svm, msr_info->index);
2679 		break;
2680 	case MSR_VM_HSAVE_PA:
2681 		msr_info->data = svm->nested.hsave_msr;
2682 		break;
2683 	case MSR_VM_CR:
2684 		msr_info->data = svm->nested.vm_cr_msr;
2685 		break;
2686 	case MSR_IA32_SPEC_CTRL:
2687 		if (!msr_info->host_initiated &&
2688 		    !guest_has_spec_ctrl_msr(vcpu))
2689 			return 1;
2690 
2691 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2692 			msr_info->data = svm->vmcb->save.spec_ctrl;
2693 		else
2694 			msr_info->data = svm->spec_ctrl;
2695 		break;
2696 	case MSR_AMD64_VIRT_SPEC_CTRL:
2697 		if (!msr_info->host_initiated &&
2698 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2699 			return 1;
2700 
2701 		msr_info->data = svm->virt_spec_ctrl;
2702 		break;
2703 	case MSR_F15H_IC_CFG: {
2704 
2705 		int family, model;
2706 
2707 		family = guest_cpuid_family(vcpu);
2708 		model  = guest_cpuid_model(vcpu);
2709 
2710 		if (family < 0 || model < 0)
2711 			return kvm_get_msr_common(vcpu, msr_info);
2712 
2713 		msr_info->data = 0;
2714 
2715 		if (family == 0x15 &&
2716 		    (model >= 0x2 && model < 0x20))
2717 			msr_info->data = 0x1E;
2718 		}
2719 		break;
2720 	case MSR_F10H_DECFG:
2721 		msr_info->data = svm->msr_decfg;
2722 		break;
2723 	default:
2724 		return kvm_get_msr_common(vcpu, msr_info);
2725 	}
2726 	return 0;
2727 }
2728 
2729 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2730 {
2731 	struct vcpu_svm *svm = to_svm(vcpu);
2732 	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2733 		return kvm_complete_insn_gp(vcpu, err);
2734 
2735 	ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1);
2736 	ghcb_set_sw_exit_info_2(svm->sev_es.ghcb,
2737 				X86_TRAP_GP |
2738 				SVM_EVTINJ_TYPE_EXEPT |
2739 				SVM_EVTINJ_VALID);
2740 	return 1;
2741 }
2742 
2743 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2744 {
2745 	struct vcpu_svm *svm = to_svm(vcpu);
2746 	int svm_dis, chg_mask;
2747 
2748 	if (data & ~SVM_VM_CR_VALID_MASK)
2749 		return 1;
2750 
2751 	chg_mask = SVM_VM_CR_VALID_MASK;
2752 
2753 	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2754 		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2755 
2756 	svm->nested.vm_cr_msr &= ~chg_mask;
2757 	svm->nested.vm_cr_msr |= (data & chg_mask);
2758 
2759 	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2760 
2761 	/* check for svm_disable while efer.svme is set */
2762 	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2763 		return 1;
2764 
2765 	return 0;
2766 }
2767 
2768 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2769 {
2770 	struct vcpu_svm *svm = to_svm(vcpu);
2771 	int r;
2772 
2773 	u32 ecx = msr->index;
2774 	u64 data = msr->data;
2775 	switch (ecx) {
2776 	case MSR_AMD64_TSC_RATIO:
2777 
2778 		if (!svm->tsc_scaling_enabled) {
2779 
2780 			if (!msr->host_initiated)
2781 				return 1;
2782 			/*
2783 			 * In case TSC scaling is not enabled, always
2784 			 * leave this MSR at the default value.
2785 			 *
2786 			 * Due to bug in qemu 6.2.0, it would try to set
2787 			 * this msr to 0 if tsc scaling is not enabled.
2788 			 * Ignore this value as well.
2789 			 */
2790 			if (data != 0 && data != svm->tsc_ratio_msr)
2791 				return 1;
2792 			break;
2793 		}
2794 
2795 		if (data & SVM_TSC_RATIO_RSVD)
2796 			return 1;
2797 
2798 		svm->tsc_ratio_msr = data;
2799 
2800 		if (svm->tsc_scaling_enabled && is_guest_mode(vcpu))
2801 			nested_svm_update_tsc_ratio_msr(vcpu);
2802 
2803 		break;
2804 	case MSR_IA32_CR_PAT:
2805 		if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
2806 			return 1;
2807 		vcpu->arch.pat = data;
2808 		svm->vmcb01.ptr->save.g_pat = data;
2809 		if (is_guest_mode(vcpu))
2810 			nested_vmcb02_compute_g_pat(svm);
2811 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
2812 		break;
2813 	case MSR_IA32_SPEC_CTRL:
2814 		if (!msr->host_initiated &&
2815 		    !guest_has_spec_ctrl_msr(vcpu))
2816 			return 1;
2817 
2818 		if (kvm_spec_ctrl_test_value(data))
2819 			return 1;
2820 
2821 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2822 			svm->vmcb->save.spec_ctrl = data;
2823 		else
2824 			svm->spec_ctrl = data;
2825 		if (!data)
2826 			break;
2827 
2828 		/*
2829 		 * For non-nested:
2830 		 * When it's written (to non-zero) for the first time, pass
2831 		 * it through.
2832 		 *
2833 		 * For nested:
2834 		 * The handling of the MSR bitmap for L2 guests is done in
2835 		 * nested_svm_vmrun_msrpm.
2836 		 * We update the L1 MSR bit as well since it will end up
2837 		 * touching the MSR anyway now.
2838 		 */
2839 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
2840 		break;
2841 	case MSR_IA32_PRED_CMD:
2842 		if (!msr->host_initiated &&
2843 		    !guest_has_pred_cmd_msr(vcpu))
2844 			return 1;
2845 
2846 		if (data & ~PRED_CMD_IBPB)
2847 			return 1;
2848 		if (!boot_cpu_has(X86_FEATURE_IBPB))
2849 			return 1;
2850 		if (!data)
2851 			break;
2852 
2853 		wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2854 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
2855 		break;
2856 	case MSR_AMD64_VIRT_SPEC_CTRL:
2857 		if (!msr->host_initiated &&
2858 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2859 			return 1;
2860 
2861 		if (data & ~SPEC_CTRL_SSBD)
2862 			return 1;
2863 
2864 		svm->virt_spec_ctrl = data;
2865 		break;
2866 	case MSR_STAR:
2867 		svm->vmcb01.ptr->save.star = data;
2868 		break;
2869 #ifdef CONFIG_X86_64
2870 	case MSR_LSTAR:
2871 		svm->vmcb01.ptr->save.lstar = data;
2872 		break;
2873 	case MSR_CSTAR:
2874 		svm->vmcb01.ptr->save.cstar = data;
2875 		break;
2876 	case MSR_KERNEL_GS_BASE:
2877 		svm->vmcb01.ptr->save.kernel_gs_base = data;
2878 		break;
2879 	case MSR_SYSCALL_MASK:
2880 		svm->vmcb01.ptr->save.sfmask = data;
2881 		break;
2882 #endif
2883 	case MSR_IA32_SYSENTER_CS:
2884 		svm->vmcb01.ptr->save.sysenter_cs = data;
2885 		break;
2886 	case MSR_IA32_SYSENTER_EIP:
2887 		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
2888 		/*
2889 		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
2890 		 * when we spoof an Intel vendor ID (for cross vendor migration).
2891 		 * In this case we use this intercept to track the high
2892 		 * 32 bit part of these msrs to support Intel's
2893 		 * implementation of SYSENTER/SYSEXIT.
2894 		 */
2895 		svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
2896 		break;
2897 	case MSR_IA32_SYSENTER_ESP:
2898 		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
2899 		svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
2900 		break;
2901 	case MSR_TSC_AUX:
2902 		/*
2903 		 * TSC_AUX is usually changed only during boot and never read
2904 		 * directly.  Intercept TSC_AUX instead of exposing it to the
2905 		 * guest via direct_access_msrs, and switch it via user return.
2906 		 */
2907 		preempt_disable();
2908 		r = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
2909 		preempt_enable();
2910 		if (r)
2911 			return 1;
2912 
2913 		svm->tsc_aux = data;
2914 		break;
2915 	case MSR_IA32_DEBUGCTLMSR:
2916 		if (!lbrv) {
2917 			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2918 				    __func__, data);
2919 			break;
2920 		}
2921 		if (data & DEBUGCTL_RESERVED_BITS)
2922 			return 1;
2923 
2924 		if (svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK)
2925 			svm->vmcb->save.dbgctl = data;
2926 		else
2927 			svm->vmcb01.ptr->save.dbgctl = data;
2928 
2929 		svm_update_lbrv(vcpu);
2930 
2931 		break;
2932 	case MSR_VM_HSAVE_PA:
2933 		/*
2934 		 * Old kernels did not validate the value written to
2935 		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
2936 		 * value to allow live migrating buggy or malicious guests
2937 		 * originating from those kernels.
2938 		 */
2939 		if (!msr->host_initiated && !page_address_valid(vcpu, data))
2940 			return 1;
2941 
2942 		svm->nested.hsave_msr = data & PAGE_MASK;
2943 		break;
2944 	case MSR_VM_CR:
2945 		return svm_set_vm_cr(vcpu, data);
2946 	case MSR_VM_IGNNE:
2947 		vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
2948 		break;
2949 	case MSR_F10H_DECFG: {
2950 		struct kvm_msr_entry msr_entry;
2951 
2952 		msr_entry.index = msr->index;
2953 		if (svm_get_msr_feature(&msr_entry))
2954 			return 1;
2955 
2956 		/* Check the supported bits */
2957 		if (data & ~msr_entry.data)
2958 			return 1;
2959 
2960 		/* Don't allow the guest to change a bit, #GP */
2961 		if (!msr->host_initiated && (data ^ msr_entry.data))
2962 			return 1;
2963 
2964 		svm->msr_decfg = data;
2965 		break;
2966 	}
2967 	default:
2968 		return kvm_set_msr_common(vcpu, msr);
2969 	}
2970 	return 0;
2971 }
2972 
2973 static int msr_interception(struct kvm_vcpu *vcpu)
2974 {
2975 	if (to_svm(vcpu)->vmcb->control.exit_info_1)
2976 		return kvm_emulate_wrmsr(vcpu);
2977 	else
2978 		return kvm_emulate_rdmsr(vcpu);
2979 }
2980 
2981 static int interrupt_window_interception(struct kvm_vcpu *vcpu)
2982 {
2983 	kvm_make_request(KVM_REQ_EVENT, vcpu);
2984 	svm_clear_vintr(to_svm(vcpu));
2985 
2986 	/*
2987 	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
2988 	 * In this case AVIC was temporarily disabled for
2989 	 * requesting the IRQ window and we have to re-enable it.
2990 	 *
2991 	 * If running nested, still remove the VM wide AVIC inhibit to
2992 	 * support case in which the interrupt window was requested when the
2993 	 * vCPU was not running nested.
2994 
2995 	 * All vCPUs which run still run nested, will remain to have their
2996 	 * AVIC still inhibited due to per-cpu AVIC inhibition.
2997 	 */
2998 	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
2999 
3000 	++vcpu->stat.irq_window_exits;
3001 	return 1;
3002 }
3003 
3004 static int pause_interception(struct kvm_vcpu *vcpu)
3005 {
3006 	bool in_kernel;
3007 	/*
3008 	 * CPL is not made available for an SEV-ES guest, therefore
3009 	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3010 	 * set in_kernel to false as well.
3011 	 */
3012 	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3013 
3014 	grow_ple_window(vcpu);
3015 
3016 	kvm_vcpu_on_spin(vcpu, in_kernel);
3017 	return kvm_skip_emulated_instruction(vcpu);
3018 }
3019 
3020 static int invpcid_interception(struct kvm_vcpu *vcpu)
3021 {
3022 	struct vcpu_svm *svm = to_svm(vcpu);
3023 	unsigned long type;
3024 	gva_t gva;
3025 
3026 	if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3027 		kvm_queue_exception(vcpu, UD_VECTOR);
3028 		return 1;
3029 	}
3030 
3031 	/*
3032 	 * For an INVPCID intercept:
3033 	 * EXITINFO1 provides the linear address of the memory operand.
3034 	 * EXITINFO2 provides the contents of the register operand.
3035 	 */
3036 	type = svm->vmcb->control.exit_info_2;
3037 	gva = svm->vmcb->control.exit_info_1;
3038 
3039 	return kvm_handle_invpcid(vcpu, type, gva);
3040 }
3041 
3042 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3043 	[SVM_EXIT_READ_CR0]			= cr_interception,
3044 	[SVM_EXIT_READ_CR3]			= cr_interception,
3045 	[SVM_EXIT_READ_CR4]			= cr_interception,
3046 	[SVM_EXIT_READ_CR8]			= cr_interception,
3047 	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3048 	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3049 	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3050 	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3051 	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3052 	[SVM_EXIT_READ_DR0]			= dr_interception,
3053 	[SVM_EXIT_READ_DR1]			= dr_interception,
3054 	[SVM_EXIT_READ_DR2]			= dr_interception,
3055 	[SVM_EXIT_READ_DR3]			= dr_interception,
3056 	[SVM_EXIT_READ_DR4]			= dr_interception,
3057 	[SVM_EXIT_READ_DR5]			= dr_interception,
3058 	[SVM_EXIT_READ_DR6]			= dr_interception,
3059 	[SVM_EXIT_READ_DR7]			= dr_interception,
3060 	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3061 	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3062 	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3063 	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3064 	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3065 	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3066 	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3067 	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3068 	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3069 	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3070 	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3071 	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3072 	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3073 	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3074 	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3075 	[SVM_EXIT_INTR]				= intr_interception,
3076 	[SVM_EXIT_NMI]				= nmi_interception,
3077 	[SVM_EXIT_SMI]				= smi_interception,
3078 	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3079 	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3080 	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3081 	[SVM_EXIT_IRET]                         = iret_interception,
3082 	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3083 	[SVM_EXIT_PAUSE]			= pause_interception,
3084 	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3085 	[SVM_EXIT_INVLPG]			= invlpg_interception,
3086 	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3087 	[SVM_EXIT_IOIO]				= io_interception,
3088 	[SVM_EXIT_MSR]				= msr_interception,
3089 	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3090 	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3091 	[SVM_EXIT_VMRUN]			= vmrun_interception,
3092 	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3093 	[SVM_EXIT_VMLOAD]			= vmload_interception,
3094 	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3095 	[SVM_EXIT_STGI]				= stgi_interception,
3096 	[SVM_EXIT_CLGI]				= clgi_interception,
3097 	[SVM_EXIT_SKINIT]			= skinit_interception,
3098 	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3099 	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3100 	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3101 	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3102 	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3103 	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3104 	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3105 	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3106 	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3107 	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3108 	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3109 	[SVM_EXIT_NPF]				= npf_interception,
3110 	[SVM_EXIT_RSM]                          = rsm_interception,
3111 	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3112 	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3113 	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3114 };
3115 
3116 static void dump_vmcb(struct kvm_vcpu *vcpu)
3117 {
3118 	struct vcpu_svm *svm = to_svm(vcpu);
3119 	struct vmcb_control_area *control = &svm->vmcb->control;
3120 	struct vmcb_save_area *save = &svm->vmcb->save;
3121 	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3122 
3123 	if (!dump_invalid_vmcb) {
3124 		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3125 		return;
3126 	}
3127 
3128 	pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3129 	       svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3130 	pr_err("VMCB Control Area:\n");
3131 	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3132 	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3133 	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3134 	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3135 	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3136 	pr_err("%-20s%08x %08x\n", "intercepts:",
3137               control->intercepts[INTERCEPT_WORD3],
3138 	       control->intercepts[INTERCEPT_WORD4]);
3139 	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3140 	pr_err("%-20s%d\n", "pause filter threshold:",
3141 	       control->pause_filter_thresh);
3142 	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3143 	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3144 	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3145 	pr_err("%-20s%d\n", "asid:", control->asid);
3146 	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3147 	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3148 	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3149 	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3150 	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3151 	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3152 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3153 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3154 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3155 	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3156 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3157 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3158 	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3159 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3160 	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3161 	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3162 	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3163 	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3164 	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3165 	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3166 	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3167 	pr_err("VMCB State Save Area:\n");
3168 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3169 	       "es:",
3170 	       save->es.selector, save->es.attrib,
3171 	       save->es.limit, save->es.base);
3172 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3173 	       "cs:",
3174 	       save->cs.selector, save->cs.attrib,
3175 	       save->cs.limit, save->cs.base);
3176 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3177 	       "ss:",
3178 	       save->ss.selector, save->ss.attrib,
3179 	       save->ss.limit, save->ss.base);
3180 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3181 	       "ds:",
3182 	       save->ds.selector, save->ds.attrib,
3183 	       save->ds.limit, save->ds.base);
3184 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3185 	       "fs:",
3186 	       save01->fs.selector, save01->fs.attrib,
3187 	       save01->fs.limit, save01->fs.base);
3188 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3189 	       "gs:",
3190 	       save01->gs.selector, save01->gs.attrib,
3191 	       save01->gs.limit, save01->gs.base);
3192 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3193 	       "gdtr:",
3194 	       save->gdtr.selector, save->gdtr.attrib,
3195 	       save->gdtr.limit, save->gdtr.base);
3196 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3197 	       "ldtr:",
3198 	       save01->ldtr.selector, save01->ldtr.attrib,
3199 	       save01->ldtr.limit, save01->ldtr.base);
3200 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3201 	       "idtr:",
3202 	       save->idtr.selector, save->idtr.attrib,
3203 	       save->idtr.limit, save->idtr.base);
3204 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3205 	       "tr:",
3206 	       save01->tr.selector, save01->tr.attrib,
3207 	       save01->tr.limit, save01->tr.base);
3208 	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3209 	       save->vmpl, save->cpl, save->efer);
3210 	pr_err("%-15s %016llx %-13s %016llx\n",
3211 	       "cr0:", save->cr0, "cr2:", save->cr2);
3212 	pr_err("%-15s %016llx %-13s %016llx\n",
3213 	       "cr3:", save->cr3, "cr4:", save->cr4);
3214 	pr_err("%-15s %016llx %-13s %016llx\n",
3215 	       "dr6:", save->dr6, "dr7:", save->dr7);
3216 	pr_err("%-15s %016llx %-13s %016llx\n",
3217 	       "rip:", save->rip, "rflags:", save->rflags);
3218 	pr_err("%-15s %016llx %-13s %016llx\n",
3219 	       "rsp:", save->rsp, "rax:", save->rax);
3220 	pr_err("%-15s %016llx %-13s %016llx\n",
3221 	       "star:", save01->star, "lstar:", save01->lstar);
3222 	pr_err("%-15s %016llx %-13s %016llx\n",
3223 	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3224 	pr_err("%-15s %016llx %-13s %016llx\n",
3225 	       "kernel_gs_base:", save01->kernel_gs_base,
3226 	       "sysenter_cs:", save01->sysenter_cs);
3227 	pr_err("%-15s %016llx %-13s %016llx\n",
3228 	       "sysenter_esp:", save01->sysenter_esp,
3229 	       "sysenter_eip:", save01->sysenter_eip);
3230 	pr_err("%-15s %016llx %-13s %016llx\n",
3231 	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3232 	pr_err("%-15s %016llx %-13s %016llx\n",
3233 	       "br_from:", save->br_from, "br_to:", save->br_to);
3234 	pr_err("%-15s %016llx %-13s %016llx\n",
3235 	       "excp_from:", save->last_excp_from,
3236 	       "excp_to:", save->last_excp_to);
3237 }
3238 
3239 static bool svm_check_exit_valid(u64 exit_code)
3240 {
3241 	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3242 		svm_exit_handlers[exit_code]);
3243 }
3244 
3245 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3246 {
3247 	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3248 	dump_vmcb(vcpu);
3249 	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3250 	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3251 	vcpu->run->internal.ndata = 2;
3252 	vcpu->run->internal.data[0] = exit_code;
3253 	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3254 	return 0;
3255 }
3256 
3257 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3258 {
3259 	if (!svm_check_exit_valid(exit_code))
3260 		return svm_handle_invalid_exit(vcpu, exit_code);
3261 
3262 #ifdef CONFIG_RETPOLINE
3263 	if (exit_code == SVM_EXIT_MSR)
3264 		return msr_interception(vcpu);
3265 	else if (exit_code == SVM_EXIT_VINTR)
3266 		return interrupt_window_interception(vcpu);
3267 	else if (exit_code == SVM_EXIT_INTR)
3268 		return intr_interception(vcpu);
3269 	else if (exit_code == SVM_EXIT_HLT)
3270 		return kvm_emulate_halt(vcpu);
3271 	else if (exit_code == SVM_EXIT_NPF)
3272 		return npf_interception(vcpu);
3273 #endif
3274 	return svm_exit_handlers[exit_code](vcpu);
3275 }
3276 
3277 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3278 			      u64 *info1, u64 *info2,
3279 			      u32 *intr_info, u32 *error_code)
3280 {
3281 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3282 
3283 	*reason = control->exit_code;
3284 	*info1 = control->exit_info_1;
3285 	*info2 = control->exit_info_2;
3286 	*intr_info = control->exit_int_info;
3287 	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3288 	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3289 		*error_code = control->exit_int_info_err;
3290 	else
3291 		*error_code = 0;
3292 }
3293 
3294 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3295 {
3296 	struct vcpu_svm *svm = to_svm(vcpu);
3297 	struct kvm_run *kvm_run = vcpu->run;
3298 	u32 exit_code = svm->vmcb->control.exit_code;
3299 
3300 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
3301 
3302 	/* SEV-ES guests must use the CR write traps to track CR registers. */
3303 	if (!sev_es_guest(vcpu->kvm)) {
3304 		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3305 			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3306 		if (npt_enabled)
3307 			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3308 	}
3309 
3310 	if (is_guest_mode(vcpu)) {
3311 		int vmexit;
3312 
3313 		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3314 
3315 		vmexit = nested_svm_exit_special(svm);
3316 
3317 		if (vmexit == NESTED_EXIT_CONTINUE)
3318 			vmexit = nested_svm_exit_handled(svm);
3319 
3320 		if (vmexit == NESTED_EXIT_DONE)
3321 			return 1;
3322 	}
3323 
3324 	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3325 		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3326 		kvm_run->fail_entry.hardware_entry_failure_reason
3327 			= svm->vmcb->control.exit_code;
3328 		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3329 		dump_vmcb(vcpu);
3330 		return 0;
3331 	}
3332 
3333 	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3334 	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3335 	    exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3336 	    exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3337 		printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3338 		       "exit_code 0x%x\n",
3339 		       __func__, svm->vmcb->control.exit_int_info,
3340 		       exit_code);
3341 
3342 	if (exit_fastpath != EXIT_FASTPATH_NONE)
3343 		return 1;
3344 
3345 	return svm_invoke_exit_handler(vcpu, exit_code);
3346 }
3347 
3348 static void reload_tss(struct kvm_vcpu *vcpu)
3349 {
3350 	struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3351 
3352 	sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3353 	load_TR_desc();
3354 }
3355 
3356 static void pre_svm_run(struct kvm_vcpu *vcpu)
3357 {
3358 	struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3359 	struct vcpu_svm *svm = to_svm(vcpu);
3360 
3361 	/*
3362 	 * If the previous vmrun of the vmcb occurred on a different physical
3363 	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3364 	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3365 	 */
3366 	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3367 		svm->current_vmcb->asid_generation = 0;
3368 		vmcb_mark_all_dirty(svm->vmcb);
3369 		svm->current_vmcb->cpu = vcpu->cpu;
3370         }
3371 
3372 	if (sev_guest(vcpu->kvm))
3373 		return pre_sev_run(svm, vcpu->cpu);
3374 
3375 	/* FIXME: handle wraparound of asid_generation */
3376 	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3377 		new_asid(svm, sd);
3378 }
3379 
3380 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3381 {
3382 	struct vcpu_svm *svm = to_svm(vcpu);
3383 
3384 	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3385 	vcpu->arch.hflags |= HF_NMI_MASK;
3386 	if (!sev_es_guest(vcpu->kvm))
3387 		svm_set_intercept(svm, INTERCEPT_IRET);
3388 	++vcpu->stat.nmi_injections;
3389 }
3390 
3391 static void svm_inject_irq(struct kvm_vcpu *vcpu)
3392 {
3393 	struct vcpu_svm *svm = to_svm(vcpu);
3394 
3395 	BUG_ON(!(gif_set(svm)));
3396 
3397 	trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3398 	++vcpu->stat.irq_injections;
3399 
3400 	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3401 		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3402 }
3403 
3404 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3405 				     int trig_mode, int vector)
3406 {
3407 	/*
3408 	 * vcpu->arch.apicv_active must be read after vcpu->mode.
3409 	 * Pairs with smp_store_release in vcpu_enter_guest.
3410 	 */
3411 	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3412 
3413 	if (!READ_ONCE(vcpu->arch.apicv_active)) {
3414 		/* Process the interrupt via inject_pending_event */
3415 		kvm_make_request(KVM_REQ_EVENT, vcpu);
3416 		kvm_vcpu_kick(vcpu);
3417 		return;
3418 	}
3419 
3420 	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3421 	if (in_guest_mode) {
3422 		/*
3423 		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3424 		 * the vCPU exits the guest before the doorbell chimes, hardware
3425 		 * will automatically process AVIC interrupts at the next VMRUN.
3426 		 */
3427 		avic_ring_doorbell(vcpu);
3428 	} else {
3429 		/*
3430 		 * Wake the vCPU if it was blocking.  KVM will then detect the
3431 		 * pending IRQ when checking if the vCPU has a wake event.
3432 		 */
3433 		kvm_vcpu_wake_up(vcpu);
3434 	}
3435 }
3436 
3437 static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3438 				  int trig_mode, int vector)
3439 {
3440 	kvm_lapic_set_irr(vector, apic);
3441 
3442 	/*
3443 	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3444 	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3445 	 * the read of guest_mode.  This guarantees that either VMRUN will see
3446 	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3447 	 * will signal the doorbell if the CPU has already entered the guest.
3448 	 */
3449 	smp_mb__after_atomic();
3450 	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3451 }
3452 
3453 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3454 {
3455 	struct vcpu_svm *svm = to_svm(vcpu);
3456 
3457 	/*
3458 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3459 	 * tracking is done using the CR write traps.
3460 	 */
3461 	if (sev_es_guest(vcpu->kvm))
3462 		return;
3463 
3464 	if (nested_svm_virtualize_tpr(vcpu))
3465 		return;
3466 
3467 	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3468 
3469 	if (irr == -1)
3470 		return;
3471 
3472 	if (tpr >= irr)
3473 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3474 }
3475 
3476 bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3477 {
3478 	struct vcpu_svm *svm = to_svm(vcpu);
3479 	struct vmcb *vmcb = svm->vmcb;
3480 	bool ret;
3481 
3482 	if (!gif_set(svm))
3483 		return true;
3484 
3485 	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3486 		return false;
3487 
3488 	ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
3489 	      (vcpu->arch.hflags & HF_NMI_MASK);
3490 
3491 	return ret;
3492 }
3493 
3494 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3495 {
3496 	struct vcpu_svm *svm = to_svm(vcpu);
3497 	if (svm->nested.nested_run_pending)
3498 		return -EBUSY;
3499 
3500 	if (svm_nmi_blocked(vcpu))
3501 		return 0;
3502 
3503 	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3504 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3505 		return -EBUSY;
3506 	return 1;
3507 }
3508 
3509 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3510 {
3511 	return !!(vcpu->arch.hflags & HF_NMI_MASK);
3512 }
3513 
3514 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3515 {
3516 	struct vcpu_svm *svm = to_svm(vcpu);
3517 
3518 	if (masked) {
3519 		vcpu->arch.hflags |= HF_NMI_MASK;
3520 		if (!sev_es_guest(vcpu->kvm))
3521 			svm_set_intercept(svm, INTERCEPT_IRET);
3522 	} else {
3523 		vcpu->arch.hflags &= ~HF_NMI_MASK;
3524 		if (!sev_es_guest(vcpu->kvm))
3525 			svm_clr_intercept(svm, INTERCEPT_IRET);
3526 	}
3527 }
3528 
3529 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3530 {
3531 	struct vcpu_svm *svm = to_svm(vcpu);
3532 	struct vmcb *vmcb = svm->vmcb;
3533 
3534 	if (!gif_set(svm))
3535 		return true;
3536 
3537 	if (is_guest_mode(vcpu)) {
3538 		/* As long as interrupts are being delivered...  */
3539 		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3540 		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3541 		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3542 			return true;
3543 
3544 		/* ... vmexits aren't blocked by the interrupt shadow  */
3545 		if (nested_exit_on_intr(svm))
3546 			return false;
3547 	} else {
3548 		if (!svm_get_if_flag(vcpu))
3549 			return true;
3550 	}
3551 
3552 	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3553 }
3554 
3555 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3556 {
3557 	struct vcpu_svm *svm = to_svm(vcpu);
3558 
3559 	if (svm->nested.nested_run_pending)
3560 		return -EBUSY;
3561 
3562 	if (svm_interrupt_blocked(vcpu))
3563 		return 0;
3564 
3565 	/*
3566 	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3567 	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3568 	 */
3569 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3570 		return -EBUSY;
3571 
3572 	return 1;
3573 }
3574 
3575 static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3576 {
3577 	struct vcpu_svm *svm = to_svm(vcpu);
3578 
3579 	/*
3580 	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3581 	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3582 	 * get that intercept, this function will be called again though and
3583 	 * we'll get the vintr intercept. However, if the vGIF feature is
3584 	 * enabled, the STGI interception will not occur. Enable the irq
3585 	 * window under the assumption that the hardware will set the GIF.
3586 	 */
3587 	if (vgif || gif_set(svm)) {
3588 		/*
3589 		 * IRQ window is not needed when AVIC is enabled,
3590 		 * unless we have pending ExtINT since it cannot be injected
3591 		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3592 		 * and fallback to injecting IRQ via V_IRQ.
3593 		 *
3594 		 * If running nested, AVIC is already locally inhibited
3595 		 * on this vCPU, therefore there is no need to request
3596 		 * the VM wide AVIC inhibition.
3597 		 */
3598 		if (!is_guest_mode(vcpu))
3599 			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3600 
3601 		svm_set_vintr(svm);
3602 	}
3603 }
3604 
3605 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3606 {
3607 	struct vcpu_svm *svm = to_svm(vcpu);
3608 
3609 	if ((vcpu->arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) == HF_NMI_MASK)
3610 		return; /* IRET will cause a vm exit */
3611 
3612 	if (!gif_set(svm)) {
3613 		if (vgif)
3614 			svm_set_intercept(svm, INTERCEPT_STGI);
3615 		return; /* STGI will cause a vm exit */
3616 	}
3617 
3618 	/*
3619 	 * Something prevents NMI from been injected. Single step over possible
3620 	 * problem (IRET or exception injection or interrupt shadow)
3621 	 */
3622 	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3623 	svm->nmi_singlestep = true;
3624 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3625 }
3626 
3627 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
3628 {
3629 	struct vcpu_svm *svm = to_svm(vcpu);
3630 
3631 	/*
3632 	 * Flush only the current ASID even if the TLB flush was invoked via
3633 	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3634 	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3635 	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3636 	 * VM-Exit (via kvm_mmu_reset_context()).
3637 	 */
3638 	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3639 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3640 	else
3641 		svm->current_vmcb->asid_generation--;
3642 }
3643 
3644 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3645 {
3646 	struct vcpu_svm *svm = to_svm(vcpu);
3647 
3648 	invlpga(gva, svm->vmcb->control.asid);
3649 }
3650 
3651 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3652 {
3653 	struct vcpu_svm *svm = to_svm(vcpu);
3654 
3655 	if (nested_svm_virtualize_tpr(vcpu))
3656 		return;
3657 
3658 	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3659 		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3660 		kvm_set_cr8(vcpu, cr8);
3661 	}
3662 }
3663 
3664 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3665 {
3666 	struct vcpu_svm *svm = to_svm(vcpu);
3667 	u64 cr8;
3668 
3669 	if (nested_svm_virtualize_tpr(vcpu) ||
3670 	    kvm_vcpu_apicv_active(vcpu))
3671 		return;
3672 
3673 	cr8 = kvm_get_cr8(vcpu);
3674 	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3675 	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3676 }
3677 
3678 static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
3679 {
3680 	struct vcpu_svm *svm = to_svm(vcpu);
3681 	u8 vector;
3682 	int type;
3683 	u32 exitintinfo = svm->vmcb->control.exit_int_info;
3684 	unsigned int3_injected = svm->int3_injected;
3685 
3686 	svm->int3_injected = 0;
3687 
3688 	/*
3689 	 * If we've made progress since setting HF_IRET_MASK, we've
3690 	 * executed an IRET and can allow NMI injection.
3691 	 */
3692 	if ((vcpu->arch.hflags & HF_IRET_MASK) &&
3693 	    (sev_es_guest(vcpu->kvm) ||
3694 	     kvm_rip_read(vcpu) != svm->nmi_iret_rip)) {
3695 		vcpu->arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3696 		kvm_make_request(KVM_REQ_EVENT, vcpu);
3697 	}
3698 
3699 	vcpu->arch.nmi_injected = false;
3700 	kvm_clear_exception_queue(vcpu);
3701 	kvm_clear_interrupt_queue(vcpu);
3702 
3703 	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3704 		return;
3705 
3706 	kvm_make_request(KVM_REQ_EVENT, vcpu);
3707 
3708 	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3709 	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3710 
3711 	switch (type) {
3712 	case SVM_EXITINTINFO_TYPE_NMI:
3713 		vcpu->arch.nmi_injected = true;
3714 		break;
3715 	case SVM_EXITINTINFO_TYPE_EXEPT:
3716 		/*
3717 		 * Never re-inject a #VC exception.
3718 		 */
3719 		if (vector == X86_TRAP_VC)
3720 			break;
3721 
3722 		/*
3723 		 * In case of software exceptions, do not reinject the vector,
3724 		 * but re-execute the instruction instead. Rewind RIP first
3725 		 * if we emulated INT3 before.
3726 		 */
3727 		if (kvm_exception_is_soft(vector)) {
3728 			if (vector == BP_VECTOR && int3_injected &&
3729 			    kvm_is_linear_rip(vcpu, svm->int3_rip))
3730 				kvm_rip_write(vcpu,
3731 					      kvm_rip_read(vcpu) - int3_injected);
3732 			break;
3733 		}
3734 		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3735 			u32 err = svm->vmcb->control.exit_int_info_err;
3736 			kvm_requeue_exception_e(vcpu, vector, err);
3737 
3738 		} else
3739 			kvm_requeue_exception(vcpu, vector);
3740 		break;
3741 	case SVM_EXITINTINFO_TYPE_INTR:
3742 		kvm_queue_interrupt(vcpu, vector, false);
3743 		break;
3744 	default:
3745 		break;
3746 	}
3747 }
3748 
3749 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3750 {
3751 	struct vcpu_svm *svm = to_svm(vcpu);
3752 	struct vmcb_control_area *control = &svm->vmcb->control;
3753 
3754 	control->exit_int_info = control->event_inj;
3755 	control->exit_int_info_err = control->event_inj_err;
3756 	control->event_inj = 0;
3757 	svm_complete_interrupts(vcpu);
3758 }
3759 
3760 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
3761 {
3762 	return 1;
3763 }
3764 
3765 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
3766 {
3767 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
3768 	    to_svm(vcpu)->vmcb->control.exit_info_1)
3769 		return handle_fastpath_set_msr_irqoff(vcpu);
3770 
3771 	return EXIT_FASTPATH_NONE;
3772 }
3773 
3774 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
3775 {
3776 	struct vcpu_svm *svm = to_svm(vcpu);
3777 	unsigned long vmcb_pa = svm->current_vmcb->pa;
3778 
3779 	guest_state_enter_irqoff();
3780 
3781 	if (sev_es_guest(vcpu->kvm)) {
3782 		__svm_sev_es_vcpu_run(vmcb_pa);
3783 	} else {
3784 		struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3785 
3786 		/*
3787 		 * Use a single vmcb (vmcb01 because it's always valid) for
3788 		 * context switching guest state via VMLOAD/VMSAVE, that way
3789 		 * the state doesn't need to be copied between vmcb01 and
3790 		 * vmcb02 when switching vmcbs for nested virtualization.
3791 		 */
3792 		vmload(svm->vmcb01.pa);
3793 		__svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs);
3794 		vmsave(svm->vmcb01.pa);
3795 
3796 		vmload(__sme_page_pa(sd->save_area));
3797 	}
3798 
3799 	guest_state_exit_irqoff();
3800 }
3801 
3802 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
3803 {
3804 	struct vcpu_svm *svm = to_svm(vcpu);
3805 
3806 	trace_kvm_entry(vcpu);
3807 
3808 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3809 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3810 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3811 
3812 	/*
3813 	 * Disable singlestep if we're injecting an interrupt/exception.
3814 	 * We don't want our modified rflags to be pushed on the stack where
3815 	 * we might not be able to easily reset them if we disabled NMI
3816 	 * singlestep later.
3817 	 */
3818 	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
3819 		/*
3820 		 * Event injection happens before external interrupts cause a
3821 		 * vmexit and interrupts are disabled here, so smp_send_reschedule
3822 		 * is enough to force an immediate vmexit.
3823 		 */
3824 		disable_nmi_singlestep(svm);
3825 		smp_send_reschedule(vcpu->cpu);
3826 	}
3827 
3828 	pre_svm_run(vcpu);
3829 
3830 	sync_lapic_to_cr8(vcpu);
3831 
3832 	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
3833 		svm->vmcb->control.asid = svm->asid;
3834 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
3835 	}
3836 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
3837 
3838 	svm_hv_update_vp_id(svm->vmcb, vcpu);
3839 
3840 	/*
3841 	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
3842 	 * of a #DB.
3843 	 */
3844 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3845 		svm_set_dr6(svm, vcpu->arch.dr6);
3846 	else
3847 		svm_set_dr6(svm, DR6_ACTIVE_LOW);
3848 
3849 	clgi();
3850 	kvm_load_guest_xsave_state(vcpu);
3851 
3852 	kvm_wait_lapic_expire(vcpu);
3853 
3854 	/*
3855 	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
3856 	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
3857 	 * is no need to worry about the conditional branch over the wrmsr
3858 	 * being speculatively taken.
3859 	 */
3860 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3861 		x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
3862 
3863 	svm_vcpu_enter_exit(vcpu);
3864 
3865 	/*
3866 	 * We do not use IBRS in the kernel. If this vCPU has used the
3867 	 * SPEC_CTRL MSR it may have left it on; save the value and
3868 	 * turn it off. This is much more efficient than blindly adding
3869 	 * it to the atomic save/restore list. Especially as the former
3870 	 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
3871 	 *
3872 	 * For non-nested case:
3873 	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
3874 	 * save it.
3875 	 *
3876 	 * For nested case:
3877 	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
3878 	 * save it.
3879 	 */
3880 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) &&
3881 	    unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
3882 		svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
3883 
3884 	if (!sev_es_guest(vcpu->kvm))
3885 		reload_tss(vcpu);
3886 
3887 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3888 		x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
3889 
3890 	if (!sev_es_guest(vcpu->kvm)) {
3891 		vcpu->arch.cr2 = svm->vmcb->save.cr2;
3892 		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3893 		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3894 		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3895 	}
3896 	vcpu->arch.regs_dirty = 0;
3897 
3898 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3899 		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
3900 
3901 	kvm_load_host_xsave_state(vcpu);
3902 	stgi();
3903 
3904 	/* Any pending NMI will happen here */
3905 
3906 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3907 		kvm_after_interrupt(vcpu);
3908 
3909 	sync_cr8_to_lapic(vcpu);
3910 
3911 	svm->next_rip = 0;
3912 	if (is_guest_mode(vcpu)) {
3913 		nested_sync_control_from_vmcb02(svm);
3914 
3915 		/* Track VMRUNs that have made past consistency checking */
3916 		if (svm->nested.nested_run_pending &&
3917 		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
3918                         ++vcpu->stat.nested_run;
3919 
3920 		svm->nested.nested_run_pending = 0;
3921 	}
3922 
3923 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3924 	vmcb_mark_all_clean(svm->vmcb);
3925 
3926 	/* if exit due to PF check for async PF */
3927 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3928 		vcpu->arch.apf.host_apf_flags =
3929 			kvm_read_and_reset_apf_flags();
3930 
3931 	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
3932 
3933 	/*
3934 	 * We need to handle MC intercepts here before the vcpu has a chance to
3935 	 * change the physical cpu
3936 	 */
3937 	if (unlikely(svm->vmcb->control.exit_code ==
3938 		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
3939 		svm_handle_mce(vcpu);
3940 
3941 	svm_complete_interrupts(vcpu);
3942 
3943 	if (is_guest_mode(vcpu))
3944 		return EXIT_FASTPATH_NONE;
3945 
3946 	return svm_exit_handlers_fastpath(vcpu);
3947 }
3948 
3949 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
3950 			     int root_level)
3951 {
3952 	struct vcpu_svm *svm = to_svm(vcpu);
3953 	unsigned long cr3;
3954 
3955 	if (npt_enabled) {
3956 		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
3957 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3958 
3959 		hv_track_root_tdp(vcpu, root_hpa);
3960 
3961 		cr3 = vcpu->arch.cr3;
3962 	} else if (vcpu->arch.mmu->root_role.level >= PT64_ROOT_4LEVEL) {
3963 		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
3964 	} else {
3965 		/* PCID in the guest should be impossible with a 32-bit MMU. */
3966 		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
3967 		cr3 = root_hpa;
3968 	}
3969 
3970 	svm->vmcb->save.cr3 = cr3;
3971 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
3972 }
3973 
3974 static int is_disabled(void)
3975 {
3976 	u64 vm_cr;
3977 
3978 	rdmsrl(MSR_VM_CR, vm_cr);
3979 	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
3980 		return 1;
3981 
3982 	return 0;
3983 }
3984 
3985 static void
3986 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3987 {
3988 	/*
3989 	 * Patch in the VMMCALL instruction:
3990 	 */
3991 	hypercall[0] = 0x0f;
3992 	hypercall[1] = 0x01;
3993 	hypercall[2] = 0xd9;
3994 }
3995 
3996 static int __init svm_check_processor_compat(void)
3997 {
3998 	return 0;
3999 }
4000 
4001 /*
4002  * The kvm parameter can be NULL (module initialization, or invocation before
4003  * VM creation). Be sure to check the kvm parameter before using it.
4004  */
4005 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4006 {
4007 	switch (index) {
4008 	case MSR_IA32_MCG_EXT_CTL:
4009 	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
4010 		return false;
4011 	case MSR_IA32_SMBASE:
4012 		/* SEV-ES guests do not support SMM, so report false */
4013 		if (kvm && sev_es_guest(kvm))
4014 			return false;
4015 		break;
4016 	default:
4017 		break;
4018 	}
4019 
4020 	return true;
4021 }
4022 
4023 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4024 {
4025 	return 0;
4026 }
4027 
4028 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4029 {
4030 	struct vcpu_svm *svm = to_svm(vcpu);
4031 	struct kvm_cpuid_entry2 *best;
4032 	struct kvm *kvm = vcpu->kvm;
4033 
4034 	vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
4035 				    boot_cpu_has(X86_FEATURE_XSAVE) &&
4036 				    boot_cpu_has(X86_FEATURE_XSAVES);
4037 
4038 	/* Update nrips enabled cache */
4039 	svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) &&
4040 			     guest_cpuid_has(vcpu, X86_FEATURE_NRIPS);
4041 
4042 	svm->tsc_scaling_enabled = tsc_scaling && guest_cpuid_has(vcpu, X86_FEATURE_TSCRATEMSR);
4043 	svm->lbrv_enabled = lbrv && guest_cpuid_has(vcpu, X86_FEATURE_LBRV);
4044 
4045 	svm->v_vmload_vmsave_enabled = vls && guest_cpuid_has(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4046 
4047 	svm->pause_filter_enabled = kvm_cpu_cap_has(X86_FEATURE_PAUSEFILTER) &&
4048 			guest_cpuid_has(vcpu, X86_FEATURE_PAUSEFILTER);
4049 
4050 	svm->pause_threshold_enabled = kvm_cpu_cap_has(X86_FEATURE_PFTHRESHOLD) &&
4051 			guest_cpuid_has(vcpu, X86_FEATURE_PFTHRESHOLD);
4052 
4053 	svm->vgif_enabled = vgif && guest_cpuid_has(vcpu, X86_FEATURE_VGIF);
4054 
4055 	svm_recalc_instruction_intercepts(vcpu, svm);
4056 
4057 	/* For sev guests, the memory encryption bit is not reserved in CR3.  */
4058 	if (sev_guest(vcpu->kvm)) {
4059 		best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);
4060 		if (best)
4061 			vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
4062 	}
4063 
4064 	if (kvm_vcpu_apicv_active(vcpu)) {
4065 		/*
4066 		 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
4067 		 * is exposed to the guest, disable AVIC.
4068 		 */
4069 		if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
4070 			kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_X2APIC);
4071 	}
4072 	init_vmcb_after_set_cpuid(vcpu);
4073 }
4074 
4075 static bool svm_has_wbinvd_exit(void)
4076 {
4077 	return true;
4078 }
4079 
4080 #define PRE_EX(exit)  { .exit_code = (exit), \
4081 			.stage = X86_ICPT_PRE_EXCEPT, }
4082 #define POST_EX(exit) { .exit_code = (exit), \
4083 			.stage = X86_ICPT_POST_EXCEPT, }
4084 #define POST_MEM(exit) { .exit_code = (exit), \
4085 			.stage = X86_ICPT_POST_MEMACCESS, }
4086 
4087 static const struct __x86_intercept {
4088 	u32 exit_code;
4089 	enum x86_intercept_stage stage;
4090 } x86_intercept_map[] = {
4091 	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4092 	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4093 	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4094 	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4095 	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4096 	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4097 	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4098 	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4099 	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4100 	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4101 	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4102 	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4103 	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4104 	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4105 	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4106 	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4107 	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4108 	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4109 	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4110 	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4111 	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4112 	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4113 	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4114 	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4115 	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4116 	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4117 	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4118 	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4119 	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4120 	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4121 	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4122 	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4123 	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4124 	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4125 	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4126 	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4127 	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4128 	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4129 	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4130 	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4131 	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4132 	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4133 	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4134 	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4135 	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4136 	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4137 	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4138 };
4139 
4140 #undef PRE_EX
4141 #undef POST_EX
4142 #undef POST_MEM
4143 
4144 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4145 			       struct x86_instruction_info *info,
4146 			       enum x86_intercept_stage stage,
4147 			       struct x86_exception *exception)
4148 {
4149 	struct vcpu_svm *svm = to_svm(vcpu);
4150 	int vmexit, ret = X86EMUL_CONTINUE;
4151 	struct __x86_intercept icpt_info;
4152 	struct vmcb *vmcb = svm->vmcb;
4153 
4154 	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4155 		goto out;
4156 
4157 	icpt_info = x86_intercept_map[info->intercept];
4158 
4159 	if (stage != icpt_info.stage)
4160 		goto out;
4161 
4162 	switch (icpt_info.exit_code) {
4163 	case SVM_EXIT_READ_CR0:
4164 		if (info->intercept == x86_intercept_cr_read)
4165 			icpt_info.exit_code += info->modrm_reg;
4166 		break;
4167 	case SVM_EXIT_WRITE_CR0: {
4168 		unsigned long cr0, val;
4169 
4170 		if (info->intercept == x86_intercept_cr_write)
4171 			icpt_info.exit_code += info->modrm_reg;
4172 
4173 		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4174 		    info->intercept == x86_intercept_clts)
4175 			break;
4176 
4177 		if (!(vmcb12_is_intercept(&svm->nested.ctl,
4178 					INTERCEPT_SELECTIVE_CR0)))
4179 			break;
4180 
4181 		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4182 		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4183 
4184 		if (info->intercept == x86_intercept_lmsw) {
4185 			cr0 &= 0xfUL;
4186 			val &= 0xfUL;
4187 			/* lmsw can't clear PE - catch this here */
4188 			if (cr0 & X86_CR0_PE)
4189 				val |= X86_CR0_PE;
4190 		}
4191 
4192 		if (cr0 ^ val)
4193 			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4194 
4195 		break;
4196 	}
4197 	case SVM_EXIT_READ_DR0:
4198 	case SVM_EXIT_WRITE_DR0:
4199 		icpt_info.exit_code += info->modrm_reg;
4200 		break;
4201 	case SVM_EXIT_MSR:
4202 		if (info->intercept == x86_intercept_wrmsr)
4203 			vmcb->control.exit_info_1 = 1;
4204 		else
4205 			vmcb->control.exit_info_1 = 0;
4206 		break;
4207 	case SVM_EXIT_PAUSE:
4208 		/*
4209 		 * We get this for NOP only, but pause
4210 		 * is rep not, check this here
4211 		 */
4212 		if (info->rep_prefix != REPE_PREFIX)
4213 			goto out;
4214 		break;
4215 	case SVM_EXIT_IOIO: {
4216 		u64 exit_info;
4217 		u32 bytes;
4218 
4219 		if (info->intercept == x86_intercept_in ||
4220 		    info->intercept == x86_intercept_ins) {
4221 			exit_info = ((info->src_val & 0xffff) << 16) |
4222 				SVM_IOIO_TYPE_MASK;
4223 			bytes = info->dst_bytes;
4224 		} else {
4225 			exit_info = (info->dst_val & 0xffff) << 16;
4226 			bytes = info->src_bytes;
4227 		}
4228 
4229 		if (info->intercept == x86_intercept_outs ||
4230 		    info->intercept == x86_intercept_ins)
4231 			exit_info |= SVM_IOIO_STR_MASK;
4232 
4233 		if (info->rep_prefix)
4234 			exit_info |= SVM_IOIO_REP_MASK;
4235 
4236 		bytes = min(bytes, 4u);
4237 
4238 		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4239 
4240 		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4241 
4242 		vmcb->control.exit_info_1 = exit_info;
4243 		vmcb->control.exit_info_2 = info->next_rip;
4244 
4245 		break;
4246 	}
4247 	default:
4248 		break;
4249 	}
4250 
4251 	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4252 	if (static_cpu_has(X86_FEATURE_NRIPS))
4253 		vmcb->control.next_rip  = info->next_rip;
4254 	vmcb->control.exit_code = icpt_info.exit_code;
4255 	vmexit = nested_svm_exit_handled(svm);
4256 
4257 	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4258 					   : X86EMUL_CONTINUE;
4259 
4260 out:
4261 	return ret;
4262 }
4263 
4264 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4265 {
4266 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
4267 		vcpu->arch.at_instruction_boundary = true;
4268 }
4269 
4270 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4271 {
4272 	if (!kvm_pause_in_guest(vcpu->kvm))
4273 		shrink_ple_window(vcpu);
4274 }
4275 
4276 static void svm_setup_mce(struct kvm_vcpu *vcpu)
4277 {
4278 	/* [63:9] are reserved. */
4279 	vcpu->arch.mcg_cap &= 0x1ff;
4280 }
4281 
4282 bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4283 {
4284 	struct vcpu_svm *svm = to_svm(vcpu);
4285 
4286 	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4287 	if (!gif_set(svm))
4288 		return true;
4289 
4290 	return is_smm(vcpu);
4291 }
4292 
4293 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4294 {
4295 	struct vcpu_svm *svm = to_svm(vcpu);
4296 	if (svm->nested.nested_run_pending)
4297 		return -EBUSY;
4298 
4299 	if (svm_smi_blocked(vcpu))
4300 		return 0;
4301 
4302 	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4303 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4304 		return -EBUSY;
4305 
4306 	return 1;
4307 }
4308 
4309 static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
4310 {
4311 	struct vcpu_svm *svm = to_svm(vcpu);
4312 	struct kvm_host_map map_save;
4313 	int ret;
4314 
4315 	if (!is_guest_mode(vcpu))
4316 		return 0;
4317 
4318 	/* FED8h - SVM Guest */
4319 	put_smstate(u64, smstate, 0x7ed8, 1);
4320 	/* FEE0h - SVM Guest VMCB Physical Address */
4321 	put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa);
4322 
4323 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4324 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4325 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4326 
4327 	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4328 	if (ret)
4329 		return ret;
4330 
4331 	/*
4332 	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4333 	 * VMCB01 is going to be used during SMM and thus the state will
4334 	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4335 	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4336 	 * format of the area is identical to guest save area offsetted
4337 	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4338 	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4339 	 * L1 hypervisor to save additional host context (e.g. KVM does
4340 	 * that, see svm_prepare_switch_to_guest()) which must be
4341 	 * preserved.
4342 	 */
4343 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr),
4344 			 &map_save) == -EINVAL)
4345 		return 1;
4346 
4347 	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4348 
4349 	svm_copy_vmrun_state(map_save.hva + 0x400,
4350 			     &svm->vmcb01.ptr->save);
4351 
4352 	kvm_vcpu_unmap(vcpu, &map_save, true);
4353 	return 0;
4354 }
4355 
4356 static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
4357 {
4358 	struct vcpu_svm *svm = to_svm(vcpu);
4359 	struct kvm_host_map map, map_save;
4360 	u64 saved_efer, vmcb12_gpa;
4361 	struct vmcb *vmcb12;
4362 	int ret;
4363 
4364 	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4365 		return 0;
4366 
4367 	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4368 	if (!GET_SMSTATE(u64, smstate, 0x7ed8))
4369 		return 0;
4370 
4371 	if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4372 		return 1;
4373 
4374 	saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0);
4375 	if (!(saved_efer & EFER_SVME))
4376 		return 1;
4377 
4378 	vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0);
4379 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL)
4380 		return 1;
4381 
4382 	ret = 1;
4383 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save) == -EINVAL)
4384 		goto unmap_map;
4385 
4386 	if (svm_allocate_nested(svm))
4387 		goto unmap_save;
4388 
4389 	/*
4390 	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4391 	 * used during SMM (see svm_enter_smm())
4392 	 */
4393 
4394 	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4395 
4396 	/*
4397 	 * Enter the nested guest now
4398 	 */
4399 
4400 	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4401 
4402 	vmcb12 = map.hva;
4403 	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4404 	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4405 	ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false);
4406 
4407 	if (ret)
4408 		goto unmap_save;
4409 
4410 	svm->nested.nested_run_pending = 1;
4411 
4412 unmap_save:
4413 	kvm_vcpu_unmap(vcpu, &map_save, true);
4414 unmap_map:
4415 	kvm_vcpu_unmap(vcpu, &map, true);
4416 	return ret;
4417 }
4418 
4419 static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4420 {
4421 	struct vcpu_svm *svm = to_svm(vcpu);
4422 
4423 	if (!gif_set(svm)) {
4424 		if (vgif)
4425 			svm_set_intercept(svm, INTERCEPT_STGI);
4426 		/* STGI will cause a vm exit */
4427 	} else {
4428 		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4429 	}
4430 }
4431 
4432 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4433 					void *insn, int insn_len)
4434 {
4435 	bool smep, smap, is_user;
4436 	unsigned long cr4;
4437 	u64 error_code;
4438 
4439 	/* Emulation is always possible when KVM has access to all guest state. */
4440 	if (!sev_guest(vcpu->kvm))
4441 		return true;
4442 
4443 	/* #UD and #GP should never be intercepted for SEV guests. */
4444 	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4445 				  EMULTYPE_TRAP_UD_FORCED |
4446 				  EMULTYPE_VMWARE_GP));
4447 
4448 	/*
4449 	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4450 	 * to guest register state.
4451 	 */
4452 	if (sev_es_guest(vcpu->kvm))
4453 		return false;
4454 
4455 	/*
4456 	 * Emulation is possible if the instruction is already decoded, e.g.
4457 	 * when completing I/O after returning from userspace.
4458 	 */
4459 	if (emul_type & EMULTYPE_NO_DECODE)
4460 		return true;
4461 
4462 	/*
4463 	 * Emulation is possible for SEV guests if and only if a prefilled
4464 	 * buffer containing the bytes of the intercepted instruction is
4465 	 * available. SEV guest memory is encrypted with a guest specific key
4466 	 * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and
4467 	 * decode garbage.
4468 	 *
4469 	 * Inject #UD if KVM reached this point without an instruction buffer.
4470 	 * In practice, this path should never be hit by a well-behaved guest,
4471 	 * e.g. KVM doesn't intercept #UD or #GP for SEV guests, but this path
4472 	 * is still theoretically reachable, e.g. via unaccelerated fault-like
4473 	 * AVIC access, and needs to be handled by KVM to avoid putting the
4474 	 * guest into an infinite loop.   Injecting #UD is somewhat arbitrary,
4475 	 * but its the least awful option given lack of insight into the guest.
4476 	 */
4477 	if (unlikely(!insn)) {
4478 		kvm_queue_exception(vcpu, UD_VECTOR);
4479 		return false;
4480 	}
4481 
4482 	/*
4483 	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4484 	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4485 	 * the faulting instruction because the code fetch itself faulted, e.g.
4486 	 * the guest attempted to fetch from emulated MMIO or a guest page
4487 	 * table used to translate CS:RIP resides in emulated MMIO.
4488 	 */
4489 	if (likely(insn_len))
4490 		return true;
4491 
4492 	/*
4493 	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4494 	 *
4495 	 * Errata:
4496 	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4497 	 * possible that CPU microcode implementing DecodeAssist will fail to
4498 	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4499 	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
4500 	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
4501 	 * gives up and does not fill the instruction bytes buffer.
4502 	 *
4503 	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4504 	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4505 	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4506 	 * GuestIntrBytes field of the VMCB.
4507 	 *
4508 	 * This does _not_ mean that the erratum has been encountered, as the
4509 	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4510 	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4511 	 * encountered a reserved/not-present #PF.
4512 	 *
4513 	 * To hit the erratum, the following conditions must be true:
4514 	 *    1. CR4.SMAP=1 (obviously).
4515 	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
4516 	 *       have been hit as the guest would have encountered a SMEP
4517 	 *       violation #PF, not a #NPF.
4518 	 *    3. The #NPF is not due to a code fetch, in which case failure to
4519 	 *       retrieve the instruction bytes is legitimate (see abvoe).
4520 	 *
4521 	 * In addition, don't apply the erratum workaround if the #NPF occurred
4522 	 * while translating guest page tables (see below).
4523 	 */
4524 	error_code = to_svm(vcpu)->vmcb->control.exit_info_1;
4525 	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
4526 		goto resume_guest;
4527 
4528 	cr4 = kvm_read_cr4(vcpu);
4529 	smep = cr4 & X86_CR4_SMEP;
4530 	smap = cr4 & X86_CR4_SMAP;
4531 	is_user = svm_get_cpl(vcpu) == 3;
4532 	if (smap && (!smep || is_user)) {
4533 		pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
4534 
4535 		/*
4536 		 * If the fault occurred in userspace, arbitrarily inject #GP
4537 		 * to avoid killing the guest and to hopefully avoid confusing
4538 		 * the guest kernel too much, e.g. injecting #PF would not be
4539 		 * coherent with respect to the guest's page tables.  Request
4540 		 * triple fault if the fault occurred in the kernel as there's
4541 		 * no fault that KVM can inject without confusing the guest.
4542 		 * In practice, the triple fault is moot as no sane SEV kernel
4543 		 * will execute from user memory while also running with SMAP=1.
4544 		 */
4545 		if (is_user)
4546 			kvm_inject_gp(vcpu, 0);
4547 		else
4548 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4549 	}
4550 
4551 resume_guest:
4552 	/*
4553 	 * If the erratum was not hit, simply resume the guest and let it fault
4554 	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
4555 	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
4556 	 * userspace will kill the guest, and letting the emulator read garbage
4557 	 * will yield random behavior and potentially corrupt the guest.
4558 	 *
4559 	 * Simply resuming the guest is technically not a violation of the SEV
4560 	 * architecture.  AMD's APM states that all code fetches and page table
4561 	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
4562 	 * APM also states that encrypted accesses to MMIO are "ignored", but
4563 	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4564 	 * the guest spin is technically "ignoring" the access.
4565 	 */
4566 	return false;
4567 }
4568 
4569 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4570 {
4571 	struct vcpu_svm *svm = to_svm(vcpu);
4572 
4573 	/*
4574 	 * TODO: Last condition latch INIT signals on vCPU when
4575 	 * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
4576 	 * To properly emulate the INIT intercept,
4577 	 * svm_check_nested_events() should call nested_svm_vmexit()
4578 	 * if an INIT signal is pending.
4579 	 */
4580 	return !gif_set(svm) ||
4581 		   (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT));
4582 }
4583 
4584 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4585 {
4586 	if (!sev_es_guest(vcpu->kvm))
4587 		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4588 
4589 	sev_vcpu_deliver_sipi_vector(vcpu, vector);
4590 }
4591 
4592 static void svm_vm_destroy(struct kvm *kvm)
4593 {
4594 	avic_vm_destroy(kvm);
4595 	sev_vm_destroy(kvm);
4596 }
4597 
4598 static int svm_vm_init(struct kvm *kvm)
4599 {
4600 	if (!pause_filter_count || !pause_filter_thresh)
4601 		kvm->arch.pause_in_guest = true;
4602 
4603 	if (enable_apicv) {
4604 		int ret = avic_vm_init(kvm);
4605 		if (ret)
4606 			return ret;
4607 	}
4608 
4609 	return 0;
4610 }
4611 
4612 static struct kvm_x86_ops svm_x86_ops __initdata = {
4613 	.name = "kvm_amd",
4614 
4615 	.hardware_unsetup = svm_hardware_unsetup,
4616 	.hardware_enable = svm_hardware_enable,
4617 	.hardware_disable = svm_hardware_disable,
4618 	.has_emulated_msr = svm_has_emulated_msr,
4619 
4620 	.vcpu_create = svm_vcpu_create,
4621 	.vcpu_free = svm_vcpu_free,
4622 	.vcpu_reset = svm_vcpu_reset,
4623 
4624 	.vm_size = sizeof(struct kvm_svm),
4625 	.vm_init = svm_vm_init,
4626 	.vm_destroy = svm_vm_destroy,
4627 
4628 	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
4629 	.vcpu_load = svm_vcpu_load,
4630 	.vcpu_put = svm_vcpu_put,
4631 	.vcpu_blocking = avic_vcpu_blocking,
4632 	.vcpu_unblocking = avic_vcpu_unblocking,
4633 
4634 	.update_exception_bitmap = svm_update_exception_bitmap,
4635 	.get_msr_feature = svm_get_msr_feature,
4636 	.get_msr = svm_get_msr,
4637 	.set_msr = svm_set_msr,
4638 	.get_segment_base = svm_get_segment_base,
4639 	.get_segment = svm_get_segment,
4640 	.set_segment = svm_set_segment,
4641 	.get_cpl = svm_get_cpl,
4642 	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
4643 	.set_cr0 = svm_set_cr0,
4644 	.post_set_cr3 = sev_post_set_cr3,
4645 	.is_valid_cr4 = svm_is_valid_cr4,
4646 	.set_cr4 = svm_set_cr4,
4647 	.set_efer = svm_set_efer,
4648 	.get_idt = svm_get_idt,
4649 	.set_idt = svm_set_idt,
4650 	.get_gdt = svm_get_gdt,
4651 	.set_gdt = svm_set_gdt,
4652 	.set_dr7 = svm_set_dr7,
4653 	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4654 	.cache_reg = svm_cache_reg,
4655 	.get_rflags = svm_get_rflags,
4656 	.set_rflags = svm_set_rflags,
4657 	.get_if_flag = svm_get_if_flag,
4658 
4659 	.flush_tlb_all = svm_flush_tlb_current,
4660 	.flush_tlb_current = svm_flush_tlb_current,
4661 	.flush_tlb_gva = svm_flush_tlb_gva,
4662 	.flush_tlb_guest = svm_flush_tlb_current,
4663 
4664 	.vcpu_pre_run = svm_vcpu_pre_run,
4665 	.vcpu_run = svm_vcpu_run,
4666 	.handle_exit = svm_handle_exit,
4667 	.skip_emulated_instruction = svm_skip_emulated_instruction,
4668 	.update_emulated_instruction = NULL,
4669 	.set_interrupt_shadow = svm_set_interrupt_shadow,
4670 	.get_interrupt_shadow = svm_get_interrupt_shadow,
4671 	.patch_hypercall = svm_patch_hypercall,
4672 	.inject_irq = svm_inject_irq,
4673 	.inject_nmi = svm_inject_nmi,
4674 	.queue_exception = svm_queue_exception,
4675 	.cancel_injection = svm_cancel_injection,
4676 	.interrupt_allowed = svm_interrupt_allowed,
4677 	.nmi_allowed = svm_nmi_allowed,
4678 	.get_nmi_mask = svm_get_nmi_mask,
4679 	.set_nmi_mask = svm_set_nmi_mask,
4680 	.enable_nmi_window = svm_enable_nmi_window,
4681 	.enable_irq_window = svm_enable_irq_window,
4682 	.update_cr8_intercept = svm_update_cr8_intercept,
4683 	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
4684 	.check_apicv_inhibit_reasons = avic_check_apicv_inhibit_reasons,
4685 	.apicv_post_state_restore = avic_apicv_post_state_restore,
4686 
4687 	.get_mt_mask = svm_get_mt_mask,
4688 	.get_exit_info = svm_get_exit_info,
4689 
4690 	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
4691 
4692 	.has_wbinvd_exit = svm_has_wbinvd_exit,
4693 
4694 	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
4695 	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
4696 	.write_tsc_offset = svm_write_tsc_offset,
4697 	.write_tsc_multiplier = svm_write_tsc_multiplier,
4698 
4699 	.load_mmu_pgd = svm_load_mmu_pgd,
4700 
4701 	.check_intercept = svm_check_intercept,
4702 	.handle_exit_irqoff = svm_handle_exit_irqoff,
4703 
4704 	.request_immediate_exit = __kvm_request_immediate_exit,
4705 
4706 	.sched_in = svm_sched_in,
4707 
4708 	.nested_ops = &svm_nested_ops,
4709 
4710 	.deliver_interrupt = svm_deliver_interrupt,
4711 	.pi_update_irte = avic_pi_update_irte,
4712 	.setup_mce = svm_setup_mce,
4713 
4714 	.smi_allowed = svm_smi_allowed,
4715 	.enter_smm = svm_enter_smm,
4716 	.leave_smm = svm_leave_smm,
4717 	.enable_smi_window = svm_enable_smi_window,
4718 
4719 	.mem_enc_ioctl = sev_mem_enc_ioctl,
4720 	.mem_enc_register_region = sev_mem_enc_register_region,
4721 	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
4722 	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
4723 
4724 	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
4725 	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
4726 
4727 	.can_emulate_instruction = svm_can_emulate_instruction,
4728 
4729 	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
4730 
4731 	.msr_filter_changed = svm_msr_filter_changed,
4732 	.complete_emulated_msr = svm_complete_emulated_msr,
4733 
4734 	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
4735 	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
4736 };
4737 
4738 /*
4739  * The default MMIO mask is a single bit (excluding the present bit),
4740  * which could conflict with the memory encryption bit. Check for
4741  * memory encryption support and override the default MMIO mask if
4742  * memory encryption is enabled.
4743  */
4744 static __init void svm_adjust_mmio_mask(void)
4745 {
4746 	unsigned int enc_bit, mask_bit;
4747 	u64 msr, mask;
4748 
4749 	/* If there is no memory encryption support, use existing mask */
4750 	if (cpuid_eax(0x80000000) < 0x8000001f)
4751 		return;
4752 
4753 	/* If memory encryption is not enabled, use existing mask */
4754 	rdmsrl(MSR_AMD64_SYSCFG, msr);
4755 	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
4756 		return;
4757 
4758 	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
4759 	mask_bit = boot_cpu_data.x86_phys_bits;
4760 
4761 	/* Increment the mask bit if it is the same as the encryption bit */
4762 	if (enc_bit == mask_bit)
4763 		mask_bit++;
4764 
4765 	/*
4766 	 * If the mask bit location is below 52, then some bits above the
4767 	 * physical addressing limit will always be reserved, so use the
4768 	 * rsvd_bits() function to generate the mask. This mask, along with
4769 	 * the present bit, will be used to generate a page fault with
4770 	 * PFER.RSV = 1.
4771 	 *
4772 	 * If the mask bit location is 52 (or above), then clear the mask.
4773 	 */
4774 	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
4775 
4776 	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
4777 }
4778 
4779 static __init void svm_set_cpu_caps(void)
4780 {
4781 	kvm_set_cpu_caps();
4782 
4783 	supported_xss = 0;
4784 
4785 	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
4786 	if (nested) {
4787 		kvm_cpu_cap_set(X86_FEATURE_SVM);
4788 		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
4789 
4790 		if (nrips)
4791 			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
4792 
4793 		if (npt_enabled)
4794 			kvm_cpu_cap_set(X86_FEATURE_NPT);
4795 
4796 		if (tsc_scaling)
4797 			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
4798 
4799 		if (vls)
4800 			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
4801 		if (lbrv)
4802 			kvm_cpu_cap_set(X86_FEATURE_LBRV);
4803 
4804 		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
4805 			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
4806 
4807 		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
4808 			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
4809 
4810 		if (vgif)
4811 			kvm_cpu_cap_set(X86_FEATURE_VGIF);
4812 
4813 		/* Nested VM can receive #VMEXIT instead of triggering #GP */
4814 		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
4815 	}
4816 
4817 	/* CPUID 0x80000008 */
4818 	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
4819 	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
4820 		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
4821 
4822 	/* AMD PMU PERFCTR_CORE CPUID */
4823 	if (enable_pmu && boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
4824 		kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE);
4825 
4826 	/* CPUID 0x8000001F (SME/SEV features) */
4827 	sev_set_cpu_caps();
4828 }
4829 
4830 static __init int svm_hardware_setup(void)
4831 {
4832 	int cpu;
4833 	struct page *iopm_pages;
4834 	void *iopm_va;
4835 	int r;
4836 	unsigned int order = get_order(IOPM_SIZE);
4837 
4838 	/*
4839 	 * NX is required for shadow paging and for NPT if the NX huge pages
4840 	 * mitigation is enabled.
4841 	 */
4842 	if (!boot_cpu_has(X86_FEATURE_NX)) {
4843 		pr_err_ratelimited("NX (Execute Disable) not supported\n");
4844 		return -EOPNOTSUPP;
4845 	}
4846 	kvm_enable_efer_bits(EFER_NX);
4847 
4848 	iopm_pages = alloc_pages(GFP_KERNEL, order);
4849 
4850 	if (!iopm_pages)
4851 		return -ENOMEM;
4852 
4853 	iopm_va = page_address(iopm_pages);
4854 	memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
4855 	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
4856 
4857 	init_msrpm_offsets();
4858 
4859 	supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
4860 
4861 	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
4862 		kvm_enable_efer_bits(EFER_FFXSR);
4863 
4864 	if (tsc_scaling) {
4865 		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
4866 			tsc_scaling = false;
4867 		} else {
4868 			pr_info("TSC scaling supported\n");
4869 			kvm_has_tsc_control = true;
4870 		}
4871 	}
4872 	kvm_max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
4873 	kvm_tsc_scaling_ratio_frac_bits = 32;
4874 
4875 	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
4876 
4877 	/* Check for pause filtering support */
4878 	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
4879 		pause_filter_count = 0;
4880 		pause_filter_thresh = 0;
4881 	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
4882 		pause_filter_thresh = 0;
4883 	}
4884 
4885 	if (nested) {
4886 		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
4887 		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
4888 	}
4889 
4890 	/*
4891 	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
4892 	 * NPT isn't supported if the host is using 2-level paging since host
4893 	 * CR4 is unchanged on VMRUN.
4894 	 */
4895 	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
4896 		npt_enabled = false;
4897 
4898 	if (!boot_cpu_has(X86_FEATURE_NPT))
4899 		npt_enabled = false;
4900 
4901 	/* Force VM NPT level equal to the host's paging level */
4902 	kvm_configure_mmu(npt_enabled, get_npt_level(),
4903 			  get_npt_level(), PG_LEVEL_1G);
4904 	pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
4905 
4906 	/* Setup shadow_me_value and shadow_me_mask */
4907 	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
4908 
4909 	/* Note, SEV setup consumes npt_enabled. */
4910 	sev_hardware_setup();
4911 
4912 	svm_hv_hardware_setup();
4913 
4914 	svm_adjust_mmio_mask();
4915 
4916 	for_each_possible_cpu(cpu) {
4917 		r = svm_cpu_init(cpu);
4918 		if (r)
4919 			goto err;
4920 	}
4921 
4922 	if (nrips) {
4923 		if (!boot_cpu_has(X86_FEATURE_NRIPS))
4924 			nrips = false;
4925 	}
4926 
4927 	enable_apicv = avic = avic && npt_enabled && (boot_cpu_has(X86_FEATURE_AVIC) || force_avic);
4928 
4929 	if (enable_apicv) {
4930 		if (!boot_cpu_has(X86_FEATURE_AVIC)) {
4931 			pr_warn("AVIC is not supported in CPUID but force enabled");
4932 			pr_warn("Your system might crash and burn");
4933 		} else
4934 			pr_info("AVIC enabled\n");
4935 
4936 		amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
4937 	} else {
4938 		svm_x86_ops.vcpu_blocking = NULL;
4939 		svm_x86_ops.vcpu_unblocking = NULL;
4940 		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
4941 	}
4942 
4943 	if (vls) {
4944 		if (!npt_enabled ||
4945 		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
4946 		    !IS_ENABLED(CONFIG_X86_64)) {
4947 			vls = false;
4948 		} else {
4949 			pr_info("Virtual VMLOAD VMSAVE supported\n");
4950 		}
4951 	}
4952 
4953 	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
4954 		svm_gp_erratum_intercept = false;
4955 
4956 	if (vgif) {
4957 		if (!boot_cpu_has(X86_FEATURE_VGIF))
4958 			vgif = false;
4959 		else
4960 			pr_info("Virtual GIF supported\n");
4961 	}
4962 
4963 	if (lbrv) {
4964 		if (!boot_cpu_has(X86_FEATURE_LBRV))
4965 			lbrv = false;
4966 		else
4967 			pr_info("LBR virtualization supported\n");
4968 	}
4969 
4970 	if (!enable_pmu)
4971 		pr_info("PMU virtualization is disabled\n");
4972 
4973 	svm_set_cpu_caps();
4974 
4975 	/*
4976 	 * It seems that on AMD processors PTE's accessed bit is
4977 	 * being set by the CPU hardware before the NPF vmexit.
4978 	 * This is not expected behaviour and our tests fail because
4979 	 * of it.
4980 	 * A workaround here is to disable support for
4981 	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
4982 	 * In this case userspace can know if there is support using
4983 	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
4984 	 * it
4985 	 * If future AMD CPU models change the behaviour described above,
4986 	 * this variable can be changed accordingly
4987 	 */
4988 	allow_smaller_maxphyaddr = !npt_enabled;
4989 
4990 	return 0;
4991 
4992 err:
4993 	svm_hardware_unsetup();
4994 	return r;
4995 }
4996 
4997 
4998 static struct kvm_x86_init_ops svm_init_ops __initdata = {
4999 	.cpu_has_kvm_support = has_svm,
5000 	.disabled_by_bios = is_disabled,
5001 	.hardware_setup = svm_hardware_setup,
5002 	.check_processor_compatibility = svm_check_processor_compat,
5003 
5004 	.runtime_ops = &svm_x86_ops,
5005 	.pmu_ops = &amd_pmu_ops,
5006 };
5007 
5008 static int __init svm_init(void)
5009 {
5010 	__unused_size_checks();
5011 
5012 	return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm),
5013 			__alignof__(struct vcpu_svm), THIS_MODULE);
5014 }
5015 
5016 static void __exit svm_exit(void)
5017 {
5018 	kvm_exit();
5019 }
5020 
5021 module_init(svm_init)
5022 module_exit(svm_exit)
5023