xref: /openbmc/linux/arch/x86/kvm/svm/svm.c (revision abe9af53)
1 #define pr_fmt(fmt) "SVM: " fmt
2 
3 #include <linux/kvm_host.h>
4 
5 #include "irq.h"
6 #include "mmu.h"
7 #include "kvm_cache_regs.h"
8 #include "x86.h"
9 #include "cpuid.h"
10 #include "pmu.h"
11 
12 #include <linux/module.h>
13 #include <linux/mod_devicetable.h>
14 #include <linux/kernel.h>
15 #include <linux/vmalloc.h>
16 #include <linux/highmem.h>
17 #include <linux/amd-iommu.h>
18 #include <linux/sched.h>
19 #include <linux/trace_events.h>
20 #include <linux/slab.h>
21 #include <linux/hashtable.h>
22 #include <linux/objtool.h>
23 #include <linux/psp-sev.h>
24 #include <linux/file.h>
25 #include <linux/pagemap.h>
26 #include <linux/swap.h>
27 #include <linux/rwsem.h>
28 
29 #include <asm/apic.h>
30 #include <asm/perf_event.h>
31 #include <asm/tlbflush.h>
32 #include <asm/desc.h>
33 #include <asm/debugreg.h>
34 #include <asm/kvm_para.h>
35 #include <asm/irq_remapping.h>
36 #include <asm/mce.h>
37 #include <asm/spec-ctrl.h>
38 #include <asm/cpu_device_id.h>
39 
40 #include <asm/virtext.h>
41 #include "trace.h"
42 
43 #include "svm.h"
44 
45 #define __ex(x) __kvm_handle_fault_on_reboot(x)
46 
47 MODULE_AUTHOR("Qumranet");
48 MODULE_LICENSE("GPL");
49 
50 #ifdef MODULE
51 static const struct x86_cpu_id svm_cpu_id[] = {
52 	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
53 	{}
54 };
55 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
56 #endif
57 
58 #define IOPM_ALLOC_ORDER 2
59 #define MSRPM_ALLOC_ORDER 1
60 
61 #define SEG_TYPE_LDT 2
62 #define SEG_TYPE_BUSY_TSS16 3
63 
64 #define SVM_FEATURE_LBRV           (1 <<  1)
65 #define SVM_FEATURE_SVML           (1 <<  2)
66 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
67 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
68 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
69 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
70 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
71 
72 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
73 
74 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
75 #define TSC_RATIO_MIN		0x0000000000000001ULL
76 #define TSC_RATIO_MAX		0x000000ffffffffffULL
77 
78 static bool erratum_383_found __read_mostly;
79 
80 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
81 
82 /*
83  * Set osvw_len to higher value when updated Revision Guides
84  * are published and we know what the new status bits are
85  */
86 static uint64_t osvw_len = 4, osvw_status;
87 
88 static DEFINE_PER_CPU(u64, current_tsc_ratio);
89 #define TSC_RATIO_DEFAULT	0x0100000000ULL
90 
91 static const struct svm_direct_access_msrs {
92 	u32 index;   /* Index of the MSR */
93 	bool always; /* True if intercept is always on */
94 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
95 	{ .index = MSR_STAR,				.always = true  },
96 	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
97 #ifdef CONFIG_X86_64
98 	{ .index = MSR_GS_BASE,				.always = true  },
99 	{ .index = MSR_FS_BASE,				.always = true  },
100 	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
101 	{ .index = MSR_LSTAR,				.always = true  },
102 	{ .index = MSR_CSTAR,				.always = true  },
103 	{ .index = MSR_SYSCALL_MASK,			.always = true  },
104 #endif
105 	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
106 	{ .index = MSR_IA32_PRED_CMD,			.always = false },
107 	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
108 	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
109 	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
110 	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
111 	{ .index = MSR_INVALID,				.always = false },
112 };
113 
114 /* enable NPT for AMD64 and X86 with PAE */
115 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
116 bool npt_enabled = true;
117 #else
118 bool npt_enabled;
119 #endif
120 
121 /*
122  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
123  * pause_filter_count: On processors that support Pause filtering(indicated
124  *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
125  *	count value. On VMRUN this value is loaded into an internal counter.
126  *	Each time a pause instruction is executed, this counter is decremented
127  *	until it reaches zero at which time a #VMEXIT is generated if pause
128  *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
129  *	Intercept Filtering for more details.
130  *	This also indicate if ple logic enabled.
131  *
132  * pause_filter_thresh: In addition, some processor families support advanced
133  *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
134  *	the amount of time a guest is allowed to execute in a pause loop.
135  *	In this mode, a 16-bit pause filter threshold field is added in the
136  *	VMCB. The threshold value is a cycle count that is used to reset the
137  *	pause counter. As with simple pause filtering, VMRUN loads the pause
138  *	count value from VMCB into an internal counter. Then, on each pause
139  *	instruction the hardware checks the elapsed number of cycles since
140  *	the most recent pause instruction against the pause filter threshold.
141  *	If the elapsed cycle count is greater than the pause filter threshold,
142  *	then the internal pause count is reloaded from the VMCB and execution
143  *	continues. If the elapsed cycle count is less than the pause filter
144  *	threshold, then the internal pause count is decremented. If the count
145  *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
146  *	triggered. If advanced pause filtering is supported and pause filter
147  *	threshold field is set to zero, the filter will operate in the simpler,
148  *	count only mode.
149  */
150 
151 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
152 module_param(pause_filter_thresh, ushort, 0444);
153 
154 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
155 module_param(pause_filter_count, ushort, 0444);
156 
157 /* Default doubles per-vcpu window every exit. */
158 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
159 module_param(pause_filter_count_grow, ushort, 0444);
160 
161 /* Default resets per-vcpu window every exit to pause_filter_count. */
162 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
163 module_param(pause_filter_count_shrink, ushort, 0444);
164 
165 /* Default is to compute the maximum so we can never overflow. */
166 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
167 module_param(pause_filter_count_max, ushort, 0444);
168 
169 /* allow nested paging (virtualized MMU) for all guests */
170 static int npt = true;
171 module_param(npt, int, S_IRUGO);
172 
173 /* allow nested virtualization in KVM/SVM */
174 static int nested = true;
175 module_param(nested, int, S_IRUGO);
176 
177 /* enable/disable Next RIP Save */
178 static int nrips = true;
179 module_param(nrips, int, 0444);
180 
181 /* enable/disable Virtual VMLOAD VMSAVE */
182 static int vls = true;
183 module_param(vls, int, 0444);
184 
185 /* enable/disable Virtual GIF */
186 static int vgif = true;
187 module_param(vgif, int, 0444);
188 
189 /* enable/disable SEV support */
190 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
191 module_param(sev, int, 0444);
192 
193 static bool __read_mostly dump_invalid_vmcb = 0;
194 module_param(dump_invalid_vmcb, bool, 0644);
195 
196 static u8 rsm_ins_bytes[] = "\x0f\xaa";
197 
198 static void svm_complete_interrupts(struct vcpu_svm *svm);
199 
200 static unsigned long iopm_base;
201 
202 struct kvm_ldttss_desc {
203 	u16 limit0;
204 	u16 base0;
205 	unsigned base1:8, type:5, dpl:2, p:1;
206 	unsigned limit1:4, zero0:3, g:1, base2:8;
207 	u32 base3;
208 	u32 zero1;
209 } __attribute__((packed));
210 
211 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
212 
213 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
214 
215 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
216 #define MSRS_RANGE_SIZE 2048
217 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
218 
219 u32 svm_msrpm_offset(u32 msr)
220 {
221 	u32 offset;
222 	int i;
223 
224 	for (i = 0; i < NUM_MSR_MAPS; i++) {
225 		if (msr < msrpm_ranges[i] ||
226 		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
227 			continue;
228 
229 		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
230 		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
231 
232 		/* Now we have the u8 offset - but need the u32 offset */
233 		return offset / 4;
234 	}
235 
236 	/* MSR not in any range */
237 	return MSR_INVALID;
238 }
239 
240 #define MAX_INST_SIZE 15
241 
242 static inline void clgi(void)
243 {
244 	asm volatile (__ex("clgi"));
245 }
246 
247 static inline void stgi(void)
248 {
249 	asm volatile (__ex("stgi"));
250 }
251 
252 static inline void invlpga(unsigned long addr, u32 asid)
253 {
254 	asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
255 }
256 
257 static int get_max_npt_level(void)
258 {
259 #ifdef CONFIG_X86_64
260 	return PT64_ROOT_4LEVEL;
261 #else
262 	return PT32E_ROOT_LEVEL;
263 #endif
264 }
265 
266 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
267 {
268 	struct vcpu_svm *svm = to_svm(vcpu);
269 	u64 old_efer = vcpu->arch.efer;
270 	vcpu->arch.efer = efer;
271 
272 	if (!npt_enabled) {
273 		/* Shadow paging assumes NX to be available.  */
274 		efer |= EFER_NX;
275 
276 		if (!(efer & EFER_LMA))
277 			efer &= ~EFER_LME;
278 	}
279 
280 	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
281 		if (!(efer & EFER_SVME)) {
282 			svm_leave_nested(svm);
283 			svm_set_gif(svm, true);
284 
285 			/*
286 			 * Free the nested guest state, unless we are in SMM.
287 			 * In this case we will return to the nested guest
288 			 * as soon as we leave SMM.
289 			 */
290 			if (!is_smm(&svm->vcpu))
291 				svm_free_nested(svm);
292 
293 		} else {
294 			int ret = svm_allocate_nested(svm);
295 
296 			if (ret) {
297 				vcpu->arch.efer = old_efer;
298 				return ret;
299 			}
300 		}
301 	}
302 
303 	svm->vmcb->save.efer = efer | EFER_SVME;
304 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
305 	return 0;
306 }
307 
308 static int is_external_interrupt(u32 info)
309 {
310 	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
311 	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
312 }
313 
314 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
315 {
316 	struct vcpu_svm *svm = to_svm(vcpu);
317 	u32 ret = 0;
318 
319 	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
320 		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
321 	return ret;
322 }
323 
324 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
325 {
326 	struct vcpu_svm *svm = to_svm(vcpu);
327 
328 	if (mask == 0)
329 		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
330 	else
331 		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
332 
333 }
334 
335 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
336 {
337 	struct vcpu_svm *svm = to_svm(vcpu);
338 
339 	if (nrips && svm->vmcb->control.next_rip != 0) {
340 		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
341 		svm->next_rip = svm->vmcb->control.next_rip;
342 	}
343 
344 	if (!svm->next_rip) {
345 		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
346 			return 0;
347 	} else {
348 		kvm_rip_write(vcpu, svm->next_rip);
349 	}
350 	svm_set_interrupt_shadow(vcpu, 0);
351 
352 	return 1;
353 }
354 
355 static void svm_queue_exception(struct kvm_vcpu *vcpu)
356 {
357 	struct vcpu_svm *svm = to_svm(vcpu);
358 	unsigned nr = vcpu->arch.exception.nr;
359 	bool has_error_code = vcpu->arch.exception.has_error_code;
360 	u32 error_code = vcpu->arch.exception.error_code;
361 
362 	kvm_deliver_exception_payload(&svm->vcpu);
363 
364 	if (nr == BP_VECTOR && !nrips) {
365 		unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
366 
367 		/*
368 		 * For guest debugging where we have to reinject #BP if some
369 		 * INT3 is guest-owned:
370 		 * Emulate nRIP by moving RIP forward. Will fail if injection
371 		 * raises a fault that is not intercepted. Still better than
372 		 * failing in all cases.
373 		 */
374 		(void)skip_emulated_instruction(&svm->vcpu);
375 		rip = kvm_rip_read(&svm->vcpu);
376 		svm->int3_rip = rip + svm->vmcb->save.cs.base;
377 		svm->int3_injected = rip - old_rip;
378 	}
379 
380 	svm->vmcb->control.event_inj = nr
381 		| SVM_EVTINJ_VALID
382 		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
383 		| SVM_EVTINJ_TYPE_EXEPT;
384 	svm->vmcb->control.event_inj_err = error_code;
385 }
386 
387 static void svm_init_erratum_383(void)
388 {
389 	u32 low, high;
390 	int err;
391 	u64 val;
392 
393 	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
394 		return;
395 
396 	/* Use _safe variants to not break nested virtualization */
397 	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
398 	if (err)
399 		return;
400 
401 	val |= (1ULL << 47);
402 
403 	low  = lower_32_bits(val);
404 	high = upper_32_bits(val);
405 
406 	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
407 
408 	erratum_383_found = true;
409 }
410 
411 static void svm_init_osvw(struct kvm_vcpu *vcpu)
412 {
413 	/*
414 	 * Guests should see errata 400 and 415 as fixed (assuming that
415 	 * HLT and IO instructions are intercepted).
416 	 */
417 	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
418 	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
419 
420 	/*
421 	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
422 	 * all osvw.status bits inside that length, including bit 0 (which is
423 	 * reserved for erratum 298), are valid. However, if host processor's
424 	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
425 	 * be conservative here and therefore we tell the guest that erratum 298
426 	 * is present (because we really don't know).
427 	 */
428 	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
429 		vcpu->arch.osvw.status |= 1;
430 }
431 
432 static int has_svm(void)
433 {
434 	const char *msg;
435 
436 	if (!cpu_has_svm(&msg)) {
437 		printk(KERN_INFO "has_svm: %s\n", msg);
438 		return 0;
439 	}
440 
441 	return 1;
442 }
443 
444 static void svm_hardware_disable(void)
445 {
446 	/* Make sure we clean up behind us */
447 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
448 		wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
449 
450 	cpu_svm_disable();
451 
452 	amd_pmu_disable_virt();
453 }
454 
455 static int svm_hardware_enable(void)
456 {
457 
458 	struct svm_cpu_data *sd;
459 	uint64_t efer;
460 	struct desc_struct *gdt;
461 	int me = raw_smp_processor_id();
462 
463 	rdmsrl(MSR_EFER, efer);
464 	if (efer & EFER_SVME)
465 		return -EBUSY;
466 
467 	if (!has_svm()) {
468 		pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
469 		return -EINVAL;
470 	}
471 	sd = per_cpu(svm_data, me);
472 	if (!sd) {
473 		pr_err("%s: svm_data is NULL on %d\n", __func__, me);
474 		return -EINVAL;
475 	}
476 
477 	sd->asid_generation = 1;
478 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
479 	sd->next_asid = sd->max_asid + 1;
480 	sd->min_asid = max_sev_asid + 1;
481 
482 	gdt = get_current_gdt_rw();
483 	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
484 
485 	wrmsrl(MSR_EFER, efer | EFER_SVME);
486 
487 	wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
488 
489 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
490 		wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
491 		__this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
492 	}
493 
494 
495 	/*
496 	 * Get OSVW bits.
497 	 *
498 	 * Note that it is possible to have a system with mixed processor
499 	 * revisions and therefore different OSVW bits. If bits are not the same
500 	 * on different processors then choose the worst case (i.e. if erratum
501 	 * is present on one processor and not on another then assume that the
502 	 * erratum is present everywhere).
503 	 */
504 	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
505 		uint64_t len, status = 0;
506 		int err;
507 
508 		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
509 		if (!err)
510 			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
511 						      &err);
512 
513 		if (err)
514 			osvw_status = osvw_len = 0;
515 		else {
516 			if (len < osvw_len)
517 				osvw_len = len;
518 			osvw_status |= status;
519 			osvw_status &= (1ULL << osvw_len) - 1;
520 		}
521 	} else
522 		osvw_status = osvw_len = 0;
523 
524 	svm_init_erratum_383();
525 
526 	amd_pmu_enable_virt();
527 
528 	return 0;
529 }
530 
531 static void svm_cpu_uninit(int cpu)
532 {
533 	struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
534 
535 	if (!sd)
536 		return;
537 
538 	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
539 	kfree(sd->sev_vmcbs);
540 	__free_page(sd->save_area);
541 	kfree(sd);
542 }
543 
544 static int svm_cpu_init(int cpu)
545 {
546 	struct svm_cpu_data *sd;
547 
548 	sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
549 	if (!sd)
550 		return -ENOMEM;
551 	sd->cpu = cpu;
552 	sd->save_area = alloc_page(GFP_KERNEL);
553 	if (!sd->save_area)
554 		goto free_cpu_data;
555 
556 	if (svm_sev_enabled()) {
557 		sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1,
558 					      sizeof(void *),
559 					      GFP_KERNEL);
560 		if (!sd->sev_vmcbs)
561 			goto free_save_area;
562 	}
563 
564 	per_cpu(svm_data, cpu) = sd;
565 
566 	return 0;
567 
568 free_save_area:
569 	__free_page(sd->save_area);
570 free_cpu_data:
571 	kfree(sd);
572 	return -ENOMEM;
573 
574 }
575 
576 static int direct_access_msr_slot(u32 msr)
577 {
578 	u32 i;
579 
580 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
581 		if (direct_access_msrs[i].index == msr)
582 			return i;
583 
584 	return -ENOENT;
585 }
586 
587 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
588 				     int write)
589 {
590 	struct vcpu_svm *svm = to_svm(vcpu);
591 	int slot = direct_access_msr_slot(msr);
592 
593 	if (slot == -ENOENT)
594 		return;
595 
596 	/* Set the shadow bitmaps to the desired intercept states */
597 	if (read)
598 		set_bit(slot, svm->shadow_msr_intercept.read);
599 	else
600 		clear_bit(slot, svm->shadow_msr_intercept.read);
601 
602 	if (write)
603 		set_bit(slot, svm->shadow_msr_intercept.write);
604 	else
605 		clear_bit(slot, svm->shadow_msr_intercept.write);
606 }
607 
608 static bool valid_msr_intercept(u32 index)
609 {
610 	return direct_access_msr_slot(index) != -ENOENT;
611 }
612 
613 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
614 {
615 	u8 bit_write;
616 	unsigned long tmp;
617 	u32 offset;
618 	u32 *msrpm;
619 
620 	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
621 				      to_svm(vcpu)->msrpm;
622 
623 	offset    = svm_msrpm_offset(msr);
624 	bit_write = 2 * (msr & 0x0f) + 1;
625 	tmp       = msrpm[offset];
626 
627 	BUG_ON(offset == MSR_INVALID);
628 
629 	return !!test_bit(bit_write,  &tmp);
630 }
631 
632 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
633 					u32 msr, int read, int write)
634 {
635 	u8 bit_read, bit_write;
636 	unsigned long tmp;
637 	u32 offset;
638 
639 	/*
640 	 * If this warning triggers extend the direct_access_msrs list at the
641 	 * beginning of the file
642 	 */
643 	WARN_ON(!valid_msr_intercept(msr));
644 
645 	/* Enforce non allowed MSRs to trap */
646 	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
647 		read = 0;
648 
649 	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
650 		write = 0;
651 
652 	offset    = svm_msrpm_offset(msr);
653 	bit_read  = 2 * (msr & 0x0f);
654 	bit_write = 2 * (msr & 0x0f) + 1;
655 	tmp       = msrpm[offset];
656 
657 	BUG_ON(offset == MSR_INVALID);
658 
659 	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
660 	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
661 
662 	msrpm[offset] = tmp;
663 }
664 
665 static void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
666 				 int read, int write)
667 {
668 	set_shadow_msr_intercept(vcpu, msr, read, write);
669 	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
670 }
671 
672 u32 *svm_vcpu_alloc_msrpm(void)
673 {
674 	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
675 	u32 *msrpm;
676 
677 	if (!pages)
678 		return NULL;
679 
680 	msrpm = page_address(pages);
681 	memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
682 
683 	return msrpm;
684 }
685 
686 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
687 {
688 	int i;
689 
690 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
691 		if (!direct_access_msrs[i].always)
692 			continue;
693 		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
694 	}
695 }
696 
697 
698 void svm_vcpu_free_msrpm(u32 *msrpm)
699 {
700 	__free_pages(virt_to_page(msrpm), MSRPM_ALLOC_ORDER);
701 }
702 
703 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
704 {
705 	struct vcpu_svm *svm = to_svm(vcpu);
706 	u32 i;
707 
708 	/*
709 	 * Set intercept permissions for all direct access MSRs again. They
710 	 * will automatically get filtered through the MSR filter, so we are
711 	 * back in sync after this.
712 	 */
713 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
714 		u32 msr = direct_access_msrs[i].index;
715 		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
716 		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
717 
718 		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
719 	}
720 }
721 
722 static void add_msr_offset(u32 offset)
723 {
724 	int i;
725 
726 	for (i = 0; i < MSRPM_OFFSETS; ++i) {
727 
728 		/* Offset already in list? */
729 		if (msrpm_offsets[i] == offset)
730 			return;
731 
732 		/* Slot used by another offset? */
733 		if (msrpm_offsets[i] != MSR_INVALID)
734 			continue;
735 
736 		/* Add offset to list */
737 		msrpm_offsets[i] = offset;
738 
739 		return;
740 	}
741 
742 	/*
743 	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
744 	 * increase MSRPM_OFFSETS in this case.
745 	 */
746 	BUG();
747 }
748 
749 static void init_msrpm_offsets(void)
750 {
751 	int i;
752 
753 	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
754 
755 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
756 		u32 offset;
757 
758 		offset = svm_msrpm_offset(direct_access_msrs[i].index);
759 		BUG_ON(offset == MSR_INVALID);
760 
761 		add_msr_offset(offset);
762 	}
763 }
764 
765 static void svm_enable_lbrv(struct kvm_vcpu *vcpu)
766 {
767 	struct vcpu_svm *svm = to_svm(vcpu);
768 
769 	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
770 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
771 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
772 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
773 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
774 }
775 
776 static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
777 {
778 	struct vcpu_svm *svm = to_svm(vcpu);
779 
780 	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
781 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
782 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
783 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
784 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
785 }
786 
787 void disable_nmi_singlestep(struct vcpu_svm *svm)
788 {
789 	svm->nmi_singlestep = false;
790 
791 	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
792 		/* Clear our flags if they were not set by the guest */
793 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
794 			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
795 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
796 			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
797 	}
798 }
799 
800 static void grow_ple_window(struct kvm_vcpu *vcpu)
801 {
802 	struct vcpu_svm *svm = to_svm(vcpu);
803 	struct vmcb_control_area *control = &svm->vmcb->control;
804 	int old = control->pause_filter_count;
805 
806 	control->pause_filter_count = __grow_ple_window(old,
807 							pause_filter_count,
808 							pause_filter_count_grow,
809 							pause_filter_count_max);
810 
811 	if (control->pause_filter_count != old) {
812 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
813 		trace_kvm_ple_window_update(vcpu->vcpu_id,
814 					    control->pause_filter_count, old);
815 	}
816 }
817 
818 static void shrink_ple_window(struct kvm_vcpu *vcpu)
819 {
820 	struct vcpu_svm *svm = to_svm(vcpu);
821 	struct vmcb_control_area *control = &svm->vmcb->control;
822 	int old = control->pause_filter_count;
823 
824 	control->pause_filter_count =
825 				__shrink_ple_window(old,
826 						    pause_filter_count,
827 						    pause_filter_count_shrink,
828 						    pause_filter_count);
829 	if (control->pause_filter_count != old) {
830 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
831 		trace_kvm_ple_window_update(vcpu->vcpu_id,
832 					    control->pause_filter_count, old);
833 	}
834 }
835 
836 /*
837  * The default MMIO mask is a single bit (excluding the present bit),
838  * which could conflict with the memory encryption bit. Check for
839  * memory encryption support and override the default MMIO mask if
840  * memory encryption is enabled.
841  */
842 static __init void svm_adjust_mmio_mask(void)
843 {
844 	unsigned int enc_bit, mask_bit;
845 	u64 msr, mask;
846 
847 	/* If there is no memory encryption support, use existing mask */
848 	if (cpuid_eax(0x80000000) < 0x8000001f)
849 		return;
850 
851 	/* If memory encryption is not enabled, use existing mask */
852 	rdmsrl(MSR_K8_SYSCFG, msr);
853 	if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT))
854 		return;
855 
856 	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
857 	mask_bit = boot_cpu_data.x86_phys_bits;
858 
859 	/* Increment the mask bit if it is the same as the encryption bit */
860 	if (enc_bit == mask_bit)
861 		mask_bit++;
862 
863 	/*
864 	 * If the mask bit location is below 52, then some bits above the
865 	 * physical addressing limit will always be reserved, so use the
866 	 * rsvd_bits() function to generate the mask. This mask, along with
867 	 * the present bit, will be used to generate a page fault with
868 	 * PFER.RSV = 1.
869 	 *
870 	 * If the mask bit location is 52 (or above), then clear the mask.
871 	 */
872 	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
873 
874 	kvm_mmu_set_mmio_spte_mask(mask, PT_WRITABLE_MASK | PT_USER_MASK);
875 }
876 
877 static void svm_hardware_teardown(void)
878 {
879 	int cpu;
880 
881 	if (svm_sev_enabled())
882 		sev_hardware_teardown();
883 
884 	for_each_possible_cpu(cpu)
885 		svm_cpu_uninit(cpu);
886 
887 	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
888 	iopm_base = 0;
889 }
890 
891 static __init void svm_set_cpu_caps(void)
892 {
893 	kvm_set_cpu_caps();
894 
895 	supported_xss = 0;
896 
897 	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
898 	if (nested) {
899 		kvm_cpu_cap_set(X86_FEATURE_SVM);
900 
901 		if (nrips)
902 			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
903 
904 		if (npt_enabled)
905 			kvm_cpu_cap_set(X86_FEATURE_NPT);
906 	}
907 
908 	/* CPUID 0x80000008 */
909 	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
910 	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
911 		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
912 
913 	/* Enable INVPCID feature */
914 	kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID);
915 }
916 
917 static __init int svm_hardware_setup(void)
918 {
919 	int cpu;
920 	struct page *iopm_pages;
921 	void *iopm_va;
922 	int r;
923 
924 	iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
925 
926 	if (!iopm_pages)
927 		return -ENOMEM;
928 
929 	iopm_va = page_address(iopm_pages);
930 	memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
931 	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
932 
933 	init_msrpm_offsets();
934 
935 	supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
936 
937 	if (boot_cpu_has(X86_FEATURE_NX))
938 		kvm_enable_efer_bits(EFER_NX);
939 
940 	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
941 		kvm_enable_efer_bits(EFER_FFXSR);
942 
943 	if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
944 		kvm_has_tsc_control = true;
945 		kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
946 		kvm_tsc_scaling_ratio_frac_bits = 32;
947 	}
948 
949 	/* Check for pause filtering support */
950 	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
951 		pause_filter_count = 0;
952 		pause_filter_thresh = 0;
953 	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
954 		pause_filter_thresh = 0;
955 	}
956 
957 	if (nested) {
958 		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
959 		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
960 	}
961 
962 	if (sev) {
963 		if (boot_cpu_has(X86_FEATURE_SEV) &&
964 		    IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
965 			r = sev_hardware_setup();
966 			if (r)
967 				sev = false;
968 		} else {
969 			sev = false;
970 		}
971 	}
972 
973 	svm_adjust_mmio_mask();
974 
975 	for_each_possible_cpu(cpu) {
976 		r = svm_cpu_init(cpu);
977 		if (r)
978 			goto err;
979 	}
980 
981 	if (!boot_cpu_has(X86_FEATURE_NPT))
982 		npt_enabled = false;
983 
984 	if (npt_enabled && !npt)
985 		npt_enabled = false;
986 
987 	kvm_configure_mmu(npt_enabled, get_max_npt_level(), PG_LEVEL_1G);
988 	pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
989 
990 	if (nrips) {
991 		if (!boot_cpu_has(X86_FEATURE_NRIPS))
992 			nrips = false;
993 	}
994 
995 	if (avic) {
996 		if (!npt_enabled ||
997 		    !boot_cpu_has(X86_FEATURE_AVIC) ||
998 		    !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
999 			avic = false;
1000 		} else {
1001 			pr_info("AVIC enabled\n");
1002 
1003 			amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1004 		}
1005 	}
1006 
1007 	if (vls) {
1008 		if (!npt_enabled ||
1009 		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1010 		    !IS_ENABLED(CONFIG_X86_64)) {
1011 			vls = false;
1012 		} else {
1013 			pr_info("Virtual VMLOAD VMSAVE supported\n");
1014 		}
1015 	}
1016 
1017 	if (vgif) {
1018 		if (!boot_cpu_has(X86_FEATURE_VGIF))
1019 			vgif = false;
1020 		else
1021 			pr_info("Virtual GIF supported\n");
1022 	}
1023 
1024 	svm_set_cpu_caps();
1025 
1026 	/*
1027 	 * It seems that on AMD processors PTE's accessed bit is
1028 	 * being set by the CPU hardware before the NPF vmexit.
1029 	 * This is not expected behaviour and our tests fail because
1030 	 * of it.
1031 	 * A workaround here is to disable support for
1032 	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
1033 	 * In this case userspace can know if there is support using
1034 	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
1035 	 * it
1036 	 * If future AMD CPU models change the behaviour described above,
1037 	 * this variable can be changed accordingly
1038 	 */
1039 	allow_smaller_maxphyaddr = !npt_enabled;
1040 
1041 	return 0;
1042 
1043 err:
1044 	svm_hardware_teardown();
1045 	return r;
1046 }
1047 
1048 static void init_seg(struct vmcb_seg *seg)
1049 {
1050 	seg->selector = 0;
1051 	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1052 		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1053 	seg->limit = 0xffff;
1054 	seg->base = 0;
1055 }
1056 
1057 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1058 {
1059 	seg->selector = 0;
1060 	seg->attrib = SVM_SELECTOR_P_MASK | type;
1061 	seg->limit = 0xffff;
1062 	seg->base = 0;
1063 }
1064 
1065 static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1066 {
1067 	struct vcpu_svm *svm = to_svm(vcpu);
1068 	u64 g_tsc_offset = 0;
1069 
1070 	if (is_guest_mode(vcpu)) {
1071 		/* Write L1's TSC offset.  */
1072 		g_tsc_offset = svm->vmcb->control.tsc_offset -
1073 			       svm->nested.hsave->control.tsc_offset;
1074 		svm->nested.hsave->control.tsc_offset = offset;
1075 	}
1076 
1077 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1078 				   svm->vmcb->control.tsc_offset - g_tsc_offset,
1079 				   offset);
1080 
1081 	svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1082 
1083 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1084 	return svm->vmcb->control.tsc_offset;
1085 }
1086 
1087 static void svm_check_invpcid(struct vcpu_svm *svm)
1088 {
1089 	/*
1090 	 * Intercept INVPCID instruction only if shadow page table is
1091 	 * enabled. Interception is not required with nested page table
1092 	 * enabled.
1093 	 */
1094 	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1095 		if (!npt_enabled)
1096 			svm_set_intercept(svm, INTERCEPT_INVPCID);
1097 		else
1098 			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1099 	}
1100 }
1101 
1102 static void init_vmcb(struct vcpu_svm *svm)
1103 {
1104 	struct vmcb_control_area *control = &svm->vmcb->control;
1105 	struct vmcb_save_area *save = &svm->vmcb->save;
1106 
1107 	svm->vcpu.arch.hflags = 0;
1108 
1109 	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1110 	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1111 	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1112 	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1113 	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1114 	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1115 	if (!kvm_vcpu_apicv_active(&svm->vcpu))
1116 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1117 
1118 	set_dr_intercepts(svm);
1119 
1120 	set_exception_intercept(svm, PF_VECTOR);
1121 	set_exception_intercept(svm, UD_VECTOR);
1122 	set_exception_intercept(svm, MC_VECTOR);
1123 	set_exception_intercept(svm, AC_VECTOR);
1124 	set_exception_intercept(svm, DB_VECTOR);
1125 	/*
1126 	 * Guest access to VMware backdoor ports could legitimately
1127 	 * trigger #GP because of TSS I/O permission bitmap.
1128 	 * We intercept those #GP and allow access to them anyway
1129 	 * as VMware does.
1130 	 */
1131 	if (enable_vmware_backdoor)
1132 		set_exception_intercept(svm, GP_VECTOR);
1133 
1134 	svm_set_intercept(svm, INTERCEPT_INTR);
1135 	svm_set_intercept(svm, INTERCEPT_NMI);
1136 	svm_set_intercept(svm, INTERCEPT_SMI);
1137 	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1138 	svm_set_intercept(svm, INTERCEPT_RDPMC);
1139 	svm_set_intercept(svm, INTERCEPT_CPUID);
1140 	svm_set_intercept(svm, INTERCEPT_INVD);
1141 	svm_set_intercept(svm, INTERCEPT_INVLPG);
1142 	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1143 	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1144 	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1145 	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1146 	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1147 	svm_set_intercept(svm, INTERCEPT_VMRUN);
1148 	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1149 	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1150 	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1151 	svm_set_intercept(svm, INTERCEPT_STGI);
1152 	svm_set_intercept(svm, INTERCEPT_CLGI);
1153 	svm_set_intercept(svm, INTERCEPT_SKINIT);
1154 	svm_set_intercept(svm, INTERCEPT_WBINVD);
1155 	svm_set_intercept(svm, INTERCEPT_XSETBV);
1156 	svm_set_intercept(svm, INTERCEPT_RDPRU);
1157 	svm_set_intercept(svm, INTERCEPT_RSM);
1158 
1159 	if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1160 		svm_set_intercept(svm, INTERCEPT_MONITOR);
1161 		svm_set_intercept(svm, INTERCEPT_MWAIT);
1162 	}
1163 
1164 	if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1165 		svm_set_intercept(svm, INTERCEPT_HLT);
1166 
1167 	control->iopm_base_pa = __sme_set(iopm_base);
1168 	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1169 	control->int_ctl = V_INTR_MASKING_MASK;
1170 
1171 	init_seg(&save->es);
1172 	init_seg(&save->ss);
1173 	init_seg(&save->ds);
1174 	init_seg(&save->fs);
1175 	init_seg(&save->gs);
1176 
1177 	save->cs.selector = 0xf000;
1178 	save->cs.base = 0xffff0000;
1179 	/* Executable/Readable Code Segment */
1180 	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1181 		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1182 	save->cs.limit = 0xffff;
1183 
1184 	save->gdtr.limit = 0xffff;
1185 	save->idtr.limit = 0xffff;
1186 
1187 	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1188 	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1189 
1190 	svm_set_efer(&svm->vcpu, 0);
1191 	save->dr6 = 0xffff0ff0;
1192 	kvm_set_rflags(&svm->vcpu, 2);
1193 	save->rip = 0x0000fff0;
1194 	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1195 
1196 	/*
1197 	 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1198 	 * It also updates the guest-visible cr0 value.
1199 	 */
1200 	svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1201 	kvm_mmu_reset_context(&svm->vcpu);
1202 
1203 	save->cr4 = X86_CR4_PAE;
1204 	/* rdx = ?? */
1205 
1206 	if (npt_enabled) {
1207 		/* Setup VMCB for Nested Paging */
1208 		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1209 		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1210 		clr_exception_intercept(svm, PF_VECTOR);
1211 		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1212 		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1213 		save->g_pat = svm->vcpu.arch.pat;
1214 		save->cr3 = 0;
1215 		save->cr4 = 0;
1216 	}
1217 	svm->asid_generation = 0;
1218 
1219 	svm->nested.vmcb12_gpa = 0;
1220 	svm->vcpu.arch.hflags = 0;
1221 
1222 	if (!kvm_pause_in_guest(svm->vcpu.kvm)) {
1223 		control->pause_filter_count = pause_filter_count;
1224 		if (pause_filter_thresh)
1225 			control->pause_filter_thresh = pause_filter_thresh;
1226 		svm_set_intercept(svm, INTERCEPT_PAUSE);
1227 	} else {
1228 		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1229 	}
1230 
1231 	svm_check_invpcid(svm);
1232 
1233 	if (kvm_vcpu_apicv_active(&svm->vcpu))
1234 		avic_init_vmcb(svm);
1235 
1236 	/*
1237 	 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1238 	 * in VMCB and clear intercepts to avoid #VMEXIT.
1239 	 */
1240 	if (vls) {
1241 		svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1242 		svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1243 		svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1244 	}
1245 
1246 	if (vgif) {
1247 		svm_clr_intercept(svm, INTERCEPT_STGI);
1248 		svm_clr_intercept(svm, INTERCEPT_CLGI);
1249 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1250 	}
1251 
1252 	if (sev_guest(svm->vcpu.kvm)) {
1253 		svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1254 		clr_exception_intercept(svm, UD_VECTOR);
1255 	}
1256 
1257 	vmcb_mark_all_dirty(svm->vmcb);
1258 
1259 	enable_gif(svm);
1260 
1261 }
1262 
1263 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1264 {
1265 	struct vcpu_svm *svm = to_svm(vcpu);
1266 	u32 dummy;
1267 	u32 eax = 1;
1268 
1269 	svm->spec_ctrl = 0;
1270 	svm->virt_spec_ctrl = 0;
1271 
1272 	if (!init_event) {
1273 		svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1274 					   MSR_IA32_APICBASE_ENABLE;
1275 		if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1276 			svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1277 	}
1278 	init_vmcb(svm);
1279 
1280 	kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, false);
1281 	kvm_rdx_write(vcpu, eax);
1282 
1283 	if (kvm_vcpu_apicv_active(vcpu) && !init_event)
1284 		avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
1285 }
1286 
1287 static int svm_create_vcpu(struct kvm_vcpu *vcpu)
1288 {
1289 	struct vcpu_svm *svm;
1290 	struct page *vmcb_page;
1291 	int err;
1292 
1293 	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1294 	svm = to_svm(vcpu);
1295 
1296 	err = -ENOMEM;
1297 	vmcb_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1298 	if (!vmcb_page)
1299 		goto out;
1300 
1301 	err = avic_init_vcpu(svm);
1302 	if (err)
1303 		goto error_free_vmcb_page;
1304 
1305 	/* We initialize this flag to true to make sure that the is_running
1306 	 * bit would be set the first time the vcpu is loaded.
1307 	 */
1308 	if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm))
1309 		svm->avic_is_running = true;
1310 
1311 	svm->msrpm = svm_vcpu_alloc_msrpm();
1312 	if (!svm->msrpm)
1313 		goto error_free_vmcb_page;
1314 
1315 	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1316 
1317 	svm->vmcb = page_address(vmcb_page);
1318 	svm->vmcb_pa = __sme_set(page_to_pfn(vmcb_page) << PAGE_SHIFT);
1319 	svm->asid_generation = 0;
1320 	init_vmcb(svm);
1321 
1322 	svm_init_osvw(vcpu);
1323 	vcpu->arch.microcode_version = 0x01000065;
1324 
1325 	return 0;
1326 
1327 error_free_vmcb_page:
1328 	__free_page(vmcb_page);
1329 out:
1330 	return err;
1331 }
1332 
1333 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1334 {
1335 	int i;
1336 
1337 	for_each_online_cpu(i)
1338 		cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
1339 }
1340 
1341 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1342 {
1343 	struct vcpu_svm *svm = to_svm(vcpu);
1344 
1345 	/*
1346 	 * The vmcb page can be recycled, causing a false negative in
1347 	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1348 	 * vmcb page recorded as its current vmcb.
1349 	 */
1350 	svm_clear_current_vmcb(svm->vmcb);
1351 
1352 	svm_free_nested(svm);
1353 
1354 	__free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
1355 	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1356 }
1357 
1358 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1359 {
1360 	struct vcpu_svm *svm = to_svm(vcpu);
1361 	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1362 	int i;
1363 
1364 	if (unlikely(cpu != vcpu->cpu)) {
1365 		svm->asid_generation = 0;
1366 		vmcb_mark_all_dirty(svm->vmcb);
1367 	}
1368 
1369 #ifdef CONFIG_X86_64
1370 	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1371 #endif
1372 	savesegment(fs, svm->host.fs);
1373 	savesegment(gs, svm->host.gs);
1374 	svm->host.ldt = kvm_read_ldt();
1375 
1376 	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1377 		rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1378 
1379 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1380 		u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1381 		if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1382 			__this_cpu_write(current_tsc_ratio, tsc_ratio);
1383 			wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1384 		}
1385 	}
1386 	/* This assumes that the kernel never uses MSR_TSC_AUX */
1387 	if (static_cpu_has(X86_FEATURE_RDTSCP))
1388 		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1389 
1390 	if (sd->current_vmcb != svm->vmcb) {
1391 		sd->current_vmcb = svm->vmcb;
1392 		indirect_branch_prediction_barrier();
1393 	}
1394 	avic_vcpu_load(vcpu, cpu);
1395 }
1396 
1397 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1398 {
1399 	struct vcpu_svm *svm = to_svm(vcpu);
1400 	int i;
1401 
1402 	avic_vcpu_put(vcpu);
1403 
1404 	++vcpu->stat.host_state_reload;
1405 	kvm_load_ldt(svm->host.ldt);
1406 #ifdef CONFIG_X86_64
1407 	loadsegment(fs, svm->host.fs);
1408 	wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
1409 	load_gs_index(svm->host.gs);
1410 #else
1411 #ifdef CONFIG_X86_32_LAZY_GS
1412 	loadsegment(gs, svm->host.gs);
1413 #endif
1414 #endif
1415 	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1416 		wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1417 }
1418 
1419 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1420 {
1421 	struct vcpu_svm *svm = to_svm(vcpu);
1422 	unsigned long rflags = svm->vmcb->save.rflags;
1423 
1424 	if (svm->nmi_singlestep) {
1425 		/* Hide our flags if they were not set by the guest */
1426 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1427 			rflags &= ~X86_EFLAGS_TF;
1428 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1429 			rflags &= ~X86_EFLAGS_RF;
1430 	}
1431 	return rflags;
1432 }
1433 
1434 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1435 {
1436 	if (to_svm(vcpu)->nmi_singlestep)
1437 		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1438 
1439        /*
1440         * Any change of EFLAGS.VM is accompanied by a reload of SS
1441         * (caused by either a task switch or an inter-privilege IRET),
1442         * so we do not need to update the CPL here.
1443         */
1444 	to_svm(vcpu)->vmcb->save.rflags = rflags;
1445 }
1446 
1447 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1448 {
1449 	switch (reg) {
1450 	case VCPU_EXREG_PDPTR:
1451 		BUG_ON(!npt_enabled);
1452 		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1453 		break;
1454 	default:
1455 		WARN_ON_ONCE(1);
1456 	}
1457 }
1458 
1459 static void svm_set_vintr(struct vcpu_svm *svm)
1460 {
1461 	struct vmcb_control_area *control;
1462 
1463 	/* The following fields are ignored when AVIC is enabled */
1464 	WARN_ON(kvm_vcpu_apicv_active(&svm->vcpu));
1465 	svm_set_intercept(svm, INTERCEPT_VINTR);
1466 
1467 	/*
1468 	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1469 	 * Actual injection of virtual interrupts happens through EVENTINJ.
1470 	 */
1471 	control = &svm->vmcb->control;
1472 	control->int_vector = 0x0;
1473 	control->int_ctl &= ~V_INTR_PRIO_MASK;
1474 	control->int_ctl |= V_IRQ_MASK |
1475 		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1476 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1477 }
1478 
1479 static void svm_clear_vintr(struct vcpu_svm *svm)
1480 {
1481 	const u32 mask = V_TPR_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK | V_INTR_MASKING_MASK;
1482 	svm_clr_intercept(svm, INTERCEPT_VINTR);
1483 
1484 	/* Drop int_ctl fields related to VINTR injection.  */
1485 	svm->vmcb->control.int_ctl &= mask;
1486 	if (is_guest_mode(&svm->vcpu)) {
1487 		svm->nested.hsave->control.int_ctl &= mask;
1488 
1489 		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1490 			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1491 		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & ~mask;
1492 	}
1493 
1494 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1495 }
1496 
1497 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1498 {
1499 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1500 
1501 	switch (seg) {
1502 	case VCPU_SREG_CS: return &save->cs;
1503 	case VCPU_SREG_DS: return &save->ds;
1504 	case VCPU_SREG_ES: return &save->es;
1505 	case VCPU_SREG_FS: return &save->fs;
1506 	case VCPU_SREG_GS: return &save->gs;
1507 	case VCPU_SREG_SS: return &save->ss;
1508 	case VCPU_SREG_TR: return &save->tr;
1509 	case VCPU_SREG_LDTR: return &save->ldtr;
1510 	}
1511 	BUG();
1512 	return NULL;
1513 }
1514 
1515 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1516 {
1517 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1518 
1519 	return s->base;
1520 }
1521 
1522 static void svm_get_segment(struct kvm_vcpu *vcpu,
1523 			    struct kvm_segment *var, int seg)
1524 {
1525 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1526 
1527 	var->base = s->base;
1528 	var->limit = s->limit;
1529 	var->selector = s->selector;
1530 	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1531 	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1532 	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1533 	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1534 	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1535 	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1536 	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1537 
1538 	/*
1539 	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1540 	 * However, the SVM spec states that the G bit is not observed by the
1541 	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1542 	 * So let's synthesize a legal G bit for all segments, this helps
1543 	 * running KVM nested. It also helps cross-vendor migration, because
1544 	 * Intel's vmentry has a check on the 'G' bit.
1545 	 */
1546 	var->g = s->limit > 0xfffff;
1547 
1548 	/*
1549 	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1550 	 * for cross vendor migration purposes by "not present"
1551 	 */
1552 	var->unusable = !var->present;
1553 
1554 	switch (seg) {
1555 	case VCPU_SREG_TR:
1556 		/*
1557 		 * Work around a bug where the busy flag in the tr selector
1558 		 * isn't exposed
1559 		 */
1560 		var->type |= 0x2;
1561 		break;
1562 	case VCPU_SREG_DS:
1563 	case VCPU_SREG_ES:
1564 	case VCPU_SREG_FS:
1565 	case VCPU_SREG_GS:
1566 		/*
1567 		 * The accessed bit must always be set in the segment
1568 		 * descriptor cache, although it can be cleared in the
1569 		 * descriptor, the cached bit always remains at 1. Since
1570 		 * Intel has a check on this, set it here to support
1571 		 * cross-vendor migration.
1572 		 */
1573 		if (!var->unusable)
1574 			var->type |= 0x1;
1575 		break;
1576 	case VCPU_SREG_SS:
1577 		/*
1578 		 * On AMD CPUs sometimes the DB bit in the segment
1579 		 * descriptor is left as 1, although the whole segment has
1580 		 * been made unusable. Clear it here to pass an Intel VMX
1581 		 * entry check when cross vendor migrating.
1582 		 */
1583 		if (var->unusable)
1584 			var->db = 0;
1585 		/* This is symmetric with svm_set_segment() */
1586 		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1587 		break;
1588 	}
1589 }
1590 
1591 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1592 {
1593 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1594 
1595 	return save->cpl;
1596 }
1597 
1598 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1599 {
1600 	struct vcpu_svm *svm = to_svm(vcpu);
1601 
1602 	dt->size = svm->vmcb->save.idtr.limit;
1603 	dt->address = svm->vmcb->save.idtr.base;
1604 }
1605 
1606 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1607 {
1608 	struct vcpu_svm *svm = to_svm(vcpu);
1609 
1610 	svm->vmcb->save.idtr.limit = dt->size;
1611 	svm->vmcb->save.idtr.base = dt->address ;
1612 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1613 }
1614 
1615 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1616 {
1617 	struct vcpu_svm *svm = to_svm(vcpu);
1618 
1619 	dt->size = svm->vmcb->save.gdtr.limit;
1620 	dt->address = svm->vmcb->save.gdtr.base;
1621 }
1622 
1623 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1624 {
1625 	struct vcpu_svm *svm = to_svm(vcpu);
1626 
1627 	svm->vmcb->save.gdtr.limit = dt->size;
1628 	svm->vmcb->save.gdtr.base = dt->address ;
1629 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1630 }
1631 
1632 static void update_cr0_intercept(struct vcpu_svm *svm)
1633 {
1634 	ulong gcr0 = svm->vcpu.arch.cr0;
1635 	u64 *hcr0 = &svm->vmcb->save.cr0;
1636 
1637 	*hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1638 		| (gcr0 & SVM_CR0_SELECTIVE_MASK);
1639 
1640 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1641 
1642 	if (gcr0 == *hcr0) {
1643 		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1644 		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1645 	} else {
1646 		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1647 		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1648 	}
1649 }
1650 
1651 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1652 {
1653 	struct vcpu_svm *svm = to_svm(vcpu);
1654 
1655 #ifdef CONFIG_X86_64
1656 	if (vcpu->arch.efer & EFER_LME) {
1657 		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1658 			vcpu->arch.efer |= EFER_LMA;
1659 			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1660 		}
1661 
1662 		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1663 			vcpu->arch.efer &= ~EFER_LMA;
1664 			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1665 		}
1666 	}
1667 #endif
1668 	vcpu->arch.cr0 = cr0;
1669 
1670 	if (!npt_enabled)
1671 		cr0 |= X86_CR0_PG | X86_CR0_WP;
1672 
1673 	/*
1674 	 * re-enable caching here because the QEMU bios
1675 	 * does not do it - this results in some delay at
1676 	 * reboot
1677 	 */
1678 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1679 		cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1680 	svm->vmcb->save.cr0 = cr0;
1681 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1682 	update_cr0_intercept(svm);
1683 }
1684 
1685 int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1686 {
1687 	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1688 	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1689 
1690 	if (cr4 & X86_CR4_VMXE)
1691 		return 1;
1692 
1693 	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1694 		svm_flush_tlb(vcpu);
1695 
1696 	vcpu->arch.cr4 = cr4;
1697 	if (!npt_enabled)
1698 		cr4 |= X86_CR4_PAE;
1699 	cr4 |= host_cr4_mce;
1700 	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1701 	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1702 	return 0;
1703 }
1704 
1705 static void svm_set_segment(struct kvm_vcpu *vcpu,
1706 			    struct kvm_segment *var, int seg)
1707 {
1708 	struct vcpu_svm *svm = to_svm(vcpu);
1709 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1710 
1711 	s->base = var->base;
1712 	s->limit = var->limit;
1713 	s->selector = var->selector;
1714 	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1715 	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1716 	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1717 	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1718 	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1719 	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1720 	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1721 	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1722 
1723 	/*
1724 	 * This is always accurate, except if SYSRET returned to a segment
1725 	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1726 	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1727 	 * would entail passing the CPL to userspace and back.
1728 	 */
1729 	if (seg == VCPU_SREG_SS)
1730 		/* This is symmetric with svm_get_segment() */
1731 		svm->vmcb->save.cpl = (var->dpl & 3);
1732 
1733 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1734 }
1735 
1736 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
1737 {
1738 	struct vcpu_svm *svm = to_svm(vcpu);
1739 
1740 	clr_exception_intercept(svm, BP_VECTOR);
1741 
1742 	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1743 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1744 			set_exception_intercept(svm, BP_VECTOR);
1745 	}
1746 }
1747 
1748 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1749 {
1750 	if (sd->next_asid > sd->max_asid) {
1751 		++sd->asid_generation;
1752 		sd->next_asid = sd->min_asid;
1753 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1754 	}
1755 
1756 	svm->asid_generation = sd->asid_generation;
1757 	svm->vmcb->control.asid = sd->next_asid++;
1758 
1759 	vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1760 }
1761 
1762 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
1763 {
1764 	struct vmcb *vmcb = svm->vmcb;
1765 
1766 	if (unlikely(value != vmcb->save.dr6)) {
1767 		vmcb->save.dr6 = value;
1768 		vmcb_mark_dirty(vmcb, VMCB_DR);
1769 	}
1770 }
1771 
1772 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1773 {
1774 	struct vcpu_svm *svm = to_svm(vcpu);
1775 
1776 	get_debugreg(vcpu->arch.db[0], 0);
1777 	get_debugreg(vcpu->arch.db[1], 1);
1778 	get_debugreg(vcpu->arch.db[2], 2);
1779 	get_debugreg(vcpu->arch.db[3], 3);
1780 	/*
1781 	 * We cannot reset svm->vmcb->save.dr6 to DR6_FIXED_1|DR6_RTM here,
1782 	 * because db_interception might need it.  We can do it before vmentry.
1783 	 */
1784 	vcpu->arch.dr6 = svm->vmcb->save.dr6;
1785 	vcpu->arch.dr7 = svm->vmcb->save.dr7;
1786 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1787 	set_dr_intercepts(svm);
1788 }
1789 
1790 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1791 {
1792 	struct vcpu_svm *svm = to_svm(vcpu);
1793 
1794 	svm->vmcb->save.dr7 = value;
1795 	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
1796 }
1797 
1798 static int pf_interception(struct vcpu_svm *svm)
1799 {
1800 	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
1801 	u64 error_code = svm->vmcb->control.exit_info_1;
1802 
1803 	return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
1804 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1805 			svm->vmcb->control.insn_bytes : NULL,
1806 			svm->vmcb->control.insn_len);
1807 }
1808 
1809 static int npf_interception(struct vcpu_svm *svm)
1810 {
1811 	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
1812 	u64 error_code = svm->vmcb->control.exit_info_1;
1813 
1814 	trace_kvm_page_fault(fault_address, error_code);
1815 	return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1816 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1817 			svm->vmcb->control.insn_bytes : NULL,
1818 			svm->vmcb->control.insn_len);
1819 }
1820 
1821 static int db_interception(struct vcpu_svm *svm)
1822 {
1823 	struct kvm_run *kvm_run = svm->vcpu.run;
1824 	struct kvm_vcpu *vcpu = &svm->vcpu;
1825 
1826 	if (!(svm->vcpu.guest_debug &
1827 	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1828 		!svm->nmi_singlestep) {
1829 		u32 payload = (svm->vmcb->save.dr6 ^ DR6_RTM) & ~DR6_FIXED_1;
1830 		kvm_queue_exception_p(&svm->vcpu, DB_VECTOR, payload);
1831 		return 1;
1832 	}
1833 
1834 	if (svm->nmi_singlestep) {
1835 		disable_nmi_singlestep(svm);
1836 		/* Make sure we check for pending NMIs upon entry */
1837 		kvm_make_request(KVM_REQ_EVENT, vcpu);
1838 	}
1839 
1840 	if (svm->vcpu.guest_debug &
1841 	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1842 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
1843 		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
1844 		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
1845 		kvm_run->debug.arch.pc =
1846 			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1847 		kvm_run->debug.arch.exception = DB_VECTOR;
1848 		return 0;
1849 	}
1850 
1851 	return 1;
1852 }
1853 
1854 static int bp_interception(struct vcpu_svm *svm)
1855 {
1856 	struct kvm_run *kvm_run = svm->vcpu.run;
1857 
1858 	kvm_run->exit_reason = KVM_EXIT_DEBUG;
1859 	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1860 	kvm_run->debug.arch.exception = BP_VECTOR;
1861 	return 0;
1862 }
1863 
1864 static int ud_interception(struct vcpu_svm *svm)
1865 {
1866 	return handle_ud(&svm->vcpu);
1867 }
1868 
1869 static int ac_interception(struct vcpu_svm *svm)
1870 {
1871 	kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
1872 	return 1;
1873 }
1874 
1875 static int gp_interception(struct vcpu_svm *svm)
1876 {
1877 	struct kvm_vcpu *vcpu = &svm->vcpu;
1878 	u32 error_code = svm->vmcb->control.exit_info_1;
1879 
1880 	WARN_ON_ONCE(!enable_vmware_backdoor);
1881 
1882 	/*
1883 	 * VMware backdoor emulation on #GP interception only handles IN{S},
1884 	 * OUT{S}, and RDPMC, none of which generate a non-zero error code.
1885 	 */
1886 	if (error_code) {
1887 		kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1888 		return 1;
1889 	}
1890 	return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
1891 }
1892 
1893 static bool is_erratum_383(void)
1894 {
1895 	int err, i;
1896 	u64 value;
1897 
1898 	if (!erratum_383_found)
1899 		return false;
1900 
1901 	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1902 	if (err)
1903 		return false;
1904 
1905 	/* Bit 62 may or may not be set for this mce */
1906 	value &= ~(1ULL << 62);
1907 
1908 	if (value != 0xb600000000010015ULL)
1909 		return false;
1910 
1911 	/* Clear MCi_STATUS registers */
1912 	for (i = 0; i < 6; ++i)
1913 		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1914 
1915 	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1916 	if (!err) {
1917 		u32 low, high;
1918 
1919 		value &= ~(1ULL << 2);
1920 		low    = lower_32_bits(value);
1921 		high   = upper_32_bits(value);
1922 
1923 		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1924 	}
1925 
1926 	/* Flush tlb to evict multi-match entries */
1927 	__flush_tlb_all();
1928 
1929 	return true;
1930 }
1931 
1932 /*
1933  * Trigger machine check on the host. We assume all the MSRs are already set up
1934  * by the CPU and that we still run on the same CPU as the MCE occurred on.
1935  * We pass a fake environment to the machine check handler because we want
1936  * the guest to be always treated like user space, no matter what context
1937  * it used internally.
1938  */
1939 static void kvm_machine_check(void)
1940 {
1941 #if defined(CONFIG_X86_MCE)
1942 	struct pt_regs regs = {
1943 		.cs = 3, /* Fake ring 3 no matter what the guest ran on */
1944 		.flags = X86_EFLAGS_IF,
1945 	};
1946 
1947 	do_machine_check(&regs);
1948 #endif
1949 }
1950 
1951 static void svm_handle_mce(struct vcpu_svm *svm)
1952 {
1953 	if (is_erratum_383()) {
1954 		/*
1955 		 * Erratum 383 triggered. Guest state is corrupt so kill the
1956 		 * guest.
1957 		 */
1958 		pr_err("KVM: Guest triggered AMD Erratum 383\n");
1959 
1960 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1961 
1962 		return;
1963 	}
1964 
1965 	/*
1966 	 * On an #MC intercept the MCE handler is not called automatically in
1967 	 * the host. So do it by hand here.
1968 	 */
1969 	kvm_machine_check();
1970 }
1971 
1972 static int mc_interception(struct vcpu_svm *svm)
1973 {
1974 	return 1;
1975 }
1976 
1977 static int shutdown_interception(struct vcpu_svm *svm)
1978 {
1979 	struct kvm_run *kvm_run = svm->vcpu.run;
1980 
1981 	/*
1982 	 * VMCB is undefined after a SHUTDOWN intercept
1983 	 * so reinitialize it.
1984 	 */
1985 	clear_page(svm->vmcb);
1986 	init_vmcb(svm);
1987 
1988 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1989 	return 0;
1990 }
1991 
1992 static int io_interception(struct vcpu_svm *svm)
1993 {
1994 	struct kvm_vcpu *vcpu = &svm->vcpu;
1995 	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1996 	int size, in, string;
1997 	unsigned port;
1998 
1999 	++svm->vcpu.stat.io_exits;
2000 	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2001 	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2002 	if (string)
2003 		return kvm_emulate_instruction(vcpu, 0);
2004 
2005 	port = io_info >> 16;
2006 	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2007 	svm->next_rip = svm->vmcb->control.exit_info_2;
2008 
2009 	return kvm_fast_pio(&svm->vcpu, size, port, in);
2010 }
2011 
2012 static int nmi_interception(struct vcpu_svm *svm)
2013 {
2014 	return 1;
2015 }
2016 
2017 static int intr_interception(struct vcpu_svm *svm)
2018 {
2019 	++svm->vcpu.stat.irq_exits;
2020 	return 1;
2021 }
2022 
2023 static int nop_on_interception(struct vcpu_svm *svm)
2024 {
2025 	return 1;
2026 }
2027 
2028 static int halt_interception(struct vcpu_svm *svm)
2029 {
2030 	return kvm_emulate_halt(&svm->vcpu);
2031 }
2032 
2033 static int vmmcall_interception(struct vcpu_svm *svm)
2034 {
2035 	return kvm_emulate_hypercall(&svm->vcpu);
2036 }
2037 
2038 static int vmload_interception(struct vcpu_svm *svm)
2039 {
2040 	struct vmcb *nested_vmcb;
2041 	struct kvm_host_map map;
2042 	int ret;
2043 
2044 	if (nested_svm_check_permissions(svm))
2045 		return 1;
2046 
2047 	ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2048 	if (ret) {
2049 		if (ret == -EINVAL)
2050 			kvm_inject_gp(&svm->vcpu, 0);
2051 		return 1;
2052 	}
2053 
2054 	nested_vmcb = map.hva;
2055 
2056 	ret = kvm_skip_emulated_instruction(&svm->vcpu);
2057 
2058 	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2059 	kvm_vcpu_unmap(&svm->vcpu, &map, true);
2060 
2061 	return ret;
2062 }
2063 
2064 static int vmsave_interception(struct vcpu_svm *svm)
2065 {
2066 	struct vmcb *nested_vmcb;
2067 	struct kvm_host_map map;
2068 	int ret;
2069 
2070 	if (nested_svm_check_permissions(svm))
2071 		return 1;
2072 
2073 	ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2074 	if (ret) {
2075 		if (ret == -EINVAL)
2076 			kvm_inject_gp(&svm->vcpu, 0);
2077 		return 1;
2078 	}
2079 
2080 	nested_vmcb = map.hva;
2081 
2082 	ret = kvm_skip_emulated_instruction(&svm->vcpu);
2083 
2084 	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2085 	kvm_vcpu_unmap(&svm->vcpu, &map, true);
2086 
2087 	return ret;
2088 }
2089 
2090 static int vmrun_interception(struct vcpu_svm *svm)
2091 {
2092 	if (nested_svm_check_permissions(svm))
2093 		return 1;
2094 
2095 	return nested_svm_vmrun(svm);
2096 }
2097 
2098 void svm_set_gif(struct vcpu_svm *svm, bool value)
2099 {
2100 	if (value) {
2101 		/*
2102 		 * If VGIF is enabled, the STGI intercept is only added to
2103 		 * detect the opening of the SMI/NMI window; remove it now.
2104 		 * Likewise, clear the VINTR intercept, we will set it
2105 		 * again while processing KVM_REQ_EVENT if needed.
2106 		 */
2107 		if (vgif_enabled(svm))
2108 			svm_clr_intercept(svm, INTERCEPT_STGI);
2109 		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2110 			svm_clear_vintr(svm);
2111 
2112 		enable_gif(svm);
2113 		if (svm->vcpu.arch.smi_pending ||
2114 		    svm->vcpu.arch.nmi_pending ||
2115 		    kvm_cpu_has_injectable_intr(&svm->vcpu))
2116 			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2117 	} else {
2118 		disable_gif(svm);
2119 
2120 		/*
2121 		 * After a CLGI no interrupts should come.  But if vGIF is
2122 		 * in use, we still rely on the VINTR intercept (rather than
2123 		 * STGI) to detect an open interrupt window.
2124 		*/
2125 		if (!vgif_enabled(svm))
2126 			svm_clear_vintr(svm);
2127 	}
2128 }
2129 
2130 static int stgi_interception(struct vcpu_svm *svm)
2131 {
2132 	int ret;
2133 
2134 	if (nested_svm_check_permissions(svm))
2135 		return 1;
2136 
2137 	ret = kvm_skip_emulated_instruction(&svm->vcpu);
2138 	svm_set_gif(svm, true);
2139 	return ret;
2140 }
2141 
2142 static int clgi_interception(struct vcpu_svm *svm)
2143 {
2144 	int ret;
2145 
2146 	if (nested_svm_check_permissions(svm))
2147 		return 1;
2148 
2149 	ret = kvm_skip_emulated_instruction(&svm->vcpu);
2150 	svm_set_gif(svm, false);
2151 	return ret;
2152 }
2153 
2154 static int invlpga_interception(struct vcpu_svm *svm)
2155 {
2156 	struct kvm_vcpu *vcpu = &svm->vcpu;
2157 
2158 	trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),
2159 			  kvm_rax_read(&svm->vcpu));
2160 
2161 	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2162 	kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
2163 
2164 	return kvm_skip_emulated_instruction(&svm->vcpu);
2165 }
2166 
2167 static int skinit_interception(struct vcpu_svm *svm)
2168 {
2169 	trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));
2170 
2171 	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2172 	return 1;
2173 }
2174 
2175 static int wbinvd_interception(struct vcpu_svm *svm)
2176 {
2177 	return kvm_emulate_wbinvd(&svm->vcpu);
2178 }
2179 
2180 static int xsetbv_interception(struct vcpu_svm *svm)
2181 {
2182 	u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2183 	u32 index = kvm_rcx_read(&svm->vcpu);
2184 
2185 	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2186 		return kvm_skip_emulated_instruction(&svm->vcpu);
2187 	}
2188 
2189 	return 1;
2190 }
2191 
2192 static int rdpru_interception(struct vcpu_svm *svm)
2193 {
2194 	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2195 	return 1;
2196 }
2197 
2198 static int task_switch_interception(struct vcpu_svm *svm)
2199 {
2200 	u16 tss_selector;
2201 	int reason;
2202 	int int_type = svm->vmcb->control.exit_int_info &
2203 		SVM_EXITINTINFO_TYPE_MASK;
2204 	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2205 	uint32_t type =
2206 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2207 	uint32_t idt_v =
2208 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2209 	bool has_error_code = false;
2210 	u32 error_code = 0;
2211 
2212 	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2213 
2214 	if (svm->vmcb->control.exit_info_2 &
2215 	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2216 		reason = TASK_SWITCH_IRET;
2217 	else if (svm->vmcb->control.exit_info_2 &
2218 		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2219 		reason = TASK_SWITCH_JMP;
2220 	else if (idt_v)
2221 		reason = TASK_SWITCH_GATE;
2222 	else
2223 		reason = TASK_SWITCH_CALL;
2224 
2225 	if (reason == TASK_SWITCH_GATE) {
2226 		switch (type) {
2227 		case SVM_EXITINTINFO_TYPE_NMI:
2228 			svm->vcpu.arch.nmi_injected = false;
2229 			break;
2230 		case SVM_EXITINTINFO_TYPE_EXEPT:
2231 			if (svm->vmcb->control.exit_info_2 &
2232 			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2233 				has_error_code = true;
2234 				error_code =
2235 					(u32)svm->vmcb->control.exit_info_2;
2236 			}
2237 			kvm_clear_exception_queue(&svm->vcpu);
2238 			break;
2239 		case SVM_EXITINTINFO_TYPE_INTR:
2240 			kvm_clear_interrupt_queue(&svm->vcpu);
2241 			break;
2242 		default:
2243 			break;
2244 		}
2245 	}
2246 
2247 	if (reason != TASK_SWITCH_GATE ||
2248 	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2249 	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2250 	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2251 		if (!skip_emulated_instruction(&svm->vcpu))
2252 			return 0;
2253 	}
2254 
2255 	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2256 		int_vec = -1;
2257 
2258 	return kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2259 			       has_error_code, error_code);
2260 }
2261 
2262 static int cpuid_interception(struct vcpu_svm *svm)
2263 {
2264 	return kvm_emulate_cpuid(&svm->vcpu);
2265 }
2266 
2267 static int iret_interception(struct vcpu_svm *svm)
2268 {
2269 	++svm->vcpu.stat.nmi_window_exits;
2270 	svm_clr_intercept(svm, INTERCEPT_IRET);
2271 	svm->vcpu.arch.hflags |= HF_IRET_MASK;
2272 	svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
2273 	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2274 	return 1;
2275 }
2276 
2277 static int invd_interception(struct vcpu_svm *svm)
2278 {
2279 	/* Treat an INVD instruction as a NOP and just skip it. */
2280 	return kvm_skip_emulated_instruction(&svm->vcpu);
2281 }
2282 
2283 static int invlpg_interception(struct vcpu_svm *svm)
2284 {
2285 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2286 		return kvm_emulate_instruction(&svm->vcpu, 0);
2287 
2288 	kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
2289 	return kvm_skip_emulated_instruction(&svm->vcpu);
2290 }
2291 
2292 static int emulate_on_interception(struct vcpu_svm *svm)
2293 {
2294 	return kvm_emulate_instruction(&svm->vcpu, 0);
2295 }
2296 
2297 static int rsm_interception(struct vcpu_svm *svm)
2298 {
2299 	return kvm_emulate_instruction_from_buffer(&svm->vcpu, rsm_ins_bytes, 2);
2300 }
2301 
2302 static int rdpmc_interception(struct vcpu_svm *svm)
2303 {
2304 	int err;
2305 
2306 	if (!nrips)
2307 		return emulate_on_interception(svm);
2308 
2309 	err = kvm_rdpmc(&svm->vcpu);
2310 	return kvm_complete_insn_gp(&svm->vcpu, err);
2311 }
2312 
2313 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2314 					    unsigned long val)
2315 {
2316 	unsigned long cr0 = svm->vcpu.arch.cr0;
2317 	bool ret = false;
2318 
2319 	if (!is_guest_mode(&svm->vcpu) ||
2320 	    (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2321 		return false;
2322 
2323 	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2324 	val &= ~SVM_CR0_SELECTIVE_MASK;
2325 
2326 	if (cr0 ^ val) {
2327 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2328 		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2329 	}
2330 
2331 	return ret;
2332 }
2333 
2334 #define CR_VALID (1ULL << 63)
2335 
2336 static int cr_interception(struct vcpu_svm *svm)
2337 {
2338 	int reg, cr;
2339 	unsigned long val;
2340 	int err;
2341 
2342 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2343 		return emulate_on_interception(svm);
2344 
2345 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2346 		return emulate_on_interception(svm);
2347 
2348 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2349 	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2350 		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2351 	else
2352 		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2353 
2354 	err = 0;
2355 	if (cr >= 16) { /* mov to cr */
2356 		cr -= 16;
2357 		val = kvm_register_read(&svm->vcpu, reg);
2358 		trace_kvm_cr_write(cr, val);
2359 		switch (cr) {
2360 		case 0:
2361 			if (!check_selective_cr0_intercepted(svm, val))
2362 				err = kvm_set_cr0(&svm->vcpu, val);
2363 			else
2364 				return 1;
2365 
2366 			break;
2367 		case 3:
2368 			err = kvm_set_cr3(&svm->vcpu, val);
2369 			break;
2370 		case 4:
2371 			err = kvm_set_cr4(&svm->vcpu, val);
2372 			break;
2373 		case 8:
2374 			err = kvm_set_cr8(&svm->vcpu, val);
2375 			break;
2376 		default:
2377 			WARN(1, "unhandled write to CR%d", cr);
2378 			kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2379 			return 1;
2380 		}
2381 	} else { /* mov from cr */
2382 		switch (cr) {
2383 		case 0:
2384 			val = kvm_read_cr0(&svm->vcpu);
2385 			break;
2386 		case 2:
2387 			val = svm->vcpu.arch.cr2;
2388 			break;
2389 		case 3:
2390 			val = kvm_read_cr3(&svm->vcpu);
2391 			break;
2392 		case 4:
2393 			val = kvm_read_cr4(&svm->vcpu);
2394 			break;
2395 		case 8:
2396 			val = kvm_get_cr8(&svm->vcpu);
2397 			break;
2398 		default:
2399 			WARN(1, "unhandled read from CR%d", cr);
2400 			kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2401 			return 1;
2402 		}
2403 		kvm_register_write(&svm->vcpu, reg, val);
2404 		trace_kvm_cr_read(cr, val);
2405 	}
2406 	return kvm_complete_insn_gp(&svm->vcpu, err);
2407 }
2408 
2409 static int dr_interception(struct vcpu_svm *svm)
2410 {
2411 	int reg, dr;
2412 	unsigned long val;
2413 
2414 	if (svm->vcpu.guest_debug == 0) {
2415 		/*
2416 		 * No more DR vmexits; force a reload of the debug registers
2417 		 * and reenter on this instruction.  The next vmexit will
2418 		 * retrieve the full state of the debug registers.
2419 		 */
2420 		clr_dr_intercepts(svm);
2421 		svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2422 		return 1;
2423 	}
2424 
2425 	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2426 		return emulate_on_interception(svm);
2427 
2428 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2429 	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2430 
2431 	if (dr >= 16) { /* mov to DRn */
2432 		if (!kvm_require_dr(&svm->vcpu, dr - 16))
2433 			return 1;
2434 		val = kvm_register_read(&svm->vcpu, reg);
2435 		kvm_set_dr(&svm->vcpu, dr - 16, val);
2436 	} else {
2437 		if (!kvm_require_dr(&svm->vcpu, dr))
2438 			return 1;
2439 		kvm_get_dr(&svm->vcpu, dr, &val);
2440 		kvm_register_write(&svm->vcpu, reg, val);
2441 	}
2442 
2443 	return kvm_skip_emulated_instruction(&svm->vcpu);
2444 }
2445 
2446 static int cr8_write_interception(struct vcpu_svm *svm)
2447 {
2448 	struct kvm_run *kvm_run = svm->vcpu.run;
2449 	int r;
2450 
2451 	u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2452 	/* instruction emulation calls kvm_set_cr8() */
2453 	r = cr_interception(svm);
2454 	if (lapic_in_kernel(&svm->vcpu))
2455 		return r;
2456 	if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2457 		return r;
2458 	kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2459 	return 0;
2460 }
2461 
2462 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2463 {
2464 	msr->data = 0;
2465 
2466 	switch (msr->index) {
2467 	case MSR_F10H_DECFG:
2468 		if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
2469 			msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
2470 		break;
2471 	case MSR_IA32_PERF_CAPABILITIES:
2472 		return 0;
2473 	default:
2474 		return KVM_MSR_RET_INVALID;
2475 	}
2476 
2477 	return 0;
2478 }
2479 
2480 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2481 {
2482 	struct vcpu_svm *svm = to_svm(vcpu);
2483 
2484 	switch (msr_info->index) {
2485 	case MSR_STAR:
2486 		msr_info->data = svm->vmcb->save.star;
2487 		break;
2488 #ifdef CONFIG_X86_64
2489 	case MSR_LSTAR:
2490 		msr_info->data = svm->vmcb->save.lstar;
2491 		break;
2492 	case MSR_CSTAR:
2493 		msr_info->data = svm->vmcb->save.cstar;
2494 		break;
2495 	case MSR_KERNEL_GS_BASE:
2496 		msr_info->data = svm->vmcb->save.kernel_gs_base;
2497 		break;
2498 	case MSR_SYSCALL_MASK:
2499 		msr_info->data = svm->vmcb->save.sfmask;
2500 		break;
2501 #endif
2502 	case MSR_IA32_SYSENTER_CS:
2503 		msr_info->data = svm->vmcb->save.sysenter_cs;
2504 		break;
2505 	case MSR_IA32_SYSENTER_EIP:
2506 		msr_info->data = svm->sysenter_eip;
2507 		break;
2508 	case MSR_IA32_SYSENTER_ESP:
2509 		msr_info->data = svm->sysenter_esp;
2510 		break;
2511 	case MSR_TSC_AUX:
2512 		if (!boot_cpu_has(X86_FEATURE_RDTSCP))
2513 			return 1;
2514 		msr_info->data = svm->tsc_aux;
2515 		break;
2516 	/*
2517 	 * Nobody will change the following 5 values in the VMCB so we can
2518 	 * safely return them on rdmsr. They will always be 0 until LBRV is
2519 	 * implemented.
2520 	 */
2521 	case MSR_IA32_DEBUGCTLMSR:
2522 		msr_info->data = svm->vmcb->save.dbgctl;
2523 		break;
2524 	case MSR_IA32_LASTBRANCHFROMIP:
2525 		msr_info->data = svm->vmcb->save.br_from;
2526 		break;
2527 	case MSR_IA32_LASTBRANCHTOIP:
2528 		msr_info->data = svm->vmcb->save.br_to;
2529 		break;
2530 	case MSR_IA32_LASTINTFROMIP:
2531 		msr_info->data = svm->vmcb->save.last_excp_from;
2532 		break;
2533 	case MSR_IA32_LASTINTTOIP:
2534 		msr_info->data = svm->vmcb->save.last_excp_to;
2535 		break;
2536 	case MSR_VM_HSAVE_PA:
2537 		msr_info->data = svm->nested.hsave_msr;
2538 		break;
2539 	case MSR_VM_CR:
2540 		msr_info->data = svm->nested.vm_cr_msr;
2541 		break;
2542 	case MSR_IA32_SPEC_CTRL:
2543 		if (!msr_info->host_initiated &&
2544 		    !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
2545 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) &&
2546 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
2547 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
2548 			return 1;
2549 
2550 		msr_info->data = svm->spec_ctrl;
2551 		break;
2552 	case MSR_AMD64_VIRT_SPEC_CTRL:
2553 		if (!msr_info->host_initiated &&
2554 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2555 			return 1;
2556 
2557 		msr_info->data = svm->virt_spec_ctrl;
2558 		break;
2559 	case MSR_F15H_IC_CFG: {
2560 
2561 		int family, model;
2562 
2563 		family = guest_cpuid_family(vcpu);
2564 		model  = guest_cpuid_model(vcpu);
2565 
2566 		if (family < 0 || model < 0)
2567 			return kvm_get_msr_common(vcpu, msr_info);
2568 
2569 		msr_info->data = 0;
2570 
2571 		if (family == 0x15 &&
2572 		    (model >= 0x2 && model < 0x20))
2573 			msr_info->data = 0x1E;
2574 		}
2575 		break;
2576 	case MSR_F10H_DECFG:
2577 		msr_info->data = svm->msr_decfg;
2578 		break;
2579 	default:
2580 		return kvm_get_msr_common(vcpu, msr_info);
2581 	}
2582 	return 0;
2583 }
2584 
2585 static int rdmsr_interception(struct vcpu_svm *svm)
2586 {
2587 	return kvm_emulate_rdmsr(&svm->vcpu);
2588 }
2589 
2590 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2591 {
2592 	struct vcpu_svm *svm = to_svm(vcpu);
2593 	int svm_dis, chg_mask;
2594 
2595 	if (data & ~SVM_VM_CR_VALID_MASK)
2596 		return 1;
2597 
2598 	chg_mask = SVM_VM_CR_VALID_MASK;
2599 
2600 	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2601 		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2602 
2603 	svm->nested.vm_cr_msr &= ~chg_mask;
2604 	svm->nested.vm_cr_msr |= (data & chg_mask);
2605 
2606 	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2607 
2608 	/* check for svm_disable while efer.svme is set */
2609 	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2610 		return 1;
2611 
2612 	return 0;
2613 }
2614 
2615 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2616 {
2617 	struct vcpu_svm *svm = to_svm(vcpu);
2618 
2619 	u32 ecx = msr->index;
2620 	u64 data = msr->data;
2621 	switch (ecx) {
2622 	case MSR_IA32_CR_PAT:
2623 		if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
2624 			return 1;
2625 		vcpu->arch.pat = data;
2626 		svm->vmcb->save.g_pat = data;
2627 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
2628 		break;
2629 	case MSR_IA32_SPEC_CTRL:
2630 		if (!msr->host_initiated &&
2631 		    !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
2632 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) &&
2633 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
2634 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
2635 			return 1;
2636 
2637 		if (kvm_spec_ctrl_test_value(data))
2638 			return 1;
2639 
2640 		svm->spec_ctrl = data;
2641 		if (!data)
2642 			break;
2643 
2644 		/*
2645 		 * For non-nested:
2646 		 * When it's written (to non-zero) for the first time, pass
2647 		 * it through.
2648 		 *
2649 		 * For nested:
2650 		 * The handling of the MSR bitmap for L2 guests is done in
2651 		 * nested_svm_vmrun_msrpm.
2652 		 * We update the L1 MSR bit as well since it will end up
2653 		 * touching the MSR anyway now.
2654 		 */
2655 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
2656 		break;
2657 	case MSR_IA32_PRED_CMD:
2658 		if (!msr->host_initiated &&
2659 		    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
2660 			return 1;
2661 
2662 		if (data & ~PRED_CMD_IBPB)
2663 			return 1;
2664 		if (!boot_cpu_has(X86_FEATURE_AMD_IBPB))
2665 			return 1;
2666 		if (!data)
2667 			break;
2668 
2669 		wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2670 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
2671 		break;
2672 	case MSR_AMD64_VIRT_SPEC_CTRL:
2673 		if (!msr->host_initiated &&
2674 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2675 			return 1;
2676 
2677 		if (data & ~SPEC_CTRL_SSBD)
2678 			return 1;
2679 
2680 		svm->virt_spec_ctrl = data;
2681 		break;
2682 	case MSR_STAR:
2683 		svm->vmcb->save.star = data;
2684 		break;
2685 #ifdef CONFIG_X86_64
2686 	case MSR_LSTAR:
2687 		svm->vmcb->save.lstar = data;
2688 		break;
2689 	case MSR_CSTAR:
2690 		svm->vmcb->save.cstar = data;
2691 		break;
2692 	case MSR_KERNEL_GS_BASE:
2693 		svm->vmcb->save.kernel_gs_base = data;
2694 		break;
2695 	case MSR_SYSCALL_MASK:
2696 		svm->vmcb->save.sfmask = data;
2697 		break;
2698 #endif
2699 	case MSR_IA32_SYSENTER_CS:
2700 		svm->vmcb->save.sysenter_cs = data;
2701 		break;
2702 	case MSR_IA32_SYSENTER_EIP:
2703 		svm->sysenter_eip = data;
2704 		svm->vmcb->save.sysenter_eip = data;
2705 		break;
2706 	case MSR_IA32_SYSENTER_ESP:
2707 		svm->sysenter_esp = data;
2708 		svm->vmcb->save.sysenter_esp = data;
2709 		break;
2710 	case MSR_TSC_AUX:
2711 		if (!boot_cpu_has(X86_FEATURE_RDTSCP))
2712 			return 1;
2713 
2714 		/*
2715 		 * This is rare, so we update the MSR here instead of using
2716 		 * direct_access_msrs.  Doing that would require a rdmsr in
2717 		 * svm_vcpu_put.
2718 		 */
2719 		svm->tsc_aux = data;
2720 		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2721 		break;
2722 	case MSR_IA32_DEBUGCTLMSR:
2723 		if (!boot_cpu_has(X86_FEATURE_LBRV)) {
2724 			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2725 				    __func__, data);
2726 			break;
2727 		}
2728 		if (data & DEBUGCTL_RESERVED_BITS)
2729 			return 1;
2730 
2731 		svm->vmcb->save.dbgctl = data;
2732 		vmcb_mark_dirty(svm->vmcb, VMCB_LBR);
2733 		if (data & (1ULL<<0))
2734 			svm_enable_lbrv(vcpu);
2735 		else
2736 			svm_disable_lbrv(vcpu);
2737 		break;
2738 	case MSR_VM_HSAVE_PA:
2739 		svm->nested.hsave_msr = data;
2740 		break;
2741 	case MSR_VM_CR:
2742 		return svm_set_vm_cr(vcpu, data);
2743 	case MSR_VM_IGNNE:
2744 		vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
2745 		break;
2746 	case MSR_F10H_DECFG: {
2747 		struct kvm_msr_entry msr_entry;
2748 
2749 		msr_entry.index = msr->index;
2750 		if (svm_get_msr_feature(&msr_entry))
2751 			return 1;
2752 
2753 		/* Check the supported bits */
2754 		if (data & ~msr_entry.data)
2755 			return 1;
2756 
2757 		/* Don't allow the guest to change a bit, #GP */
2758 		if (!msr->host_initiated && (data ^ msr_entry.data))
2759 			return 1;
2760 
2761 		svm->msr_decfg = data;
2762 		break;
2763 	}
2764 	case MSR_IA32_APICBASE:
2765 		if (kvm_vcpu_apicv_active(vcpu))
2766 			avic_update_vapic_bar(to_svm(vcpu), data);
2767 		fallthrough;
2768 	default:
2769 		return kvm_set_msr_common(vcpu, msr);
2770 	}
2771 	return 0;
2772 }
2773 
2774 static int wrmsr_interception(struct vcpu_svm *svm)
2775 {
2776 	return kvm_emulate_wrmsr(&svm->vcpu);
2777 }
2778 
2779 static int msr_interception(struct vcpu_svm *svm)
2780 {
2781 	if (svm->vmcb->control.exit_info_1)
2782 		return wrmsr_interception(svm);
2783 	else
2784 		return rdmsr_interception(svm);
2785 }
2786 
2787 static int interrupt_window_interception(struct vcpu_svm *svm)
2788 {
2789 	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2790 	svm_clear_vintr(svm);
2791 
2792 	/*
2793 	 * For AVIC, the only reason to end up here is ExtINTs.
2794 	 * In this case AVIC was temporarily disabled for
2795 	 * requesting the IRQ window and we have to re-enable it.
2796 	 */
2797 	svm_toggle_avic_for_irq_window(&svm->vcpu, true);
2798 
2799 	++svm->vcpu.stat.irq_window_exits;
2800 	return 1;
2801 }
2802 
2803 static int pause_interception(struct vcpu_svm *svm)
2804 {
2805 	struct kvm_vcpu *vcpu = &svm->vcpu;
2806 	bool in_kernel = (svm_get_cpl(vcpu) == 0);
2807 
2808 	if (!kvm_pause_in_guest(vcpu->kvm))
2809 		grow_ple_window(vcpu);
2810 
2811 	kvm_vcpu_on_spin(vcpu, in_kernel);
2812 	return 1;
2813 }
2814 
2815 static int nop_interception(struct vcpu_svm *svm)
2816 {
2817 	return kvm_skip_emulated_instruction(&(svm->vcpu));
2818 }
2819 
2820 static int monitor_interception(struct vcpu_svm *svm)
2821 {
2822 	printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
2823 	return nop_interception(svm);
2824 }
2825 
2826 static int mwait_interception(struct vcpu_svm *svm)
2827 {
2828 	printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
2829 	return nop_interception(svm);
2830 }
2831 
2832 static int invpcid_interception(struct vcpu_svm *svm)
2833 {
2834 	struct kvm_vcpu *vcpu = &svm->vcpu;
2835 	unsigned long type;
2836 	gva_t gva;
2837 
2838 	if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
2839 		kvm_queue_exception(vcpu, UD_VECTOR);
2840 		return 1;
2841 	}
2842 
2843 	/*
2844 	 * For an INVPCID intercept:
2845 	 * EXITINFO1 provides the linear address of the memory operand.
2846 	 * EXITINFO2 provides the contents of the register operand.
2847 	 */
2848 	type = svm->vmcb->control.exit_info_2;
2849 	gva = svm->vmcb->control.exit_info_1;
2850 
2851 	if (type > 3) {
2852 		kvm_inject_gp(vcpu, 0);
2853 		return 1;
2854 	}
2855 
2856 	return kvm_handle_invpcid(vcpu, type, gva);
2857 }
2858 
2859 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
2860 	[SVM_EXIT_READ_CR0]			= cr_interception,
2861 	[SVM_EXIT_READ_CR3]			= cr_interception,
2862 	[SVM_EXIT_READ_CR4]			= cr_interception,
2863 	[SVM_EXIT_READ_CR8]			= cr_interception,
2864 	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
2865 	[SVM_EXIT_WRITE_CR0]			= cr_interception,
2866 	[SVM_EXIT_WRITE_CR3]			= cr_interception,
2867 	[SVM_EXIT_WRITE_CR4]			= cr_interception,
2868 	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
2869 	[SVM_EXIT_READ_DR0]			= dr_interception,
2870 	[SVM_EXIT_READ_DR1]			= dr_interception,
2871 	[SVM_EXIT_READ_DR2]			= dr_interception,
2872 	[SVM_EXIT_READ_DR3]			= dr_interception,
2873 	[SVM_EXIT_READ_DR4]			= dr_interception,
2874 	[SVM_EXIT_READ_DR5]			= dr_interception,
2875 	[SVM_EXIT_READ_DR6]			= dr_interception,
2876 	[SVM_EXIT_READ_DR7]			= dr_interception,
2877 	[SVM_EXIT_WRITE_DR0]			= dr_interception,
2878 	[SVM_EXIT_WRITE_DR1]			= dr_interception,
2879 	[SVM_EXIT_WRITE_DR2]			= dr_interception,
2880 	[SVM_EXIT_WRITE_DR3]			= dr_interception,
2881 	[SVM_EXIT_WRITE_DR4]			= dr_interception,
2882 	[SVM_EXIT_WRITE_DR5]			= dr_interception,
2883 	[SVM_EXIT_WRITE_DR6]			= dr_interception,
2884 	[SVM_EXIT_WRITE_DR7]			= dr_interception,
2885 	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
2886 	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
2887 	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
2888 	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
2889 	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
2890 	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
2891 	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
2892 	[SVM_EXIT_INTR]				= intr_interception,
2893 	[SVM_EXIT_NMI]				= nmi_interception,
2894 	[SVM_EXIT_SMI]				= nop_on_interception,
2895 	[SVM_EXIT_INIT]				= nop_on_interception,
2896 	[SVM_EXIT_VINTR]			= interrupt_window_interception,
2897 	[SVM_EXIT_RDPMC]			= rdpmc_interception,
2898 	[SVM_EXIT_CPUID]			= cpuid_interception,
2899 	[SVM_EXIT_IRET]                         = iret_interception,
2900 	[SVM_EXIT_INVD]                         = invd_interception,
2901 	[SVM_EXIT_PAUSE]			= pause_interception,
2902 	[SVM_EXIT_HLT]				= halt_interception,
2903 	[SVM_EXIT_INVLPG]			= invlpg_interception,
2904 	[SVM_EXIT_INVLPGA]			= invlpga_interception,
2905 	[SVM_EXIT_IOIO]				= io_interception,
2906 	[SVM_EXIT_MSR]				= msr_interception,
2907 	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
2908 	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
2909 	[SVM_EXIT_VMRUN]			= vmrun_interception,
2910 	[SVM_EXIT_VMMCALL]			= vmmcall_interception,
2911 	[SVM_EXIT_VMLOAD]			= vmload_interception,
2912 	[SVM_EXIT_VMSAVE]			= vmsave_interception,
2913 	[SVM_EXIT_STGI]				= stgi_interception,
2914 	[SVM_EXIT_CLGI]				= clgi_interception,
2915 	[SVM_EXIT_SKINIT]			= skinit_interception,
2916 	[SVM_EXIT_WBINVD]                       = wbinvd_interception,
2917 	[SVM_EXIT_MONITOR]			= monitor_interception,
2918 	[SVM_EXIT_MWAIT]			= mwait_interception,
2919 	[SVM_EXIT_XSETBV]			= xsetbv_interception,
2920 	[SVM_EXIT_RDPRU]			= rdpru_interception,
2921 	[SVM_EXIT_INVPCID]                      = invpcid_interception,
2922 	[SVM_EXIT_NPF]				= npf_interception,
2923 	[SVM_EXIT_RSM]                          = rsm_interception,
2924 	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
2925 	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
2926 };
2927 
2928 static void dump_vmcb(struct kvm_vcpu *vcpu)
2929 {
2930 	struct vcpu_svm *svm = to_svm(vcpu);
2931 	struct vmcb_control_area *control = &svm->vmcb->control;
2932 	struct vmcb_save_area *save = &svm->vmcb->save;
2933 
2934 	if (!dump_invalid_vmcb) {
2935 		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
2936 		return;
2937 	}
2938 
2939 	pr_err("VMCB Control Area:\n");
2940 	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
2941 	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
2942 	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
2943 	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
2944 	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
2945 	pr_err("%-20s%08x %08x\n", "intercepts:",
2946               control->intercepts[INTERCEPT_WORD3],
2947 	       control->intercepts[INTERCEPT_WORD4]);
2948 	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
2949 	pr_err("%-20s%d\n", "pause filter threshold:",
2950 	       control->pause_filter_thresh);
2951 	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
2952 	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
2953 	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
2954 	pr_err("%-20s%d\n", "asid:", control->asid);
2955 	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
2956 	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
2957 	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
2958 	pr_err("%-20s%08x\n", "int_state:", control->int_state);
2959 	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
2960 	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
2961 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
2962 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
2963 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
2964 	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
2965 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
2966 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
2967 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
2968 	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
2969 	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
2970 	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
2971 	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
2972 	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
2973 	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
2974 	pr_err("VMCB State Save Area:\n");
2975 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2976 	       "es:",
2977 	       save->es.selector, save->es.attrib,
2978 	       save->es.limit, save->es.base);
2979 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2980 	       "cs:",
2981 	       save->cs.selector, save->cs.attrib,
2982 	       save->cs.limit, save->cs.base);
2983 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2984 	       "ss:",
2985 	       save->ss.selector, save->ss.attrib,
2986 	       save->ss.limit, save->ss.base);
2987 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2988 	       "ds:",
2989 	       save->ds.selector, save->ds.attrib,
2990 	       save->ds.limit, save->ds.base);
2991 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2992 	       "fs:",
2993 	       save->fs.selector, save->fs.attrib,
2994 	       save->fs.limit, save->fs.base);
2995 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2996 	       "gs:",
2997 	       save->gs.selector, save->gs.attrib,
2998 	       save->gs.limit, save->gs.base);
2999 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3000 	       "gdtr:",
3001 	       save->gdtr.selector, save->gdtr.attrib,
3002 	       save->gdtr.limit, save->gdtr.base);
3003 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3004 	       "ldtr:",
3005 	       save->ldtr.selector, save->ldtr.attrib,
3006 	       save->ldtr.limit, save->ldtr.base);
3007 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3008 	       "idtr:",
3009 	       save->idtr.selector, save->idtr.attrib,
3010 	       save->idtr.limit, save->idtr.base);
3011 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3012 	       "tr:",
3013 	       save->tr.selector, save->tr.attrib,
3014 	       save->tr.limit, save->tr.base);
3015 	pr_err("cpl:            %d                efer:         %016llx\n",
3016 		save->cpl, save->efer);
3017 	pr_err("%-15s %016llx %-13s %016llx\n",
3018 	       "cr0:", save->cr0, "cr2:", save->cr2);
3019 	pr_err("%-15s %016llx %-13s %016llx\n",
3020 	       "cr3:", save->cr3, "cr4:", save->cr4);
3021 	pr_err("%-15s %016llx %-13s %016llx\n",
3022 	       "dr6:", save->dr6, "dr7:", save->dr7);
3023 	pr_err("%-15s %016llx %-13s %016llx\n",
3024 	       "rip:", save->rip, "rflags:", save->rflags);
3025 	pr_err("%-15s %016llx %-13s %016llx\n",
3026 	       "rsp:", save->rsp, "rax:", save->rax);
3027 	pr_err("%-15s %016llx %-13s %016llx\n",
3028 	       "star:", save->star, "lstar:", save->lstar);
3029 	pr_err("%-15s %016llx %-13s %016llx\n",
3030 	       "cstar:", save->cstar, "sfmask:", save->sfmask);
3031 	pr_err("%-15s %016llx %-13s %016llx\n",
3032 	       "kernel_gs_base:", save->kernel_gs_base,
3033 	       "sysenter_cs:", save->sysenter_cs);
3034 	pr_err("%-15s %016llx %-13s %016llx\n",
3035 	       "sysenter_esp:", save->sysenter_esp,
3036 	       "sysenter_eip:", save->sysenter_eip);
3037 	pr_err("%-15s %016llx %-13s %016llx\n",
3038 	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3039 	pr_err("%-15s %016llx %-13s %016llx\n",
3040 	       "br_from:", save->br_from, "br_to:", save->br_to);
3041 	pr_err("%-15s %016llx %-13s %016llx\n",
3042 	       "excp_from:", save->last_excp_from,
3043 	       "excp_to:", save->last_excp_to);
3044 }
3045 
3046 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2,
3047 			      u32 *intr_info, u32 *error_code)
3048 {
3049 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3050 
3051 	*info1 = control->exit_info_1;
3052 	*info2 = control->exit_info_2;
3053 	*intr_info = control->exit_int_info;
3054 	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3055 	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3056 		*error_code = control->exit_int_info_err;
3057 	else
3058 		*error_code = 0;
3059 }
3060 
3061 static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3062 {
3063 	struct vcpu_svm *svm = to_svm(vcpu);
3064 	struct kvm_run *kvm_run = vcpu->run;
3065 	u32 exit_code = svm->vmcb->control.exit_code;
3066 
3067 	trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3068 
3069 	if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3070 		vcpu->arch.cr0 = svm->vmcb->save.cr0;
3071 	if (npt_enabled)
3072 		vcpu->arch.cr3 = svm->vmcb->save.cr3;
3073 
3074 	if (is_guest_mode(vcpu)) {
3075 		int vmexit;
3076 
3077 		trace_kvm_nested_vmexit(exit_code, vcpu, KVM_ISA_SVM);
3078 
3079 		vmexit = nested_svm_exit_special(svm);
3080 
3081 		if (vmexit == NESTED_EXIT_CONTINUE)
3082 			vmexit = nested_svm_exit_handled(svm);
3083 
3084 		if (vmexit == NESTED_EXIT_DONE)
3085 			return 1;
3086 	}
3087 
3088 	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3089 		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3090 		kvm_run->fail_entry.hardware_entry_failure_reason
3091 			= svm->vmcb->control.exit_code;
3092 		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3093 		dump_vmcb(vcpu);
3094 		return 0;
3095 	}
3096 
3097 	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3098 	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3099 	    exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3100 	    exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3101 		printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3102 		       "exit_code 0x%x\n",
3103 		       __func__, svm->vmcb->control.exit_int_info,
3104 		       exit_code);
3105 
3106 	if (exit_fastpath != EXIT_FASTPATH_NONE)
3107 		return 1;
3108 
3109 	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3110 	    || !svm_exit_handlers[exit_code]) {
3111 		vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
3112 		dump_vmcb(vcpu);
3113 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3114 		vcpu->run->internal.suberror =
3115 			KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3116 		vcpu->run->internal.ndata = 2;
3117 		vcpu->run->internal.data[0] = exit_code;
3118 		vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3119 		return 0;
3120 	}
3121 
3122 #ifdef CONFIG_RETPOLINE
3123 	if (exit_code == SVM_EXIT_MSR)
3124 		return msr_interception(svm);
3125 	else if (exit_code == SVM_EXIT_VINTR)
3126 		return interrupt_window_interception(svm);
3127 	else if (exit_code == SVM_EXIT_INTR)
3128 		return intr_interception(svm);
3129 	else if (exit_code == SVM_EXIT_HLT)
3130 		return halt_interception(svm);
3131 	else if (exit_code == SVM_EXIT_NPF)
3132 		return npf_interception(svm);
3133 #endif
3134 	return svm_exit_handlers[exit_code](svm);
3135 }
3136 
3137 static void reload_tss(struct kvm_vcpu *vcpu)
3138 {
3139 	struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3140 
3141 	sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3142 	load_TR_desc();
3143 }
3144 
3145 static void pre_svm_run(struct vcpu_svm *svm)
3146 {
3147 	struct svm_cpu_data *sd = per_cpu(svm_data, svm->vcpu.cpu);
3148 
3149 	if (sev_guest(svm->vcpu.kvm))
3150 		return pre_sev_run(svm, svm->vcpu.cpu);
3151 
3152 	/* FIXME: handle wraparound of asid_generation */
3153 	if (svm->asid_generation != sd->asid_generation)
3154 		new_asid(svm, sd);
3155 }
3156 
3157 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3158 {
3159 	struct vcpu_svm *svm = to_svm(vcpu);
3160 
3161 	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3162 	vcpu->arch.hflags |= HF_NMI_MASK;
3163 	svm_set_intercept(svm, INTERCEPT_IRET);
3164 	++vcpu->stat.nmi_injections;
3165 }
3166 
3167 static void svm_set_irq(struct kvm_vcpu *vcpu)
3168 {
3169 	struct vcpu_svm *svm = to_svm(vcpu);
3170 
3171 	BUG_ON(!(gif_set(svm)));
3172 
3173 	trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3174 	++vcpu->stat.irq_injections;
3175 
3176 	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3177 		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3178 }
3179 
3180 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3181 {
3182 	struct vcpu_svm *svm = to_svm(vcpu);
3183 
3184 	if (nested_svm_virtualize_tpr(vcpu))
3185 		return;
3186 
3187 	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3188 
3189 	if (irr == -1)
3190 		return;
3191 
3192 	if (tpr >= irr)
3193 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3194 }
3195 
3196 bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3197 {
3198 	struct vcpu_svm *svm = to_svm(vcpu);
3199 	struct vmcb *vmcb = svm->vmcb;
3200 	bool ret;
3201 
3202 	if (!gif_set(svm))
3203 		return true;
3204 
3205 	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3206 		return false;
3207 
3208 	ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
3209 	      (svm->vcpu.arch.hflags & HF_NMI_MASK);
3210 
3211 	return ret;
3212 }
3213 
3214 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3215 {
3216 	struct vcpu_svm *svm = to_svm(vcpu);
3217 	if (svm->nested.nested_run_pending)
3218 		return -EBUSY;
3219 
3220 	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3221 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3222 		return -EBUSY;
3223 
3224 	return !svm_nmi_blocked(vcpu);
3225 }
3226 
3227 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3228 {
3229 	struct vcpu_svm *svm = to_svm(vcpu);
3230 
3231 	return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3232 }
3233 
3234 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3235 {
3236 	struct vcpu_svm *svm = to_svm(vcpu);
3237 
3238 	if (masked) {
3239 		svm->vcpu.arch.hflags |= HF_NMI_MASK;
3240 		svm_set_intercept(svm, INTERCEPT_IRET);
3241 	} else {
3242 		svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3243 		svm_clr_intercept(svm, INTERCEPT_IRET);
3244 	}
3245 }
3246 
3247 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3248 {
3249 	struct vcpu_svm *svm = to_svm(vcpu);
3250 	struct vmcb *vmcb = svm->vmcb;
3251 
3252 	if (!gif_set(svm))
3253 		return true;
3254 
3255 	if (is_guest_mode(vcpu)) {
3256 		/* As long as interrupts are being delivered...  */
3257 		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3258 		    ? !(svm->nested.hsave->save.rflags & X86_EFLAGS_IF)
3259 		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3260 			return true;
3261 
3262 		/* ... vmexits aren't blocked by the interrupt shadow  */
3263 		if (nested_exit_on_intr(svm))
3264 			return false;
3265 	} else {
3266 		if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3267 			return true;
3268 	}
3269 
3270 	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3271 }
3272 
3273 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3274 {
3275 	struct vcpu_svm *svm = to_svm(vcpu);
3276 	if (svm->nested.nested_run_pending)
3277 		return -EBUSY;
3278 
3279 	/*
3280 	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3281 	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3282 	 */
3283 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3284 		return -EBUSY;
3285 
3286 	return !svm_interrupt_blocked(vcpu);
3287 }
3288 
3289 static void enable_irq_window(struct kvm_vcpu *vcpu)
3290 {
3291 	struct vcpu_svm *svm = to_svm(vcpu);
3292 
3293 	/*
3294 	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3295 	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3296 	 * get that intercept, this function will be called again though and
3297 	 * we'll get the vintr intercept. However, if the vGIF feature is
3298 	 * enabled, the STGI interception will not occur. Enable the irq
3299 	 * window under the assumption that the hardware will set the GIF.
3300 	 */
3301 	if (vgif_enabled(svm) || gif_set(svm)) {
3302 		/*
3303 		 * IRQ window is not needed when AVIC is enabled,
3304 		 * unless we have pending ExtINT since it cannot be injected
3305 		 * via AVIC. In such case, we need to temporarily disable AVIC,
3306 		 * and fallback to injecting IRQ via V_IRQ.
3307 		 */
3308 		svm_toggle_avic_for_irq_window(vcpu, false);
3309 		svm_set_vintr(svm);
3310 	}
3311 }
3312 
3313 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3314 {
3315 	struct vcpu_svm *svm = to_svm(vcpu);
3316 
3317 	if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3318 	    == HF_NMI_MASK)
3319 		return; /* IRET will cause a vm exit */
3320 
3321 	if (!gif_set(svm)) {
3322 		if (vgif_enabled(svm))
3323 			svm_set_intercept(svm, INTERCEPT_STGI);
3324 		return; /* STGI will cause a vm exit */
3325 	}
3326 
3327 	/*
3328 	 * Something prevents NMI from been injected. Single step over possible
3329 	 * problem (IRET or exception injection or interrupt shadow)
3330 	 */
3331 	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3332 	svm->nmi_singlestep = true;
3333 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3334 }
3335 
3336 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3337 {
3338 	return 0;
3339 }
3340 
3341 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
3342 {
3343 	return 0;
3344 }
3345 
3346 void svm_flush_tlb(struct kvm_vcpu *vcpu)
3347 {
3348 	struct vcpu_svm *svm = to_svm(vcpu);
3349 
3350 	/*
3351 	 * Flush only the current ASID even if the TLB flush was invoked via
3352 	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3353 	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3354 	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3355 	 * VM-Exit (via kvm_mmu_reset_context()).
3356 	 */
3357 	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3358 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3359 	else
3360 		svm->asid_generation--;
3361 }
3362 
3363 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3364 {
3365 	struct vcpu_svm *svm = to_svm(vcpu);
3366 
3367 	invlpga(gva, svm->vmcb->control.asid);
3368 }
3369 
3370 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3371 {
3372 }
3373 
3374 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3375 {
3376 	struct vcpu_svm *svm = to_svm(vcpu);
3377 
3378 	if (nested_svm_virtualize_tpr(vcpu))
3379 		return;
3380 
3381 	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3382 		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3383 		kvm_set_cr8(vcpu, cr8);
3384 	}
3385 }
3386 
3387 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3388 {
3389 	struct vcpu_svm *svm = to_svm(vcpu);
3390 	u64 cr8;
3391 
3392 	if (nested_svm_virtualize_tpr(vcpu) ||
3393 	    kvm_vcpu_apicv_active(vcpu))
3394 		return;
3395 
3396 	cr8 = kvm_get_cr8(vcpu);
3397 	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3398 	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3399 }
3400 
3401 static void svm_complete_interrupts(struct vcpu_svm *svm)
3402 {
3403 	u8 vector;
3404 	int type;
3405 	u32 exitintinfo = svm->vmcb->control.exit_int_info;
3406 	unsigned int3_injected = svm->int3_injected;
3407 
3408 	svm->int3_injected = 0;
3409 
3410 	/*
3411 	 * If we've made progress since setting HF_IRET_MASK, we've
3412 	 * executed an IRET and can allow NMI injection.
3413 	 */
3414 	if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
3415 	    && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
3416 		svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3417 		kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3418 	}
3419 
3420 	svm->vcpu.arch.nmi_injected = false;
3421 	kvm_clear_exception_queue(&svm->vcpu);
3422 	kvm_clear_interrupt_queue(&svm->vcpu);
3423 
3424 	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3425 		return;
3426 
3427 	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3428 
3429 	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3430 	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3431 
3432 	switch (type) {
3433 	case SVM_EXITINTINFO_TYPE_NMI:
3434 		svm->vcpu.arch.nmi_injected = true;
3435 		break;
3436 	case SVM_EXITINTINFO_TYPE_EXEPT:
3437 		/*
3438 		 * In case of software exceptions, do not reinject the vector,
3439 		 * but re-execute the instruction instead. Rewind RIP first
3440 		 * if we emulated INT3 before.
3441 		 */
3442 		if (kvm_exception_is_soft(vector)) {
3443 			if (vector == BP_VECTOR && int3_injected &&
3444 			    kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3445 				kvm_rip_write(&svm->vcpu,
3446 					      kvm_rip_read(&svm->vcpu) -
3447 					      int3_injected);
3448 			break;
3449 		}
3450 		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3451 			u32 err = svm->vmcb->control.exit_int_info_err;
3452 			kvm_requeue_exception_e(&svm->vcpu, vector, err);
3453 
3454 		} else
3455 			kvm_requeue_exception(&svm->vcpu, vector);
3456 		break;
3457 	case SVM_EXITINTINFO_TYPE_INTR:
3458 		kvm_queue_interrupt(&svm->vcpu, vector, false);
3459 		break;
3460 	default:
3461 		break;
3462 	}
3463 }
3464 
3465 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3466 {
3467 	struct vcpu_svm *svm = to_svm(vcpu);
3468 	struct vmcb_control_area *control = &svm->vmcb->control;
3469 
3470 	control->exit_int_info = control->event_inj;
3471 	control->exit_int_info_err = control->event_inj_err;
3472 	control->event_inj = 0;
3473 	svm_complete_interrupts(svm);
3474 }
3475 
3476 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
3477 {
3478 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
3479 	    to_svm(vcpu)->vmcb->control.exit_info_1)
3480 		return handle_fastpath_set_msr_irqoff(vcpu);
3481 
3482 	return EXIT_FASTPATH_NONE;
3483 }
3484 
3485 void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs);
3486 
3487 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
3488 					struct vcpu_svm *svm)
3489 {
3490 	/*
3491 	 * VMENTER enables interrupts (host state), but the kernel state is
3492 	 * interrupts disabled when this is invoked. Also tell RCU about
3493 	 * it. This is the same logic as for exit_to_user_mode().
3494 	 *
3495 	 * This ensures that e.g. latency analysis on the host observes
3496 	 * guest mode as interrupt enabled.
3497 	 *
3498 	 * guest_enter_irqoff() informs context tracking about the
3499 	 * transition to guest mode and if enabled adjusts RCU state
3500 	 * accordingly.
3501 	 */
3502 	instrumentation_begin();
3503 	trace_hardirqs_on_prepare();
3504 	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
3505 	instrumentation_end();
3506 
3507 	guest_enter_irqoff();
3508 	lockdep_hardirqs_on(CALLER_ADDR0);
3509 
3510 	__svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);
3511 
3512 #ifdef CONFIG_X86_64
3513 	native_wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3514 #else
3515 	loadsegment(fs, svm->host.fs);
3516 #ifndef CONFIG_X86_32_LAZY_GS
3517 	loadsegment(gs, svm->host.gs);
3518 #endif
3519 #endif
3520 
3521 	/*
3522 	 * VMEXIT disables interrupts (host state), but tracing and lockdep
3523 	 * have them in state 'on' as recorded before entering guest mode.
3524 	 * Same as enter_from_user_mode().
3525 	 *
3526 	 * guest_exit_irqoff() restores host context and reinstates RCU if
3527 	 * enabled and required.
3528 	 *
3529 	 * This needs to be done before the below as native_read_msr()
3530 	 * contains a tracepoint and x86_spec_ctrl_restore_host() calls
3531 	 * into world and some more.
3532 	 */
3533 	lockdep_hardirqs_off(CALLER_ADDR0);
3534 	guest_exit_irqoff();
3535 
3536 	instrumentation_begin();
3537 	trace_hardirqs_off_finish();
3538 	instrumentation_end();
3539 }
3540 
3541 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
3542 {
3543 	struct vcpu_svm *svm = to_svm(vcpu);
3544 
3545 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3546 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3547 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3548 
3549 	/*
3550 	 * Disable singlestep if we're injecting an interrupt/exception.
3551 	 * We don't want our modified rflags to be pushed on the stack where
3552 	 * we might not be able to easily reset them if we disabled NMI
3553 	 * singlestep later.
3554 	 */
3555 	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
3556 		/*
3557 		 * Event injection happens before external interrupts cause a
3558 		 * vmexit and interrupts are disabled here, so smp_send_reschedule
3559 		 * is enough to force an immediate vmexit.
3560 		 */
3561 		disable_nmi_singlestep(svm);
3562 		smp_send_reschedule(vcpu->cpu);
3563 	}
3564 
3565 	pre_svm_run(svm);
3566 
3567 	sync_lapic_to_cr8(vcpu);
3568 
3569 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
3570 
3571 	/*
3572 	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
3573 	 * of a #DB.
3574 	 */
3575 	if (unlikely(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3576 		svm_set_dr6(svm, vcpu->arch.dr6);
3577 	else
3578 		svm_set_dr6(svm, DR6_FIXED_1 | DR6_RTM);
3579 
3580 	clgi();
3581 	kvm_load_guest_xsave_state(vcpu);
3582 
3583 	kvm_wait_lapic_expire(vcpu);
3584 
3585 	/*
3586 	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
3587 	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
3588 	 * is no need to worry about the conditional branch over the wrmsr
3589 	 * being speculatively taken.
3590 	 */
3591 	x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
3592 
3593 	svm_vcpu_enter_exit(vcpu, svm);
3594 
3595 	/*
3596 	 * We do not use IBRS in the kernel. If this vCPU has used the
3597 	 * SPEC_CTRL MSR it may have left it on; save the value and
3598 	 * turn it off. This is much more efficient than blindly adding
3599 	 * it to the atomic save/restore list. Especially as the former
3600 	 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
3601 	 *
3602 	 * For non-nested case:
3603 	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
3604 	 * save it.
3605 	 *
3606 	 * For nested case:
3607 	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
3608 	 * save it.
3609 	 */
3610 	if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
3611 		svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
3612 
3613 	reload_tss(vcpu);
3614 
3615 	x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
3616 
3617 	vcpu->arch.cr2 = svm->vmcb->save.cr2;
3618 	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3619 	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3620 	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3621 
3622 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3623 		kvm_before_interrupt(&svm->vcpu);
3624 
3625 	kvm_load_host_xsave_state(vcpu);
3626 	stgi();
3627 
3628 	/* Any pending NMI will happen here */
3629 
3630 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3631 		kvm_after_interrupt(&svm->vcpu);
3632 
3633 	sync_cr8_to_lapic(vcpu);
3634 
3635 	svm->next_rip = 0;
3636 	if (is_guest_mode(&svm->vcpu)) {
3637 		sync_nested_vmcb_control(svm);
3638 		svm->nested.nested_run_pending = 0;
3639 	}
3640 
3641 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3642 	vmcb_mark_all_clean(svm->vmcb);
3643 
3644 	/* if exit due to PF check for async PF */
3645 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3646 		svm->vcpu.arch.apf.host_apf_flags =
3647 			kvm_read_and_reset_apf_flags();
3648 
3649 	if (npt_enabled) {
3650 		vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
3651 		vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
3652 	}
3653 
3654 	/*
3655 	 * We need to handle MC intercepts here before the vcpu has a chance to
3656 	 * change the physical cpu
3657 	 */
3658 	if (unlikely(svm->vmcb->control.exit_code ==
3659 		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
3660 		svm_handle_mce(svm);
3661 
3662 	svm_complete_interrupts(svm);
3663 
3664 	if (is_guest_mode(vcpu))
3665 		return EXIT_FASTPATH_NONE;
3666 
3667 	return svm_exit_handlers_fastpath(vcpu);
3668 }
3669 
3670 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root,
3671 			     int root_level)
3672 {
3673 	struct vcpu_svm *svm = to_svm(vcpu);
3674 	unsigned long cr3;
3675 
3676 	cr3 = __sme_set(root);
3677 	if (npt_enabled) {
3678 		svm->vmcb->control.nested_cr3 = cr3;
3679 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3680 
3681 		/* Loading L2's CR3 is handled by enter_svm_guest_mode.  */
3682 		if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
3683 			return;
3684 		cr3 = vcpu->arch.cr3;
3685 	}
3686 
3687 	svm->vmcb->save.cr3 = cr3;
3688 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
3689 }
3690 
3691 static int is_disabled(void)
3692 {
3693 	u64 vm_cr;
3694 
3695 	rdmsrl(MSR_VM_CR, vm_cr);
3696 	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
3697 		return 1;
3698 
3699 	return 0;
3700 }
3701 
3702 static void
3703 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3704 {
3705 	/*
3706 	 * Patch in the VMMCALL instruction:
3707 	 */
3708 	hypercall[0] = 0x0f;
3709 	hypercall[1] = 0x01;
3710 	hypercall[2] = 0xd9;
3711 }
3712 
3713 static int __init svm_check_processor_compat(void)
3714 {
3715 	return 0;
3716 }
3717 
3718 static bool svm_cpu_has_accelerated_tpr(void)
3719 {
3720 	return false;
3721 }
3722 
3723 static bool svm_has_emulated_msr(u32 index)
3724 {
3725 	switch (index) {
3726 	case MSR_IA32_MCG_EXT_CTL:
3727 	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3728 		return false;
3729 	default:
3730 		break;
3731 	}
3732 
3733 	return true;
3734 }
3735 
3736 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
3737 {
3738 	return 0;
3739 }
3740 
3741 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
3742 {
3743 	struct vcpu_svm *svm = to_svm(vcpu);
3744 	struct kvm_cpuid_entry2 *best;
3745 
3746 	vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
3747 				    boot_cpu_has(X86_FEATURE_XSAVE) &&
3748 				    boot_cpu_has(X86_FEATURE_XSAVES);
3749 
3750 	/* Update nrips enabled cache */
3751 	svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) &&
3752 			     guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
3753 
3754 	/* Check again if INVPCID interception if required */
3755 	svm_check_invpcid(svm);
3756 
3757 	/* For sev guests, the memory encryption bit is not reserved in CR3.  */
3758 	if (sev_guest(vcpu->kvm)) {
3759 		best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);
3760 		if (best)
3761 			vcpu->arch.cr3_lm_rsvd_bits &= ~(1UL << (best->ebx & 0x3f));
3762 	}
3763 
3764 	if (!kvm_vcpu_apicv_active(vcpu))
3765 		return;
3766 
3767 	/*
3768 	 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
3769 	 * is exposed to the guest, disable AVIC.
3770 	 */
3771 	if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
3772 		kvm_request_apicv_update(vcpu->kvm, false,
3773 					 APICV_INHIBIT_REASON_X2APIC);
3774 
3775 	/*
3776 	 * Currently, AVIC does not work with nested virtualization.
3777 	 * So, we disable AVIC when cpuid for SVM is set in the L1 guest.
3778 	 */
3779 	if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM))
3780 		kvm_request_apicv_update(vcpu->kvm, false,
3781 					 APICV_INHIBIT_REASON_NESTED);
3782 }
3783 
3784 static bool svm_has_wbinvd_exit(void)
3785 {
3786 	return true;
3787 }
3788 
3789 #define PRE_EX(exit)  { .exit_code = (exit), \
3790 			.stage = X86_ICPT_PRE_EXCEPT, }
3791 #define POST_EX(exit) { .exit_code = (exit), \
3792 			.stage = X86_ICPT_POST_EXCEPT, }
3793 #define POST_MEM(exit) { .exit_code = (exit), \
3794 			.stage = X86_ICPT_POST_MEMACCESS, }
3795 
3796 static const struct __x86_intercept {
3797 	u32 exit_code;
3798 	enum x86_intercept_stage stage;
3799 } x86_intercept_map[] = {
3800 	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
3801 	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
3802 	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
3803 	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
3804 	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
3805 	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
3806 	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
3807 	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
3808 	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
3809 	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
3810 	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
3811 	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
3812 	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
3813 	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
3814 	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
3815 	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
3816 	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
3817 	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
3818 	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
3819 	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
3820 	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
3821 	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
3822 	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
3823 	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
3824 	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
3825 	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
3826 	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
3827 	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
3828 	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
3829 	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
3830 	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
3831 	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
3832 	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
3833 	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
3834 	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
3835 	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
3836 	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
3837 	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
3838 	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
3839 	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
3840 	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
3841 	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
3842 	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
3843 	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
3844 	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
3845 	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
3846 	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
3847 };
3848 
3849 #undef PRE_EX
3850 #undef POST_EX
3851 #undef POST_MEM
3852 
3853 static int svm_check_intercept(struct kvm_vcpu *vcpu,
3854 			       struct x86_instruction_info *info,
3855 			       enum x86_intercept_stage stage,
3856 			       struct x86_exception *exception)
3857 {
3858 	struct vcpu_svm *svm = to_svm(vcpu);
3859 	int vmexit, ret = X86EMUL_CONTINUE;
3860 	struct __x86_intercept icpt_info;
3861 	struct vmcb *vmcb = svm->vmcb;
3862 
3863 	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
3864 		goto out;
3865 
3866 	icpt_info = x86_intercept_map[info->intercept];
3867 
3868 	if (stage != icpt_info.stage)
3869 		goto out;
3870 
3871 	switch (icpt_info.exit_code) {
3872 	case SVM_EXIT_READ_CR0:
3873 		if (info->intercept == x86_intercept_cr_read)
3874 			icpt_info.exit_code += info->modrm_reg;
3875 		break;
3876 	case SVM_EXIT_WRITE_CR0: {
3877 		unsigned long cr0, val;
3878 
3879 		if (info->intercept == x86_intercept_cr_write)
3880 			icpt_info.exit_code += info->modrm_reg;
3881 
3882 		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
3883 		    info->intercept == x86_intercept_clts)
3884 			break;
3885 
3886 		if (!(vmcb_is_intercept(&svm->nested.ctl,
3887 					INTERCEPT_SELECTIVE_CR0)))
3888 			break;
3889 
3890 		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
3891 		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
3892 
3893 		if (info->intercept == x86_intercept_lmsw) {
3894 			cr0 &= 0xfUL;
3895 			val &= 0xfUL;
3896 			/* lmsw can't clear PE - catch this here */
3897 			if (cr0 & X86_CR0_PE)
3898 				val |= X86_CR0_PE;
3899 		}
3900 
3901 		if (cr0 ^ val)
3902 			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3903 
3904 		break;
3905 	}
3906 	case SVM_EXIT_READ_DR0:
3907 	case SVM_EXIT_WRITE_DR0:
3908 		icpt_info.exit_code += info->modrm_reg;
3909 		break;
3910 	case SVM_EXIT_MSR:
3911 		if (info->intercept == x86_intercept_wrmsr)
3912 			vmcb->control.exit_info_1 = 1;
3913 		else
3914 			vmcb->control.exit_info_1 = 0;
3915 		break;
3916 	case SVM_EXIT_PAUSE:
3917 		/*
3918 		 * We get this for NOP only, but pause
3919 		 * is rep not, check this here
3920 		 */
3921 		if (info->rep_prefix != REPE_PREFIX)
3922 			goto out;
3923 		break;
3924 	case SVM_EXIT_IOIO: {
3925 		u64 exit_info;
3926 		u32 bytes;
3927 
3928 		if (info->intercept == x86_intercept_in ||
3929 		    info->intercept == x86_intercept_ins) {
3930 			exit_info = ((info->src_val & 0xffff) << 16) |
3931 				SVM_IOIO_TYPE_MASK;
3932 			bytes = info->dst_bytes;
3933 		} else {
3934 			exit_info = (info->dst_val & 0xffff) << 16;
3935 			bytes = info->src_bytes;
3936 		}
3937 
3938 		if (info->intercept == x86_intercept_outs ||
3939 		    info->intercept == x86_intercept_ins)
3940 			exit_info |= SVM_IOIO_STR_MASK;
3941 
3942 		if (info->rep_prefix)
3943 			exit_info |= SVM_IOIO_REP_MASK;
3944 
3945 		bytes = min(bytes, 4u);
3946 
3947 		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
3948 
3949 		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
3950 
3951 		vmcb->control.exit_info_1 = exit_info;
3952 		vmcb->control.exit_info_2 = info->next_rip;
3953 
3954 		break;
3955 	}
3956 	default:
3957 		break;
3958 	}
3959 
3960 	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
3961 	if (static_cpu_has(X86_FEATURE_NRIPS))
3962 		vmcb->control.next_rip  = info->next_rip;
3963 	vmcb->control.exit_code = icpt_info.exit_code;
3964 	vmexit = nested_svm_exit_handled(svm);
3965 
3966 	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
3967 					   : X86EMUL_CONTINUE;
3968 
3969 out:
3970 	return ret;
3971 }
3972 
3973 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
3974 {
3975 }
3976 
3977 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
3978 {
3979 	if (!kvm_pause_in_guest(vcpu->kvm))
3980 		shrink_ple_window(vcpu);
3981 }
3982 
3983 static void svm_setup_mce(struct kvm_vcpu *vcpu)
3984 {
3985 	/* [63:9] are reserved. */
3986 	vcpu->arch.mcg_cap &= 0x1ff;
3987 }
3988 
3989 bool svm_smi_blocked(struct kvm_vcpu *vcpu)
3990 {
3991 	struct vcpu_svm *svm = to_svm(vcpu);
3992 
3993 	/* Per APM Vol.2 15.22.2 "Response to SMI" */
3994 	if (!gif_set(svm))
3995 		return true;
3996 
3997 	return is_smm(vcpu);
3998 }
3999 
4000 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4001 {
4002 	struct vcpu_svm *svm = to_svm(vcpu);
4003 	if (svm->nested.nested_run_pending)
4004 		return -EBUSY;
4005 
4006 	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4007 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4008 		return -EBUSY;
4009 
4010 	return !svm_smi_blocked(vcpu);
4011 }
4012 
4013 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
4014 {
4015 	struct vcpu_svm *svm = to_svm(vcpu);
4016 	int ret;
4017 
4018 	if (is_guest_mode(vcpu)) {
4019 		/* FED8h - SVM Guest */
4020 		put_smstate(u64, smstate, 0x7ed8, 1);
4021 		/* FEE0h - SVM Guest VMCB Physical Address */
4022 		put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa);
4023 
4024 		svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4025 		svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4026 		svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4027 
4028 		ret = nested_svm_vmexit(svm);
4029 		if (ret)
4030 			return ret;
4031 	}
4032 	return 0;
4033 }
4034 
4035 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
4036 {
4037 	struct vcpu_svm *svm = to_svm(vcpu);
4038 	struct kvm_host_map map;
4039 	int ret = 0;
4040 
4041 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) {
4042 		u64 saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0);
4043 		u64 guest = GET_SMSTATE(u64, smstate, 0x7ed8);
4044 		u64 vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0);
4045 
4046 		if (guest) {
4047 			if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4048 				return 1;
4049 
4050 			if (!(saved_efer & EFER_SVME))
4051 				return 1;
4052 
4053 			if (kvm_vcpu_map(&svm->vcpu,
4054 					 gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL)
4055 				return 1;
4056 
4057 			if (svm_allocate_nested(svm))
4058 				return 1;
4059 
4060 			ret = enter_svm_guest_mode(svm, vmcb12_gpa, map.hva);
4061 			kvm_vcpu_unmap(&svm->vcpu, &map, true);
4062 		}
4063 	}
4064 
4065 	return ret;
4066 }
4067 
4068 static void enable_smi_window(struct kvm_vcpu *vcpu)
4069 {
4070 	struct vcpu_svm *svm = to_svm(vcpu);
4071 
4072 	if (!gif_set(svm)) {
4073 		if (vgif_enabled(svm))
4074 			svm_set_intercept(svm, INTERCEPT_STGI);
4075 		/* STGI will cause a vm exit */
4076 	} else {
4077 		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4078 	}
4079 }
4080 
4081 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len)
4082 {
4083 	bool smep, smap, is_user;
4084 	unsigned long cr4;
4085 
4086 	/*
4087 	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4088 	 *
4089 	 * Errata:
4090 	 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
4091 	 * possible that CPU microcode implementing DecodeAssist will fail
4092 	 * to read bytes of instruction which caused #NPF. In this case,
4093 	 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
4094 	 * return 0 instead of the correct guest instruction bytes.
4095 	 *
4096 	 * This happens because CPU microcode reading instruction bytes
4097 	 * uses a special opcode which attempts to read data using CPL=0
4098 	 * priviledges. The microcode reads CS:RIP and if it hits a SMAP
4099 	 * fault, it gives up and returns no instruction bytes.
4100 	 *
4101 	 * Detection:
4102 	 * We reach here in case CPU supports DecodeAssist, raised #NPF and
4103 	 * returned 0 in GuestIntrBytes field of the VMCB.
4104 	 * First, errata can only be triggered in case vCPU CR4.SMAP=1.
4105 	 * Second, if vCPU CR4.SMEP=1, errata could only be triggered
4106 	 * in case vCPU CPL==3 (Because otherwise guest would have triggered
4107 	 * a SMEP fault instead of #NPF).
4108 	 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
4109 	 * As most guests enable SMAP if they have also enabled SMEP, use above
4110 	 * logic in order to attempt minimize false-positive of detecting errata
4111 	 * while still preserving all cases semantic correctness.
4112 	 *
4113 	 * Workaround:
4114 	 * To determine what instruction the guest was executing, the hypervisor
4115 	 * will have to decode the instruction at the instruction pointer.
4116 	 *
4117 	 * In non SEV guest, hypervisor will be able to read the guest
4118 	 * memory to decode the instruction pointer when insn_len is zero
4119 	 * so we return true to indicate that decoding is possible.
4120 	 *
4121 	 * But in the SEV guest, the guest memory is encrypted with the
4122 	 * guest specific key and hypervisor will not be able to decode the
4123 	 * instruction pointer so we will not able to workaround it. Lets
4124 	 * print the error and request to kill the guest.
4125 	 */
4126 	if (likely(!insn || insn_len))
4127 		return true;
4128 
4129 	/*
4130 	 * If RIP is invalid, go ahead with emulation which will cause an
4131 	 * internal error exit.
4132 	 */
4133 	if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT))
4134 		return true;
4135 
4136 	cr4 = kvm_read_cr4(vcpu);
4137 	smep = cr4 & X86_CR4_SMEP;
4138 	smap = cr4 & X86_CR4_SMAP;
4139 	is_user = svm_get_cpl(vcpu) == 3;
4140 	if (smap && (!smep || is_user)) {
4141 		if (!sev_guest(vcpu->kvm))
4142 			return true;
4143 
4144 		pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
4145 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4146 	}
4147 
4148 	return false;
4149 }
4150 
4151 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4152 {
4153 	struct vcpu_svm *svm = to_svm(vcpu);
4154 
4155 	/*
4156 	 * TODO: Last condition latch INIT signals on vCPU when
4157 	 * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
4158 	 * To properly emulate the INIT intercept,
4159 	 * svm_check_nested_events() should call nested_svm_vmexit()
4160 	 * if an INIT signal is pending.
4161 	 */
4162 	return !gif_set(svm) ||
4163 		   (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT));
4164 }
4165 
4166 static void svm_vm_destroy(struct kvm *kvm)
4167 {
4168 	avic_vm_destroy(kvm);
4169 	sev_vm_destroy(kvm);
4170 }
4171 
4172 static int svm_vm_init(struct kvm *kvm)
4173 {
4174 	if (!pause_filter_count || !pause_filter_thresh)
4175 		kvm->arch.pause_in_guest = true;
4176 
4177 	if (avic) {
4178 		int ret = avic_vm_init(kvm);
4179 		if (ret)
4180 			return ret;
4181 	}
4182 
4183 	kvm_apicv_init(kvm, avic);
4184 	return 0;
4185 }
4186 
4187 static struct kvm_x86_ops svm_x86_ops __initdata = {
4188 	.hardware_unsetup = svm_hardware_teardown,
4189 	.hardware_enable = svm_hardware_enable,
4190 	.hardware_disable = svm_hardware_disable,
4191 	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4192 	.has_emulated_msr = svm_has_emulated_msr,
4193 
4194 	.vcpu_create = svm_create_vcpu,
4195 	.vcpu_free = svm_free_vcpu,
4196 	.vcpu_reset = svm_vcpu_reset,
4197 
4198 	.vm_size = sizeof(struct kvm_svm),
4199 	.vm_init = svm_vm_init,
4200 	.vm_destroy = svm_vm_destroy,
4201 
4202 	.prepare_guest_switch = svm_prepare_guest_switch,
4203 	.vcpu_load = svm_vcpu_load,
4204 	.vcpu_put = svm_vcpu_put,
4205 	.vcpu_blocking = svm_vcpu_blocking,
4206 	.vcpu_unblocking = svm_vcpu_unblocking,
4207 
4208 	.update_exception_bitmap = update_exception_bitmap,
4209 	.get_msr_feature = svm_get_msr_feature,
4210 	.get_msr = svm_get_msr,
4211 	.set_msr = svm_set_msr,
4212 	.get_segment_base = svm_get_segment_base,
4213 	.get_segment = svm_get_segment,
4214 	.set_segment = svm_set_segment,
4215 	.get_cpl = svm_get_cpl,
4216 	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4217 	.set_cr0 = svm_set_cr0,
4218 	.set_cr4 = svm_set_cr4,
4219 	.set_efer = svm_set_efer,
4220 	.get_idt = svm_get_idt,
4221 	.set_idt = svm_set_idt,
4222 	.get_gdt = svm_get_gdt,
4223 	.set_gdt = svm_set_gdt,
4224 	.set_dr7 = svm_set_dr7,
4225 	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4226 	.cache_reg = svm_cache_reg,
4227 	.get_rflags = svm_get_rflags,
4228 	.set_rflags = svm_set_rflags,
4229 
4230 	.tlb_flush_all = svm_flush_tlb,
4231 	.tlb_flush_current = svm_flush_tlb,
4232 	.tlb_flush_gva = svm_flush_tlb_gva,
4233 	.tlb_flush_guest = svm_flush_tlb,
4234 
4235 	.run = svm_vcpu_run,
4236 	.handle_exit = handle_exit,
4237 	.skip_emulated_instruction = skip_emulated_instruction,
4238 	.update_emulated_instruction = NULL,
4239 	.set_interrupt_shadow = svm_set_interrupt_shadow,
4240 	.get_interrupt_shadow = svm_get_interrupt_shadow,
4241 	.patch_hypercall = svm_patch_hypercall,
4242 	.set_irq = svm_set_irq,
4243 	.set_nmi = svm_inject_nmi,
4244 	.queue_exception = svm_queue_exception,
4245 	.cancel_injection = svm_cancel_injection,
4246 	.interrupt_allowed = svm_interrupt_allowed,
4247 	.nmi_allowed = svm_nmi_allowed,
4248 	.get_nmi_mask = svm_get_nmi_mask,
4249 	.set_nmi_mask = svm_set_nmi_mask,
4250 	.enable_nmi_window = enable_nmi_window,
4251 	.enable_irq_window = enable_irq_window,
4252 	.update_cr8_intercept = update_cr8_intercept,
4253 	.set_virtual_apic_mode = svm_set_virtual_apic_mode,
4254 	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4255 	.check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
4256 	.pre_update_apicv_exec_ctrl = svm_pre_update_apicv_exec_ctrl,
4257 	.load_eoi_exitmap = svm_load_eoi_exitmap,
4258 	.hwapic_irr_update = svm_hwapic_irr_update,
4259 	.hwapic_isr_update = svm_hwapic_isr_update,
4260 	.sync_pir_to_irr = kvm_lapic_find_highest_irr,
4261 	.apicv_post_state_restore = avic_post_state_restore,
4262 
4263 	.set_tss_addr = svm_set_tss_addr,
4264 	.set_identity_map_addr = svm_set_identity_map_addr,
4265 	.get_mt_mask = svm_get_mt_mask,
4266 
4267 	.get_exit_info = svm_get_exit_info,
4268 
4269 	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
4270 
4271 	.has_wbinvd_exit = svm_has_wbinvd_exit,
4272 
4273 	.write_l1_tsc_offset = svm_write_l1_tsc_offset,
4274 
4275 	.load_mmu_pgd = svm_load_mmu_pgd,
4276 
4277 	.check_intercept = svm_check_intercept,
4278 	.handle_exit_irqoff = svm_handle_exit_irqoff,
4279 
4280 	.request_immediate_exit = __kvm_request_immediate_exit,
4281 
4282 	.sched_in = svm_sched_in,
4283 
4284 	.pmu_ops = &amd_pmu_ops,
4285 	.nested_ops = &svm_nested_ops,
4286 
4287 	.deliver_posted_interrupt = svm_deliver_avic_intr,
4288 	.dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
4289 	.update_pi_irte = svm_update_pi_irte,
4290 	.setup_mce = svm_setup_mce,
4291 
4292 	.smi_allowed = svm_smi_allowed,
4293 	.pre_enter_smm = svm_pre_enter_smm,
4294 	.pre_leave_smm = svm_pre_leave_smm,
4295 	.enable_smi_window = enable_smi_window,
4296 
4297 	.mem_enc_op = svm_mem_enc_op,
4298 	.mem_enc_reg_region = svm_register_enc_region,
4299 	.mem_enc_unreg_region = svm_unregister_enc_region,
4300 
4301 	.can_emulate_instruction = svm_can_emulate_instruction,
4302 
4303 	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
4304 
4305 	.msr_filter_changed = svm_msr_filter_changed,
4306 };
4307 
4308 static struct kvm_x86_init_ops svm_init_ops __initdata = {
4309 	.cpu_has_kvm_support = has_svm,
4310 	.disabled_by_bios = is_disabled,
4311 	.hardware_setup = svm_hardware_setup,
4312 	.check_processor_compatibility = svm_check_processor_compat,
4313 
4314 	.runtime_ops = &svm_x86_ops,
4315 };
4316 
4317 static int __init svm_init(void)
4318 {
4319 	__unused_size_checks();
4320 
4321 	return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm),
4322 			__alignof__(struct vcpu_svm), THIS_MODULE);
4323 }
4324 
4325 static void __exit svm_exit(void)
4326 {
4327 	kvm_exit();
4328 }
4329 
4330 module_init(svm_init)
4331 module_exit(svm_exit)
4332