xref: /openbmc/linux/tools/testing/selftests/kvm/lib/x86_64/processor.c (revision b1a792601f264df7172a728f1a83a05b6b399dfb)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * tools/testing/selftests/kvm/lib/x86_64/processor.c
4  *
5  * Copyright (C) 2018, Google LLC.
6  */
7 
8 #include "test_util.h"
9 #include "kvm_util.h"
10 #include "../kvm_util_internal.h"
11 #include "processor.h"
12 
13 #ifndef NUM_INTERRUPTS
14 #define NUM_INTERRUPTS 256
15 #endif
16 
17 #define DEFAULT_CODE_SELECTOR 0x8
18 #define DEFAULT_DATA_SELECTOR 0x10
19 
20 /* Minimum physical address used for virtual translation tables. */
21 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000
22 
23 vm_vaddr_t exception_handlers;
24 
25 /* Virtual translation table structure declarations */
26 struct pageMapL4Entry {
27 	uint64_t present:1;
28 	uint64_t writable:1;
29 	uint64_t user:1;
30 	uint64_t write_through:1;
31 	uint64_t cache_disable:1;
32 	uint64_t accessed:1;
33 	uint64_t ignored_06:1;
34 	uint64_t page_size:1;
35 	uint64_t ignored_11_08:4;
36 	uint64_t address:40;
37 	uint64_t ignored_62_52:11;
38 	uint64_t execute_disable:1;
39 };
40 
41 struct pageDirectoryPointerEntry {
42 	uint64_t present:1;
43 	uint64_t writable:1;
44 	uint64_t user:1;
45 	uint64_t write_through:1;
46 	uint64_t cache_disable:1;
47 	uint64_t accessed:1;
48 	uint64_t ignored_06:1;
49 	uint64_t page_size:1;
50 	uint64_t ignored_11_08:4;
51 	uint64_t address:40;
52 	uint64_t ignored_62_52:11;
53 	uint64_t execute_disable:1;
54 };
55 
56 struct pageDirectoryEntry {
57 	uint64_t present:1;
58 	uint64_t writable:1;
59 	uint64_t user:1;
60 	uint64_t write_through:1;
61 	uint64_t cache_disable:1;
62 	uint64_t accessed:1;
63 	uint64_t ignored_06:1;
64 	uint64_t page_size:1;
65 	uint64_t ignored_11_08:4;
66 	uint64_t address:40;
67 	uint64_t ignored_62_52:11;
68 	uint64_t execute_disable:1;
69 };
70 
71 struct pageTableEntry {
72 	uint64_t present:1;
73 	uint64_t writable:1;
74 	uint64_t user:1;
75 	uint64_t write_through:1;
76 	uint64_t cache_disable:1;
77 	uint64_t accessed:1;
78 	uint64_t dirty:1;
79 	uint64_t reserved_07:1;
80 	uint64_t global:1;
81 	uint64_t ignored_11_09:3;
82 	uint64_t address:40;
83 	uint64_t ignored_62_52:11;
84 	uint64_t execute_disable:1;
85 };
86 
87 void regs_dump(FILE *stream, struct kvm_regs *regs,
88 	       uint8_t indent)
89 {
90 	fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
91 		"rcx: 0x%.16llx rdx: 0x%.16llx\n",
92 		indent, "",
93 		regs->rax, regs->rbx, regs->rcx, regs->rdx);
94 	fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
95 		"rsp: 0x%.16llx rbp: 0x%.16llx\n",
96 		indent, "",
97 		regs->rsi, regs->rdi, regs->rsp, regs->rbp);
98 	fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "
99 		"r10: 0x%.16llx r11: 0x%.16llx\n",
100 		indent, "",
101 		regs->r8, regs->r9, regs->r10, regs->r11);
102 	fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
103 		"r14: 0x%.16llx r15: 0x%.16llx\n",
104 		indent, "",
105 		regs->r12, regs->r13, regs->r14, regs->r15);
106 	fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
107 		indent, "",
108 		regs->rip, regs->rflags);
109 }
110 
111 /*
112  * Segment Dump
113  *
114  * Input Args:
115  *   stream  - Output FILE stream
116  *   segment - KVM segment
117  *   indent  - Left margin indent amount
118  *
119  * Output Args: None
120  *
121  * Return: None
122  *
123  * Dumps the state of the KVM segment given by @segment, to the FILE stream
124  * given by @stream.
125  */
126 static void segment_dump(FILE *stream, struct kvm_segment *segment,
127 			 uint8_t indent)
128 {
129 	fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
130 		"selector: 0x%.4x type: 0x%.2x\n",
131 		indent, "", segment->base, segment->limit,
132 		segment->selector, segment->type);
133 	fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
134 		"db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
135 		indent, "", segment->present, segment->dpl,
136 		segment->db, segment->s, segment->l);
137 	fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
138 		"unusable: 0x%.2x padding: 0x%.2x\n",
139 		indent, "", segment->g, segment->avl,
140 		segment->unusable, segment->padding);
141 }
142 
143 /*
144  * dtable Dump
145  *
146  * Input Args:
147  *   stream - Output FILE stream
148  *   dtable - KVM dtable
149  *   indent - Left margin indent amount
150  *
151  * Output Args: None
152  *
153  * Return: None
154  *
155  * Dumps the state of the KVM dtable given by @dtable, to the FILE stream
156  * given by @stream.
157  */
158 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
159 			uint8_t indent)
160 {
161 	fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
162 		"padding: 0x%.4x 0x%.4x 0x%.4x\n",
163 		indent, "", dtable->base, dtable->limit,
164 		dtable->padding[0], dtable->padding[1], dtable->padding[2]);
165 }
166 
167 void sregs_dump(FILE *stream, struct kvm_sregs *sregs,
168 		uint8_t indent)
169 {
170 	unsigned int i;
171 
172 	fprintf(stream, "%*scs:\n", indent, "");
173 	segment_dump(stream, &sregs->cs, indent + 2);
174 	fprintf(stream, "%*sds:\n", indent, "");
175 	segment_dump(stream, &sregs->ds, indent + 2);
176 	fprintf(stream, "%*ses:\n", indent, "");
177 	segment_dump(stream, &sregs->es, indent + 2);
178 	fprintf(stream, "%*sfs:\n", indent, "");
179 	segment_dump(stream, &sregs->fs, indent + 2);
180 	fprintf(stream, "%*sgs:\n", indent, "");
181 	segment_dump(stream, &sregs->gs, indent + 2);
182 	fprintf(stream, "%*sss:\n", indent, "");
183 	segment_dump(stream, &sregs->ss, indent + 2);
184 	fprintf(stream, "%*str:\n", indent, "");
185 	segment_dump(stream, &sregs->tr, indent + 2);
186 	fprintf(stream, "%*sldt:\n", indent, "");
187 	segment_dump(stream, &sregs->ldt, indent + 2);
188 
189 	fprintf(stream, "%*sgdt:\n", indent, "");
190 	dtable_dump(stream, &sregs->gdt, indent + 2);
191 	fprintf(stream, "%*sidt:\n", indent, "");
192 	dtable_dump(stream, &sregs->idt, indent + 2);
193 
194 	fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
195 		"cr3: 0x%.16llx cr4: 0x%.16llx\n",
196 		indent, "",
197 		sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
198 	fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
199 		"apic_base: 0x%.16llx\n",
200 		indent, "",
201 		sregs->cr8, sregs->efer, sregs->apic_base);
202 
203 	fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
204 	for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
205 		fprintf(stream, "%*s%.16llx\n", indent + 2, "",
206 			sregs->interrupt_bitmap[i]);
207 	}
208 }
209 
210 void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot)
211 {
212 	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
213 		"unknown or unsupported guest mode, mode: 0x%x", vm->mode);
214 
215 	/* If needed, create page map l4 table. */
216 	if (!vm->pgd_created) {
217 		vm_paddr_t paddr = vm_phy_page_alloc(vm,
218 			KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot);
219 		vm->pgd = paddr;
220 		vm->pgd_created = true;
221 	}
222 }
223 
224 void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
225 	uint32_t pgd_memslot)
226 {
227 	uint16_t index[4];
228 	struct pageMapL4Entry *pml4e;
229 
230 	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
231 		"unknown or unsupported guest mode, mode: 0x%x", vm->mode);
232 
233 	TEST_ASSERT((vaddr % vm->page_size) == 0,
234 		"Virtual address not on page boundary,\n"
235 		"  vaddr: 0x%lx vm->page_size: 0x%x",
236 		vaddr, vm->page_size);
237 	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
238 		(vaddr >> vm->page_shift)),
239 		"Invalid virtual address, vaddr: 0x%lx",
240 		vaddr);
241 	TEST_ASSERT((paddr % vm->page_size) == 0,
242 		"Physical address not on page boundary,\n"
243 		"  paddr: 0x%lx vm->page_size: 0x%x",
244 		paddr, vm->page_size);
245 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
246 		"Physical address beyond beyond maximum supported,\n"
247 		"  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
248 		paddr, vm->max_gfn, vm->page_size);
249 
250 	index[0] = (vaddr >> 12) & 0x1ffu;
251 	index[1] = (vaddr >> 21) & 0x1ffu;
252 	index[2] = (vaddr >> 30) & 0x1ffu;
253 	index[3] = (vaddr >> 39) & 0x1ffu;
254 
255 	/* Allocate page directory pointer table if not present. */
256 	pml4e = addr_gpa2hva(vm, vm->pgd);
257 	if (!pml4e[index[3]].present) {
258 		pml4e[index[3]].address = vm_phy_page_alloc(vm,
259 			KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
260 			>> vm->page_shift;
261 		pml4e[index[3]].writable = true;
262 		pml4e[index[3]].present = true;
263 	}
264 
265 	/* Allocate page directory table if not present. */
266 	struct pageDirectoryPointerEntry *pdpe;
267 	pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size);
268 	if (!pdpe[index[2]].present) {
269 		pdpe[index[2]].address = vm_phy_page_alloc(vm,
270 			KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
271 			>> vm->page_shift;
272 		pdpe[index[2]].writable = true;
273 		pdpe[index[2]].present = true;
274 	}
275 
276 	/* Allocate page table if not present. */
277 	struct pageDirectoryEntry *pde;
278 	pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size);
279 	if (!pde[index[1]].present) {
280 		pde[index[1]].address = vm_phy_page_alloc(vm,
281 			KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
282 			>> vm->page_shift;
283 		pde[index[1]].writable = true;
284 		pde[index[1]].present = true;
285 	}
286 
287 	/* Fill in page table entry. */
288 	struct pageTableEntry *pte;
289 	pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size);
290 	pte[index[0]].address = paddr >> vm->page_shift;
291 	pte[index[0]].writable = true;
292 	pte[index[0]].present = 1;
293 }
294 
295 void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
296 {
297 	struct pageMapL4Entry *pml4e, *pml4e_start;
298 	struct pageDirectoryPointerEntry *pdpe, *pdpe_start;
299 	struct pageDirectoryEntry *pde, *pde_start;
300 	struct pageTableEntry *pte, *pte_start;
301 
302 	if (!vm->pgd_created)
303 		return;
304 
305 	fprintf(stream, "%*s                                          "
306 		"                no\n", indent, "");
307 	fprintf(stream, "%*s      index hvaddr         gpaddr         "
308 		"addr         w exec dirty\n",
309 		indent, "");
310 	pml4e_start = (struct pageMapL4Entry *) addr_gpa2hva(vm,
311 		vm->pgd);
312 	for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
313 		pml4e = &pml4e_start[n1];
314 		if (!pml4e->present)
315 			continue;
316 		fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10lx %u "
317 			" %u\n",
318 			indent, "",
319 			pml4e - pml4e_start, pml4e,
320 			addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->address,
321 			pml4e->writable, pml4e->execute_disable);
322 
323 		pdpe_start = addr_gpa2hva(vm, pml4e->address
324 			* vm->page_size);
325 		for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
326 			pdpe = &pdpe_start[n2];
327 			if (!pdpe->present)
328 				continue;
329 			fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10lx "
330 				"%u  %u\n",
331 				indent, "",
332 				pdpe - pdpe_start, pdpe,
333 				addr_hva2gpa(vm, pdpe),
334 				(uint64_t) pdpe->address, pdpe->writable,
335 				pdpe->execute_disable);
336 
337 			pde_start = addr_gpa2hva(vm,
338 				pdpe->address * vm->page_size);
339 			for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
340 				pde = &pde_start[n3];
341 				if (!pde->present)
342 					continue;
343 				fprintf(stream, "%*spde   0x%-3zx %p "
344 					"0x%-12lx 0x%-10lx %u  %u\n",
345 					indent, "", pde - pde_start, pde,
346 					addr_hva2gpa(vm, pde),
347 					(uint64_t) pde->address, pde->writable,
348 					pde->execute_disable);
349 
350 				pte_start = addr_gpa2hva(vm,
351 					pde->address * vm->page_size);
352 				for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
353 					pte = &pte_start[n4];
354 					if (!pte->present)
355 						continue;
356 					fprintf(stream, "%*spte   0x%-3zx %p "
357 						"0x%-12lx 0x%-10lx %u  %u "
358 						"    %u    0x%-10lx\n",
359 						indent, "",
360 						pte - pte_start, pte,
361 						addr_hva2gpa(vm, pte),
362 						(uint64_t) pte->address,
363 						pte->writable,
364 						pte->execute_disable,
365 						pte->dirty,
366 						((uint64_t) n1 << 27)
367 							| ((uint64_t) n2 << 18)
368 							| ((uint64_t) n3 << 9)
369 							| ((uint64_t) n4));
370 				}
371 			}
372 		}
373 	}
374 }
375 
376 /*
377  * Set Unusable Segment
378  *
379  * Input Args: None
380  *
381  * Output Args:
382  *   segp - Pointer to segment register
383  *
384  * Return: None
385  *
386  * Sets the segment register pointed to by @segp to an unusable state.
387  */
388 static void kvm_seg_set_unusable(struct kvm_segment *segp)
389 {
390 	memset(segp, 0, sizeof(*segp));
391 	segp->unusable = true;
392 }
393 
394 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
395 {
396 	void *gdt = addr_gva2hva(vm, vm->gdt);
397 	struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
398 
399 	desc->limit0 = segp->limit & 0xFFFF;
400 	desc->base0 = segp->base & 0xFFFF;
401 	desc->base1 = segp->base >> 16;
402 	desc->type = segp->type;
403 	desc->s = segp->s;
404 	desc->dpl = segp->dpl;
405 	desc->p = segp->present;
406 	desc->limit1 = segp->limit >> 16;
407 	desc->avl = segp->avl;
408 	desc->l = segp->l;
409 	desc->db = segp->db;
410 	desc->g = segp->g;
411 	desc->base2 = segp->base >> 24;
412 	if (!segp->s)
413 		desc->base3 = segp->base >> 32;
414 }
415 
416 
417 /*
418  * Set Long Mode Flat Kernel Code Segment
419  *
420  * Input Args:
421  *   vm - VM whose GDT is being filled, or NULL to only write segp
422  *   selector - selector value
423  *
424  * Output Args:
425  *   segp - Pointer to KVM segment
426  *
427  * Return: None
428  *
429  * Sets up the KVM segment pointed to by @segp, to be a code segment
430  * with the selector value given by @selector.
431  */
432 static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
433 	struct kvm_segment *segp)
434 {
435 	memset(segp, 0, sizeof(*segp));
436 	segp->selector = selector;
437 	segp->limit = 0xFFFFFFFFu;
438 	segp->s = 0x1; /* kTypeCodeData */
439 	segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
440 					  * | kFlagCodeReadable
441 					  */
442 	segp->g = true;
443 	segp->l = true;
444 	segp->present = 1;
445 	if (vm)
446 		kvm_seg_fill_gdt_64bit(vm, segp);
447 }
448 
449 /*
450  * Set Long Mode Flat Kernel Data Segment
451  *
452  * Input Args:
453  *   vm - VM whose GDT is being filled, or NULL to only write segp
454  *   selector - selector value
455  *
456  * Output Args:
457  *   segp - Pointer to KVM segment
458  *
459  * Return: None
460  *
461  * Sets up the KVM segment pointed to by @segp, to be a data segment
462  * with the selector value given by @selector.
463  */
464 static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
465 	struct kvm_segment *segp)
466 {
467 	memset(segp, 0, sizeof(*segp));
468 	segp->selector = selector;
469 	segp->limit = 0xFFFFFFFFu;
470 	segp->s = 0x1; /* kTypeCodeData */
471 	segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
472 					  * | kFlagDataWritable
473 					  */
474 	segp->g = true;
475 	segp->present = true;
476 	if (vm)
477 		kvm_seg_fill_gdt_64bit(vm, segp);
478 }
479 
480 vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
481 {
482 	uint16_t index[4];
483 	struct pageMapL4Entry *pml4e;
484 	struct pageDirectoryPointerEntry *pdpe;
485 	struct pageDirectoryEntry *pde;
486 	struct pageTableEntry *pte;
487 
488 	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
489 		"unknown or unsupported guest mode, mode: 0x%x", vm->mode);
490 
491 	index[0] = (gva >> 12) & 0x1ffu;
492 	index[1] = (gva >> 21) & 0x1ffu;
493 	index[2] = (gva >> 30) & 0x1ffu;
494 	index[3] = (gva >> 39) & 0x1ffu;
495 
496 	if (!vm->pgd_created)
497 		goto unmapped_gva;
498 	pml4e = addr_gpa2hva(vm, vm->pgd);
499 	if (!pml4e[index[3]].present)
500 		goto unmapped_gva;
501 
502 	pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size);
503 	if (!pdpe[index[2]].present)
504 		goto unmapped_gva;
505 
506 	pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size);
507 	if (!pde[index[1]].present)
508 		goto unmapped_gva;
509 
510 	pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size);
511 	if (!pte[index[0]].present)
512 		goto unmapped_gva;
513 
514 	return (pte[index[0]].address * vm->page_size) + (gva & 0xfffu);
515 
516 unmapped_gva:
517 	TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
518 	exit(EXIT_FAILURE);
519 }
520 
521 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt, int gdt_memslot,
522 			  int pgd_memslot)
523 {
524 	if (!vm->gdt)
525 		vm->gdt = vm_vaddr_alloc(vm, getpagesize(),
526 			KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot);
527 
528 	dt->base = vm->gdt;
529 	dt->limit = getpagesize();
530 }
531 
532 static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
533 				int selector, int gdt_memslot,
534 				int pgd_memslot)
535 {
536 	if (!vm->tss)
537 		vm->tss = vm_vaddr_alloc(vm, getpagesize(),
538 			KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot);
539 
540 	memset(segp, 0, sizeof(*segp));
541 	segp->base = vm->tss;
542 	segp->limit = 0x67;
543 	segp->selector = selector;
544 	segp->type = 0xb;
545 	segp->present = 1;
546 	kvm_seg_fill_gdt_64bit(vm, segp);
547 }
548 
549 static void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_memslot)
550 {
551 	struct kvm_sregs sregs;
552 
553 	/* Set mode specific system register values. */
554 	vcpu_sregs_get(vm, vcpuid, &sregs);
555 
556 	sregs.idt.limit = 0;
557 
558 	kvm_setup_gdt(vm, &sregs.gdt, gdt_memslot, pgd_memslot);
559 
560 	switch (vm->mode) {
561 	case VM_MODE_PXXV48_4K:
562 		sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
563 		sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
564 		sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
565 
566 		kvm_seg_set_unusable(&sregs.ldt);
567 		kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs);
568 		kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds);
569 		kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es);
570 		kvm_setup_tss_64bit(vm, &sregs.tr, 0x18, gdt_memslot, pgd_memslot);
571 		break;
572 
573 	default:
574 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
575 	}
576 
577 	sregs.cr3 = vm->pgd;
578 	vcpu_sregs_set(vm, vcpuid, &sregs);
579 }
580 
581 void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
582 {
583 	struct kvm_mp_state mp_state;
584 	struct kvm_regs regs;
585 	vm_vaddr_t stack_vaddr;
586 	stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
587 				     DEFAULT_GUEST_STACK_VADDR_MIN, 0, 0);
588 
589 	/* Create VCPU */
590 	vm_vcpu_add(vm, vcpuid);
591 	vcpu_setup(vm, vcpuid, 0, 0);
592 
593 	/* Setup guest general purpose registers */
594 	vcpu_regs_get(vm, vcpuid, &regs);
595 	regs.rflags = regs.rflags | 0x2;
596 	regs.rsp = stack_vaddr + (DEFAULT_STACK_PGS * getpagesize());
597 	regs.rip = (unsigned long) guest_code;
598 	vcpu_regs_set(vm, vcpuid, &regs);
599 
600 	/* Setup the MP state */
601 	mp_state.mp_state = 0;
602 	vcpu_set_mp_state(vm, vcpuid, &mp_state);
603 }
604 
605 /*
606  * Allocate an instance of struct kvm_cpuid2
607  *
608  * Input Args: None
609  *
610  * Output Args: None
611  *
612  * Return: A pointer to the allocated struct. The caller is responsible
613  * for freeing this struct.
614  *
615  * Since kvm_cpuid2 uses a 0-length array to allow a the size of the
616  * array to be decided at allocation time, allocation is slightly
617  * complicated. This function uses a reasonable default length for
618  * the array and performs the appropriate allocation.
619  */
620 static struct kvm_cpuid2 *allocate_kvm_cpuid2(void)
621 {
622 	struct kvm_cpuid2 *cpuid;
623 	int nent = 100;
624 	size_t size;
625 
626 	size = sizeof(*cpuid);
627 	size += nent * sizeof(struct kvm_cpuid_entry2);
628 	cpuid = malloc(size);
629 	if (!cpuid) {
630 		perror("malloc");
631 		abort();
632 	}
633 
634 	cpuid->nent = nent;
635 
636 	return cpuid;
637 }
638 
639 /*
640  * KVM Supported CPUID Get
641  *
642  * Input Args: None
643  *
644  * Output Args:
645  *
646  * Return: The supported KVM CPUID
647  *
648  * Get the guest CPUID supported by KVM.
649  */
650 struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
651 {
652 	static struct kvm_cpuid2 *cpuid;
653 	int ret;
654 	int kvm_fd;
655 
656 	if (cpuid)
657 		return cpuid;
658 
659 	cpuid = allocate_kvm_cpuid2();
660 	kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
661 	if (kvm_fd < 0)
662 		exit(KSFT_SKIP);
663 
664 	ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, cpuid);
665 	TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_CPUID failed %d %d\n",
666 		    ret, errno);
667 
668 	close(kvm_fd);
669 	return cpuid;
670 }
671 
672 /*
673  * KVM Get MSR
674  *
675  * Input Args:
676  *   msr_index - Index of MSR
677  *
678  * Output Args: None
679  *
680  * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
681  *
682  * Get value of MSR for VCPU.
683  */
684 uint64_t kvm_get_feature_msr(uint64_t msr_index)
685 {
686 	struct {
687 		struct kvm_msrs header;
688 		struct kvm_msr_entry entry;
689 	} buffer = {};
690 	int r, kvm_fd;
691 
692 	buffer.header.nmsrs = 1;
693 	buffer.entry.index = msr_index;
694 	kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
695 	if (kvm_fd < 0)
696 		exit(KSFT_SKIP);
697 
698 	r = ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
699 	TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
700 		"  rc: %i errno: %i", r, errno);
701 
702 	close(kvm_fd);
703 	return buffer.entry.data;
704 }
705 
706 /*
707  * VM VCPU CPUID Set
708  *
709  * Input Args:
710  *   vm - Virtual Machine
711  *   vcpuid - VCPU id
712  *
713  * Output Args: None
714  *
715  * Return: KVM CPUID (KVM_GET_CPUID2)
716  *
717  * Set the VCPU's CPUID.
718  */
719 struct kvm_cpuid2 *vcpu_get_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
720 {
721 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
722 	struct kvm_cpuid2 *cpuid;
723 	int max_ent;
724 	int rc = -1;
725 
726 	TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
727 
728 	cpuid = allocate_kvm_cpuid2();
729 	max_ent = cpuid->nent;
730 
731 	for (cpuid->nent = 1; cpuid->nent <= max_ent; cpuid->nent++) {
732 		rc = ioctl(vcpu->fd, KVM_GET_CPUID2, cpuid);
733 		if (!rc)
734 			break;
735 
736 		TEST_ASSERT(rc == -1 && errno == E2BIG,
737 			    "KVM_GET_CPUID2 should either succeed or give E2BIG: %d %d",
738 			    rc, errno);
739 	}
740 
741 	TEST_ASSERT(rc == 0, "KVM_GET_CPUID2 failed, rc: %i errno: %i",
742 		    rc, errno);
743 
744 	return cpuid;
745 }
746 
747 
748 
749 /*
750  * Locate a cpuid entry.
751  *
752  * Input Args:
753  *   function: The function of the cpuid entry to find.
754  *   index: The index of the cpuid entry.
755  *
756  * Output Args: None
757  *
758  * Return: A pointer to the cpuid entry. Never returns NULL.
759  */
760 struct kvm_cpuid_entry2 *
761 kvm_get_supported_cpuid_index(uint32_t function, uint32_t index)
762 {
763 	struct kvm_cpuid2 *cpuid;
764 	struct kvm_cpuid_entry2 *entry = NULL;
765 	int i;
766 
767 	cpuid = kvm_get_supported_cpuid();
768 	for (i = 0; i < cpuid->nent; i++) {
769 		if (cpuid->entries[i].function == function &&
770 		    cpuid->entries[i].index == index) {
771 			entry = &cpuid->entries[i];
772 			break;
773 		}
774 	}
775 
776 	TEST_ASSERT(entry, "Guest CPUID entry not found: (EAX=%x, ECX=%x).",
777 		    function, index);
778 	return entry;
779 }
780 
781 /*
782  * VM VCPU CPUID Set
783  *
784  * Input Args:
785  *   vm - Virtual Machine
786  *   vcpuid - VCPU id
787  *   cpuid - The CPUID values to set.
788  *
789  * Output Args: None
790  *
791  * Return: void
792  *
793  * Set the VCPU's CPUID.
794  */
795 void vcpu_set_cpuid(struct kvm_vm *vm,
796 		uint32_t vcpuid, struct kvm_cpuid2 *cpuid)
797 {
798 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
799 	int rc;
800 
801 	TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
802 
803 	rc = ioctl(vcpu->fd, KVM_SET_CPUID2, cpuid);
804 	TEST_ASSERT(rc == 0, "KVM_SET_CPUID2 failed, rc: %i errno: %i",
805 		    rc, errno);
806 
807 }
808 
809 /*
810  * VCPU Get MSR
811  *
812  * Input Args:
813  *   vm - Virtual Machine
814  *   vcpuid - VCPU ID
815  *   msr_index - Index of MSR
816  *
817  * Output Args: None
818  *
819  * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
820  *
821  * Get value of MSR for VCPU.
822  */
823 uint64_t vcpu_get_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index)
824 {
825 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
826 	struct {
827 		struct kvm_msrs header;
828 		struct kvm_msr_entry entry;
829 	} buffer = {};
830 	int r;
831 
832 	TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
833 	buffer.header.nmsrs = 1;
834 	buffer.entry.index = msr_index;
835 	r = ioctl(vcpu->fd, KVM_GET_MSRS, &buffer.header);
836 	TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
837 		"  rc: %i errno: %i", r, errno);
838 
839 	return buffer.entry.data;
840 }
841 
842 /*
843  * _VCPU Set MSR
844  *
845  * Input Args:
846  *   vm - Virtual Machine
847  *   vcpuid - VCPU ID
848  *   msr_index - Index of MSR
849  *   msr_value - New value of MSR
850  *
851  * Output Args: None
852  *
853  * Return: The result of KVM_SET_MSRS.
854  *
855  * Sets the value of an MSR for the given VCPU.
856  */
857 int _vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index,
858 		  uint64_t msr_value)
859 {
860 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
861 	struct {
862 		struct kvm_msrs header;
863 		struct kvm_msr_entry entry;
864 	} buffer = {};
865 	int r;
866 
867 	TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
868 	memset(&buffer, 0, sizeof(buffer));
869 	buffer.header.nmsrs = 1;
870 	buffer.entry.index = msr_index;
871 	buffer.entry.data = msr_value;
872 	r = ioctl(vcpu->fd, KVM_SET_MSRS, &buffer.header);
873 	return r;
874 }
875 
876 /*
877  * VCPU Set MSR
878  *
879  * Input Args:
880  *   vm - Virtual Machine
881  *   vcpuid - VCPU ID
882  *   msr_index - Index of MSR
883  *   msr_value - New value of MSR
884  *
885  * Output Args: None
886  *
887  * Return: On success, nothing. On failure a TEST_ASSERT is produced.
888  *
889  * Set value of MSR for VCPU.
890  */
891 void vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index,
892 	uint64_t msr_value)
893 {
894 	int r;
895 
896 	r = _vcpu_set_msr(vm, vcpuid, msr_index, msr_value);
897 	TEST_ASSERT(r == 1, "KVM_SET_MSRS IOCTL failed,\n"
898 		"  rc: %i errno: %i", r, errno);
899 }
900 
901 void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)
902 {
903 	va_list ap;
904 	struct kvm_regs regs;
905 
906 	TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
907 		    "  num: %u\n",
908 		    num);
909 
910 	va_start(ap, num);
911 	vcpu_regs_get(vm, vcpuid, &regs);
912 
913 	if (num >= 1)
914 		regs.rdi = va_arg(ap, uint64_t);
915 
916 	if (num >= 2)
917 		regs.rsi = va_arg(ap, uint64_t);
918 
919 	if (num >= 3)
920 		regs.rdx = va_arg(ap, uint64_t);
921 
922 	if (num >= 4)
923 		regs.rcx = va_arg(ap, uint64_t);
924 
925 	if (num >= 5)
926 		regs.r8 = va_arg(ap, uint64_t);
927 
928 	if (num >= 6)
929 		regs.r9 = va_arg(ap, uint64_t);
930 
931 	vcpu_regs_set(vm, vcpuid, &regs);
932 	va_end(ap);
933 }
934 
935 void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
936 {
937 	struct kvm_regs regs;
938 	struct kvm_sregs sregs;
939 
940 	fprintf(stream, "%*scpuid: %u\n", indent, "", vcpuid);
941 
942 	fprintf(stream, "%*sregs:\n", indent + 2, "");
943 	vcpu_regs_get(vm, vcpuid, &regs);
944 	regs_dump(stream, &regs, indent + 4);
945 
946 	fprintf(stream, "%*ssregs:\n", indent + 2, "");
947 	vcpu_sregs_get(vm, vcpuid, &sregs);
948 	sregs_dump(stream, &sregs, indent + 4);
949 }
950 
951 struct kvm_x86_state {
952 	struct kvm_vcpu_events events;
953 	struct kvm_mp_state mp_state;
954 	struct kvm_regs regs;
955 	struct kvm_xsave xsave;
956 	struct kvm_xcrs xcrs;
957 	struct kvm_sregs sregs;
958 	struct kvm_debugregs debugregs;
959 	union {
960 		struct kvm_nested_state nested;
961 		char nested_[16384];
962 	};
963 	struct kvm_msrs msrs;
964 };
965 
966 static int kvm_get_num_msrs_fd(int kvm_fd)
967 {
968 	struct kvm_msr_list nmsrs;
969 	int r;
970 
971 	nmsrs.nmsrs = 0;
972 	r = ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
973 	TEST_ASSERT(r == -1 && errno == E2BIG, "Unexpected result from KVM_GET_MSR_INDEX_LIST probe, r: %i",
974 		r);
975 
976 	return nmsrs.nmsrs;
977 }
978 
979 static int kvm_get_num_msrs(struct kvm_vm *vm)
980 {
981 	return kvm_get_num_msrs_fd(vm->kvm_fd);
982 }
983 
984 struct kvm_msr_list *kvm_get_msr_index_list(void)
985 {
986 	struct kvm_msr_list *list;
987 	int nmsrs, r, kvm_fd;
988 
989 	kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
990 	if (kvm_fd < 0)
991 		exit(KSFT_SKIP);
992 
993 	nmsrs = kvm_get_num_msrs_fd(kvm_fd);
994 	list = malloc(sizeof(*list) + nmsrs * sizeof(list->indices[0]));
995 	list->nmsrs = nmsrs;
996 	r = ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
997 	close(kvm_fd);
998 
999 	TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MSR_INDEX_LIST, r: %i",
1000 		r);
1001 
1002 	return list;
1003 }
1004 
1005 struct kvm_x86_state *vcpu_save_state(struct kvm_vm *vm, uint32_t vcpuid)
1006 {
1007 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1008 	struct kvm_msr_list *list;
1009 	struct kvm_x86_state *state;
1010 	int nmsrs, r, i;
1011 	static int nested_size = -1;
1012 
1013 	if (nested_size == -1) {
1014 		nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
1015 		TEST_ASSERT(nested_size <= sizeof(state->nested_),
1016 			    "Nested state size too big, %i > %zi",
1017 			    nested_size, sizeof(state->nested_));
1018 	}
1019 
1020 	/*
1021 	 * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
1022 	 * guest state is consistent only after userspace re-enters the
1023 	 * kernel with KVM_RUN.  Complete IO prior to migrating state
1024 	 * to a new VM.
1025 	 */
1026 	vcpu_run_complete_io(vm, vcpuid);
1027 
1028 	nmsrs = kvm_get_num_msrs(vm);
1029 	list = malloc(sizeof(*list) + nmsrs * sizeof(list->indices[0]));
1030 	list->nmsrs = nmsrs;
1031 	r = ioctl(vm->kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
1032         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MSR_INDEX_LIST, r: %i",
1033                 r);
1034 
1035 	state = malloc(sizeof(*state) + nmsrs * sizeof(state->msrs.entries[0]));
1036 	r = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, &state->events);
1037         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_VCPU_EVENTS, r: %i",
1038                 r);
1039 
1040 	r = ioctl(vcpu->fd, KVM_GET_MP_STATE, &state->mp_state);
1041         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MP_STATE, r: %i",
1042                 r);
1043 
1044 	r = ioctl(vcpu->fd, KVM_GET_REGS, &state->regs);
1045         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_REGS, r: %i",
1046                 r);
1047 
1048 	r = ioctl(vcpu->fd, KVM_GET_XSAVE, &state->xsave);
1049         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XSAVE, r: %i",
1050                 r);
1051 
1052 	if (kvm_check_cap(KVM_CAP_XCRS)) {
1053 		r = ioctl(vcpu->fd, KVM_GET_XCRS, &state->xcrs);
1054 		TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XCRS, r: %i",
1055 			    r);
1056 	}
1057 
1058 	r = ioctl(vcpu->fd, KVM_GET_SREGS, &state->sregs);
1059         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_SREGS, r: %i",
1060                 r);
1061 
1062 	if (nested_size) {
1063 		state->nested.size = sizeof(state->nested_);
1064 		r = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, &state->nested);
1065 		TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_NESTED_STATE, r: %i",
1066 			r);
1067 		TEST_ASSERT(state->nested.size <= nested_size,
1068 			"Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
1069 			state->nested.size, nested_size);
1070 	} else
1071 		state->nested.size = 0;
1072 
1073 	state->msrs.nmsrs = nmsrs;
1074 	for (i = 0; i < nmsrs; i++)
1075 		state->msrs.entries[i].index = list->indices[i];
1076 	r = ioctl(vcpu->fd, KVM_GET_MSRS, &state->msrs);
1077         TEST_ASSERT(r == nmsrs, "Unexpected result from KVM_GET_MSRS, r: %i (failed MSR was 0x%x)",
1078                 r, r == nmsrs ? -1 : list->indices[r]);
1079 
1080 	r = ioctl(vcpu->fd, KVM_GET_DEBUGREGS, &state->debugregs);
1081         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_DEBUGREGS, r: %i",
1082                 r);
1083 
1084 	free(list);
1085 	return state;
1086 }
1087 
1088 void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *state)
1089 {
1090 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1091 	int r;
1092 
1093 	r = ioctl(vcpu->fd, KVM_SET_XSAVE, &state->xsave);
1094         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XSAVE, r: %i",
1095                 r);
1096 
1097 	if (kvm_check_cap(KVM_CAP_XCRS)) {
1098 		r = ioctl(vcpu->fd, KVM_SET_XCRS, &state->xcrs);
1099 		TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XCRS, r: %i",
1100 			    r);
1101 	}
1102 
1103 	r = ioctl(vcpu->fd, KVM_SET_SREGS, &state->sregs);
1104         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_SREGS, r: %i",
1105                 r);
1106 
1107 	r = ioctl(vcpu->fd, KVM_SET_MSRS, &state->msrs);
1108         TEST_ASSERT(r == state->msrs.nmsrs, "Unexpected result from KVM_SET_MSRS, r: %i (failed at %x)",
1109                 r, r == state->msrs.nmsrs ? -1 : state->msrs.entries[r].index);
1110 
1111 	r = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, &state->events);
1112         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_VCPU_EVENTS, r: %i",
1113                 r);
1114 
1115 	r = ioctl(vcpu->fd, KVM_SET_MP_STATE, &state->mp_state);
1116         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_MP_STATE, r: %i",
1117                 r);
1118 
1119 	r = ioctl(vcpu->fd, KVM_SET_DEBUGREGS, &state->debugregs);
1120         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_DEBUGREGS, r: %i",
1121                 r);
1122 
1123 	r = ioctl(vcpu->fd, KVM_SET_REGS, &state->regs);
1124         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_REGS, r: %i",
1125                 r);
1126 
1127 	if (state->nested.size) {
1128 		r = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, &state->nested);
1129 		TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_NESTED_STATE, r: %i",
1130 			r);
1131 	}
1132 }
1133 
1134 bool is_intel_cpu(void)
1135 {
1136 	int eax, ebx, ecx, edx;
1137 	const uint32_t *chunk;
1138 	const int leaf = 0;
1139 
1140 	__asm__ __volatile__(
1141 		"cpuid"
1142 		: /* output */ "=a"(eax), "=b"(ebx),
1143 		  "=c"(ecx), "=d"(edx)
1144 		: /* input */ "0"(leaf), "2"(0));
1145 
1146 	chunk = (const uint32_t *)("GenuineIntel");
1147 	return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]);
1148 }
1149 
1150 uint32_t kvm_get_cpuid_max_basic(void)
1151 {
1152 	return kvm_get_supported_cpuid_entry(0)->eax;
1153 }
1154 
1155 uint32_t kvm_get_cpuid_max_extended(void)
1156 {
1157 	return kvm_get_supported_cpuid_entry(0x80000000)->eax;
1158 }
1159 
1160 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits)
1161 {
1162 	struct kvm_cpuid_entry2 *entry;
1163 	bool pae;
1164 
1165 	/* SDM 4.1.4 */
1166 	if (kvm_get_cpuid_max_extended() < 0x80000008) {
1167 		pae = kvm_get_supported_cpuid_entry(1)->edx & (1 << 6);
1168 		*pa_bits = pae ? 36 : 32;
1169 		*va_bits = 32;
1170 	} else {
1171 		entry = kvm_get_supported_cpuid_entry(0x80000008);
1172 		*pa_bits = entry->eax & 0xff;
1173 		*va_bits = (entry->eax >> 8) & 0xff;
1174 	}
1175 }
1176 
1177 struct idt_entry {
1178 	uint16_t offset0;
1179 	uint16_t selector;
1180 	uint16_t ist : 3;
1181 	uint16_t : 5;
1182 	uint16_t type : 4;
1183 	uint16_t : 1;
1184 	uint16_t dpl : 2;
1185 	uint16_t p : 1;
1186 	uint16_t offset1;
1187 	uint32_t offset2; uint32_t reserved;
1188 };
1189 
1190 static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr,
1191 			  int dpl, unsigned short selector)
1192 {
1193 	struct idt_entry *base =
1194 		(struct idt_entry *)addr_gva2hva(vm, vm->idt);
1195 	struct idt_entry *e = &base[vector];
1196 
1197 	memset(e, 0, sizeof(*e));
1198 	e->offset0 = addr;
1199 	e->selector = selector;
1200 	e->ist = 0;
1201 	e->type = 14;
1202 	e->dpl = dpl;
1203 	e->p = 1;
1204 	e->offset1 = addr >> 16;
1205 	e->offset2 = addr >> 32;
1206 }
1207 
1208 void kvm_exit_unexpected_vector(uint32_t value)
1209 {
1210 	outl(UNEXPECTED_VECTOR_PORT, value);
1211 }
1212 
1213 void route_exception(struct ex_regs *regs)
1214 {
1215 	typedef void(*handler)(struct ex_regs *);
1216 	handler *handlers = (handler *)exception_handlers;
1217 
1218 	if (handlers && handlers[regs->vector]) {
1219 		handlers[regs->vector](regs);
1220 		return;
1221 	}
1222 
1223 	kvm_exit_unexpected_vector(regs->vector);
1224 }
1225 
1226 void vm_init_descriptor_tables(struct kvm_vm *vm)
1227 {
1228 	extern void *idt_handlers;
1229 	int i;
1230 
1231 	vm->idt = vm_vaddr_alloc(vm, getpagesize(), 0x2000, 0, 0);
1232 	vm->handlers = vm_vaddr_alloc(vm, 256 * sizeof(void *), 0x2000, 0, 0);
1233 	/* Handlers have the same address in both address spaces.*/
1234 	for (i = 0; i < NUM_INTERRUPTS; i++)
1235 		set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,
1236 			DEFAULT_CODE_SELECTOR);
1237 }
1238 
1239 void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid)
1240 {
1241 	struct kvm_sregs sregs;
1242 
1243 	vcpu_sregs_get(vm, vcpuid, &sregs);
1244 	sregs.idt.base = vm->idt;
1245 	sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1;
1246 	sregs.gdt.base = vm->gdt;
1247 	sregs.gdt.limit = getpagesize() - 1;
1248 	kvm_seg_set_kernel_data_64bit(NULL, DEFAULT_DATA_SELECTOR, &sregs.gs);
1249 	vcpu_sregs_set(vm, vcpuid, &sregs);
1250 	*(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
1251 }
1252 
1253 void vm_handle_exception(struct kvm_vm *vm, int vector,
1254 			 void (*handler)(struct ex_regs *))
1255 {
1256 	vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers);
1257 
1258 	handlers[vector] = (vm_vaddr_t)handler;
1259 }
1260 
1261 void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid)
1262 {
1263 	if (vcpu_state(vm, vcpuid)->exit_reason == KVM_EXIT_IO
1264 		&& vcpu_state(vm, vcpuid)->io.port == UNEXPECTED_VECTOR_PORT
1265 		&& vcpu_state(vm, vcpuid)->io.size == 4) {
1266 		/* Grab pointer to io data */
1267 		uint32_t *data = (void *)vcpu_state(vm, vcpuid)
1268 			+ vcpu_state(vm, vcpuid)->io.data_offset;
1269 
1270 		TEST_ASSERT(false,
1271 			    "Unexpected vectored event in guest (vector:0x%x)",
1272 			    *data);
1273 	}
1274 }
1275 
1276 bool set_cpuid(struct kvm_cpuid2 *cpuid,
1277 	       struct kvm_cpuid_entry2 *ent)
1278 {
1279 	int i;
1280 
1281 	for (i = 0; i < cpuid->nent; i++) {
1282 		struct kvm_cpuid_entry2 *cur = &cpuid->entries[i];
1283 
1284 		if (cur->function != ent->function || cur->index != ent->index)
1285 			continue;
1286 
1287 		memcpy(cur, ent, sizeof(struct kvm_cpuid_entry2));
1288 		return true;
1289 	}
1290 
1291 	return false;
1292 }
1293 
1294 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
1295 		       uint64_t a3)
1296 {
1297 	uint64_t r;
1298 
1299 	asm volatile("vmcall"
1300 		     : "=a"(r)
1301 		     : "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1302 	return r;
1303 }
1304 
1305 struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void)
1306 {
1307 	static struct kvm_cpuid2 *cpuid;
1308 	int ret;
1309 	int kvm_fd;
1310 
1311 	if (cpuid)
1312 		return cpuid;
1313 
1314 	cpuid = allocate_kvm_cpuid2();
1315 	kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
1316 	if (kvm_fd < 0)
1317 		exit(KSFT_SKIP);
1318 
1319 	ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1320 	TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_HV_CPUID failed %d %d\n",
1321 		    ret, errno);
1322 
1323 	close(kvm_fd);
1324 	return cpuid;
1325 }
1326 
1327 void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
1328 {
1329 	static struct kvm_cpuid2 *cpuid_full;
1330 	struct kvm_cpuid2 *cpuid_sys, *cpuid_hv;
1331 	int i, nent = 0;
1332 
1333 	if (!cpuid_full) {
1334 		cpuid_sys = kvm_get_supported_cpuid();
1335 		cpuid_hv = kvm_get_supported_hv_cpuid();
1336 
1337 		cpuid_full = malloc(sizeof(*cpuid_full) +
1338 				    (cpuid_sys->nent + cpuid_hv->nent) *
1339 				    sizeof(struct kvm_cpuid_entry2));
1340 		if (!cpuid_full) {
1341 			perror("malloc");
1342 			abort();
1343 		}
1344 
1345 		/* Need to skip KVM CPUID leaves 0x400000xx */
1346 		for (i = 0; i < cpuid_sys->nent; i++) {
1347 			if (cpuid_sys->entries[i].function >= 0x40000000 &&
1348 			    cpuid_sys->entries[i].function < 0x40000100)
1349 				continue;
1350 			cpuid_full->entries[nent] = cpuid_sys->entries[i];
1351 			nent++;
1352 		}
1353 
1354 		memcpy(&cpuid_full->entries[nent], cpuid_hv->entries,
1355 		       cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2));
1356 		cpuid_full->nent = nent + cpuid_hv->nent;
1357 	}
1358 
1359 	vcpu_set_cpuid(vm, vcpuid, cpuid_full);
1360 }
1361 
1362 struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
1363 {
1364 	static struct kvm_cpuid2 *cpuid;
1365 
1366 	cpuid = allocate_kvm_cpuid2();
1367 
1368 	vcpu_ioctl(vm, vcpuid, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1369 
1370 	return cpuid;
1371 }
1372