xref: /openbmc/linux/virt/kvm/kvm_main.c (revision 97da55fc)
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9  *
10  * Authors:
11  *   Avi Kivity   <avi@qumranet.com>
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *
14  * This work is licensed under the terms of the GNU GPL, version 2.  See
15  * the COPYING file in the top-level directory.
16  *
17  */
18 
19 #include "iodev.h"
20 
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/syscore_ops.h>
34 #include <linux/cpu.h>
35 #include <linux/sched.h>
36 #include <linux/cpumask.h>
37 #include <linux/smp.h>
38 #include <linux/anon_inodes.h>
39 #include <linux/profile.h>
40 #include <linux/kvm_para.h>
41 #include <linux/pagemap.h>
42 #include <linux/mman.h>
43 #include <linux/swap.h>
44 #include <linux/bitops.h>
45 #include <linux/spinlock.h>
46 #include <linux/compat.h>
47 #include <linux/srcu.h>
48 #include <linux/hugetlb.h>
49 #include <linux/slab.h>
50 #include <linux/sort.h>
51 #include <linux/bsearch.h>
52 
53 #include <asm/processor.h>
54 #include <asm/io.h>
55 #include <asm/uaccess.h>
56 #include <asm/pgtable.h>
57 
58 #include "coalesced_mmio.h"
59 #include "async_pf.h"
60 
61 #define CREATE_TRACE_POINTS
62 #include <trace/events/kvm.h>
63 
64 MODULE_AUTHOR("Qumranet");
65 MODULE_LICENSE("GPL");
66 
67 /*
68  * Ordering of locks:
69  *
70  * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
71  */
72 
73 DEFINE_RAW_SPINLOCK(kvm_lock);
74 LIST_HEAD(vm_list);
75 
76 static cpumask_var_t cpus_hardware_enabled;
77 static int kvm_usage_count = 0;
78 static atomic_t hardware_enable_failed;
79 
80 struct kmem_cache *kvm_vcpu_cache;
81 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
82 
83 static __read_mostly struct preempt_ops kvm_preempt_ops;
84 
85 struct dentry *kvm_debugfs_dir;
86 
87 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
88 			   unsigned long arg);
89 #ifdef CONFIG_COMPAT
90 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
91 				  unsigned long arg);
92 #endif
93 static int hardware_enable_all(void);
94 static void hardware_disable_all(void);
95 
96 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
97 
98 bool kvm_rebooting;
99 EXPORT_SYMBOL_GPL(kvm_rebooting);
100 
101 static bool largepages_enabled = true;
102 
103 bool kvm_is_mmio_pfn(pfn_t pfn)
104 {
105 	if (pfn_valid(pfn)) {
106 		int reserved;
107 		struct page *tail = pfn_to_page(pfn);
108 		struct page *head = compound_trans_head(tail);
109 		reserved = PageReserved(head);
110 		if (head != tail) {
111 			/*
112 			 * "head" is not a dangling pointer
113 			 * (compound_trans_head takes care of that)
114 			 * but the hugepage may have been splitted
115 			 * from under us (and we may not hold a
116 			 * reference count on the head page so it can
117 			 * be reused before we run PageReferenced), so
118 			 * we've to check PageTail before returning
119 			 * what we just read.
120 			 */
121 			smp_rmb();
122 			if (PageTail(tail))
123 				return reserved;
124 		}
125 		return PageReserved(tail);
126 	}
127 
128 	return true;
129 }
130 
131 /*
132  * Switches to specified vcpu, until a matching vcpu_put()
133  */
134 int vcpu_load(struct kvm_vcpu *vcpu)
135 {
136 	int cpu;
137 
138 	if (mutex_lock_killable(&vcpu->mutex))
139 		return -EINTR;
140 	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
141 		/* The thread running this VCPU changed. */
142 		struct pid *oldpid = vcpu->pid;
143 		struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
144 		rcu_assign_pointer(vcpu->pid, newpid);
145 		synchronize_rcu();
146 		put_pid(oldpid);
147 	}
148 	cpu = get_cpu();
149 	preempt_notifier_register(&vcpu->preempt_notifier);
150 	kvm_arch_vcpu_load(vcpu, cpu);
151 	put_cpu();
152 	return 0;
153 }
154 
155 void vcpu_put(struct kvm_vcpu *vcpu)
156 {
157 	preempt_disable();
158 	kvm_arch_vcpu_put(vcpu);
159 	preempt_notifier_unregister(&vcpu->preempt_notifier);
160 	preempt_enable();
161 	mutex_unlock(&vcpu->mutex);
162 }
163 
164 static void ack_flush(void *_completed)
165 {
166 }
167 
168 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
169 {
170 	int i, cpu, me;
171 	cpumask_var_t cpus;
172 	bool called = true;
173 	struct kvm_vcpu *vcpu;
174 
175 	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
176 
177 	me = get_cpu();
178 	kvm_for_each_vcpu(i, vcpu, kvm) {
179 		kvm_make_request(req, vcpu);
180 		cpu = vcpu->cpu;
181 
182 		/* Set ->requests bit before we read ->mode */
183 		smp_mb();
184 
185 		if (cpus != NULL && cpu != -1 && cpu != me &&
186 		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
187 			cpumask_set_cpu(cpu, cpus);
188 	}
189 	if (unlikely(cpus == NULL))
190 		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
191 	else if (!cpumask_empty(cpus))
192 		smp_call_function_many(cpus, ack_flush, NULL, 1);
193 	else
194 		called = false;
195 	put_cpu();
196 	free_cpumask_var(cpus);
197 	return called;
198 }
199 
200 void kvm_flush_remote_tlbs(struct kvm *kvm)
201 {
202 	long dirty_count = kvm->tlbs_dirty;
203 
204 	smp_mb();
205 	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
206 		++kvm->stat.remote_tlb_flush;
207 	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
208 }
209 
210 void kvm_reload_remote_mmus(struct kvm *kvm)
211 {
212 	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
213 }
214 
215 void kvm_make_mclock_inprogress_request(struct kvm *kvm)
216 {
217 	make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
218 }
219 
220 void kvm_make_update_eoibitmap_request(struct kvm *kvm)
221 {
222 	make_all_cpus_request(kvm, KVM_REQ_EOIBITMAP);
223 }
224 
225 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
226 {
227 	struct page *page;
228 	int r;
229 
230 	mutex_init(&vcpu->mutex);
231 	vcpu->cpu = -1;
232 	vcpu->kvm = kvm;
233 	vcpu->vcpu_id = id;
234 	vcpu->pid = NULL;
235 	init_waitqueue_head(&vcpu->wq);
236 	kvm_async_pf_vcpu_init(vcpu);
237 
238 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
239 	if (!page) {
240 		r = -ENOMEM;
241 		goto fail;
242 	}
243 	vcpu->run = page_address(page);
244 
245 	kvm_vcpu_set_in_spin_loop(vcpu, false);
246 	kvm_vcpu_set_dy_eligible(vcpu, false);
247 
248 	r = kvm_arch_vcpu_init(vcpu);
249 	if (r < 0)
250 		goto fail_free_run;
251 	return 0;
252 
253 fail_free_run:
254 	free_page((unsigned long)vcpu->run);
255 fail:
256 	return r;
257 }
258 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
259 
260 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
261 {
262 	put_pid(vcpu->pid);
263 	kvm_arch_vcpu_uninit(vcpu);
264 	free_page((unsigned long)vcpu->run);
265 }
266 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
267 
268 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
269 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
270 {
271 	return container_of(mn, struct kvm, mmu_notifier);
272 }
273 
274 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
275 					     struct mm_struct *mm,
276 					     unsigned long address)
277 {
278 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
279 	int need_tlb_flush, idx;
280 
281 	/*
282 	 * When ->invalidate_page runs, the linux pte has been zapped
283 	 * already but the page is still allocated until
284 	 * ->invalidate_page returns. So if we increase the sequence
285 	 * here the kvm page fault will notice if the spte can't be
286 	 * established because the page is going to be freed. If
287 	 * instead the kvm page fault establishes the spte before
288 	 * ->invalidate_page runs, kvm_unmap_hva will release it
289 	 * before returning.
290 	 *
291 	 * The sequence increase only need to be seen at spin_unlock
292 	 * time, and not at spin_lock time.
293 	 *
294 	 * Increasing the sequence after the spin_unlock would be
295 	 * unsafe because the kvm page fault could then establish the
296 	 * pte after kvm_unmap_hva returned, without noticing the page
297 	 * is going to be freed.
298 	 */
299 	idx = srcu_read_lock(&kvm->srcu);
300 	spin_lock(&kvm->mmu_lock);
301 
302 	kvm->mmu_notifier_seq++;
303 	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
304 	/* we've to flush the tlb before the pages can be freed */
305 	if (need_tlb_flush)
306 		kvm_flush_remote_tlbs(kvm);
307 
308 	spin_unlock(&kvm->mmu_lock);
309 	srcu_read_unlock(&kvm->srcu, idx);
310 }
311 
312 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
313 					struct mm_struct *mm,
314 					unsigned long address,
315 					pte_t pte)
316 {
317 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
318 	int idx;
319 
320 	idx = srcu_read_lock(&kvm->srcu);
321 	spin_lock(&kvm->mmu_lock);
322 	kvm->mmu_notifier_seq++;
323 	kvm_set_spte_hva(kvm, address, pte);
324 	spin_unlock(&kvm->mmu_lock);
325 	srcu_read_unlock(&kvm->srcu, idx);
326 }
327 
328 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
329 						    struct mm_struct *mm,
330 						    unsigned long start,
331 						    unsigned long end)
332 {
333 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
334 	int need_tlb_flush = 0, idx;
335 
336 	idx = srcu_read_lock(&kvm->srcu);
337 	spin_lock(&kvm->mmu_lock);
338 	/*
339 	 * The count increase must become visible at unlock time as no
340 	 * spte can be established without taking the mmu_lock and
341 	 * count is also read inside the mmu_lock critical section.
342 	 */
343 	kvm->mmu_notifier_count++;
344 	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
345 	need_tlb_flush |= kvm->tlbs_dirty;
346 	/* we've to flush the tlb before the pages can be freed */
347 	if (need_tlb_flush)
348 		kvm_flush_remote_tlbs(kvm);
349 
350 	spin_unlock(&kvm->mmu_lock);
351 	srcu_read_unlock(&kvm->srcu, idx);
352 }
353 
354 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
355 						  struct mm_struct *mm,
356 						  unsigned long start,
357 						  unsigned long end)
358 {
359 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
360 
361 	spin_lock(&kvm->mmu_lock);
362 	/*
363 	 * This sequence increase will notify the kvm page fault that
364 	 * the page that is going to be mapped in the spte could have
365 	 * been freed.
366 	 */
367 	kvm->mmu_notifier_seq++;
368 	smp_wmb();
369 	/*
370 	 * The above sequence increase must be visible before the
371 	 * below count decrease, which is ensured by the smp_wmb above
372 	 * in conjunction with the smp_rmb in mmu_notifier_retry().
373 	 */
374 	kvm->mmu_notifier_count--;
375 	spin_unlock(&kvm->mmu_lock);
376 
377 	BUG_ON(kvm->mmu_notifier_count < 0);
378 }
379 
380 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
381 					      struct mm_struct *mm,
382 					      unsigned long address)
383 {
384 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
385 	int young, idx;
386 
387 	idx = srcu_read_lock(&kvm->srcu);
388 	spin_lock(&kvm->mmu_lock);
389 
390 	young = kvm_age_hva(kvm, address);
391 	if (young)
392 		kvm_flush_remote_tlbs(kvm);
393 
394 	spin_unlock(&kvm->mmu_lock);
395 	srcu_read_unlock(&kvm->srcu, idx);
396 
397 	return young;
398 }
399 
400 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
401 				       struct mm_struct *mm,
402 				       unsigned long address)
403 {
404 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
405 	int young, idx;
406 
407 	idx = srcu_read_lock(&kvm->srcu);
408 	spin_lock(&kvm->mmu_lock);
409 	young = kvm_test_age_hva(kvm, address);
410 	spin_unlock(&kvm->mmu_lock);
411 	srcu_read_unlock(&kvm->srcu, idx);
412 
413 	return young;
414 }
415 
416 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
417 				     struct mm_struct *mm)
418 {
419 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
420 	int idx;
421 
422 	idx = srcu_read_lock(&kvm->srcu);
423 	kvm_arch_flush_shadow_all(kvm);
424 	srcu_read_unlock(&kvm->srcu, idx);
425 }
426 
427 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
428 	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
429 	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
430 	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
431 	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
432 	.test_young		= kvm_mmu_notifier_test_young,
433 	.change_pte		= kvm_mmu_notifier_change_pte,
434 	.release		= kvm_mmu_notifier_release,
435 };
436 
437 static int kvm_init_mmu_notifier(struct kvm *kvm)
438 {
439 	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
440 	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
441 }
442 
443 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
444 
445 static int kvm_init_mmu_notifier(struct kvm *kvm)
446 {
447 	return 0;
448 }
449 
450 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
451 
452 static void kvm_init_memslots_id(struct kvm *kvm)
453 {
454 	int i;
455 	struct kvm_memslots *slots = kvm->memslots;
456 
457 	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
458 		slots->id_to_index[i] = slots->memslots[i].id = i;
459 }
460 
461 static struct kvm *kvm_create_vm(unsigned long type)
462 {
463 	int r, i;
464 	struct kvm *kvm = kvm_arch_alloc_vm();
465 
466 	if (!kvm)
467 		return ERR_PTR(-ENOMEM);
468 
469 	r = kvm_arch_init_vm(kvm, type);
470 	if (r)
471 		goto out_err_nodisable;
472 
473 	r = hardware_enable_all();
474 	if (r)
475 		goto out_err_nodisable;
476 
477 #ifdef CONFIG_HAVE_KVM_IRQCHIP
478 	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
479 	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
480 #endif
481 
482 	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
483 
484 	r = -ENOMEM;
485 	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
486 	if (!kvm->memslots)
487 		goto out_err_nosrcu;
488 	kvm_init_memslots_id(kvm);
489 	if (init_srcu_struct(&kvm->srcu))
490 		goto out_err_nosrcu;
491 	for (i = 0; i < KVM_NR_BUSES; i++) {
492 		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
493 					GFP_KERNEL);
494 		if (!kvm->buses[i])
495 			goto out_err;
496 	}
497 
498 	spin_lock_init(&kvm->mmu_lock);
499 	kvm->mm = current->mm;
500 	atomic_inc(&kvm->mm->mm_count);
501 	kvm_eventfd_init(kvm);
502 	mutex_init(&kvm->lock);
503 	mutex_init(&kvm->irq_lock);
504 	mutex_init(&kvm->slots_lock);
505 	atomic_set(&kvm->users_count, 1);
506 
507 	r = kvm_init_mmu_notifier(kvm);
508 	if (r)
509 		goto out_err;
510 
511 	raw_spin_lock(&kvm_lock);
512 	list_add(&kvm->vm_list, &vm_list);
513 	raw_spin_unlock(&kvm_lock);
514 
515 	return kvm;
516 
517 out_err:
518 	cleanup_srcu_struct(&kvm->srcu);
519 out_err_nosrcu:
520 	hardware_disable_all();
521 out_err_nodisable:
522 	for (i = 0; i < KVM_NR_BUSES; i++)
523 		kfree(kvm->buses[i]);
524 	kfree(kvm->memslots);
525 	kvm_arch_free_vm(kvm);
526 	return ERR_PTR(r);
527 }
528 
529 /*
530  * Avoid using vmalloc for a small buffer.
531  * Should not be used when the size is statically known.
532  */
533 void *kvm_kvzalloc(unsigned long size)
534 {
535 	if (size > PAGE_SIZE)
536 		return vzalloc(size);
537 	else
538 		return kzalloc(size, GFP_KERNEL);
539 }
540 
541 void kvm_kvfree(const void *addr)
542 {
543 	if (is_vmalloc_addr(addr))
544 		vfree(addr);
545 	else
546 		kfree(addr);
547 }
548 
549 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
550 {
551 	if (!memslot->dirty_bitmap)
552 		return;
553 
554 	kvm_kvfree(memslot->dirty_bitmap);
555 	memslot->dirty_bitmap = NULL;
556 }
557 
558 /*
559  * Free any memory in @free but not in @dont.
560  */
561 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
562 				  struct kvm_memory_slot *dont)
563 {
564 	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
565 		kvm_destroy_dirty_bitmap(free);
566 
567 	kvm_arch_free_memslot(free, dont);
568 
569 	free->npages = 0;
570 }
571 
572 void kvm_free_physmem(struct kvm *kvm)
573 {
574 	struct kvm_memslots *slots = kvm->memslots;
575 	struct kvm_memory_slot *memslot;
576 
577 	kvm_for_each_memslot(memslot, slots)
578 		kvm_free_physmem_slot(memslot, NULL);
579 
580 	kfree(kvm->memslots);
581 }
582 
583 static void kvm_destroy_vm(struct kvm *kvm)
584 {
585 	int i;
586 	struct mm_struct *mm = kvm->mm;
587 
588 	kvm_arch_sync_events(kvm);
589 	raw_spin_lock(&kvm_lock);
590 	list_del(&kvm->vm_list);
591 	raw_spin_unlock(&kvm_lock);
592 	kvm_free_irq_routing(kvm);
593 	for (i = 0; i < KVM_NR_BUSES; i++)
594 		kvm_io_bus_destroy(kvm->buses[i]);
595 	kvm_coalesced_mmio_free(kvm);
596 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
597 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
598 #else
599 	kvm_arch_flush_shadow_all(kvm);
600 #endif
601 	kvm_arch_destroy_vm(kvm);
602 	kvm_free_physmem(kvm);
603 	cleanup_srcu_struct(&kvm->srcu);
604 	kvm_arch_free_vm(kvm);
605 	hardware_disable_all();
606 	mmdrop(mm);
607 }
608 
609 void kvm_get_kvm(struct kvm *kvm)
610 {
611 	atomic_inc(&kvm->users_count);
612 }
613 EXPORT_SYMBOL_GPL(kvm_get_kvm);
614 
615 void kvm_put_kvm(struct kvm *kvm)
616 {
617 	if (atomic_dec_and_test(&kvm->users_count))
618 		kvm_destroy_vm(kvm);
619 }
620 EXPORT_SYMBOL_GPL(kvm_put_kvm);
621 
622 
623 static int kvm_vm_release(struct inode *inode, struct file *filp)
624 {
625 	struct kvm *kvm = filp->private_data;
626 
627 	kvm_irqfd_release(kvm);
628 
629 	kvm_put_kvm(kvm);
630 	return 0;
631 }
632 
633 /*
634  * Allocation size is twice as large as the actual dirty bitmap size.
635  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
636  */
637 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
638 {
639 #ifndef CONFIG_S390
640 	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
641 
642 	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
643 	if (!memslot->dirty_bitmap)
644 		return -ENOMEM;
645 
646 #endif /* !CONFIG_S390 */
647 	return 0;
648 }
649 
650 static int cmp_memslot(const void *slot1, const void *slot2)
651 {
652 	struct kvm_memory_slot *s1, *s2;
653 
654 	s1 = (struct kvm_memory_slot *)slot1;
655 	s2 = (struct kvm_memory_slot *)slot2;
656 
657 	if (s1->npages < s2->npages)
658 		return 1;
659 	if (s1->npages > s2->npages)
660 		return -1;
661 
662 	return 0;
663 }
664 
665 /*
666  * Sort the memslots base on its size, so the larger slots
667  * will get better fit.
668  */
669 static void sort_memslots(struct kvm_memslots *slots)
670 {
671 	int i;
672 
673 	sort(slots->memslots, KVM_MEM_SLOTS_NUM,
674 	      sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
675 
676 	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
677 		slots->id_to_index[slots->memslots[i].id] = i;
678 }
679 
680 void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
681 		     u64 last_generation)
682 {
683 	if (new) {
684 		int id = new->id;
685 		struct kvm_memory_slot *old = id_to_memslot(slots, id);
686 		unsigned long npages = old->npages;
687 
688 		*old = *new;
689 		if (new->npages != npages)
690 			sort_memslots(slots);
691 	}
692 
693 	slots->generation = last_generation + 1;
694 }
695 
696 static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
697 {
698 	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
699 
700 #ifdef KVM_CAP_READONLY_MEM
701 	valid_flags |= KVM_MEM_READONLY;
702 #endif
703 
704 	if (mem->flags & ~valid_flags)
705 		return -EINVAL;
706 
707 	return 0;
708 }
709 
710 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
711 		struct kvm_memslots *slots, struct kvm_memory_slot *new)
712 {
713 	struct kvm_memslots *old_memslots = kvm->memslots;
714 
715 	update_memslots(slots, new, kvm->memslots->generation);
716 	rcu_assign_pointer(kvm->memslots, slots);
717 	synchronize_srcu_expedited(&kvm->srcu);
718 	return old_memslots;
719 }
720 
721 /*
722  * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations:
723  * - create a new memory slot
724  * - delete an existing memory slot
725  * - modify an existing memory slot
726  *   -- move it in the guest physical memory space
727  *   -- just change its flags
728  *
729  * Since flags can be changed by some of these operations, the following
730  * differentiation is the best we can do for __kvm_set_memory_region():
731  */
732 enum kvm_mr_change {
733 	KVM_MR_CREATE,
734 	KVM_MR_DELETE,
735 	KVM_MR_MOVE,
736 	KVM_MR_FLAGS_ONLY,
737 };
738 
739 /*
740  * Allocate some memory and give it an address in the guest physical address
741  * space.
742  *
743  * Discontiguous memory is allowed, mostly for framebuffers.
744  *
745  * Must be called holding mmap_sem for write.
746  */
747 int __kvm_set_memory_region(struct kvm *kvm,
748 			    struct kvm_userspace_memory_region *mem,
749 			    bool user_alloc)
750 {
751 	int r;
752 	gfn_t base_gfn;
753 	unsigned long npages;
754 	struct kvm_memory_slot *slot;
755 	struct kvm_memory_slot old, new;
756 	struct kvm_memslots *slots = NULL, *old_memslots;
757 	enum kvm_mr_change change;
758 
759 	r = check_memory_region_flags(mem);
760 	if (r)
761 		goto out;
762 
763 	r = -EINVAL;
764 	/* General sanity checks */
765 	if (mem->memory_size & (PAGE_SIZE - 1))
766 		goto out;
767 	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
768 		goto out;
769 	/* We can read the guest memory with __xxx_user() later on. */
770 	if (user_alloc &&
771 	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
772 	     !access_ok(VERIFY_WRITE,
773 			(void __user *)(unsigned long)mem->userspace_addr,
774 			mem->memory_size)))
775 		goto out;
776 	if (mem->slot >= KVM_MEM_SLOTS_NUM)
777 		goto out;
778 	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
779 		goto out;
780 
781 	slot = id_to_memslot(kvm->memslots, mem->slot);
782 	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
783 	npages = mem->memory_size >> PAGE_SHIFT;
784 
785 	r = -EINVAL;
786 	if (npages > KVM_MEM_MAX_NR_PAGES)
787 		goto out;
788 
789 	if (!npages)
790 		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
791 
792 	new = old = *slot;
793 
794 	new.id = mem->slot;
795 	new.base_gfn = base_gfn;
796 	new.npages = npages;
797 	new.flags = mem->flags;
798 
799 	r = -EINVAL;
800 	if (npages) {
801 		if (!old.npages)
802 			change = KVM_MR_CREATE;
803 		else { /* Modify an existing slot. */
804 			if ((mem->userspace_addr != old.userspace_addr) ||
805 			    (npages != old.npages) ||
806 			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
807 				goto out;
808 
809 			if (base_gfn != old.base_gfn)
810 				change = KVM_MR_MOVE;
811 			else if (new.flags != old.flags)
812 				change = KVM_MR_FLAGS_ONLY;
813 			else { /* Nothing to change. */
814 				r = 0;
815 				goto out;
816 			}
817 		}
818 	} else if (old.npages) {
819 		change = KVM_MR_DELETE;
820 	} else /* Modify a non-existent slot: disallowed. */
821 		goto out;
822 
823 	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
824 		/* Check for overlaps */
825 		r = -EEXIST;
826 		kvm_for_each_memslot(slot, kvm->memslots) {
827 			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
828 			    (slot->id == mem->slot))
829 				continue;
830 			if (!((base_gfn + npages <= slot->base_gfn) ||
831 			      (base_gfn >= slot->base_gfn + slot->npages)))
832 				goto out;
833 		}
834 	}
835 
836 	/* Free page dirty bitmap if unneeded */
837 	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
838 		new.dirty_bitmap = NULL;
839 
840 	r = -ENOMEM;
841 	if (change == KVM_MR_CREATE) {
842 		new.userspace_addr = mem->userspace_addr;
843 
844 		if (kvm_arch_create_memslot(&new, npages))
845 			goto out_free;
846 	}
847 
848 	/* Allocate page dirty bitmap if needed */
849 	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
850 		if (kvm_create_dirty_bitmap(&new) < 0)
851 			goto out_free;
852 	}
853 
854 	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
855 		r = -ENOMEM;
856 		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
857 				GFP_KERNEL);
858 		if (!slots)
859 			goto out_free;
860 		slot = id_to_memslot(slots, mem->slot);
861 		slot->flags |= KVM_MEMSLOT_INVALID;
862 
863 		old_memslots = install_new_memslots(kvm, slots, NULL);
864 
865 		/* slot was deleted or moved, clear iommu mapping */
866 		kvm_iommu_unmap_pages(kvm, &old);
867 		/* From this point no new shadow pages pointing to a deleted,
868 		 * or moved, memslot will be created.
869 		 *
870 		 * validation of sp->gfn happens in:
871 		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
872 		 * 	- kvm_is_visible_gfn (mmu_check_roots)
873 		 */
874 		kvm_arch_flush_shadow_memslot(kvm, slot);
875 		slots = old_memslots;
876 	}
877 
878 	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
879 	if (r)
880 		goto out_slots;
881 
882 	r = -ENOMEM;
883 	/*
884 	 * We can re-use the old_memslots from above, the only difference
885 	 * from the currently installed memslots is the invalid flag.  This
886 	 * will get overwritten by update_memslots anyway.
887 	 */
888 	if (!slots) {
889 		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
890 				GFP_KERNEL);
891 		if (!slots)
892 			goto out_free;
893 	}
894 
895 	/*
896 	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
897 	 * un-mapped and re-mapped if their base changes.  Since base change
898 	 * unmapping is handled above with slot deletion, mapping alone is
899 	 * needed here.  Anything else the iommu might care about for existing
900 	 * slots (size changes, userspace addr changes and read-only flag
901 	 * changes) is disallowed above, so any other attribute changes getting
902 	 * here can be skipped.
903 	 */
904 	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
905 		r = kvm_iommu_map_pages(kvm, &new);
906 		if (r)
907 			goto out_slots;
908 	}
909 
910 	/* actual memory is freed via old in kvm_free_physmem_slot below */
911 	if (change == KVM_MR_DELETE) {
912 		new.dirty_bitmap = NULL;
913 		memset(&new.arch, 0, sizeof(new.arch));
914 	}
915 
916 	old_memslots = install_new_memslots(kvm, slots, &new);
917 
918 	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
919 
920 	kvm_free_physmem_slot(&old, &new);
921 	kfree(old_memslots);
922 
923 	return 0;
924 
925 out_slots:
926 	kfree(slots);
927 out_free:
928 	kvm_free_physmem_slot(&new, &old);
929 out:
930 	return r;
931 }
932 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
933 
934 int kvm_set_memory_region(struct kvm *kvm,
935 			  struct kvm_userspace_memory_region *mem,
936 			  bool user_alloc)
937 {
938 	int r;
939 
940 	mutex_lock(&kvm->slots_lock);
941 	r = __kvm_set_memory_region(kvm, mem, user_alloc);
942 	mutex_unlock(&kvm->slots_lock);
943 	return r;
944 }
945 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
946 
947 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
948 				   struct
949 				   kvm_userspace_memory_region *mem,
950 				   bool user_alloc)
951 {
952 	if (mem->slot >= KVM_USER_MEM_SLOTS)
953 		return -EINVAL;
954 	return kvm_set_memory_region(kvm, mem, user_alloc);
955 }
956 
957 int kvm_get_dirty_log(struct kvm *kvm,
958 			struct kvm_dirty_log *log, int *is_dirty)
959 {
960 	struct kvm_memory_slot *memslot;
961 	int r, i;
962 	unsigned long n;
963 	unsigned long any = 0;
964 
965 	r = -EINVAL;
966 	if (log->slot >= KVM_USER_MEM_SLOTS)
967 		goto out;
968 
969 	memslot = id_to_memslot(kvm->memslots, log->slot);
970 	r = -ENOENT;
971 	if (!memslot->dirty_bitmap)
972 		goto out;
973 
974 	n = kvm_dirty_bitmap_bytes(memslot);
975 
976 	for (i = 0; !any && i < n/sizeof(long); ++i)
977 		any = memslot->dirty_bitmap[i];
978 
979 	r = -EFAULT;
980 	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
981 		goto out;
982 
983 	if (any)
984 		*is_dirty = 1;
985 
986 	r = 0;
987 out:
988 	return r;
989 }
990 
991 bool kvm_largepages_enabled(void)
992 {
993 	return largepages_enabled;
994 }
995 
996 void kvm_disable_largepages(void)
997 {
998 	largepages_enabled = false;
999 }
1000 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1001 
1002 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1003 {
1004 	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1005 }
1006 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1007 
1008 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1009 {
1010 	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1011 
1012 	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1013 	      memslot->flags & KVM_MEMSLOT_INVALID)
1014 		return 0;
1015 
1016 	return 1;
1017 }
1018 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1019 
1020 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1021 {
1022 	struct vm_area_struct *vma;
1023 	unsigned long addr, size;
1024 
1025 	size = PAGE_SIZE;
1026 
1027 	addr = gfn_to_hva(kvm, gfn);
1028 	if (kvm_is_error_hva(addr))
1029 		return PAGE_SIZE;
1030 
1031 	down_read(&current->mm->mmap_sem);
1032 	vma = find_vma(current->mm, addr);
1033 	if (!vma)
1034 		goto out;
1035 
1036 	size = vma_kernel_pagesize(vma);
1037 
1038 out:
1039 	up_read(&current->mm->mmap_sem);
1040 
1041 	return size;
1042 }
1043 
1044 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1045 {
1046 	return slot->flags & KVM_MEM_READONLY;
1047 }
1048 
1049 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1050 				       gfn_t *nr_pages, bool write)
1051 {
1052 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1053 		return KVM_HVA_ERR_BAD;
1054 
1055 	if (memslot_is_readonly(slot) && write)
1056 		return KVM_HVA_ERR_RO_BAD;
1057 
1058 	if (nr_pages)
1059 		*nr_pages = slot->npages - (gfn - slot->base_gfn);
1060 
1061 	return __gfn_to_hva_memslot(slot, gfn);
1062 }
1063 
1064 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1065 				     gfn_t *nr_pages)
1066 {
1067 	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1068 }
1069 
1070 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1071 				 gfn_t gfn)
1072 {
1073 	return gfn_to_hva_many(slot, gfn, NULL);
1074 }
1075 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1076 
1077 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1078 {
1079 	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1080 }
1081 EXPORT_SYMBOL_GPL(gfn_to_hva);
1082 
1083 /*
1084  * The hva returned by this function is only allowed to be read.
1085  * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
1086  */
1087 static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
1088 {
1089 	return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
1090 }
1091 
1092 static int kvm_read_hva(void *data, void __user *hva, int len)
1093 {
1094 	return __copy_from_user(data, hva, len);
1095 }
1096 
1097 static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
1098 {
1099 	return __copy_from_user_inatomic(data, hva, len);
1100 }
1101 
1102 int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1103 	unsigned long start, int write, struct page **page)
1104 {
1105 	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1106 
1107 	if (write)
1108 		flags |= FOLL_WRITE;
1109 
1110 	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1111 }
1112 
1113 static inline int check_user_page_hwpoison(unsigned long addr)
1114 {
1115 	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1116 
1117 	rc = __get_user_pages(current, current->mm, addr, 1,
1118 			      flags, NULL, NULL, NULL);
1119 	return rc == -EHWPOISON;
1120 }
1121 
1122 /*
1123  * The atomic path to get the writable pfn which will be stored in @pfn,
1124  * true indicates success, otherwise false is returned.
1125  */
1126 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1127 			    bool write_fault, bool *writable, pfn_t *pfn)
1128 {
1129 	struct page *page[1];
1130 	int npages;
1131 
1132 	if (!(async || atomic))
1133 		return false;
1134 
1135 	/*
1136 	 * Fast pin a writable pfn only if it is a write fault request
1137 	 * or the caller allows to map a writable pfn for a read fault
1138 	 * request.
1139 	 */
1140 	if (!(write_fault || writable))
1141 		return false;
1142 
1143 	npages = __get_user_pages_fast(addr, 1, 1, page);
1144 	if (npages == 1) {
1145 		*pfn = page_to_pfn(page[0]);
1146 
1147 		if (writable)
1148 			*writable = true;
1149 		return true;
1150 	}
1151 
1152 	return false;
1153 }
1154 
1155 /*
1156  * The slow path to get the pfn of the specified host virtual address,
1157  * 1 indicates success, -errno is returned if error is detected.
1158  */
1159 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1160 			   bool *writable, pfn_t *pfn)
1161 {
1162 	struct page *page[1];
1163 	int npages = 0;
1164 
1165 	might_sleep();
1166 
1167 	if (writable)
1168 		*writable = write_fault;
1169 
1170 	if (async) {
1171 		down_read(&current->mm->mmap_sem);
1172 		npages = get_user_page_nowait(current, current->mm,
1173 					      addr, write_fault, page);
1174 		up_read(&current->mm->mmap_sem);
1175 	} else
1176 		npages = get_user_pages_fast(addr, 1, write_fault,
1177 					     page);
1178 	if (npages != 1)
1179 		return npages;
1180 
1181 	/* map read fault as writable if possible */
1182 	if (unlikely(!write_fault) && writable) {
1183 		struct page *wpage[1];
1184 
1185 		npages = __get_user_pages_fast(addr, 1, 1, wpage);
1186 		if (npages == 1) {
1187 			*writable = true;
1188 			put_page(page[0]);
1189 			page[0] = wpage[0];
1190 		}
1191 
1192 		npages = 1;
1193 	}
1194 	*pfn = page_to_pfn(page[0]);
1195 	return npages;
1196 }
1197 
1198 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1199 {
1200 	if (unlikely(!(vma->vm_flags & VM_READ)))
1201 		return false;
1202 
1203 	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1204 		return false;
1205 
1206 	return true;
1207 }
1208 
1209 /*
1210  * Pin guest page in memory and return its pfn.
1211  * @addr: host virtual address which maps memory to the guest
1212  * @atomic: whether this function can sleep
1213  * @async: whether this function need to wait IO complete if the
1214  *         host page is not in the memory
1215  * @write_fault: whether we should get a writable host page
1216  * @writable: whether it allows to map a writable host page for !@write_fault
1217  *
1218  * The function will map a writable host page for these two cases:
1219  * 1): @write_fault = true
1220  * 2): @write_fault = false && @writable, @writable will tell the caller
1221  *     whether the mapping is writable.
1222  */
1223 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1224 			bool write_fault, bool *writable)
1225 {
1226 	struct vm_area_struct *vma;
1227 	pfn_t pfn = 0;
1228 	int npages;
1229 
1230 	/* we can do it either atomically or asynchronously, not both */
1231 	BUG_ON(atomic && async);
1232 
1233 	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1234 		return pfn;
1235 
1236 	if (atomic)
1237 		return KVM_PFN_ERR_FAULT;
1238 
1239 	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1240 	if (npages == 1)
1241 		return pfn;
1242 
1243 	down_read(&current->mm->mmap_sem);
1244 	if (npages == -EHWPOISON ||
1245 	      (!async && check_user_page_hwpoison(addr))) {
1246 		pfn = KVM_PFN_ERR_HWPOISON;
1247 		goto exit;
1248 	}
1249 
1250 	vma = find_vma_intersection(current->mm, addr, addr + 1);
1251 
1252 	if (vma == NULL)
1253 		pfn = KVM_PFN_ERR_FAULT;
1254 	else if ((vma->vm_flags & VM_PFNMAP)) {
1255 		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1256 			vma->vm_pgoff;
1257 		BUG_ON(!kvm_is_mmio_pfn(pfn));
1258 	} else {
1259 		if (async && vma_is_valid(vma, write_fault))
1260 			*async = true;
1261 		pfn = KVM_PFN_ERR_FAULT;
1262 	}
1263 exit:
1264 	up_read(&current->mm->mmap_sem);
1265 	return pfn;
1266 }
1267 
1268 static pfn_t
1269 __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
1270 		     bool *async, bool write_fault, bool *writable)
1271 {
1272 	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1273 
1274 	if (addr == KVM_HVA_ERR_RO_BAD)
1275 		return KVM_PFN_ERR_RO_FAULT;
1276 
1277 	if (kvm_is_error_hva(addr))
1278 		return KVM_PFN_NOSLOT;
1279 
1280 	/* Do not map writable pfn in the readonly memslot. */
1281 	if (writable && memslot_is_readonly(slot)) {
1282 		*writable = false;
1283 		writable = NULL;
1284 	}
1285 
1286 	return hva_to_pfn(addr, atomic, async, write_fault,
1287 			  writable);
1288 }
1289 
1290 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1291 			  bool write_fault, bool *writable)
1292 {
1293 	struct kvm_memory_slot *slot;
1294 
1295 	if (async)
1296 		*async = false;
1297 
1298 	slot = gfn_to_memslot(kvm, gfn);
1299 
1300 	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
1301 				    writable);
1302 }
1303 
1304 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1305 {
1306 	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1307 }
1308 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1309 
1310 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
1311 		       bool write_fault, bool *writable)
1312 {
1313 	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1314 }
1315 EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
1316 
1317 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1318 {
1319 	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1320 }
1321 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1322 
1323 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1324 		      bool *writable)
1325 {
1326 	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1327 }
1328 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1329 
1330 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1331 {
1332 	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1333 }
1334 
1335 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1336 {
1337 	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1338 }
1339 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1340 
1341 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1342 								  int nr_pages)
1343 {
1344 	unsigned long addr;
1345 	gfn_t entry;
1346 
1347 	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1348 	if (kvm_is_error_hva(addr))
1349 		return -1;
1350 
1351 	if (entry < nr_pages)
1352 		return 0;
1353 
1354 	return __get_user_pages_fast(addr, nr_pages, 1, pages);
1355 }
1356 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1357 
1358 static struct page *kvm_pfn_to_page(pfn_t pfn)
1359 {
1360 	if (is_error_noslot_pfn(pfn))
1361 		return KVM_ERR_PTR_BAD_PAGE;
1362 
1363 	if (kvm_is_mmio_pfn(pfn)) {
1364 		WARN_ON(1);
1365 		return KVM_ERR_PTR_BAD_PAGE;
1366 	}
1367 
1368 	return pfn_to_page(pfn);
1369 }
1370 
1371 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1372 {
1373 	pfn_t pfn;
1374 
1375 	pfn = gfn_to_pfn(kvm, gfn);
1376 
1377 	return kvm_pfn_to_page(pfn);
1378 }
1379 
1380 EXPORT_SYMBOL_GPL(gfn_to_page);
1381 
1382 void kvm_release_page_clean(struct page *page)
1383 {
1384 	WARN_ON(is_error_page(page));
1385 
1386 	kvm_release_pfn_clean(page_to_pfn(page));
1387 }
1388 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1389 
1390 void kvm_release_pfn_clean(pfn_t pfn)
1391 {
1392 	if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
1393 		put_page(pfn_to_page(pfn));
1394 }
1395 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1396 
1397 void kvm_release_page_dirty(struct page *page)
1398 {
1399 	WARN_ON(is_error_page(page));
1400 
1401 	kvm_release_pfn_dirty(page_to_pfn(page));
1402 }
1403 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1404 
1405 void kvm_release_pfn_dirty(pfn_t pfn)
1406 {
1407 	kvm_set_pfn_dirty(pfn);
1408 	kvm_release_pfn_clean(pfn);
1409 }
1410 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1411 
1412 void kvm_set_page_dirty(struct page *page)
1413 {
1414 	kvm_set_pfn_dirty(page_to_pfn(page));
1415 }
1416 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1417 
1418 void kvm_set_pfn_dirty(pfn_t pfn)
1419 {
1420 	if (!kvm_is_mmio_pfn(pfn)) {
1421 		struct page *page = pfn_to_page(pfn);
1422 		if (!PageReserved(page))
1423 			SetPageDirty(page);
1424 	}
1425 }
1426 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1427 
1428 void kvm_set_pfn_accessed(pfn_t pfn)
1429 {
1430 	if (!kvm_is_mmio_pfn(pfn))
1431 		mark_page_accessed(pfn_to_page(pfn));
1432 }
1433 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1434 
1435 void kvm_get_pfn(pfn_t pfn)
1436 {
1437 	if (!kvm_is_mmio_pfn(pfn))
1438 		get_page(pfn_to_page(pfn));
1439 }
1440 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1441 
1442 static int next_segment(unsigned long len, int offset)
1443 {
1444 	if (len > PAGE_SIZE - offset)
1445 		return PAGE_SIZE - offset;
1446 	else
1447 		return len;
1448 }
1449 
1450 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1451 			int len)
1452 {
1453 	int r;
1454 	unsigned long addr;
1455 
1456 	addr = gfn_to_hva_read(kvm, gfn);
1457 	if (kvm_is_error_hva(addr))
1458 		return -EFAULT;
1459 	r = kvm_read_hva(data, (void __user *)addr + offset, len);
1460 	if (r)
1461 		return -EFAULT;
1462 	return 0;
1463 }
1464 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1465 
1466 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1467 {
1468 	gfn_t gfn = gpa >> PAGE_SHIFT;
1469 	int seg;
1470 	int offset = offset_in_page(gpa);
1471 	int ret;
1472 
1473 	while ((seg = next_segment(len, offset)) != 0) {
1474 		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1475 		if (ret < 0)
1476 			return ret;
1477 		offset = 0;
1478 		len -= seg;
1479 		data += seg;
1480 		++gfn;
1481 	}
1482 	return 0;
1483 }
1484 EXPORT_SYMBOL_GPL(kvm_read_guest);
1485 
1486 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1487 			  unsigned long len)
1488 {
1489 	int r;
1490 	unsigned long addr;
1491 	gfn_t gfn = gpa >> PAGE_SHIFT;
1492 	int offset = offset_in_page(gpa);
1493 
1494 	addr = gfn_to_hva_read(kvm, gfn);
1495 	if (kvm_is_error_hva(addr))
1496 		return -EFAULT;
1497 	pagefault_disable();
1498 	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1499 	pagefault_enable();
1500 	if (r)
1501 		return -EFAULT;
1502 	return 0;
1503 }
1504 EXPORT_SYMBOL(kvm_read_guest_atomic);
1505 
1506 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1507 			 int offset, int len)
1508 {
1509 	int r;
1510 	unsigned long addr;
1511 
1512 	addr = gfn_to_hva(kvm, gfn);
1513 	if (kvm_is_error_hva(addr))
1514 		return -EFAULT;
1515 	r = __copy_to_user((void __user *)addr + offset, data, len);
1516 	if (r)
1517 		return -EFAULT;
1518 	mark_page_dirty(kvm, gfn);
1519 	return 0;
1520 }
1521 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1522 
1523 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1524 		    unsigned long len)
1525 {
1526 	gfn_t gfn = gpa >> PAGE_SHIFT;
1527 	int seg;
1528 	int offset = offset_in_page(gpa);
1529 	int ret;
1530 
1531 	while ((seg = next_segment(len, offset)) != 0) {
1532 		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1533 		if (ret < 0)
1534 			return ret;
1535 		offset = 0;
1536 		len -= seg;
1537 		data += seg;
1538 		++gfn;
1539 	}
1540 	return 0;
1541 }
1542 
1543 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1544 			      gpa_t gpa)
1545 {
1546 	struct kvm_memslots *slots = kvm_memslots(kvm);
1547 	int offset = offset_in_page(gpa);
1548 	gfn_t gfn = gpa >> PAGE_SHIFT;
1549 
1550 	ghc->gpa = gpa;
1551 	ghc->generation = slots->generation;
1552 	ghc->memslot = gfn_to_memslot(kvm, gfn);
1553 	ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
1554 	if (!kvm_is_error_hva(ghc->hva))
1555 		ghc->hva += offset;
1556 	else
1557 		return -EFAULT;
1558 
1559 	return 0;
1560 }
1561 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1562 
1563 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1564 			   void *data, unsigned long len)
1565 {
1566 	struct kvm_memslots *slots = kvm_memslots(kvm);
1567 	int r;
1568 
1569 	if (slots->generation != ghc->generation)
1570 		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1571 
1572 	if (kvm_is_error_hva(ghc->hva))
1573 		return -EFAULT;
1574 
1575 	r = __copy_to_user((void __user *)ghc->hva, data, len);
1576 	if (r)
1577 		return -EFAULT;
1578 	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1579 
1580 	return 0;
1581 }
1582 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1583 
1584 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1585 			   void *data, unsigned long len)
1586 {
1587 	struct kvm_memslots *slots = kvm_memslots(kvm);
1588 	int r;
1589 
1590 	if (slots->generation != ghc->generation)
1591 		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1592 
1593 	if (kvm_is_error_hva(ghc->hva))
1594 		return -EFAULT;
1595 
1596 	r = __copy_from_user(data, (void __user *)ghc->hva, len);
1597 	if (r)
1598 		return -EFAULT;
1599 
1600 	return 0;
1601 }
1602 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1603 
1604 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1605 {
1606 	return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
1607 				    offset, len);
1608 }
1609 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1610 
1611 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1612 {
1613 	gfn_t gfn = gpa >> PAGE_SHIFT;
1614 	int seg;
1615 	int offset = offset_in_page(gpa);
1616 	int ret;
1617 
1618         while ((seg = next_segment(len, offset)) != 0) {
1619 		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1620 		if (ret < 0)
1621 			return ret;
1622 		offset = 0;
1623 		len -= seg;
1624 		++gfn;
1625 	}
1626 	return 0;
1627 }
1628 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1629 
1630 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
1631 			     gfn_t gfn)
1632 {
1633 	if (memslot && memslot->dirty_bitmap) {
1634 		unsigned long rel_gfn = gfn - memslot->base_gfn;
1635 
1636 		set_bit_le(rel_gfn, memslot->dirty_bitmap);
1637 	}
1638 }
1639 
1640 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1641 {
1642 	struct kvm_memory_slot *memslot;
1643 
1644 	memslot = gfn_to_memslot(kvm, gfn);
1645 	mark_page_dirty_in_slot(kvm, memslot, gfn);
1646 }
1647 
1648 /*
1649  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1650  */
1651 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1652 {
1653 	DEFINE_WAIT(wait);
1654 
1655 	for (;;) {
1656 		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1657 
1658 		if (kvm_arch_vcpu_runnable(vcpu)) {
1659 			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1660 			break;
1661 		}
1662 		if (kvm_cpu_has_pending_timer(vcpu))
1663 			break;
1664 		if (signal_pending(current))
1665 			break;
1666 
1667 		schedule();
1668 	}
1669 
1670 	finish_wait(&vcpu->wq, &wait);
1671 }
1672 
1673 #ifndef CONFIG_S390
1674 /*
1675  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
1676  */
1677 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
1678 {
1679 	int me;
1680 	int cpu = vcpu->cpu;
1681 	wait_queue_head_t *wqp;
1682 
1683 	wqp = kvm_arch_vcpu_wq(vcpu);
1684 	if (waitqueue_active(wqp)) {
1685 		wake_up_interruptible(wqp);
1686 		++vcpu->stat.halt_wakeup;
1687 	}
1688 
1689 	me = get_cpu();
1690 	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
1691 		if (kvm_arch_vcpu_should_kick(vcpu))
1692 			smp_send_reschedule(cpu);
1693 	put_cpu();
1694 }
1695 #endif /* !CONFIG_S390 */
1696 
1697 void kvm_resched(struct kvm_vcpu *vcpu)
1698 {
1699 	if (!need_resched())
1700 		return;
1701 	cond_resched();
1702 }
1703 EXPORT_SYMBOL_GPL(kvm_resched);
1704 
1705 bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
1706 {
1707 	struct pid *pid;
1708 	struct task_struct *task = NULL;
1709 	bool ret = false;
1710 
1711 	rcu_read_lock();
1712 	pid = rcu_dereference(target->pid);
1713 	if (pid)
1714 		task = get_pid_task(target->pid, PIDTYPE_PID);
1715 	rcu_read_unlock();
1716 	if (!task)
1717 		return ret;
1718 	if (task->flags & PF_VCPU) {
1719 		put_task_struct(task);
1720 		return ret;
1721 	}
1722 	ret = yield_to(task, 1);
1723 	put_task_struct(task);
1724 
1725 	return ret;
1726 }
1727 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
1728 
1729 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1730 /*
1731  * Helper that checks whether a VCPU is eligible for directed yield.
1732  * Most eligible candidate to yield is decided by following heuristics:
1733  *
1734  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
1735  *  (preempted lock holder), indicated by @in_spin_loop.
1736  *  Set at the beiginning and cleared at the end of interception/PLE handler.
1737  *
1738  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
1739  *  chance last time (mostly it has become eligible now since we have probably
1740  *  yielded to lockholder in last iteration. This is done by toggling
1741  *  @dy_eligible each time a VCPU checked for eligibility.)
1742  *
1743  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
1744  *  to preempted lock-holder could result in wrong VCPU selection and CPU
1745  *  burning. Giving priority for a potential lock-holder increases lock
1746  *  progress.
1747  *
1748  *  Since algorithm is based on heuristics, accessing another VCPU data without
1749  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
1750  *  and continue with next VCPU and so on.
1751  */
1752 bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
1753 {
1754 	bool eligible;
1755 
1756 	eligible = !vcpu->spin_loop.in_spin_loop ||
1757 			(vcpu->spin_loop.in_spin_loop &&
1758 			 vcpu->spin_loop.dy_eligible);
1759 
1760 	if (vcpu->spin_loop.in_spin_loop)
1761 		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
1762 
1763 	return eligible;
1764 }
1765 #endif
1766 
1767 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1768 {
1769 	struct kvm *kvm = me->kvm;
1770 	struct kvm_vcpu *vcpu;
1771 	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1772 	int yielded = 0;
1773 	int try = 3;
1774 	int pass;
1775 	int i;
1776 
1777 	kvm_vcpu_set_in_spin_loop(me, true);
1778 	/*
1779 	 * We boost the priority of a VCPU that is runnable but not
1780 	 * currently running, because it got preempted by something
1781 	 * else and called schedule in __vcpu_run.  Hopefully that
1782 	 * VCPU is holding the lock that we need and will release it.
1783 	 * We approximate round-robin by starting at the last boosted VCPU.
1784 	 */
1785 	for (pass = 0; pass < 2 && !yielded && try; pass++) {
1786 		kvm_for_each_vcpu(i, vcpu, kvm) {
1787 			if (!pass && i <= last_boosted_vcpu) {
1788 				i = last_boosted_vcpu;
1789 				continue;
1790 			} else if (pass && i > last_boosted_vcpu)
1791 				break;
1792 			if (vcpu == me)
1793 				continue;
1794 			if (waitqueue_active(&vcpu->wq))
1795 				continue;
1796 			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
1797 				continue;
1798 
1799 			yielded = kvm_vcpu_yield_to(vcpu);
1800 			if (yielded > 0) {
1801 				kvm->last_boosted_vcpu = i;
1802 				break;
1803 			} else if (yielded < 0) {
1804 				try--;
1805 				if (!try)
1806 					break;
1807 			}
1808 		}
1809 	}
1810 	kvm_vcpu_set_in_spin_loop(me, false);
1811 
1812 	/* Ensure vcpu is not eligible during next spinloop */
1813 	kvm_vcpu_set_dy_eligible(me, false);
1814 }
1815 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1816 
1817 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1818 {
1819 	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1820 	struct page *page;
1821 
1822 	if (vmf->pgoff == 0)
1823 		page = virt_to_page(vcpu->run);
1824 #ifdef CONFIG_X86
1825 	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1826 		page = virt_to_page(vcpu->arch.pio_data);
1827 #endif
1828 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1829 	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1830 		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1831 #endif
1832 	else
1833 		return kvm_arch_vcpu_fault(vcpu, vmf);
1834 	get_page(page);
1835 	vmf->page = page;
1836 	return 0;
1837 }
1838 
1839 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1840 	.fault = kvm_vcpu_fault,
1841 };
1842 
1843 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1844 {
1845 	vma->vm_ops = &kvm_vcpu_vm_ops;
1846 	return 0;
1847 }
1848 
1849 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1850 {
1851 	struct kvm_vcpu *vcpu = filp->private_data;
1852 
1853 	kvm_put_kvm(vcpu->kvm);
1854 	return 0;
1855 }
1856 
1857 static struct file_operations kvm_vcpu_fops = {
1858 	.release        = kvm_vcpu_release,
1859 	.unlocked_ioctl = kvm_vcpu_ioctl,
1860 #ifdef CONFIG_COMPAT
1861 	.compat_ioctl   = kvm_vcpu_compat_ioctl,
1862 #endif
1863 	.mmap           = kvm_vcpu_mmap,
1864 	.llseek		= noop_llseek,
1865 };
1866 
1867 /*
1868  * Allocates an inode for the vcpu.
1869  */
1870 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1871 {
1872 	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
1873 }
1874 
1875 /*
1876  * Creates some virtual cpus.  Good luck creating more than one.
1877  */
1878 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1879 {
1880 	int r;
1881 	struct kvm_vcpu *vcpu, *v;
1882 
1883 	vcpu = kvm_arch_vcpu_create(kvm, id);
1884 	if (IS_ERR(vcpu))
1885 		return PTR_ERR(vcpu);
1886 
1887 	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1888 
1889 	r = kvm_arch_vcpu_setup(vcpu);
1890 	if (r)
1891 		goto vcpu_destroy;
1892 
1893 	mutex_lock(&kvm->lock);
1894 	if (!kvm_vcpu_compatible(vcpu)) {
1895 		r = -EINVAL;
1896 		goto unlock_vcpu_destroy;
1897 	}
1898 	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
1899 		r = -EINVAL;
1900 		goto unlock_vcpu_destroy;
1901 	}
1902 
1903 	kvm_for_each_vcpu(r, v, kvm)
1904 		if (v->vcpu_id == id) {
1905 			r = -EEXIST;
1906 			goto unlock_vcpu_destroy;
1907 		}
1908 
1909 	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1910 
1911 	/* Now it's all set up, let userspace reach it */
1912 	kvm_get_kvm(kvm);
1913 	r = create_vcpu_fd(vcpu);
1914 	if (r < 0) {
1915 		kvm_put_kvm(kvm);
1916 		goto unlock_vcpu_destroy;
1917 	}
1918 
1919 	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
1920 	smp_wmb();
1921 	atomic_inc(&kvm->online_vcpus);
1922 
1923 	mutex_unlock(&kvm->lock);
1924 	kvm_arch_vcpu_postcreate(vcpu);
1925 	return r;
1926 
1927 unlock_vcpu_destroy:
1928 	mutex_unlock(&kvm->lock);
1929 vcpu_destroy:
1930 	kvm_arch_vcpu_destroy(vcpu);
1931 	return r;
1932 }
1933 
1934 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1935 {
1936 	if (sigset) {
1937 		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1938 		vcpu->sigset_active = 1;
1939 		vcpu->sigset = *sigset;
1940 	} else
1941 		vcpu->sigset_active = 0;
1942 	return 0;
1943 }
1944 
1945 static long kvm_vcpu_ioctl(struct file *filp,
1946 			   unsigned int ioctl, unsigned long arg)
1947 {
1948 	struct kvm_vcpu *vcpu = filp->private_data;
1949 	void __user *argp = (void __user *)arg;
1950 	int r;
1951 	struct kvm_fpu *fpu = NULL;
1952 	struct kvm_sregs *kvm_sregs = NULL;
1953 
1954 	if (vcpu->kvm->mm != current->mm)
1955 		return -EIO;
1956 
1957 #if defined(CONFIG_S390) || defined(CONFIG_PPC)
1958 	/*
1959 	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
1960 	 * so vcpu_load() would break it.
1961 	 */
1962 	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
1963 		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1964 #endif
1965 
1966 
1967 	r = vcpu_load(vcpu);
1968 	if (r)
1969 		return r;
1970 	switch (ioctl) {
1971 	case KVM_RUN:
1972 		r = -EINVAL;
1973 		if (arg)
1974 			goto out;
1975 		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1976 		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
1977 		break;
1978 	case KVM_GET_REGS: {
1979 		struct kvm_regs *kvm_regs;
1980 
1981 		r = -ENOMEM;
1982 		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1983 		if (!kvm_regs)
1984 			goto out;
1985 		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
1986 		if (r)
1987 			goto out_free1;
1988 		r = -EFAULT;
1989 		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1990 			goto out_free1;
1991 		r = 0;
1992 out_free1:
1993 		kfree(kvm_regs);
1994 		break;
1995 	}
1996 	case KVM_SET_REGS: {
1997 		struct kvm_regs *kvm_regs;
1998 
1999 		r = -ENOMEM;
2000 		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2001 		if (IS_ERR(kvm_regs)) {
2002 			r = PTR_ERR(kvm_regs);
2003 			goto out;
2004 		}
2005 		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2006 		kfree(kvm_regs);
2007 		break;
2008 	}
2009 	case KVM_GET_SREGS: {
2010 		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2011 		r = -ENOMEM;
2012 		if (!kvm_sregs)
2013 			goto out;
2014 		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2015 		if (r)
2016 			goto out;
2017 		r = -EFAULT;
2018 		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2019 			goto out;
2020 		r = 0;
2021 		break;
2022 	}
2023 	case KVM_SET_SREGS: {
2024 		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2025 		if (IS_ERR(kvm_sregs)) {
2026 			r = PTR_ERR(kvm_sregs);
2027 			kvm_sregs = NULL;
2028 			goto out;
2029 		}
2030 		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2031 		break;
2032 	}
2033 	case KVM_GET_MP_STATE: {
2034 		struct kvm_mp_state mp_state;
2035 
2036 		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2037 		if (r)
2038 			goto out;
2039 		r = -EFAULT;
2040 		if (copy_to_user(argp, &mp_state, sizeof mp_state))
2041 			goto out;
2042 		r = 0;
2043 		break;
2044 	}
2045 	case KVM_SET_MP_STATE: {
2046 		struct kvm_mp_state mp_state;
2047 
2048 		r = -EFAULT;
2049 		if (copy_from_user(&mp_state, argp, sizeof mp_state))
2050 			goto out;
2051 		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2052 		break;
2053 	}
2054 	case KVM_TRANSLATE: {
2055 		struct kvm_translation tr;
2056 
2057 		r = -EFAULT;
2058 		if (copy_from_user(&tr, argp, sizeof tr))
2059 			goto out;
2060 		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2061 		if (r)
2062 			goto out;
2063 		r = -EFAULT;
2064 		if (copy_to_user(argp, &tr, sizeof tr))
2065 			goto out;
2066 		r = 0;
2067 		break;
2068 	}
2069 	case KVM_SET_GUEST_DEBUG: {
2070 		struct kvm_guest_debug dbg;
2071 
2072 		r = -EFAULT;
2073 		if (copy_from_user(&dbg, argp, sizeof dbg))
2074 			goto out;
2075 		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2076 		break;
2077 	}
2078 	case KVM_SET_SIGNAL_MASK: {
2079 		struct kvm_signal_mask __user *sigmask_arg = argp;
2080 		struct kvm_signal_mask kvm_sigmask;
2081 		sigset_t sigset, *p;
2082 
2083 		p = NULL;
2084 		if (argp) {
2085 			r = -EFAULT;
2086 			if (copy_from_user(&kvm_sigmask, argp,
2087 					   sizeof kvm_sigmask))
2088 				goto out;
2089 			r = -EINVAL;
2090 			if (kvm_sigmask.len != sizeof sigset)
2091 				goto out;
2092 			r = -EFAULT;
2093 			if (copy_from_user(&sigset, sigmask_arg->sigset,
2094 					   sizeof sigset))
2095 				goto out;
2096 			p = &sigset;
2097 		}
2098 		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2099 		break;
2100 	}
2101 	case KVM_GET_FPU: {
2102 		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2103 		r = -ENOMEM;
2104 		if (!fpu)
2105 			goto out;
2106 		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2107 		if (r)
2108 			goto out;
2109 		r = -EFAULT;
2110 		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2111 			goto out;
2112 		r = 0;
2113 		break;
2114 	}
2115 	case KVM_SET_FPU: {
2116 		fpu = memdup_user(argp, sizeof(*fpu));
2117 		if (IS_ERR(fpu)) {
2118 			r = PTR_ERR(fpu);
2119 			fpu = NULL;
2120 			goto out;
2121 		}
2122 		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2123 		break;
2124 	}
2125 	default:
2126 		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2127 	}
2128 out:
2129 	vcpu_put(vcpu);
2130 	kfree(fpu);
2131 	kfree(kvm_sregs);
2132 	return r;
2133 }
2134 
2135 #ifdef CONFIG_COMPAT
2136 static long kvm_vcpu_compat_ioctl(struct file *filp,
2137 				  unsigned int ioctl, unsigned long arg)
2138 {
2139 	struct kvm_vcpu *vcpu = filp->private_data;
2140 	void __user *argp = compat_ptr(arg);
2141 	int r;
2142 
2143 	if (vcpu->kvm->mm != current->mm)
2144 		return -EIO;
2145 
2146 	switch (ioctl) {
2147 	case KVM_SET_SIGNAL_MASK: {
2148 		struct kvm_signal_mask __user *sigmask_arg = argp;
2149 		struct kvm_signal_mask kvm_sigmask;
2150 		compat_sigset_t csigset;
2151 		sigset_t sigset;
2152 
2153 		if (argp) {
2154 			r = -EFAULT;
2155 			if (copy_from_user(&kvm_sigmask, argp,
2156 					   sizeof kvm_sigmask))
2157 				goto out;
2158 			r = -EINVAL;
2159 			if (kvm_sigmask.len != sizeof csigset)
2160 				goto out;
2161 			r = -EFAULT;
2162 			if (copy_from_user(&csigset, sigmask_arg->sigset,
2163 					   sizeof csigset))
2164 				goto out;
2165 			sigset_from_compat(&sigset, &csigset);
2166 			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2167 		} else
2168 			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2169 		break;
2170 	}
2171 	default:
2172 		r = kvm_vcpu_ioctl(filp, ioctl, arg);
2173 	}
2174 
2175 out:
2176 	return r;
2177 }
2178 #endif
2179 
2180 static long kvm_vm_ioctl(struct file *filp,
2181 			   unsigned int ioctl, unsigned long arg)
2182 {
2183 	struct kvm *kvm = filp->private_data;
2184 	void __user *argp = (void __user *)arg;
2185 	int r;
2186 
2187 	if (kvm->mm != current->mm)
2188 		return -EIO;
2189 	switch (ioctl) {
2190 	case KVM_CREATE_VCPU:
2191 		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2192 		break;
2193 	case KVM_SET_USER_MEMORY_REGION: {
2194 		struct kvm_userspace_memory_region kvm_userspace_mem;
2195 
2196 		r = -EFAULT;
2197 		if (copy_from_user(&kvm_userspace_mem, argp,
2198 						sizeof kvm_userspace_mem))
2199 			goto out;
2200 
2201 		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, true);
2202 		break;
2203 	}
2204 	case KVM_GET_DIRTY_LOG: {
2205 		struct kvm_dirty_log log;
2206 
2207 		r = -EFAULT;
2208 		if (copy_from_user(&log, argp, sizeof log))
2209 			goto out;
2210 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2211 		break;
2212 	}
2213 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2214 	case KVM_REGISTER_COALESCED_MMIO: {
2215 		struct kvm_coalesced_mmio_zone zone;
2216 		r = -EFAULT;
2217 		if (copy_from_user(&zone, argp, sizeof zone))
2218 			goto out;
2219 		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2220 		break;
2221 	}
2222 	case KVM_UNREGISTER_COALESCED_MMIO: {
2223 		struct kvm_coalesced_mmio_zone zone;
2224 		r = -EFAULT;
2225 		if (copy_from_user(&zone, argp, sizeof zone))
2226 			goto out;
2227 		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2228 		break;
2229 	}
2230 #endif
2231 	case KVM_IRQFD: {
2232 		struct kvm_irqfd data;
2233 
2234 		r = -EFAULT;
2235 		if (copy_from_user(&data, argp, sizeof data))
2236 			goto out;
2237 		r = kvm_irqfd(kvm, &data);
2238 		break;
2239 	}
2240 	case KVM_IOEVENTFD: {
2241 		struct kvm_ioeventfd data;
2242 
2243 		r = -EFAULT;
2244 		if (copy_from_user(&data, argp, sizeof data))
2245 			goto out;
2246 		r = kvm_ioeventfd(kvm, &data);
2247 		break;
2248 	}
2249 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
2250 	case KVM_SET_BOOT_CPU_ID:
2251 		r = 0;
2252 		mutex_lock(&kvm->lock);
2253 		if (atomic_read(&kvm->online_vcpus) != 0)
2254 			r = -EBUSY;
2255 		else
2256 			kvm->bsp_vcpu_id = arg;
2257 		mutex_unlock(&kvm->lock);
2258 		break;
2259 #endif
2260 #ifdef CONFIG_HAVE_KVM_MSI
2261 	case KVM_SIGNAL_MSI: {
2262 		struct kvm_msi msi;
2263 
2264 		r = -EFAULT;
2265 		if (copy_from_user(&msi, argp, sizeof msi))
2266 			goto out;
2267 		r = kvm_send_userspace_msi(kvm, &msi);
2268 		break;
2269 	}
2270 #endif
2271 #ifdef __KVM_HAVE_IRQ_LINE
2272 	case KVM_IRQ_LINE_STATUS:
2273 	case KVM_IRQ_LINE: {
2274 		struct kvm_irq_level irq_event;
2275 
2276 		r = -EFAULT;
2277 		if (copy_from_user(&irq_event, argp, sizeof irq_event))
2278 			goto out;
2279 
2280 		r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
2281 		if (r)
2282 			goto out;
2283 
2284 		r = -EFAULT;
2285 		if (ioctl == KVM_IRQ_LINE_STATUS) {
2286 			if (copy_to_user(argp, &irq_event, sizeof irq_event))
2287 				goto out;
2288 		}
2289 
2290 		r = 0;
2291 		break;
2292 	}
2293 #endif
2294 	default:
2295 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2296 		if (r == -ENOTTY)
2297 			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2298 	}
2299 out:
2300 	return r;
2301 }
2302 
2303 #ifdef CONFIG_COMPAT
2304 struct compat_kvm_dirty_log {
2305 	__u32 slot;
2306 	__u32 padding1;
2307 	union {
2308 		compat_uptr_t dirty_bitmap; /* one bit per page */
2309 		__u64 padding2;
2310 	};
2311 };
2312 
2313 static long kvm_vm_compat_ioctl(struct file *filp,
2314 			   unsigned int ioctl, unsigned long arg)
2315 {
2316 	struct kvm *kvm = filp->private_data;
2317 	int r;
2318 
2319 	if (kvm->mm != current->mm)
2320 		return -EIO;
2321 	switch (ioctl) {
2322 	case KVM_GET_DIRTY_LOG: {
2323 		struct compat_kvm_dirty_log compat_log;
2324 		struct kvm_dirty_log log;
2325 
2326 		r = -EFAULT;
2327 		if (copy_from_user(&compat_log, (void __user *)arg,
2328 				   sizeof(compat_log)))
2329 			goto out;
2330 		log.slot	 = compat_log.slot;
2331 		log.padding1	 = compat_log.padding1;
2332 		log.padding2	 = compat_log.padding2;
2333 		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2334 
2335 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2336 		break;
2337 	}
2338 	default:
2339 		r = kvm_vm_ioctl(filp, ioctl, arg);
2340 	}
2341 
2342 out:
2343 	return r;
2344 }
2345 #endif
2346 
2347 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2348 {
2349 	struct page *page[1];
2350 	unsigned long addr;
2351 	int npages;
2352 	gfn_t gfn = vmf->pgoff;
2353 	struct kvm *kvm = vma->vm_file->private_data;
2354 
2355 	addr = gfn_to_hva(kvm, gfn);
2356 	if (kvm_is_error_hva(addr))
2357 		return VM_FAULT_SIGBUS;
2358 
2359 	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
2360 				NULL);
2361 	if (unlikely(npages != 1))
2362 		return VM_FAULT_SIGBUS;
2363 
2364 	vmf->page = page[0];
2365 	return 0;
2366 }
2367 
2368 static const struct vm_operations_struct kvm_vm_vm_ops = {
2369 	.fault = kvm_vm_fault,
2370 };
2371 
2372 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2373 {
2374 	vma->vm_ops = &kvm_vm_vm_ops;
2375 	return 0;
2376 }
2377 
2378 static struct file_operations kvm_vm_fops = {
2379 	.release        = kvm_vm_release,
2380 	.unlocked_ioctl = kvm_vm_ioctl,
2381 #ifdef CONFIG_COMPAT
2382 	.compat_ioctl   = kvm_vm_compat_ioctl,
2383 #endif
2384 	.mmap           = kvm_vm_mmap,
2385 	.llseek		= noop_llseek,
2386 };
2387 
2388 static int kvm_dev_ioctl_create_vm(unsigned long type)
2389 {
2390 	int r;
2391 	struct kvm *kvm;
2392 
2393 	kvm = kvm_create_vm(type);
2394 	if (IS_ERR(kvm))
2395 		return PTR_ERR(kvm);
2396 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2397 	r = kvm_coalesced_mmio_init(kvm);
2398 	if (r < 0) {
2399 		kvm_put_kvm(kvm);
2400 		return r;
2401 	}
2402 #endif
2403 	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
2404 	if (r < 0)
2405 		kvm_put_kvm(kvm);
2406 
2407 	return r;
2408 }
2409 
2410 static long kvm_dev_ioctl_check_extension_generic(long arg)
2411 {
2412 	switch (arg) {
2413 	case KVM_CAP_USER_MEMORY:
2414 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2415 	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2416 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
2417 	case KVM_CAP_SET_BOOT_CPU_ID:
2418 #endif
2419 	case KVM_CAP_INTERNAL_ERROR_DATA:
2420 #ifdef CONFIG_HAVE_KVM_MSI
2421 	case KVM_CAP_SIGNAL_MSI:
2422 #endif
2423 		return 1;
2424 #ifdef KVM_CAP_IRQ_ROUTING
2425 	case KVM_CAP_IRQ_ROUTING:
2426 		return KVM_MAX_IRQ_ROUTES;
2427 #endif
2428 	default:
2429 		break;
2430 	}
2431 	return kvm_dev_ioctl_check_extension(arg);
2432 }
2433 
2434 static long kvm_dev_ioctl(struct file *filp,
2435 			  unsigned int ioctl, unsigned long arg)
2436 {
2437 	long r = -EINVAL;
2438 
2439 	switch (ioctl) {
2440 	case KVM_GET_API_VERSION:
2441 		r = -EINVAL;
2442 		if (arg)
2443 			goto out;
2444 		r = KVM_API_VERSION;
2445 		break;
2446 	case KVM_CREATE_VM:
2447 		r = kvm_dev_ioctl_create_vm(arg);
2448 		break;
2449 	case KVM_CHECK_EXTENSION:
2450 		r = kvm_dev_ioctl_check_extension_generic(arg);
2451 		break;
2452 	case KVM_GET_VCPU_MMAP_SIZE:
2453 		r = -EINVAL;
2454 		if (arg)
2455 			goto out;
2456 		r = PAGE_SIZE;     /* struct kvm_run */
2457 #ifdef CONFIG_X86
2458 		r += PAGE_SIZE;    /* pio data page */
2459 #endif
2460 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2461 		r += PAGE_SIZE;    /* coalesced mmio ring page */
2462 #endif
2463 		break;
2464 	case KVM_TRACE_ENABLE:
2465 	case KVM_TRACE_PAUSE:
2466 	case KVM_TRACE_DISABLE:
2467 		r = -EOPNOTSUPP;
2468 		break;
2469 	default:
2470 		return kvm_arch_dev_ioctl(filp, ioctl, arg);
2471 	}
2472 out:
2473 	return r;
2474 }
2475 
2476 static struct file_operations kvm_chardev_ops = {
2477 	.unlocked_ioctl = kvm_dev_ioctl,
2478 	.compat_ioctl   = kvm_dev_ioctl,
2479 	.llseek		= noop_llseek,
2480 };
2481 
2482 static struct miscdevice kvm_dev = {
2483 	KVM_MINOR,
2484 	"kvm",
2485 	&kvm_chardev_ops,
2486 };
2487 
2488 static void hardware_enable_nolock(void *junk)
2489 {
2490 	int cpu = raw_smp_processor_id();
2491 	int r;
2492 
2493 	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2494 		return;
2495 
2496 	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2497 
2498 	r = kvm_arch_hardware_enable(NULL);
2499 
2500 	if (r) {
2501 		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2502 		atomic_inc(&hardware_enable_failed);
2503 		printk(KERN_INFO "kvm: enabling virtualization on "
2504 				 "CPU%d failed\n", cpu);
2505 	}
2506 }
2507 
2508 static void hardware_enable(void *junk)
2509 {
2510 	raw_spin_lock(&kvm_lock);
2511 	hardware_enable_nolock(junk);
2512 	raw_spin_unlock(&kvm_lock);
2513 }
2514 
2515 static void hardware_disable_nolock(void *junk)
2516 {
2517 	int cpu = raw_smp_processor_id();
2518 
2519 	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2520 		return;
2521 	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2522 	kvm_arch_hardware_disable(NULL);
2523 }
2524 
2525 static void hardware_disable(void *junk)
2526 {
2527 	raw_spin_lock(&kvm_lock);
2528 	hardware_disable_nolock(junk);
2529 	raw_spin_unlock(&kvm_lock);
2530 }
2531 
2532 static void hardware_disable_all_nolock(void)
2533 {
2534 	BUG_ON(!kvm_usage_count);
2535 
2536 	kvm_usage_count--;
2537 	if (!kvm_usage_count)
2538 		on_each_cpu(hardware_disable_nolock, NULL, 1);
2539 }
2540 
2541 static void hardware_disable_all(void)
2542 {
2543 	raw_spin_lock(&kvm_lock);
2544 	hardware_disable_all_nolock();
2545 	raw_spin_unlock(&kvm_lock);
2546 }
2547 
2548 static int hardware_enable_all(void)
2549 {
2550 	int r = 0;
2551 
2552 	raw_spin_lock(&kvm_lock);
2553 
2554 	kvm_usage_count++;
2555 	if (kvm_usage_count == 1) {
2556 		atomic_set(&hardware_enable_failed, 0);
2557 		on_each_cpu(hardware_enable_nolock, NULL, 1);
2558 
2559 		if (atomic_read(&hardware_enable_failed)) {
2560 			hardware_disable_all_nolock();
2561 			r = -EBUSY;
2562 		}
2563 	}
2564 
2565 	raw_spin_unlock(&kvm_lock);
2566 
2567 	return r;
2568 }
2569 
2570 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2571 			   void *v)
2572 {
2573 	int cpu = (long)v;
2574 
2575 	if (!kvm_usage_count)
2576 		return NOTIFY_OK;
2577 
2578 	val &= ~CPU_TASKS_FROZEN;
2579 	switch (val) {
2580 	case CPU_DYING:
2581 		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2582 		       cpu);
2583 		hardware_disable(NULL);
2584 		break;
2585 	case CPU_STARTING:
2586 		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2587 		       cpu);
2588 		hardware_enable(NULL);
2589 		break;
2590 	}
2591 	return NOTIFY_OK;
2592 }
2593 
2594 
2595 asmlinkage void kvm_spurious_fault(void)
2596 {
2597 	/* Fault while not rebooting.  We want the trace. */
2598 	BUG();
2599 }
2600 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2601 
2602 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2603 		      void *v)
2604 {
2605 	/*
2606 	 * Some (well, at least mine) BIOSes hang on reboot if
2607 	 * in vmx root mode.
2608 	 *
2609 	 * And Intel TXT required VMX off for all cpu when system shutdown.
2610 	 */
2611 	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2612 	kvm_rebooting = true;
2613 	on_each_cpu(hardware_disable_nolock, NULL, 1);
2614 	return NOTIFY_OK;
2615 }
2616 
2617 static struct notifier_block kvm_reboot_notifier = {
2618 	.notifier_call = kvm_reboot,
2619 	.priority = 0,
2620 };
2621 
2622 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2623 {
2624 	int i;
2625 
2626 	for (i = 0; i < bus->dev_count; i++) {
2627 		struct kvm_io_device *pos = bus->range[i].dev;
2628 
2629 		kvm_iodevice_destructor(pos);
2630 	}
2631 	kfree(bus);
2632 }
2633 
2634 int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
2635 {
2636 	const struct kvm_io_range *r1 = p1;
2637 	const struct kvm_io_range *r2 = p2;
2638 
2639 	if (r1->addr < r2->addr)
2640 		return -1;
2641 	if (r1->addr + r1->len > r2->addr + r2->len)
2642 		return 1;
2643 	return 0;
2644 }
2645 
2646 int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
2647 			  gpa_t addr, int len)
2648 {
2649 	bus->range[bus->dev_count++] = (struct kvm_io_range) {
2650 		.addr = addr,
2651 		.len = len,
2652 		.dev = dev,
2653 	};
2654 
2655 	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
2656 		kvm_io_bus_sort_cmp, NULL);
2657 
2658 	return 0;
2659 }
2660 
2661 int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2662 			     gpa_t addr, int len)
2663 {
2664 	struct kvm_io_range *range, key;
2665 	int off;
2666 
2667 	key = (struct kvm_io_range) {
2668 		.addr = addr,
2669 		.len = len,
2670 	};
2671 
2672 	range = bsearch(&key, bus->range, bus->dev_count,
2673 			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
2674 	if (range == NULL)
2675 		return -ENOENT;
2676 
2677 	off = range - bus->range;
2678 
2679 	while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
2680 		off--;
2681 
2682 	return off;
2683 }
2684 
2685 /* kvm_io_bus_write - called under kvm->slots_lock */
2686 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2687 		     int len, const void *val)
2688 {
2689 	int idx;
2690 	struct kvm_io_bus *bus;
2691 	struct kvm_io_range range;
2692 
2693 	range = (struct kvm_io_range) {
2694 		.addr = addr,
2695 		.len = len,
2696 	};
2697 
2698 	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2699 	idx = kvm_io_bus_get_first_dev(bus, addr, len);
2700 	if (idx < 0)
2701 		return -EOPNOTSUPP;
2702 
2703 	while (idx < bus->dev_count &&
2704 		kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
2705 		if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
2706 			return 0;
2707 		idx++;
2708 	}
2709 
2710 	return -EOPNOTSUPP;
2711 }
2712 
2713 /* kvm_io_bus_read - called under kvm->slots_lock */
2714 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2715 		    int len, void *val)
2716 {
2717 	int idx;
2718 	struct kvm_io_bus *bus;
2719 	struct kvm_io_range range;
2720 
2721 	range = (struct kvm_io_range) {
2722 		.addr = addr,
2723 		.len = len,
2724 	};
2725 
2726 	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2727 	idx = kvm_io_bus_get_first_dev(bus, addr, len);
2728 	if (idx < 0)
2729 		return -EOPNOTSUPP;
2730 
2731 	while (idx < bus->dev_count &&
2732 		kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
2733 		if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
2734 			return 0;
2735 		idx++;
2736 	}
2737 
2738 	return -EOPNOTSUPP;
2739 }
2740 
2741 /* Caller must hold slots_lock. */
2742 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2743 			    int len, struct kvm_io_device *dev)
2744 {
2745 	struct kvm_io_bus *new_bus, *bus;
2746 
2747 	bus = kvm->buses[bus_idx];
2748 	if (bus->dev_count > NR_IOBUS_DEVS - 1)
2749 		return -ENOSPC;
2750 
2751 	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
2752 			  sizeof(struct kvm_io_range)), GFP_KERNEL);
2753 	if (!new_bus)
2754 		return -ENOMEM;
2755 	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
2756 	       sizeof(struct kvm_io_range)));
2757 	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
2758 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2759 	synchronize_srcu_expedited(&kvm->srcu);
2760 	kfree(bus);
2761 
2762 	return 0;
2763 }
2764 
2765 /* Caller must hold slots_lock. */
2766 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2767 			      struct kvm_io_device *dev)
2768 {
2769 	int i, r;
2770 	struct kvm_io_bus *new_bus, *bus;
2771 
2772 	bus = kvm->buses[bus_idx];
2773 	r = -ENOENT;
2774 	for (i = 0; i < bus->dev_count; i++)
2775 		if (bus->range[i].dev == dev) {
2776 			r = 0;
2777 			break;
2778 		}
2779 
2780 	if (r)
2781 		return r;
2782 
2783 	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
2784 			  sizeof(struct kvm_io_range)), GFP_KERNEL);
2785 	if (!new_bus)
2786 		return -ENOMEM;
2787 
2788 	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
2789 	new_bus->dev_count--;
2790 	memcpy(new_bus->range + i, bus->range + i + 1,
2791 	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
2792 
2793 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2794 	synchronize_srcu_expedited(&kvm->srcu);
2795 	kfree(bus);
2796 	return r;
2797 }
2798 
2799 static struct notifier_block kvm_cpu_notifier = {
2800 	.notifier_call = kvm_cpu_hotplug,
2801 };
2802 
2803 static int vm_stat_get(void *_offset, u64 *val)
2804 {
2805 	unsigned offset = (long)_offset;
2806 	struct kvm *kvm;
2807 
2808 	*val = 0;
2809 	raw_spin_lock(&kvm_lock);
2810 	list_for_each_entry(kvm, &vm_list, vm_list)
2811 		*val += *(u32 *)((void *)kvm + offset);
2812 	raw_spin_unlock(&kvm_lock);
2813 	return 0;
2814 }
2815 
2816 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
2817 
2818 static int vcpu_stat_get(void *_offset, u64 *val)
2819 {
2820 	unsigned offset = (long)_offset;
2821 	struct kvm *kvm;
2822 	struct kvm_vcpu *vcpu;
2823 	int i;
2824 
2825 	*val = 0;
2826 	raw_spin_lock(&kvm_lock);
2827 	list_for_each_entry(kvm, &vm_list, vm_list)
2828 		kvm_for_each_vcpu(i, vcpu, kvm)
2829 			*val += *(u32 *)((void *)vcpu + offset);
2830 
2831 	raw_spin_unlock(&kvm_lock);
2832 	return 0;
2833 }
2834 
2835 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
2836 
2837 static const struct file_operations *stat_fops[] = {
2838 	[KVM_STAT_VCPU] = &vcpu_stat_fops,
2839 	[KVM_STAT_VM]   = &vm_stat_fops,
2840 };
2841 
2842 static int kvm_init_debug(void)
2843 {
2844 	int r = -EFAULT;
2845 	struct kvm_stats_debugfs_item *p;
2846 
2847 	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2848 	if (kvm_debugfs_dir == NULL)
2849 		goto out;
2850 
2851 	for (p = debugfs_entries; p->name; ++p) {
2852 		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
2853 						(void *)(long)p->offset,
2854 						stat_fops[p->kind]);
2855 		if (p->dentry == NULL)
2856 			goto out_dir;
2857 	}
2858 
2859 	return 0;
2860 
2861 out_dir:
2862 	debugfs_remove_recursive(kvm_debugfs_dir);
2863 out:
2864 	return r;
2865 }
2866 
2867 static void kvm_exit_debug(void)
2868 {
2869 	struct kvm_stats_debugfs_item *p;
2870 
2871 	for (p = debugfs_entries; p->name; ++p)
2872 		debugfs_remove(p->dentry);
2873 	debugfs_remove(kvm_debugfs_dir);
2874 }
2875 
2876 static int kvm_suspend(void)
2877 {
2878 	if (kvm_usage_count)
2879 		hardware_disable_nolock(NULL);
2880 	return 0;
2881 }
2882 
2883 static void kvm_resume(void)
2884 {
2885 	if (kvm_usage_count) {
2886 		WARN_ON(raw_spin_is_locked(&kvm_lock));
2887 		hardware_enable_nolock(NULL);
2888 	}
2889 }
2890 
2891 static struct syscore_ops kvm_syscore_ops = {
2892 	.suspend = kvm_suspend,
2893 	.resume = kvm_resume,
2894 };
2895 
2896 static inline
2897 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2898 {
2899 	return container_of(pn, struct kvm_vcpu, preempt_notifier);
2900 }
2901 
2902 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2903 {
2904 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2905 
2906 	kvm_arch_vcpu_load(vcpu, cpu);
2907 }
2908 
2909 static void kvm_sched_out(struct preempt_notifier *pn,
2910 			  struct task_struct *next)
2911 {
2912 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2913 
2914 	kvm_arch_vcpu_put(vcpu);
2915 }
2916 
2917 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2918 		  struct module *module)
2919 {
2920 	int r;
2921 	int cpu;
2922 
2923 	r = kvm_arch_init(opaque);
2924 	if (r)
2925 		goto out_fail;
2926 
2927 	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2928 		r = -ENOMEM;
2929 		goto out_free_0;
2930 	}
2931 
2932 	r = kvm_arch_hardware_setup();
2933 	if (r < 0)
2934 		goto out_free_0a;
2935 
2936 	for_each_online_cpu(cpu) {
2937 		smp_call_function_single(cpu,
2938 				kvm_arch_check_processor_compat,
2939 				&r, 1);
2940 		if (r < 0)
2941 			goto out_free_1;
2942 	}
2943 
2944 	r = register_cpu_notifier(&kvm_cpu_notifier);
2945 	if (r)
2946 		goto out_free_2;
2947 	register_reboot_notifier(&kvm_reboot_notifier);
2948 
2949 	/* A kmem cache lets us meet the alignment requirements of fx_save. */
2950 	if (!vcpu_align)
2951 		vcpu_align = __alignof__(struct kvm_vcpu);
2952 	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
2953 					   0, NULL);
2954 	if (!kvm_vcpu_cache) {
2955 		r = -ENOMEM;
2956 		goto out_free_3;
2957 	}
2958 
2959 	r = kvm_async_pf_init();
2960 	if (r)
2961 		goto out_free;
2962 
2963 	kvm_chardev_ops.owner = module;
2964 	kvm_vm_fops.owner = module;
2965 	kvm_vcpu_fops.owner = module;
2966 
2967 	r = misc_register(&kvm_dev);
2968 	if (r) {
2969 		printk(KERN_ERR "kvm: misc device register failed\n");
2970 		goto out_unreg;
2971 	}
2972 
2973 	register_syscore_ops(&kvm_syscore_ops);
2974 
2975 	kvm_preempt_ops.sched_in = kvm_sched_in;
2976 	kvm_preempt_ops.sched_out = kvm_sched_out;
2977 
2978 	r = kvm_init_debug();
2979 	if (r) {
2980 		printk(KERN_ERR "kvm: create debugfs files failed\n");
2981 		goto out_undebugfs;
2982 	}
2983 
2984 	return 0;
2985 
2986 out_undebugfs:
2987 	unregister_syscore_ops(&kvm_syscore_ops);
2988 out_unreg:
2989 	kvm_async_pf_deinit();
2990 out_free:
2991 	kmem_cache_destroy(kvm_vcpu_cache);
2992 out_free_3:
2993 	unregister_reboot_notifier(&kvm_reboot_notifier);
2994 	unregister_cpu_notifier(&kvm_cpu_notifier);
2995 out_free_2:
2996 out_free_1:
2997 	kvm_arch_hardware_unsetup();
2998 out_free_0a:
2999 	free_cpumask_var(cpus_hardware_enabled);
3000 out_free_0:
3001 	kvm_arch_exit();
3002 out_fail:
3003 	return r;
3004 }
3005 EXPORT_SYMBOL_GPL(kvm_init);
3006 
3007 void kvm_exit(void)
3008 {
3009 	kvm_exit_debug();
3010 	misc_deregister(&kvm_dev);
3011 	kmem_cache_destroy(kvm_vcpu_cache);
3012 	kvm_async_pf_deinit();
3013 	unregister_syscore_ops(&kvm_syscore_ops);
3014 	unregister_reboot_notifier(&kvm_reboot_notifier);
3015 	unregister_cpu_notifier(&kvm_cpu_notifier);
3016 	on_each_cpu(hardware_disable_nolock, NULL, 1);
3017 	kvm_arch_hardware_unsetup();
3018 	kvm_arch_exit();
3019 	free_cpumask_var(cpus_hardware_enabled);
3020 }
3021 EXPORT_SYMBOL_GPL(kvm_exit);
3022