xref: /openbmc/linux/virt/kvm/kvm_main.c (revision 174cd4b1)
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9  *
10  * Authors:
11  *   Avi Kivity   <avi@qumranet.com>
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *
14  * This work is licensed under the terms of the GNU GPL, version 2.  See
15  * the COPYING file in the top-level directory.
16  *
17  */
18 
19 #include <kvm/iodev.h>
20 
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/syscore_ops.h>
34 #include <linux/cpu.h>
35 #include <linux/sched/signal.h>
36 #include <linux/sched/mm.h>
37 #include <linux/cpumask.h>
38 #include <linux/smp.h>
39 #include <linux/anon_inodes.h>
40 #include <linux/profile.h>
41 #include <linux/kvm_para.h>
42 #include <linux/pagemap.h>
43 #include <linux/mman.h>
44 #include <linux/swap.h>
45 #include <linux/bitops.h>
46 #include <linux/spinlock.h>
47 #include <linux/compat.h>
48 #include <linux/srcu.h>
49 #include <linux/hugetlb.h>
50 #include <linux/slab.h>
51 #include <linux/sort.h>
52 #include <linux/bsearch.h>
53 
54 #include <asm/processor.h>
55 #include <asm/io.h>
56 #include <asm/ioctl.h>
57 #include <linux/uaccess.h>
58 #include <asm/pgtable.h>
59 
60 #include "coalesced_mmio.h"
61 #include "async_pf.h"
62 #include "vfio.h"
63 
64 #define CREATE_TRACE_POINTS
65 #include <trace/events/kvm.h>
66 
67 /* Worst case buffer size needed for holding an integer. */
68 #define ITOA_MAX_LEN 12
69 
70 MODULE_AUTHOR("Qumranet");
71 MODULE_LICENSE("GPL");
72 
73 /* Architectures should define their poll value according to the halt latency */
74 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
75 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
76 EXPORT_SYMBOL_GPL(halt_poll_ns);
77 
78 /* Default doubles per-vcpu halt_poll_ns. */
79 unsigned int halt_poll_ns_grow = 2;
80 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR);
81 EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
82 
83 /* Default resets per-vcpu halt_poll_ns . */
84 unsigned int halt_poll_ns_shrink;
85 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR);
86 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
87 
88 /*
89  * Ordering of locks:
90  *
91  *	kvm->lock --> kvm->slots_lock --> kvm->irq_lock
92  */
93 
94 DEFINE_SPINLOCK(kvm_lock);
95 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
96 LIST_HEAD(vm_list);
97 
98 static cpumask_var_t cpus_hardware_enabled;
99 static int kvm_usage_count;
100 static atomic_t hardware_enable_failed;
101 
102 struct kmem_cache *kvm_vcpu_cache;
103 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
104 
105 static __read_mostly struct preempt_ops kvm_preempt_ops;
106 
107 struct dentry *kvm_debugfs_dir;
108 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
109 
110 static int kvm_debugfs_num_entries;
111 static const struct file_operations *stat_fops_per_vm[];
112 
113 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
114 			   unsigned long arg);
115 #ifdef CONFIG_KVM_COMPAT
116 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
117 				  unsigned long arg);
118 #endif
119 static int hardware_enable_all(void);
120 static void hardware_disable_all(void);
121 
122 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
123 
124 static void kvm_release_pfn_dirty(kvm_pfn_t pfn);
125 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
126 
127 __visible bool kvm_rebooting;
128 EXPORT_SYMBOL_GPL(kvm_rebooting);
129 
130 static bool largepages_enabled = true;
131 
132 bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
133 {
134 	if (pfn_valid(pfn))
135 		return PageReserved(pfn_to_page(pfn));
136 
137 	return true;
138 }
139 
140 /*
141  * Switches to specified vcpu, until a matching vcpu_put()
142  */
143 int vcpu_load(struct kvm_vcpu *vcpu)
144 {
145 	int cpu;
146 
147 	if (mutex_lock_killable(&vcpu->mutex))
148 		return -EINTR;
149 	cpu = get_cpu();
150 	preempt_notifier_register(&vcpu->preempt_notifier);
151 	kvm_arch_vcpu_load(vcpu, cpu);
152 	put_cpu();
153 	return 0;
154 }
155 EXPORT_SYMBOL_GPL(vcpu_load);
156 
157 void vcpu_put(struct kvm_vcpu *vcpu)
158 {
159 	preempt_disable();
160 	kvm_arch_vcpu_put(vcpu);
161 	preempt_notifier_unregister(&vcpu->preempt_notifier);
162 	preempt_enable();
163 	mutex_unlock(&vcpu->mutex);
164 }
165 EXPORT_SYMBOL_GPL(vcpu_put);
166 
167 static void ack_flush(void *_completed)
168 {
169 }
170 
171 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
172 {
173 	int i, cpu, me;
174 	cpumask_var_t cpus;
175 	bool called = true;
176 	struct kvm_vcpu *vcpu;
177 
178 	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
179 
180 	me = get_cpu();
181 	kvm_for_each_vcpu(i, vcpu, kvm) {
182 		kvm_make_request(req, vcpu);
183 		cpu = vcpu->cpu;
184 
185 		/* Set ->requests bit before we read ->mode. */
186 		smp_mb__after_atomic();
187 
188 		if (cpus != NULL && cpu != -1 && cpu != me &&
189 		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
190 			cpumask_set_cpu(cpu, cpus);
191 	}
192 	if (unlikely(cpus == NULL))
193 		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
194 	else if (!cpumask_empty(cpus))
195 		smp_call_function_many(cpus, ack_flush, NULL, 1);
196 	else
197 		called = false;
198 	put_cpu();
199 	free_cpumask_var(cpus);
200 	return called;
201 }
202 
203 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
204 void kvm_flush_remote_tlbs(struct kvm *kvm)
205 {
206 	/*
207 	 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
208 	 * kvm_make_all_cpus_request.
209 	 */
210 	long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
211 
212 	/*
213 	 * We want to publish modifications to the page tables before reading
214 	 * mode. Pairs with a memory barrier in arch-specific code.
215 	 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
216 	 * and smp_mb in walk_shadow_page_lockless_begin/end.
217 	 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
218 	 *
219 	 * There is already an smp_mb__after_atomic() before
220 	 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
221 	 * barrier here.
222 	 */
223 	if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
224 		++kvm->stat.remote_tlb_flush;
225 	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
226 }
227 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
228 #endif
229 
230 void kvm_reload_remote_mmus(struct kvm *kvm)
231 {
232 	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
233 }
234 
235 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
236 {
237 	struct page *page;
238 	int r;
239 
240 	mutex_init(&vcpu->mutex);
241 	vcpu->cpu = -1;
242 	vcpu->kvm = kvm;
243 	vcpu->vcpu_id = id;
244 	vcpu->pid = NULL;
245 	init_swait_queue_head(&vcpu->wq);
246 	kvm_async_pf_vcpu_init(vcpu);
247 
248 	vcpu->pre_pcpu = -1;
249 	INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
250 
251 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
252 	if (!page) {
253 		r = -ENOMEM;
254 		goto fail;
255 	}
256 	vcpu->run = page_address(page);
257 
258 	kvm_vcpu_set_in_spin_loop(vcpu, false);
259 	kvm_vcpu_set_dy_eligible(vcpu, false);
260 	vcpu->preempted = false;
261 
262 	r = kvm_arch_vcpu_init(vcpu);
263 	if (r < 0)
264 		goto fail_free_run;
265 	return 0;
266 
267 fail_free_run:
268 	free_page((unsigned long)vcpu->run);
269 fail:
270 	return r;
271 }
272 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
273 
274 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
275 {
276 	put_pid(vcpu->pid);
277 	kvm_arch_vcpu_uninit(vcpu);
278 	free_page((unsigned long)vcpu->run);
279 }
280 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
281 
282 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
283 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
284 {
285 	return container_of(mn, struct kvm, mmu_notifier);
286 }
287 
288 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
289 					     struct mm_struct *mm,
290 					     unsigned long address)
291 {
292 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
293 	int need_tlb_flush, idx;
294 
295 	/*
296 	 * When ->invalidate_page runs, the linux pte has been zapped
297 	 * already but the page is still allocated until
298 	 * ->invalidate_page returns. So if we increase the sequence
299 	 * here the kvm page fault will notice if the spte can't be
300 	 * established because the page is going to be freed. If
301 	 * instead the kvm page fault establishes the spte before
302 	 * ->invalidate_page runs, kvm_unmap_hva will release it
303 	 * before returning.
304 	 *
305 	 * The sequence increase only need to be seen at spin_unlock
306 	 * time, and not at spin_lock time.
307 	 *
308 	 * Increasing the sequence after the spin_unlock would be
309 	 * unsafe because the kvm page fault could then establish the
310 	 * pte after kvm_unmap_hva returned, without noticing the page
311 	 * is going to be freed.
312 	 */
313 	idx = srcu_read_lock(&kvm->srcu);
314 	spin_lock(&kvm->mmu_lock);
315 
316 	kvm->mmu_notifier_seq++;
317 	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
318 	/* we've to flush the tlb before the pages can be freed */
319 	if (need_tlb_flush)
320 		kvm_flush_remote_tlbs(kvm);
321 
322 	spin_unlock(&kvm->mmu_lock);
323 
324 	kvm_arch_mmu_notifier_invalidate_page(kvm, address);
325 
326 	srcu_read_unlock(&kvm->srcu, idx);
327 }
328 
329 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
330 					struct mm_struct *mm,
331 					unsigned long address,
332 					pte_t pte)
333 {
334 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
335 	int idx;
336 
337 	idx = srcu_read_lock(&kvm->srcu);
338 	spin_lock(&kvm->mmu_lock);
339 	kvm->mmu_notifier_seq++;
340 	kvm_set_spte_hva(kvm, address, pte);
341 	spin_unlock(&kvm->mmu_lock);
342 	srcu_read_unlock(&kvm->srcu, idx);
343 }
344 
345 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
346 						    struct mm_struct *mm,
347 						    unsigned long start,
348 						    unsigned long end)
349 {
350 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
351 	int need_tlb_flush = 0, idx;
352 
353 	idx = srcu_read_lock(&kvm->srcu);
354 	spin_lock(&kvm->mmu_lock);
355 	/*
356 	 * The count increase must become visible at unlock time as no
357 	 * spte can be established without taking the mmu_lock and
358 	 * count is also read inside the mmu_lock critical section.
359 	 */
360 	kvm->mmu_notifier_count++;
361 	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
362 	need_tlb_flush |= kvm->tlbs_dirty;
363 	/* we've to flush the tlb before the pages can be freed */
364 	if (need_tlb_flush)
365 		kvm_flush_remote_tlbs(kvm);
366 
367 	spin_unlock(&kvm->mmu_lock);
368 	srcu_read_unlock(&kvm->srcu, idx);
369 }
370 
371 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
372 						  struct mm_struct *mm,
373 						  unsigned long start,
374 						  unsigned long end)
375 {
376 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
377 
378 	spin_lock(&kvm->mmu_lock);
379 	/*
380 	 * This sequence increase will notify the kvm page fault that
381 	 * the page that is going to be mapped in the spte could have
382 	 * been freed.
383 	 */
384 	kvm->mmu_notifier_seq++;
385 	smp_wmb();
386 	/*
387 	 * The above sequence increase must be visible before the
388 	 * below count decrease, which is ensured by the smp_wmb above
389 	 * in conjunction with the smp_rmb in mmu_notifier_retry().
390 	 */
391 	kvm->mmu_notifier_count--;
392 	spin_unlock(&kvm->mmu_lock);
393 
394 	BUG_ON(kvm->mmu_notifier_count < 0);
395 }
396 
397 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
398 					      struct mm_struct *mm,
399 					      unsigned long start,
400 					      unsigned long end)
401 {
402 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
403 	int young, idx;
404 
405 	idx = srcu_read_lock(&kvm->srcu);
406 	spin_lock(&kvm->mmu_lock);
407 
408 	young = kvm_age_hva(kvm, start, end);
409 	if (young)
410 		kvm_flush_remote_tlbs(kvm);
411 
412 	spin_unlock(&kvm->mmu_lock);
413 	srcu_read_unlock(&kvm->srcu, idx);
414 
415 	return young;
416 }
417 
418 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
419 					struct mm_struct *mm,
420 					unsigned long start,
421 					unsigned long end)
422 {
423 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
424 	int young, idx;
425 
426 	idx = srcu_read_lock(&kvm->srcu);
427 	spin_lock(&kvm->mmu_lock);
428 	/*
429 	 * Even though we do not flush TLB, this will still adversely
430 	 * affect performance on pre-Haswell Intel EPT, where there is
431 	 * no EPT Access Bit to clear so that we have to tear down EPT
432 	 * tables instead. If we find this unacceptable, we can always
433 	 * add a parameter to kvm_age_hva so that it effectively doesn't
434 	 * do anything on clear_young.
435 	 *
436 	 * Also note that currently we never issue secondary TLB flushes
437 	 * from clear_young, leaving this job up to the regular system
438 	 * cadence. If we find this inaccurate, we might come up with a
439 	 * more sophisticated heuristic later.
440 	 */
441 	young = kvm_age_hva(kvm, start, end);
442 	spin_unlock(&kvm->mmu_lock);
443 	srcu_read_unlock(&kvm->srcu, idx);
444 
445 	return young;
446 }
447 
448 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
449 				       struct mm_struct *mm,
450 				       unsigned long address)
451 {
452 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
453 	int young, idx;
454 
455 	idx = srcu_read_lock(&kvm->srcu);
456 	spin_lock(&kvm->mmu_lock);
457 	young = kvm_test_age_hva(kvm, address);
458 	spin_unlock(&kvm->mmu_lock);
459 	srcu_read_unlock(&kvm->srcu, idx);
460 
461 	return young;
462 }
463 
464 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
465 				     struct mm_struct *mm)
466 {
467 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
468 	int idx;
469 
470 	idx = srcu_read_lock(&kvm->srcu);
471 	kvm_arch_flush_shadow_all(kvm);
472 	srcu_read_unlock(&kvm->srcu, idx);
473 }
474 
475 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
476 	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
477 	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
478 	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
479 	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
480 	.clear_young		= kvm_mmu_notifier_clear_young,
481 	.test_young		= kvm_mmu_notifier_test_young,
482 	.change_pte		= kvm_mmu_notifier_change_pte,
483 	.release		= kvm_mmu_notifier_release,
484 };
485 
486 static int kvm_init_mmu_notifier(struct kvm *kvm)
487 {
488 	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
489 	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
490 }
491 
492 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
493 
494 static int kvm_init_mmu_notifier(struct kvm *kvm)
495 {
496 	return 0;
497 }
498 
499 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
500 
501 static struct kvm_memslots *kvm_alloc_memslots(void)
502 {
503 	int i;
504 	struct kvm_memslots *slots;
505 
506 	slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
507 	if (!slots)
508 		return NULL;
509 
510 	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
511 		slots->id_to_index[i] = slots->memslots[i].id = i;
512 
513 	return slots;
514 }
515 
516 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
517 {
518 	if (!memslot->dirty_bitmap)
519 		return;
520 
521 	kvfree(memslot->dirty_bitmap);
522 	memslot->dirty_bitmap = NULL;
523 }
524 
525 /*
526  * Free any memory in @free but not in @dont.
527  */
528 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
529 			      struct kvm_memory_slot *dont)
530 {
531 	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
532 		kvm_destroy_dirty_bitmap(free);
533 
534 	kvm_arch_free_memslot(kvm, free, dont);
535 
536 	free->npages = 0;
537 }
538 
539 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
540 {
541 	struct kvm_memory_slot *memslot;
542 
543 	if (!slots)
544 		return;
545 
546 	kvm_for_each_memslot(memslot, slots)
547 		kvm_free_memslot(kvm, memslot, NULL);
548 
549 	kvfree(slots);
550 }
551 
552 static void kvm_destroy_vm_debugfs(struct kvm *kvm)
553 {
554 	int i;
555 
556 	if (!kvm->debugfs_dentry)
557 		return;
558 
559 	debugfs_remove_recursive(kvm->debugfs_dentry);
560 
561 	if (kvm->debugfs_stat_data) {
562 		for (i = 0; i < kvm_debugfs_num_entries; i++)
563 			kfree(kvm->debugfs_stat_data[i]);
564 		kfree(kvm->debugfs_stat_data);
565 	}
566 }
567 
568 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
569 {
570 	char dir_name[ITOA_MAX_LEN * 2];
571 	struct kvm_stat_data *stat_data;
572 	struct kvm_stats_debugfs_item *p;
573 
574 	if (!debugfs_initialized())
575 		return 0;
576 
577 	snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
578 	kvm->debugfs_dentry = debugfs_create_dir(dir_name,
579 						 kvm_debugfs_dir);
580 	if (!kvm->debugfs_dentry)
581 		return -ENOMEM;
582 
583 	kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
584 					 sizeof(*kvm->debugfs_stat_data),
585 					 GFP_KERNEL);
586 	if (!kvm->debugfs_stat_data)
587 		return -ENOMEM;
588 
589 	for (p = debugfs_entries; p->name; p++) {
590 		stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL);
591 		if (!stat_data)
592 			return -ENOMEM;
593 
594 		stat_data->kvm = kvm;
595 		stat_data->offset = p->offset;
596 		kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
597 		if (!debugfs_create_file(p->name, 0644,
598 					 kvm->debugfs_dentry,
599 					 stat_data,
600 					 stat_fops_per_vm[p->kind]))
601 			return -ENOMEM;
602 	}
603 	return 0;
604 }
605 
606 static struct kvm *kvm_create_vm(unsigned long type)
607 {
608 	int r, i;
609 	struct kvm *kvm = kvm_arch_alloc_vm();
610 
611 	if (!kvm)
612 		return ERR_PTR(-ENOMEM);
613 
614 	spin_lock_init(&kvm->mmu_lock);
615 	mmgrab(current->mm);
616 	kvm->mm = current->mm;
617 	kvm_eventfd_init(kvm);
618 	mutex_init(&kvm->lock);
619 	mutex_init(&kvm->irq_lock);
620 	mutex_init(&kvm->slots_lock);
621 	atomic_set(&kvm->users_count, 1);
622 	INIT_LIST_HEAD(&kvm->devices);
623 
624 	r = kvm_arch_init_vm(kvm, type);
625 	if (r)
626 		goto out_err_no_disable;
627 
628 	r = hardware_enable_all();
629 	if (r)
630 		goto out_err_no_disable;
631 
632 #ifdef CONFIG_HAVE_KVM_IRQFD
633 	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
634 #endif
635 
636 	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
637 
638 	r = -ENOMEM;
639 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
640 		struct kvm_memslots *slots = kvm_alloc_memslots();
641 		if (!slots)
642 			goto out_err_no_srcu;
643 		/*
644 		 * Generations must be different for each address space.
645 		 * Init kvm generation close to the maximum to easily test the
646 		 * code of handling generation number wrap-around.
647 		 */
648 		slots->generation = i * 2 - 150;
649 		rcu_assign_pointer(kvm->memslots[i], slots);
650 	}
651 
652 	if (init_srcu_struct(&kvm->srcu))
653 		goto out_err_no_srcu;
654 	if (init_srcu_struct(&kvm->irq_srcu))
655 		goto out_err_no_irq_srcu;
656 	for (i = 0; i < KVM_NR_BUSES; i++) {
657 		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
658 					GFP_KERNEL);
659 		if (!kvm->buses[i])
660 			goto out_err;
661 	}
662 
663 	r = kvm_init_mmu_notifier(kvm);
664 	if (r)
665 		goto out_err;
666 
667 	spin_lock(&kvm_lock);
668 	list_add(&kvm->vm_list, &vm_list);
669 	spin_unlock(&kvm_lock);
670 
671 	preempt_notifier_inc();
672 
673 	return kvm;
674 
675 out_err:
676 	cleanup_srcu_struct(&kvm->irq_srcu);
677 out_err_no_irq_srcu:
678 	cleanup_srcu_struct(&kvm->srcu);
679 out_err_no_srcu:
680 	hardware_disable_all();
681 out_err_no_disable:
682 	for (i = 0; i < KVM_NR_BUSES; i++)
683 		kfree(kvm->buses[i]);
684 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
685 		kvm_free_memslots(kvm, kvm->memslots[i]);
686 	kvm_arch_free_vm(kvm);
687 	mmdrop(current->mm);
688 	return ERR_PTR(r);
689 }
690 
691 /*
692  * Avoid using vmalloc for a small buffer.
693  * Should not be used when the size is statically known.
694  */
695 void *kvm_kvzalloc(unsigned long size)
696 {
697 	if (size > PAGE_SIZE)
698 		return vzalloc(size);
699 	else
700 		return kzalloc(size, GFP_KERNEL);
701 }
702 
703 static void kvm_destroy_devices(struct kvm *kvm)
704 {
705 	struct kvm_device *dev, *tmp;
706 
707 	/*
708 	 * We do not need to take the kvm->lock here, because nobody else
709 	 * has a reference to the struct kvm at this point and therefore
710 	 * cannot access the devices list anyhow.
711 	 */
712 	list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
713 		list_del(&dev->vm_node);
714 		dev->ops->destroy(dev);
715 	}
716 }
717 
718 static void kvm_destroy_vm(struct kvm *kvm)
719 {
720 	int i;
721 	struct mm_struct *mm = kvm->mm;
722 
723 	kvm_destroy_vm_debugfs(kvm);
724 	kvm_arch_sync_events(kvm);
725 	spin_lock(&kvm_lock);
726 	list_del(&kvm->vm_list);
727 	spin_unlock(&kvm_lock);
728 	kvm_free_irq_routing(kvm);
729 	for (i = 0; i < KVM_NR_BUSES; i++)
730 		kvm_io_bus_destroy(kvm->buses[i]);
731 	kvm_coalesced_mmio_free(kvm);
732 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
733 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
734 #else
735 	kvm_arch_flush_shadow_all(kvm);
736 #endif
737 	kvm_arch_destroy_vm(kvm);
738 	kvm_destroy_devices(kvm);
739 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
740 		kvm_free_memslots(kvm, kvm->memslots[i]);
741 	cleanup_srcu_struct(&kvm->irq_srcu);
742 	cleanup_srcu_struct(&kvm->srcu);
743 	kvm_arch_free_vm(kvm);
744 	preempt_notifier_dec();
745 	hardware_disable_all();
746 	mmdrop(mm);
747 }
748 
749 void kvm_get_kvm(struct kvm *kvm)
750 {
751 	atomic_inc(&kvm->users_count);
752 }
753 EXPORT_SYMBOL_GPL(kvm_get_kvm);
754 
755 void kvm_put_kvm(struct kvm *kvm)
756 {
757 	if (atomic_dec_and_test(&kvm->users_count))
758 		kvm_destroy_vm(kvm);
759 }
760 EXPORT_SYMBOL_GPL(kvm_put_kvm);
761 
762 
763 static int kvm_vm_release(struct inode *inode, struct file *filp)
764 {
765 	struct kvm *kvm = filp->private_data;
766 
767 	kvm_irqfd_release(kvm);
768 
769 	kvm_put_kvm(kvm);
770 	return 0;
771 }
772 
773 /*
774  * Allocation size is twice as large as the actual dirty bitmap size.
775  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
776  */
777 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
778 {
779 	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
780 
781 	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
782 	if (!memslot->dirty_bitmap)
783 		return -ENOMEM;
784 
785 	return 0;
786 }
787 
788 /*
789  * Insert memslot and re-sort memslots based on their GFN,
790  * so binary search could be used to lookup GFN.
791  * Sorting algorithm takes advantage of having initially
792  * sorted array and known changed memslot position.
793  */
794 static void update_memslots(struct kvm_memslots *slots,
795 			    struct kvm_memory_slot *new)
796 {
797 	int id = new->id;
798 	int i = slots->id_to_index[id];
799 	struct kvm_memory_slot *mslots = slots->memslots;
800 
801 	WARN_ON(mslots[i].id != id);
802 	if (!new->npages) {
803 		WARN_ON(!mslots[i].npages);
804 		if (mslots[i].npages)
805 			slots->used_slots--;
806 	} else {
807 		if (!mslots[i].npages)
808 			slots->used_slots++;
809 	}
810 
811 	while (i < KVM_MEM_SLOTS_NUM - 1 &&
812 	       new->base_gfn <= mslots[i + 1].base_gfn) {
813 		if (!mslots[i + 1].npages)
814 			break;
815 		mslots[i] = mslots[i + 1];
816 		slots->id_to_index[mslots[i].id] = i;
817 		i++;
818 	}
819 
820 	/*
821 	 * The ">=" is needed when creating a slot with base_gfn == 0,
822 	 * so that it moves before all those with base_gfn == npages == 0.
823 	 *
824 	 * On the other hand, if new->npages is zero, the above loop has
825 	 * already left i pointing to the beginning of the empty part of
826 	 * mslots, and the ">=" would move the hole backwards in this
827 	 * case---which is wrong.  So skip the loop when deleting a slot.
828 	 */
829 	if (new->npages) {
830 		while (i > 0 &&
831 		       new->base_gfn >= mslots[i - 1].base_gfn) {
832 			mslots[i] = mslots[i - 1];
833 			slots->id_to_index[mslots[i].id] = i;
834 			i--;
835 		}
836 	} else
837 		WARN_ON_ONCE(i != slots->used_slots);
838 
839 	mslots[i] = *new;
840 	slots->id_to_index[mslots[i].id] = i;
841 }
842 
843 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
844 {
845 	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
846 
847 #ifdef __KVM_HAVE_READONLY_MEM
848 	valid_flags |= KVM_MEM_READONLY;
849 #endif
850 
851 	if (mem->flags & ~valid_flags)
852 		return -EINVAL;
853 
854 	return 0;
855 }
856 
857 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
858 		int as_id, struct kvm_memslots *slots)
859 {
860 	struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
861 
862 	/*
863 	 * Set the low bit in the generation, which disables SPTE caching
864 	 * until the end of synchronize_srcu_expedited.
865 	 */
866 	WARN_ON(old_memslots->generation & 1);
867 	slots->generation = old_memslots->generation + 1;
868 
869 	rcu_assign_pointer(kvm->memslots[as_id], slots);
870 	synchronize_srcu_expedited(&kvm->srcu);
871 
872 	/*
873 	 * Increment the new memslot generation a second time. This prevents
874 	 * vm exits that race with memslot updates from caching a memslot
875 	 * generation that will (potentially) be valid forever.
876 	 *
877 	 * Generations must be unique even across address spaces.  We do not need
878 	 * a global counter for that, instead the generation space is evenly split
879 	 * across address spaces.  For example, with two address spaces, address
880 	 * space 0 will use generations 0, 4, 8, ... while * address space 1 will
881 	 * use generations 2, 6, 10, 14, ...
882 	 */
883 	slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1;
884 
885 	kvm_arch_memslots_updated(kvm, slots);
886 
887 	return old_memslots;
888 }
889 
890 /*
891  * Allocate some memory and give it an address in the guest physical address
892  * space.
893  *
894  * Discontiguous memory is allowed, mostly for framebuffers.
895  *
896  * Must be called holding kvm->slots_lock for write.
897  */
898 int __kvm_set_memory_region(struct kvm *kvm,
899 			    const struct kvm_userspace_memory_region *mem)
900 {
901 	int r;
902 	gfn_t base_gfn;
903 	unsigned long npages;
904 	struct kvm_memory_slot *slot;
905 	struct kvm_memory_slot old, new;
906 	struct kvm_memslots *slots = NULL, *old_memslots;
907 	int as_id, id;
908 	enum kvm_mr_change change;
909 
910 	r = check_memory_region_flags(mem);
911 	if (r)
912 		goto out;
913 
914 	r = -EINVAL;
915 	as_id = mem->slot >> 16;
916 	id = (u16)mem->slot;
917 
918 	/* General sanity checks */
919 	if (mem->memory_size & (PAGE_SIZE - 1))
920 		goto out;
921 	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
922 		goto out;
923 	/* We can read the guest memory with __xxx_user() later on. */
924 	if ((id < KVM_USER_MEM_SLOTS) &&
925 	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
926 	     !access_ok(VERIFY_WRITE,
927 			(void __user *)(unsigned long)mem->userspace_addr,
928 			mem->memory_size)))
929 		goto out;
930 	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
931 		goto out;
932 	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
933 		goto out;
934 
935 	slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
936 	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
937 	npages = mem->memory_size >> PAGE_SHIFT;
938 
939 	if (npages > KVM_MEM_MAX_NR_PAGES)
940 		goto out;
941 
942 	new = old = *slot;
943 
944 	new.id = id;
945 	new.base_gfn = base_gfn;
946 	new.npages = npages;
947 	new.flags = mem->flags;
948 
949 	if (npages) {
950 		if (!old.npages)
951 			change = KVM_MR_CREATE;
952 		else { /* Modify an existing slot. */
953 			if ((mem->userspace_addr != old.userspace_addr) ||
954 			    (npages != old.npages) ||
955 			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
956 				goto out;
957 
958 			if (base_gfn != old.base_gfn)
959 				change = KVM_MR_MOVE;
960 			else if (new.flags != old.flags)
961 				change = KVM_MR_FLAGS_ONLY;
962 			else { /* Nothing to change. */
963 				r = 0;
964 				goto out;
965 			}
966 		}
967 	} else {
968 		if (!old.npages)
969 			goto out;
970 
971 		change = KVM_MR_DELETE;
972 		new.base_gfn = 0;
973 		new.flags = 0;
974 	}
975 
976 	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
977 		/* Check for overlaps */
978 		r = -EEXIST;
979 		kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
980 			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
981 			    (slot->id == id))
982 				continue;
983 			if (!((base_gfn + npages <= slot->base_gfn) ||
984 			      (base_gfn >= slot->base_gfn + slot->npages)))
985 				goto out;
986 		}
987 	}
988 
989 	/* Free page dirty bitmap if unneeded */
990 	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
991 		new.dirty_bitmap = NULL;
992 
993 	r = -ENOMEM;
994 	if (change == KVM_MR_CREATE) {
995 		new.userspace_addr = mem->userspace_addr;
996 
997 		if (kvm_arch_create_memslot(kvm, &new, npages))
998 			goto out_free;
999 	}
1000 
1001 	/* Allocate page dirty bitmap if needed */
1002 	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
1003 		if (kvm_create_dirty_bitmap(&new) < 0)
1004 			goto out_free;
1005 	}
1006 
1007 	slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
1008 	if (!slots)
1009 		goto out_free;
1010 	memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
1011 
1012 	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
1013 		slot = id_to_memslot(slots, id);
1014 		slot->flags |= KVM_MEMSLOT_INVALID;
1015 
1016 		old_memslots = install_new_memslots(kvm, as_id, slots);
1017 
1018 		/* slot was deleted or moved, clear iommu mapping */
1019 		kvm_iommu_unmap_pages(kvm, &old);
1020 		/* From this point no new shadow pages pointing to a deleted,
1021 		 * or moved, memslot will be created.
1022 		 *
1023 		 * validation of sp->gfn happens in:
1024 		 *	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1025 		 *	- kvm_is_visible_gfn (mmu_check_roots)
1026 		 */
1027 		kvm_arch_flush_shadow_memslot(kvm, slot);
1028 
1029 		/*
1030 		 * We can re-use the old_memslots from above, the only difference
1031 		 * from the currently installed memslots is the invalid flag.  This
1032 		 * will get overwritten by update_memslots anyway.
1033 		 */
1034 		slots = old_memslots;
1035 	}
1036 
1037 	r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1038 	if (r)
1039 		goto out_slots;
1040 
1041 	/* actual memory is freed via old in kvm_free_memslot below */
1042 	if (change == KVM_MR_DELETE) {
1043 		new.dirty_bitmap = NULL;
1044 		memset(&new.arch, 0, sizeof(new.arch));
1045 	}
1046 
1047 	update_memslots(slots, &new);
1048 	old_memslots = install_new_memslots(kvm, as_id, slots);
1049 
1050 	kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1051 
1052 	kvm_free_memslot(kvm, &old, &new);
1053 	kvfree(old_memslots);
1054 
1055 	/*
1056 	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
1057 	 * un-mapped and re-mapped if their base changes.  Since base change
1058 	 * unmapping is handled above with slot deletion, mapping alone is
1059 	 * needed here.  Anything else the iommu might care about for existing
1060 	 * slots (size changes, userspace addr changes and read-only flag
1061 	 * changes) is disallowed above, so any other attribute changes getting
1062 	 * here can be skipped.
1063 	 */
1064 	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1065 		r = kvm_iommu_map_pages(kvm, &new);
1066 		return r;
1067 	}
1068 
1069 	return 0;
1070 
1071 out_slots:
1072 	kvfree(slots);
1073 out_free:
1074 	kvm_free_memslot(kvm, &new, &old);
1075 out:
1076 	return r;
1077 }
1078 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1079 
1080 int kvm_set_memory_region(struct kvm *kvm,
1081 			  const struct kvm_userspace_memory_region *mem)
1082 {
1083 	int r;
1084 
1085 	mutex_lock(&kvm->slots_lock);
1086 	r = __kvm_set_memory_region(kvm, mem);
1087 	mutex_unlock(&kvm->slots_lock);
1088 	return r;
1089 }
1090 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1091 
1092 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1093 					  struct kvm_userspace_memory_region *mem)
1094 {
1095 	if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1096 		return -EINVAL;
1097 
1098 	return kvm_set_memory_region(kvm, mem);
1099 }
1100 
1101 int kvm_get_dirty_log(struct kvm *kvm,
1102 			struct kvm_dirty_log *log, int *is_dirty)
1103 {
1104 	struct kvm_memslots *slots;
1105 	struct kvm_memory_slot *memslot;
1106 	int i, as_id, id;
1107 	unsigned long n;
1108 	unsigned long any = 0;
1109 
1110 	as_id = log->slot >> 16;
1111 	id = (u16)log->slot;
1112 	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1113 		return -EINVAL;
1114 
1115 	slots = __kvm_memslots(kvm, as_id);
1116 	memslot = id_to_memslot(slots, id);
1117 	if (!memslot->dirty_bitmap)
1118 		return -ENOENT;
1119 
1120 	n = kvm_dirty_bitmap_bytes(memslot);
1121 
1122 	for (i = 0; !any && i < n/sizeof(long); ++i)
1123 		any = memslot->dirty_bitmap[i];
1124 
1125 	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1126 		return -EFAULT;
1127 
1128 	if (any)
1129 		*is_dirty = 1;
1130 	return 0;
1131 }
1132 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1133 
1134 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1135 /**
1136  * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1137  *	are dirty write protect them for next write.
1138  * @kvm:	pointer to kvm instance
1139  * @log:	slot id and address to which we copy the log
1140  * @is_dirty:	flag set if any page is dirty
1141  *
1142  * We need to keep it in mind that VCPU threads can write to the bitmap
1143  * concurrently. So, to avoid losing track of dirty pages we keep the
1144  * following order:
1145  *
1146  *    1. Take a snapshot of the bit and clear it if needed.
1147  *    2. Write protect the corresponding page.
1148  *    3. Copy the snapshot to the userspace.
1149  *    4. Upon return caller flushes TLB's if needed.
1150  *
1151  * Between 2 and 4, the guest may write to the page using the remaining TLB
1152  * entry.  This is not a problem because the page is reported dirty using
1153  * the snapshot taken before and step 4 ensures that writes done after
1154  * exiting to userspace will be logged for the next call.
1155  *
1156  */
1157 int kvm_get_dirty_log_protect(struct kvm *kvm,
1158 			struct kvm_dirty_log *log, bool *is_dirty)
1159 {
1160 	struct kvm_memslots *slots;
1161 	struct kvm_memory_slot *memslot;
1162 	int i, as_id, id;
1163 	unsigned long n;
1164 	unsigned long *dirty_bitmap;
1165 	unsigned long *dirty_bitmap_buffer;
1166 
1167 	as_id = log->slot >> 16;
1168 	id = (u16)log->slot;
1169 	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1170 		return -EINVAL;
1171 
1172 	slots = __kvm_memslots(kvm, as_id);
1173 	memslot = id_to_memslot(slots, id);
1174 
1175 	dirty_bitmap = memslot->dirty_bitmap;
1176 	if (!dirty_bitmap)
1177 		return -ENOENT;
1178 
1179 	n = kvm_dirty_bitmap_bytes(memslot);
1180 
1181 	dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
1182 	memset(dirty_bitmap_buffer, 0, n);
1183 
1184 	spin_lock(&kvm->mmu_lock);
1185 	*is_dirty = false;
1186 	for (i = 0; i < n / sizeof(long); i++) {
1187 		unsigned long mask;
1188 		gfn_t offset;
1189 
1190 		if (!dirty_bitmap[i])
1191 			continue;
1192 
1193 		*is_dirty = true;
1194 
1195 		mask = xchg(&dirty_bitmap[i], 0);
1196 		dirty_bitmap_buffer[i] = mask;
1197 
1198 		if (mask) {
1199 			offset = i * BITS_PER_LONG;
1200 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1201 								offset, mask);
1202 		}
1203 	}
1204 
1205 	spin_unlock(&kvm->mmu_lock);
1206 	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1207 		return -EFAULT;
1208 	return 0;
1209 }
1210 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1211 #endif
1212 
1213 bool kvm_largepages_enabled(void)
1214 {
1215 	return largepages_enabled;
1216 }
1217 
1218 void kvm_disable_largepages(void)
1219 {
1220 	largepages_enabled = false;
1221 }
1222 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1223 
1224 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1225 {
1226 	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1227 }
1228 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1229 
1230 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1231 {
1232 	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1233 }
1234 
1235 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1236 {
1237 	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1238 
1239 	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1240 	      memslot->flags & KVM_MEMSLOT_INVALID)
1241 		return false;
1242 
1243 	return true;
1244 }
1245 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1246 
1247 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1248 {
1249 	struct vm_area_struct *vma;
1250 	unsigned long addr, size;
1251 
1252 	size = PAGE_SIZE;
1253 
1254 	addr = gfn_to_hva(kvm, gfn);
1255 	if (kvm_is_error_hva(addr))
1256 		return PAGE_SIZE;
1257 
1258 	down_read(&current->mm->mmap_sem);
1259 	vma = find_vma(current->mm, addr);
1260 	if (!vma)
1261 		goto out;
1262 
1263 	size = vma_kernel_pagesize(vma);
1264 
1265 out:
1266 	up_read(&current->mm->mmap_sem);
1267 
1268 	return size;
1269 }
1270 
1271 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1272 {
1273 	return slot->flags & KVM_MEM_READONLY;
1274 }
1275 
1276 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1277 				       gfn_t *nr_pages, bool write)
1278 {
1279 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1280 		return KVM_HVA_ERR_BAD;
1281 
1282 	if (memslot_is_readonly(slot) && write)
1283 		return KVM_HVA_ERR_RO_BAD;
1284 
1285 	if (nr_pages)
1286 		*nr_pages = slot->npages - (gfn - slot->base_gfn);
1287 
1288 	return __gfn_to_hva_memslot(slot, gfn);
1289 }
1290 
1291 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1292 				     gfn_t *nr_pages)
1293 {
1294 	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1295 }
1296 
1297 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1298 					gfn_t gfn)
1299 {
1300 	return gfn_to_hva_many(slot, gfn, NULL);
1301 }
1302 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1303 
1304 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1305 {
1306 	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1307 }
1308 EXPORT_SYMBOL_GPL(gfn_to_hva);
1309 
1310 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1311 {
1312 	return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1313 }
1314 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1315 
1316 /*
1317  * If writable is set to false, the hva returned by this function is only
1318  * allowed to be read.
1319  */
1320 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1321 				      gfn_t gfn, bool *writable)
1322 {
1323 	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1324 
1325 	if (!kvm_is_error_hva(hva) && writable)
1326 		*writable = !memslot_is_readonly(slot);
1327 
1328 	return hva;
1329 }
1330 
1331 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1332 {
1333 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1334 
1335 	return gfn_to_hva_memslot_prot(slot, gfn, writable);
1336 }
1337 
1338 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1339 {
1340 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1341 
1342 	return gfn_to_hva_memslot_prot(slot, gfn, writable);
1343 }
1344 
1345 static int get_user_page_nowait(unsigned long start, int write,
1346 		struct page **page)
1347 {
1348 	int flags = FOLL_NOWAIT | FOLL_HWPOISON;
1349 
1350 	if (write)
1351 		flags |= FOLL_WRITE;
1352 
1353 	return get_user_pages(start, 1, flags, page, NULL);
1354 }
1355 
1356 static inline int check_user_page_hwpoison(unsigned long addr)
1357 {
1358 	int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1359 
1360 	rc = get_user_pages(addr, 1, flags, NULL, NULL);
1361 	return rc == -EHWPOISON;
1362 }
1363 
1364 /*
1365  * The atomic path to get the writable pfn which will be stored in @pfn,
1366  * true indicates success, otherwise false is returned.
1367  */
1368 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1369 			    bool write_fault, bool *writable, kvm_pfn_t *pfn)
1370 {
1371 	struct page *page[1];
1372 	int npages;
1373 
1374 	if (!(async || atomic))
1375 		return false;
1376 
1377 	/*
1378 	 * Fast pin a writable pfn only if it is a write fault request
1379 	 * or the caller allows to map a writable pfn for a read fault
1380 	 * request.
1381 	 */
1382 	if (!(write_fault || writable))
1383 		return false;
1384 
1385 	npages = __get_user_pages_fast(addr, 1, 1, page);
1386 	if (npages == 1) {
1387 		*pfn = page_to_pfn(page[0]);
1388 
1389 		if (writable)
1390 			*writable = true;
1391 		return true;
1392 	}
1393 
1394 	return false;
1395 }
1396 
1397 /*
1398  * The slow path to get the pfn of the specified host virtual address,
1399  * 1 indicates success, -errno is returned if error is detected.
1400  */
1401 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1402 			   bool *writable, kvm_pfn_t *pfn)
1403 {
1404 	struct page *page[1];
1405 	int npages = 0;
1406 
1407 	might_sleep();
1408 
1409 	if (writable)
1410 		*writable = write_fault;
1411 
1412 	if (async) {
1413 		down_read(&current->mm->mmap_sem);
1414 		npages = get_user_page_nowait(addr, write_fault, page);
1415 		up_read(&current->mm->mmap_sem);
1416 	} else {
1417 		unsigned int flags = FOLL_HWPOISON;
1418 
1419 		if (write_fault)
1420 			flags |= FOLL_WRITE;
1421 
1422 		npages = get_user_pages_unlocked(addr, 1, page, flags);
1423 	}
1424 	if (npages != 1)
1425 		return npages;
1426 
1427 	/* map read fault as writable if possible */
1428 	if (unlikely(!write_fault) && writable) {
1429 		struct page *wpage[1];
1430 
1431 		npages = __get_user_pages_fast(addr, 1, 1, wpage);
1432 		if (npages == 1) {
1433 			*writable = true;
1434 			put_page(page[0]);
1435 			page[0] = wpage[0];
1436 		}
1437 
1438 		npages = 1;
1439 	}
1440 	*pfn = page_to_pfn(page[0]);
1441 	return npages;
1442 }
1443 
1444 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1445 {
1446 	if (unlikely(!(vma->vm_flags & VM_READ)))
1447 		return false;
1448 
1449 	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1450 		return false;
1451 
1452 	return true;
1453 }
1454 
1455 static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1456 			       unsigned long addr, bool *async,
1457 			       bool write_fault, kvm_pfn_t *p_pfn)
1458 {
1459 	unsigned long pfn;
1460 	int r;
1461 
1462 	r = follow_pfn(vma, addr, &pfn);
1463 	if (r) {
1464 		/*
1465 		 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1466 		 * not call the fault handler, so do it here.
1467 		 */
1468 		bool unlocked = false;
1469 		r = fixup_user_fault(current, current->mm, addr,
1470 				     (write_fault ? FAULT_FLAG_WRITE : 0),
1471 				     &unlocked);
1472 		if (unlocked)
1473 			return -EAGAIN;
1474 		if (r)
1475 			return r;
1476 
1477 		r = follow_pfn(vma, addr, &pfn);
1478 		if (r)
1479 			return r;
1480 
1481 	}
1482 
1483 
1484 	/*
1485 	 * Get a reference here because callers of *hva_to_pfn* and
1486 	 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1487 	 * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
1488 	 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1489 	 * simply do nothing for reserved pfns.
1490 	 *
1491 	 * Whoever called remap_pfn_range is also going to call e.g.
1492 	 * unmap_mapping_range before the underlying pages are freed,
1493 	 * causing a call to our MMU notifier.
1494 	 */
1495 	kvm_get_pfn(pfn);
1496 
1497 	*p_pfn = pfn;
1498 	return 0;
1499 }
1500 
1501 /*
1502  * Pin guest page in memory and return its pfn.
1503  * @addr: host virtual address which maps memory to the guest
1504  * @atomic: whether this function can sleep
1505  * @async: whether this function need to wait IO complete if the
1506  *         host page is not in the memory
1507  * @write_fault: whether we should get a writable host page
1508  * @writable: whether it allows to map a writable host page for !@write_fault
1509  *
1510  * The function will map a writable host page for these two cases:
1511  * 1): @write_fault = true
1512  * 2): @write_fault = false && @writable, @writable will tell the caller
1513  *     whether the mapping is writable.
1514  */
1515 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1516 			bool write_fault, bool *writable)
1517 {
1518 	struct vm_area_struct *vma;
1519 	kvm_pfn_t pfn = 0;
1520 	int npages, r;
1521 
1522 	/* we can do it either atomically or asynchronously, not both */
1523 	BUG_ON(atomic && async);
1524 
1525 	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1526 		return pfn;
1527 
1528 	if (atomic)
1529 		return KVM_PFN_ERR_FAULT;
1530 
1531 	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1532 	if (npages == 1)
1533 		return pfn;
1534 
1535 	down_read(&current->mm->mmap_sem);
1536 	if (npages == -EHWPOISON ||
1537 	      (!async && check_user_page_hwpoison(addr))) {
1538 		pfn = KVM_PFN_ERR_HWPOISON;
1539 		goto exit;
1540 	}
1541 
1542 retry:
1543 	vma = find_vma_intersection(current->mm, addr, addr + 1);
1544 
1545 	if (vma == NULL)
1546 		pfn = KVM_PFN_ERR_FAULT;
1547 	else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1548 		r = hva_to_pfn_remapped(vma, addr, async, write_fault, &pfn);
1549 		if (r == -EAGAIN)
1550 			goto retry;
1551 		if (r < 0)
1552 			pfn = KVM_PFN_ERR_FAULT;
1553 	} else {
1554 		if (async && vma_is_valid(vma, write_fault))
1555 			*async = true;
1556 		pfn = KVM_PFN_ERR_FAULT;
1557 	}
1558 exit:
1559 	up_read(&current->mm->mmap_sem);
1560 	return pfn;
1561 }
1562 
1563 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1564 			       bool atomic, bool *async, bool write_fault,
1565 			       bool *writable)
1566 {
1567 	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1568 
1569 	if (addr == KVM_HVA_ERR_RO_BAD) {
1570 		if (writable)
1571 			*writable = false;
1572 		return KVM_PFN_ERR_RO_FAULT;
1573 	}
1574 
1575 	if (kvm_is_error_hva(addr)) {
1576 		if (writable)
1577 			*writable = false;
1578 		return KVM_PFN_NOSLOT;
1579 	}
1580 
1581 	/* Do not map writable pfn in the readonly memslot. */
1582 	if (writable && memslot_is_readonly(slot)) {
1583 		*writable = false;
1584 		writable = NULL;
1585 	}
1586 
1587 	return hva_to_pfn(addr, atomic, async, write_fault,
1588 			  writable);
1589 }
1590 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1591 
1592 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1593 		      bool *writable)
1594 {
1595 	return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1596 				    write_fault, writable);
1597 }
1598 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1599 
1600 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1601 {
1602 	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1603 }
1604 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1605 
1606 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1607 {
1608 	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1609 }
1610 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1611 
1612 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1613 {
1614 	return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1615 }
1616 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1617 
1618 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1619 {
1620 	return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1621 }
1622 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1623 
1624 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1625 {
1626 	return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1627 }
1628 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1629 
1630 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1631 {
1632 	return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1633 }
1634 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1635 
1636 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1637 			    struct page **pages, int nr_pages)
1638 {
1639 	unsigned long addr;
1640 	gfn_t entry;
1641 
1642 	addr = gfn_to_hva_many(slot, gfn, &entry);
1643 	if (kvm_is_error_hva(addr))
1644 		return -1;
1645 
1646 	if (entry < nr_pages)
1647 		return 0;
1648 
1649 	return __get_user_pages_fast(addr, nr_pages, 1, pages);
1650 }
1651 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1652 
1653 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
1654 {
1655 	if (is_error_noslot_pfn(pfn))
1656 		return KVM_ERR_PTR_BAD_PAGE;
1657 
1658 	if (kvm_is_reserved_pfn(pfn)) {
1659 		WARN_ON(1);
1660 		return KVM_ERR_PTR_BAD_PAGE;
1661 	}
1662 
1663 	return pfn_to_page(pfn);
1664 }
1665 
1666 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1667 {
1668 	kvm_pfn_t pfn;
1669 
1670 	pfn = gfn_to_pfn(kvm, gfn);
1671 
1672 	return kvm_pfn_to_page(pfn);
1673 }
1674 EXPORT_SYMBOL_GPL(gfn_to_page);
1675 
1676 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1677 {
1678 	kvm_pfn_t pfn;
1679 
1680 	pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1681 
1682 	return kvm_pfn_to_page(pfn);
1683 }
1684 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1685 
1686 void kvm_release_page_clean(struct page *page)
1687 {
1688 	WARN_ON(is_error_page(page));
1689 
1690 	kvm_release_pfn_clean(page_to_pfn(page));
1691 }
1692 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1693 
1694 void kvm_release_pfn_clean(kvm_pfn_t pfn)
1695 {
1696 	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1697 		put_page(pfn_to_page(pfn));
1698 }
1699 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1700 
1701 void kvm_release_page_dirty(struct page *page)
1702 {
1703 	WARN_ON(is_error_page(page));
1704 
1705 	kvm_release_pfn_dirty(page_to_pfn(page));
1706 }
1707 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1708 
1709 static void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1710 {
1711 	kvm_set_pfn_dirty(pfn);
1712 	kvm_release_pfn_clean(pfn);
1713 }
1714 
1715 void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1716 {
1717 	if (!kvm_is_reserved_pfn(pfn)) {
1718 		struct page *page = pfn_to_page(pfn);
1719 
1720 		if (!PageReserved(page))
1721 			SetPageDirty(page);
1722 	}
1723 }
1724 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1725 
1726 void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1727 {
1728 	if (!kvm_is_reserved_pfn(pfn))
1729 		mark_page_accessed(pfn_to_page(pfn));
1730 }
1731 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1732 
1733 void kvm_get_pfn(kvm_pfn_t pfn)
1734 {
1735 	if (!kvm_is_reserved_pfn(pfn))
1736 		get_page(pfn_to_page(pfn));
1737 }
1738 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1739 
1740 static int next_segment(unsigned long len, int offset)
1741 {
1742 	if (len > PAGE_SIZE - offset)
1743 		return PAGE_SIZE - offset;
1744 	else
1745 		return len;
1746 }
1747 
1748 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1749 				 void *data, int offset, int len)
1750 {
1751 	int r;
1752 	unsigned long addr;
1753 
1754 	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1755 	if (kvm_is_error_hva(addr))
1756 		return -EFAULT;
1757 	r = __copy_from_user(data, (void __user *)addr + offset, len);
1758 	if (r)
1759 		return -EFAULT;
1760 	return 0;
1761 }
1762 
1763 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1764 			int len)
1765 {
1766 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1767 
1768 	return __kvm_read_guest_page(slot, gfn, data, offset, len);
1769 }
1770 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1771 
1772 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1773 			     int offset, int len)
1774 {
1775 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1776 
1777 	return __kvm_read_guest_page(slot, gfn, data, offset, len);
1778 }
1779 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1780 
1781 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1782 {
1783 	gfn_t gfn = gpa >> PAGE_SHIFT;
1784 	int seg;
1785 	int offset = offset_in_page(gpa);
1786 	int ret;
1787 
1788 	while ((seg = next_segment(len, offset)) != 0) {
1789 		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1790 		if (ret < 0)
1791 			return ret;
1792 		offset = 0;
1793 		len -= seg;
1794 		data += seg;
1795 		++gfn;
1796 	}
1797 	return 0;
1798 }
1799 EXPORT_SYMBOL_GPL(kvm_read_guest);
1800 
1801 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1802 {
1803 	gfn_t gfn = gpa >> PAGE_SHIFT;
1804 	int seg;
1805 	int offset = offset_in_page(gpa);
1806 	int ret;
1807 
1808 	while ((seg = next_segment(len, offset)) != 0) {
1809 		ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1810 		if (ret < 0)
1811 			return ret;
1812 		offset = 0;
1813 		len -= seg;
1814 		data += seg;
1815 		++gfn;
1816 	}
1817 	return 0;
1818 }
1819 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1820 
1821 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1822 			           void *data, int offset, unsigned long len)
1823 {
1824 	int r;
1825 	unsigned long addr;
1826 
1827 	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1828 	if (kvm_is_error_hva(addr))
1829 		return -EFAULT;
1830 	pagefault_disable();
1831 	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1832 	pagefault_enable();
1833 	if (r)
1834 		return -EFAULT;
1835 	return 0;
1836 }
1837 
1838 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1839 			  unsigned long len)
1840 {
1841 	gfn_t gfn = gpa >> PAGE_SHIFT;
1842 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1843 	int offset = offset_in_page(gpa);
1844 
1845 	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1846 }
1847 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1848 
1849 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1850 			       void *data, unsigned long len)
1851 {
1852 	gfn_t gfn = gpa >> PAGE_SHIFT;
1853 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1854 	int offset = offset_in_page(gpa);
1855 
1856 	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1857 }
1858 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1859 
1860 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1861 			          const void *data, int offset, int len)
1862 {
1863 	int r;
1864 	unsigned long addr;
1865 
1866 	addr = gfn_to_hva_memslot(memslot, gfn);
1867 	if (kvm_is_error_hva(addr))
1868 		return -EFAULT;
1869 	r = __copy_to_user((void __user *)addr + offset, data, len);
1870 	if (r)
1871 		return -EFAULT;
1872 	mark_page_dirty_in_slot(memslot, gfn);
1873 	return 0;
1874 }
1875 
1876 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1877 			 const void *data, int offset, int len)
1878 {
1879 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1880 
1881 	return __kvm_write_guest_page(slot, gfn, data, offset, len);
1882 }
1883 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1884 
1885 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1886 			      const void *data, int offset, int len)
1887 {
1888 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1889 
1890 	return __kvm_write_guest_page(slot, gfn, data, offset, len);
1891 }
1892 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1893 
1894 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1895 		    unsigned long len)
1896 {
1897 	gfn_t gfn = gpa >> PAGE_SHIFT;
1898 	int seg;
1899 	int offset = offset_in_page(gpa);
1900 	int ret;
1901 
1902 	while ((seg = next_segment(len, offset)) != 0) {
1903 		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1904 		if (ret < 0)
1905 			return ret;
1906 		offset = 0;
1907 		len -= seg;
1908 		data += seg;
1909 		++gfn;
1910 	}
1911 	return 0;
1912 }
1913 EXPORT_SYMBOL_GPL(kvm_write_guest);
1914 
1915 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1916 		         unsigned long len)
1917 {
1918 	gfn_t gfn = gpa >> PAGE_SHIFT;
1919 	int seg;
1920 	int offset = offset_in_page(gpa);
1921 	int ret;
1922 
1923 	while ((seg = next_segment(len, offset)) != 0) {
1924 		ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1925 		if (ret < 0)
1926 			return ret;
1927 		offset = 0;
1928 		len -= seg;
1929 		data += seg;
1930 		++gfn;
1931 	}
1932 	return 0;
1933 }
1934 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
1935 
1936 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
1937 				       struct gfn_to_hva_cache *ghc,
1938 				       gpa_t gpa, unsigned long len)
1939 {
1940 	int offset = offset_in_page(gpa);
1941 	gfn_t start_gfn = gpa >> PAGE_SHIFT;
1942 	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1943 	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1944 	gfn_t nr_pages_avail;
1945 
1946 	ghc->gpa = gpa;
1947 	ghc->generation = slots->generation;
1948 	ghc->len = len;
1949 	ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1950 	ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1951 	if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1952 		ghc->hva += offset;
1953 	} else {
1954 		/*
1955 		 * If the requested region crosses two memslots, we still
1956 		 * verify that the entire region is valid here.
1957 		 */
1958 		while (start_gfn <= end_gfn) {
1959 			ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1960 			ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1961 						   &nr_pages_avail);
1962 			if (kvm_is_error_hva(ghc->hva))
1963 				return -EFAULT;
1964 			start_gfn += nr_pages_avail;
1965 		}
1966 		/* Use the slow path for cross page reads and writes. */
1967 		ghc->memslot = NULL;
1968 	}
1969 	return 0;
1970 }
1971 
1972 int kvm_vcpu_gfn_to_hva_cache_init(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc,
1973 			      gpa_t gpa, unsigned long len)
1974 {
1975 	struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
1976 	return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
1977 }
1978 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva_cache_init);
1979 
1980 int kvm_vcpu_write_guest_offset_cached(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc,
1981 				       void *data, int offset, unsigned long len)
1982 {
1983 	struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
1984 	int r;
1985 	gpa_t gpa = ghc->gpa + offset;
1986 
1987 	BUG_ON(len + offset > ghc->len);
1988 
1989 	if (slots->generation != ghc->generation)
1990 		__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
1991 
1992 	if (unlikely(!ghc->memslot))
1993 		return kvm_vcpu_write_guest(vcpu, gpa, data, len);
1994 
1995 	if (kvm_is_error_hva(ghc->hva))
1996 		return -EFAULT;
1997 
1998 	r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
1999 	if (r)
2000 		return -EFAULT;
2001 	mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
2002 
2003 	return 0;
2004 }
2005 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_offset_cached);
2006 
2007 int kvm_vcpu_write_guest_cached(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc,
2008 			       void *data, unsigned long len)
2009 {
2010 	return kvm_vcpu_write_guest_offset_cached(vcpu, ghc, data, 0, len);
2011 }
2012 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_cached);
2013 
2014 int kvm_vcpu_read_guest_cached(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc,
2015 			       void *data, unsigned long len)
2016 {
2017 	struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
2018 	int r;
2019 
2020 	BUG_ON(len > ghc->len);
2021 
2022 	if (slots->generation != ghc->generation)
2023 		__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2024 
2025 	if (unlikely(!ghc->memslot))
2026 		return kvm_vcpu_read_guest(vcpu, ghc->gpa, data, len);
2027 
2028 	if (kvm_is_error_hva(ghc->hva))
2029 		return -EFAULT;
2030 
2031 	r = __copy_from_user(data, (void __user *)ghc->hva, len);
2032 	if (r)
2033 		return -EFAULT;
2034 
2035 	return 0;
2036 }
2037 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_cached);
2038 
2039 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2040 {
2041 	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2042 
2043 	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2044 }
2045 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2046 
2047 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2048 {
2049 	gfn_t gfn = gpa >> PAGE_SHIFT;
2050 	int seg;
2051 	int offset = offset_in_page(gpa);
2052 	int ret;
2053 
2054 	while ((seg = next_segment(len, offset)) != 0) {
2055 		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2056 		if (ret < 0)
2057 			return ret;
2058 		offset = 0;
2059 		len -= seg;
2060 		++gfn;
2061 	}
2062 	return 0;
2063 }
2064 EXPORT_SYMBOL_GPL(kvm_clear_guest);
2065 
2066 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2067 				    gfn_t gfn)
2068 {
2069 	if (memslot && memslot->dirty_bitmap) {
2070 		unsigned long rel_gfn = gfn - memslot->base_gfn;
2071 
2072 		set_bit_le(rel_gfn, memslot->dirty_bitmap);
2073 	}
2074 }
2075 
2076 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2077 {
2078 	struct kvm_memory_slot *memslot;
2079 
2080 	memslot = gfn_to_memslot(kvm, gfn);
2081 	mark_page_dirty_in_slot(memslot, gfn);
2082 }
2083 EXPORT_SYMBOL_GPL(mark_page_dirty);
2084 
2085 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2086 {
2087 	struct kvm_memory_slot *memslot;
2088 
2089 	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2090 	mark_page_dirty_in_slot(memslot, gfn);
2091 }
2092 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2093 
2094 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2095 {
2096 	unsigned int old, val, grow;
2097 
2098 	old = val = vcpu->halt_poll_ns;
2099 	grow = READ_ONCE(halt_poll_ns_grow);
2100 	/* 10us base */
2101 	if (val == 0 && grow)
2102 		val = 10000;
2103 	else
2104 		val *= grow;
2105 
2106 	if (val > halt_poll_ns)
2107 		val = halt_poll_ns;
2108 
2109 	vcpu->halt_poll_ns = val;
2110 	trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
2111 }
2112 
2113 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2114 {
2115 	unsigned int old, val, shrink;
2116 
2117 	old = val = vcpu->halt_poll_ns;
2118 	shrink = READ_ONCE(halt_poll_ns_shrink);
2119 	if (shrink == 0)
2120 		val = 0;
2121 	else
2122 		val /= shrink;
2123 
2124 	vcpu->halt_poll_ns = val;
2125 	trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
2126 }
2127 
2128 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2129 {
2130 	if (kvm_arch_vcpu_runnable(vcpu)) {
2131 		kvm_make_request(KVM_REQ_UNHALT, vcpu);
2132 		return -EINTR;
2133 	}
2134 	if (kvm_cpu_has_pending_timer(vcpu))
2135 		return -EINTR;
2136 	if (signal_pending(current))
2137 		return -EINTR;
2138 
2139 	return 0;
2140 }
2141 
2142 /*
2143  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2144  */
2145 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2146 {
2147 	ktime_t start, cur;
2148 	DECLARE_SWAITQUEUE(wait);
2149 	bool waited = false;
2150 	u64 block_ns;
2151 
2152 	start = cur = ktime_get();
2153 	if (vcpu->halt_poll_ns) {
2154 		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2155 
2156 		++vcpu->stat.halt_attempted_poll;
2157 		do {
2158 			/*
2159 			 * This sets KVM_REQ_UNHALT if an interrupt
2160 			 * arrives.
2161 			 */
2162 			if (kvm_vcpu_check_block(vcpu) < 0) {
2163 				++vcpu->stat.halt_successful_poll;
2164 				if (!vcpu_valid_wakeup(vcpu))
2165 					++vcpu->stat.halt_poll_invalid;
2166 				goto out;
2167 			}
2168 			cur = ktime_get();
2169 		} while (single_task_running() && ktime_before(cur, stop));
2170 	}
2171 
2172 	kvm_arch_vcpu_blocking(vcpu);
2173 
2174 	for (;;) {
2175 		prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2176 
2177 		if (kvm_vcpu_check_block(vcpu) < 0)
2178 			break;
2179 
2180 		waited = true;
2181 		schedule();
2182 	}
2183 
2184 	finish_swait(&vcpu->wq, &wait);
2185 	cur = ktime_get();
2186 
2187 	kvm_arch_vcpu_unblocking(vcpu);
2188 out:
2189 	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2190 
2191 	if (!vcpu_valid_wakeup(vcpu))
2192 		shrink_halt_poll_ns(vcpu);
2193 	else if (halt_poll_ns) {
2194 		if (block_ns <= vcpu->halt_poll_ns)
2195 			;
2196 		/* we had a long block, shrink polling */
2197 		else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2198 			shrink_halt_poll_ns(vcpu);
2199 		/* we had a short halt and our poll time is too small */
2200 		else if (vcpu->halt_poll_ns < halt_poll_ns &&
2201 			block_ns < halt_poll_ns)
2202 			grow_halt_poll_ns(vcpu);
2203 	} else
2204 		vcpu->halt_poll_ns = 0;
2205 
2206 	trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2207 	kvm_arch_vcpu_block_finish(vcpu);
2208 }
2209 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2210 
2211 #ifndef CONFIG_S390
2212 void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2213 {
2214 	struct swait_queue_head *wqp;
2215 
2216 	wqp = kvm_arch_vcpu_wq(vcpu);
2217 	if (swait_active(wqp)) {
2218 		swake_up(wqp);
2219 		++vcpu->stat.halt_wakeup;
2220 	}
2221 
2222 }
2223 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2224 
2225 /*
2226  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2227  */
2228 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2229 {
2230 	int me;
2231 	int cpu = vcpu->cpu;
2232 
2233 	kvm_vcpu_wake_up(vcpu);
2234 	me = get_cpu();
2235 	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2236 		if (kvm_arch_vcpu_should_kick(vcpu))
2237 			smp_send_reschedule(cpu);
2238 	put_cpu();
2239 }
2240 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2241 #endif /* !CONFIG_S390 */
2242 
2243 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2244 {
2245 	struct pid *pid;
2246 	struct task_struct *task = NULL;
2247 	int ret = 0;
2248 
2249 	rcu_read_lock();
2250 	pid = rcu_dereference(target->pid);
2251 	if (pid)
2252 		task = get_pid_task(pid, PIDTYPE_PID);
2253 	rcu_read_unlock();
2254 	if (!task)
2255 		return ret;
2256 	ret = yield_to(task, 1);
2257 	put_task_struct(task);
2258 
2259 	return ret;
2260 }
2261 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2262 
2263 /*
2264  * Helper that checks whether a VCPU is eligible for directed yield.
2265  * Most eligible candidate to yield is decided by following heuristics:
2266  *
2267  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2268  *  (preempted lock holder), indicated by @in_spin_loop.
2269  *  Set at the beiginning and cleared at the end of interception/PLE handler.
2270  *
2271  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2272  *  chance last time (mostly it has become eligible now since we have probably
2273  *  yielded to lockholder in last iteration. This is done by toggling
2274  *  @dy_eligible each time a VCPU checked for eligibility.)
2275  *
2276  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2277  *  to preempted lock-holder could result in wrong VCPU selection and CPU
2278  *  burning. Giving priority for a potential lock-holder increases lock
2279  *  progress.
2280  *
2281  *  Since algorithm is based on heuristics, accessing another VCPU data without
2282  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
2283  *  and continue with next VCPU and so on.
2284  */
2285 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2286 {
2287 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2288 	bool eligible;
2289 
2290 	eligible = !vcpu->spin_loop.in_spin_loop ||
2291 		    vcpu->spin_loop.dy_eligible;
2292 
2293 	if (vcpu->spin_loop.in_spin_loop)
2294 		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2295 
2296 	return eligible;
2297 #else
2298 	return true;
2299 #endif
2300 }
2301 
2302 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
2303 {
2304 	struct kvm *kvm = me->kvm;
2305 	struct kvm_vcpu *vcpu;
2306 	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2307 	int yielded = 0;
2308 	int try = 3;
2309 	int pass;
2310 	int i;
2311 
2312 	kvm_vcpu_set_in_spin_loop(me, true);
2313 	/*
2314 	 * We boost the priority of a VCPU that is runnable but not
2315 	 * currently running, because it got preempted by something
2316 	 * else and called schedule in __vcpu_run.  Hopefully that
2317 	 * VCPU is holding the lock that we need and will release it.
2318 	 * We approximate round-robin by starting at the last boosted VCPU.
2319 	 */
2320 	for (pass = 0; pass < 2 && !yielded && try; pass++) {
2321 		kvm_for_each_vcpu(i, vcpu, kvm) {
2322 			if (!pass && i <= last_boosted_vcpu) {
2323 				i = last_boosted_vcpu;
2324 				continue;
2325 			} else if (pass && i > last_boosted_vcpu)
2326 				break;
2327 			if (!ACCESS_ONCE(vcpu->preempted))
2328 				continue;
2329 			if (vcpu == me)
2330 				continue;
2331 			if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2332 				continue;
2333 			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2334 				continue;
2335 
2336 			yielded = kvm_vcpu_yield_to(vcpu);
2337 			if (yielded > 0) {
2338 				kvm->last_boosted_vcpu = i;
2339 				break;
2340 			} else if (yielded < 0) {
2341 				try--;
2342 				if (!try)
2343 					break;
2344 			}
2345 		}
2346 	}
2347 	kvm_vcpu_set_in_spin_loop(me, false);
2348 
2349 	/* Ensure vcpu is not eligible during next spinloop */
2350 	kvm_vcpu_set_dy_eligible(me, false);
2351 }
2352 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2353 
2354 static int kvm_vcpu_fault(struct vm_fault *vmf)
2355 {
2356 	struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2357 	struct page *page;
2358 
2359 	if (vmf->pgoff == 0)
2360 		page = virt_to_page(vcpu->run);
2361 #ifdef CONFIG_X86
2362 	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2363 		page = virt_to_page(vcpu->arch.pio_data);
2364 #endif
2365 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2366 	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2367 		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2368 #endif
2369 	else
2370 		return kvm_arch_vcpu_fault(vcpu, vmf);
2371 	get_page(page);
2372 	vmf->page = page;
2373 	return 0;
2374 }
2375 
2376 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2377 	.fault = kvm_vcpu_fault,
2378 };
2379 
2380 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2381 {
2382 	vma->vm_ops = &kvm_vcpu_vm_ops;
2383 	return 0;
2384 }
2385 
2386 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2387 {
2388 	struct kvm_vcpu *vcpu = filp->private_data;
2389 
2390 	debugfs_remove_recursive(vcpu->debugfs_dentry);
2391 	kvm_put_kvm(vcpu->kvm);
2392 	return 0;
2393 }
2394 
2395 static struct file_operations kvm_vcpu_fops = {
2396 	.release        = kvm_vcpu_release,
2397 	.unlocked_ioctl = kvm_vcpu_ioctl,
2398 #ifdef CONFIG_KVM_COMPAT
2399 	.compat_ioctl   = kvm_vcpu_compat_ioctl,
2400 #endif
2401 	.mmap           = kvm_vcpu_mmap,
2402 	.llseek		= noop_llseek,
2403 };
2404 
2405 /*
2406  * Allocates an inode for the vcpu.
2407  */
2408 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2409 {
2410 	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2411 }
2412 
2413 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2414 {
2415 	char dir_name[ITOA_MAX_LEN * 2];
2416 	int ret;
2417 
2418 	if (!kvm_arch_has_vcpu_debugfs())
2419 		return 0;
2420 
2421 	if (!debugfs_initialized())
2422 		return 0;
2423 
2424 	snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
2425 	vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2426 								vcpu->kvm->debugfs_dentry);
2427 	if (!vcpu->debugfs_dentry)
2428 		return -ENOMEM;
2429 
2430 	ret = kvm_arch_create_vcpu_debugfs(vcpu);
2431 	if (ret < 0) {
2432 		debugfs_remove_recursive(vcpu->debugfs_dentry);
2433 		return ret;
2434 	}
2435 
2436 	return 0;
2437 }
2438 
2439 /*
2440  * Creates some virtual cpus.  Good luck creating more than one.
2441  */
2442 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2443 {
2444 	int r;
2445 	struct kvm_vcpu *vcpu;
2446 
2447 	if (id >= KVM_MAX_VCPU_ID)
2448 		return -EINVAL;
2449 
2450 	mutex_lock(&kvm->lock);
2451 	if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2452 		mutex_unlock(&kvm->lock);
2453 		return -EINVAL;
2454 	}
2455 
2456 	kvm->created_vcpus++;
2457 	mutex_unlock(&kvm->lock);
2458 
2459 	vcpu = kvm_arch_vcpu_create(kvm, id);
2460 	if (IS_ERR(vcpu)) {
2461 		r = PTR_ERR(vcpu);
2462 		goto vcpu_decrement;
2463 	}
2464 
2465 	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2466 
2467 	r = kvm_arch_vcpu_setup(vcpu);
2468 	if (r)
2469 		goto vcpu_destroy;
2470 
2471 	r = kvm_create_vcpu_debugfs(vcpu);
2472 	if (r)
2473 		goto vcpu_destroy;
2474 
2475 	mutex_lock(&kvm->lock);
2476 	if (kvm_get_vcpu_by_id(kvm, id)) {
2477 		r = -EEXIST;
2478 		goto unlock_vcpu_destroy;
2479 	}
2480 
2481 	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2482 
2483 	/* Now it's all set up, let userspace reach it */
2484 	kvm_get_kvm(kvm);
2485 	r = create_vcpu_fd(vcpu);
2486 	if (r < 0) {
2487 		kvm_put_kvm(kvm);
2488 		goto unlock_vcpu_destroy;
2489 	}
2490 
2491 	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2492 
2493 	/*
2494 	 * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
2495 	 * before kvm->online_vcpu's incremented value.
2496 	 */
2497 	smp_wmb();
2498 	atomic_inc(&kvm->online_vcpus);
2499 
2500 	mutex_unlock(&kvm->lock);
2501 	kvm_arch_vcpu_postcreate(vcpu);
2502 	return r;
2503 
2504 unlock_vcpu_destroy:
2505 	mutex_unlock(&kvm->lock);
2506 	debugfs_remove_recursive(vcpu->debugfs_dentry);
2507 vcpu_destroy:
2508 	kvm_arch_vcpu_destroy(vcpu);
2509 vcpu_decrement:
2510 	mutex_lock(&kvm->lock);
2511 	kvm->created_vcpus--;
2512 	mutex_unlock(&kvm->lock);
2513 	return r;
2514 }
2515 
2516 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2517 {
2518 	if (sigset) {
2519 		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2520 		vcpu->sigset_active = 1;
2521 		vcpu->sigset = *sigset;
2522 	} else
2523 		vcpu->sigset_active = 0;
2524 	return 0;
2525 }
2526 
2527 static long kvm_vcpu_ioctl(struct file *filp,
2528 			   unsigned int ioctl, unsigned long arg)
2529 {
2530 	struct kvm_vcpu *vcpu = filp->private_data;
2531 	void __user *argp = (void __user *)arg;
2532 	int r;
2533 	struct kvm_fpu *fpu = NULL;
2534 	struct kvm_sregs *kvm_sregs = NULL;
2535 
2536 	if (vcpu->kvm->mm != current->mm)
2537 		return -EIO;
2538 
2539 	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2540 		return -EINVAL;
2541 
2542 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2543 	/*
2544 	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
2545 	 * so vcpu_load() would break it.
2546 	 */
2547 	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
2548 		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2549 #endif
2550 
2551 
2552 	r = vcpu_load(vcpu);
2553 	if (r)
2554 		return r;
2555 	switch (ioctl) {
2556 	case KVM_RUN:
2557 		r = -EINVAL;
2558 		if (arg)
2559 			goto out;
2560 		if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
2561 			/* The thread running this VCPU changed. */
2562 			struct pid *oldpid = vcpu->pid;
2563 			struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
2564 
2565 			rcu_assign_pointer(vcpu->pid, newpid);
2566 			if (oldpid)
2567 				synchronize_rcu();
2568 			put_pid(oldpid);
2569 		}
2570 		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2571 		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2572 		break;
2573 	case KVM_GET_REGS: {
2574 		struct kvm_regs *kvm_regs;
2575 
2576 		r = -ENOMEM;
2577 		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2578 		if (!kvm_regs)
2579 			goto out;
2580 		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2581 		if (r)
2582 			goto out_free1;
2583 		r = -EFAULT;
2584 		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2585 			goto out_free1;
2586 		r = 0;
2587 out_free1:
2588 		kfree(kvm_regs);
2589 		break;
2590 	}
2591 	case KVM_SET_REGS: {
2592 		struct kvm_regs *kvm_regs;
2593 
2594 		r = -ENOMEM;
2595 		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2596 		if (IS_ERR(kvm_regs)) {
2597 			r = PTR_ERR(kvm_regs);
2598 			goto out;
2599 		}
2600 		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2601 		kfree(kvm_regs);
2602 		break;
2603 	}
2604 	case KVM_GET_SREGS: {
2605 		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2606 		r = -ENOMEM;
2607 		if (!kvm_sregs)
2608 			goto out;
2609 		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2610 		if (r)
2611 			goto out;
2612 		r = -EFAULT;
2613 		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2614 			goto out;
2615 		r = 0;
2616 		break;
2617 	}
2618 	case KVM_SET_SREGS: {
2619 		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2620 		if (IS_ERR(kvm_sregs)) {
2621 			r = PTR_ERR(kvm_sregs);
2622 			kvm_sregs = NULL;
2623 			goto out;
2624 		}
2625 		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2626 		break;
2627 	}
2628 	case KVM_GET_MP_STATE: {
2629 		struct kvm_mp_state mp_state;
2630 
2631 		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2632 		if (r)
2633 			goto out;
2634 		r = -EFAULT;
2635 		if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2636 			goto out;
2637 		r = 0;
2638 		break;
2639 	}
2640 	case KVM_SET_MP_STATE: {
2641 		struct kvm_mp_state mp_state;
2642 
2643 		r = -EFAULT;
2644 		if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2645 			goto out;
2646 		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2647 		break;
2648 	}
2649 	case KVM_TRANSLATE: {
2650 		struct kvm_translation tr;
2651 
2652 		r = -EFAULT;
2653 		if (copy_from_user(&tr, argp, sizeof(tr)))
2654 			goto out;
2655 		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2656 		if (r)
2657 			goto out;
2658 		r = -EFAULT;
2659 		if (copy_to_user(argp, &tr, sizeof(tr)))
2660 			goto out;
2661 		r = 0;
2662 		break;
2663 	}
2664 	case KVM_SET_GUEST_DEBUG: {
2665 		struct kvm_guest_debug dbg;
2666 
2667 		r = -EFAULT;
2668 		if (copy_from_user(&dbg, argp, sizeof(dbg)))
2669 			goto out;
2670 		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2671 		break;
2672 	}
2673 	case KVM_SET_SIGNAL_MASK: {
2674 		struct kvm_signal_mask __user *sigmask_arg = argp;
2675 		struct kvm_signal_mask kvm_sigmask;
2676 		sigset_t sigset, *p;
2677 
2678 		p = NULL;
2679 		if (argp) {
2680 			r = -EFAULT;
2681 			if (copy_from_user(&kvm_sigmask, argp,
2682 					   sizeof(kvm_sigmask)))
2683 				goto out;
2684 			r = -EINVAL;
2685 			if (kvm_sigmask.len != sizeof(sigset))
2686 				goto out;
2687 			r = -EFAULT;
2688 			if (copy_from_user(&sigset, sigmask_arg->sigset,
2689 					   sizeof(sigset)))
2690 				goto out;
2691 			p = &sigset;
2692 		}
2693 		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2694 		break;
2695 	}
2696 	case KVM_GET_FPU: {
2697 		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2698 		r = -ENOMEM;
2699 		if (!fpu)
2700 			goto out;
2701 		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2702 		if (r)
2703 			goto out;
2704 		r = -EFAULT;
2705 		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2706 			goto out;
2707 		r = 0;
2708 		break;
2709 	}
2710 	case KVM_SET_FPU: {
2711 		fpu = memdup_user(argp, sizeof(*fpu));
2712 		if (IS_ERR(fpu)) {
2713 			r = PTR_ERR(fpu);
2714 			fpu = NULL;
2715 			goto out;
2716 		}
2717 		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2718 		break;
2719 	}
2720 	default:
2721 		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2722 	}
2723 out:
2724 	vcpu_put(vcpu);
2725 	kfree(fpu);
2726 	kfree(kvm_sregs);
2727 	return r;
2728 }
2729 
2730 #ifdef CONFIG_KVM_COMPAT
2731 static long kvm_vcpu_compat_ioctl(struct file *filp,
2732 				  unsigned int ioctl, unsigned long arg)
2733 {
2734 	struct kvm_vcpu *vcpu = filp->private_data;
2735 	void __user *argp = compat_ptr(arg);
2736 	int r;
2737 
2738 	if (vcpu->kvm->mm != current->mm)
2739 		return -EIO;
2740 
2741 	switch (ioctl) {
2742 	case KVM_SET_SIGNAL_MASK: {
2743 		struct kvm_signal_mask __user *sigmask_arg = argp;
2744 		struct kvm_signal_mask kvm_sigmask;
2745 		compat_sigset_t csigset;
2746 		sigset_t sigset;
2747 
2748 		if (argp) {
2749 			r = -EFAULT;
2750 			if (copy_from_user(&kvm_sigmask, argp,
2751 					   sizeof(kvm_sigmask)))
2752 				goto out;
2753 			r = -EINVAL;
2754 			if (kvm_sigmask.len != sizeof(csigset))
2755 				goto out;
2756 			r = -EFAULT;
2757 			if (copy_from_user(&csigset, sigmask_arg->sigset,
2758 					   sizeof(csigset)))
2759 				goto out;
2760 			sigset_from_compat(&sigset, &csigset);
2761 			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2762 		} else
2763 			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2764 		break;
2765 	}
2766 	default:
2767 		r = kvm_vcpu_ioctl(filp, ioctl, arg);
2768 	}
2769 
2770 out:
2771 	return r;
2772 }
2773 #endif
2774 
2775 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2776 				 int (*accessor)(struct kvm_device *dev,
2777 						 struct kvm_device_attr *attr),
2778 				 unsigned long arg)
2779 {
2780 	struct kvm_device_attr attr;
2781 
2782 	if (!accessor)
2783 		return -EPERM;
2784 
2785 	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2786 		return -EFAULT;
2787 
2788 	return accessor(dev, &attr);
2789 }
2790 
2791 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2792 			     unsigned long arg)
2793 {
2794 	struct kvm_device *dev = filp->private_data;
2795 
2796 	switch (ioctl) {
2797 	case KVM_SET_DEVICE_ATTR:
2798 		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2799 	case KVM_GET_DEVICE_ATTR:
2800 		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2801 	case KVM_HAS_DEVICE_ATTR:
2802 		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2803 	default:
2804 		if (dev->ops->ioctl)
2805 			return dev->ops->ioctl(dev, ioctl, arg);
2806 
2807 		return -ENOTTY;
2808 	}
2809 }
2810 
2811 static int kvm_device_release(struct inode *inode, struct file *filp)
2812 {
2813 	struct kvm_device *dev = filp->private_data;
2814 	struct kvm *kvm = dev->kvm;
2815 
2816 	kvm_put_kvm(kvm);
2817 	return 0;
2818 }
2819 
2820 static const struct file_operations kvm_device_fops = {
2821 	.unlocked_ioctl = kvm_device_ioctl,
2822 #ifdef CONFIG_KVM_COMPAT
2823 	.compat_ioctl = kvm_device_ioctl,
2824 #endif
2825 	.release = kvm_device_release,
2826 };
2827 
2828 struct kvm_device *kvm_device_from_filp(struct file *filp)
2829 {
2830 	if (filp->f_op != &kvm_device_fops)
2831 		return NULL;
2832 
2833 	return filp->private_data;
2834 }
2835 
2836 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2837 #ifdef CONFIG_KVM_MPIC
2838 	[KVM_DEV_TYPE_FSL_MPIC_20]	= &kvm_mpic_ops,
2839 	[KVM_DEV_TYPE_FSL_MPIC_42]	= &kvm_mpic_ops,
2840 #endif
2841 
2842 #ifdef CONFIG_KVM_XICS
2843 	[KVM_DEV_TYPE_XICS]		= &kvm_xics_ops,
2844 #endif
2845 };
2846 
2847 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2848 {
2849 	if (type >= ARRAY_SIZE(kvm_device_ops_table))
2850 		return -ENOSPC;
2851 
2852 	if (kvm_device_ops_table[type] != NULL)
2853 		return -EEXIST;
2854 
2855 	kvm_device_ops_table[type] = ops;
2856 	return 0;
2857 }
2858 
2859 void kvm_unregister_device_ops(u32 type)
2860 {
2861 	if (kvm_device_ops_table[type] != NULL)
2862 		kvm_device_ops_table[type] = NULL;
2863 }
2864 
2865 static int kvm_ioctl_create_device(struct kvm *kvm,
2866 				   struct kvm_create_device *cd)
2867 {
2868 	struct kvm_device_ops *ops = NULL;
2869 	struct kvm_device *dev;
2870 	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2871 	int ret;
2872 
2873 	if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
2874 		return -ENODEV;
2875 
2876 	ops = kvm_device_ops_table[cd->type];
2877 	if (ops == NULL)
2878 		return -ENODEV;
2879 
2880 	if (test)
2881 		return 0;
2882 
2883 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2884 	if (!dev)
2885 		return -ENOMEM;
2886 
2887 	dev->ops = ops;
2888 	dev->kvm = kvm;
2889 
2890 	mutex_lock(&kvm->lock);
2891 	ret = ops->create(dev, cd->type);
2892 	if (ret < 0) {
2893 		mutex_unlock(&kvm->lock);
2894 		kfree(dev);
2895 		return ret;
2896 	}
2897 	list_add(&dev->vm_node, &kvm->devices);
2898 	mutex_unlock(&kvm->lock);
2899 
2900 	if (ops->init)
2901 		ops->init(dev);
2902 
2903 	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2904 	if (ret < 0) {
2905 		mutex_lock(&kvm->lock);
2906 		list_del(&dev->vm_node);
2907 		mutex_unlock(&kvm->lock);
2908 		ops->destroy(dev);
2909 		return ret;
2910 	}
2911 
2912 	kvm_get_kvm(kvm);
2913 	cd->fd = ret;
2914 	return 0;
2915 }
2916 
2917 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2918 {
2919 	switch (arg) {
2920 	case KVM_CAP_USER_MEMORY:
2921 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2922 	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2923 	case KVM_CAP_INTERNAL_ERROR_DATA:
2924 #ifdef CONFIG_HAVE_KVM_MSI
2925 	case KVM_CAP_SIGNAL_MSI:
2926 #endif
2927 #ifdef CONFIG_HAVE_KVM_IRQFD
2928 	case KVM_CAP_IRQFD:
2929 	case KVM_CAP_IRQFD_RESAMPLE:
2930 #endif
2931 	case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2932 	case KVM_CAP_CHECK_EXTENSION_VM:
2933 		return 1;
2934 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2935 	case KVM_CAP_IRQ_ROUTING:
2936 		return KVM_MAX_IRQ_ROUTES;
2937 #endif
2938 #if KVM_ADDRESS_SPACE_NUM > 1
2939 	case KVM_CAP_MULTI_ADDRESS_SPACE:
2940 		return KVM_ADDRESS_SPACE_NUM;
2941 #endif
2942 	case KVM_CAP_MAX_VCPU_ID:
2943 		return KVM_MAX_VCPU_ID;
2944 	default:
2945 		break;
2946 	}
2947 	return kvm_vm_ioctl_check_extension(kvm, arg);
2948 }
2949 
2950 static long kvm_vm_ioctl(struct file *filp,
2951 			   unsigned int ioctl, unsigned long arg)
2952 {
2953 	struct kvm *kvm = filp->private_data;
2954 	void __user *argp = (void __user *)arg;
2955 	int r;
2956 
2957 	if (kvm->mm != current->mm)
2958 		return -EIO;
2959 	switch (ioctl) {
2960 	case KVM_CREATE_VCPU:
2961 		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2962 		break;
2963 	case KVM_SET_USER_MEMORY_REGION: {
2964 		struct kvm_userspace_memory_region kvm_userspace_mem;
2965 
2966 		r = -EFAULT;
2967 		if (copy_from_user(&kvm_userspace_mem, argp,
2968 						sizeof(kvm_userspace_mem)))
2969 			goto out;
2970 
2971 		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2972 		break;
2973 	}
2974 	case KVM_GET_DIRTY_LOG: {
2975 		struct kvm_dirty_log log;
2976 
2977 		r = -EFAULT;
2978 		if (copy_from_user(&log, argp, sizeof(log)))
2979 			goto out;
2980 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2981 		break;
2982 	}
2983 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2984 	case KVM_REGISTER_COALESCED_MMIO: {
2985 		struct kvm_coalesced_mmio_zone zone;
2986 
2987 		r = -EFAULT;
2988 		if (copy_from_user(&zone, argp, sizeof(zone)))
2989 			goto out;
2990 		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2991 		break;
2992 	}
2993 	case KVM_UNREGISTER_COALESCED_MMIO: {
2994 		struct kvm_coalesced_mmio_zone zone;
2995 
2996 		r = -EFAULT;
2997 		if (copy_from_user(&zone, argp, sizeof(zone)))
2998 			goto out;
2999 		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
3000 		break;
3001 	}
3002 #endif
3003 	case KVM_IRQFD: {
3004 		struct kvm_irqfd data;
3005 
3006 		r = -EFAULT;
3007 		if (copy_from_user(&data, argp, sizeof(data)))
3008 			goto out;
3009 		r = kvm_irqfd(kvm, &data);
3010 		break;
3011 	}
3012 	case KVM_IOEVENTFD: {
3013 		struct kvm_ioeventfd data;
3014 
3015 		r = -EFAULT;
3016 		if (copy_from_user(&data, argp, sizeof(data)))
3017 			goto out;
3018 		r = kvm_ioeventfd(kvm, &data);
3019 		break;
3020 	}
3021 #ifdef CONFIG_HAVE_KVM_MSI
3022 	case KVM_SIGNAL_MSI: {
3023 		struct kvm_msi msi;
3024 
3025 		r = -EFAULT;
3026 		if (copy_from_user(&msi, argp, sizeof(msi)))
3027 			goto out;
3028 		r = kvm_send_userspace_msi(kvm, &msi);
3029 		break;
3030 	}
3031 #endif
3032 #ifdef __KVM_HAVE_IRQ_LINE
3033 	case KVM_IRQ_LINE_STATUS:
3034 	case KVM_IRQ_LINE: {
3035 		struct kvm_irq_level irq_event;
3036 
3037 		r = -EFAULT;
3038 		if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3039 			goto out;
3040 
3041 		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
3042 					ioctl == KVM_IRQ_LINE_STATUS);
3043 		if (r)
3044 			goto out;
3045 
3046 		r = -EFAULT;
3047 		if (ioctl == KVM_IRQ_LINE_STATUS) {
3048 			if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3049 				goto out;
3050 		}
3051 
3052 		r = 0;
3053 		break;
3054 	}
3055 #endif
3056 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3057 	case KVM_SET_GSI_ROUTING: {
3058 		struct kvm_irq_routing routing;
3059 		struct kvm_irq_routing __user *urouting;
3060 		struct kvm_irq_routing_entry *entries = NULL;
3061 
3062 		r = -EFAULT;
3063 		if (copy_from_user(&routing, argp, sizeof(routing)))
3064 			goto out;
3065 		r = -EINVAL;
3066 		if (routing.nr > KVM_MAX_IRQ_ROUTES)
3067 			goto out;
3068 		if (routing.flags)
3069 			goto out;
3070 		if (routing.nr) {
3071 			r = -ENOMEM;
3072 			entries = vmalloc(routing.nr * sizeof(*entries));
3073 			if (!entries)
3074 				goto out;
3075 			r = -EFAULT;
3076 			urouting = argp;
3077 			if (copy_from_user(entries, urouting->entries,
3078 					   routing.nr * sizeof(*entries)))
3079 				goto out_free_irq_routing;
3080 		}
3081 		r = kvm_set_irq_routing(kvm, entries, routing.nr,
3082 					routing.flags);
3083 out_free_irq_routing:
3084 		vfree(entries);
3085 		break;
3086 	}
3087 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
3088 	case KVM_CREATE_DEVICE: {
3089 		struct kvm_create_device cd;
3090 
3091 		r = -EFAULT;
3092 		if (copy_from_user(&cd, argp, sizeof(cd)))
3093 			goto out;
3094 
3095 		r = kvm_ioctl_create_device(kvm, &cd);
3096 		if (r)
3097 			goto out;
3098 
3099 		r = -EFAULT;
3100 		if (copy_to_user(argp, &cd, sizeof(cd)))
3101 			goto out;
3102 
3103 		r = 0;
3104 		break;
3105 	}
3106 	case KVM_CHECK_EXTENSION:
3107 		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3108 		break;
3109 	default:
3110 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3111 	}
3112 out:
3113 	return r;
3114 }
3115 
3116 #ifdef CONFIG_KVM_COMPAT
3117 struct compat_kvm_dirty_log {
3118 	__u32 slot;
3119 	__u32 padding1;
3120 	union {
3121 		compat_uptr_t dirty_bitmap; /* one bit per page */
3122 		__u64 padding2;
3123 	};
3124 };
3125 
3126 static long kvm_vm_compat_ioctl(struct file *filp,
3127 			   unsigned int ioctl, unsigned long arg)
3128 {
3129 	struct kvm *kvm = filp->private_data;
3130 	int r;
3131 
3132 	if (kvm->mm != current->mm)
3133 		return -EIO;
3134 	switch (ioctl) {
3135 	case KVM_GET_DIRTY_LOG: {
3136 		struct compat_kvm_dirty_log compat_log;
3137 		struct kvm_dirty_log log;
3138 
3139 		if (copy_from_user(&compat_log, (void __user *)arg,
3140 				   sizeof(compat_log)))
3141 			return -EFAULT;
3142 		log.slot	 = compat_log.slot;
3143 		log.padding1	 = compat_log.padding1;
3144 		log.padding2	 = compat_log.padding2;
3145 		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3146 
3147 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3148 		break;
3149 	}
3150 	default:
3151 		r = kvm_vm_ioctl(filp, ioctl, arg);
3152 	}
3153 	return r;
3154 }
3155 #endif
3156 
3157 static struct file_operations kvm_vm_fops = {
3158 	.release        = kvm_vm_release,
3159 	.unlocked_ioctl = kvm_vm_ioctl,
3160 #ifdef CONFIG_KVM_COMPAT
3161 	.compat_ioctl   = kvm_vm_compat_ioctl,
3162 #endif
3163 	.llseek		= noop_llseek,
3164 };
3165 
3166 static int kvm_dev_ioctl_create_vm(unsigned long type)
3167 {
3168 	int r;
3169 	struct kvm *kvm;
3170 	struct file *file;
3171 
3172 	kvm = kvm_create_vm(type);
3173 	if (IS_ERR(kvm))
3174 		return PTR_ERR(kvm);
3175 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
3176 	r = kvm_coalesced_mmio_init(kvm);
3177 	if (r < 0) {
3178 		kvm_put_kvm(kvm);
3179 		return r;
3180 	}
3181 #endif
3182 	r = get_unused_fd_flags(O_CLOEXEC);
3183 	if (r < 0) {
3184 		kvm_put_kvm(kvm);
3185 		return r;
3186 	}
3187 	file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3188 	if (IS_ERR(file)) {
3189 		put_unused_fd(r);
3190 		kvm_put_kvm(kvm);
3191 		return PTR_ERR(file);
3192 	}
3193 
3194 	if (kvm_create_vm_debugfs(kvm, r) < 0) {
3195 		put_unused_fd(r);
3196 		fput(file);
3197 		return -ENOMEM;
3198 	}
3199 
3200 	fd_install(r, file);
3201 	return r;
3202 }
3203 
3204 static long kvm_dev_ioctl(struct file *filp,
3205 			  unsigned int ioctl, unsigned long arg)
3206 {
3207 	long r = -EINVAL;
3208 
3209 	switch (ioctl) {
3210 	case KVM_GET_API_VERSION:
3211 		if (arg)
3212 			goto out;
3213 		r = KVM_API_VERSION;
3214 		break;
3215 	case KVM_CREATE_VM:
3216 		r = kvm_dev_ioctl_create_vm(arg);
3217 		break;
3218 	case KVM_CHECK_EXTENSION:
3219 		r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3220 		break;
3221 	case KVM_GET_VCPU_MMAP_SIZE:
3222 		if (arg)
3223 			goto out;
3224 		r = PAGE_SIZE;     /* struct kvm_run */
3225 #ifdef CONFIG_X86
3226 		r += PAGE_SIZE;    /* pio data page */
3227 #endif
3228 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
3229 		r += PAGE_SIZE;    /* coalesced mmio ring page */
3230 #endif
3231 		break;
3232 	case KVM_TRACE_ENABLE:
3233 	case KVM_TRACE_PAUSE:
3234 	case KVM_TRACE_DISABLE:
3235 		r = -EOPNOTSUPP;
3236 		break;
3237 	default:
3238 		return kvm_arch_dev_ioctl(filp, ioctl, arg);
3239 	}
3240 out:
3241 	return r;
3242 }
3243 
3244 static struct file_operations kvm_chardev_ops = {
3245 	.unlocked_ioctl = kvm_dev_ioctl,
3246 	.compat_ioctl   = kvm_dev_ioctl,
3247 	.llseek		= noop_llseek,
3248 };
3249 
3250 static struct miscdevice kvm_dev = {
3251 	KVM_MINOR,
3252 	"kvm",
3253 	&kvm_chardev_ops,
3254 };
3255 
3256 static void hardware_enable_nolock(void *junk)
3257 {
3258 	int cpu = raw_smp_processor_id();
3259 	int r;
3260 
3261 	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3262 		return;
3263 
3264 	cpumask_set_cpu(cpu, cpus_hardware_enabled);
3265 
3266 	r = kvm_arch_hardware_enable();
3267 
3268 	if (r) {
3269 		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3270 		atomic_inc(&hardware_enable_failed);
3271 		pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3272 	}
3273 }
3274 
3275 static int kvm_starting_cpu(unsigned int cpu)
3276 {
3277 	raw_spin_lock(&kvm_count_lock);
3278 	if (kvm_usage_count)
3279 		hardware_enable_nolock(NULL);
3280 	raw_spin_unlock(&kvm_count_lock);
3281 	return 0;
3282 }
3283 
3284 static void hardware_disable_nolock(void *junk)
3285 {
3286 	int cpu = raw_smp_processor_id();
3287 
3288 	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3289 		return;
3290 	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3291 	kvm_arch_hardware_disable();
3292 }
3293 
3294 static int kvm_dying_cpu(unsigned int cpu)
3295 {
3296 	raw_spin_lock(&kvm_count_lock);
3297 	if (kvm_usage_count)
3298 		hardware_disable_nolock(NULL);
3299 	raw_spin_unlock(&kvm_count_lock);
3300 	return 0;
3301 }
3302 
3303 static void hardware_disable_all_nolock(void)
3304 {
3305 	BUG_ON(!kvm_usage_count);
3306 
3307 	kvm_usage_count--;
3308 	if (!kvm_usage_count)
3309 		on_each_cpu(hardware_disable_nolock, NULL, 1);
3310 }
3311 
3312 static void hardware_disable_all(void)
3313 {
3314 	raw_spin_lock(&kvm_count_lock);
3315 	hardware_disable_all_nolock();
3316 	raw_spin_unlock(&kvm_count_lock);
3317 }
3318 
3319 static int hardware_enable_all(void)
3320 {
3321 	int r = 0;
3322 
3323 	raw_spin_lock(&kvm_count_lock);
3324 
3325 	kvm_usage_count++;
3326 	if (kvm_usage_count == 1) {
3327 		atomic_set(&hardware_enable_failed, 0);
3328 		on_each_cpu(hardware_enable_nolock, NULL, 1);
3329 
3330 		if (atomic_read(&hardware_enable_failed)) {
3331 			hardware_disable_all_nolock();
3332 			r = -EBUSY;
3333 		}
3334 	}
3335 
3336 	raw_spin_unlock(&kvm_count_lock);
3337 
3338 	return r;
3339 }
3340 
3341 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3342 		      void *v)
3343 {
3344 	/*
3345 	 * Some (well, at least mine) BIOSes hang on reboot if
3346 	 * in vmx root mode.
3347 	 *
3348 	 * And Intel TXT required VMX off for all cpu when system shutdown.
3349 	 */
3350 	pr_info("kvm: exiting hardware virtualization\n");
3351 	kvm_rebooting = true;
3352 	on_each_cpu(hardware_disable_nolock, NULL, 1);
3353 	return NOTIFY_OK;
3354 }
3355 
3356 static struct notifier_block kvm_reboot_notifier = {
3357 	.notifier_call = kvm_reboot,
3358 	.priority = 0,
3359 };
3360 
3361 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3362 {
3363 	int i;
3364 
3365 	for (i = 0; i < bus->dev_count; i++) {
3366 		struct kvm_io_device *pos = bus->range[i].dev;
3367 
3368 		kvm_iodevice_destructor(pos);
3369 	}
3370 	kfree(bus);
3371 }
3372 
3373 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3374 				 const struct kvm_io_range *r2)
3375 {
3376 	gpa_t addr1 = r1->addr;
3377 	gpa_t addr2 = r2->addr;
3378 
3379 	if (addr1 < addr2)
3380 		return -1;
3381 
3382 	/* If r2->len == 0, match the exact address.  If r2->len != 0,
3383 	 * accept any overlapping write.  Any order is acceptable for
3384 	 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3385 	 * we process all of them.
3386 	 */
3387 	if (r2->len) {
3388 		addr1 += r1->len;
3389 		addr2 += r2->len;
3390 	}
3391 
3392 	if (addr1 > addr2)
3393 		return 1;
3394 
3395 	return 0;
3396 }
3397 
3398 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3399 {
3400 	return kvm_io_bus_cmp(p1, p2);
3401 }
3402 
3403 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
3404 			  gpa_t addr, int len)
3405 {
3406 	bus->range[bus->dev_count++] = (struct kvm_io_range) {
3407 		.addr = addr,
3408 		.len = len,
3409 		.dev = dev,
3410 	};
3411 
3412 	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
3413 		kvm_io_bus_sort_cmp, NULL);
3414 
3415 	return 0;
3416 }
3417 
3418 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3419 			     gpa_t addr, int len)
3420 {
3421 	struct kvm_io_range *range, key;
3422 	int off;
3423 
3424 	key = (struct kvm_io_range) {
3425 		.addr = addr,
3426 		.len = len,
3427 	};
3428 
3429 	range = bsearch(&key, bus->range, bus->dev_count,
3430 			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3431 	if (range == NULL)
3432 		return -ENOENT;
3433 
3434 	off = range - bus->range;
3435 
3436 	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3437 		off--;
3438 
3439 	return off;
3440 }
3441 
3442 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3443 			      struct kvm_io_range *range, const void *val)
3444 {
3445 	int idx;
3446 
3447 	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3448 	if (idx < 0)
3449 		return -EOPNOTSUPP;
3450 
3451 	while (idx < bus->dev_count &&
3452 		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3453 		if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3454 					range->len, val))
3455 			return idx;
3456 		idx++;
3457 	}
3458 
3459 	return -EOPNOTSUPP;
3460 }
3461 
3462 /* kvm_io_bus_write - called under kvm->slots_lock */
3463 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3464 		     int len, const void *val)
3465 {
3466 	struct kvm_io_bus *bus;
3467 	struct kvm_io_range range;
3468 	int r;
3469 
3470 	range = (struct kvm_io_range) {
3471 		.addr = addr,
3472 		.len = len,
3473 	};
3474 
3475 	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3476 	r = __kvm_io_bus_write(vcpu, bus, &range, val);
3477 	return r < 0 ? r : 0;
3478 }
3479 
3480 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3481 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3482 			    gpa_t addr, int len, const void *val, long cookie)
3483 {
3484 	struct kvm_io_bus *bus;
3485 	struct kvm_io_range range;
3486 
3487 	range = (struct kvm_io_range) {
3488 		.addr = addr,
3489 		.len = len,
3490 	};
3491 
3492 	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3493 
3494 	/* First try the device referenced by cookie. */
3495 	if ((cookie >= 0) && (cookie < bus->dev_count) &&
3496 	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3497 		if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3498 					val))
3499 			return cookie;
3500 
3501 	/*
3502 	 * cookie contained garbage; fall back to search and return the
3503 	 * correct cookie value.
3504 	 */
3505 	return __kvm_io_bus_write(vcpu, bus, &range, val);
3506 }
3507 
3508 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3509 			     struct kvm_io_range *range, void *val)
3510 {
3511 	int idx;
3512 
3513 	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3514 	if (idx < 0)
3515 		return -EOPNOTSUPP;
3516 
3517 	while (idx < bus->dev_count &&
3518 		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3519 		if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3520 				       range->len, val))
3521 			return idx;
3522 		idx++;
3523 	}
3524 
3525 	return -EOPNOTSUPP;
3526 }
3527 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3528 
3529 /* kvm_io_bus_read - called under kvm->slots_lock */
3530 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3531 		    int len, void *val)
3532 {
3533 	struct kvm_io_bus *bus;
3534 	struct kvm_io_range range;
3535 	int r;
3536 
3537 	range = (struct kvm_io_range) {
3538 		.addr = addr,
3539 		.len = len,
3540 	};
3541 
3542 	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3543 	r = __kvm_io_bus_read(vcpu, bus, &range, val);
3544 	return r < 0 ? r : 0;
3545 }
3546 
3547 
3548 /* Caller must hold slots_lock. */
3549 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3550 			    int len, struct kvm_io_device *dev)
3551 {
3552 	struct kvm_io_bus *new_bus, *bus;
3553 
3554 	bus = kvm->buses[bus_idx];
3555 	/* exclude ioeventfd which is limited by maximum fd */
3556 	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3557 		return -ENOSPC;
3558 
3559 	new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
3560 			  sizeof(struct kvm_io_range)), GFP_KERNEL);
3561 	if (!new_bus)
3562 		return -ENOMEM;
3563 	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
3564 	       sizeof(struct kvm_io_range)));
3565 	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
3566 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3567 	synchronize_srcu_expedited(&kvm->srcu);
3568 	kfree(bus);
3569 
3570 	return 0;
3571 }
3572 
3573 /* Caller must hold slots_lock. */
3574 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3575 			      struct kvm_io_device *dev)
3576 {
3577 	int i, r;
3578 	struct kvm_io_bus *new_bus, *bus;
3579 
3580 	bus = kvm->buses[bus_idx];
3581 	r = -ENOENT;
3582 	for (i = 0; i < bus->dev_count; i++)
3583 		if (bus->range[i].dev == dev) {
3584 			r = 0;
3585 			break;
3586 		}
3587 
3588 	if (r)
3589 		return r;
3590 
3591 	new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
3592 			  sizeof(struct kvm_io_range)), GFP_KERNEL);
3593 	if (!new_bus)
3594 		return -ENOMEM;
3595 
3596 	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3597 	new_bus->dev_count--;
3598 	memcpy(new_bus->range + i, bus->range + i + 1,
3599 	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
3600 
3601 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3602 	synchronize_srcu_expedited(&kvm->srcu);
3603 	kfree(bus);
3604 	return r;
3605 }
3606 
3607 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3608 					 gpa_t addr)
3609 {
3610 	struct kvm_io_bus *bus;
3611 	int dev_idx, srcu_idx;
3612 	struct kvm_io_device *iodev = NULL;
3613 
3614 	srcu_idx = srcu_read_lock(&kvm->srcu);
3615 
3616 	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
3617 
3618 	dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3619 	if (dev_idx < 0)
3620 		goto out_unlock;
3621 
3622 	iodev = bus->range[dev_idx].dev;
3623 
3624 out_unlock:
3625 	srcu_read_unlock(&kvm->srcu, srcu_idx);
3626 
3627 	return iodev;
3628 }
3629 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3630 
3631 static int kvm_debugfs_open(struct inode *inode, struct file *file,
3632 			   int (*get)(void *, u64 *), int (*set)(void *, u64),
3633 			   const char *fmt)
3634 {
3635 	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3636 					  inode->i_private;
3637 
3638 	/* The debugfs files are a reference to the kvm struct which
3639 	 * is still valid when kvm_destroy_vm is called.
3640 	 * To avoid the race between open and the removal of the debugfs
3641 	 * directory we test against the users count.
3642 	 */
3643 	if (!atomic_add_unless(&stat_data->kvm->users_count, 1, 0))
3644 		return -ENOENT;
3645 
3646 	if (simple_attr_open(inode, file, get, set, fmt)) {
3647 		kvm_put_kvm(stat_data->kvm);
3648 		return -ENOMEM;
3649 	}
3650 
3651 	return 0;
3652 }
3653 
3654 static int kvm_debugfs_release(struct inode *inode, struct file *file)
3655 {
3656 	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3657 					  inode->i_private;
3658 
3659 	simple_attr_release(inode, file);
3660 	kvm_put_kvm(stat_data->kvm);
3661 
3662 	return 0;
3663 }
3664 
3665 static int vm_stat_get_per_vm(void *data, u64 *val)
3666 {
3667 	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3668 
3669 	*val = *(ulong *)((void *)stat_data->kvm + stat_data->offset);
3670 
3671 	return 0;
3672 }
3673 
3674 static int vm_stat_clear_per_vm(void *data, u64 val)
3675 {
3676 	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3677 
3678 	if (val)
3679 		return -EINVAL;
3680 
3681 	*(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;
3682 
3683 	return 0;
3684 }
3685 
3686 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
3687 {
3688 	__simple_attr_check_format("%llu\n", 0ull);
3689 	return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
3690 				vm_stat_clear_per_vm, "%llu\n");
3691 }
3692 
3693 static const struct file_operations vm_stat_get_per_vm_fops = {
3694 	.owner   = THIS_MODULE,
3695 	.open    = vm_stat_get_per_vm_open,
3696 	.release = kvm_debugfs_release,
3697 	.read    = simple_attr_read,
3698 	.write   = simple_attr_write,
3699 	.llseek  = generic_file_llseek,
3700 };
3701 
3702 static int vcpu_stat_get_per_vm(void *data, u64 *val)
3703 {
3704 	int i;
3705 	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3706 	struct kvm_vcpu *vcpu;
3707 
3708 	*val = 0;
3709 
3710 	kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3711 		*val += *(u64 *)((void *)vcpu + stat_data->offset);
3712 
3713 	return 0;
3714 }
3715 
3716 static int vcpu_stat_clear_per_vm(void *data, u64 val)
3717 {
3718 	int i;
3719 	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3720 	struct kvm_vcpu *vcpu;
3721 
3722 	if (val)
3723 		return -EINVAL;
3724 
3725 	kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3726 		*(u64 *)((void *)vcpu + stat_data->offset) = 0;
3727 
3728 	return 0;
3729 }
3730 
3731 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
3732 {
3733 	__simple_attr_check_format("%llu\n", 0ull);
3734 	return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
3735 				 vcpu_stat_clear_per_vm, "%llu\n");
3736 }
3737 
3738 static const struct file_operations vcpu_stat_get_per_vm_fops = {
3739 	.owner   = THIS_MODULE,
3740 	.open    = vcpu_stat_get_per_vm_open,
3741 	.release = kvm_debugfs_release,
3742 	.read    = simple_attr_read,
3743 	.write   = simple_attr_write,
3744 	.llseek  = generic_file_llseek,
3745 };
3746 
3747 static const struct file_operations *stat_fops_per_vm[] = {
3748 	[KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
3749 	[KVM_STAT_VM]   = &vm_stat_get_per_vm_fops,
3750 };
3751 
3752 static int vm_stat_get(void *_offset, u64 *val)
3753 {
3754 	unsigned offset = (long)_offset;
3755 	struct kvm *kvm;
3756 	struct kvm_stat_data stat_tmp = {.offset = offset};
3757 	u64 tmp_val;
3758 
3759 	*val = 0;
3760 	spin_lock(&kvm_lock);
3761 	list_for_each_entry(kvm, &vm_list, vm_list) {
3762 		stat_tmp.kvm = kvm;
3763 		vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3764 		*val += tmp_val;
3765 	}
3766 	spin_unlock(&kvm_lock);
3767 	return 0;
3768 }
3769 
3770 static int vm_stat_clear(void *_offset, u64 val)
3771 {
3772 	unsigned offset = (long)_offset;
3773 	struct kvm *kvm;
3774 	struct kvm_stat_data stat_tmp = {.offset = offset};
3775 
3776 	if (val)
3777 		return -EINVAL;
3778 
3779 	spin_lock(&kvm_lock);
3780 	list_for_each_entry(kvm, &vm_list, vm_list) {
3781 		stat_tmp.kvm = kvm;
3782 		vm_stat_clear_per_vm((void *)&stat_tmp, 0);
3783 	}
3784 	spin_unlock(&kvm_lock);
3785 
3786 	return 0;
3787 }
3788 
3789 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
3790 
3791 static int vcpu_stat_get(void *_offset, u64 *val)
3792 {
3793 	unsigned offset = (long)_offset;
3794 	struct kvm *kvm;
3795 	struct kvm_stat_data stat_tmp = {.offset = offset};
3796 	u64 tmp_val;
3797 
3798 	*val = 0;
3799 	spin_lock(&kvm_lock);
3800 	list_for_each_entry(kvm, &vm_list, vm_list) {
3801 		stat_tmp.kvm = kvm;
3802 		vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3803 		*val += tmp_val;
3804 	}
3805 	spin_unlock(&kvm_lock);
3806 	return 0;
3807 }
3808 
3809 static int vcpu_stat_clear(void *_offset, u64 val)
3810 {
3811 	unsigned offset = (long)_offset;
3812 	struct kvm *kvm;
3813 	struct kvm_stat_data stat_tmp = {.offset = offset};
3814 
3815 	if (val)
3816 		return -EINVAL;
3817 
3818 	spin_lock(&kvm_lock);
3819 	list_for_each_entry(kvm, &vm_list, vm_list) {
3820 		stat_tmp.kvm = kvm;
3821 		vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
3822 	}
3823 	spin_unlock(&kvm_lock);
3824 
3825 	return 0;
3826 }
3827 
3828 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
3829 			"%llu\n");
3830 
3831 static const struct file_operations *stat_fops[] = {
3832 	[KVM_STAT_VCPU] = &vcpu_stat_fops,
3833 	[KVM_STAT_VM]   = &vm_stat_fops,
3834 };
3835 
3836 static int kvm_init_debug(void)
3837 {
3838 	int r = -EEXIST;
3839 	struct kvm_stats_debugfs_item *p;
3840 
3841 	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3842 	if (kvm_debugfs_dir == NULL)
3843 		goto out;
3844 
3845 	kvm_debugfs_num_entries = 0;
3846 	for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
3847 		if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
3848 					 (void *)(long)p->offset,
3849 					 stat_fops[p->kind]))
3850 			goto out_dir;
3851 	}
3852 
3853 	return 0;
3854 
3855 out_dir:
3856 	debugfs_remove_recursive(kvm_debugfs_dir);
3857 out:
3858 	return r;
3859 }
3860 
3861 static int kvm_suspend(void)
3862 {
3863 	if (kvm_usage_count)
3864 		hardware_disable_nolock(NULL);
3865 	return 0;
3866 }
3867 
3868 static void kvm_resume(void)
3869 {
3870 	if (kvm_usage_count) {
3871 		WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3872 		hardware_enable_nolock(NULL);
3873 	}
3874 }
3875 
3876 static struct syscore_ops kvm_syscore_ops = {
3877 	.suspend = kvm_suspend,
3878 	.resume = kvm_resume,
3879 };
3880 
3881 static inline
3882 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3883 {
3884 	return container_of(pn, struct kvm_vcpu, preempt_notifier);
3885 }
3886 
3887 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3888 {
3889 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3890 
3891 	if (vcpu->preempted)
3892 		vcpu->preempted = false;
3893 
3894 	kvm_arch_sched_in(vcpu, cpu);
3895 
3896 	kvm_arch_vcpu_load(vcpu, cpu);
3897 }
3898 
3899 static void kvm_sched_out(struct preempt_notifier *pn,
3900 			  struct task_struct *next)
3901 {
3902 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3903 
3904 	if (current->state == TASK_RUNNING)
3905 		vcpu->preempted = true;
3906 	kvm_arch_vcpu_put(vcpu);
3907 }
3908 
3909 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3910 		  struct module *module)
3911 {
3912 	int r;
3913 	int cpu;
3914 
3915 	r = kvm_arch_init(opaque);
3916 	if (r)
3917 		goto out_fail;
3918 
3919 	/*
3920 	 * kvm_arch_init makes sure there's at most one caller
3921 	 * for architectures that support multiple implementations,
3922 	 * like intel and amd on x86.
3923 	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
3924 	 * conflicts in case kvm is already setup for another implementation.
3925 	 */
3926 	r = kvm_irqfd_init();
3927 	if (r)
3928 		goto out_irqfd;
3929 
3930 	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3931 		r = -ENOMEM;
3932 		goto out_free_0;
3933 	}
3934 
3935 	r = kvm_arch_hardware_setup();
3936 	if (r < 0)
3937 		goto out_free_0a;
3938 
3939 	for_each_online_cpu(cpu) {
3940 		smp_call_function_single(cpu,
3941 				kvm_arch_check_processor_compat,
3942 				&r, 1);
3943 		if (r < 0)
3944 			goto out_free_1;
3945 	}
3946 
3947 	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
3948 				      kvm_starting_cpu, kvm_dying_cpu);
3949 	if (r)
3950 		goto out_free_2;
3951 	register_reboot_notifier(&kvm_reboot_notifier);
3952 
3953 	/* A kmem cache lets us meet the alignment requirements of fx_save. */
3954 	if (!vcpu_align)
3955 		vcpu_align = __alignof__(struct kvm_vcpu);
3956 	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
3957 					   0, NULL);
3958 	if (!kvm_vcpu_cache) {
3959 		r = -ENOMEM;
3960 		goto out_free_3;
3961 	}
3962 
3963 	r = kvm_async_pf_init();
3964 	if (r)
3965 		goto out_free;
3966 
3967 	kvm_chardev_ops.owner = module;
3968 	kvm_vm_fops.owner = module;
3969 	kvm_vcpu_fops.owner = module;
3970 
3971 	r = misc_register(&kvm_dev);
3972 	if (r) {
3973 		pr_err("kvm: misc device register failed\n");
3974 		goto out_unreg;
3975 	}
3976 
3977 	register_syscore_ops(&kvm_syscore_ops);
3978 
3979 	kvm_preempt_ops.sched_in = kvm_sched_in;
3980 	kvm_preempt_ops.sched_out = kvm_sched_out;
3981 
3982 	r = kvm_init_debug();
3983 	if (r) {
3984 		pr_err("kvm: create debugfs files failed\n");
3985 		goto out_undebugfs;
3986 	}
3987 
3988 	r = kvm_vfio_ops_init();
3989 	WARN_ON(r);
3990 
3991 	return 0;
3992 
3993 out_undebugfs:
3994 	unregister_syscore_ops(&kvm_syscore_ops);
3995 	misc_deregister(&kvm_dev);
3996 out_unreg:
3997 	kvm_async_pf_deinit();
3998 out_free:
3999 	kmem_cache_destroy(kvm_vcpu_cache);
4000 out_free_3:
4001 	unregister_reboot_notifier(&kvm_reboot_notifier);
4002 	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4003 out_free_2:
4004 out_free_1:
4005 	kvm_arch_hardware_unsetup();
4006 out_free_0a:
4007 	free_cpumask_var(cpus_hardware_enabled);
4008 out_free_0:
4009 	kvm_irqfd_exit();
4010 out_irqfd:
4011 	kvm_arch_exit();
4012 out_fail:
4013 	return r;
4014 }
4015 EXPORT_SYMBOL_GPL(kvm_init);
4016 
4017 void kvm_exit(void)
4018 {
4019 	debugfs_remove_recursive(kvm_debugfs_dir);
4020 	misc_deregister(&kvm_dev);
4021 	kmem_cache_destroy(kvm_vcpu_cache);
4022 	kvm_async_pf_deinit();
4023 	unregister_syscore_ops(&kvm_syscore_ops);
4024 	unregister_reboot_notifier(&kvm_reboot_notifier);
4025 	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4026 	on_each_cpu(hardware_disable_nolock, NULL, 1);
4027 	kvm_arch_hardware_unsetup();
4028 	kvm_arch_exit();
4029 	kvm_irqfd_exit();
4030 	free_cpumask_var(cpus_hardware_enabled);
4031 	kvm_vfio_ops_exit();
4032 }
4033 EXPORT_SYMBOL_GPL(kvm_exit);
4034