1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched/signal.h> 36 #include <linux/sched/mm.h> 37 #include <linux/cpumask.h> 38 #include <linux/smp.h> 39 #include <linux/anon_inodes.h> 40 #include <linux/profile.h> 41 #include <linux/kvm_para.h> 42 #include <linux/pagemap.h> 43 #include <linux/mman.h> 44 #include <linux/swap.h> 45 #include <linux/bitops.h> 46 #include <linux/spinlock.h> 47 #include <linux/compat.h> 48 #include <linux/srcu.h> 49 #include <linux/hugetlb.h> 50 #include <linux/slab.h> 51 #include <linux/sort.h> 52 #include <linux/bsearch.h> 53 54 #include <asm/processor.h> 55 #include <asm/io.h> 56 #include <asm/ioctl.h> 57 #include <linux/uaccess.h> 58 #include <asm/pgtable.h> 59 60 #include "coalesced_mmio.h" 61 #include "async_pf.h" 62 #include "vfio.h" 63 64 #define CREATE_TRACE_POINTS 65 #include <trace/events/kvm.h> 66 67 /* Worst case buffer size needed for holding an integer. */ 68 #define ITOA_MAX_LEN 12 69 70 MODULE_AUTHOR("Qumranet"); 71 MODULE_LICENSE("GPL"); 72 73 /* Architectures should define their poll value according to the halt latency */ 74 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 75 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 76 EXPORT_SYMBOL_GPL(halt_poll_ns); 77 78 /* Default doubles per-vcpu halt_poll_ns. */ 79 unsigned int halt_poll_ns_grow = 2; 80 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR); 81 EXPORT_SYMBOL_GPL(halt_poll_ns_grow); 82 83 /* Default resets per-vcpu halt_poll_ns . */ 84 unsigned int halt_poll_ns_shrink; 85 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR); 86 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); 87 88 /* 89 * Ordering of locks: 90 * 91 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 92 */ 93 94 DEFINE_SPINLOCK(kvm_lock); 95 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 96 LIST_HEAD(vm_list); 97 98 static cpumask_var_t cpus_hardware_enabled; 99 static int kvm_usage_count; 100 static atomic_t hardware_enable_failed; 101 102 struct kmem_cache *kvm_vcpu_cache; 103 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 104 105 static __read_mostly struct preempt_ops kvm_preempt_ops; 106 107 struct dentry *kvm_debugfs_dir; 108 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 109 110 static int kvm_debugfs_num_entries; 111 static const struct file_operations *stat_fops_per_vm[]; 112 113 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 114 unsigned long arg); 115 #ifdef CONFIG_KVM_COMPAT 116 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 117 unsigned long arg); 118 #endif 119 static int hardware_enable_all(void); 120 static void hardware_disable_all(void); 121 122 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 123 124 static void kvm_release_pfn_dirty(kvm_pfn_t pfn); 125 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 126 127 __visible bool kvm_rebooting; 128 EXPORT_SYMBOL_GPL(kvm_rebooting); 129 130 static bool largepages_enabled = true; 131 132 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 133 { 134 if (pfn_valid(pfn)) 135 return PageReserved(pfn_to_page(pfn)); 136 137 return true; 138 } 139 140 /* 141 * Switches to specified vcpu, until a matching vcpu_put() 142 */ 143 int vcpu_load(struct kvm_vcpu *vcpu) 144 { 145 int cpu; 146 147 if (mutex_lock_killable(&vcpu->mutex)) 148 return -EINTR; 149 cpu = get_cpu(); 150 preempt_notifier_register(&vcpu->preempt_notifier); 151 kvm_arch_vcpu_load(vcpu, cpu); 152 put_cpu(); 153 return 0; 154 } 155 EXPORT_SYMBOL_GPL(vcpu_load); 156 157 void vcpu_put(struct kvm_vcpu *vcpu) 158 { 159 preempt_disable(); 160 kvm_arch_vcpu_put(vcpu); 161 preempt_notifier_unregister(&vcpu->preempt_notifier); 162 preempt_enable(); 163 mutex_unlock(&vcpu->mutex); 164 } 165 EXPORT_SYMBOL_GPL(vcpu_put); 166 167 static void ack_flush(void *_completed) 168 { 169 } 170 171 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 172 { 173 int i, cpu, me; 174 cpumask_var_t cpus; 175 bool called = true; 176 struct kvm_vcpu *vcpu; 177 178 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 179 180 me = get_cpu(); 181 kvm_for_each_vcpu(i, vcpu, kvm) { 182 kvm_make_request(req, vcpu); 183 cpu = vcpu->cpu; 184 185 /* Set ->requests bit before we read ->mode. */ 186 smp_mb__after_atomic(); 187 188 if (cpus != NULL && cpu != -1 && cpu != me && 189 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 190 cpumask_set_cpu(cpu, cpus); 191 } 192 if (unlikely(cpus == NULL)) 193 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 194 else if (!cpumask_empty(cpus)) 195 smp_call_function_many(cpus, ack_flush, NULL, 1); 196 else 197 called = false; 198 put_cpu(); 199 free_cpumask_var(cpus); 200 return called; 201 } 202 203 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 204 void kvm_flush_remote_tlbs(struct kvm *kvm) 205 { 206 /* 207 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 208 * kvm_make_all_cpus_request. 209 */ 210 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 211 212 /* 213 * We want to publish modifications to the page tables before reading 214 * mode. Pairs with a memory barrier in arch-specific code. 215 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 216 * and smp_mb in walk_shadow_page_lockless_begin/end. 217 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 218 * 219 * There is already an smp_mb__after_atomic() before 220 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 221 * barrier here. 222 */ 223 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 224 ++kvm->stat.remote_tlb_flush; 225 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 226 } 227 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 228 #endif 229 230 void kvm_reload_remote_mmus(struct kvm *kvm) 231 { 232 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 233 } 234 235 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 236 { 237 struct page *page; 238 int r; 239 240 mutex_init(&vcpu->mutex); 241 vcpu->cpu = -1; 242 vcpu->kvm = kvm; 243 vcpu->vcpu_id = id; 244 vcpu->pid = NULL; 245 init_swait_queue_head(&vcpu->wq); 246 kvm_async_pf_vcpu_init(vcpu); 247 248 vcpu->pre_pcpu = -1; 249 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 250 251 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 252 if (!page) { 253 r = -ENOMEM; 254 goto fail; 255 } 256 vcpu->run = page_address(page); 257 258 kvm_vcpu_set_in_spin_loop(vcpu, false); 259 kvm_vcpu_set_dy_eligible(vcpu, false); 260 vcpu->preempted = false; 261 262 r = kvm_arch_vcpu_init(vcpu); 263 if (r < 0) 264 goto fail_free_run; 265 return 0; 266 267 fail_free_run: 268 free_page((unsigned long)vcpu->run); 269 fail: 270 return r; 271 } 272 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 273 274 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 275 { 276 put_pid(vcpu->pid); 277 kvm_arch_vcpu_uninit(vcpu); 278 free_page((unsigned long)vcpu->run); 279 } 280 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 281 282 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 283 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 284 { 285 return container_of(mn, struct kvm, mmu_notifier); 286 } 287 288 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 289 struct mm_struct *mm, 290 unsigned long address) 291 { 292 struct kvm *kvm = mmu_notifier_to_kvm(mn); 293 int need_tlb_flush, idx; 294 295 /* 296 * When ->invalidate_page runs, the linux pte has been zapped 297 * already but the page is still allocated until 298 * ->invalidate_page returns. So if we increase the sequence 299 * here the kvm page fault will notice if the spte can't be 300 * established because the page is going to be freed. If 301 * instead the kvm page fault establishes the spte before 302 * ->invalidate_page runs, kvm_unmap_hva will release it 303 * before returning. 304 * 305 * The sequence increase only need to be seen at spin_unlock 306 * time, and not at spin_lock time. 307 * 308 * Increasing the sequence after the spin_unlock would be 309 * unsafe because the kvm page fault could then establish the 310 * pte after kvm_unmap_hva returned, without noticing the page 311 * is going to be freed. 312 */ 313 idx = srcu_read_lock(&kvm->srcu); 314 spin_lock(&kvm->mmu_lock); 315 316 kvm->mmu_notifier_seq++; 317 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 318 /* we've to flush the tlb before the pages can be freed */ 319 if (need_tlb_flush) 320 kvm_flush_remote_tlbs(kvm); 321 322 spin_unlock(&kvm->mmu_lock); 323 324 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 325 326 srcu_read_unlock(&kvm->srcu, idx); 327 } 328 329 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 330 struct mm_struct *mm, 331 unsigned long address, 332 pte_t pte) 333 { 334 struct kvm *kvm = mmu_notifier_to_kvm(mn); 335 int idx; 336 337 idx = srcu_read_lock(&kvm->srcu); 338 spin_lock(&kvm->mmu_lock); 339 kvm->mmu_notifier_seq++; 340 kvm_set_spte_hva(kvm, address, pte); 341 spin_unlock(&kvm->mmu_lock); 342 srcu_read_unlock(&kvm->srcu, idx); 343 } 344 345 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 346 struct mm_struct *mm, 347 unsigned long start, 348 unsigned long end) 349 { 350 struct kvm *kvm = mmu_notifier_to_kvm(mn); 351 int need_tlb_flush = 0, idx; 352 353 idx = srcu_read_lock(&kvm->srcu); 354 spin_lock(&kvm->mmu_lock); 355 /* 356 * The count increase must become visible at unlock time as no 357 * spte can be established without taking the mmu_lock and 358 * count is also read inside the mmu_lock critical section. 359 */ 360 kvm->mmu_notifier_count++; 361 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 362 need_tlb_flush |= kvm->tlbs_dirty; 363 /* we've to flush the tlb before the pages can be freed */ 364 if (need_tlb_flush) 365 kvm_flush_remote_tlbs(kvm); 366 367 spin_unlock(&kvm->mmu_lock); 368 srcu_read_unlock(&kvm->srcu, idx); 369 } 370 371 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 372 struct mm_struct *mm, 373 unsigned long start, 374 unsigned long end) 375 { 376 struct kvm *kvm = mmu_notifier_to_kvm(mn); 377 378 spin_lock(&kvm->mmu_lock); 379 /* 380 * This sequence increase will notify the kvm page fault that 381 * the page that is going to be mapped in the spte could have 382 * been freed. 383 */ 384 kvm->mmu_notifier_seq++; 385 smp_wmb(); 386 /* 387 * The above sequence increase must be visible before the 388 * below count decrease, which is ensured by the smp_wmb above 389 * in conjunction with the smp_rmb in mmu_notifier_retry(). 390 */ 391 kvm->mmu_notifier_count--; 392 spin_unlock(&kvm->mmu_lock); 393 394 BUG_ON(kvm->mmu_notifier_count < 0); 395 } 396 397 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 398 struct mm_struct *mm, 399 unsigned long start, 400 unsigned long end) 401 { 402 struct kvm *kvm = mmu_notifier_to_kvm(mn); 403 int young, idx; 404 405 idx = srcu_read_lock(&kvm->srcu); 406 spin_lock(&kvm->mmu_lock); 407 408 young = kvm_age_hva(kvm, start, end); 409 if (young) 410 kvm_flush_remote_tlbs(kvm); 411 412 spin_unlock(&kvm->mmu_lock); 413 srcu_read_unlock(&kvm->srcu, idx); 414 415 return young; 416 } 417 418 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 419 struct mm_struct *mm, 420 unsigned long start, 421 unsigned long end) 422 { 423 struct kvm *kvm = mmu_notifier_to_kvm(mn); 424 int young, idx; 425 426 idx = srcu_read_lock(&kvm->srcu); 427 spin_lock(&kvm->mmu_lock); 428 /* 429 * Even though we do not flush TLB, this will still adversely 430 * affect performance on pre-Haswell Intel EPT, where there is 431 * no EPT Access Bit to clear so that we have to tear down EPT 432 * tables instead. If we find this unacceptable, we can always 433 * add a parameter to kvm_age_hva so that it effectively doesn't 434 * do anything on clear_young. 435 * 436 * Also note that currently we never issue secondary TLB flushes 437 * from clear_young, leaving this job up to the regular system 438 * cadence. If we find this inaccurate, we might come up with a 439 * more sophisticated heuristic later. 440 */ 441 young = kvm_age_hva(kvm, start, end); 442 spin_unlock(&kvm->mmu_lock); 443 srcu_read_unlock(&kvm->srcu, idx); 444 445 return young; 446 } 447 448 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 449 struct mm_struct *mm, 450 unsigned long address) 451 { 452 struct kvm *kvm = mmu_notifier_to_kvm(mn); 453 int young, idx; 454 455 idx = srcu_read_lock(&kvm->srcu); 456 spin_lock(&kvm->mmu_lock); 457 young = kvm_test_age_hva(kvm, address); 458 spin_unlock(&kvm->mmu_lock); 459 srcu_read_unlock(&kvm->srcu, idx); 460 461 return young; 462 } 463 464 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 465 struct mm_struct *mm) 466 { 467 struct kvm *kvm = mmu_notifier_to_kvm(mn); 468 int idx; 469 470 idx = srcu_read_lock(&kvm->srcu); 471 kvm_arch_flush_shadow_all(kvm); 472 srcu_read_unlock(&kvm->srcu, idx); 473 } 474 475 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 476 .invalidate_page = kvm_mmu_notifier_invalidate_page, 477 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 478 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 479 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 480 .clear_young = kvm_mmu_notifier_clear_young, 481 .test_young = kvm_mmu_notifier_test_young, 482 .change_pte = kvm_mmu_notifier_change_pte, 483 .release = kvm_mmu_notifier_release, 484 }; 485 486 static int kvm_init_mmu_notifier(struct kvm *kvm) 487 { 488 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 489 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 490 } 491 492 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 493 494 static int kvm_init_mmu_notifier(struct kvm *kvm) 495 { 496 return 0; 497 } 498 499 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 500 501 static struct kvm_memslots *kvm_alloc_memslots(void) 502 { 503 int i; 504 struct kvm_memslots *slots; 505 506 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 507 if (!slots) 508 return NULL; 509 510 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 511 slots->id_to_index[i] = slots->memslots[i].id = i; 512 513 return slots; 514 } 515 516 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 517 { 518 if (!memslot->dirty_bitmap) 519 return; 520 521 kvfree(memslot->dirty_bitmap); 522 memslot->dirty_bitmap = NULL; 523 } 524 525 /* 526 * Free any memory in @free but not in @dont. 527 */ 528 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 529 struct kvm_memory_slot *dont) 530 { 531 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 532 kvm_destroy_dirty_bitmap(free); 533 534 kvm_arch_free_memslot(kvm, free, dont); 535 536 free->npages = 0; 537 } 538 539 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 540 { 541 struct kvm_memory_slot *memslot; 542 543 if (!slots) 544 return; 545 546 kvm_for_each_memslot(memslot, slots) 547 kvm_free_memslot(kvm, memslot, NULL); 548 549 kvfree(slots); 550 } 551 552 static void kvm_destroy_vm_debugfs(struct kvm *kvm) 553 { 554 int i; 555 556 if (!kvm->debugfs_dentry) 557 return; 558 559 debugfs_remove_recursive(kvm->debugfs_dentry); 560 561 if (kvm->debugfs_stat_data) { 562 for (i = 0; i < kvm_debugfs_num_entries; i++) 563 kfree(kvm->debugfs_stat_data[i]); 564 kfree(kvm->debugfs_stat_data); 565 } 566 } 567 568 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) 569 { 570 char dir_name[ITOA_MAX_LEN * 2]; 571 struct kvm_stat_data *stat_data; 572 struct kvm_stats_debugfs_item *p; 573 574 if (!debugfs_initialized()) 575 return 0; 576 577 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd); 578 kvm->debugfs_dentry = debugfs_create_dir(dir_name, 579 kvm_debugfs_dir); 580 if (!kvm->debugfs_dentry) 581 return -ENOMEM; 582 583 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, 584 sizeof(*kvm->debugfs_stat_data), 585 GFP_KERNEL); 586 if (!kvm->debugfs_stat_data) 587 return -ENOMEM; 588 589 for (p = debugfs_entries; p->name; p++) { 590 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL); 591 if (!stat_data) 592 return -ENOMEM; 593 594 stat_data->kvm = kvm; 595 stat_data->offset = p->offset; 596 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; 597 if (!debugfs_create_file(p->name, 0644, 598 kvm->debugfs_dentry, 599 stat_data, 600 stat_fops_per_vm[p->kind])) 601 return -ENOMEM; 602 } 603 return 0; 604 } 605 606 static struct kvm *kvm_create_vm(unsigned long type) 607 { 608 int r, i; 609 struct kvm *kvm = kvm_arch_alloc_vm(); 610 611 if (!kvm) 612 return ERR_PTR(-ENOMEM); 613 614 spin_lock_init(&kvm->mmu_lock); 615 mmgrab(current->mm); 616 kvm->mm = current->mm; 617 kvm_eventfd_init(kvm); 618 mutex_init(&kvm->lock); 619 mutex_init(&kvm->irq_lock); 620 mutex_init(&kvm->slots_lock); 621 atomic_set(&kvm->users_count, 1); 622 INIT_LIST_HEAD(&kvm->devices); 623 624 r = kvm_arch_init_vm(kvm, type); 625 if (r) 626 goto out_err_no_disable; 627 628 r = hardware_enable_all(); 629 if (r) 630 goto out_err_no_disable; 631 632 #ifdef CONFIG_HAVE_KVM_IRQFD 633 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 634 #endif 635 636 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 637 638 r = -ENOMEM; 639 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 640 struct kvm_memslots *slots = kvm_alloc_memslots(); 641 if (!slots) 642 goto out_err_no_srcu; 643 /* 644 * Generations must be different for each address space. 645 * Init kvm generation close to the maximum to easily test the 646 * code of handling generation number wrap-around. 647 */ 648 slots->generation = i * 2 - 150; 649 rcu_assign_pointer(kvm->memslots[i], slots); 650 } 651 652 if (init_srcu_struct(&kvm->srcu)) 653 goto out_err_no_srcu; 654 if (init_srcu_struct(&kvm->irq_srcu)) 655 goto out_err_no_irq_srcu; 656 for (i = 0; i < KVM_NR_BUSES; i++) { 657 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 658 GFP_KERNEL); 659 if (!kvm->buses[i]) 660 goto out_err; 661 } 662 663 r = kvm_init_mmu_notifier(kvm); 664 if (r) 665 goto out_err; 666 667 spin_lock(&kvm_lock); 668 list_add(&kvm->vm_list, &vm_list); 669 spin_unlock(&kvm_lock); 670 671 preempt_notifier_inc(); 672 673 return kvm; 674 675 out_err: 676 cleanup_srcu_struct(&kvm->irq_srcu); 677 out_err_no_irq_srcu: 678 cleanup_srcu_struct(&kvm->srcu); 679 out_err_no_srcu: 680 hardware_disable_all(); 681 out_err_no_disable: 682 for (i = 0; i < KVM_NR_BUSES; i++) 683 kfree(kvm->buses[i]); 684 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 685 kvm_free_memslots(kvm, kvm->memslots[i]); 686 kvm_arch_free_vm(kvm); 687 mmdrop(current->mm); 688 return ERR_PTR(r); 689 } 690 691 /* 692 * Avoid using vmalloc for a small buffer. 693 * Should not be used when the size is statically known. 694 */ 695 void *kvm_kvzalloc(unsigned long size) 696 { 697 if (size > PAGE_SIZE) 698 return vzalloc(size); 699 else 700 return kzalloc(size, GFP_KERNEL); 701 } 702 703 static void kvm_destroy_devices(struct kvm *kvm) 704 { 705 struct kvm_device *dev, *tmp; 706 707 /* 708 * We do not need to take the kvm->lock here, because nobody else 709 * has a reference to the struct kvm at this point and therefore 710 * cannot access the devices list anyhow. 711 */ 712 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 713 list_del(&dev->vm_node); 714 dev->ops->destroy(dev); 715 } 716 } 717 718 static void kvm_destroy_vm(struct kvm *kvm) 719 { 720 int i; 721 struct mm_struct *mm = kvm->mm; 722 723 kvm_destroy_vm_debugfs(kvm); 724 kvm_arch_sync_events(kvm); 725 spin_lock(&kvm_lock); 726 list_del(&kvm->vm_list); 727 spin_unlock(&kvm_lock); 728 kvm_free_irq_routing(kvm); 729 for (i = 0; i < KVM_NR_BUSES; i++) 730 kvm_io_bus_destroy(kvm->buses[i]); 731 kvm_coalesced_mmio_free(kvm); 732 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 733 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 734 #else 735 kvm_arch_flush_shadow_all(kvm); 736 #endif 737 kvm_arch_destroy_vm(kvm); 738 kvm_destroy_devices(kvm); 739 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 740 kvm_free_memslots(kvm, kvm->memslots[i]); 741 cleanup_srcu_struct(&kvm->irq_srcu); 742 cleanup_srcu_struct(&kvm->srcu); 743 kvm_arch_free_vm(kvm); 744 preempt_notifier_dec(); 745 hardware_disable_all(); 746 mmdrop(mm); 747 } 748 749 void kvm_get_kvm(struct kvm *kvm) 750 { 751 atomic_inc(&kvm->users_count); 752 } 753 EXPORT_SYMBOL_GPL(kvm_get_kvm); 754 755 void kvm_put_kvm(struct kvm *kvm) 756 { 757 if (atomic_dec_and_test(&kvm->users_count)) 758 kvm_destroy_vm(kvm); 759 } 760 EXPORT_SYMBOL_GPL(kvm_put_kvm); 761 762 763 static int kvm_vm_release(struct inode *inode, struct file *filp) 764 { 765 struct kvm *kvm = filp->private_data; 766 767 kvm_irqfd_release(kvm); 768 769 kvm_put_kvm(kvm); 770 return 0; 771 } 772 773 /* 774 * Allocation size is twice as large as the actual dirty bitmap size. 775 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 776 */ 777 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 778 { 779 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 780 781 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 782 if (!memslot->dirty_bitmap) 783 return -ENOMEM; 784 785 return 0; 786 } 787 788 /* 789 * Insert memslot and re-sort memslots based on their GFN, 790 * so binary search could be used to lookup GFN. 791 * Sorting algorithm takes advantage of having initially 792 * sorted array and known changed memslot position. 793 */ 794 static void update_memslots(struct kvm_memslots *slots, 795 struct kvm_memory_slot *new) 796 { 797 int id = new->id; 798 int i = slots->id_to_index[id]; 799 struct kvm_memory_slot *mslots = slots->memslots; 800 801 WARN_ON(mslots[i].id != id); 802 if (!new->npages) { 803 WARN_ON(!mslots[i].npages); 804 if (mslots[i].npages) 805 slots->used_slots--; 806 } else { 807 if (!mslots[i].npages) 808 slots->used_slots++; 809 } 810 811 while (i < KVM_MEM_SLOTS_NUM - 1 && 812 new->base_gfn <= mslots[i + 1].base_gfn) { 813 if (!mslots[i + 1].npages) 814 break; 815 mslots[i] = mslots[i + 1]; 816 slots->id_to_index[mslots[i].id] = i; 817 i++; 818 } 819 820 /* 821 * The ">=" is needed when creating a slot with base_gfn == 0, 822 * so that it moves before all those with base_gfn == npages == 0. 823 * 824 * On the other hand, if new->npages is zero, the above loop has 825 * already left i pointing to the beginning of the empty part of 826 * mslots, and the ">=" would move the hole backwards in this 827 * case---which is wrong. So skip the loop when deleting a slot. 828 */ 829 if (new->npages) { 830 while (i > 0 && 831 new->base_gfn >= mslots[i - 1].base_gfn) { 832 mslots[i] = mslots[i - 1]; 833 slots->id_to_index[mslots[i].id] = i; 834 i--; 835 } 836 } else 837 WARN_ON_ONCE(i != slots->used_slots); 838 839 mslots[i] = *new; 840 slots->id_to_index[mslots[i].id] = i; 841 } 842 843 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 844 { 845 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 846 847 #ifdef __KVM_HAVE_READONLY_MEM 848 valid_flags |= KVM_MEM_READONLY; 849 #endif 850 851 if (mem->flags & ~valid_flags) 852 return -EINVAL; 853 854 return 0; 855 } 856 857 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 858 int as_id, struct kvm_memslots *slots) 859 { 860 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 861 862 /* 863 * Set the low bit in the generation, which disables SPTE caching 864 * until the end of synchronize_srcu_expedited. 865 */ 866 WARN_ON(old_memslots->generation & 1); 867 slots->generation = old_memslots->generation + 1; 868 869 rcu_assign_pointer(kvm->memslots[as_id], slots); 870 synchronize_srcu_expedited(&kvm->srcu); 871 872 /* 873 * Increment the new memslot generation a second time. This prevents 874 * vm exits that race with memslot updates from caching a memslot 875 * generation that will (potentially) be valid forever. 876 * 877 * Generations must be unique even across address spaces. We do not need 878 * a global counter for that, instead the generation space is evenly split 879 * across address spaces. For example, with two address spaces, address 880 * space 0 will use generations 0, 4, 8, ... while * address space 1 will 881 * use generations 2, 6, 10, 14, ... 882 */ 883 slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1; 884 885 kvm_arch_memslots_updated(kvm, slots); 886 887 return old_memslots; 888 } 889 890 /* 891 * Allocate some memory and give it an address in the guest physical address 892 * space. 893 * 894 * Discontiguous memory is allowed, mostly for framebuffers. 895 * 896 * Must be called holding kvm->slots_lock for write. 897 */ 898 int __kvm_set_memory_region(struct kvm *kvm, 899 const struct kvm_userspace_memory_region *mem) 900 { 901 int r; 902 gfn_t base_gfn; 903 unsigned long npages; 904 struct kvm_memory_slot *slot; 905 struct kvm_memory_slot old, new; 906 struct kvm_memslots *slots = NULL, *old_memslots; 907 int as_id, id; 908 enum kvm_mr_change change; 909 910 r = check_memory_region_flags(mem); 911 if (r) 912 goto out; 913 914 r = -EINVAL; 915 as_id = mem->slot >> 16; 916 id = (u16)mem->slot; 917 918 /* General sanity checks */ 919 if (mem->memory_size & (PAGE_SIZE - 1)) 920 goto out; 921 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 922 goto out; 923 /* We can read the guest memory with __xxx_user() later on. */ 924 if ((id < KVM_USER_MEM_SLOTS) && 925 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 926 !access_ok(VERIFY_WRITE, 927 (void __user *)(unsigned long)mem->userspace_addr, 928 mem->memory_size))) 929 goto out; 930 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 931 goto out; 932 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 933 goto out; 934 935 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 936 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 937 npages = mem->memory_size >> PAGE_SHIFT; 938 939 if (npages > KVM_MEM_MAX_NR_PAGES) 940 goto out; 941 942 new = old = *slot; 943 944 new.id = id; 945 new.base_gfn = base_gfn; 946 new.npages = npages; 947 new.flags = mem->flags; 948 949 if (npages) { 950 if (!old.npages) 951 change = KVM_MR_CREATE; 952 else { /* Modify an existing slot. */ 953 if ((mem->userspace_addr != old.userspace_addr) || 954 (npages != old.npages) || 955 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 956 goto out; 957 958 if (base_gfn != old.base_gfn) 959 change = KVM_MR_MOVE; 960 else if (new.flags != old.flags) 961 change = KVM_MR_FLAGS_ONLY; 962 else { /* Nothing to change. */ 963 r = 0; 964 goto out; 965 } 966 } 967 } else { 968 if (!old.npages) 969 goto out; 970 971 change = KVM_MR_DELETE; 972 new.base_gfn = 0; 973 new.flags = 0; 974 } 975 976 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 977 /* Check for overlaps */ 978 r = -EEXIST; 979 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 980 if ((slot->id >= KVM_USER_MEM_SLOTS) || 981 (slot->id == id)) 982 continue; 983 if (!((base_gfn + npages <= slot->base_gfn) || 984 (base_gfn >= slot->base_gfn + slot->npages))) 985 goto out; 986 } 987 } 988 989 /* Free page dirty bitmap if unneeded */ 990 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 991 new.dirty_bitmap = NULL; 992 993 r = -ENOMEM; 994 if (change == KVM_MR_CREATE) { 995 new.userspace_addr = mem->userspace_addr; 996 997 if (kvm_arch_create_memslot(kvm, &new, npages)) 998 goto out_free; 999 } 1000 1001 /* Allocate page dirty bitmap if needed */ 1002 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 1003 if (kvm_create_dirty_bitmap(&new) < 0) 1004 goto out_free; 1005 } 1006 1007 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 1008 if (!slots) 1009 goto out_free; 1010 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 1011 1012 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 1013 slot = id_to_memslot(slots, id); 1014 slot->flags |= KVM_MEMSLOT_INVALID; 1015 1016 old_memslots = install_new_memslots(kvm, as_id, slots); 1017 1018 /* slot was deleted or moved, clear iommu mapping */ 1019 kvm_iommu_unmap_pages(kvm, &old); 1020 /* From this point no new shadow pages pointing to a deleted, 1021 * or moved, memslot will be created. 1022 * 1023 * validation of sp->gfn happens in: 1024 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 1025 * - kvm_is_visible_gfn (mmu_check_roots) 1026 */ 1027 kvm_arch_flush_shadow_memslot(kvm, slot); 1028 1029 /* 1030 * We can re-use the old_memslots from above, the only difference 1031 * from the currently installed memslots is the invalid flag. This 1032 * will get overwritten by update_memslots anyway. 1033 */ 1034 slots = old_memslots; 1035 } 1036 1037 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 1038 if (r) 1039 goto out_slots; 1040 1041 /* actual memory is freed via old in kvm_free_memslot below */ 1042 if (change == KVM_MR_DELETE) { 1043 new.dirty_bitmap = NULL; 1044 memset(&new.arch, 0, sizeof(new.arch)); 1045 } 1046 1047 update_memslots(slots, &new); 1048 old_memslots = install_new_memslots(kvm, as_id, slots); 1049 1050 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 1051 1052 kvm_free_memslot(kvm, &old, &new); 1053 kvfree(old_memslots); 1054 1055 /* 1056 * IOMMU mapping: New slots need to be mapped. Old slots need to be 1057 * un-mapped and re-mapped if their base changes. Since base change 1058 * unmapping is handled above with slot deletion, mapping alone is 1059 * needed here. Anything else the iommu might care about for existing 1060 * slots (size changes, userspace addr changes and read-only flag 1061 * changes) is disallowed above, so any other attribute changes getting 1062 * here can be skipped. 1063 */ 1064 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 1065 r = kvm_iommu_map_pages(kvm, &new); 1066 return r; 1067 } 1068 1069 return 0; 1070 1071 out_slots: 1072 kvfree(slots); 1073 out_free: 1074 kvm_free_memslot(kvm, &new, &old); 1075 out: 1076 return r; 1077 } 1078 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 1079 1080 int kvm_set_memory_region(struct kvm *kvm, 1081 const struct kvm_userspace_memory_region *mem) 1082 { 1083 int r; 1084 1085 mutex_lock(&kvm->slots_lock); 1086 r = __kvm_set_memory_region(kvm, mem); 1087 mutex_unlock(&kvm->slots_lock); 1088 return r; 1089 } 1090 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1091 1092 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1093 struct kvm_userspace_memory_region *mem) 1094 { 1095 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1096 return -EINVAL; 1097 1098 return kvm_set_memory_region(kvm, mem); 1099 } 1100 1101 int kvm_get_dirty_log(struct kvm *kvm, 1102 struct kvm_dirty_log *log, int *is_dirty) 1103 { 1104 struct kvm_memslots *slots; 1105 struct kvm_memory_slot *memslot; 1106 int i, as_id, id; 1107 unsigned long n; 1108 unsigned long any = 0; 1109 1110 as_id = log->slot >> 16; 1111 id = (u16)log->slot; 1112 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1113 return -EINVAL; 1114 1115 slots = __kvm_memslots(kvm, as_id); 1116 memslot = id_to_memslot(slots, id); 1117 if (!memslot->dirty_bitmap) 1118 return -ENOENT; 1119 1120 n = kvm_dirty_bitmap_bytes(memslot); 1121 1122 for (i = 0; !any && i < n/sizeof(long); ++i) 1123 any = memslot->dirty_bitmap[i]; 1124 1125 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1126 return -EFAULT; 1127 1128 if (any) 1129 *is_dirty = 1; 1130 return 0; 1131 } 1132 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1133 1134 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1135 /** 1136 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1137 * are dirty write protect them for next write. 1138 * @kvm: pointer to kvm instance 1139 * @log: slot id and address to which we copy the log 1140 * @is_dirty: flag set if any page is dirty 1141 * 1142 * We need to keep it in mind that VCPU threads can write to the bitmap 1143 * concurrently. So, to avoid losing track of dirty pages we keep the 1144 * following order: 1145 * 1146 * 1. Take a snapshot of the bit and clear it if needed. 1147 * 2. Write protect the corresponding page. 1148 * 3. Copy the snapshot to the userspace. 1149 * 4. Upon return caller flushes TLB's if needed. 1150 * 1151 * Between 2 and 4, the guest may write to the page using the remaining TLB 1152 * entry. This is not a problem because the page is reported dirty using 1153 * the snapshot taken before and step 4 ensures that writes done after 1154 * exiting to userspace will be logged for the next call. 1155 * 1156 */ 1157 int kvm_get_dirty_log_protect(struct kvm *kvm, 1158 struct kvm_dirty_log *log, bool *is_dirty) 1159 { 1160 struct kvm_memslots *slots; 1161 struct kvm_memory_slot *memslot; 1162 int i, as_id, id; 1163 unsigned long n; 1164 unsigned long *dirty_bitmap; 1165 unsigned long *dirty_bitmap_buffer; 1166 1167 as_id = log->slot >> 16; 1168 id = (u16)log->slot; 1169 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1170 return -EINVAL; 1171 1172 slots = __kvm_memslots(kvm, as_id); 1173 memslot = id_to_memslot(slots, id); 1174 1175 dirty_bitmap = memslot->dirty_bitmap; 1176 if (!dirty_bitmap) 1177 return -ENOENT; 1178 1179 n = kvm_dirty_bitmap_bytes(memslot); 1180 1181 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1182 memset(dirty_bitmap_buffer, 0, n); 1183 1184 spin_lock(&kvm->mmu_lock); 1185 *is_dirty = false; 1186 for (i = 0; i < n / sizeof(long); i++) { 1187 unsigned long mask; 1188 gfn_t offset; 1189 1190 if (!dirty_bitmap[i]) 1191 continue; 1192 1193 *is_dirty = true; 1194 1195 mask = xchg(&dirty_bitmap[i], 0); 1196 dirty_bitmap_buffer[i] = mask; 1197 1198 if (mask) { 1199 offset = i * BITS_PER_LONG; 1200 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1201 offset, mask); 1202 } 1203 } 1204 1205 spin_unlock(&kvm->mmu_lock); 1206 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1207 return -EFAULT; 1208 return 0; 1209 } 1210 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1211 #endif 1212 1213 bool kvm_largepages_enabled(void) 1214 { 1215 return largepages_enabled; 1216 } 1217 1218 void kvm_disable_largepages(void) 1219 { 1220 largepages_enabled = false; 1221 } 1222 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1223 1224 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1225 { 1226 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1227 } 1228 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1229 1230 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1231 { 1232 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1233 } 1234 1235 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1236 { 1237 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1238 1239 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1240 memslot->flags & KVM_MEMSLOT_INVALID) 1241 return false; 1242 1243 return true; 1244 } 1245 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1246 1247 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1248 { 1249 struct vm_area_struct *vma; 1250 unsigned long addr, size; 1251 1252 size = PAGE_SIZE; 1253 1254 addr = gfn_to_hva(kvm, gfn); 1255 if (kvm_is_error_hva(addr)) 1256 return PAGE_SIZE; 1257 1258 down_read(¤t->mm->mmap_sem); 1259 vma = find_vma(current->mm, addr); 1260 if (!vma) 1261 goto out; 1262 1263 size = vma_kernel_pagesize(vma); 1264 1265 out: 1266 up_read(¤t->mm->mmap_sem); 1267 1268 return size; 1269 } 1270 1271 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1272 { 1273 return slot->flags & KVM_MEM_READONLY; 1274 } 1275 1276 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1277 gfn_t *nr_pages, bool write) 1278 { 1279 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1280 return KVM_HVA_ERR_BAD; 1281 1282 if (memslot_is_readonly(slot) && write) 1283 return KVM_HVA_ERR_RO_BAD; 1284 1285 if (nr_pages) 1286 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1287 1288 return __gfn_to_hva_memslot(slot, gfn); 1289 } 1290 1291 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1292 gfn_t *nr_pages) 1293 { 1294 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1295 } 1296 1297 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1298 gfn_t gfn) 1299 { 1300 return gfn_to_hva_many(slot, gfn, NULL); 1301 } 1302 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1303 1304 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1305 { 1306 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1307 } 1308 EXPORT_SYMBOL_GPL(gfn_to_hva); 1309 1310 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1311 { 1312 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1313 } 1314 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1315 1316 /* 1317 * If writable is set to false, the hva returned by this function is only 1318 * allowed to be read. 1319 */ 1320 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1321 gfn_t gfn, bool *writable) 1322 { 1323 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1324 1325 if (!kvm_is_error_hva(hva) && writable) 1326 *writable = !memslot_is_readonly(slot); 1327 1328 return hva; 1329 } 1330 1331 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1332 { 1333 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1334 1335 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1336 } 1337 1338 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1339 { 1340 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1341 1342 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1343 } 1344 1345 static int get_user_page_nowait(unsigned long start, int write, 1346 struct page **page) 1347 { 1348 int flags = FOLL_NOWAIT | FOLL_HWPOISON; 1349 1350 if (write) 1351 flags |= FOLL_WRITE; 1352 1353 return get_user_pages(start, 1, flags, page, NULL); 1354 } 1355 1356 static inline int check_user_page_hwpoison(unsigned long addr) 1357 { 1358 int rc, flags = FOLL_HWPOISON | FOLL_WRITE; 1359 1360 rc = get_user_pages(addr, 1, flags, NULL, NULL); 1361 return rc == -EHWPOISON; 1362 } 1363 1364 /* 1365 * The atomic path to get the writable pfn which will be stored in @pfn, 1366 * true indicates success, otherwise false is returned. 1367 */ 1368 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1369 bool write_fault, bool *writable, kvm_pfn_t *pfn) 1370 { 1371 struct page *page[1]; 1372 int npages; 1373 1374 if (!(async || atomic)) 1375 return false; 1376 1377 /* 1378 * Fast pin a writable pfn only if it is a write fault request 1379 * or the caller allows to map a writable pfn for a read fault 1380 * request. 1381 */ 1382 if (!(write_fault || writable)) 1383 return false; 1384 1385 npages = __get_user_pages_fast(addr, 1, 1, page); 1386 if (npages == 1) { 1387 *pfn = page_to_pfn(page[0]); 1388 1389 if (writable) 1390 *writable = true; 1391 return true; 1392 } 1393 1394 return false; 1395 } 1396 1397 /* 1398 * The slow path to get the pfn of the specified host virtual address, 1399 * 1 indicates success, -errno is returned if error is detected. 1400 */ 1401 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1402 bool *writable, kvm_pfn_t *pfn) 1403 { 1404 struct page *page[1]; 1405 int npages = 0; 1406 1407 might_sleep(); 1408 1409 if (writable) 1410 *writable = write_fault; 1411 1412 if (async) { 1413 down_read(¤t->mm->mmap_sem); 1414 npages = get_user_page_nowait(addr, write_fault, page); 1415 up_read(¤t->mm->mmap_sem); 1416 } else { 1417 unsigned int flags = FOLL_HWPOISON; 1418 1419 if (write_fault) 1420 flags |= FOLL_WRITE; 1421 1422 npages = get_user_pages_unlocked(addr, 1, page, flags); 1423 } 1424 if (npages != 1) 1425 return npages; 1426 1427 /* map read fault as writable if possible */ 1428 if (unlikely(!write_fault) && writable) { 1429 struct page *wpage[1]; 1430 1431 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1432 if (npages == 1) { 1433 *writable = true; 1434 put_page(page[0]); 1435 page[0] = wpage[0]; 1436 } 1437 1438 npages = 1; 1439 } 1440 *pfn = page_to_pfn(page[0]); 1441 return npages; 1442 } 1443 1444 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1445 { 1446 if (unlikely(!(vma->vm_flags & VM_READ))) 1447 return false; 1448 1449 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1450 return false; 1451 1452 return true; 1453 } 1454 1455 static int hva_to_pfn_remapped(struct vm_area_struct *vma, 1456 unsigned long addr, bool *async, 1457 bool write_fault, kvm_pfn_t *p_pfn) 1458 { 1459 unsigned long pfn; 1460 int r; 1461 1462 r = follow_pfn(vma, addr, &pfn); 1463 if (r) { 1464 /* 1465 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does 1466 * not call the fault handler, so do it here. 1467 */ 1468 bool unlocked = false; 1469 r = fixup_user_fault(current, current->mm, addr, 1470 (write_fault ? FAULT_FLAG_WRITE : 0), 1471 &unlocked); 1472 if (unlocked) 1473 return -EAGAIN; 1474 if (r) 1475 return r; 1476 1477 r = follow_pfn(vma, addr, &pfn); 1478 if (r) 1479 return r; 1480 1481 } 1482 1483 1484 /* 1485 * Get a reference here because callers of *hva_to_pfn* and 1486 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the 1487 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP 1488 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will 1489 * simply do nothing for reserved pfns. 1490 * 1491 * Whoever called remap_pfn_range is also going to call e.g. 1492 * unmap_mapping_range before the underlying pages are freed, 1493 * causing a call to our MMU notifier. 1494 */ 1495 kvm_get_pfn(pfn); 1496 1497 *p_pfn = pfn; 1498 return 0; 1499 } 1500 1501 /* 1502 * Pin guest page in memory and return its pfn. 1503 * @addr: host virtual address which maps memory to the guest 1504 * @atomic: whether this function can sleep 1505 * @async: whether this function need to wait IO complete if the 1506 * host page is not in the memory 1507 * @write_fault: whether we should get a writable host page 1508 * @writable: whether it allows to map a writable host page for !@write_fault 1509 * 1510 * The function will map a writable host page for these two cases: 1511 * 1): @write_fault = true 1512 * 2): @write_fault = false && @writable, @writable will tell the caller 1513 * whether the mapping is writable. 1514 */ 1515 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1516 bool write_fault, bool *writable) 1517 { 1518 struct vm_area_struct *vma; 1519 kvm_pfn_t pfn = 0; 1520 int npages, r; 1521 1522 /* we can do it either atomically or asynchronously, not both */ 1523 BUG_ON(atomic && async); 1524 1525 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1526 return pfn; 1527 1528 if (atomic) 1529 return KVM_PFN_ERR_FAULT; 1530 1531 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1532 if (npages == 1) 1533 return pfn; 1534 1535 down_read(¤t->mm->mmap_sem); 1536 if (npages == -EHWPOISON || 1537 (!async && check_user_page_hwpoison(addr))) { 1538 pfn = KVM_PFN_ERR_HWPOISON; 1539 goto exit; 1540 } 1541 1542 retry: 1543 vma = find_vma_intersection(current->mm, addr, addr + 1); 1544 1545 if (vma == NULL) 1546 pfn = KVM_PFN_ERR_FAULT; 1547 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { 1548 r = hva_to_pfn_remapped(vma, addr, async, write_fault, &pfn); 1549 if (r == -EAGAIN) 1550 goto retry; 1551 if (r < 0) 1552 pfn = KVM_PFN_ERR_FAULT; 1553 } else { 1554 if (async && vma_is_valid(vma, write_fault)) 1555 *async = true; 1556 pfn = KVM_PFN_ERR_FAULT; 1557 } 1558 exit: 1559 up_read(¤t->mm->mmap_sem); 1560 return pfn; 1561 } 1562 1563 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 1564 bool atomic, bool *async, bool write_fault, 1565 bool *writable) 1566 { 1567 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1568 1569 if (addr == KVM_HVA_ERR_RO_BAD) { 1570 if (writable) 1571 *writable = false; 1572 return KVM_PFN_ERR_RO_FAULT; 1573 } 1574 1575 if (kvm_is_error_hva(addr)) { 1576 if (writable) 1577 *writable = false; 1578 return KVM_PFN_NOSLOT; 1579 } 1580 1581 /* Do not map writable pfn in the readonly memslot. */ 1582 if (writable && memslot_is_readonly(slot)) { 1583 *writable = false; 1584 writable = NULL; 1585 } 1586 1587 return hva_to_pfn(addr, atomic, async, write_fault, 1588 writable); 1589 } 1590 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1591 1592 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1593 bool *writable) 1594 { 1595 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1596 write_fault, writable); 1597 } 1598 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1599 1600 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1601 { 1602 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1603 } 1604 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1605 1606 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1607 { 1608 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1609 } 1610 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1611 1612 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1613 { 1614 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1615 } 1616 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1617 1618 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1619 { 1620 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1621 } 1622 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1623 1624 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1625 { 1626 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1627 } 1628 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1629 1630 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1631 { 1632 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1633 } 1634 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1635 1636 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1637 struct page **pages, int nr_pages) 1638 { 1639 unsigned long addr; 1640 gfn_t entry; 1641 1642 addr = gfn_to_hva_many(slot, gfn, &entry); 1643 if (kvm_is_error_hva(addr)) 1644 return -1; 1645 1646 if (entry < nr_pages) 1647 return 0; 1648 1649 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1650 } 1651 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1652 1653 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 1654 { 1655 if (is_error_noslot_pfn(pfn)) 1656 return KVM_ERR_PTR_BAD_PAGE; 1657 1658 if (kvm_is_reserved_pfn(pfn)) { 1659 WARN_ON(1); 1660 return KVM_ERR_PTR_BAD_PAGE; 1661 } 1662 1663 return pfn_to_page(pfn); 1664 } 1665 1666 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1667 { 1668 kvm_pfn_t pfn; 1669 1670 pfn = gfn_to_pfn(kvm, gfn); 1671 1672 return kvm_pfn_to_page(pfn); 1673 } 1674 EXPORT_SYMBOL_GPL(gfn_to_page); 1675 1676 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1677 { 1678 kvm_pfn_t pfn; 1679 1680 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1681 1682 return kvm_pfn_to_page(pfn); 1683 } 1684 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1685 1686 void kvm_release_page_clean(struct page *page) 1687 { 1688 WARN_ON(is_error_page(page)); 1689 1690 kvm_release_pfn_clean(page_to_pfn(page)); 1691 } 1692 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1693 1694 void kvm_release_pfn_clean(kvm_pfn_t pfn) 1695 { 1696 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1697 put_page(pfn_to_page(pfn)); 1698 } 1699 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1700 1701 void kvm_release_page_dirty(struct page *page) 1702 { 1703 WARN_ON(is_error_page(page)); 1704 1705 kvm_release_pfn_dirty(page_to_pfn(page)); 1706 } 1707 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1708 1709 static void kvm_release_pfn_dirty(kvm_pfn_t pfn) 1710 { 1711 kvm_set_pfn_dirty(pfn); 1712 kvm_release_pfn_clean(pfn); 1713 } 1714 1715 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 1716 { 1717 if (!kvm_is_reserved_pfn(pfn)) { 1718 struct page *page = pfn_to_page(pfn); 1719 1720 if (!PageReserved(page)) 1721 SetPageDirty(page); 1722 } 1723 } 1724 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1725 1726 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 1727 { 1728 if (!kvm_is_reserved_pfn(pfn)) 1729 mark_page_accessed(pfn_to_page(pfn)); 1730 } 1731 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1732 1733 void kvm_get_pfn(kvm_pfn_t pfn) 1734 { 1735 if (!kvm_is_reserved_pfn(pfn)) 1736 get_page(pfn_to_page(pfn)); 1737 } 1738 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1739 1740 static int next_segment(unsigned long len, int offset) 1741 { 1742 if (len > PAGE_SIZE - offset) 1743 return PAGE_SIZE - offset; 1744 else 1745 return len; 1746 } 1747 1748 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1749 void *data, int offset, int len) 1750 { 1751 int r; 1752 unsigned long addr; 1753 1754 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1755 if (kvm_is_error_hva(addr)) 1756 return -EFAULT; 1757 r = __copy_from_user(data, (void __user *)addr + offset, len); 1758 if (r) 1759 return -EFAULT; 1760 return 0; 1761 } 1762 1763 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1764 int len) 1765 { 1766 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1767 1768 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1769 } 1770 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1771 1772 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1773 int offset, int len) 1774 { 1775 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1776 1777 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1778 } 1779 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1780 1781 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1782 { 1783 gfn_t gfn = gpa >> PAGE_SHIFT; 1784 int seg; 1785 int offset = offset_in_page(gpa); 1786 int ret; 1787 1788 while ((seg = next_segment(len, offset)) != 0) { 1789 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1790 if (ret < 0) 1791 return ret; 1792 offset = 0; 1793 len -= seg; 1794 data += seg; 1795 ++gfn; 1796 } 1797 return 0; 1798 } 1799 EXPORT_SYMBOL_GPL(kvm_read_guest); 1800 1801 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1802 { 1803 gfn_t gfn = gpa >> PAGE_SHIFT; 1804 int seg; 1805 int offset = offset_in_page(gpa); 1806 int ret; 1807 1808 while ((seg = next_segment(len, offset)) != 0) { 1809 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1810 if (ret < 0) 1811 return ret; 1812 offset = 0; 1813 len -= seg; 1814 data += seg; 1815 ++gfn; 1816 } 1817 return 0; 1818 } 1819 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1820 1821 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1822 void *data, int offset, unsigned long len) 1823 { 1824 int r; 1825 unsigned long addr; 1826 1827 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1828 if (kvm_is_error_hva(addr)) 1829 return -EFAULT; 1830 pagefault_disable(); 1831 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1832 pagefault_enable(); 1833 if (r) 1834 return -EFAULT; 1835 return 0; 1836 } 1837 1838 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1839 unsigned long len) 1840 { 1841 gfn_t gfn = gpa >> PAGE_SHIFT; 1842 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1843 int offset = offset_in_page(gpa); 1844 1845 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1846 } 1847 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1848 1849 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1850 void *data, unsigned long len) 1851 { 1852 gfn_t gfn = gpa >> PAGE_SHIFT; 1853 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1854 int offset = offset_in_page(gpa); 1855 1856 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1857 } 1858 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1859 1860 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1861 const void *data, int offset, int len) 1862 { 1863 int r; 1864 unsigned long addr; 1865 1866 addr = gfn_to_hva_memslot(memslot, gfn); 1867 if (kvm_is_error_hva(addr)) 1868 return -EFAULT; 1869 r = __copy_to_user((void __user *)addr + offset, data, len); 1870 if (r) 1871 return -EFAULT; 1872 mark_page_dirty_in_slot(memslot, gfn); 1873 return 0; 1874 } 1875 1876 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1877 const void *data, int offset, int len) 1878 { 1879 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1880 1881 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1882 } 1883 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1884 1885 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1886 const void *data, int offset, int len) 1887 { 1888 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1889 1890 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1891 } 1892 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1893 1894 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1895 unsigned long len) 1896 { 1897 gfn_t gfn = gpa >> PAGE_SHIFT; 1898 int seg; 1899 int offset = offset_in_page(gpa); 1900 int ret; 1901 1902 while ((seg = next_segment(len, offset)) != 0) { 1903 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1904 if (ret < 0) 1905 return ret; 1906 offset = 0; 1907 len -= seg; 1908 data += seg; 1909 ++gfn; 1910 } 1911 return 0; 1912 } 1913 EXPORT_SYMBOL_GPL(kvm_write_guest); 1914 1915 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1916 unsigned long len) 1917 { 1918 gfn_t gfn = gpa >> PAGE_SHIFT; 1919 int seg; 1920 int offset = offset_in_page(gpa); 1921 int ret; 1922 1923 while ((seg = next_segment(len, offset)) != 0) { 1924 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1925 if (ret < 0) 1926 return ret; 1927 offset = 0; 1928 len -= seg; 1929 data += seg; 1930 ++gfn; 1931 } 1932 return 0; 1933 } 1934 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1935 1936 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots, 1937 struct gfn_to_hva_cache *ghc, 1938 gpa_t gpa, unsigned long len) 1939 { 1940 int offset = offset_in_page(gpa); 1941 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1942 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1943 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1944 gfn_t nr_pages_avail; 1945 1946 ghc->gpa = gpa; 1947 ghc->generation = slots->generation; 1948 ghc->len = len; 1949 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 1950 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1951 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1952 ghc->hva += offset; 1953 } else { 1954 /* 1955 * If the requested region crosses two memslots, we still 1956 * verify that the entire region is valid here. 1957 */ 1958 while (start_gfn <= end_gfn) { 1959 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 1960 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1961 &nr_pages_avail); 1962 if (kvm_is_error_hva(ghc->hva)) 1963 return -EFAULT; 1964 start_gfn += nr_pages_avail; 1965 } 1966 /* Use the slow path for cross page reads and writes. */ 1967 ghc->memslot = NULL; 1968 } 1969 return 0; 1970 } 1971 1972 int kvm_vcpu_gfn_to_hva_cache_init(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc, 1973 gpa_t gpa, unsigned long len) 1974 { 1975 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu); 1976 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len); 1977 } 1978 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva_cache_init); 1979 1980 int kvm_vcpu_write_guest_offset_cached(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc, 1981 void *data, int offset, unsigned long len) 1982 { 1983 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu); 1984 int r; 1985 gpa_t gpa = ghc->gpa + offset; 1986 1987 BUG_ON(len + offset > ghc->len); 1988 1989 if (slots->generation != ghc->generation) 1990 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len); 1991 1992 if (unlikely(!ghc->memslot)) 1993 return kvm_vcpu_write_guest(vcpu, gpa, data, len); 1994 1995 if (kvm_is_error_hva(ghc->hva)) 1996 return -EFAULT; 1997 1998 r = __copy_to_user((void __user *)ghc->hva + offset, data, len); 1999 if (r) 2000 return -EFAULT; 2001 mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT); 2002 2003 return 0; 2004 } 2005 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_offset_cached); 2006 2007 int kvm_vcpu_write_guest_cached(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc, 2008 void *data, unsigned long len) 2009 { 2010 return kvm_vcpu_write_guest_offset_cached(vcpu, ghc, data, 0, len); 2011 } 2012 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_cached); 2013 2014 int kvm_vcpu_read_guest_cached(struct kvm_vcpu *vcpu, struct gfn_to_hva_cache *ghc, 2015 void *data, unsigned long len) 2016 { 2017 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu); 2018 int r; 2019 2020 BUG_ON(len > ghc->len); 2021 2022 if (slots->generation != ghc->generation) 2023 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len); 2024 2025 if (unlikely(!ghc->memslot)) 2026 return kvm_vcpu_read_guest(vcpu, ghc->gpa, data, len); 2027 2028 if (kvm_is_error_hva(ghc->hva)) 2029 return -EFAULT; 2030 2031 r = __copy_from_user(data, (void __user *)ghc->hva, len); 2032 if (r) 2033 return -EFAULT; 2034 2035 return 0; 2036 } 2037 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_cached); 2038 2039 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 2040 { 2041 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 2042 2043 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 2044 } 2045 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 2046 2047 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 2048 { 2049 gfn_t gfn = gpa >> PAGE_SHIFT; 2050 int seg; 2051 int offset = offset_in_page(gpa); 2052 int ret; 2053 2054 while ((seg = next_segment(len, offset)) != 0) { 2055 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 2056 if (ret < 0) 2057 return ret; 2058 offset = 0; 2059 len -= seg; 2060 ++gfn; 2061 } 2062 return 0; 2063 } 2064 EXPORT_SYMBOL_GPL(kvm_clear_guest); 2065 2066 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 2067 gfn_t gfn) 2068 { 2069 if (memslot && memslot->dirty_bitmap) { 2070 unsigned long rel_gfn = gfn - memslot->base_gfn; 2071 2072 set_bit_le(rel_gfn, memslot->dirty_bitmap); 2073 } 2074 } 2075 2076 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 2077 { 2078 struct kvm_memory_slot *memslot; 2079 2080 memslot = gfn_to_memslot(kvm, gfn); 2081 mark_page_dirty_in_slot(memslot, gfn); 2082 } 2083 EXPORT_SYMBOL_GPL(mark_page_dirty); 2084 2085 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 2086 { 2087 struct kvm_memory_slot *memslot; 2088 2089 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2090 mark_page_dirty_in_slot(memslot, gfn); 2091 } 2092 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 2093 2094 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 2095 { 2096 unsigned int old, val, grow; 2097 2098 old = val = vcpu->halt_poll_ns; 2099 grow = READ_ONCE(halt_poll_ns_grow); 2100 /* 10us base */ 2101 if (val == 0 && grow) 2102 val = 10000; 2103 else 2104 val *= grow; 2105 2106 if (val > halt_poll_ns) 2107 val = halt_poll_ns; 2108 2109 vcpu->halt_poll_ns = val; 2110 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 2111 } 2112 2113 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 2114 { 2115 unsigned int old, val, shrink; 2116 2117 old = val = vcpu->halt_poll_ns; 2118 shrink = READ_ONCE(halt_poll_ns_shrink); 2119 if (shrink == 0) 2120 val = 0; 2121 else 2122 val /= shrink; 2123 2124 vcpu->halt_poll_ns = val; 2125 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 2126 } 2127 2128 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 2129 { 2130 if (kvm_arch_vcpu_runnable(vcpu)) { 2131 kvm_make_request(KVM_REQ_UNHALT, vcpu); 2132 return -EINTR; 2133 } 2134 if (kvm_cpu_has_pending_timer(vcpu)) 2135 return -EINTR; 2136 if (signal_pending(current)) 2137 return -EINTR; 2138 2139 return 0; 2140 } 2141 2142 /* 2143 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2144 */ 2145 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2146 { 2147 ktime_t start, cur; 2148 DECLARE_SWAITQUEUE(wait); 2149 bool waited = false; 2150 u64 block_ns; 2151 2152 start = cur = ktime_get(); 2153 if (vcpu->halt_poll_ns) { 2154 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2155 2156 ++vcpu->stat.halt_attempted_poll; 2157 do { 2158 /* 2159 * This sets KVM_REQ_UNHALT if an interrupt 2160 * arrives. 2161 */ 2162 if (kvm_vcpu_check_block(vcpu) < 0) { 2163 ++vcpu->stat.halt_successful_poll; 2164 if (!vcpu_valid_wakeup(vcpu)) 2165 ++vcpu->stat.halt_poll_invalid; 2166 goto out; 2167 } 2168 cur = ktime_get(); 2169 } while (single_task_running() && ktime_before(cur, stop)); 2170 } 2171 2172 kvm_arch_vcpu_blocking(vcpu); 2173 2174 for (;;) { 2175 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2176 2177 if (kvm_vcpu_check_block(vcpu) < 0) 2178 break; 2179 2180 waited = true; 2181 schedule(); 2182 } 2183 2184 finish_swait(&vcpu->wq, &wait); 2185 cur = ktime_get(); 2186 2187 kvm_arch_vcpu_unblocking(vcpu); 2188 out: 2189 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2190 2191 if (!vcpu_valid_wakeup(vcpu)) 2192 shrink_halt_poll_ns(vcpu); 2193 else if (halt_poll_ns) { 2194 if (block_ns <= vcpu->halt_poll_ns) 2195 ; 2196 /* we had a long block, shrink polling */ 2197 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2198 shrink_halt_poll_ns(vcpu); 2199 /* we had a short halt and our poll time is too small */ 2200 else if (vcpu->halt_poll_ns < halt_poll_ns && 2201 block_ns < halt_poll_ns) 2202 grow_halt_poll_ns(vcpu); 2203 } else 2204 vcpu->halt_poll_ns = 0; 2205 2206 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu)); 2207 kvm_arch_vcpu_block_finish(vcpu); 2208 } 2209 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2210 2211 #ifndef CONFIG_S390 2212 void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 2213 { 2214 struct swait_queue_head *wqp; 2215 2216 wqp = kvm_arch_vcpu_wq(vcpu); 2217 if (swait_active(wqp)) { 2218 swake_up(wqp); 2219 ++vcpu->stat.halt_wakeup; 2220 } 2221 2222 } 2223 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 2224 2225 /* 2226 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2227 */ 2228 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2229 { 2230 int me; 2231 int cpu = vcpu->cpu; 2232 2233 kvm_vcpu_wake_up(vcpu); 2234 me = get_cpu(); 2235 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2236 if (kvm_arch_vcpu_should_kick(vcpu)) 2237 smp_send_reschedule(cpu); 2238 put_cpu(); 2239 } 2240 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2241 #endif /* !CONFIG_S390 */ 2242 2243 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2244 { 2245 struct pid *pid; 2246 struct task_struct *task = NULL; 2247 int ret = 0; 2248 2249 rcu_read_lock(); 2250 pid = rcu_dereference(target->pid); 2251 if (pid) 2252 task = get_pid_task(pid, PIDTYPE_PID); 2253 rcu_read_unlock(); 2254 if (!task) 2255 return ret; 2256 ret = yield_to(task, 1); 2257 put_task_struct(task); 2258 2259 return ret; 2260 } 2261 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2262 2263 /* 2264 * Helper that checks whether a VCPU is eligible for directed yield. 2265 * Most eligible candidate to yield is decided by following heuristics: 2266 * 2267 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2268 * (preempted lock holder), indicated by @in_spin_loop. 2269 * Set at the beiginning and cleared at the end of interception/PLE handler. 2270 * 2271 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2272 * chance last time (mostly it has become eligible now since we have probably 2273 * yielded to lockholder in last iteration. This is done by toggling 2274 * @dy_eligible each time a VCPU checked for eligibility.) 2275 * 2276 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2277 * to preempted lock-holder could result in wrong VCPU selection and CPU 2278 * burning. Giving priority for a potential lock-holder increases lock 2279 * progress. 2280 * 2281 * Since algorithm is based on heuristics, accessing another VCPU data without 2282 * locking does not harm. It may result in trying to yield to same VCPU, fail 2283 * and continue with next VCPU and so on. 2284 */ 2285 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2286 { 2287 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2288 bool eligible; 2289 2290 eligible = !vcpu->spin_loop.in_spin_loop || 2291 vcpu->spin_loop.dy_eligible; 2292 2293 if (vcpu->spin_loop.in_spin_loop) 2294 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2295 2296 return eligible; 2297 #else 2298 return true; 2299 #endif 2300 } 2301 2302 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 2303 { 2304 struct kvm *kvm = me->kvm; 2305 struct kvm_vcpu *vcpu; 2306 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2307 int yielded = 0; 2308 int try = 3; 2309 int pass; 2310 int i; 2311 2312 kvm_vcpu_set_in_spin_loop(me, true); 2313 /* 2314 * We boost the priority of a VCPU that is runnable but not 2315 * currently running, because it got preempted by something 2316 * else and called schedule in __vcpu_run. Hopefully that 2317 * VCPU is holding the lock that we need and will release it. 2318 * We approximate round-robin by starting at the last boosted VCPU. 2319 */ 2320 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2321 kvm_for_each_vcpu(i, vcpu, kvm) { 2322 if (!pass && i <= last_boosted_vcpu) { 2323 i = last_boosted_vcpu; 2324 continue; 2325 } else if (pass && i > last_boosted_vcpu) 2326 break; 2327 if (!ACCESS_ONCE(vcpu->preempted)) 2328 continue; 2329 if (vcpu == me) 2330 continue; 2331 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2332 continue; 2333 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2334 continue; 2335 2336 yielded = kvm_vcpu_yield_to(vcpu); 2337 if (yielded > 0) { 2338 kvm->last_boosted_vcpu = i; 2339 break; 2340 } else if (yielded < 0) { 2341 try--; 2342 if (!try) 2343 break; 2344 } 2345 } 2346 } 2347 kvm_vcpu_set_in_spin_loop(me, false); 2348 2349 /* Ensure vcpu is not eligible during next spinloop */ 2350 kvm_vcpu_set_dy_eligible(me, false); 2351 } 2352 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2353 2354 static int kvm_vcpu_fault(struct vm_fault *vmf) 2355 { 2356 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data; 2357 struct page *page; 2358 2359 if (vmf->pgoff == 0) 2360 page = virt_to_page(vcpu->run); 2361 #ifdef CONFIG_X86 2362 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2363 page = virt_to_page(vcpu->arch.pio_data); 2364 #endif 2365 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2366 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2367 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2368 #endif 2369 else 2370 return kvm_arch_vcpu_fault(vcpu, vmf); 2371 get_page(page); 2372 vmf->page = page; 2373 return 0; 2374 } 2375 2376 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2377 .fault = kvm_vcpu_fault, 2378 }; 2379 2380 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2381 { 2382 vma->vm_ops = &kvm_vcpu_vm_ops; 2383 return 0; 2384 } 2385 2386 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2387 { 2388 struct kvm_vcpu *vcpu = filp->private_data; 2389 2390 debugfs_remove_recursive(vcpu->debugfs_dentry); 2391 kvm_put_kvm(vcpu->kvm); 2392 return 0; 2393 } 2394 2395 static struct file_operations kvm_vcpu_fops = { 2396 .release = kvm_vcpu_release, 2397 .unlocked_ioctl = kvm_vcpu_ioctl, 2398 #ifdef CONFIG_KVM_COMPAT 2399 .compat_ioctl = kvm_vcpu_compat_ioctl, 2400 #endif 2401 .mmap = kvm_vcpu_mmap, 2402 .llseek = noop_llseek, 2403 }; 2404 2405 /* 2406 * Allocates an inode for the vcpu. 2407 */ 2408 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2409 { 2410 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2411 } 2412 2413 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 2414 { 2415 char dir_name[ITOA_MAX_LEN * 2]; 2416 int ret; 2417 2418 if (!kvm_arch_has_vcpu_debugfs()) 2419 return 0; 2420 2421 if (!debugfs_initialized()) 2422 return 0; 2423 2424 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); 2425 vcpu->debugfs_dentry = debugfs_create_dir(dir_name, 2426 vcpu->kvm->debugfs_dentry); 2427 if (!vcpu->debugfs_dentry) 2428 return -ENOMEM; 2429 2430 ret = kvm_arch_create_vcpu_debugfs(vcpu); 2431 if (ret < 0) { 2432 debugfs_remove_recursive(vcpu->debugfs_dentry); 2433 return ret; 2434 } 2435 2436 return 0; 2437 } 2438 2439 /* 2440 * Creates some virtual cpus. Good luck creating more than one. 2441 */ 2442 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2443 { 2444 int r; 2445 struct kvm_vcpu *vcpu; 2446 2447 if (id >= KVM_MAX_VCPU_ID) 2448 return -EINVAL; 2449 2450 mutex_lock(&kvm->lock); 2451 if (kvm->created_vcpus == KVM_MAX_VCPUS) { 2452 mutex_unlock(&kvm->lock); 2453 return -EINVAL; 2454 } 2455 2456 kvm->created_vcpus++; 2457 mutex_unlock(&kvm->lock); 2458 2459 vcpu = kvm_arch_vcpu_create(kvm, id); 2460 if (IS_ERR(vcpu)) { 2461 r = PTR_ERR(vcpu); 2462 goto vcpu_decrement; 2463 } 2464 2465 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2466 2467 r = kvm_arch_vcpu_setup(vcpu); 2468 if (r) 2469 goto vcpu_destroy; 2470 2471 r = kvm_create_vcpu_debugfs(vcpu); 2472 if (r) 2473 goto vcpu_destroy; 2474 2475 mutex_lock(&kvm->lock); 2476 if (kvm_get_vcpu_by_id(kvm, id)) { 2477 r = -EEXIST; 2478 goto unlock_vcpu_destroy; 2479 } 2480 2481 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2482 2483 /* Now it's all set up, let userspace reach it */ 2484 kvm_get_kvm(kvm); 2485 r = create_vcpu_fd(vcpu); 2486 if (r < 0) { 2487 kvm_put_kvm(kvm); 2488 goto unlock_vcpu_destroy; 2489 } 2490 2491 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2492 2493 /* 2494 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2495 * before kvm->online_vcpu's incremented value. 2496 */ 2497 smp_wmb(); 2498 atomic_inc(&kvm->online_vcpus); 2499 2500 mutex_unlock(&kvm->lock); 2501 kvm_arch_vcpu_postcreate(vcpu); 2502 return r; 2503 2504 unlock_vcpu_destroy: 2505 mutex_unlock(&kvm->lock); 2506 debugfs_remove_recursive(vcpu->debugfs_dentry); 2507 vcpu_destroy: 2508 kvm_arch_vcpu_destroy(vcpu); 2509 vcpu_decrement: 2510 mutex_lock(&kvm->lock); 2511 kvm->created_vcpus--; 2512 mutex_unlock(&kvm->lock); 2513 return r; 2514 } 2515 2516 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2517 { 2518 if (sigset) { 2519 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2520 vcpu->sigset_active = 1; 2521 vcpu->sigset = *sigset; 2522 } else 2523 vcpu->sigset_active = 0; 2524 return 0; 2525 } 2526 2527 static long kvm_vcpu_ioctl(struct file *filp, 2528 unsigned int ioctl, unsigned long arg) 2529 { 2530 struct kvm_vcpu *vcpu = filp->private_data; 2531 void __user *argp = (void __user *)arg; 2532 int r; 2533 struct kvm_fpu *fpu = NULL; 2534 struct kvm_sregs *kvm_sregs = NULL; 2535 2536 if (vcpu->kvm->mm != current->mm) 2537 return -EIO; 2538 2539 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2540 return -EINVAL; 2541 2542 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2543 /* 2544 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2545 * so vcpu_load() would break it. 2546 */ 2547 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2548 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2549 #endif 2550 2551 2552 r = vcpu_load(vcpu); 2553 if (r) 2554 return r; 2555 switch (ioctl) { 2556 case KVM_RUN: 2557 r = -EINVAL; 2558 if (arg) 2559 goto out; 2560 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 2561 /* The thread running this VCPU changed. */ 2562 struct pid *oldpid = vcpu->pid; 2563 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2564 2565 rcu_assign_pointer(vcpu->pid, newpid); 2566 if (oldpid) 2567 synchronize_rcu(); 2568 put_pid(oldpid); 2569 } 2570 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2571 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2572 break; 2573 case KVM_GET_REGS: { 2574 struct kvm_regs *kvm_regs; 2575 2576 r = -ENOMEM; 2577 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2578 if (!kvm_regs) 2579 goto out; 2580 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2581 if (r) 2582 goto out_free1; 2583 r = -EFAULT; 2584 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2585 goto out_free1; 2586 r = 0; 2587 out_free1: 2588 kfree(kvm_regs); 2589 break; 2590 } 2591 case KVM_SET_REGS: { 2592 struct kvm_regs *kvm_regs; 2593 2594 r = -ENOMEM; 2595 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2596 if (IS_ERR(kvm_regs)) { 2597 r = PTR_ERR(kvm_regs); 2598 goto out; 2599 } 2600 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2601 kfree(kvm_regs); 2602 break; 2603 } 2604 case KVM_GET_SREGS: { 2605 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2606 r = -ENOMEM; 2607 if (!kvm_sregs) 2608 goto out; 2609 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2610 if (r) 2611 goto out; 2612 r = -EFAULT; 2613 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2614 goto out; 2615 r = 0; 2616 break; 2617 } 2618 case KVM_SET_SREGS: { 2619 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2620 if (IS_ERR(kvm_sregs)) { 2621 r = PTR_ERR(kvm_sregs); 2622 kvm_sregs = NULL; 2623 goto out; 2624 } 2625 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2626 break; 2627 } 2628 case KVM_GET_MP_STATE: { 2629 struct kvm_mp_state mp_state; 2630 2631 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2632 if (r) 2633 goto out; 2634 r = -EFAULT; 2635 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2636 goto out; 2637 r = 0; 2638 break; 2639 } 2640 case KVM_SET_MP_STATE: { 2641 struct kvm_mp_state mp_state; 2642 2643 r = -EFAULT; 2644 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2645 goto out; 2646 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2647 break; 2648 } 2649 case KVM_TRANSLATE: { 2650 struct kvm_translation tr; 2651 2652 r = -EFAULT; 2653 if (copy_from_user(&tr, argp, sizeof(tr))) 2654 goto out; 2655 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2656 if (r) 2657 goto out; 2658 r = -EFAULT; 2659 if (copy_to_user(argp, &tr, sizeof(tr))) 2660 goto out; 2661 r = 0; 2662 break; 2663 } 2664 case KVM_SET_GUEST_DEBUG: { 2665 struct kvm_guest_debug dbg; 2666 2667 r = -EFAULT; 2668 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2669 goto out; 2670 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2671 break; 2672 } 2673 case KVM_SET_SIGNAL_MASK: { 2674 struct kvm_signal_mask __user *sigmask_arg = argp; 2675 struct kvm_signal_mask kvm_sigmask; 2676 sigset_t sigset, *p; 2677 2678 p = NULL; 2679 if (argp) { 2680 r = -EFAULT; 2681 if (copy_from_user(&kvm_sigmask, argp, 2682 sizeof(kvm_sigmask))) 2683 goto out; 2684 r = -EINVAL; 2685 if (kvm_sigmask.len != sizeof(sigset)) 2686 goto out; 2687 r = -EFAULT; 2688 if (copy_from_user(&sigset, sigmask_arg->sigset, 2689 sizeof(sigset))) 2690 goto out; 2691 p = &sigset; 2692 } 2693 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2694 break; 2695 } 2696 case KVM_GET_FPU: { 2697 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2698 r = -ENOMEM; 2699 if (!fpu) 2700 goto out; 2701 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2702 if (r) 2703 goto out; 2704 r = -EFAULT; 2705 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2706 goto out; 2707 r = 0; 2708 break; 2709 } 2710 case KVM_SET_FPU: { 2711 fpu = memdup_user(argp, sizeof(*fpu)); 2712 if (IS_ERR(fpu)) { 2713 r = PTR_ERR(fpu); 2714 fpu = NULL; 2715 goto out; 2716 } 2717 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2718 break; 2719 } 2720 default: 2721 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2722 } 2723 out: 2724 vcpu_put(vcpu); 2725 kfree(fpu); 2726 kfree(kvm_sregs); 2727 return r; 2728 } 2729 2730 #ifdef CONFIG_KVM_COMPAT 2731 static long kvm_vcpu_compat_ioctl(struct file *filp, 2732 unsigned int ioctl, unsigned long arg) 2733 { 2734 struct kvm_vcpu *vcpu = filp->private_data; 2735 void __user *argp = compat_ptr(arg); 2736 int r; 2737 2738 if (vcpu->kvm->mm != current->mm) 2739 return -EIO; 2740 2741 switch (ioctl) { 2742 case KVM_SET_SIGNAL_MASK: { 2743 struct kvm_signal_mask __user *sigmask_arg = argp; 2744 struct kvm_signal_mask kvm_sigmask; 2745 compat_sigset_t csigset; 2746 sigset_t sigset; 2747 2748 if (argp) { 2749 r = -EFAULT; 2750 if (copy_from_user(&kvm_sigmask, argp, 2751 sizeof(kvm_sigmask))) 2752 goto out; 2753 r = -EINVAL; 2754 if (kvm_sigmask.len != sizeof(csigset)) 2755 goto out; 2756 r = -EFAULT; 2757 if (copy_from_user(&csigset, sigmask_arg->sigset, 2758 sizeof(csigset))) 2759 goto out; 2760 sigset_from_compat(&sigset, &csigset); 2761 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2762 } else 2763 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2764 break; 2765 } 2766 default: 2767 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2768 } 2769 2770 out: 2771 return r; 2772 } 2773 #endif 2774 2775 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2776 int (*accessor)(struct kvm_device *dev, 2777 struct kvm_device_attr *attr), 2778 unsigned long arg) 2779 { 2780 struct kvm_device_attr attr; 2781 2782 if (!accessor) 2783 return -EPERM; 2784 2785 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2786 return -EFAULT; 2787 2788 return accessor(dev, &attr); 2789 } 2790 2791 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2792 unsigned long arg) 2793 { 2794 struct kvm_device *dev = filp->private_data; 2795 2796 switch (ioctl) { 2797 case KVM_SET_DEVICE_ATTR: 2798 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2799 case KVM_GET_DEVICE_ATTR: 2800 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2801 case KVM_HAS_DEVICE_ATTR: 2802 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2803 default: 2804 if (dev->ops->ioctl) 2805 return dev->ops->ioctl(dev, ioctl, arg); 2806 2807 return -ENOTTY; 2808 } 2809 } 2810 2811 static int kvm_device_release(struct inode *inode, struct file *filp) 2812 { 2813 struct kvm_device *dev = filp->private_data; 2814 struct kvm *kvm = dev->kvm; 2815 2816 kvm_put_kvm(kvm); 2817 return 0; 2818 } 2819 2820 static const struct file_operations kvm_device_fops = { 2821 .unlocked_ioctl = kvm_device_ioctl, 2822 #ifdef CONFIG_KVM_COMPAT 2823 .compat_ioctl = kvm_device_ioctl, 2824 #endif 2825 .release = kvm_device_release, 2826 }; 2827 2828 struct kvm_device *kvm_device_from_filp(struct file *filp) 2829 { 2830 if (filp->f_op != &kvm_device_fops) 2831 return NULL; 2832 2833 return filp->private_data; 2834 } 2835 2836 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2837 #ifdef CONFIG_KVM_MPIC 2838 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2839 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2840 #endif 2841 2842 #ifdef CONFIG_KVM_XICS 2843 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2844 #endif 2845 }; 2846 2847 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2848 { 2849 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2850 return -ENOSPC; 2851 2852 if (kvm_device_ops_table[type] != NULL) 2853 return -EEXIST; 2854 2855 kvm_device_ops_table[type] = ops; 2856 return 0; 2857 } 2858 2859 void kvm_unregister_device_ops(u32 type) 2860 { 2861 if (kvm_device_ops_table[type] != NULL) 2862 kvm_device_ops_table[type] = NULL; 2863 } 2864 2865 static int kvm_ioctl_create_device(struct kvm *kvm, 2866 struct kvm_create_device *cd) 2867 { 2868 struct kvm_device_ops *ops = NULL; 2869 struct kvm_device *dev; 2870 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2871 int ret; 2872 2873 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2874 return -ENODEV; 2875 2876 ops = kvm_device_ops_table[cd->type]; 2877 if (ops == NULL) 2878 return -ENODEV; 2879 2880 if (test) 2881 return 0; 2882 2883 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2884 if (!dev) 2885 return -ENOMEM; 2886 2887 dev->ops = ops; 2888 dev->kvm = kvm; 2889 2890 mutex_lock(&kvm->lock); 2891 ret = ops->create(dev, cd->type); 2892 if (ret < 0) { 2893 mutex_unlock(&kvm->lock); 2894 kfree(dev); 2895 return ret; 2896 } 2897 list_add(&dev->vm_node, &kvm->devices); 2898 mutex_unlock(&kvm->lock); 2899 2900 if (ops->init) 2901 ops->init(dev); 2902 2903 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2904 if (ret < 0) { 2905 mutex_lock(&kvm->lock); 2906 list_del(&dev->vm_node); 2907 mutex_unlock(&kvm->lock); 2908 ops->destroy(dev); 2909 return ret; 2910 } 2911 2912 kvm_get_kvm(kvm); 2913 cd->fd = ret; 2914 return 0; 2915 } 2916 2917 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2918 { 2919 switch (arg) { 2920 case KVM_CAP_USER_MEMORY: 2921 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2922 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2923 case KVM_CAP_INTERNAL_ERROR_DATA: 2924 #ifdef CONFIG_HAVE_KVM_MSI 2925 case KVM_CAP_SIGNAL_MSI: 2926 #endif 2927 #ifdef CONFIG_HAVE_KVM_IRQFD 2928 case KVM_CAP_IRQFD: 2929 case KVM_CAP_IRQFD_RESAMPLE: 2930 #endif 2931 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 2932 case KVM_CAP_CHECK_EXTENSION_VM: 2933 return 1; 2934 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2935 case KVM_CAP_IRQ_ROUTING: 2936 return KVM_MAX_IRQ_ROUTES; 2937 #endif 2938 #if KVM_ADDRESS_SPACE_NUM > 1 2939 case KVM_CAP_MULTI_ADDRESS_SPACE: 2940 return KVM_ADDRESS_SPACE_NUM; 2941 #endif 2942 case KVM_CAP_MAX_VCPU_ID: 2943 return KVM_MAX_VCPU_ID; 2944 default: 2945 break; 2946 } 2947 return kvm_vm_ioctl_check_extension(kvm, arg); 2948 } 2949 2950 static long kvm_vm_ioctl(struct file *filp, 2951 unsigned int ioctl, unsigned long arg) 2952 { 2953 struct kvm *kvm = filp->private_data; 2954 void __user *argp = (void __user *)arg; 2955 int r; 2956 2957 if (kvm->mm != current->mm) 2958 return -EIO; 2959 switch (ioctl) { 2960 case KVM_CREATE_VCPU: 2961 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2962 break; 2963 case KVM_SET_USER_MEMORY_REGION: { 2964 struct kvm_userspace_memory_region kvm_userspace_mem; 2965 2966 r = -EFAULT; 2967 if (copy_from_user(&kvm_userspace_mem, argp, 2968 sizeof(kvm_userspace_mem))) 2969 goto out; 2970 2971 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2972 break; 2973 } 2974 case KVM_GET_DIRTY_LOG: { 2975 struct kvm_dirty_log log; 2976 2977 r = -EFAULT; 2978 if (copy_from_user(&log, argp, sizeof(log))) 2979 goto out; 2980 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2981 break; 2982 } 2983 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2984 case KVM_REGISTER_COALESCED_MMIO: { 2985 struct kvm_coalesced_mmio_zone zone; 2986 2987 r = -EFAULT; 2988 if (copy_from_user(&zone, argp, sizeof(zone))) 2989 goto out; 2990 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2991 break; 2992 } 2993 case KVM_UNREGISTER_COALESCED_MMIO: { 2994 struct kvm_coalesced_mmio_zone zone; 2995 2996 r = -EFAULT; 2997 if (copy_from_user(&zone, argp, sizeof(zone))) 2998 goto out; 2999 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 3000 break; 3001 } 3002 #endif 3003 case KVM_IRQFD: { 3004 struct kvm_irqfd data; 3005 3006 r = -EFAULT; 3007 if (copy_from_user(&data, argp, sizeof(data))) 3008 goto out; 3009 r = kvm_irqfd(kvm, &data); 3010 break; 3011 } 3012 case KVM_IOEVENTFD: { 3013 struct kvm_ioeventfd data; 3014 3015 r = -EFAULT; 3016 if (copy_from_user(&data, argp, sizeof(data))) 3017 goto out; 3018 r = kvm_ioeventfd(kvm, &data); 3019 break; 3020 } 3021 #ifdef CONFIG_HAVE_KVM_MSI 3022 case KVM_SIGNAL_MSI: { 3023 struct kvm_msi msi; 3024 3025 r = -EFAULT; 3026 if (copy_from_user(&msi, argp, sizeof(msi))) 3027 goto out; 3028 r = kvm_send_userspace_msi(kvm, &msi); 3029 break; 3030 } 3031 #endif 3032 #ifdef __KVM_HAVE_IRQ_LINE 3033 case KVM_IRQ_LINE_STATUS: 3034 case KVM_IRQ_LINE: { 3035 struct kvm_irq_level irq_event; 3036 3037 r = -EFAULT; 3038 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 3039 goto out; 3040 3041 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 3042 ioctl == KVM_IRQ_LINE_STATUS); 3043 if (r) 3044 goto out; 3045 3046 r = -EFAULT; 3047 if (ioctl == KVM_IRQ_LINE_STATUS) { 3048 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 3049 goto out; 3050 } 3051 3052 r = 0; 3053 break; 3054 } 3055 #endif 3056 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 3057 case KVM_SET_GSI_ROUTING: { 3058 struct kvm_irq_routing routing; 3059 struct kvm_irq_routing __user *urouting; 3060 struct kvm_irq_routing_entry *entries = NULL; 3061 3062 r = -EFAULT; 3063 if (copy_from_user(&routing, argp, sizeof(routing))) 3064 goto out; 3065 r = -EINVAL; 3066 if (routing.nr > KVM_MAX_IRQ_ROUTES) 3067 goto out; 3068 if (routing.flags) 3069 goto out; 3070 if (routing.nr) { 3071 r = -ENOMEM; 3072 entries = vmalloc(routing.nr * sizeof(*entries)); 3073 if (!entries) 3074 goto out; 3075 r = -EFAULT; 3076 urouting = argp; 3077 if (copy_from_user(entries, urouting->entries, 3078 routing.nr * sizeof(*entries))) 3079 goto out_free_irq_routing; 3080 } 3081 r = kvm_set_irq_routing(kvm, entries, routing.nr, 3082 routing.flags); 3083 out_free_irq_routing: 3084 vfree(entries); 3085 break; 3086 } 3087 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 3088 case KVM_CREATE_DEVICE: { 3089 struct kvm_create_device cd; 3090 3091 r = -EFAULT; 3092 if (copy_from_user(&cd, argp, sizeof(cd))) 3093 goto out; 3094 3095 r = kvm_ioctl_create_device(kvm, &cd); 3096 if (r) 3097 goto out; 3098 3099 r = -EFAULT; 3100 if (copy_to_user(argp, &cd, sizeof(cd))) 3101 goto out; 3102 3103 r = 0; 3104 break; 3105 } 3106 case KVM_CHECK_EXTENSION: 3107 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 3108 break; 3109 default: 3110 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 3111 } 3112 out: 3113 return r; 3114 } 3115 3116 #ifdef CONFIG_KVM_COMPAT 3117 struct compat_kvm_dirty_log { 3118 __u32 slot; 3119 __u32 padding1; 3120 union { 3121 compat_uptr_t dirty_bitmap; /* one bit per page */ 3122 __u64 padding2; 3123 }; 3124 }; 3125 3126 static long kvm_vm_compat_ioctl(struct file *filp, 3127 unsigned int ioctl, unsigned long arg) 3128 { 3129 struct kvm *kvm = filp->private_data; 3130 int r; 3131 3132 if (kvm->mm != current->mm) 3133 return -EIO; 3134 switch (ioctl) { 3135 case KVM_GET_DIRTY_LOG: { 3136 struct compat_kvm_dirty_log compat_log; 3137 struct kvm_dirty_log log; 3138 3139 if (copy_from_user(&compat_log, (void __user *)arg, 3140 sizeof(compat_log))) 3141 return -EFAULT; 3142 log.slot = compat_log.slot; 3143 log.padding1 = compat_log.padding1; 3144 log.padding2 = compat_log.padding2; 3145 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 3146 3147 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 3148 break; 3149 } 3150 default: 3151 r = kvm_vm_ioctl(filp, ioctl, arg); 3152 } 3153 return r; 3154 } 3155 #endif 3156 3157 static struct file_operations kvm_vm_fops = { 3158 .release = kvm_vm_release, 3159 .unlocked_ioctl = kvm_vm_ioctl, 3160 #ifdef CONFIG_KVM_COMPAT 3161 .compat_ioctl = kvm_vm_compat_ioctl, 3162 #endif 3163 .llseek = noop_llseek, 3164 }; 3165 3166 static int kvm_dev_ioctl_create_vm(unsigned long type) 3167 { 3168 int r; 3169 struct kvm *kvm; 3170 struct file *file; 3171 3172 kvm = kvm_create_vm(type); 3173 if (IS_ERR(kvm)) 3174 return PTR_ERR(kvm); 3175 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3176 r = kvm_coalesced_mmio_init(kvm); 3177 if (r < 0) { 3178 kvm_put_kvm(kvm); 3179 return r; 3180 } 3181 #endif 3182 r = get_unused_fd_flags(O_CLOEXEC); 3183 if (r < 0) { 3184 kvm_put_kvm(kvm); 3185 return r; 3186 } 3187 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 3188 if (IS_ERR(file)) { 3189 put_unused_fd(r); 3190 kvm_put_kvm(kvm); 3191 return PTR_ERR(file); 3192 } 3193 3194 if (kvm_create_vm_debugfs(kvm, r) < 0) { 3195 put_unused_fd(r); 3196 fput(file); 3197 return -ENOMEM; 3198 } 3199 3200 fd_install(r, file); 3201 return r; 3202 } 3203 3204 static long kvm_dev_ioctl(struct file *filp, 3205 unsigned int ioctl, unsigned long arg) 3206 { 3207 long r = -EINVAL; 3208 3209 switch (ioctl) { 3210 case KVM_GET_API_VERSION: 3211 if (arg) 3212 goto out; 3213 r = KVM_API_VERSION; 3214 break; 3215 case KVM_CREATE_VM: 3216 r = kvm_dev_ioctl_create_vm(arg); 3217 break; 3218 case KVM_CHECK_EXTENSION: 3219 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 3220 break; 3221 case KVM_GET_VCPU_MMAP_SIZE: 3222 if (arg) 3223 goto out; 3224 r = PAGE_SIZE; /* struct kvm_run */ 3225 #ifdef CONFIG_X86 3226 r += PAGE_SIZE; /* pio data page */ 3227 #endif 3228 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3229 r += PAGE_SIZE; /* coalesced mmio ring page */ 3230 #endif 3231 break; 3232 case KVM_TRACE_ENABLE: 3233 case KVM_TRACE_PAUSE: 3234 case KVM_TRACE_DISABLE: 3235 r = -EOPNOTSUPP; 3236 break; 3237 default: 3238 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3239 } 3240 out: 3241 return r; 3242 } 3243 3244 static struct file_operations kvm_chardev_ops = { 3245 .unlocked_ioctl = kvm_dev_ioctl, 3246 .compat_ioctl = kvm_dev_ioctl, 3247 .llseek = noop_llseek, 3248 }; 3249 3250 static struct miscdevice kvm_dev = { 3251 KVM_MINOR, 3252 "kvm", 3253 &kvm_chardev_ops, 3254 }; 3255 3256 static void hardware_enable_nolock(void *junk) 3257 { 3258 int cpu = raw_smp_processor_id(); 3259 int r; 3260 3261 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3262 return; 3263 3264 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3265 3266 r = kvm_arch_hardware_enable(); 3267 3268 if (r) { 3269 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3270 atomic_inc(&hardware_enable_failed); 3271 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3272 } 3273 } 3274 3275 static int kvm_starting_cpu(unsigned int cpu) 3276 { 3277 raw_spin_lock(&kvm_count_lock); 3278 if (kvm_usage_count) 3279 hardware_enable_nolock(NULL); 3280 raw_spin_unlock(&kvm_count_lock); 3281 return 0; 3282 } 3283 3284 static void hardware_disable_nolock(void *junk) 3285 { 3286 int cpu = raw_smp_processor_id(); 3287 3288 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3289 return; 3290 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3291 kvm_arch_hardware_disable(); 3292 } 3293 3294 static int kvm_dying_cpu(unsigned int cpu) 3295 { 3296 raw_spin_lock(&kvm_count_lock); 3297 if (kvm_usage_count) 3298 hardware_disable_nolock(NULL); 3299 raw_spin_unlock(&kvm_count_lock); 3300 return 0; 3301 } 3302 3303 static void hardware_disable_all_nolock(void) 3304 { 3305 BUG_ON(!kvm_usage_count); 3306 3307 kvm_usage_count--; 3308 if (!kvm_usage_count) 3309 on_each_cpu(hardware_disable_nolock, NULL, 1); 3310 } 3311 3312 static void hardware_disable_all(void) 3313 { 3314 raw_spin_lock(&kvm_count_lock); 3315 hardware_disable_all_nolock(); 3316 raw_spin_unlock(&kvm_count_lock); 3317 } 3318 3319 static int hardware_enable_all(void) 3320 { 3321 int r = 0; 3322 3323 raw_spin_lock(&kvm_count_lock); 3324 3325 kvm_usage_count++; 3326 if (kvm_usage_count == 1) { 3327 atomic_set(&hardware_enable_failed, 0); 3328 on_each_cpu(hardware_enable_nolock, NULL, 1); 3329 3330 if (atomic_read(&hardware_enable_failed)) { 3331 hardware_disable_all_nolock(); 3332 r = -EBUSY; 3333 } 3334 } 3335 3336 raw_spin_unlock(&kvm_count_lock); 3337 3338 return r; 3339 } 3340 3341 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3342 void *v) 3343 { 3344 /* 3345 * Some (well, at least mine) BIOSes hang on reboot if 3346 * in vmx root mode. 3347 * 3348 * And Intel TXT required VMX off for all cpu when system shutdown. 3349 */ 3350 pr_info("kvm: exiting hardware virtualization\n"); 3351 kvm_rebooting = true; 3352 on_each_cpu(hardware_disable_nolock, NULL, 1); 3353 return NOTIFY_OK; 3354 } 3355 3356 static struct notifier_block kvm_reboot_notifier = { 3357 .notifier_call = kvm_reboot, 3358 .priority = 0, 3359 }; 3360 3361 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3362 { 3363 int i; 3364 3365 for (i = 0; i < bus->dev_count; i++) { 3366 struct kvm_io_device *pos = bus->range[i].dev; 3367 3368 kvm_iodevice_destructor(pos); 3369 } 3370 kfree(bus); 3371 } 3372 3373 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3374 const struct kvm_io_range *r2) 3375 { 3376 gpa_t addr1 = r1->addr; 3377 gpa_t addr2 = r2->addr; 3378 3379 if (addr1 < addr2) 3380 return -1; 3381 3382 /* If r2->len == 0, match the exact address. If r2->len != 0, 3383 * accept any overlapping write. Any order is acceptable for 3384 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3385 * we process all of them. 3386 */ 3387 if (r2->len) { 3388 addr1 += r1->len; 3389 addr2 += r2->len; 3390 } 3391 3392 if (addr1 > addr2) 3393 return 1; 3394 3395 return 0; 3396 } 3397 3398 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3399 { 3400 return kvm_io_bus_cmp(p1, p2); 3401 } 3402 3403 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3404 gpa_t addr, int len) 3405 { 3406 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3407 .addr = addr, 3408 .len = len, 3409 .dev = dev, 3410 }; 3411 3412 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3413 kvm_io_bus_sort_cmp, NULL); 3414 3415 return 0; 3416 } 3417 3418 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3419 gpa_t addr, int len) 3420 { 3421 struct kvm_io_range *range, key; 3422 int off; 3423 3424 key = (struct kvm_io_range) { 3425 .addr = addr, 3426 .len = len, 3427 }; 3428 3429 range = bsearch(&key, bus->range, bus->dev_count, 3430 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3431 if (range == NULL) 3432 return -ENOENT; 3433 3434 off = range - bus->range; 3435 3436 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3437 off--; 3438 3439 return off; 3440 } 3441 3442 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3443 struct kvm_io_range *range, const void *val) 3444 { 3445 int idx; 3446 3447 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3448 if (idx < 0) 3449 return -EOPNOTSUPP; 3450 3451 while (idx < bus->dev_count && 3452 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3453 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3454 range->len, val)) 3455 return idx; 3456 idx++; 3457 } 3458 3459 return -EOPNOTSUPP; 3460 } 3461 3462 /* kvm_io_bus_write - called under kvm->slots_lock */ 3463 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3464 int len, const void *val) 3465 { 3466 struct kvm_io_bus *bus; 3467 struct kvm_io_range range; 3468 int r; 3469 3470 range = (struct kvm_io_range) { 3471 .addr = addr, 3472 .len = len, 3473 }; 3474 3475 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3476 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3477 return r < 0 ? r : 0; 3478 } 3479 3480 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3481 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3482 gpa_t addr, int len, const void *val, long cookie) 3483 { 3484 struct kvm_io_bus *bus; 3485 struct kvm_io_range range; 3486 3487 range = (struct kvm_io_range) { 3488 .addr = addr, 3489 .len = len, 3490 }; 3491 3492 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3493 3494 /* First try the device referenced by cookie. */ 3495 if ((cookie >= 0) && (cookie < bus->dev_count) && 3496 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3497 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3498 val)) 3499 return cookie; 3500 3501 /* 3502 * cookie contained garbage; fall back to search and return the 3503 * correct cookie value. 3504 */ 3505 return __kvm_io_bus_write(vcpu, bus, &range, val); 3506 } 3507 3508 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3509 struct kvm_io_range *range, void *val) 3510 { 3511 int idx; 3512 3513 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3514 if (idx < 0) 3515 return -EOPNOTSUPP; 3516 3517 while (idx < bus->dev_count && 3518 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3519 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3520 range->len, val)) 3521 return idx; 3522 idx++; 3523 } 3524 3525 return -EOPNOTSUPP; 3526 } 3527 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3528 3529 /* kvm_io_bus_read - called under kvm->slots_lock */ 3530 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3531 int len, void *val) 3532 { 3533 struct kvm_io_bus *bus; 3534 struct kvm_io_range range; 3535 int r; 3536 3537 range = (struct kvm_io_range) { 3538 .addr = addr, 3539 .len = len, 3540 }; 3541 3542 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3543 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3544 return r < 0 ? r : 0; 3545 } 3546 3547 3548 /* Caller must hold slots_lock. */ 3549 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3550 int len, struct kvm_io_device *dev) 3551 { 3552 struct kvm_io_bus *new_bus, *bus; 3553 3554 bus = kvm->buses[bus_idx]; 3555 /* exclude ioeventfd which is limited by maximum fd */ 3556 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3557 return -ENOSPC; 3558 3559 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3560 sizeof(struct kvm_io_range)), GFP_KERNEL); 3561 if (!new_bus) 3562 return -ENOMEM; 3563 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3564 sizeof(struct kvm_io_range))); 3565 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3566 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3567 synchronize_srcu_expedited(&kvm->srcu); 3568 kfree(bus); 3569 3570 return 0; 3571 } 3572 3573 /* Caller must hold slots_lock. */ 3574 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3575 struct kvm_io_device *dev) 3576 { 3577 int i, r; 3578 struct kvm_io_bus *new_bus, *bus; 3579 3580 bus = kvm->buses[bus_idx]; 3581 r = -ENOENT; 3582 for (i = 0; i < bus->dev_count; i++) 3583 if (bus->range[i].dev == dev) { 3584 r = 0; 3585 break; 3586 } 3587 3588 if (r) 3589 return r; 3590 3591 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3592 sizeof(struct kvm_io_range)), GFP_KERNEL); 3593 if (!new_bus) 3594 return -ENOMEM; 3595 3596 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3597 new_bus->dev_count--; 3598 memcpy(new_bus->range + i, bus->range + i + 1, 3599 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3600 3601 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3602 synchronize_srcu_expedited(&kvm->srcu); 3603 kfree(bus); 3604 return r; 3605 } 3606 3607 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3608 gpa_t addr) 3609 { 3610 struct kvm_io_bus *bus; 3611 int dev_idx, srcu_idx; 3612 struct kvm_io_device *iodev = NULL; 3613 3614 srcu_idx = srcu_read_lock(&kvm->srcu); 3615 3616 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3617 3618 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); 3619 if (dev_idx < 0) 3620 goto out_unlock; 3621 3622 iodev = bus->range[dev_idx].dev; 3623 3624 out_unlock: 3625 srcu_read_unlock(&kvm->srcu, srcu_idx); 3626 3627 return iodev; 3628 } 3629 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev); 3630 3631 static int kvm_debugfs_open(struct inode *inode, struct file *file, 3632 int (*get)(void *, u64 *), int (*set)(void *, u64), 3633 const char *fmt) 3634 { 3635 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3636 inode->i_private; 3637 3638 /* The debugfs files are a reference to the kvm struct which 3639 * is still valid when kvm_destroy_vm is called. 3640 * To avoid the race between open and the removal of the debugfs 3641 * directory we test against the users count. 3642 */ 3643 if (!atomic_add_unless(&stat_data->kvm->users_count, 1, 0)) 3644 return -ENOENT; 3645 3646 if (simple_attr_open(inode, file, get, set, fmt)) { 3647 kvm_put_kvm(stat_data->kvm); 3648 return -ENOMEM; 3649 } 3650 3651 return 0; 3652 } 3653 3654 static int kvm_debugfs_release(struct inode *inode, struct file *file) 3655 { 3656 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3657 inode->i_private; 3658 3659 simple_attr_release(inode, file); 3660 kvm_put_kvm(stat_data->kvm); 3661 3662 return 0; 3663 } 3664 3665 static int vm_stat_get_per_vm(void *data, u64 *val) 3666 { 3667 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3668 3669 *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset); 3670 3671 return 0; 3672 } 3673 3674 static int vm_stat_clear_per_vm(void *data, u64 val) 3675 { 3676 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3677 3678 if (val) 3679 return -EINVAL; 3680 3681 *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0; 3682 3683 return 0; 3684 } 3685 3686 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file) 3687 { 3688 __simple_attr_check_format("%llu\n", 0ull); 3689 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm, 3690 vm_stat_clear_per_vm, "%llu\n"); 3691 } 3692 3693 static const struct file_operations vm_stat_get_per_vm_fops = { 3694 .owner = THIS_MODULE, 3695 .open = vm_stat_get_per_vm_open, 3696 .release = kvm_debugfs_release, 3697 .read = simple_attr_read, 3698 .write = simple_attr_write, 3699 .llseek = generic_file_llseek, 3700 }; 3701 3702 static int vcpu_stat_get_per_vm(void *data, u64 *val) 3703 { 3704 int i; 3705 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3706 struct kvm_vcpu *vcpu; 3707 3708 *val = 0; 3709 3710 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3711 *val += *(u64 *)((void *)vcpu + stat_data->offset); 3712 3713 return 0; 3714 } 3715 3716 static int vcpu_stat_clear_per_vm(void *data, u64 val) 3717 { 3718 int i; 3719 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3720 struct kvm_vcpu *vcpu; 3721 3722 if (val) 3723 return -EINVAL; 3724 3725 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3726 *(u64 *)((void *)vcpu + stat_data->offset) = 0; 3727 3728 return 0; 3729 } 3730 3731 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file) 3732 { 3733 __simple_attr_check_format("%llu\n", 0ull); 3734 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm, 3735 vcpu_stat_clear_per_vm, "%llu\n"); 3736 } 3737 3738 static const struct file_operations vcpu_stat_get_per_vm_fops = { 3739 .owner = THIS_MODULE, 3740 .open = vcpu_stat_get_per_vm_open, 3741 .release = kvm_debugfs_release, 3742 .read = simple_attr_read, 3743 .write = simple_attr_write, 3744 .llseek = generic_file_llseek, 3745 }; 3746 3747 static const struct file_operations *stat_fops_per_vm[] = { 3748 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops, 3749 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops, 3750 }; 3751 3752 static int vm_stat_get(void *_offset, u64 *val) 3753 { 3754 unsigned offset = (long)_offset; 3755 struct kvm *kvm; 3756 struct kvm_stat_data stat_tmp = {.offset = offset}; 3757 u64 tmp_val; 3758 3759 *val = 0; 3760 spin_lock(&kvm_lock); 3761 list_for_each_entry(kvm, &vm_list, vm_list) { 3762 stat_tmp.kvm = kvm; 3763 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3764 *val += tmp_val; 3765 } 3766 spin_unlock(&kvm_lock); 3767 return 0; 3768 } 3769 3770 static int vm_stat_clear(void *_offset, u64 val) 3771 { 3772 unsigned offset = (long)_offset; 3773 struct kvm *kvm; 3774 struct kvm_stat_data stat_tmp = {.offset = offset}; 3775 3776 if (val) 3777 return -EINVAL; 3778 3779 spin_lock(&kvm_lock); 3780 list_for_each_entry(kvm, &vm_list, vm_list) { 3781 stat_tmp.kvm = kvm; 3782 vm_stat_clear_per_vm((void *)&stat_tmp, 0); 3783 } 3784 spin_unlock(&kvm_lock); 3785 3786 return 0; 3787 } 3788 3789 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); 3790 3791 static int vcpu_stat_get(void *_offset, u64 *val) 3792 { 3793 unsigned offset = (long)_offset; 3794 struct kvm *kvm; 3795 struct kvm_stat_data stat_tmp = {.offset = offset}; 3796 u64 tmp_val; 3797 3798 *val = 0; 3799 spin_lock(&kvm_lock); 3800 list_for_each_entry(kvm, &vm_list, vm_list) { 3801 stat_tmp.kvm = kvm; 3802 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3803 *val += tmp_val; 3804 } 3805 spin_unlock(&kvm_lock); 3806 return 0; 3807 } 3808 3809 static int vcpu_stat_clear(void *_offset, u64 val) 3810 { 3811 unsigned offset = (long)_offset; 3812 struct kvm *kvm; 3813 struct kvm_stat_data stat_tmp = {.offset = offset}; 3814 3815 if (val) 3816 return -EINVAL; 3817 3818 spin_lock(&kvm_lock); 3819 list_for_each_entry(kvm, &vm_list, vm_list) { 3820 stat_tmp.kvm = kvm; 3821 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0); 3822 } 3823 spin_unlock(&kvm_lock); 3824 3825 return 0; 3826 } 3827 3828 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, 3829 "%llu\n"); 3830 3831 static const struct file_operations *stat_fops[] = { 3832 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3833 [KVM_STAT_VM] = &vm_stat_fops, 3834 }; 3835 3836 static int kvm_init_debug(void) 3837 { 3838 int r = -EEXIST; 3839 struct kvm_stats_debugfs_item *p; 3840 3841 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3842 if (kvm_debugfs_dir == NULL) 3843 goto out; 3844 3845 kvm_debugfs_num_entries = 0; 3846 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { 3847 if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir, 3848 (void *)(long)p->offset, 3849 stat_fops[p->kind])) 3850 goto out_dir; 3851 } 3852 3853 return 0; 3854 3855 out_dir: 3856 debugfs_remove_recursive(kvm_debugfs_dir); 3857 out: 3858 return r; 3859 } 3860 3861 static int kvm_suspend(void) 3862 { 3863 if (kvm_usage_count) 3864 hardware_disable_nolock(NULL); 3865 return 0; 3866 } 3867 3868 static void kvm_resume(void) 3869 { 3870 if (kvm_usage_count) { 3871 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3872 hardware_enable_nolock(NULL); 3873 } 3874 } 3875 3876 static struct syscore_ops kvm_syscore_ops = { 3877 .suspend = kvm_suspend, 3878 .resume = kvm_resume, 3879 }; 3880 3881 static inline 3882 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3883 { 3884 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3885 } 3886 3887 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3888 { 3889 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3890 3891 if (vcpu->preempted) 3892 vcpu->preempted = false; 3893 3894 kvm_arch_sched_in(vcpu, cpu); 3895 3896 kvm_arch_vcpu_load(vcpu, cpu); 3897 } 3898 3899 static void kvm_sched_out(struct preempt_notifier *pn, 3900 struct task_struct *next) 3901 { 3902 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3903 3904 if (current->state == TASK_RUNNING) 3905 vcpu->preempted = true; 3906 kvm_arch_vcpu_put(vcpu); 3907 } 3908 3909 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3910 struct module *module) 3911 { 3912 int r; 3913 int cpu; 3914 3915 r = kvm_arch_init(opaque); 3916 if (r) 3917 goto out_fail; 3918 3919 /* 3920 * kvm_arch_init makes sure there's at most one caller 3921 * for architectures that support multiple implementations, 3922 * like intel and amd on x86. 3923 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3924 * conflicts in case kvm is already setup for another implementation. 3925 */ 3926 r = kvm_irqfd_init(); 3927 if (r) 3928 goto out_irqfd; 3929 3930 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3931 r = -ENOMEM; 3932 goto out_free_0; 3933 } 3934 3935 r = kvm_arch_hardware_setup(); 3936 if (r < 0) 3937 goto out_free_0a; 3938 3939 for_each_online_cpu(cpu) { 3940 smp_call_function_single(cpu, 3941 kvm_arch_check_processor_compat, 3942 &r, 1); 3943 if (r < 0) 3944 goto out_free_1; 3945 } 3946 3947 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting", 3948 kvm_starting_cpu, kvm_dying_cpu); 3949 if (r) 3950 goto out_free_2; 3951 register_reboot_notifier(&kvm_reboot_notifier); 3952 3953 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3954 if (!vcpu_align) 3955 vcpu_align = __alignof__(struct kvm_vcpu); 3956 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3957 0, NULL); 3958 if (!kvm_vcpu_cache) { 3959 r = -ENOMEM; 3960 goto out_free_3; 3961 } 3962 3963 r = kvm_async_pf_init(); 3964 if (r) 3965 goto out_free; 3966 3967 kvm_chardev_ops.owner = module; 3968 kvm_vm_fops.owner = module; 3969 kvm_vcpu_fops.owner = module; 3970 3971 r = misc_register(&kvm_dev); 3972 if (r) { 3973 pr_err("kvm: misc device register failed\n"); 3974 goto out_unreg; 3975 } 3976 3977 register_syscore_ops(&kvm_syscore_ops); 3978 3979 kvm_preempt_ops.sched_in = kvm_sched_in; 3980 kvm_preempt_ops.sched_out = kvm_sched_out; 3981 3982 r = kvm_init_debug(); 3983 if (r) { 3984 pr_err("kvm: create debugfs files failed\n"); 3985 goto out_undebugfs; 3986 } 3987 3988 r = kvm_vfio_ops_init(); 3989 WARN_ON(r); 3990 3991 return 0; 3992 3993 out_undebugfs: 3994 unregister_syscore_ops(&kvm_syscore_ops); 3995 misc_deregister(&kvm_dev); 3996 out_unreg: 3997 kvm_async_pf_deinit(); 3998 out_free: 3999 kmem_cache_destroy(kvm_vcpu_cache); 4000 out_free_3: 4001 unregister_reboot_notifier(&kvm_reboot_notifier); 4002 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4003 out_free_2: 4004 out_free_1: 4005 kvm_arch_hardware_unsetup(); 4006 out_free_0a: 4007 free_cpumask_var(cpus_hardware_enabled); 4008 out_free_0: 4009 kvm_irqfd_exit(); 4010 out_irqfd: 4011 kvm_arch_exit(); 4012 out_fail: 4013 return r; 4014 } 4015 EXPORT_SYMBOL_GPL(kvm_init); 4016 4017 void kvm_exit(void) 4018 { 4019 debugfs_remove_recursive(kvm_debugfs_dir); 4020 misc_deregister(&kvm_dev); 4021 kmem_cache_destroy(kvm_vcpu_cache); 4022 kvm_async_pf_deinit(); 4023 unregister_syscore_ops(&kvm_syscore_ops); 4024 unregister_reboot_notifier(&kvm_reboot_notifier); 4025 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4026 on_each_cpu(hardware_disable_nolock, NULL, 1); 4027 kvm_arch_hardware_unsetup(); 4028 kvm_arch_exit(); 4029 kvm_irqfd_exit(); 4030 free_cpumask_var(cpus_hardware_enabled); 4031 kvm_vfio_ops_exit(); 4032 } 4033 EXPORT_SYMBOL_GPL(kvm_exit); 4034