1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched/signal.h> 36 #include <linux/sched/mm.h> 37 #include <linux/sched/stat.h> 38 #include <linux/cpumask.h> 39 #include <linux/smp.h> 40 #include <linux/anon_inodes.h> 41 #include <linux/profile.h> 42 #include <linux/kvm_para.h> 43 #include <linux/pagemap.h> 44 #include <linux/mman.h> 45 #include <linux/swap.h> 46 #include <linux/bitops.h> 47 #include <linux/spinlock.h> 48 #include <linux/compat.h> 49 #include <linux/srcu.h> 50 #include <linux/hugetlb.h> 51 #include <linux/slab.h> 52 #include <linux/sort.h> 53 #include <linux/bsearch.h> 54 55 #include <asm/processor.h> 56 #include <asm/io.h> 57 #include <asm/ioctl.h> 58 #include <linux/uaccess.h> 59 #include <asm/pgtable.h> 60 61 #include "coalesced_mmio.h" 62 #include "async_pf.h" 63 #include "vfio.h" 64 65 #define CREATE_TRACE_POINTS 66 #include <trace/events/kvm.h> 67 68 /* Worst case buffer size needed for holding an integer. */ 69 #define ITOA_MAX_LEN 12 70 71 MODULE_AUTHOR("Qumranet"); 72 MODULE_LICENSE("GPL"); 73 74 /* Architectures should define their poll value according to the halt latency */ 75 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 76 module_param(halt_poll_ns, uint, 0644); 77 EXPORT_SYMBOL_GPL(halt_poll_ns); 78 79 /* Default doubles per-vcpu halt_poll_ns. */ 80 unsigned int halt_poll_ns_grow = 2; 81 module_param(halt_poll_ns_grow, uint, 0644); 82 EXPORT_SYMBOL_GPL(halt_poll_ns_grow); 83 84 /* Default resets per-vcpu halt_poll_ns . */ 85 unsigned int halt_poll_ns_shrink; 86 module_param(halt_poll_ns_shrink, uint, 0644); 87 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); 88 89 /* 90 * Ordering of locks: 91 * 92 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 93 */ 94 95 DEFINE_SPINLOCK(kvm_lock); 96 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 97 LIST_HEAD(vm_list); 98 99 static cpumask_var_t cpus_hardware_enabled; 100 static int kvm_usage_count; 101 static atomic_t hardware_enable_failed; 102 103 struct kmem_cache *kvm_vcpu_cache; 104 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 105 106 static __read_mostly struct preempt_ops kvm_preempt_ops; 107 108 struct dentry *kvm_debugfs_dir; 109 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 110 111 static int kvm_debugfs_num_entries; 112 static const struct file_operations *stat_fops_per_vm[]; 113 114 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 115 unsigned long arg); 116 #ifdef CONFIG_KVM_COMPAT 117 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 118 unsigned long arg); 119 #endif 120 static int hardware_enable_all(void); 121 static void hardware_disable_all(void); 122 123 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 124 125 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 126 127 __visible bool kvm_rebooting; 128 EXPORT_SYMBOL_GPL(kvm_rebooting); 129 130 static bool largepages_enabled = true; 131 132 #define KVM_EVENT_CREATE_VM 0 133 #define KVM_EVENT_DESTROY_VM 1 134 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm); 135 static unsigned long long kvm_createvm_count; 136 static unsigned long long kvm_active_vms; 137 138 __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, 139 unsigned long start, unsigned long end) 140 { 141 } 142 143 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 144 { 145 if (pfn_valid(pfn)) 146 return PageReserved(pfn_to_page(pfn)); 147 148 return true; 149 } 150 151 /* 152 * Switches to specified vcpu, until a matching vcpu_put() 153 */ 154 void vcpu_load(struct kvm_vcpu *vcpu) 155 { 156 int cpu = get_cpu(); 157 preempt_notifier_register(&vcpu->preempt_notifier); 158 kvm_arch_vcpu_load(vcpu, cpu); 159 put_cpu(); 160 } 161 EXPORT_SYMBOL_GPL(vcpu_load); 162 163 void vcpu_put(struct kvm_vcpu *vcpu) 164 { 165 preempt_disable(); 166 kvm_arch_vcpu_put(vcpu); 167 preempt_notifier_unregister(&vcpu->preempt_notifier); 168 preempt_enable(); 169 } 170 EXPORT_SYMBOL_GPL(vcpu_put); 171 172 /* TODO: merge with kvm_arch_vcpu_should_kick */ 173 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req) 174 { 175 int mode = kvm_vcpu_exiting_guest_mode(vcpu); 176 177 /* 178 * We need to wait for the VCPU to reenable interrupts and get out of 179 * READING_SHADOW_PAGE_TABLES mode. 180 */ 181 if (req & KVM_REQUEST_WAIT) 182 return mode != OUTSIDE_GUEST_MODE; 183 184 /* 185 * Need to kick a running VCPU, but otherwise there is nothing to do. 186 */ 187 return mode == IN_GUEST_MODE; 188 } 189 190 static void ack_flush(void *_completed) 191 { 192 } 193 194 static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait) 195 { 196 if (unlikely(!cpus)) 197 cpus = cpu_online_mask; 198 199 if (cpumask_empty(cpus)) 200 return false; 201 202 smp_call_function_many(cpus, ack_flush, NULL, wait); 203 return true; 204 } 205 206 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, 207 unsigned long *vcpu_bitmap, cpumask_var_t tmp) 208 { 209 int i, cpu, me; 210 struct kvm_vcpu *vcpu; 211 bool called; 212 213 me = get_cpu(); 214 215 kvm_for_each_vcpu(i, vcpu, kvm) { 216 if (!test_bit(i, vcpu_bitmap)) 217 continue; 218 219 kvm_make_request(req, vcpu); 220 cpu = vcpu->cpu; 221 222 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu)) 223 continue; 224 225 if (tmp != NULL && cpu != -1 && cpu != me && 226 kvm_request_needs_ipi(vcpu, req)) 227 __cpumask_set_cpu(cpu, tmp); 228 } 229 230 called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT)); 231 put_cpu(); 232 233 return called; 234 } 235 236 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 237 { 238 cpumask_var_t cpus; 239 bool called; 240 static unsigned long vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)] 241 = {[0 ... BITS_TO_LONGS(KVM_MAX_VCPUS)-1] = ULONG_MAX}; 242 243 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 244 245 called = kvm_make_vcpus_request_mask(kvm, req, vcpu_bitmap, cpus); 246 247 free_cpumask_var(cpus); 248 return called; 249 } 250 251 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 252 void kvm_flush_remote_tlbs(struct kvm *kvm) 253 { 254 /* 255 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 256 * kvm_make_all_cpus_request. 257 */ 258 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 259 260 /* 261 * We want to publish modifications to the page tables before reading 262 * mode. Pairs with a memory barrier in arch-specific code. 263 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 264 * and smp_mb in walk_shadow_page_lockless_begin/end. 265 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 266 * 267 * There is already an smp_mb__after_atomic() before 268 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 269 * barrier here. 270 */ 271 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 272 ++kvm->stat.remote_tlb_flush; 273 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 274 } 275 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 276 #endif 277 278 void kvm_reload_remote_mmus(struct kvm *kvm) 279 { 280 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 281 } 282 283 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 284 { 285 struct page *page; 286 int r; 287 288 mutex_init(&vcpu->mutex); 289 vcpu->cpu = -1; 290 vcpu->kvm = kvm; 291 vcpu->vcpu_id = id; 292 vcpu->pid = NULL; 293 init_swait_queue_head(&vcpu->wq); 294 kvm_async_pf_vcpu_init(vcpu); 295 296 vcpu->pre_pcpu = -1; 297 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 298 299 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 300 if (!page) { 301 r = -ENOMEM; 302 goto fail; 303 } 304 vcpu->run = page_address(page); 305 306 kvm_vcpu_set_in_spin_loop(vcpu, false); 307 kvm_vcpu_set_dy_eligible(vcpu, false); 308 vcpu->preempted = false; 309 310 r = kvm_arch_vcpu_init(vcpu); 311 if (r < 0) 312 goto fail_free_run; 313 return 0; 314 315 fail_free_run: 316 free_page((unsigned long)vcpu->run); 317 fail: 318 return r; 319 } 320 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 321 322 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 323 { 324 /* 325 * no need for rcu_read_lock as VCPU_RUN is the only place that 326 * will change the vcpu->pid pointer and on uninit all file 327 * descriptors are already gone. 328 */ 329 put_pid(rcu_dereference_protected(vcpu->pid, 1)); 330 kvm_arch_vcpu_uninit(vcpu); 331 free_page((unsigned long)vcpu->run); 332 } 333 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 334 335 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 336 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 337 { 338 return container_of(mn, struct kvm, mmu_notifier); 339 } 340 341 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 342 struct mm_struct *mm, 343 unsigned long address, 344 pte_t pte) 345 { 346 struct kvm *kvm = mmu_notifier_to_kvm(mn); 347 int idx; 348 349 idx = srcu_read_lock(&kvm->srcu); 350 spin_lock(&kvm->mmu_lock); 351 kvm->mmu_notifier_seq++; 352 kvm_set_spte_hva(kvm, address, pte); 353 spin_unlock(&kvm->mmu_lock); 354 srcu_read_unlock(&kvm->srcu, idx); 355 } 356 357 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 358 struct mm_struct *mm, 359 unsigned long start, 360 unsigned long end) 361 { 362 struct kvm *kvm = mmu_notifier_to_kvm(mn); 363 int need_tlb_flush = 0, idx; 364 365 idx = srcu_read_lock(&kvm->srcu); 366 spin_lock(&kvm->mmu_lock); 367 /* 368 * The count increase must become visible at unlock time as no 369 * spte can be established without taking the mmu_lock and 370 * count is also read inside the mmu_lock critical section. 371 */ 372 kvm->mmu_notifier_count++; 373 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 374 need_tlb_flush |= kvm->tlbs_dirty; 375 /* we've to flush the tlb before the pages can be freed */ 376 if (need_tlb_flush) 377 kvm_flush_remote_tlbs(kvm); 378 379 spin_unlock(&kvm->mmu_lock); 380 381 kvm_arch_mmu_notifier_invalidate_range(kvm, start, end); 382 383 srcu_read_unlock(&kvm->srcu, idx); 384 } 385 386 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 387 struct mm_struct *mm, 388 unsigned long start, 389 unsigned long end) 390 { 391 struct kvm *kvm = mmu_notifier_to_kvm(mn); 392 393 spin_lock(&kvm->mmu_lock); 394 /* 395 * This sequence increase will notify the kvm page fault that 396 * the page that is going to be mapped in the spte could have 397 * been freed. 398 */ 399 kvm->mmu_notifier_seq++; 400 smp_wmb(); 401 /* 402 * The above sequence increase must be visible before the 403 * below count decrease, which is ensured by the smp_wmb above 404 * in conjunction with the smp_rmb in mmu_notifier_retry(). 405 */ 406 kvm->mmu_notifier_count--; 407 spin_unlock(&kvm->mmu_lock); 408 409 BUG_ON(kvm->mmu_notifier_count < 0); 410 } 411 412 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 413 struct mm_struct *mm, 414 unsigned long start, 415 unsigned long end) 416 { 417 struct kvm *kvm = mmu_notifier_to_kvm(mn); 418 int young, idx; 419 420 idx = srcu_read_lock(&kvm->srcu); 421 spin_lock(&kvm->mmu_lock); 422 423 young = kvm_age_hva(kvm, start, end); 424 if (young) 425 kvm_flush_remote_tlbs(kvm); 426 427 spin_unlock(&kvm->mmu_lock); 428 srcu_read_unlock(&kvm->srcu, idx); 429 430 return young; 431 } 432 433 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 434 struct mm_struct *mm, 435 unsigned long start, 436 unsigned long end) 437 { 438 struct kvm *kvm = mmu_notifier_to_kvm(mn); 439 int young, idx; 440 441 idx = srcu_read_lock(&kvm->srcu); 442 spin_lock(&kvm->mmu_lock); 443 /* 444 * Even though we do not flush TLB, this will still adversely 445 * affect performance on pre-Haswell Intel EPT, where there is 446 * no EPT Access Bit to clear so that we have to tear down EPT 447 * tables instead. If we find this unacceptable, we can always 448 * add a parameter to kvm_age_hva so that it effectively doesn't 449 * do anything on clear_young. 450 * 451 * Also note that currently we never issue secondary TLB flushes 452 * from clear_young, leaving this job up to the regular system 453 * cadence. If we find this inaccurate, we might come up with a 454 * more sophisticated heuristic later. 455 */ 456 young = kvm_age_hva(kvm, start, end); 457 spin_unlock(&kvm->mmu_lock); 458 srcu_read_unlock(&kvm->srcu, idx); 459 460 return young; 461 } 462 463 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 464 struct mm_struct *mm, 465 unsigned long address) 466 { 467 struct kvm *kvm = mmu_notifier_to_kvm(mn); 468 int young, idx; 469 470 idx = srcu_read_lock(&kvm->srcu); 471 spin_lock(&kvm->mmu_lock); 472 young = kvm_test_age_hva(kvm, address); 473 spin_unlock(&kvm->mmu_lock); 474 srcu_read_unlock(&kvm->srcu, idx); 475 476 return young; 477 } 478 479 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 480 struct mm_struct *mm) 481 { 482 struct kvm *kvm = mmu_notifier_to_kvm(mn); 483 int idx; 484 485 idx = srcu_read_lock(&kvm->srcu); 486 kvm_arch_flush_shadow_all(kvm); 487 srcu_read_unlock(&kvm->srcu, idx); 488 } 489 490 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 491 .flags = MMU_INVALIDATE_DOES_NOT_BLOCK, 492 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 493 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 494 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 495 .clear_young = kvm_mmu_notifier_clear_young, 496 .test_young = kvm_mmu_notifier_test_young, 497 .change_pte = kvm_mmu_notifier_change_pte, 498 .release = kvm_mmu_notifier_release, 499 }; 500 501 static int kvm_init_mmu_notifier(struct kvm *kvm) 502 { 503 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 504 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 505 } 506 507 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 508 509 static int kvm_init_mmu_notifier(struct kvm *kvm) 510 { 511 return 0; 512 } 513 514 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 515 516 static struct kvm_memslots *kvm_alloc_memslots(void) 517 { 518 int i; 519 struct kvm_memslots *slots; 520 521 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 522 if (!slots) 523 return NULL; 524 525 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 526 slots->id_to_index[i] = slots->memslots[i].id = i; 527 528 return slots; 529 } 530 531 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 532 { 533 if (!memslot->dirty_bitmap) 534 return; 535 536 kvfree(memslot->dirty_bitmap); 537 memslot->dirty_bitmap = NULL; 538 } 539 540 /* 541 * Free any memory in @free but not in @dont. 542 */ 543 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 544 struct kvm_memory_slot *dont) 545 { 546 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 547 kvm_destroy_dirty_bitmap(free); 548 549 kvm_arch_free_memslot(kvm, free, dont); 550 551 free->npages = 0; 552 } 553 554 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 555 { 556 struct kvm_memory_slot *memslot; 557 558 if (!slots) 559 return; 560 561 kvm_for_each_memslot(memslot, slots) 562 kvm_free_memslot(kvm, memslot, NULL); 563 564 kvfree(slots); 565 } 566 567 static void kvm_destroy_vm_debugfs(struct kvm *kvm) 568 { 569 int i; 570 571 if (!kvm->debugfs_dentry) 572 return; 573 574 debugfs_remove_recursive(kvm->debugfs_dentry); 575 576 if (kvm->debugfs_stat_data) { 577 for (i = 0; i < kvm_debugfs_num_entries; i++) 578 kfree(kvm->debugfs_stat_data[i]); 579 kfree(kvm->debugfs_stat_data); 580 } 581 } 582 583 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) 584 { 585 char dir_name[ITOA_MAX_LEN * 2]; 586 struct kvm_stat_data *stat_data; 587 struct kvm_stats_debugfs_item *p; 588 589 if (!debugfs_initialized()) 590 return 0; 591 592 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd); 593 kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir); 594 595 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, 596 sizeof(*kvm->debugfs_stat_data), 597 GFP_KERNEL); 598 if (!kvm->debugfs_stat_data) 599 return -ENOMEM; 600 601 for (p = debugfs_entries; p->name; p++) { 602 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL); 603 if (!stat_data) 604 return -ENOMEM; 605 606 stat_data->kvm = kvm; 607 stat_data->offset = p->offset; 608 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; 609 debugfs_create_file(p->name, 0644, kvm->debugfs_dentry, 610 stat_data, stat_fops_per_vm[p->kind]); 611 } 612 return 0; 613 } 614 615 static struct kvm *kvm_create_vm(unsigned long type) 616 { 617 int r, i; 618 struct kvm *kvm = kvm_arch_alloc_vm(); 619 620 if (!kvm) 621 return ERR_PTR(-ENOMEM); 622 623 spin_lock_init(&kvm->mmu_lock); 624 mmgrab(current->mm); 625 kvm->mm = current->mm; 626 kvm_eventfd_init(kvm); 627 mutex_init(&kvm->lock); 628 mutex_init(&kvm->irq_lock); 629 mutex_init(&kvm->slots_lock); 630 refcount_set(&kvm->users_count, 1); 631 INIT_LIST_HEAD(&kvm->devices); 632 633 r = kvm_arch_init_vm(kvm, type); 634 if (r) 635 goto out_err_no_disable; 636 637 r = hardware_enable_all(); 638 if (r) 639 goto out_err_no_disable; 640 641 #ifdef CONFIG_HAVE_KVM_IRQFD 642 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 643 #endif 644 645 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 646 647 r = -ENOMEM; 648 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 649 struct kvm_memslots *slots = kvm_alloc_memslots(); 650 if (!slots) 651 goto out_err_no_srcu; 652 /* 653 * Generations must be different for each address space. 654 * Init kvm generation close to the maximum to easily test the 655 * code of handling generation number wrap-around. 656 */ 657 slots->generation = i * 2 - 150; 658 rcu_assign_pointer(kvm->memslots[i], slots); 659 } 660 661 if (init_srcu_struct(&kvm->srcu)) 662 goto out_err_no_srcu; 663 if (init_srcu_struct(&kvm->irq_srcu)) 664 goto out_err_no_irq_srcu; 665 for (i = 0; i < KVM_NR_BUSES; i++) { 666 rcu_assign_pointer(kvm->buses[i], 667 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL)); 668 if (!kvm->buses[i]) 669 goto out_err; 670 } 671 672 r = kvm_init_mmu_notifier(kvm); 673 if (r) 674 goto out_err; 675 676 spin_lock(&kvm_lock); 677 list_add(&kvm->vm_list, &vm_list); 678 spin_unlock(&kvm_lock); 679 680 preempt_notifier_inc(); 681 682 return kvm; 683 684 out_err: 685 cleanup_srcu_struct(&kvm->irq_srcu); 686 out_err_no_irq_srcu: 687 cleanup_srcu_struct(&kvm->srcu); 688 out_err_no_srcu: 689 hardware_disable_all(); 690 out_err_no_disable: 691 refcount_set(&kvm->users_count, 0); 692 for (i = 0; i < KVM_NR_BUSES; i++) 693 kfree(kvm_get_bus(kvm, i)); 694 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 695 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); 696 kvm_arch_free_vm(kvm); 697 mmdrop(current->mm); 698 return ERR_PTR(r); 699 } 700 701 static void kvm_destroy_devices(struct kvm *kvm) 702 { 703 struct kvm_device *dev, *tmp; 704 705 /* 706 * We do not need to take the kvm->lock here, because nobody else 707 * has a reference to the struct kvm at this point and therefore 708 * cannot access the devices list anyhow. 709 */ 710 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 711 list_del(&dev->vm_node); 712 dev->ops->destroy(dev); 713 } 714 } 715 716 static void kvm_destroy_vm(struct kvm *kvm) 717 { 718 int i; 719 struct mm_struct *mm = kvm->mm; 720 721 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); 722 kvm_destroy_vm_debugfs(kvm); 723 kvm_arch_sync_events(kvm); 724 spin_lock(&kvm_lock); 725 list_del(&kvm->vm_list); 726 spin_unlock(&kvm_lock); 727 kvm_free_irq_routing(kvm); 728 for (i = 0; i < KVM_NR_BUSES; i++) { 729 struct kvm_io_bus *bus = kvm_get_bus(kvm, i); 730 731 if (bus) 732 kvm_io_bus_destroy(bus); 733 kvm->buses[i] = NULL; 734 } 735 kvm_coalesced_mmio_free(kvm); 736 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 737 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 738 #else 739 kvm_arch_flush_shadow_all(kvm); 740 #endif 741 kvm_arch_destroy_vm(kvm); 742 kvm_destroy_devices(kvm); 743 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 744 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); 745 cleanup_srcu_struct(&kvm->irq_srcu); 746 cleanup_srcu_struct(&kvm->srcu); 747 kvm_arch_free_vm(kvm); 748 preempt_notifier_dec(); 749 hardware_disable_all(); 750 mmdrop(mm); 751 } 752 753 void kvm_get_kvm(struct kvm *kvm) 754 { 755 refcount_inc(&kvm->users_count); 756 } 757 EXPORT_SYMBOL_GPL(kvm_get_kvm); 758 759 void kvm_put_kvm(struct kvm *kvm) 760 { 761 if (refcount_dec_and_test(&kvm->users_count)) 762 kvm_destroy_vm(kvm); 763 } 764 EXPORT_SYMBOL_GPL(kvm_put_kvm); 765 766 767 static int kvm_vm_release(struct inode *inode, struct file *filp) 768 { 769 struct kvm *kvm = filp->private_data; 770 771 kvm_irqfd_release(kvm); 772 773 kvm_put_kvm(kvm); 774 return 0; 775 } 776 777 /* 778 * Allocation size is twice as large as the actual dirty bitmap size. 779 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 780 */ 781 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 782 { 783 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 784 785 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL); 786 if (!memslot->dirty_bitmap) 787 return -ENOMEM; 788 789 return 0; 790 } 791 792 /* 793 * Insert memslot and re-sort memslots based on their GFN, 794 * so binary search could be used to lookup GFN. 795 * Sorting algorithm takes advantage of having initially 796 * sorted array and known changed memslot position. 797 */ 798 static void update_memslots(struct kvm_memslots *slots, 799 struct kvm_memory_slot *new) 800 { 801 int id = new->id; 802 int i = slots->id_to_index[id]; 803 struct kvm_memory_slot *mslots = slots->memslots; 804 805 WARN_ON(mslots[i].id != id); 806 if (!new->npages) { 807 WARN_ON(!mslots[i].npages); 808 if (mslots[i].npages) 809 slots->used_slots--; 810 } else { 811 if (!mslots[i].npages) 812 slots->used_slots++; 813 } 814 815 while (i < KVM_MEM_SLOTS_NUM - 1 && 816 new->base_gfn <= mslots[i + 1].base_gfn) { 817 if (!mslots[i + 1].npages) 818 break; 819 mslots[i] = mslots[i + 1]; 820 slots->id_to_index[mslots[i].id] = i; 821 i++; 822 } 823 824 /* 825 * The ">=" is needed when creating a slot with base_gfn == 0, 826 * so that it moves before all those with base_gfn == npages == 0. 827 * 828 * On the other hand, if new->npages is zero, the above loop has 829 * already left i pointing to the beginning of the empty part of 830 * mslots, and the ">=" would move the hole backwards in this 831 * case---which is wrong. So skip the loop when deleting a slot. 832 */ 833 if (new->npages) { 834 while (i > 0 && 835 new->base_gfn >= mslots[i - 1].base_gfn) { 836 mslots[i] = mslots[i - 1]; 837 slots->id_to_index[mslots[i].id] = i; 838 i--; 839 } 840 } else 841 WARN_ON_ONCE(i != slots->used_slots); 842 843 mslots[i] = *new; 844 slots->id_to_index[mslots[i].id] = i; 845 } 846 847 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 848 { 849 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 850 851 #ifdef __KVM_HAVE_READONLY_MEM 852 valid_flags |= KVM_MEM_READONLY; 853 #endif 854 855 if (mem->flags & ~valid_flags) 856 return -EINVAL; 857 858 return 0; 859 } 860 861 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 862 int as_id, struct kvm_memslots *slots) 863 { 864 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 865 866 /* 867 * Set the low bit in the generation, which disables SPTE caching 868 * until the end of synchronize_srcu_expedited. 869 */ 870 WARN_ON(old_memslots->generation & 1); 871 slots->generation = old_memslots->generation + 1; 872 873 rcu_assign_pointer(kvm->memslots[as_id], slots); 874 synchronize_srcu_expedited(&kvm->srcu); 875 876 /* 877 * Increment the new memslot generation a second time. This prevents 878 * vm exits that race with memslot updates from caching a memslot 879 * generation that will (potentially) be valid forever. 880 * 881 * Generations must be unique even across address spaces. We do not need 882 * a global counter for that, instead the generation space is evenly split 883 * across address spaces. For example, with two address spaces, address 884 * space 0 will use generations 0, 4, 8, ... while * address space 1 will 885 * use generations 2, 6, 10, 14, ... 886 */ 887 slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1; 888 889 kvm_arch_memslots_updated(kvm, slots); 890 891 return old_memslots; 892 } 893 894 /* 895 * Allocate some memory and give it an address in the guest physical address 896 * space. 897 * 898 * Discontiguous memory is allowed, mostly for framebuffers. 899 * 900 * Must be called holding kvm->slots_lock for write. 901 */ 902 int __kvm_set_memory_region(struct kvm *kvm, 903 const struct kvm_userspace_memory_region *mem) 904 { 905 int r; 906 gfn_t base_gfn; 907 unsigned long npages; 908 struct kvm_memory_slot *slot; 909 struct kvm_memory_slot old, new; 910 struct kvm_memslots *slots = NULL, *old_memslots; 911 int as_id, id; 912 enum kvm_mr_change change; 913 914 r = check_memory_region_flags(mem); 915 if (r) 916 goto out; 917 918 r = -EINVAL; 919 as_id = mem->slot >> 16; 920 id = (u16)mem->slot; 921 922 /* General sanity checks */ 923 if (mem->memory_size & (PAGE_SIZE - 1)) 924 goto out; 925 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 926 goto out; 927 /* We can read the guest memory with __xxx_user() later on. */ 928 if ((id < KVM_USER_MEM_SLOTS) && 929 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 930 !access_ok(VERIFY_WRITE, 931 (void __user *)(unsigned long)mem->userspace_addr, 932 mem->memory_size))) 933 goto out; 934 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 935 goto out; 936 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 937 goto out; 938 939 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 940 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 941 npages = mem->memory_size >> PAGE_SHIFT; 942 943 if (npages > KVM_MEM_MAX_NR_PAGES) 944 goto out; 945 946 new = old = *slot; 947 948 new.id = id; 949 new.base_gfn = base_gfn; 950 new.npages = npages; 951 new.flags = mem->flags; 952 953 if (npages) { 954 if (!old.npages) 955 change = KVM_MR_CREATE; 956 else { /* Modify an existing slot. */ 957 if ((mem->userspace_addr != old.userspace_addr) || 958 (npages != old.npages) || 959 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 960 goto out; 961 962 if (base_gfn != old.base_gfn) 963 change = KVM_MR_MOVE; 964 else if (new.flags != old.flags) 965 change = KVM_MR_FLAGS_ONLY; 966 else { /* Nothing to change. */ 967 r = 0; 968 goto out; 969 } 970 } 971 } else { 972 if (!old.npages) 973 goto out; 974 975 change = KVM_MR_DELETE; 976 new.base_gfn = 0; 977 new.flags = 0; 978 } 979 980 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 981 /* Check for overlaps */ 982 r = -EEXIST; 983 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 984 if (slot->id == id) 985 continue; 986 if (!((base_gfn + npages <= slot->base_gfn) || 987 (base_gfn >= slot->base_gfn + slot->npages))) 988 goto out; 989 } 990 } 991 992 /* Free page dirty bitmap if unneeded */ 993 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 994 new.dirty_bitmap = NULL; 995 996 r = -ENOMEM; 997 if (change == KVM_MR_CREATE) { 998 new.userspace_addr = mem->userspace_addr; 999 1000 if (kvm_arch_create_memslot(kvm, &new, npages)) 1001 goto out_free; 1002 } 1003 1004 /* Allocate page dirty bitmap if needed */ 1005 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 1006 if (kvm_create_dirty_bitmap(&new) < 0) 1007 goto out_free; 1008 } 1009 1010 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 1011 if (!slots) 1012 goto out_free; 1013 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 1014 1015 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 1016 slot = id_to_memslot(slots, id); 1017 slot->flags |= KVM_MEMSLOT_INVALID; 1018 1019 old_memslots = install_new_memslots(kvm, as_id, slots); 1020 1021 /* From this point no new shadow pages pointing to a deleted, 1022 * or moved, memslot will be created. 1023 * 1024 * validation of sp->gfn happens in: 1025 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 1026 * - kvm_is_visible_gfn (mmu_check_roots) 1027 */ 1028 kvm_arch_flush_shadow_memslot(kvm, slot); 1029 1030 /* 1031 * We can re-use the old_memslots from above, the only difference 1032 * from the currently installed memslots is the invalid flag. This 1033 * will get overwritten by update_memslots anyway. 1034 */ 1035 slots = old_memslots; 1036 } 1037 1038 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 1039 if (r) 1040 goto out_slots; 1041 1042 /* actual memory is freed via old in kvm_free_memslot below */ 1043 if (change == KVM_MR_DELETE) { 1044 new.dirty_bitmap = NULL; 1045 memset(&new.arch, 0, sizeof(new.arch)); 1046 } 1047 1048 update_memslots(slots, &new); 1049 old_memslots = install_new_memslots(kvm, as_id, slots); 1050 1051 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 1052 1053 kvm_free_memslot(kvm, &old, &new); 1054 kvfree(old_memslots); 1055 return 0; 1056 1057 out_slots: 1058 kvfree(slots); 1059 out_free: 1060 kvm_free_memslot(kvm, &new, &old); 1061 out: 1062 return r; 1063 } 1064 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 1065 1066 int kvm_set_memory_region(struct kvm *kvm, 1067 const struct kvm_userspace_memory_region *mem) 1068 { 1069 int r; 1070 1071 mutex_lock(&kvm->slots_lock); 1072 r = __kvm_set_memory_region(kvm, mem); 1073 mutex_unlock(&kvm->slots_lock); 1074 return r; 1075 } 1076 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1077 1078 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1079 struct kvm_userspace_memory_region *mem) 1080 { 1081 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1082 return -EINVAL; 1083 1084 return kvm_set_memory_region(kvm, mem); 1085 } 1086 1087 int kvm_get_dirty_log(struct kvm *kvm, 1088 struct kvm_dirty_log *log, int *is_dirty) 1089 { 1090 struct kvm_memslots *slots; 1091 struct kvm_memory_slot *memslot; 1092 int i, as_id, id; 1093 unsigned long n; 1094 unsigned long any = 0; 1095 1096 as_id = log->slot >> 16; 1097 id = (u16)log->slot; 1098 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1099 return -EINVAL; 1100 1101 slots = __kvm_memslots(kvm, as_id); 1102 memslot = id_to_memslot(slots, id); 1103 if (!memslot->dirty_bitmap) 1104 return -ENOENT; 1105 1106 n = kvm_dirty_bitmap_bytes(memslot); 1107 1108 for (i = 0; !any && i < n/sizeof(long); ++i) 1109 any = memslot->dirty_bitmap[i]; 1110 1111 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1112 return -EFAULT; 1113 1114 if (any) 1115 *is_dirty = 1; 1116 return 0; 1117 } 1118 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1119 1120 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1121 /** 1122 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1123 * are dirty write protect them for next write. 1124 * @kvm: pointer to kvm instance 1125 * @log: slot id and address to which we copy the log 1126 * @is_dirty: flag set if any page is dirty 1127 * 1128 * We need to keep it in mind that VCPU threads can write to the bitmap 1129 * concurrently. So, to avoid losing track of dirty pages we keep the 1130 * following order: 1131 * 1132 * 1. Take a snapshot of the bit and clear it if needed. 1133 * 2. Write protect the corresponding page. 1134 * 3. Copy the snapshot to the userspace. 1135 * 4. Upon return caller flushes TLB's if needed. 1136 * 1137 * Between 2 and 4, the guest may write to the page using the remaining TLB 1138 * entry. This is not a problem because the page is reported dirty using 1139 * the snapshot taken before and step 4 ensures that writes done after 1140 * exiting to userspace will be logged for the next call. 1141 * 1142 */ 1143 int kvm_get_dirty_log_protect(struct kvm *kvm, 1144 struct kvm_dirty_log *log, bool *is_dirty) 1145 { 1146 struct kvm_memslots *slots; 1147 struct kvm_memory_slot *memslot; 1148 int i, as_id, id; 1149 unsigned long n; 1150 unsigned long *dirty_bitmap; 1151 unsigned long *dirty_bitmap_buffer; 1152 1153 as_id = log->slot >> 16; 1154 id = (u16)log->slot; 1155 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1156 return -EINVAL; 1157 1158 slots = __kvm_memslots(kvm, as_id); 1159 memslot = id_to_memslot(slots, id); 1160 1161 dirty_bitmap = memslot->dirty_bitmap; 1162 if (!dirty_bitmap) 1163 return -ENOENT; 1164 1165 n = kvm_dirty_bitmap_bytes(memslot); 1166 1167 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1168 memset(dirty_bitmap_buffer, 0, n); 1169 1170 spin_lock(&kvm->mmu_lock); 1171 *is_dirty = false; 1172 for (i = 0; i < n / sizeof(long); i++) { 1173 unsigned long mask; 1174 gfn_t offset; 1175 1176 if (!dirty_bitmap[i]) 1177 continue; 1178 1179 *is_dirty = true; 1180 1181 mask = xchg(&dirty_bitmap[i], 0); 1182 dirty_bitmap_buffer[i] = mask; 1183 1184 if (mask) { 1185 offset = i * BITS_PER_LONG; 1186 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1187 offset, mask); 1188 } 1189 } 1190 1191 spin_unlock(&kvm->mmu_lock); 1192 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1193 return -EFAULT; 1194 return 0; 1195 } 1196 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1197 #endif 1198 1199 bool kvm_largepages_enabled(void) 1200 { 1201 return largepages_enabled; 1202 } 1203 1204 void kvm_disable_largepages(void) 1205 { 1206 largepages_enabled = false; 1207 } 1208 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1209 1210 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1211 { 1212 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1213 } 1214 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1215 1216 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1217 { 1218 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1219 } 1220 1221 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1222 { 1223 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1224 1225 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1226 memslot->flags & KVM_MEMSLOT_INVALID) 1227 return false; 1228 1229 return true; 1230 } 1231 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1232 1233 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1234 { 1235 struct vm_area_struct *vma; 1236 unsigned long addr, size; 1237 1238 size = PAGE_SIZE; 1239 1240 addr = gfn_to_hva(kvm, gfn); 1241 if (kvm_is_error_hva(addr)) 1242 return PAGE_SIZE; 1243 1244 down_read(¤t->mm->mmap_sem); 1245 vma = find_vma(current->mm, addr); 1246 if (!vma) 1247 goto out; 1248 1249 size = vma_kernel_pagesize(vma); 1250 1251 out: 1252 up_read(¤t->mm->mmap_sem); 1253 1254 return size; 1255 } 1256 1257 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1258 { 1259 return slot->flags & KVM_MEM_READONLY; 1260 } 1261 1262 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1263 gfn_t *nr_pages, bool write) 1264 { 1265 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1266 return KVM_HVA_ERR_BAD; 1267 1268 if (memslot_is_readonly(slot) && write) 1269 return KVM_HVA_ERR_RO_BAD; 1270 1271 if (nr_pages) 1272 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1273 1274 return __gfn_to_hva_memslot(slot, gfn); 1275 } 1276 1277 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1278 gfn_t *nr_pages) 1279 { 1280 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1281 } 1282 1283 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1284 gfn_t gfn) 1285 { 1286 return gfn_to_hva_many(slot, gfn, NULL); 1287 } 1288 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1289 1290 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1291 { 1292 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1293 } 1294 EXPORT_SYMBOL_GPL(gfn_to_hva); 1295 1296 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1297 { 1298 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1299 } 1300 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1301 1302 /* 1303 * If writable is set to false, the hva returned by this function is only 1304 * allowed to be read. 1305 */ 1306 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1307 gfn_t gfn, bool *writable) 1308 { 1309 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1310 1311 if (!kvm_is_error_hva(hva) && writable) 1312 *writable = !memslot_is_readonly(slot); 1313 1314 return hva; 1315 } 1316 1317 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1318 { 1319 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1320 1321 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1322 } 1323 1324 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1325 { 1326 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1327 1328 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1329 } 1330 1331 static inline int check_user_page_hwpoison(unsigned long addr) 1332 { 1333 int rc, flags = FOLL_HWPOISON | FOLL_WRITE; 1334 1335 rc = get_user_pages(addr, 1, flags, NULL, NULL); 1336 return rc == -EHWPOISON; 1337 } 1338 1339 /* 1340 * The atomic path to get the writable pfn which will be stored in @pfn, 1341 * true indicates success, otherwise false is returned. 1342 */ 1343 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1344 bool write_fault, bool *writable, kvm_pfn_t *pfn) 1345 { 1346 struct page *page[1]; 1347 int npages; 1348 1349 if (!(async || atomic)) 1350 return false; 1351 1352 /* 1353 * Fast pin a writable pfn only if it is a write fault request 1354 * or the caller allows to map a writable pfn for a read fault 1355 * request. 1356 */ 1357 if (!(write_fault || writable)) 1358 return false; 1359 1360 npages = __get_user_pages_fast(addr, 1, 1, page); 1361 if (npages == 1) { 1362 *pfn = page_to_pfn(page[0]); 1363 1364 if (writable) 1365 *writable = true; 1366 return true; 1367 } 1368 1369 return false; 1370 } 1371 1372 /* 1373 * The slow path to get the pfn of the specified host virtual address, 1374 * 1 indicates success, -errno is returned if error is detected. 1375 */ 1376 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1377 bool *writable, kvm_pfn_t *pfn) 1378 { 1379 unsigned int flags = FOLL_HWPOISON; 1380 struct page *page; 1381 int npages = 0; 1382 1383 might_sleep(); 1384 1385 if (writable) 1386 *writable = write_fault; 1387 1388 if (write_fault) 1389 flags |= FOLL_WRITE; 1390 if (async) 1391 flags |= FOLL_NOWAIT; 1392 1393 npages = get_user_pages_unlocked(addr, 1, &page, flags); 1394 if (npages != 1) 1395 return npages; 1396 1397 /* map read fault as writable if possible */ 1398 if (unlikely(!write_fault) && writable) { 1399 struct page *wpage; 1400 1401 if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) { 1402 *writable = true; 1403 put_page(page); 1404 page = wpage; 1405 } 1406 } 1407 *pfn = page_to_pfn(page); 1408 return npages; 1409 } 1410 1411 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1412 { 1413 if (unlikely(!(vma->vm_flags & VM_READ))) 1414 return false; 1415 1416 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1417 return false; 1418 1419 return true; 1420 } 1421 1422 static int hva_to_pfn_remapped(struct vm_area_struct *vma, 1423 unsigned long addr, bool *async, 1424 bool write_fault, bool *writable, 1425 kvm_pfn_t *p_pfn) 1426 { 1427 unsigned long pfn; 1428 int r; 1429 1430 r = follow_pfn(vma, addr, &pfn); 1431 if (r) { 1432 /* 1433 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does 1434 * not call the fault handler, so do it here. 1435 */ 1436 bool unlocked = false; 1437 r = fixup_user_fault(current, current->mm, addr, 1438 (write_fault ? FAULT_FLAG_WRITE : 0), 1439 &unlocked); 1440 if (unlocked) 1441 return -EAGAIN; 1442 if (r) 1443 return r; 1444 1445 r = follow_pfn(vma, addr, &pfn); 1446 if (r) 1447 return r; 1448 1449 } 1450 1451 if (writable) 1452 *writable = true; 1453 1454 /* 1455 * Get a reference here because callers of *hva_to_pfn* and 1456 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the 1457 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP 1458 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will 1459 * simply do nothing for reserved pfns. 1460 * 1461 * Whoever called remap_pfn_range is also going to call e.g. 1462 * unmap_mapping_range before the underlying pages are freed, 1463 * causing a call to our MMU notifier. 1464 */ 1465 kvm_get_pfn(pfn); 1466 1467 *p_pfn = pfn; 1468 return 0; 1469 } 1470 1471 /* 1472 * Pin guest page in memory and return its pfn. 1473 * @addr: host virtual address which maps memory to the guest 1474 * @atomic: whether this function can sleep 1475 * @async: whether this function need to wait IO complete if the 1476 * host page is not in the memory 1477 * @write_fault: whether we should get a writable host page 1478 * @writable: whether it allows to map a writable host page for !@write_fault 1479 * 1480 * The function will map a writable host page for these two cases: 1481 * 1): @write_fault = true 1482 * 2): @write_fault = false && @writable, @writable will tell the caller 1483 * whether the mapping is writable. 1484 */ 1485 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1486 bool write_fault, bool *writable) 1487 { 1488 struct vm_area_struct *vma; 1489 kvm_pfn_t pfn = 0; 1490 int npages, r; 1491 1492 /* we can do it either atomically or asynchronously, not both */ 1493 BUG_ON(atomic && async); 1494 1495 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1496 return pfn; 1497 1498 if (atomic) 1499 return KVM_PFN_ERR_FAULT; 1500 1501 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1502 if (npages == 1) 1503 return pfn; 1504 1505 down_read(¤t->mm->mmap_sem); 1506 if (npages == -EHWPOISON || 1507 (!async && check_user_page_hwpoison(addr))) { 1508 pfn = KVM_PFN_ERR_HWPOISON; 1509 goto exit; 1510 } 1511 1512 retry: 1513 vma = find_vma_intersection(current->mm, addr, addr + 1); 1514 1515 if (vma == NULL) 1516 pfn = KVM_PFN_ERR_FAULT; 1517 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { 1518 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn); 1519 if (r == -EAGAIN) 1520 goto retry; 1521 if (r < 0) 1522 pfn = KVM_PFN_ERR_FAULT; 1523 } else { 1524 if (async && vma_is_valid(vma, write_fault)) 1525 *async = true; 1526 pfn = KVM_PFN_ERR_FAULT; 1527 } 1528 exit: 1529 up_read(¤t->mm->mmap_sem); 1530 return pfn; 1531 } 1532 1533 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 1534 bool atomic, bool *async, bool write_fault, 1535 bool *writable) 1536 { 1537 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1538 1539 if (addr == KVM_HVA_ERR_RO_BAD) { 1540 if (writable) 1541 *writable = false; 1542 return KVM_PFN_ERR_RO_FAULT; 1543 } 1544 1545 if (kvm_is_error_hva(addr)) { 1546 if (writable) 1547 *writable = false; 1548 return KVM_PFN_NOSLOT; 1549 } 1550 1551 /* Do not map writable pfn in the readonly memslot. */ 1552 if (writable && memslot_is_readonly(slot)) { 1553 *writable = false; 1554 writable = NULL; 1555 } 1556 1557 return hva_to_pfn(addr, atomic, async, write_fault, 1558 writable); 1559 } 1560 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1561 1562 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1563 bool *writable) 1564 { 1565 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1566 write_fault, writable); 1567 } 1568 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1569 1570 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1571 { 1572 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1573 } 1574 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1575 1576 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1577 { 1578 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1579 } 1580 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1581 1582 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1583 { 1584 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1585 } 1586 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1587 1588 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1589 { 1590 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1591 } 1592 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1593 1594 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1595 { 1596 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1597 } 1598 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1599 1600 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1601 { 1602 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1603 } 1604 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1605 1606 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1607 struct page **pages, int nr_pages) 1608 { 1609 unsigned long addr; 1610 gfn_t entry = 0; 1611 1612 addr = gfn_to_hva_many(slot, gfn, &entry); 1613 if (kvm_is_error_hva(addr)) 1614 return -1; 1615 1616 if (entry < nr_pages) 1617 return 0; 1618 1619 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1620 } 1621 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1622 1623 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 1624 { 1625 if (is_error_noslot_pfn(pfn)) 1626 return KVM_ERR_PTR_BAD_PAGE; 1627 1628 if (kvm_is_reserved_pfn(pfn)) { 1629 WARN_ON(1); 1630 return KVM_ERR_PTR_BAD_PAGE; 1631 } 1632 1633 return pfn_to_page(pfn); 1634 } 1635 1636 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1637 { 1638 kvm_pfn_t pfn; 1639 1640 pfn = gfn_to_pfn(kvm, gfn); 1641 1642 return kvm_pfn_to_page(pfn); 1643 } 1644 EXPORT_SYMBOL_GPL(gfn_to_page); 1645 1646 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1647 { 1648 kvm_pfn_t pfn; 1649 1650 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1651 1652 return kvm_pfn_to_page(pfn); 1653 } 1654 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1655 1656 void kvm_release_page_clean(struct page *page) 1657 { 1658 WARN_ON(is_error_page(page)); 1659 1660 kvm_release_pfn_clean(page_to_pfn(page)); 1661 } 1662 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1663 1664 void kvm_release_pfn_clean(kvm_pfn_t pfn) 1665 { 1666 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1667 put_page(pfn_to_page(pfn)); 1668 } 1669 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1670 1671 void kvm_release_page_dirty(struct page *page) 1672 { 1673 WARN_ON(is_error_page(page)); 1674 1675 kvm_release_pfn_dirty(page_to_pfn(page)); 1676 } 1677 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1678 1679 void kvm_release_pfn_dirty(kvm_pfn_t pfn) 1680 { 1681 kvm_set_pfn_dirty(pfn); 1682 kvm_release_pfn_clean(pfn); 1683 } 1684 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 1685 1686 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 1687 { 1688 if (!kvm_is_reserved_pfn(pfn)) { 1689 struct page *page = pfn_to_page(pfn); 1690 1691 if (!PageReserved(page)) 1692 SetPageDirty(page); 1693 } 1694 } 1695 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1696 1697 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 1698 { 1699 if (!kvm_is_reserved_pfn(pfn)) 1700 mark_page_accessed(pfn_to_page(pfn)); 1701 } 1702 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1703 1704 void kvm_get_pfn(kvm_pfn_t pfn) 1705 { 1706 if (!kvm_is_reserved_pfn(pfn)) 1707 get_page(pfn_to_page(pfn)); 1708 } 1709 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1710 1711 static int next_segment(unsigned long len, int offset) 1712 { 1713 if (len > PAGE_SIZE - offset) 1714 return PAGE_SIZE - offset; 1715 else 1716 return len; 1717 } 1718 1719 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1720 void *data, int offset, int len) 1721 { 1722 int r; 1723 unsigned long addr; 1724 1725 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1726 if (kvm_is_error_hva(addr)) 1727 return -EFAULT; 1728 r = __copy_from_user(data, (void __user *)addr + offset, len); 1729 if (r) 1730 return -EFAULT; 1731 return 0; 1732 } 1733 1734 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1735 int len) 1736 { 1737 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1738 1739 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1740 } 1741 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1742 1743 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1744 int offset, int len) 1745 { 1746 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1747 1748 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1749 } 1750 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1751 1752 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1753 { 1754 gfn_t gfn = gpa >> PAGE_SHIFT; 1755 int seg; 1756 int offset = offset_in_page(gpa); 1757 int ret; 1758 1759 while ((seg = next_segment(len, offset)) != 0) { 1760 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1761 if (ret < 0) 1762 return ret; 1763 offset = 0; 1764 len -= seg; 1765 data += seg; 1766 ++gfn; 1767 } 1768 return 0; 1769 } 1770 EXPORT_SYMBOL_GPL(kvm_read_guest); 1771 1772 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1773 { 1774 gfn_t gfn = gpa >> PAGE_SHIFT; 1775 int seg; 1776 int offset = offset_in_page(gpa); 1777 int ret; 1778 1779 while ((seg = next_segment(len, offset)) != 0) { 1780 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1781 if (ret < 0) 1782 return ret; 1783 offset = 0; 1784 len -= seg; 1785 data += seg; 1786 ++gfn; 1787 } 1788 return 0; 1789 } 1790 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1791 1792 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1793 void *data, int offset, unsigned long len) 1794 { 1795 int r; 1796 unsigned long addr; 1797 1798 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1799 if (kvm_is_error_hva(addr)) 1800 return -EFAULT; 1801 pagefault_disable(); 1802 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1803 pagefault_enable(); 1804 if (r) 1805 return -EFAULT; 1806 return 0; 1807 } 1808 1809 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1810 unsigned long len) 1811 { 1812 gfn_t gfn = gpa >> PAGE_SHIFT; 1813 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1814 int offset = offset_in_page(gpa); 1815 1816 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1817 } 1818 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1819 1820 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1821 void *data, unsigned long len) 1822 { 1823 gfn_t gfn = gpa >> PAGE_SHIFT; 1824 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1825 int offset = offset_in_page(gpa); 1826 1827 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1828 } 1829 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1830 1831 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1832 const void *data, int offset, int len) 1833 { 1834 int r; 1835 unsigned long addr; 1836 1837 addr = gfn_to_hva_memslot(memslot, gfn); 1838 if (kvm_is_error_hva(addr)) 1839 return -EFAULT; 1840 r = __copy_to_user((void __user *)addr + offset, data, len); 1841 if (r) 1842 return -EFAULT; 1843 mark_page_dirty_in_slot(memslot, gfn); 1844 return 0; 1845 } 1846 1847 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1848 const void *data, int offset, int len) 1849 { 1850 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1851 1852 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1853 } 1854 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1855 1856 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1857 const void *data, int offset, int len) 1858 { 1859 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1860 1861 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1862 } 1863 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1864 1865 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1866 unsigned long len) 1867 { 1868 gfn_t gfn = gpa >> PAGE_SHIFT; 1869 int seg; 1870 int offset = offset_in_page(gpa); 1871 int ret; 1872 1873 while ((seg = next_segment(len, offset)) != 0) { 1874 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1875 if (ret < 0) 1876 return ret; 1877 offset = 0; 1878 len -= seg; 1879 data += seg; 1880 ++gfn; 1881 } 1882 return 0; 1883 } 1884 EXPORT_SYMBOL_GPL(kvm_write_guest); 1885 1886 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1887 unsigned long len) 1888 { 1889 gfn_t gfn = gpa >> PAGE_SHIFT; 1890 int seg; 1891 int offset = offset_in_page(gpa); 1892 int ret; 1893 1894 while ((seg = next_segment(len, offset)) != 0) { 1895 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1896 if (ret < 0) 1897 return ret; 1898 offset = 0; 1899 len -= seg; 1900 data += seg; 1901 ++gfn; 1902 } 1903 return 0; 1904 } 1905 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1906 1907 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots, 1908 struct gfn_to_hva_cache *ghc, 1909 gpa_t gpa, unsigned long len) 1910 { 1911 int offset = offset_in_page(gpa); 1912 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1913 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1914 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1915 gfn_t nr_pages_avail; 1916 1917 ghc->gpa = gpa; 1918 ghc->generation = slots->generation; 1919 ghc->len = len; 1920 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 1921 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1922 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1923 ghc->hva += offset; 1924 } else { 1925 /* 1926 * If the requested region crosses two memslots, we still 1927 * verify that the entire region is valid here. 1928 */ 1929 while (start_gfn <= end_gfn) { 1930 nr_pages_avail = 0; 1931 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 1932 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1933 &nr_pages_avail); 1934 if (kvm_is_error_hva(ghc->hva)) 1935 return -EFAULT; 1936 start_gfn += nr_pages_avail; 1937 } 1938 /* Use the slow path for cross page reads and writes. */ 1939 ghc->memslot = NULL; 1940 } 1941 return 0; 1942 } 1943 1944 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1945 gpa_t gpa, unsigned long len) 1946 { 1947 struct kvm_memslots *slots = kvm_memslots(kvm); 1948 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len); 1949 } 1950 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1951 1952 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1953 void *data, int offset, unsigned long len) 1954 { 1955 struct kvm_memslots *slots = kvm_memslots(kvm); 1956 int r; 1957 gpa_t gpa = ghc->gpa + offset; 1958 1959 BUG_ON(len + offset > ghc->len); 1960 1961 if (slots->generation != ghc->generation) 1962 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len); 1963 1964 if (unlikely(!ghc->memslot)) 1965 return kvm_write_guest(kvm, gpa, data, len); 1966 1967 if (kvm_is_error_hva(ghc->hva)) 1968 return -EFAULT; 1969 1970 r = __copy_to_user((void __user *)ghc->hva + offset, data, len); 1971 if (r) 1972 return -EFAULT; 1973 mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT); 1974 1975 return 0; 1976 } 1977 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached); 1978 1979 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1980 void *data, unsigned long len) 1981 { 1982 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); 1983 } 1984 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1985 1986 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1987 void *data, unsigned long len) 1988 { 1989 struct kvm_memslots *slots = kvm_memslots(kvm); 1990 int r; 1991 1992 BUG_ON(len > ghc->len); 1993 1994 if (slots->generation != ghc->generation) 1995 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len); 1996 1997 if (unlikely(!ghc->memslot)) 1998 return kvm_read_guest(kvm, ghc->gpa, data, len); 1999 2000 if (kvm_is_error_hva(ghc->hva)) 2001 return -EFAULT; 2002 2003 r = __copy_from_user(data, (void __user *)ghc->hva, len); 2004 if (r) 2005 return -EFAULT; 2006 2007 return 0; 2008 } 2009 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 2010 2011 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 2012 { 2013 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 2014 2015 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 2016 } 2017 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 2018 2019 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 2020 { 2021 gfn_t gfn = gpa >> PAGE_SHIFT; 2022 int seg; 2023 int offset = offset_in_page(gpa); 2024 int ret; 2025 2026 while ((seg = next_segment(len, offset)) != 0) { 2027 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 2028 if (ret < 0) 2029 return ret; 2030 offset = 0; 2031 len -= seg; 2032 ++gfn; 2033 } 2034 return 0; 2035 } 2036 EXPORT_SYMBOL_GPL(kvm_clear_guest); 2037 2038 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 2039 gfn_t gfn) 2040 { 2041 if (memslot && memslot->dirty_bitmap) { 2042 unsigned long rel_gfn = gfn - memslot->base_gfn; 2043 2044 set_bit_le(rel_gfn, memslot->dirty_bitmap); 2045 } 2046 } 2047 2048 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 2049 { 2050 struct kvm_memory_slot *memslot; 2051 2052 memslot = gfn_to_memslot(kvm, gfn); 2053 mark_page_dirty_in_slot(memslot, gfn); 2054 } 2055 EXPORT_SYMBOL_GPL(mark_page_dirty); 2056 2057 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 2058 { 2059 struct kvm_memory_slot *memslot; 2060 2061 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2062 mark_page_dirty_in_slot(memslot, gfn); 2063 } 2064 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 2065 2066 void kvm_sigset_activate(struct kvm_vcpu *vcpu) 2067 { 2068 if (!vcpu->sigset_active) 2069 return; 2070 2071 /* 2072 * This does a lockless modification of ->real_blocked, which is fine 2073 * because, only current can change ->real_blocked and all readers of 2074 * ->real_blocked don't care as long ->real_blocked is always a subset 2075 * of ->blocked. 2076 */ 2077 sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked); 2078 } 2079 2080 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu) 2081 { 2082 if (!vcpu->sigset_active) 2083 return; 2084 2085 sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL); 2086 sigemptyset(¤t->real_blocked); 2087 } 2088 2089 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 2090 { 2091 unsigned int old, val, grow; 2092 2093 old = val = vcpu->halt_poll_ns; 2094 grow = READ_ONCE(halt_poll_ns_grow); 2095 /* 10us base */ 2096 if (val == 0 && grow) 2097 val = 10000; 2098 else 2099 val *= grow; 2100 2101 if (val > halt_poll_ns) 2102 val = halt_poll_ns; 2103 2104 vcpu->halt_poll_ns = val; 2105 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 2106 } 2107 2108 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 2109 { 2110 unsigned int old, val, shrink; 2111 2112 old = val = vcpu->halt_poll_ns; 2113 shrink = READ_ONCE(halt_poll_ns_shrink); 2114 if (shrink == 0) 2115 val = 0; 2116 else 2117 val /= shrink; 2118 2119 vcpu->halt_poll_ns = val; 2120 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 2121 } 2122 2123 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 2124 { 2125 if (kvm_arch_vcpu_runnable(vcpu)) { 2126 kvm_make_request(KVM_REQ_UNHALT, vcpu); 2127 return -EINTR; 2128 } 2129 if (kvm_cpu_has_pending_timer(vcpu)) 2130 return -EINTR; 2131 if (signal_pending(current)) 2132 return -EINTR; 2133 2134 return 0; 2135 } 2136 2137 /* 2138 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2139 */ 2140 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2141 { 2142 ktime_t start, cur; 2143 DECLARE_SWAITQUEUE(wait); 2144 bool waited = false; 2145 u64 block_ns; 2146 2147 start = cur = ktime_get(); 2148 if (vcpu->halt_poll_ns) { 2149 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2150 2151 ++vcpu->stat.halt_attempted_poll; 2152 do { 2153 /* 2154 * This sets KVM_REQ_UNHALT if an interrupt 2155 * arrives. 2156 */ 2157 if (kvm_vcpu_check_block(vcpu) < 0) { 2158 ++vcpu->stat.halt_successful_poll; 2159 if (!vcpu_valid_wakeup(vcpu)) 2160 ++vcpu->stat.halt_poll_invalid; 2161 goto out; 2162 } 2163 cur = ktime_get(); 2164 } while (single_task_running() && ktime_before(cur, stop)); 2165 } 2166 2167 kvm_arch_vcpu_blocking(vcpu); 2168 2169 for (;;) { 2170 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2171 2172 if (kvm_vcpu_check_block(vcpu) < 0) 2173 break; 2174 2175 waited = true; 2176 schedule(); 2177 } 2178 2179 finish_swait(&vcpu->wq, &wait); 2180 cur = ktime_get(); 2181 2182 kvm_arch_vcpu_unblocking(vcpu); 2183 out: 2184 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2185 2186 if (!vcpu_valid_wakeup(vcpu)) 2187 shrink_halt_poll_ns(vcpu); 2188 else if (halt_poll_ns) { 2189 if (block_ns <= vcpu->halt_poll_ns) 2190 ; 2191 /* we had a long block, shrink polling */ 2192 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2193 shrink_halt_poll_ns(vcpu); 2194 /* we had a short halt and our poll time is too small */ 2195 else if (vcpu->halt_poll_ns < halt_poll_ns && 2196 block_ns < halt_poll_ns) 2197 grow_halt_poll_ns(vcpu); 2198 } else 2199 vcpu->halt_poll_ns = 0; 2200 2201 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu)); 2202 kvm_arch_vcpu_block_finish(vcpu); 2203 } 2204 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2205 2206 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 2207 { 2208 struct swait_queue_head *wqp; 2209 2210 wqp = kvm_arch_vcpu_wq(vcpu); 2211 if (swq_has_sleeper(wqp)) { 2212 swake_up(wqp); 2213 ++vcpu->stat.halt_wakeup; 2214 return true; 2215 } 2216 2217 return false; 2218 } 2219 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 2220 2221 #ifndef CONFIG_S390 2222 /* 2223 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2224 */ 2225 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2226 { 2227 int me; 2228 int cpu = vcpu->cpu; 2229 2230 if (kvm_vcpu_wake_up(vcpu)) 2231 return; 2232 2233 me = get_cpu(); 2234 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2235 if (kvm_arch_vcpu_should_kick(vcpu)) 2236 smp_send_reschedule(cpu); 2237 put_cpu(); 2238 } 2239 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2240 #endif /* !CONFIG_S390 */ 2241 2242 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2243 { 2244 struct pid *pid; 2245 struct task_struct *task = NULL; 2246 int ret = 0; 2247 2248 rcu_read_lock(); 2249 pid = rcu_dereference(target->pid); 2250 if (pid) 2251 task = get_pid_task(pid, PIDTYPE_PID); 2252 rcu_read_unlock(); 2253 if (!task) 2254 return ret; 2255 ret = yield_to(task, 1); 2256 put_task_struct(task); 2257 2258 return ret; 2259 } 2260 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2261 2262 /* 2263 * Helper that checks whether a VCPU is eligible for directed yield. 2264 * Most eligible candidate to yield is decided by following heuristics: 2265 * 2266 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2267 * (preempted lock holder), indicated by @in_spin_loop. 2268 * Set at the beiginning and cleared at the end of interception/PLE handler. 2269 * 2270 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2271 * chance last time (mostly it has become eligible now since we have probably 2272 * yielded to lockholder in last iteration. This is done by toggling 2273 * @dy_eligible each time a VCPU checked for eligibility.) 2274 * 2275 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2276 * to preempted lock-holder could result in wrong VCPU selection and CPU 2277 * burning. Giving priority for a potential lock-holder increases lock 2278 * progress. 2279 * 2280 * Since algorithm is based on heuristics, accessing another VCPU data without 2281 * locking does not harm. It may result in trying to yield to same VCPU, fail 2282 * and continue with next VCPU and so on. 2283 */ 2284 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2285 { 2286 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2287 bool eligible; 2288 2289 eligible = !vcpu->spin_loop.in_spin_loop || 2290 vcpu->spin_loop.dy_eligible; 2291 2292 if (vcpu->spin_loop.in_spin_loop) 2293 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2294 2295 return eligible; 2296 #else 2297 return true; 2298 #endif 2299 } 2300 2301 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) 2302 { 2303 struct kvm *kvm = me->kvm; 2304 struct kvm_vcpu *vcpu; 2305 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2306 int yielded = 0; 2307 int try = 3; 2308 int pass; 2309 int i; 2310 2311 kvm_vcpu_set_in_spin_loop(me, true); 2312 /* 2313 * We boost the priority of a VCPU that is runnable but not 2314 * currently running, because it got preempted by something 2315 * else and called schedule in __vcpu_run. Hopefully that 2316 * VCPU is holding the lock that we need and will release it. 2317 * We approximate round-robin by starting at the last boosted VCPU. 2318 */ 2319 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2320 kvm_for_each_vcpu(i, vcpu, kvm) { 2321 if (!pass && i <= last_boosted_vcpu) { 2322 i = last_boosted_vcpu; 2323 continue; 2324 } else if (pass && i > last_boosted_vcpu) 2325 break; 2326 if (!READ_ONCE(vcpu->preempted)) 2327 continue; 2328 if (vcpu == me) 2329 continue; 2330 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2331 continue; 2332 if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu)) 2333 continue; 2334 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2335 continue; 2336 2337 yielded = kvm_vcpu_yield_to(vcpu); 2338 if (yielded > 0) { 2339 kvm->last_boosted_vcpu = i; 2340 break; 2341 } else if (yielded < 0) { 2342 try--; 2343 if (!try) 2344 break; 2345 } 2346 } 2347 } 2348 kvm_vcpu_set_in_spin_loop(me, false); 2349 2350 /* Ensure vcpu is not eligible during next spinloop */ 2351 kvm_vcpu_set_dy_eligible(me, false); 2352 } 2353 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2354 2355 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf) 2356 { 2357 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data; 2358 struct page *page; 2359 2360 if (vmf->pgoff == 0) 2361 page = virt_to_page(vcpu->run); 2362 #ifdef CONFIG_X86 2363 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2364 page = virt_to_page(vcpu->arch.pio_data); 2365 #endif 2366 #ifdef CONFIG_KVM_MMIO 2367 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2368 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2369 #endif 2370 else 2371 return kvm_arch_vcpu_fault(vcpu, vmf); 2372 get_page(page); 2373 vmf->page = page; 2374 return 0; 2375 } 2376 2377 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2378 .fault = kvm_vcpu_fault, 2379 }; 2380 2381 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2382 { 2383 vma->vm_ops = &kvm_vcpu_vm_ops; 2384 return 0; 2385 } 2386 2387 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2388 { 2389 struct kvm_vcpu *vcpu = filp->private_data; 2390 2391 debugfs_remove_recursive(vcpu->debugfs_dentry); 2392 kvm_put_kvm(vcpu->kvm); 2393 return 0; 2394 } 2395 2396 static struct file_operations kvm_vcpu_fops = { 2397 .release = kvm_vcpu_release, 2398 .unlocked_ioctl = kvm_vcpu_ioctl, 2399 #ifdef CONFIG_KVM_COMPAT 2400 .compat_ioctl = kvm_vcpu_compat_ioctl, 2401 #endif 2402 .mmap = kvm_vcpu_mmap, 2403 .llseek = noop_llseek, 2404 }; 2405 2406 /* 2407 * Allocates an inode for the vcpu. 2408 */ 2409 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2410 { 2411 char name[8 + 1 + ITOA_MAX_LEN + 1]; 2412 2413 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); 2414 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2415 } 2416 2417 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 2418 { 2419 char dir_name[ITOA_MAX_LEN * 2]; 2420 int ret; 2421 2422 if (!kvm_arch_has_vcpu_debugfs()) 2423 return 0; 2424 2425 if (!debugfs_initialized()) 2426 return 0; 2427 2428 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); 2429 vcpu->debugfs_dentry = debugfs_create_dir(dir_name, 2430 vcpu->kvm->debugfs_dentry); 2431 if (!vcpu->debugfs_dentry) 2432 return -ENOMEM; 2433 2434 ret = kvm_arch_create_vcpu_debugfs(vcpu); 2435 if (ret < 0) { 2436 debugfs_remove_recursive(vcpu->debugfs_dentry); 2437 return ret; 2438 } 2439 2440 return 0; 2441 } 2442 2443 /* 2444 * Creates some virtual cpus. Good luck creating more than one. 2445 */ 2446 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2447 { 2448 int r; 2449 struct kvm_vcpu *vcpu; 2450 2451 if (id >= KVM_MAX_VCPU_ID) 2452 return -EINVAL; 2453 2454 mutex_lock(&kvm->lock); 2455 if (kvm->created_vcpus == KVM_MAX_VCPUS) { 2456 mutex_unlock(&kvm->lock); 2457 return -EINVAL; 2458 } 2459 2460 kvm->created_vcpus++; 2461 mutex_unlock(&kvm->lock); 2462 2463 vcpu = kvm_arch_vcpu_create(kvm, id); 2464 if (IS_ERR(vcpu)) { 2465 r = PTR_ERR(vcpu); 2466 goto vcpu_decrement; 2467 } 2468 2469 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2470 2471 r = kvm_arch_vcpu_setup(vcpu); 2472 if (r) 2473 goto vcpu_destroy; 2474 2475 r = kvm_create_vcpu_debugfs(vcpu); 2476 if (r) 2477 goto vcpu_destroy; 2478 2479 mutex_lock(&kvm->lock); 2480 if (kvm_get_vcpu_by_id(kvm, id)) { 2481 r = -EEXIST; 2482 goto unlock_vcpu_destroy; 2483 } 2484 2485 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2486 2487 /* Now it's all set up, let userspace reach it */ 2488 kvm_get_kvm(kvm); 2489 r = create_vcpu_fd(vcpu); 2490 if (r < 0) { 2491 kvm_put_kvm(kvm); 2492 goto unlock_vcpu_destroy; 2493 } 2494 2495 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2496 2497 /* 2498 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2499 * before kvm->online_vcpu's incremented value. 2500 */ 2501 smp_wmb(); 2502 atomic_inc(&kvm->online_vcpus); 2503 2504 mutex_unlock(&kvm->lock); 2505 kvm_arch_vcpu_postcreate(vcpu); 2506 return r; 2507 2508 unlock_vcpu_destroy: 2509 mutex_unlock(&kvm->lock); 2510 debugfs_remove_recursive(vcpu->debugfs_dentry); 2511 vcpu_destroy: 2512 kvm_arch_vcpu_destroy(vcpu); 2513 vcpu_decrement: 2514 mutex_lock(&kvm->lock); 2515 kvm->created_vcpus--; 2516 mutex_unlock(&kvm->lock); 2517 return r; 2518 } 2519 2520 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2521 { 2522 if (sigset) { 2523 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2524 vcpu->sigset_active = 1; 2525 vcpu->sigset = *sigset; 2526 } else 2527 vcpu->sigset_active = 0; 2528 return 0; 2529 } 2530 2531 static long kvm_vcpu_ioctl(struct file *filp, 2532 unsigned int ioctl, unsigned long arg) 2533 { 2534 struct kvm_vcpu *vcpu = filp->private_data; 2535 void __user *argp = (void __user *)arg; 2536 int r; 2537 struct kvm_fpu *fpu = NULL; 2538 struct kvm_sregs *kvm_sregs = NULL; 2539 2540 if (vcpu->kvm->mm != current->mm) 2541 return -EIO; 2542 2543 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2544 return -EINVAL; 2545 2546 /* 2547 * Some architectures have vcpu ioctls that are asynchronous to vcpu 2548 * execution; mutex_lock() would break them. 2549 */ 2550 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg); 2551 if (r != -ENOIOCTLCMD) 2552 return r; 2553 2554 if (mutex_lock_killable(&vcpu->mutex)) 2555 return -EINTR; 2556 switch (ioctl) { 2557 case KVM_RUN: { 2558 struct pid *oldpid; 2559 r = -EINVAL; 2560 if (arg) 2561 goto out; 2562 oldpid = rcu_access_pointer(vcpu->pid); 2563 if (unlikely(oldpid != current->pids[PIDTYPE_PID].pid)) { 2564 /* The thread running this VCPU changed. */ 2565 struct pid *newpid; 2566 2567 r = kvm_arch_vcpu_run_pid_change(vcpu); 2568 if (r) 2569 break; 2570 2571 newpid = get_task_pid(current, PIDTYPE_PID); 2572 rcu_assign_pointer(vcpu->pid, newpid); 2573 if (oldpid) 2574 synchronize_rcu(); 2575 put_pid(oldpid); 2576 } 2577 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2578 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2579 break; 2580 } 2581 case KVM_GET_REGS: { 2582 struct kvm_regs *kvm_regs; 2583 2584 r = -ENOMEM; 2585 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2586 if (!kvm_regs) 2587 goto out; 2588 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2589 if (r) 2590 goto out_free1; 2591 r = -EFAULT; 2592 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2593 goto out_free1; 2594 r = 0; 2595 out_free1: 2596 kfree(kvm_regs); 2597 break; 2598 } 2599 case KVM_SET_REGS: { 2600 struct kvm_regs *kvm_regs; 2601 2602 r = -ENOMEM; 2603 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2604 if (IS_ERR(kvm_regs)) { 2605 r = PTR_ERR(kvm_regs); 2606 goto out; 2607 } 2608 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2609 kfree(kvm_regs); 2610 break; 2611 } 2612 case KVM_GET_SREGS: { 2613 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2614 r = -ENOMEM; 2615 if (!kvm_sregs) 2616 goto out; 2617 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2618 if (r) 2619 goto out; 2620 r = -EFAULT; 2621 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2622 goto out; 2623 r = 0; 2624 break; 2625 } 2626 case KVM_SET_SREGS: { 2627 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2628 if (IS_ERR(kvm_sregs)) { 2629 r = PTR_ERR(kvm_sregs); 2630 kvm_sregs = NULL; 2631 goto out; 2632 } 2633 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2634 break; 2635 } 2636 case KVM_GET_MP_STATE: { 2637 struct kvm_mp_state mp_state; 2638 2639 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2640 if (r) 2641 goto out; 2642 r = -EFAULT; 2643 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2644 goto out; 2645 r = 0; 2646 break; 2647 } 2648 case KVM_SET_MP_STATE: { 2649 struct kvm_mp_state mp_state; 2650 2651 r = -EFAULT; 2652 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2653 goto out; 2654 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2655 break; 2656 } 2657 case KVM_TRANSLATE: { 2658 struct kvm_translation tr; 2659 2660 r = -EFAULT; 2661 if (copy_from_user(&tr, argp, sizeof(tr))) 2662 goto out; 2663 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2664 if (r) 2665 goto out; 2666 r = -EFAULT; 2667 if (copy_to_user(argp, &tr, sizeof(tr))) 2668 goto out; 2669 r = 0; 2670 break; 2671 } 2672 case KVM_SET_GUEST_DEBUG: { 2673 struct kvm_guest_debug dbg; 2674 2675 r = -EFAULT; 2676 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2677 goto out; 2678 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2679 break; 2680 } 2681 case KVM_SET_SIGNAL_MASK: { 2682 struct kvm_signal_mask __user *sigmask_arg = argp; 2683 struct kvm_signal_mask kvm_sigmask; 2684 sigset_t sigset, *p; 2685 2686 p = NULL; 2687 if (argp) { 2688 r = -EFAULT; 2689 if (copy_from_user(&kvm_sigmask, argp, 2690 sizeof(kvm_sigmask))) 2691 goto out; 2692 r = -EINVAL; 2693 if (kvm_sigmask.len != sizeof(sigset)) 2694 goto out; 2695 r = -EFAULT; 2696 if (copy_from_user(&sigset, sigmask_arg->sigset, 2697 sizeof(sigset))) 2698 goto out; 2699 p = &sigset; 2700 } 2701 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2702 break; 2703 } 2704 case KVM_GET_FPU: { 2705 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2706 r = -ENOMEM; 2707 if (!fpu) 2708 goto out; 2709 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2710 if (r) 2711 goto out; 2712 r = -EFAULT; 2713 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2714 goto out; 2715 r = 0; 2716 break; 2717 } 2718 case KVM_SET_FPU: { 2719 fpu = memdup_user(argp, sizeof(*fpu)); 2720 if (IS_ERR(fpu)) { 2721 r = PTR_ERR(fpu); 2722 fpu = NULL; 2723 goto out; 2724 } 2725 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2726 break; 2727 } 2728 default: 2729 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2730 } 2731 out: 2732 mutex_unlock(&vcpu->mutex); 2733 kfree(fpu); 2734 kfree(kvm_sregs); 2735 return r; 2736 } 2737 2738 #ifdef CONFIG_KVM_COMPAT 2739 static long kvm_vcpu_compat_ioctl(struct file *filp, 2740 unsigned int ioctl, unsigned long arg) 2741 { 2742 struct kvm_vcpu *vcpu = filp->private_data; 2743 void __user *argp = compat_ptr(arg); 2744 int r; 2745 2746 if (vcpu->kvm->mm != current->mm) 2747 return -EIO; 2748 2749 switch (ioctl) { 2750 case KVM_SET_SIGNAL_MASK: { 2751 struct kvm_signal_mask __user *sigmask_arg = argp; 2752 struct kvm_signal_mask kvm_sigmask; 2753 sigset_t sigset; 2754 2755 if (argp) { 2756 r = -EFAULT; 2757 if (copy_from_user(&kvm_sigmask, argp, 2758 sizeof(kvm_sigmask))) 2759 goto out; 2760 r = -EINVAL; 2761 if (kvm_sigmask.len != sizeof(compat_sigset_t)) 2762 goto out; 2763 r = -EFAULT; 2764 if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset)) 2765 goto out; 2766 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2767 } else 2768 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2769 break; 2770 } 2771 default: 2772 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2773 } 2774 2775 out: 2776 return r; 2777 } 2778 #endif 2779 2780 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2781 int (*accessor)(struct kvm_device *dev, 2782 struct kvm_device_attr *attr), 2783 unsigned long arg) 2784 { 2785 struct kvm_device_attr attr; 2786 2787 if (!accessor) 2788 return -EPERM; 2789 2790 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2791 return -EFAULT; 2792 2793 return accessor(dev, &attr); 2794 } 2795 2796 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2797 unsigned long arg) 2798 { 2799 struct kvm_device *dev = filp->private_data; 2800 2801 switch (ioctl) { 2802 case KVM_SET_DEVICE_ATTR: 2803 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2804 case KVM_GET_DEVICE_ATTR: 2805 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2806 case KVM_HAS_DEVICE_ATTR: 2807 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2808 default: 2809 if (dev->ops->ioctl) 2810 return dev->ops->ioctl(dev, ioctl, arg); 2811 2812 return -ENOTTY; 2813 } 2814 } 2815 2816 static int kvm_device_release(struct inode *inode, struct file *filp) 2817 { 2818 struct kvm_device *dev = filp->private_data; 2819 struct kvm *kvm = dev->kvm; 2820 2821 kvm_put_kvm(kvm); 2822 return 0; 2823 } 2824 2825 static const struct file_operations kvm_device_fops = { 2826 .unlocked_ioctl = kvm_device_ioctl, 2827 #ifdef CONFIG_KVM_COMPAT 2828 .compat_ioctl = kvm_device_ioctl, 2829 #endif 2830 .release = kvm_device_release, 2831 }; 2832 2833 struct kvm_device *kvm_device_from_filp(struct file *filp) 2834 { 2835 if (filp->f_op != &kvm_device_fops) 2836 return NULL; 2837 2838 return filp->private_data; 2839 } 2840 2841 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2842 #ifdef CONFIG_KVM_MPIC 2843 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2844 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2845 #endif 2846 }; 2847 2848 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2849 { 2850 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2851 return -ENOSPC; 2852 2853 if (kvm_device_ops_table[type] != NULL) 2854 return -EEXIST; 2855 2856 kvm_device_ops_table[type] = ops; 2857 return 0; 2858 } 2859 2860 void kvm_unregister_device_ops(u32 type) 2861 { 2862 if (kvm_device_ops_table[type] != NULL) 2863 kvm_device_ops_table[type] = NULL; 2864 } 2865 2866 static int kvm_ioctl_create_device(struct kvm *kvm, 2867 struct kvm_create_device *cd) 2868 { 2869 struct kvm_device_ops *ops = NULL; 2870 struct kvm_device *dev; 2871 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2872 int ret; 2873 2874 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2875 return -ENODEV; 2876 2877 ops = kvm_device_ops_table[cd->type]; 2878 if (ops == NULL) 2879 return -ENODEV; 2880 2881 if (test) 2882 return 0; 2883 2884 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2885 if (!dev) 2886 return -ENOMEM; 2887 2888 dev->ops = ops; 2889 dev->kvm = kvm; 2890 2891 mutex_lock(&kvm->lock); 2892 ret = ops->create(dev, cd->type); 2893 if (ret < 0) { 2894 mutex_unlock(&kvm->lock); 2895 kfree(dev); 2896 return ret; 2897 } 2898 list_add(&dev->vm_node, &kvm->devices); 2899 mutex_unlock(&kvm->lock); 2900 2901 if (ops->init) 2902 ops->init(dev); 2903 2904 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2905 if (ret < 0) { 2906 mutex_lock(&kvm->lock); 2907 list_del(&dev->vm_node); 2908 mutex_unlock(&kvm->lock); 2909 ops->destroy(dev); 2910 return ret; 2911 } 2912 2913 kvm_get_kvm(kvm); 2914 cd->fd = ret; 2915 return 0; 2916 } 2917 2918 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2919 { 2920 switch (arg) { 2921 case KVM_CAP_USER_MEMORY: 2922 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2923 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2924 case KVM_CAP_INTERNAL_ERROR_DATA: 2925 #ifdef CONFIG_HAVE_KVM_MSI 2926 case KVM_CAP_SIGNAL_MSI: 2927 #endif 2928 #ifdef CONFIG_HAVE_KVM_IRQFD 2929 case KVM_CAP_IRQFD: 2930 case KVM_CAP_IRQFD_RESAMPLE: 2931 #endif 2932 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 2933 case KVM_CAP_CHECK_EXTENSION_VM: 2934 return 1; 2935 #ifdef CONFIG_KVM_MMIO 2936 case KVM_CAP_COALESCED_MMIO: 2937 return KVM_COALESCED_MMIO_PAGE_OFFSET; 2938 #endif 2939 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2940 case KVM_CAP_IRQ_ROUTING: 2941 return KVM_MAX_IRQ_ROUTES; 2942 #endif 2943 #if KVM_ADDRESS_SPACE_NUM > 1 2944 case KVM_CAP_MULTI_ADDRESS_SPACE: 2945 return KVM_ADDRESS_SPACE_NUM; 2946 #endif 2947 case KVM_CAP_MAX_VCPU_ID: 2948 return KVM_MAX_VCPU_ID; 2949 default: 2950 break; 2951 } 2952 return kvm_vm_ioctl_check_extension(kvm, arg); 2953 } 2954 2955 static long kvm_vm_ioctl(struct file *filp, 2956 unsigned int ioctl, unsigned long arg) 2957 { 2958 struct kvm *kvm = filp->private_data; 2959 void __user *argp = (void __user *)arg; 2960 int r; 2961 2962 if (kvm->mm != current->mm) 2963 return -EIO; 2964 switch (ioctl) { 2965 case KVM_CREATE_VCPU: 2966 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2967 break; 2968 case KVM_SET_USER_MEMORY_REGION: { 2969 struct kvm_userspace_memory_region kvm_userspace_mem; 2970 2971 r = -EFAULT; 2972 if (copy_from_user(&kvm_userspace_mem, argp, 2973 sizeof(kvm_userspace_mem))) 2974 goto out; 2975 2976 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2977 break; 2978 } 2979 case KVM_GET_DIRTY_LOG: { 2980 struct kvm_dirty_log log; 2981 2982 r = -EFAULT; 2983 if (copy_from_user(&log, argp, sizeof(log))) 2984 goto out; 2985 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2986 break; 2987 } 2988 #ifdef CONFIG_KVM_MMIO 2989 case KVM_REGISTER_COALESCED_MMIO: { 2990 struct kvm_coalesced_mmio_zone zone; 2991 2992 r = -EFAULT; 2993 if (copy_from_user(&zone, argp, sizeof(zone))) 2994 goto out; 2995 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2996 break; 2997 } 2998 case KVM_UNREGISTER_COALESCED_MMIO: { 2999 struct kvm_coalesced_mmio_zone zone; 3000 3001 r = -EFAULT; 3002 if (copy_from_user(&zone, argp, sizeof(zone))) 3003 goto out; 3004 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 3005 break; 3006 } 3007 #endif 3008 case KVM_IRQFD: { 3009 struct kvm_irqfd data; 3010 3011 r = -EFAULT; 3012 if (copy_from_user(&data, argp, sizeof(data))) 3013 goto out; 3014 r = kvm_irqfd(kvm, &data); 3015 break; 3016 } 3017 case KVM_IOEVENTFD: { 3018 struct kvm_ioeventfd data; 3019 3020 r = -EFAULT; 3021 if (copy_from_user(&data, argp, sizeof(data))) 3022 goto out; 3023 r = kvm_ioeventfd(kvm, &data); 3024 break; 3025 } 3026 #ifdef CONFIG_HAVE_KVM_MSI 3027 case KVM_SIGNAL_MSI: { 3028 struct kvm_msi msi; 3029 3030 r = -EFAULT; 3031 if (copy_from_user(&msi, argp, sizeof(msi))) 3032 goto out; 3033 r = kvm_send_userspace_msi(kvm, &msi); 3034 break; 3035 } 3036 #endif 3037 #ifdef __KVM_HAVE_IRQ_LINE 3038 case KVM_IRQ_LINE_STATUS: 3039 case KVM_IRQ_LINE: { 3040 struct kvm_irq_level irq_event; 3041 3042 r = -EFAULT; 3043 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 3044 goto out; 3045 3046 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 3047 ioctl == KVM_IRQ_LINE_STATUS); 3048 if (r) 3049 goto out; 3050 3051 r = -EFAULT; 3052 if (ioctl == KVM_IRQ_LINE_STATUS) { 3053 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 3054 goto out; 3055 } 3056 3057 r = 0; 3058 break; 3059 } 3060 #endif 3061 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 3062 case KVM_SET_GSI_ROUTING: { 3063 struct kvm_irq_routing routing; 3064 struct kvm_irq_routing __user *urouting; 3065 struct kvm_irq_routing_entry *entries = NULL; 3066 3067 r = -EFAULT; 3068 if (copy_from_user(&routing, argp, sizeof(routing))) 3069 goto out; 3070 r = -EINVAL; 3071 if (!kvm_arch_can_set_irq_routing(kvm)) 3072 goto out; 3073 if (routing.nr > KVM_MAX_IRQ_ROUTES) 3074 goto out; 3075 if (routing.flags) 3076 goto out; 3077 if (routing.nr) { 3078 r = -ENOMEM; 3079 entries = vmalloc(array_size(sizeof(*entries), 3080 routing.nr)); 3081 if (!entries) 3082 goto out; 3083 r = -EFAULT; 3084 urouting = argp; 3085 if (copy_from_user(entries, urouting->entries, 3086 routing.nr * sizeof(*entries))) 3087 goto out_free_irq_routing; 3088 } 3089 r = kvm_set_irq_routing(kvm, entries, routing.nr, 3090 routing.flags); 3091 out_free_irq_routing: 3092 vfree(entries); 3093 break; 3094 } 3095 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 3096 case KVM_CREATE_DEVICE: { 3097 struct kvm_create_device cd; 3098 3099 r = -EFAULT; 3100 if (copy_from_user(&cd, argp, sizeof(cd))) 3101 goto out; 3102 3103 r = kvm_ioctl_create_device(kvm, &cd); 3104 if (r) 3105 goto out; 3106 3107 r = -EFAULT; 3108 if (copy_to_user(argp, &cd, sizeof(cd))) 3109 goto out; 3110 3111 r = 0; 3112 break; 3113 } 3114 case KVM_CHECK_EXTENSION: 3115 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 3116 break; 3117 default: 3118 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 3119 } 3120 out: 3121 return r; 3122 } 3123 3124 #ifdef CONFIG_KVM_COMPAT 3125 struct compat_kvm_dirty_log { 3126 __u32 slot; 3127 __u32 padding1; 3128 union { 3129 compat_uptr_t dirty_bitmap; /* one bit per page */ 3130 __u64 padding2; 3131 }; 3132 }; 3133 3134 static long kvm_vm_compat_ioctl(struct file *filp, 3135 unsigned int ioctl, unsigned long arg) 3136 { 3137 struct kvm *kvm = filp->private_data; 3138 int r; 3139 3140 if (kvm->mm != current->mm) 3141 return -EIO; 3142 switch (ioctl) { 3143 case KVM_GET_DIRTY_LOG: { 3144 struct compat_kvm_dirty_log compat_log; 3145 struct kvm_dirty_log log; 3146 3147 if (copy_from_user(&compat_log, (void __user *)arg, 3148 sizeof(compat_log))) 3149 return -EFAULT; 3150 log.slot = compat_log.slot; 3151 log.padding1 = compat_log.padding1; 3152 log.padding2 = compat_log.padding2; 3153 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 3154 3155 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 3156 break; 3157 } 3158 default: 3159 r = kvm_vm_ioctl(filp, ioctl, arg); 3160 } 3161 return r; 3162 } 3163 #endif 3164 3165 static struct file_operations kvm_vm_fops = { 3166 .release = kvm_vm_release, 3167 .unlocked_ioctl = kvm_vm_ioctl, 3168 #ifdef CONFIG_KVM_COMPAT 3169 .compat_ioctl = kvm_vm_compat_ioctl, 3170 #endif 3171 .llseek = noop_llseek, 3172 }; 3173 3174 static int kvm_dev_ioctl_create_vm(unsigned long type) 3175 { 3176 int r; 3177 struct kvm *kvm; 3178 struct file *file; 3179 3180 kvm = kvm_create_vm(type); 3181 if (IS_ERR(kvm)) 3182 return PTR_ERR(kvm); 3183 #ifdef CONFIG_KVM_MMIO 3184 r = kvm_coalesced_mmio_init(kvm); 3185 if (r < 0) 3186 goto put_kvm; 3187 #endif 3188 r = get_unused_fd_flags(O_CLOEXEC); 3189 if (r < 0) 3190 goto put_kvm; 3191 3192 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 3193 if (IS_ERR(file)) { 3194 put_unused_fd(r); 3195 r = PTR_ERR(file); 3196 goto put_kvm; 3197 } 3198 3199 /* 3200 * Don't call kvm_put_kvm anymore at this point; file->f_op is 3201 * already set, with ->release() being kvm_vm_release(). In error 3202 * cases it will be called by the final fput(file) and will take 3203 * care of doing kvm_put_kvm(kvm). 3204 */ 3205 if (kvm_create_vm_debugfs(kvm, r) < 0) { 3206 put_unused_fd(r); 3207 fput(file); 3208 return -ENOMEM; 3209 } 3210 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); 3211 3212 fd_install(r, file); 3213 return r; 3214 3215 put_kvm: 3216 kvm_put_kvm(kvm); 3217 return r; 3218 } 3219 3220 static long kvm_dev_ioctl(struct file *filp, 3221 unsigned int ioctl, unsigned long arg) 3222 { 3223 long r = -EINVAL; 3224 3225 switch (ioctl) { 3226 case KVM_GET_API_VERSION: 3227 if (arg) 3228 goto out; 3229 r = KVM_API_VERSION; 3230 break; 3231 case KVM_CREATE_VM: 3232 r = kvm_dev_ioctl_create_vm(arg); 3233 break; 3234 case KVM_CHECK_EXTENSION: 3235 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 3236 break; 3237 case KVM_GET_VCPU_MMAP_SIZE: 3238 if (arg) 3239 goto out; 3240 r = PAGE_SIZE; /* struct kvm_run */ 3241 #ifdef CONFIG_X86 3242 r += PAGE_SIZE; /* pio data page */ 3243 #endif 3244 #ifdef CONFIG_KVM_MMIO 3245 r += PAGE_SIZE; /* coalesced mmio ring page */ 3246 #endif 3247 break; 3248 case KVM_TRACE_ENABLE: 3249 case KVM_TRACE_PAUSE: 3250 case KVM_TRACE_DISABLE: 3251 r = -EOPNOTSUPP; 3252 break; 3253 default: 3254 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3255 } 3256 out: 3257 return r; 3258 } 3259 3260 static struct file_operations kvm_chardev_ops = { 3261 .unlocked_ioctl = kvm_dev_ioctl, 3262 .compat_ioctl = kvm_dev_ioctl, 3263 .llseek = noop_llseek, 3264 }; 3265 3266 static struct miscdevice kvm_dev = { 3267 KVM_MINOR, 3268 "kvm", 3269 &kvm_chardev_ops, 3270 }; 3271 3272 static void hardware_enable_nolock(void *junk) 3273 { 3274 int cpu = raw_smp_processor_id(); 3275 int r; 3276 3277 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3278 return; 3279 3280 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3281 3282 r = kvm_arch_hardware_enable(); 3283 3284 if (r) { 3285 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3286 atomic_inc(&hardware_enable_failed); 3287 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3288 } 3289 } 3290 3291 static int kvm_starting_cpu(unsigned int cpu) 3292 { 3293 raw_spin_lock(&kvm_count_lock); 3294 if (kvm_usage_count) 3295 hardware_enable_nolock(NULL); 3296 raw_spin_unlock(&kvm_count_lock); 3297 return 0; 3298 } 3299 3300 static void hardware_disable_nolock(void *junk) 3301 { 3302 int cpu = raw_smp_processor_id(); 3303 3304 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3305 return; 3306 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3307 kvm_arch_hardware_disable(); 3308 } 3309 3310 static int kvm_dying_cpu(unsigned int cpu) 3311 { 3312 raw_spin_lock(&kvm_count_lock); 3313 if (kvm_usage_count) 3314 hardware_disable_nolock(NULL); 3315 raw_spin_unlock(&kvm_count_lock); 3316 return 0; 3317 } 3318 3319 static void hardware_disable_all_nolock(void) 3320 { 3321 BUG_ON(!kvm_usage_count); 3322 3323 kvm_usage_count--; 3324 if (!kvm_usage_count) 3325 on_each_cpu(hardware_disable_nolock, NULL, 1); 3326 } 3327 3328 static void hardware_disable_all(void) 3329 { 3330 raw_spin_lock(&kvm_count_lock); 3331 hardware_disable_all_nolock(); 3332 raw_spin_unlock(&kvm_count_lock); 3333 } 3334 3335 static int hardware_enable_all(void) 3336 { 3337 int r = 0; 3338 3339 raw_spin_lock(&kvm_count_lock); 3340 3341 kvm_usage_count++; 3342 if (kvm_usage_count == 1) { 3343 atomic_set(&hardware_enable_failed, 0); 3344 on_each_cpu(hardware_enable_nolock, NULL, 1); 3345 3346 if (atomic_read(&hardware_enable_failed)) { 3347 hardware_disable_all_nolock(); 3348 r = -EBUSY; 3349 } 3350 } 3351 3352 raw_spin_unlock(&kvm_count_lock); 3353 3354 return r; 3355 } 3356 3357 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3358 void *v) 3359 { 3360 /* 3361 * Some (well, at least mine) BIOSes hang on reboot if 3362 * in vmx root mode. 3363 * 3364 * And Intel TXT required VMX off for all cpu when system shutdown. 3365 */ 3366 pr_info("kvm: exiting hardware virtualization\n"); 3367 kvm_rebooting = true; 3368 on_each_cpu(hardware_disable_nolock, NULL, 1); 3369 return NOTIFY_OK; 3370 } 3371 3372 static struct notifier_block kvm_reboot_notifier = { 3373 .notifier_call = kvm_reboot, 3374 .priority = 0, 3375 }; 3376 3377 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3378 { 3379 int i; 3380 3381 for (i = 0; i < bus->dev_count; i++) { 3382 struct kvm_io_device *pos = bus->range[i].dev; 3383 3384 kvm_iodevice_destructor(pos); 3385 } 3386 kfree(bus); 3387 } 3388 3389 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3390 const struct kvm_io_range *r2) 3391 { 3392 gpa_t addr1 = r1->addr; 3393 gpa_t addr2 = r2->addr; 3394 3395 if (addr1 < addr2) 3396 return -1; 3397 3398 /* If r2->len == 0, match the exact address. If r2->len != 0, 3399 * accept any overlapping write. Any order is acceptable for 3400 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3401 * we process all of them. 3402 */ 3403 if (r2->len) { 3404 addr1 += r1->len; 3405 addr2 += r2->len; 3406 } 3407 3408 if (addr1 > addr2) 3409 return 1; 3410 3411 return 0; 3412 } 3413 3414 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3415 { 3416 return kvm_io_bus_cmp(p1, p2); 3417 } 3418 3419 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3420 gpa_t addr, int len) 3421 { 3422 struct kvm_io_range *range, key; 3423 int off; 3424 3425 key = (struct kvm_io_range) { 3426 .addr = addr, 3427 .len = len, 3428 }; 3429 3430 range = bsearch(&key, bus->range, bus->dev_count, 3431 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3432 if (range == NULL) 3433 return -ENOENT; 3434 3435 off = range - bus->range; 3436 3437 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3438 off--; 3439 3440 return off; 3441 } 3442 3443 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3444 struct kvm_io_range *range, const void *val) 3445 { 3446 int idx; 3447 3448 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3449 if (idx < 0) 3450 return -EOPNOTSUPP; 3451 3452 while (idx < bus->dev_count && 3453 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3454 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3455 range->len, val)) 3456 return idx; 3457 idx++; 3458 } 3459 3460 return -EOPNOTSUPP; 3461 } 3462 3463 /* kvm_io_bus_write - called under kvm->slots_lock */ 3464 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3465 int len, const void *val) 3466 { 3467 struct kvm_io_bus *bus; 3468 struct kvm_io_range range; 3469 int r; 3470 3471 range = (struct kvm_io_range) { 3472 .addr = addr, 3473 .len = len, 3474 }; 3475 3476 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3477 if (!bus) 3478 return -ENOMEM; 3479 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3480 return r < 0 ? r : 0; 3481 } 3482 3483 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3484 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3485 gpa_t addr, int len, const void *val, long cookie) 3486 { 3487 struct kvm_io_bus *bus; 3488 struct kvm_io_range range; 3489 3490 range = (struct kvm_io_range) { 3491 .addr = addr, 3492 .len = len, 3493 }; 3494 3495 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3496 if (!bus) 3497 return -ENOMEM; 3498 3499 /* First try the device referenced by cookie. */ 3500 if ((cookie >= 0) && (cookie < bus->dev_count) && 3501 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3502 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3503 val)) 3504 return cookie; 3505 3506 /* 3507 * cookie contained garbage; fall back to search and return the 3508 * correct cookie value. 3509 */ 3510 return __kvm_io_bus_write(vcpu, bus, &range, val); 3511 } 3512 3513 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3514 struct kvm_io_range *range, void *val) 3515 { 3516 int idx; 3517 3518 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3519 if (idx < 0) 3520 return -EOPNOTSUPP; 3521 3522 while (idx < bus->dev_count && 3523 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3524 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3525 range->len, val)) 3526 return idx; 3527 idx++; 3528 } 3529 3530 return -EOPNOTSUPP; 3531 } 3532 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3533 3534 /* kvm_io_bus_read - called under kvm->slots_lock */ 3535 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3536 int len, void *val) 3537 { 3538 struct kvm_io_bus *bus; 3539 struct kvm_io_range range; 3540 int r; 3541 3542 range = (struct kvm_io_range) { 3543 .addr = addr, 3544 .len = len, 3545 }; 3546 3547 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3548 if (!bus) 3549 return -ENOMEM; 3550 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3551 return r < 0 ? r : 0; 3552 } 3553 3554 3555 /* Caller must hold slots_lock. */ 3556 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3557 int len, struct kvm_io_device *dev) 3558 { 3559 int i; 3560 struct kvm_io_bus *new_bus, *bus; 3561 struct kvm_io_range range; 3562 3563 bus = kvm_get_bus(kvm, bus_idx); 3564 if (!bus) 3565 return -ENOMEM; 3566 3567 /* exclude ioeventfd which is limited by maximum fd */ 3568 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3569 return -ENOSPC; 3570 3571 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3572 sizeof(struct kvm_io_range)), GFP_KERNEL); 3573 if (!new_bus) 3574 return -ENOMEM; 3575 3576 range = (struct kvm_io_range) { 3577 .addr = addr, 3578 .len = len, 3579 .dev = dev, 3580 }; 3581 3582 for (i = 0; i < bus->dev_count; i++) 3583 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0) 3584 break; 3585 3586 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3587 new_bus->dev_count++; 3588 new_bus->range[i] = range; 3589 memcpy(new_bus->range + i + 1, bus->range + i, 3590 (bus->dev_count - i) * sizeof(struct kvm_io_range)); 3591 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3592 synchronize_srcu_expedited(&kvm->srcu); 3593 kfree(bus); 3594 3595 return 0; 3596 } 3597 3598 /* Caller must hold slots_lock. */ 3599 void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3600 struct kvm_io_device *dev) 3601 { 3602 int i; 3603 struct kvm_io_bus *new_bus, *bus; 3604 3605 bus = kvm_get_bus(kvm, bus_idx); 3606 if (!bus) 3607 return; 3608 3609 for (i = 0; i < bus->dev_count; i++) 3610 if (bus->range[i].dev == dev) { 3611 break; 3612 } 3613 3614 if (i == bus->dev_count) 3615 return; 3616 3617 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3618 sizeof(struct kvm_io_range)), GFP_KERNEL); 3619 if (!new_bus) { 3620 pr_err("kvm: failed to shrink bus, removing it completely\n"); 3621 goto broken; 3622 } 3623 3624 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3625 new_bus->dev_count--; 3626 memcpy(new_bus->range + i, bus->range + i + 1, 3627 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3628 3629 broken: 3630 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3631 synchronize_srcu_expedited(&kvm->srcu); 3632 kfree(bus); 3633 return; 3634 } 3635 3636 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3637 gpa_t addr) 3638 { 3639 struct kvm_io_bus *bus; 3640 int dev_idx, srcu_idx; 3641 struct kvm_io_device *iodev = NULL; 3642 3643 srcu_idx = srcu_read_lock(&kvm->srcu); 3644 3645 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3646 if (!bus) 3647 goto out_unlock; 3648 3649 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); 3650 if (dev_idx < 0) 3651 goto out_unlock; 3652 3653 iodev = bus->range[dev_idx].dev; 3654 3655 out_unlock: 3656 srcu_read_unlock(&kvm->srcu, srcu_idx); 3657 3658 return iodev; 3659 } 3660 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev); 3661 3662 static int kvm_debugfs_open(struct inode *inode, struct file *file, 3663 int (*get)(void *, u64 *), int (*set)(void *, u64), 3664 const char *fmt) 3665 { 3666 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3667 inode->i_private; 3668 3669 /* The debugfs files are a reference to the kvm struct which 3670 * is still valid when kvm_destroy_vm is called. 3671 * To avoid the race between open and the removal of the debugfs 3672 * directory we test against the users count. 3673 */ 3674 if (!refcount_inc_not_zero(&stat_data->kvm->users_count)) 3675 return -ENOENT; 3676 3677 if (simple_attr_open(inode, file, get, set, fmt)) { 3678 kvm_put_kvm(stat_data->kvm); 3679 return -ENOMEM; 3680 } 3681 3682 return 0; 3683 } 3684 3685 static int kvm_debugfs_release(struct inode *inode, struct file *file) 3686 { 3687 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3688 inode->i_private; 3689 3690 simple_attr_release(inode, file); 3691 kvm_put_kvm(stat_data->kvm); 3692 3693 return 0; 3694 } 3695 3696 static int vm_stat_get_per_vm(void *data, u64 *val) 3697 { 3698 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3699 3700 *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset); 3701 3702 return 0; 3703 } 3704 3705 static int vm_stat_clear_per_vm(void *data, u64 val) 3706 { 3707 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3708 3709 if (val) 3710 return -EINVAL; 3711 3712 *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0; 3713 3714 return 0; 3715 } 3716 3717 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file) 3718 { 3719 __simple_attr_check_format("%llu\n", 0ull); 3720 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm, 3721 vm_stat_clear_per_vm, "%llu\n"); 3722 } 3723 3724 static const struct file_operations vm_stat_get_per_vm_fops = { 3725 .owner = THIS_MODULE, 3726 .open = vm_stat_get_per_vm_open, 3727 .release = kvm_debugfs_release, 3728 .read = simple_attr_read, 3729 .write = simple_attr_write, 3730 .llseek = no_llseek, 3731 }; 3732 3733 static int vcpu_stat_get_per_vm(void *data, u64 *val) 3734 { 3735 int i; 3736 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3737 struct kvm_vcpu *vcpu; 3738 3739 *val = 0; 3740 3741 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3742 *val += *(u64 *)((void *)vcpu + stat_data->offset); 3743 3744 return 0; 3745 } 3746 3747 static int vcpu_stat_clear_per_vm(void *data, u64 val) 3748 { 3749 int i; 3750 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3751 struct kvm_vcpu *vcpu; 3752 3753 if (val) 3754 return -EINVAL; 3755 3756 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3757 *(u64 *)((void *)vcpu + stat_data->offset) = 0; 3758 3759 return 0; 3760 } 3761 3762 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file) 3763 { 3764 __simple_attr_check_format("%llu\n", 0ull); 3765 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm, 3766 vcpu_stat_clear_per_vm, "%llu\n"); 3767 } 3768 3769 static const struct file_operations vcpu_stat_get_per_vm_fops = { 3770 .owner = THIS_MODULE, 3771 .open = vcpu_stat_get_per_vm_open, 3772 .release = kvm_debugfs_release, 3773 .read = simple_attr_read, 3774 .write = simple_attr_write, 3775 .llseek = no_llseek, 3776 }; 3777 3778 static const struct file_operations *stat_fops_per_vm[] = { 3779 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops, 3780 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops, 3781 }; 3782 3783 static int vm_stat_get(void *_offset, u64 *val) 3784 { 3785 unsigned offset = (long)_offset; 3786 struct kvm *kvm; 3787 struct kvm_stat_data stat_tmp = {.offset = offset}; 3788 u64 tmp_val; 3789 3790 *val = 0; 3791 spin_lock(&kvm_lock); 3792 list_for_each_entry(kvm, &vm_list, vm_list) { 3793 stat_tmp.kvm = kvm; 3794 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3795 *val += tmp_val; 3796 } 3797 spin_unlock(&kvm_lock); 3798 return 0; 3799 } 3800 3801 static int vm_stat_clear(void *_offset, u64 val) 3802 { 3803 unsigned offset = (long)_offset; 3804 struct kvm *kvm; 3805 struct kvm_stat_data stat_tmp = {.offset = offset}; 3806 3807 if (val) 3808 return -EINVAL; 3809 3810 spin_lock(&kvm_lock); 3811 list_for_each_entry(kvm, &vm_list, vm_list) { 3812 stat_tmp.kvm = kvm; 3813 vm_stat_clear_per_vm((void *)&stat_tmp, 0); 3814 } 3815 spin_unlock(&kvm_lock); 3816 3817 return 0; 3818 } 3819 3820 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); 3821 3822 static int vcpu_stat_get(void *_offset, u64 *val) 3823 { 3824 unsigned offset = (long)_offset; 3825 struct kvm *kvm; 3826 struct kvm_stat_data stat_tmp = {.offset = offset}; 3827 u64 tmp_val; 3828 3829 *val = 0; 3830 spin_lock(&kvm_lock); 3831 list_for_each_entry(kvm, &vm_list, vm_list) { 3832 stat_tmp.kvm = kvm; 3833 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3834 *val += tmp_val; 3835 } 3836 spin_unlock(&kvm_lock); 3837 return 0; 3838 } 3839 3840 static int vcpu_stat_clear(void *_offset, u64 val) 3841 { 3842 unsigned offset = (long)_offset; 3843 struct kvm *kvm; 3844 struct kvm_stat_data stat_tmp = {.offset = offset}; 3845 3846 if (val) 3847 return -EINVAL; 3848 3849 spin_lock(&kvm_lock); 3850 list_for_each_entry(kvm, &vm_list, vm_list) { 3851 stat_tmp.kvm = kvm; 3852 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0); 3853 } 3854 spin_unlock(&kvm_lock); 3855 3856 return 0; 3857 } 3858 3859 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, 3860 "%llu\n"); 3861 3862 static const struct file_operations *stat_fops[] = { 3863 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3864 [KVM_STAT_VM] = &vm_stat_fops, 3865 }; 3866 3867 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) 3868 { 3869 struct kobj_uevent_env *env; 3870 unsigned long long created, active; 3871 3872 if (!kvm_dev.this_device || !kvm) 3873 return; 3874 3875 spin_lock(&kvm_lock); 3876 if (type == KVM_EVENT_CREATE_VM) { 3877 kvm_createvm_count++; 3878 kvm_active_vms++; 3879 } else if (type == KVM_EVENT_DESTROY_VM) { 3880 kvm_active_vms--; 3881 } 3882 created = kvm_createvm_count; 3883 active = kvm_active_vms; 3884 spin_unlock(&kvm_lock); 3885 3886 env = kzalloc(sizeof(*env), GFP_KERNEL); 3887 if (!env) 3888 return; 3889 3890 add_uevent_var(env, "CREATED=%llu", created); 3891 add_uevent_var(env, "COUNT=%llu", active); 3892 3893 if (type == KVM_EVENT_CREATE_VM) { 3894 add_uevent_var(env, "EVENT=create"); 3895 kvm->userspace_pid = task_pid_nr(current); 3896 } else if (type == KVM_EVENT_DESTROY_VM) { 3897 add_uevent_var(env, "EVENT=destroy"); 3898 } 3899 add_uevent_var(env, "PID=%d", kvm->userspace_pid); 3900 3901 if (kvm->debugfs_dentry) { 3902 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL); 3903 3904 if (p) { 3905 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); 3906 if (!IS_ERR(tmp)) 3907 add_uevent_var(env, "STATS_PATH=%s", tmp); 3908 kfree(p); 3909 } 3910 } 3911 /* no need for checks, since we are adding at most only 5 keys */ 3912 env->envp[env->envp_idx++] = NULL; 3913 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp); 3914 kfree(env); 3915 } 3916 3917 static void kvm_init_debug(void) 3918 { 3919 struct kvm_stats_debugfs_item *p; 3920 3921 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3922 3923 kvm_debugfs_num_entries = 0; 3924 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { 3925 debugfs_create_file(p->name, 0644, kvm_debugfs_dir, 3926 (void *)(long)p->offset, 3927 stat_fops[p->kind]); 3928 } 3929 } 3930 3931 static int kvm_suspend(void) 3932 { 3933 if (kvm_usage_count) 3934 hardware_disable_nolock(NULL); 3935 return 0; 3936 } 3937 3938 static void kvm_resume(void) 3939 { 3940 if (kvm_usage_count) { 3941 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3942 hardware_enable_nolock(NULL); 3943 } 3944 } 3945 3946 static struct syscore_ops kvm_syscore_ops = { 3947 .suspend = kvm_suspend, 3948 .resume = kvm_resume, 3949 }; 3950 3951 static inline 3952 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3953 { 3954 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3955 } 3956 3957 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3958 { 3959 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3960 3961 if (vcpu->preempted) 3962 vcpu->preempted = false; 3963 3964 kvm_arch_sched_in(vcpu, cpu); 3965 3966 kvm_arch_vcpu_load(vcpu, cpu); 3967 } 3968 3969 static void kvm_sched_out(struct preempt_notifier *pn, 3970 struct task_struct *next) 3971 { 3972 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3973 3974 if (current->state == TASK_RUNNING) 3975 vcpu->preempted = true; 3976 kvm_arch_vcpu_put(vcpu); 3977 } 3978 3979 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3980 struct module *module) 3981 { 3982 int r; 3983 int cpu; 3984 3985 r = kvm_arch_init(opaque); 3986 if (r) 3987 goto out_fail; 3988 3989 /* 3990 * kvm_arch_init makes sure there's at most one caller 3991 * for architectures that support multiple implementations, 3992 * like intel and amd on x86. 3993 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3994 * conflicts in case kvm is already setup for another implementation. 3995 */ 3996 r = kvm_irqfd_init(); 3997 if (r) 3998 goto out_irqfd; 3999 4000 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 4001 r = -ENOMEM; 4002 goto out_free_0; 4003 } 4004 4005 r = kvm_arch_hardware_setup(); 4006 if (r < 0) 4007 goto out_free_0a; 4008 4009 for_each_online_cpu(cpu) { 4010 smp_call_function_single(cpu, 4011 kvm_arch_check_processor_compat, 4012 &r, 1); 4013 if (r < 0) 4014 goto out_free_1; 4015 } 4016 4017 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting", 4018 kvm_starting_cpu, kvm_dying_cpu); 4019 if (r) 4020 goto out_free_2; 4021 register_reboot_notifier(&kvm_reboot_notifier); 4022 4023 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 4024 if (!vcpu_align) 4025 vcpu_align = __alignof__(struct kvm_vcpu); 4026 kvm_vcpu_cache = 4027 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, 4028 SLAB_ACCOUNT, 4029 offsetof(struct kvm_vcpu, arch), 4030 sizeof_field(struct kvm_vcpu, arch), 4031 NULL); 4032 if (!kvm_vcpu_cache) { 4033 r = -ENOMEM; 4034 goto out_free_3; 4035 } 4036 4037 r = kvm_async_pf_init(); 4038 if (r) 4039 goto out_free; 4040 4041 kvm_chardev_ops.owner = module; 4042 kvm_vm_fops.owner = module; 4043 kvm_vcpu_fops.owner = module; 4044 4045 r = misc_register(&kvm_dev); 4046 if (r) { 4047 pr_err("kvm: misc device register failed\n"); 4048 goto out_unreg; 4049 } 4050 4051 register_syscore_ops(&kvm_syscore_ops); 4052 4053 kvm_preempt_ops.sched_in = kvm_sched_in; 4054 kvm_preempt_ops.sched_out = kvm_sched_out; 4055 4056 kvm_init_debug(); 4057 4058 r = kvm_vfio_ops_init(); 4059 WARN_ON(r); 4060 4061 return 0; 4062 4063 out_unreg: 4064 kvm_async_pf_deinit(); 4065 out_free: 4066 kmem_cache_destroy(kvm_vcpu_cache); 4067 out_free_3: 4068 unregister_reboot_notifier(&kvm_reboot_notifier); 4069 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4070 out_free_2: 4071 out_free_1: 4072 kvm_arch_hardware_unsetup(); 4073 out_free_0a: 4074 free_cpumask_var(cpus_hardware_enabled); 4075 out_free_0: 4076 kvm_irqfd_exit(); 4077 out_irqfd: 4078 kvm_arch_exit(); 4079 out_fail: 4080 return r; 4081 } 4082 EXPORT_SYMBOL_GPL(kvm_init); 4083 4084 void kvm_exit(void) 4085 { 4086 debugfs_remove_recursive(kvm_debugfs_dir); 4087 misc_deregister(&kvm_dev); 4088 kmem_cache_destroy(kvm_vcpu_cache); 4089 kvm_async_pf_deinit(); 4090 unregister_syscore_ops(&kvm_syscore_ops); 4091 unregister_reboot_notifier(&kvm_reboot_notifier); 4092 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4093 on_each_cpu(hardware_disable_nolock, NULL, 1); 4094 kvm_arch_hardware_unsetup(); 4095 kvm_arch_exit(); 4096 kvm_irqfd_exit(); 4097 free_cpumask_var(cpus_hardware_enabled); 4098 kvm_vfio_ops_exit(); 4099 } 4100 EXPORT_SYMBOL_GPL(kvm_exit); 4101