1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched/signal.h> 36 #include <linux/sched/mm.h> 37 #include <linux/sched/stat.h> 38 #include <linux/cpumask.h> 39 #include <linux/smp.h> 40 #include <linux/anon_inodes.h> 41 #include <linux/profile.h> 42 #include <linux/kvm_para.h> 43 #include <linux/pagemap.h> 44 #include <linux/mman.h> 45 #include <linux/swap.h> 46 #include <linux/bitops.h> 47 #include <linux/spinlock.h> 48 #include <linux/compat.h> 49 #include <linux/srcu.h> 50 #include <linux/hugetlb.h> 51 #include <linux/slab.h> 52 #include <linux/sort.h> 53 #include <linux/bsearch.h> 54 55 #include <asm/processor.h> 56 #include <asm/io.h> 57 #include <asm/ioctl.h> 58 #include <linux/uaccess.h> 59 #include <asm/pgtable.h> 60 61 #include "coalesced_mmio.h" 62 #include "async_pf.h" 63 #include "vfio.h" 64 65 #define CREATE_TRACE_POINTS 66 #include <trace/events/kvm.h> 67 68 /* Worst case buffer size needed for holding an integer. */ 69 #define ITOA_MAX_LEN 12 70 71 MODULE_AUTHOR("Qumranet"); 72 MODULE_LICENSE("GPL"); 73 74 /* Architectures should define their poll value according to the halt latency */ 75 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 76 module_param(halt_poll_ns, uint, 0644); 77 EXPORT_SYMBOL_GPL(halt_poll_ns); 78 79 /* Default doubles per-vcpu halt_poll_ns. */ 80 unsigned int halt_poll_ns_grow = 2; 81 module_param(halt_poll_ns_grow, uint, 0644); 82 EXPORT_SYMBOL_GPL(halt_poll_ns_grow); 83 84 /* Default resets per-vcpu halt_poll_ns . */ 85 unsigned int halt_poll_ns_shrink; 86 module_param(halt_poll_ns_shrink, uint, 0644); 87 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); 88 89 /* 90 * Ordering of locks: 91 * 92 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 93 */ 94 95 DEFINE_SPINLOCK(kvm_lock); 96 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 97 LIST_HEAD(vm_list); 98 99 static cpumask_var_t cpus_hardware_enabled; 100 static int kvm_usage_count; 101 static atomic_t hardware_enable_failed; 102 103 struct kmem_cache *kvm_vcpu_cache; 104 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 105 106 static __read_mostly struct preempt_ops kvm_preempt_ops; 107 108 struct dentry *kvm_debugfs_dir; 109 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 110 111 static int kvm_debugfs_num_entries; 112 static const struct file_operations *stat_fops_per_vm[]; 113 114 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 115 unsigned long arg); 116 #ifdef CONFIG_KVM_COMPAT 117 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 118 unsigned long arg); 119 #endif 120 static int hardware_enable_all(void); 121 static void hardware_disable_all(void); 122 123 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 124 125 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 126 127 __visible bool kvm_rebooting; 128 EXPORT_SYMBOL_GPL(kvm_rebooting); 129 130 static bool largepages_enabled = true; 131 132 #define KVM_EVENT_CREATE_VM 0 133 #define KVM_EVENT_DESTROY_VM 1 134 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm); 135 static unsigned long long kvm_createvm_count; 136 static unsigned long long kvm_active_vms; 137 138 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 139 { 140 if (pfn_valid(pfn)) 141 return PageReserved(pfn_to_page(pfn)); 142 143 return true; 144 } 145 146 /* 147 * Switches to specified vcpu, until a matching vcpu_put() 148 */ 149 int vcpu_load(struct kvm_vcpu *vcpu) 150 { 151 int cpu; 152 153 if (mutex_lock_killable(&vcpu->mutex)) 154 return -EINTR; 155 cpu = get_cpu(); 156 preempt_notifier_register(&vcpu->preempt_notifier); 157 kvm_arch_vcpu_load(vcpu, cpu); 158 put_cpu(); 159 return 0; 160 } 161 EXPORT_SYMBOL_GPL(vcpu_load); 162 163 void vcpu_put(struct kvm_vcpu *vcpu) 164 { 165 preempt_disable(); 166 kvm_arch_vcpu_put(vcpu); 167 preempt_notifier_unregister(&vcpu->preempt_notifier); 168 preempt_enable(); 169 mutex_unlock(&vcpu->mutex); 170 } 171 EXPORT_SYMBOL_GPL(vcpu_put); 172 173 /* TODO: merge with kvm_arch_vcpu_should_kick */ 174 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req) 175 { 176 int mode = kvm_vcpu_exiting_guest_mode(vcpu); 177 178 /* 179 * We need to wait for the VCPU to reenable interrupts and get out of 180 * READING_SHADOW_PAGE_TABLES mode. 181 */ 182 if (req & KVM_REQUEST_WAIT) 183 return mode != OUTSIDE_GUEST_MODE; 184 185 /* 186 * Need to kick a running VCPU, but otherwise there is nothing to do. 187 */ 188 return mode == IN_GUEST_MODE; 189 } 190 191 static void ack_flush(void *_completed) 192 { 193 } 194 195 static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait) 196 { 197 if (unlikely(!cpus)) 198 cpus = cpu_online_mask; 199 200 if (cpumask_empty(cpus)) 201 return false; 202 203 smp_call_function_many(cpus, ack_flush, NULL, wait); 204 return true; 205 } 206 207 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 208 { 209 int i, cpu, me; 210 cpumask_var_t cpus; 211 bool called; 212 struct kvm_vcpu *vcpu; 213 214 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 215 216 me = get_cpu(); 217 kvm_for_each_vcpu(i, vcpu, kvm) { 218 kvm_make_request(req, vcpu); 219 cpu = vcpu->cpu; 220 221 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu)) 222 continue; 223 224 if (cpus != NULL && cpu != -1 && cpu != me && 225 kvm_request_needs_ipi(vcpu, req)) 226 __cpumask_set_cpu(cpu, cpus); 227 } 228 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT)); 229 put_cpu(); 230 free_cpumask_var(cpus); 231 return called; 232 } 233 234 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 235 void kvm_flush_remote_tlbs(struct kvm *kvm) 236 { 237 /* 238 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 239 * kvm_make_all_cpus_request. 240 */ 241 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 242 243 /* 244 * We want to publish modifications to the page tables before reading 245 * mode. Pairs with a memory barrier in arch-specific code. 246 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 247 * and smp_mb in walk_shadow_page_lockless_begin/end. 248 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 249 * 250 * There is already an smp_mb__after_atomic() before 251 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 252 * barrier here. 253 */ 254 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 255 ++kvm->stat.remote_tlb_flush; 256 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 257 } 258 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 259 #endif 260 261 void kvm_reload_remote_mmus(struct kvm *kvm) 262 { 263 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 264 } 265 266 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 267 { 268 struct page *page; 269 int r; 270 271 mutex_init(&vcpu->mutex); 272 vcpu->cpu = -1; 273 vcpu->kvm = kvm; 274 vcpu->vcpu_id = id; 275 vcpu->pid = NULL; 276 init_swait_queue_head(&vcpu->wq); 277 kvm_async_pf_vcpu_init(vcpu); 278 279 vcpu->pre_pcpu = -1; 280 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 281 282 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 283 if (!page) { 284 r = -ENOMEM; 285 goto fail; 286 } 287 vcpu->run = page_address(page); 288 289 kvm_vcpu_set_in_spin_loop(vcpu, false); 290 kvm_vcpu_set_dy_eligible(vcpu, false); 291 vcpu->preempted = false; 292 293 r = kvm_arch_vcpu_init(vcpu); 294 if (r < 0) 295 goto fail_free_run; 296 return 0; 297 298 fail_free_run: 299 free_page((unsigned long)vcpu->run); 300 fail: 301 return r; 302 } 303 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 304 305 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 306 { 307 /* 308 * no need for rcu_read_lock as VCPU_RUN is the only place that 309 * will change the vcpu->pid pointer and on uninit all file 310 * descriptors are already gone. 311 */ 312 put_pid(rcu_dereference_protected(vcpu->pid, 1)); 313 kvm_arch_vcpu_uninit(vcpu); 314 free_page((unsigned long)vcpu->run); 315 } 316 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 317 318 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 319 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 320 { 321 return container_of(mn, struct kvm, mmu_notifier); 322 } 323 324 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 325 struct mm_struct *mm, 326 unsigned long address, 327 pte_t pte) 328 { 329 struct kvm *kvm = mmu_notifier_to_kvm(mn); 330 int idx; 331 332 idx = srcu_read_lock(&kvm->srcu); 333 spin_lock(&kvm->mmu_lock); 334 kvm->mmu_notifier_seq++; 335 kvm_set_spte_hva(kvm, address, pte); 336 spin_unlock(&kvm->mmu_lock); 337 srcu_read_unlock(&kvm->srcu, idx); 338 } 339 340 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 341 struct mm_struct *mm, 342 unsigned long start, 343 unsigned long end) 344 { 345 struct kvm *kvm = mmu_notifier_to_kvm(mn); 346 int need_tlb_flush = 0, idx; 347 348 idx = srcu_read_lock(&kvm->srcu); 349 spin_lock(&kvm->mmu_lock); 350 /* 351 * The count increase must become visible at unlock time as no 352 * spte can be established without taking the mmu_lock and 353 * count is also read inside the mmu_lock critical section. 354 */ 355 kvm->mmu_notifier_count++; 356 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 357 need_tlb_flush |= kvm->tlbs_dirty; 358 /* we've to flush the tlb before the pages can be freed */ 359 if (need_tlb_flush) 360 kvm_flush_remote_tlbs(kvm); 361 362 spin_unlock(&kvm->mmu_lock); 363 srcu_read_unlock(&kvm->srcu, idx); 364 } 365 366 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 367 struct mm_struct *mm, 368 unsigned long start, 369 unsigned long end) 370 { 371 struct kvm *kvm = mmu_notifier_to_kvm(mn); 372 373 spin_lock(&kvm->mmu_lock); 374 /* 375 * This sequence increase will notify the kvm page fault that 376 * the page that is going to be mapped in the spte could have 377 * been freed. 378 */ 379 kvm->mmu_notifier_seq++; 380 smp_wmb(); 381 /* 382 * The above sequence increase must be visible before the 383 * below count decrease, which is ensured by the smp_wmb above 384 * in conjunction with the smp_rmb in mmu_notifier_retry(). 385 */ 386 kvm->mmu_notifier_count--; 387 spin_unlock(&kvm->mmu_lock); 388 389 BUG_ON(kvm->mmu_notifier_count < 0); 390 } 391 392 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 393 struct mm_struct *mm, 394 unsigned long start, 395 unsigned long end) 396 { 397 struct kvm *kvm = mmu_notifier_to_kvm(mn); 398 int young, idx; 399 400 idx = srcu_read_lock(&kvm->srcu); 401 spin_lock(&kvm->mmu_lock); 402 403 young = kvm_age_hva(kvm, start, end); 404 if (young) 405 kvm_flush_remote_tlbs(kvm); 406 407 spin_unlock(&kvm->mmu_lock); 408 srcu_read_unlock(&kvm->srcu, idx); 409 410 return young; 411 } 412 413 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 414 struct mm_struct *mm, 415 unsigned long start, 416 unsigned long end) 417 { 418 struct kvm *kvm = mmu_notifier_to_kvm(mn); 419 int young, idx; 420 421 idx = srcu_read_lock(&kvm->srcu); 422 spin_lock(&kvm->mmu_lock); 423 /* 424 * Even though we do not flush TLB, this will still adversely 425 * affect performance on pre-Haswell Intel EPT, where there is 426 * no EPT Access Bit to clear so that we have to tear down EPT 427 * tables instead. If we find this unacceptable, we can always 428 * add a parameter to kvm_age_hva so that it effectively doesn't 429 * do anything on clear_young. 430 * 431 * Also note that currently we never issue secondary TLB flushes 432 * from clear_young, leaving this job up to the regular system 433 * cadence. If we find this inaccurate, we might come up with a 434 * more sophisticated heuristic later. 435 */ 436 young = kvm_age_hva(kvm, start, end); 437 spin_unlock(&kvm->mmu_lock); 438 srcu_read_unlock(&kvm->srcu, idx); 439 440 return young; 441 } 442 443 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 444 struct mm_struct *mm, 445 unsigned long address) 446 { 447 struct kvm *kvm = mmu_notifier_to_kvm(mn); 448 int young, idx; 449 450 idx = srcu_read_lock(&kvm->srcu); 451 spin_lock(&kvm->mmu_lock); 452 young = kvm_test_age_hva(kvm, address); 453 spin_unlock(&kvm->mmu_lock); 454 srcu_read_unlock(&kvm->srcu, idx); 455 456 return young; 457 } 458 459 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 460 struct mm_struct *mm) 461 { 462 struct kvm *kvm = mmu_notifier_to_kvm(mn); 463 int idx; 464 465 idx = srcu_read_lock(&kvm->srcu); 466 kvm_arch_flush_shadow_all(kvm); 467 srcu_read_unlock(&kvm->srcu, idx); 468 } 469 470 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 471 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 472 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 473 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 474 .clear_young = kvm_mmu_notifier_clear_young, 475 .test_young = kvm_mmu_notifier_test_young, 476 .change_pte = kvm_mmu_notifier_change_pte, 477 .release = kvm_mmu_notifier_release, 478 }; 479 480 static int kvm_init_mmu_notifier(struct kvm *kvm) 481 { 482 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 483 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 484 } 485 486 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 487 488 static int kvm_init_mmu_notifier(struct kvm *kvm) 489 { 490 return 0; 491 } 492 493 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 494 495 static struct kvm_memslots *kvm_alloc_memslots(void) 496 { 497 int i; 498 struct kvm_memslots *slots; 499 500 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 501 if (!slots) 502 return NULL; 503 504 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 505 slots->id_to_index[i] = slots->memslots[i].id = i; 506 507 return slots; 508 } 509 510 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 511 { 512 if (!memslot->dirty_bitmap) 513 return; 514 515 kvfree(memslot->dirty_bitmap); 516 memslot->dirty_bitmap = NULL; 517 } 518 519 /* 520 * Free any memory in @free but not in @dont. 521 */ 522 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 523 struct kvm_memory_slot *dont) 524 { 525 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 526 kvm_destroy_dirty_bitmap(free); 527 528 kvm_arch_free_memslot(kvm, free, dont); 529 530 free->npages = 0; 531 } 532 533 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 534 { 535 struct kvm_memory_slot *memslot; 536 537 if (!slots) 538 return; 539 540 kvm_for_each_memslot(memslot, slots) 541 kvm_free_memslot(kvm, memslot, NULL); 542 543 kvfree(slots); 544 } 545 546 static void kvm_destroy_vm_debugfs(struct kvm *kvm) 547 { 548 int i; 549 550 if (!kvm->debugfs_dentry) 551 return; 552 553 debugfs_remove_recursive(kvm->debugfs_dentry); 554 555 if (kvm->debugfs_stat_data) { 556 for (i = 0; i < kvm_debugfs_num_entries; i++) 557 kfree(kvm->debugfs_stat_data[i]); 558 kfree(kvm->debugfs_stat_data); 559 } 560 } 561 562 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) 563 { 564 char dir_name[ITOA_MAX_LEN * 2]; 565 struct kvm_stat_data *stat_data; 566 struct kvm_stats_debugfs_item *p; 567 568 if (!debugfs_initialized()) 569 return 0; 570 571 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd); 572 kvm->debugfs_dentry = debugfs_create_dir(dir_name, 573 kvm_debugfs_dir); 574 if (!kvm->debugfs_dentry) 575 return -ENOMEM; 576 577 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, 578 sizeof(*kvm->debugfs_stat_data), 579 GFP_KERNEL); 580 if (!kvm->debugfs_stat_data) 581 return -ENOMEM; 582 583 for (p = debugfs_entries; p->name; p++) { 584 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL); 585 if (!stat_data) 586 return -ENOMEM; 587 588 stat_data->kvm = kvm; 589 stat_data->offset = p->offset; 590 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; 591 if (!debugfs_create_file(p->name, 0644, 592 kvm->debugfs_dentry, 593 stat_data, 594 stat_fops_per_vm[p->kind])) 595 return -ENOMEM; 596 } 597 return 0; 598 } 599 600 static struct kvm *kvm_create_vm(unsigned long type) 601 { 602 int r, i; 603 struct kvm *kvm = kvm_arch_alloc_vm(); 604 605 if (!kvm) 606 return ERR_PTR(-ENOMEM); 607 608 spin_lock_init(&kvm->mmu_lock); 609 mmgrab(current->mm); 610 kvm->mm = current->mm; 611 kvm_eventfd_init(kvm); 612 mutex_init(&kvm->lock); 613 mutex_init(&kvm->irq_lock); 614 mutex_init(&kvm->slots_lock); 615 refcount_set(&kvm->users_count, 1); 616 INIT_LIST_HEAD(&kvm->devices); 617 618 r = kvm_arch_init_vm(kvm, type); 619 if (r) 620 goto out_err_no_disable; 621 622 r = hardware_enable_all(); 623 if (r) 624 goto out_err_no_disable; 625 626 #ifdef CONFIG_HAVE_KVM_IRQFD 627 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 628 #endif 629 630 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 631 632 r = -ENOMEM; 633 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 634 struct kvm_memslots *slots = kvm_alloc_memslots(); 635 if (!slots) 636 goto out_err_no_srcu; 637 /* 638 * Generations must be different for each address space. 639 * Init kvm generation close to the maximum to easily test the 640 * code of handling generation number wrap-around. 641 */ 642 slots->generation = i * 2 - 150; 643 rcu_assign_pointer(kvm->memslots[i], slots); 644 } 645 646 if (init_srcu_struct(&kvm->srcu)) 647 goto out_err_no_srcu; 648 if (init_srcu_struct(&kvm->irq_srcu)) 649 goto out_err_no_irq_srcu; 650 for (i = 0; i < KVM_NR_BUSES; i++) { 651 rcu_assign_pointer(kvm->buses[i], 652 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL)); 653 if (!kvm->buses[i]) 654 goto out_err; 655 } 656 657 r = kvm_init_mmu_notifier(kvm); 658 if (r) 659 goto out_err; 660 661 spin_lock(&kvm_lock); 662 list_add(&kvm->vm_list, &vm_list); 663 spin_unlock(&kvm_lock); 664 665 preempt_notifier_inc(); 666 667 return kvm; 668 669 out_err: 670 cleanup_srcu_struct(&kvm->irq_srcu); 671 out_err_no_irq_srcu: 672 cleanup_srcu_struct(&kvm->srcu); 673 out_err_no_srcu: 674 hardware_disable_all(); 675 out_err_no_disable: 676 refcount_set(&kvm->users_count, 0); 677 for (i = 0; i < KVM_NR_BUSES; i++) 678 kfree(kvm_get_bus(kvm, i)); 679 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 680 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); 681 kvm_arch_free_vm(kvm); 682 mmdrop(current->mm); 683 return ERR_PTR(r); 684 } 685 686 static void kvm_destroy_devices(struct kvm *kvm) 687 { 688 struct kvm_device *dev, *tmp; 689 690 /* 691 * We do not need to take the kvm->lock here, because nobody else 692 * has a reference to the struct kvm at this point and therefore 693 * cannot access the devices list anyhow. 694 */ 695 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 696 list_del(&dev->vm_node); 697 dev->ops->destroy(dev); 698 } 699 } 700 701 static void kvm_destroy_vm(struct kvm *kvm) 702 { 703 int i; 704 struct mm_struct *mm = kvm->mm; 705 706 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); 707 kvm_destroy_vm_debugfs(kvm); 708 kvm_arch_sync_events(kvm); 709 spin_lock(&kvm_lock); 710 list_del(&kvm->vm_list); 711 spin_unlock(&kvm_lock); 712 kvm_free_irq_routing(kvm); 713 for (i = 0; i < KVM_NR_BUSES; i++) { 714 struct kvm_io_bus *bus = kvm_get_bus(kvm, i); 715 716 if (bus) 717 kvm_io_bus_destroy(bus); 718 kvm->buses[i] = NULL; 719 } 720 kvm_coalesced_mmio_free(kvm); 721 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 722 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 723 #else 724 kvm_arch_flush_shadow_all(kvm); 725 #endif 726 kvm_arch_destroy_vm(kvm); 727 kvm_destroy_devices(kvm); 728 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 729 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); 730 cleanup_srcu_struct(&kvm->irq_srcu); 731 cleanup_srcu_struct(&kvm->srcu); 732 kvm_arch_free_vm(kvm); 733 preempt_notifier_dec(); 734 hardware_disable_all(); 735 mmdrop(mm); 736 } 737 738 void kvm_get_kvm(struct kvm *kvm) 739 { 740 refcount_inc(&kvm->users_count); 741 } 742 EXPORT_SYMBOL_GPL(kvm_get_kvm); 743 744 void kvm_put_kvm(struct kvm *kvm) 745 { 746 if (refcount_dec_and_test(&kvm->users_count)) 747 kvm_destroy_vm(kvm); 748 } 749 EXPORT_SYMBOL_GPL(kvm_put_kvm); 750 751 752 static int kvm_vm_release(struct inode *inode, struct file *filp) 753 { 754 struct kvm *kvm = filp->private_data; 755 756 kvm_irqfd_release(kvm); 757 758 kvm_put_kvm(kvm); 759 return 0; 760 } 761 762 /* 763 * Allocation size is twice as large as the actual dirty bitmap size. 764 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 765 */ 766 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 767 { 768 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 769 770 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL); 771 if (!memslot->dirty_bitmap) 772 return -ENOMEM; 773 774 return 0; 775 } 776 777 /* 778 * Insert memslot and re-sort memslots based on their GFN, 779 * so binary search could be used to lookup GFN. 780 * Sorting algorithm takes advantage of having initially 781 * sorted array and known changed memslot position. 782 */ 783 static void update_memslots(struct kvm_memslots *slots, 784 struct kvm_memory_slot *new) 785 { 786 int id = new->id; 787 int i = slots->id_to_index[id]; 788 struct kvm_memory_slot *mslots = slots->memslots; 789 790 WARN_ON(mslots[i].id != id); 791 if (!new->npages) { 792 WARN_ON(!mslots[i].npages); 793 if (mslots[i].npages) 794 slots->used_slots--; 795 } else { 796 if (!mslots[i].npages) 797 slots->used_slots++; 798 } 799 800 while (i < KVM_MEM_SLOTS_NUM - 1 && 801 new->base_gfn <= mslots[i + 1].base_gfn) { 802 if (!mslots[i + 1].npages) 803 break; 804 mslots[i] = mslots[i + 1]; 805 slots->id_to_index[mslots[i].id] = i; 806 i++; 807 } 808 809 /* 810 * The ">=" is needed when creating a slot with base_gfn == 0, 811 * so that it moves before all those with base_gfn == npages == 0. 812 * 813 * On the other hand, if new->npages is zero, the above loop has 814 * already left i pointing to the beginning of the empty part of 815 * mslots, and the ">=" would move the hole backwards in this 816 * case---which is wrong. So skip the loop when deleting a slot. 817 */ 818 if (new->npages) { 819 while (i > 0 && 820 new->base_gfn >= mslots[i - 1].base_gfn) { 821 mslots[i] = mslots[i - 1]; 822 slots->id_to_index[mslots[i].id] = i; 823 i--; 824 } 825 } else 826 WARN_ON_ONCE(i != slots->used_slots); 827 828 mslots[i] = *new; 829 slots->id_to_index[mslots[i].id] = i; 830 } 831 832 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 833 { 834 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 835 836 #ifdef __KVM_HAVE_READONLY_MEM 837 valid_flags |= KVM_MEM_READONLY; 838 #endif 839 840 if (mem->flags & ~valid_flags) 841 return -EINVAL; 842 843 return 0; 844 } 845 846 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 847 int as_id, struct kvm_memslots *slots) 848 { 849 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 850 851 /* 852 * Set the low bit in the generation, which disables SPTE caching 853 * until the end of synchronize_srcu_expedited. 854 */ 855 WARN_ON(old_memslots->generation & 1); 856 slots->generation = old_memslots->generation + 1; 857 858 rcu_assign_pointer(kvm->memslots[as_id], slots); 859 synchronize_srcu_expedited(&kvm->srcu); 860 861 /* 862 * Increment the new memslot generation a second time. This prevents 863 * vm exits that race with memslot updates from caching a memslot 864 * generation that will (potentially) be valid forever. 865 * 866 * Generations must be unique even across address spaces. We do not need 867 * a global counter for that, instead the generation space is evenly split 868 * across address spaces. For example, with two address spaces, address 869 * space 0 will use generations 0, 4, 8, ... while * address space 1 will 870 * use generations 2, 6, 10, 14, ... 871 */ 872 slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1; 873 874 kvm_arch_memslots_updated(kvm, slots); 875 876 return old_memslots; 877 } 878 879 /* 880 * Allocate some memory and give it an address in the guest physical address 881 * space. 882 * 883 * Discontiguous memory is allowed, mostly for framebuffers. 884 * 885 * Must be called holding kvm->slots_lock for write. 886 */ 887 int __kvm_set_memory_region(struct kvm *kvm, 888 const struct kvm_userspace_memory_region *mem) 889 { 890 int r; 891 gfn_t base_gfn; 892 unsigned long npages; 893 struct kvm_memory_slot *slot; 894 struct kvm_memory_slot old, new; 895 struct kvm_memslots *slots = NULL, *old_memslots; 896 int as_id, id; 897 enum kvm_mr_change change; 898 899 r = check_memory_region_flags(mem); 900 if (r) 901 goto out; 902 903 r = -EINVAL; 904 as_id = mem->slot >> 16; 905 id = (u16)mem->slot; 906 907 /* General sanity checks */ 908 if (mem->memory_size & (PAGE_SIZE - 1)) 909 goto out; 910 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 911 goto out; 912 /* We can read the guest memory with __xxx_user() later on. */ 913 if ((id < KVM_USER_MEM_SLOTS) && 914 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 915 !access_ok(VERIFY_WRITE, 916 (void __user *)(unsigned long)mem->userspace_addr, 917 mem->memory_size))) 918 goto out; 919 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 920 goto out; 921 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 922 goto out; 923 924 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 925 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 926 npages = mem->memory_size >> PAGE_SHIFT; 927 928 if (npages > KVM_MEM_MAX_NR_PAGES) 929 goto out; 930 931 new = old = *slot; 932 933 new.id = id; 934 new.base_gfn = base_gfn; 935 new.npages = npages; 936 new.flags = mem->flags; 937 938 if (npages) { 939 if (!old.npages) 940 change = KVM_MR_CREATE; 941 else { /* Modify an existing slot. */ 942 if ((mem->userspace_addr != old.userspace_addr) || 943 (npages != old.npages) || 944 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 945 goto out; 946 947 if (base_gfn != old.base_gfn) 948 change = KVM_MR_MOVE; 949 else if (new.flags != old.flags) 950 change = KVM_MR_FLAGS_ONLY; 951 else { /* Nothing to change. */ 952 r = 0; 953 goto out; 954 } 955 } 956 } else { 957 if (!old.npages) 958 goto out; 959 960 change = KVM_MR_DELETE; 961 new.base_gfn = 0; 962 new.flags = 0; 963 } 964 965 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 966 /* Check for overlaps */ 967 r = -EEXIST; 968 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 969 if ((slot->id >= KVM_USER_MEM_SLOTS) || 970 (slot->id == id)) 971 continue; 972 if (!((base_gfn + npages <= slot->base_gfn) || 973 (base_gfn >= slot->base_gfn + slot->npages))) 974 goto out; 975 } 976 } 977 978 /* Free page dirty bitmap if unneeded */ 979 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 980 new.dirty_bitmap = NULL; 981 982 r = -ENOMEM; 983 if (change == KVM_MR_CREATE) { 984 new.userspace_addr = mem->userspace_addr; 985 986 if (kvm_arch_create_memslot(kvm, &new, npages)) 987 goto out_free; 988 } 989 990 /* Allocate page dirty bitmap if needed */ 991 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 992 if (kvm_create_dirty_bitmap(&new) < 0) 993 goto out_free; 994 } 995 996 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 997 if (!slots) 998 goto out_free; 999 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 1000 1001 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 1002 slot = id_to_memslot(slots, id); 1003 slot->flags |= KVM_MEMSLOT_INVALID; 1004 1005 old_memslots = install_new_memslots(kvm, as_id, slots); 1006 1007 /* From this point no new shadow pages pointing to a deleted, 1008 * or moved, memslot will be created. 1009 * 1010 * validation of sp->gfn happens in: 1011 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 1012 * - kvm_is_visible_gfn (mmu_check_roots) 1013 */ 1014 kvm_arch_flush_shadow_memslot(kvm, slot); 1015 1016 /* 1017 * We can re-use the old_memslots from above, the only difference 1018 * from the currently installed memslots is the invalid flag. This 1019 * will get overwritten by update_memslots anyway. 1020 */ 1021 slots = old_memslots; 1022 } 1023 1024 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 1025 if (r) 1026 goto out_slots; 1027 1028 /* actual memory is freed via old in kvm_free_memslot below */ 1029 if (change == KVM_MR_DELETE) { 1030 new.dirty_bitmap = NULL; 1031 memset(&new.arch, 0, sizeof(new.arch)); 1032 } 1033 1034 update_memslots(slots, &new); 1035 old_memslots = install_new_memslots(kvm, as_id, slots); 1036 1037 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 1038 1039 kvm_free_memslot(kvm, &old, &new); 1040 kvfree(old_memslots); 1041 return 0; 1042 1043 out_slots: 1044 kvfree(slots); 1045 out_free: 1046 kvm_free_memslot(kvm, &new, &old); 1047 out: 1048 return r; 1049 } 1050 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 1051 1052 int kvm_set_memory_region(struct kvm *kvm, 1053 const struct kvm_userspace_memory_region *mem) 1054 { 1055 int r; 1056 1057 mutex_lock(&kvm->slots_lock); 1058 r = __kvm_set_memory_region(kvm, mem); 1059 mutex_unlock(&kvm->slots_lock); 1060 return r; 1061 } 1062 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1063 1064 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1065 struct kvm_userspace_memory_region *mem) 1066 { 1067 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1068 return -EINVAL; 1069 1070 return kvm_set_memory_region(kvm, mem); 1071 } 1072 1073 int kvm_get_dirty_log(struct kvm *kvm, 1074 struct kvm_dirty_log *log, int *is_dirty) 1075 { 1076 struct kvm_memslots *slots; 1077 struct kvm_memory_slot *memslot; 1078 int i, as_id, id; 1079 unsigned long n; 1080 unsigned long any = 0; 1081 1082 as_id = log->slot >> 16; 1083 id = (u16)log->slot; 1084 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1085 return -EINVAL; 1086 1087 slots = __kvm_memslots(kvm, as_id); 1088 memslot = id_to_memslot(slots, id); 1089 if (!memslot->dirty_bitmap) 1090 return -ENOENT; 1091 1092 n = kvm_dirty_bitmap_bytes(memslot); 1093 1094 for (i = 0; !any && i < n/sizeof(long); ++i) 1095 any = memslot->dirty_bitmap[i]; 1096 1097 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1098 return -EFAULT; 1099 1100 if (any) 1101 *is_dirty = 1; 1102 return 0; 1103 } 1104 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1105 1106 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1107 /** 1108 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1109 * are dirty write protect them for next write. 1110 * @kvm: pointer to kvm instance 1111 * @log: slot id and address to which we copy the log 1112 * @is_dirty: flag set if any page is dirty 1113 * 1114 * We need to keep it in mind that VCPU threads can write to the bitmap 1115 * concurrently. So, to avoid losing track of dirty pages we keep the 1116 * following order: 1117 * 1118 * 1. Take a snapshot of the bit and clear it if needed. 1119 * 2. Write protect the corresponding page. 1120 * 3. Copy the snapshot to the userspace. 1121 * 4. Upon return caller flushes TLB's if needed. 1122 * 1123 * Between 2 and 4, the guest may write to the page using the remaining TLB 1124 * entry. This is not a problem because the page is reported dirty using 1125 * the snapshot taken before and step 4 ensures that writes done after 1126 * exiting to userspace will be logged for the next call. 1127 * 1128 */ 1129 int kvm_get_dirty_log_protect(struct kvm *kvm, 1130 struct kvm_dirty_log *log, bool *is_dirty) 1131 { 1132 struct kvm_memslots *slots; 1133 struct kvm_memory_slot *memslot; 1134 int i, as_id, id; 1135 unsigned long n; 1136 unsigned long *dirty_bitmap; 1137 unsigned long *dirty_bitmap_buffer; 1138 1139 as_id = log->slot >> 16; 1140 id = (u16)log->slot; 1141 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1142 return -EINVAL; 1143 1144 slots = __kvm_memslots(kvm, as_id); 1145 memslot = id_to_memslot(slots, id); 1146 1147 dirty_bitmap = memslot->dirty_bitmap; 1148 if (!dirty_bitmap) 1149 return -ENOENT; 1150 1151 n = kvm_dirty_bitmap_bytes(memslot); 1152 1153 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1154 memset(dirty_bitmap_buffer, 0, n); 1155 1156 spin_lock(&kvm->mmu_lock); 1157 *is_dirty = false; 1158 for (i = 0; i < n / sizeof(long); i++) { 1159 unsigned long mask; 1160 gfn_t offset; 1161 1162 if (!dirty_bitmap[i]) 1163 continue; 1164 1165 *is_dirty = true; 1166 1167 mask = xchg(&dirty_bitmap[i], 0); 1168 dirty_bitmap_buffer[i] = mask; 1169 1170 if (mask) { 1171 offset = i * BITS_PER_LONG; 1172 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1173 offset, mask); 1174 } 1175 } 1176 1177 spin_unlock(&kvm->mmu_lock); 1178 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1179 return -EFAULT; 1180 return 0; 1181 } 1182 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1183 #endif 1184 1185 bool kvm_largepages_enabled(void) 1186 { 1187 return largepages_enabled; 1188 } 1189 1190 void kvm_disable_largepages(void) 1191 { 1192 largepages_enabled = false; 1193 } 1194 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1195 1196 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1197 { 1198 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1199 } 1200 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1201 1202 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1203 { 1204 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1205 } 1206 1207 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1208 { 1209 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1210 1211 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1212 memslot->flags & KVM_MEMSLOT_INVALID) 1213 return false; 1214 1215 return true; 1216 } 1217 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1218 1219 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1220 { 1221 struct vm_area_struct *vma; 1222 unsigned long addr, size; 1223 1224 size = PAGE_SIZE; 1225 1226 addr = gfn_to_hva(kvm, gfn); 1227 if (kvm_is_error_hva(addr)) 1228 return PAGE_SIZE; 1229 1230 down_read(¤t->mm->mmap_sem); 1231 vma = find_vma(current->mm, addr); 1232 if (!vma) 1233 goto out; 1234 1235 size = vma_kernel_pagesize(vma); 1236 1237 out: 1238 up_read(¤t->mm->mmap_sem); 1239 1240 return size; 1241 } 1242 1243 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1244 { 1245 return slot->flags & KVM_MEM_READONLY; 1246 } 1247 1248 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1249 gfn_t *nr_pages, bool write) 1250 { 1251 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1252 return KVM_HVA_ERR_BAD; 1253 1254 if (memslot_is_readonly(slot) && write) 1255 return KVM_HVA_ERR_RO_BAD; 1256 1257 if (nr_pages) 1258 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1259 1260 return __gfn_to_hva_memslot(slot, gfn); 1261 } 1262 1263 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1264 gfn_t *nr_pages) 1265 { 1266 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1267 } 1268 1269 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1270 gfn_t gfn) 1271 { 1272 return gfn_to_hva_many(slot, gfn, NULL); 1273 } 1274 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1275 1276 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1277 { 1278 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1279 } 1280 EXPORT_SYMBOL_GPL(gfn_to_hva); 1281 1282 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1283 { 1284 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1285 } 1286 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1287 1288 /* 1289 * If writable is set to false, the hva returned by this function is only 1290 * allowed to be read. 1291 */ 1292 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1293 gfn_t gfn, bool *writable) 1294 { 1295 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1296 1297 if (!kvm_is_error_hva(hva) && writable) 1298 *writable = !memslot_is_readonly(slot); 1299 1300 return hva; 1301 } 1302 1303 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1304 { 1305 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1306 1307 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1308 } 1309 1310 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1311 { 1312 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1313 1314 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1315 } 1316 1317 static int get_user_page_nowait(unsigned long start, int write, 1318 struct page **page) 1319 { 1320 int flags = FOLL_NOWAIT | FOLL_HWPOISON; 1321 1322 if (write) 1323 flags |= FOLL_WRITE; 1324 1325 return get_user_pages(start, 1, flags, page, NULL); 1326 } 1327 1328 static inline int check_user_page_hwpoison(unsigned long addr) 1329 { 1330 int rc, flags = FOLL_HWPOISON | FOLL_WRITE; 1331 1332 rc = get_user_pages(addr, 1, flags, NULL, NULL); 1333 return rc == -EHWPOISON; 1334 } 1335 1336 /* 1337 * The atomic path to get the writable pfn which will be stored in @pfn, 1338 * true indicates success, otherwise false is returned. 1339 */ 1340 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1341 bool write_fault, bool *writable, kvm_pfn_t *pfn) 1342 { 1343 struct page *page[1]; 1344 int npages; 1345 1346 if (!(async || atomic)) 1347 return false; 1348 1349 /* 1350 * Fast pin a writable pfn only if it is a write fault request 1351 * or the caller allows to map a writable pfn for a read fault 1352 * request. 1353 */ 1354 if (!(write_fault || writable)) 1355 return false; 1356 1357 npages = __get_user_pages_fast(addr, 1, 1, page); 1358 if (npages == 1) { 1359 *pfn = page_to_pfn(page[0]); 1360 1361 if (writable) 1362 *writable = true; 1363 return true; 1364 } 1365 1366 return false; 1367 } 1368 1369 /* 1370 * The slow path to get the pfn of the specified host virtual address, 1371 * 1 indicates success, -errno is returned if error is detected. 1372 */ 1373 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1374 bool *writable, kvm_pfn_t *pfn) 1375 { 1376 struct page *page[1]; 1377 int npages = 0; 1378 1379 might_sleep(); 1380 1381 if (writable) 1382 *writable = write_fault; 1383 1384 if (async) { 1385 down_read(¤t->mm->mmap_sem); 1386 npages = get_user_page_nowait(addr, write_fault, page); 1387 up_read(¤t->mm->mmap_sem); 1388 } else { 1389 unsigned int flags = FOLL_HWPOISON; 1390 1391 if (write_fault) 1392 flags |= FOLL_WRITE; 1393 1394 npages = get_user_pages_unlocked(addr, 1, page, flags); 1395 } 1396 if (npages != 1) 1397 return npages; 1398 1399 /* map read fault as writable if possible */ 1400 if (unlikely(!write_fault) && writable) { 1401 struct page *wpage[1]; 1402 1403 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1404 if (npages == 1) { 1405 *writable = true; 1406 put_page(page[0]); 1407 page[0] = wpage[0]; 1408 } 1409 1410 npages = 1; 1411 } 1412 *pfn = page_to_pfn(page[0]); 1413 return npages; 1414 } 1415 1416 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1417 { 1418 if (unlikely(!(vma->vm_flags & VM_READ))) 1419 return false; 1420 1421 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1422 return false; 1423 1424 return true; 1425 } 1426 1427 static int hva_to_pfn_remapped(struct vm_area_struct *vma, 1428 unsigned long addr, bool *async, 1429 bool write_fault, kvm_pfn_t *p_pfn) 1430 { 1431 unsigned long pfn; 1432 int r; 1433 1434 r = follow_pfn(vma, addr, &pfn); 1435 if (r) { 1436 /* 1437 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does 1438 * not call the fault handler, so do it here. 1439 */ 1440 bool unlocked = false; 1441 r = fixup_user_fault(current, current->mm, addr, 1442 (write_fault ? FAULT_FLAG_WRITE : 0), 1443 &unlocked); 1444 if (unlocked) 1445 return -EAGAIN; 1446 if (r) 1447 return r; 1448 1449 r = follow_pfn(vma, addr, &pfn); 1450 if (r) 1451 return r; 1452 1453 } 1454 1455 1456 /* 1457 * Get a reference here because callers of *hva_to_pfn* and 1458 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the 1459 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP 1460 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will 1461 * simply do nothing for reserved pfns. 1462 * 1463 * Whoever called remap_pfn_range is also going to call e.g. 1464 * unmap_mapping_range before the underlying pages are freed, 1465 * causing a call to our MMU notifier. 1466 */ 1467 kvm_get_pfn(pfn); 1468 1469 *p_pfn = pfn; 1470 return 0; 1471 } 1472 1473 /* 1474 * Pin guest page in memory and return its pfn. 1475 * @addr: host virtual address which maps memory to the guest 1476 * @atomic: whether this function can sleep 1477 * @async: whether this function need to wait IO complete if the 1478 * host page is not in the memory 1479 * @write_fault: whether we should get a writable host page 1480 * @writable: whether it allows to map a writable host page for !@write_fault 1481 * 1482 * The function will map a writable host page for these two cases: 1483 * 1): @write_fault = true 1484 * 2): @write_fault = false && @writable, @writable will tell the caller 1485 * whether the mapping is writable. 1486 */ 1487 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1488 bool write_fault, bool *writable) 1489 { 1490 struct vm_area_struct *vma; 1491 kvm_pfn_t pfn = 0; 1492 int npages, r; 1493 1494 /* we can do it either atomically or asynchronously, not both */ 1495 BUG_ON(atomic && async); 1496 1497 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1498 return pfn; 1499 1500 if (atomic) 1501 return KVM_PFN_ERR_FAULT; 1502 1503 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1504 if (npages == 1) 1505 return pfn; 1506 1507 down_read(¤t->mm->mmap_sem); 1508 if (npages == -EHWPOISON || 1509 (!async && check_user_page_hwpoison(addr))) { 1510 pfn = KVM_PFN_ERR_HWPOISON; 1511 goto exit; 1512 } 1513 1514 retry: 1515 vma = find_vma_intersection(current->mm, addr, addr + 1); 1516 1517 if (vma == NULL) 1518 pfn = KVM_PFN_ERR_FAULT; 1519 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { 1520 r = hva_to_pfn_remapped(vma, addr, async, write_fault, &pfn); 1521 if (r == -EAGAIN) 1522 goto retry; 1523 if (r < 0) 1524 pfn = KVM_PFN_ERR_FAULT; 1525 } else { 1526 if (async && vma_is_valid(vma, write_fault)) 1527 *async = true; 1528 pfn = KVM_PFN_ERR_FAULT; 1529 } 1530 exit: 1531 up_read(¤t->mm->mmap_sem); 1532 return pfn; 1533 } 1534 1535 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 1536 bool atomic, bool *async, bool write_fault, 1537 bool *writable) 1538 { 1539 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1540 1541 if (addr == KVM_HVA_ERR_RO_BAD) { 1542 if (writable) 1543 *writable = false; 1544 return KVM_PFN_ERR_RO_FAULT; 1545 } 1546 1547 if (kvm_is_error_hva(addr)) { 1548 if (writable) 1549 *writable = false; 1550 return KVM_PFN_NOSLOT; 1551 } 1552 1553 /* Do not map writable pfn in the readonly memslot. */ 1554 if (writable && memslot_is_readonly(slot)) { 1555 *writable = false; 1556 writable = NULL; 1557 } 1558 1559 return hva_to_pfn(addr, atomic, async, write_fault, 1560 writable); 1561 } 1562 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1563 1564 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1565 bool *writable) 1566 { 1567 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1568 write_fault, writable); 1569 } 1570 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1571 1572 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1573 { 1574 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1575 } 1576 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1577 1578 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1579 { 1580 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1581 } 1582 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1583 1584 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1585 { 1586 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1587 } 1588 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1589 1590 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1591 { 1592 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1593 } 1594 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1595 1596 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1597 { 1598 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1599 } 1600 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1601 1602 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1603 { 1604 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1605 } 1606 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1607 1608 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1609 struct page **pages, int nr_pages) 1610 { 1611 unsigned long addr; 1612 gfn_t entry = 0; 1613 1614 addr = gfn_to_hva_many(slot, gfn, &entry); 1615 if (kvm_is_error_hva(addr)) 1616 return -1; 1617 1618 if (entry < nr_pages) 1619 return 0; 1620 1621 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1622 } 1623 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1624 1625 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 1626 { 1627 if (is_error_noslot_pfn(pfn)) 1628 return KVM_ERR_PTR_BAD_PAGE; 1629 1630 if (kvm_is_reserved_pfn(pfn)) { 1631 WARN_ON(1); 1632 return KVM_ERR_PTR_BAD_PAGE; 1633 } 1634 1635 return pfn_to_page(pfn); 1636 } 1637 1638 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1639 { 1640 kvm_pfn_t pfn; 1641 1642 pfn = gfn_to_pfn(kvm, gfn); 1643 1644 return kvm_pfn_to_page(pfn); 1645 } 1646 EXPORT_SYMBOL_GPL(gfn_to_page); 1647 1648 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1649 { 1650 kvm_pfn_t pfn; 1651 1652 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1653 1654 return kvm_pfn_to_page(pfn); 1655 } 1656 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1657 1658 void kvm_release_page_clean(struct page *page) 1659 { 1660 WARN_ON(is_error_page(page)); 1661 1662 kvm_release_pfn_clean(page_to_pfn(page)); 1663 } 1664 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1665 1666 void kvm_release_pfn_clean(kvm_pfn_t pfn) 1667 { 1668 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1669 put_page(pfn_to_page(pfn)); 1670 } 1671 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1672 1673 void kvm_release_page_dirty(struct page *page) 1674 { 1675 WARN_ON(is_error_page(page)); 1676 1677 kvm_release_pfn_dirty(page_to_pfn(page)); 1678 } 1679 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1680 1681 void kvm_release_pfn_dirty(kvm_pfn_t pfn) 1682 { 1683 kvm_set_pfn_dirty(pfn); 1684 kvm_release_pfn_clean(pfn); 1685 } 1686 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 1687 1688 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 1689 { 1690 if (!kvm_is_reserved_pfn(pfn)) { 1691 struct page *page = pfn_to_page(pfn); 1692 1693 if (!PageReserved(page)) 1694 SetPageDirty(page); 1695 } 1696 } 1697 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1698 1699 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 1700 { 1701 if (!kvm_is_reserved_pfn(pfn)) 1702 mark_page_accessed(pfn_to_page(pfn)); 1703 } 1704 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1705 1706 void kvm_get_pfn(kvm_pfn_t pfn) 1707 { 1708 if (!kvm_is_reserved_pfn(pfn)) 1709 get_page(pfn_to_page(pfn)); 1710 } 1711 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1712 1713 static int next_segment(unsigned long len, int offset) 1714 { 1715 if (len > PAGE_SIZE - offset) 1716 return PAGE_SIZE - offset; 1717 else 1718 return len; 1719 } 1720 1721 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1722 void *data, int offset, int len) 1723 { 1724 int r; 1725 unsigned long addr; 1726 1727 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1728 if (kvm_is_error_hva(addr)) 1729 return -EFAULT; 1730 r = __copy_from_user(data, (void __user *)addr + offset, len); 1731 if (r) 1732 return -EFAULT; 1733 return 0; 1734 } 1735 1736 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1737 int len) 1738 { 1739 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1740 1741 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1742 } 1743 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1744 1745 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1746 int offset, int len) 1747 { 1748 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1749 1750 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1751 } 1752 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1753 1754 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1755 { 1756 gfn_t gfn = gpa >> PAGE_SHIFT; 1757 int seg; 1758 int offset = offset_in_page(gpa); 1759 int ret; 1760 1761 while ((seg = next_segment(len, offset)) != 0) { 1762 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1763 if (ret < 0) 1764 return ret; 1765 offset = 0; 1766 len -= seg; 1767 data += seg; 1768 ++gfn; 1769 } 1770 return 0; 1771 } 1772 EXPORT_SYMBOL_GPL(kvm_read_guest); 1773 1774 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1775 { 1776 gfn_t gfn = gpa >> PAGE_SHIFT; 1777 int seg; 1778 int offset = offset_in_page(gpa); 1779 int ret; 1780 1781 while ((seg = next_segment(len, offset)) != 0) { 1782 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1783 if (ret < 0) 1784 return ret; 1785 offset = 0; 1786 len -= seg; 1787 data += seg; 1788 ++gfn; 1789 } 1790 return 0; 1791 } 1792 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1793 1794 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1795 void *data, int offset, unsigned long len) 1796 { 1797 int r; 1798 unsigned long addr; 1799 1800 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1801 if (kvm_is_error_hva(addr)) 1802 return -EFAULT; 1803 pagefault_disable(); 1804 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1805 pagefault_enable(); 1806 if (r) 1807 return -EFAULT; 1808 return 0; 1809 } 1810 1811 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1812 unsigned long len) 1813 { 1814 gfn_t gfn = gpa >> PAGE_SHIFT; 1815 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1816 int offset = offset_in_page(gpa); 1817 1818 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1819 } 1820 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1821 1822 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1823 void *data, unsigned long len) 1824 { 1825 gfn_t gfn = gpa >> PAGE_SHIFT; 1826 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1827 int offset = offset_in_page(gpa); 1828 1829 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1830 } 1831 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1832 1833 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1834 const void *data, int offset, int len) 1835 { 1836 int r; 1837 unsigned long addr; 1838 1839 addr = gfn_to_hva_memslot(memslot, gfn); 1840 if (kvm_is_error_hva(addr)) 1841 return -EFAULT; 1842 r = __copy_to_user((void __user *)addr + offset, data, len); 1843 if (r) 1844 return -EFAULT; 1845 mark_page_dirty_in_slot(memslot, gfn); 1846 return 0; 1847 } 1848 1849 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1850 const void *data, int offset, int len) 1851 { 1852 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1853 1854 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1855 } 1856 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1857 1858 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1859 const void *data, int offset, int len) 1860 { 1861 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1862 1863 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1864 } 1865 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1866 1867 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1868 unsigned long len) 1869 { 1870 gfn_t gfn = gpa >> PAGE_SHIFT; 1871 int seg; 1872 int offset = offset_in_page(gpa); 1873 int ret; 1874 1875 while ((seg = next_segment(len, offset)) != 0) { 1876 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1877 if (ret < 0) 1878 return ret; 1879 offset = 0; 1880 len -= seg; 1881 data += seg; 1882 ++gfn; 1883 } 1884 return 0; 1885 } 1886 EXPORT_SYMBOL_GPL(kvm_write_guest); 1887 1888 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1889 unsigned long len) 1890 { 1891 gfn_t gfn = gpa >> PAGE_SHIFT; 1892 int seg; 1893 int offset = offset_in_page(gpa); 1894 int ret; 1895 1896 while ((seg = next_segment(len, offset)) != 0) { 1897 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1898 if (ret < 0) 1899 return ret; 1900 offset = 0; 1901 len -= seg; 1902 data += seg; 1903 ++gfn; 1904 } 1905 return 0; 1906 } 1907 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1908 1909 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots, 1910 struct gfn_to_hva_cache *ghc, 1911 gpa_t gpa, unsigned long len) 1912 { 1913 int offset = offset_in_page(gpa); 1914 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1915 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1916 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1917 gfn_t nr_pages_avail; 1918 1919 ghc->gpa = gpa; 1920 ghc->generation = slots->generation; 1921 ghc->len = len; 1922 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 1923 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1924 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1925 ghc->hva += offset; 1926 } else { 1927 /* 1928 * If the requested region crosses two memslots, we still 1929 * verify that the entire region is valid here. 1930 */ 1931 while (start_gfn <= end_gfn) { 1932 nr_pages_avail = 0; 1933 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 1934 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1935 &nr_pages_avail); 1936 if (kvm_is_error_hva(ghc->hva)) 1937 return -EFAULT; 1938 start_gfn += nr_pages_avail; 1939 } 1940 /* Use the slow path for cross page reads and writes. */ 1941 ghc->memslot = NULL; 1942 } 1943 return 0; 1944 } 1945 1946 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1947 gpa_t gpa, unsigned long len) 1948 { 1949 struct kvm_memslots *slots = kvm_memslots(kvm); 1950 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len); 1951 } 1952 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1953 1954 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1955 void *data, int offset, unsigned long len) 1956 { 1957 struct kvm_memslots *slots = kvm_memslots(kvm); 1958 int r; 1959 gpa_t gpa = ghc->gpa + offset; 1960 1961 BUG_ON(len + offset > ghc->len); 1962 1963 if (slots->generation != ghc->generation) 1964 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len); 1965 1966 if (unlikely(!ghc->memslot)) 1967 return kvm_write_guest(kvm, gpa, data, len); 1968 1969 if (kvm_is_error_hva(ghc->hva)) 1970 return -EFAULT; 1971 1972 r = __copy_to_user((void __user *)ghc->hva + offset, data, len); 1973 if (r) 1974 return -EFAULT; 1975 mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT); 1976 1977 return 0; 1978 } 1979 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached); 1980 1981 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1982 void *data, unsigned long len) 1983 { 1984 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); 1985 } 1986 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1987 1988 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1989 void *data, unsigned long len) 1990 { 1991 struct kvm_memslots *slots = kvm_memslots(kvm); 1992 int r; 1993 1994 BUG_ON(len > ghc->len); 1995 1996 if (slots->generation != ghc->generation) 1997 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len); 1998 1999 if (unlikely(!ghc->memslot)) 2000 return kvm_read_guest(kvm, ghc->gpa, data, len); 2001 2002 if (kvm_is_error_hva(ghc->hva)) 2003 return -EFAULT; 2004 2005 r = __copy_from_user(data, (void __user *)ghc->hva, len); 2006 if (r) 2007 return -EFAULT; 2008 2009 return 0; 2010 } 2011 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 2012 2013 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 2014 { 2015 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 2016 2017 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 2018 } 2019 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 2020 2021 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 2022 { 2023 gfn_t gfn = gpa >> PAGE_SHIFT; 2024 int seg; 2025 int offset = offset_in_page(gpa); 2026 int ret; 2027 2028 while ((seg = next_segment(len, offset)) != 0) { 2029 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 2030 if (ret < 0) 2031 return ret; 2032 offset = 0; 2033 len -= seg; 2034 ++gfn; 2035 } 2036 return 0; 2037 } 2038 EXPORT_SYMBOL_GPL(kvm_clear_guest); 2039 2040 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 2041 gfn_t gfn) 2042 { 2043 if (memslot && memslot->dirty_bitmap) { 2044 unsigned long rel_gfn = gfn - memslot->base_gfn; 2045 2046 set_bit_le(rel_gfn, memslot->dirty_bitmap); 2047 } 2048 } 2049 2050 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 2051 { 2052 struct kvm_memory_slot *memslot; 2053 2054 memslot = gfn_to_memslot(kvm, gfn); 2055 mark_page_dirty_in_slot(memslot, gfn); 2056 } 2057 EXPORT_SYMBOL_GPL(mark_page_dirty); 2058 2059 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 2060 { 2061 struct kvm_memory_slot *memslot; 2062 2063 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2064 mark_page_dirty_in_slot(memslot, gfn); 2065 } 2066 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 2067 2068 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 2069 { 2070 unsigned int old, val, grow; 2071 2072 old = val = vcpu->halt_poll_ns; 2073 grow = READ_ONCE(halt_poll_ns_grow); 2074 /* 10us base */ 2075 if (val == 0 && grow) 2076 val = 10000; 2077 else 2078 val *= grow; 2079 2080 if (val > halt_poll_ns) 2081 val = halt_poll_ns; 2082 2083 vcpu->halt_poll_ns = val; 2084 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 2085 } 2086 2087 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 2088 { 2089 unsigned int old, val, shrink; 2090 2091 old = val = vcpu->halt_poll_ns; 2092 shrink = READ_ONCE(halt_poll_ns_shrink); 2093 if (shrink == 0) 2094 val = 0; 2095 else 2096 val /= shrink; 2097 2098 vcpu->halt_poll_ns = val; 2099 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 2100 } 2101 2102 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 2103 { 2104 if (kvm_arch_vcpu_runnable(vcpu)) { 2105 kvm_make_request(KVM_REQ_UNHALT, vcpu); 2106 return -EINTR; 2107 } 2108 if (kvm_cpu_has_pending_timer(vcpu)) 2109 return -EINTR; 2110 if (signal_pending(current)) 2111 return -EINTR; 2112 2113 return 0; 2114 } 2115 2116 /* 2117 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2118 */ 2119 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2120 { 2121 ktime_t start, cur; 2122 DECLARE_SWAITQUEUE(wait); 2123 bool waited = false; 2124 u64 block_ns; 2125 2126 start = cur = ktime_get(); 2127 if (vcpu->halt_poll_ns) { 2128 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2129 2130 ++vcpu->stat.halt_attempted_poll; 2131 do { 2132 /* 2133 * This sets KVM_REQ_UNHALT if an interrupt 2134 * arrives. 2135 */ 2136 if (kvm_vcpu_check_block(vcpu) < 0) { 2137 ++vcpu->stat.halt_successful_poll; 2138 if (!vcpu_valid_wakeup(vcpu)) 2139 ++vcpu->stat.halt_poll_invalid; 2140 goto out; 2141 } 2142 cur = ktime_get(); 2143 } while (single_task_running() && ktime_before(cur, stop)); 2144 } 2145 2146 kvm_arch_vcpu_blocking(vcpu); 2147 2148 for (;;) { 2149 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2150 2151 if (kvm_vcpu_check_block(vcpu) < 0) 2152 break; 2153 2154 waited = true; 2155 schedule(); 2156 } 2157 2158 finish_swait(&vcpu->wq, &wait); 2159 cur = ktime_get(); 2160 2161 kvm_arch_vcpu_unblocking(vcpu); 2162 out: 2163 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2164 2165 if (!vcpu_valid_wakeup(vcpu)) 2166 shrink_halt_poll_ns(vcpu); 2167 else if (halt_poll_ns) { 2168 if (block_ns <= vcpu->halt_poll_ns) 2169 ; 2170 /* we had a long block, shrink polling */ 2171 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2172 shrink_halt_poll_ns(vcpu); 2173 /* we had a short halt and our poll time is too small */ 2174 else if (vcpu->halt_poll_ns < halt_poll_ns && 2175 block_ns < halt_poll_ns) 2176 grow_halt_poll_ns(vcpu); 2177 } else 2178 vcpu->halt_poll_ns = 0; 2179 2180 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu)); 2181 kvm_arch_vcpu_block_finish(vcpu); 2182 } 2183 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2184 2185 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 2186 { 2187 struct swait_queue_head *wqp; 2188 2189 wqp = kvm_arch_vcpu_wq(vcpu); 2190 if (swq_has_sleeper(wqp)) { 2191 swake_up(wqp); 2192 ++vcpu->stat.halt_wakeup; 2193 return true; 2194 } 2195 2196 return false; 2197 } 2198 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 2199 2200 #ifndef CONFIG_S390 2201 /* 2202 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2203 */ 2204 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2205 { 2206 int me; 2207 int cpu = vcpu->cpu; 2208 2209 if (kvm_vcpu_wake_up(vcpu)) 2210 return; 2211 2212 me = get_cpu(); 2213 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2214 if (kvm_arch_vcpu_should_kick(vcpu)) 2215 smp_send_reschedule(cpu); 2216 put_cpu(); 2217 } 2218 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2219 #endif /* !CONFIG_S390 */ 2220 2221 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2222 { 2223 struct pid *pid; 2224 struct task_struct *task = NULL; 2225 int ret = 0; 2226 2227 rcu_read_lock(); 2228 pid = rcu_dereference(target->pid); 2229 if (pid) 2230 task = get_pid_task(pid, PIDTYPE_PID); 2231 rcu_read_unlock(); 2232 if (!task) 2233 return ret; 2234 ret = yield_to(task, 1); 2235 put_task_struct(task); 2236 2237 return ret; 2238 } 2239 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2240 2241 /* 2242 * Helper that checks whether a VCPU is eligible for directed yield. 2243 * Most eligible candidate to yield is decided by following heuristics: 2244 * 2245 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2246 * (preempted lock holder), indicated by @in_spin_loop. 2247 * Set at the beiginning and cleared at the end of interception/PLE handler. 2248 * 2249 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2250 * chance last time (mostly it has become eligible now since we have probably 2251 * yielded to lockholder in last iteration. This is done by toggling 2252 * @dy_eligible each time a VCPU checked for eligibility.) 2253 * 2254 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2255 * to preempted lock-holder could result in wrong VCPU selection and CPU 2256 * burning. Giving priority for a potential lock-holder increases lock 2257 * progress. 2258 * 2259 * Since algorithm is based on heuristics, accessing another VCPU data without 2260 * locking does not harm. It may result in trying to yield to same VCPU, fail 2261 * and continue with next VCPU and so on. 2262 */ 2263 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2264 { 2265 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2266 bool eligible; 2267 2268 eligible = !vcpu->spin_loop.in_spin_loop || 2269 vcpu->spin_loop.dy_eligible; 2270 2271 if (vcpu->spin_loop.in_spin_loop) 2272 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2273 2274 return eligible; 2275 #else 2276 return true; 2277 #endif 2278 } 2279 2280 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) 2281 { 2282 struct kvm *kvm = me->kvm; 2283 struct kvm_vcpu *vcpu; 2284 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2285 int yielded = 0; 2286 int try = 3; 2287 int pass; 2288 int i; 2289 2290 kvm_vcpu_set_in_spin_loop(me, true); 2291 /* 2292 * We boost the priority of a VCPU that is runnable but not 2293 * currently running, because it got preempted by something 2294 * else and called schedule in __vcpu_run. Hopefully that 2295 * VCPU is holding the lock that we need and will release it. 2296 * We approximate round-robin by starting at the last boosted VCPU. 2297 */ 2298 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2299 kvm_for_each_vcpu(i, vcpu, kvm) { 2300 if (!pass && i <= last_boosted_vcpu) { 2301 i = last_boosted_vcpu; 2302 continue; 2303 } else if (pass && i > last_boosted_vcpu) 2304 break; 2305 if (!READ_ONCE(vcpu->preempted)) 2306 continue; 2307 if (vcpu == me) 2308 continue; 2309 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2310 continue; 2311 if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu)) 2312 continue; 2313 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2314 continue; 2315 2316 yielded = kvm_vcpu_yield_to(vcpu); 2317 if (yielded > 0) { 2318 kvm->last_boosted_vcpu = i; 2319 break; 2320 } else if (yielded < 0) { 2321 try--; 2322 if (!try) 2323 break; 2324 } 2325 } 2326 } 2327 kvm_vcpu_set_in_spin_loop(me, false); 2328 2329 /* Ensure vcpu is not eligible during next spinloop */ 2330 kvm_vcpu_set_dy_eligible(me, false); 2331 } 2332 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2333 2334 static int kvm_vcpu_fault(struct vm_fault *vmf) 2335 { 2336 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data; 2337 struct page *page; 2338 2339 if (vmf->pgoff == 0) 2340 page = virt_to_page(vcpu->run); 2341 #ifdef CONFIG_X86 2342 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2343 page = virt_to_page(vcpu->arch.pio_data); 2344 #endif 2345 #ifdef CONFIG_KVM_MMIO 2346 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2347 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2348 #endif 2349 else 2350 return kvm_arch_vcpu_fault(vcpu, vmf); 2351 get_page(page); 2352 vmf->page = page; 2353 return 0; 2354 } 2355 2356 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2357 .fault = kvm_vcpu_fault, 2358 }; 2359 2360 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2361 { 2362 vma->vm_ops = &kvm_vcpu_vm_ops; 2363 return 0; 2364 } 2365 2366 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2367 { 2368 struct kvm_vcpu *vcpu = filp->private_data; 2369 2370 debugfs_remove_recursive(vcpu->debugfs_dentry); 2371 kvm_put_kvm(vcpu->kvm); 2372 return 0; 2373 } 2374 2375 static struct file_operations kvm_vcpu_fops = { 2376 .release = kvm_vcpu_release, 2377 .unlocked_ioctl = kvm_vcpu_ioctl, 2378 #ifdef CONFIG_KVM_COMPAT 2379 .compat_ioctl = kvm_vcpu_compat_ioctl, 2380 #endif 2381 .mmap = kvm_vcpu_mmap, 2382 .llseek = noop_llseek, 2383 }; 2384 2385 /* 2386 * Allocates an inode for the vcpu. 2387 */ 2388 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2389 { 2390 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2391 } 2392 2393 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 2394 { 2395 char dir_name[ITOA_MAX_LEN * 2]; 2396 int ret; 2397 2398 if (!kvm_arch_has_vcpu_debugfs()) 2399 return 0; 2400 2401 if (!debugfs_initialized()) 2402 return 0; 2403 2404 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); 2405 vcpu->debugfs_dentry = debugfs_create_dir(dir_name, 2406 vcpu->kvm->debugfs_dentry); 2407 if (!vcpu->debugfs_dentry) 2408 return -ENOMEM; 2409 2410 ret = kvm_arch_create_vcpu_debugfs(vcpu); 2411 if (ret < 0) { 2412 debugfs_remove_recursive(vcpu->debugfs_dentry); 2413 return ret; 2414 } 2415 2416 return 0; 2417 } 2418 2419 /* 2420 * Creates some virtual cpus. Good luck creating more than one. 2421 */ 2422 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2423 { 2424 int r; 2425 struct kvm_vcpu *vcpu; 2426 2427 if (id >= KVM_MAX_VCPU_ID) 2428 return -EINVAL; 2429 2430 mutex_lock(&kvm->lock); 2431 if (kvm->created_vcpus == KVM_MAX_VCPUS) { 2432 mutex_unlock(&kvm->lock); 2433 return -EINVAL; 2434 } 2435 2436 kvm->created_vcpus++; 2437 mutex_unlock(&kvm->lock); 2438 2439 vcpu = kvm_arch_vcpu_create(kvm, id); 2440 if (IS_ERR(vcpu)) { 2441 r = PTR_ERR(vcpu); 2442 goto vcpu_decrement; 2443 } 2444 2445 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2446 2447 r = kvm_arch_vcpu_setup(vcpu); 2448 if (r) 2449 goto vcpu_destroy; 2450 2451 r = kvm_create_vcpu_debugfs(vcpu); 2452 if (r) 2453 goto vcpu_destroy; 2454 2455 mutex_lock(&kvm->lock); 2456 if (kvm_get_vcpu_by_id(kvm, id)) { 2457 r = -EEXIST; 2458 goto unlock_vcpu_destroy; 2459 } 2460 2461 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2462 2463 /* Now it's all set up, let userspace reach it */ 2464 kvm_get_kvm(kvm); 2465 r = create_vcpu_fd(vcpu); 2466 if (r < 0) { 2467 kvm_put_kvm(kvm); 2468 goto unlock_vcpu_destroy; 2469 } 2470 2471 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2472 2473 /* 2474 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2475 * before kvm->online_vcpu's incremented value. 2476 */ 2477 smp_wmb(); 2478 atomic_inc(&kvm->online_vcpus); 2479 2480 mutex_unlock(&kvm->lock); 2481 kvm_arch_vcpu_postcreate(vcpu); 2482 return r; 2483 2484 unlock_vcpu_destroy: 2485 mutex_unlock(&kvm->lock); 2486 debugfs_remove_recursive(vcpu->debugfs_dentry); 2487 vcpu_destroy: 2488 kvm_arch_vcpu_destroy(vcpu); 2489 vcpu_decrement: 2490 mutex_lock(&kvm->lock); 2491 kvm->created_vcpus--; 2492 mutex_unlock(&kvm->lock); 2493 return r; 2494 } 2495 2496 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2497 { 2498 if (sigset) { 2499 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2500 vcpu->sigset_active = 1; 2501 vcpu->sigset = *sigset; 2502 } else 2503 vcpu->sigset_active = 0; 2504 return 0; 2505 } 2506 2507 static long kvm_vcpu_ioctl(struct file *filp, 2508 unsigned int ioctl, unsigned long arg) 2509 { 2510 struct kvm_vcpu *vcpu = filp->private_data; 2511 void __user *argp = (void __user *)arg; 2512 int r; 2513 struct kvm_fpu *fpu = NULL; 2514 struct kvm_sregs *kvm_sregs = NULL; 2515 2516 if (vcpu->kvm->mm != current->mm) 2517 return -EIO; 2518 2519 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2520 return -EINVAL; 2521 2522 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2523 /* 2524 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2525 * so vcpu_load() would break it. 2526 */ 2527 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2528 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2529 #endif 2530 2531 2532 r = vcpu_load(vcpu); 2533 if (r) 2534 return r; 2535 switch (ioctl) { 2536 case KVM_RUN: { 2537 struct pid *oldpid; 2538 r = -EINVAL; 2539 if (arg) 2540 goto out; 2541 oldpid = rcu_access_pointer(vcpu->pid); 2542 if (unlikely(oldpid != current->pids[PIDTYPE_PID].pid)) { 2543 /* The thread running this VCPU changed. */ 2544 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2545 2546 rcu_assign_pointer(vcpu->pid, newpid); 2547 if (oldpid) 2548 synchronize_rcu(); 2549 put_pid(oldpid); 2550 } 2551 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2552 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2553 break; 2554 } 2555 case KVM_GET_REGS: { 2556 struct kvm_regs *kvm_regs; 2557 2558 r = -ENOMEM; 2559 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2560 if (!kvm_regs) 2561 goto out; 2562 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2563 if (r) 2564 goto out_free1; 2565 r = -EFAULT; 2566 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2567 goto out_free1; 2568 r = 0; 2569 out_free1: 2570 kfree(kvm_regs); 2571 break; 2572 } 2573 case KVM_SET_REGS: { 2574 struct kvm_regs *kvm_regs; 2575 2576 r = -ENOMEM; 2577 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2578 if (IS_ERR(kvm_regs)) { 2579 r = PTR_ERR(kvm_regs); 2580 goto out; 2581 } 2582 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2583 kfree(kvm_regs); 2584 break; 2585 } 2586 case KVM_GET_SREGS: { 2587 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2588 r = -ENOMEM; 2589 if (!kvm_sregs) 2590 goto out; 2591 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2592 if (r) 2593 goto out; 2594 r = -EFAULT; 2595 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2596 goto out; 2597 r = 0; 2598 break; 2599 } 2600 case KVM_SET_SREGS: { 2601 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2602 if (IS_ERR(kvm_sregs)) { 2603 r = PTR_ERR(kvm_sregs); 2604 kvm_sregs = NULL; 2605 goto out; 2606 } 2607 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2608 break; 2609 } 2610 case KVM_GET_MP_STATE: { 2611 struct kvm_mp_state mp_state; 2612 2613 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2614 if (r) 2615 goto out; 2616 r = -EFAULT; 2617 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2618 goto out; 2619 r = 0; 2620 break; 2621 } 2622 case KVM_SET_MP_STATE: { 2623 struct kvm_mp_state mp_state; 2624 2625 r = -EFAULT; 2626 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2627 goto out; 2628 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2629 break; 2630 } 2631 case KVM_TRANSLATE: { 2632 struct kvm_translation tr; 2633 2634 r = -EFAULT; 2635 if (copy_from_user(&tr, argp, sizeof(tr))) 2636 goto out; 2637 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2638 if (r) 2639 goto out; 2640 r = -EFAULT; 2641 if (copy_to_user(argp, &tr, sizeof(tr))) 2642 goto out; 2643 r = 0; 2644 break; 2645 } 2646 case KVM_SET_GUEST_DEBUG: { 2647 struct kvm_guest_debug dbg; 2648 2649 r = -EFAULT; 2650 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2651 goto out; 2652 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2653 break; 2654 } 2655 case KVM_SET_SIGNAL_MASK: { 2656 struct kvm_signal_mask __user *sigmask_arg = argp; 2657 struct kvm_signal_mask kvm_sigmask; 2658 sigset_t sigset, *p; 2659 2660 p = NULL; 2661 if (argp) { 2662 r = -EFAULT; 2663 if (copy_from_user(&kvm_sigmask, argp, 2664 sizeof(kvm_sigmask))) 2665 goto out; 2666 r = -EINVAL; 2667 if (kvm_sigmask.len != sizeof(sigset)) 2668 goto out; 2669 r = -EFAULT; 2670 if (copy_from_user(&sigset, sigmask_arg->sigset, 2671 sizeof(sigset))) 2672 goto out; 2673 p = &sigset; 2674 } 2675 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2676 break; 2677 } 2678 case KVM_GET_FPU: { 2679 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2680 r = -ENOMEM; 2681 if (!fpu) 2682 goto out; 2683 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2684 if (r) 2685 goto out; 2686 r = -EFAULT; 2687 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2688 goto out; 2689 r = 0; 2690 break; 2691 } 2692 case KVM_SET_FPU: { 2693 fpu = memdup_user(argp, sizeof(*fpu)); 2694 if (IS_ERR(fpu)) { 2695 r = PTR_ERR(fpu); 2696 fpu = NULL; 2697 goto out; 2698 } 2699 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2700 break; 2701 } 2702 default: 2703 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2704 } 2705 out: 2706 vcpu_put(vcpu); 2707 kfree(fpu); 2708 kfree(kvm_sregs); 2709 return r; 2710 } 2711 2712 #ifdef CONFIG_KVM_COMPAT 2713 static long kvm_vcpu_compat_ioctl(struct file *filp, 2714 unsigned int ioctl, unsigned long arg) 2715 { 2716 struct kvm_vcpu *vcpu = filp->private_data; 2717 void __user *argp = compat_ptr(arg); 2718 int r; 2719 2720 if (vcpu->kvm->mm != current->mm) 2721 return -EIO; 2722 2723 switch (ioctl) { 2724 case KVM_SET_SIGNAL_MASK: { 2725 struct kvm_signal_mask __user *sigmask_arg = argp; 2726 struct kvm_signal_mask kvm_sigmask; 2727 compat_sigset_t csigset; 2728 sigset_t sigset; 2729 2730 if (argp) { 2731 r = -EFAULT; 2732 if (copy_from_user(&kvm_sigmask, argp, 2733 sizeof(kvm_sigmask))) 2734 goto out; 2735 r = -EINVAL; 2736 if (kvm_sigmask.len != sizeof(csigset)) 2737 goto out; 2738 r = -EFAULT; 2739 if (copy_from_user(&csigset, sigmask_arg->sigset, 2740 sizeof(csigset))) 2741 goto out; 2742 sigset_from_compat(&sigset, &csigset); 2743 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2744 } else 2745 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2746 break; 2747 } 2748 default: 2749 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2750 } 2751 2752 out: 2753 return r; 2754 } 2755 #endif 2756 2757 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2758 int (*accessor)(struct kvm_device *dev, 2759 struct kvm_device_attr *attr), 2760 unsigned long arg) 2761 { 2762 struct kvm_device_attr attr; 2763 2764 if (!accessor) 2765 return -EPERM; 2766 2767 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2768 return -EFAULT; 2769 2770 return accessor(dev, &attr); 2771 } 2772 2773 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2774 unsigned long arg) 2775 { 2776 struct kvm_device *dev = filp->private_data; 2777 2778 switch (ioctl) { 2779 case KVM_SET_DEVICE_ATTR: 2780 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2781 case KVM_GET_DEVICE_ATTR: 2782 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2783 case KVM_HAS_DEVICE_ATTR: 2784 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2785 default: 2786 if (dev->ops->ioctl) 2787 return dev->ops->ioctl(dev, ioctl, arg); 2788 2789 return -ENOTTY; 2790 } 2791 } 2792 2793 static int kvm_device_release(struct inode *inode, struct file *filp) 2794 { 2795 struct kvm_device *dev = filp->private_data; 2796 struct kvm *kvm = dev->kvm; 2797 2798 kvm_put_kvm(kvm); 2799 return 0; 2800 } 2801 2802 static const struct file_operations kvm_device_fops = { 2803 .unlocked_ioctl = kvm_device_ioctl, 2804 #ifdef CONFIG_KVM_COMPAT 2805 .compat_ioctl = kvm_device_ioctl, 2806 #endif 2807 .release = kvm_device_release, 2808 }; 2809 2810 struct kvm_device *kvm_device_from_filp(struct file *filp) 2811 { 2812 if (filp->f_op != &kvm_device_fops) 2813 return NULL; 2814 2815 return filp->private_data; 2816 } 2817 2818 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2819 #ifdef CONFIG_KVM_MPIC 2820 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2821 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2822 #endif 2823 }; 2824 2825 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2826 { 2827 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2828 return -ENOSPC; 2829 2830 if (kvm_device_ops_table[type] != NULL) 2831 return -EEXIST; 2832 2833 kvm_device_ops_table[type] = ops; 2834 return 0; 2835 } 2836 2837 void kvm_unregister_device_ops(u32 type) 2838 { 2839 if (kvm_device_ops_table[type] != NULL) 2840 kvm_device_ops_table[type] = NULL; 2841 } 2842 2843 static int kvm_ioctl_create_device(struct kvm *kvm, 2844 struct kvm_create_device *cd) 2845 { 2846 struct kvm_device_ops *ops = NULL; 2847 struct kvm_device *dev; 2848 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2849 int ret; 2850 2851 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2852 return -ENODEV; 2853 2854 ops = kvm_device_ops_table[cd->type]; 2855 if (ops == NULL) 2856 return -ENODEV; 2857 2858 if (test) 2859 return 0; 2860 2861 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2862 if (!dev) 2863 return -ENOMEM; 2864 2865 dev->ops = ops; 2866 dev->kvm = kvm; 2867 2868 mutex_lock(&kvm->lock); 2869 ret = ops->create(dev, cd->type); 2870 if (ret < 0) { 2871 mutex_unlock(&kvm->lock); 2872 kfree(dev); 2873 return ret; 2874 } 2875 list_add(&dev->vm_node, &kvm->devices); 2876 mutex_unlock(&kvm->lock); 2877 2878 if (ops->init) 2879 ops->init(dev); 2880 2881 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2882 if (ret < 0) { 2883 mutex_lock(&kvm->lock); 2884 list_del(&dev->vm_node); 2885 mutex_unlock(&kvm->lock); 2886 ops->destroy(dev); 2887 return ret; 2888 } 2889 2890 kvm_get_kvm(kvm); 2891 cd->fd = ret; 2892 return 0; 2893 } 2894 2895 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2896 { 2897 switch (arg) { 2898 case KVM_CAP_USER_MEMORY: 2899 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2900 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2901 case KVM_CAP_INTERNAL_ERROR_DATA: 2902 #ifdef CONFIG_HAVE_KVM_MSI 2903 case KVM_CAP_SIGNAL_MSI: 2904 #endif 2905 #ifdef CONFIG_HAVE_KVM_IRQFD 2906 case KVM_CAP_IRQFD: 2907 case KVM_CAP_IRQFD_RESAMPLE: 2908 #endif 2909 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 2910 case KVM_CAP_CHECK_EXTENSION_VM: 2911 return 1; 2912 #ifdef CONFIG_KVM_MMIO 2913 case KVM_CAP_COALESCED_MMIO: 2914 return KVM_COALESCED_MMIO_PAGE_OFFSET; 2915 #endif 2916 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2917 case KVM_CAP_IRQ_ROUTING: 2918 return KVM_MAX_IRQ_ROUTES; 2919 #endif 2920 #if KVM_ADDRESS_SPACE_NUM > 1 2921 case KVM_CAP_MULTI_ADDRESS_SPACE: 2922 return KVM_ADDRESS_SPACE_NUM; 2923 #endif 2924 case KVM_CAP_MAX_VCPU_ID: 2925 return KVM_MAX_VCPU_ID; 2926 default: 2927 break; 2928 } 2929 return kvm_vm_ioctl_check_extension(kvm, arg); 2930 } 2931 2932 static long kvm_vm_ioctl(struct file *filp, 2933 unsigned int ioctl, unsigned long arg) 2934 { 2935 struct kvm *kvm = filp->private_data; 2936 void __user *argp = (void __user *)arg; 2937 int r; 2938 2939 if (kvm->mm != current->mm) 2940 return -EIO; 2941 switch (ioctl) { 2942 case KVM_CREATE_VCPU: 2943 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2944 break; 2945 case KVM_SET_USER_MEMORY_REGION: { 2946 struct kvm_userspace_memory_region kvm_userspace_mem; 2947 2948 r = -EFAULT; 2949 if (copy_from_user(&kvm_userspace_mem, argp, 2950 sizeof(kvm_userspace_mem))) 2951 goto out; 2952 2953 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2954 break; 2955 } 2956 case KVM_GET_DIRTY_LOG: { 2957 struct kvm_dirty_log log; 2958 2959 r = -EFAULT; 2960 if (copy_from_user(&log, argp, sizeof(log))) 2961 goto out; 2962 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2963 break; 2964 } 2965 #ifdef CONFIG_KVM_MMIO 2966 case KVM_REGISTER_COALESCED_MMIO: { 2967 struct kvm_coalesced_mmio_zone zone; 2968 2969 r = -EFAULT; 2970 if (copy_from_user(&zone, argp, sizeof(zone))) 2971 goto out; 2972 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2973 break; 2974 } 2975 case KVM_UNREGISTER_COALESCED_MMIO: { 2976 struct kvm_coalesced_mmio_zone zone; 2977 2978 r = -EFAULT; 2979 if (copy_from_user(&zone, argp, sizeof(zone))) 2980 goto out; 2981 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2982 break; 2983 } 2984 #endif 2985 case KVM_IRQFD: { 2986 struct kvm_irqfd data; 2987 2988 r = -EFAULT; 2989 if (copy_from_user(&data, argp, sizeof(data))) 2990 goto out; 2991 r = kvm_irqfd(kvm, &data); 2992 break; 2993 } 2994 case KVM_IOEVENTFD: { 2995 struct kvm_ioeventfd data; 2996 2997 r = -EFAULT; 2998 if (copy_from_user(&data, argp, sizeof(data))) 2999 goto out; 3000 r = kvm_ioeventfd(kvm, &data); 3001 break; 3002 } 3003 #ifdef CONFIG_HAVE_KVM_MSI 3004 case KVM_SIGNAL_MSI: { 3005 struct kvm_msi msi; 3006 3007 r = -EFAULT; 3008 if (copy_from_user(&msi, argp, sizeof(msi))) 3009 goto out; 3010 r = kvm_send_userspace_msi(kvm, &msi); 3011 break; 3012 } 3013 #endif 3014 #ifdef __KVM_HAVE_IRQ_LINE 3015 case KVM_IRQ_LINE_STATUS: 3016 case KVM_IRQ_LINE: { 3017 struct kvm_irq_level irq_event; 3018 3019 r = -EFAULT; 3020 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 3021 goto out; 3022 3023 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 3024 ioctl == KVM_IRQ_LINE_STATUS); 3025 if (r) 3026 goto out; 3027 3028 r = -EFAULT; 3029 if (ioctl == KVM_IRQ_LINE_STATUS) { 3030 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 3031 goto out; 3032 } 3033 3034 r = 0; 3035 break; 3036 } 3037 #endif 3038 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 3039 case KVM_SET_GSI_ROUTING: { 3040 struct kvm_irq_routing routing; 3041 struct kvm_irq_routing __user *urouting; 3042 struct kvm_irq_routing_entry *entries = NULL; 3043 3044 r = -EFAULT; 3045 if (copy_from_user(&routing, argp, sizeof(routing))) 3046 goto out; 3047 r = -EINVAL; 3048 if (!kvm_arch_can_set_irq_routing(kvm)) 3049 goto out; 3050 if (routing.nr > KVM_MAX_IRQ_ROUTES) 3051 goto out; 3052 if (routing.flags) 3053 goto out; 3054 if (routing.nr) { 3055 r = -ENOMEM; 3056 entries = vmalloc(routing.nr * sizeof(*entries)); 3057 if (!entries) 3058 goto out; 3059 r = -EFAULT; 3060 urouting = argp; 3061 if (copy_from_user(entries, urouting->entries, 3062 routing.nr * sizeof(*entries))) 3063 goto out_free_irq_routing; 3064 } 3065 r = kvm_set_irq_routing(kvm, entries, routing.nr, 3066 routing.flags); 3067 out_free_irq_routing: 3068 vfree(entries); 3069 break; 3070 } 3071 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 3072 case KVM_CREATE_DEVICE: { 3073 struct kvm_create_device cd; 3074 3075 r = -EFAULT; 3076 if (copy_from_user(&cd, argp, sizeof(cd))) 3077 goto out; 3078 3079 r = kvm_ioctl_create_device(kvm, &cd); 3080 if (r) 3081 goto out; 3082 3083 r = -EFAULT; 3084 if (copy_to_user(argp, &cd, sizeof(cd))) 3085 goto out; 3086 3087 r = 0; 3088 break; 3089 } 3090 case KVM_CHECK_EXTENSION: 3091 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 3092 break; 3093 default: 3094 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 3095 } 3096 out: 3097 return r; 3098 } 3099 3100 #ifdef CONFIG_KVM_COMPAT 3101 struct compat_kvm_dirty_log { 3102 __u32 slot; 3103 __u32 padding1; 3104 union { 3105 compat_uptr_t dirty_bitmap; /* one bit per page */ 3106 __u64 padding2; 3107 }; 3108 }; 3109 3110 static long kvm_vm_compat_ioctl(struct file *filp, 3111 unsigned int ioctl, unsigned long arg) 3112 { 3113 struct kvm *kvm = filp->private_data; 3114 int r; 3115 3116 if (kvm->mm != current->mm) 3117 return -EIO; 3118 switch (ioctl) { 3119 case KVM_GET_DIRTY_LOG: { 3120 struct compat_kvm_dirty_log compat_log; 3121 struct kvm_dirty_log log; 3122 3123 if (copy_from_user(&compat_log, (void __user *)arg, 3124 sizeof(compat_log))) 3125 return -EFAULT; 3126 log.slot = compat_log.slot; 3127 log.padding1 = compat_log.padding1; 3128 log.padding2 = compat_log.padding2; 3129 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 3130 3131 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 3132 break; 3133 } 3134 default: 3135 r = kvm_vm_ioctl(filp, ioctl, arg); 3136 } 3137 return r; 3138 } 3139 #endif 3140 3141 static struct file_operations kvm_vm_fops = { 3142 .release = kvm_vm_release, 3143 .unlocked_ioctl = kvm_vm_ioctl, 3144 #ifdef CONFIG_KVM_COMPAT 3145 .compat_ioctl = kvm_vm_compat_ioctl, 3146 #endif 3147 .llseek = noop_llseek, 3148 }; 3149 3150 static int kvm_dev_ioctl_create_vm(unsigned long type) 3151 { 3152 int r; 3153 struct kvm *kvm; 3154 struct file *file; 3155 3156 kvm = kvm_create_vm(type); 3157 if (IS_ERR(kvm)) 3158 return PTR_ERR(kvm); 3159 #ifdef CONFIG_KVM_MMIO 3160 r = kvm_coalesced_mmio_init(kvm); 3161 if (r < 0) { 3162 kvm_put_kvm(kvm); 3163 return r; 3164 } 3165 #endif 3166 r = get_unused_fd_flags(O_CLOEXEC); 3167 if (r < 0) { 3168 kvm_put_kvm(kvm); 3169 return r; 3170 } 3171 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 3172 if (IS_ERR(file)) { 3173 put_unused_fd(r); 3174 kvm_put_kvm(kvm); 3175 return PTR_ERR(file); 3176 } 3177 3178 /* 3179 * Don't call kvm_put_kvm anymore at this point; file->f_op is 3180 * already set, with ->release() being kvm_vm_release(). In error 3181 * cases it will be called by the final fput(file) and will take 3182 * care of doing kvm_put_kvm(kvm). 3183 */ 3184 if (kvm_create_vm_debugfs(kvm, r) < 0) { 3185 put_unused_fd(r); 3186 fput(file); 3187 return -ENOMEM; 3188 } 3189 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); 3190 3191 fd_install(r, file); 3192 return r; 3193 } 3194 3195 static long kvm_dev_ioctl(struct file *filp, 3196 unsigned int ioctl, unsigned long arg) 3197 { 3198 long r = -EINVAL; 3199 3200 switch (ioctl) { 3201 case KVM_GET_API_VERSION: 3202 if (arg) 3203 goto out; 3204 r = KVM_API_VERSION; 3205 break; 3206 case KVM_CREATE_VM: 3207 r = kvm_dev_ioctl_create_vm(arg); 3208 break; 3209 case KVM_CHECK_EXTENSION: 3210 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 3211 break; 3212 case KVM_GET_VCPU_MMAP_SIZE: 3213 if (arg) 3214 goto out; 3215 r = PAGE_SIZE; /* struct kvm_run */ 3216 #ifdef CONFIG_X86 3217 r += PAGE_SIZE; /* pio data page */ 3218 #endif 3219 #ifdef CONFIG_KVM_MMIO 3220 r += PAGE_SIZE; /* coalesced mmio ring page */ 3221 #endif 3222 break; 3223 case KVM_TRACE_ENABLE: 3224 case KVM_TRACE_PAUSE: 3225 case KVM_TRACE_DISABLE: 3226 r = -EOPNOTSUPP; 3227 break; 3228 default: 3229 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3230 } 3231 out: 3232 return r; 3233 } 3234 3235 static struct file_operations kvm_chardev_ops = { 3236 .unlocked_ioctl = kvm_dev_ioctl, 3237 .compat_ioctl = kvm_dev_ioctl, 3238 .llseek = noop_llseek, 3239 }; 3240 3241 static struct miscdevice kvm_dev = { 3242 KVM_MINOR, 3243 "kvm", 3244 &kvm_chardev_ops, 3245 }; 3246 3247 static void hardware_enable_nolock(void *junk) 3248 { 3249 int cpu = raw_smp_processor_id(); 3250 int r; 3251 3252 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3253 return; 3254 3255 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3256 3257 r = kvm_arch_hardware_enable(); 3258 3259 if (r) { 3260 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3261 atomic_inc(&hardware_enable_failed); 3262 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3263 } 3264 } 3265 3266 static int kvm_starting_cpu(unsigned int cpu) 3267 { 3268 raw_spin_lock(&kvm_count_lock); 3269 if (kvm_usage_count) 3270 hardware_enable_nolock(NULL); 3271 raw_spin_unlock(&kvm_count_lock); 3272 return 0; 3273 } 3274 3275 static void hardware_disable_nolock(void *junk) 3276 { 3277 int cpu = raw_smp_processor_id(); 3278 3279 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3280 return; 3281 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3282 kvm_arch_hardware_disable(); 3283 } 3284 3285 static int kvm_dying_cpu(unsigned int cpu) 3286 { 3287 raw_spin_lock(&kvm_count_lock); 3288 if (kvm_usage_count) 3289 hardware_disable_nolock(NULL); 3290 raw_spin_unlock(&kvm_count_lock); 3291 return 0; 3292 } 3293 3294 static void hardware_disable_all_nolock(void) 3295 { 3296 BUG_ON(!kvm_usage_count); 3297 3298 kvm_usage_count--; 3299 if (!kvm_usage_count) 3300 on_each_cpu(hardware_disable_nolock, NULL, 1); 3301 } 3302 3303 static void hardware_disable_all(void) 3304 { 3305 raw_spin_lock(&kvm_count_lock); 3306 hardware_disable_all_nolock(); 3307 raw_spin_unlock(&kvm_count_lock); 3308 } 3309 3310 static int hardware_enable_all(void) 3311 { 3312 int r = 0; 3313 3314 raw_spin_lock(&kvm_count_lock); 3315 3316 kvm_usage_count++; 3317 if (kvm_usage_count == 1) { 3318 atomic_set(&hardware_enable_failed, 0); 3319 on_each_cpu(hardware_enable_nolock, NULL, 1); 3320 3321 if (atomic_read(&hardware_enable_failed)) { 3322 hardware_disable_all_nolock(); 3323 r = -EBUSY; 3324 } 3325 } 3326 3327 raw_spin_unlock(&kvm_count_lock); 3328 3329 return r; 3330 } 3331 3332 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3333 void *v) 3334 { 3335 /* 3336 * Some (well, at least mine) BIOSes hang on reboot if 3337 * in vmx root mode. 3338 * 3339 * And Intel TXT required VMX off for all cpu when system shutdown. 3340 */ 3341 pr_info("kvm: exiting hardware virtualization\n"); 3342 kvm_rebooting = true; 3343 on_each_cpu(hardware_disable_nolock, NULL, 1); 3344 return NOTIFY_OK; 3345 } 3346 3347 static struct notifier_block kvm_reboot_notifier = { 3348 .notifier_call = kvm_reboot, 3349 .priority = 0, 3350 }; 3351 3352 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3353 { 3354 int i; 3355 3356 for (i = 0; i < bus->dev_count; i++) { 3357 struct kvm_io_device *pos = bus->range[i].dev; 3358 3359 kvm_iodevice_destructor(pos); 3360 } 3361 kfree(bus); 3362 } 3363 3364 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3365 const struct kvm_io_range *r2) 3366 { 3367 gpa_t addr1 = r1->addr; 3368 gpa_t addr2 = r2->addr; 3369 3370 if (addr1 < addr2) 3371 return -1; 3372 3373 /* If r2->len == 0, match the exact address. If r2->len != 0, 3374 * accept any overlapping write. Any order is acceptable for 3375 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3376 * we process all of them. 3377 */ 3378 if (r2->len) { 3379 addr1 += r1->len; 3380 addr2 += r2->len; 3381 } 3382 3383 if (addr1 > addr2) 3384 return 1; 3385 3386 return 0; 3387 } 3388 3389 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3390 { 3391 return kvm_io_bus_cmp(p1, p2); 3392 } 3393 3394 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3395 gpa_t addr, int len) 3396 { 3397 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3398 .addr = addr, 3399 .len = len, 3400 .dev = dev, 3401 }; 3402 3403 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3404 kvm_io_bus_sort_cmp, NULL); 3405 3406 return 0; 3407 } 3408 3409 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3410 gpa_t addr, int len) 3411 { 3412 struct kvm_io_range *range, key; 3413 int off; 3414 3415 key = (struct kvm_io_range) { 3416 .addr = addr, 3417 .len = len, 3418 }; 3419 3420 range = bsearch(&key, bus->range, bus->dev_count, 3421 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3422 if (range == NULL) 3423 return -ENOENT; 3424 3425 off = range - bus->range; 3426 3427 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3428 off--; 3429 3430 return off; 3431 } 3432 3433 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3434 struct kvm_io_range *range, const void *val) 3435 { 3436 int idx; 3437 3438 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3439 if (idx < 0) 3440 return -EOPNOTSUPP; 3441 3442 while (idx < bus->dev_count && 3443 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3444 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3445 range->len, val)) 3446 return idx; 3447 idx++; 3448 } 3449 3450 return -EOPNOTSUPP; 3451 } 3452 3453 /* kvm_io_bus_write - called under kvm->slots_lock */ 3454 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3455 int len, const void *val) 3456 { 3457 struct kvm_io_bus *bus; 3458 struct kvm_io_range range; 3459 int r; 3460 3461 range = (struct kvm_io_range) { 3462 .addr = addr, 3463 .len = len, 3464 }; 3465 3466 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3467 if (!bus) 3468 return -ENOMEM; 3469 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3470 return r < 0 ? r : 0; 3471 } 3472 3473 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3474 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3475 gpa_t addr, int len, const void *val, long cookie) 3476 { 3477 struct kvm_io_bus *bus; 3478 struct kvm_io_range range; 3479 3480 range = (struct kvm_io_range) { 3481 .addr = addr, 3482 .len = len, 3483 }; 3484 3485 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3486 if (!bus) 3487 return -ENOMEM; 3488 3489 /* First try the device referenced by cookie. */ 3490 if ((cookie >= 0) && (cookie < bus->dev_count) && 3491 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3492 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3493 val)) 3494 return cookie; 3495 3496 /* 3497 * cookie contained garbage; fall back to search and return the 3498 * correct cookie value. 3499 */ 3500 return __kvm_io_bus_write(vcpu, bus, &range, val); 3501 } 3502 3503 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3504 struct kvm_io_range *range, void *val) 3505 { 3506 int idx; 3507 3508 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3509 if (idx < 0) 3510 return -EOPNOTSUPP; 3511 3512 while (idx < bus->dev_count && 3513 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3514 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3515 range->len, val)) 3516 return idx; 3517 idx++; 3518 } 3519 3520 return -EOPNOTSUPP; 3521 } 3522 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3523 3524 /* kvm_io_bus_read - called under kvm->slots_lock */ 3525 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3526 int len, void *val) 3527 { 3528 struct kvm_io_bus *bus; 3529 struct kvm_io_range range; 3530 int r; 3531 3532 range = (struct kvm_io_range) { 3533 .addr = addr, 3534 .len = len, 3535 }; 3536 3537 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3538 if (!bus) 3539 return -ENOMEM; 3540 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3541 return r < 0 ? r : 0; 3542 } 3543 3544 3545 /* Caller must hold slots_lock. */ 3546 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3547 int len, struct kvm_io_device *dev) 3548 { 3549 struct kvm_io_bus *new_bus, *bus; 3550 3551 bus = kvm_get_bus(kvm, bus_idx); 3552 if (!bus) 3553 return -ENOMEM; 3554 3555 /* exclude ioeventfd which is limited by maximum fd */ 3556 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3557 return -ENOSPC; 3558 3559 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3560 sizeof(struct kvm_io_range)), GFP_KERNEL); 3561 if (!new_bus) 3562 return -ENOMEM; 3563 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3564 sizeof(struct kvm_io_range))); 3565 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3566 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3567 synchronize_srcu_expedited(&kvm->srcu); 3568 kfree(bus); 3569 3570 return 0; 3571 } 3572 3573 /* Caller must hold slots_lock. */ 3574 void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3575 struct kvm_io_device *dev) 3576 { 3577 int i; 3578 struct kvm_io_bus *new_bus, *bus; 3579 3580 bus = kvm_get_bus(kvm, bus_idx); 3581 if (!bus) 3582 return; 3583 3584 for (i = 0; i < bus->dev_count; i++) 3585 if (bus->range[i].dev == dev) { 3586 break; 3587 } 3588 3589 if (i == bus->dev_count) 3590 return; 3591 3592 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3593 sizeof(struct kvm_io_range)), GFP_KERNEL); 3594 if (!new_bus) { 3595 pr_err("kvm: failed to shrink bus, removing it completely\n"); 3596 goto broken; 3597 } 3598 3599 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3600 new_bus->dev_count--; 3601 memcpy(new_bus->range + i, bus->range + i + 1, 3602 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3603 3604 broken: 3605 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3606 synchronize_srcu_expedited(&kvm->srcu); 3607 kfree(bus); 3608 return; 3609 } 3610 3611 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3612 gpa_t addr) 3613 { 3614 struct kvm_io_bus *bus; 3615 int dev_idx, srcu_idx; 3616 struct kvm_io_device *iodev = NULL; 3617 3618 srcu_idx = srcu_read_lock(&kvm->srcu); 3619 3620 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3621 if (!bus) 3622 goto out_unlock; 3623 3624 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); 3625 if (dev_idx < 0) 3626 goto out_unlock; 3627 3628 iodev = bus->range[dev_idx].dev; 3629 3630 out_unlock: 3631 srcu_read_unlock(&kvm->srcu, srcu_idx); 3632 3633 return iodev; 3634 } 3635 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev); 3636 3637 static int kvm_debugfs_open(struct inode *inode, struct file *file, 3638 int (*get)(void *, u64 *), int (*set)(void *, u64), 3639 const char *fmt) 3640 { 3641 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3642 inode->i_private; 3643 3644 /* The debugfs files are a reference to the kvm struct which 3645 * is still valid when kvm_destroy_vm is called. 3646 * To avoid the race between open and the removal of the debugfs 3647 * directory we test against the users count. 3648 */ 3649 if (!refcount_inc_not_zero(&stat_data->kvm->users_count)) 3650 return -ENOENT; 3651 3652 if (simple_attr_open(inode, file, get, set, fmt)) { 3653 kvm_put_kvm(stat_data->kvm); 3654 return -ENOMEM; 3655 } 3656 3657 return 0; 3658 } 3659 3660 static int kvm_debugfs_release(struct inode *inode, struct file *file) 3661 { 3662 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3663 inode->i_private; 3664 3665 simple_attr_release(inode, file); 3666 kvm_put_kvm(stat_data->kvm); 3667 3668 return 0; 3669 } 3670 3671 static int vm_stat_get_per_vm(void *data, u64 *val) 3672 { 3673 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3674 3675 *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset); 3676 3677 return 0; 3678 } 3679 3680 static int vm_stat_clear_per_vm(void *data, u64 val) 3681 { 3682 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3683 3684 if (val) 3685 return -EINVAL; 3686 3687 *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0; 3688 3689 return 0; 3690 } 3691 3692 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file) 3693 { 3694 __simple_attr_check_format("%llu\n", 0ull); 3695 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm, 3696 vm_stat_clear_per_vm, "%llu\n"); 3697 } 3698 3699 static const struct file_operations vm_stat_get_per_vm_fops = { 3700 .owner = THIS_MODULE, 3701 .open = vm_stat_get_per_vm_open, 3702 .release = kvm_debugfs_release, 3703 .read = simple_attr_read, 3704 .write = simple_attr_write, 3705 .llseek = no_llseek, 3706 }; 3707 3708 static int vcpu_stat_get_per_vm(void *data, u64 *val) 3709 { 3710 int i; 3711 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3712 struct kvm_vcpu *vcpu; 3713 3714 *val = 0; 3715 3716 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3717 *val += *(u64 *)((void *)vcpu + stat_data->offset); 3718 3719 return 0; 3720 } 3721 3722 static int vcpu_stat_clear_per_vm(void *data, u64 val) 3723 { 3724 int i; 3725 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3726 struct kvm_vcpu *vcpu; 3727 3728 if (val) 3729 return -EINVAL; 3730 3731 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3732 *(u64 *)((void *)vcpu + stat_data->offset) = 0; 3733 3734 return 0; 3735 } 3736 3737 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file) 3738 { 3739 __simple_attr_check_format("%llu\n", 0ull); 3740 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm, 3741 vcpu_stat_clear_per_vm, "%llu\n"); 3742 } 3743 3744 static const struct file_operations vcpu_stat_get_per_vm_fops = { 3745 .owner = THIS_MODULE, 3746 .open = vcpu_stat_get_per_vm_open, 3747 .release = kvm_debugfs_release, 3748 .read = simple_attr_read, 3749 .write = simple_attr_write, 3750 .llseek = no_llseek, 3751 }; 3752 3753 static const struct file_operations *stat_fops_per_vm[] = { 3754 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops, 3755 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops, 3756 }; 3757 3758 static int vm_stat_get(void *_offset, u64 *val) 3759 { 3760 unsigned offset = (long)_offset; 3761 struct kvm *kvm; 3762 struct kvm_stat_data stat_tmp = {.offset = offset}; 3763 u64 tmp_val; 3764 3765 *val = 0; 3766 spin_lock(&kvm_lock); 3767 list_for_each_entry(kvm, &vm_list, vm_list) { 3768 stat_tmp.kvm = kvm; 3769 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3770 *val += tmp_val; 3771 } 3772 spin_unlock(&kvm_lock); 3773 return 0; 3774 } 3775 3776 static int vm_stat_clear(void *_offset, u64 val) 3777 { 3778 unsigned offset = (long)_offset; 3779 struct kvm *kvm; 3780 struct kvm_stat_data stat_tmp = {.offset = offset}; 3781 3782 if (val) 3783 return -EINVAL; 3784 3785 spin_lock(&kvm_lock); 3786 list_for_each_entry(kvm, &vm_list, vm_list) { 3787 stat_tmp.kvm = kvm; 3788 vm_stat_clear_per_vm((void *)&stat_tmp, 0); 3789 } 3790 spin_unlock(&kvm_lock); 3791 3792 return 0; 3793 } 3794 3795 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); 3796 3797 static int vcpu_stat_get(void *_offset, u64 *val) 3798 { 3799 unsigned offset = (long)_offset; 3800 struct kvm *kvm; 3801 struct kvm_stat_data stat_tmp = {.offset = offset}; 3802 u64 tmp_val; 3803 3804 *val = 0; 3805 spin_lock(&kvm_lock); 3806 list_for_each_entry(kvm, &vm_list, vm_list) { 3807 stat_tmp.kvm = kvm; 3808 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3809 *val += tmp_val; 3810 } 3811 spin_unlock(&kvm_lock); 3812 return 0; 3813 } 3814 3815 static int vcpu_stat_clear(void *_offset, u64 val) 3816 { 3817 unsigned offset = (long)_offset; 3818 struct kvm *kvm; 3819 struct kvm_stat_data stat_tmp = {.offset = offset}; 3820 3821 if (val) 3822 return -EINVAL; 3823 3824 spin_lock(&kvm_lock); 3825 list_for_each_entry(kvm, &vm_list, vm_list) { 3826 stat_tmp.kvm = kvm; 3827 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0); 3828 } 3829 spin_unlock(&kvm_lock); 3830 3831 return 0; 3832 } 3833 3834 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, 3835 "%llu\n"); 3836 3837 static const struct file_operations *stat_fops[] = { 3838 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3839 [KVM_STAT_VM] = &vm_stat_fops, 3840 }; 3841 3842 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) 3843 { 3844 struct kobj_uevent_env *env; 3845 unsigned long long created, active; 3846 3847 if (!kvm_dev.this_device || !kvm) 3848 return; 3849 3850 spin_lock(&kvm_lock); 3851 if (type == KVM_EVENT_CREATE_VM) { 3852 kvm_createvm_count++; 3853 kvm_active_vms++; 3854 } else if (type == KVM_EVENT_DESTROY_VM) { 3855 kvm_active_vms--; 3856 } 3857 created = kvm_createvm_count; 3858 active = kvm_active_vms; 3859 spin_unlock(&kvm_lock); 3860 3861 env = kzalloc(sizeof(*env), GFP_KERNEL); 3862 if (!env) 3863 return; 3864 3865 add_uevent_var(env, "CREATED=%llu", created); 3866 add_uevent_var(env, "COUNT=%llu", active); 3867 3868 if (type == KVM_EVENT_CREATE_VM) { 3869 add_uevent_var(env, "EVENT=create"); 3870 kvm->userspace_pid = task_pid_nr(current); 3871 } else if (type == KVM_EVENT_DESTROY_VM) { 3872 add_uevent_var(env, "EVENT=destroy"); 3873 } 3874 add_uevent_var(env, "PID=%d", kvm->userspace_pid); 3875 3876 if (kvm->debugfs_dentry) { 3877 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL); 3878 3879 if (p) { 3880 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); 3881 if (!IS_ERR(tmp)) 3882 add_uevent_var(env, "STATS_PATH=%s", tmp); 3883 kfree(p); 3884 } 3885 } 3886 /* no need for checks, since we are adding at most only 5 keys */ 3887 env->envp[env->envp_idx++] = NULL; 3888 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp); 3889 kfree(env); 3890 } 3891 3892 static int kvm_init_debug(void) 3893 { 3894 int r = -EEXIST; 3895 struct kvm_stats_debugfs_item *p; 3896 3897 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3898 if (kvm_debugfs_dir == NULL) 3899 goto out; 3900 3901 kvm_debugfs_num_entries = 0; 3902 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { 3903 if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir, 3904 (void *)(long)p->offset, 3905 stat_fops[p->kind])) 3906 goto out_dir; 3907 } 3908 3909 return 0; 3910 3911 out_dir: 3912 debugfs_remove_recursive(kvm_debugfs_dir); 3913 out: 3914 return r; 3915 } 3916 3917 static int kvm_suspend(void) 3918 { 3919 if (kvm_usage_count) 3920 hardware_disable_nolock(NULL); 3921 return 0; 3922 } 3923 3924 static void kvm_resume(void) 3925 { 3926 if (kvm_usage_count) { 3927 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3928 hardware_enable_nolock(NULL); 3929 } 3930 } 3931 3932 static struct syscore_ops kvm_syscore_ops = { 3933 .suspend = kvm_suspend, 3934 .resume = kvm_resume, 3935 }; 3936 3937 static inline 3938 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3939 { 3940 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3941 } 3942 3943 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3944 { 3945 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3946 3947 if (vcpu->preempted) 3948 vcpu->preempted = false; 3949 3950 kvm_arch_sched_in(vcpu, cpu); 3951 3952 kvm_arch_vcpu_load(vcpu, cpu); 3953 } 3954 3955 static void kvm_sched_out(struct preempt_notifier *pn, 3956 struct task_struct *next) 3957 { 3958 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3959 3960 if (current->state == TASK_RUNNING) 3961 vcpu->preempted = true; 3962 kvm_arch_vcpu_put(vcpu); 3963 } 3964 3965 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3966 struct module *module) 3967 { 3968 int r; 3969 int cpu; 3970 3971 r = kvm_arch_init(opaque); 3972 if (r) 3973 goto out_fail; 3974 3975 /* 3976 * kvm_arch_init makes sure there's at most one caller 3977 * for architectures that support multiple implementations, 3978 * like intel and amd on x86. 3979 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3980 * conflicts in case kvm is already setup for another implementation. 3981 */ 3982 r = kvm_irqfd_init(); 3983 if (r) 3984 goto out_irqfd; 3985 3986 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3987 r = -ENOMEM; 3988 goto out_free_0; 3989 } 3990 3991 r = kvm_arch_hardware_setup(); 3992 if (r < 0) 3993 goto out_free_0a; 3994 3995 for_each_online_cpu(cpu) { 3996 smp_call_function_single(cpu, 3997 kvm_arch_check_processor_compat, 3998 &r, 1); 3999 if (r < 0) 4000 goto out_free_1; 4001 } 4002 4003 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting", 4004 kvm_starting_cpu, kvm_dying_cpu); 4005 if (r) 4006 goto out_free_2; 4007 register_reboot_notifier(&kvm_reboot_notifier); 4008 4009 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 4010 if (!vcpu_align) 4011 vcpu_align = __alignof__(struct kvm_vcpu); 4012 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 4013 SLAB_ACCOUNT, NULL); 4014 if (!kvm_vcpu_cache) { 4015 r = -ENOMEM; 4016 goto out_free_3; 4017 } 4018 4019 r = kvm_async_pf_init(); 4020 if (r) 4021 goto out_free; 4022 4023 kvm_chardev_ops.owner = module; 4024 kvm_vm_fops.owner = module; 4025 kvm_vcpu_fops.owner = module; 4026 4027 r = misc_register(&kvm_dev); 4028 if (r) { 4029 pr_err("kvm: misc device register failed\n"); 4030 goto out_unreg; 4031 } 4032 4033 register_syscore_ops(&kvm_syscore_ops); 4034 4035 kvm_preempt_ops.sched_in = kvm_sched_in; 4036 kvm_preempt_ops.sched_out = kvm_sched_out; 4037 4038 r = kvm_init_debug(); 4039 if (r) { 4040 pr_err("kvm: create debugfs files failed\n"); 4041 goto out_undebugfs; 4042 } 4043 4044 r = kvm_vfio_ops_init(); 4045 WARN_ON(r); 4046 4047 return 0; 4048 4049 out_undebugfs: 4050 unregister_syscore_ops(&kvm_syscore_ops); 4051 misc_deregister(&kvm_dev); 4052 out_unreg: 4053 kvm_async_pf_deinit(); 4054 out_free: 4055 kmem_cache_destroy(kvm_vcpu_cache); 4056 out_free_3: 4057 unregister_reboot_notifier(&kvm_reboot_notifier); 4058 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4059 out_free_2: 4060 out_free_1: 4061 kvm_arch_hardware_unsetup(); 4062 out_free_0a: 4063 free_cpumask_var(cpus_hardware_enabled); 4064 out_free_0: 4065 kvm_irqfd_exit(); 4066 out_irqfd: 4067 kvm_arch_exit(); 4068 out_fail: 4069 return r; 4070 } 4071 EXPORT_SYMBOL_GPL(kvm_init); 4072 4073 void kvm_exit(void) 4074 { 4075 debugfs_remove_recursive(kvm_debugfs_dir); 4076 misc_deregister(&kvm_dev); 4077 kmem_cache_destroy(kvm_vcpu_cache); 4078 kvm_async_pf_deinit(); 4079 unregister_syscore_ops(&kvm_syscore_ops); 4080 unregister_reboot_notifier(&kvm_reboot_notifier); 4081 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4082 on_each_cpu(hardware_disable_nolock, NULL, 1); 4083 kvm_arch_hardware_unsetup(); 4084 kvm_arch_exit(); 4085 kvm_irqfd_exit(); 4086 free_cpumask_var(cpus_hardware_enabled); 4087 kvm_vfio_ops_exit(); 4088 } 4089 EXPORT_SYMBOL_GPL(kvm_exit); 4090