1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 16 #include <kvm/iodev.h> 17 18 #include <linux/kvm_host.h> 19 #include <linux/kvm.h> 20 #include <linux/module.h> 21 #include <linux/errno.h> 22 #include <linux/percpu.h> 23 #include <linux/mm.h> 24 #include <linux/miscdevice.h> 25 #include <linux/vmalloc.h> 26 #include <linux/reboot.h> 27 #include <linux/debugfs.h> 28 #include <linux/highmem.h> 29 #include <linux/file.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/cpu.h> 32 #include <linux/sched/signal.h> 33 #include <linux/sched/mm.h> 34 #include <linux/sched/stat.h> 35 #include <linux/cpumask.h> 36 #include <linux/smp.h> 37 #include <linux/anon_inodes.h> 38 #include <linux/profile.h> 39 #include <linux/kvm_para.h> 40 #include <linux/pagemap.h> 41 #include <linux/mman.h> 42 #include <linux/swap.h> 43 #include <linux/bitops.h> 44 #include <linux/spinlock.h> 45 #include <linux/compat.h> 46 #include <linux/srcu.h> 47 #include <linux/hugetlb.h> 48 #include <linux/slab.h> 49 #include <linux/sort.h> 50 #include <linux/bsearch.h> 51 #include <linux/io.h> 52 #include <linux/lockdep.h> 53 #include <linux/kthread.h> 54 55 #include <asm/processor.h> 56 #include <asm/ioctl.h> 57 #include <linux/uaccess.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 #include <linux/kvm_dirty_ring.h> 67 68 /* Worst case buffer size needed for holding an integer. */ 69 #define ITOA_MAX_LEN 12 70 71 MODULE_AUTHOR("Qumranet"); 72 MODULE_LICENSE("GPL"); 73 74 /* Architectures should define their poll value according to the halt latency */ 75 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 76 module_param(halt_poll_ns, uint, 0644); 77 EXPORT_SYMBOL_GPL(halt_poll_ns); 78 79 /* Default doubles per-vcpu halt_poll_ns. */ 80 unsigned int halt_poll_ns_grow = 2; 81 module_param(halt_poll_ns_grow, uint, 0644); 82 EXPORT_SYMBOL_GPL(halt_poll_ns_grow); 83 84 /* The start value to grow halt_poll_ns from */ 85 unsigned int halt_poll_ns_grow_start = 10000; /* 10us */ 86 module_param(halt_poll_ns_grow_start, uint, 0644); 87 EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start); 88 89 /* Default resets per-vcpu halt_poll_ns . */ 90 unsigned int halt_poll_ns_shrink; 91 module_param(halt_poll_ns_shrink, uint, 0644); 92 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); 93 94 /* 95 * Ordering of locks: 96 * 97 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 98 */ 99 100 DEFINE_MUTEX(kvm_lock); 101 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 102 LIST_HEAD(vm_list); 103 104 static cpumask_var_t cpus_hardware_enabled; 105 static int kvm_usage_count; 106 static atomic_t hardware_enable_failed; 107 108 static struct kmem_cache *kvm_vcpu_cache; 109 110 static __read_mostly struct preempt_ops kvm_preempt_ops; 111 static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu); 112 113 struct dentry *kvm_debugfs_dir; 114 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 115 116 static int kvm_debugfs_num_entries; 117 static const struct file_operations stat_fops_per_vm; 118 119 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 120 unsigned long arg); 121 #ifdef CONFIG_KVM_COMPAT 122 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 123 unsigned long arg); 124 #define KVM_COMPAT(c) .compat_ioctl = (c) 125 #else 126 /* 127 * For architectures that don't implement a compat infrastructure, 128 * adopt a double line of defense: 129 * - Prevent a compat task from opening /dev/kvm 130 * - If the open has been done by a 64bit task, and the KVM fd 131 * passed to a compat task, let the ioctls fail. 132 */ 133 static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl, 134 unsigned long arg) { return -EINVAL; } 135 136 static int kvm_no_compat_open(struct inode *inode, struct file *file) 137 { 138 return is_compat_task() ? -ENODEV : 0; 139 } 140 #define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \ 141 .open = kvm_no_compat_open 142 #endif 143 static int hardware_enable_all(void); 144 static void hardware_disable_all(void); 145 146 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 147 148 __visible bool kvm_rebooting; 149 EXPORT_SYMBOL_GPL(kvm_rebooting); 150 151 #define KVM_EVENT_CREATE_VM 0 152 #define KVM_EVENT_DESTROY_VM 1 153 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm); 154 static unsigned long long kvm_createvm_count; 155 static unsigned long long kvm_active_vms; 156 157 __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, 158 unsigned long start, unsigned long end) 159 { 160 } 161 162 bool kvm_is_zone_device_pfn(kvm_pfn_t pfn) 163 { 164 /* 165 * The metadata used by is_zone_device_page() to determine whether or 166 * not a page is ZONE_DEVICE is guaranteed to be valid if and only if 167 * the device has been pinned, e.g. by get_user_pages(). WARN if the 168 * page_count() is zero to help detect bad usage of this helper. 169 */ 170 if (!pfn_valid(pfn) || WARN_ON_ONCE(!page_count(pfn_to_page(pfn)))) 171 return false; 172 173 return is_zone_device_page(pfn_to_page(pfn)); 174 } 175 176 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 177 { 178 /* 179 * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting 180 * perspective they are "normal" pages, albeit with slightly different 181 * usage rules. 182 */ 183 if (pfn_valid(pfn)) 184 return PageReserved(pfn_to_page(pfn)) && 185 !is_zero_pfn(pfn) && 186 !kvm_is_zone_device_pfn(pfn); 187 188 return true; 189 } 190 191 bool kvm_is_transparent_hugepage(kvm_pfn_t pfn) 192 { 193 struct page *page = pfn_to_page(pfn); 194 195 if (!PageTransCompoundMap(page)) 196 return false; 197 198 return is_transparent_hugepage(compound_head(page)); 199 } 200 201 /* 202 * Switches to specified vcpu, until a matching vcpu_put() 203 */ 204 void vcpu_load(struct kvm_vcpu *vcpu) 205 { 206 int cpu = get_cpu(); 207 208 __this_cpu_write(kvm_running_vcpu, vcpu); 209 preempt_notifier_register(&vcpu->preempt_notifier); 210 kvm_arch_vcpu_load(vcpu, cpu); 211 put_cpu(); 212 } 213 EXPORT_SYMBOL_GPL(vcpu_load); 214 215 void vcpu_put(struct kvm_vcpu *vcpu) 216 { 217 preempt_disable(); 218 kvm_arch_vcpu_put(vcpu); 219 preempt_notifier_unregister(&vcpu->preempt_notifier); 220 __this_cpu_write(kvm_running_vcpu, NULL); 221 preempt_enable(); 222 } 223 EXPORT_SYMBOL_GPL(vcpu_put); 224 225 /* TODO: merge with kvm_arch_vcpu_should_kick */ 226 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req) 227 { 228 int mode = kvm_vcpu_exiting_guest_mode(vcpu); 229 230 /* 231 * We need to wait for the VCPU to reenable interrupts and get out of 232 * READING_SHADOW_PAGE_TABLES mode. 233 */ 234 if (req & KVM_REQUEST_WAIT) 235 return mode != OUTSIDE_GUEST_MODE; 236 237 /* 238 * Need to kick a running VCPU, but otherwise there is nothing to do. 239 */ 240 return mode == IN_GUEST_MODE; 241 } 242 243 static void ack_flush(void *_completed) 244 { 245 } 246 247 static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait) 248 { 249 if (unlikely(!cpus)) 250 cpus = cpu_online_mask; 251 252 if (cpumask_empty(cpus)) 253 return false; 254 255 smp_call_function_many(cpus, ack_flush, NULL, wait); 256 return true; 257 } 258 259 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, 260 struct kvm_vcpu *except, 261 unsigned long *vcpu_bitmap, cpumask_var_t tmp) 262 { 263 int i, cpu, me; 264 struct kvm_vcpu *vcpu; 265 bool called; 266 267 me = get_cpu(); 268 269 kvm_for_each_vcpu(i, vcpu, kvm) { 270 if ((vcpu_bitmap && !test_bit(i, vcpu_bitmap)) || 271 vcpu == except) 272 continue; 273 274 kvm_make_request(req, vcpu); 275 cpu = vcpu->cpu; 276 277 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu)) 278 continue; 279 280 if (tmp != NULL && cpu != -1 && cpu != me && 281 kvm_request_needs_ipi(vcpu, req)) 282 __cpumask_set_cpu(cpu, tmp); 283 } 284 285 called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT)); 286 put_cpu(); 287 288 return called; 289 } 290 291 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req, 292 struct kvm_vcpu *except) 293 { 294 cpumask_var_t cpus; 295 bool called; 296 297 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 298 299 called = kvm_make_vcpus_request_mask(kvm, req, except, NULL, cpus); 300 301 free_cpumask_var(cpus); 302 return called; 303 } 304 305 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 306 { 307 return kvm_make_all_cpus_request_except(kvm, req, NULL); 308 } 309 310 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 311 void kvm_flush_remote_tlbs(struct kvm *kvm) 312 { 313 /* 314 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 315 * kvm_make_all_cpus_request. 316 */ 317 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 318 319 /* 320 * We want to publish modifications to the page tables before reading 321 * mode. Pairs with a memory barrier in arch-specific code. 322 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 323 * and smp_mb in walk_shadow_page_lockless_begin/end. 324 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 325 * 326 * There is already an smp_mb__after_atomic() before 327 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 328 * barrier here. 329 */ 330 if (!kvm_arch_flush_remote_tlb(kvm) 331 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 332 ++kvm->stat.remote_tlb_flush; 333 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 334 } 335 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 336 #endif 337 338 void kvm_reload_remote_mmus(struct kvm *kvm) 339 { 340 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 341 } 342 343 #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 344 static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc, 345 gfp_t gfp_flags) 346 { 347 gfp_flags |= mc->gfp_zero; 348 349 if (mc->kmem_cache) 350 return kmem_cache_alloc(mc->kmem_cache, gfp_flags); 351 else 352 return (void *)__get_free_page(gfp_flags); 353 } 354 355 int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min) 356 { 357 void *obj; 358 359 if (mc->nobjs >= min) 360 return 0; 361 while (mc->nobjs < ARRAY_SIZE(mc->objects)) { 362 obj = mmu_memory_cache_alloc_obj(mc, GFP_KERNEL_ACCOUNT); 363 if (!obj) 364 return mc->nobjs >= min ? 0 : -ENOMEM; 365 mc->objects[mc->nobjs++] = obj; 366 } 367 return 0; 368 } 369 370 int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc) 371 { 372 return mc->nobjs; 373 } 374 375 void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) 376 { 377 while (mc->nobjs) { 378 if (mc->kmem_cache) 379 kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]); 380 else 381 free_page((unsigned long)mc->objects[--mc->nobjs]); 382 } 383 } 384 385 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) 386 { 387 void *p; 388 389 if (WARN_ON(!mc->nobjs)) 390 p = mmu_memory_cache_alloc_obj(mc, GFP_ATOMIC | __GFP_ACCOUNT); 391 else 392 p = mc->objects[--mc->nobjs]; 393 BUG_ON(!p); 394 return p; 395 } 396 #endif 397 398 static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 399 { 400 mutex_init(&vcpu->mutex); 401 vcpu->cpu = -1; 402 vcpu->kvm = kvm; 403 vcpu->vcpu_id = id; 404 vcpu->pid = NULL; 405 rcuwait_init(&vcpu->wait); 406 kvm_async_pf_vcpu_init(vcpu); 407 408 vcpu->pre_pcpu = -1; 409 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 410 411 kvm_vcpu_set_in_spin_loop(vcpu, false); 412 kvm_vcpu_set_dy_eligible(vcpu, false); 413 vcpu->preempted = false; 414 vcpu->ready = false; 415 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 416 } 417 418 void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) 419 { 420 kvm_dirty_ring_free(&vcpu->dirty_ring); 421 kvm_arch_vcpu_destroy(vcpu); 422 423 /* 424 * No need for rcu_read_lock as VCPU_RUN is the only place that changes 425 * the vcpu->pid pointer, and at destruction time all file descriptors 426 * are already gone. 427 */ 428 put_pid(rcu_dereference_protected(vcpu->pid, 1)); 429 430 free_page((unsigned long)vcpu->run); 431 kmem_cache_free(kvm_vcpu_cache, vcpu); 432 } 433 EXPORT_SYMBOL_GPL(kvm_vcpu_destroy); 434 435 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 436 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 437 { 438 return container_of(mn, struct kvm, mmu_notifier); 439 } 440 441 static void kvm_mmu_notifier_invalidate_range(struct mmu_notifier *mn, 442 struct mm_struct *mm, 443 unsigned long start, unsigned long end) 444 { 445 struct kvm *kvm = mmu_notifier_to_kvm(mn); 446 int idx; 447 448 idx = srcu_read_lock(&kvm->srcu); 449 kvm_arch_mmu_notifier_invalidate_range(kvm, start, end); 450 srcu_read_unlock(&kvm->srcu, idx); 451 } 452 453 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 454 struct mm_struct *mm, 455 unsigned long address, 456 pte_t pte) 457 { 458 struct kvm *kvm = mmu_notifier_to_kvm(mn); 459 int idx; 460 461 idx = srcu_read_lock(&kvm->srcu); 462 spin_lock(&kvm->mmu_lock); 463 kvm->mmu_notifier_seq++; 464 465 if (kvm_set_spte_hva(kvm, address, pte)) 466 kvm_flush_remote_tlbs(kvm); 467 468 spin_unlock(&kvm->mmu_lock); 469 srcu_read_unlock(&kvm->srcu, idx); 470 } 471 472 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 473 const struct mmu_notifier_range *range) 474 { 475 struct kvm *kvm = mmu_notifier_to_kvm(mn); 476 int need_tlb_flush = 0, idx; 477 478 idx = srcu_read_lock(&kvm->srcu); 479 spin_lock(&kvm->mmu_lock); 480 /* 481 * The count increase must become visible at unlock time as no 482 * spte can be established without taking the mmu_lock and 483 * count is also read inside the mmu_lock critical section. 484 */ 485 kvm->mmu_notifier_count++; 486 need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end, 487 range->flags); 488 /* we've to flush the tlb before the pages can be freed */ 489 if (need_tlb_flush || kvm->tlbs_dirty) 490 kvm_flush_remote_tlbs(kvm); 491 492 spin_unlock(&kvm->mmu_lock); 493 srcu_read_unlock(&kvm->srcu, idx); 494 495 return 0; 496 } 497 498 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 499 const struct mmu_notifier_range *range) 500 { 501 struct kvm *kvm = mmu_notifier_to_kvm(mn); 502 503 spin_lock(&kvm->mmu_lock); 504 /* 505 * This sequence increase will notify the kvm page fault that 506 * the page that is going to be mapped in the spte could have 507 * been freed. 508 */ 509 kvm->mmu_notifier_seq++; 510 smp_wmb(); 511 /* 512 * The above sequence increase must be visible before the 513 * below count decrease, which is ensured by the smp_wmb above 514 * in conjunction with the smp_rmb in mmu_notifier_retry(). 515 */ 516 kvm->mmu_notifier_count--; 517 spin_unlock(&kvm->mmu_lock); 518 519 BUG_ON(kvm->mmu_notifier_count < 0); 520 } 521 522 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 523 struct mm_struct *mm, 524 unsigned long start, 525 unsigned long end) 526 { 527 struct kvm *kvm = mmu_notifier_to_kvm(mn); 528 int young, idx; 529 530 idx = srcu_read_lock(&kvm->srcu); 531 spin_lock(&kvm->mmu_lock); 532 533 young = kvm_age_hva(kvm, start, end); 534 if (young) 535 kvm_flush_remote_tlbs(kvm); 536 537 spin_unlock(&kvm->mmu_lock); 538 srcu_read_unlock(&kvm->srcu, idx); 539 540 return young; 541 } 542 543 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 544 struct mm_struct *mm, 545 unsigned long start, 546 unsigned long end) 547 { 548 struct kvm *kvm = mmu_notifier_to_kvm(mn); 549 int young, idx; 550 551 idx = srcu_read_lock(&kvm->srcu); 552 spin_lock(&kvm->mmu_lock); 553 /* 554 * Even though we do not flush TLB, this will still adversely 555 * affect performance on pre-Haswell Intel EPT, where there is 556 * no EPT Access Bit to clear so that we have to tear down EPT 557 * tables instead. If we find this unacceptable, we can always 558 * add a parameter to kvm_age_hva so that it effectively doesn't 559 * do anything on clear_young. 560 * 561 * Also note that currently we never issue secondary TLB flushes 562 * from clear_young, leaving this job up to the regular system 563 * cadence. If we find this inaccurate, we might come up with a 564 * more sophisticated heuristic later. 565 */ 566 young = kvm_age_hva(kvm, start, end); 567 spin_unlock(&kvm->mmu_lock); 568 srcu_read_unlock(&kvm->srcu, idx); 569 570 return young; 571 } 572 573 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 574 struct mm_struct *mm, 575 unsigned long address) 576 { 577 struct kvm *kvm = mmu_notifier_to_kvm(mn); 578 int young, idx; 579 580 idx = srcu_read_lock(&kvm->srcu); 581 spin_lock(&kvm->mmu_lock); 582 young = kvm_test_age_hva(kvm, address); 583 spin_unlock(&kvm->mmu_lock); 584 srcu_read_unlock(&kvm->srcu, idx); 585 586 return young; 587 } 588 589 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 590 struct mm_struct *mm) 591 { 592 struct kvm *kvm = mmu_notifier_to_kvm(mn); 593 int idx; 594 595 idx = srcu_read_lock(&kvm->srcu); 596 kvm_arch_flush_shadow_all(kvm); 597 srcu_read_unlock(&kvm->srcu, idx); 598 } 599 600 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 601 .invalidate_range = kvm_mmu_notifier_invalidate_range, 602 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 603 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 604 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 605 .clear_young = kvm_mmu_notifier_clear_young, 606 .test_young = kvm_mmu_notifier_test_young, 607 .change_pte = kvm_mmu_notifier_change_pte, 608 .release = kvm_mmu_notifier_release, 609 }; 610 611 static int kvm_init_mmu_notifier(struct kvm *kvm) 612 { 613 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 614 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 615 } 616 617 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 618 619 static int kvm_init_mmu_notifier(struct kvm *kvm) 620 { 621 return 0; 622 } 623 624 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 625 626 static struct kvm_memslots *kvm_alloc_memslots(void) 627 { 628 int i; 629 struct kvm_memslots *slots; 630 631 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT); 632 if (!slots) 633 return NULL; 634 635 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 636 slots->id_to_index[i] = -1; 637 638 return slots; 639 } 640 641 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 642 { 643 if (!memslot->dirty_bitmap) 644 return; 645 646 kvfree(memslot->dirty_bitmap); 647 memslot->dirty_bitmap = NULL; 648 } 649 650 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 651 { 652 kvm_destroy_dirty_bitmap(slot); 653 654 kvm_arch_free_memslot(kvm, slot); 655 656 slot->flags = 0; 657 slot->npages = 0; 658 } 659 660 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 661 { 662 struct kvm_memory_slot *memslot; 663 664 if (!slots) 665 return; 666 667 kvm_for_each_memslot(memslot, slots) 668 kvm_free_memslot(kvm, memslot); 669 670 kvfree(slots); 671 } 672 673 static void kvm_destroy_vm_debugfs(struct kvm *kvm) 674 { 675 int i; 676 677 if (!kvm->debugfs_dentry) 678 return; 679 680 debugfs_remove_recursive(kvm->debugfs_dentry); 681 682 if (kvm->debugfs_stat_data) { 683 for (i = 0; i < kvm_debugfs_num_entries; i++) 684 kfree(kvm->debugfs_stat_data[i]); 685 kfree(kvm->debugfs_stat_data); 686 } 687 } 688 689 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) 690 { 691 char dir_name[ITOA_MAX_LEN * 2]; 692 struct kvm_stat_data *stat_data; 693 struct kvm_stats_debugfs_item *p; 694 695 if (!debugfs_initialized()) 696 return 0; 697 698 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd); 699 kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir); 700 701 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, 702 sizeof(*kvm->debugfs_stat_data), 703 GFP_KERNEL_ACCOUNT); 704 if (!kvm->debugfs_stat_data) 705 return -ENOMEM; 706 707 for (p = debugfs_entries; p->name; p++) { 708 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT); 709 if (!stat_data) 710 return -ENOMEM; 711 712 stat_data->kvm = kvm; 713 stat_data->dbgfs_item = p; 714 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; 715 debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p), 716 kvm->debugfs_dentry, stat_data, 717 &stat_fops_per_vm); 718 } 719 return 0; 720 } 721 722 /* 723 * Called after the VM is otherwise initialized, but just before adding it to 724 * the vm_list. 725 */ 726 int __weak kvm_arch_post_init_vm(struct kvm *kvm) 727 { 728 return 0; 729 } 730 731 /* 732 * Called just after removing the VM from the vm_list, but before doing any 733 * other destruction. 734 */ 735 void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm) 736 { 737 } 738 739 static struct kvm *kvm_create_vm(unsigned long type) 740 { 741 struct kvm *kvm = kvm_arch_alloc_vm(); 742 int r = -ENOMEM; 743 int i; 744 745 if (!kvm) 746 return ERR_PTR(-ENOMEM); 747 748 spin_lock_init(&kvm->mmu_lock); 749 mmgrab(current->mm); 750 kvm->mm = current->mm; 751 kvm_eventfd_init(kvm); 752 mutex_init(&kvm->lock); 753 mutex_init(&kvm->irq_lock); 754 mutex_init(&kvm->slots_lock); 755 INIT_LIST_HEAD(&kvm->devices); 756 757 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 758 759 if (init_srcu_struct(&kvm->srcu)) 760 goto out_err_no_srcu; 761 if (init_srcu_struct(&kvm->irq_srcu)) 762 goto out_err_no_irq_srcu; 763 764 refcount_set(&kvm->users_count, 1); 765 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 766 struct kvm_memslots *slots = kvm_alloc_memslots(); 767 768 if (!slots) 769 goto out_err_no_arch_destroy_vm; 770 /* Generations must be different for each address space. */ 771 slots->generation = i; 772 rcu_assign_pointer(kvm->memslots[i], slots); 773 } 774 775 for (i = 0; i < KVM_NR_BUSES; i++) { 776 rcu_assign_pointer(kvm->buses[i], 777 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT)); 778 if (!kvm->buses[i]) 779 goto out_err_no_arch_destroy_vm; 780 } 781 782 kvm->max_halt_poll_ns = halt_poll_ns; 783 784 r = kvm_arch_init_vm(kvm, type); 785 if (r) 786 goto out_err_no_arch_destroy_vm; 787 788 r = hardware_enable_all(); 789 if (r) 790 goto out_err_no_disable; 791 792 #ifdef CONFIG_HAVE_KVM_IRQFD 793 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 794 #endif 795 796 r = kvm_init_mmu_notifier(kvm); 797 if (r) 798 goto out_err_no_mmu_notifier; 799 800 r = kvm_arch_post_init_vm(kvm); 801 if (r) 802 goto out_err; 803 804 mutex_lock(&kvm_lock); 805 list_add(&kvm->vm_list, &vm_list); 806 mutex_unlock(&kvm_lock); 807 808 preempt_notifier_inc(); 809 810 return kvm; 811 812 out_err: 813 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 814 if (kvm->mmu_notifier.ops) 815 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm); 816 #endif 817 out_err_no_mmu_notifier: 818 hardware_disable_all(); 819 out_err_no_disable: 820 kvm_arch_destroy_vm(kvm); 821 out_err_no_arch_destroy_vm: 822 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count)); 823 for (i = 0; i < KVM_NR_BUSES; i++) 824 kfree(kvm_get_bus(kvm, i)); 825 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 826 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); 827 cleanup_srcu_struct(&kvm->irq_srcu); 828 out_err_no_irq_srcu: 829 cleanup_srcu_struct(&kvm->srcu); 830 out_err_no_srcu: 831 kvm_arch_free_vm(kvm); 832 mmdrop(current->mm); 833 return ERR_PTR(r); 834 } 835 836 static void kvm_destroy_devices(struct kvm *kvm) 837 { 838 struct kvm_device *dev, *tmp; 839 840 /* 841 * We do not need to take the kvm->lock here, because nobody else 842 * has a reference to the struct kvm at this point and therefore 843 * cannot access the devices list anyhow. 844 */ 845 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 846 list_del(&dev->vm_node); 847 dev->ops->destroy(dev); 848 } 849 } 850 851 static void kvm_destroy_vm(struct kvm *kvm) 852 { 853 int i; 854 struct mm_struct *mm = kvm->mm; 855 856 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); 857 kvm_destroy_vm_debugfs(kvm); 858 kvm_arch_sync_events(kvm); 859 mutex_lock(&kvm_lock); 860 list_del(&kvm->vm_list); 861 mutex_unlock(&kvm_lock); 862 kvm_arch_pre_destroy_vm(kvm); 863 864 kvm_free_irq_routing(kvm); 865 for (i = 0; i < KVM_NR_BUSES; i++) { 866 struct kvm_io_bus *bus = kvm_get_bus(kvm, i); 867 868 if (bus) 869 kvm_io_bus_destroy(bus); 870 kvm->buses[i] = NULL; 871 } 872 kvm_coalesced_mmio_free(kvm); 873 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 874 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 875 #else 876 kvm_arch_flush_shadow_all(kvm); 877 #endif 878 kvm_arch_destroy_vm(kvm); 879 kvm_destroy_devices(kvm); 880 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 881 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); 882 cleanup_srcu_struct(&kvm->irq_srcu); 883 cleanup_srcu_struct(&kvm->srcu); 884 kvm_arch_free_vm(kvm); 885 preempt_notifier_dec(); 886 hardware_disable_all(); 887 mmdrop(mm); 888 } 889 890 void kvm_get_kvm(struct kvm *kvm) 891 { 892 refcount_inc(&kvm->users_count); 893 } 894 EXPORT_SYMBOL_GPL(kvm_get_kvm); 895 896 void kvm_put_kvm(struct kvm *kvm) 897 { 898 if (refcount_dec_and_test(&kvm->users_count)) 899 kvm_destroy_vm(kvm); 900 } 901 EXPORT_SYMBOL_GPL(kvm_put_kvm); 902 903 /* 904 * Used to put a reference that was taken on behalf of an object associated 905 * with a user-visible file descriptor, e.g. a vcpu or device, if installation 906 * of the new file descriptor fails and the reference cannot be transferred to 907 * its final owner. In such cases, the caller is still actively using @kvm and 908 * will fail miserably if the refcount unexpectedly hits zero. 909 */ 910 void kvm_put_kvm_no_destroy(struct kvm *kvm) 911 { 912 WARN_ON(refcount_dec_and_test(&kvm->users_count)); 913 } 914 EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy); 915 916 static int kvm_vm_release(struct inode *inode, struct file *filp) 917 { 918 struct kvm *kvm = filp->private_data; 919 920 kvm_irqfd_release(kvm); 921 922 kvm_put_kvm(kvm); 923 return 0; 924 } 925 926 /* 927 * Allocation size is twice as large as the actual dirty bitmap size. 928 * See kvm_vm_ioctl_get_dirty_log() why this is needed. 929 */ 930 static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) 931 { 932 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 933 934 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT); 935 if (!memslot->dirty_bitmap) 936 return -ENOMEM; 937 938 return 0; 939 } 940 941 /* 942 * Delete a memslot by decrementing the number of used slots and shifting all 943 * other entries in the array forward one spot. 944 */ 945 static inline void kvm_memslot_delete(struct kvm_memslots *slots, 946 struct kvm_memory_slot *memslot) 947 { 948 struct kvm_memory_slot *mslots = slots->memslots; 949 int i; 950 951 if (WARN_ON(slots->id_to_index[memslot->id] == -1)) 952 return; 953 954 slots->used_slots--; 955 956 if (atomic_read(&slots->lru_slot) >= slots->used_slots) 957 atomic_set(&slots->lru_slot, 0); 958 959 for (i = slots->id_to_index[memslot->id]; i < slots->used_slots; i++) { 960 mslots[i] = mslots[i + 1]; 961 slots->id_to_index[mslots[i].id] = i; 962 } 963 mslots[i] = *memslot; 964 slots->id_to_index[memslot->id] = -1; 965 } 966 967 /* 968 * "Insert" a new memslot by incrementing the number of used slots. Returns 969 * the new slot's initial index into the memslots array. 970 */ 971 static inline int kvm_memslot_insert_back(struct kvm_memslots *slots) 972 { 973 return slots->used_slots++; 974 } 975 976 /* 977 * Move a changed memslot backwards in the array by shifting existing slots 978 * with a higher GFN toward the front of the array. Note, the changed memslot 979 * itself is not preserved in the array, i.e. not swapped at this time, only 980 * its new index into the array is tracked. Returns the changed memslot's 981 * current index into the memslots array. 982 */ 983 static inline int kvm_memslot_move_backward(struct kvm_memslots *slots, 984 struct kvm_memory_slot *memslot) 985 { 986 struct kvm_memory_slot *mslots = slots->memslots; 987 int i; 988 989 if (WARN_ON_ONCE(slots->id_to_index[memslot->id] == -1) || 990 WARN_ON_ONCE(!slots->used_slots)) 991 return -1; 992 993 /* 994 * Move the target memslot backward in the array by shifting existing 995 * memslots with a higher GFN (than the target memslot) towards the 996 * front of the array. 997 */ 998 for (i = slots->id_to_index[memslot->id]; i < slots->used_slots - 1; i++) { 999 if (memslot->base_gfn > mslots[i + 1].base_gfn) 1000 break; 1001 1002 WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn); 1003 1004 /* Shift the next memslot forward one and update its index. */ 1005 mslots[i] = mslots[i + 1]; 1006 slots->id_to_index[mslots[i].id] = i; 1007 } 1008 return i; 1009 } 1010 1011 /* 1012 * Move a changed memslot forwards in the array by shifting existing slots with 1013 * a lower GFN toward the back of the array. Note, the changed memslot itself 1014 * is not preserved in the array, i.e. not swapped at this time, only its new 1015 * index into the array is tracked. Returns the changed memslot's final index 1016 * into the memslots array. 1017 */ 1018 static inline int kvm_memslot_move_forward(struct kvm_memslots *slots, 1019 struct kvm_memory_slot *memslot, 1020 int start) 1021 { 1022 struct kvm_memory_slot *mslots = slots->memslots; 1023 int i; 1024 1025 for (i = start; i > 0; i--) { 1026 if (memslot->base_gfn < mslots[i - 1].base_gfn) 1027 break; 1028 1029 WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn); 1030 1031 /* Shift the next memslot back one and update its index. */ 1032 mslots[i] = mslots[i - 1]; 1033 slots->id_to_index[mslots[i].id] = i; 1034 } 1035 return i; 1036 } 1037 1038 /* 1039 * Re-sort memslots based on their GFN to account for an added, deleted, or 1040 * moved memslot. Sorting memslots by GFN allows using a binary search during 1041 * memslot lookup. 1042 * 1043 * IMPORTANT: Slots are sorted from highest GFN to lowest GFN! I.e. the entry 1044 * at memslots[0] has the highest GFN. 1045 * 1046 * The sorting algorithm takes advantage of having initially sorted memslots 1047 * and knowing the position of the changed memslot. Sorting is also optimized 1048 * by not swapping the updated memslot and instead only shifting other memslots 1049 * and tracking the new index for the update memslot. Only once its final 1050 * index is known is the updated memslot copied into its position in the array. 1051 * 1052 * - When deleting a memslot, the deleted memslot simply needs to be moved to 1053 * the end of the array. 1054 * 1055 * - When creating a memslot, the algorithm "inserts" the new memslot at the 1056 * end of the array and then it forward to its correct location. 1057 * 1058 * - When moving a memslot, the algorithm first moves the updated memslot 1059 * backward to handle the scenario where the memslot's GFN was changed to a 1060 * lower value. update_memslots() then falls through and runs the same flow 1061 * as creating a memslot to move the memslot forward to handle the scenario 1062 * where its GFN was changed to a higher value. 1063 * 1064 * Note, slots are sorted from highest->lowest instead of lowest->highest for 1065 * historical reasons. Originally, invalid memslots where denoted by having 1066 * GFN=0, thus sorting from highest->lowest naturally sorted invalid memslots 1067 * to the end of the array. The current algorithm uses dedicated logic to 1068 * delete a memslot and thus does not rely on invalid memslots having GFN=0. 1069 * 1070 * The other historical motiviation for highest->lowest was to improve the 1071 * performance of memslot lookup. KVM originally used a linear search starting 1072 * at memslots[0]. On x86, the largest memslot usually has one of the highest, 1073 * if not *the* highest, GFN, as the bulk of the guest's RAM is located in a 1074 * single memslot above the 4gb boundary. As the largest memslot is also the 1075 * most likely to be referenced, sorting it to the front of the array was 1076 * advantageous. The current binary search starts from the middle of the array 1077 * and uses an LRU pointer to improve performance for all memslots and GFNs. 1078 */ 1079 static void update_memslots(struct kvm_memslots *slots, 1080 struct kvm_memory_slot *memslot, 1081 enum kvm_mr_change change) 1082 { 1083 int i; 1084 1085 if (change == KVM_MR_DELETE) { 1086 kvm_memslot_delete(slots, memslot); 1087 } else { 1088 if (change == KVM_MR_CREATE) 1089 i = kvm_memslot_insert_back(slots); 1090 else 1091 i = kvm_memslot_move_backward(slots, memslot); 1092 i = kvm_memslot_move_forward(slots, memslot, i); 1093 1094 /* 1095 * Copy the memslot to its new position in memslots and update 1096 * its index accordingly. 1097 */ 1098 slots->memslots[i] = *memslot; 1099 slots->id_to_index[memslot->id] = i; 1100 } 1101 } 1102 1103 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 1104 { 1105 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 1106 1107 #ifdef __KVM_HAVE_READONLY_MEM 1108 valid_flags |= KVM_MEM_READONLY; 1109 #endif 1110 1111 if (mem->flags & ~valid_flags) 1112 return -EINVAL; 1113 1114 return 0; 1115 } 1116 1117 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 1118 int as_id, struct kvm_memslots *slots) 1119 { 1120 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 1121 u64 gen = old_memslots->generation; 1122 1123 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); 1124 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; 1125 1126 rcu_assign_pointer(kvm->memslots[as_id], slots); 1127 synchronize_srcu_expedited(&kvm->srcu); 1128 1129 /* 1130 * Increment the new memslot generation a second time, dropping the 1131 * update in-progress flag and incrementing the generation based on 1132 * the number of address spaces. This provides a unique and easily 1133 * identifiable generation number while the memslots are in flux. 1134 */ 1135 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; 1136 1137 /* 1138 * Generations must be unique even across address spaces. We do not need 1139 * a global counter for that, instead the generation space is evenly split 1140 * across address spaces. For example, with two address spaces, address 1141 * space 0 will use generations 0, 2, 4, ... while address space 1 will 1142 * use generations 1, 3, 5, ... 1143 */ 1144 gen += KVM_ADDRESS_SPACE_NUM; 1145 1146 kvm_arch_memslots_updated(kvm, gen); 1147 1148 slots->generation = gen; 1149 1150 return old_memslots; 1151 } 1152 1153 /* 1154 * Note, at a minimum, the current number of used slots must be allocated, even 1155 * when deleting a memslot, as we need a complete duplicate of the memslots for 1156 * use when invalidating a memslot prior to deleting/moving the memslot. 1157 */ 1158 static struct kvm_memslots *kvm_dup_memslots(struct kvm_memslots *old, 1159 enum kvm_mr_change change) 1160 { 1161 struct kvm_memslots *slots; 1162 size_t old_size, new_size; 1163 1164 old_size = sizeof(struct kvm_memslots) + 1165 (sizeof(struct kvm_memory_slot) * old->used_slots); 1166 1167 if (change == KVM_MR_CREATE) 1168 new_size = old_size + sizeof(struct kvm_memory_slot); 1169 else 1170 new_size = old_size; 1171 1172 slots = kvzalloc(new_size, GFP_KERNEL_ACCOUNT); 1173 if (likely(slots)) 1174 memcpy(slots, old, old_size); 1175 1176 return slots; 1177 } 1178 1179 static int kvm_set_memslot(struct kvm *kvm, 1180 const struct kvm_userspace_memory_region *mem, 1181 struct kvm_memory_slot *old, 1182 struct kvm_memory_slot *new, int as_id, 1183 enum kvm_mr_change change) 1184 { 1185 struct kvm_memory_slot *slot; 1186 struct kvm_memslots *slots; 1187 int r; 1188 1189 slots = kvm_dup_memslots(__kvm_memslots(kvm, as_id), change); 1190 if (!slots) 1191 return -ENOMEM; 1192 1193 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { 1194 /* 1195 * Note, the INVALID flag needs to be in the appropriate entry 1196 * in the freshly allocated memslots, not in @old or @new. 1197 */ 1198 slot = id_to_memslot(slots, old->id); 1199 slot->flags |= KVM_MEMSLOT_INVALID; 1200 1201 /* 1202 * We can re-use the old memslots, the only difference from the 1203 * newly installed memslots is the invalid flag, which will get 1204 * dropped by update_memslots anyway. We'll also revert to the 1205 * old memslots if preparing the new memory region fails. 1206 */ 1207 slots = install_new_memslots(kvm, as_id, slots); 1208 1209 /* From this point no new shadow pages pointing to a deleted, 1210 * or moved, memslot will be created. 1211 * 1212 * validation of sp->gfn happens in: 1213 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 1214 * - kvm_is_visible_gfn (mmu_check_root) 1215 */ 1216 kvm_arch_flush_shadow_memslot(kvm, slot); 1217 } 1218 1219 r = kvm_arch_prepare_memory_region(kvm, new, mem, change); 1220 if (r) 1221 goto out_slots; 1222 1223 update_memslots(slots, new, change); 1224 slots = install_new_memslots(kvm, as_id, slots); 1225 1226 kvm_arch_commit_memory_region(kvm, mem, old, new, change); 1227 1228 kvfree(slots); 1229 return 0; 1230 1231 out_slots: 1232 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) 1233 slots = install_new_memslots(kvm, as_id, slots); 1234 kvfree(slots); 1235 return r; 1236 } 1237 1238 static int kvm_delete_memslot(struct kvm *kvm, 1239 const struct kvm_userspace_memory_region *mem, 1240 struct kvm_memory_slot *old, int as_id) 1241 { 1242 struct kvm_memory_slot new; 1243 int r; 1244 1245 if (!old->npages) 1246 return -EINVAL; 1247 1248 memset(&new, 0, sizeof(new)); 1249 new.id = old->id; 1250 /* 1251 * This is only for debugging purpose; it should never be referenced 1252 * for a removed memslot. 1253 */ 1254 new.as_id = as_id; 1255 1256 r = kvm_set_memslot(kvm, mem, old, &new, as_id, KVM_MR_DELETE); 1257 if (r) 1258 return r; 1259 1260 kvm_free_memslot(kvm, old); 1261 return 0; 1262 } 1263 1264 /* 1265 * Allocate some memory and give it an address in the guest physical address 1266 * space. 1267 * 1268 * Discontiguous memory is allowed, mostly for framebuffers. 1269 * 1270 * Must be called holding kvm->slots_lock for write. 1271 */ 1272 int __kvm_set_memory_region(struct kvm *kvm, 1273 const struct kvm_userspace_memory_region *mem) 1274 { 1275 struct kvm_memory_slot old, new; 1276 struct kvm_memory_slot *tmp; 1277 enum kvm_mr_change change; 1278 int as_id, id; 1279 int r; 1280 1281 r = check_memory_region_flags(mem); 1282 if (r) 1283 return r; 1284 1285 as_id = mem->slot >> 16; 1286 id = (u16)mem->slot; 1287 1288 /* General sanity checks */ 1289 if (mem->memory_size & (PAGE_SIZE - 1)) 1290 return -EINVAL; 1291 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 1292 return -EINVAL; 1293 /* We can read the guest memory with __xxx_user() later on. */ 1294 if ((mem->userspace_addr & (PAGE_SIZE - 1)) || 1295 !access_ok((void __user *)(unsigned long)mem->userspace_addr, 1296 mem->memory_size)) 1297 return -EINVAL; 1298 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 1299 return -EINVAL; 1300 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 1301 return -EINVAL; 1302 1303 /* 1304 * Make a full copy of the old memslot, the pointer will become stale 1305 * when the memslots are re-sorted by update_memslots(), and the old 1306 * memslot needs to be referenced after calling update_memslots(), e.g. 1307 * to free its resources and for arch specific behavior. 1308 */ 1309 tmp = id_to_memslot(__kvm_memslots(kvm, as_id), id); 1310 if (tmp) { 1311 old = *tmp; 1312 tmp = NULL; 1313 } else { 1314 memset(&old, 0, sizeof(old)); 1315 old.id = id; 1316 } 1317 1318 if (!mem->memory_size) 1319 return kvm_delete_memslot(kvm, mem, &old, as_id); 1320 1321 new.as_id = as_id; 1322 new.id = id; 1323 new.base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 1324 new.npages = mem->memory_size >> PAGE_SHIFT; 1325 new.flags = mem->flags; 1326 new.userspace_addr = mem->userspace_addr; 1327 1328 if (new.npages > KVM_MEM_MAX_NR_PAGES) 1329 return -EINVAL; 1330 1331 if (!old.npages) { 1332 change = KVM_MR_CREATE; 1333 new.dirty_bitmap = NULL; 1334 memset(&new.arch, 0, sizeof(new.arch)); 1335 } else { /* Modify an existing slot. */ 1336 if ((new.userspace_addr != old.userspace_addr) || 1337 (new.npages != old.npages) || 1338 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 1339 return -EINVAL; 1340 1341 if (new.base_gfn != old.base_gfn) 1342 change = KVM_MR_MOVE; 1343 else if (new.flags != old.flags) 1344 change = KVM_MR_FLAGS_ONLY; 1345 else /* Nothing to change. */ 1346 return 0; 1347 1348 /* Copy dirty_bitmap and arch from the current memslot. */ 1349 new.dirty_bitmap = old.dirty_bitmap; 1350 memcpy(&new.arch, &old.arch, sizeof(new.arch)); 1351 } 1352 1353 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 1354 /* Check for overlaps */ 1355 kvm_for_each_memslot(tmp, __kvm_memslots(kvm, as_id)) { 1356 if (tmp->id == id) 1357 continue; 1358 if (!((new.base_gfn + new.npages <= tmp->base_gfn) || 1359 (new.base_gfn >= tmp->base_gfn + tmp->npages))) 1360 return -EEXIST; 1361 } 1362 } 1363 1364 /* Allocate/free page dirty bitmap as needed */ 1365 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 1366 new.dirty_bitmap = NULL; 1367 else if (!new.dirty_bitmap && !kvm->dirty_ring_size) { 1368 r = kvm_alloc_dirty_bitmap(&new); 1369 if (r) 1370 return r; 1371 1372 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) 1373 bitmap_set(new.dirty_bitmap, 0, new.npages); 1374 } 1375 1376 r = kvm_set_memslot(kvm, mem, &old, &new, as_id, change); 1377 if (r) 1378 goto out_bitmap; 1379 1380 if (old.dirty_bitmap && !new.dirty_bitmap) 1381 kvm_destroy_dirty_bitmap(&old); 1382 return 0; 1383 1384 out_bitmap: 1385 if (new.dirty_bitmap && !old.dirty_bitmap) 1386 kvm_destroy_dirty_bitmap(&new); 1387 return r; 1388 } 1389 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 1390 1391 int kvm_set_memory_region(struct kvm *kvm, 1392 const struct kvm_userspace_memory_region *mem) 1393 { 1394 int r; 1395 1396 mutex_lock(&kvm->slots_lock); 1397 r = __kvm_set_memory_region(kvm, mem); 1398 mutex_unlock(&kvm->slots_lock); 1399 return r; 1400 } 1401 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1402 1403 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1404 struct kvm_userspace_memory_region *mem) 1405 { 1406 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1407 return -EINVAL; 1408 1409 return kvm_set_memory_region(kvm, mem); 1410 } 1411 1412 #ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1413 /** 1414 * kvm_get_dirty_log - get a snapshot of dirty pages 1415 * @kvm: pointer to kvm instance 1416 * @log: slot id and address to which we copy the log 1417 * @is_dirty: set to '1' if any dirty pages were found 1418 * @memslot: set to the associated memslot, always valid on success 1419 */ 1420 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, 1421 int *is_dirty, struct kvm_memory_slot **memslot) 1422 { 1423 struct kvm_memslots *slots; 1424 int i, as_id, id; 1425 unsigned long n; 1426 unsigned long any = 0; 1427 1428 /* Dirty ring tracking is exclusive to dirty log tracking */ 1429 if (kvm->dirty_ring_size) 1430 return -ENXIO; 1431 1432 *memslot = NULL; 1433 *is_dirty = 0; 1434 1435 as_id = log->slot >> 16; 1436 id = (u16)log->slot; 1437 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1438 return -EINVAL; 1439 1440 slots = __kvm_memslots(kvm, as_id); 1441 *memslot = id_to_memslot(slots, id); 1442 if (!(*memslot) || !(*memslot)->dirty_bitmap) 1443 return -ENOENT; 1444 1445 kvm_arch_sync_dirty_log(kvm, *memslot); 1446 1447 n = kvm_dirty_bitmap_bytes(*memslot); 1448 1449 for (i = 0; !any && i < n/sizeof(long); ++i) 1450 any = (*memslot)->dirty_bitmap[i]; 1451 1452 if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n)) 1453 return -EFAULT; 1454 1455 if (any) 1456 *is_dirty = 1; 1457 return 0; 1458 } 1459 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1460 1461 #else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ 1462 /** 1463 * kvm_get_dirty_log_protect - get a snapshot of dirty pages 1464 * and reenable dirty page tracking for the corresponding pages. 1465 * @kvm: pointer to kvm instance 1466 * @log: slot id and address to which we copy the log 1467 * 1468 * We need to keep it in mind that VCPU threads can write to the bitmap 1469 * concurrently. So, to avoid losing track of dirty pages we keep the 1470 * following order: 1471 * 1472 * 1. Take a snapshot of the bit and clear it if needed. 1473 * 2. Write protect the corresponding page. 1474 * 3. Copy the snapshot to the userspace. 1475 * 4. Upon return caller flushes TLB's if needed. 1476 * 1477 * Between 2 and 4, the guest may write to the page using the remaining TLB 1478 * entry. This is not a problem because the page is reported dirty using 1479 * the snapshot taken before and step 4 ensures that writes done after 1480 * exiting to userspace will be logged for the next call. 1481 * 1482 */ 1483 static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) 1484 { 1485 struct kvm_memslots *slots; 1486 struct kvm_memory_slot *memslot; 1487 int i, as_id, id; 1488 unsigned long n; 1489 unsigned long *dirty_bitmap; 1490 unsigned long *dirty_bitmap_buffer; 1491 bool flush; 1492 1493 /* Dirty ring tracking is exclusive to dirty log tracking */ 1494 if (kvm->dirty_ring_size) 1495 return -ENXIO; 1496 1497 as_id = log->slot >> 16; 1498 id = (u16)log->slot; 1499 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1500 return -EINVAL; 1501 1502 slots = __kvm_memslots(kvm, as_id); 1503 memslot = id_to_memslot(slots, id); 1504 if (!memslot || !memslot->dirty_bitmap) 1505 return -ENOENT; 1506 1507 dirty_bitmap = memslot->dirty_bitmap; 1508 1509 kvm_arch_sync_dirty_log(kvm, memslot); 1510 1511 n = kvm_dirty_bitmap_bytes(memslot); 1512 flush = false; 1513 if (kvm->manual_dirty_log_protect) { 1514 /* 1515 * Unlike kvm_get_dirty_log, we always return false in *flush, 1516 * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There 1517 * is some code duplication between this function and 1518 * kvm_get_dirty_log, but hopefully all architecture 1519 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log 1520 * can be eliminated. 1521 */ 1522 dirty_bitmap_buffer = dirty_bitmap; 1523 } else { 1524 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot); 1525 memset(dirty_bitmap_buffer, 0, n); 1526 1527 spin_lock(&kvm->mmu_lock); 1528 for (i = 0; i < n / sizeof(long); i++) { 1529 unsigned long mask; 1530 gfn_t offset; 1531 1532 if (!dirty_bitmap[i]) 1533 continue; 1534 1535 flush = true; 1536 mask = xchg(&dirty_bitmap[i], 0); 1537 dirty_bitmap_buffer[i] = mask; 1538 1539 offset = i * BITS_PER_LONG; 1540 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1541 offset, mask); 1542 } 1543 spin_unlock(&kvm->mmu_lock); 1544 } 1545 1546 if (flush) 1547 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 1548 1549 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1550 return -EFAULT; 1551 return 0; 1552 } 1553 1554 1555 /** 1556 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot 1557 * @kvm: kvm instance 1558 * @log: slot id and address to which we copy the log 1559 * 1560 * Steps 1-4 below provide general overview of dirty page logging. See 1561 * kvm_get_dirty_log_protect() function description for additional details. 1562 * 1563 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we 1564 * always flush the TLB (step 4) even if previous step failed and the dirty 1565 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API 1566 * does not preclude user space subsequent dirty log read. Flushing TLB ensures 1567 * writes will be marked dirty for next log read. 1568 * 1569 * 1. Take a snapshot of the bit and clear it if needed. 1570 * 2. Write protect the corresponding page. 1571 * 3. Copy the snapshot to the userspace. 1572 * 4. Flush TLB's if needed. 1573 */ 1574 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 1575 struct kvm_dirty_log *log) 1576 { 1577 int r; 1578 1579 mutex_lock(&kvm->slots_lock); 1580 1581 r = kvm_get_dirty_log_protect(kvm, log); 1582 1583 mutex_unlock(&kvm->slots_lock); 1584 return r; 1585 } 1586 1587 /** 1588 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap 1589 * and reenable dirty page tracking for the corresponding pages. 1590 * @kvm: pointer to kvm instance 1591 * @log: slot id and address from which to fetch the bitmap of dirty pages 1592 */ 1593 static int kvm_clear_dirty_log_protect(struct kvm *kvm, 1594 struct kvm_clear_dirty_log *log) 1595 { 1596 struct kvm_memslots *slots; 1597 struct kvm_memory_slot *memslot; 1598 int as_id, id; 1599 gfn_t offset; 1600 unsigned long i, n; 1601 unsigned long *dirty_bitmap; 1602 unsigned long *dirty_bitmap_buffer; 1603 bool flush; 1604 1605 /* Dirty ring tracking is exclusive to dirty log tracking */ 1606 if (kvm->dirty_ring_size) 1607 return -ENXIO; 1608 1609 as_id = log->slot >> 16; 1610 id = (u16)log->slot; 1611 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1612 return -EINVAL; 1613 1614 if (log->first_page & 63) 1615 return -EINVAL; 1616 1617 slots = __kvm_memslots(kvm, as_id); 1618 memslot = id_to_memslot(slots, id); 1619 if (!memslot || !memslot->dirty_bitmap) 1620 return -ENOENT; 1621 1622 dirty_bitmap = memslot->dirty_bitmap; 1623 1624 n = ALIGN(log->num_pages, BITS_PER_LONG) / 8; 1625 1626 if (log->first_page > memslot->npages || 1627 log->num_pages > memslot->npages - log->first_page || 1628 (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63))) 1629 return -EINVAL; 1630 1631 kvm_arch_sync_dirty_log(kvm, memslot); 1632 1633 flush = false; 1634 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot); 1635 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n)) 1636 return -EFAULT; 1637 1638 spin_lock(&kvm->mmu_lock); 1639 for (offset = log->first_page, i = offset / BITS_PER_LONG, 1640 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--; 1641 i++, offset += BITS_PER_LONG) { 1642 unsigned long mask = *dirty_bitmap_buffer++; 1643 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i]; 1644 if (!mask) 1645 continue; 1646 1647 mask &= atomic_long_fetch_andnot(mask, p); 1648 1649 /* 1650 * mask contains the bits that really have been cleared. This 1651 * never includes any bits beyond the length of the memslot (if 1652 * the length is not aligned to 64 pages), therefore it is not 1653 * a problem if userspace sets them in log->dirty_bitmap. 1654 */ 1655 if (mask) { 1656 flush = true; 1657 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1658 offset, mask); 1659 } 1660 } 1661 spin_unlock(&kvm->mmu_lock); 1662 1663 if (flush) 1664 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 1665 1666 return 0; 1667 } 1668 1669 static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, 1670 struct kvm_clear_dirty_log *log) 1671 { 1672 int r; 1673 1674 mutex_lock(&kvm->slots_lock); 1675 1676 r = kvm_clear_dirty_log_protect(kvm, log); 1677 1678 mutex_unlock(&kvm->slots_lock); 1679 return r; 1680 } 1681 #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ 1682 1683 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1684 { 1685 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1686 } 1687 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1688 1689 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1690 { 1691 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1692 } 1693 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot); 1694 1695 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1696 { 1697 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1698 1699 return kvm_is_visible_memslot(memslot); 1700 } 1701 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1702 1703 bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1704 { 1705 struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1706 1707 return kvm_is_visible_memslot(memslot); 1708 } 1709 EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn); 1710 1711 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn) 1712 { 1713 struct vm_area_struct *vma; 1714 unsigned long addr, size; 1715 1716 size = PAGE_SIZE; 1717 1718 addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL); 1719 if (kvm_is_error_hva(addr)) 1720 return PAGE_SIZE; 1721 1722 mmap_read_lock(current->mm); 1723 vma = find_vma(current->mm, addr); 1724 if (!vma) 1725 goto out; 1726 1727 size = vma_kernel_pagesize(vma); 1728 1729 out: 1730 mmap_read_unlock(current->mm); 1731 1732 return size; 1733 } 1734 1735 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1736 { 1737 return slot->flags & KVM_MEM_READONLY; 1738 } 1739 1740 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1741 gfn_t *nr_pages, bool write) 1742 { 1743 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1744 return KVM_HVA_ERR_BAD; 1745 1746 if (memslot_is_readonly(slot) && write) 1747 return KVM_HVA_ERR_RO_BAD; 1748 1749 if (nr_pages) 1750 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1751 1752 return __gfn_to_hva_memslot(slot, gfn); 1753 } 1754 1755 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1756 gfn_t *nr_pages) 1757 { 1758 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1759 } 1760 1761 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1762 gfn_t gfn) 1763 { 1764 return gfn_to_hva_many(slot, gfn, NULL); 1765 } 1766 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1767 1768 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1769 { 1770 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1771 } 1772 EXPORT_SYMBOL_GPL(gfn_to_hva); 1773 1774 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1775 { 1776 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1777 } 1778 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1779 1780 /* 1781 * Return the hva of a @gfn and the R/W attribute if possible. 1782 * 1783 * @slot: the kvm_memory_slot which contains @gfn 1784 * @gfn: the gfn to be translated 1785 * @writable: used to return the read/write attribute of the @slot if the hva 1786 * is valid and @writable is not NULL 1787 */ 1788 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1789 gfn_t gfn, bool *writable) 1790 { 1791 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1792 1793 if (!kvm_is_error_hva(hva) && writable) 1794 *writable = !memslot_is_readonly(slot); 1795 1796 return hva; 1797 } 1798 1799 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1800 { 1801 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1802 1803 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1804 } 1805 1806 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1807 { 1808 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1809 1810 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1811 } 1812 1813 static inline int check_user_page_hwpoison(unsigned long addr) 1814 { 1815 int rc, flags = FOLL_HWPOISON | FOLL_WRITE; 1816 1817 rc = get_user_pages(addr, 1, flags, NULL, NULL); 1818 return rc == -EHWPOISON; 1819 } 1820 1821 /* 1822 * The fast path to get the writable pfn which will be stored in @pfn, 1823 * true indicates success, otherwise false is returned. It's also the 1824 * only part that runs if we can in atomic context. 1825 */ 1826 static bool hva_to_pfn_fast(unsigned long addr, bool write_fault, 1827 bool *writable, kvm_pfn_t *pfn) 1828 { 1829 struct page *page[1]; 1830 1831 /* 1832 * Fast pin a writable pfn only if it is a write fault request 1833 * or the caller allows to map a writable pfn for a read fault 1834 * request. 1835 */ 1836 if (!(write_fault || writable)) 1837 return false; 1838 1839 if (get_user_page_fast_only(addr, FOLL_WRITE, page)) { 1840 *pfn = page_to_pfn(page[0]); 1841 1842 if (writable) 1843 *writable = true; 1844 return true; 1845 } 1846 1847 return false; 1848 } 1849 1850 /* 1851 * The slow path to get the pfn of the specified host virtual address, 1852 * 1 indicates success, -errno is returned if error is detected. 1853 */ 1854 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1855 bool *writable, kvm_pfn_t *pfn) 1856 { 1857 unsigned int flags = FOLL_HWPOISON; 1858 struct page *page; 1859 int npages = 0; 1860 1861 might_sleep(); 1862 1863 if (writable) 1864 *writable = write_fault; 1865 1866 if (write_fault) 1867 flags |= FOLL_WRITE; 1868 if (async) 1869 flags |= FOLL_NOWAIT; 1870 1871 npages = get_user_pages_unlocked(addr, 1, &page, flags); 1872 if (npages != 1) 1873 return npages; 1874 1875 /* map read fault as writable if possible */ 1876 if (unlikely(!write_fault) && writable) { 1877 struct page *wpage; 1878 1879 if (get_user_page_fast_only(addr, FOLL_WRITE, &wpage)) { 1880 *writable = true; 1881 put_page(page); 1882 page = wpage; 1883 } 1884 } 1885 *pfn = page_to_pfn(page); 1886 return npages; 1887 } 1888 1889 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1890 { 1891 if (unlikely(!(vma->vm_flags & VM_READ))) 1892 return false; 1893 1894 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1895 return false; 1896 1897 return true; 1898 } 1899 1900 static int hva_to_pfn_remapped(struct vm_area_struct *vma, 1901 unsigned long addr, bool *async, 1902 bool write_fault, bool *writable, 1903 kvm_pfn_t *p_pfn) 1904 { 1905 unsigned long pfn; 1906 int r; 1907 1908 r = follow_pfn(vma, addr, &pfn); 1909 if (r) { 1910 /* 1911 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does 1912 * not call the fault handler, so do it here. 1913 */ 1914 bool unlocked = false; 1915 r = fixup_user_fault(current->mm, addr, 1916 (write_fault ? FAULT_FLAG_WRITE : 0), 1917 &unlocked); 1918 if (unlocked) 1919 return -EAGAIN; 1920 if (r) 1921 return r; 1922 1923 r = follow_pfn(vma, addr, &pfn); 1924 if (r) 1925 return r; 1926 1927 } 1928 1929 if (writable) 1930 *writable = true; 1931 1932 /* 1933 * Get a reference here because callers of *hva_to_pfn* and 1934 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the 1935 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP 1936 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will 1937 * simply do nothing for reserved pfns. 1938 * 1939 * Whoever called remap_pfn_range is also going to call e.g. 1940 * unmap_mapping_range before the underlying pages are freed, 1941 * causing a call to our MMU notifier. 1942 */ 1943 kvm_get_pfn(pfn); 1944 1945 *p_pfn = pfn; 1946 return 0; 1947 } 1948 1949 /* 1950 * Pin guest page in memory and return its pfn. 1951 * @addr: host virtual address which maps memory to the guest 1952 * @atomic: whether this function can sleep 1953 * @async: whether this function need to wait IO complete if the 1954 * host page is not in the memory 1955 * @write_fault: whether we should get a writable host page 1956 * @writable: whether it allows to map a writable host page for !@write_fault 1957 * 1958 * The function will map a writable host page for these two cases: 1959 * 1): @write_fault = true 1960 * 2): @write_fault = false && @writable, @writable will tell the caller 1961 * whether the mapping is writable. 1962 */ 1963 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1964 bool write_fault, bool *writable) 1965 { 1966 struct vm_area_struct *vma; 1967 kvm_pfn_t pfn = 0; 1968 int npages, r; 1969 1970 /* we can do it either atomically or asynchronously, not both */ 1971 BUG_ON(atomic && async); 1972 1973 if (hva_to_pfn_fast(addr, write_fault, writable, &pfn)) 1974 return pfn; 1975 1976 if (atomic) 1977 return KVM_PFN_ERR_FAULT; 1978 1979 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1980 if (npages == 1) 1981 return pfn; 1982 1983 mmap_read_lock(current->mm); 1984 if (npages == -EHWPOISON || 1985 (!async && check_user_page_hwpoison(addr))) { 1986 pfn = KVM_PFN_ERR_HWPOISON; 1987 goto exit; 1988 } 1989 1990 retry: 1991 vma = find_vma_intersection(current->mm, addr, addr + 1); 1992 1993 if (vma == NULL) 1994 pfn = KVM_PFN_ERR_FAULT; 1995 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { 1996 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn); 1997 if (r == -EAGAIN) 1998 goto retry; 1999 if (r < 0) 2000 pfn = KVM_PFN_ERR_FAULT; 2001 } else { 2002 if (async && vma_is_valid(vma, write_fault)) 2003 *async = true; 2004 pfn = KVM_PFN_ERR_FAULT; 2005 } 2006 exit: 2007 mmap_read_unlock(current->mm); 2008 return pfn; 2009 } 2010 2011 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 2012 bool atomic, bool *async, bool write_fault, 2013 bool *writable) 2014 { 2015 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 2016 2017 if (addr == KVM_HVA_ERR_RO_BAD) { 2018 if (writable) 2019 *writable = false; 2020 return KVM_PFN_ERR_RO_FAULT; 2021 } 2022 2023 if (kvm_is_error_hva(addr)) { 2024 if (writable) 2025 *writable = false; 2026 return KVM_PFN_NOSLOT; 2027 } 2028 2029 /* Do not map writable pfn in the readonly memslot. */ 2030 if (writable && memslot_is_readonly(slot)) { 2031 *writable = false; 2032 writable = NULL; 2033 } 2034 2035 return hva_to_pfn(addr, atomic, async, write_fault, 2036 writable); 2037 } 2038 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 2039 2040 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 2041 bool *writable) 2042 { 2043 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 2044 write_fault, writable); 2045 } 2046 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 2047 2048 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 2049 { 2050 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 2051 } 2052 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 2053 2054 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 2055 { 2056 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 2057 } 2058 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 2059 2060 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 2061 { 2062 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 2063 } 2064 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 2065 2066 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 2067 { 2068 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 2069 } 2070 EXPORT_SYMBOL_GPL(gfn_to_pfn); 2071 2072 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 2073 { 2074 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 2075 } 2076 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 2077 2078 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 2079 struct page **pages, int nr_pages) 2080 { 2081 unsigned long addr; 2082 gfn_t entry = 0; 2083 2084 addr = gfn_to_hva_many(slot, gfn, &entry); 2085 if (kvm_is_error_hva(addr)) 2086 return -1; 2087 2088 if (entry < nr_pages) 2089 return 0; 2090 2091 return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages); 2092 } 2093 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 2094 2095 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 2096 { 2097 if (is_error_noslot_pfn(pfn)) 2098 return KVM_ERR_PTR_BAD_PAGE; 2099 2100 if (kvm_is_reserved_pfn(pfn)) { 2101 WARN_ON(1); 2102 return KVM_ERR_PTR_BAD_PAGE; 2103 } 2104 2105 return pfn_to_page(pfn); 2106 } 2107 2108 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 2109 { 2110 kvm_pfn_t pfn; 2111 2112 pfn = gfn_to_pfn(kvm, gfn); 2113 2114 return kvm_pfn_to_page(pfn); 2115 } 2116 EXPORT_SYMBOL_GPL(gfn_to_page); 2117 2118 void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache) 2119 { 2120 if (pfn == 0) 2121 return; 2122 2123 if (cache) 2124 cache->pfn = cache->gfn = 0; 2125 2126 if (dirty) 2127 kvm_release_pfn_dirty(pfn); 2128 else 2129 kvm_release_pfn_clean(pfn); 2130 } 2131 2132 static void kvm_cache_gfn_to_pfn(struct kvm_memory_slot *slot, gfn_t gfn, 2133 struct gfn_to_pfn_cache *cache, u64 gen) 2134 { 2135 kvm_release_pfn(cache->pfn, cache->dirty, cache); 2136 2137 cache->pfn = gfn_to_pfn_memslot(slot, gfn); 2138 cache->gfn = gfn; 2139 cache->dirty = false; 2140 cache->generation = gen; 2141 } 2142 2143 static int __kvm_map_gfn(struct kvm_memslots *slots, gfn_t gfn, 2144 struct kvm_host_map *map, 2145 struct gfn_to_pfn_cache *cache, 2146 bool atomic) 2147 { 2148 kvm_pfn_t pfn; 2149 void *hva = NULL; 2150 struct page *page = KVM_UNMAPPED_PAGE; 2151 struct kvm_memory_slot *slot = __gfn_to_memslot(slots, gfn); 2152 u64 gen = slots->generation; 2153 2154 if (!map) 2155 return -EINVAL; 2156 2157 if (cache) { 2158 if (!cache->pfn || cache->gfn != gfn || 2159 cache->generation != gen) { 2160 if (atomic) 2161 return -EAGAIN; 2162 kvm_cache_gfn_to_pfn(slot, gfn, cache, gen); 2163 } 2164 pfn = cache->pfn; 2165 } else { 2166 if (atomic) 2167 return -EAGAIN; 2168 pfn = gfn_to_pfn_memslot(slot, gfn); 2169 } 2170 if (is_error_noslot_pfn(pfn)) 2171 return -EINVAL; 2172 2173 if (pfn_valid(pfn)) { 2174 page = pfn_to_page(pfn); 2175 if (atomic) 2176 hva = kmap_atomic(page); 2177 else 2178 hva = kmap(page); 2179 #ifdef CONFIG_HAS_IOMEM 2180 } else if (!atomic) { 2181 hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB); 2182 } else { 2183 return -EINVAL; 2184 #endif 2185 } 2186 2187 if (!hva) 2188 return -EFAULT; 2189 2190 map->page = page; 2191 map->hva = hva; 2192 map->pfn = pfn; 2193 map->gfn = gfn; 2194 2195 return 0; 2196 } 2197 2198 int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, 2199 struct gfn_to_pfn_cache *cache, bool atomic) 2200 { 2201 return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map, 2202 cache, atomic); 2203 } 2204 EXPORT_SYMBOL_GPL(kvm_map_gfn); 2205 2206 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map) 2207 { 2208 return __kvm_map_gfn(kvm_vcpu_memslots(vcpu), gfn, map, 2209 NULL, false); 2210 } 2211 EXPORT_SYMBOL_GPL(kvm_vcpu_map); 2212 2213 static void __kvm_unmap_gfn(struct kvm *kvm, 2214 struct kvm_memory_slot *memslot, 2215 struct kvm_host_map *map, 2216 struct gfn_to_pfn_cache *cache, 2217 bool dirty, bool atomic) 2218 { 2219 if (!map) 2220 return; 2221 2222 if (!map->hva) 2223 return; 2224 2225 if (map->page != KVM_UNMAPPED_PAGE) { 2226 if (atomic) 2227 kunmap_atomic(map->hva); 2228 else 2229 kunmap(map->page); 2230 } 2231 #ifdef CONFIG_HAS_IOMEM 2232 else if (!atomic) 2233 memunmap(map->hva); 2234 else 2235 WARN_ONCE(1, "Unexpected unmapping in atomic context"); 2236 #endif 2237 2238 if (dirty) 2239 mark_page_dirty_in_slot(kvm, memslot, map->gfn); 2240 2241 if (cache) 2242 cache->dirty |= dirty; 2243 else 2244 kvm_release_pfn(map->pfn, dirty, NULL); 2245 2246 map->hva = NULL; 2247 map->page = NULL; 2248 } 2249 2250 int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, 2251 struct gfn_to_pfn_cache *cache, bool dirty, bool atomic) 2252 { 2253 __kvm_unmap_gfn(vcpu->kvm, gfn_to_memslot(vcpu->kvm, map->gfn), map, 2254 cache, dirty, atomic); 2255 return 0; 2256 } 2257 EXPORT_SYMBOL_GPL(kvm_unmap_gfn); 2258 2259 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty) 2260 { 2261 __kvm_unmap_gfn(vcpu->kvm, kvm_vcpu_gfn_to_memslot(vcpu, map->gfn), 2262 map, NULL, dirty, false); 2263 } 2264 EXPORT_SYMBOL_GPL(kvm_vcpu_unmap); 2265 2266 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 2267 { 2268 kvm_pfn_t pfn; 2269 2270 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 2271 2272 return kvm_pfn_to_page(pfn); 2273 } 2274 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 2275 2276 void kvm_release_page_clean(struct page *page) 2277 { 2278 WARN_ON(is_error_page(page)); 2279 2280 kvm_release_pfn_clean(page_to_pfn(page)); 2281 } 2282 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 2283 2284 void kvm_release_pfn_clean(kvm_pfn_t pfn) 2285 { 2286 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 2287 put_page(pfn_to_page(pfn)); 2288 } 2289 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 2290 2291 void kvm_release_page_dirty(struct page *page) 2292 { 2293 WARN_ON(is_error_page(page)); 2294 2295 kvm_release_pfn_dirty(page_to_pfn(page)); 2296 } 2297 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 2298 2299 void kvm_release_pfn_dirty(kvm_pfn_t pfn) 2300 { 2301 kvm_set_pfn_dirty(pfn); 2302 kvm_release_pfn_clean(pfn); 2303 } 2304 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 2305 2306 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 2307 { 2308 if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn)) 2309 SetPageDirty(pfn_to_page(pfn)); 2310 } 2311 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 2312 2313 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 2314 { 2315 if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn)) 2316 mark_page_accessed(pfn_to_page(pfn)); 2317 } 2318 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 2319 2320 void kvm_get_pfn(kvm_pfn_t pfn) 2321 { 2322 if (!kvm_is_reserved_pfn(pfn)) 2323 get_page(pfn_to_page(pfn)); 2324 } 2325 EXPORT_SYMBOL_GPL(kvm_get_pfn); 2326 2327 static int next_segment(unsigned long len, int offset) 2328 { 2329 if (len > PAGE_SIZE - offset) 2330 return PAGE_SIZE - offset; 2331 else 2332 return len; 2333 } 2334 2335 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 2336 void *data, int offset, int len) 2337 { 2338 int r; 2339 unsigned long addr; 2340 2341 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 2342 if (kvm_is_error_hva(addr)) 2343 return -EFAULT; 2344 r = __copy_from_user(data, (void __user *)addr + offset, len); 2345 if (r) 2346 return -EFAULT; 2347 return 0; 2348 } 2349 2350 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 2351 int len) 2352 { 2353 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 2354 2355 return __kvm_read_guest_page(slot, gfn, data, offset, len); 2356 } 2357 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 2358 2359 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 2360 int offset, int len) 2361 { 2362 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2363 2364 return __kvm_read_guest_page(slot, gfn, data, offset, len); 2365 } 2366 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 2367 2368 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 2369 { 2370 gfn_t gfn = gpa >> PAGE_SHIFT; 2371 int seg; 2372 int offset = offset_in_page(gpa); 2373 int ret; 2374 2375 while ((seg = next_segment(len, offset)) != 0) { 2376 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 2377 if (ret < 0) 2378 return ret; 2379 offset = 0; 2380 len -= seg; 2381 data += seg; 2382 ++gfn; 2383 } 2384 return 0; 2385 } 2386 EXPORT_SYMBOL_GPL(kvm_read_guest); 2387 2388 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 2389 { 2390 gfn_t gfn = gpa >> PAGE_SHIFT; 2391 int seg; 2392 int offset = offset_in_page(gpa); 2393 int ret; 2394 2395 while ((seg = next_segment(len, offset)) != 0) { 2396 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 2397 if (ret < 0) 2398 return ret; 2399 offset = 0; 2400 len -= seg; 2401 data += seg; 2402 ++gfn; 2403 } 2404 return 0; 2405 } 2406 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 2407 2408 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 2409 void *data, int offset, unsigned long len) 2410 { 2411 int r; 2412 unsigned long addr; 2413 2414 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 2415 if (kvm_is_error_hva(addr)) 2416 return -EFAULT; 2417 pagefault_disable(); 2418 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 2419 pagefault_enable(); 2420 if (r) 2421 return -EFAULT; 2422 return 0; 2423 } 2424 2425 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 2426 void *data, unsigned long len) 2427 { 2428 gfn_t gfn = gpa >> PAGE_SHIFT; 2429 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2430 int offset = offset_in_page(gpa); 2431 2432 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 2433 } 2434 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 2435 2436 static int __kvm_write_guest_page(struct kvm *kvm, 2437 struct kvm_memory_slot *memslot, gfn_t gfn, 2438 const void *data, int offset, int len) 2439 { 2440 int r; 2441 unsigned long addr; 2442 2443 addr = gfn_to_hva_memslot(memslot, gfn); 2444 if (kvm_is_error_hva(addr)) 2445 return -EFAULT; 2446 r = __copy_to_user((void __user *)addr + offset, data, len); 2447 if (r) 2448 return -EFAULT; 2449 mark_page_dirty_in_slot(kvm, memslot, gfn); 2450 return 0; 2451 } 2452 2453 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 2454 const void *data, int offset, int len) 2455 { 2456 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 2457 2458 return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len); 2459 } 2460 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 2461 2462 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 2463 const void *data, int offset, int len) 2464 { 2465 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2466 2467 return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len); 2468 } 2469 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 2470 2471 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 2472 unsigned long len) 2473 { 2474 gfn_t gfn = gpa >> PAGE_SHIFT; 2475 int seg; 2476 int offset = offset_in_page(gpa); 2477 int ret; 2478 2479 while ((seg = next_segment(len, offset)) != 0) { 2480 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 2481 if (ret < 0) 2482 return ret; 2483 offset = 0; 2484 len -= seg; 2485 data += seg; 2486 ++gfn; 2487 } 2488 return 0; 2489 } 2490 EXPORT_SYMBOL_GPL(kvm_write_guest); 2491 2492 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 2493 unsigned long len) 2494 { 2495 gfn_t gfn = gpa >> PAGE_SHIFT; 2496 int seg; 2497 int offset = offset_in_page(gpa); 2498 int ret; 2499 2500 while ((seg = next_segment(len, offset)) != 0) { 2501 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 2502 if (ret < 0) 2503 return ret; 2504 offset = 0; 2505 len -= seg; 2506 data += seg; 2507 ++gfn; 2508 } 2509 return 0; 2510 } 2511 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 2512 2513 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots, 2514 struct gfn_to_hva_cache *ghc, 2515 gpa_t gpa, unsigned long len) 2516 { 2517 int offset = offset_in_page(gpa); 2518 gfn_t start_gfn = gpa >> PAGE_SHIFT; 2519 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 2520 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 2521 gfn_t nr_pages_avail; 2522 2523 /* Update ghc->generation before performing any error checks. */ 2524 ghc->generation = slots->generation; 2525 2526 if (start_gfn > end_gfn) { 2527 ghc->hva = KVM_HVA_ERR_BAD; 2528 return -EINVAL; 2529 } 2530 2531 /* 2532 * If the requested region crosses two memslots, we still 2533 * verify that the entire region is valid here. 2534 */ 2535 for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) { 2536 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 2537 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 2538 &nr_pages_avail); 2539 if (kvm_is_error_hva(ghc->hva)) 2540 return -EFAULT; 2541 } 2542 2543 /* Use the slow path for cross page reads and writes. */ 2544 if (nr_pages_needed == 1) 2545 ghc->hva += offset; 2546 else 2547 ghc->memslot = NULL; 2548 2549 ghc->gpa = gpa; 2550 ghc->len = len; 2551 return 0; 2552 } 2553 2554 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 2555 gpa_t gpa, unsigned long len) 2556 { 2557 struct kvm_memslots *slots = kvm_memslots(kvm); 2558 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len); 2559 } 2560 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 2561 2562 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 2563 void *data, unsigned int offset, 2564 unsigned long len) 2565 { 2566 struct kvm_memslots *slots = kvm_memslots(kvm); 2567 int r; 2568 gpa_t gpa = ghc->gpa + offset; 2569 2570 BUG_ON(len + offset > ghc->len); 2571 2572 if (slots->generation != ghc->generation) { 2573 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) 2574 return -EFAULT; 2575 } 2576 2577 if (kvm_is_error_hva(ghc->hva)) 2578 return -EFAULT; 2579 2580 if (unlikely(!ghc->memslot)) 2581 return kvm_write_guest(kvm, gpa, data, len); 2582 2583 r = __copy_to_user((void __user *)ghc->hva + offset, data, len); 2584 if (r) 2585 return -EFAULT; 2586 mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT); 2587 2588 return 0; 2589 } 2590 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached); 2591 2592 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 2593 void *data, unsigned long len) 2594 { 2595 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); 2596 } 2597 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 2598 2599 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 2600 void *data, unsigned int offset, 2601 unsigned long len) 2602 { 2603 struct kvm_memslots *slots = kvm_memslots(kvm); 2604 int r; 2605 gpa_t gpa = ghc->gpa + offset; 2606 2607 BUG_ON(len + offset > ghc->len); 2608 2609 if (slots->generation != ghc->generation) { 2610 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) 2611 return -EFAULT; 2612 } 2613 2614 if (kvm_is_error_hva(ghc->hva)) 2615 return -EFAULT; 2616 2617 if (unlikely(!ghc->memslot)) 2618 return kvm_read_guest(kvm, gpa, data, len); 2619 2620 r = __copy_from_user(data, (void __user *)ghc->hva + offset, len); 2621 if (r) 2622 return -EFAULT; 2623 2624 return 0; 2625 } 2626 EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached); 2627 2628 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 2629 void *data, unsigned long len) 2630 { 2631 return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len); 2632 } 2633 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 2634 2635 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 2636 { 2637 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 2638 gfn_t gfn = gpa >> PAGE_SHIFT; 2639 int seg; 2640 int offset = offset_in_page(gpa); 2641 int ret; 2642 2643 while ((seg = next_segment(len, offset)) != 0) { 2644 ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 2645 if (ret < 0) 2646 return ret; 2647 offset = 0; 2648 len -= seg; 2649 ++gfn; 2650 } 2651 return 0; 2652 } 2653 EXPORT_SYMBOL_GPL(kvm_clear_guest); 2654 2655 void mark_page_dirty_in_slot(struct kvm *kvm, 2656 struct kvm_memory_slot *memslot, 2657 gfn_t gfn) 2658 { 2659 if (memslot && kvm_slot_dirty_track_enabled(memslot)) { 2660 unsigned long rel_gfn = gfn - memslot->base_gfn; 2661 u32 slot = (memslot->as_id << 16) | memslot->id; 2662 2663 if (kvm->dirty_ring_size) 2664 kvm_dirty_ring_push(kvm_dirty_ring_get(kvm), 2665 slot, rel_gfn); 2666 else 2667 set_bit_le(rel_gfn, memslot->dirty_bitmap); 2668 } 2669 } 2670 EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot); 2671 2672 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 2673 { 2674 struct kvm_memory_slot *memslot; 2675 2676 memslot = gfn_to_memslot(kvm, gfn); 2677 mark_page_dirty_in_slot(kvm, memslot, gfn); 2678 } 2679 EXPORT_SYMBOL_GPL(mark_page_dirty); 2680 2681 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 2682 { 2683 struct kvm_memory_slot *memslot; 2684 2685 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2686 mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn); 2687 } 2688 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 2689 2690 void kvm_sigset_activate(struct kvm_vcpu *vcpu) 2691 { 2692 if (!vcpu->sigset_active) 2693 return; 2694 2695 /* 2696 * This does a lockless modification of ->real_blocked, which is fine 2697 * because, only current can change ->real_blocked and all readers of 2698 * ->real_blocked don't care as long ->real_blocked is always a subset 2699 * of ->blocked. 2700 */ 2701 sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked); 2702 } 2703 2704 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu) 2705 { 2706 if (!vcpu->sigset_active) 2707 return; 2708 2709 sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL); 2710 sigemptyset(¤t->real_blocked); 2711 } 2712 2713 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 2714 { 2715 unsigned int old, val, grow, grow_start; 2716 2717 old = val = vcpu->halt_poll_ns; 2718 grow_start = READ_ONCE(halt_poll_ns_grow_start); 2719 grow = READ_ONCE(halt_poll_ns_grow); 2720 if (!grow) 2721 goto out; 2722 2723 val *= grow; 2724 if (val < grow_start) 2725 val = grow_start; 2726 2727 if (val > halt_poll_ns) 2728 val = halt_poll_ns; 2729 2730 vcpu->halt_poll_ns = val; 2731 out: 2732 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 2733 } 2734 2735 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 2736 { 2737 unsigned int old, val, shrink; 2738 2739 old = val = vcpu->halt_poll_ns; 2740 shrink = READ_ONCE(halt_poll_ns_shrink); 2741 if (shrink == 0) 2742 val = 0; 2743 else 2744 val /= shrink; 2745 2746 vcpu->halt_poll_ns = val; 2747 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 2748 } 2749 2750 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 2751 { 2752 int ret = -EINTR; 2753 int idx = srcu_read_lock(&vcpu->kvm->srcu); 2754 2755 if (kvm_arch_vcpu_runnable(vcpu)) { 2756 kvm_make_request(KVM_REQ_UNHALT, vcpu); 2757 goto out; 2758 } 2759 if (kvm_cpu_has_pending_timer(vcpu)) 2760 goto out; 2761 if (signal_pending(current)) 2762 goto out; 2763 2764 ret = 0; 2765 out: 2766 srcu_read_unlock(&vcpu->kvm->srcu, idx); 2767 return ret; 2768 } 2769 2770 static inline void 2771 update_halt_poll_stats(struct kvm_vcpu *vcpu, u64 poll_ns, bool waited) 2772 { 2773 if (waited) 2774 vcpu->stat.halt_poll_fail_ns += poll_ns; 2775 else 2776 vcpu->stat.halt_poll_success_ns += poll_ns; 2777 } 2778 2779 /* 2780 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2781 */ 2782 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2783 { 2784 ktime_t start, cur, poll_end; 2785 bool waited = false; 2786 u64 block_ns; 2787 2788 kvm_arch_vcpu_blocking(vcpu); 2789 2790 start = cur = poll_end = ktime_get(); 2791 if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) { 2792 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2793 2794 ++vcpu->stat.halt_attempted_poll; 2795 do { 2796 /* 2797 * This sets KVM_REQ_UNHALT if an interrupt 2798 * arrives. 2799 */ 2800 if (kvm_vcpu_check_block(vcpu) < 0) { 2801 ++vcpu->stat.halt_successful_poll; 2802 if (!vcpu_valid_wakeup(vcpu)) 2803 ++vcpu->stat.halt_poll_invalid; 2804 goto out; 2805 } 2806 poll_end = cur = ktime_get(); 2807 } while (single_task_running() && ktime_before(cur, stop)); 2808 } 2809 2810 prepare_to_rcuwait(&vcpu->wait); 2811 for (;;) { 2812 set_current_state(TASK_INTERRUPTIBLE); 2813 2814 if (kvm_vcpu_check_block(vcpu) < 0) 2815 break; 2816 2817 waited = true; 2818 schedule(); 2819 } 2820 finish_rcuwait(&vcpu->wait); 2821 cur = ktime_get(); 2822 out: 2823 kvm_arch_vcpu_unblocking(vcpu); 2824 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2825 2826 update_halt_poll_stats( 2827 vcpu, ktime_to_ns(ktime_sub(poll_end, start)), waited); 2828 2829 if (!kvm_arch_no_poll(vcpu)) { 2830 if (!vcpu_valid_wakeup(vcpu)) { 2831 shrink_halt_poll_ns(vcpu); 2832 } else if (vcpu->kvm->max_halt_poll_ns) { 2833 if (block_ns <= vcpu->halt_poll_ns) 2834 ; 2835 /* we had a long block, shrink polling */ 2836 else if (vcpu->halt_poll_ns && 2837 block_ns > vcpu->kvm->max_halt_poll_ns) 2838 shrink_halt_poll_ns(vcpu); 2839 /* we had a short halt and our poll time is too small */ 2840 else if (vcpu->halt_poll_ns < vcpu->kvm->max_halt_poll_ns && 2841 block_ns < vcpu->kvm->max_halt_poll_ns) 2842 grow_halt_poll_ns(vcpu); 2843 } else { 2844 vcpu->halt_poll_ns = 0; 2845 } 2846 } 2847 2848 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu)); 2849 kvm_arch_vcpu_block_finish(vcpu); 2850 } 2851 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2852 2853 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 2854 { 2855 struct rcuwait *waitp; 2856 2857 waitp = kvm_arch_vcpu_get_wait(vcpu); 2858 if (rcuwait_wake_up(waitp)) { 2859 WRITE_ONCE(vcpu->ready, true); 2860 ++vcpu->stat.halt_wakeup; 2861 return true; 2862 } 2863 2864 return false; 2865 } 2866 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 2867 2868 #ifndef CONFIG_S390 2869 /* 2870 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2871 */ 2872 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2873 { 2874 int me; 2875 int cpu = vcpu->cpu; 2876 2877 if (kvm_vcpu_wake_up(vcpu)) 2878 return; 2879 2880 me = get_cpu(); 2881 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2882 if (kvm_arch_vcpu_should_kick(vcpu)) 2883 smp_send_reschedule(cpu); 2884 put_cpu(); 2885 } 2886 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2887 #endif /* !CONFIG_S390 */ 2888 2889 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2890 { 2891 struct pid *pid; 2892 struct task_struct *task = NULL; 2893 int ret = 0; 2894 2895 rcu_read_lock(); 2896 pid = rcu_dereference(target->pid); 2897 if (pid) 2898 task = get_pid_task(pid, PIDTYPE_PID); 2899 rcu_read_unlock(); 2900 if (!task) 2901 return ret; 2902 ret = yield_to(task, 1); 2903 put_task_struct(task); 2904 2905 return ret; 2906 } 2907 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2908 2909 /* 2910 * Helper that checks whether a VCPU is eligible for directed yield. 2911 * Most eligible candidate to yield is decided by following heuristics: 2912 * 2913 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2914 * (preempted lock holder), indicated by @in_spin_loop. 2915 * Set at the beginning and cleared at the end of interception/PLE handler. 2916 * 2917 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2918 * chance last time (mostly it has become eligible now since we have probably 2919 * yielded to lockholder in last iteration. This is done by toggling 2920 * @dy_eligible each time a VCPU checked for eligibility.) 2921 * 2922 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2923 * to preempted lock-holder could result in wrong VCPU selection and CPU 2924 * burning. Giving priority for a potential lock-holder increases lock 2925 * progress. 2926 * 2927 * Since algorithm is based on heuristics, accessing another VCPU data without 2928 * locking does not harm. It may result in trying to yield to same VCPU, fail 2929 * and continue with next VCPU and so on. 2930 */ 2931 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2932 { 2933 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2934 bool eligible; 2935 2936 eligible = !vcpu->spin_loop.in_spin_loop || 2937 vcpu->spin_loop.dy_eligible; 2938 2939 if (vcpu->spin_loop.in_spin_loop) 2940 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2941 2942 return eligible; 2943 #else 2944 return true; 2945 #endif 2946 } 2947 2948 /* 2949 * Unlike kvm_arch_vcpu_runnable, this function is called outside 2950 * a vcpu_load/vcpu_put pair. However, for most architectures 2951 * kvm_arch_vcpu_runnable does not require vcpu_load. 2952 */ 2953 bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) 2954 { 2955 return kvm_arch_vcpu_runnable(vcpu); 2956 } 2957 2958 static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu) 2959 { 2960 if (kvm_arch_dy_runnable(vcpu)) 2961 return true; 2962 2963 #ifdef CONFIG_KVM_ASYNC_PF 2964 if (!list_empty_careful(&vcpu->async_pf.done)) 2965 return true; 2966 #endif 2967 2968 return false; 2969 } 2970 2971 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) 2972 { 2973 struct kvm *kvm = me->kvm; 2974 struct kvm_vcpu *vcpu; 2975 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2976 int yielded = 0; 2977 int try = 3; 2978 int pass; 2979 int i; 2980 2981 kvm_vcpu_set_in_spin_loop(me, true); 2982 /* 2983 * We boost the priority of a VCPU that is runnable but not 2984 * currently running, because it got preempted by something 2985 * else and called schedule in __vcpu_run. Hopefully that 2986 * VCPU is holding the lock that we need and will release it. 2987 * We approximate round-robin by starting at the last boosted VCPU. 2988 */ 2989 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2990 kvm_for_each_vcpu(i, vcpu, kvm) { 2991 if (!pass && i <= last_boosted_vcpu) { 2992 i = last_boosted_vcpu; 2993 continue; 2994 } else if (pass && i > last_boosted_vcpu) 2995 break; 2996 if (!READ_ONCE(vcpu->ready)) 2997 continue; 2998 if (vcpu == me) 2999 continue; 3000 if (rcuwait_active(&vcpu->wait) && 3001 !vcpu_dy_runnable(vcpu)) 3002 continue; 3003 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode && 3004 !kvm_arch_vcpu_in_kernel(vcpu)) 3005 continue; 3006 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 3007 continue; 3008 3009 yielded = kvm_vcpu_yield_to(vcpu); 3010 if (yielded > 0) { 3011 kvm->last_boosted_vcpu = i; 3012 break; 3013 } else if (yielded < 0) { 3014 try--; 3015 if (!try) 3016 break; 3017 } 3018 } 3019 } 3020 kvm_vcpu_set_in_spin_loop(me, false); 3021 3022 /* Ensure vcpu is not eligible during next spinloop */ 3023 kvm_vcpu_set_dy_eligible(me, false); 3024 } 3025 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 3026 3027 static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff) 3028 { 3029 #if KVM_DIRTY_LOG_PAGE_OFFSET > 0 3030 return (pgoff >= KVM_DIRTY_LOG_PAGE_OFFSET) && 3031 (pgoff < KVM_DIRTY_LOG_PAGE_OFFSET + 3032 kvm->dirty_ring_size / PAGE_SIZE); 3033 #else 3034 return false; 3035 #endif 3036 } 3037 3038 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf) 3039 { 3040 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data; 3041 struct page *page; 3042 3043 if (vmf->pgoff == 0) 3044 page = virt_to_page(vcpu->run); 3045 #ifdef CONFIG_X86 3046 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 3047 page = virt_to_page(vcpu->arch.pio_data); 3048 #endif 3049 #ifdef CONFIG_KVM_MMIO 3050 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 3051 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 3052 #endif 3053 else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff)) 3054 page = kvm_dirty_ring_get_page( 3055 &vcpu->dirty_ring, 3056 vmf->pgoff - KVM_DIRTY_LOG_PAGE_OFFSET); 3057 else 3058 return kvm_arch_vcpu_fault(vcpu, vmf); 3059 get_page(page); 3060 vmf->page = page; 3061 return 0; 3062 } 3063 3064 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 3065 .fault = kvm_vcpu_fault, 3066 }; 3067 3068 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 3069 { 3070 struct kvm_vcpu *vcpu = file->private_data; 3071 unsigned long pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 3072 3073 if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) || 3074 kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) && 3075 ((vma->vm_flags & VM_EXEC) || !(vma->vm_flags & VM_SHARED))) 3076 return -EINVAL; 3077 3078 vma->vm_ops = &kvm_vcpu_vm_ops; 3079 return 0; 3080 } 3081 3082 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 3083 { 3084 struct kvm_vcpu *vcpu = filp->private_data; 3085 3086 kvm_put_kvm(vcpu->kvm); 3087 return 0; 3088 } 3089 3090 static struct file_operations kvm_vcpu_fops = { 3091 .release = kvm_vcpu_release, 3092 .unlocked_ioctl = kvm_vcpu_ioctl, 3093 .mmap = kvm_vcpu_mmap, 3094 .llseek = noop_llseek, 3095 KVM_COMPAT(kvm_vcpu_compat_ioctl), 3096 }; 3097 3098 /* 3099 * Allocates an inode for the vcpu. 3100 */ 3101 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 3102 { 3103 char name[8 + 1 + ITOA_MAX_LEN + 1]; 3104 3105 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); 3106 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 3107 } 3108 3109 static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 3110 { 3111 #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS 3112 struct dentry *debugfs_dentry; 3113 char dir_name[ITOA_MAX_LEN * 2]; 3114 3115 if (!debugfs_initialized()) 3116 return; 3117 3118 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); 3119 debugfs_dentry = debugfs_create_dir(dir_name, 3120 vcpu->kvm->debugfs_dentry); 3121 3122 kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry); 3123 #endif 3124 } 3125 3126 /* 3127 * Creates some virtual cpus. Good luck creating more than one. 3128 */ 3129 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 3130 { 3131 int r; 3132 struct kvm_vcpu *vcpu; 3133 struct page *page; 3134 3135 if (id >= KVM_MAX_VCPU_ID) 3136 return -EINVAL; 3137 3138 mutex_lock(&kvm->lock); 3139 if (kvm->created_vcpus == KVM_MAX_VCPUS) { 3140 mutex_unlock(&kvm->lock); 3141 return -EINVAL; 3142 } 3143 3144 kvm->created_vcpus++; 3145 mutex_unlock(&kvm->lock); 3146 3147 r = kvm_arch_vcpu_precreate(kvm, id); 3148 if (r) 3149 goto vcpu_decrement; 3150 3151 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 3152 if (!vcpu) { 3153 r = -ENOMEM; 3154 goto vcpu_decrement; 3155 } 3156 3157 BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE); 3158 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 3159 if (!page) { 3160 r = -ENOMEM; 3161 goto vcpu_free; 3162 } 3163 vcpu->run = page_address(page); 3164 3165 kvm_vcpu_init(vcpu, kvm, id); 3166 3167 r = kvm_arch_vcpu_create(vcpu); 3168 if (r) 3169 goto vcpu_free_run_page; 3170 3171 if (kvm->dirty_ring_size) { 3172 r = kvm_dirty_ring_alloc(&vcpu->dirty_ring, 3173 id, kvm->dirty_ring_size); 3174 if (r) 3175 goto arch_vcpu_destroy; 3176 } 3177 3178 mutex_lock(&kvm->lock); 3179 if (kvm_get_vcpu_by_id(kvm, id)) { 3180 r = -EEXIST; 3181 goto unlock_vcpu_destroy; 3182 } 3183 3184 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); 3185 BUG_ON(kvm->vcpus[vcpu->vcpu_idx]); 3186 3187 /* Now it's all set up, let userspace reach it */ 3188 kvm_get_kvm(kvm); 3189 r = create_vcpu_fd(vcpu); 3190 if (r < 0) { 3191 kvm_put_kvm_no_destroy(kvm); 3192 goto unlock_vcpu_destroy; 3193 } 3194 3195 kvm->vcpus[vcpu->vcpu_idx] = vcpu; 3196 3197 /* 3198 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 3199 * before kvm->online_vcpu's incremented value. 3200 */ 3201 smp_wmb(); 3202 atomic_inc(&kvm->online_vcpus); 3203 3204 mutex_unlock(&kvm->lock); 3205 kvm_arch_vcpu_postcreate(vcpu); 3206 kvm_create_vcpu_debugfs(vcpu); 3207 return r; 3208 3209 unlock_vcpu_destroy: 3210 mutex_unlock(&kvm->lock); 3211 kvm_dirty_ring_free(&vcpu->dirty_ring); 3212 arch_vcpu_destroy: 3213 kvm_arch_vcpu_destroy(vcpu); 3214 vcpu_free_run_page: 3215 free_page((unsigned long)vcpu->run); 3216 vcpu_free: 3217 kmem_cache_free(kvm_vcpu_cache, vcpu); 3218 vcpu_decrement: 3219 mutex_lock(&kvm->lock); 3220 kvm->created_vcpus--; 3221 mutex_unlock(&kvm->lock); 3222 return r; 3223 } 3224 3225 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 3226 { 3227 if (sigset) { 3228 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 3229 vcpu->sigset_active = 1; 3230 vcpu->sigset = *sigset; 3231 } else 3232 vcpu->sigset_active = 0; 3233 return 0; 3234 } 3235 3236 static long kvm_vcpu_ioctl(struct file *filp, 3237 unsigned int ioctl, unsigned long arg) 3238 { 3239 struct kvm_vcpu *vcpu = filp->private_data; 3240 void __user *argp = (void __user *)arg; 3241 int r; 3242 struct kvm_fpu *fpu = NULL; 3243 struct kvm_sregs *kvm_sregs = NULL; 3244 3245 if (vcpu->kvm->mm != current->mm) 3246 return -EIO; 3247 3248 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 3249 return -EINVAL; 3250 3251 /* 3252 * Some architectures have vcpu ioctls that are asynchronous to vcpu 3253 * execution; mutex_lock() would break them. 3254 */ 3255 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg); 3256 if (r != -ENOIOCTLCMD) 3257 return r; 3258 3259 if (mutex_lock_killable(&vcpu->mutex)) 3260 return -EINTR; 3261 switch (ioctl) { 3262 case KVM_RUN: { 3263 struct pid *oldpid; 3264 r = -EINVAL; 3265 if (arg) 3266 goto out; 3267 oldpid = rcu_access_pointer(vcpu->pid); 3268 if (unlikely(oldpid != task_pid(current))) { 3269 /* The thread running this VCPU changed. */ 3270 struct pid *newpid; 3271 3272 r = kvm_arch_vcpu_run_pid_change(vcpu); 3273 if (r) 3274 break; 3275 3276 newpid = get_task_pid(current, PIDTYPE_PID); 3277 rcu_assign_pointer(vcpu->pid, newpid); 3278 if (oldpid) 3279 synchronize_rcu(); 3280 put_pid(oldpid); 3281 } 3282 r = kvm_arch_vcpu_ioctl_run(vcpu); 3283 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 3284 break; 3285 } 3286 case KVM_GET_REGS: { 3287 struct kvm_regs *kvm_regs; 3288 3289 r = -ENOMEM; 3290 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT); 3291 if (!kvm_regs) 3292 goto out; 3293 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 3294 if (r) 3295 goto out_free1; 3296 r = -EFAULT; 3297 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 3298 goto out_free1; 3299 r = 0; 3300 out_free1: 3301 kfree(kvm_regs); 3302 break; 3303 } 3304 case KVM_SET_REGS: { 3305 struct kvm_regs *kvm_regs; 3306 3307 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 3308 if (IS_ERR(kvm_regs)) { 3309 r = PTR_ERR(kvm_regs); 3310 goto out; 3311 } 3312 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 3313 kfree(kvm_regs); 3314 break; 3315 } 3316 case KVM_GET_SREGS: { 3317 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), 3318 GFP_KERNEL_ACCOUNT); 3319 r = -ENOMEM; 3320 if (!kvm_sregs) 3321 goto out; 3322 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 3323 if (r) 3324 goto out; 3325 r = -EFAULT; 3326 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 3327 goto out; 3328 r = 0; 3329 break; 3330 } 3331 case KVM_SET_SREGS: { 3332 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 3333 if (IS_ERR(kvm_sregs)) { 3334 r = PTR_ERR(kvm_sregs); 3335 kvm_sregs = NULL; 3336 goto out; 3337 } 3338 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 3339 break; 3340 } 3341 case KVM_GET_MP_STATE: { 3342 struct kvm_mp_state mp_state; 3343 3344 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 3345 if (r) 3346 goto out; 3347 r = -EFAULT; 3348 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 3349 goto out; 3350 r = 0; 3351 break; 3352 } 3353 case KVM_SET_MP_STATE: { 3354 struct kvm_mp_state mp_state; 3355 3356 r = -EFAULT; 3357 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 3358 goto out; 3359 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 3360 break; 3361 } 3362 case KVM_TRANSLATE: { 3363 struct kvm_translation tr; 3364 3365 r = -EFAULT; 3366 if (copy_from_user(&tr, argp, sizeof(tr))) 3367 goto out; 3368 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 3369 if (r) 3370 goto out; 3371 r = -EFAULT; 3372 if (copy_to_user(argp, &tr, sizeof(tr))) 3373 goto out; 3374 r = 0; 3375 break; 3376 } 3377 case KVM_SET_GUEST_DEBUG: { 3378 struct kvm_guest_debug dbg; 3379 3380 r = -EFAULT; 3381 if (copy_from_user(&dbg, argp, sizeof(dbg))) 3382 goto out; 3383 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 3384 break; 3385 } 3386 case KVM_SET_SIGNAL_MASK: { 3387 struct kvm_signal_mask __user *sigmask_arg = argp; 3388 struct kvm_signal_mask kvm_sigmask; 3389 sigset_t sigset, *p; 3390 3391 p = NULL; 3392 if (argp) { 3393 r = -EFAULT; 3394 if (copy_from_user(&kvm_sigmask, argp, 3395 sizeof(kvm_sigmask))) 3396 goto out; 3397 r = -EINVAL; 3398 if (kvm_sigmask.len != sizeof(sigset)) 3399 goto out; 3400 r = -EFAULT; 3401 if (copy_from_user(&sigset, sigmask_arg->sigset, 3402 sizeof(sigset))) 3403 goto out; 3404 p = &sigset; 3405 } 3406 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 3407 break; 3408 } 3409 case KVM_GET_FPU: { 3410 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT); 3411 r = -ENOMEM; 3412 if (!fpu) 3413 goto out; 3414 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 3415 if (r) 3416 goto out; 3417 r = -EFAULT; 3418 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 3419 goto out; 3420 r = 0; 3421 break; 3422 } 3423 case KVM_SET_FPU: { 3424 fpu = memdup_user(argp, sizeof(*fpu)); 3425 if (IS_ERR(fpu)) { 3426 r = PTR_ERR(fpu); 3427 fpu = NULL; 3428 goto out; 3429 } 3430 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 3431 break; 3432 } 3433 default: 3434 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 3435 } 3436 out: 3437 mutex_unlock(&vcpu->mutex); 3438 kfree(fpu); 3439 kfree(kvm_sregs); 3440 return r; 3441 } 3442 3443 #ifdef CONFIG_KVM_COMPAT 3444 static long kvm_vcpu_compat_ioctl(struct file *filp, 3445 unsigned int ioctl, unsigned long arg) 3446 { 3447 struct kvm_vcpu *vcpu = filp->private_data; 3448 void __user *argp = compat_ptr(arg); 3449 int r; 3450 3451 if (vcpu->kvm->mm != current->mm) 3452 return -EIO; 3453 3454 switch (ioctl) { 3455 case KVM_SET_SIGNAL_MASK: { 3456 struct kvm_signal_mask __user *sigmask_arg = argp; 3457 struct kvm_signal_mask kvm_sigmask; 3458 sigset_t sigset; 3459 3460 if (argp) { 3461 r = -EFAULT; 3462 if (copy_from_user(&kvm_sigmask, argp, 3463 sizeof(kvm_sigmask))) 3464 goto out; 3465 r = -EINVAL; 3466 if (kvm_sigmask.len != sizeof(compat_sigset_t)) 3467 goto out; 3468 r = -EFAULT; 3469 if (get_compat_sigset(&sigset, 3470 (compat_sigset_t __user *)sigmask_arg->sigset)) 3471 goto out; 3472 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 3473 } else 3474 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 3475 break; 3476 } 3477 default: 3478 r = kvm_vcpu_ioctl(filp, ioctl, arg); 3479 } 3480 3481 out: 3482 return r; 3483 } 3484 #endif 3485 3486 static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma) 3487 { 3488 struct kvm_device *dev = filp->private_data; 3489 3490 if (dev->ops->mmap) 3491 return dev->ops->mmap(dev, vma); 3492 3493 return -ENODEV; 3494 } 3495 3496 static int kvm_device_ioctl_attr(struct kvm_device *dev, 3497 int (*accessor)(struct kvm_device *dev, 3498 struct kvm_device_attr *attr), 3499 unsigned long arg) 3500 { 3501 struct kvm_device_attr attr; 3502 3503 if (!accessor) 3504 return -EPERM; 3505 3506 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 3507 return -EFAULT; 3508 3509 return accessor(dev, &attr); 3510 } 3511 3512 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 3513 unsigned long arg) 3514 { 3515 struct kvm_device *dev = filp->private_data; 3516 3517 if (dev->kvm->mm != current->mm) 3518 return -EIO; 3519 3520 switch (ioctl) { 3521 case KVM_SET_DEVICE_ATTR: 3522 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 3523 case KVM_GET_DEVICE_ATTR: 3524 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 3525 case KVM_HAS_DEVICE_ATTR: 3526 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 3527 default: 3528 if (dev->ops->ioctl) 3529 return dev->ops->ioctl(dev, ioctl, arg); 3530 3531 return -ENOTTY; 3532 } 3533 } 3534 3535 static int kvm_device_release(struct inode *inode, struct file *filp) 3536 { 3537 struct kvm_device *dev = filp->private_data; 3538 struct kvm *kvm = dev->kvm; 3539 3540 if (dev->ops->release) { 3541 mutex_lock(&kvm->lock); 3542 list_del(&dev->vm_node); 3543 dev->ops->release(dev); 3544 mutex_unlock(&kvm->lock); 3545 } 3546 3547 kvm_put_kvm(kvm); 3548 return 0; 3549 } 3550 3551 static const struct file_operations kvm_device_fops = { 3552 .unlocked_ioctl = kvm_device_ioctl, 3553 .release = kvm_device_release, 3554 KVM_COMPAT(kvm_device_ioctl), 3555 .mmap = kvm_device_mmap, 3556 }; 3557 3558 struct kvm_device *kvm_device_from_filp(struct file *filp) 3559 { 3560 if (filp->f_op != &kvm_device_fops) 3561 return NULL; 3562 3563 return filp->private_data; 3564 } 3565 3566 static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 3567 #ifdef CONFIG_KVM_MPIC 3568 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 3569 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 3570 #endif 3571 }; 3572 3573 int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type) 3574 { 3575 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 3576 return -ENOSPC; 3577 3578 if (kvm_device_ops_table[type] != NULL) 3579 return -EEXIST; 3580 3581 kvm_device_ops_table[type] = ops; 3582 return 0; 3583 } 3584 3585 void kvm_unregister_device_ops(u32 type) 3586 { 3587 if (kvm_device_ops_table[type] != NULL) 3588 kvm_device_ops_table[type] = NULL; 3589 } 3590 3591 static int kvm_ioctl_create_device(struct kvm *kvm, 3592 struct kvm_create_device *cd) 3593 { 3594 const struct kvm_device_ops *ops = NULL; 3595 struct kvm_device *dev; 3596 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 3597 int type; 3598 int ret; 3599 3600 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 3601 return -ENODEV; 3602 3603 type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table)); 3604 ops = kvm_device_ops_table[type]; 3605 if (ops == NULL) 3606 return -ENODEV; 3607 3608 if (test) 3609 return 0; 3610 3611 dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT); 3612 if (!dev) 3613 return -ENOMEM; 3614 3615 dev->ops = ops; 3616 dev->kvm = kvm; 3617 3618 mutex_lock(&kvm->lock); 3619 ret = ops->create(dev, type); 3620 if (ret < 0) { 3621 mutex_unlock(&kvm->lock); 3622 kfree(dev); 3623 return ret; 3624 } 3625 list_add(&dev->vm_node, &kvm->devices); 3626 mutex_unlock(&kvm->lock); 3627 3628 if (ops->init) 3629 ops->init(dev); 3630 3631 kvm_get_kvm(kvm); 3632 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 3633 if (ret < 0) { 3634 kvm_put_kvm_no_destroy(kvm); 3635 mutex_lock(&kvm->lock); 3636 list_del(&dev->vm_node); 3637 mutex_unlock(&kvm->lock); 3638 ops->destroy(dev); 3639 return ret; 3640 } 3641 3642 cd->fd = ret; 3643 return 0; 3644 } 3645 3646 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 3647 { 3648 switch (arg) { 3649 case KVM_CAP_USER_MEMORY: 3650 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 3651 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 3652 case KVM_CAP_INTERNAL_ERROR_DATA: 3653 #ifdef CONFIG_HAVE_KVM_MSI 3654 case KVM_CAP_SIGNAL_MSI: 3655 #endif 3656 #ifdef CONFIG_HAVE_KVM_IRQFD 3657 case KVM_CAP_IRQFD: 3658 case KVM_CAP_IRQFD_RESAMPLE: 3659 #endif 3660 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 3661 case KVM_CAP_CHECK_EXTENSION_VM: 3662 case KVM_CAP_ENABLE_CAP_VM: 3663 case KVM_CAP_HALT_POLL: 3664 return 1; 3665 #ifdef CONFIG_KVM_MMIO 3666 case KVM_CAP_COALESCED_MMIO: 3667 return KVM_COALESCED_MMIO_PAGE_OFFSET; 3668 case KVM_CAP_COALESCED_PIO: 3669 return 1; 3670 #endif 3671 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 3672 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: 3673 return KVM_DIRTY_LOG_MANUAL_CAPS; 3674 #endif 3675 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 3676 case KVM_CAP_IRQ_ROUTING: 3677 return KVM_MAX_IRQ_ROUTES; 3678 #endif 3679 #if KVM_ADDRESS_SPACE_NUM > 1 3680 case KVM_CAP_MULTI_ADDRESS_SPACE: 3681 return KVM_ADDRESS_SPACE_NUM; 3682 #endif 3683 case KVM_CAP_NR_MEMSLOTS: 3684 return KVM_USER_MEM_SLOTS; 3685 case KVM_CAP_DIRTY_LOG_RING: 3686 #if KVM_DIRTY_LOG_PAGE_OFFSET > 0 3687 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn); 3688 #else 3689 return 0; 3690 #endif 3691 default: 3692 break; 3693 } 3694 return kvm_vm_ioctl_check_extension(kvm, arg); 3695 } 3696 3697 static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size) 3698 { 3699 int r; 3700 3701 if (!KVM_DIRTY_LOG_PAGE_OFFSET) 3702 return -EINVAL; 3703 3704 /* the size should be power of 2 */ 3705 if (!size || (size & (size - 1))) 3706 return -EINVAL; 3707 3708 /* Should be bigger to keep the reserved entries, or a page */ 3709 if (size < kvm_dirty_ring_get_rsvd_entries() * 3710 sizeof(struct kvm_dirty_gfn) || size < PAGE_SIZE) 3711 return -EINVAL; 3712 3713 if (size > KVM_DIRTY_RING_MAX_ENTRIES * 3714 sizeof(struct kvm_dirty_gfn)) 3715 return -E2BIG; 3716 3717 /* We only allow it to set once */ 3718 if (kvm->dirty_ring_size) 3719 return -EINVAL; 3720 3721 mutex_lock(&kvm->lock); 3722 3723 if (kvm->created_vcpus) { 3724 /* We don't allow to change this value after vcpu created */ 3725 r = -EINVAL; 3726 } else { 3727 kvm->dirty_ring_size = size; 3728 r = 0; 3729 } 3730 3731 mutex_unlock(&kvm->lock); 3732 return r; 3733 } 3734 3735 static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm) 3736 { 3737 int i; 3738 struct kvm_vcpu *vcpu; 3739 int cleared = 0; 3740 3741 if (!kvm->dirty_ring_size) 3742 return -EINVAL; 3743 3744 mutex_lock(&kvm->slots_lock); 3745 3746 kvm_for_each_vcpu(i, vcpu, kvm) 3747 cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring); 3748 3749 mutex_unlock(&kvm->slots_lock); 3750 3751 if (cleared) 3752 kvm_flush_remote_tlbs(kvm); 3753 3754 return cleared; 3755 } 3756 3757 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm, 3758 struct kvm_enable_cap *cap) 3759 { 3760 return -EINVAL; 3761 } 3762 3763 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, 3764 struct kvm_enable_cap *cap) 3765 { 3766 switch (cap->cap) { 3767 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 3768 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: { 3769 u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE; 3770 3771 if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE) 3772 allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS; 3773 3774 if (cap->flags || (cap->args[0] & ~allowed_options)) 3775 return -EINVAL; 3776 kvm->manual_dirty_log_protect = cap->args[0]; 3777 return 0; 3778 } 3779 #endif 3780 case KVM_CAP_HALT_POLL: { 3781 if (cap->flags || cap->args[0] != (unsigned int)cap->args[0]) 3782 return -EINVAL; 3783 3784 kvm->max_halt_poll_ns = cap->args[0]; 3785 return 0; 3786 } 3787 case KVM_CAP_DIRTY_LOG_RING: 3788 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]); 3789 default: 3790 return kvm_vm_ioctl_enable_cap(kvm, cap); 3791 } 3792 } 3793 3794 static long kvm_vm_ioctl(struct file *filp, 3795 unsigned int ioctl, unsigned long arg) 3796 { 3797 struct kvm *kvm = filp->private_data; 3798 void __user *argp = (void __user *)arg; 3799 int r; 3800 3801 if (kvm->mm != current->mm) 3802 return -EIO; 3803 switch (ioctl) { 3804 case KVM_CREATE_VCPU: 3805 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 3806 break; 3807 case KVM_ENABLE_CAP: { 3808 struct kvm_enable_cap cap; 3809 3810 r = -EFAULT; 3811 if (copy_from_user(&cap, argp, sizeof(cap))) 3812 goto out; 3813 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap); 3814 break; 3815 } 3816 case KVM_SET_USER_MEMORY_REGION: { 3817 struct kvm_userspace_memory_region kvm_userspace_mem; 3818 3819 r = -EFAULT; 3820 if (copy_from_user(&kvm_userspace_mem, argp, 3821 sizeof(kvm_userspace_mem))) 3822 goto out; 3823 3824 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 3825 break; 3826 } 3827 case KVM_GET_DIRTY_LOG: { 3828 struct kvm_dirty_log log; 3829 3830 r = -EFAULT; 3831 if (copy_from_user(&log, argp, sizeof(log))) 3832 goto out; 3833 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 3834 break; 3835 } 3836 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 3837 case KVM_CLEAR_DIRTY_LOG: { 3838 struct kvm_clear_dirty_log log; 3839 3840 r = -EFAULT; 3841 if (copy_from_user(&log, argp, sizeof(log))) 3842 goto out; 3843 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); 3844 break; 3845 } 3846 #endif 3847 #ifdef CONFIG_KVM_MMIO 3848 case KVM_REGISTER_COALESCED_MMIO: { 3849 struct kvm_coalesced_mmio_zone zone; 3850 3851 r = -EFAULT; 3852 if (copy_from_user(&zone, argp, sizeof(zone))) 3853 goto out; 3854 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 3855 break; 3856 } 3857 case KVM_UNREGISTER_COALESCED_MMIO: { 3858 struct kvm_coalesced_mmio_zone zone; 3859 3860 r = -EFAULT; 3861 if (copy_from_user(&zone, argp, sizeof(zone))) 3862 goto out; 3863 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 3864 break; 3865 } 3866 #endif 3867 case KVM_IRQFD: { 3868 struct kvm_irqfd data; 3869 3870 r = -EFAULT; 3871 if (copy_from_user(&data, argp, sizeof(data))) 3872 goto out; 3873 r = kvm_irqfd(kvm, &data); 3874 break; 3875 } 3876 case KVM_IOEVENTFD: { 3877 struct kvm_ioeventfd data; 3878 3879 r = -EFAULT; 3880 if (copy_from_user(&data, argp, sizeof(data))) 3881 goto out; 3882 r = kvm_ioeventfd(kvm, &data); 3883 break; 3884 } 3885 #ifdef CONFIG_HAVE_KVM_MSI 3886 case KVM_SIGNAL_MSI: { 3887 struct kvm_msi msi; 3888 3889 r = -EFAULT; 3890 if (copy_from_user(&msi, argp, sizeof(msi))) 3891 goto out; 3892 r = kvm_send_userspace_msi(kvm, &msi); 3893 break; 3894 } 3895 #endif 3896 #ifdef __KVM_HAVE_IRQ_LINE 3897 case KVM_IRQ_LINE_STATUS: 3898 case KVM_IRQ_LINE: { 3899 struct kvm_irq_level irq_event; 3900 3901 r = -EFAULT; 3902 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 3903 goto out; 3904 3905 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 3906 ioctl == KVM_IRQ_LINE_STATUS); 3907 if (r) 3908 goto out; 3909 3910 r = -EFAULT; 3911 if (ioctl == KVM_IRQ_LINE_STATUS) { 3912 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 3913 goto out; 3914 } 3915 3916 r = 0; 3917 break; 3918 } 3919 #endif 3920 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 3921 case KVM_SET_GSI_ROUTING: { 3922 struct kvm_irq_routing routing; 3923 struct kvm_irq_routing __user *urouting; 3924 struct kvm_irq_routing_entry *entries = NULL; 3925 3926 r = -EFAULT; 3927 if (copy_from_user(&routing, argp, sizeof(routing))) 3928 goto out; 3929 r = -EINVAL; 3930 if (!kvm_arch_can_set_irq_routing(kvm)) 3931 goto out; 3932 if (routing.nr > KVM_MAX_IRQ_ROUTES) 3933 goto out; 3934 if (routing.flags) 3935 goto out; 3936 if (routing.nr) { 3937 urouting = argp; 3938 entries = vmemdup_user(urouting->entries, 3939 array_size(sizeof(*entries), 3940 routing.nr)); 3941 if (IS_ERR(entries)) { 3942 r = PTR_ERR(entries); 3943 goto out; 3944 } 3945 } 3946 r = kvm_set_irq_routing(kvm, entries, routing.nr, 3947 routing.flags); 3948 kvfree(entries); 3949 break; 3950 } 3951 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 3952 case KVM_CREATE_DEVICE: { 3953 struct kvm_create_device cd; 3954 3955 r = -EFAULT; 3956 if (copy_from_user(&cd, argp, sizeof(cd))) 3957 goto out; 3958 3959 r = kvm_ioctl_create_device(kvm, &cd); 3960 if (r) 3961 goto out; 3962 3963 r = -EFAULT; 3964 if (copy_to_user(argp, &cd, sizeof(cd))) 3965 goto out; 3966 3967 r = 0; 3968 break; 3969 } 3970 case KVM_CHECK_EXTENSION: 3971 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 3972 break; 3973 case KVM_RESET_DIRTY_RINGS: 3974 r = kvm_vm_ioctl_reset_dirty_pages(kvm); 3975 break; 3976 default: 3977 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 3978 } 3979 out: 3980 return r; 3981 } 3982 3983 #ifdef CONFIG_KVM_COMPAT 3984 struct compat_kvm_dirty_log { 3985 __u32 slot; 3986 __u32 padding1; 3987 union { 3988 compat_uptr_t dirty_bitmap; /* one bit per page */ 3989 __u64 padding2; 3990 }; 3991 }; 3992 3993 static long kvm_vm_compat_ioctl(struct file *filp, 3994 unsigned int ioctl, unsigned long arg) 3995 { 3996 struct kvm *kvm = filp->private_data; 3997 int r; 3998 3999 if (kvm->mm != current->mm) 4000 return -EIO; 4001 switch (ioctl) { 4002 case KVM_GET_DIRTY_LOG: { 4003 struct compat_kvm_dirty_log compat_log; 4004 struct kvm_dirty_log log; 4005 4006 if (copy_from_user(&compat_log, (void __user *)arg, 4007 sizeof(compat_log))) 4008 return -EFAULT; 4009 log.slot = compat_log.slot; 4010 log.padding1 = compat_log.padding1; 4011 log.padding2 = compat_log.padding2; 4012 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 4013 4014 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 4015 break; 4016 } 4017 default: 4018 r = kvm_vm_ioctl(filp, ioctl, arg); 4019 } 4020 return r; 4021 } 4022 #endif 4023 4024 static struct file_operations kvm_vm_fops = { 4025 .release = kvm_vm_release, 4026 .unlocked_ioctl = kvm_vm_ioctl, 4027 .llseek = noop_llseek, 4028 KVM_COMPAT(kvm_vm_compat_ioctl), 4029 }; 4030 4031 static int kvm_dev_ioctl_create_vm(unsigned long type) 4032 { 4033 int r; 4034 struct kvm *kvm; 4035 struct file *file; 4036 4037 kvm = kvm_create_vm(type); 4038 if (IS_ERR(kvm)) 4039 return PTR_ERR(kvm); 4040 #ifdef CONFIG_KVM_MMIO 4041 r = kvm_coalesced_mmio_init(kvm); 4042 if (r < 0) 4043 goto put_kvm; 4044 #endif 4045 r = get_unused_fd_flags(O_CLOEXEC); 4046 if (r < 0) 4047 goto put_kvm; 4048 4049 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 4050 if (IS_ERR(file)) { 4051 put_unused_fd(r); 4052 r = PTR_ERR(file); 4053 goto put_kvm; 4054 } 4055 4056 /* 4057 * Don't call kvm_put_kvm anymore at this point; file->f_op is 4058 * already set, with ->release() being kvm_vm_release(). In error 4059 * cases it will be called by the final fput(file) and will take 4060 * care of doing kvm_put_kvm(kvm). 4061 */ 4062 if (kvm_create_vm_debugfs(kvm, r) < 0) { 4063 put_unused_fd(r); 4064 fput(file); 4065 return -ENOMEM; 4066 } 4067 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); 4068 4069 fd_install(r, file); 4070 return r; 4071 4072 put_kvm: 4073 kvm_put_kvm(kvm); 4074 return r; 4075 } 4076 4077 static long kvm_dev_ioctl(struct file *filp, 4078 unsigned int ioctl, unsigned long arg) 4079 { 4080 long r = -EINVAL; 4081 4082 switch (ioctl) { 4083 case KVM_GET_API_VERSION: 4084 if (arg) 4085 goto out; 4086 r = KVM_API_VERSION; 4087 break; 4088 case KVM_CREATE_VM: 4089 r = kvm_dev_ioctl_create_vm(arg); 4090 break; 4091 case KVM_CHECK_EXTENSION: 4092 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 4093 break; 4094 case KVM_GET_VCPU_MMAP_SIZE: 4095 if (arg) 4096 goto out; 4097 r = PAGE_SIZE; /* struct kvm_run */ 4098 #ifdef CONFIG_X86 4099 r += PAGE_SIZE; /* pio data page */ 4100 #endif 4101 #ifdef CONFIG_KVM_MMIO 4102 r += PAGE_SIZE; /* coalesced mmio ring page */ 4103 #endif 4104 break; 4105 case KVM_TRACE_ENABLE: 4106 case KVM_TRACE_PAUSE: 4107 case KVM_TRACE_DISABLE: 4108 r = -EOPNOTSUPP; 4109 break; 4110 default: 4111 return kvm_arch_dev_ioctl(filp, ioctl, arg); 4112 } 4113 out: 4114 return r; 4115 } 4116 4117 static struct file_operations kvm_chardev_ops = { 4118 .unlocked_ioctl = kvm_dev_ioctl, 4119 .llseek = noop_llseek, 4120 KVM_COMPAT(kvm_dev_ioctl), 4121 }; 4122 4123 static struct miscdevice kvm_dev = { 4124 KVM_MINOR, 4125 "kvm", 4126 &kvm_chardev_ops, 4127 }; 4128 4129 static void hardware_enable_nolock(void *junk) 4130 { 4131 int cpu = raw_smp_processor_id(); 4132 int r; 4133 4134 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 4135 return; 4136 4137 cpumask_set_cpu(cpu, cpus_hardware_enabled); 4138 4139 r = kvm_arch_hardware_enable(); 4140 4141 if (r) { 4142 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 4143 atomic_inc(&hardware_enable_failed); 4144 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 4145 } 4146 } 4147 4148 static int kvm_starting_cpu(unsigned int cpu) 4149 { 4150 raw_spin_lock(&kvm_count_lock); 4151 if (kvm_usage_count) 4152 hardware_enable_nolock(NULL); 4153 raw_spin_unlock(&kvm_count_lock); 4154 return 0; 4155 } 4156 4157 static void hardware_disable_nolock(void *junk) 4158 { 4159 int cpu = raw_smp_processor_id(); 4160 4161 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 4162 return; 4163 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 4164 kvm_arch_hardware_disable(); 4165 } 4166 4167 static int kvm_dying_cpu(unsigned int cpu) 4168 { 4169 raw_spin_lock(&kvm_count_lock); 4170 if (kvm_usage_count) 4171 hardware_disable_nolock(NULL); 4172 raw_spin_unlock(&kvm_count_lock); 4173 return 0; 4174 } 4175 4176 static void hardware_disable_all_nolock(void) 4177 { 4178 BUG_ON(!kvm_usage_count); 4179 4180 kvm_usage_count--; 4181 if (!kvm_usage_count) 4182 on_each_cpu(hardware_disable_nolock, NULL, 1); 4183 } 4184 4185 static void hardware_disable_all(void) 4186 { 4187 raw_spin_lock(&kvm_count_lock); 4188 hardware_disable_all_nolock(); 4189 raw_spin_unlock(&kvm_count_lock); 4190 } 4191 4192 static int hardware_enable_all(void) 4193 { 4194 int r = 0; 4195 4196 raw_spin_lock(&kvm_count_lock); 4197 4198 kvm_usage_count++; 4199 if (kvm_usage_count == 1) { 4200 atomic_set(&hardware_enable_failed, 0); 4201 on_each_cpu(hardware_enable_nolock, NULL, 1); 4202 4203 if (atomic_read(&hardware_enable_failed)) { 4204 hardware_disable_all_nolock(); 4205 r = -EBUSY; 4206 } 4207 } 4208 4209 raw_spin_unlock(&kvm_count_lock); 4210 4211 return r; 4212 } 4213 4214 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 4215 void *v) 4216 { 4217 /* 4218 * Some (well, at least mine) BIOSes hang on reboot if 4219 * in vmx root mode. 4220 * 4221 * And Intel TXT required VMX off for all cpu when system shutdown. 4222 */ 4223 pr_info("kvm: exiting hardware virtualization\n"); 4224 kvm_rebooting = true; 4225 on_each_cpu(hardware_disable_nolock, NULL, 1); 4226 return NOTIFY_OK; 4227 } 4228 4229 static struct notifier_block kvm_reboot_notifier = { 4230 .notifier_call = kvm_reboot, 4231 .priority = 0, 4232 }; 4233 4234 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 4235 { 4236 int i; 4237 4238 for (i = 0; i < bus->dev_count; i++) { 4239 struct kvm_io_device *pos = bus->range[i].dev; 4240 4241 kvm_iodevice_destructor(pos); 4242 } 4243 kfree(bus); 4244 } 4245 4246 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 4247 const struct kvm_io_range *r2) 4248 { 4249 gpa_t addr1 = r1->addr; 4250 gpa_t addr2 = r2->addr; 4251 4252 if (addr1 < addr2) 4253 return -1; 4254 4255 /* If r2->len == 0, match the exact address. If r2->len != 0, 4256 * accept any overlapping write. Any order is acceptable for 4257 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 4258 * we process all of them. 4259 */ 4260 if (r2->len) { 4261 addr1 += r1->len; 4262 addr2 += r2->len; 4263 } 4264 4265 if (addr1 > addr2) 4266 return 1; 4267 4268 return 0; 4269 } 4270 4271 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 4272 { 4273 return kvm_io_bus_cmp(p1, p2); 4274 } 4275 4276 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 4277 gpa_t addr, int len) 4278 { 4279 struct kvm_io_range *range, key; 4280 int off; 4281 4282 key = (struct kvm_io_range) { 4283 .addr = addr, 4284 .len = len, 4285 }; 4286 4287 range = bsearch(&key, bus->range, bus->dev_count, 4288 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 4289 if (range == NULL) 4290 return -ENOENT; 4291 4292 off = range - bus->range; 4293 4294 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 4295 off--; 4296 4297 return off; 4298 } 4299 4300 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 4301 struct kvm_io_range *range, const void *val) 4302 { 4303 int idx; 4304 4305 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 4306 if (idx < 0) 4307 return -EOPNOTSUPP; 4308 4309 while (idx < bus->dev_count && 4310 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 4311 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 4312 range->len, val)) 4313 return idx; 4314 idx++; 4315 } 4316 4317 return -EOPNOTSUPP; 4318 } 4319 4320 /* kvm_io_bus_write - called under kvm->slots_lock */ 4321 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 4322 int len, const void *val) 4323 { 4324 struct kvm_io_bus *bus; 4325 struct kvm_io_range range; 4326 int r; 4327 4328 range = (struct kvm_io_range) { 4329 .addr = addr, 4330 .len = len, 4331 }; 4332 4333 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 4334 if (!bus) 4335 return -ENOMEM; 4336 r = __kvm_io_bus_write(vcpu, bus, &range, val); 4337 return r < 0 ? r : 0; 4338 } 4339 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 4340 4341 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 4342 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 4343 gpa_t addr, int len, const void *val, long cookie) 4344 { 4345 struct kvm_io_bus *bus; 4346 struct kvm_io_range range; 4347 4348 range = (struct kvm_io_range) { 4349 .addr = addr, 4350 .len = len, 4351 }; 4352 4353 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 4354 if (!bus) 4355 return -ENOMEM; 4356 4357 /* First try the device referenced by cookie. */ 4358 if ((cookie >= 0) && (cookie < bus->dev_count) && 4359 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 4360 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 4361 val)) 4362 return cookie; 4363 4364 /* 4365 * cookie contained garbage; fall back to search and return the 4366 * correct cookie value. 4367 */ 4368 return __kvm_io_bus_write(vcpu, bus, &range, val); 4369 } 4370 4371 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 4372 struct kvm_io_range *range, void *val) 4373 { 4374 int idx; 4375 4376 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 4377 if (idx < 0) 4378 return -EOPNOTSUPP; 4379 4380 while (idx < bus->dev_count && 4381 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 4382 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 4383 range->len, val)) 4384 return idx; 4385 idx++; 4386 } 4387 4388 return -EOPNOTSUPP; 4389 } 4390 4391 /* kvm_io_bus_read - called under kvm->slots_lock */ 4392 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 4393 int len, void *val) 4394 { 4395 struct kvm_io_bus *bus; 4396 struct kvm_io_range range; 4397 int r; 4398 4399 range = (struct kvm_io_range) { 4400 .addr = addr, 4401 .len = len, 4402 }; 4403 4404 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 4405 if (!bus) 4406 return -ENOMEM; 4407 r = __kvm_io_bus_read(vcpu, bus, &range, val); 4408 return r < 0 ? r : 0; 4409 } 4410 4411 /* Caller must hold slots_lock. */ 4412 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 4413 int len, struct kvm_io_device *dev) 4414 { 4415 int i; 4416 struct kvm_io_bus *new_bus, *bus; 4417 struct kvm_io_range range; 4418 4419 bus = kvm_get_bus(kvm, bus_idx); 4420 if (!bus) 4421 return -ENOMEM; 4422 4423 /* exclude ioeventfd which is limited by maximum fd */ 4424 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 4425 return -ENOSPC; 4426 4427 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1), 4428 GFP_KERNEL_ACCOUNT); 4429 if (!new_bus) 4430 return -ENOMEM; 4431 4432 range = (struct kvm_io_range) { 4433 .addr = addr, 4434 .len = len, 4435 .dev = dev, 4436 }; 4437 4438 for (i = 0; i < bus->dev_count; i++) 4439 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0) 4440 break; 4441 4442 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 4443 new_bus->dev_count++; 4444 new_bus->range[i] = range; 4445 memcpy(new_bus->range + i + 1, bus->range + i, 4446 (bus->dev_count - i) * sizeof(struct kvm_io_range)); 4447 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 4448 synchronize_srcu_expedited(&kvm->srcu); 4449 kfree(bus); 4450 4451 return 0; 4452 } 4453 4454 /* Caller must hold slots_lock. */ 4455 void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 4456 struct kvm_io_device *dev) 4457 { 4458 int i, j; 4459 struct kvm_io_bus *new_bus, *bus; 4460 4461 bus = kvm_get_bus(kvm, bus_idx); 4462 if (!bus) 4463 return; 4464 4465 for (i = 0; i < bus->dev_count; i++) 4466 if (bus->range[i].dev == dev) { 4467 break; 4468 } 4469 4470 if (i == bus->dev_count) 4471 return; 4472 4473 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1), 4474 GFP_KERNEL_ACCOUNT); 4475 if (new_bus) { 4476 memcpy(new_bus, bus, struct_size(bus, range, i)); 4477 new_bus->dev_count--; 4478 memcpy(new_bus->range + i, bus->range + i + 1, 4479 flex_array_size(new_bus, range, new_bus->dev_count - i)); 4480 } else { 4481 pr_err("kvm: failed to shrink bus, removing it completely\n"); 4482 for (j = 0; j < bus->dev_count; j++) { 4483 if (j == i) 4484 continue; 4485 kvm_iodevice_destructor(bus->range[j].dev); 4486 } 4487 } 4488 4489 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 4490 synchronize_srcu_expedited(&kvm->srcu); 4491 kfree(bus); 4492 return; 4493 } 4494 4495 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 4496 gpa_t addr) 4497 { 4498 struct kvm_io_bus *bus; 4499 int dev_idx, srcu_idx; 4500 struct kvm_io_device *iodev = NULL; 4501 4502 srcu_idx = srcu_read_lock(&kvm->srcu); 4503 4504 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 4505 if (!bus) 4506 goto out_unlock; 4507 4508 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); 4509 if (dev_idx < 0) 4510 goto out_unlock; 4511 4512 iodev = bus->range[dev_idx].dev; 4513 4514 out_unlock: 4515 srcu_read_unlock(&kvm->srcu, srcu_idx); 4516 4517 return iodev; 4518 } 4519 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev); 4520 4521 static int kvm_debugfs_open(struct inode *inode, struct file *file, 4522 int (*get)(void *, u64 *), int (*set)(void *, u64), 4523 const char *fmt) 4524 { 4525 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 4526 inode->i_private; 4527 4528 /* The debugfs files are a reference to the kvm struct which 4529 * is still valid when kvm_destroy_vm is called. 4530 * To avoid the race between open and the removal of the debugfs 4531 * directory we test against the users count. 4532 */ 4533 if (!refcount_inc_not_zero(&stat_data->kvm->users_count)) 4534 return -ENOENT; 4535 4536 if (simple_attr_open(inode, file, get, 4537 KVM_DBGFS_GET_MODE(stat_data->dbgfs_item) & 0222 4538 ? set : NULL, 4539 fmt)) { 4540 kvm_put_kvm(stat_data->kvm); 4541 return -ENOMEM; 4542 } 4543 4544 return 0; 4545 } 4546 4547 static int kvm_debugfs_release(struct inode *inode, struct file *file) 4548 { 4549 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 4550 inode->i_private; 4551 4552 simple_attr_release(inode, file); 4553 kvm_put_kvm(stat_data->kvm); 4554 4555 return 0; 4556 } 4557 4558 static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val) 4559 { 4560 *val = *(ulong *)((void *)kvm + offset); 4561 4562 return 0; 4563 } 4564 4565 static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset) 4566 { 4567 *(ulong *)((void *)kvm + offset) = 0; 4568 4569 return 0; 4570 } 4571 4572 static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val) 4573 { 4574 int i; 4575 struct kvm_vcpu *vcpu; 4576 4577 *val = 0; 4578 4579 kvm_for_each_vcpu(i, vcpu, kvm) 4580 *val += *(u64 *)((void *)vcpu + offset); 4581 4582 return 0; 4583 } 4584 4585 static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset) 4586 { 4587 int i; 4588 struct kvm_vcpu *vcpu; 4589 4590 kvm_for_each_vcpu(i, vcpu, kvm) 4591 *(u64 *)((void *)vcpu + offset) = 0; 4592 4593 return 0; 4594 } 4595 4596 static int kvm_stat_data_get(void *data, u64 *val) 4597 { 4598 int r = -EFAULT; 4599 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 4600 4601 switch (stat_data->dbgfs_item->kind) { 4602 case KVM_STAT_VM: 4603 r = kvm_get_stat_per_vm(stat_data->kvm, 4604 stat_data->dbgfs_item->offset, val); 4605 break; 4606 case KVM_STAT_VCPU: 4607 r = kvm_get_stat_per_vcpu(stat_data->kvm, 4608 stat_data->dbgfs_item->offset, val); 4609 break; 4610 } 4611 4612 return r; 4613 } 4614 4615 static int kvm_stat_data_clear(void *data, u64 val) 4616 { 4617 int r = -EFAULT; 4618 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 4619 4620 if (val) 4621 return -EINVAL; 4622 4623 switch (stat_data->dbgfs_item->kind) { 4624 case KVM_STAT_VM: 4625 r = kvm_clear_stat_per_vm(stat_data->kvm, 4626 stat_data->dbgfs_item->offset); 4627 break; 4628 case KVM_STAT_VCPU: 4629 r = kvm_clear_stat_per_vcpu(stat_data->kvm, 4630 stat_data->dbgfs_item->offset); 4631 break; 4632 } 4633 4634 return r; 4635 } 4636 4637 static int kvm_stat_data_open(struct inode *inode, struct file *file) 4638 { 4639 __simple_attr_check_format("%llu\n", 0ull); 4640 return kvm_debugfs_open(inode, file, kvm_stat_data_get, 4641 kvm_stat_data_clear, "%llu\n"); 4642 } 4643 4644 static const struct file_operations stat_fops_per_vm = { 4645 .owner = THIS_MODULE, 4646 .open = kvm_stat_data_open, 4647 .release = kvm_debugfs_release, 4648 .read = simple_attr_read, 4649 .write = simple_attr_write, 4650 .llseek = no_llseek, 4651 }; 4652 4653 static int vm_stat_get(void *_offset, u64 *val) 4654 { 4655 unsigned offset = (long)_offset; 4656 struct kvm *kvm; 4657 u64 tmp_val; 4658 4659 *val = 0; 4660 mutex_lock(&kvm_lock); 4661 list_for_each_entry(kvm, &vm_list, vm_list) { 4662 kvm_get_stat_per_vm(kvm, offset, &tmp_val); 4663 *val += tmp_val; 4664 } 4665 mutex_unlock(&kvm_lock); 4666 return 0; 4667 } 4668 4669 static int vm_stat_clear(void *_offset, u64 val) 4670 { 4671 unsigned offset = (long)_offset; 4672 struct kvm *kvm; 4673 4674 if (val) 4675 return -EINVAL; 4676 4677 mutex_lock(&kvm_lock); 4678 list_for_each_entry(kvm, &vm_list, vm_list) { 4679 kvm_clear_stat_per_vm(kvm, offset); 4680 } 4681 mutex_unlock(&kvm_lock); 4682 4683 return 0; 4684 } 4685 4686 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); 4687 4688 static int vcpu_stat_get(void *_offset, u64 *val) 4689 { 4690 unsigned offset = (long)_offset; 4691 struct kvm *kvm; 4692 u64 tmp_val; 4693 4694 *val = 0; 4695 mutex_lock(&kvm_lock); 4696 list_for_each_entry(kvm, &vm_list, vm_list) { 4697 kvm_get_stat_per_vcpu(kvm, offset, &tmp_val); 4698 *val += tmp_val; 4699 } 4700 mutex_unlock(&kvm_lock); 4701 return 0; 4702 } 4703 4704 static int vcpu_stat_clear(void *_offset, u64 val) 4705 { 4706 unsigned offset = (long)_offset; 4707 struct kvm *kvm; 4708 4709 if (val) 4710 return -EINVAL; 4711 4712 mutex_lock(&kvm_lock); 4713 list_for_each_entry(kvm, &vm_list, vm_list) { 4714 kvm_clear_stat_per_vcpu(kvm, offset); 4715 } 4716 mutex_unlock(&kvm_lock); 4717 4718 return 0; 4719 } 4720 4721 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, 4722 "%llu\n"); 4723 4724 static const struct file_operations *stat_fops[] = { 4725 [KVM_STAT_VCPU] = &vcpu_stat_fops, 4726 [KVM_STAT_VM] = &vm_stat_fops, 4727 }; 4728 4729 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) 4730 { 4731 struct kobj_uevent_env *env; 4732 unsigned long long created, active; 4733 4734 if (!kvm_dev.this_device || !kvm) 4735 return; 4736 4737 mutex_lock(&kvm_lock); 4738 if (type == KVM_EVENT_CREATE_VM) { 4739 kvm_createvm_count++; 4740 kvm_active_vms++; 4741 } else if (type == KVM_EVENT_DESTROY_VM) { 4742 kvm_active_vms--; 4743 } 4744 created = kvm_createvm_count; 4745 active = kvm_active_vms; 4746 mutex_unlock(&kvm_lock); 4747 4748 env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT); 4749 if (!env) 4750 return; 4751 4752 add_uevent_var(env, "CREATED=%llu", created); 4753 add_uevent_var(env, "COUNT=%llu", active); 4754 4755 if (type == KVM_EVENT_CREATE_VM) { 4756 add_uevent_var(env, "EVENT=create"); 4757 kvm->userspace_pid = task_pid_nr(current); 4758 } else if (type == KVM_EVENT_DESTROY_VM) { 4759 add_uevent_var(env, "EVENT=destroy"); 4760 } 4761 add_uevent_var(env, "PID=%d", kvm->userspace_pid); 4762 4763 if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) { 4764 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT); 4765 4766 if (p) { 4767 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); 4768 if (!IS_ERR(tmp)) 4769 add_uevent_var(env, "STATS_PATH=%s", tmp); 4770 kfree(p); 4771 } 4772 } 4773 /* no need for checks, since we are adding at most only 5 keys */ 4774 env->envp[env->envp_idx++] = NULL; 4775 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp); 4776 kfree(env); 4777 } 4778 4779 static void kvm_init_debug(void) 4780 { 4781 struct kvm_stats_debugfs_item *p; 4782 4783 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 4784 4785 kvm_debugfs_num_entries = 0; 4786 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { 4787 debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p), 4788 kvm_debugfs_dir, (void *)(long)p->offset, 4789 stat_fops[p->kind]); 4790 } 4791 } 4792 4793 static int kvm_suspend(void) 4794 { 4795 if (kvm_usage_count) 4796 hardware_disable_nolock(NULL); 4797 return 0; 4798 } 4799 4800 static void kvm_resume(void) 4801 { 4802 if (kvm_usage_count) { 4803 #ifdef CONFIG_LOCKDEP 4804 WARN_ON(lockdep_is_held(&kvm_count_lock)); 4805 #endif 4806 hardware_enable_nolock(NULL); 4807 } 4808 } 4809 4810 static struct syscore_ops kvm_syscore_ops = { 4811 .suspend = kvm_suspend, 4812 .resume = kvm_resume, 4813 }; 4814 4815 static inline 4816 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 4817 { 4818 return container_of(pn, struct kvm_vcpu, preempt_notifier); 4819 } 4820 4821 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 4822 { 4823 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 4824 4825 WRITE_ONCE(vcpu->preempted, false); 4826 WRITE_ONCE(vcpu->ready, false); 4827 4828 __this_cpu_write(kvm_running_vcpu, vcpu); 4829 kvm_arch_sched_in(vcpu, cpu); 4830 kvm_arch_vcpu_load(vcpu, cpu); 4831 } 4832 4833 static void kvm_sched_out(struct preempt_notifier *pn, 4834 struct task_struct *next) 4835 { 4836 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 4837 4838 if (current->state == TASK_RUNNING) { 4839 WRITE_ONCE(vcpu->preempted, true); 4840 WRITE_ONCE(vcpu->ready, true); 4841 } 4842 kvm_arch_vcpu_put(vcpu); 4843 __this_cpu_write(kvm_running_vcpu, NULL); 4844 } 4845 4846 /** 4847 * kvm_get_running_vcpu - get the vcpu running on the current CPU. 4848 * 4849 * We can disable preemption locally around accessing the per-CPU variable, 4850 * and use the resolved vcpu pointer after enabling preemption again, 4851 * because even if the current thread is migrated to another CPU, reading 4852 * the per-CPU value later will give us the same value as we update the 4853 * per-CPU variable in the preempt notifier handlers. 4854 */ 4855 struct kvm_vcpu *kvm_get_running_vcpu(void) 4856 { 4857 struct kvm_vcpu *vcpu; 4858 4859 preempt_disable(); 4860 vcpu = __this_cpu_read(kvm_running_vcpu); 4861 preempt_enable(); 4862 4863 return vcpu; 4864 } 4865 EXPORT_SYMBOL_GPL(kvm_get_running_vcpu); 4866 4867 /** 4868 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus. 4869 */ 4870 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void) 4871 { 4872 return &kvm_running_vcpu; 4873 } 4874 4875 struct kvm_cpu_compat_check { 4876 void *opaque; 4877 int *ret; 4878 }; 4879 4880 static void check_processor_compat(void *data) 4881 { 4882 struct kvm_cpu_compat_check *c = data; 4883 4884 *c->ret = kvm_arch_check_processor_compat(c->opaque); 4885 } 4886 4887 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 4888 struct module *module) 4889 { 4890 struct kvm_cpu_compat_check c; 4891 int r; 4892 int cpu; 4893 4894 r = kvm_arch_init(opaque); 4895 if (r) 4896 goto out_fail; 4897 4898 /* 4899 * kvm_arch_init makes sure there's at most one caller 4900 * for architectures that support multiple implementations, 4901 * like intel and amd on x86. 4902 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 4903 * conflicts in case kvm is already setup for another implementation. 4904 */ 4905 r = kvm_irqfd_init(); 4906 if (r) 4907 goto out_irqfd; 4908 4909 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 4910 r = -ENOMEM; 4911 goto out_free_0; 4912 } 4913 4914 r = kvm_arch_hardware_setup(opaque); 4915 if (r < 0) 4916 goto out_free_1; 4917 4918 c.ret = &r; 4919 c.opaque = opaque; 4920 for_each_online_cpu(cpu) { 4921 smp_call_function_single(cpu, check_processor_compat, &c, 1); 4922 if (r < 0) 4923 goto out_free_2; 4924 } 4925 4926 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting", 4927 kvm_starting_cpu, kvm_dying_cpu); 4928 if (r) 4929 goto out_free_2; 4930 register_reboot_notifier(&kvm_reboot_notifier); 4931 4932 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 4933 if (!vcpu_align) 4934 vcpu_align = __alignof__(struct kvm_vcpu); 4935 kvm_vcpu_cache = 4936 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, 4937 SLAB_ACCOUNT, 4938 offsetof(struct kvm_vcpu, arch), 4939 sizeof_field(struct kvm_vcpu, arch), 4940 NULL); 4941 if (!kvm_vcpu_cache) { 4942 r = -ENOMEM; 4943 goto out_free_3; 4944 } 4945 4946 r = kvm_async_pf_init(); 4947 if (r) 4948 goto out_free; 4949 4950 kvm_chardev_ops.owner = module; 4951 kvm_vm_fops.owner = module; 4952 kvm_vcpu_fops.owner = module; 4953 4954 r = misc_register(&kvm_dev); 4955 if (r) { 4956 pr_err("kvm: misc device register failed\n"); 4957 goto out_unreg; 4958 } 4959 4960 register_syscore_ops(&kvm_syscore_ops); 4961 4962 kvm_preempt_ops.sched_in = kvm_sched_in; 4963 kvm_preempt_ops.sched_out = kvm_sched_out; 4964 4965 kvm_init_debug(); 4966 4967 r = kvm_vfio_ops_init(); 4968 WARN_ON(r); 4969 4970 return 0; 4971 4972 out_unreg: 4973 kvm_async_pf_deinit(); 4974 out_free: 4975 kmem_cache_destroy(kvm_vcpu_cache); 4976 out_free_3: 4977 unregister_reboot_notifier(&kvm_reboot_notifier); 4978 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 4979 out_free_2: 4980 kvm_arch_hardware_unsetup(); 4981 out_free_1: 4982 free_cpumask_var(cpus_hardware_enabled); 4983 out_free_0: 4984 kvm_irqfd_exit(); 4985 out_irqfd: 4986 kvm_arch_exit(); 4987 out_fail: 4988 return r; 4989 } 4990 EXPORT_SYMBOL_GPL(kvm_init); 4991 4992 void kvm_exit(void) 4993 { 4994 debugfs_remove_recursive(kvm_debugfs_dir); 4995 misc_deregister(&kvm_dev); 4996 kmem_cache_destroy(kvm_vcpu_cache); 4997 kvm_async_pf_deinit(); 4998 unregister_syscore_ops(&kvm_syscore_ops); 4999 unregister_reboot_notifier(&kvm_reboot_notifier); 5000 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING); 5001 on_each_cpu(hardware_disable_nolock, NULL, 1); 5002 kvm_arch_hardware_unsetup(); 5003 kvm_arch_exit(); 5004 kvm_irqfd_exit(); 5005 free_cpumask_var(cpus_hardware_enabled); 5006 kvm_vfio_ops_exit(); 5007 } 5008 EXPORT_SYMBOL_GPL(kvm_exit); 5009 5010 struct kvm_vm_worker_thread_context { 5011 struct kvm *kvm; 5012 struct task_struct *parent; 5013 struct completion init_done; 5014 kvm_vm_thread_fn_t thread_fn; 5015 uintptr_t data; 5016 int err; 5017 }; 5018 5019 static int kvm_vm_worker_thread(void *context) 5020 { 5021 /* 5022 * The init_context is allocated on the stack of the parent thread, so 5023 * we have to locally copy anything that is needed beyond initialization 5024 */ 5025 struct kvm_vm_worker_thread_context *init_context = context; 5026 struct kvm *kvm = init_context->kvm; 5027 kvm_vm_thread_fn_t thread_fn = init_context->thread_fn; 5028 uintptr_t data = init_context->data; 5029 int err; 5030 5031 err = kthread_park(current); 5032 /* kthread_park(current) is never supposed to return an error */ 5033 WARN_ON(err != 0); 5034 if (err) 5035 goto init_complete; 5036 5037 err = cgroup_attach_task_all(init_context->parent, current); 5038 if (err) { 5039 kvm_err("%s: cgroup_attach_task_all failed with err %d\n", 5040 __func__, err); 5041 goto init_complete; 5042 } 5043 5044 set_user_nice(current, task_nice(init_context->parent)); 5045 5046 init_complete: 5047 init_context->err = err; 5048 complete(&init_context->init_done); 5049 init_context = NULL; 5050 5051 if (err) 5052 return err; 5053 5054 /* Wait to be woken up by the spawner before proceeding. */ 5055 kthread_parkme(); 5056 5057 if (!kthread_should_stop()) 5058 err = thread_fn(kvm, data); 5059 5060 return err; 5061 } 5062 5063 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, 5064 uintptr_t data, const char *name, 5065 struct task_struct **thread_ptr) 5066 { 5067 struct kvm_vm_worker_thread_context init_context = {}; 5068 struct task_struct *thread; 5069 5070 *thread_ptr = NULL; 5071 init_context.kvm = kvm; 5072 init_context.parent = current; 5073 init_context.thread_fn = thread_fn; 5074 init_context.data = data; 5075 init_completion(&init_context.init_done); 5076 5077 thread = kthread_run(kvm_vm_worker_thread, &init_context, 5078 "%s-%d", name, task_pid_nr(current)); 5079 if (IS_ERR(thread)) 5080 return PTR_ERR(thread); 5081 5082 /* kthread_run is never supposed to return NULL */ 5083 WARN_ON(thread == NULL); 5084 5085 wait_for_completion(&init_context.init_done); 5086 5087 if (!init_context.err) 5088 *thread_ptr = thread; 5089 5090 return init_context.err; 5091 } 5092