1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 16 #include <kvm/iodev.h> 17 18 #include <linux/kvm_host.h> 19 #include <linux/kvm.h> 20 #include <linux/module.h> 21 #include <linux/errno.h> 22 #include <linux/percpu.h> 23 #include <linux/mm.h> 24 #include <linux/miscdevice.h> 25 #include <linux/vmalloc.h> 26 #include <linux/reboot.h> 27 #include <linux/debugfs.h> 28 #include <linux/highmem.h> 29 #include <linux/file.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/cpu.h> 32 #include <linux/sched/signal.h> 33 #include <linux/sched/mm.h> 34 #include <linux/sched/stat.h> 35 #include <linux/cpumask.h> 36 #include <linux/smp.h> 37 #include <linux/anon_inodes.h> 38 #include <linux/profile.h> 39 #include <linux/kvm_para.h> 40 #include <linux/pagemap.h> 41 #include <linux/mman.h> 42 #include <linux/swap.h> 43 #include <linux/bitops.h> 44 #include <linux/spinlock.h> 45 #include <linux/compat.h> 46 #include <linux/srcu.h> 47 #include <linux/hugetlb.h> 48 #include <linux/slab.h> 49 #include <linux/sort.h> 50 #include <linux/bsearch.h> 51 #include <linux/io.h> 52 #include <linux/lockdep.h> 53 #include <linux/kthread.h> 54 #include <linux/suspend.h> 55 56 #include <asm/processor.h> 57 #include <asm/ioctl.h> 58 #include <linux/uaccess.h> 59 60 #include "coalesced_mmio.h" 61 #include "async_pf.h" 62 #include "kvm_mm.h" 63 #include "vfio.h" 64 65 #include <trace/events/ipi.h> 66 67 #define CREATE_TRACE_POINTS 68 #include <trace/events/kvm.h> 69 70 #include <linux/kvm_dirty_ring.h> 71 72 73 /* Worst case buffer size needed for holding an integer. */ 74 #define ITOA_MAX_LEN 12 75 76 MODULE_AUTHOR("Qumranet"); 77 MODULE_LICENSE("GPL"); 78 79 /* Architectures should define their poll value according to the halt latency */ 80 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 81 module_param(halt_poll_ns, uint, 0644); 82 EXPORT_SYMBOL_GPL(halt_poll_ns); 83 84 /* Default doubles per-vcpu halt_poll_ns. */ 85 unsigned int halt_poll_ns_grow = 2; 86 module_param(halt_poll_ns_grow, uint, 0644); 87 EXPORT_SYMBOL_GPL(halt_poll_ns_grow); 88 89 /* The start value to grow halt_poll_ns from */ 90 unsigned int halt_poll_ns_grow_start = 10000; /* 10us */ 91 module_param(halt_poll_ns_grow_start, uint, 0644); 92 EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start); 93 94 /* Default resets per-vcpu halt_poll_ns . */ 95 unsigned int halt_poll_ns_shrink; 96 module_param(halt_poll_ns_shrink, uint, 0644); 97 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); 98 99 /* 100 * Ordering of locks: 101 * 102 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 103 */ 104 105 DEFINE_MUTEX(kvm_lock); 106 LIST_HEAD(vm_list); 107 108 static struct kmem_cache *kvm_vcpu_cache; 109 110 static __read_mostly struct preempt_ops kvm_preempt_ops; 111 static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu); 112 113 struct dentry *kvm_debugfs_dir; 114 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 115 116 static const struct file_operations stat_fops_per_vm; 117 118 static struct file_operations kvm_chardev_ops; 119 120 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 121 unsigned long arg); 122 #ifdef CONFIG_KVM_COMPAT 123 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 124 unsigned long arg); 125 #define KVM_COMPAT(c) .compat_ioctl = (c) 126 #else 127 /* 128 * For architectures that don't implement a compat infrastructure, 129 * adopt a double line of defense: 130 * - Prevent a compat task from opening /dev/kvm 131 * - If the open has been done by a 64bit task, and the KVM fd 132 * passed to a compat task, let the ioctls fail. 133 */ 134 static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl, 135 unsigned long arg) { return -EINVAL; } 136 137 static int kvm_no_compat_open(struct inode *inode, struct file *file) 138 { 139 return is_compat_task() ? -ENODEV : 0; 140 } 141 #define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \ 142 .open = kvm_no_compat_open 143 #endif 144 static int hardware_enable_all(void); 145 static void hardware_disable_all(void); 146 147 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 148 149 #define KVM_EVENT_CREATE_VM 0 150 #define KVM_EVENT_DESTROY_VM 1 151 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm); 152 static unsigned long long kvm_createvm_count; 153 static unsigned long long kvm_active_vms; 154 155 static DEFINE_PER_CPU(cpumask_var_t, cpu_kick_mask); 156 157 __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, 158 unsigned long start, unsigned long end) 159 { 160 } 161 162 __weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) 163 { 164 } 165 166 bool kvm_is_zone_device_page(struct page *page) 167 { 168 /* 169 * The metadata used by is_zone_device_page() to determine whether or 170 * not a page is ZONE_DEVICE is guaranteed to be valid if and only if 171 * the device has been pinned, e.g. by get_user_pages(). WARN if the 172 * page_count() is zero to help detect bad usage of this helper. 173 */ 174 if (WARN_ON_ONCE(!page_count(page))) 175 return false; 176 177 return is_zone_device_page(page); 178 } 179 180 /* 181 * Returns a 'struct page' if the pfn is "valid" and backed by a refcounted 182 * page, NULL otherwise. Note, the list of refcounted PG_reserved page types 183 * is likely incomplete, it has been compiled purely through people wanting to 184 * back guest with a certain type of memory and encountering issues. 185 */ 186 struct page *kvm_pfn_to_refcounted_page(kvm_pfn_t pfn) 187 { 188 struct page *page; 189 190 if (!pfn_valid(pfn)) 191 return NULL; 192 193 page = pfn_to_page(pfn); 194 if (!PageReserved(page)) 195 return page; 196 197 /* The ZERO_PAGE(s) is marked PG_reserved, but is refcounted. */ 198 if (is_zero_pfn(pfn)) 199 return page; 200 201 /* 202 * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting 203 * perspective they are "normal" pages, albeit with slightly different 204 * usage rules. 205 */ 206 if (kvm_is_zone_device_page(page)) 207 return page; 208 209 return NULL; 210 } 211 212 /* 213 * Switches to specified vcpu, until a matching vcpu_put() 214 */ 215 void vcpu_load(struct kvm_vcpu *vcpu) 216 { 217 int cpu = get_cpu(); 218 219 __this_cpu_write(kvm_running_vcpu, vcpu); 220 preempt_notifier_register(&vcpu->preempt_notifier); 221 kvm_arch_vcpu_load(vcpu, cpu); 222 put_cpu(); 223 } 224 EXPORT_SYMBOL_GPL(vcpu_load); 225 226 void vcpu_put(struct kvm_vcpu *vcpu) 227 { 228 preempt_disable(); 229 kvm_arch_vcpu_put(vcpu); 230 preempt_notifier_unregister(&vcpu->preempt_notifier); 231 __this_cpu_write(kvm_running_vcpu, NULL); 232 preempt_enable(); 233 } 234 EXPORT_SYMBOL_GPL(vcpu_put); 235 236 /* TODO: merge with kvm_arch_vcpu_should_kick */ 237 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req) 238 { 239 int mode = kvm_vcpu_exiting_guest_mode(vcpu); 240 241 /* 242 * We need to wait for the VCPU to reenable interrupts and get out of 243 * READING_SHADOW_PAGE_TABLES mode. 244 */ 245 if (req & KVM_REQUEST_WAIT) 246 return mode != OUTSIDE_GUEST_MODE; 247 248 /* 249 * Need to kick a running VCPU, but otherwise there is nothing to do. 250 */ 251 return mode == IN_GUEST_MODE; 252 } 253 254 static void ack_kick(void *_completed) 255 { 256 } 257 258 static inline bool kvm_kick_many_cpus(struct cpumask *cpus, bool wait) 259 { 260 if (cpumask_empty(cpus)) 261 return false; 262 263 smp_call_function_many(cpus, ack_kick, NULL, wait); 264 return true; 265 } 266 267 static void kvm_make_vcpu_request(struct kvm_vcpu *vcpu, unsigned int req, 268 struct cpumask *tmp, int current_cpu) 269 { 270 int cpu; 271 272 if (likely(!(req & KVM_REQUEST_NO_ACTION))) 273 __kvm_make_request(req, vcpu); 274 275 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu)) 276 return; 277 278 /* 279 * Note, the vCPU could get migrated to a different pCPU at any point 280 * after kvm_request_needs_ipi(), which could result in sending an IPI 281 * to the previous pCPU. But, that's OK because the purpose of the IPI 282 * is to ensure the vCPU returns to OUTSIDE_GUEST_MODE, which is 283 * satisfied if the vCPU migrates. Entering READING_SHADOW_PAGE_TABLES 284 * after this point is also OK, as the requirement is only that KVM wait 285 * for vCPUs that were reading SPTEs _before_ any changes were 286 * finalized. See kvm_vcpu_kick() for more details on handling requests. 287 */ 288 if (kvm_request_needs_ipi(vcpu, req)) { 289 cpu = READ_ONCE(vcpu->cpu); 290 if (cpu != -1 && cpu != current_cpu) 291 __cpumask_set_cpu(cpu, tmp); 292 } 293 } 294 295 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, 296 unsigned long *vcpu_bitmap) 297 { 298 struct kvm_vcpu *vcpu; 299 struct cpumask *cpus; 300 int i, me; 301 bool called; 302 303 me = get_cpu(); 304 305 cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask); 306 cpumask_clear(cpus); 307 308 for_each_set_bit(i, vcpu_bitmap, KVM_MAX_VCPUS) { 309 vcpu = kvm_get_vcpu(kvm, i); 310 if (!vcpu) 311 continue; 312 kvm_make_vcpu_request(vcpu, req, cpus, me); 313 } 314 315 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT)); 316 put_cpu(); 317 318 return called; 319 } 320 321 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req, 322 struct kvm_vcpu *except) 323 { 324 struct kvm_vcpu *vcpu; 325 struct cpumask *cpus; 326 unsigned long i; 327 bool called; 328 int me; 329 330 me = get_cpu(); 331 332 cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask); 333 cpumask_clear(cpus); 334 335 kvm_for_each_vcpu(i, vcpu, kvm) { 336 if (vcpu == except) 337 continue; 338 kvm_make_vcpu_request(vcpu, req, cpus, me); 339 } 340 341 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT)); 342 put_cpu(); 343 344 return called; 345 } 346 347 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 348 { 349 return kvm_make_all_cpus_request_except(kvm, req, NULL); 350 } 351 EXPORT_SYMBOL_GPL(kvm_make_all_cpus_request); 352 353 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 354 void kvm_flush_remote_tlbs(struct kvm *kvm) 355 { 356 ++kvm->stat.generic.remote_tlb_flush_requests; 357 358 /* 359 * We want to publish modifications to the page tables before reading 360 * mode. Pairs with a memory barrier in arch-specific code. 361 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 362 * and smp_mb in walk_shadow_page_lockless_begin/end. 363 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 364 * 365 * There is already an smp_mb__after_atomic() before 366 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 367 * barrier here. 368 */ 369 if (!kvm_arch_flush_remote_tlb(kvm) 370 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 371 ++kvm->stat.generic.remote_tlb_flush; 372 } 373 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 374 #endif 375 376 static void kvm_flush_shadow_all(struct kvm *kvm) 377 { 378 kvm_arch_flush_shadow_all(kvm); 379 kvm_arch_guest_memory_reclaimed(kvm); 380 } 381 382 #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 383 static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc, 384 gfp_t gfp_flags) 385 { 386 gfp_flags |= mc->gfp_zero; 387 388 if (mc->kmem_cache) 389 return kmem_cache_alloc(mc->kmem_cache, gfp_flags); 390 else 391 return (void *)__get_free_page(gfp_flags); 392 } 393 394 int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min) 395 { 396 gfp_t gfp = mc->gfp_custom ? mc->gfp_custom : GFP_KERNEL_ACCOUNT; 397 void *obj; 398 399 if (mc->nobjs >= min) 400 return 0; 401 402 if (unlikely(!mc->objects)) { 403 if (WARN_ON_ONCE(!capacity)) 404 return -EIO; 405 406 mc->objects = kvmalloc_array(sizeof(void *), capacity, gfp); 407 if (!mc->objects) 408 return -ENOMEM; 409 410 mc->capacity = capacity; 411 } 412 413 /* It is illegal to request a different capacity across topups. */ 414 if (WARN_ON_ONCE(mc->capacity != capacity)) 415 return -EIO; 416 417 while (mc->nobjs < mc->capacity) { 418 obj = mmu_memory_cache_alloc_obj(mc, gfp); 419 if (!obj) 420 return mc->nobjs >= min ? 0 : -ENOMEM; 421 mc->objects[mc->nobjs++] = obj; 422 } 423 return 0; 424 } 425 426 int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min) 427 { 428 return __kvm_mmu_topup_memory_cache(mc, KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE, min); 429 } 430 431 int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc) 432 { 433 return mc->nobjs; 434 } 435 436 void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) 437 { 438 while (mc->nobjs) { 439 if (mc->kmem_cache) 440 kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]); 441 else 442 free_page((unsigned long)mc->objects[--mc->nobjs]); 443 } 444 445 kvfree(mc->objects); 446 447 mc->objects = NULL; 448 mc->capacity = 0; 449 } 450 451 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) 452 { 453 void *p; 454 455 if (WARN_ON(!mc->nobjs)) 456 p = mmu_memory_cache_alloc_obj(mc, GFP_ATOMIC | __GFP_ACCOUNT); 457 else 458 p = mc->objects[--mc->nobjs]; 459 BUG_ON(!p); 460 return p; 461 } 462 #endif 463 464 static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 465 { 466 mutex_init(&vcpu->mutex); 467 vcpu->cpu = -1; 468 vcpu->kvm = kvm; 469 vcpu->vcpu_id = id; 470 vcpu->pid = NULL; 471 #ifndef __KVM_HAVE_ARCH_WQP 472 rcuwait_init(&vcpu->wait); 473 #endif 474 kvm_async_pf_vcpu_init(vcpu); 475 476 kvm_vcpu_set_in_spin_loop(vcpu, false); 477 kvm_vcpu_set_dy_eligible(vcpu, false); 478 vcpu->preempted = false; 479 vcpu->ready = false; 480 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 481 vcpu->last_used_slot = NULL; 482 483 /* Fill the stats id string for the vcpu */ 484 snprintf(vcpu->stats_id, sizeof(vcpu->stats_id), "kvm-%d/vcpu-%d", 485 task_pid_nr(current), id); 486 } 487 488 static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) 489 { 490 kvm_arch_vcpu_destroy(vcpu); 491 kvm_dirty_ring_free(&vcpu->dirty_ring); 492 493 /* 494 * No need for rcu_read_lock as VCPU_RUN is the only place that changes 495 * the vcpu->pid pointer, and at destruction time all file descriptors 496 * are already gone. 497 */ 498 put_pid(rcu_dereference_protected(vcpu->pid, 1)); 499 500 free_page((unsigned long)vcpu->run); 501 kmem_cache_free(kvm_vcpu_cache, vcpu); 502 } 503 504 void kvm_destroy_vcpus(struct kvm *kvm) 505 { 506 unsigned long i; 507 struct kvm_vcpu *vcpu; 508 509 kvm_for_each_vcpu(i, vcpu, kvm) { 510 kvm_vcpu_destroy(vcpu); 511 xa_erase(&kvm->vcpu_array, i); 512 } 513 514 atomic_set(&kvm->online_vcpus, 0); 515 } 516 EXPORT_SYMBOL_GPL(kvm_destroy_vcpus); 517 518 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 519 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 520 { 521 return container_of(mn, struct kvm, mmu_notifier); 522 } 523 524 static void kvm_mmu_notifier_invalidate_range(struct mmu_notifier *mn, 525 struct mm_struct *mm, 526 unsigned long start, unsigned long end) 527 { 528 struct kvm *kvm = mmu_notifier_to_kvm(mn); 529 int idx; 530 531 idx = srcu_read_lock(&kvm->srcu); 532 kvm_arch_mmu_notifier_invalidate_range(kvm, start, end); 533 srcu_read_unlock(&kvm->srcu, idx); 534 } 535 536 typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range); 537 538 typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start, 539 unsigned long end); 540 541 typedef void (*on_unlock_fn_t)(struct kvm *kvm); 542 543 struct kvm_hva_range { 544 unsigned long start; 545 unsigned long end; 546 pte_t pte; 547 hva_handler_t handler; 548 on_lock_fn_t on_lock; 549 on_unlock_fn_t on_unlock; 550 bool flush_on_ret; 551 bool may_block; 552 }; 553 554 /* 555 * Use a dedicated stub instead of NULL to indicate that there is no callback 556 * function/handler. The compiler technically can't guarantee that a real 557 * function will have a non-zero address, and so it will generate code to 558 * check for !NULL, whereas comparing against a stub will be elided at compile 559 * time (unless the compiler is getting long in the tooth, e.g. gcc 4.9). 560 */ 561 static void kvm_null_fn(void) 562 { 563 564 } 565 #define IS_KVM_NULL_FN(fn) ((fn) == (void *)kvm_null_fn) 566 567 /* Iterate over each memslot intersecting [start, last] (inclusive) range */ 568 #define kvm_for_each_memslot_in_hva_range(node, slots, start, last) \ 569 for (node = interval_tree_iter_first(&slots->hva_tree, start, last); \ 570 node; \ 571 node = interval_tree_iter_next(node, start, last)) \ 572 573 static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, 574 const struct kvm_hva_range *range) 575 { 576 bool ret = false, locked = false; 577 struct kvm_gfn_range gfn_range; 578 struct kvm_memory_slot *slot; 579 struct kvm_memslots *slots; 580 int i, idx; 581 582 if (WARN_ON_ONCE(range->end <= range->start)) 583 return 0; 584 585 /* A null handler is allowed if and only if on_lock() is provided. */ 586 if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) && 587 IS_KVM_NULL_FN(range->handler))) 588 return 0; 589 590 idx = srcu_read_lock(&kvm->srcu); 591 592 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 593 struct interval_tree_node *node; 594 595 slots = __kvm_memslots(kvm, i); 596 kvm_for_each_memslot_in_hva_range(node, slots, 597 range->start, range->end - 1) { 598 unsigned long hva_start, hva_end; 599 600 slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]); 601 hva_start = max(range->start, slot->userspace_addr); 602 hva_end = min(range->end, slot->userspace_addr + 603 (slot->npages << PAGE_SHIFT)); 604 605 /* 606 * To optimize for the likely case where the address 607 * range is covered by zero or one memslots, don't 608 * bother making these conditional (to avoid writes on 609 * the second or later invocation of the handler). 610 */ 611 gfn_range.pte = range->pte; 612 gfn_range.may_block = range->may_block; 613 614 /* 615 * {gfn(page) | page intersects with [hva_start, hva_end)} = 616 * {gfn_start, gfn_start+1, ..., gfn_end-1}. 617 */ 618 gfn_range.start = hva_to_gfn_memslot(hva_start, slot); 619 gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot); 620 gfn_range.slot = slot; 621 622 if (!locked) { 623 locked = true; 624 KVM_MMU_LOCK(kvm); 625 if (!IS_KVM_NULL_FN(range->on_lock)) 626 range->on_lock(kvm, range->start, range->end); 627 if (IS_KVM_NULL_FN(range->handler)) 628 break; 629 } 630 ret |= range->handler(kvm, &gfn_range); 631 } 632 } 633 634 if (range->flush_on_ret && ret) 635 kvm_flush_remote_tlbs(kvm); 636 637 if (locked) { 638 KVM_MMU_UNLOCK(kvm); 639 if (!IS_KVM_NULL_FN(range->on_unlock)) 640 range->on_unlock(kvm); 641 } 642 643 srcu_read_unlock(&kvm->srcu, idx); 644 645 /* The notifiers are averse to booleans. :-( */ 646 return (int)ret; 647 } 648 649 static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn, 650 unsigned long start, 651 unsigned long end, 652 pte_t pte, 653 hva_handler_t handler) 654 { 655 struct kvm *kvm = mmu_notifier_to_kvm(mn); 656 const struct kvm_hva_range range = { 657 .start = start, 658 .end = end, 659 .pte = pte, 660 .handler = handler, 661 .on_lock = (void *)kvm_null_fn, 662 .on_unlock = (void *)kvm_null_fn, 663 .flush_on_ret = true, 664 .may_block = false, 665 }; 666 667 return __kvm_handle_hva_range(kvm, &range); 668 } 669 670 static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn, 671 unsigned long start, 672 unsigned long end, 673 hva_handler_t handler) 674 { 675 struct kvm *kvm = mmu_notifier_to_kvm(mn); 676 const struct kvm_hva_range range = { 677 .start = start, 678 .end = end, 679 .pte = __pte(0), 680 .handler = handler, 681 .on_lock = (void *)kvm_null_fn, 682 .on_unlock = (void *)kvm_null_fn, 683 .flush_on_ret = false, 684 .may_block = false, 685 }; 686 687 return __kvm_handle_hva_range(kvm, &range); 688 } 689 690 static bool kvm_change_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 691 { 692 /* 693 * Skipping invalid memslots is correct if and only change_pte() is 694 * surrounded by invalidate_range_{start,end}(), which is currently 695 * guaranteed by the primary MMU. If that ever changes, KVM needs to 696 * unmap the memslot instead of skipping the memslot to ensure that KVM 697 * doesn't hold references to the old PFN. 698 */ 699 WARN_ON_ONCE(!READ_ONCE(kvm->mn_active_invalidate_count)); 700 701 if (range->slot->flags & KVM_MEMSLOT_INVALID) 702 return false; 703 704 return kvm_set_spte_gfn(kvm, range); 705 } 706 707 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 708 struct mm_struct *mm, 709 unsigned long address, 710 pte_t pte) 711 { 712 struct kvm *kvm = mmu_notifier_to_kvm(mn); 713 714 trace_kvm_set_spte_hva(address); 715 716 /* 717 * .change_pte() must be surrounded by .invalidate_range_{start,end}(). 718 * If mmu_invalidate_in_progress is zero, then no in-progress 719 * invalidations, including this one, found a relevant memslot at 720 * start(); rechecking memslots here is unnecessary. Note, a false 721 * positive (count elevated by a different invalidation) is sub-optimal 722 * but functionally ok. 723 */ 724 WARN_ON_ONCE(!READ_ONCE(kvm->mn_active_invalidate_count)); 725 if (!READ_ONCE(kvm->mmu_invalidate_in_progress)) 726 return; 727 728 kvm_handle_hva_range(mn, address, address + 1, pte, kvm_change_spte_gfn); 729 } 730 731 void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start, 732 unsigned long end) 733 { 734 /* 735 * The count increase must become visible at unlock time as no 736 * spte can be established without taking the mmu_lock and 737 * count is also read inside the mmu_lock critical section. 738 */ 739 kvm->mmu_invalidate_in_progress++; 740 if (likely(kvm->mmu_invalidate_in_progress == 1)) { 741 kvm->mmu_invalidate_range_start = start; 742 kvm->mmu_invalidate_range_end = end; 743 } else { 744 /* 745 * Fully tracking multiple concurrent ranges has diminishing 746 * returns. Keep things simple and just find the minimal range 747 * which includes the current and new ranges. As there won't be 748 * enough information to subtract a range after its invalidate 749 * completes, any ranges invalidated concurrently will 750 * accumulate and persist until all outstanding invalidates 751 * complete. 752 */ 753 kvm->mmu_invalidate_range_start = 754 min(kvm->mmu_invalidate_range_start, start); 755 kvm->mmu_invalidate_range_end = 756 max(kvm->mmu_invalidate_range_end, end); 757 } 758 } 759 760 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 761 const struct mmu_notifier_range *range) 762 { 763 struct kvm *kvm = mmu_notifier_to_kvm(mn); 764 const struct kvm_hva_range hva_range = { 765 .start = range->start, 766 .end = range->end, 767 .pte = __pte(0), 768 .handler = kvm_unmap_gfn_range, 769 .on_lock = kvm_mmu_invalidate_begin, 770 .on_unlock = kvm_arch_guest_memory_reclaimed, 771 .flush_on_ret = true, 772 .may_block = mmu_notifier_range_blockable(range), 773 }; 774 775 trace_kvm_unmap_hva_range(range->start, range->end); 776 777 /* 778 * Prevent memslot modification between range_start() and range_end() 779 * so that conditionally locking provides the same result in both 780 * functions. Without that guarantee, the mmu_invalidate_in_progress 781 * adjustments will be imbalanced. 782 * 783 * Pairs with the decrement in range_end(). 784 */ 785 spin_lock(&kvm->mn_invalidate_lock); 786 kvm->mn_active_invalidate_count++; 787 spin_unlock(&kvm->mn_invalidate_lock); 788 789 /* 790 * Invalidate pfn caches _before_ invalidating the secondary MMUs, i.e. 791 * before acquiring mmu_lock, to avoid holding mmu_lock while acquiring 792 * each cache's lock. There are relatively few caches in existence at 793 * any given time, and the caches themselves can check for hva overlap, 794 * i.e. don't need to rely on memslot overlap checks for performance. 795 * Because this runs without holding mmu_lock, the pfn caches must use 796 * mn_active_invalidate_count (see above) instead of 797 * mmu_invalidate_in_progress. 798 */ 799 gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end, 800 hva_range.may_block); 801 802 __kvm_handle_hva_range(kvm, &hva_range); 803 804 return 0; 805 } 806 807 void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start, 808 unsigned long end) 809 { 810 /* 811 * This sequence increase will notify the kvm page fault that 812 * the page that is going to be mapped in the spte could have 813 * been freed. 814 */ 815 kvm->mmu_invalidate_seq++; 816 smp_wmb(); 817 /* 818 * The above sequence increase must be visible before the 819 * below count decrease, which is ensured by the smp_wmb above 820 * in conjunction with the smp_rmb in mmu_invalidate_retry(). 821 */ 822 kvm->mmu_invalidate_in_progress--; 823 } 824 825 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 826 const struct mmu_notifier_range *range) 827 { 828 struct kvm *kvm = mmu_notifier_to_kvm(mn); 829 const struct kvm_hva_range hva_range = { 830 .start = range->start, 831 .end = range->end, 832 .pte = __pte(0), 833 .handler = (void *)kvm_null_fn, 834 .on_lock = kvm_mmu_invalidate_end, 835 .on_unlock = (void *)kvm_null_fn, 836 .flush_on_ret = false, 837 .may_block = mmu_notifier_range_blockable(range), 838 }; 839 bool wake; 840 841 __kvm_handle_hva_range(kvm, &hva_range); 842 843 /* Pairs with the increment in range_start(). */ 844 spin_lock(&kvm->mn_invalidate_lock); 845 wake = (--kvm->mn_active_invalidate_count == 0); 846 spin_unlock(&kvm->mn_invalidate_lock); 847 848 /* 849 * There can only be one waiter, since the wait happens under 850 * slots_lock. 851 */ 852 if (wake) 853 rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait); 854 855 BUG_ON(kvm->mmu_invalidate_in_progress < 0); 856 } 857 858 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 859 struct mm_struct *mm, 860 unsigned long start, 861 unsigned long end) 862 { 863 trace_kvm_age_hva(start, end); 864 865 return kvm_handle_hva_range(mn, start, end, __pte(0), kvm_age_gfn); 866 } 867 868 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 869 struct mm_struct *mm, 870 unsigned long start, 871 unsigned long end) 872 { 873 trace_kvm_age_hva(start, end); 874 875 /* 876 * Even though we do not flush TLB, this will still adversely 877 * affect performance on pre-Haswell Intel EPT, where there is 878 * no EPT Access Bit to clear so that we have to tear down EPT 879 * tables instead. If we find this unacceptable, we can always 880 * add a parameter to kvm_age_hva so that it effectively doesn't 881 * do anything on clear_young. 882 * 883 * Also note that currently we never issue secondary TLB flushes 884 * from clear_young, leaving this job up to the regular system 885 * cadence. If we find this inaccurate, we might come up with a 886 * more sophisticated heuristic later. 887 */ 888 return kvm_handle_hva_range_no_flush(mn, start, end, kvm_age_gfn); 889 } 890 891 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 892 struct mm_struct *mm, 893 unsigned long address) 894 { 895 trace_kvm_test_age_hva(address); 896 897 return kvm_handle_hva_range_no_flush(mn, address, address + 1, 898 kvm_test_age_gfn); 899 } 900 901 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 902 struct mm_struct *mm) 903 { 904 struct kvm *kvm = mmu_notifier_to_kvm(mn); 905 int idx; 906 907 idx = srcu_read_lock(&kvm->srcu); 908 kvm_flush_shadow_all(kvm); 909 srcu_read_unlock(&kvm->srcu, idx); 910 } 911 912 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 913 .invalidate_range = kvm_mmu_notifier_invalidate_range, 914 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 915 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 916 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 917 .clear_young = kvm_mmu_notifier_clear_young, 918 .test_young = kvm_mmu_notifier_test_young, 919 .change_pte = kvm_mmu_notifier_change_pte, 920 .release = kvm_mmu_notifier_release, 921 }; 922 923 static int kvm_init_mmu_notifier(struct kvm *kvm) 924 { 925 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 926 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 927 } 928 929 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 930 931 static int kvm_init_mmu_notifier(struct kvm *kvm) 932 { 933 return 0; 934 } 935 936 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 937 938 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 939 static int kvm_pm_notifier_call(struct notifier_block *bl, 940 unsigned long state, 941 void *unused) 942 { 943 struct kvm *kvm = container_of(bl, struct kvm, pm_notifier); 944 945 return kvm_arch_pm_notifier(kvm, state); 946 } 947 948 static void kvm_init_pm_notifier(struct kvm *kvm) 949 { 950 kvm->pm_notifier.notifier_call = kvm_pm_notifier_call; 951 /* Suspend KVM before we suspend ftrace, RCU, etc. */ 952 kvm->pm_notifier.priority = INT_MAX; 953 register_pm_notifier(&kvm->pm_notifier); 954 } 955 956 static void kvm_destroy_pm_notifier(struct kvm *kvm) 957 { 958 unregister_pm_notifier(&kvm->pm_notifier); 959 } 960 #else /* !CONFIG_HAVE_KVM_PM_NOTIFIER */ 961 static void kvm_init_pm_notifier(struct kvm *kvm) 962 { 963 } 964 965 static void kvm_destroy_pm_notifier(struct kvm *kvm) 966 { 967 } 968 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ 969 970 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 971 { 972 if (!memslot->dirty_bitmap) 973 return; 974 975 kvfree(memslot->dirty_bitmap); 976 memslot->dirty_bitmap = NULL; 977 } 978 979 /* This does not remove the slot from struct kvm_memslots data structures */ 980 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 981 { 982 kvm_destroy_dirty_bitmap(slot); 983 984 kvm_arch_free_memslot(kvm, slot); 985 986 kfree(slot); 987 } 988 989 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 990 { 991 struct hlist_node *idnode; 992 struct kvm_memory_slot *memslot; 993 int bkt; 994 995 /* 996 * The same memslot objects live in both active and inactive sets, 997 * arbitrarily free using index '1' so the second invocation of this 998 * function isn't operating over a structure with dangling pointers 999 * (even though this function isn't actually touching them). 1000 */ 1001 if (!slots->node_idx) 1002 return; 1003 1004 hash_for_each_safe(slots->id_hash, bkt, idnode, memslot, id_node[1]) 1005 kvm_free_memslot(kvm, memslot); 1006 } 1007 1008 static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc) 1009 { 1010 switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) { 1011 case KVM_STATS_TYPE_INSTANT: 1012 return 0444; 1013 case KVM_STATS_TYPE_CUMULATIVE: 1014 case KVM_STATS_TYPE_PEAK: 1015 default: 1016 return 0644; 1017 } 1018 } 1019 1020 1021 static void kvm_destroy_vm_debugfs(struct kvm *kvm) 1022 { 1023 int i; 1024 int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc + 1025 kvm_vcpu_stats_header.num_desc; 1026 1027 if (IS_ERR(kvm->debugfs_dentry)) 1028 return; 1029 1030 debugfs_remove_recursive(kvm->debugfs_dentry); 1031 1032 if (kvm->debugfs_stat_data) { 1033 for (i = 0; i < kvm_debugfs_num_entries; i++) 1034 kfree(kvm->debugfs_stat_data[i]); 1035 kfree(kvm->debugfs_stat_data); 1036 } 1037 } 1038 1039 static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) 1040 { 1041 static DEFINE_MUTEX(kvm_debugfs_lock); 1042 struct dentry *dent; 1043 char dir_name[ITOA_MAX_LEN * 2]; 1044 struct kvm_stat_data *stat_data; 1045 const struct _kvm_stats_desc *pdesc; 1046 int i, ret = -ENOMEM; 1047 int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc + 1048 kvm_vcpu_stats_header.num_desc; 1049 1050 if (!debugfs_initialized()) 1051 return 0; 1052 1053 snprintf(dir_name, sizeof(dir_name), "%d-%s", task_pid_nr(current), fdname); 1054 mutex_lock(&kvm_debugfs_lock); 1055 dent = debugfs_lookup(dir_name, kvm_debugfs_dir); 1056 if (dent) { 1057 pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name); 1058 dput(dent); 1059 mutex_unlock(&kvm_debugfs_lock); 1060 return 0; 1061 } 1062 dent = debugfs_create_dir(dir_name, kvm_debugfs_dir); 1063 mutex_unlock(&kvm_debugfs_lock); 1064 if (IS_ERR(dent)) 1065 return 0; 1066 1067 kvm->debugfs_dentry = dent; 1068 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, 1069 sizeof(*kvm->debugfs_stat_data), 1070 GFP_KERNEL_ACCOUNT); 1071 if (!kvm->debugfs_stat_data) 1072 goto out_err; 1073 1074 for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { 1075 pdesc = &kvm_vm_stats_desc[i]; 1076 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT); 1077 if (!stat_data) 1078 goto out_err; 1079 1080 stat_data->kvm = kvm; 1081 stat_data->desc = pdesc; 1082 stat_data->kind = KVM_STAT_VM; 1083 kvm->debugfs_stat_data[i] = stat_data; 1084 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), 1085 kvm->debugfs_dentry, stat_data, 1086 &stat_fops_per_vm); 1087 } 1088 1089 for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) { 1090 pdesc = &kvm_vcpu_stats_desc[i]; 1091 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT); 1092 if (!stat_data) 1093 goto out_err; 1094 1095 stat_data->kvm = kvm; 1096 stat_data->desc = pdesc; 1097 stat_data->kind = KVM_STAT_VCPU; 1098 kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data; 1099 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), 1100 kvm->debugfs_dentry, stat_data, 1101 &stat_fops_per_vm); 1102 } 1103 1104 ret = kvm_arch_create_vm_debugfs(kvm); 1105 if (ret) 1106 goto out_err; 1107 1108 return 0; 1109 out_err: 1110 kvm_destroy_vm_debugfs(kvm); 1111 return ret; 1112 } 1113 1114 /* 1115 * Called after the VM is otherwise initialized, but just before adding it to 1116 * the vm_list. 1117 */ 1118 int __weak kvm_arch_post_init_vm(struct kvm *kvm) 1119 { 1120 return 0; 1121 } 1122 1123 /* 1124 * Called just after removing the VM from the vm_list, but before doing any 1125 * other destruction. 1126 */ 1127 void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm) 1128 { 1129 } 1130 1131 /* 1132 * Called after per-vm debugfs created. When called kvm->debugfs_dentry should 1133 * be setup already, so we can create arch-specific debugfs entries under it. 1134 * Cleanup should be automatic done in kvm_destroy_vm_debugfs() recursively, so 1135 * a per-arch destroy interface is not needed. 1136 */ 1137 int __weak kvm_arch_create_vm_debugfs(struct kvm *kvm) 1138 { 1139 return 0; 1140 } 1141 1142 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) 1143 { 1144 struct kvm *kvm = kvm_arch_alloc_vm(); 1145 struct kvm_memslots *slots; 1146 int r = -ENOMEM; 1147 int i, j; 1148 1149 if (!kvm) 1150 return ERR_PTR(-ENOMEM); 1151 1152 /* KVM is pinned via open("/dev/kvm"), the fd passed to this ioctl(). */ 1153 __module_get(kvm_chardev_ops.owner); 1154 1155 KVM_MMU_LOCK_INIT(kvm); 1156 mmgrab(current->mm); 1157 kvm->mm = current->mm; 1158 kvm_eventfd_init(kvm); 1159 mutex_init(&kvm->lock); 1160 mutex_init(&kvm->irq_lock); 1161 mutex_init(&kvm->slots_lock); 1162 mutex_init(&kvm->slots_arch_lock); 1163 spin_lock_init(&kvm->mn_invalidate_lock); 1164 rcuwait_init(&kvm->mn_memslots_update_rcuwait); 1165 xa_init(&kvm->vcpu_array); 1166 1167 INIT_LIST_HEAD(&kvm->gpc_list); 1168 spin_lock_init(&kvm->gpc_lock); 1169 1170 INIT_LIST_HEAD(&kvm->devices); 1171 kvm->max_vcpus = KVM_MAX_VCPUS; 1172 1173 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 1174 1175 /* 1176 * Force subsequent debugfs file creations to fail if the VM directory 1177 * is not created (by kvm_create_vm_debugfs()). 1178 */ 1179 kvm->debugfs_dentry = ERR_PTR(-ENOENT); 1180 1181 snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d", 1182 task_pid_nr(current)); 1183 1184 if (init_srcu_struct(&kvm->srcu)) 1185 goto out_err_no_srcu; 1186 if (init_srcu_struct(&kvm->irq_srcu)) 1187 goto out_err_no_irq_srcu; 1188 1189 refcount_set(&kvm->users_count, 1); 1190 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 1191 for (j = 0; j < 2; j++) { 1192 slots = &kvm->__memslots[i][j]; 1193 1194 atomic_long_set(&slots->last_used_slot, (unsigned long)NULL); 1195 slots->hva_tree = RB_ROOT_CACHED; 1196 slots->gfn_tree = RB_ROOT; 1197 hash_init(slots->id_hash); 1198 slots->node_idx = j; 1199 1200 /* Generations must be different for each address space. */ 1201 slots->generation = i; 1202 } 1203 1204 rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]); 1205 } 1206 1207 for (i = 0; i < KVM_NR_BUSES; i++) { 1208 rcu_assign_pointer(kvm->buses[i], 1209 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT)); 1210 if (!kvm->buses[i]) 1211 goto out_err_no_arch_destroy_vm; 1212 } 1213 1214 r = kvm_arch_init_vm(kvm, type); 1215 if (r) 1216 goto out_err_no_arch_destroy_vm; 1217 1218 r = hardware_enable_all(); 1219 if (r) 1220 goto out_err_no_disable; 1221 1222 #ifdef CONFIG_HAVE_KVM_IRQFD 1223 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 1224 #endif 1225 1226 r = kvm_init_mmu_notifier(kvm); 1227 if (r) 1228 goto out_err_no_mmu_notifier; 1229 1230 r = kvm_coalesced_mmio_init(kvm); 1231 if (r < 0) 1232 goto out_no_coalesced_mmio; 1233 1234 r = kvm_create_vm_debugfs(kvm, fdname); 1235 if (r) 1236 goto out_err_no_debugfs; 1237 1238 r = kvm_arch_post_init_vm(kvm); 1239 if (r) 1240 goto out_err; 1241 1242 mutex_lock(&kvm_lock); 1243 list_add(&kvm->vm_list, &vm_list); 1244 mutex_unlock(&kvm_lock); 1245 1246 preempt_notifier_inc(); 1247 kvm_init_pm_notifier(kvm); 1248 1249 return kvm; 1250 1251 out_err: 1252 kvm_destroy_vm_debugfs(kvm); 1253 out_err_no_debugfs: 1254 kvm_coalesced_mmio_free(kvm); 1255 out_no_coalesced_mmio: 1256 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 1257 if (kvm->mmu_notifier.ops) 1258 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm); 1259 #endif 1260 out_err_no_mmu_notifier: 1261 hardware_disable_all(); 1262 out_err_no_disable: 1263 kvm_arch_destroy_vm(kvm); 1264 out_err_no_arch_destroy_vm: 1265 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count)); 1266 for (i = 0; i < KVM_NR_BUSES; i++) 1267 kfree(kvm_get_bus(kvm, i)); 1268 cleanup_srcu_struct(&kvm->irq_srcu); 1269 out_err_no_irq_srcu: 1270 cleanup_srcu_struct(&kvm->srcu); 1271 out_err_no_srcu: 1272 kvm_arch_free_vm(kvm); 1273 mmdrop(current->mm); 1274 module_put(kvm_chardev_ops.owner); 1275 return ERR_PTR(r); 1276 } 1277 1278 static void kvm_destroy_devices(struct kvm *kvm) 1279 { 1280 struct kvm_device *dev, *tmp; 1281 1282 /* 1283 * We do not need to take the kvm->lock here, because nobody else 1284 * has a reference to the struct kvm at this point and therefore 1285 * cannot access the devices list anyhow. 1286 */ 1287 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 1288 list_del(&dev->vm_node); 1289 dev->ops->destroy(dev); 1290 } 1291 } 1292 1293 static void kvm_destroy_vm(struct kvm *kvm) 1294 { 1295 int i; 1296 struct mm_struct *mm = kvm->mm; 1297 1298 kvm_destroy_pm_notifier(kvm); 1299 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); 1300 kvm_destroy_vm_debugfs(kvm); 1301 kvm_arch_sync_events(kvm); 1302 mutex_lock(&kvm_lock); 1303 list_del(&kvm->vm_list); 1304 mutex_unlock(&kvm_lock); 1305 kvm_arch_pre_destroy_vm(kvm); 1306 1307 kvm_free_irq_routing(kvm); 1308 for (i = 0; i < KVM_NR_BUSES; i++) { 1309 struct kvm_io_bus *bus = kvm_get_bus(kvm, i); 1310 1311 if (bus) 1312 kvm_io_bus_destroy(bus); 1313 kvm->buses[i] = NULL; 1314 } 1315 kvm_coalesced_mmio_free(kvm); 1316 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 1317 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 1318 /* 1319 * At this point, pending calls to invalidate_range_start() 1320 * have completed but no more MMU notifiers will run, so 1321 * mn_active_invalidate_count may remain unbalanced. 1322 * No threads can be waiting in kvm_swap_active_memslots() as the 1323 * last reference on KVM has been dropped, but freeing 1324 * memslots would deadlock without this manual intervention. 1325 */ 1326 WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait)); 1327 kvm->mn_active_invalidate_count = 0; 1328 #else 1329 kvm_flush_shadow_all(kvm); 1330 #endif 1331 kvm_arch_destroy_vm(kvm); 1332 kvm_destroy_devices(kvm); 1333 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 1334 kvm_free_memslots(kvm, &kvm->__memslots[i][0]); 1335 kvm_free_memslots(kvm, &kvm->__memslots[i][1]); 1336 } 1337 cleanup_srcu_struct(&kvm->irq_srcu); 1338 cleanup_srcu_struct(&kvm->srcu); 1339 kvm_arch_free_vm(kvm); 1340 preempt_notifier_dec(); 1341 hardware_disable_all(); 1342 mmdrop(mm); 1343 module_put(kvm_chardev_ops.owner); 1344 } 1345 1346 void kvm_get_kvm(struct kvm *kvm) 1347 { 1348 refcount_inc(&kvm->users_count); 1349 } 1350 EXPORT_SYMBOL_GPL(kvm_get_kvm); 1351 1352 /* 1353 * Make sure the vm is not during destruction, which is a safe version of 1354 * kvm_get_kvm(). Return true if kvm referenced successfully, false otherwise. 1355 */ 1356 bool kvm_get_kvm_safe(struct kvm *kvm) 1357 { 1358 return refcount_inc_not_zero(&kvm->users_count); 1359 } 1360 EXPORT_SYMBOL_GPL(kvm_get_kvm_safe); 1361 1362 void kvm_put_kvm(struct kvm *kvm) 1363 { 1364 if (refcount_dec_and_test(&kvm->users_count)) 1365 kvm_destroy_vm(kvm); 1366 } 1367 EXPORT_SYMBOL_GPL(kvm_put_kvm); 1368 1369 /* 1370 * Used to put a reference that was taken on behalf of an object associated 1371 * with a user-visible file descriptor, e.g. a vcpu or device, if installation 1372 * of the new file descriptor fails and the reference cannot be transferred to 1373 * its final owner. In such cases, the caller is still actively using @kvm and 1374 * will fail miserably if the refcount unexpectedly hits zero. 1375 */ 1376 void kvm_put_kvm_no_destroy(struct kvm *kvm) 1377 { 1378 WARN_ON(refcount_dec_and_test(&kvm->users_count)); 1379 } 1380 EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy); 1381 1382 static int kvm_vm_release(struct inode *inode, struct file *filp) 1383 { 1384 struct kvm *kvm = filp->private_data; 1385 1386 kvm_irqfd_release(kvm); 1387 1388 kvm_put_kvm(kvm); 1389 return 0; 1390 } 1391 1392 /* 1393 * Allocation size is twice as large as the actual dirty bitmap size. 1394 * See kvm_vm_ioctl_get_dirty_log() why this is needed. 1395 */ 1396 static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) 1397 { 1398 unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot); 1399 1400 memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT); 1401 if (!memslot->dirty_bitmap) 1402 return -ENOMEM; 1403 1404 return 0; 1405 } 1406 1407 static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id) 1408 { 1409 struct kvm_memslots *active = __kvm_memslots(kvm, as_id); 1410 int node_idx_inactive = active->node_idx ^ 1; 1411 1412 return &kvm->__memslots[as_id][node_idx_inactive]; 1413 } 1414 1415 /* 1416 * Helper to get the address space ID when one of memslot pointers may be NULL. 1417 * This also serves as a sanity that at least one of the pointers is non-NULL, 1418 * and that their address space IDs don't diverge. 1419 */ 1420 static int kvm_memslots_get_as_id(struct kvm_memory_slot *a, 1421 struct kvm_memory_slot *b) 1422 { 1423 if (WARN_ON_ONCE(!a && !b)) 1424 return 0; 1425 1426 if (!a) 1427 return b->as_id; 1428 if (!b) 1429 return a->as_id; 1430 1431 WARN_ON_ONCE(a->as_id != b->as_id); 1432 return a->as_id; 1433 } 1434 1435 static void kvm_insert_gfn_node(struct kvm_memslots *slots, 1436 struct kvm_memory_slot *slot) 1437 { 1438 struct rb_root *gfn_tree = &slots->gfn_tree; 1439 struct rb_node **node, *parent; 1440 int idx = slots->node_idx; 1441 1442 parent = NULL; 1443 for (node = &gfn_tree->rb_node; *node; ) { 1444 struct kvm_memory_slot *tmp; 1445 1446 tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]); 1447 parent = *node; 1448 if (slot->base_gfn < tmp->base_gfn) 1449 node = &(*node)->rb_left; 1450 else if (slot->base_gfn > tmp->base_gfn) 1451 node = &(*node)->rb_right; 1452 else 1453 BUG(); 1454 } 1455 1456 rb_link_node(&slot->gfn_node[idx], parent, node); 1457 rb_insert_color(&slot->gfn_node[idx], gfn_tree); 1458 } 1459 1460 static void kvm_erase_gfn_node(struct kvm_memslots *slots, 1461 struct kvm_memory_slot *slot) 1462 { 1463 rb_erase(&slot->gfn_node[slots->node_idx], &slots->gfn_tree); 1464 } 1465 1466 static void kvm_replace_gfn_node(struct kvm_memslots *slots, 1467 struct kvm_memory_slot *old, 1468 struct kvm_memory_slot *new) 1469 { 1470 int idx = slots->node_idx; 1471 1472 WARN_ON_ONCE(old->base_gfn != new->base_gfn); 1473 1474 rb_replace_node(&old->gfn_node[idx], &new->gfn_node[idx], 1475 &slots->gfn_tree); 1476 } 1477 1478 /* 1479 * Replace @old with @new in the inactive memslots. 1480 * 1481 * With NULL @old this simply adds @new. 1482 * With NULL @new this simply removes @old. 1483 * 1484 * If @new is non-NULL its hva_node[slots_idx] range has to be set 1485 * appropriately. 1486 */ 1487 static void kvm_replace_memslot(struct kvm *kvm, 1488 struct kvm_memory_slot *old, 1489 struct kvm_memory_slot *new) 1490 { 1491 int as_id = kvm_memslots_get_as_id(old, new); 1492 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id); 1493 int idx = slots->node_idx; 1494 1495 if (old) { 1496 hash_del(&old->id_node[idx]); 1497 interval_tree_remove(&old->hva_node[idx], &slots->hva_tree); 1498 1499 if ((long)old == atomic_long_read(&slots->last_used_slot)) 1500 atomic_long_set(&slots->last_used_slot, (long)new); 1501 1502 if (!new) { 1503 kvm_erase_gfn_node(slots, old); 1504 return; 1505 } 1506 } 1507 1508 /* 1509 * Initialize @new's hva range. Do this even when replacing an @old 1510 * slot, kvm_copy_memslot() deliberately does not touch node data. 1511 */ 1512 new->hva_node[idx].start = new->userspace_addr; 1513 new->hva_node[idx].last = new->userspace_addr + 1514 (new->npages << PAGE_SHIFT) - 1; 1515 1516 /* 1517 * (Re)Add the new memslot. There is no O(1) interval_tree_replace(), 1518 * hva_node needs to be swapped with remove+insert even though hva can't 1519 * change when replacing an existing slot. 1520 */ 1521 hash_add(slots->id_hash, &new->id_node[idx], new->id); 1522 interval_tree_insert(&new->hva_node[idx], &slots->hva_tree); 1523 1524 /* 1525 * If the memslot gfn is unchanged, rb_replace_node() can be used to 1526 * switch the node in the gfn tree instead of removing the old and 1527 * inserting the new as two separate operations. Replacement is a 1528 * single O(1) operation versus two O(log(n)) operations for 1529 * remove+insert. 1530 */ 1531 if (old && old->base_gfn == new->base_gfn) { 1532 kvm_replace_gfn_node(slots, old, new); 1533 } else { 1534 if (old) 1535 kvm_erase_gfn_node(slots, old); 1536 kvm_insert_gfn_node(slots, new); 1537 } 1538 } 1539 1540 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 1541 { 1542 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 1543 1544 #ifdef __KVM_HAVE_READONLY_MEM 1545 valid_flags |= KVM_MEM_READONLY; 1546 #endif 1547 1548 if (mem->flags & ~valid_flags) 1549 return -EINVAL; 1550 1551 return 0; 1552 } 1553 1554 static void kvm_swap_active_memslots(struct kvm *kvm, int as_id) 1555 { 1556 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id); 1557 1558 /* Grab the generation from the activate memslots. */ 1559 u64 gen = __kvm_memslots(kvm, as_id)->generation; 1560 1561 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); 1562 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; 1563 1564 /* 1565 * Do not store the new memslots while there are invalidations in 1566 * progress, otherwise the locking in invalidate_range_start and 1567 * invalidate_range_end will be unbalanced. 1568 */ 1569 spin_lock(&kvm->mn_invalidate_lock); 1570 prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait); 1571 while (kvm->mn_active_invalidate_count) { 1572 set_current_state(TASK_UNINTERRUPTIBLE); 1573 spin_unlock(&kvm->mn_invalidate_lock); 1574 schedule(); 1575 spin_lock(&kvm->mn_invalidate_lock); 1576 } 1577 finish_rcuwait(&kvm->mn_memslots_update_rcuwait); 1578 rcu_assign_pointer(kvm->memslots[as_id], slots); 1579 spin_unlock(&kvm->mn_invalidate_lock); 1580 1581 /* 1582 * Acquired in kvm_set_memslot. Must be released before synchronize 1583 * SRCU below in order to avoid deadlock with another thread 1584 * acquiring the slots_arch_lock in an srcu critical section. 1585 */ 1586 mutex_unlock(&kvm->slots_arch_lock); 1587 1588 synchronize_srcu_expedited(&kvm->srcu); 1589 1590 /* 1591 * Increment the new memslot generation a second time, dropping the 1592 * update in-progress flag and incrementing the generation based on 1593 * the number of address spaces. This provides a unique and easily 1594 * identifiable generation number while the memslots are in flux. 1595 */ 1596 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; 1597 1598 /* 1599 * Generations must be unique even across address spaces. We do not need 1600 * a global counter for that, instead the generation space is evenly split 1601 * across address spaces. For example, with two address spaces, address 1602 * space 0 will use generations 0, 2, 4, ... while address space 1 will 1603 * use generations 1, 3, 5, ... 1604 */ 1605 gen += KVM_ADDRESS_SPACE_NUM; 1606 1607 kvm_arch_memslots_updated(kvm, gen); 1608 1609 slots->generation = gen; 1610 } 1611 1612 static int kvm_prepare_memory_region(struct kvm *kvm, 1613 const struct kvm_memory_slot *old, 1614 struct kvm_memory_slot *new, 1615 enum kvm_mr_change change) 1616 { 1617 int r; 1618 1619 /* 1620 * If dirty logging is disabled, nullify the bitmap; the old bitmap 1621 * will be freed on "commit". If logging is enabled in both old and 1622 * new, reuse the existing bitmap. If logging is enabled only in the 1623 * new and KVM isn't using a ring buffer, allocate and initialize a 1624 * new bitmap. 1625 */ 1626 if (change != KVM_MR_DELETE) { 1627 if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) 1628 new->dirty_bitmap = NULL; 1629 else if (old && old->dirty_bitmap) 1630 new->dirty_bitmap = old->dirty_bitmap; 1631 else if (kvm_use_dirty_bitmap(kvm)) { 1632 r = kvm_alloc_dirty_bitmap(new); 1633 if (r) 1634 return r; 1635 1636 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) 1637 bitmap_set(new->dirty_bitmap, 0, new->npages); 1638 } 1639 } 1640 1641 r = kvm_arch_prepare_memory_region(kvm, old, new, change); 1642 1643 /* Free the bitmap on failure if it was allocated above. */ 1644 if (r && new && new->dirty_bitmap && (!old || !old->dirty_bitmap)) 1645 kvm_destroy_dirty_bitmap(new); 1646 1647 return r; 1648 } 1649 1650 static void kvm_commit_memory_region(struct kvm *kvm, 1651 struct kvm_memory_slot *old, 1652 const struct kvm_memory_slot *new, 1653 enum kvm_mr_change change) 1654 { 1655 int old_flags = old ? old->flags : 0; 1656 int new_flags = new ? new->flags : 0; 1657 /* 1658 * Update the total number of memslot pages before calling the arch 1659 * hook so that architectures can consume the result directly. 1660 */ 1661 if (change == KVM_MR_DELETE) 1662 kvm->nr_memslot_pages -= old->npages; 1663 else if (change == KVM_MR_CREATE) 1664 kvm->nr_memslot_pages += new->npages; 1665 1666 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) { 1667 int change = (new_flags & KVM_MEM_LOG_DIRTY_PAGES) ? 1 : -1; 1668 atomic_set(&kvm->nr_memslots_dirty_logging, 1669 atomic_read(&kvm->nr_memslots_dirty_logging) + change); 1670 } 1671 1672 kvm_arch_commit_memory_region(kvm, old, new, change); 1673 1674 switch (change) { 1675 case KVM_MR_CREATE: 1676 /* Nothing more to do. */ 1677 break; 1678 case KVM_MR_DELETE: 1679 /* Free the old memslot and all its metadata. */ 1680 kvm_free_memslot(kvm, old); 1681 break; 1682 case KVM_MR_MOVE: 1683 case KVM_MR_FLAGS_ONLY: 1684 /* 1685 * Free the dirty bitmap as needed; the below check encompasses 1686 * both the flags and whether a ring buffer is being used) 1687 */ 1688 if (old->dirty_bitmap && !new->dirty_bitmap) 1689 kvm_destroy_dirty_bitmap(old); 1690 1691 /* 1692 * The final quirk. Free the detached, old slot, but only its 1693 * memory, not any metadata. Metadata, including arch specific 1694 * data, may be reused by @new. 1695 */ 1696 kfree(old); 1697 break; 1698 default: 1699 BUG(); 1700 } 1701 } 1702 1703 /* 1704 * Activate @new, which must be installed in the inactive slots by the caller, 1705 * by swapping the active slots and then propagating @new to @old once @old is 1706 * unreachable and can be safely modified. 1707 * 1708 * With NULL @old this simply adds @new to @active (while swapping the sets). 1709 * With NULL @new this simply removes @old from @active and frees it 1710 * (while also swapping the sets). 1711 */ 1712 static void kvm_activate_memslot(struct kvm *kvm, 1713 struct kvm_memory_slot *old, 1714 struct kvm_memory_slot *new) 1715 { 1716 int as_id = kvm_memslots_get_as_id(old, new); 1717 1718 kvm_swap_active_memslots(kvm, as_id); 1719 1720 /* Propagate the new memslot to the now inactive memslots. */ 1721 kvm_replace_memslot(kvm, old, new); 1722 } 1723 1724 static void kvm_copy_memslot(struct kvm_memory_slot *dest, 1725 const struct kvm_memory_slot *src) 1726 { 1727 dest->base_gfn = src->base_gfn; 1728 dest->npages = src->npages; 1729 dest->dirty_bitmap = src->dirty_bitmap; 1730 dest->arch = src->arch; 1731 dest->userspace_addr = src->userspace_addr; 1732 dest->flags = src->flags; 1733 dest->id = src->id; 1734 dest->as_id = src->as_id; 1735 } 1736 1737 static void kvm_invalidate_memslot(struct kvm *kvm, 1738 struct kvm_memory_slot *old, 1739 struct kvm_memory_slot *invalid_slot) 1740 { 1741 /* 1742 * Mark the current slot INVALID. As with all memslot modifications, 1743 * this must be done on an unreachable slot to avoid modifying the 1744 * current slot in the active tree. 1745 */ 1746 kvm_copy_memslot(invalid_slot, old); 1747 invalid_slot->flags |= KVM_MEMSLOT_INVALID; 1748 kvm_replace_memslot(kvm, old, invalid_slot); 1749 1750 /* 1751 * Activate the slot that is now marked INVALID, but don't propagate 1752 * the slot to the now inactive slots. The slot is either going to be 1753 * deleted or recreated as a new slot. 1754 */ 1755 kvm_swap_active_memslots(kvm, old->as_id); 1756 1757 /* 1758 * From this point no new shadow pages pointing to a deleted, or moved, 1759 * memslot will be created. Validation of sp->gfn happens in: 1760 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 1761 * - kvm_is_visible_gfn (mmu_check_root) 1762 */ 1763 kvm_arch_flush_shadow_memslot(kvm, old); 1764 kvm_arch_guest_memory_reclaimed(kvm); 1765 1766 /* Was released by kvm_swap_active_memslots(), reacquire. */ 1767 mutex_lock(&kvm->slots_arch_lock); 1768 1769 /* 1770 * Copy the arch-specific field of the newly-installed slot back to the 1771 * old slot as the arch data could have changed between releasing 1772 * slots_arch_lock in kvm_swap_active_memslots() and re-acquiring the lock 1773 * above. Writers are required to retrieve memslots *after* acquiring 1774 * slots_arch_lock, thus the active slot's data is guaranteed to be fresh. 1775 */ 1776 old->arch = invalid_slot->arch; 1777 } 1778 1779 static void kvm_create_memslot(struct kvm *kvm, 1780 struct kvm_memory_slot *new) 1781 { 1782 /* Add the new memslot to the inactive set and activate. */ 1783 kvm_replace_memslot(kvm, NULL, new); 1784 kvm_activate_memslot(kvm, NULL, new); 1785 } 1786 1787 static void kvm_delete_memslot(struct kvm *kvm, 1788 struct kvm_memory_slot *old, 1789 struct kvm_memory_slot *invalid_slot) 1790 { 1791 /* 1792 * Remove the old memslot (in the inactive memslots) by passing NULL as 1793 * the "new" slot, and for the invalid version in the active slots. 1794 */ 1795 kvm_replace_memslot(kvm, old, NULL); 1796 kvm_activate_memslot(kvm, invalid_slot, NULL); 1797 } 1798 1799 static void kvm_move_memslot(struct kvm *kvm, 1800 struct kvm_memory_slot *old, 1801 struct kvm_memory_slot *new, 1802 struct kvm_memory_slot *invalid_slot) 1803 { 1804 /* 1805 * Replace the old memslot in the inactive slots, and then swap slots 1806 * and replace the current INVALID with the new as well. 1807 */ 1808 kvm_replace_memslot(kvm, old, new); 1809 kvm_activate_memslot(kvm, invalid_slot, new); 1810 } 1811 1812 static void kvm_update_flags_memslot(struct kvm *kvm, 1813 struct kvm_memory_slot *old, 1814 struct kvm_memory_slot *new) 1815 { 1816 /* 1817 * Similar to the MOVE case, but the slot doesn't need to be zapped as 1818 * an intermediate step. Instead, the old memslot is simply replaced 1819 * with a new, updated copy in both memslot sets. 1820 */ 1821 kvm_replace_memslot(kvm, old, new); 1822 kvm_activate_memslot(kvm, old, new); 1823 } 1824 1825 static int kvm_set_memslot(struct kvm *kvm, 1826 struct kvm_memory_slot *old, 1827 struct kvm_memory_slot *new, 1828 enum kvm_mr_change change) 1829 { 1830 struct kvm_memory_slot *invalid_slot; 1831 int r; 1832 1833 /* 1834 * Released in kvm_swap_active_memslots(). 1835 * 1836 * Must be held from before the current memslots are copied until after 1837 * the new memslots are installed with rcu_assign_pointer, then 1838 * released before the synchronize srcu in kvm_swap_active_memslots(). 1839 * 1840 * When modifying memslots outside of the slots_lock, must be held 1841 * before reading the pointer to the current memslots until after all 1842 * changes to those memslots are complete. 1843 * 1844 * These rules ensure that installing new memslots does not lose 1845 * changes made to the previous memslots. 1846 */ 1847 mutex_lock(&kvm->slots_arch_lock); 1848 1849 /* 1850 * Invalidate the old slot if it's being deleted or moved. This is 1851 * done prior to actually deleting/moving the memslot to allow vCPUs to 1852 * continue running by ensuring there are no mappings or shadow pages 1853 * for the memslot when it is deleted/moved. Without pre-invalidation 1854 * (and without a lock), a window would exist between effecting the 1855 * delete/move and committing the changes in arch code where KVM or a 1856 * guest could access a non-existent memslot. 1857 * 1858 * Modifications are done on a temporary, unreachable slot. The old 1859 * slot needs to be preserved in case a later step fails and the 1860 * invalidation needs to be reverted. 1861 */ 1862 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { 1863 invalid_slot = kzalloc(sizeof(*invalid_slot), GFP_KERNEL_ACCOUNT); 1864 if (!invalid_slot) { 1865 mutex_unlock(&kvm->slots_arch_lock); 1866 return -ENOMEM; 1867 } 1868 kvm_invalidate_memslot(kvm, old, invalid_slot); 1869 } 1870 1871 r = kvm_prepare_memory_region(kvm, old, new, change); 1872 if (r) { 1873 /* 1874 * For DELETE/MOVE, revert the above INVALID change. No 1875 * modifications required since the original slot was preserved 1876 * in the inactive slots. Changing the active memslots also 1877 * release slots_arch_lock. 1878 */ 1879 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { 1880 kvm_activate_memslot(kvm, invalid_slot, old); 1881 kfree(invalid_slot); 1882 } else { 1883 mutex_unlock(&kvm->slots_arch_lock); 1884 } 1885 return r; 1886 } 1887 1888 /* 1889 * For DELETE and MOVE, the working slot is now active as the INVALID 1890 * version of the old slot. MOVE is particularly special as it reuses 1891 * the old slot and returns a copy of the old slot (in working_slot). 1892 * For CREATE, there is no old slot. For DELETE and FLAGS_ONLY, the 1893 * old slot is detached but otherwise preserved. 1894 */ 1895 if (change == KVM_MR_CREATE) 1896 kvm_create_memslot(kvm, new); 1897 else if (change == KVM_MR_DELETE) 1898 kvm_delete_memslot(kvm, old, invalid_slot); 1899 else if (change == KVM_MR_MOVE) 1900 kvm_move_memslot(kvm, old, new, invalid_slot); 1901 else if (change == KVM_MR_FLAGS_ONLY) 1902 kvm_update_flags_memslot(kvm, old, new); 1903 else 1904 BUG(); 1905 1906 /* Free the temporary INVALID slot used for DELETE and MOVE. */ 1907 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) 1908 kfree(invalid_slot); 1909 1910 /* 1911 * No need to refresh new->arch, changes after dropping slots_arch_lock 1912 * will directly hit the final, active memslot. Architectures are 1913 * responsible for knowing that new->arch may be stale. 1914 */ 1915 kvm_commit_memory_region(kvm, old, new, change); 1916 1917 return 0; 1918 } 1919 1920 static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id, 1921 gfn_t start, gfn_t end) 1922 { 1923 struct kvm_memslot_iter iter; 1924 1925 kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { 1926 if (iter.slot->id != id) 1927 return true; 1928 } 1929 1930 return false; 1931 } 1932 1933 /* 1934 * Allocate some memory and give it an address in the guest physical address 1935 * space. 1936 * 1937 * Discontiguous memory is allowed, mostly for framebuffers. 1938 * 1939 * Must be called holding kvm->slots_lock for write. 1940 */ 1941 int __kvm_set_memory_region(struct kvm *kvm, 1942 const struct kvm_userspace_memory_region *mem) 1943 { 1944 struct kvm_memory_slot *old, *new; 1945 struct kvm_memslots *slots; 1946 enum kvm_mr_change change; 1947 unsigned long npages; 1948 gfn_t base_gfn; 1949 int as_id, id; 1950 int r; 1951 1952 r = check_memory_region_flags(mem); 1953 if (r) 1954 return r; 1955 1956 as_id = mem->slot >> 16; 1957 id = (u16)mem->slot; 1958 1959 /* General sanity checks */ 1960 if ((mem->memory_size & (PAGE_SIZE - 1)) || 1961 (mem->memory_size != (unsigned long)mem->memory_size)) 1962 return -EINVAL; 1963 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 1964 return -EINVAL; 1965 /* We can read the guest memory with __xxx_user() later on. */ 1966 if ((mem->userspace_addr & (PAGE_SIZE - 1)) || 1967 (mem->userspace_addr != untagged_addr(mem->userspace_addr)) || 1968 !access_ok((void __user *)(unsigned long)mem->userspace_addr, 1969 mem->memory_size)) 1970 return -EINVAL; 1971 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 1972 return -EINVAL; 1973 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 1974 return -EINVAL; 1975 if ((mem->memory_size >> PAGE_SHIFT) > KVM_MEM_MAX_NR_PAGES) 1976 return -EINVAL; 1977 1978 slots = __kvm_memslots(kvm, as_id); 1979 1980 /* 1981 * Note, the old memslot (and the pointer itself!) may be invalidated 1982 * and/or destroyed by kvm_set_memslot(). 1983 */ 1984 old = id_to_memslot(slots, id); 1985 1986 if (!mem->memory_size) { 1987 if (!old || !old->npages) 1988 return -EINVAL; 1989 1990 if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages)) 1991 return -EIO; 1992 1993 return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE); 1994 } 1995 1996 base_gfn = (mem->guest_phys_addr >> PAGE_SHIFT); 1997 npages = (mem->memory_size >> PAGE_SHIFT); 1998 1999 if (!old || !old->npages) { 2000 change = KVM_MR_CREATE; 2001 2002 /* 2003 * To simplify KVM internals, the total number of pages across 2004 * all memslots must fit in an unsigned long. 2005 */ 2006 if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages) 2007 return -EINVAL; 2008 } else { /* Modify an existing slot. */ 2009 if ((mem->userspace_addr != old->userspace_addr) || 2010 (npages != old->npages) || 2011 ((mem->flags ^ old->flags) & KVM_MEM_READONLY)) 2012 return -EINVAL; 2013 2014 if (base_gfn != old->base_gfn) 2015 change = KVM_MR_MOVE; 2016 else if (mem->flags != old->flags) 2017 change = KVM_MR_FLAGS_ONLY; 2018 else /* Nothing to change. */ 2019 return 0; 2020 } 2021 2022 if ((change == KVM_MR_CREATE || change == KVM_MR_MOVE) && 2023 kvm_check_memslot_overlap(slots, id, base_gfn, base_gfn + npages)) 2024 return -EEXIST; 2025 2026 /* Allocate a slot that will persist in the memslot. */ 2027 new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT); 2028 if (!new) 2029 return -ENOMEM; 2030 2031 new->as_id = as_id; 2032 new->id = id; 2033 new->base_gfn = base_gfn; 2034 new->npages = npages; 2035 new->flags = mem->flags; 2036 new->userspace_addr = mem->userspace_addr; 2037 2038 r = kvm_set_memslot(kvm, old, new, change); 2039 if (r) 2040 kfree(new); 2041 return r; 2042 } 2043 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 2044 2045 int kvm_set_memory_region(struct kvm *kvm, 2046 const struct kvm_userspace_memory_region *mem) 2047 { 2048 int r; 2049 2050 mutex_lock(&kvm->slots_lock); 2051 r = __kvm_set_memory_region(kvm, mem); 2052 mutex_unlock(&kvm->slots_lock); 2053 return r; 2054 } 2055 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 2056 2057 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 2058 struct kvm_userspace_memory_region *mem) 2059 { 2060 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 2061 return -EINVAL; 2062 2063 return kvm_set_memory_region(kvm, mem); 2064 } 2065 2066 #ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 2067 /** 2068 * kvm_get_dirty_log - get a snapshot of dirty pages 2069 * @kvm: pointer to kvm instance 2070 * @log: slot id and address to which we copy the log 2071 * @is_dirty: set to '1' if any dirty pages were found 2072 * @memslot: set to the associated memslot, always valid on success 2073 */ 2074 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, 2075 int *is_dirty, struct kvm_memory_slot **memslot) 2076 { 2077 struct kvm_memslots *slots; 2078 int i, as_id, id; 2079 unsigned long n; 2080 unsigned long any = 0; 2081 2082 /* Dirty ring tracking may be exclusive to dirty log tracking */ 2083 if (!kvm_use_dirty_bitmap(kvm)) 2084 return -ENXIO; 2085 2086 *memslot = NULL; 2087 *is_dirty = 0; 2088 2089 as_id = log->slot >> 16; 2090 id = (u16)log->slot; 2091 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 2092 return -EINVAL; 2093 2094 slots = __kvm_memslots(kvm, as_id); 2095 *memslot = id_to_memslot(slots, id); 2096 if (!(*memslot) || !(*memslot)->dirty_bitmap) 2097 return -ENOENT; 2098 2099 kvm_arch_sync_dirty_log(kvm, *memslot); 2100 2101 n = kvm_dirty_bitmap_bytes(*memslot); 2102 2103 for (i = 0; !any && i < n/sizeof(long); ++i) 2104 any = (*memslot)->dirty_bitmap[i]; 2105 2106 if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n)) 2107 return -EFAULT; 2108 2109 if (any) 2110 *is_dirty = 1; 2111 return 0; 2112 } 2113 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 2114 2115 #else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ 2116 /** 2117 * kvm_get_dirty_log_protect - get a snapshot of dirty pages 2118 * and reenable dirty page tracking for the corresponding pages. 2119 * @kvm: pointer to kvm instance 2120 * @log: slot id and address to which we copy the log 2121 * 2122 * We need to keep it in mind that VCPU threads can write to the bitmap 2123 * concurrently. So, to avoid losing track of dirty pages we keep the 2124 * following order: 2125 * 2126 * 1. Take a snapshot of the bit and clear it if needed. 2127 * 2. Write protect the corresponding page. 2128 * 3. Copy the snapshot to the userspace. 2129 * 4. Upon return caller flushes TLB's if needed. 2130 * 2131 * Between 2 and 4, the guest may write to the page using the remaining TLB 2132 * entry. This is not a problem because the page is reported dirty using 2133 * the snapshot taken before and step 4 ensures that writes done after 2134 * exiting to userspace will be logged for the next call. 2135 * 2136 */ 2137 static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) 2138 { 2139 struct kvm_memslots *slots; 2140 struct kvm_memory_slot *memslot; 2141 int i, as_id, id; 2142 unsigned long n; 2143 unsigned long *dirty_bitmap; 2144 unsigned long *dirty_bitmap_buffer; 2145 bool flush; 2146 2147 /* Dirty ring tracking may be exclusive to dirty log tracking */ 2148 if (!kvm_use_dirty_bitmap(kvm)) 2149 return -ENXIO; 2150 2151 as_id = log->slot >> 16; 2152 id = (u16)log->slot; 2153 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 2154 return -EINVAL; 2155 2156 slots = __kvm_memslots(kvm, as_id); 2157 memslot = id_to_memslot(slots, id); 2158 if (!memslot || !memslot->dirty_bitmap) 2159 return -ENOENT; 2160 2161 dirty_bitmap = memslot->dirty_bitmap; 2162 2163 kvm_arch_sync_dirty_log(kvm, memslot); 2164 2165 n = kvm_dirty_bitmap_bytes(memslot); 2166 flush = false; 2167 if (kvm->manual_dirty_log_protect) { 2168 /* 2169 * Unlike kvm_get_dirty_log, we always return false in *flush, 2170 * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There 2171 * is some code duplication between this function and 2172 * kvm_get_dirty_log, but hopefully all architecture 2173 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log 2174 * can be eliminated. 2175 */ 2176 dirty_bitmap_buffer = dirty_bitmap; 2177 } else { 2178 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot); 2179 memset(dirty_bitmap_buffer, 0, n); 2180 2181 KVM_MMU_LOCK(kvm); 2182 for (i = 0; i < n / sizeof(long); i++) { 2183 unsigned long mask; 2184 gfn_t offset; 2185 2186 if (!dirty_bitmap[i]) 2187 continue; 2188 2189 flush = true; 2190 mask = xchg(&dirty_bitmap[i], 0); 2191 dirty_bitmap_buffer[i] = mask; 2192 2193 offset = i * BITS_PER_LONG; 2194 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 2195 offset, mask); 2196 } 2197 KVM_MMU_UNLOCK(kvm); 2198 } 2199 2200 if (flush) 2201 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 2202 2203 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 2204 return -EFAULT; 2205 return 0; 2206 } 2207 2208 2209 /** 2210 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot 2211 * @kvm: kvm instance 2212 * @log: slot id and address to which we copy the log 2213 * 2214 * Steps 1-4 below provide general overview of dirty page logging. See 2215 * kvm_get_dirty_log_protect() function description for additional details. 2216 * 2217 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we 2218 * always flush the TLB (step 4) even if previous step failed and the dirty 2219 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API 2220 * does not preclude user space subsequent dirty log read. Flushing TLB ensures 2221 * writes will be marked dirty for next log read. 2222 * 2223 * 1. Take a snapshot of the bit and clear it if needed. 2224 * 2. Write protect the corresponding page. 2225 * 3. Copy the snapshot to the userspace. 2226 * 4. Flush TLB's if needed. 2227 */ 2228 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 2229 struct kvm_dirty_log *log) 2230 { 2231 int r; 2232 2233 mutex_lock(&kvm->slots_lock); 2234 2235 r = kvm_get_dirty_log_protect(kvm, log); 2236 2237 mutex_unlock(&kvm->slots_lock); 2238 return r; 2239 } 2240 2241 /** 2242 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap 2243 * and reenable dirty page tracking for the corresponding pages. 2244 * @kvm: pointer to kvm instance 2245 * @log: slot id and address from which to fetch the bitmap of dirty pages 2246 */ 2247 static int kvm_clear_dirty_log_protect(struct kvm *kvm, 2248 struct kvm_clear_dirty_log *log) 2249 { 2250 struct kvm_memslots *slots; 2251 struct kvm_memory_slot *memslot; 2252 int as_id, id; 2253 gfn_t offset; 2254 unsigned long i, n; 2255 unsigned long *dirty_bitmap; 2256 unsigned long *dirty_bitmap_buffer; 2257 bool flush; 2258 2259 /* Dirty ring tracking may be exclusive to dirty log tracking */ 2260 if (!kvm_use_dirty_bitmap(kvm)) 2261 return -ENXIO; 2262 2263 as_id = log->slot >> 16; 2264 id = (u16)log->slot; 2265 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 2266 return -EINVAL; 2267 2268 if (log->first_page & 63) 2269 return -EINVAL; 2270 2271 slots = __kvm_memslots(kvm, as_id); 2272 memslot = id_to_memslot(slots, id); 2273 if (!memslot || !memslot->dirty_bitmap) 2274 return -ENOENT; 2275 2276 dirty_bitmap = memslot->dirty_bitmap; 2277 2278 n = ALIGN(log->num_pages, BITS_PER_LONG) / 8; 2279 2280 if (log->first_page > memslot->npages || 2281 log->num_pages > memslot->npages - log->first_page || 2282 (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63))) 2283 return -EINVAL; 2284 2285 kvm_arch_sync_dirty_log(kvm, memslot); 2286 2287 flush = false; 2288 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot); 2289 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n)) 2290 return -EFAULT; 2291 2292 KVM_MMU_LOCK(kvm); 2293 for (offset = log->first_page, i = offset / BITS_PER_LONG, 2294 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--; 2295 i++, offset += BITS_PER_LONG) { 2296 unsigned long mask = *dirty_bitmap_buffer++; 2297 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i]; 2298 if (!mask) 2299 continue; 2300 2301 mask &= atomic_long_fetch_andnot(mask, p); 2302 2303 /* 2304 * mask contains the bits that really have been cleared. This 2305 * never includes any bits beyond the length of the memslot (if 2306 * the length is not aligned to 64 pages), therefore it is not 2307 * a problem if userspace sets them in log->dirty_bitmap. 2308 */ 2309 if (mask) { 2310 flush = true; 2311 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 2312 offset, mask); 2313 } 2314 } 2315 KVM_MMU_UNLOCK(kvm); 2316 2317 if (flush) 2318 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 2319 2320 return 0; 2321 } 2322 2323 static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, 2324 struct kvm_clear_dirty_log *log) 2325 { 2326 int r; 2327 2328 mutex_lock(&kvm->slots_lock); 2329 2330 r = kvm_clear_dirty_log_protect(kvm, log); 2331 2332 mutex_unlock(&kvm->slots_lock); 2333 return r; 2334 } 2335 #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ 2336 2337 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 2338 { 2339 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 2340 } 2341 EXPORT_SYMBOL_GPL(gfn_to_memslot); 2342 2343 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 2344 { 2345 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu); 2346 u64 gen = slots->generation; 2347 struct kvm_memory_slot *slot; 2348 2349 /* 2350 * This also protects against using a memslot from a different address space, 2351 * since different address spaces have different generation numbers. 2352 */ 2353 if (unlikely(gen != vcpu->last_used_slot_gen)) { 2354 vcpu->last_used_slot = NULL; 2355 vcpu->last_used_slot_gen = gen; 2356 } 2357 2358 slot = try_get_memslot(vcpu->last_used_slot, gfn); 2359 if (slot) 2360 return slot; 2361 2362 /* 2363 * Fall back to searching all memslots. We purposely use 2364 * search_memslots() instead of __gfn_to_memslot() to avoid 2365 * thrashing the VM-wide last_used_slot in kvm_memslots. 2366 */ 2367 slot = search_memslots(slots, gfn, false); 2368 if (slot) { 2369 vcpu->last_used_slot = slot; 2370 return slot; 2371 } 2372 2373 return NULL; 2374 } 2375 2376 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 2377 { 2378 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 2379 2380 return kvm_is_visible_memslot(memslot); 2381 } 2382 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 2383 2384 bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 2385 { 2386 struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2387 2388 return kvm_is_visible_memslot(memslot); 2389 } 2390 EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn); 2391 2392 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn) 2393 { 2394 struct vm_area_struct *vma; 2395 unsigned long addr, size; 2396 2397 size = PAGE_SIZE; 2398 2399 addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL); 2400 if (kvm_is_error_hva(addr)) 2401 return PAGE_SIZE; 2402 2403 mmap_read_lock(current->mm); 2404 vma = find_vma(current->mm, addr); 2405 if (!vma) 2406 goto out; 2407 2408 size = vma_kernel_pagesize(vma); 2409 2410 out: 2411 mmap_read_unlock(current->mm); 2412 2413 return size; 2414 } 2415 2416 static bool memslot_is_readonly(const struct kvm_memory_slot *slot) 2417 { 2418 return slot->flags & KVM_MEM_READONLY; 2419 } 2420 2421 static unsigned long __gfn_to_hva_many(const struct kvm_memory_slot *slot, gfn_t gfn, 2422 gfn_t *nr_pages, bool write) 2423 { 2424 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 2425 return KVM_HVA_ERR_BAD; 2426 2427 if (memslot_is_readonly(slot) && write) 2428 return KVM_HVA_ERR_RO_BAD; 2429 2430 if (nr_pages) 2431 *nr_pages = slot->npages - (gfn - slot->base_gfn); 2432 2433 return __gfn_to_hva_memslot(slot, gfn); 2434 } 2435 2436 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 2437 gfn_t *nr_pages) 2438 { 2439 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 2440 } 2441 2442 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 2443 gfn_t gfn) 2444 { 2445 return gfn_to_hva_many(slot, gfn, NULL); 2446 } 2447 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 2448 2449 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 2450 { 2451 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 2452 } 2453 EXPORT_SYMBOL_GPL(gfn_to_hva); 2454 2455 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 2456 { 2457 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 2458 } 2459 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 2460 2461 /* 2462 * Return the hva of a @gfn and the R/W attribute if possible. 2463 * 2464 * @slot: the kvm_memory_slot which contains @gfn 2465 * @gfn: the gfn to be translated 2466 * @writable: used to return the read/write attribute of the @slot if the hva 2467 * is valid and @writable is not NULL 2468 */ 2469 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 2470 gfn_t gfn, bool *writable) 2471 { 2472 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 2473 2474 if (!kvm_is_error_hva(hva) && writable) 2475 *writable = !memslot_is_readonly(slot); 2476 2477 return hva; 2478 } 2479 2480 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 2481 { 2482 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 2483 2484 return gfn_to_hva_memslot_prot(slot, gfn, writable); 2485 } 2486 2487 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 2488 { 2489 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2490 2491 return gfn_to_hva_memslot_prot(slot, gfn, writable); 2492 } 2493 2494 static inline int check_user_page_hwpoison(unsigned long addr) 2495 { 2496 int rc, flags = FOLL_HWPOISON | FOLL_WRITE; 2497 2498 rc = get_user_pages(addr, 1, flags, NULL); 2499 return rc == -EHWPOISON; 2500 } 2501 2502 /* 2503 * The fast path to get the writable pfn which will be stored in @pfn, 2504 * true indicates success, otherwise false is returned. It's also the 2505 * only part that runs if we can in atomic context. 2506 */ 2507 static bool hva_to_pfn_fast(unsigned long addr, bool write_fault, 2508 bool *writable, kvm_pfn_t *pfn) 2509 { 2510 struct page *page[1]; 2511 2512 /* 2513 * Fast pin a writable pfn only if it is a write fault request 2514 * or the caller allows to map a writable pfn for a read fault 2515 * request. 2516 */ 2517 if (!(write_fault || writable)) 2518 return false; 2519 2520 if (get_user_page_fast_only(addr, FOLL_WRITE, page)) { 2521 *pfn = page_to_pfn(page[0]); 2522 2523 if (writable) 2524 *writable = true; 2525 return true; 2526 } 2527 2528 return false; 2529 } 2530 2531 /* 2532 * The slow path to get the pfn of the specified host virtual address, 2533 * 1 indicates success, -errno is returned if error is detected. 2534 */ 2535 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 2536 bool interruptible, bool *writable, kvm_pfn_t *pfn) 2537 { 2538 unsigned int flags = FOLL_HWPOISON; 2539 struct page *page; 2540 int npages; 2541 2542 might_sleep(); 2543 2544 if (writable) 2545 *writable = write_fault; 2546 2547 if (write_fault) 2548 flags |= FOLL_WRITE; 2549 if (async) 2550 flags |= FOLL_NOWAIT; 2551 if (interruptible) 2552 flags |= FOLL_INTERRUPTIBLE; 2553 2554 npages = get_user_pages_unlocked(addr, 1, &page, flags); 2555 if (npages != 1) 2556 return npages; 2557 2558 /* map read fault as writable if possible */ 2559 if (unlikely(!write_fault) && writable) { 2560 struct page *wpage; 2561 2562 if (get_user_page_fast_only(addr, FOLL_WRITE, &wpage)) { 2563 *writable = true; 2564 put_page(page); 2565 page = wpage; 2566 } 2567 } 2568 *pfn = page_to_pfn(page); 2569 return npages; 2570 } 2571 2572 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 2573 { 2574 if (unlikely(!(vma->vm_flags & VM_READ))) 2575 return false; 2576 2577 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 2578 return false; 2579 2580 return true; 2581 } 2582 2583 static int kvm_try_get_pfn(kvm_pfn_t pfn) 2584 { 2585 struct page *page = kvm_pfn_to_refcounted_page(pfn); 2586 2587 if (!page) 2588 return 1; 2589 2590 return get_page_unless_zero(page); 2591 } 2592 2593 static int hva_to_pfn_remapped(struct vm_area_struct *vma, 2594 unsigned long addr, bool write_fault, 2595 bool *writable, kvm_pfn_t *p_pfn) 2596 { 2597 kvm_pfn_t pfn; 2598 pte_t *ptep; 2599 pte_t pte; 2600 spinlock_t *ptl; 2601 int r; 2602 2603 r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); 2604 if (r) { 2605 /* 2606 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does 2607 * not call the fault handler, so do it here. 2608 */ 2609 bool unlocked = false; 2610 r = fixup_user_fault(current->mm, addr, 2611 (write_fault ? FAULT_FLAG_WRITE : 0), 2612 &unlocked); 2613 if (unlocked) 2614 return -EAGAIN; 2615 if (r) 2616 return r; 2617 2618 r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); 2619 if (r) 2620 return r; 2621 } 2622 2623 pte = ptep_get(ptep); 2624 2625 if (write_fault && !pte_write(pte)) { 2626 pfn = KVM_PFN_ERR_RO_FAULT; 2627 goto out; 2628 } 2629 2630 if (writable) 2631 *writable = pte_write(pte); 2632 pfn = pte_pfn(pte); 2633 2634 /* 2635 * Get a reference here because callers of *hva_to_pfn* and 2636 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the 2637 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP 2638 * set, but the kvm_try_get_pfn/kvm_release_pfn_clean pair will 2639 * simply do nothing for reserved pfns. 2640 * 2641 * Whoever called remap_pfn_range is also going to call e.g. 2642 * unmap_mapping_range before the underlying pages are freed, 2643 * causing a call to our MMU notifier. 2644 * 2645 * Certain IO or PFNMAP mappings can be backed with valid 2646 * struct pages, but be allocated without refcounting e.g., 2647 * tail pages of non-compound higher order allocations, which 2648 * would then underflow the refcount when the caller does the 2649 * required put_page. Don't allow those pages here. 2650 */ 2651 if (!kvm_try_get_pfn(pfn)) 2652 r = -EFAULT; 2653 2654 out: 2655 pte_unmap_unlock(ptep, ptl); 2656 *p_pfn = pfn; 2657 2658 return r; 2659 } 2660 2661 /* 2662 * Pin guest page in memory and return its pfn. 2663 * @addr: host virtual address which maps memory to the guest 2664 * @atomic: whether this function can sleep 2665 * @interruptible: whether the process can be interrupted by non-fatal signals 2666 * @async: whether this function need to wait IO complete if the 2667 * host page is not in the memory 2668 * @write_fault: whether we should get a writable host page 2669 * @writable: whether it allows to map a writable host page for !@write_fault 2670 * 2671 * The function will map a writable host page for these two cases: 2672 * 1): @write_fault = true 2673 * 2): @write_fault = false && @writable, @writable will tell the caller 2674 * whether the mapping is writable. 2675 */ 2676 kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool interruptible, 2677 bool *async, bool write_fault, bool *writable) 2678 { 2679 struct vm_area_struct *vma; 2680 kvm_pfn_t pfn; 2681 int npages, r; 2682 2683 /* we can do it either atomically or asynchronously, not both */ 2684 BUG_ON(atomic && async); 2685 2686 if (hva_to_pfn_fast(addr, write_fault, writable, &pfn)) 2687 return pfn; 2688 2689 if (atomic) 2690 return KVM_PFN_ERR_FAULT; 2691 2692 npages = hva_to_pfn_slow(addr, async, write_fault, interruptible, 2693 writable, &pfn); 2694 if (npages == 1) 2695 return pfn; 2696 if (npages == -EINTR) 2697 return KVM_PFN_ERR_SIGPENDING; 2698 2699 mmap_read_lock(current->mm); 2700 if (npages == -EHWPOISON || 2701 (!async && check_user_page_hwpoison(addr))) { 2702 pfn = KVM_PFN_ERR_HWPOISON; 2703 goto exit; 2704 } 2705 2706 retry: 2707 vma = vma_lookup(current->mm, addr); 2708 2709 if (vma == NULL) 2710 pfn = KVM_PFN_ERR_FAULT; 2711 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { 2712 r = hva_to_pfn_remapped(vma, addr, write_fault, writable, &pfn); 2713 if (r == -EAGAIN) 2714 goto retry; 2715 if (r < 0) 2716 pfn = KVM_PFN_ERR_FAULT; 2717 } else { 2718 if (async && vma_is_valid(vma, write_fault)) 2719 *async = true; 2720 pfn = KVM_PFN_ERR_FAULT; 2721 } 2722 exit: 2723 mmap_read_unlock(current->mm); 2724 return pfn; 2725 } 2726 2727 kvm_pfn_t __gfn_to_pfn_memslot(const struct kvm_memory_slot *slot, gfn_t gfn, 2728 bool atomic, bool interruptible, bool *async, 2729 bool write_fault, bool *writable, hva_t *hva) 2730 { 2731 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 2732 2733 if (hva) 2734 *hva = addr; 2735 2736 if (addr == KVM_HVA_ERR_RO_BAD) { 2737 if (writable) 2738 *writable = false; 2739 return KVM_PFN_ERR_RO_FAULT; 2740 } 2741 2742 if (kvm_is_error_hva(addr)) { 2743 if (writable) 2744 *writable = false; 2745 return KVM_PFN_NOSLOT; 2746 } 2747 2748 /* Do not map writable pfn in the readonly memslot. */ 2749 if (writable && memslot_is_readonly(slot)) { 2750 *writable = false; 2751 writable = NULL; 2752 } 2753 2754 return hva_to_pfn(addr, atomic, interruptible, async, write_fault, 2755 writable); 2756 } 2757 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 2758 2759 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 2760 bool *writable) 2761 { 2762 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, false, 2763 NULL, write_fault, writable, NULL); 2764 } 2765 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 2766 2767 kvm_pfn_t gfn_to_pfn_memslot(const struct kvm_memory_slot *slot, gfn_t gfn) 2768 { 2769 return __gfn_to_pfn_memslot(slot, gfn, false, false, NULL, true, 2770 NULL, NULL); 2771 } 2772 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 2773 2774 kvm_pfn_t gfn_to_pfn_memslot_atomic(const struct kvm_memory_slot *slot, gfn_t gfn) 2775 { 2776 return __gfn_to_pfn_memslot(slot, gfn, true, false, NULL, true, 2777 NULL, NULL); 2778 } 2779 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 2780 2781 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 2782 { 2783 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 2784 } 2785 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 2786 2787 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 2788 { 2789 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 2790 } 2791 EXPORT_SYMBOL_GPL(gfn_to_pfn); 2792 2793 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 2794 { 2795 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 2796 } 2797 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 2798 2799 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 2800 struct page **pages, int nr_pages) 2801 { 2802 unsigned long addr; 2803 gfn_t entry = 0; 2804 2805 addr = gfn_to_hva_many(slot, gfn, &entry); 2806 if (kvm_is_error_hva(addr)) 2807 return -1; 2808 2809 if (entry < nr_pages) 2810 return 0; 2811 2812 return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages); 2813 } 2814 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 2815 2816 /* 2817 * Do not use this helper unless you are absolutely certain the gfn _must_ be 2818 * backed by 'struct page'. A valid example is if the backing memslot is 2819 * controlled by KVM. Note, if the returned page is valid, it's refcount has 2820 * been elevated by gfn_to_pfn(). 2821 */ 2822 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 2823 { 2824 struct page *page; 2825 kvm_pfn_t pfn; 2826 2827 pfn = gfn_to_pfn(kvm, gfn); 2828 2829 if (is_error_noslot_pfn(pfn)) 2830 return KVM_ERR_PTR_BAD_PAGE; 2831 2832 page = kvm_pfn_to_refcounted_page(pfn); 2833 if (!page) 2834 return KVM_ERR_PTR_BAD_PAGE; 2835 2836 return page; 2837 } 2838 EXPORT_SYMBOL_GPL(gfn_to_page); 2839 2840 void kvm_release_pfn(kvm_pfn_t pfn, bool dirty) 2841 { 2842 if (dirty) 2843 kvm_release_pfn_dirty(pfn); 2844 else 2845 kvm_release_pfn_clean(pfn); 2846 } 2847 2848 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map) 2849 { 2850 kvm_pfn_t pfn; 2851 void *hva = NULL; 2852 struct page *page = KVM_UNMAPPED_PAGE; 2853 2854 if (!map) 2855 return -EINVAL; 2856 2857 pfn = gfn_to_pfn(vcpu->kvm, gfn); 2858 if (is_error_noslot_pfn(pfn)) 2859 return -EINVAL; 2860 2861 if (pfn_valid(pfn)) { 2862 page = pfn_to_page(pfn); 2863 hva = kmap(page); 2864 #ifdef CONFIG_HAS_IOMEM 2865 } else { 2866 hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB); 2867 #endif 2868 } 2869 2870 if (!hva) 2871 return -EFAULT; 2872 2873 map->page = page; 2874 map->hva = hva; 2875 map->pfn = pfn; 2876 map->gfn = gfn; 2877 2878 return 0; 2879 } 2880 EXPORT_SYMBOL_GPL(kvm_vcpu_map); 2881 2882 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty) 2883 { 2884 if (!map) 2885 return; 2886 2887 if (!map->hva) 2888 return; 2889 2890 if (map->page != KVM_UNMAPPED_PAGE) 2891 kunmap(map->page); 2892 #ifdef CONFIG_HAS_IOMEM 2893 else 2894 memunmap(map->hva); 2895 #endif 2896 2897 if (dirty) 2898 kvm_vcpu_mark_page_dirty(vcpu, map->gfn); 2899 2900 kvm_release_pfn(map->pfn, dirty); 2901 2902 map->hva = NULL; 2903 map->page = NULL; 2904 } 2905 EXPORT_SYMBOL_GPL(kvm_vcpu_unmap); 2906 2907 static bool kvm_is_ad_tracked_page(struct page *page) 2908 { 2909 /* 2910 * Per page-flags.h, pages tagged PG_reserved "should in general not be 2911 * touched (e.g. set dirty) except by its owner". 2912 */ 2913 return !PageReserved(page); 2914 } 2915 2916 static void kvm_set_page_dirty(struct page *page) 2917 { 2918 if (kvm_is_ad_tracked_page(page)) 2919 SetPageDirty(page); 2920 } 2921 2922 static void kvm_set_page_accessed(struct page *page) 2923 { 2924 if (kvm_is_ad_tracked_page(page)) 2925 mark_page_accessed(page); 2926 } 2927 2928 void kvm_release_page_clean(struct page *page) 2929 { 2930 WARN_ON(is_error_page(page)); 2931 2932 kvm_set_page_accessed(page); 2933 put_page(page); 2934 } 2935 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 2936 2937 void kvm_release_pfn_clean(kvm_pfn_t pfn) 2938 { 2939 struct page *page; 2940 2941 if (is_error_noslot_pfn(pfn)) 2942 return; 2943 2944 page = kvm_pfn_to_refcounted_page(pfn); 2945 if (!page) 2946 return; 2947 2948 kvm_release_page_clean(page); 2949 } 2950 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 2951 2952 void kvm_release_page_dirty(struct page *page) 2953 { 2954 WARN_ON(is_error_page(page)); 2955 2956 kvm_set_page_dirty(page); 2957 kvm_release_page_clean(page); 2958 } 2959 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 2960 2961 void kvm_release_pfn_dirty(kvm_pfn_t pfn) 2962 { 2963 struct page *page; 2964 2965 if (is_error_noslot_pfn(pfn)) 2966 return; 2967 2968 page = kvm_pfn_to_refcounted_page(pfn); 2969 if (!page) 2970 return; 2971 2972 kvm_release_page_dirty(page); 2973 } 2974 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 2975 2976 /* 2977 * Note, checking for an error/noslot pfn is the caller's responsibility when 2978 * directly marking a page dirty/accessed. Unlike the "release" helpers, the 2979 * "set" helpers are not to be used when the pfn might point at garbage. 2980 */ 2981 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 2982 { 2983 if (WARN_ON(is_error_noslot_pfn(pfn))) 2984 return; 2985 2986 if (pfn_valid(pfn)) 2987 kvm_set_page_dirty(pfn_to_page(pfn)); 2988 } 2989 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 2990 2991 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 2992 { 2993 if (WARN_ON(is_error_noslot_pfn(pfn))) 2994 return; 2995 2996 if (pfn_valid(pfn)) 2997 kvm_set_page_accessed(pfn_to_page(pfn)); 2998 } 2999 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 3000 3001 static int next_segment(unsigned long len, int offset) 3002 { 3003 if (len > PAGE_SIZE - offset) 3004 return PAGE_SIZE - offset; 3005 else 3006 return len; 3007 } 3008 3009 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 3010 void *data, int offset, int len) 3011 { 3012 int r; 3013 unsigned long addr; 3014 3015 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 3016 if (kvm_is_error_hva(addr)) 3017 return -EFAULT; 3018 r = __copy_from_user(data, (void __user *)addr + offset, len); 3019 if (r) 3020 return -EFAULT; 3021 return 0; 3022 } 3023 3024 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 3025 int len) 3026 { 3027 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 3028 3029 return __kvm_read_guest_page(slot, gfn, data, offset, len); 3030 } 3031 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 3032 3033 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 3034 int offset, int len) 3035 { 3036 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 3037 3038 return __kvm_read_guest_page(slot, gfn, data, offset, len); 3039 } 3040 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 3041 3042 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 3043 { 3044 gfn_t gfn = gpa >> PAGE_SHIFT; 3045 int seg; 3046 int offset = offset_in_page(gpa); 3047 int ret; 3048 3049 while ((seg = next_segment(len, offset)) != 0) { 3050 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 3051 if (ret < 0) 3052 return ret; 3053 offset = 0; 3054 len -= seg; 3055 data += seg; 3056 ++gfn; 3057 } 3058 return 0; 3059 } 3060 EXPORT_SYMBOL_GPL(kvm_read_guest); 3061 3062 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 3063 { 3064 gfn_t gfn = gpa >> PAGE_SHIFT; 3065 int seg; 3066 int offset = offset_in_page(gpa); 3067 int ret; 3068 3069 while ((seg = next_segment(len, offset)) != 0) { 3070 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 3071 if (ret < 0) 3072 return ret; 3073 offset = 0; 3074 len -= seg; 3075 data += seg; 3076 ++gfn; 3077 } 3078 return 0; 3079 } 3080 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 3081 3082 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 3083 void *data, int offset, unsigned long len) 3084 { 3085 int r; 3086 unsigned long addr; 3087 3088 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 3089 if (kvm_is_error_hva(addr)) 3090 return -EFAULT; 3091 pagefault_disable(); 3092 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 3093 pagefault_enable(); 3094 if (r) 3095 return -EFAULT; 3096 return 0; 3097 } 3098 3099 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 3100 void *data, unsigned long len) 3101 { 3102 gfn_t gfn = gpa >> PAGE_SHIFT; 3103 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 3104 int offset = offset_in_page(gpa); 3105 3106 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 3107 } 3108 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 3109 3110 static int __kvm_write_guest_page(struct kvm *kvm, 3111 struct kvm_memory_slot *memslot, gfn_t gfn, 3112 const void *data, int offset, int len) 3113 { 3114 int r; 3115 unsigned long addr; 3116 3117 addr = gfn_to_hva_memslot(memslot, gfn); 3118 if (kvm_is_error_hva(addr)) 3119 return -EFAULT; 3120 r = __copy_to_user((void __user *)addr + offset, data, len); 3121 if (r) 3122 return -EFAULT; 3123 mark_page_dirty_in_slot(kvm, memslot, gfn); 3124 return 0; 3125 } 3126 3127 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 3128 const void *data, int offset, int len) 3129 { 3130 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 3131 3132 return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len); 3133 } 3134 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 3135 3136 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 3137 const void *data, int offset, int len) 3138 { 3139 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 3140 3141 return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len); 3142 } 3143 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 3144 3145 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 3146 unsigned long len) 3147 { 3148 gfn_t gfn = gpa >> PAGE_SHIFT; 3149 int seg; 3150 int offset = offset_in_page(gpa); 3151 int ret; 3152 3153 while ((seg = next_segment(len, offset)) != 0) { 3154 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 3155 if (ret < 0) 3156 return ret; 3157 offset = 0; 3158 len -= seg; 3159 data += seg; 3160 ++gfn; 3161 } 3162 return 0; 3163 } 3164 EXPORT_SYMBOL_GPL(kvm_write_guest); 3165 3166 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 3167 unsigned long len) 3168 { 3169 gfn_t gfn = gpa >> PAGE_SHIFT; 3170 int seg; 3171 int offset = offset_in_page(gpa); 3172 int ret; 3173 3174 while ((seg = next_segment(len, offset)) != 0) { 3175 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 3176 if (ret < 0) 3177 return ret; 3178 offset = 0; 3179 len -= seg; 3180 data += seg; 3181 ++gfn; 3182 } 3183 return 0; 3184 } 3185 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 3186 3187 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots, 3188 struct gfn_to_hva_cache *ghc, 3189 gpa_t gpa, unsigned long len) 3190 { 3191 int offset = offset_in_page(gpa); 3192 gfn_t start_gfn = gpa >> PAGE_SHIFT; 3193 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 3194 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 3195 gfn_t nr_pages_avail; 3196 3197 /* Update ghc->generation before performing any error checks. */ 3198 ghc->generation = slots->generation; 3199 3200 if (start_gfn > end_gfn) { 3201 ghc->hva = KVM_HVA_ERR_BAD; 3202 return -EINVAL; 3203 } 3204 3205 /* 3206 * If the requested region crosses two memslots, we still 3207 * verify that the entire region is valid here. 3208 */ 3209 for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) { 3210 ghc->memslot = __gfn_to_memslot(slots, start_gfn); 3211 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 3212 &nr_pages_avail); 3213 if (kvm_is_error_hva(ghc->hva)) 3214 return -EFAULT; 3215 } 3216 3217 /* Use the slow path for cross page reads and writes. */ 3218 if (nr_pages_needed == 1) 3219 ghc->hva += offset; 3220 else 3221 ghc->memslot = NULL; 3222 3223 ghc->gpa = gpa; 3224 ghc->len = len; 3225 return 0; 3226 } 3227 3228 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 3229 gpa_t gpa, unsigned long len) 3230 { 3231 struct kvm_memslots *slots = kvm_memslots(kvm); 3232 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len); 3233 } 3234 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 3235 3236 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 3237 void *data, unsigned int offset, 3238 unsigned long len) 3239 { 3240 struct kvm_memslots *slots = kvm_memslots(kvm); 3241 int r; 3242 gpa_t gpa = ghc->gpa + offset; 3243 3244 if (WARN_ON_ONCE(len + offset > ghc->len)) 3245 return -EINVAL; 3246 3247 if (slots->generation != ghc->generation) { 3248 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) 3249 return -EFAULT; 3250 } 3251 3252 if (kvm_is_error_hva(ghc->hva)) 3253 return -EFAULT; 3254 3255 if (unlikely(!ghc->memslot)) 3256 return kvm_write_guest(kvm, gpa, data, len); 3257 3258 r = __copy_to_user((void __user *)ghc->hva + offset, data, len); 3259 if (r) 3260 return -EFAULT; 3261 mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT); 3262 3263 return 0; 3264 } 3265 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached); 3266 3267 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 3268 void *data, unsigned long len) 3269 { 3270 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); 3271 } 3272 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 3273 3274 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 3275 void *data, unsigned int offset, 3276 unsigned long len) 3277 { 3278 struct kvm_memslots *slots = kvm_memslots(kvm); 3279 int r; 3280 gpa_t gpa = ghc->gpa + offset; 3281 3282 if (WARN_ON_ONCE(len + offset > ghc->len)) 3283 return -EINVAL; 3284 3285 if (slots->generation != ghc->generation) { 3286 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) 3287 return -EFAULT; 3288 } 3289 3290 if (kvm_is_error_hva(ghc->hva)) 3291 return -EFAULT; 3292 3293 if (unlikely(!ghc->memslot)) 3294 return kvm_read_guest(kvm, gpa, data, len); 3295 3296 r = __copy_from_user(data, (void __user *)ghc->hva + offset, len); 3297 if (r) 3298 return -EFAULT; 3299 3300 return 0; 3301 } 3302 EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached); 3303 3304 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 3305 void *data, unsigned long len) 3306 { 3307 return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len); 3308 } 3309 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 3310 3311 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 3312 { 3313 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 3314 gfn_t gfn = gpa >> PAGE_SHIFT; 3315 int seg; 3316 int offset = offset_in_page(gpa); 3317 int ret; 3318 3319 while ((seg = next_segment(len, offset)) != 0) { 3320 ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 3321 if (ret < 0) 3322 return ret; 3323 offset = 0; 3324 len -= seg; 3325 ++gfn; 3326 } 3327 return 0; 3328 } 3329 EXPORT_SYMBOL_GPL(kvm_clear_guest); 3330 3331 void mark_page_dirty_in_slot(struct kvm *kvm, 3332 const struct kvm_memory_slot *memslot, 3333 gfn_t gfn) 3334 { 3335 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 3336 3337 #ifdef CONFIG_HAVE_KVM_DIRTY_RING 3338 if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm)) 3339 return; 3340 3341 WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm)); 3342 #endif 3343 3344 if (memslot && kvm_slot_dirty_track_enabled(memslot)) { 3345 unsigned long rel_gfn = gfn - memslot->base_gfn; 3346 u32 slot = (memslot->as_id << 16) | memslot->id; 3347 3348 if (kvm->dirty_ring_size && vcpu) 3349 kvm_dirty_ring_push(vcpu, slot, rel_gfn); 3350 else if (memslot->dirty_bitmap) 3351 set_bit_le(rel_gfn, memslot->dirty_bitmap); 3352 } 3353 } 3354 EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot); 3355 3356 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 3357 { 3358 struct kvm_memory_slot *memslot; 3359 3360 memslot = gfn_to_memslot(kvm, gfn); 3361 mark_page_dirty_in_slot(kvm, memslot, gfn); 3362 } 3363 EXPORT_SYMBOL_GPL(mark_page_dirty); 3364 3365 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 3366 { 3367 struct kvm_memory_slot *memslot; 3368 3369 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 3370 mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn); 3371 } 3372 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 3373 3374 void kvm_sigset_activate(struct kvm_vcpu *vcpu) 3375 { 3376 if (!vcpu->sigset_active) 3377 return; 3378 3379 /* 3380 * This does a lockless modification of ->real_blocked, which is fine 3381 * because, only current can change ->real_blocked and all readers of 3382 * ->real_blocked don't care as long ->real_blocked is always a subset 3383 * of ->blocked. 3384 */ 3385 sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked); 3386 } 3387 3388 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu) 3389 { 3390 if (!vcpu->sigset_active) 3391 return; 3392 3393 sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL); 3394 sigemptyset(¤t->real_blocked); 3395 } 3396 3397 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 3398 { 3399 unsigned int old, val, grow, grow_start; 3400 3401 old = val = vcpu->halt_poll_ns; 3402 grow_start = READ_ONCE(halt_poll_ns_grow_start); 3403 grow = READ_ONCE(halt_poll_ns_grow); 3404 if (!grow) 3405 goto out; 3406 3407 val *= grow; 3408 if (val < grow_start) 3409 val = grow_start; 3410 3411 vcpu->halt_poll_ns = val; 3412 out: 3413 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 3414 } 3415 3416 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 3417 { 3418 unsigned int old, val, shrink, grow_start; 3419 3420 old = val = vcpu->halt_poll_ns; 3421 shrink = READ_ONCE(halt_poll_ns_shrink); 3422 grow_start = READ_ONCE(halt_poll_ns_grow_start); 3423 if (shrink == 0) 3424 val = 0; 3425 else 3426 val /= shrink; 3427 3428 if (val < grow_start) 3429 val = 0; 3430 3431 vcpu->halt_poll_ns = val; 3432 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 3433 } 3434 3435 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 3436 { 3437 int ret = -EINTR; 3438 int idx = srcu_read_lock(&vcpu->kvm->srcu); 3439 3440 if (kvm_arch_vcpu_runnable(vcpu)) 3441 goto out; 3442 if (kvm_cpu_has_pending_timer(vcpu)) 3443 goto out; 3444 if (signal_pending(current)) 3445 goto out; 3446 if (kvm_check_request(KVM_REQ_UNBLOCK, vcpu)) 3447 goto out; 3448 3449 ret = 0; 3450 out: 3451 srcu_read_unlock(&vcpu->kvm->srcu, idx); 3452 return ret; 3453 } 3454 3455 /* 3456 * Block the vCPU until the vCPU is runnable, an event arrives, or a signal is 3457 * pending. This is mostly used when halting a vCPU, but may also be used 3458 * directly for other vCPU non-runnable states, e.g. x86's Wait-For-SIPI. 3459 */ 3460 bool kvm_vcpu_block(struct kvm_vcpu *vcpu) 3461 { 3462 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 3463 bool waited = false; 3464 3465 vcpu->stat.generic.blocking = 1; 3466 3467 preempt_disable(); 3468 kvm_arch_vcpu_blocking(vcpu); 3469 prepare_to_rcuwait(wait); 3470 preempt_enable(); 3471 3472 for (;;) { 3473 set_current_state(TASK_INTERRUPTIBLE); 3474 3475 if (kvm_vcpu_check_block(vcpu) < 0) 3476 break; 3477 3478 waited = true; 3479 schedule(); 3480 } 3481 3482 preempt_disable(); 3483 finish_rcuwait(wait); 3484 kvm_arch_vcpu_unblocking(vcpu); 3485 preempt_enable(); 3486 3487 vcpu->stat.generic.blocking = 0; 3488 3489 return waited; 3490 } 3491 3492 static inline void update_halt_poll_stats(struct kvm_vcpu *vcpu, ktime_t start, 3493 ktime_t end, bool success) 3494 { 3495 struct kvm_vcpu_stat_generic *stats = &vcpu->stat.generic; 3496 u64 poll_ns = ktime_to_ns(ktime_sub(end, start)); 3497 3498 ++vcpu->stat.generic.halt_attempted_poll; 3499 3500 if (success) { 3501 ++vcpu->stat.generic.halt_successful_poll; 3502 3503 if (!vcpu_valid_wakeup(vcpu)) 3504 ++vcpu->stat.generic.halt_poll_invalid; 3505 3506 stats->halt_poll_success_ns += poll_ns; 3507 KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_success_hist, poll_ns); 3508 } else { 3509 stats->halt_poll_fail_ns += poll_ns; 3510 KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_fail_hist, poll_ns); 3511 } 3512 } 3513 3514 static unsigned int kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu *vcpu) 3515 { 3516 struct kvm *kvm = vcpu->kvm; 3517 3518 if (kvm->override_halt_poll_ns) { 3519 /* 3520 * Ensure kvm->max_halt_poll_ns is not read before 3521 * kvm->override_halt_poll_ns. 3522 * 3523 * Pairs with the smp_wmb() when enabling KVM_CAP_HALT_POLL. 3524 */ 3525 smp_rmb(); 3526 return READ_ONCE(kvm->max_halt_poll_ns); 3527 } 3528 3529 return READ_ONCE(halt_poll_ns); 3530 } 3531 3532 /* 3533 * Emulate a vCPU halt condition, e.g. HLT on x86, WFI on arm, etc... If halt 3534 * polling is enabled, busy wait for a short time before blocking to avoid the 3535 * expensive block+unblock sequence if a wake event arrives soon after the vCPU 3536 * is halted. 3537 */ 3538 void kvm_vcpu_halt(struct kvm_vcpu *vcpu) 3539 { 3540 unsigned int max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu); 3541 bool halt_poll_allowed = !kvm_arch_no_poll(vcpu); 3542 ktime_t start, cur, poll_end; 3543 bool waited = false; 3544 bool do_halt_poll; 3545 u64 halt_ns; 3546 3547 if (vcpu->halt_poll_ns > max_halt_poll_ns) 3548 vcpu->halt_poll_ns = max_halt_poll_ns; 3549 3550 do_halt_poll = halt_poll_allowed && vcpu->halt_poll_ns; 3551 3552 start = cur = poll_end = ktime_get(); 3553 if (do_halt_poll) { 3554 ktime_t stop = ktime_add_ns(start, vcpu->halt_poll_ns); 3555 3556 do { 3557 if (kvm_vcpu_check_block(vcpu) < 0) 3558 goto out; 3559 cpu_relax(); 3560 poll_end = cur = ktime_get(); 3561 } while (kvm_vcpu_can_poll(cur, stop)); 3562 } 3563 3564 waited = kvm_vcpu_block(vcpu); 3565 3566 cur = ktime_get(); 3567 if (waited) { 3568 vcpu->stat.generic.halt_wait_ns += 3569 ktime_to_ns(cur) - ktime_to_ns(poll_end); 3570 KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.generic.halt_wait_hist, 3571 ktime_to_ns(cur) - ktime_to_ns(poll_end)); 3572 } 3573 out: 3574 /* The total time the vCPU was "halted", including polling time. */ 3575 halt_ns = ktime_to_ns(cur) - ktime_to_ns(start); 3576 3577 /* 3578 * Note, halt-polling is considered successful so long as the vCPU was 3579 * never actually scheduled out, i.e. even if the wake event arrived 3580 * after of the halt-polling loop itself, but before the full wait. 3581 */ 3582 if (do_halt_poll) 3583 update_halt_poll_stats(vcpu, start, poll_end, !waited); 3584 3585 if (halt_poll_allowed) { 3586 /* Recompute the max halt poll time in case it changed. */ 3587 max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu); 3588 3589 if (!vcpu_valid_wakeup(vcpu)) { 3590 shrink_halt_poll_ns(vcpu); 3591 } else if (max_halt_poll_ns) { 3592 if (halt_ns <= vcpu->halt_poll_ns) 3593 ; 3594 /* we had a long block, shrink polling */ 3595 else if (vcpu->halt_poll_ns && 3596 halt_ns > max_halt_poll_ns) 3597 shrink_halt_poll_ns(vcpu); 3598 /* we had a short halt and our poll time is too small */ 3599 else if (vcpu->halt_poll_ns < max_halt_poll_ns && 3600 halt_ns < max_halt_poll_ns) 3601 grow_halt_poll_ns(vcpu); 3602 } else { 3603 vcpu->halt_poll_ns = 0; 3604 } 3605 } 3606 3607 trace_kvm_vcpu_wakeup(halt_ns, waited, vcpu_valid_wakeup(vcpu)); 3608 } 3609 EXPORT_SYMBOL_GPL(kvm_vcpu_halt); 3610 3611 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 3612 { 3613 if (__kvm_vcpu_wake_up(vcpu)) { 3614 WRITE_ONCE(vcpu->ready, true); 3615 ++vcpu->stat.generic.halt_wakeup; 3616 return true; 3617 } 3618 3619 return false; 3620 } 3621 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 3622 3623 #ifndef CONFIG_S390 3624 /* 3625 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 3626 */ 3627 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 3628 { 3629 int me, cpu; 3630 3631 if (kvm_vcpu_wake_up(vcpu)) 3632 return; 3633 3634 me = get_cpu(); 3635 /* 3636 * The only state change done outside the vcpu mutex is IN_GUEST_MODE 3637 * to EXITING_GUEST_MODE. Therefore the moderately expensive "should 3638 * kick" check does not need atomic operations if kvm_vcpu_kick is used 3639 * within the vCPU thread itself. 3640 */ 3641 if (vcpu == __this_cpu_read(kvm_running_vcpu)) { 3642 if (vcpu->mode == IN_GUEST_MODE) 3643 WRITE_ONCE(vcpu->mode, EXITING_GUEST_MODE); 3644 goto out; 3645 } 3646 3647 /* 3648 * Note, the vCPU could get migrated to a different pCPU at any point 3649 * after kvm_arch_vcpu_should_kick(), which could result in sending an 3650 * IPI to the previous pCPU. But, that's ok because the purpose of the 3651 * IPI is to force the vCPU to leave IN_GUEST_MODE, and migrating the 3652 * vCPU also requires it to leave IN_GUEST_MODE. 3653 */ 3654 if (kvm_arch_vcpu_should_kick(vcpu)) { 3655 cpu = READ_ONCE(vcpu->cpu); 3656 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 3657 smp_send_reschedule(cpu); 3658 } 3659 out: 3660 put_cpu(); 3661 } 3662 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 3663 #endif /* !CONFIG_S390 */ 3664 3665 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 3666 { 3667 struct pid *pid; 3668 struct task_struct *task = NULL; 3669 int ret = 0; 3670 3671 rcu_read_lock(); 3672 pid = rcu_dereference(target->pid); 3673 if (pid) 3674 task = get_pid_task(pid, PIDTYPE_PID); 3675 rcu_read_unlock(); 3676 if (!task) 3677 return ret; 3678 ret = yield_to(task, 1); 3679 put_task_struct(task); 3680 3681 return ret; 3682 } 3683 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 3684 3685 /* 3686 * Helper that checks whether a VCPU is eligible for directed yield. 3687 * Most eligible candidate to yield is decided by following heuristics: 3688 * 3689 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 3690 * (preempted lock holder), indicated by @in_spin_loop. 3691 * Set at the beginning and cleared at the end of interception/PLE handler. 3692 * 3693 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 3694 * chance last time (mostly it has become eligible now since we have probably 3695 * yielded to lockholder in last iteration. This is done by toggling 3696 * @dy_eligible each time a VCPU checked for eligibility.) 3697 * 3698 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 3699 * to preempted lock-holder could result in wrong VCPU selection and CPU 3700 * burning. Giving priority for a potential lock-holder increases lock 3701 * progress. 3702 * 3703 * Since algorithm is based on heuristics, accessing another VCPU data without 3704 * locking does not harm. It may result in trying to yield to same VCPU, fail 3705 * and continue with next VCPU and so on. 3706 */ 3707 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 3708 { 3709 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 3710 bool eligible; 3711 3712 eligible = !vcpu->spin_loop.in_spin_loop || 3713 vcpu->spin_loop.dy_eligible; 3714 3715 if (vcpu->spin_loop.in_spin_loop) 3716 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 3717 3718 return eligible; 3719 #else 3720 return true; 3721 #endif 3722 } 3723 3724 /* 3725 * Unlike kvm_arch_vcpu_runnable, this function is called outside 3726 * a vcpu_load/vcpu_put pair. However, for most architectures 3727 * kvm_arch_vcpu_runnable does not require vcpu_load. 3728 */ 3729 bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) 3730 { 3731 return kvm_arch_vcpu_runnable(vcpu); 3732 } 3733 3734 static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu) 3735 { 3736 if (kvm_arch_dy_runnable(vcpu)) 3737 return true; 3738 3739 #ifdef CONFIG_KVM_ASYNC_PF 3740 if (!list_empty_careful(&vcpu->async_pf.done)) 3741 return true; 3742 #endif 3743 3744 return false; 3745 } 3746 3747 bool __weak kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) 3748 { 3749 return false; 3750 } 3751 3752 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) 3753 { 3754 struct kvm *kvm = me->kvm; 3755 struct kvm_vcpu *vcpu; 3756 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 3757 unsigned long i; 3758 int yielded = 0; 3759 int try = 3; 3760 int pass; 3761 3762 kvm_vcpu_set_in_spin_loop(me, true); 3763 /* 3764 * We boost the priority of a VCPU that is runnable but not 3765 * currently running, because it got preempted by something 3766 * else and called schedule in __vcpu_run. Hopefully that 3767 * VCPU is holding the lock that we need and will release it. 3768 * We approximate round-robin by starting at the last boosted VCPU. 3769 */ 3770 for (pass = 0; pass < 2 && !yielded && try; pass++) { 3771 kvm_for_each_vcpu(i, vcpu, kvm) { 3772 if (!pass && i <= last_boosted_vcpu) { 3773 i = last_boosted_vcpu; 3774 continue; 3775 } else if (pass && i > last_boosted_vcpu) 3776 break; 3777 if (!READ_ONCE(vcpu->ready)) 3778 continue; 3779 if (vcpu == me) 3780 continue; 3781 if (kvm_vcpu_is_blocking(vcpu) && !vcpu_dy_runnable(vcpu)) 3782 continue; 3783 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode && 3784 !kvm_arch_dy_has_pending_interrupt(vcpu) && 3785 !kvm_arch_vcpu_in_kernel(vcpu)) 3786 continue; 3787 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 3788 continue; 3789 3790 yielded = kvm_vcpu_yield_to(vcpu); 3791 if (yielded > 0) { 3792 kvm->last_boosted_vcpu = i; 3793 break; 3794 } else if (yielded < 0) { 3795 try--; 3796 if (!try) 3797 break; 3798 } 3799 } 3800 } 3801 kvm_vcpu_set_in_spin_loop(me, false); 3802 3803 /* Ensure vcpu is not eligible during next spinloop */ 3804 kvm_vcpu_set_dy_eligible(me, false); 3805 } 3806 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 3807 3808 static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff) 3809 { 3810 #ifdef CONFIG_HAVE_KVM_DIRTY_RING 3811 return (pgoff >= KVM_DIRTY_LOG_PAGE_OFFSET) && 3812 (pgoff < KVM_DIRTY_LOG_PAGE_OFFSET + 3813 kvm->dirty_ring_size / PAGE_SIZE); 3814 #else 3815 return false; 3816 #endif 3817 } 3818 3819 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf) 3820 { 3821 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data; 3822 struct page *page; 3823 3824 if (vmf->pgoff == 0) 3825 page = virt_to_page(vcpu->run); 3826 #ifdef CONFIG_X86 3827 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 3828 page = virt_to_page(vcpu->arch.pio_data); 3829 #endif 3830 #ifdef CONFIG_KVM_MMIO 3831 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 3832 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 3833 #endif 3834 else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff)) 3835 page = kvm_dirty_ring_get_page( 3836 &vcpu->dirty_ring, 3837 vmf->pgoff - KVM_DIRTY_LOG_PAGE_OFFSET); 3838 else 3839 return kvm_arch_vcpu_fault(vcpu, vmf); 3840 get_page(page); 3841 vmf->page = page; 3842 return 0; 3843 } 3844 3845 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 3846 .fault = kvm_vcpu_fault, 3847 }; 3848 3849 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 3850 { 3851 struct kvm_vcpu *vcpu = file->private_data; 3852 unsigned long pages = vma_pages(vma); 3853 3854 if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) || 3855 kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) && 3856 ((vma->vm_flags & VM_EXEC) || !(vma->vm_flags & VM_SHARED))) 3857 return -EINVAL; 3858 3859 vma->vm_ops = &kvm_vcpu_vm_ops; 3860 return 0; 3861 } 3862 3863 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 3864 { 3865 struct kvm_vcpu *vcpu = filp->private_data; 3866 3867 kvm_put_kvm(vcpu->kvm); 3868 return 0; 3869 } 3870 3871 static const struct file_operations kvm_vcpu_fops = { 3872 .release = kvm_vcpu_release, 3873 .unlocked_ioctl = kvm_vcpu_ioctl, 3874 .mmap = kvm_vcpu_mmap, 3875 .llseek = noop_llseek, 3876 KVM_COMPAT(kvm_vcpu_compat_ioctl), 3877 }; 3878 3879 /* 3880 * Allocates an inode for the vcpu. 3881 */ 3882 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 3883 { 3884 char name[8 + 1 + ITOA_MAX_LEN + 1]; 3885 3886 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); 3887 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 3888 } 3889 3890 #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS 3891 static int vcpu_get_pid(void *data, u64 *val) 3892 { 3893 struct kvm_vcpu *vcpu = data; 3894 *val = pid_nr(rcu_access_pointer(vcpu->pid)); 3895 return 0; 3896 } 3897 3898 DEFINE_SIMPLE_ATTRIBUTE(vcpu_get_pid_fops, vcpu_get_pid, NULL, "%llu\n"); 3899 3900 static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 3901 { 3902 struct dentry *debugfs_dentry; 3903 char dir_name[ITOA_MAX_LEN * 2]; 3904 3905 if (!debugfs_initialized()) 3906 return; 3907 3908 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); 3909 debugfs_dentry = debugfs_create_dir(dir_name, 3910 vcpu->kvm->debugfs_dentry); 3911 debugfs_create_file("pid", 0444, debugfs_dentry, vcpu, 3912 &vcpu_get_pid_fops); 3913 3914 kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry); 3915 } 3916 #endif 3917 3918 /* 3919 * Creates some virtual cpus. Good luck creating more than one. 3920 */ 3921 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 3922 { 3923 int r; 3924 struct kvm_vcpu *vcpu; 3925 struct page *page; 3926 3927 if (id >= KVM_MAX_VCPU_IDS) 3928 return -EINVAL; 3929 3930 mutex_lock(&kvm->lock); 3931 if (kvm->created_vcpus >= kvm->max_vcpus) { 3932 mutex_unlock(&kvm->lock); 3933 return -EINVAL; 3934 } 3935 3936 r = kvm_arch_vcpu_precreate(kvm, id); 3937 if (r) { 3938 mutex_unlock(&kvm->lock); 3939 return r; 3940 } 3941 3942 kvm->created_vcpus++; 3943 mutex_unlock(&kvm->lock); 3944 3945 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); 3946 if (!vcpu) { 3947 r = -ENOMEM; 3948 goto vcpu_decrement; 3949 } 3950 3951 BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE); 3952 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 3953 if (!page) { 3954 r = -ENOMEM; 3955 goto vcpu_free; 3956 } 3957 vcpu->run = page_address(page); 3958 3959 kvm_vcpu_init(vcpu, kvm, id); 3960 3961 r = kvm_arch_vcpu_create(vcpu); 3962 if (r) 3963 goto vcpu_free_run_page; 3964 3965 if (kvm->dirty_ring_size) { 3966 r = kvm_dirty_ring_alloc(&vcpu->dirty_ring, 3967 id, kvm->dirty_ring_size); 3968 if (r) 3969 goto arch_vcpu_destroy; 3970 } 3971 3972 mutex_lock(&kvm->lock); 3973 3974 #ifdef CONFIG_LOCKDEP 3975 /* Ensure that lockdep knows vcpu->mutex is taken *inside* kvm->lock */ 3976 mutex_lock(&vcpu->mutex); 3977 mutex_unlock(&vcpu->mutex); 3978 #endif 3979 3980 if (kvm_get_vcpu_by_id(kvm, id)) { 3981 r = -EEXIST; 3982 goto unlock_vcpu_destroy; 3983 } 3984 3985 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); 3986 r = xa_reserve(&kvm->vcpu_array, vcpu->vcpu_idx, GFP_KERNEL_ACCOUNT); 3987 if (r) 3988 goto unlock_vcpu_destroy; 3989 3990 /* Now it's all set up, let userspace reach it */ 3991 kvm_get_kvm(kvm); 3992 r = create_vcpu_fd(vcpu); 3993 if (r < 0) 3994 goto kvm_put_xa_release; 3995 3996 if (KVM_BUG_ON(!!xa_store(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, 0), kvm)) { 3997 r = -EINVAL; 3998 goto kvm_put_xa_release; 3999 } 4000 4001 /* 4002 * Pairs with smp_rmb() in kvm_get_vcpu. Store the vcpu 4003 * pointer before kvm->online_vcpu's incremented value. 4004 */ 4005 smp_wmb(); 4006 atomic_inc(&kvm->online_vcpus); 4007 4008 mutex_unlock(&kvm->lock); 4009 kvm_arch_vcpu_postcreate(vcpu); 4010 kvm_create_vcpu_debugfs(vcpu); 4011 return r; 4012 4013 kvm_put_xa_release: 4014 kvm_put_kvm_no_destroy(kvm); 4015 xa_release(&kvm->vcpu_array, vcpu->vcpu_idx); 4016 unlock_vcpu_destroy: 4017 mutex_unlock(&kvm->lock); 4018 kvm_dirty_ring_free(&vcpu->dirty_ring); 4019 arch_vcpu_destroy: 4020 kvm_arch_vcpu_destroy(vcpu); 4021 vcpu_free_run_page: 4022 free_page((unsigned long)vcpu->run); 4023 vcpu_free: 4024 kmem_cache_free(kvm_vcpu_cache, vcpu); 4025 vcpu_decrement: 4026 mutex_lock(&kvm->lock); 4027 kvm->created_vcpus--; 4028 mutex_unlock(&kvm->lock); 4029 return r; 4030 } 4031 4032 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 4033 { 4034 if (sigset) { 4035 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 4036 vcpu->sigset_active = 1; 4037 vcpu->sigset = *sigset; 4038 } else 4039 vcpu->sigset_active = 0; 4040 return 0; 4041 } 4042 4043 static ssize_t kvm_vcpu_stats_read(struct file *file, char __user *user_buffer, 4044 size_t size, loff_t *offset) 4045 { 4046 struct kvm_vcpu *vcpu = file->private_data; 4047 4048 return kvm_stats_read(vcpu->stats_id, &kvm_vcpu_stats_header, 4049 &kvm_vcpu_stats_desc[0], &vcpu->stat, 4050 sizeof(vcpu->stat), user_buffer, size, offset); 4051 } 4052 4053 static const struct file_operations kvm_vcpu_stats_fops = { 4054 .read = kvm_vcpu_stats_read, 4055 .llseek = noop_llseek, 4056 }; 4057 4058 static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) 4059 { 4060 int fd; 4061 struct file *file; 4062 char name[15 + ITOA_MAX_LEN + 1]; 4063 4064 snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id); 4065 4066 fd = get_unused_fd_flags(O_CLOEXEC); 4067 if (fd < 0) 4068 return fd; 4069 4070 file = anon_inode_getfile(name, &kvm_vcpu_stats_fops, vcpu, O_RDONLY); 4071 if (IS_ERR(file)) { 4072 put_unused_fd(fd); 4073 return PTR_ERR(file); 4074 } 4075 file->f_mode |= FMODE_PREAD; 4076 fd_install(fd, file); 4077 4078 return fd; 4079 } 4080 4081 static long kvm_vcpu_ioctl(struct file *filp, 4082 unsigned int ioctl, unsigned long arg) 4083 { 4084 struct kvm_vcpu *vcpu = filp->private_data; 4085 void __user *argp = (void __user *)arg; 4086 int r; 4087 struct kvm_fpu *fpu = NULL; 4088 struct kvm_sregs *kvm_sregs = NULL; 4089 4090 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead) 4091 return -EIO; 4092 4093 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 4094 return -EINVAL; 4095 4096 /* 4097 * Some architectures have vcpu ioctls that are asynchronous to vcpu 4098 * execution; mutex_lock() would break them. 4099 */ 4100 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg); 4101 if (r != -ENOIOCTLCMD) 4102 return r; 4103 4104 if (mutex_lock_killable(&vcpu->mutex)) 4105 return -EINTR; 4106 switch (ioctl) { 4107 case KVM_RUN: { 4108 struct pid *oldpid; 4109 r = -EINVAL; 4110 if (arg) 4111 goto out; 4112 oldpid = rcu_access_pointer(vcpu->pid); 4113 if (unlikely(oldpid != task_pid(current))) { 4114 /* The thread running this VCPU changed. */ 4115 struct pid *newpid; 4116 4117 r = kvm_arch_vcpu_run_pid_change(vcpu); 4118 if (r) 4119 break; 4120 4121 newpid = get_task_pid(current, PIDTYPE_PID); 4122 rcu_assign_pointer(vcpu->pid, newpid); 4123 if (oldpid) 4124 synchronize_rcu(); 4125 put_pid(oldpid); 4126 } 4127 r = kvm_arch_vcpu_ioctl_run(vcpu); 4128 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 4129 break; 4130 } 4131 case KVM_GET_REGS: { 4132 struct kvm_regs *kvm_regs; 4133 4134 r = -ENOMEM; 4135 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT); 4136 if (!kvm_regs) 4137 goto out; 4138 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 4139 if (r) 4140 goto out_free1; 4141 r = -EFAULT; 4142 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 4143 goto out_free1; 4144 r = 0; 4145 out_free1: 4146 kfree(kvm_regs); 4147 break; 4148 } 4149 case KVM_SET_REGS: { 4150 struct kvm_regs *kvm_regs; 4151 4152 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 4153 if (IS_ERR(kvm_regs)) { 4154 r = PTR_ERR(kvm_regs); 4155 goto out; 4156 } 4157 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 4158 kfree(kvm_regs); 4159 break; 4160 } 4161 case KVM_GET_SREGS: { 4162 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), 4163 GFP_KERNEL_ACCOUNT); 4164 r = -ENOMEM; 4165 if (!kvm_sregs) 4166 goto out; 4167 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 4168 if (r) 4169 goto out; 4170 r = -EFAULT; 4171 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 4172 goto out; 4173 r = 0; 4174 break; 4175 } 4176 case KVM_SET_SREGS: { 4177 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 4178 if (IS_ERR(kvm_sregs)) { 4179 r = PTR_ERR(kvm_sregs); 4180 kvm_sregs = NULL; 4181 goto out; 4182 } 4183 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 4184 break; 4185 } 4186 case KVM_GET_MP_STATE: { 4187 struct kvm_mp_state mp_state; 4188 4189 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 4190 if (r) 4191 goto out; 4192 r = -EFAULT; 4193 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 4194 goto out; 4195 r = 0; 4196 break; 4197 } 4198 case KVM_SET_MP_STATE: { 4199 struct kvm_mp_state mp_state; 4200 4201 r = -EFAULT; 4202 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 4203 goto out; 4204 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 4205 break; 4206 } 4207 case KVM_TRANSLATE: { 4208 struct kvm_translation tr; 4209 4210 r = -EFAULT; 4211 if (copy_from_user(&tr, argp, sizeof(tr))) 4212 goto out; 4213 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 4214 if (r) 4215 goto out; 4216 r = -EFAULT; 4217 if (copy_to_user(argp, &tr, sizeof(tr))) 4218 goto out; 4219 r = 0; 4220 break; 4221 } 4222 case KVM_SET_GUEST_DEBUG: { 4223 struct kvm_guest_debug dbg; 4224 4225 r = -EFAULT; 4226 if (copy_from_user(&dbg, argp, sizeof(dbg))) 4227 goto out; 4228 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 4229 break; 4230 } 4231 case KVM_SET_SIGNAL_MASK: { 4232 struct kvm_signal_mask __user *sigmask_arg = argp; 4233 struct kvm_signal_mask kvm_sigmask; 4234 sigset_t sigset, *p; 4235 4236 p = NULL; 4237 if (argp) { 4238 r = -EFAULT; 4239 if (copy_from_user(&kvm_sigmask, argp, 4240 sizeof(kvm_sigmask))) 4241 goto out; 4242 r = -EINVAL; 4243 if (kvm_sigmask.len != sizeof(sigset)) 4244 goto out; 4245 r = -EFAULT; 4246 if (copy_from_user(&sigset, sigmask_arg->sigset, 4247 sizeof(sigset))) 4248 goto out; 4249 p = &sigset; 4250 } 4251 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 4252 break; 4253 } 4254 case KVM_GET_FPU: { 4255 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT); 4256 r = -ENOMEM; 4257 if (!fpu) 4258 goto out; 4259 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 4260 if (r) 4261 goto out; 4262 r = -EFAULT; 4263 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 4264 goto out; 4265 r = 0; 4266 break; 4267 } 4268 case KVM_SET_FPU: { 4269 fpu = memdup_user(argp, sizeof(*fpu)); 4270 if (IS_ERR(fpu)) { 4271 r = PTR_ERR(fpu); 4272 fpu = NULL; 4273 goto out; 4274 } 4275 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 4276 break; 4277 } 4278 case KVM_GET_STATS_FD: { 4279 r = kvm_vcpu_ioctl_get_stats_fd(vcpu); 4280 break; 4281 } 4282 default: 4283 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 4284 } 4285 out: 4286 mutex_unlock(&vcpu->mutex); 4287 kfree(fpu); 4288 kfree(kvm_sregs); 4289 return r; 4290 } 4291 4292 #ifdef CONFIG_KVM_COMPAT 4293 static long kvm_vcpu_compat_ioctl(struct file *filp, 4294 unsigned int ioctl, unsigned long arg) 4295 { 4296 struct kvm_vcpu *vcpu = filp->private_data; 4297 void __user *argp = compat_ptr(arg); 4298 int r; 4299 4300 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead) 4301 return -EIO; 4302 4303 switch (ioctl) { 4304 case KVM_SET_SIGNAL_MASK: { 4305 struct kvm_signal_mask __user *sigmask_arg = argp; 4306 struct kvm_signal_mask kvm_sigmask; 4307 sigset_t sigset; 4308 4309 if (argp) { 4310 r = -EFAULT; 4311 if (copy_from_user(&kvm_sigmask, argp, 4312 sizeof(kvm_sigmask))) 4313 goto out; 4314 r = -EINVAL; 4315 if (kvm_sigmask.len != sizeof(compat_sigset_t)) 4316 goto out; 4317 r = -EFAULT; 4318 if (get_compat_sigset(&sigset, 4319 (compat_sigset_t __user *)sigmask_arg->sigset)) 4320 goto out; 4321 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 4322 } else 4323 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 4324 break; 4325 } 4326 default: 4327 r = kvm_vcpu_ioctl(filp, ioctl, arg); 4328 } 4329 4330 out: 4331 return r; 4332 } 4333 #endif 4334 4335 static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma) 4336 { 4337 struct kvm_device *dev = filp->private_data; 4338 4339 if (dev->ops->mmap) 4340 return dev->ops->mmap(dev, vma); 4341 4342 return -ENODEV; 4343 } 4344 4345 static int kvm_device_ioctl_attr(struct kvm_device *dev, 4346 int (*accessor)(struct kvm_device *dev, 4347 struct kvm_device_attr *attr), 4348 unsigned long arg) 4349 { 4350 struct kvm_device_attr attr; 4351 4352 if (!accessor) 4353 return -EPERM; 4354 4355 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 4356 return -EFAULT; 4357 4358 return accessor(dev, &attr); 4359 } 4360 4361 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 4362 unsigned long arg) 4363 { 4364 struct kvm_device *dev = filp->private_data; 4365 4366 if (dev->kvm->mm != current->mm || dev->kvm->vm_dead) 4367 return -EIO; 4368 4369 switch (ioctl) { 4370 case KVM_SET_DEVICE_ATTR: 4371 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 4372 case KVM_GET_DEVICE_ATTR: 4373 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 4374 case KVM_HAS_DEVICE_ATTR: 4375 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 4376 default: 4377 if (dev->ops->ioctl) 4378 return dev->ops->ioctl(dev, ioctl, arg); 4379 4380 return -ENOTTY; 4381 } 4382 } 4383 4384 static int kvm_device_release(struct inode *inode, struct file *filp) 4385 { 4386 struct kvm_device *dev = filp->private_data; 4387 struct kvm *kvm = dev->kvm; 4388 4389 if (dev->ops->release) { 4390 mutex_lock(&kvm->lock); 4391 list_del(&dev->vm_node); 4392 dev->ops->release(dev); 4393 mutex_unlock(&kvm->lock); 4394 } 4395 4396 kvm_put_kvm(kvm); 4397 return 0; 4398 } 4399 4400 static const struct file_operations kvm_device_fops = { 4401 .unlocked_ioctl = kvm_device_ioctl, 4402 .release = kvm_device_release, 4403 KVM_COMPAT(kvm_device_ioctl), 4404 .mmap = kvm_device_mmap, 4405 }; 4406 4407 struct kvm_device *kvm_device_from_filp(struct file *filp) 4408 { 4409 if (filp->f_op != &kvm_device_fops) 4410 return NULL; 4411 4412 return filp->private_data; 4413 } 4414 4415 static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 4416 #ifdef CONFIG_KVM_MPIC 4417 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 4418 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 4419 #endif 4420 }; 4421 4422 int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type) 4423 { 4424 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 4425 return -ENOSPC; 4426 4427 if (kvm_device_ops_table[type] != NULL) 4428 return -EEXIST; 4429 4430 kvm_device_ops_table[type] = ops; 4431 return 0; 4432 } 4433 4434 void kvm_unregister_device_ops(u32 type) 4435 { 4436 if (kvm_device_ops_table[type] != NULL) 4437 kvm_device_ops_table[type] = NULL; 4438 } 4439 4440 static int kvm_ioctl_create_device(struct kvm *kvm, 4441 struct kvm_create_device *cd) 4442 { 4443 const struct kvm_device_ops *ops; 4444 struct kvm_device *dev; 4445 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 4446 int type; 4447 int ret; 4448 4449 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 4450 return -ENODEV; 4451 4452 type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table)); 4453 ops = kvm_device_ops_table[type]; 4454 if (ops == NULL) 4455 return -ENODEV; 4456 4457 if (test) 4458 return 0; 4459 4460 dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT); 4461 if (!dev) 4462 return -ENOMEM; 4463 4464 dev->ops = ops; 4465 dev->kvm = kvm; 4466 4467 mutex_lock(&kvm->lock); 4468 ret = ops->create(dev, type); 4469 if (ret < 0) { 4470 mutex_unlock(&kvm->lock); 4471 kfree(dev); 4472 return ret; 4473 } 4474 list_add(&dev->vm_node, &kvm->devices); 4475 mutex_unlock(&kvm->lock); 4476 4477 if (ops->init) 4478 ops->init(dev); 4479 4480 kvm_get_kvm(kvm); 4481 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 4482 if (ret < 0) { 4483 kvm_put_kvm_no_destroy(kvm); 4484 mutex_lock(&kvm->lock); 4485 list_del(&dev->vm_node); 4486 if (ops->release) 4487 ops->release(dev); 4488 mutex_unlock(&kvm->lock); 4489 if (ops->destroy) 4490 ops->destroy(dev); 4491 return ret; 4492 } 4493 4494 cd->fd = ret; 4495 return 0; 4496 } 4497 4498 static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 4499 { 4500 switch (arg) { 4501 case KVM_CAP_USER_MEMORY: 4502 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 4503 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 4504 case KVM_CAP_INTERNAL_ERROR_DATA: 4505 #ifdef CONFIG_HAVE_KVM_MSI 4506 case KVM_CAP_SIGNAL_MSI: 4507 #endif 4508 #ifdef CONFIG_HAVE_KVM_IRQFD 4509 case KVM_CAP_IRQFD: 4510 #endif 4511 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 4512 case KVM_CAP_CHECK_EXTENSION_VM: 4513 case KVM_CAP_ENABLE_CAP_VM: 4514 case KVM_CAP_HALT_POLL: 4515 return 1; 4516 #ifdef CONFIG_KVM_MMIO 4517 case KVM_CAP_COALESCED_MMIO: 4518 return KVM_COALESCED_MMIO_PAGE_OFFSET; 4519 case KVM_CAP_COALESCED_PIO: 4520 return 1; 4521 #endif 4522 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 4523 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: 4524 return KVM_DIRTY_LOG_MANUAL_CAPS; 4525 #endif 4526 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 4527 case KVM_CAP_IRQ_ROUTING: 4528 return KVM_MAX_IRQ_ROUTES; 4529 #endif 4530 #if KVM_ADDRESS_SPACE_NUM > 1 4531 case KVM_CAP_MULTI_ADDRESS_SPACE: 4532 return KVM_ADDRESS_SPACE_NUM; 4533 #endif 4534 case KVM_CAP_NR_MEMSLOTS: 4535 return KVM_USER_MEM_SLOTS; 4536 case KVM_CAP_DIRTY_LOG_RING: 4537 #ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO 4538 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn); 4539 #else 4540 return 0; 4541 #endif 4542 case KVM_CAP_DIRTY_LOG_RING_ACQ_REL: 4543 #ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL 4544 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn); 4545 #else 4546 return 0; 4547 #endif 4548 #ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP 4549 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: 4550 #endif 4551 case KVM_CAP_BINARY_STATS_FD: 4552 case KVM_CAP_SYSTEM_EVENT_DATA: 4553 return 1; 4554 default: 4555 break; 4556 } 4557 return kvm_vm_ioctl_check_extension(kvm, arg); 4558 } 4559 4560 static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size) 4561 { 4562 int r; 4563 4564 if (!KVM_DIRTY_LOG_PAGE_OFFSET) 4565 return -EINVAL; 4566 4567 /* the size should be power of 2 */ 4568 if (!size || (size & (size - 1))) 4569 return -EINVAL; 4570 4571 /* Should be bigger to keep the reserved entries, or a page */ 4572 if (size < kvm_dirty_ring_get_rsvd_entries() * 4573 sizeof(struct kvm_dirty_gfn) || size < PAGE_SIZE) 4574 return -EINVAL; 4575 4576 if (size > KVM_DIRTY_RING_MAX_ENTRIES * 4577 sizeof(struct kvm_dirty_gfn)) 4578 return -E2BIG; 4579 4580 /* We only allow it to set once */ 4581 if (kvm->dirty_ring_size) 4582 return -EINVAL; 4583 4584 mutex_lock(&kvm->lock); 4585 4586 if (kvm->created_vcpus) { 4587 /* We don't allow to change this value after vcpu created */ 4588 r = -EINVAL; 4589 } else { 4590 kvm->dirty_ring_size = size; 4591 r = 0; 4592 } 4593 4594 mutex_unlock(&kvm->lock); 4595 return r; 4596 } 4597 4598 static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm) 4599 { 4600 unsigned long i; 4601 struct kvm_vcpu *vcpu; 4602 int cleared = 0; 4603 4604 if (!kvm->dirty_ring_size) 4605 return -EINVAL; 4606 4607 mutex_lock(&kvm->slots_lock); 4608 4609 kvm_for_each_vcpu(i, vcpu, kvm) 4610 cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring); 4611 4612 mutex_unlock(&kvm->slots_lock); 4613 4614 if (cleared) 4615 kvm_flush_remote_tlbs(kvm); 4616 4617 return cleared; 4618 } 4619 4620 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm, 4621 struct kvm_enable_cap *cap) 4622 { 4623 return -EINVAL; 4624 } 4625 4626 static bool kvm_are_all_memslots_empty(struct kvm *kvm) 4627 { 4628 int i; 4629 4630 lockdep_assert_held(&kvm->slots_lock); 4631 4632 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 4633 if (!kvm_memslots_empty(__kvm_memslots(kvm, i))) 4634 return false; 4635 } 4636 4637 return true; 4638 } 4639 4640 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, 4641 struct kvm_enable_cap *cap) 4642 { 4643 switch (cap->cap) { 4644 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 4645 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: { 4646 u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE; 4647 4648 if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE) 4649 allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS; 4650 4651 if (cap->flags || (cap->args[0] & ~allowed_options)) 4652 return -EINVAL; 4653 kvm->manual_dirty_log_protect = cap->args[0]; 4654 return 0; 4655 } 4656 #endif 4657 case KVM_CAP_HALT_POLL: { 4658 if (cap->flags || cap->args[0] != (unsigned int)cap->args[0]) 4659 return -EINVAL; 4660 4661 kvm->max_halt_poll_ns = cap->args[0]; 4662 4663 /* 4664 * Ensure kvm->override_halt_poll_ns does not become visible 4665 * before kvm->max_halt_poll_ns. 4666 * 4667 * Pairs with the smp_rmb() in kvm_vcpu_max_halt_poll_ns(). 4668 */ 4669 smp_wmb(); 4670 kvm->override_halt_poll_ns = true; 4671 4672 return 0; 4673 } 4674 case KVM_CAP_DIRTY_LOG_RING: 4675 case KVM_CAP_DIRTY_LOG_RING_ACQ_REL: 4676 if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap)) 4677 return -EINVAL; 4678 4679 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]); 4680 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: { 4681 int r = -EINVAL; 4682 4683 if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) || 4684 !kvm->dirty_ring_size || cap->flags) 4685 return r; 4686 4687 mutex_lock(&kvm->slots_lock); 4688 4689 /* 4690 * For simplicity, allow enabling ring+bitmap if and only if 4691 * there are no memslots, e.g. to ensure all memslots allocate 4692 * a bitmap after the capability is enabled. 4693 */ 4694 if (kvm_are_all_memslots_empty(kvm)) { 4695 kvm->dirty_ring_with_bitmap = true; 4696 r = 0; 4697 } 4698 4699 mutex_unlock(&kvm->slots_lock); 4700 4701 return r; 4702 } 4703 default: 4704 return kvm_vm_ioctl_enable_cap(kvm, cap); 4705 } 4706 } 4707 4708 static ssize_t kvm_vm_stats_read(struct file *file, char __user *user_buffer, 4709 size_t size, loff_t *offset) 4710 { 4711 struct kvm *kvm = file->private_data; 4712 4713 return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header, 4714 &kvm_vm_stats_desc[0], &kvm->stat, 4715 sizeof(kvm->stat), user_buffer, size, offset); 4716 } 4717 4718 static const struct file_operations kvm_vm_stats_fops = { 4719 .read = kvm_vm_stats_read, 4720 .llseek = noop_llseek, 4721 }; 4722 4723 static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) 4724 { 4725 int fd; 4726 struct file *file; 4727 4728 fd = get_unused_fd_flags(O_CLOEXEC); 4729 if (fd < 0) 4730 return fd; 4731 4732 file = anon_inode_getfile("kvm-vm-stats", 4733 &kvm_vm_stats_fops, kvm, O_RDONLY); 4734 if (IS_ERR(file)) { 4735 put_unused_fd(fd); 4736 return PTR_ERR(file); 4737 } 4738 file->f_mode |= FMODE_PREAD; 4739 fd_install(fd, file); 4740 4741 return fd; 4742 } 4743 4744 static long kvm_vm_ioctl(struct file *filp, 4745 unsigned int ioctl, unsigned long arg) 4746 { 4747 struct kvm *kvm = filp->private_data; 4748 void __user *argp = (void __user *)arg; 4749 int r; 4750 4751 if (kvm->mm != current->mm || kvm->vm_dead) 4752 return -EIO; 4753 switch (ioctl) { 4754 case KVM_CREATE_VCPU: 4755 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 4756 break; 4757 case KVM_ENABLE_CAP: { 4758 struct kvm_enable_cap cap; 4759 4760 r = -EFAULT; 4761 if (copy_from_user(&cap, argp, sizeof(cap))) 4762 goto out; 4763 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap); 4764 break; 4765 } 4766 case KVM_SET_USER_MEMORY_REGION: { 4767 struct kvm_userspace_memory_region kvm_userspace_mem; 4768 4769 r = -EFAULT; 4770 if (copy_from_user(&kvm_userspace_mem, argp, 4771 sizeof(kvm_userspace_mem))) 4772 goto out; 4773 4774 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 4775 break; 4776 } 4777 case KVM_GET_DIRTY_LOG: { 4778 struct kvm_dirty_log log; 4779 4780 r = -EFAULT; 4781 if (copy_from_user(&log, argp, sizeof(log))) 4782 goto out; 4783 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 4784 break; 4785 } 4786 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 4787 case KVM_CLEAR_DIRTY_LOG: { 4788 struct kvm_clear_dirty_log log; 4789 4790 r = -EFAULT; 4791 if (copy_from_user(&log, argp, sizeof(log))) 4792 goto out; 4793 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); 4794 break; 4795 } 4796 #endif 4797 #ifdef CONFIG_KVM_MMIO 4798 case KVM_REGISTER_COALESCED_MMIO: { 4799 struct kvm_coalesced_mmio_zone zone; 4800 4801 r = -EFAULT; 4802 if (copy_from_user(&zone, argp, sizeof(zone))) 4803 goto out; 4804 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 4805 break; 4806 } 4807 case KVM_UNREGISTER_COALESCED_MMIO: { 4808 struct kvm_coalesced_mmio_zone zone; 4809 4810 r = -EFAULT; 4811 if (copy_from_user(&zone, argp, sizeof(zone))) 4812 goto out; 4813 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 4814 break; 4815 } 4816 #endif 4817 case KVM_IRQFD: { 4818 struct kvm_irqfd data; 4819 4820 r = -EFAULT; 4821 if (copy_from_user(&data, argp, sizeof(data))) 4822 goto out; 4823 r = kvm_irqfd(kvm, &data); 4824 break; 4825 } 4826 case KVM_IOEVENTFD: { 4827 struct kvm_ioeventfd data; 4828 4829 r = -EFAULT; 4830 if (copy_from_user(&data, argp, sizeof(data))) 4831 goto out; 4832 r = kvm_ioeventfd(kvm, &data); 4833 break; 4834 } 4835 #ifdef CONFIG_HAVE_KVM_MSI 4836 case KVM_SIGNAL_MSI: { 4837 struct kvm_msi msi; 4838 4839 r = -EFAULT; 4840 if (copy_from_user(&msi, argp, sizeof(msi))) 4841 goto out; 4842 r = kvm_send_userspace_msi(kvm, &msi); 4843 break; 4844 } 4845 #endif 4846 #ifdef __KVM_HAVE_IRQ_LINE 4847 case KVM_IRQ_LINE_STATUS: 4848 case KVM_IRQ_LINE: { 4849 struct kvm_irq_level irq_event; 4850 4851 r = -EFAULT; 4852 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 4853 goto out; 4854 4855 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 4856 ioctl == KVM_IRQ_LINE_STATUS); 4857 if (r) 4858 goto out; 4859 4860 r = -EFAULT; 4861 if (ioctl == KVM_IRQ_LINE_STATUS) { 4862 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 4863 goto out; 4864 } 4865 4866 r = 0; 4867 break; 4868 } 4869 #endif 4870 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 4871 case KVM_SET_GSI_ROUTING: { 4872 struct kvm_irq_routing routing; 4873 struct kvm_irq_routing __user *urouting; 4874 struct kvm_irq_routing_entry *entries = NULL; 4875 4876 r = -EFAULT; 4877 if (copy_from_user(&routing, argp, sizeof(routing))) 4878 goto out; 4879 r = -EINVAL; 4880 if (!kvm_arch_can_set_irq_routing(kvm)) 4881 goto out; 4882 if (routing.nr > KVM_MAX_IRQ_ROUTES) 4883 goto out; 4884 if (routing.flags) 4885 goto out; 4886 if (routing.nr) { 4887 urouting = argp; 4888 entries = vmemdup_user(urouting->entries, 4889 array_size(sizeof(*entries), 4890 routing.nr)); 4891 if (IS_ERR(entries)) { 4892 r = PTR_ERR(entries); 4893 goto out; 4894 } 4895 } 4896 r = kvm_set_irq_routing(kvm, entries, routing.nr, 4897 routing.flags); 4898 kvfree(entries); 4899 break; 4900 } 4901 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 4902 case KVM_CREATE_DEVICE: { 4903 struct kvm_create_device cd; 4904 4905 r = -EFAULT; 4906 if (copy_from_user(&cd, argp, sizeof(cd))) 4907 goto out; 4908 4909 r = kvm_ioctl_create_device(kvm, &cd); 4910 if (r) 4911 goto out; 4912 4913 r = -EFAULT; 4914 if (copy_to_user(argp, &cd, sizeof(cd))) 4915 goto out; 4916 4917 r = 0; 4918 break; 4919 } 4920 case KVM_CHECK_EXTENSION: 4921 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 4922 break; 4923 case KVM_RESET_DIRTY_RINGS: 4924 r = kvm_vm_ioctl_reset_dirty_pages(kvm); 4925 break; 4926 case KVM_GET_STATS_FD: 4927 r = kvm_vm_ioctl_get_stats_fd(kvm); 4928 break; 4929 default: 4930 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 4931 } 4932 out: 4933 return r; 4934 } 4935 4936 #ifdef CONFIG_KVM_COMPAT 4937 struct compat_kvm_dirty_log { 4938 __u32 slot; 4939 __u32 padding1; 4940 union { 4941 compat_uptr_t dirty_bitmap; /* one bit per page */ 4942 __u64 padding2; 4943 }; 4944 }; 4945 4946 struct compat_kvm_clear_dirty_log { 4947 __u32 slot; 4948 __u32 num_pages; 4949 __u64 first_page; 4950 union { 4951 compat_uptr_t dirty_bitmap; /* one bit per page */ 4952 __u64 padding2; 4953 }; 4954 }; 4955 4956 long __weak kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl, 4957 unsigned long arg) 4958 { 4959 return -ENOTTY; 4960 } 4961 4962 static long kvm_vm_compat_ioctl(struct file *filp, 4963 unsigned int ioctl, unsigned long arg) 4964 { 4965 struct kvm *kvm = filp->private_data; 4966 int r; 4967 4968 if (kvm->mm != current->mm || kvm->vm_dead) 4969 return -EIO; 4970 4971 r = kvm_arch_vm_compat_ioctl(filp, ioctl, arg); 4972 if (r != -ENOTTY) 4973 return r; 4974 4975 switch (ioctl) { 4976 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 4977 case KVM_CLEAR_DIRTY_LOG: { 4978 struct compat_kvm_clear_dirty_log compat_log; 4979 struct kvm_clear_dirty_log log; 4980 4981 if (copy_from_user(&compat_log, (void __user *)arg, 4982 sizeof(compat_log))) 4983 return -EFAULT; 4984 log.slot = compat_log.slot; 4985 log.num_pages = compat_log.num_pages; 4986 log.first_page = compat_log.first_page; 4987 log.padding2 = compat_log.padding2; 4988 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 4989 4990 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); 4991 break; 4992 } 4993 #endif 4994 case KVM_GET_DIRTY_LOG: { 4995 struct compat_kvm_dirty_log compat_log; 4996 struct kvm_dirty_log log; 4997 4998 if (copy_from_user(&compat_log, (void __user *)arg, 4999 sizeof(compat_log))) 5000 return -EFAULT; 5001 log.slot = compat_log.slot; 5002 log.padding1 = compat_log.padding1; 5003 log.padding2 = compat_log.padding2; 5004 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 5005 5006 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 5007 break; 5008 } 5009 default: 5010 r = kvm_vm_ioctl(filp, ioctl, arg); 5011 } 5012 return r; 5013 } 5014 #endif 5015 5016 static const struct file_operations kvm_vm_fops = { 5017 .release = kvm_vm_release, 5018 .unlocked_ioctl = kvm_vm_ioctl, 5019 .llseek = noop_llseek, 5020 KVM_COMPAT(kvm_vm_compat_ioctl), 5021 }; 5022 5023 bool file_is_kvm(struct file *file) 5024 { 5025 return file && file->f_op == &kvm_vm_fops; 5026 } 5027 EXPORT_SYMBOL_GPL(file_is_kvm); 5028 5029 static int kvm_dev_ioctl_create_vm(unsigned long type) 5030 { 5031 char fdname[ITOA_MAX_LEN + 1]; 5032 int r, fd; 5033 struct kvm *kvm; 5034 struct file *file; 5035 5036 fd = get_unused_fd_flags(O_CLOEXEC); 5037 if (fd < 0) 5038 return fd; 5039 5040 snprintf(fdname, sizeof(fdname), "%d", fd); 5041 5042 kvm = kvm_create_vm(type, fdname); 5043 if (IS_ERR(kvm)) { 5044 r = PTR_ERR(kvm); 5045 goto put_fd; 5046 } 5047 5048 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 5049 if (IS_ERR(file)) { 5050 r = PTR_ERR(file); 5051 goto put_kvm; 5052 } 5053 5054 /* 5055 * Don't call kvm_put_kvm anymore at this point; file->f_op is 5056 * already set, with ->release() being kvm_vm_release(). In error 5057 * cases it will be called by the final fput(file) and will take 5058 * care of doing kvm_put_kvm(kvm). 5059 */ 5060 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); 5061 5062 fd_install(fd, file); 5063 return fd; 5064 5065 put_kvm: 5066 kvm_put_kvm(kvm); 5067 put_fd: 5068 put_unused_fd(fd); 5069 return r; 5070 } 5071 5072 static long kvm_dev_ioctl(struct file *filp, 5073 unsigned int ioctl, unsigned long arg) 5074 { 5075 int r = -EINVAL; 5076 5077 switch (ioctl) { 5078 case KVM_GET_API_VERSION: 5079 if (arg) 5080 goto out; 5081 r = KVM_API_VERSION; 5082 break; 5083 case KVM_CREATE_VM: 5084 r = kvm_dev_ioctl_create_vm(arg); 5085 break; 5086 case KVM_CHECK_EXTENSION: 5087 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 5088 break; 5089 case KVM_GET_VCPU_MMAP_SIZE: 5090 if (arg) 5091 goto out; 5092 r = PAGE_SIZE; /* struct kvm_run */ 5093 #ifdef CONFIG_X86 5094 r += PAGE_SIZE; /* pio data page */ 5095 #endif 5096 #ifdef CONFIG_KVM_MMIO 5097 r += PAGE_SIZE; /* coalesced mmio ring page */ 5098 #endif 5099 break; 5100 case KVM_TRACE_ENABLE: 5101 case KVM_TRACE_PAUSE: 5102 case KVM_TRACE_DISABLE: 5103 r = -EOPNOTSUPP; 5104 break; 5105 default: 5106 return kvm_arch_dev_ioctl(filp, ioctl, arg); 5107 } 5108 out: 5109 return r; 5110 } 5111 5112 static struct file_operations kvm_chardev_ops = { 5113 .unlocked_ioctl = kvm_dev_ioctl, 5114 .llseek = noop_llseek, 5115 KVM_COMPAT(kvm_dev_ioctl), 5116 }; 5117 5118 static struct miscdevice kvm_dev = { 5119 KVM_MINOR, 5120 "kvm", 5121 &kvm_chardev_ops, 5122 }; 5123 5124 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 5125 __visible bool kvm_rebooting; 5126 EXPORT_SYMBOL_GPL(kvm_rebooting); 5127 5128 static DEFINE_PER_CPU(bool, hardware_enabled); 5129 static int kvm_usage_count; 5130 5131 static int __hardware_enable_nolock(void) 5132 { 5133 if (__this_cpu_read(hardware_enabled)) 5134 return 0; 5135 5136 if (kvm_arch_hardware_enable()) { 5137 pr_info("kvm: enabling virtualization on CPU%d failed\n", 5138 raw_smp_processor_id()); 5139 return -EIO; 5140 } 5141 5142 __this_cpu_write(hardware_enabled, true); 5143 return 0; 5144 } 5145 5146 static void hardware_enable_nolock(void *failed) 5147 { 5148 if (__hardware_enable_nolock()) 5149 atomic_inc(failed); 5150 } 5151 5152 static int kvm_online_cpu(unsigned int cpu) 5153 { 5154 int ret = 0; 5155 5156 /* 5157 * Abort the CPU online process if hardware virtualization cannot 5158 * be enabled. Otherwise running VMs would encounter unrecoverable 5159 * errors when scheduled to this CPU. 5160 */ 5161 mutex_lock(&kvm_lock); 5162 if (kvm_usage_count) 5163 ret = __hardware_enable_nolock(); 5164 mutex_unlock(&kvm_lock); 5165 return ret; 5166 } 5167 5168 static void hardware_disable_nolock(void *junk) 5169 { 5170 /* 5171 * Note, hardware_disable_all_nolock() tells all online CPUs to disable 5172 * hardware, not just CPUs that successfully enabled hardware! 5173 */ 5174 if (!__this_cpu_read(hardware_enabled)) 5175 return; 5176 5177 kvm_arch_hardware_disable(); 5178 5179 __this_cpu_write(hardware_enabled, false); 5180 } 5181 5182 static int kvm_offline_cpu(unsigned int cpu) 5183 { 5184 mutex_lock(&kvm_lock); 5185 if (kvm_usage_count) 5186 hardware_disable_nolock(NULL); 5187 mutex_unlock(&kvm_lock); 5188 return 0; 5189 } 5190 5191 static void hardware_disable_all_nolock(void) 5192 { 5193 BUG_ON(!kvm_usage_count); 5194 5195 kvm_usage_count--; 5196 if (!kvm_usage_count) 5197 on_each_cpu(hardware_disable_nolock, NULL, 1); 5198 } 5199 5200 static void hardware_disable_all(void) 5201 { 5202 cpus_read_lock(); 5203 mutex_lock(&kvm_lock); 5204 hardware_disable_all_nolock(); 5205 mutex_unlock(&kvm_lock); 5206 cpus_read_unlock(); 5207 } 5208 5209 static int hardware_enable_all(void) 5210 { 5211 atomic_t failed = ATOMIC_INIT(0); 5212 int r; 5213 5214 /* 5215 * Do not enable hardware virtualization if the system is going down. 5216 * If userspace initiated a forced reboot, e.g. reboot -f, then it's 5217 * possible for an in-flight KVM_CREATE_VM to trigger hardware enabling 5218 * after kvm_reboot() is called. Note, this relies on system_state 5219 * being set _before_ kvm_reboot(), which is why KVM uses a syscore ops 5220 * hook instead of registering a dedicated reboot notifier (the latter 5221 * runs before system_state is updated). 5222 */ 5223 if (system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF || 5224 system_state == SYSTEM_RESTART) 5225 return -EBUSY; 5226 5227 /* 5228 * When onlining a CPU, cpu_online_mask is set before kvm_online_cpu() 5229 * is called, and so on_each_cpu() between them includes the CPU that 5230 * is being onlined. As a result, hardware_enable_nolock() may get 5231 * invoked before kvm_online_cpu(), which also enables hardware if the 5232 * usage count is non-zero. Disable CPU hotplug to avoid attempting to 5233 * enable hardware multiple times. 5234 */ 5235 cpus_read_lock(); 5236 mutex_lock(&kvm_lock); 5237 5238 r = 0; 5239 5240 kvm_usage_count++; 5241 if (kvm_usage_count == 1) { 5242 on_each_cpu(hardware_enable_nolock, &failed, 1); 5243 5244 if (atomic_read(&failed)) { 5245 hardware_disable_all_nolock(); 5246 r = -EBUSY; 5247 } 5248 } 5249 5250 mutex_unlock(&kvm_lock); 5251 cpus_read_unlock(); 5252 5253 return r; 5254 } 5255 5256 static void kvm_shutdown(void) 5257 { 5258 /* 5259 * Disable hardware virtualization and set kvm_rebooting to indicate 5260 * that KVM has asynchronously disabled hardware virtualization, i.e. 5261 * that relevant errors and exceptions aren't entirely unexpected. 5262 * Some flavors of hardware virtualization need to be disabled before 5263 * transferring control to firmware (to perform shutdown/reboot), e.g. 5264 * on x86, virtualization can block INIT interrupts, which are used by 5265 * firmware to pull APs back under firmware control. Note, this path 5266 * is used for both shutdown and reboot scenarios, i.e. neither name is 5267 * 100% comprehensive. 5268 */ 5269 pr_info("kvm: exiting hardware virtualization\n"); 5270 kvm_rebooting = true; 5271 on_each_cpu(hardware_disable_nolock, NULL, 1); 5272 } 5273 5274 static int kvm_suspend(void) 5275 { 5276 /* 5277 * Secondary CPUs and CPU hotplug are disabled across the suspend/resume 5278 * callbacks, i.e. no need to acquire kvm_lock to ensure the usage count 5279 * is stable. Assert that kvm_lock is not held to ensure the system 5280 * isn't suspended while KVM is enabling hardware. Hardware enabling 5281 * can be preempted, but the task cannot be frozen until it has dropped 5282 * all locks (userspace tasks are frozen via a fake signal). 5283 */ 5284 lockdep_assert_not_held(&kvm_lock); 5285 lockdep_assert_irqs_disabled(); 5286 5287 if (kvm_usage_count) 5288 hardware_disable_nolock(NULL); 5289 return 0; 5290 } 5291 5292 static void kvm_resume(void) 5293 { 5294 lockdep_assert_not_held(&kvm_lock); 5295 lockdep_assert_irqs_disabled(); 5296 5297 if (kvm_usage_count) 5298 WARN_ON_ONCE(__hardware_enable_nolock()); 5299 } 5300 5301 static struct syscore_ops kvm_syscore_ops = { 5302 .suspend = kvm_suspend, 5303 .resume = kvm_resume, 5304 .shutdown = kvm_shutdown, 5305 }; 5306 #else /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */ 5307 static int hardware_enable_all(void) 5308 { 5309 return 0; 5310 } 5311 5312 static void hardware_disable_all(void) 5313 { 5314 5315 } 5316 #endif /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */ 5317 5318 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 5319 { 5320 int i; 5321 5322 for (i = 0; i < bus->dev_count; i++) { 5323 struct kvm_io_device *pos = bus->range[i].dev; 5324 5325 kvm_iodevice_destructor(pos); 5326 } 5327 kfree(bus); 5328 } 5329 5330 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 5331 const struct kvm_io_range *r2) 5332 { 5333 gpa_t addr1 = r1->addr; 5334 gpa_t addr2 = r2->addr; 5335 5336 if (addr1 < addr2) 5337 return -1; 5338 5339 /* If r2->len == 0, match the exact address. If r2->len != 0, 5340 * accept any overlapping write. Any order is acceptable for 5341 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 5342 * we process all of them. 5343 */ 5344 if (r2->len) { 5345 addr1 += r1->len; 5346 addr2 += r2->len; 5347 } 5348 5349 if (addr1 > addr2) 5350 return 1; 5351 5352 return 0; 5353 } 5354 5355 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 5356 { 5357 return kvm_io_bus_cmp(p1, p2); 5358 } 5359 5360 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 5361 gpa_t addr, int len) 5362 { 5363 struct kvm_io_range *range, key; 5364 int off; 5365 5366 key = (struct kvm_io_range) { 5367 .addr = addr, 5368 .len = len, 5369 }; 5370 5371 range = bsearch(&key, bus->range, bus->dev_count, 5372 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 5373 if (range == NULL) 5374 return -ENOENT; 5375 5376 off = range - bus->range; 5377 5378 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 5379 off--; 5380 5381 return off; 5382 } 5383 5384 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 5385 struct kvm_io_range *range, const void *val) 5386 { 5387 int idx; 5388 5389 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 5390 if (idx < 0) 5391 return -EOPNOTSUPP; 5392 5393 while (idx < bus->dev_count && 5394 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 5395 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 5396 range->len, val)) 5397 return idx; 5398 idx++; 5399 } 5400 5401 return -EOPNOTSUPP; 5402 } 5403 5404 /* kvm_io_bus_write - called under kvm->slots_lock */ 5405 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 5406 int len, const void *val) 5407 { 5408 struct kvm_io_bus *bus; 5409 struct kvm_io_range range; 5410 int r; 5411 5412 range = (struct kvm_io_range) { 5413 .addr = addr, 5414 .len = len, 5415 }; 5416 5417 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 5418 if (!bus) 5419 return -ENOMEM; 5420 r = __kvm_io_bus_write(vcpu, bus, &range, val); 5421 return r < 0 ? r : 0; 5422 } 5423 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 5424 5425 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 5426 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 5427 gpa_t addr, int len, const void *val, long cookie) 5428 { 5429 struct kvm_io_bus *bus; 5430 struct kvm_io_range range; 5431 5432 range = (struct kvm_io_range) { 5433 .addr = addr, 5434 .len = len, 5435 }; 5436 5437 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 5438 if (!bus) 5439 return -ENOMEM; 5440 5441 /* First try the device referenced by cookie. */ 5442 if ((cookie >= 0) && (cookie < bus->dev_count) && 5443 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 5444 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 5445 val)) 5446 return cookie; 5447 5448 /* 5449 * cookie contained garbage; fall back to search and return the 5450 * correct cookie value. 5451 */ 5452 return __kvm_io_bus_write(vcpu, bus, &range, val); 5453 } 5454 5455 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 5456 struct kvm_io_range *range, void *val) 5457 { 5458 int idx; 5459 5460 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 5461 if (idx < 0) 5462 return -EOPNOTSUPP; 5463 5464 while (idx < bus->dev_count && 5465 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 5466 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 5467 range->len, val)) 5468 return idx; 5469 idx++; 5470 } 5471 5472 return -EOPNOTSUPP; 5473 } 5474 5475 /* kvm_io_bus_read - called under kvm->slots_lock */ 5476 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 5477 int len, void *val) 5478 { 5479 struct kvm_io_bus *bus; 5480 struct kvm_io_range range; 5481 int r; 5482 5483 range = (struct kvm_io_range) { 5484 .addr = addr, 5485 .len = len, 5486 }; 5487 5488 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 5489 if (!bus) 5490 return -ENOMEM; 5491 r = __kvm_io_bus_read(vcpu, bus, &range, val); 5492 return r < 0 ? r : 0; 5493 } 5494 5495 /* Caller must hold slots_lock. */ 5496 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 5497 int len, struct kvm_io_device *dev) 5498 { 5499 int i; 5500 struct kvm_io_bus *new_bus, *bus; 5501 struct kvm_io_range range; 5502 5503 bus = kvm_get_bus(kvm, bus_idx); 5504 if (!bus) 5505 return -ENOMEM; 5506 5507 /* exclude ioeventfd which is limited by maximum fd */ 5508 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 5509 return -ENOSPC; 5510 5511 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1), 5512 GFP_KERNEL_ACCOUNT); 5513 if (!new_bus) 5514 return -ENOMEM; 5515 5516 range = (struct kvm_io_range) { 5517 .addr = addr, 5518 .len = len, 5519 .dev = dev, 5520 }; 5521 5522 for (i = 0; i < bus->dev_count; i++) 5523 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0) 5524 break; 5525 5526 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 5527 new_bus->dev_count++; 5528 new_bus->range[i] = range; 5529 memcpy(new_bus->range + i + 1, bus->range + i, 5530 (bus->dev_count - i) * sizeof(struct kvm_io_range)); 5531 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 5532 synchronize_srcu_expedited(&kvm->srcu); 5533 kfree(bus); 5534 5535 return 0; 5536 } 5537 5538 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 5539 struct kvm_io_device *dev) 5540 { 5541 int i, j; 5542 struct kvm_io_bus *new_bus, *bus; 5543 5544 lockdep_assert_held(&kvm->slots_lock); 5545 5546 bus = kvm_get_bus(kvm, bus_idx); 5547 if (!bus) 5548 return 0; 5549 5550 for (i = 0; i < bus->dev_count; i++) { 5551 if (bus->range[i].dev == dev) { 5552 break; 5553 } 5554 } 5555 5556 if (i == bus->dev_count) 5557 return 0; 5558 5559 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1), 5560 GFP_KERNEL_ACCOUNT); 5561 if (new_bus) { 5562 memcpy(new_bus, bus, struct_size(bus, range, i)); 5563 new_bus->dev_count--; 5564 memcpy(new_bus->range + i, bus->range + i + 1, 5565 flex_array_size(new_bus, range, new_bus->dev_count - i)); 5566 } 5567 5568 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 5569 synchronize_srcu_expedited(&kvm->srcu); 5570 5571 /* Destroy the old bus _after_ installing the (null) bus. */ 5572 if (!new_bus) { 5573 pr_err("kvm: failed to shrink bus, removing it completely\n"); 5574 for (j = 0; j < bus->dev_count; j++) { 5575 if (j == i) 5576 continue; 5577 kvm_iodevice_destructor(bus->range[j].dev); 5578 } 5579 } 5580 5581 kfree(bus); 5582 return new_bus ? 0 : -ENOMEM; 5583 } 5584 5585 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 5586 gpa_t addr) 5587 { 5588 struct kvm_io_bus *bus; 5589 int dev_idx, srcu_idx; 5590 struct kvm_io_device *iodev = NULL; 5591 5592 srcu_idx = srcu_read_lock(&kvm->srcu); 5593 5594 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 5595 if (!bus) 5596 goto out_unlock; 5597 5598 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); 5599 if (dev_idx < 0) 5600 goto out_unlock; 5601 5602 iodev = bus->range[dev_idx].dev; 5603 5604 out_unlock: 5605 srcu_read_unlock(&kvm->srcu, srcu_idx); 5606 5607 return iodev; 5608 } 5609 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev); 5610 5611 static int kvm_debugfs_open(struct inode *inode, struct file *file, 5612 int (*get)(void *, u64 *), int (*set)(void *, u64), 5613 const char *fmt) 5614 { 5615 int ret; 5616 struct kvm_stat_data *stat_data = inode->i_private; 5617 5618 /* 5619 * The debugfs files are a reference to the kvm struct which 5620 * is still valid when kvm_destroy_vm is called. kvm_get_kvm_safe 5621 * avoids the race between open and the removal of the debugfs directory. 5622 */ 5623 if (!kvm_get_kvm_safe(stat_data->kvm)) 5624 return -ENOENT; 5625 5626 ret = simple_attr_open(inode, file, get, 5627 kvm_stats_debugfs_mode(stat_data->desc) & 0222 5628 ? set : NULL, fmt); 5629 if (ret) 5630 kvm_put_kvm(stat_data->kvm); 5631 5632 return ret; 5633 } 5634 5635 static int kvm_debugfs_release(struct inode *inode, struct file *file) 5636 { 5637 struct kvm_stat_data *stat_data = inode->i_private; 5638 5639 simple_attr_release(inode, file); 5640 kvm_put_kvm(stat_data->kvm); 5641 5642 return 0; 5643 } 5644 5645 static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val) 5646 { 5647 *val = *(u64 *)((void *)(&kvm->stat) + offset); 5648 5649 return 0; 5650 } 5651 5652 static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset) 5653 { 5654 *(u64 *)((void *)(&kvm->stat) + offset) = 0; 5655 5656 return 0; 5657 } 5658 5659 static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val) 5660 { 5661 unsigned long i; 5662 struct kvm_vcpu *vcpu; 5663 5664 *val = 0; 5665 5666 kvm_for_each_vcpu(i, vcpu, kvm) 5667 *val += *(u64 *)((void *)(&vcpu->stat) + offset); 5668 5669 return 0; 5670 } 5671 5672 static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset) 5673 { 5674 unsigned long i; 5675 struct kvm_vcpu *vcpu; 5676 5677 kvm_for_each_vcpu(i, vcpu, kvm) 5678 *(u64 *)((void *)(&vcpu->stat) + offset) = 0; 5679 5680 return 0; 5681 } 5682 5683 static int kvm_stat_data_get(void *data, u64 *val) 5684 { 5685 int r = -EFAULT; 5686 struct kvm_stat_data *stat_data = data; 5687 5688 switch (stat_data->kind) { 5689 case KVM_STAT_VM: 5690 r = kvm_get_stat_per_vm(stat_data->kvm, 5691 stat_data->desc->desc.offset, val); 5692 break; 5693 case KVM_STAT_VCPU: 5694 r = kvm_get_stat_per_vcpu(stat_data->kvm, 5695 stat_data->desc->desc.offset, val); 5696 break; 5697 } 5698 5699 return r; 5700 } 5701 5702 static int kvm_stat_data_clear(void *data, u64 val) 5703 { 5704 int r = -EFAULT; 5705 struct kvm_stat_data *stat_data = data; 5706 5707 if (val) 5708 return -EINVAL; 5709 5710 switch (stat_data->kind) { 5711 case KVM_STAT_VM: 5712 r = kvm_clear_stat_per_vm(stat_data->kvm, 5713 stat_data->desc->desc.offset); 5714 break; 5715 case KVM_STAT_VCPU: 5716 r = kvm_clear_stat_per_vcpu(stat_data->kvm, 5717 stat_data->desc->desc.offset); 5718 break; 5719 } 5720 5721 return r; 5722 } 5723 5724 static int kvm_stat_data_open(struct inode *inode, struct file *file) 5725 { 5726 __simple_attr_check_format("%llu\n", 0ull); 5727 return kvm_debugfs_open(inode, file, kvm_stat_data_get, 5728 kvm_stat_data_clear, "%llu\n"); 5729 } 5730 5731 static const struct file_operations stat_fops_per_vm = { 5732 .owner = THIS_MODULE, 5733 .open = kvm_stat_data_open, 5734 .release = kvm_debugfs_release, 5735 .read = simple_attr_read, 5736 .write = simple_attr_write, 5737 .llseek = no_llseek, 5738 }; 5739 5740 static int vm_stat_get(void *_offset, u64 *val) 5741 { 5742 unsigned offset = (long)_offset; 5743 struct kvm *kvm; 5744 u64 tmp_val; 5745 5746 *val = 0; 5747 mutex_lock(&kvm_lock); 5748 list_for_each_entry(kvm, &vm_list, vm_list) { 5749 kvm_get_stat_per_vm(kvm, offset, &tmp_val); 5750 *val += tmp_val; 5751 } 5752 mutex_unlock(&kvm_lock); 5753 return 0; 5754 } 5755 5756 static int vm_stat_clear(void *_offset, u64 val) 5757 { 5758 unsigned offset = (long)_offset; 5759 struct kvm *kvm; 5760 5761 if (val) 5762 return -EINVAL; 5763 5764 mutex_lock(&kvm_lock); 5765 list_for_each_entry(kvm, &vm_list, vm_list) { 5766 kvm_clear_stat_per_vm(kvm, offset); 5767 } 5768 mutex_unlock(&kvm_lock); 5769 5770 return 0; 5771 } 5772 5773 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); 5774 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_readonly_fops, vm_stat_get, NULL, "%llu\n"); 5775 5776 static int vcpu_stat_get(void *_offset, u64 *val) 5777 { 5778 unsigned offset = (long)_offset; 5779 struct kvm *kvm; 5780 u64 tmp_val; 5781 5782 *val = 0; 5783 mutex_lock(&kvm_lock); 5784 list_for_each_entry(kvm, &vm_list, vm_list) { 5785 kvm_get_stat_per_vcpu(kvm, offset, &tmp_val); 5786 *val += tmp_val; 5787 } 5788 mutex_unlock(&kvm_lock); 5789 return 0; 5790 } 5791 5792 static int vcpu_stat_clear(void *_offset, u64 val) 5793 { 5794 unsigned offset = (long)_offset; 5795 struct kvm *kvm; 5796 5797 if (val) 5798 return -EINVAL; 5799 5800 mutex_lock(&kvm_lock); 5801 list_for_each_entry(kvm, &vm_list, vm_list) { 5802 kvm_clear_stat_per_vcpu(kvm, offset); 5803 } 5804 mutex_unlock(&kvm_lock); 5805 5806 return 0; 5807 } 5808 5809 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, 5810 "%llu\n"); 5811 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n"); 5812 5813 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) 5814 { 5815 struct kobj_uevent_env *env; 5816 unsigned long long created, active; 5817 5818 if (!kvm_dev.this_device || !kvm) 5819 return; 5820 5821 mutex_lock(&kvm_lock); 5822 if (type == KVM_EVENT_CREATE_VM) { 5823 kvm_createvm_count++; 5824 kvm_active_vms++; 5825 } else if (type == KVM_EVENT_DESTROY_VM) { 5826 kvm_active_vms--; 5827 } 5828 created = kvm_createvm_count; 5829 active = kvm_active_vms; 5830 mutex_unlock(&kvm_lock); 5831 5832 env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT); 5833 if (!env) 5834 return; 5835 5836 add_uevent_var(env, "CREATED=%llu", created); 5837 add_uevent_var(env, "COUNT=%llu", active); 5838 5839 if (type == KVM_EVENT_CREATE_VM) { 5840 add_uevent_var(env, "EVENT=create"); 5841 kvm->userspace_pid = task_pid_nr(current); 5842 } else if (type == KVM_EVENT_DESTROY_VM) { 5843 add_uevent_var(env, "EVENT=destroy"); 5844 } 5845 add_uevent_var(env, "PID=%d", kvm->userspace_pid); 5846 5847 if (!IS_ERR(kvm->debugfs_dentry)) { 5848 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT); 5849 5850 if (p) { 5851 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); 5852 if (!IS_ERR(tmp)) 5853 add_uevent_var(env, "STATS_PATH=%s", tmp); 5854 kfree(p); 5855 } 5856 } 5857 /* no need for checks, since we are adding at most only 5 keys */ 5858 env->envp[env->envp_idx++] = NULL; 5859 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp); 5860 kfree(env); 5861 } 5862 5863 static void kvm_init_debug(void) 5864 { 5865 const struct file_operations *fops; 5866 const struct _kvm_stats_desc *pdesc; 5867 int i; 5868 5869 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 5870 5871 for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { 5872 pdesc = &kvm_vm_stats_desc[i]; 5873 if (kvm_stats_debugfs_mode(pdesc) & 0222) 5874 fops = &vm_stat_fops; 5875 else 5876 fops = &vm_stat_readonly_fops; 5877 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), 5878 kvm_debugfs_dir, 5879 (void *)(long)pdesc->desc.offset, fops); 5880 } 5881 5882 for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) { 5883 pdesc = &kvm_vcpu_stats_desc[i]; 5884 if (kvm_stats_debugfs_mode(pdesc) & 0222) 5885 fops = &vcpu_stat_fops; 5886 else 5887 fops = &vcpu_stat_readonly_fops; 5888 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), 5889 kvm_debugfs_dir, 5890 (void *)(long)pdesc->desc.offset, fops); 5891 } 5892 } 5893 5894 static inline 5895 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 5896 { 5897 return container_of(pn, struct kvm_vcpu, preempt_notifier); 5898 } 5899 5900 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 5901 { 5902 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 5903 5904 WRITE_ONCE(vcpu->preempted, false); 5905 WRITE_ONCE(vcpu->ready, false); 5906 5907 __this_cpu_write(kvm_running_vcpu, vcpu); 5908 kvm_arch_sched_in(vcpu, cpu); 5909 kvm_arch_vcpu_load(vcpu, cpu); 5910 } 5911 5912 static void kvm_sched_out(struct preempt_notifier *pn, 5913 struct task_struct *next) 5914 { 5915 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 5916 5917 if (current->on_rq) { 5918 WRITE_ONCE(vcpu->preempted, true); 5919 WRITE_ONCE(vcpu->ready, true); 5920 } 5921 kvm_arch_vcpu_put(vcpu); 5922 __this_cpu_write(kvm_running_vcpu, NULL); 5923 } 5924 5925 /** 5926 * kvm_get_running_vcpu - get the vcpu running on the current CPU. 5927 * 5928 * We can disable preemption locally around accessing the per-CPU variable, 5929 * and use the resolved vcpu pointer after enabling preemption again, 5930 * because even if the current thread is migrated to another CPU, reading 5931 * the per-CPU value later will give us the same value as we update the 5932 * per-CPU variable in the preempt notifier handlers. 5933 */ 5934 struct kvm_vcpu *kvm_get_running_vcpu(void) 5935 { 5936 struct kvm_vcpu *vcpu; 5937 5938 preempt_disable(); 5939 vcpu = __this_cpu_read(kvm_running_vcpu); 5940 preempt_enable(); 5941 5942 return vcpu; 5943 } 5944 EXPORT_SYMBOL_GPL(kvm_get_running_vcpu); 5945 5946 /** 5947 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus. 5948 */ 5949 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void) 5950 { 5951 return &kvm_running_vcpu; 5952 } 5953 5954 #ifdef CONFIG_GUEST_PERF_EVENTS 5955 static unsigned int kvm_guest_state(void) 5956 { 5957 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 5958 unsigned int state; 5959 5960 if (!kvm_arch_pmi_in_guest(vcpu)) 5961 return 0; 5962 5963 state = PERF_GUEST_ACTIVE; 5964 if (!kvm_arch_vcpu_in_kernel(vcpu)) 5965 state |= PERF_GUEST_USER; 5966 5967 return state; 5968 } 5969 5970 static unsigned long kvm_guest_get_ip(void) 5971 { 5972 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 5973 5974 /* Retrieving the IP must be guarded by a call to kvm_guest_state(). */ 5975 if (WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu))) 5976 return 0; 5977 5978 return kvm_arch_vcpu_get_ip(vcpu); 5979 } 5980 5981 static struct perf_guest_info_callbacks kvm_guest_cbs = { 5982 .state = kvm_guest_state, 5983 .get_ip = kvm_guest_get_ip, 5984 .handle_intel_pt_intr = NULL, 5985 }; 5986 5987 void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void)) 5988 { 5989 kvm_guest_cbs.handle_intel_pt_intr = pt_intr_handler; 5990 perf_register_guest_info_callbacks(&kvm_guest_cbs); 5991 } 5992 void kvm_unregister_perf_callbacks(void) 5993 { 5994 perf_unregister_guest_info_callbacks(&kvm_guest_cbs); 5995 } 5996 #endif 5997 5998 int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) 5999 { 6000 int r; 6001 int cpu; 6002 6003 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 6004 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_ONLINE, "kvm/cpu:online", 6005 kvm_online_cpu, kvm_offline_cpu); 6006 if (r) 6007 return r; 6008 6009 register_syscore_ops(&kvm_syscore_ops); 6010 #endif 6011 6012 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 6013 if (!vcpu_align) 6014 vcpu_align = __alignof__(struct kvm_vcpu); 6015 kvm_vcpu_cache = 6016 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, 6017 SLAB_ACCOUNT, 6018 offsetof(struct kvm_vcpu, arch), 6019 offsetofend(struct kvm_vcpu, stats_id) 6020 - offsetof(struct kvm_vcpu, arch), 6021 NULL); 6022 if (!kvm_vcpu_cache) { 6023 r = -ENOMEM; 6024 goto err_vcpu_cache; 6025 } 6026 6027 for_each_possible_cpu(cpu) { 6028 if (!alloc_cpumask_var_node(&per_cpu(cpu_kick_mask, cpu), 6029 GFP_KERNEL, cpu_to_node(cpu))) { 6030 r = -ENOMEM; 6031 goto err_cpu_kick_mask; 6032 } 6033 } 6034 6035 r = kvm_irqfd_init(); 6036 if (r) 6037 goto err_irqfd; 6038 6039 r = kvm_async_pf_init(); 6040 if (r) 6041 goto err_async_pf; 6042 6043 kvm_chardev_ops.owner = module; 6044 6045 kvm_preempt_ops.sched_in = kvm_sched_in; 6046 kvm_preempt_ops.sched_out = kvm_sched_out; 6047 6048 kvm_init_debug(); 6049 6050 r = kvm_vfio_ops_init(); 6051 if (WARN_ON_ONCE(r)) 6052 goto err_vfio; 6053 6054 /* 6055 * Registration _must_ be the very last thing done, as this exposes 6056 * /dev/kvm to userspace, i.e. all infrastructure must be setup! 6057 */ 6058 r = misc_register(&kvm_dev); 6059 if (r) { 6060 pr_err("kvm: misc device register failed\n"); 6061 goto err_register; 6062 } 6063 6064 return 0; 6065 6066 err_register: 6067 kvm_vfio_ops_exit(); 6068 err_vfio: 6069 kvm_async_pf_deinit(); 6070 err_async_pf: 6071 kvm_irqfd_exit(); 6072 err_irqfd: 6073 err_cpu_kick_mask: 6074 for_each_possible_cpu(cpu) 6075 free_cpumask_var(per_cpu(cpu_kick_mask, cpu)); 6076 kmem_cache_destroy(kvm_vcpu_cache); 6077 err_vcpu_cache: 6078 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 6079 unregister_syscore_ops(&kvm_syscore_ops); 6080 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_ONLINE); 6081 #endif 6082 return r; 6083 } 6084 EXPORT_SYMBOL_GPL(kvm_init); 6085 6086 void kvm_exit(void) 6087 { 6088 int cpu; 6089 6090 /* 6091 * Note, unregistering /dev/kvm doesn't strictly need to come first, 6092 * fops_get(), a.k.a. try_module_get(), prevents acquiring references 6093 * to KVM while the module is being stopped. 6094 */ 6095 misc_deregister(&kvm_dev); 6096 6097 debugfs_remove_recursive(kvm_debugfs_dir); 6098 for_each_possible_cpu(cpu) 6099 free_cpumask_var(per_cpu(cpu_kick_mask, cpu)); 6100 kmem_cache_destroy(kvm_vcpu_cache); 6101 kvm_vfio_ops_exit(); 6102 kvm_async_pf_deinit(); 6103 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 6104 unregister_syscore_ops(&kvm_syscore_ops); 6105 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_ONLINE); 6106 #endif 6107 kvm_irqfd_exit(); 6108 } 6109 EXPORT_SYMBOL_GPL(kvm_exit); 6110 6111 struct kvm_vm_worker_thread_context { 6112 struct kvm *kvm; 6113 struct task_struct *parent; 6114 struct completion init_done; 6115 kvm_vm_thread_fn_t thread_fn; 6116 uintptr_t data; 6117 int err; 6118 }; 6119 6120 static int kvm_vm_worker_thread(void *context) 6121 { 6122 /* 6123 * The init_context is allocated on the stack of the parent thread, so 6124 * we have to locally copy anything that is needed beyond initialization 6125 */ 6126 struct kvm_vm_worker_thread_context *init_context = context; 6127 struct task_struct *parent; 6128 struct kvm *kvm = init_context->kvm; 6129 kvm_vm_thread_fn_t thread_fn = init_context->thread_fn; 6130 uintptr_t data = init_context->data; 6131 int err; 6132 6133 err = kthread_park(current); 6134 /* kthread_park(current) is never supposed to return an error */ 6135 WARN_ON(err != 0); 6136 if (err) 6137 goto init_complete; 6138 6139 err = cgroup_attach_task_all(init_context->parent, current); 6140 if (err) { 6141 kvm_err("%s: cgroup_attach_task_all failed with err %d\n", 6142 __func__, err); 6143 goto init_complete; 6144 } 6145 6146 set_user_nice(current, task_nice(init_context->parent)); 6147 6148 init_complete: 6149 init_context->err = err; 6150 complete(&init_context->init_done); 6151 init_context = NULL; 6152 6153 if (err) 6154 goto out; 6155 6156 /* Wait to be woken up by the spawner before proceeding. */ 6157 kthread_parkme(); 6158 6159 if (!kthread_should_stop()) 6160 err = thread_fn(kvm, data); 6161 6162 out: 6163 /* 6164 * Move kthread back to its original cgroup to prevent it lingering in 6165 * the cgroup of the VM process, after the latter finishes its 6166 * execution. 6167 * 6168 * kthread_stop() waits on the 'exited' completion condition which is 6169 * set in exit_mm(), via mm_release(), in do_exit(). However, the 6170 * kthread is removed from the cgroup in the cgroup_exit() which is 6171 * called after the exit_mm(). This causes the kthread_stop() to return 6172 * before the kthread actually quits the cgroup. 6173 */ 6174 rcu_read_lock(); 6175 parent = rcu_dereference(current->real_parent); 6176 get_task_struct(parent); 6177 rcu_read_unlock(); 6178 cgroup_attach_task_all(parent, current); 6179 put_task_struct(parent); 6180 6181 return err; 6182 } 6183 6184 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, 6185 uintptr_t data, const char *name, 6186 struct task_struct **thread_ptr) 6187 { 6188 struct kvm_vm_worker_thread_context init_context = {}; 6189 struct task_struct *thread; 6190 6191 *thread_ptr = NULL; 6192 init_context.kvm = kvm; 6193 init_context.parent = current; 6194 init_context.thread_fn = thread_fn; 6195 init_context.data = data; 6196 init_completion(&init_context.init_done); 6197 6198 thread = kthread_run(kvm_vm_worker_thread, &init_context, 6199 "%s-%d", name, task_pid_nr(current)); 6200 if (IS_ERR(thread)) 6201 return PTR_ERR(thread); 6202 6203 /* kthread_run is never supposed to return NULL */ 6204 WARN_ON(thread == NULL); 6205 6206 wait_for_completion(&init_context.init_done); 6207 6208 if (!init_context.err) 6209 *thread_ptr = thread; 6210 6211 return init_context.err; 6212 } 6213