1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef __KVM_HOST_H 3 #define __KVM_HOST_H 4 5 6 #include <linux/types.h> 7 #include <linux/hardirq.h> 8 #include <linux/list.h> 9 #include <linux/mutex.h> 10 #include <linux/spinlock.h> 11 #include <linux/signal.h> 12 #include <linux/sched.h> 13 #include <linux/sched/stat.h> 14 #include <linux/bug.h> 15 #include <linux/minmax.h> 16 #include <linux/mm.h> 17 #include <linux/mmu_notifier.h> 18 #include <linux/preempt.h> 19 #include <linux/msi.h> 20 #include <linux/slab.h> 21 #include <linux/vmalloc.h> 22 #include <linux/rcupdate.h> 23 #include <linux/ratelimit.h> 24 #include <linux/err.h> 25 #include <linux/irqflags.h> 26 #include <linux/context_tracking.h> 27 #include <linux/irqbypass.h> 28 #include <linux/rcuwait.h> 29 #include <linux/refcount.h> 30 #include <linux/nospec.h> 31 #include <linux/notifier.h> 32 #include <linux/hashtable.h> 33 #include <linux/interval_tree.h> 34 #include <linux/rbtree.h> 35 #include <linux/xarray.h> 36 #include <asm/signal.h> 37 38 #include <linux/kvm.h> 39 #include <linux/kvm_para.h> 40 41 #include <linux/kvm_types.h> 42 43 #include <asm/kvm_host.h> 44 #include <linux/kvm_dirty_ring.h> 45 46 #ifndef KVM_MAX_VCPU_IDS 47 #define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS 48 #endif 49 50 /* 51 * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used 52 * in kvm, other bits are visible for userspace which are defined in 53 * include/linux/kvm_h. 54 */ 55 #define KVM_MEMSLOT_INVALID (1UL << 16) 56 57 /* 58 * Bit 63 of the memslot generation number is an "update in-progress flag", 59 * e.g. is temporarily set for the duration of install_new_memslots(). 60 * This flag effectively creates a unique generation number that is used to 61 * mark cached memslot data, e.g. MMIO accesses, as potentially being stale, 62 * i.e. may (or may not) have come from the previous memslots generation. 63 * 64 * This is necessary because the actual memslots update is not atomic with 65 * respect to the generation number update. Updating the generation number 66 * first would allow a vCPU to cache a spte from the old memslots using the 67 * new generation number, and updating the generation number after switching 68 * to the new memslots would allow cache hits using the old generation number 69 * to reference the defunct memslots. 70 * 71 * This mechanism is used to prevent getting hits in KVM's caches while a 72 * memslot update is in-progress, and to prevent cache hits *after* updating 73 * the actual generation number against accesses that were inserted into the 74 * cache *before* the memslots were updated. 75 */ 76 #define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS BIT_ULL(63) 77 78 /* Two fragments for cross MMIO pages. */ 79 #define KVM_MAX_MMIO_FRAGMENTS 2 80 81 #ifndef KVM_ADDRESS_SPACE_NUM 82 #define KVM_ADDRESS_SPACE_NUM 1 83 #endif 84 85 /* 86 * For the normal pfn, the highest 12 bits should be zero, 87 * so we can mask bit 62 ~ bit 52 to indicate the error pfn, 88 * mask bit 63 to indicate the noslot pfn. 89 */ 90 #define KVM_PFN_ERR_MASK (0x7ffULL << 52) 91 #define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52) 92 #define KVM_PFN_NOSLOT (0x1ULL << 63) 93 94 #define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK) 95 #define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1) 96 #define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2) 97 98 /* 99 * error pfns indicate that the gfn is in slot but faild to 100 * translate it to pfn on host. 101 */ 102 static inline bool is_error_pfn(kvm_pfn_t pfn) 103 { 104 return !!(pfn & KVM_PFN_ERR_MASK); 105 } 106 107 /* 108 * error_noslot pfns indicate that the gfn can not be 109 * translated to pfn - it is not in slot or failed to 110 * translate it to pfn. 111 */ 112 static inline bool is_error_noslot_pfn(kvm_pfn_t pfn) 113 { 114 return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK); 115 } 116 117 /* noslot pfn indicates that the gfn is not in slot. */ 118 static inline bool is_noslot_pfn(kvm_pfn_t pfn) 119 { 120 return pfn == KVM_PFN_NOSLOT; 121 } 122 123 /* 124 * architectures with KVM_HVA_ERR_BAD other than PAGE_OFFSET (e.g. s390) 125 * provide own defines and kvm_is_error_hva 126 */ 127 #ifndef KVM_HVA_ERR_BAD 128 129 #define KVM_HVA_ERR_BAD (PAGE_OFFSET) 130 #define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE) 131 132 static inline bool kvm_is_error_hva(unsigned long addr) 133 { 134 return addr >= PAGE_OFFSET; 135 } 136 137 #endif 138 139 #define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT)) 140 141 static inline bool is_error_page(struct page *page) 142 { 143 return IS_ERR(page); 144 } 145 146 #define KVM_REQUEST_MASK GENMASK(7,0) 147 #define KVM_REQUEST_NO_WAKEUP BIT(8) 148 #define KVM_REQUEST_WAIT BIT(9) 149 /* 150 * Architecture-independent vcpu->requests bit members 151 * Bits 4-7 are reserved for more arch-independent bits. 152 */ 153 #define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 154 #define KVM_REQ_MMU_RELOAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 155 #define KVM_REQ_UNBLOCK 2 156 #define KVM_REQ_UNHALT 3 157 #define KVM_REQ_VM_DEAD (4 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 158 #define KVM_REQ_GPC_INVALIDATE (5 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 159 #define KVM_REQUEST_ARCH_BASE 8 160 161 #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \ 162 BUILD_BUG_ON((unsigned)(nr) >= (sizeof_field(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \ 163 (unsigned)(((nr) + KVM_REQUEST_ARCH_BASE) | (flags)); \ 164 }) 165 #define KVM_ARCH_REQ(nr) KVM_ARCH_REQ_FLAGS(nr, 0) 166 167 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, 168 unsigned long *vcpu_bitmap); 169 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req); 170 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req, 171 struct kvm_vcpu *except); 172 bool kvm_make_cpus_request_mask(struct kvm *kvm, unsigned int req, 173 unsigned long *vcpu_bitmap); 174 175 #define KVM_USERSPACE_IRQ_SOURCE_ID 0 176 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1 177 178 extern struct mutex kvm_lock; 179 extern struct list_head vm_list; 180 181 struct kvm_io_range { 182 gpa_t addr; 183 int len; 184 struct kvm_io_device *dev; 185 }; 186 187 #define NR_IOBUS_DEVS 1000 188 189 struct kvm_io_bus { 190 int dev_count; 191 int ioeventfd_count; 192 struct kvm_io_range range[]; 193 }; 194 195 enum kvm_bus { 196 KVM_MMIO_BUS, 197 KVM_PIO_BUS, 198 KVM_VIRTIO_CCW_NOTIFY_BUS, 199 KVM_FAST_MMIO_BUS, 200 KVM_NR_BUSES 201 }; 202 203 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 204 int len, const void *val); 205 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 206 gpa_t addr, int len, const void *val, long cookie); 207 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 208 int len, void *val); 209 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 210 int len, struct kvm_io_device *dev); 211 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 212 struct kvm_io_device *dev); 213 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 214 gpa_t addr); 215 216 #ifdef CONFIG_KVM_ASYNC_PF 217 struct kvm_async_pf { 218 struct work_struct work; 219 struct list_head link; 220 struct list_head queue; 221 struct kvm_vcpu *vcpu; 222 struct mm_struct *mm; 223 gpa_t cr2_or_gpa; 224 unsigned long addr; 225 struct kvm_arch_async_pf arch; 226 bool wakeup_all; 227 bool notpresent_injected; 228 }; 229 230 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu); 231 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu); 232 bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 233 unsigned long hva, struct kvm_arch_async_pf *arch); 234 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); 235 #endif 236 237 #ifdef KVM_ARCH_WANT_MMU_NOTIFIER 238 struct kvm_gfn_range { 239 struct kvm_memory_slot *slot; 240 gfn_t start; 241 gfn_t end; 242 pte_t pte; 243 bool may_block; 244 }; 245 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); 246 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range); 247 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range); 248 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range); 249 #endif 250 251 enum { 252 OUTSIDE_GUEST_MODE, 253 IN_GUEST_MODE, 254 EXITING_GUEST_MODE, 255 READING_SHADOW_PAGE_TABLES, 256 }; 257 258 #define KVM_UNMAPPED_PAGE ((void *) 0x500 + POISON_POINTER_DELTA) 259 260 struct kvm_host_map { 261 /* 262 * Only valid if the 'pfn' is managed by the host kernel (i.e. There is 263 * a 'struct page' for it. When using mem= kernel parameter some memory 264 * can be used as guest memory but they are not managed by host 265 * kernel). 266 * If 'pfn' is not managed by the host kernel, this field is 267 * initialized to KVM_UNMAPPED_PAGE. 268 */ 269 struct page *page; 270 void *hva; 271 kvm_pfn_t pfn; 272 kvm_pfn_t gfn; 273 }; 274 275 /* 276 * Used to check if the mapping is valid or not. Never use 'kvm_host_map' 277 * directly to check for that. 278 */ 279 static inline bool kvm_vcpu_mapped(struct kvm_host_map *map) 280 { 281 return !!map->hva; 282 } 283 284 static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop) 285 { 286 return single_task_running() && !need_resched() && ktime_before(cur, stop); 287 } 288 289 /* 290 * Sometimes a large or cross-page mmio needs to be broken up into separate 291 * exits for userspace servicing. 292 */ 293 struct kvm_mmio_fragment { 294 gpa_t gpa; 295 void *data; 296 unsigned len; 297 }; 298 299 struct kvm_vcpu { 300 struct kvm *kvm; 301 #ifdef CONFIG_PREEMPT_NOTIFIERS 302 struct preempt_notifier preempt_notifier; 303 #endif 304 int cpu; 305 int vcpu_id; /* id given by userspace at creation */ 306 int vcpu_idx; /* index in kvm->vcpus array */ 307 int srcu_idx; 308 int mode; 309 u64 requests; 310 unsigned long guest_debug; 311 312 struct mutex mutex; 313 struct kvm_run *run; 314 315 #ifndef __KVM_HAVE_ARCH_WQP 316 struct rcuwait wait; 317 #endif 318 struct pid __rcu *pid; 319 int sigset_active; 320 sigset_t sigset; 321 unsigned int halt_poll_ns; 322 bool valid_wakeup; 323 324 #ifdef CONFIG_HAS_IOMEM 325 int mmio_needed; 326 int mmio_read_completed; 327 int mmio_is_write; 328 int mmio_cur_fragment; 329 int mmio_nr_fragments; 330 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS]; 331 #endif 332 333 #ifdef CONFIG_KVM_ASYNC_PF 334 struct { 335 u32 queued; 336 struct list_head queue; 337 struct list_head done; 338 spinlock_t lock; 339 } async_pf; 340 #endif 341 342 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 343 /* 344 * Cpu relax intercept or pause loop exit optimization 345 * in_spin_loop: set when a vcpu does a pause loop exit 346 * or cpu relax intercepted. 347 * dy_eligible: indicates whether vcpu is eligible for directed yield. 348 */ 349 struct { 350 bool in_spin_loop; 351 bool dy_eligible; 352 } spin_loop; 353 #endif 354 bool preempted; 355 bool ready; 356 struct kvm_vcpu_arch arch; 357 struct kvm_vcpu_stat stat; 358 char stats_id[KVM_STATS_NAME_SIZE]; 359 struct kvm_dirty_ring dirty_ring; 360 361 /* 362 * The most recently used memslot by this vCPU and the slots generation 363 * for which it is valid. 364 * No wraparound protection is needed since generations won't overflow in 365 * thousands of years, even assuming 1M memslot operations per second. 366 */ 367 struct kvm_memory_slot *last_used_slot; 368 u64 last_used_slot_gen; 369 }; 370 371 /* must be called with irqs disabled */ 372 static __always_inline void guest_enter_irqoff(void) 373 { 374 /* 375 * This is running in ioctl context so its safe to assume that it's the 376 * stime pending cputime to flush. 377 */ 378 instrumentation_begin(); 379 vtime_account_guest_enter(); 380 instrumentation_end(); 381 382 /* 383 * KVM does not hold any references to rcu protected data when it 384 * switches CPU into a guest mode. In fact switching to a guest mode 385 * is very similar to exiting to userspace from rcu point of view. In 386 * addition CPU may stay in a guest mode for quite a long time (up to 387 * one time slice). Lets treat guest mode as quiescent state, just like 388 * we do with user-mode execution. 389 */ 390 if (!context_tracking_guest_enter()) { 391 instrumentation_begin(); 392 rcu_virt_note_context_switch(smp_processor_id()); 393 instrumentation_end(); 394 } 395 } 396 397 static __always_inline void guest_exit_irqoff(void) 398 { 399 context_tracking_guest_exit(); 400 401 instrumentation_begin(); 402 /* Flush the guest cputime we spent on the guest */ 403 vtime_account_guest_exit(); 404 instrumentation_end(); 405 } 406 407 static inline void guest_exit(void) 408 { 409 unsigned long flags; 410 411 local_irq_save(flags); 412 guest_exit_irqoff(); 413 local_irq_restore(flags); 414 } 415 416 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) 417 { 418 /* 419 * The memory barrier ensures a previous write to vcpu->requests cannot 420 * be reordered with the read of vcpu->mode. It pairs with the general 421 * memory barrier following the write of vcpu->mode in VCPU RUN. 422 */ 423 smp_mb__before_atomic(); 424 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); 425 } 426 427 /* 428 * Some of the bitops functions do not support too long bitmaps. 429 * This number must be determined not to exceed such limits. 430 */ 431 #define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1) 432 433 /* 434 * Since at idle each memslot belongs to two memslot sets it has to contain 435 * two embedded nodes for each data structure that it forms a part of. 436 * 437 * Two memslot sets (one active and one inactive) are necessary so the VM 438 * continues to run on one memslot set while the other is being modified. 439 * 440 * These two memslot sets normally point to the same set of memslots. 441 * They can, however, be desynchronized when performing a memslot management 442 * operation by replacing the memslot to be modified by its copy. 443 * After the operation is complete, both memslot sets once again point to 444 * the same, common set of memslot data. 445 * 446 * The memslots themselves are independent of each other so they can be 447 * individually added or deleted. 448 */ 449 struct kvm_memory_slot { 450 struct hlist_node id_node[2]; 451 struct interval_tree_node hva_node[2]; 452 struct rb_node gfn_node[2]; 453 gfn_t base_gfn; 454 unsigned long npages; 455 unsigned long *dirty_bitmap; 456 struct kvm_arch_memory_slot arch; 457 unsigned long userspace_addr; 458 u32 flags; 459 short id; 460 u16 as_id; 461 }; 462 463 static inline bool kvm_slot_dirty_track_enabled(const struct kvm_memory_slot *slot) 464 { 465 return slot->flags & KVM_MEM_LOG_DIRTY_PAGES; 466 } 467 468 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot) 469 { 470 return ALIGN(memslot->npages, BITS_PER_LONG) / 8; 471 } 472 473 static inline unsigned long *kvm_second_dirty_bitmap(struct kvm_memory_slot *memslot) 474 { 475 unsigned long len = kvm_dirty_bitmap_bytes(memslot); 476 477 return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap); 478 } 479 480 #ifndef KVM_DIRTY_LOG_MANUAL_CAPS 481 #define KVM_DIRTY_LOG_MANUAL_CAPS KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE 482 #endif 483 484 struct kvm_s390_adapter_int { 485 u64 ind_addr; 486 u64 summary_addr; 487 u64 ind_offset; 488 u32 summary_offset; 489 u32 adapter_id; 490 }; 491 492 struct kvm_hv_sint { 493 u32 vcpu; 494 u32 sint; 495 }; 496 497 struct kvm_xen_evtchn { 498 u32 port; 499 u32 vcpu; 500 u32 priority; 501 }; 502 503 struct kvm_kernel_irq_routing_entry { 504 u32 gsi; 505 u32 type; 506 int (*set)(struct kvm_kernel_irq_routing_entry *e, 507 struct kvm *kvm, int irq_source_id, int level, 508 bool line_status); 509 union { 510 struct { 511 unsigned irqchip; 512 unsigned pin; 513 } irqchip; 514 struct { 515 u32 address_lo; 516 u32 address_hi; 517 u32 data; 518 u32 flags; 519 u32 devid; 520 } msi; 521 struct kvm_s390_adapter_int adapter; 522 struct kvm_hv_sint hv_sint; 523 struct kvm_xen_evtchn xen_evtchn; 524 }; 525 struct hlist_node link; 526 }; 527 528 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 529 struct kvm_irq_routing_table { 530 int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS]; 531 u32 nr_rt_entries; 532 /* 533 * Array indexed by gsi. Each entry contains list of irq chips 534 * the gsi is connected to. 535 */ 536 struct hlist_head map[]; 537 }; 538 #endif 539 540 #ifndef KVM_PRIVATE_MEM_SLOTS 541 #define KVM_PRIVATE_MEM_SLOTS 0 542 #endif 543 544 #define KVM_MEM_SLOTS_NUM SHRT_MAX 545 #define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_PRIVATE_MEM_SLOTS) 546 547 #ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE 548 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) 549 { 550 return 0; 551 } 552 #endif 553 554 struct kvm_memslots { 555 u64 generation; 556 atomic_long_t last_used_slot; 557 struct rb_root_cached hva_tree; 558 struct rb_root gfn_tree; 559 /* 560 * The mapping table from slot id to memslot. 561 * 562 * 7-bit bucket count matches the size of the old id to index array for 563 * 512 slots, while giving good performance with this slot count. 564 * Higher bucket counts bring only small performance improvements but 565 * always result in higher memory usage (even for lower memslot counts). 566 */ 567 DECLARE_HASHTABLE(id_hash, 7); 568 int node_idx; 569 }; 570 571 struct kvm { 572 #ifdef KVM_HAVE_MMU_RWLOCK 573 rwlock_t mmu_lock; 574 #else 575 spinlock_t mmu_lock; 576 #endif /* KVM_HAVE_MMU_RWLOCK */ 577 578 struct mutex slots_lock; 579 580 /* 581 * Protects the arch-specific fields of struct kvm_memory_slots in 582 * use by the VM. To be used under the slots_lock (above) or in a 583 * kvm->srcu critical section where acquiring the slots_lock would 584 * lead to deadlock with the synchronize_srcu in 585 * install_new_memslots. 586 */ 587 struct mutex slots_arch_lock; 588 struct mm_struct *mm; /* userspace tied to this vm */ 589 unsigned long nr_memslot_pages; 590 /* The two memslot sets - active and inactive (per address space) */ 591 struct kvm_memslots __memslots[KVM_ADDRESS_SPACE_NUM][2]; 592 /* The current active memslot set for each address space */ 593 struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM]; 594 struct xarray vcpu_array; 595 596 /* Used to wait for completion of MMU notifiers. */ 597 spinlock_t mn_invalidate_lock; 598 unsigned long mn_active_invalidate_count; 599 struct rcuwait mn_memslots_update_rcuwait; 600 601 /* For management / invalidation of gfn_to_pfn_caches */ 602 spinlock_t gpc_lock; 603 struct list_head gpc_list; 604 605 /* 606 * created_vcpus is protected by kvm->lock, and is incremented 607 * at the beginning of KVM_CREATE_VCPU. online_vcpus is only 608 * incremented after storing the kvm_vcpu pointer in vcpus, 609 * and is accessed atomically. 610 */ 611 atomic_t online_vcpus; 612 int created_vcpus; 613 int last_boosted_vcpu; 614 struct list_head vm_list; 615 struct mutex lock; 616 struct kvm_io_bus __rcu *buses[KVM_NR_BUSES]; 617 #ifdef CONFIG_HAVE_KVM_EVENTFD 618 struct { 619 spinlock_t lock; 620 struct list_head items; 621 struct list_head resampler_list; 622 struct mutex resampler_lock; 623 } irqfds; 624 struct list_head ioeventfds; 625 #endif 626 struct kvm_vm_stat stat; 627 struct kvm_arch arch; 628 refcount_t users_count; 629 #ifdef CONFIG_KVM_MMIO 630 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 631 spinlock_t ring_lock; 632 struct list_head coalesced_zones; 633 #endif 634 635 struct mutex irq_lock; 636 #ifdef CONFIG_HAVE_KVM_IRQCHIP 637 /* 638 * Update side is protected by irq_lock. 639 */ 640 struct kvm_irq_routing_table __rcu *irq_routing; 641 #endif 642 #ifdef CONFIG_HAVE_KVM_IRQFD 643 struct hlist_head irq_ack_notifier_list; 644 #endif 645 646 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 647 struct mmu_notifier mmu_notifier; 648 unsigned long mmu_notifier_seq; 649 long mmu_notifier_count; 650 unsigned long mmu_notifier_range_start; 651 unsigned long mmu_notifier_range_end; 652 #endif 653 struct list_head devices; 654 u64 manual_dirty_log_protect; 655 struct dentry *debugfs_dentry; 656 struct kvm_stat_data **debugfs_stat_data; 657 struct srcu_struct srcu; 658 struct srcu_struct irq_srcu; 659 pid_t userspace_pid; 660 unsigned int max_halt_poll_ns; 661 u32 dirty_ring_size; 662 bool vm_bugged; 663 bool vm_dead; 664 665 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 666 struct notifier_block pm_notifier; 667 #endif 668 char stats_id[KVM_STATS_NAME_SIZE]; 669 }; 670 671 #define kvm_err(fmt, ...) \ 672 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 673 #define kvm_info(fmt, ...) \ 674 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 675 #define kvm_debug(fmt, ...) \ 676 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 677 #define kvm_debug_ratelimited(fmt, ...) \ 678 pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \ 679 ## __VA_ARGS__) 680 #define kvm_pr_unimpl(fmt, ...) \ 681 pr_err_ratelimited("kvm [%i]: " fmt, \ 682 task_tgid_nr(current), ## __VA_ARGS__) 683 684 /* The guest did something we don't support. */ 685 #define vcpu_unimpl(vcpu, fmt, ...) \ 686 kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt, \ 687 (vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__) 688 689 #define vcpu_debug(vcpu, fmt, ...) \ 690 kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 691 #define vcpu_debug_ratelimited(vcpu, fmt, ...) \ 692 kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id, \ 693 ## __VA_ARGS__) 694 #define vcpu_err(vcpu, fmt, ...) \ 695 kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 696 697 static inline void kvm_vm_dead(struct kvm *kvm) 698 { 699 kvm->vm_dead = true; 700 kvm_make_all_cpus_request(kvm, KVM_REQ_VM_DEAD); 701 } 702 703 static inline void kvm_vm_bugged(struct kvm *kvm) 704 { 705 kvm->vm_bugged = true; 706 kvm_vm_dead(kvm); 707 } 708 709 710 #define KVM_BUG(cond, kvm, fmt...) \ 711 ({ \ 712 int __ret = (cond); \ 713 \ 714 if (WARN_ONCE(__ret && !(kvm)->vm_bugged, fmt)) \ 715 kvm_vm_bugged(kvm); \ 716 unlikely(__ret); \ 717 }) 718 719 #define KVM_BUG_ON(cond, kvm) \ 720 ({ \ 721 int __ret = (cond); \ 722 \ 723 if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged)) \ 724 kvm_vm_bugged(kvm); \ 725 unlikely(__ret); \ 726 }) 727 728 static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) 729 { 730 return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET); 731 } 732 733 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) 734 { 735 return srcu_dereference_check(kvm->buses[idx], &kvm->srcu, 736 lockdep_is_held(&kvm->slots_lock) || 737 !refcount_read(&kvm->users_count)); 738 } 739 740 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) 741 { 742 int num_vcpus = atomic_read(&kvm->online_vcpus); 743 i = array_index_nospec(i, num_vcpus); 744 745 /* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu. */ 746 smp_rmb(); 747 return xa_load(&kvm->vcpu_array, i); 748 } 749 750 #define kvm_for_each_vcpu(idx, vcpup, kvm) \ 751 xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0, \ 752 (atomic_read(&kvm->online_vcpus) - 1)) 753 754 static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) 755 { 756 struct kvm_vcpu *vcpu = NULL; 757 unsigned long i; 758 759 if (id < 0) 760 return NULL; 761 if (id < KVM_MAX_VCPUS) 762 vcpu = kvm_get_vcpu(kvm, id); 763 if (vcpu && vcpu->vcpu_id == id) 764 return vcpu; 765 kvm_for_each_vcpu(i, vcpu, kvm) 766 if (vcpu->vcpu_id == id) 767 return vcpu; 768 return NULL; 769 } 770 771 static inline int kvm_vcpu_get_idx(struct kvm_vcpu *vcpu) 772 { 773 return vcpu->vcpu_idx; 774 } 775 776 void kvm_destroy_vcpus(struct kvm *kvm); 777 778 void vcpu_load(struct kvm_vcpu *vcpu); 779 void vcpu_put(struct kvm_vcpu *vcpu); 780 781 #ifdef __KVM_HAVE_IOAPIC 782 void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm); 783 void kvm_arch_post_irq_routing_update(struct kvm *kvm); 784 #else 785 static inline void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm) 786 { 787 } 788 static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm) 789 { 790 } 791 #endif 792 793 #ifdef CONFIG_HAVE_KVM_IRQFD 794 int kvm_irqfd_init(void); 795 void kvm_irqfd_exit(void); 796 #else 797 static inline int kvm_irqfd_init(void) 798 { 799 return 0; 800 } 801 802 static inline void kvm_irqfd_exit(void) 803 { 804 } 805 #endif 806 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 807 struct module *module); 808 void kvm_exit(void); 809 810 void kvm_get_kvm(struct kvm *kvm); 811 bool kvm_get_kvm_safe(struct kvm *kvm); 812 void kvm_put_kvm(struct kvm *kvm); 813 bool file_is_kvm(struct file *file); 814 void kvm_put_kvm_no_destroy(struct kvm *kvm); 815 816 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) 817 { 818 as_id = array_index_nospec(as_id, KVM_ADDRESS_SPACE_NUM); 819 return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu, 820 lockdep_is_held(&kvm->slots_lock) || 821 !refcount_read(&kvm->users_count)); 822 } 823 824 static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm) 825 { 826 return __kvm_memslots(kvm, 0); 827 } 828 829 static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu) 830 { 831 int as_id = kvm_arch_vcpu_memslots_id(vcpu); 832 833 return __kvm_memslots(vcpu->kvm, as_id); 834 } 835 836 static inline bool kvm_memslots_empty(struct kvm_memslots *slots) 837 { 838 return RB_EMPTY_ROOT(&slots->gfn_tree); 839 } 840 841 #define kvm_for_each_memslot(memslot, bkt, slots) \ 842 hash_for_each(slots->id_hash, bkt, memslot, id_node[slots->node_idx]) \ 843 if (WARN_ON_ONCE(!memslot->npages)) { \ 844 } else 845 846 static inline 847 struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id) 848 { 849 struct kvm_memory_slot *slot; 850 int idx = slots->node_idx; 851 852 hash_for_each_possible(slots->id_hash, slot, id_node[idx], id) { 853 if (slot->id == id) 854 return slot; 855 } 856 857 return NULL; 858 } 859 860 /* Iterator used for walking memslots that overlap a gfn range. */ 861 struct kvm_memslot_iter { 862 struct kvm_memslots *slots; 863 struct rb_node *node; 864 struct kvm_memory_slot *slot; 865 }; 866 867 static inline void kvm_memslot_iter_next(struct kvm_memslot_iter *iter) 868 { 869 iter->node = rb_next(iter->node); 870 if (!iter->node) 871 return; 872 873 iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[iter->slots->node_idx]); 874 } 875 876 static inline void kvm_memslot_iter_start(struct kvm_memslot_iter *iter, 877 struct kvm_memslots *slots, 878 gfn_t start) 879 { 880 int idx = slots->node_idx; 881 struct rb_node *tmp; 882 struct kvm_memory_slot *slot; 883 884 iter->slots = slots; 885 886 /* 887 * Find the so called "upper bound" of a key - the first node that has 888 * its key strictly greater than the searched one (the start gfn in our case). 889 */ 890 iter->node = NULL; 891 for (tmp = slots->gfn_tree.rb_node; tmp; ) { 892 slot = container_of(tmp, struct kvm_memory_slot, gfn_node[idx]); 893 if (start < slot->base_gfn) { 894 iter->node = tmp; 895 tmp = tmp->rb_left; 896 } else { 897 tmp = tmp->rb_right; 898 } 899 } 900 901 /* 902 * Find the slot with the lowest gfn that can possibly intersect with 903 * the range, so we'll ideally have slot start <= range start 904 */ 905 if (iter->node) { 906 /* 907 * A NULL previous node means that the very first slot 908 * already has a higher start gfn. 909 * In this case slot start > range start. 910 */ 911 tmp = rb_prev(iter->node); 912 if (tmp) 913 iter->node = tmp; 914 } else { 915 /* a NULL node below means no slots */ 916 iter->node = rb_last(&slots->gfn_tree); 917 } 918 919 if (iter->node) { 920 iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[idx]); 921 922 /* 923 * It is possible in the slot start < range start case that the 924 * found slot ends before or at range start (slot end <= range start) 925 * and so it does not overlap the requested range. 926 * 927 * In such non-overlapping case the next slot (if it exists) will 928 * already have slot start > range start, otherwise the logic above 929 * would have found it instead of the current slot. 930 */ 931 if (iter->slot->base_gfn + iter->slot->npages <= start) 932 kvm_memslot_iter_next(iter); 933 } 934 } 935 936 static inline bool kvm_memslot_iter_is_valid(struct kvm_memslot_iter *iter, gfn_t end) 937 { 938 if (!iter->node) 939 return false; 940 941 /* 942 * If this slot starts beyond or at the end of the range so does 943 * every next one 944 */ 945 return iter->slot->base_gfn < end; 946 } 947 948 /* Iterate over each memslot at least partially intersecting [start, end) range */ 949 #define kvm_for_each_memslot_in_gfn_range(iter, slots, start, end) \ 950 for (kvm_memslot_iter_start(iter, slots, start); \ 951 kvm_memslot_iter_is_valid(iter, end); \ 952 kvm_memslot_iter_next(iter)) 953 954 /* 955 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations: 956 * - create a new memory slot 957 * - delete an existing memory slot 958 * - modify an existing memory slot 959 * -- move it in the guest physical memory space 960 * -- just change its flags 961 * 962 * Since flags can be changed by some of these operations, the following 963 * differentiation is the best we can do for __kvm_set_memory_region(): 964 */ 965 enum kvm_mr_change { 966 KVM_MR_CREATE, 967 KVM_MR_DELETE, 968 KVM_MR_MOVE, 969 KVM_MR_FLAGS_ONLY, 970 }; 971 972 int kvm_set_memory_region(struct kvm *kvm, 973 const struct kvm_userspace_memory_region *mem); 974 int __kvm_set_memory_region(struct kvm *kvm, 975 const struct kvm_userspace_memory_region *mem); 976 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot); 977 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen); 978 int kvm_arch_prepare_memory_region(struct kvm *kvm, 979 const struct kvm_memory_slot *old, 980 struct kvm_memory_slot *new, 981 enum kvm_mr_change change); 982 void kvm_arch_commit_memory_region(struct kvm *kvm, 983 struct kvm_memory_slot *old, 984 const struct kvm_memory_slot *new, 985 enum kvm_mr_change change); 986 /* flush all memory translations */ 987 void kvm_arch_flush_shadow_all(struct kvm *kvm); 988 /* flush memory translations pointing to 'slot' */ 989 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 990 struct kvm_memory_slot *slot); 991 992 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 993 struct page **pages, int nr_pages); 994 995 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); 996 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn); 997 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable); 998 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 999 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn, 1000 bool *writable); 1001 void kvm_release_page_clean(struct page *page); 1002 void kvm_release_page_dirty(struct page *page); 1003 void kvm_set_page_accessed(struct page *page); 1004 1005 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); 1006 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1007 bool *writable); 1008 kvm_pfn_t gfn_to_pfn_memslot(const struct kvm_memory_slot *slot, gfn_t gfn); 1009 kvm_pfn_t gfn_to_pfn_memslot_atomic(const struct kvm_memory_slot *slot, gfn_t gfn); 1010 kvm_pfn_t __gfn_to_pfn_memslot(const struct kvm_memory_slot *slot, gfn_t gfn, 1011 bool atomic, bool *async, bool write_fault, 1012 bool *writable, hva_t *hva); 1013 1014 void kvm_release_pfn_clean(kvm_pfn_t pfn); 1015 void kvm_release_pfn_dirty(kvm_pfn_t pfn); 1016 void kvm_set_pfn_dirty(kvm_pfn_t pfn); 1017 void kvm_set_pfn_accessed(kvm_pfn_t pfn); 1018 1019 void kvm_release_pfn(kvm_pfn_t pfn, bool dirty); 1020 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1021 int len); 1022 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); 1023 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1024 void *data, unsigned long len); 1025 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1026 void *data, unsigned int offset, 1027 unsigned long len); 1028 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1029 int offset, int len); 1030 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1031 unsigned long len); 1032 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1033 void *data, unsigned long len); 1034 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1035 void *data, unsigned int offset, 1036 unsigned long len); 1037 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1038 gpa_t gpa, unsigned long len); 1039 1040 #define __kvm_get_guest(kvm, gfn, offset, v) \ 1041 ({ \ 1042 unsigned long __addr = gfn_to_hva(kvm, gfn); \ 1043 typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset); \ 1044 int __ret = -EFAULT; \ 1045 \ 1046 if (!kvm_is_error_hva(__addr)) \ 1047 __ret = get_user(v, __uaddr); \ 1048 __ret; \ 1049 }) 1050 1051 #define kvm_get_guest(kvm, gpa, v) \ 1052 ({ \ 1053 gpa_t __gpa = gpa; \ 1054 struct kvm *__kvm = kvm; \ 1055 \ 1056 __kvm_get_guest(__kvm, __gpa >> PAGE_SHIFT, \ 1057 offset_in_page(__gpa), v); \ 1058 }) 1059 1060 #define __kvm_put_guest(kvm, gfn, offset, v) \ 1061 ({ \ 1062 unsigned long __addr = gfn_to_hva(kvm, gfn); \ 1063 typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset); \ 1064 int __ret = -EFAULT; \ 1065 \ 1066 if (!kvm_is_error_hva(__addr)) \ 1067 __ret = put_user(v, __uaddr); \ 1068 if (!__ret) \ 1069 mark_page_dirty(kvm, gfn); \ 1070 __ret; \ 1071 }) 1072 1073 #define kvm_put_guest(kvm, gpa, v) \ 1074 ({ \ 1075 gpa_t __gpa = gpa; \ 1076 struct kvm *__kvm = kvm; \ 1077 \ 1078 __kvm_put_guest(__kvm, __gpa >> PAGE_SHIFT, \ 1079 offset_in_page(__gpa), v); \ 1080 }) 1081 1082 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); 1083 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); 1084 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); 1085 bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); 1086 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn); 1087 void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn); 1088 void mark_page_dirty(struct kvm *kvm, gfn_t gfn); 1089 1090 struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu); 1091 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn); 1092 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn); 1093 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); 1094 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map); 1095 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn); 1096 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty); 1097 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); 1098 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); 1099 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, 1100 int len); 1101 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 1102 unsigned long len); 1103 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 1104 unsigned long len); 1105 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data, 1106 int offset, int len); 1107 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1108 unsigned long len); 1109 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); 1110 1111 /** 1112 * kvm_gfn_to_pfn_cache_init - prepare a cached kernel mapping and HPA for a 1113 * given guest physical address. 1114 * 1115 * @kvm: pointer to kvm instance. 1116 * @gpc: struct gfn_to_pfn_cache object. 1117 * @vcpu: vCPU to be used for marking pages dirty and to be woken on 1118 * invalidation. 1119 * @guest_uses_pa: indicates that the resulting host physical PFN is used while 1120 * @vcpu is IN_GUEST_MODE so invalidations should wake it. 1121 * @kernel_map: requests a kernel virtual mapping (kmap / memremap). 1122 * @gpa: guest physical address to map. 1123 * @len: sanity check; the range being access must fit a single page. 1124 * @dirty: mark the cache dirty immediately. 1125 * 1126 * @return: 0 for success. 1127 * -EINVAL for a mapping which would cross a page boundary. 1128 * -EFAULT for an untranslatable guest physical address. 1129 * 1130 * This primes a gfn_to_pfn_cache and links it into the @kvm's list for 1131 * invalidations to be processed. Invalidation callbacks to @vcpu using 1132 * %KVM_REQ_GPC_INVALIDATE will occur only for MMU notifiers, not for KVM 1133 * memslot changes. Callers are required to use kvm_gfn_to_pfn_cache_check() 1134 * to ensure that the cache is valid before accessing the target page. 1135 */ 1136 int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, 1137 struct kvm_vcpu *vcpu, bool guest_uses_pa, 1138 bool kernel_map, gpa_t gpa, unsigned long len, 1139 bool dirty); 1140 1141 /** 1142 * kvm_gfn_to_pfn_cache_check - check validity of a gfn_to_pfn_cache. 1143 * 1144 * @kvm: pointer to kvm instance. 1145 * @gpc: struct gfn_to_pfn_cache object. 1146 * @gpa: current guest physical address to map. 1147 * @len: sanity check; the range being access must fit a single page. 1148 * @dirty: mark the cache dirty immediately. 1149 * 1150 * @return: %true if the cache is still valid and the address matches. 1151 * %false if the cache is not valid. 1152 * 1153 * Callers outside IN_GUEST_MODE context should hold a read lock on @gpc->lock 1154 * while calling this function, and then continue to hold the lock until the 1155 * access is complete. 1156 * 1157 * Callers in IN_GUEST_MODE may do so without locking, although they should 1158 * still hold a read lock on kvm->scru for the memslot checks. 1159 */ 1160 bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, 1161 gpa_t gpa, unsigned long len); 1162 1163 /** 1164 * kvm_gfn_to_pfn_cache_refresh - update a previously initialized cache. 1165 * 1166 * @kvm: pointer to kvm instance. 1167 * @gpc: struct gfn_to_pfn_cache object. 1168 * @gpa: updated guest physical address to map. 1169 * @len: sanity check; the range being access must fit a single page. 1170 * @dirty: mark the cache dirty immediately. 1171 * 1172 * @return: 0 for success. 1173 * -EINVAL for a mapping which would cross a page boundary. 1174 * -EFAULT for an untranslatable guest physical address. 1175 * 1176 * This will attempt to refresh a gfn_to_pfn_cache. Note that a successful 1177 * returm from this function does not mean the page can be immediately 1178 * accessed because it may have raced with an invalidation. Callers must 1179 * still lock and check the cache status, as this function does not return 1180 * with the lock still held to permit access. 1181 */ 1182 int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, 1183 gpa_t gpa, unsigned long len, bool dirty); 1184 1185 /** 1186 * kvm_gfn_to_pfn_cache_unmap - temporarily unmap a gfn_to_pfn_cache. 1187 * 1188 * @kvm: pointer to kvm instance. 1189 * @gpc: struct gfn_to_pfn_cache object. 1190 * 1191 * This unmaps the referenced page and marks it dirty, if appropriate. The 1192 * cache is left in the invalid state but at least the mapping from GPA to 1193 * userspace HVA will remain cached and can be reused on a subsequent 1194 * refresh. 1195 */ 1196 void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); 1197 1198 /** 1199 * kvm_gfn_to_pfn_cache_destroy - destroy and unlink a gfn_to_pfn_cache. 1200 * 1201 * @kvm: pointer to kvm instance. 1202 * @gpc: struct gfn_to_pfn_cache object. 1203 * 1204 * This removes a cache from the @kvm's list to be processed on MMU notifier 1205 * invocation. 1206 */ 1207 void kvm_gfn_to_pfn_cache_destroy(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); 1208 1209 void kvm_sigset_activate(struct kvm_vcpu *vcpu); 1210 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu); 1211 1212 void kvm_vcpu_halt(struct kvm_vcpu *vcpu); 1213 bool kvm_vcpu_block(struct kvm_vcpu *vcpu); 1214 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu); 1215 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu); 1216 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu); 1217 void kvm_vcpu_kick(struct kvm_vcpu *vcpu); 1218 int kvm_vcpu_yield_to(struct kvm_vcpu *target); 1219 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu, bool usermode_vcpu_not_eligible); 1220 1221 void kvm_flush_remote_tlbs(struct kvm *kvm); 1222 void kvm_reload_remote_mmus(struct kvm *kvm); 1223 1224 #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 1225 int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min); 1226 int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc); 1227 void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc); 1228 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); 1229 #endif 1230 1231 void kvm_inc_notifier_count(struct kvm *kvm, unsigned long start, 1232 unsigned long end); 1233 void kvm_dec_notifier_count(struct kvm *kvm, unsigned long start, 1234 unsigned long end); 1235 1236 long kvm_arch_dev_ioctl(struct file *filp, 1237 unsigned int ioctl, unsigned long arg); 1238 long kvm_arch_vcpu_ioctl(struct file *filp, 1239 unsigned int ioctl, unsigned long arg); 1240 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf); 1241 1242 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext); 1243 1244 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 1245 struct kvm_memory_slot *slot, 1246 gfn_t gfn_offset, 1247 unsigned long mask); 1248 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot); 1249 1250 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1251 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 1252 const struct kvm_memory_slot *memslot); 1253 #else /* !CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ 1254 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); 1255 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, 1256 int *is_dirty, struct kvm_memory_slot **memslot); 1257 #endif 1258 1259 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 1260 bool line_status); 1261 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 1262 struct kvm_enable_cap *cap); 1263 long kvm_arch_vm_ioctl(struct file *filp, 1264 unsigned int ioctl, unsigned long arg); 1265 1266 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 1267 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 1268 1269 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 1270 struct kvm_translation *tr); 1271 1272 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 1273 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 1274 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 1275 struct kvm_sregs *sregs); 1276 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 1277 struct kvm_sregs *sregs); 1278 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 1279 struct kvm_mp_state *mp_state); 1280 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 1281 struct kvm_mp_state *mp_state); 1282 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 1283 struct kvm_guest_debug *dbg); 1284 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu); 1285 1286 int kvm_arch_init(void *opaque); 1287 void kvm_arch_exit(void); 1288 1289 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu); 1290 1291 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); 1292 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); 1293 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id); 1294 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu); 1295 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu); 1296 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); 1297 1298 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 1299 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state); 1300 #endif 1301 1302 #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS 1303 void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); 1304 #endif 1305 1306 int kvm_arch_hardware_enable(void); 1307 void kvm_arch_hardware_disable(void); 1308 int kvm_arch_hardware_setup(void *opaque); 1309 void kvm_arch_hardware_unsetup(void); 1310 int kvm_arch_check_processor_compat(void *opaque); 1311 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); 1312 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu); 1313 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); 1314 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu); 1315 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu); 1316 int kvm_arch_post_init_vm(struct kvm *kvm); 1317 void kvm_arch_pre_destroy_vm(struct kvm *kvm); 1318 int kvm_arch_create_vm_debugfs(struct kvm *kvm); 1319 1320 #ifndef __KVM_HAVE_ARCH_VM_ALLOC 1321 /* 1322 * All architectures that want to use vzalloc currently also 1323 * need their own kvm_arch_alloc_vm implementation. 1324 */ 1325 static inline struct kvm *kvm_arch_alloc_vm(void) 1326 { 1327 return kzalloc(sizeof(struct kvm), GFP_KERNEL); 1328 } 1329 #endif 1330 1331 static inline void __kvm_arch_free_vm(struct kvm *kvm) 1332 { 1333 kvfree(kvm); 1334 } 1335 1336 #ifndef __KVM_HAVE_ARCH_VM_FREE 1337 static inline void kvm_arch_free_vm(struct kvm *kvm) 1338 { 1339 __kvm_arch_free_vm(kvm); 1340 } 1341 #endif 1342 1343 #ifndef __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB 1344 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) 1345 { 1346 return -ENOTSUPP; 1347 } 1348 #endif 1349 1350 #ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA 1351 void kvm_arch_register_noncoherent_dma(struct kvm *kvm); 1352 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm); 1353 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm); 1354 #else 1355 static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 1356 { 1357 } 1358 1359 static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 1360 { 1361 } 1362 1363 static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 1364 { 1365 return false; 1366 } 1367 #endif 1368 #ifdef __KVM_HAVE_ARCH_ASSIGNED_DEVICE 1369 void kvm_arch_start_assignment(struct kvm *kvm); 1370 void kvm_arch_end_assignment(struct kvm *kvm); 1371 bool kvm_arch_has_assigned_device(struct kvm *kvm); 1372 #else 1373 static inline void kvm_arch_start_assignment(struct kvm *kvm) 1374 { 1375 } 1376 1377 static inline void kvm_arch_end_assignment(struct kvm *kvm) 1378 { 1379 } 1380 1381 static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) 1382 { 1383 return false; 1384 } 1385 #endif 1386 1387 static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu) 1388 { 1389 #ifdef __KVM_HAVE_ARCH_WQP 1390 return vcpu->arch.waitp; 1391 #else 1392 return &vcpu->wait; 1393 #endif 1394 } 1395 1396 /* 1397 * Wake a vCPU if necessary, but don't do any stats/metadata updates. Returns 1398 * true if the vCPU was blocking and was awakened, false otherwise. 1399 */ 1400 static inline bool __kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 1401 { 1402 return !!rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu)); 1403 } 1404 1405 static inline bool kvm_vcpu_is_blocking(struct kvm_vcpu *vcpu) 1406 { 1407 return rcuwait_active(kvm_arch_vcpu_get_wait(vcpu)); 1408 } 1409 1410 #ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED 1411 /* 1412 * returns true if the virtual interrupt controller is initialized and 1413 * ready to accept virtual IRQ. On some architectures the virtual interrupt 1414 * controller is dynamically instantiated and this is not always true. 1415 */ 1416 bool kvm_arch_intc_initialized(struct kvm *kvm); 1417 #else 1418 static inline bool kvm_arch_intc_initialized(struct kvm *kvm) 1419 { 1420 return true; 1421 } 1422 #endif 1423 1424 #ifdef CONFIG_GUEST_PERF_EVENTS 1425 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu); 1426 1427 void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void)); 1428 void kvm_unregister_perf_callbacks(void); 1429 #else 1430 static inline void kvm_register_perf_callbacks(void *ign) {} 1431 static inline void kvm_unregister_perf_callbacks(void) {} 1432 #endif /* CONFIG_GUEST_PERF_EVENTS */ 1433 1434 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); 1435 void kvm_arch_destroy_vm(struct kvm *kvm); 1436 void kvm_arch_sync_events(struct kvm *kvm); 1437 1438 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); 1439 1440 bool kvm_is_reserved_pfn(kvm_pfn_t pfn); 1441 bool kvm_is_zone_device_pfn(kvm_pfn_t pfn); 1442 1443 struct kvm_irq_ack_notifier { 1444 struct hlist_node link; 1445 unsigned gsi; 1446 void (*irq_acked)(struct kvm_irq_ack_notifier *kian); 1447 }; 1448 1449 int kvm_irq_map_gsi(struct kvm *kvm, 1450 struct kvm_kernel_irq_routing_entry *entries, int gsi); 1451 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin); 1452 1453 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 1454 bool line_status); 1455 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm, 1456 int irq_source_id, int level, bool line_status); 1457 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 1458 struct kvm *kvm, int irq_source_id, 1459 int level, bool line_status); 1460 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin); 1461 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi); 1462 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin); 1463 void kvm_register_irq_ack_notifier(struct kvm *kvm, 1464 struct kvm_irq_ack_notifier *kian); 1465 void kvm_unregister_irq_ack_notifier(struct kvm *kvm, 1466 struct kvm_irq_ack_notifier *kian); 1467 int kvm_request_irq_source_id(struct kvm *kvm); 1468 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id); 1469 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args); 1470 1471 /* 1472 * Returns a pointer to the memslot if it contains gfn. 1473 * Otherwise returns NULL. 1474 */ 1475 static inline struct kvm_memory_slot * 1476 try_get_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1477 { 1478 if (!slot) 1479 return NULL; 1480 1481 if (gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages) 1482 return slot; 1483 else 1484 return NULL; 1485 } 1486 1487 /* 1488 * Returns a pointer to the memslot that contains gfn. Otherwise returns NULL. 1489 * 1490 * With "approx" set returns the memslot also when the address falls 1491 * in a hole. In that case one of the memslots bordering the hole is 1492 * returned. 1493 */ 1494 static inline struct kvm_memory_slot * 1495 search_memslots(struct kvm_memslots *slots, gfn_t gfn, bool approx) 1496 { 1497 struct kvm_memory_slot *slot; 1498 struct rb_node *node; 1499 int idx = slots->node_idx; 1500 1501 slot = NULL; 1502 for (node = slots->gfn_tree.rb_node; node; ) { 1503 slot = container_of(node, struct kvm_memory_slot, gfn_node[idx]); 1504 if (gfn >= slot->base_gfn) { 1505 if (gfn < slot->base_gfn + slot->npages) 1506 return slot; 1507 node = node->rb_right; 1508 } else 1509 node = node->rb_left; 1510 } 1511 1512 return approx ? slot : NULL; 1513 } 1514 1515 static inline struct kvm_memory_slot * 1516 ____gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn, bool approx) 1517 { 1518 struct kvm_memory_slot *slot; 1519 1520 slot = (struct kvm_memory_slot *)atomic_long_read(&slots->last_used_slot); 1521 slot = try_get_memslot(slot, gfn); 1522 if (slot) 1523 return slot; 1524 1525 slot = search_memslots(slots, gfn, approx); 1526 if (slot) { 1527 atomic_long_set(&slots->last_used_slot, (unsigned long)slot); 1528 return slot; 1529 } 1530 1531 return NULL; 1532 } 1533 1534 /* 1535 * __gfn_to_memslot() and its descendants are here to allow arch code to inline 1536 * the lookups in hot paths. gfn_to_memslot() itself isn't here as an inline 1537 * because that would bloat other code too much. 1538 */ 1539 static inline struct kvm_memory_slot * 1540 __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn) 1541 { 1542 return ____gfn_to_memslot(slots, gfn, false); 1543 } 1544 1545 static inline unsigned long 1546 __gfn_to_hva_memslot(const struct kvm_memory_slot *slot, gfn_t gfn) 1547 { 1548 /* 1549 * The index was checked originally in search_memslots. To avoid 1550 * that a malicious guest builds a Spectre gadget out of e.g. page 1551 * table walks, do not let the processor speculate loads outside 1552 * the guest's registered memslots. 1553 */ 1554 unsigned long offset = gfn - slot->base_gfn; 1555 offset = array_index_nospec(offset, slot->npages); 1556 return slot->userspace_addr + offset * PAGE_SIZE; 1557 } 1558 1559 static inline int memslot_id(struct kvm *kvm, gfn_t gfn) 1560 { 1561 return gfn_to_memslot(kvm, gfn)->id; 1562 } 1563 1564 static inline gfn_t 1565 hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot) 1566 { 1567 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT; 1568 1569 return slot->base_gfn + gfn_offset; 1570 } 1571 1572 static inline gpa_t gfn_to_gpa(gfn_t gfn) 1573 { 1574 return (gpa_t)gfn << PAGE_SHIFT; 1575 } 1576 1577 static inline gfn_t gpa_to_gfn(gpa_t gpa) 1578 { 1579 return (gfn_t)(gpa >> PAGE_SHIFT); 1580 } 1581 1582 static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn) 1583 { 1584 return (hpa_t)pfn << PAGE_SHIFT; 1585 } 1586 1587 static inline struct page *kvm_vcpu_gpa_to_page(struct kvm_vcpu *vcpu, 1588 gpa_t gpa) 1589 { 1590 return kvm_vcpu_gfn_to_page(vcpu, gpa_to_gfn(gpa)); 1591 } 1592 1593 static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) 1594 { 1595 unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); 1596 1597 return kvm_is_error_hva(hva); 1598 } 1599 1600 enum kvm_stat_kind { 1601 KVM_STAT_VM, 1602 KVM_STAT_VCPU, 1603 }; 1604 1605 struct kvm_stat_data { 1606 struct kvm *kvm; 1607 const struct _kvm_stats_desc *desc; 1608 enum kvm_stat_kind kind; 1609 }; 1610 1611 struct _kvm_stats_desc { 1612 struct kvm_stats_desc desc; 1613 char name[KVM_STATS_NAME_SIZE]; 1614 }; 1615 1616 #define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz) \ 1617 .flags = type | unit | base | \ 1618 BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ 1619 BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ 1620 BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ 1621 .exponent = exp, \ 1622 .size = sz, \ 1623 .bucket_size = bsz 1624 1625 #define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1626 { \ 1627 { \ 1628 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1629 .offset = offsetof(struct kvm_vm_stat, generic.stat) \ 1630 }, \ 1631 .name = #stat, \ 1632 } 1633 #define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1634 { \ 1635 { \ 1636 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1637 .offset = offsetof(struct kvm_vcpu_stat, generic.stat) \ 1638 }, \ 1639 .name = #stat, \ 1640 } 1641 #define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1642 { \ 1643 { \ 1644 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1645 .offset = offsetof(struct kvm_vm_stat, stat) \ 1646 }, \ 1647 .name = #stat, \ 1648 } 1649 #define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1650 { \ 1651 { \ 1652 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1653 .offset = offsetof(struct kvm_vcpu_stat, stat) \ 1654 }, \ 1655 .name = #stat, \ 1656 } 1657 /* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */ 1658 #define STATS_DESC(SCOPE, stat, type, unit, base, exp, sz, bsz) \ 1659 SCOPE##_STATS_DESC(stat, type, unit, base, exp, sz, bsz) 1660 1661 #define STATS_DESC_CUMULATIVE(SCOPE, name, unit, base, exponent) \ 1662 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_CUMULATIVE, \ 1663 unit, base, exponent, 1, 0) 1664 #define STATS_DESC_INSTANT(SCOPE, name, unit, base, exponent) \ 1665 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_INSTANT, \ 1666 unit, base, exponent, 1, 0) 1667 #define STATS_DESC_PEAK(SCOPE, name, unit, base, exponent) \ 1668 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_PEAK, \ 1669 unit, base, exponent, 1, 0) 1670 #define STATS_DESC_LINEAR_HIST(SCOPE, name, unit, base, exponent, sz, bsz) \ 1671 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_LINEAR_HIST, \ 1672 unit, base, exponent, sz, bsz) 1673 #define STATS_DESC_LOG_HIST(SCOPE, name, unit, base, exponent, sz) \ 1674 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_LOG_HIST, \ 1675 unit, base, exponent, sz, 0) 1676 1677 /* Cumulative counter, read/write */ 1678 #define STATS_DESC_COUNTER(SCOPE, name) \ 1679 STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_NONE, \ 1680 KVM_STATS_BASE_POW10, 0) 1681 /* Instantaneous counter, read only */ 1682 #define STATS_DESC_ICOUNTER(SCOPE, name) \ 1683 STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_NONE, \ 1684 KVM_STATS_BASE_POW10, 0) 1685 /* Peak counter, read/write */ 1686 #define STATS_DESC_PCOUNTER(SCOPE, name) \ 1687 STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_NONE, \ 1688 KVM_STATS_BASE_POW10, 0) 1689 1690 /* Cumulative time in nanosecond */ 1691 #define STATS_DESC_TIME_NSEC(SCOPE, name) \ 1692 STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ 1693 KVM_STATS_BASE_POW10, -9) 1694 /* Linear histogram for time in nanosecond */ 1695 #define STATS_DESC_LINHIST_TIME_NSEC(SCOPE, name, sz, bsz) \ 1696 STATS_DESC_LINEAR_HIST(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ 1697 KVM_STATS_BASE_POW10, -9, sz, bsz) 1698 /* Logarithmic histogram for time in nanosecond */ 1699 #define STATS_DESC_LOGHIST_TIME_NSEC(SCOPE, name, sz) \ 1700 STATS_DESC_LOG_HIST(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ 1701 KVM_STATS_BASE_POW10, -9, sz) 1702 1703 #define KVM_GENERIC_VM_STATS() \ 1704 STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush), \ 1705 STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush_requests) 1706 1707 #define KVM_GENERIC_VCPU_STATS() \ 1708 STATS_DESC_COUNTER(VCPU_GENERIC, halt_successful_poll), \ 1709 STATS_DESC_COUNTER(VCPU_GENERIC, halt_attempted_poll), \ 1710 STATS_DESC_COUNTER(VCPU_GENERIC, halt_poll_invalid), \ 1711 STATS_DESC_COUNTER(VCPU_GENERIC, halt_wakeup), \ 1712 STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_success_ns), \ 1713 STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_ns), \ 1714 STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_wait_ns), \ 1715 STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_poll_success_hist, \ 1716 HALT_POLL_HIST_COUNT), \ 1717 STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_hist, \ 1718 HALT_POLL_HIST_COUNT), \ 1719 STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_wait_hist, \ 1720 HALT_POLL_HIST_COUNT), \ 1721 STATS_DESC_ICOUNTER(VCPU_GENERIC, blocking) 1722 1723 extern struct dentry *kvm_debugfs_dir; 1724 1725 ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, 1726 const struct _kvm_stats_desc *desc, 1727 void *stats, size_t size_stats, 1728 char __user *user_buffer, size_t size, loff_t *offset); 1729 1730 /** 1731 * kvm_stats_linear_hist_update() - Update bucket value for linear histogram 1732 * statistics data. 1733 * 1734 * @data: start address of the stats data 1735 * @size: the number of bucket of the stats data 1736 * @value: the new value used to update the linear histogram's bucket 1737 * @bucket_size: the size (width) of a bucket 1738 */ 1739 static inline void kvm_stats_linear_hist_update(u64 *data, size_t size, 1740 u64 value, size_t bucket_size) 1741 { 1742 size_t index = div64_u64(value, bucket_size); 1743 1744 index = min(index, size - 1); 1745 ++data[index]; 1746 } 1747 1748 /** 1749 * kvm_stats_log_hist_update() - Update bucket value for logarithmic histogram 1750 * statistics data. 1751 * 1752 * @data: start address of the stats data 1753 * @size: the number of bucket of the stats data 1754 * @value: the new value used to update the logarithmic histogram's bucket 1755 */ 1756 static inline void kvm_stats_log_hist_update(u64 *data, size_t size, u64 value) 1757 { 1758 size_t index = fls64(value); 1759 1760 index = min(index, size - 1); 1761 ++data[index]; 1762 } 1763 1764 #define KVM_STATS_LINEAR_HIST_UPDATE(array, value, bsize) \ 1765 kvm_stats_linear_hist_update(array, ARRAY_SIZE(array), value, bsize) 1766 #define KVM_STATS_LOG_HIST_UPDATE(array, value) \ 1767 kvm_stats_log_hist_update(array, ARRAY_SIZE(array), value) 1768 1769 1770 extern const struct kvm_stats_header kvm_vm_stats_header; 1771 extern const struct _kvm_stats_desc kvm_vm_stats_desc[]; 1772 extern const struct kvm_stats_header kvm_vcpu_stats_header; 1773 extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[]; 1774 1775 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 1776 static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq) 1777 { 1778 if (unlikely(kvm->mmu_notifier_count)) 1779 return 1; 1780 /* 1781 * Ensure the read of mmu_notifier_count happens before the read 1782 * of mmu_notifier_seq. This interacts with the smp_wmb() in 1783 * mmu_notifier_invalidate_range_end to make sure that the caller 1784 * either sees the old (non-zero) value of mmu_notifier_count or 1785 * the new (incremented) value of mmu_notifier_seq. 1786 * PowerPC Book3s HV KVM calls this under a per-page lock 1787 * rather than under kvm->mmu_lock, for scalability, so 1788 * can't rely on kvm->mmu_lock to keep things ordered. 1789 */ 1790 smp_rmb(); 1791 if (kvm->mmu_notifier_seq != mmu_seq) 1792 return 1; 1793 return 0; 1794 } 1795 1796 static inline int mmu_notifier_retry_hva(struct kvm *kvm, 1797 unsigned long mmu_seq, 1798 unsigned long hva) 1799 { 1800 lockdep_assert_held(&kvm->mmu_lock); 1801 /* 1802 * If mmu_notifier_count is non-zero, then the range maintained by 1803 * kvm_mmu_notifier_invalidate_range_start contains all addresses that 1804 * might be being invalidated. Note that it may include some false 1805 * positives, due to shortcuts when handing concurrent invalidations. 1806 */ 1807 if (unlikely(kvm->mmu_notifier_count) && 1808 hva >= kvm->mmu_notifier_range_start && 1809 hva < kvm->mmu_notifier_range_end) 1810 return 1; 1811 if (kvm->mmu_notifier_seq != mmu_seq) 1812 return 1; 1813 return 0; 1814 } 1815 #endif 1816 1817 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 1818 1819 #define KVM_MAX_IRQ_ROUTES 4096 /* might need extension/rework in the future */ 1820 1821 bool kvm_arch_can_set_irq_routing(struct kvm *kvm); 1822 int kvm_set_irq_routing(struct kvm *kvm, 1823 const struct kvm_irq_routing_entry *entries, 1824 unsigned nr, 1825 unsigned flags); 1826 int kvm_set_routing_entry(struct kvm *kvm, 1827 struct kvm_kernel_irq_routing_entry *e, 1828 const struct kvm_irq_routing_entry *ue); 1829 void kvm_free_irq_routing(struct kvm *kvm); 1830 1831 #else 1832 1833 static inline void kvm_free_irq_routing(struct kvm *kvm) {} 1834 1835 #endif 1836 1837 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); 1838 1839 #ifdef CONFIG_HAVE_KVM_EVENTFD 1840 1841 void kvm_eventfd_init(struct kvm *kvm); 1842 int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); 1843 1844 #ifdef CONFIG_HAVE_KVM_IRQFD 1845 int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args); 1846 void kvm_irqfd_release(struct kvm *kvm); 1847 void kvm_irq_routing_update(struct kvm *); 1848 #else 1849 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 1850 { 1851 return -EINVAL; 1852 } 1853 1854 static inline void kvm_irqfd_release(struct kvm *kvm) {} 1855 #endif 1856 1857 #else 1858 1859 static inline void kvm_eventfd_init(struct kvm *kvm) {} 1860 1861 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 1862 { 1863 return -EINVAL; 1864 } 1865 1866 static inline void kvm_irqfd_release(struct kvm *kvm) {} 1867 1868 #ifdef CONFIG_HAVE_KVM_IRQCHIP 1869 static inline void kvm_irq_routing_update(struct kvm *kvm) 1870 { 1871 } 1872 #endif 1873 1874 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 1875 { 1876 return -ENOSYS; 1877 } 1878 1879 #endif /* CONFIG_HAVE_KVM_EVENTFD */ 1880 1881 void kvm_arch_irq_routing_update(struct kvm *kvm); 1882 1883 static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) 1884 { 1885 /* 1886 * Ensure the rest of the request is published to kvm_check_request's 1887 * caller. Paired with the smp_mb__after_atomic in kvm_check_request. 1888 */ 1889 smp_wmb(); 1890 set_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 1891 } 1892 1893 static inline bool kvm_request_pending(struct kvm_vcpu *vcpu) 1894 { 1895 return READ_ONCE(vcpu->requests); 1896 } 1897 1898 static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu) 1899 { 1900 return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 1901 } 1902 1903 static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) 1904 { 1905 clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 1906 } 1907 1908 static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) 1909 { 1910 if (kvm_test_request(req, vcpu)) { 1911 kvm_clear_request(req, vcpu); 1912 1913 /* 1914 * Ensure the rest of the request is visible to kvm_check_request's 1915 * caller. Paired with the smp_wmb in kvm_make_request. 1916 */ 1917 smp_mb__after_atomic(); 1918 return true; 1919 } else { 1920 return false; 1921 } 1922 } 1923 1924 extern bool kvm_rebooting; 1925 1926 extern unsigned int halt_poll_ns; 1927 extern unsigned int halt_poll_ns_grow; 1928 extern unsigned int halt_poll_ns_grow_start; 1929 extern unsigned int halt_poll_ns_shrink; 1930 1931 struct kvm_device { 1932 const struct kvm_device_ops *ops; 1933 struct kvm *kvm; 1934 void *private; 1935 struct list_head vm_node; 1936 }; 1937 1938 /* create, destroy, and name are mandatory */ 1939 struct kvm_device_ops { 1940 const char *name; 1941 1942 /* 1943 * create is called holding kvm->lock and any operations not suitable 1944 * to do while holding the lock should be deferred to init (see 1945 * below). 1946 */ 1947 int (*create)(struct kvm_device *dev, u32 type); 1948 1949 /* 1950 * init is called after create if create is successful and is called 1951 * outside of holding kvm->lock. 1952 */ 1953 void (*init)(struct kvm_device *dev); 1954 1955 /* 1956 * Destroy is responsible for freeing dev. 1957 * 1958 * Destroy may be called before or after destructors are called 1959 * on emulated I/O regions, depending on whether a reference is 1960 * held by a vcpu or other kvm component that gets destroyed 1961 * after the emulated I/O. 1962 */ 1963 void (*destroy)(struct kvm_device *dev); 1964 1965 /* 1966 * Release is an alternative method to free the device. It is 1967 * called when the device file descriptor is closed. Once 1968 * release is called, the destroy method will not be called 1969 * anymore as the device is removed from the device list of 1970 * the VM. kvm->lock is held. 1971 */ 1972 void (*release)(struct kvm_device *dev); 1973 1974 int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1975 int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1976 int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1977 long (*ioctl)(struct kvm_device *dev, unsigned int ioctl, 1978 unsigned long arg); 1979 int (*mmap)(struct kvm_device *dev, struct vm_area_struct *vma); 1980 }; 1981 1982 void kvm_device_get(struct kvm_device *dev); 1983 void kvm_device_put(struct kvm_device *dev); 1984 struct kvm_device *kvm_device_from_filp(struct file *filp); 1985 int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type); 1986 void kvm_unregister_device_ops(u32 type); 1987 1988 extern struct kvm_device_ops kvm_mpic_ops; 1989 extern struct kvm_device_ops kvm_arm_vgic_v2_ops; 1990 extern struct kvm_device_ops kvm_arm_vgic_v3_ops; 1991 1992 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 1993 1994 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 1995 { 1996 vcpu->spin_loop.in_spin_loop = val; 1997 } 1998 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 1999 { 2000 vcpu->spin_loop.dy_eligible = val; 2001 } 2002 2003 #else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 2004 2005 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 2006 { 2007 } 2008 2009 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 2010 { 2011 } 2012 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 2013 2014 static inline bool kvm_is_visible_memslot(struct kvm_memory_slot *memslot) 2015 { 2016 return (memslot && memslot->id < KVM_USER_MEM_SLOTS && 2017 !(memslot->flags & KVM_MEMSLOT_INVALID)); 2018 } 2019 2020 struct kvm_vcpu *kvm_get_running_vcpu(void); 2021 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); 2022 2023 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 2024 bool kvm_arch_has_irq_bypass(void); 2025 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *, 2026 struct irq_bypass_producer *); 2027 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *, 2028 struct irq_bypass_producer *); 2029 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *); 2030 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *); 2031 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, 2032 uint32_t guest_irq, bool set); 2033 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *, 2034 struct kvm_kernel_irq_routing_entry *); 2035 #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ 2036 2037 #ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS 2038 /* If we wakeup during the poll time, was it a sucessful poll? */ 2039 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 2040 { 2041 return vcpu->valid_wakeup; 2042 } 2043 2044 #else 2045 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 2046 { 2047 return true; 2048 } 2049 #endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ 2050 2051 #ifdef CONFIG_HAVE_KVM_NO_POLL 2052 /* Callback that tells if we must not poll */ 2053 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); 2054 #else 2055 static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 2056 { 2057 return false; 2058 } 2059 #endif /* CONFIG_HAVE_KVM_NO_POLL */ 2060 2061 #ifdef CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL 2062 long kvm_arch_vcpu_async_ioctl(struct file *filp, 2063 unsigned int ioctl, unsigned long arg); 2064 #else 2065 static inline long kvm_arch_vcpu_async_ioctl(struct file *filp, 2066 unsigned int ioctl, 2067 unsigned long arg) 2068 { 2069 return -ENOIOCTLCMD; 2070 } 2071 #endif /* CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL */ 2072 2073 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, 2074 unsigned long start, unsigned long end); 2075 2076 #ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE 2077 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); 2078 #else 2079 static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 2080 { 2081 return 0; 2082 } 2083 #endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ 2084 2085 typedef int (*kvm_vm_thread_fn_t)(struct kvm *kvm, uintptr_t data); 2086 2087 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, 2088 uintptr_t data, const char *name, 2089 struct task_struct **thread_ptr); 2090 2091 #ifdef CONFIG_KVM_XFER_TO_GUEST_WORK 2092 static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) 2093 { 2094 vcpu->run->exit_reason = KVM_EXIT_INTR; 2095 vcpu->stat.signal_exits++; 2096 } 2097 #endif /* CONFIG_KVM_XFER_TO_GUEST_WORK */ 2098 2099 /* 2100 * This defines how many reserved entries we want to keep before we 2101 * kick the vcpu to the userspace to avoid dirty ring full. This 2102 * value can be tuned to higher if e.g. PML is enabled on the host. 2103 */ 2104 #define KVM_DIRTY_RING_RSVD_ENTRIES 64 2105 2106 /* Max number of entries allowed for each kvm dirty ring */ 2107 #define KVM_DIRTY_RING_MAX_ENTRIES 65536 2108 2109 #endif 2110