1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef __KVM_HOST_H 3 #define __KVM_HOST_H 4 5 6 #include <linux/types.h> 7 #include <linux/hardirq.h> 8 #include <linux/list.h> 9 #include <linux/mutex.h> 10 #include <linux/spinlock.h> 11 #include <linux/signal.h> 12 #include <linux/sched.h> 13 #include <linux/bug.h> 14 #include <linux/mm.h> 15 #include <linux/mmu_notifier.h> 16 #include <linux/preempt.h> 17 #include <linux/msi.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/rcupdate.h> 21 #include <linux/ratelimit.h> 22 #include <linux/err.h> 23 #include <linux/irqflags.h> 24 #include <linux/context_tracking.h> 25 #include <linux/irqbypass.h> 26 #include <linux/rcuwait.h> 27 #include <linux/refcount.h> 28 #include <linux/nospec.h> 29 #include <asm/signal.h> 30 31 #include <linux/kvm.h> 32 #include <linux/kvm_para.h> 33 34 #include <linux/kvm_types.h> 35 36 #include <asm/kvm_host.h> 37 #include <linux/kvm_dirty_ring.h> 38 39 #ifndef KVM_MAX_VCPU_ID 40 #define KVM_MAX_VCPU_ID KVM_MAX_VCPUS 41 #endif 42 43 /* 44 * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used 45 * in kvm, other bits are visible for userspace which are defined in 46 * include/linux/kvm_h. 47 */ 48 #define KVM_MEMSLOT_INVALID (1UL << 16) 49 50 /* 51 * Bit 63 of the memslot generation number is an "update in-progress flag", 52 * e.g. is temporarily set for the duration of install_new_memslots(). 53 * This flag effectively creates a unique generation number that is used to 54 * mark cached memslot data, e.g. MMIO accesses, as potentially being stale, 55 * i.e. may (or may not) have come from the previous memslots generation. 56 * 57 * This is necessary because the actual memslots update is not atomic with 58 * respect to the generation number update. Updating the generation number 59 * first would allow a vCPU to cache a spte from the old memslots using the 60 * new generation number, and updating the generation number after switching 61 * to the new memslots would allow cache hits using the old generation number 62 * to reference the defunct memslots. 63 * 64 * This mechanism is used to prevent getting hits in KVM's caches while a 65 * memslot update is in-progress, and to prevent cache hits *after* updating 66 * the actual generation number against accesses that were inserted into the 67 * cache *before* the memslots were updated. 68 */ 69 #define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS BIT_ULL(63) 70 71 /* Two fragments for cross MMIO pages. */ 72 #define KVM_MAX_MMIO_FRAGMENTS 2 73 74 #ifndef KVM_ADDRESS_SPACE_NUM 75 #define KVM_ADDRESS_SPACE_NUM 1 76 #endif 77 78 /* 79 * For the normal pfn, the highest 12 bits should be zero, 80 * so we can mask bit 62 ~ bit 52 to indicate the error pfn, 81 * mask bit 63 to indicate the noslot pfn. 82 */ 83 #define KVM_PFN_ERR_MASK (0x7ffULL << 52) 84 #define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52) 85 #define KVM_PFN_NOSLOT (0x1ULL << 63) 86 87 #define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK) 88 #define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1) 89 #define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2) 90 91 /* 92 * error pfns indicate that the gfn is in slot but faild to 93 * translate it to pfn on host. 94 */ 95 static inline bool is_error_pfn(kvm_pfn_t pfn) 96 { 97 return !!(pfn & KVM_PFN_ERR_MASK); 98 } 99 100 /* 101 * error_noslot pfns indicate that the gfn can not be 102 * translated to pfn - it is not in slot or failed to 103 * translate it to pfn. 104 */ 105 static inline bool is_error_noslot_pfn(kvm_pfn_t pfn) 106 { 107 return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK); 108 } 109 110 /* noslot pfn indicates that the gfn is not in slot. */ 111 static inline bool is_noslot_pfn(kvm_pfn_t pfn) 112 { 113 return pfn == KVM_PFN_NOSLOT; 114 } 115 116 /* 117 * architectures with KVM_HVA_ERR_BAD other than PAGE_OFFSET (e.g. s390) 118 * provide own defines and kvm_is_error_hva 119 */ 120 #ifndef KVM_HVA_ERR_BAD 121 122 #define KVM_HVA_ERR_BAD (PAGE_OFFSET) 123 #define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE) 124 125 static inline bool kvm_is_error_hva(unsigned long addr) 126 { 127 return addr >= PAGE_OFFSET; 128 } 129 130 #endif 131 132 #define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT)) 133 134 static inline bool is_error_page(struct page *page) 135 { 136 return IS_ERR(page); 137 } 138 139 #define KVM_REQUEST_MASK GENMASK(7,0) 140 #define KVM_REQUEST_NO_WAKEUP BIT(8) 141 #define KVM_REQUEST_WAIT BIT(9) 142 /* 143 * Architecture-independent vcpu->requests bit members 144 * Bits 4-7 are reserved for more arch-independent bits. 145 */ 146 #define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 147 #define KVM_REQ_MMU_RELOAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 148 #define KVM_REQ_PENDING_TIMER 2 149 #define KVM_REQ_UNHALT 3 150 #define KVM_REQUEST_ARCH_BASE 8 151 152 #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \ 153 BUILD_BUG_ON((unsigned)(nr) >= (sizeof_field(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \ 154 (unsigned)(((nr) + KVM_REQUEST_ARCH_BASE) | (flags)); \ 155 }) 156 #define KVM_ARCH_REQ(nr) KVM_ARCH_REQ_FLAGS(nr, 0) 157 158 #define KVM_USERSPACE_IRQ_SOURCE_ID 0 159 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1 160 161 extern struct mutex kvm_lock; 162 extern struct list_head vm_list; 163 164 struct kvm_io_range { 165 gpa_t addr; 166 int len; 167 struct kvm_io_device *dev; 168 }; 169 170 #define NR_IOBUS_DEVS 1000 171 172 struct kvm_io_bus { 173 int dev_count; 174 int ioeventfd_count; 175 struct kvm_io_range range[]; 176 }; 177 178 enum kvm_bus { 179 KVM_MMIO_BUS, 180 KVM_PIO_BUS, 181 KVM_VIRTIO_CCW_NOTIFY_BUS, 182 KVM_FAST_MMIO_BUS, 183 KVM_NR_BUSES 184 }; 185 186 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 187 int len, const void *val); 188 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 189 gpa_t addr, int len, const void *val, long cookie); 190 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 191 int len, void *val); 192 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 193 int len, struct kvm_io_device *dev); 194 void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 195 struct kvm_io_device *dev); 196 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 197 gpa_t addr); 198 199 #ifdef CONFIG_KVM_ASYNC_PF 200 struct kvm_async_pf { 201 struct work_struct work; 202 struct list_head link; 203 struct list_head queue; 204 struct kvm_vcpu *vcpu; 205 struct mm_struct *mm; 206 gpa_t cr2_or_gpa; 207 unsigned long addr; 208 struct kvm_arch_async_pf arch; 209 bool wakeup_all; 210 bool notpresent_injected; 211 }; 212 213 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu); 214 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu); 215 bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 216 unsigned long hva, struct kvm_arch_async_pf *arch); 217 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); 218 #endif 219 220 enum { 221 OUTSIDE_GUEST_MODE, 222 IN_GUEST_MODE, 223 EXITING_GUEST_MODE, 224 READING_SHADOW_PAGE_TABLES, 225 }; 226 227 #define KVM_UNMAPPED_PAGE ((void *) 0x500 + POISON_POINTER_DELTA) 228 229 struct kvm_host_map { 230 /* 231 * Only valid if the 'pfn' is managed by the host kernel (i.e. There is 232 * a 'struct page' for it. When using mem= kernel parameter some memory 233 * can be used as guest memory but they are not managed by host 234 * kernel). 235 * If 'pfn' is not managed by the host kernel, this field is 236 * initialized to KVM_UNMAPPED_PAGE. 237 */ 238 struct page *page; 239 void *hva; 240 kvm_pfn_t pfn; 241 kvm_pfn_t gfn; 242 }; 243 244 /* 245 * Used to check if the mapping is valid or not. Never use 'kvm_host_map' 246 * directly to check for that. 247 */ 248 static inline bool kvm_vcpu_mapped(struct kvm_host_map *map) 249 { 250 return !!map->hva; 251 } 252 253 /* 254 * Sometimes a large or cross-page mmio needs to be broken up into separate 255 * exits for userspace servicing. 256 */ 257 struct kvm_mmio_fragment { 258 gpa_t gpa; 259 void *data; 260 unsigned len; 261 }; 262 263 struct kvm_vcpu { 264 struct kvm *kvm; 265 #ifdef CONFIG_PREEMPT_NOTIFIERS 266 struct preempt_notifier preempt_notifier; 267 #endif 268 int cpu; 269 int vcpu_id; /* id given by userspace at creation */ 270 int vcpu_idx; /* index in kvm->vcpus array */ 271 int srcu_idx; 272 int mode; 273 u64 requests; 274 unsigned long guest_debug; 275 276 int pre_pcpu; 277 struct list_head blocked_vcpu_list; 278 279 struct mutex mutex; 280 struct kvm_run *run; 281 282 struct rcuwait wait; 283 struct pid __rcu *pid; 284 int sigset_active; 285 sigset_t sigset; 286 struct kvm_vcpu_stat stat; 287 unsigned int halt_poll_ns; 288 bool valid_wakeup; 289 290 #ifdef CONFIG_HAS_IOMEM 291 int mmio_needed; 292 int mmio_read_completed; 293 int mmio_is_write; 294 int mmio_cur_fragment; 295 int mmio_nr_fragments; 296 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS]; 297 #endif 298 299 #ifdef CONFIG_KVM_ASYNC_PF 300 struct { 301 u32 queued; 302 struct list_head queue; 303 struct list_head done; 304 spinlock_t lock; 305 } async_pf; 306 #endif 307 308 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 309 /* 310 * Cpu relax intercept or pause loop exit optimization 311 * in_spin_loop: set when a vcpu does a pause loop exit 312 * or cpu relax intercepted. 313 * dy_eligible: indicates whether vcpu is eligible for directed yield. 314 */ 315 struct { 316 bool in_spin_loop; 317 bool dy_eligible; 318 } spin_loop; 319 #endif 320 bool preempted; 321 bool ready; 322 struct kvm_vcpu_arch arch; 323 struct kvm_dirty_ring dirty_ring; 324 }; 325 326 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) 327 { 328 /* 329 * The memory barrier ensures a previous write to vcpu->requests cannot 330 * be reordered with the read of vcpu->mode. It pairs with the general 331 * memory barrier following the write of vcpu->mode in VCPU RUN. 332 */ 333 smp_mb__before_atomic(); 334 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); 335 } 336 337 /* 338 * Some of the bitops functions do not support too long bitmaps. 339 * This number must be determined not to exceed such limits. 340 */ 341 #define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1) 342 343 struct kvm_memory_slot { 344 gfn_t base_gfn; 345 unsigned long npages; 346 unsigned long *dirty_bitmap; 347 struct kvm_arch_memory_slot arch; 348 unsigned long userspace_addr; 349 u32 flags; 350 short id; 351 u16 as_id; 352 }; 353 354 static inline bool kvm_slot_dirty_track_enabled(struct kvm_memory_slot *slot) 355 { 356 return slot->flags & KVM_MEM_LOG_DIRTY_PAGES; 357 } 358 359 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot) 360 { 361 return ALIGN(memslot->npages, BITS_PER_LONG) / 8; 362 } 363 364 static inline unsigned long *kvm_second_dirty_bitmap(struct kvm_memory_slot *memslot) 365 { 366 unsigned long len = kvm_dirty_bitmap_bytes(memslot); 367 368 return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap); 369 } 370 371 #ifndef KVM_DIRTY_LOG_MANUAL_CAPS 372 #define KVM_DIRTY_LOG_MANUAL_CAPS KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE 373 #endif 374 375 struct kvm_s390_adapter_int { 376 u64 ind_addr; 377 u64 summary_addr; 378 u64 ind_offset; 379 u32 summary_offset; 380 u32 adapter_id; 381 }; 382 383 struct kvm_hv_sint { 384 u32 vcpu; 385 u32 sint; 386 }; 387 388 struct kvm_kernel_irq_routing_entry { 389 u32 gsi; 390 u32 type; 391 int (*set)(struct kvm_kernel_irq_routing_entry *e, 392 struct kvm *kvm, int irq_source_id, int level, 393 bool line_status); 394 union { 395 struct { 396 unsigned irqchip; 397 unsigned pin; 398 } irqchip; 399 struct { 400 u32 address_lo; 401 u32 address_hi; 402 u32 data; 403 u32 flags; 404 u32 devid; 405 } msi; 406 struct kvm_s390_adapter_int adapter; 407 struct kvm_hv_sint hv_sint; 408 }; 409 struct hlist_node link; 410 }; 411 412 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 413 struct kvm_irq_routing_table { 414 int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS]; 415 u32 nr_rt_entries; 416 /* 417 * Array indexed by gsi. Each entry contains list of irq chips 418 * the gsi is connected to. 419 */ 420 struct hlist_head map[]; 421 }; 422 #endif 423 424 #ifndef KVM_PRIVATE_MEM_SLOTS 425 #define KVM_PRIVATE_MEM_SLOTS 0 426 #endif 427 428 #ifndef KVM_MEM_SLOTS_NUM 429 #define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS) 430 #endif 431 432 #ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE 433 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) 434 { 435 return 0; 436 } 437 #endif 438 439 /* 440 * Note: 441 * memslots are not sorted by id anymore, please use id_to_memslot() 442 * to get the memslot by its id. 443 */ 444 struct kvm_memslots { 445 u64 generation; 446 /* The mapping table from slot id to the index in memslots[]. */ 447 short id_to_index[KVM_MEM_SLOTS_NUM]; 448 atomic_t lru_slot; 449 int used_slots; 450 struct kvm_memory_slot memslots[]; 451 }; 452 453 struct kvm { 454 spinlock_t mmu_lock; 455 struct mutex slots_lock; 456 struct mm_struct *mm; /* userspace tied to this vm */ 457 struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM]; 458 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; 459 460 /* 461 * created_vcpus is protected by kvm->lock, and is incremented 462 * at the beginning of KVM_CREATE_VCPU. online_vcpus is only 463 * incremented after storing the kvm_vcpu pointer in vcpus, 464 * and is accessed atomically. 465 */ 466 atomic_t online_vcpus; 467 int created_vcpus; 468 int last_boosted_vcpu; 469 struct list_head vm_list; 470 struct mutex lock; 471 struct kvm_io_bus __rcu *buses[KVM_NR_BUSES]; 472 #ifdef CONFIG_HAVE_KVM_EVENTFD 473 struct { 474 spinlock_t lock; 475 struct list_head items; 476 struct list_head resampler_list; 477 struct mutex resampler_lock; 478 } irqfds; 479 struct list_head ioeventfds; 480 #endif 481 struct kvm_vm_stat stat; 482 struct kvm_arch arch; 483 refcount_t users_count; 484 #ifdef CONFIG_KVM_MMIO 485 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 486 spinlock_t ring_lock; 487 struct list_head coalesced_zones; 488 #endif 489 490 struct mutex irq_lock; 491 #ifdef CONFIG_HAVE_KVM_IRQCHIP 492 /* 493 * Update side is protected by irq_lock. 494 */ 495 struct kvm_irq_routing_table __rcu *irq_routing; 496 #endif 497 #ifdef CONFIG_HAVE_KVM_IRQFD 498 struct hlist_head irq_ack_notifier_list; 499 #endif 500 501 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 502 struct mmu_notifier mmu_notifier; 503 unsigned long mmu_notifier_seq; 504 long mmu_notifier_count; 505 #endif 506 long tlbs_dirty; 507 struct list_head devices; 508 u64 manual_dirty_log_protect; 509 struct dentry *debugfs_dentry; 510 struct kvm_stat_data **debugfs_stat_data; 511 struct srcu_struct srcu; 512 struct srcu_struct irq_srcu; 513 pid_t userspace_pid; 514 unsigned int max_halt_poll_ns; 515 u32 dirty_ring_size; 516 }; 517 518 #define kvm_err(fmt, ...) \ 519 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 520 #define kvm_info(fmt, ...) \ 521 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 522 #define kvm_debug(fmt, ...) \ 523 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 524 #define kvm_debug_ratelimited(fmt, ...) \ 525 pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \ 526 ## __VA_ARGS__) 527 #define kvm_pr_unimpl(fmt, ...) \ 528 pr_err_ratelimited("kvm [%i]: " fmt, \ 529 task_tgid_nr(current), ## __VA_ARGS__) 530 531 /* The guest did something we don't support. */ 532 #define vcpu_unimpl(vcpu, fmt, ...) \ 533 kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt, \ 534 (vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__) 535 536 #define vcpu_debug(vcpu, fmt, ...) \ 537 kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 538 #define vcpu_debug_ratelimited(vcpu, fmt, ...) \ 539 kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id, \ 540 ## __VA_ARGS__) 541 #define vcpu_err(vcpu, fmt, ...) \ 542 kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 543 544 static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) 545 { 546 return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET); 547 } 548 549 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) 550 { 551 return srcu_dereference_check(kvm->buses[idx], &kvm->srcu, 552 lockdep_is_held(&kvm->slots_lock) || 553 !refcount_read(&kvm->users_count)); 554 } 555 556 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) 557 { 558 int num_vcpus = atomic_read(&kvm->online_vcpus); 559 i = array_index_nospec(i, num_vcpus); 560 561 /* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu. */ 562 smp_rmb(); 563 return kvm->vcpus[i]; 564 } 565 566 #define kvm_for_each_vcpu(idx, vcpup, kvm) \ 567 for (idx = 0; \ 568 idx < atomic_read(&kvm->online_vcpus) && \ 569 (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \ 570 idx++) 571 572 static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) 573 { 574 struct kvm_vcpu *vcpu = NULL; 575 int i; 576 577 if (id < 0) 578 return NULL; 579 if (id < KVM_MAX_VCPUS) 580 vcpu = kvm_get_vcpu(kvm, id); 581 if (vcpu && vcpu->vcpu_id == id) 582 return vcpu; 583 kvm_for_each_vcpu(i, vcpu, kvm) 584 if (vcpu->vcpu_id == id) 585 return vcpu; 586 return NULL; 587 } 588 589 static inline int kvm_vcpu_get_idx(struct kvm_vcpu *vcpu) 590 { 591 return vcpu->vcpu_idx; 592 } 593 594 #define kvm_for_each_memslot(memslot, slots) \ 595 for (memslot = &slots->memslots[0]; \ 596 memslot < slots->memslots + slots->used_slots; memslot++) \ 597 if (WARN_ON_ONCE(!memslot->npages)) { \ 598 } else 599 600 void kvm_vcpu_destroy(struct kvm_vcpu *vcpu); 601 602 void vcpu_load(struct kvm_vcpu *vcpu); 603 void vcpu_put(struct kvm_vcpu *vcpu); 604 605 #ifdef __KVM_HAVE_IOAPIC 606 void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm); 607 void kvm_arch_post_irq_routing_update(struct kvm *kvm); 608 #else 609 static inline void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm) 610 { 611 } 612 static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm) 613 { 614 } 615 #endif 616 617 #ifdef CONFIG_HAVE_KVM_IRQFD 618 int kvm_irqfd_init(void); 619 void kvm_irqfd_exit(void); 620 #else 621 static inline int kvm_irqfd_init(void) 622 { 623 return 0; 624 } 625 626 static inline void kvm_irqfd_exit(void) 627 { 628 } 629 #endif 630 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 631 struct module *module); 632 void kvm_exit(void); 633 634 void kvm_get_kvm(struct kvm *kvm); 635 void kvm_put_kvm(struct kvm *kvm); 636 void kvm_put_kvm_no_destroy(struct kvm *kvm); 637 638 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) 639 { 640 as_id = array_index_nospec(as_id, KVM_ADDRESS_SPACE_NUM); 641 return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu, 642 lockdep_is_held(&kvm->slots_lock) || 643 !refcount_read(&kvm->users_count)); 644 } 645 646 static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm) 647 { 648 return __kvm_memslots(kvm, 0); 649 } 650 651 static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu) 652 { 653 int as_id = kvm_arch_vcpu_memslots_id(vcpu); 654 655 return __kvm_memslots(vcpu->kvm, as_id); 656 } 657 658 static inline 659 struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id) 660 { 661 int index = slots->id_to_index[id]; 662 struct kvm_memory_slot *slot; 663 664 if (index < 0) 665 return NULL; 666 667 slot = &slots->memslots[index]; 668 669 WARN_ON(slot->id != id); 670 return slot; 671 } 672 673 /* 674 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations: 675 * - create a new memory slot 676 * - delete an existing memory slot 677 * - modify an existing memory slot 678 * -- move it in the guest physical memory space 679 * -- just change its flags 680 * 681 * Since flags can be changed by some of these operations, the following 682 * differentiation is the best we can do for __kvm_set_memory_region(): 683 */ 684 enum kvm_mr_change { 685 KVM_MR_CREATE, 686 KVM_MR_DELETE, 687 KVM_MR_MOVE, 688 KVM_MR_FLAGS_ONLY, 689 }; 690 691 int kvm_set_memory_region(struct kvm *kvm, 692 const struct kvm_userspace_memory_region *mem); 693 int __kvm_set_memory_region(struct kvm *kvm, 694 const struct kvm_userspace_memory_region *mem); 695 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot); 696 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen); 697 int kvm_arch_prepare_memory_region(struct kvm *kvm, 698 struct kvm_memory_slot *memslot, 699 const struct kvm_userspace_memory_region *mem, 700 enum kvm_mr_change change); 701 void kvm_arch_commit_memory_region(struct kvm *kvm, 702 const struct kvm_userspace_memory_region *mem, 703 struct kvm_memory_slot *old, 704 const struct kvm_memory_slot *new, 705 enum kvm_mr_change change); 706 /* flush all memory translations */ 707 void kvm_arch_flush_shadow_all(struct kvm *kvm); 708 /* flush memory translations pointing to 'slot' */ 709 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 710 struct kvm_memory_slot *slot); 711 712 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 713 struct page **pages, int nr_pages); 714 715 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); 716 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn); 717 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable); 718 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 719 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn, 720 bool *writable); 721 void kvm_release_page_clean(struct page *page); 722 void kvm_release_page_dirty(struct page *page); 723 void kvm_set_page_accessed(struct page *page); 724 725 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); 726 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 727 bool *writable); 728 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 729 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn); 730 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 731 bool atomic, bool *async, bool write_fault, 732 bool *writable); 733 734 void kvm_release_pfn_clean(kvm_pfn_t pfn); 735 void kvm_release_pfn_dirty(kvm_pfn_t pfn); 736 void kvm_set_pfn_dirty(kvm_pfn_t pfn); 737 void kvm_set_pfn_accessed(kvm_pfn_t pfn); 738 void kvm_get_pfn(kvm_pfn_t pfn); 739 740 void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache); 741 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 742 int len); 743 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); 744 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 745 void *data, unsigned long len); 746 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 747 void *data, unsigned int offset, 748 unsigned long len); 749 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 750 int offset, int len); 751 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 752 unsigned long len); 753 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 754 void *data, unsigned long len); 755 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 756 void *data, unsigned int offset, 757 unsigned long len); 758 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 759 gpa_t gpa, unsigned long len); 760 761 #define __kvm_get_guest(kvm, gfn, offset, v) \ 762 ({ \ 763 unsigned long __addr = gfn_to_hva(kvm, gfn); \ 764 typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset); \ 765 int __ret = -EFAULT; \ 766 \ 767 if (!kvm_is_error_hva(__addr)) \ 768 __ret = get_user(v, __uaddr); \ 769 __ret; \ 770 }) 771 772 #define kvm_get_guest(kvm, gpa, v) \ 773 ({ \ 774 gpa_t __gpa = gpa; \ 775 struct kvm *__kvm = kvm; \ 776 \ 777 __kvm_get_guest(__kvm, __gpa >> PAGE_SHIFT, \ 778 offset_in_page(__gpa), v); \ 779 }) 780 781 #define __kvm_put_guest(kvm, gfn, offset, v) \ 782 ({ \ 783 unsigned long __addr = gfn_to_hva(kvm, gfn); \ 784 typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset); \ 785 int __ret = -EFAULT; \ 786 \ 787 if (!kvm_is_error_hva(__addr)) \ 788 __ret = put_user(v, __uaddr); \ 789 if (!__ret) \ 790 mark_page_dirty(kvm, gfn); \ 791 __ret; \ 792 }) 793 794 #define kvm_put_guest(kvm, gpa, v) \ 795 ({ \ 796 gpa_t __gpa = gpa; \ 797 struct kvm *__kvm = kvm; \ 798 \ 799 __kvm_put_guest(__kvm, __gpa >> PAGE_SHIFT, \ 800 offset_in_page(__gpa), v); \ 801 }) 802 803 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); 804 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); 805 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); 806 bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); 807 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn); 808 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot, gfn_t gfn); 809 void mark_page_dirty(struct kvm *kvm, gfn_t gfn); 810 811 struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu); 812 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn); 813 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn); 814 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); 815 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map); 816 int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, 817 struct gfn_to_pfn_cache *cache, bool atomic); 818 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn); 819 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty); 820 int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, 821 struct gfn_to_pfn_cache *cache, bool dirty, bool atomic); 822 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); 823 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); 824 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, 825 int len); 826 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 827 unsigned long len); 828 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 829 unsigned long len); 830 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data, 831 int offset, int len); 832 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 833 unsigned long len); 834 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); 835 836 void kvm_sigset_activate(struct kvm_vcpu *vcpu); 837 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu); 838 839 void kvm_vcpu_block(struct kvm_vcpu *vcpu); 840 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu); 841 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu); 842 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu); 843 void kvm_vcpu_kick(struct kvm_vcpu *vcpu); 844 int kvm_vcpu_yield_to(struct kvm_vcpu *target); 845 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu, bool usermode_vcpu_not_eligible); 846 847 void kvm_flush_remote_tlbs(struct kvm *kvm); 848 void kvm_reload_remote_mmus(struct kvm *kvm); 849 850 #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 851 int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min); 852 int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc); 853 void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc); 854 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); 855 #endif 856 857 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, 858 struct kvm_vcpu *except, 859 unsigned long *vcpu_bitmap, cpumask_var_t tmp); 860 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req); 861 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req, 862 struct kvm_vcpu *except); 863 bool kvm_make_cpus_request_mask(struct kvm *kvm, unsigned int req, 864 unsigned long *vcpu_bitmap); 865 866 long kvm_arch_dev_ioctl(struct file *filp, 867 unsigned int ioctl, unsigned long arg); 868 long kvm_arch_vcpu_ioctl(struct file *filp, 869 unsigned int ioctl, unsigned long arg); 870 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf); 871 872 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext); 873 874 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 875 struct kvm_memory_slot *slot, 876 gfn_t gfn_offset, 877 unsigned long mask); 878 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot); 879 880 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 881 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 882 struct kvm_memory_slot *memslot); 883 #else /* !CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ 884 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); 885 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, 886 int *is_dirty, struct kvm_memory_slot **memslot); 887 #endif 888 889 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 890 bool line_status); 891 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 892 struct kvm_enable_cap *cap); 893 long kvm_arch_vm_ioctl(struct file *filp, 894 unsigned int ioctl, unsigned long arg); 895 896 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 897 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 898 899 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 900 struct kvm_translation *tr); 901 902 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 903 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 904 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 905 struct kvm_sregs *sregs); 906 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 907 struct kvm_sregs *sregs); 908 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 909 struct kvm_mp_state *mp_state); 910 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 911 struct kvm_mp_state *mp_state); 912 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 913 struct kvm_guest_debug *dbg); 914 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu); 915 916 int kvm_arch_init(void *opaque); 917 void kvm_arch_exit(void); 918 919 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu); 920 921 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); 922 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); 923 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id); 924 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu); 925 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu); 926 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); 927 928 #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS 929 void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); 930 #endif 931 932 int kvm_arch_hardware_enable(void); 933 void kvm_arch_hardware_disable(void); 934 int kvm_arch_hardware_setup(void *opaque); 935 void kvm_arch_hardware_unsetup(void); 936 int kvm_arch_check_processor_compat(void *opaque); 937 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); 938 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu); 939 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); 940 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu); 941 int kvm_arch_post_init_vm(struct kvm *kvm); 942 void kvm_arch_pre_destroy_vm(struct kvm *kvm); 943 944 #ifndef __KVM_HAVE_ARCH_VM_ALLOC 945 /* 946 * All architectures that want to use vzalloc currently also 947 * need their own kvm_arch_alloc_vm implementation. 948 */ 949 static inline struct kvm *kvm_arch_alloc_vm(void) 950 { 951 return kzalloc(sizeof(struct kvm), GFP_KERNEL); 952 } 953 954 static inline void kvm_arch_free_vm(struct kvm *kvm) 955 { 956 kfree(kvm); 957 } 958 #endif 959 960 #ifndef __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB 961 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) 962 { 963 return -ENOTSUPP; 964 } 965 #endif 966 967 #ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA 968 void kvm_arch_register_noncoherent_dma(struct kvm *kvm); 969 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm); 970 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm); 971 #else 972 static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 973 { 974 } 975 976 static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 977 { 978 } 979 980 static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 981 { 982 return false; 983 } 984 #endif 985 #ifdef __KVM_HAVE_ARCH_ASSIGNED_DEVICE 986 void kvm_arch_start_assignment(struct kvm *kvm); 987 void kvm_arch_end_assignment(struct kvm *kvm); 988 bool kvm_arch_has_assigned_device(struct kvm *kvm); 989 #else 990 static inline void kvm_arch_start_assignment(struct kvm *kvm) 991 { 992 } 993 994 static inline void kvm_arch_end_assignment(struct kvm *kvm) 995 { 996 } 997 998 static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) 999 { 1000 return false; 1001 } 1002 #endif 1003 1004 static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu) 1005 { 1006 #ifdef __KVM_HAVE_ARCH_WQP 1007 return vcpu->arch.waitp; 1008 #else 1009 return &vcpu->wait; 1010 #endif 1011 } 1012 1013 #ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED 1014 /* 1015 * returns true if the virtual interrupt controller is initialized and 1016 * ready to accept virtual IRQ. On some architectures the virtual interrupt 1017 * controller is dynamically instantiated and this is not always true. 1018 */ 1019 bool kvm_arch_intc_initialized(struct kvm *kvm); 1020 #else 1021 static inline bool kvm_arch_intc_initialized(struct kvm *kvm) 1022 { 1023 return true; 1024 } 1025 #endif 1026 1027 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); 1028 void kvm_arch_destroy_vm(struct kvm *kvm); 1029 void kvm_arch_sync_events(struct kvm *kvm); 1030 1031 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); 1032 1033 bool kvm_is_reserved_pfn(kvm_pfn_t pfn); 1034 bool kvm_is_zone_device_pfn(kvm_pfn_t pfn); 1035 bool kvm_is_transparent_hugepage(kvm_pfn_t pfn); 1036 1037 struct kvm_irq_ack_notifier { 1038 struct hlist_node link; 1039 unsigned gsi; 1040 void (*irq_acked)(struct kvm_irq_ack_notifier *kian); 1041 }; 1042 1043 int kvm_irq_map_gsi(struct kvm *kvm, 1044 struct kvm_kernel_irq_routing_entry *entries, int gsi); 1045 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin); 1046 1047 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 1048 bool line_status); 1049 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm, 1050 int irq_source_id, int level, bool line_status); 1051 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 1052 struct kvm *kvm, int irq_source_id, 1053 int level, bool line_status); 1054 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin); 1055 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi); 1056 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin); 1057 void kvm_register_irq_ack_notifier(struct kvm *kvm, 1058 struct kvm_irq_ack_notifier *kian); 1059 void kvm_unregister_irq_ack_notifier(struct kvm *kvm, 1060 struct kvm_irq_ack_notifier *kian); 1061 int kvm_request_irq_source_id(struct kvm *kvm); 1062 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id); 1063 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args); 1064 1065 /* 1066 * search_memslots() and __gfn_to_memslot() are here because they are 1067 * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c. 1068 * gfn_to_memslot() itself isn't here as an inline because that would 1069 * bloat other code too much. 1070 * 1071 * IMPORTANT: Slots are sorted from highest GFN to lowest GFN! 1072 */ 1073 static inline struct kvm_memory_slot * 1074 search_memslots(struct kvm_memslots *slots, gfn_t gfn) 1075 { 1076 int start = 0, end = slots->used_slots; 1077 int slot = atomic_read(&slots->lru_slot); 1078 struct kvm_memory_slot *memslots = slots->memslots; 1079 1080 if (unlikely(!slots->used_slots)) 1081 return NULL; 1082 1083 if (gfn >= memslots[slot].base_gfn && 1084 gfn < memslots[slot].base_gfn + memslots[slot].npages) 1085 return &memslots[slot]; 1086 1087 while (start < end) { 1088 slot = start + (end - start) / 2; 1089 1090 if (gfn >= memslots[slot].base_gfn) 1091 end = slot; 1092 else 1093 start = slot + 1; 1094 } 1095 1096 if (start < slots->used_slots && gfn >= memslots[start].base_gfn && 1097 gfn < memslots[start].base_gfn + memslots[start].npages) { 1098 atomic_set(&slots->lru_slot, start); 1099 return &memslots[start]; 1100 } 1101 1102 return NULL; 1103 } 1104 1105 static inline struct kvm_memory_slot * 1106 __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn) 1107 { 1108 return search_memslots(slots, gfn); 1109 } 1110 1111 static inline unsigned long 1112 __gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1113 { 1114 return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE; 1115 } 1116 1117 static inline int memslot_id(struct kvm *kvm, gfn_t gfn) 1118 { 1119 return gfn_to_memslot(kvm, gfn)->id; 1120 } 1121 1122 static inline gfn_t 1123 hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot) 1124 { 1125 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT; 1126 1127 return slot->base_gfn + gfn_offset; 1128 } 1129 1130 static inline gpa_t gfn_to_gpa(gfn_t gfn) 1131 { 1132 return (gpa_t)gfn << PAGE_SHIFT; 1133 } 1134 1135 static inline gfn_t gpa_to_gfn(gpa_t gpa) 1136 { 1137 return (gfn_t)(gpa >> PAGE_SHIFT); 1138 } 1139 1140 static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn) 1141 { 1142 return (hpa_t)pfn << PAGE_SHIFT; 1143 } 1144 1145 static inline struct page *kvm_vcpu_gpa_to_page(struct kvm_vcpu *vcpu, 1146 gpa_t gpa) 1147 { 1148 return kvm_vcpu_gfn_to_page(vcpu, gpa_to_gfn(gpa)); 1149 } 1150 1151 static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) 1152 { 1153 unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); 1154 1155 return kvm_is_error_hva(hva); 1156 } 1157 1158 enum kvm_stat_kind { 1159 KVM_STAT_VM, 1160 KVM_STAT_VCPU, 1161 }; 1162 1163 struct kvm_stat_data { 1164 struct kvm *kvm; 1165 struct kvm_stats_debugfs_item *dbgfs_item; 1166 }; 1167 1168 struct kvm_stats_debugfs_item { 1169 const char *name; 1170 int offset; 1171 enum kvm_stat_kind kind; 1172 int mode; 1173 }; 1174 1175 #define KVM_DBGFS_GET_MODE(dbgfs_item) \ 1176 ((dbgfs_item)->mode ? (dbgfs_item)->mode : 0644) 1177 1178 #define VM_STAT(n, x, ...) \ 1179 { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } 1180 #define VCPU_STAT(n, x, ...) \ 1181 { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } 1182 1183 extern struct kvm_stats_debugfs_item debugfs_entries[]; 1184 extern struct dentry *kvm_debugfs_dir; 1185 1186 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 1187 static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq) 1188 { 1189 if (unlikely(kvm->mmu_notifier_count)) 1190 return 1; 1191 /* 1192 * Ensure the read of mmu_notifier_count happens before the read 1193 * of mmu_notifier_seq. This interacts with the smp_wmb() in 1194 * mmu_notifier_invalidate_range_end to make sure that the caller 1195 * either sees the old (non-zero) value of mmu_notifier_count or 1196 * the new (incremented) value of mmu_notifier_seq. 1197 * PowerPC Book3s HV KVM calls this under a per-page lock 1198 * rather than under kvm->mmu_lock, for scalability, so 1199 * can't rely on kvm->mmu_lock to keep things ordered. 1200 */ 1201 smp_rmb(); 1202 if (kvm->mmu_notifier_seq != mmu_seq) 1203 return 1; 1204 return 0; 1205 } 1206 #endif 1207 1208 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 1209 1210 #define KVM_MAX_IRQ_ROUTES 4096 /* might need extension/rework in the future */ 1211 1212 bool kvm_arch_can_set_irq_routing(struct kvm *kvm); 1213 int kvm_set_irq_routing(struct kvm *kvm, 1214 const struct kvm_irq_routing_entry *entries, 1215 unsigned nr, 1216 unsigned flags); 1217 int kvm_set_routing_entry(struct kvm *kvm, 1218 struct kvm_kernel_irq_routing_entry *e, 1219 const struct kvm_irq_routing_entry *ue); 1220 void kvm_free_irq_routing(struct kvm *kvm); 1221 1222 #else 1223 1224 static inline void kvm_free_irq_routing(struct kvm *kvm) {} 1225 1226 #endif 1227 1228 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); 1229 1230 #ifdef CONFIG_HAVE_KVM_EVENTFD 1231 1232 void kvm_eventfd_init(struct kvm *kvm); 1233 int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); 1234 1235 #ifdef CONFIG_HAVE_KVM_IRQFD 1236 int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args); 1237 void kvm_irqfd_release(struct kvm *kvm); 1238 void kvm_irq_routing_update(struct kvm *); 1239 #else 1240 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 1241 { 1242 return -EINVAL; 1243 } 1244 1245 static inline void kvm_irqfd_release(struct kvm *kvm) {} 1246 #endif 1247 1248 #else 1249 1250 static inline void kvm_eventfd_init(struct kvm *kvm) {} 1251 1252 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 1253 { 1254 return -EINVAL; 1255 } 1256 1257 static inline void kvm_irqfd_release(struct kvm *kvm) {} 1258 1259 #ifdef CONFIG_HAVE_KVM_IRQCHIP 1260 static inline void kvm_irq_routing_update(struct kvm *kvm) 1261 { 1262 } 1263 #endif 1264 1265 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 1266 { 1267 return -ENOSYS; 1268 } 1269 1270 #endif /* CONFIG_HAVE_KVM_EVENTFD */ 1271 1272 void kvm_arch_irq_routing_update(struct kvm *kvm); 1273 1274 static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) 1275 { 1276 /* 1277 * Ensure the rest of the request is published to kvm_check_request's 1278 * caller. Paired with the smp_mb__after_atomic in kvm_check_request. 1279 */ 1280 smp_wmb(); 1281 set_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 1282 } 1283 1284 static inline bool kvm_request_pending(struct kvm_vcpu *vcpu) 1285 { 1286 return READ_ONCE(vcpu->requests); 1287 } 1288 1289 static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu) 1290 { 1291 return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 1292 } 1293 1294 static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) 1295 { 1296 clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 1297 } 1298 1299 static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) 1300 { 1301 if (kvm_test_request(req, vcpu)) { 1302 kvm_clear_request(req, vcpu); 1303 1304 /* 1305 * Ensure the rest of the request is visible to kvm_check_request's 1306 * caller. Paired with the smp_wmb in kvm_make_request. 1307 */ 1308 smp_mb__after_atomic(); 1309 return true; 1310 } else { 1311 return false; 1312 } 1313 } 1314 1315 extern bool kvm_rebooting; 1316 1317 extern unsigned int halt_poll_ns; 1318 extern unsigned int halt_poll_ns_grow; 1319 extern unsigned int halt_poll_ns_grow_start; 1320 extern unsigned int halt_poll_ns_shrink; 1321 1322 struct kvm_device { 1323 const struct kvm_device_ops *ops; 1324 struct kvm *kvm; 1325 void *private; 1326 struct list_head vm_node; 1327 }; 1328 1329 /* create, destroy, and name are mandatory */ 1330 struct kvm_device_ops { 1331 const char *name; 1332 1333 /* 1334 * create is called holding kvm->lock and any operations not suitable 1335 * to do while holding the lock should be deferred to init (see 1336 * below). 1337 */ 1338 int (*create)(struct kvm_device *dev, u32 type); 1339 1340 /* 1341 * init is called after create if create is successful and is called 1342 * outside of holding kvm->lock. 1343 */ 1344 void (*init)(struct kvm_device *dev); 1345 1346 /* 1347 * Destroy is responsible for freeing dev. 1348 * 1349 * Destroy may be called before or after destructors are called 1350 * on emulated I/O regions, depending on whether a reference is 1351 * held by a vcpu or other kvm component that gets destroyed 1352 * after the emulated I/O. 1353 */ 1354 void (*destroy)(struct kvm_device *dev); 1355 1356 /* 1357 * Release is an alternative method to free the device. It is 1358 * called when the device file descriptor is closed. Once 1359 * release is called, the destroy method will not be called 1360 * anymore as the device is removed from the device list of 1361 * the VM. kvm->lock is held. 1362 */ 1363 void (*release)(struct kvm_device *dev); 1364 1365 int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1366 int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1367 int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1368 long (*ioctl)(struct kvm_device *dev, unsigned int ioctl, 1369 unsigned long arg); 1370 int (*mmap)(struct kvm_device *dev, struct vm_area_struct *vma); 1371 }; 1372 1373 void kvm_device_get(struct kvm_device *dev); 1374 void kvm_device_put(struct kvm_device *dev); 1375 struct kvm_device *kvm_device_from_filp(struct file *filp); 1376 int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type); 1377 void kvm_unregister_device_ops(u32 type); 1378 1379 extern struct kvm_device_ops kvm_mpic_ops; 1380 extern struct kvm_device_ops kvm_arm_vgic_v2_ops; 1381 extern struct kvm_device_ops kvm_arm_vgic_v3_ops; 1382 1383 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 1384 1385 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 1386 { 1387 vcpu->spin_loop.in_spin_loop = val; 1388 } 1389 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 1390 { 1391 vcpu->spin_loop.dy_eligible = val; 1392 } 1393 1394 #else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 1395 1396 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 1397 { 1398 } 1399 1400 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 1401 { 1402 } 1403 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 1404 1405 static inline bool kvm_is_visible_memslot(struct kvm_memory_slot *memslot) 1406 { 1407 return (memslot && memslot->id < KVM_USER_MEM_SLOTS && 1408 !(memslot->flags & KVM_MEMSLOT_INVALID)); 1409 } 1410 1411 struct kvm_vcpu *kvm_get_running_vcpu(void); 1412 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); 1413 1414 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 1415 bool kvm_arch_has_irq_bypass(void); 1416 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *, 1417 struct irq_bypass_producer *); 1418 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *, 1419 struct irq_bypass_producer *); 1420 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *); 1421 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *); 1422 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, 1423 uint32_t guest_irq, bool set); 1424 #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ 1425 1426 #ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS 1427 /* If we wakeup during the poll time, was it a sucessful poll? */ 1428 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 1429 { 1430 return vcpu->valid_wakeup; 1431 } 1432 1433 #else 1434 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 1435 { 1436 return true; 1437 } 1438 #endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ 1439 1440 #ifdef CONFIG_HAVE_KVM_NO_POLL 1441 /* Callback that tells if we must not poll */ 1442 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); 1443 #else 1444 static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 1445 { 1446 return false; 1447 } 1448 #endif /* CONFIG_HAVE_KVM_NO_POLL */ 1449 1450 #ifdef CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL 1451 long kvm_arch_vcpu_async_ioctl(struct file *filp, 1452 unsigned int ioctl, unsigned long arg); 1453 #else 1454 static inline long kvm_arch_vcpu_async_ioctl(struct file *filp, 1455 unsigned int ioctl, 1456 unsigned long arg) 1457 { 1458 return -ENOIOCTLCMD; 1459 } 1460 #endif /* CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL */ 1461 1462 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, 1463 unsigned long start, unsigned long end); 1464 1465 #ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE 1466 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); 1467 #else 1468 static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 1469 { 1470 return 0; 1471 } 1472 #endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ 1473 1474 typedef int (*kvm_vm_thread_fn_t)(struct kvm *kvm, uintptr_t data); 1475 1476 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, 1477 uintptr_t data, const char *name, 1478 struct task_struct **thread_ptr); 1479 1480 #ifdef CONFIG_KVM_XFER_TO_GUEST_WORK 1481 static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) 1482 { 1483 vcpu->run->exit_reason = KVM_EXIT_INTR; 1484 vcpu->stat.signal_exits++; 1485 } 1486 #endif /* CONFIG_KVM_XFER_TO_GUEST_WORK */ 1487 1488 /* 1489 * This defines how many reserved entries we want to keep before we 1490 * kick the vcpu to the userspace to avoid dirty ring full. This 1491 * value can be tuned to higher if e.g. PML is enabled on the host. 1492 */ 1493 #define KVM_DIRTY_RING_RSVD_ENTRIES 64 1494 1495 /* Max number of entries allowed for each kvm dirty ring */ 1496 #define KVM_DIRTY_RING_MAX_ENTRIES 65536 1497 1498 #endif 1499