1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * MMU support 9 * 10 * Copyright (C) 2006 Qumranet, Inc. 11 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 12 * 13 * Authors: 14 * Yaniv Kamay <yaniv@qumranet.com> 15 * Avi Kivity <avi@qumranet.com> 16 */ 17 18 #include "irq.h" 19 #include "ioapic.h" 20 #include "mmu.h" 21 #include "mmu_internal.h" 22 #include "tdp_mmu.h" 23 #include "x86.h" 24 #include "kvm_cache_regs.h" 25 #include "kvm_emulate.h" 26 #include "cpuid.h" 27 #include "spte.h" 28 29 #include <linux/kvm_host.h> 30 #include <linux/types.h> 31 #include <linux/string.h> 32 #include <linux/mm.h> 33 #include <linux/highmem.h> 34 #include <linux/moduleparam.h> 35 #include <linux/export.h> 36 #include <linux/swap.h> 37 #include <linux/hugetlb.h> 38 #include <linux/compiler.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/sched/signal.h> 42 #include <linux/uaccess.h> 43 #include <linux/hash.h> 44 #include <linux/kern_levels.h> 45 #include <linux/kthread.h> 46 47 #include <asm/page.h> 48 #include <asm/memtype.h> 49 #include <asm/cmpxchg.h> 50 #include <asm/io.h> 51 #include <asm/vmx.h> 52 #include <asm/kvm_page_track.h> 53 #include "trace.h" 54 55 extern bool itlb_multihit_kvm_mitigation; 56 57 static int __read_mostly nx_huge_pages = -1; 58 #ifdef CONFIG_PREEMPT_RT 59 /* Recovery can cause latency spikes, disable it for PREEMPT_RT. */ 60 static uint __read_mostly nx_huge_pages_recovery_ratio = 0; 61 #else 62 static uint __read_mostly nx_huge_pages_recovery_ratio = 60; 63 #endif 64 65 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp); 66 static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp); 67 68 static const struct kernel_param_ops nx_huge_pages_ops = { 69 .set = set_nx_huge_pages, 70 .get = param_get_bool, 71 }; 72 73 static const struct kernel_param_ops nx_huge_pages_recovery_ratio_ops = { 74 .set = set_nx_huge_pages_recovery_ratio, 75 .get = param_get_uint, 76 }; 77 78 module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644); 79 __MODULE_PARM_TYPE(nx_huge_pages, "bool"); 80 module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_ratio_ops, 81 &nx_huge_pages_recovery_ratio, 0644); 82 __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint"); 83 84 static bool __read_mostly force_flush_and_sync_on_reuse; 85 module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); 86 87 /* 88 * When setting this variable to true it enables Two-Dimensional-Paging 89 * where the hardware walks 2 page tables: 90 * 1. the guest-virtual to guest-physical 91 * 2. while doing 1. it walks guest-physical to host-physical 92 * If the hardware supports that we don't need to do shadow paging. 93 */ 94 bool tdp_enabled = false; 95 96 static int max_huge_page_level __read_mostly; 97 static int max_tdp_level __read_mostly; 98 99 enum { 100 AUDIT_PRE_PAGE_FAULT, 101 AUDIT_POST_PAGE_FAULT, 102 AUDIT_PRE_PTE_WRITE, 103 AUDIT_POST_PTE_WRITE, 104 AUDIT_PRE_SYNC, 105 AUDIT_POST_SYNC 106 }; 107 108 #ifdef MMU_DEBUG 109 bool dbg = 0; 110 module_param(dbg, bool, 0644); 111 #endif 112 113 #define PTE_PREFETCH_NUM 8 114 115 #define PT32_LEVEL_BITS 10 116 117 #define PT32_LEVEL_SHIFT(level) \ 118 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) 119 120 #define PT32_LVL_OFFSET_MASK(level) \ 121 (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \ 122 * PT32_LEVEL_BITS))) - 1)) 123 124 #define PT32_INDEX(address, level)\ 125 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1)) 126 127 128 #define PT32_BASE_ADDR_MASK PAGE_MASK 129 #define PT32_DIR_BASE_ADDR_MASK \ 130 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) 131 #define PT32_LVL_ADDR_MASK(level) \ 132 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \ 133 * PT32_LEVEL_BITS))) - 1)) 134 135 #include <trace/events/kvm.h> 136 137 /* make pte_list_desc fit well in cache line */ 138 #define PTE_LIST_EXT 3 139 140 struct pte_list_desc { 141 u64 *sptes[PTE_LIST_EXT]; 142 struct pte_list_desc *more; 143 }; 144 145 struct kvm_shadow_walk_iterator { 146 u64 addr; 147 hpa_t shadow_addr; 148 u64 *sptep; 149 int level; 150 unsigned index; 151 }; 152 153 #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker) \ 154 for (shadow_walk_init_using_root(&(_walker), (_vcpu), \ 155 (_root), (_addr)); \ 156 shadow_walk_okay(&(_walker)); \ 157 shadow_walk_next(&(_walker))) 158 159 #define for_each_shadow_entry(_vcpu, _addr, _walker) \ 160 for (shadow_walk_init(&(_walker), _vcpu, _addr); \ 161 shadow_walk_okay(&(_walker)); \ 162 shadow_walk_next(&(_walker))) 163 164 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \ 165 for (shadow_walk_init(&(_walker), _vcpu, _addr); \ 166 shadow_walk_okay(&(_walker)) && \ 167 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \ 168 __shadow_walk_next(&(_walker), spte)) 169 170 static struct kmem_cache *pte_list_desc_cache; 171 struct kmem_cache *mmu_page_header_cache; 172 static struct percpu_counter kvm_total_used_mmu_pages; 173 174 static void mmu_spte_set(u64 *sptep, u64 spte); 175 static union kvm_mmu_page_role 176 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu); 177 178 #define CREATE_TRACE_POINTS 179 #include "mmutrace.h" 180 181 182 static inline bool kvm_available_flush_tlb_with_range(void) 183 { 184 return kvm_x86_ops.tlb_remote_flush_with_range; 185 } 186 187 static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, 188 struct kvm_tlb_range *range) 189 { 190 int ret = -ENOTSUPP; 191 192 if (range && kvm_x86_ops.tlb_remote_flush_with_range) 193 ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range); 194 195 if (ret) 196 kvm_flush_remote_tlbs(kvm); 197 } 198 199 void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, 200 u64 start_gfn, u64 pages) 201 { 202 struct kvm_tlb_range range; 203 204 range.start_gfn = start_gfn; 205 range.pages = pages; 206 207 kvm_flush_remote_tlbs_with_range(kvm, &range); 208 } 209 210 bool is_nx_huge_page_enabled(void) 211 { 212 return READ_ONCE(nx_huge_pages); 213 } 214 215 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, 216 unsigned int access) 217 { 218 u64 mask = make_mmio_spte(vcpu, gfn, access); 219 220 trace_mark_mmio_spte(sptep, gfn, mask); 221 mmu_spte_set(sptep, mask); 222 } 223 224 static gfn_t get_mmio_spte_gfn(u64 spte) 225 { 226 u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask; 227 228 gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN) 229 & shadow_nonpresent_or_rsvd_mask; 230 231 return gpa >> PAGE_SHIFT; 232 } 233 234 static unsigned get_mmio_spte_access(u64 spte) 235 { 236 return spte & shadow_mmio_access_mask; 237 } 238 239 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, 240 kvm_pfn_t pfn, unsigned int access) 241 { 242 if (unlikely(is_noslot_pfn(pfn))) { 243 mark_mmio_spte(vcpu, sptep, gfn, access); 244 return true; 245 } 246 247 return false; 248 } 249 250 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte) 251 { 252 u64 kvm_gen, spte_gen, gen; 253 254 gen = kvm_vcpu_memslots(vcpu)->generation; 255 if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS)) 256 return false; 257 258 kvm_gen = gen & MMIO_SPTE_GEN_MASK; 259 spte_gen = get_mmio_spte_generation(spte); 260 261 trace_check_mmio_spte(spte, kvm_gen, spte_gen); 262 return likely(kvm_gen == spte_gen); 263 } 264 265 static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, 266 struct x86_exception *exception) 267 { 268 /* Check if guest physical address doesn't exceed guest maximum */ 269 if (kvm_vcpu_is_illegal_gpa(vcpu, gpa)) { 270 exception->error_code |= PFERR_RSVD_MASK; 271 return UNMAPPED_GVA; 272 } 273 274 return gpa; 275 } 276 277 static int is_cpuid_PSE36(void) 278 { 279 return 1; 280 } 281 282 static int is_nx(struct kvm_vcpu *vcpu) 283 { 284 return vcpu->arch.efer & EFER_NX; 285 } 286 287 static gfn_t pse36_gfn_delta(u32 gpte) 288 { 289 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; 290 291 return (gpte & PT32_DIR_PSE36_MASK) << shift; 292 } 293 294 #ifdef CONFIG_X86_64 295 static void __set_spte(u64 *sptep, u64 spte) 296 { 297 WRITE_ONCE(*sptep, spte); 298 } 299 300 static void __update_clear_spte_fast(u64 *sptep, u64 spte) 301 { 302 WRITE_ONCE(*sptep, spte); 303 } 304 305 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) 306 { 307 return xchg(sptep, spte); 308 } 309 310 static u64 __get_spte_lockless(u64 *sptep) 311 { 312 return READ_ONCE(*sptep); 313 } 314 #else 315 union split_spte { 316 struct { 317 u32 spte_low; 318 u32 spte_high; 319 }; 320 u64 spte; 321 }; 322 323 static void count_spte_clear(u64 *sptep, u64 spte) 324 { 325 struct kvm_mmu_page *sp = sptep_to_sp(sptep); 326 327 if (is_shadow_present_pte(spte)) 328 return; 329 330 /* Ensure the spte is completely set before we increase the count */ 331 smp_wmb(); 332 sp->clear_spte_count++; 333 } 334 335 static void __set_spte(u64 *sptep, u64 spte) 336 { 337 union split_spte *ssptep, sspte; 338 339 ssptep = (union split_spte *)sptep; 340 sspte = (union split_spte)spte; 341 342 ssptep->spte_high = sspte.spte_high; 343 344 /* 345 * If we map the spte from nonpresent to present, We should store 346 * the high bits firstly, then set present bit, so cpu can not 347 * fetch this spte while we are setting the spte. 348 */ 349 smp_wmb(); 350 351 WRITE_ONCE(ssptep->spte_low, sspte.spte_low); 352 } 353 354 static void __update_clear_spte_fast(u64 *sptep, u64 spte) 355 { 356 union split_spte *ssptep, sspte; 357 358 ssptep = (union split_spte *)sptep; 359 sspte = (union split_spte)spte; 360 361 WRITE_ONCE(ssptep->spte_low, sspte.spte_low); 362 363 /* 364 * If we map the spte from present to nonpresent, we should clear 365 * present bit firstly to avoid vcpu fetch the old high bits. 366 */ 367 smp_wmb(); 368 369 ssptep->spte_high = sspte.spte_high; 370 count_spte_clear(sptep, spte); 371 } 372 373 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) 374 { 375 union split_spte *ssptep, sspte, orig; 376 377 ssptep = (union split_spte *)sptep; 378 sspte = (union split_spte)spte; 379 380 /* xchg acts as a barrier before the setting of the high bits */ 381 orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low); 382 orig.spte_high = ssptep->spte_high; 383 ssptep->spte_high = sspte.spte_high; 384 count_spte_clear(sptep, spte); 385 386 return orig.spte; 387 } 388 389 /* 390 * The idea using the light way get the spte on x86_32 guest is from 391 * gup_get_pte (mm/gup.c). 392 * 393 * An spte tlb flush may be pending, because kvm_set_pte_rmapp 394 * coalesces them and we are running out of the MMU lock. Therefore 395 * we need to protect against in-progress updates of the spte. 396 * 397 * Reading the spte while an update is in progress may get the old value 398 * for the high part of the spte. The race is fine for a present->non-present 399 * change (because the high part of the spte is ignored for non-present spte), 400 * but for a present->present change we must reread the spte. 401 * 402 * All such changes are done in two steps (present->non-present and 403 * non-present->present), hence it is enough to count the number of 404 * present->non-present updates: if it changed while reading the spte, 405 * we might have hit the race. This is done using clear_spte_count. 406 */ 407 static u64 __get_spte_lockless(u64 *sptep) 408 { 409 struct kvm_mmu_page *sp = sptep_to_sp(sptep); 410 union split_spte spte, *orig = (union split_spte *)sptep; 411 int count; 412 413 retry: 414 count = sp->clear_spte_count; 415 smp_rmb(); 416 417 spte.spte_low = orig->spte_low; 418 smp_rmb(); 419 420 spte.spte_high = orig->spte_high; 421 smp_rmb(); 422 423 if (unlikely(spte.spte_low != orig->spte_low || 424 count != sp->clear_spte_count)) 425 goto retry; 426 427 return spte.spte; 428 } 429 #endif 430 431 static bool spte_has_volatile_bits(u64 spte) 432 { 433 if (!is_shadow_present_pte(spte)) 434 return false; 435 436 /* 437 * Always atomically update spte if it can be updated 438 * out of mmu-lock, it can ensure dirty bit is not lost, 439 * also, it can help us to get a stable is_writable_pte() 440 * to ensure tlb flush is not missed. 441 */ 442 if (spte_can_locklessly_be_made_writable(spte) || 443 is_access_track_spte(spte)) 444 return true; 445 446 if (spte_ad_enabled(spte)) { 447 if ((spte & shadow_accessed_mask) == 0 || 448 (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0)) 449 return true; 450 } 451 452 return false; 453 } 454 455 /* Rules for using mmu_spte_set: 456 * Set the sptep from nonpresent to present. 457 * Note: the sptep being assigned *must* be either not present 458 * or in a state where the hardware will not attempt to update 459 * the spte. 460 */ 461 static void mmu_spte_set(u64 *sptep, u64 new_spte) 462 { 463 WARN_ON(is_shadow_present_pte(*sptep)); 464 __set_spte(sptep, new_spte); 465 } 466 467 /* 468 * Update the SPTE (excluding the PFN), but do not track changes in its 469 * accessed/dirty status. 470 */ 471 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte) 472 { 473 u64 old_spte = *sptep; 474 475 WARN_ON(!is_shadow_present_pte(new_spte)); 476 477 if (!is_shadow_present_pte(old_spte)) { 478 mmu_spte_set(sptep, new_spte); 479 return old_spte; 480 } 481 482 if (!spte_has_volatile_bits(old_spte)) 483 __update_clear_spte_fast(sptep, new_spte); 484 else 485 old_spte = __update_clear_spte_slow(sptep, new_spte); 486 487 WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte)); 488 489 return old_spte; 490 } 491 492 /* Rules for using mmu_spte_update: 493 * Update the state bits, it means the mapped pfn is not changed. 494 * 495 * Whenever we overwrite a writable spte with a read-only one we 496 * should flush remote TLBs. Otherwise rmap_write_protect 497 * will find a read-only spte, even though the writable spte 498 * might be cached on a CPU's TLB, the return value indicates this 499 * case. 500 * 501 * Returns true if the TLB needs to be flushed 502 */ 503 static bool mmu_spte_update(u64 *sptep, u64 new_spte) 504 { 505 bool flush = false; 506 u64 old_spte = mmu_spte_update_no_track(sptep, new_spte); 507 508 if (!is_shadow_present_pte(old_spte)) 509 return false; 510 511 /* 512 * For the spte updated out of mmu-lock is safe, since 513 * we always atomically update it, see the comments in 514 * spte_has_volatile_bits(). 515 */ 516 if (spte_can_locklessly_be_made_writable(old_spte) && 517 !is_writable_pte(new_spte)) 518 flush = true; 519 520 /* 521 * Flush TLB when accessed/dirty states are changed in the page tables, 522 * to guarantee consistency between TLB and page tables. 523 */ 524 525 if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) { 526 flush = true; 527 kvm_set_pfn_accessed(spte_to_pfn(old_spte)); 528 } 529 530 if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) { 531 flush = true; 532 kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 533 } 534 535 return flush; 536 } 537 538 /* 539 * Rules for using mmu_spte_clear_track_bits: 540 * It sets the sptep from present to nonpresent, and track the 541 * state bits, it is used to clear the last level sptep. 542 * Returns non-zero if the PTE was previously valid. 543 */ 544 static int mmu_spte_clear_track_bits(u64 *sptep) 545 { 546 kvm_pfn_t pfn; 547 u64 old_spte = *sptep; 548 549 if (!spte_has_volatile_bits(old_spte)) 550 __update_clear_spte_fast(sptep, 0ull); 551 else 552 old_spte = __update_clear_spte_slow(sptep, 0ull); 553 554 if (!is_shadow_present_pte(old_spte)) 555 return 0; 556 557 pfn = spte_to_pfn(old_spte); 558 559 /* 560 * KVM does not hold the refcount of the page used by 561 * kvm mmu, before reclaiming the page, we should 562 * unmap it from mmu first. 563 */ 564 WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn))); 565 566 if (is_accessed_spte(old_spte)) 567 kvm_set_pfn_accessed(pfn); 568 569 if (is_dirty_spte(old_spte)) 570 kvm_set_pfn_dirty(pfn); 571 572 return 1; 573 } 574 575 /* 576 * Rules for using mmu_spte_clear_no_track: 577 * Directly clear spte without caring the state bits of sptep, 578 * it is used to set the upper level spte. 579 */ 580 static void mmu_spte_clear_no_track(u64 *sptep) 581 { 582 __update_clear_spte_fast(sptep, 0ull); 583 } 584 585 static u64 mmu_spte_get_lockless(u64 *sptep) 586 { 587 return __get_spte_lockless(sptep); 588 } 589 590 /* Restore an acc-track PTE back to a regular PTE */ 591 static u64 restore_acc_track_spte(u64 spte) 592 { 593 u64 new_spte = spte; 594 u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT) 595 & SHADOW_ACC_TRACK_SAVED_BITS_MASK; 596 597 WARN_ON_ONCE(spte_ad_enabled(spte)); 598 WARN_ON_ONCE(!is_access_track_spte(spte)); 599 600 new_spte &= ~shadow_acc_track_mask; 601 new_spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK << 602 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT); 603 new_spte |= saved_bits; 604 605 return new_spte; 606 } 607 608 /* Returns the Accessed status of the PTE and resets it at the same time. */ 609 static bool mmu_spte_age(u64 *sptep) 610 { 611 u64 spte = mmu_spte_get_lockless(sptep); 612 613 if (!is_accessed_spte(spte)) 614 return false; 615 616 if (spte_ad_enabled(spte)) { 617 clear_bit((ffs(shadow_accessed_mask) - 1), 618 (unsigned long *)sptep); 619 } else { 620 /* 621 * Capture the dirty status of the page, so that it doesn't get 622 * lost when the SPTE is marked for access tracking. 623 */ 624 if (is_writable_pte(spte)) 625 kvm_set_pfn_dirty(spte_to_pfn(spte)); 626 627 spte = mark_spte_for_access_track(spte); 628 mmu_spte_update_no_track(sptep, spte); 629 } 630 631 return true; 632 } 633 634 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu) 635 { 636 /* 637 * Prevent page table teardown by making any free-er wait during 638 * kvm_flush_remote_tlbs() IPI to all active vcpus. 639 */ 640 local_irq_disable(); 641 642 /* 643 * Make sure a following spte read is not reordered ahead of the write 644 * to vcpu->mode. 645 */ 646 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES); 647 } 648 649 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu) 650 { 651 /* 652 * Make sure the write to vcpu->mode is not reordered in front of 653 * reads to sptes. If it does, kvm_mmu_commit_zap_page() can see us 654 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table. 655 */ 656 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); 657 local_irq_enable(); 658 } 659 660 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect) 661 { 662 int r; 663 664 /* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */ 665 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache, 666 1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM); 667 if (r) 668 return r; 669 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache, 670 PT64_ROOT_MAX_LEVEL); 671 if (r) 672 return r; 673 if (maybe_indirect) { 674 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_gfn_array_cache, 675 PT64_ROOT_MAX_LEVEL); 676 if (r) 677 return r; 678 } 679 return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, 680 PT64_ROOT_MAX_LEVEL); 681 } 682 683 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) 684 { 685 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache); 686 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache); 687 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_gfn_array_cache); 688 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache); 689 } 690 691 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu) 692 { 693 return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache); 694 } 695 696 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc) 697 { 698 kmem_cache_free(pte_list_desc_cache, pte_list_desc); 699 } 700 701 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index) 702 { 703 if (!sp->role.direct) 704 return sp->gfns[index]; 705 706 return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS)); 707 } 708 709 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) 710 { 711 if (!sp->role.direct) { 712 sp->gfns[index] = gfn; 713 return; 714 } 715 716 if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index))) 717 pr_err_ratelimited("gfn mismatch under direct page %llx " 718 "(expected %llx, got %llx)\n", 719 sp->gfn, 720 kvm_mmu_page_get_gfn(sp, index), gfn); 721 } 722 723 /* 724 * Return the pointer to the large page information for a given gfn, 725 * handling slots that are not large page aligned. 726 */ 727 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, 728 struct kvm_memory_slot *slot, 729 int level) 730 { 731 unsigned long idx; 732 733 idx = gfn_to_index(gfn, slot->base_gfn, level); 734 return &slot->arch.lpage_info[level - 2][idx]; 735 } 736 737 static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot, 738 gfn_t gfn, int count) 739 { 740 struct kvm_lpage_info *linfo; 741 int i; 742 743 for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { 744 linfo = lpage_info_slot(gfn, slot, i); 745 linfo->disallow_lpage += count; 746 WARN_ON(linfo->disallow_lpage < 0); 747 } 748 } 749 750 void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn) 751 { 752 update_gfn_disallow_lpage_count(slot, gfn, 1); 753 } 754 755 void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn) 756 { 757 update_gfn_disallow_lpage_count(slot, gfn, -1); 758 } 759 760 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) 761 { 762 struct kvm_memslots *slots; 763 struct kvm_memory_slot *slot; 764 gfn_t gfn; 765 766 kvm->arch.indirect_shadow_pages++; 767 gfn = sp->gfn; 768 slots = kvm_memslots_for_spte_role(kvm, sp->role); 769 slot = __gfn_to_memslot(slots, gfn); 770 771 /* the non-leaf shadow pages are keeping readonly. */ 772 if (sp->role.level > PG_LEVEL_4K) 773 return kvm_slot_page_track_add_page(kvm, slot, gfn, 774 KVM_PAGE_TRACK_WRITE); 775 776 kvm_mmu_gfn_disallow_lpage(slot, gfn); 777 } 778 779 void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp) 780 { 781 if (sp->lpage_disallowed) 782 return; 783 784 ++kvm->stat.nx_lpage_splits; 785 list_add_tail(&sp->lpage_disallowed_link, 786 &kvm->arch.lpage_disallowed_mmu_pages); 787 sp->lpage_disallowed = true; 788 } 789 790 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) 791 { 792 struct kvm_memslots *slots; 793 struct kvm_memory_slot *slot; 794 gfn_t gfn; 795 796 kvm->arch.indirect_shadow_pages--; 797 gfn = sp->gfn; 798 slots = kvm_memslots_for_spte_role(kvm, sp->role); 799 slot = __gfn_to_memslot(slots, gfn); 800 if (sp->role.level > PG_LEVEL_4K) 801 return kvm_slot_page_track_remove_page(kvm, slot, gfn, 802 KVM_PAGE_TRACK_WRITE); 803 804 kvm_mmu_gfn_allow_lpage(slot, gfn); 805 } 806 807 void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp) 808 { 809 --kvm->stat.nx_lpage_splits; 810 sp->lpage_disallowed = false; 811 list_del(&sp->lpage_disallowed_link); 812 } 813 814 static struct kvm_memory_slot * 815 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn, 816 bool no_dirty_log) 817 { 818 struct kvm_memory_slot *slot; 819 820 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 821 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 822 return NULL; 823 if (no_dirty_log && slot->dirty_bitmap) 824 return NULL; 825 826 return slot; 827 } 828 829 /* 830 * About rmap_head encoding: 831 * 832 * If the bit zero of rmap_head->val is clear, then it points to the only spte 833 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct 834 * pte_list_desc containing more mappings. 835 */ 836 837 /* 838 * Returns the number of pointers in the rmap chain, not counting the new one. 839 */ 840 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte, 841 struct kvm_rmap_head *rmap_head) 842 { 843 struct pte_list_desc *desc; 844 int i, count = 0; 845 846 if (!rmap_head->val) { 847 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte); 848 rmap_head->val = (unsigned long)spte; 849 } else if (!(rmap_head->val & 1)) { 850 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte); 851 desc = mmu_alloc_pte_list_desc(vcpu); 852 desc->sptes[0] = (u64 *)rmap_head->val; 853 desc->sptes[1] = spte; 854 rmap_head->val = (unsigned long)desc | 1; 855 ++count; 856 } else { 857 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte); 858 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 859 while (desc->sptes[PTE_LIST_EXT-1]) { 860 count += PTE_LIST_EXT; 861 862 if (!desc->more) { 863 desc->more = mmu_alloc_pte_list_desc(vcpu); 864 desc = desc->more; 865 break; 866 } 867 desc = desc->more; 868 } 869 for (i = 0; desc->sptes[i]; ++i) 870 ++count; 871 desc->sptes[i] = spte; 872 } 873 return count; 874 } 875 876 static void 877 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head, 878 struct pte_list_desc *desc, int i, 879 struct pte_list_desc *prev_desc) 880 { 881 int j; 882 883 for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j) 884 ; 885 desc->sptes[i] = desc->sptes[j]; 886 desc->sptes[j] = NULL; 887 if (j != 0) 888 return; 889 if (!prev_desc && !desc->more) 890 rmap_head->val = 0; 891 else 892 if (prev_desc) 893 prev_desc->more = desc->more; 894 else 895 rmap_head->val = (unsigned long)desc->more | 1; 896 mmu_free_pte_list_desc(desc); 897 } 898 899 static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head) 900 { 901 struct pte_list_desc *desc; 902 struct pte_list_desc *prev_desc; 903 int i; 904 905 if (!rmap_head->val) { 906 pr_err("%s: %p 0->BUG\n", __func__, spte); 907 BUG(); 908 } else if (!(rmap_head->val & 1)) { 909 rmap_printk("%s: %p 1->0\n", __func__, spte); 910 if ((u64 *)rmap_head->val != spte) { 911 pr_err("%s: %p 1->BUG\n", __func__, spte); 912 BUG(); 913 } 914 rmap_head->val = 0; 915 } else { 916 rmap_printk("%s: %p many->many\n", __func__, spte); 917 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 918 prev_desc = NULL; 919 while (desc) { 920 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) { 921 if (desc->sptes[i] == spte) { 922 pte_list_desc_remove_entry(rmap_head, 923 desc, i, prev_desc); 924 return; 925 } 926 } 927 prev_desc = desc; 928 desc = desc->more; 929 } 930 pr_err("%s: %p many->many\n", __func__, spte); 931 BUG(); 932 } 933 } 934 935 static void pte_list_remove(struct kvm_rmap_head *rmap_head, u64 *sptep) 936 { 937 mmu_spte_clear_track_bits(sptep); 938 __pte_list_remove(sptep, rmap_head); 939 } 940 941 static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level, 942 struct kvm_memory_slot *slot) 943 { 944 unsigned long idx; 945 946 idx = gfn_to_index(gfn, slot->base_gfn, level); 947 return &slot->arch.rmap[level - PG_LEVEL_4K][idx]; 948 } 949 950 static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, 951 struct kvm_mmu_page *sp) 952 { 953 struct kvm_memslots *slots; 954 struct kvm_memory_slot *slot; 955 956 slots = kvm_memslots_for_spte_role(kvm, sp->role); 957 slot = __gfn_to_memslot(slots, gfn); 958 return __gfn_to_rmap(gfn, sp->role.level, slot); 959 } 960 961 static bool rmap_can_add(struct kvm_vcpu *vcpu) 962 { 963 struct kvm_mmu_memory_cache *mc; 964 965 mc = &vcpu->arch.mmu_pte_list_desc_cache; 966 return kvm_mmu_memory_cache_nr_free_objects(mc); 967 } 968 969 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) 970 { 971 struct kvm_mmu_page *sp; 972 struct kvm_rmap_head *rmap_head; 973 974 sp = sptep_to_sp(spte); 975 kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn); 976 rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp); 977 return pte_list_add(vcpu, spte, rmap_head); 978 } 979 980 static void rmap_remove(struct kvm *kvm, u64 *spte) 981 { 982 struct kvm_mmu_page *sp; 983 gfn_t gfn; 984 struct kvm_rmap_head *rmap_head; 985 986 sp = sptep_to_sp(spte); 987 gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt); 988 rmap_head = gfn_to_rmap(kvm, gfn, sp); 989 __pte_list_remove(spte, rmap_head); 990 } 991 992 /* 993 * Used by the following functions to iterate through the sptes linked by a 994 * rmap. All fields are private and not assumed to be used outside. 995 */ 996 struct rmap_iterator { 997 /* private fields */ 998 struct pte_list_desc *desc; /* holds the sptep if not NULL */ 999 int pos; /* index of the sptep */ 1000 }; 1001 1002 /* 1003 * Iteration must be started by this function. This should also be used after 1004 * removing/dropping sptes from the rmap link because in such cases the 1005 * information in the iterator may not be valid. 1006 * 1007 * Returns sptep if found, NULL otherwise. 1008 */ 1009 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, 1010 struct rmap_iterator *iter) 1011 { 1012 u64 *sptep; 1013 1014 if (!rmap_head->val) 1015 return NULL; 1016 1017 if (!(rmap_head->val & 1)) { 1018 iter->desc = NULL; 1019 sptep = (u64 *)rmap_head->val; 1020 goto out; 1021 } 1022 1023 iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 1024 iter->pos = 0; 1025 sptep = iter->desc->sptes[iter->pos]; 1026 out: 1027 BUG_ON(!is_shadow_present_pte(*sptep)); 1028 return sptep; 1029 } 1030 1031 /* 1032 * Must be used with a valid iterator: e.g. after rmap_get_first(). 1033 * 1034 * Returns sptep if found, NULL otherwise. 1035 */ 1036 static u64 *rmap_get_next(struct rmap_iterator *iter) 1037 { 1038 u64 *sptep; 1039 1040 if (iter->desc) { 1041 if (iter->pos < PTE_LIST_EXT - 1) { 1042 ++iter->pos; 1043 sptep = iter->desc->sptes[iter->pos]; 1044 if (sptep) 1045 goto out; 1046 } 1047 1048 iter->desc = iter->desc->more; 1049 1050 if (iter->desc) { 1051 iter->pos = 0; 1052 /* desc->sptes[0] cannot be NULL */ 1053 sptep = iter->desc->sptes[iter->pos]; 1054 goto out; 1055 } 1056 } 1057 1058 return NULL; 1059 out: 1060 BUG_ON(!is_shadow_present_pte(*sptep)); 1061 return sptep; 1062 } 1063 1064 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_) \ 1065 for (_spte_ = rmap_get_first(_rmap_head_, _iter_); \ 1066 _spte_; _spte_ = rmap_get_next(_iter_)) 1067 1068 static void drop_spte(struct kvm *kvm, u64 *sptep) 1069 { 1070 if (mmu_spte_clear_track_bits(sptep)) 1071 rmap_remove(kvm, sptep); 1072 } 1073 1074 1075 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep) 1076 { 1077 if (is_large_pte(*sptep)) { 1078 WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K); 1079 drop_spte(kvm, sptep); 1080 --kvm->stat.lpages; 1081 return true; 1082 } 1083 1084 return false; 1085 } 1086 1087 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) 1088 { 1089 if (__drop_large_spte(vcpu->kvm, sptep)) { 1090 struct kvm_mmu_page *sp = sptep_to_sp(sptep); 1091 1092 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, 1093 KVM_PAGES_PER_HPAGE(sp->role.level)); 1094 } 1095 } 1096 1097 /* 1098 * Write-protect on the specified @sptep, @pt_protect indicates whether 1099 * spte write-protection is caused by protecting shadow page table. 1100 * 1101 * Note: write protection is difference between dirty logging and spte 1102 * protection: 1103 * - for dirty logging, the spte can be set to writable at anytime if 1104 * its dirty bitmap is properly set. 1105 * - for spte protection, the spte can be writable only after unsync-ing 1106 * shadow page. 1107 * 1108 * Return true if tlb need be flushed. 1109 */ 1110 static bool spte_write_protect(u64 *sptep, bool pt_protect) 1111 { 1112 u64 spte = *sptep; 1113 1114 if (!is_writable_pte(spte) && 1115 !(pt_protect && spte_can_locklessly_be_made_writable(spte))) 1116 return false; 1117 1118 rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep); 1119 1120 if (pt_protect) 1121 spte &= ~SPTE_MMU_WRITEABLE; 1122 spte = spte & ~PT_WRITABLE_MASK; 1123 1124 return mmu_spte_update(sptep, spte); 1125 } 1126 1127 static bool __rmap_write_protect(struct kvm *kvm, 1128 struct kvm_rmap_head *rmap_head, 1129 bool pt_protect) 1130 { 1131 u64 *sptep; 1132 struct rmap_iterator iter; 1133 bool flush = false; 1134 1135 for_each_rmap_spte(rmap_head, &iter, sptep) 1136 flush |= spte_write_protect(sptep, pt_protect); 1137 1138 return flush; 1139 } 1140 1141 static bool spte_clear_dirty(u64 *sptep) 1142 { 1143 u64 spte = *sptep; 1144 1145 rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep); 1146 1147 MMU_WARN_ON(!spte_ad_enabled(spte)); 1148 spte &= ~shadow_dirty_mask; 1149 return mmu_spte_update(sptep, spte); 1150 } 1151 1152 static bool spte_wrprot_for_clear_dirty(u64 *sptep) 1153 { 1154 bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT, 1155 (unsigned long *)sptep); 1156 if (was_writable && !spte_ad_enabled(*sptep)) 1157 kvm_set_pfn_dirty(spte_to_pfn(*sptep)); 1158 1159 return was_writable; 1160 } 1161 1162 /* 1163 * Gets the GFN ready for another round of dirty logging by clearing the 1164 * - D bit on ad-enabled SPTEs, and 1165 * - W bit on ad-disabled SPTEs. 1166 * Returns true iff any D or W bits were cleared. 1167 */ 1168 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) 1169 { 1170 u64 *sptep; 1171 struct rmap_iterator iter; 1172 bool flush = false; 1173 1174 for_each_rmap_spte(rmap_head, &iter, sptep) 1175 if (spte_ad_need_write_protect(*sptep)) 1176 flush |= spte_wrprot_for_clear_dirty(sptep); 1177 else 1178 flush |= spte_clear_dirty(sptep); 1179 1180 return flush; 1181 } 1182 1183 static bool spte_set_dirty(u64 *sptep) 1184 { 1185 u64 spte = *sptep; 1186 1187 rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep); 1188 1189 /* 1190 * Similar to the !kvm_x86_ops.slot_disable_log_dirty case, 1191 * do not bother adding back write access to pages marked 1192 * SPTE_AD_WRPROT_ONLY_MASK. 1193 */ 1194 spte |= shadow_dirty_mask; 1195 1196 return mmu_spte_update(sptep, spte); 1197 } 1198 1199 static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) 1200 { 1201 u64 *sptep; 1202 struct rmap_iterator iter; 1203 bool flush = false; 1204 1205 for_each_rmap_spte(rmap_head, &iter, sptep) 1206 if (spte_ad_enabled(*sptep)) 1207 flush |= spte_set_dirty(sptep); 1208 1209 return flush; 1210 } 1211 1212 /** 1213 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages 1214 * @kvm: kvm instance 1215 * @slot: slot to protect 1216 * @gfn_offset: start of the BITS_PER_LONG pages we care about 1217 * @mask: indicates which pages we should protect 1218 * 1219 * Used when we do not need to care about huge page mappings: e.g. during dirty 1220 * logging we do not have any such mappings. 1221 */ 1222 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, 1223 struct kvm_memory_slot *slot, 1224 gfn_t gfn_offset, unsigned long mask) 1225 { 1226 struct kvm_rmap_head *rmap_head; 1227 1228 if (kvm->arch.tdp_mmu_enabled) 1229 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot, 1230 slot->base_gfn + gfn_offset, mask, true); 1231 while (mask) { 1232 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), 1233 PG_LEVEL_4K, slot); 1234 __rmap_write_protect(kvm, rmap_head, false); 1235 1236 /* clear the first set bit */ 1237 mask &= mask - 1; 1238 } 1239 } 1240 1241 /** 1242 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write 1243 * protect the page if the D-bit isn't supported. 1244 * @kvm: kvm instance 1245 * @slot: slot to clear D-bit 1246 * @gfn_offset: start of the BITS_PER_LONG pages we care about 1247 * @mask: indicates which pages we should clear D-bit 1248 * 1249 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap. 1250 */ 1251 void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1252 struct kvm_memory_slot *slot, 1253 gfn_t gfn_offset, unsigned long mask) 1254 { 1255 struct kvm_rmap_head *rmap_head; 1256 1257 if (kvm->arch.tdp_mmu_enabled) 1258 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot, 1259 slot->base_gfn + gfn_offset, mask, false); 1260 while (mask) { 1261 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), 1262 PG_LEVEL_4K, slot); 1263 __rmap_clear_dirty(kvm, rmap_head); 1264 1265 /* clear the first set bit */ 1266 mask &= mask - 1; 1267 } 1268 } 1269 EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked); 1270 1271 /** 1272 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected 1273 * PT level pages. 1274 * 1275 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to 1276 * enable dirty logging for them. 1277 * 1278 * Used when we do not need to care about huge page mappings: e.g. during dirty 1279 * logging we do not have any such mappings. 1280 */ 1281 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 1282 struct kvm_memory_slot *slot, 1283 gfn_t gfn_offset, unsigned long mask) 1284 { 1285 if (kvm_x86_ops.enable_log_dirty_pt_masked) 1286 kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset, 1287 mask); 1288 else 1289 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); 1290 } 1291 1292 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, 1293 struct kvm_memory_slot *slot, u64 gfn) 1294 { 1295 struct kvm_rmap_head *rmap_head; 1296 int i; 1297 bool write_protected = false; 1298 1299 for (i = PG_LEVEL_4K; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { 1300 rmap_head = __gfn_to_rmap(gfn, i, slot); 1301 write_protected |= __rmap_write_protect(kvm, rmap_head, true); 1302 } 1303 1304 if (kvm->arch.tdp_mmu_enabled) 1305 write_protected |= 1306 kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn); 1307 1308 return write_protected; 1309 } 1310 1311 static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) 1312 { 1313 struct kvm_memory_slot *slot; 1314 1315 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1316 return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn); 1317 } 1318 1319 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head) 1320 { 1321 u64 *sptep; 1322 struct rmap_iterator iter; 1323 bool flush = false; 1324 1325 while ((sptep = rmap_get_first(rmap_head, &iter))) { 1326 rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep); 1327 1328 pte_list_remove(rmap_head, sptep); 1329 flush = true; 1330 } 1331 1332 return flush; 1333 } 1334 1335 static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1336 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1337 unsigned long data) 1338 { 1339 return kvm_zap_rmapp(kvm, rmap_head); 1340 } 1341 1342 static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1343 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1344 unsigned long data) 1345 { 1346 u64 *sptep; 1347 struct rmap_iterator iter; 1348 int need_flush = 0; 1349 u64 new_spte; 1350 pte_t *ptep = (pte_t *)data; 1351 kvm_pfn_t new_pfn; 1352 1353 WARN_ON(pte_huge(*ptep)); 1354 new_pfn = pte_pfn(*ptep); 1355 1356 restart: 1357 for_each_rmap_spte(rmap_head, &iter, sptep) { 1358 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n", 1359 sptep, *sptep, gfn, level); 1360 1361 need_flush = 1; 1362 1363 if (pte_write(*ptep)) { 1364 pte_list_remove(rmap_head, sptep); 1365 goto restart; 1366 } else { 1367 new_spte = kvm_mmu_changed_pte_notifier_make_spte( 1368 *sptep, new_pfn); 1369 1370 mmu_spte_clear_track_bits(sptep); 1371 mmu_spte_set(sptep, new_spte); 1372 } 1373 } 1374 1375 if (need_flush && kvm_available_flush_tlb_with_range()) { 1376 kvm_flush_remote_tlbs_with_address(kvm, gfn, 1); 1377 return 0; 1378 } 1379 1380 return need_flush; 1381 } 1382 1383 struct slot_rmap_walk_iterator { 1384 /* input fields. */ 1385 struct kvm_memory_slot *slot; 1386 gfn_t start_gfn; 1387 gfn_t end_gfn; 1388 int start_level; 1389 int end_level; 1390 1391 /* output fields. */ 1392 gfn_t gfn; 1393 struct kvm_rmap_head *rmap; 1394 int level; 1395 1396 /* private field. */ 1397 struct kvm_rmap_head *end_rmap; 1398 }; 1399 1400 static void 1401 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level) 1402 { 1403 iterator->level = level; 1404 iterator->gfn = iterator->start_gfn; 1405 iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot); 1406 iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level, 1407 iterator->slot); 1408 } 1409 1410 static void 1411 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator, 1412 struct kvm_memory_slot *slot, int start_level, 1413 int end_level, gfn_t start_gfn, gfn_t end_gfn) 1414 { 1415 iterator->slot = slot; 1416 iterator->start_level = start_level; 1417 iterator->end_level = end_level; 1418 iterator->start_gfn = start_gfn; 1419 iterator->end_gfn = end_gfn; 1420 1421 rmap_walk_init_level(iterator, iterator->start_level); 1422 } 1423 1424 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator) 1425 { 1426 return !!iterator->rmap; 1427 } 1428 1429 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator) 1430 { 1431 if (++iterator->rmap <= iterator->end_rmap) { 1432 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level)); 1433 return; 1434 } 1435 1436 if (++iterator->level > iterator->end_level) { 1437 iterator->rmap = NULL; 1438 return; 1439 } 1440 1441 rmap_walk_init_level(iterator, iterator->level); 1442 } 1443 1444 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_, \ 1445 _start_gfn, _end_gfn, _iter_) \ 1446 for (slot_rmap_walk_init(_iter_, _slot_, _start_level_, \ 1447 _end_level_, _start_gfn, _end_gfn); \ 1448 slot_rmap_walk_okay(_iter_); \ 1449 slot_rmap_walk_next(_iter_)) 1450 1451 static int kvm_handle_hva_range(struct kvm *kvm, 1452 unsigned long start, 1453 unsigned long end, 1454 unsigned long data, 1455 int (*handler)(struct kvm *kvm, 1456 struct kvm_rmap_head *rmap_head, 1457 struct kvm_memory_slot *slot, 1458 gfn_t gfn, 1459 int level, 1460 unsigned long data)) 1461 { 1462 struct kvm_memslots *slots; 1463 struct kvm_memory_slot *memslot; 1464 struct slot_rmap_walk_iterator iterator; 1465 int ret = 0; 1466 int i; 1467 1468 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 1469 slots = __kvm_memslots(kvm, i); 1470 kvm_for_each_memslot(memslot, slots) { 1471 unsigned long hva_start, hva_end; 1472 gfn_t gfn_start, gfn_end; 1473 1474 hva_start = max(start, memslot->userspace_addr); 1475 hva_end = min(end, memslot->userspace_addr + 1476 (memslot->npages << PAGE_SHIFT)); 1477 if (hva_start >= hva_end) 1478 continue; 1479 /* 1480 * {gfn(page) | page intersects with [hva_start, hva_end)} = 1481 * {gfn_start, gfn_start+1, ..., gfn_end-1}. 1482 */ 1483 gfn_start = hva_to_gfn_memslot(hva_start, memslot); 1484 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); 1485 1486 for_each_slot_rmap_range(memslot, PG_LEVEL_4K, 1487 KVM_MAX_HUGEPAGE_LEVEL, 1488 gfn_start, gfn_end - 1, 1489 &iterator) 1490 ret |= handler(kvm, iterator.rmap, memslot, 1491 iterator.gfn, iterator.level, data); 1492 } 1493 } 1494 1495 return ret; 1496 } 1497 1498 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, 1499 unsigned long data, 1500 int (*handler)(struct kvm *kvm, 1501 struct kvm_rmap_head *rmap_head, 1502 struct kvm_memory_slot *slot, 1503 gfn_t gfn, int level, 1504 unsigned long data)) 1505 { 1506 return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler); 1507 } 1508 1509 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, 1510 unsigned flags) 1511 { 1512 int r; 1513 1514 r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp); 1515 1516 if (kvm->arch.tdp_mmu_enabled) 1517 r |= kvm_tdp_mmu_zap_hva_range(kvm, start, end); 1518 1519 return r; 1520 } 1521 1522 int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) 1523 { 1524 int r; 1525 1526 r = kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp); 1527 1528 if (kvm->arch.tdp_mmu_enabled) 1529 r |= kvm_tdp_mmu_set_spte_hva(kvm, hva, &pte); 1530 1531 return r; 1532 } 1533 1534 static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1535 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1536 unsigned long data) 1537 { 1538 u64 *sptep; 1539 struct rmap_iterator iter; 1540 int young = 0; 1541 1542 for_each_rmap_spte(rmap_head, &iter, sptep) 1543 young |= mmu_spte_age(sptep); 1544 1545 trace_kvm_age_page(gfn, level, slot, young); 1546 return young; 1547 } 1548 1549 static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1550 struct kvm_memory_slot *slot, gfn_t gfn, 1551 int level, unsigned long data) 1552 { 1553 u64 *sptep; 1554 struct rmap_iterator iter; 1555 1556 for_each_rmap_spte(rmap_head, &iter, sptep) 1557 if (is_accessed_spte(*sptep)) 1558 return 1; 1559 return 0; 1560 } 1561 1562 #define RMAP_RECYCLE_THRESHOLD 1000 1563 1564 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) 1565 { 1566 struct kvm_rmap_head *rmap_head; 1567 struct kvm_mmu_page *sp; 1568 1569 sp = sptep_to_sp(spte); 1570 1571 rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp); 1572 1573 kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0); 1574 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, 1575 KVM_PAGES_PER_HPAGE(sp->role.level)); 1576 } 1577 1578 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) 1579 { 1580 int young = false; 1581 1582 young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp); 1583 if (kvm->arch.tdp_mmu_enabled) 1584 young |= kvm_tdp_mmu_age_hva_range(kvm, start, end); 1585 1586 return young; 1587 } 1588 1589 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) 1590 { 1591 int young = false; 1592 1593 young = kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp); 1594 if (kvm->arch.tdp_mmu_enabled) 1595 young |= kvm_tdp_mmu_test_age_hva(kvm, hva); 1596 1597 return young; 1598 } 1599 1600 #ifdef MMU_DEBUG 1601 static int is_empty_shadow_page(u64 *spt) 1602 { 1603 u64 *pos; 1604 u64 *end; 1605 1606 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) 1607 if (is_shadow_present_pte(*pos)) { 1608 printk(KERN_ERR "%s: %p %llx\n", __func__, 1609 pos, *pos); 1610 return 0; 1611 } 1612 return 1; 1613 } 1614 #endif 1615 1616 /* 1617 * This value is the sum of all of the kvm instances's 1618 * kvm->arch.n_used_mmu_pages values. We need a global, 1619 * aggregate version in order to make the slab shrinker 1620 * faster 1621 */ 1622 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, unsigned long nr) 1623 { 1624 kvm->arch.n_used_mmu_pages += nr; 1625 percpu_counter_add(&kvm_total_used_mmu_pages, nr); 1626 } 1627 1628 static void kvm_mmu_free_page(struct kvm_mmu_page *sp) 1629 { 1630 MMU_WARN_ON(!is_empty_shadow_page(sp->spt)); 1631 hlist_del(&sp->hash_link); 1632 list_del(&sp->link); 1633 free_page((unsigned long)sp->spt); 1634 if (!sp->role.direct) 1635 free_page((unsigned long)sp->gfns); 1636 kmem_cache_free(mmu_page_header_cache, sp); 1637 } 1638 1639 static unsigned kvm_page_table_hashfn(gfn_t gfn) 1640 { 1641 return hash_64(gfn, KVM_MMU_HASH_SHIFT); 1642 } 1643 1644 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, 1645 struct kvm_mmu_page *sp, u64 *parent_pte) 1646 { 1647 if (!parent_pte) 1648 return; 1649 1650 pte_list_add(vcpu, parent_pte, &sp->parent_ptes); 1651 } 1652 1653 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, 1654 u64 *parent_pte) 1655 { 1656 __pte_list_remove(parent_pte, &sp->parent_ptes); 1657 } 1658 1659 static void drop_parent_pte(struct kvm_mmu_page *sp, 1660 u64 *parent_pte) 1661 { 1662 mmu_page_remove_parent_pte(sp, parent_pte); 1663 mmu_spte_clear_no_track(parent_pte); 1664 } 1665 1666 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct) 1667 { 1668 struct kvm_mmu_page *sp; 1669 1670 sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 1671 sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 1672 if (!direct) 1673 sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache); 1674 set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 1675 1676 /* 1677 * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages() 1678 * depends on valid pages being added to the head of the list. See 1679 * comments in kvm_zap_obsolete_pages(). 1680 */ 1681 sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen; 1682 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); 1683 kvm_mod_used_mmu_pages(vcpu->kvm, +1); 1684 return sp; 1685 } 1686 1687 static void mark_unsync(u64 *spte); 1688 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp) 1689 { 1690 u64 *sptep; 1691 struct rmap_iterator iter; 1692 1693 for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) { 1694 mark_unsync(sptep); 1695 } 1696 } 1697 1698 static void mark_unsync(u64 *spte) 1699 { 1700 struct kvm_mmu_page *sp; 1701 unsigned int index; 1702 1703 sp = sptep_to_sp(spte); 1704 index = spte - sp->spt; 1705 if (__test_and_set_bit(index, sp->unsync_child_bitmap)) 1706 return; 1707 if (sp->unsync_children++) 1708 return; 1709 kvm_mmu_mark_parents_unsync(sp); 1710 } 1711 1712 static int nonpaging_sync_page(struct kvm_vcpu *vcpu, 1713 struct kvm_mmu_page *sp) 1714 { 1715 return 0; 1716 } 1717 1718 static void nonpaging_update_pte(struct kvm_vcpu *vcpu, 1719 struct kvm_mmu_page *sp, u64 *spte, 1720 const void *pte) 1721 { 1722 WARN_ON(1); 1723 } 1724 1725 #define KVM_PAGE_ARRAY_NR 16 1726 1727 struct kvm_mmu_pages { 1728 struct mmu_page_and_offset { 1729 struct kvm_mmu_page *sp; 1730 unsigned int idx; 1731 } page[KVM_PAGE_ARRAY_NR]; 1732 unsigned int nr; 1733 }; 1734 1735 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp, 1736 int idx) 1737 { 1738 int i; 1739 1740 if (sp->unsync) 1741 for (i=0; i < pvec->nr; i++) 1742 if (pvec->page[i].sp == sp) 1743 return 0; 1744 1745 pvec->page[pvec->nr].sp = sp; 1746 pvec->page[pvec->nr].idx = idx; 1747 pvec->nr++; 1748 return (pvec->nr == KVM_PAGE_ARRAY_NR); 1749 } 1750 1751 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx) 1752 { 1753 --sp->unsync_children; 1754 WARN_ON((int)sp->unsync_children < 0); 1755 __clear_bit(idx, sp->unsync_child_bitmap); 1756 } 1757 1758 static int __mmu_unsync_walk(struct kvm_mmu_page *sp, 1759 struct kvm_mmu_pages *pvec) 1760 { 1761 int i, ret, nr_unsync_leaf = 0; 1762 1763 for_each_set_bit(i, sp->unsync_child_bitmap, 512) { 1764 struct kvm_mmu_page *child; 1765 u64 ent = sp->spt[i]; 1766 1767 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) { 1768 clear_unsync_child_bit(sp, i); 1769 continue; 1770 } 1771 1772 child = to_shadow_page(ent & PT64_BASE_ADDR_MASK); 1773 1774 if (child->unsync_children) { 1775 if (mmu_pages_add(pvec, child, i)) 1776 return -ENOSPC; 1777 1778 ret = __mmu_unsync_walk(child, pvec); 1779 if (!ret) { 1780 clear_unsync_child_bit(sp, i); 1781 continue; 1782 } else if (ret > 0) { 1783 nr_unsync_leaf += ret; 1784 } else 1785 return ret; 1786 } else if (child->unsync) { 1787 nr_unsync_leaf++; 1788 if (mmu_pages_add(pvec, child, i)) 1789 return -ENOSPC; 1790 } else 1791 clear_unsync_child_bit(sp, i); 1792 } 1793 1794 return nr_unsync_leaf; 1795 } 1796 1797 #define INVALID_INDEX (-1) 1798 1799 static int mmu_unsync_walk(struct kvm_mmu_page *sp, 1800 struct kvm_mmu_pages *pvec) 1801 { 1802 pvec->nr = 0; 1803 if (!sp->unsync_children) 1804 return 0; 1805 1806 mmu_pages_add(pvec, sp, INVALID_INDEX); 1807 return __mmu_unsync_walk(sp, pvec); 1808 } 1809 1810 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp) 1811 { 1812 WARN_ON(!sp->unsync); 1813 trace_kvm_mmu_sync_page(sp); 1814 sp->unsync = 0; 1815 --kvm->stat.mmu_unsync; 1816 } 1817 1818 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, 1819 struct list_head *invalid_list); 1820 static void kvm_mmu_commit_zap_page(struct kvm *kvm, 1821 struct list_head *invalid_list); 1822 1823 #define for_each_valid_sp(_kvm, _sp, _list) \ 1824 hlist_for_each_entry(_sp, _list, hash_link) \ 1825 if (is_obsolete_sp((_kvm), (_sp))) { \ 1826 } else 1827 1828 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \ 1829 for_each_valid_sp(_kvm, _sp, \ 1830 &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)]) \ 1831 if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else 1832 1833 static inline bool is_ept_sp(struct kvm_mmu_page *sp) 1834 { 1835 return sp->role.cr0_wp && sp->role.smap_andnot_wp; 1836 } 1837 1838 /* @sp->gfn should be write-protected at the call site */ 1839 static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 1840 struct list_head *invalid_list) 1841 { 1842 if ((!is_ept_sp(sp) && sp->role.gpte_is_8_bytes != !!is_pae(vcpu)) || 1843 vcpu->arch.mmu->sync_page(vcpu, sp) == 0) { 1844 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); 1845 return false; 1846 } 1847 1848 return true; 1849 } 1850 1851 static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, 1852 struct list_head *invalid_list, 1853 bool remote_flush) 1854 { 1855 if (!remote_flush && list_empty(invalid_list)) 1856 return false; 1857 1858 if (!list_empty(invalid_list)) 1859 kvm_mmu_commit_zap_page(kvm, invalid_list); 1860 else 1861 kvm_flush_remote_tlbs(kvm); 1862 return true; 1863 } 1864 1865 static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu, 1866 struct list_head *invalid_list, 1867 bool remote_flush, bool local_flush) 1868 { 1869 if (kvm_mmu_remote_flush_or_zap(vcpu->kvm, invalid_list, remote_flush)) 1870 return; 1871 1872 if (local_flush) 1873 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1874 } 1875 1876 #ifdef CONFIG_KVM_MMU_AUDIT 1877 #include "mmu_audit.c" 1878 #else 1879 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { } 1880 static void mmu_audit_disable(void) { } 1881 #endif 1882 1883 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 1884 { 1885 return sp->role.invalid || 1886 unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); 1887 } 1888 1889 static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 1890 struct list_head *invalid_list) 1891 { 1892 kvm_unlink_unsync_page(vcpu->kvm, sp); 1893 return __kvm_sync_page(vcpu, sp, invalid_list); 1894 } 1895 1896 /* @gfn should be write-protected at the call site */ 1897 static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, 1898 struct list_head *invalid_list) 1899 { 1900 struct kvm_mmu_page *s; 1901 bool ret = false; 1902 1903 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) { 1904 if (!s->unsync) 1905 continue; 1906 1907 WARN_ON(s->role.level != PG_LEVEL_4K); 1908 ret |= kvm_sync_page(vcpu, s, invalid_list); 1909 } 1910 1911 return ret; 1912 } 1913 1914 struct mmu_page_path { 1915 struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL]; 1916 unsigned int idx[PT64_ROOT_MAX_LEVEL]; 1917 }; 1918 1919 #define for_each_sp(pvec, sp, parents, i) \ 1920 for (i = mmu_pages_first(&pvec, &parents); \ 1921 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \ 1922 i = mmu_pages_next(&pvec, &parents, i)) 1923 1924 static int mmu_pages_next(struct kvm_mmu_pages *pvec, 1925 struct mmu_page_path *parents, 1926 int i) 1927 { 1928 int n; 1929 1930 for (n = i+1; n < pvec->nr; n++) { 1931 struct kvm_mmu_page *sp = pvec->page[n].sp; 1932 unsigned idx = pvec->page[n].idx; 1933 int level = sp->role.level; 1934 1935 parents->idx[level-1] = idx; 1936 if (level == PG_LEVEL_4K) 1937 break; 1938 1939 parents->parent[level-2] = sp; 1940 } 1941 1942 return n; 1943 } 1944 1945 static int mmu_pages_first(struct kvm_mmu_pages *pvec, 1946 struct mmu_page_path *parents) 1947 { 1948 struct kvm_mmu_page *sp; 1949 int level; 1950 1951 if (pvec->nr == 0) 1952 return 0; 1953 1954 WARN_ON(pvec->page[0].idx != INVALID_INDEX); 1955 1956 sp = pvec->page[0].sp; 1957 level = sp->role.level; 1958 WARN_ON(level == PG_LEVEL_4K); 1959 1960 parents->parent[level-2] = sp; 1961 1962 /* Also set up a sentinel. Further entries in pvec are all 1963 * children of sp, so this element is never overwritten. 1964 */ 1965 parents->parent[level-1] = NULL; 1966 return mmu_pages_next(pvec, parents, 0); 1967 } 1968 1969 static void mmu_pages_clear_parents(struct mmu_page_path *parents) 1970 { 1971 struct kvm_mmu_page *sp; 1972 unsigned int level = 0; 1973 1974 do { 1975 unsigned int idx = parents->idx[level]; 1976 sp = parents->parent[level]; 1977 if (!sp) 1978 return; 1979 1980 WARN_ON(idx == INVALID_INDEX); 1981 clear_unsync_child_bit(sp, idx); 1982 level++; 1983 } while (!sp->unsync_children); 1984 } 1985 1986 static void mmu_sync_children(struct kvm_vcpu *vcpu, 1987 struct kvm_mmu_page *parent) 1988 { 1989 int i; 1990 struct kvm_mmu_page *sp; 1991 struct mmu_page_path parents; 1992 struct kvm_mmu_pages pages; 1993 LIST_HEAD(invalid_list); 1994 bool flush = false; 1995 1996 while (mmu_unsync_walk(parent, &pages)) { 1997 bool protected = false; 1998 1999 for_each_sp(pages, sp, parents, i) 2000 protected |= rmap_write_protect(vcpu, sp->gfn); 2001 2002 if (protected) { 2003 kvm_flush_remote_tlbs(vcpu->kvm); 2004 flush = false; 2005 } 2006 2007 for_each_sp(pages, sp, parents, i) { 2008 flush |= kvm_sync_page(vcpu, sp, &invalid_list); 2009 mmu_pages_clear_parents(&parents); 2010 } 2011 if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) { 2012 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2013 cond_resched_lock(&vcpu->kvm->mmu_lock); 2014 flush = false; 2015 } 2016 } 2017 2018 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2019 } 2020 2021 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp) 2022 { 2023 atomic_set(&sp->write_flooding_count, 0); 2024 } 2025 2026 static void clear_sp_write_flooding_count(u64 *spte) 2027 { 2028 __clear_sp_write_flooding_count(sptep_to_sp(spte)); 2029 } 2030 2031 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, 2032 gfn_t gfn, 2033 gva_t gaddr, 2034 unsigned level, 2035 int direct, 2036 unsigned int access) 2037 { 2038 bool direct_mmu = vcpu->arch.mmu->direct_map; 2039 union kvm_mmu_page_role role; 2040 struct hlist_head *sp_list; 2041 unsigned quadrant; 2042 struct kvm_mmu_page *sp; 2043 bool need_sync = false; 2044 bool flush = false; 2045 int collisions = 0; 2046 LIST_HEAD(invalid_list); 2047 2048 role = vcpu->arch.mmu->mmu_role.base; 2049 role.level = level; 2050 role.direct = direct; 2051 if (role.direct) 2052 role.gpte_is_8_bytes = true; 2053 role.access = access; 2054 if (!direct_mmu && vcpu->arch.mmu->root_level <= PT32_ROOT_LEVEL) { 2055 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); 2056 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; 2057 role.quadrant = quadrant; 2058 } 2059 2060 sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]; 2061 for_each_valid_sp(vcpu->kvm, sp, sp_list) { 2062 if (sp->gfn != gfn) { 2063 collisions++; 2064 continue; 2065 } 2066 2067 if (!need_sync && sp->unsync) 2068 need_sync = true; 2069 2070 if (sp->role.word != role.word) 2071 continue; 2072 2073 if (direct_mmu) 2074 goto trace_get_page; 2075 2076 if (sp->unsync) { 2077 /* The page is good, but __kvm_sync_page might still end 2078 * up zapping it. If so, break in order to rebuild it. 2079 */ 2080 if (!__kvm_sync_page(vcpu, sp, &invalid_list)) 2081 break; 2082 2083 WARN_ON(!list_empty(&invalid_list)); 2084 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2085 } 2086 2087 if (sp->unsync_children) 2088 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 2089 2090 __clear_sp_write_flooding_count(sp); 2091 2092 trace_get_page: 2093 trace_kvm_mmu_get_page(sp, false); 2094 goto out; 2095 } 2096 2097 ++vcpu->kvm->stat.mmu_cache_miss; 2098 2099 sp = kvm_mmu_alloc_page(vcpu, direct); 2100 2101 sp->gfn = gfn; 2102 sp->role = role; 2103 hlist_add_head(&sp->hash_link, sp_list); 2104 if (!direct) { 2105 /* 2106 * we should do write protection before syncing pages 2107 * otherwise the content of the synced shadow page may 2108 * be inconsistent with guest page table. 2109 */ 2110 account_shadowed(vcpu->kvm, sp); 2111 if (level == PG_LEVEL_4K && rmap_write_protect(vcpu, gfn)) 2112 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1); 2113 2114 if (level > PG_LEVEL_4K && need_sync) 2115 flush |= kvm_sync_pages(vcpu, gfn, &invalid_list); 2116 } 2117 trace_kvm_mmu_get_page(sp, true); 2118 2119 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2120 out: 2121 if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions) 2122 vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions; 2123 return sp; 2124 } 2125 2126 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator, 2127 struct kvm_vcpu *vcpu, hpa_t root, 2128 u64 addr) 2129 { 2130 iterator->addr = addr; 2131 iterator->shadow_addr = root; 2132 iterator->level = vcpu->arch.mmu->shadow_root_level; 2133 2134 if (iterator->level == PT64_ROOT_4LEVEL && 2135 vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL && 2136 !vcpu->arch.mmu->direct_map) 2137 --iterator->level; 2138 2139 if (iterator->level == PT32E_ROOT_LEVEL) { 2140 /* 2141 * prev_root is currently only used for 64-bit hosts. So only 2142 * the active root_hpa is valid here. 2143 */ 2144 BUG_ON(root != vcpu->arch.mmu->root_hpa); 2145 2146 iterator->shadow_addr 2147 = vcpu->arch.mmu->pae_root[(addr >> 30) & 3]; 2148 iterator->shadow_addr &= PT64_BASE_ADDR_MASK; 2149 --iterator->level; 2150 if (!iterator->shadow_addr) 2151 iterator->level = 0; 2152 } 2153 } 2154 2155 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator, 2156 struct kvm_vcpu *vcpu, u64 addr) 2157 { 2158 shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa, 2159 addr); 2160 } 2161 2162 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator) 2163 { 2164 if (iterator->level < PG_LEVEL_4K) 2165 return false; 2166 2167 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level); 2168 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index; 2169 return true; 2170 } 2171 2172 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator, 2173 u64 spte) 2174 { 2175 if (is_last_spte(spte, iterator->level)) { 2176 iterator->level = 0; 2177 return; 2178 } 2179 2180 iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK; 2181 --iterator->level; 2182 } 2183 2184 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator) 2185 { 2186 __shadow_walk_next(iterator, *iterator->sptep); 2187 } 2188 2189 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep, 2190 struct kvm_mmu_page *sp) 2191 { 2192 u64 spte; 2193 2194 BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK); 2195 2196 spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp)); 2197 2198 mmu_spte_set(sptep, spte); 2199 2200 mmu_page_add_parent_pte(vcpu, sp, sptep); 2201 2202 if (sp->unsync_children || sp->unsync) 2203 mark_unsync(sptep); 2204 } 2205 2206 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2207 unsigned direct_access) 2208 { 2209 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) { 2210 struct kvm_mmu_page *child; 2211 2212 /* 2213 * For the direct sp, if the guest pte's dirty bit 2214 * changed form clean to dirty, it will corrupt the 2215 * sp's access: allow writable in the read-only sp, 2216 * so we should update the spte at this point to get 2217 * a new sp with the correct access. 2218 */ 2219 child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK); 2220 if (child->role.access == direct_access) 2221 return; 2222 2223 drop_parent_pte(child, sptep); 2224 kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1); 2225 } 2226 } 2227 2228 /* Returns the number of zapped non-leaf child shadow pages. */ 2229 static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, 2230 u64 *spte, struct list_head *invalid_list) 2231 { 2232 u64 pte; 2233 struct kvm_mmu_page *child; 2234 2235 pte = *spte; 2236 if (is_shadow_present_pte(pte)) { 2237 if (is_last_spte(pte, sp->role.level)) { 2238 drop_spte(kvm, spte); 2239 if (is_large_pte(pte)) 2240 --kvm->stat.lpages; 2241 } else { 2242 child = to_shadow_page(pte & PT64_BASE_ADDR_MASK); 2243 drop_parent_pte(child, spte); 2244 2245 /* 2246 * Recursively zap nested TDP SPs, parentless SPs are 2247 * unlikely to be used again in the near future. This 2248 * avoids retaining a large number of stale nested SPs. 2249 */ 2250 if (tdp_enabled && invalid_list && 2251 child->role.guest_mode && !child->parent_ptes.val) 2252 return kvm_mmu_prepare_zap_page(kvm, child, 2253 invalid_list); 2254 } 2255 } else if (is_mmio_spte(pte)) { 2256 mmu_spte_clear_no_track(spte); 2257 } 2258 return 0; 2259 } 2260 2261 static int kvm_mmu_page_unlink_children(struct kvm *kvm, 2262 struct kvm_mmu_page *sp, 2263 struct list_head *invalid_list) 2264 { 2265 int zapped = 0; 2266 unsigned i; 2267 2268 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) 2269 zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list); 2270 2271 return zapped; 2272 } 2273 2274 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp) 2275 { 2276 u64 *sptep; 2277 struct rmap_iterator iter; 2278 2279 while ((sptep = rmap_get_first(&sp->parent_ptes, &iter))) 2280 drop_parent_pte(sp, sptep); 2281 } 2282 2283 static int mmu_zap_unsync_children(struct kvm *kvm, 2284 struct kvm_mmu_page *parent, 2285 struct list_head *invalid_list) 2286 { 2287 int i, zapped = 0; 2288 struct mmu_page_path parents; 2289 struct kvm_mmu_pages pages; 2290 2291 if (parent->role.level == PG_LEVEL_4K) 2292 return 0; 2293 2294 while (mmu_unsync_walk(parent, &pages)) { 2295 struct kvm_mmu_page *sp; 2296 2297 for_each_sp(pages, sp, parents, i) { 2298 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 2299 mmu_pages_clear_parents(&parents); 2300 zapped++; 2301 } 2302 } 2303 2304 return zapped; 2305 } 2306 2307 static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, 2308 struct kvm_mmu_page *sp, 2309 struct list_head *invalid_list, 2310 int *nr_zapped) 2311 { 2312 bool list_unstable; 2313 2314 trace_kvm_mmu_prepare_zap_page(sp); 2315 ++kvm->stat.mmu_shadow_zapped; 2316 *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list); 2317 *nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list); 2318 kvm_mmu_unlink_parents(kvm, sp); 2319 2320 /* Zapping children means active_mmu_pages has become unstable. */ 2321 list_unstable = *nr_zapped; 2322 2323 if (!sp->role.invalid && !sp->role.direct) 2324 unaccount_shadowed(kvm, sp); 2325 2326 if (sp->unsync) 2327 kvm_unlink_unsync_page(kvm, sp); 2328 if (!sp->root_count) { 2329 /* Count self */ 2330 (*nr_zapped)++; 2331 2332 /* 2333 * Already invalid pages (previously active roots) are not on 2334 * the active page list. See list_del() in the "else" case of 2335 * !sp->root_count. 2336 */ 2337 if (sp->role.invalid) 2338 list_add(&sp->link, invalid_list); 2339 else 2340 list_move(&sp->link, invalid_list); 2341 kvm_mod_used_mmu_pages(kvm, -1); 2342 } else { 2343 /* 2344 * Remove the active root from the active page list, the root 2345 * will be explicitly freed when the root_count hits zero. 2346 */ 2347 list_del(&sp->link); 2348 2349 /* 2350 * Obsolete pages cannot be used on any vCPUs, see the comment 2351 * in kvm_mmu_zap_all_fast(). Note, is_obsolete_sp() also 2352 * treats invalid shadow pages as being obsolete. 2353 */ 2354 if (!is_obsolete_sp(kvm, sp)) 2355 kvm_reload_remote_mmus(kvm); 2356 } 2357 2358 if (sp->lpage_disallowed) 2359 unaccount_huge_nx_page(kvm, sp); 2360 2361 sp->role.invalid = 1; 2362 return list_unstable; 2363 } 2364 2365 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, 2366 struct list_head *invalid_list) 2367 { 2368 int nr_zapped; 2369 2370 __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped); 2371 return nr_zapped; 2372 } 2373 2374 static void kvm_mmu_commit_zap_page(struct kvm *kvm, 2375 struct list_head *invalid_list) 2376 { 2377 struct kvm_mmu_page *sp, *nsp; 2378 2379 if (list_empty(invalid_list)) 2380 return; 2381 2382 /* 2383 * We need to make sure everyone sees our modifications to 2384 * the page tables and see changes to vcpu->mode here. The barrier 2385 * in the kvm_flush_remote_tlbs() achieves this. This pairs 2386 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end. 2387 * 2388 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit 2389 * guest mode and/or lockless shadow page table walks. 2390 */ 2391 kvm_flush_remote_tlbs(kvm); 2392 2393 list_for_each_entry_safe(sp, nsp, invalid_list, link) { 2394 WARN_ON(!sp->role.invalid || sp->root_count); 2395 kvm_mmu_free_page(sp); 2396 } 2397 } 2398 2399 static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm, 2400 unsigned long nr_to_zap) 2401 { 2402 unsigned long total_zapped = 0; 2403 struct kvm_mmu_page *sp, *tmp; 2404 LIST_HEAD(invalid_list); 2405 bool unstable; 2406 int nr_zapped; 2407 2408 if (list_empty(&kvm->arch.active_mmu_pages)) 2409 return 0; 2410 2411 restart: 2412 list_for_each_entry_safe(sp, tmp, &kvm->arch.active_mmu_pages, link) { 2413 /* 2414 * Don't zap active root pages, the page itself can't be freed 2415 * and zapping it will just force vCPUs to realloc and reload. 2416 */ 2417 if (sp->root_count) 2418 continue; 2419 2420 unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, 2421 &nr_zapped); 2422 total_zapped += nr_zapped; 2423 if (total_zapped >= nr_to_zap) 2424 break; 2425 2426 if (unstable) 2427 goto restart; 2428 } 2429 2430 kvm_mmu_commit_zap_page(kvm, &invalid_list); 2431 2432 kvm->stat.mmu_recycled += total_zapped; 2433 return total_zapped; 2434 } 2435 2436 static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm) 2437 { 2438 if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages) 2439 return kvm->arch.n_max_mmu_pages - 2440 kvm->arch.n_used_mmu_pages; 2441 2442 return 0; 2443 } 2444 2445 static int make_mmu_pages_available(struct kvm_vcpu *vcpu) 2446 { 2447 unsigned long avail = kvm_mmu_available_pages(vcpu->kvm); 2448 2449 if (likely(avail >= KVM_MIN_FREE_MMU_PAGES)) 2450 return 0; 2451 2452 kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail); 2453 2454 if (!kvm_mmu_available_pages(vcpu->kvm)) 2455 return -ENOSPC; 2456 return 0; 2457 } 2458 2459 /* 2460 * Changing the number of mmu pages allocated to the vm 2461 * Note: if goal_nr_mmu_pages is too small, you will get dead lock 2462 */ 2463 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) 2464 { 2465 spin_lock(&kvm->mmu_lock); 2466 2467 if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) { 2468 kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages - 2469 goal_nr_mmu_pages); 2470 2471 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages; 2472 } 2473 2474 kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages; 2475 2476 spin_unlock(&kvm->mmu_lock); 2477 } 2478 2479 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) 2480 { 2481 struct kvm_mmu_page *sp; 2482 LIST_HEAD(invalid_list); 2483 int r; 2484 2485 pgprintk("%s: looking for gfn %llx\n", __func__, gfn); 2486 r = 0; 2487 spin_lock(&kvm->mmu_lock); 2488 for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { 2489 pgprintk("%s: gfn %llx role %x\n", __func__, gfn, 2490 sp->role.word); 2491 r = 1; 2492 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); 2493 } 2494 kvm_mmu_commit_zap_page(kvm, &invalid_list); 2495 spin_unlock(&kvm->mmu_lock); 2496 2497 return r; 2498 } 2499 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page); 2500 2501 static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) 2502 { 2503 trace_kvm_mmu_unsync_page(sp); 2504 ++vcpu->kvm->stat.mmu_unsync; 2505 sp->unsync = 1; 2506 2507 kvm_mmu_mark_parents_unsync(sp); 2508 } 2509 2510 bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, 2511 bool can_unsync) 2512 { 2513 struct kvm_mmu_page *sp; 2514 2515 if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) 2516 return true; 2517 2518 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { 2519 if (!can_unsync) 2520 return true; 2521 2522 if (sp->unsync) 2523 continue; 2524 2525 WARN_ON(sp->role.level != PG_LEVEL_4K); 2526 kvm_unsync_page(vcpu, sp); 2527 } 2528 2529 /* 2530 * We need to ensure that the marking of unsync pages is visible 2531 * before the SPTE is updated to allow writes because 2532 * kvm_mmu_sync_roots() checks the unsync flags without holding 2533 * the MMU lock and so can race with this. If the SPTE was updated 2534 * before the page had been marked as unsync-ed, something like the 2535 * following could happen: 2536 * 2537 * CPU 1 CPU 2 2538 * --------------------------------------------------------------------- 2539 * 1.2 Host updates SPTE 2540 * to be writable 2541 * 2.1 Guest writes a GPTE for GVA X. 2542 * (GPTE being in the guest page table shadowed 2543 * by the SP from CPU 1.) 2544 * This reads SPTE during the page table walk. 2545 * Since SPTE.W is read as 1, there is no 2546 * fault. 2547 * 2548 * 2.2 Guest issues TLB flush. 2549 * That causes a VM Exit. 2550 * 2551 * 2.3 kvm_mmu_sync_pages() reads sp->unsync. 2552 * Since it is false, so it just returns. 2553 * 2554 * 2.4 Guest accesses GVA X. 2555 * Since the mapping in the SP was not updated, 2556 * so the old mapping for GVA X incorrectly 2557 * gets used. 2558 * 1.1 Host marks SP 2559 * as unsync 2560 * (sp->unsync = true) 2561 * 2562 * The write barrier below ensures that 1.1 happens before 1.2 and thus 2563 * the situation in 2.4 does not arise. The implicit barrier in 2.2 2564 * pairs with this write barrier. 2565 */ 2566 smp_wmb(); 2567 2568 return false; 2569 } 2570 2571 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2572 unsigned int pte_access, int level, 2573 gfn_t gfn, kvm_pfn_t pfn, bool speculative, 2574 bool can_unsync, bool host_writable) 2575 { 2576 u64 spte; 2577 struct kvm_mmu_page *sp; 2578 int ret; 2579 2580 if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access)) 2581 return 0; 2582 2583 sp = sptep_to_sp(sptep); 2584 2585 ret = make_spte(vcpu, pte_access, level, gfn, pfn, *sptep, speculative, 2586 can_unsync, host_writable, sp_ad_disabled(sp), &spte); 2587 2588 if (spte & PT_WRITABLE_MASK) 2589 kvm_vcpu_mark_page_dirty(vcpu, gfn); 2590 2591 if (*sptep == spte) 2592 ret |= SET_SPTE_SPURIOUS; 2593 else if (mmu_spte_update(sptep, spte)) 2594 ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH; 2595 return ret; 2596 } 2597 2598 static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2599 unsigned int pte_access, bool write_fault, int level, 2600 gfn_t gfn, kvm_pfn_t pfn, bool speculative, 2601 bool host_writable) 2602 { 2603 int was_rmapped = 0; 2604 int rmap_count; 2605 int set_spte_ret; 2606 int ret = RET_PF_FIXED; 2607 bool flush = false; 2608 2609 pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__, 2610 *sptep, write_fault, gfn); 2611 2612 if (is_shadow_present_pte(*sptep)) { 2613 /* 2614 * If we overwrite a PTE page pointer with a 2MB PMD, unlink 2615 * the parent of the now unreachable PTE. 2616 */ 2617 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) { 2618 struct kvm_mmu_page *child; 2619 u64 pte = *sptep; 2620 2621 child = to_shadow_page(pte & PT64_BASE_ADDR_MASK); 2622 drop_parent_pte(child, sptep); 2623 flush = true; 2624 } else if (pfn != spte_to_pfn(*sptep)) { 2625 pgprintk("hfn old %llx new %llx\n", 2626 spte_to_pfn(*sptep), pfn); 2627 drop_spte(vcpu->kvm, sptep); 2628 flush = true; 2629 } else 2630 was_rmapped = 1; 2631 } 2632 2633 set_spte_ret = set_spte(vcpu, sptep, pte_access, level, gfn, pfn, 2634 speculative, true, host_writable); 2635 if (set_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { 2636 if (write_fault) 2637 ret = RET_PF_EMULATE; 2638 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2639 } 2640 2641 if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH || flush) 2642 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 2643 KVM_PAGES_PER_HPAGE(level)); 2644 2645 if (unlikely(is_mmio_spte(*sptep))) 2646 ret = RET_PF_EMULATE; 2647 2648 /* 2649 * The fault is fully spurious if and only if the new SPTE and old SPTE 2650 * are identical, and emulation is not required. 2651 */ 2652 if ((set_spte_ret & SET_SPTE_SPURIOUS) && ret == RET_PF_FIXED) { 2653 WARN_ON_ONCE(!was_rmapped); 2654 return RET_PF_SPURIOUS; 2655 } 2656 2657 pgprintk("%s: setting spte %llx\n", __func__, *sptep); 2658 trace_kvm_mmu_set_spte(level, gfn, sptep); 2659 if (!was_rmapped && is_large_pte(*sptep)) 2660 ++vcpu->kvm->stat.lpages; 2661 2662 if (is_shadow_present_pte(*sptep)) { 2663 if (!was_rmapped) { 2664 rmap_count = rmap_add(vcpu, sptep, gfn); 2665 if (rmap_count > RMAP_RECYCLE_THRESHOLD) 2666 rmap_recycle(vcpu, sptep, gfn); 2667 } 2668 } 2669 2670 return ret; 2671 } 2672 2673 static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, 2674 bool no_dirty_log) 2675 { 2676 struct kvm_memory_slot *slot; 2677 2678 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); 2679 if (!slot) 2680 return KVM_PFN_ERR_FAULT; 2681 2682 return gfn_to_pfn_memslot_atomic(slot, gfn); 2683 } 2684 2685 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, 2686 struct kvm_mmu_page *sp, 2687 u64 *start, u64 *end) 2688 { 2689 struct page *pages[PTE_PREFETCH_NUM]; 2690 struct kvm_memory_slot *slot; 2691 unsigned int access = sp->role.access; 2692 int i, ret; 2693 gfn_t gfn; 2694 2695 gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt); 2696 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK); 2697 if (!slot) 2698 return -1; 2699 2700 ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start); 2701 if (ret <= 0) 2702 return -1; 2703 2704 for (i = 0; i < ret; i++, gfn++, start++) { 2705 mmu_set_spte(vcpu, start, access, false, sp->role.level, gfn, 2706 page_to_pfn(pages[i]), true, true); 2707 put_page(pages[i]); 2708 } 2709 2710 return 0; 2711 } 2712 2713 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu, 2714 struct kvm_mmu_page *sp, u64 *sptep) 2715 { 2716 u64 *spte, *start = NULL; 2717 int i; 2718 2719 WARN_ON(!sp->role.direct); 2720 2721 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1); 2722 spte = sp->spt + i; 2723 2724 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) { 2725 if (is_shadow_present_pte(*spte) || spte == sptep) { 2726 if (!start) 2727 continue; 2728 if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0) 2729 break; 2730 start = NULL; 2731 } else if (!start) 2732 start = spte; 2733 } 2734 } 2735 2736 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) 2737 { 2738 struct kvm_mmu_page *sp; 2739 2740 sp = sptep_to_sp(sptep); 2741 2742 /* 2743 * Without accessed bits, there's no way to distinguish between 2744 * actually accessed translations and prefetched, so disable pte 2745 * prefetch if accessed bits aren't available. 2746 */ 2747 if (sp_ad_disabled(sp)) 2748 return; 2749 2750 if (sp->role.level > PG_LEVEL_4K) 2751 return; 2752 2753 __direct_pte_prefetch(vcpu, sp, sptep); 2754 } 2755 2756 static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, 2757 kvm_pfn_t pfn, struct kvm_memory_slot *slot) 2758 { 2759 unsigned long hva; 2760 pte_t *pte; 2761 int level; 2762 2763 if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn)) 2764 return PG_LEVEL_4K; 2765 2766 /* 2767 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot() 2768 * is not solely for performance, it's also necessary to avoid the 2769 * "writable" check in __gfn_to_hva_many(), which will always fail on 2770 * read-only memslots due to gfn_to_hva() assuming writes. Earlier 2771 * page fault steps have already verified the guest isn't writing a 2772 * read-only memslot. 2773 */ 2774 hva = __gfn_to_hva_memslot(slot, gfn); 2775 2776 pte = lookup_address_in_mm(vcpu->kvm->mm, hva, &level); 2777 if (unlikely(!pte)) 2778 return PG_LEVEL_4K; 2779 2780 return level; 2781 } 2782 2783 int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, 2784 int max_level, kvm_pfn_t *pfnp, 2785 bool huge_page_disallowed, int *req_level) 2786 { 2787 struct kvm_memory_slot *slot; 2788 struct kvm_lpage_info *linfo; 2789 kvm_pfn_t pfn = *pfnp; 2790 kvm_pfn_t mask; 2791 int level; 2792 2793 *req_level = PG_LEVEL_4K; 2794 2795 if (unlikely(max_level == PG_LEVEL_4K)) 2796 return PG_LEVEL_4K; 2797 2798 if (is_error_noslot_pfn(pfn) || kvm_is_reserved_pfn(pfn)) 2799 return PG_LEVEL_4K; 2800 2801 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, true); 2802 if (!slot) 2803 return PG_LEVEL_4K; 2804 2805 max_level = min(max_level, max_huge_page_level); 2806 for ( ; max_level > PG_LEVEL_4K; max_level--) { 2807 linfo = lpage_info_slot(gfn, slot, max_level); 2808 if (!linfo->disallow_lpage) 2809 break; 2810 } 2811 2812 if (max_level == PG_LEVEL_4K) 2813 return PG_LEVEL_4K; 2814 2815 level = host_pfn_mapping_level(vcpu, gfn, pfn, slot); 2816 if (level == PG_LEVEL_4K) 2817 return level; 2818 2819 *req_level = level = min(level, max_level); 2820 2821 /* 2822 * Enforce the iTLB multihit workaround after capturing the requested 2823 * level, which will be used to do precise, accurate accounting. 2824 */ 2825 if (huge_page_disallowed) 2826 return PG_LEVEL_4K; 2827 2828 /* 2829 * mmu_notifier_retry() was successful and mmu_lock is held, so 2830 * the pmd can't be split from under us. 2831 */ 2832 mask = KVM_PAGES_PER_HPAGE(level) - 1; 2833 VM_BUG_ON((gfn & mask) != (pfn & mask)); 2834 *pfnp = pfn & ~mask; 2835 2836 return level; 2837 } 2838 2839 void disallowed_hugepage_adjust(u64 spte, gfn_t gfn, int cur_level, 2840 kvm_pfn_t *pfnp, int *goal_levelp) 2841 { 2842 int level = *goal_levelp; 2843 2844 if (cur_level == level && level > PG_LEVEL_4K && 2845 is_shadow_present_pte(spte) && 2846 !is_large_pte(spte)) { 2847 /* 2848 * A small SPTE exists for this pfn, but FNAME(fetch) 2849 * and __direct_map would like to create a large PTE 2850 * instead: just force them to go down another level, 2851 * patching back for them into pfn the next 9 bits of 2852 * the address. 2853 */ 2854 u64 page_mask = KVM_PAGES_PER_HPAGE(level) - 2855 KVM_PAGES_PER_HPAGE(level - 1); 2856 *pfnp |= gfn & page_mask; 2857 (*goal_levelp)--; 2858 } 2859 } 2860 2861 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 2862 int map_writable, int max_level, kvm_pfn_t pfn, 2863 bool prefault, bool is_tdp) 2864 { 2865 bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(); 2866 bool write = error_code & PFERR_WRITE_MASK; 2867 bool exec = error_code & PFERR_FETCH_MASK; 2868 bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled; 2869 struct kvm_shadow_walk_iterator it; 2870 struct kvm_mmu_page *sp; 2871 int level, req_level, ret; 2872 gfn_t gfn = gpa >> PAGE_SHIFT; 2873 gfn_t base_gfn = gfn; 2874 2875 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 2876 return RET_PF_RETRY; 2877 2878 level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, 2879 huge_page_disallowed, &req_level); 2880 2881 trace_kvm_mmu_spte_requested(gpa, level, pfn); 2882 for_each_shadow_entry(vcpu, gpa, it) { 2883 /* 2884 * We cannot overwrite existing page tables with an NX 2885 * large page, as the leaf could be executable. 2886 */ 2887 if (nx_huge_page_workaround_enabled) 2888 disallowed_hugepage_adjust(*it.sptep, gfn, it.level, 2889 &pfn, &level); 2890 2891 base_gfn = gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); 2892 if (it.level == level) 2893 break; 2894 2895 drop_large_spte(vcpu, it.sptep); 2896 if (!is_shadow_present_pte(*it.sptep)) { 2897 sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr, 2898 it.level - 1, true, ACC_ALL); 2899 2900 link_shadow_page(vcpu, it.sptep, sp); 2901 if (is_tdp && huge_page_disallowed && 2902 req_level >= it.level) 2903 account_huge_nx_page(vcpu->kvm, sp); 2904 } 2905 } 2906 2907 ret = mmu_set_spte(vcpu, it.sptep, ACC_ALL, 2908 write, level, base_gfn, pfn, prefault, 2909 map_writable); 2910 if (ret == RET_PF_SPURIOUS) 2911 return ret; 2912 2913 direct_pte_prefetch(vcpu, it.sptep); 2914 ++vcpu->stat.pf_fixed; 2915 return ret; 2916 } 2917 2918 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk) 2919 { 2920 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk); 2921 } 2922 2923 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) 2924 { 2925 /* 2926 * Do not cache the mmio info caused by writing the readonly gfn 2927 * into the spte otherwise read access on readonly gfn also can 2928 * caused mmio page fault and treat it as mmio access. 2929 */ 2930 if (pfn == KVM_PFN_ERR_RO_FAULT) 2931 return RET_PF_EMULATE; 2932 2933 if (pfn == KVM_PFN_ERR_HWPOISON) { 2934 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current); 2935 return RET_PF_RETRY; 2936 } 2937 2938 return -EFAULT; 2939 } 2940 2941 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, 2942 kvm_pfn_t pfn, unsigned int access, 2943 int *ret_val) 2944 { 2945 /* The pfn is invalid, report the error! */ 2946 if (unlikely(is_error_pfn(pfn))) { 2947 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn); 2948 return true; 2949 } 2950 2951 if (unlikely(is_noslot_pfn(pfn))) 2952 vcpu_cache_mmio_info(vcpu, gva, gfn, 2953 access & shadow_mmio_access_mask); 2954 2955 return false; 2956 } 2957 2958 static bool page_fault_can_be_fast(u32 error_code) 2959 { 2960 /* 2961 * Do not fix the mmio spte with invalid generation number which 2962 * need to be updated by slow page fault path. 2963 */ 2964 if (unlikely(error_code & PFERR_RSVD_MASK)) 2965 return false; 2966 2967 /* See if the page fault is due to an NX violation */ 2968 if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)) 2969 == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)))) 2970 return false; 2971 2972 /* 2973 * #PF can be fast if: 2974 * 1. The shadow page table entry is not present, which could mean that 2975 * the fault is potentially caused by access tracking (if enabled). 2976 * 2. The shadow page table entry is present and the fault 2977 * is caused by write-protect, that means we just need change the W 2978 * bit of the spte which can be done out of mmu-lock. 2979 * 2980 * However, if access tracking is disabled we know that a non-present 2981 * page must be a genuine page fault where we have to create a new SPTE. 2982 * So, if access tracking is disabled, we return true only for write 2983 * accesses to a present page. 2984 */ 2985 2986 return shadow_acc_track_mask != 0 || 2987 ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)) 2988 == (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)); 2989 } 2990 2991 /* 2992 * Returns true if the SPTE was fixed successfully. Otherwise, 2993 * someone else modified the SPTE from its original value. 2994 */ 2995 static bool 2996 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 2997 u64 *sptep, u64 old_spte, u64 new_spte) 2998 { 2999 gfn_t gfn; 3000 3001 WARN_ON(!sp->role.direct); 3002 3003 /* 3004 * Theoretically we could also set dirty bit (and flush TLB) here in 3005 * order to eliminate unnecessary PML logging. See comments in 3006 * set_spte. But fast_page_fault is very unlikely to happen with PML 3007 * enabled, so we do not do this. This might result in the same GPA 3008 * to be logged in PML buffer again when the write really happens, and 3009 * eventually to be called by mark_page_dirty twice. But it's also no 3010 * harm. This also avoids the TLB flush needed after setting dirty bit 3011 * so non-PML cases won't be impacted. 3012 * 3013 * Compare with set_spte where instead shadow_dirty_mask is set. 3014 */ 3015 if (cmpxchg64(sptep, old_spte, new_spte) != old_spte) 3016 return false; 3017 3018 if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) { 3019 /* 3020 * The gfn of direct spte is stable since it is 3021 * calculated by sp->gfn. 3022 */ 3023 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); 3024 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3025 } 3026 3027 return true; 3028 } 3029 3030 static bool is_access_allowed(u32 fault_err_code, u64 spte) 3031 { 3032 if (fault_err_code & PFERR_FETCH_MASK) 3033 return is_executable_pte(spte); 3034 3035 if (fault_err_code & PFERR_WRITE_MASK) 3036 return is_writable_pte(spte); 3037 3038 /* Fault was on Read access */ 3039 return spte & PT_PRESENT_MASK; 3040 } 3041 3042 /* 3043 * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS. 3044 */ 3045 static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 3046 u32 error_code) 3047 { 3048 struct kvm_shadow_walk_iterator iterator; 3049 struct kvm_mmu_page *sp; 3050 int ret = RET_PF_INVALID; 3051 u64 spte = 0ull; 3052 uint retry_count = 0; 3053 3054 if (!page_fault_can_be_fast(error_code)) 3055 return ret; 3056 3057 walk_shadow_page_lockless_begin(vcpu); 3058 3059 do { 3060 u64 new_spte; 3061 3062 for_each_shadow_entry_lockless(vcpu, cr2_or_gpa, iterator, spte) 3063 if (!is_shadow_present_pte(spte)) 3064 break; 3065 3066 sp = sptep_to_sp(iterator.sptep); 3067 if (!is_last_spte(spte, sp->role.level)) 3068 break; 3069 3070 /* 3071 * Check whether the memory access that caused the fault would 3072 * still cause it if it were to be performed right now. If not, 3073 * then this is a spurious fault caused by TLB lazily flushed, 3074 * or some other CPU has already fixed the PTE after the 3075 * current CPU took the fault. 3076 * 3077 * Need not check the access of upper level table entries since 3078 * they are always ACC_ALL. 3079 */ 3080 if (is_access_allowed(error_code, spte)) { 3081 ret = RET_PF_SPURIOUS; 3082 break; 3083 } 3084 3085 new_spte = spte; 3086 3087 if (is_access_track_spte(spte)) 3088 new_spte = restore_acc_track_spte(new_spte); 3089 3090 /* 3091 * Currently, to simplify the code, write-protection can 3092 * be removed in the fast path only if the SPTE was 3093 * write-protected for dirty-logging or access tracking. 3094 */ 3095 if ((error_code & PFERR_WRITE_MASK) && 3096 spte_can_locklessly_be_made_writable(spte)) { 3097 new_spte |= PT_WRITABLE_MASK; 3098 3099 /* 3100 * Do not fix write-permission on the large spte. Since 3101 * we only dirty the first page into the dirty-bitmap in 3102 * fast_pf_fix_direct_spte(), other pages are missed 3103 * if its slot has dirty logging enabled. 3104 * 3105 * Instead, we let the slow page fault path create a 3106 * normal spte to fix the access. 3107 * 3108 * See the comments in kvm_arch_commit_memory_region(). 3109 */ 3110 if (sp->role.level > PG_LEVEL_4K) 3111 break; 3112 } 3113 3114 /* Verify that the fault can be handled in the fast path */ 3115 if (new_spte == spte || 3116 !is_access_allowed(error_code, new_spte)) 3117 break; 3118 3119 /* 3120 * Currently, fast page fault only works for direct mapping 3121 * since the gfn is not stable for indirect shadow page. See 3122 * Documentation/virt/kvm/locking.rst to get more detail. 3123 */ 3124 if (fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte, 3125 new_spte)) { 3126 ret = RET_PF_FIXED; 3127 break; 3128 } 3129 3130 if (++retry_count > 4) { 3131 printk_once(KERN_WARNING 3132 "kvm: Fast #PF retrying more than 4 times.\n"); 3133 break; 3134 } 3135 3136 } while (true); 3137 3138 trace_fast_page_fault(vcpu, cr2_or_gpa, error_code, iterator.sptep, 3139 spte, ret); 3140 walk_shadow_page_lockless_end(vcpu); 3141 3142 return ret; 3143 } 3144 3145 static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa, 3146 struct list_head *invalid_list) 3147 { 3148 struct kvm_mmu_page *sp; 3149 3150 if (!VALID_PAGE(*root_hpa)) 3151 return; 3152 3153 sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK); 3154 3155 if (kvm_mmu_put_root(kvm, sp)) { 3156 if (sp->tdp_mmu_page) 3157 kvm_tdp_mmu_free_root(kvm, sp); 3158 else if (sp->role.invalid) 3159 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 3160 } 3161 3162 *root_hpa = INVALID_PAGE; 3163 } 3164 3165 /* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */ 3166 void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 3167 ulong roots_to_free) 3168 { 3169 struct kvm *kvm = vcpu->kvm; 3170 int i; 3171 LIST_HEAD(invalid_list); 3172 bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT; 3173 3174 BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG); 3175 3176 /* Before acquiring the MMU lock, see if we need to do any real work. */ 3177 if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) { 3178 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 3179 if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) && 3180 VALID_PAGE(mmu->prev_roots[i].hpa)) 3181 break; 3182 3183 if (i == KVM_MMU_NUM_PREV_ROOTS) 3184 return; 3185 } 3186 3187 spin_lock(&kvm->mmu_lock); 3188 3189 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 3190 if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) 3191 mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa, 3192 &invalid_list); 3193 3194 if (free_active_root) { 3195 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && 3196 (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) { 3197 mmu_free_root_page(kvm, &mmu->root_hpa, &invalid_list); 3198 } else { 3199 for (i = 0; i < 4; ++i) 3200 if (mmu->pae_root[i] != 0) 3201 mmu_free_root_page(kvm, 3202 &mmu->pae_root[i], 3203 &invalid_list); 3204 mmu->root_hpa = INVALID_PAGE; 3205 } 3206 mmu->root_pgd = 0; 3207 } 3208 3209 kvm_mmu_commit_zap_page(kvm, &invalid_list); 3210 spin_unlock(&kvm->mmu_lock); 3211 } 3212 EXPORT_SYMBOL_GPL(kvm_mmu_free_roots); 3213 3214 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn) 3215 { 3216 int ret = 0; 3217 3218 if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) { 3219 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3220 ret = 1; 3221 } 3222 3223 return ret; 3224 } 3225 3226 static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, 3227 u8 level, bool direct) 3228 { 3229 struct kvm_mmu_page *sp; 3230 3231 spin_lock(&vcpu->kvm->mmu_lock); 3232 3233 if (make_mmu_pages_available(vcpu)) { 3234 spin_unlock(&vcpu->kvm->mmu_lock); 3235 return INVALID_PAGE; 3236 } 3237 sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL); 3238 ++sp->root_count; 3239 3240 spin_unlock(&vcpu->kvm->mmu_lock); 3241 return __pa(sp->spt); 3242 } 3243 3244 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) 3245 { 3246 u8 shadow_root_level = vcpu->arch.mmu->shadow_root_level; 3247 hpa_t root; 3248 unsigned i; 3249 3250 if (vcpu->kvm->arch.tdp_mmu_enabled) { 3251 root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu); 3252 3253 if (!VALID_PAGE(root)) 3254 return -ENOSPC; 3255 vcpu->arch.mmu->root_hpa = root; 3256 } else if (shadow_root_level >= PT64_ROOT_4LEVEL) { 3257 root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, 3258 true); 3259 3260 if (!VALID_PAGE(root)) 3261 return -ENOSPC; 3262 vcpu->arch.mmu->root_hpa = root; 3263 } else if (shadow_root_level == PT32E_ROOT_LEVEL) { 3264 for (i = 0; i < 4; ++i) { 3265 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); 3266 3267 root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), 3268 i << 30, PT32_ROOT_LEVEL, true); 3269 if (!VALID_PAGE(root)) 3270 return -ENOSPC; 3271 vcpu->arch.mmu->pae_root[i] = root | PT_PRESENT_MASK; 3272 } 3273 vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); 3274 } else 3275 BUG(); 3276 3277 /* root_pgd is ignored for direct MMUs. */ 3278 vcpu->arch.mmu->root_pgd = 0; 3279 3280 return 0; 3281 } 3282 3283 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) 3284 { 3285 u64 pdptr, pm_mask; 3286 gfn_t root_gfn, root_pgd; 3287 hpa_t root; 3288 int i; 3289 3290 root_pgd = vcpu->arch.mmu->get_guest_pgd(vcpu); 3291 root_gfn = root_pgd >> PAGE_SHIFT; 3292 3293 if (mmu_check_root(vcpu, root_gfn)) 3294 return 1; 3295 3296 /* 3297 * Do we shadow a long mode page table? If so we need to 3298 * write-protect the guests page table root. 3299 */ 3300 if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { 3301 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->root_hpa)); 3302 3303 root = mmu_alloc_root(vcpu, root_gfn, 0, 3304 vcpu->arch.mmu->shadow_root_level, false); 3305 if (!VALID_PAGE(root)) 3306 return -ENOSPC; 3307 vcpu->arch.mmu->root_hpa = root; 3308 goto set_root_pgd; 3309 } 3310 3311 /* 3312 * We shadow a 32 bit page table. This may be a legacy 2-level 3313 * or a PAE 3-level page table. In either case we need to be aware that 3314 * the shadow page table may be a PAE or a long mode page table. 3315 */ 3316 pm_mask = PT_PRESENT_MASK; 3317 if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) 3318 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 3319 3320 for (i = 0; i < 4; ++i) { 3321 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); 3322 if (vcpu->arch.mmu->root_level == PT32E_ROOT_LEVEL) { 3323 pdptr = vcpu->arch.mmu->get_pdptr(vcpu, i); 3324 if (!(pdptr & PT_PRESENT_MASK)) { 3325 vcpu->arch.mmu->pae_root[i] = 0; 3326 continue; 3327 } 3328 root_gfn = pdptr >> PAGE_SHIFT; 3329 if (mmu_check_root(vcpu, root_gfn)) 3330 return 1; 3331 } 3332 3333 root = mmu_alloc_root(vcpu, root_gfn, i << 30, 3334 PT32_ROOT_LEVEL, false); 3335 if (!VALID_PAGE(root)) 3336 return -ENOSPC; 3337 vcpu->arch.mmu->pae_root[i] = root | pm_mask; 3338 } 3339 vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); 3340 3341 /* 3342 * If we shadow a 32 bit page table with a long mode page 3343 * table we enter this path. 3344 */ 3345 if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { 3346 if (vcpu->arch.mmu->lm_root == NULL) { 3347 /* 3348 * The additional page necessary for this is only 3349 * allocated on demand. 3350 */ 3351 3352 u64 *lm_root; 3353 3354 lm_root = (void*)get_zeroed_page(GFP_KERNEL_ACCOUNT); 3355 if (lm_root == NULL) 3356 return 1; 3357 3358 lm_root[0] = __pa(vcpu->arch.mmu->pae_root) | pm_mask; 3359 3360 vcpu->arch.mmu->lm_root = lm_root; 3361 } 3362 3363 vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); 3364 } 3365 3366 set_root_pgd: 3367 vcpu->arch.mmu->root_pgd = root_pgd; 3368 3369 return 0; 3370 } 3371 3372 static int mmu_alloc_roots(struct kvm_vcpu *vcpu) 3373 { 3374 if (vcpu->arch.mmu->direct_map) 3375 return mmu_alloc_direct_roots(vcpu); 3376 else 3377 return mmu_alloc_shadow_roots(vcpu); 3378 } 3379 3380 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) 3381 { 3382 int i; 3383 struct kvm_mmu_page *sp; 3384 3385 if (vcpu->arch.mmu->direct_map) 3386 return; 3387 3388 if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) 3389 return; 3390 3391 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); 3392 3393 if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { 3394 hpa_t root = vcpu->arch.mmu->root_hpa; 3395 sp = to_shadow_page(root); 3396 3397 /* 3398 * Even if another CPU was marking the SP as unsync-ed 3399 * simultaneously, any guest page table changes are not 3400 * guaranteed to be visible anyway until this VCPU issues a TLB 3401 * flush strictly after those changes are made. We only need to 3402 * ensure that the other CPU sets these flags before any actual 3403 * changes to the page tables are made. The comments in 3404 * mmu_need_write_protect() describe what could go wrong if this 3405 * requirement isn't satisfied. 3406 */ 3407 if (!smp_load_acquire(&sp->unsync) && 3408 !smp_load_acquire(&sp->unsync_children)) 3409 return; 3410 3411 spin_lock(&vcpu->kvm->mmu_lock); 3412 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); 3413 3414 mmu_sync_children(vcpu, sp); 3415 3416 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); 3417 spin_unlock(&vcpu->kvm->mmu_lock); 3418 return; 3419 } 3420 3421 spin_lock(&vcpu->kvm->mmu_lock); 3422 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); 3423 3424 for (i = 0; i < 4; ++i) { 3425 hpa_t root = vcpu->arch.mmu->pae_root[i]; 3426 3427 if (root && VALID_PAGE(root)) { 3428 root &= PT64_BASE_ADDR_MASK; 3429 sp = to_shadow_page(root); 3430 mmu_sync_children(vcpu, sp); 3431 } 3432 } 3433 3434 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); 3435 spin_unlock(&vcpu->kvm->mmu_lock); 3436 } 3437 EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots); 3438 3439 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr, 3440 u32 access, struct x86_exception *exception) 3441 { 3442 if (exception) 3443 exception->error_code = 0; 3444 return vaddr; 3445 } 3446 3447 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gpa_t vaddr, 3448 u32 access, 3449 struct x86_exception *exception) 3450 { 3451 if (exception) 3452 exception->error_code = 0; 3453 return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception); 3454 } 3455 3456 static bool 3457 __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level) 3458 { 3459 int bit7 = (pte >> 7) & 1; 3460 3461 return pte & rsvd_check->rsvd_bits_mask[bit7][level-1]; 3462 } 3463 3464 static bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check, u64 pte) 3465 { 3466 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f); 3467 } 3468 3469 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct) 3470 { 3471 /* 3472 * A nested guest cannot use the MMIO cache if it is using nested 3473 * page tables, because cr2 is a nGPA while the cache stores GPAs. 3474 */ 3475 if (mmu_is_nested(vcpu)) 3476 return false; 3477 3478 if (direct) 3479 return vcpu_match_mmio_gpa(vcpu, addr); 3480 3481 return vcpu_match_mmio_gva(vcpu, addr); 3482 } 3483 3484 /* 3485 * Return the level of the lowest level SPTE added to sptes. 3486 * That SPTE may be non-present. 3487 */ 3488 static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes) 3489 { 3490 struct kvm_shadow_walk_iterator iterator; 3491 int leaf = vcpu->arch.mmu->root_level; 3492 u64 spte; 3493 3494 3495 walk_shadow_page_lockless_begin(vcpu); 3496 3497 for (shadow_walk_init(&iterator, vcpu, addr); 3498 shadow_walk_okay(&iterator); 3499 __shadow_walk_next(&iterator, spte)) { 3500 leaf = iterator.level; 3501 spte = mmu_spte_get_lockless(iterator.sptep); 3502 3503 sptes[leaf - 1] = spte; 3504 3505 if (!is_shadow_present_pte(spte)) 3506 break; 3507 3508 } 3509 3510 walk_shadow_page_lockless_end(vcpu); 3511 3512 return leaf; 3513 } 3514 3515 /* return true if reserved bit is detected on spte. */ 3516 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) 3517 { 3518 u64 sptes[PT64_ROOT_MAX_LEVEL]; 3519 struct rsvd_bits_validate *rsvd_check; 3520 int root = vcpu->arch.mmu->shadow_root_level; 3521 int leaf; 3522 int level; 3523 bool reserved = false; 3524 3525 if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) { 3526 *sptep = 0ull; 3527 return reserved; 3528 } 3529 3530 if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) 3531 leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes); 3532 else 3533 leaf = get_walk(vcpu, addr, sptes); 3534 3535 rsvd_check = &vcpu->arch.mmu->shadow_zero_check; 3536 3537 for (level = root; level >= leaf; level--) { 3538 if (!is_shadow_present_pte(sptes[level - 1])) 3539 break; 3540 /* 3541 * Use a bitwise-OR instead of a logical-OR to aggregate the 3542 * reserved bit and EPT's invalid memtype/XWR checks to avoid 3543 * adding a Jcc in the loop. 3544 */ 3545 reserved |= __is_bad_mt_xwr(rsvd_check, sptes[level - 1]) | 3546 __is_rsvd_bits_set(rsvd_check, sptes[level - 1], 3547 level); 3548 } 3549 3550 if (reserved) { 3551 pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n", 3552 __func__, addr); 3553 for (level = root; level >= leaf; level--) 3554 pr_err("------ spte 0x%llx level %d.\n", 3555 sptes[level - 1], level); 3556 } 3557 3558 *sptep = sptes[leaf - 1]; 3559 3560 return reserved; 3561 } 3562 3563 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) 3564 { 3565 u64 spte; 3566 bool reserved; 3567 3568 if (mmio_info_in_cache(vcpu, addr, direct)) 3569 return RET_PF_EMULATE; 3570 3571 reserved = get_mmio_spte(vcpu, addr, &spte); 3572 if (WARN_ON(reserved)) 3573 return -EINVAL; 3574 3575 if (is_mmio_spte(spte)) { 3576 gfn_t gfn = get_mmio_spte_gfn(spte); 3577 unsigned int access = get_mmio_spte_access(spte); 3578 3579 if (!check_mmio_spte(vcpu, spte)) 3580 return RET_PF_INVALID; 3581 3582 if (direct) 3583 addr = 0; 3584 3585 trace_handle_mmio_page_fault(addr, gfn, access); 3586 vcpu_cache_mmio_info(vcpu, addr, gfn, access); 3587 return RET_PF_EMULATE; 3588 } 3589 3590 /* 3591 * If the page table is zapped by other cpus, let CPU fault again on 3592 * the address. 3593 */ 3594 return RET_PF_RETRY; 3595 } 3596 3597 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, 3598 u32 error_code, gfn_t gfn) 3599 { 3600 if (unlikely(error_code & PFERR_RSVD_MASK)) 3601 return false; 3602 3603 if (!(error_code & PFERR_PRESENT_MASK) || 3604 !(error_code & PFERR_WRITE_MASK)) 3605 return false; 3606 3607 /* 3608 * guest is writing the page which is write tracked which can 3609 * not be fixed by page fault handler. 3610 */ 3611 if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) 3612 return true; 3613 3614 return false; 3615 } 3616 3617 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr) 3618 { 3619 struct kvm_shadow_walk_iterator iterator; 3620 u64 spte; 3621 3622 walk_shadow_page_lockless_begin(vcpu); 3623 for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) { 3624 clear_sp_write_flooding_count(iterator.sptep); 3625 if (!is_shadow_present_pte(spte)) 3626 break; 3627 } 3628 walk_shadow_page_lockless_end(vcpu); 3629 } 3630 3631 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 3632 gfn_t gfn) 3633 { 3634 struct kvm_arch_async_pf arch; 3635 3636 arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; 3637 arch.gfn = gfn; 3638 arch.direct_map = vcpu->arch.mmu->direct_map; 3639 arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); 3640 3641 return kvm_setup_async_pf(vcpu, cr2_or_gpa, 3642 kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); 3643 } 3644 3645 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, 3646 gpa_t cr2_or_gpa, kvm_pfn_t *pfn, bool write, 3647 bool *writable) 3648 { 3649 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 3650 bool async; 3651 3652 /* Don't expose private memslots to L2. */ 3653 if (is_guest_mode(vcpu) && !kvm_is_visible_memslot(slot)) { 3654 *pfn = KVM_PFN_NOSLOT; 3655 *writable = false; 3656 return false; 3657 } 3658 3659 async = false; 3660 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable); 3661 if (!async) 3662 return false; /* *pfn has correct page already */ 3663 3664 if (!prefault && kvm_can_do_async_pf(vcpu)) { 3665 trace_kvm_try_async_get_page(cr2_or_gpa, gfn); 3666 if (kvm_find_async_pf_gfn(vcpu, gfn)) { 3667 trace_kvm_async_pf_doublefault(cr2_or_gpa, gfn); 3668 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 3669 return true; 3670 } else if (kvm_arch_setup_async_pf(vcpu, cr2_or_gpa, gfn)) 3671 return true; 3672 } 3673 3674 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable); 3675 return false; 3676 } 3677 3678 static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 3679 bool prefault, int max_level, bool is_tdp) 3680 { 3681 bool write = error_code & PFERR_WRITE_MASK; 3682 bool map_writable; 3683 3684 gfn_t gfn = gpa >> PAGE_SHIFT; 3685 unsigned long mmu_seq; 3686 kvm_pfn_t pfn; 3687 int r; 3688 3689 if (page_fault_handle_page_track(vcpu, error_code, gfn)) 3690 return RET_PF_EMULATE; 3691 3692 if (!is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) { 3693 r = fast_page_fault(vcpu, gpa, error_code); 3694 if (r != RET_PF_INVALID) 3695 return r; 3696 } 3697 3698 r = mmu_topup_memory_caches(vcpu, false); 3699 if (r) 3700 return r; 3701 3702 mmu_seq = vcpu->kvm->mmu_notifier_seq; 3703 smp_rmb(); 3704 3705 if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable)) 3706 return RET_PF_RETRY; 3707 3708 if (handle_abnormal_pfn(vcpu, is_tdp ? 0 : gpa, gfn, pfn, ACC_ALL, &r)) 3709 return r; 3710 3711 r = RET_PF_RETRY; 3712 spin_lock(&vcpu->kvm->mmu_lock); 3713 if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) 3714 goto out_unlock; 3715 r = make_mmu_pages_available(vcpu); 3716 if (r) 3717 goto out_unlock; 3718 3719 if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) 3720 r = kvm_tdp_mmu_map(vcpu, gpa, error_code, map_writable, max_level, 3721 pfn, prefault); 3722 else 3723 r = __direct_map(vcpu, gpa, error_code, map_writable, max_level, pfn, 3724 prefault, is_tdp); 3725 3726 out_unlock: 3727 spin_unlock(&vcpu->kvm->mmu_lock); 3728 kvm_release_pfn_clean(pfn); 3729 return r; 3730 } 3731 3732 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, 3733 u32 error_code, bool prefault) 3734 { 3735 pgprintk("%s: gva %lx error %x\n", __func__, gpa, error_code); 3736 3737 /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */ 3738 return direct_page_fault(vcpu, gpa & PAGE_MASK, error_code, prefault, 3739 PG_LEVEL_2M, false); 3740 } 3741 3742 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, 3743 u64 fault_address, char *insn, int insn_len) 3744 { 3745 int r = 1; 3746 u32 flags = vcpu->arch.apf.host_apf_flags; 3747 3748 #ifndef CONFIG_X86_64 3749 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ 3750 if (WARN_ON_ONCE(fault_address >> 32)) 3751 return -EFAULT; 3752 #endif 3753 3754 vcpu->arch.l1tf_flush_l1d = true; 3755 if (!flags) { 3756 trace_kvm_page_fault(fault_address, error_code); 3757 3758 if (kvm_event_needs_reinjection(vcpu)) 3759 kvm_mmu_unprotect_page_virt(vcpu, fault_address); 3760 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn, 3761 insn_len); 3762 } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) { 3763 vcpu->arch.apf.host_apf_flags = 0; 3764 local_irq_disable(); 3765 kvm_async_pf_task_wait_schedule(fault_address); 3766 local_irq_enable(); 3767 } else { 3768 WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags); 3769 } 3770 3771 return r; 3772 } 3773 EXPORT_SYMBOL_GPL(kvm_handle_page_fault); 3774 3775 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 3776 bool prefault) 3777 { 3778 int max_level; 3779 3780 for (max_level = KVM_MAX_HUGEPAGE_LEVEL; 3781 max_level > PG_LEVEL_4K; 3782 max_level--) { 3783 int page_num = KVM_PAGES_PER_HPAGE(max_level); 3784 gfn_t base = (gpa >> PAGE_SHIFT) & ~(page_num - 1); 3785 3786 if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num)) 3787 break; 3788 } 3789 3790 return direct_page_fault(vcpu, gpa, error_code, prefault, 3791 max_level, true); 3792 } 3793 3794 static void nonpaging_init_context(struct kvm_vcpu *vcpu, 3795 struct kvm_mmu *context) 3796 { 3797 context->page_fault = nonpaging_page_fault; 3798 context->gva_to_gpa = nonpaging_gva_to_gpa; 3799 context->sync_page = nonpaging_sync_page; 3800 context->invlpg = NULL; 3801 context->update_pte = nonpaging_update_pte; 3802 context->root_level = 0; 3803 context->shadow_root_level = PT32E_ROOT_LEVEL; 3804 context->direct_map = true; 3805 context->nx = false; 3806 } 3807 3808 static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd, 3809 union kvm_mmu_page_role role) 3810 { 3811 return (role.direct || pgd == root->pgd) && 3812 VALID_PAGE(root->hpa) && to_shadow_page(root->hpa) && 3813 role.word == to_shadow_page(root->hpa)->role.word; 3814 } 3815 3816 /* 3817 * Find out if a previously cached root matching the new pgd/role is available. 3818 * The current root is also inserted into the cache. 3819 * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is 3820 * returned. 3821 * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and 3822 * false is returned. This root should now be freed by the caller. 3823 */ 3824 static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_pgd, 3825 union kvm_mmu_page_role new_role) 3826 { 3827 uint i; 3828 struct kvm_mmu_root_info root; 3829 struct kvm_mmu *mmu = vcpu->arch.mmu; 3830 3831 root.pgd = mmu->root_pgd; 3832 root.hpa = mmu->root_hpa; 3833 3834 if (is_root_usable(&root, new_pgd, new_role)) 3835 return true; 3836 3837 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 3838 swap(root, mmu->prev_roots[i]); 3839 3840 if (is_root_usable(&root, new_pgd, new_role)) 3841 break; 3842 } 3843 3844 mmu->root_hpa = root.hpa; 3845 mmu->root_pgd = root.pgd; 3846 3847 return i < KVM_MMU_NUM_PREV_ROOTS; 3848 } 3849 3850 static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd, 3851 union kvm_mmu_page_role new_role) 3852 { 3853 struct kvm_mmu *mmu = vcpu->arch.mmu; 3854 3855 /* 3856 * For now, limit the fast switch to 64-bit hosts+VMs in order to avoid 3857 * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs 3858 * later if necessary. 3859 */ 3860 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && 3861 mmu->root_level >= PT64_ROOT_4LEVEL) 3862 return cached_root_available(vcpu, new_pgd, new_role); 3863 3864 return false; 3865 } 3866 3867 static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, 3868 union kvm_mmu_page_role new_role, 3869 bool skip_tlb_flush, bool skip_mmu_sync) 3870 { 3871 if (!fast_pgd_switch(vcpu, new_pgd, new_role)) { 3872 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); 3873 return; 3874 } 3875 3876 /* 3877 * It's possible that the cached previous root page is obsolete because 3878 * of a change in the MMU generation number. However, changing the 3879 * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will 3880 * free the root set here and allocate a new one. 3881 */ 3882 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); 3883 3884 if (!skip_mmu_sync || force_flush_and_sync_on_reuse) 3885 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 3886 if (!skip_tlb_flush || force_flush_and_sync_on_reuse) 3887 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 3888 3889 /* 3890 * The last MMIO access's GVA and GPA are cached in the VCPU. When 3891 * switching to a new CR3, that GVA->GPA mapping may no longer be 3892 * valid. So clear any cached MMIO info even when we don't need to sync 3893 * the shadow page tables. 3894 */ 3895 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); 3896 3897 /* 3898 * If this is a direct root page, it doesn't have a write flooding 3899 * count. Otherwise, clear the write flooding count. 3900 */ 3901 if (!new_role.direct) 3902 __clear_sp_write_flooding_count( 3903 to_shadow_page(vcpu->arch.mmu->root_hpa)); 3904 } 3905 3906 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, 3907 bool skip_mmu_sync) 3908 { 3909 __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu), 3910 skip_tlb_flush, skip_mmu_sync); 3911 } 3912 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd); 3913 3914 static unsigned long get_cr3(struct kvm_vcpu *vcpu) 3915 { 3916 return kvm_read_cr3(vcpu); 3917 } 3918 3919 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, 3920 unsigned int access, int *nr_present) 3921 { 3922 if (unlikely(is_mmio_spte(*sptep))) { 3923 if (gfn != get_mmio_spte_gfn(*sptep)) { 3924 mmu_spte_clear_no_track(sptep); 3925 return true; 3926 } 3927 3928 (*nr_present)++; 3929 mark_mmio_spte(vcpu, sptep, gfn, access); 3930 return true; 3931 } 3932 3933 return false; 3934 } 3935 3936 static inline bool is_last_gpte(struct kvm_mmu *mmu, 3937 unsigned level, unsigned gpte) 3938 { 3939 /* 3940 * The RHS has bit 7 set iff level < mmu->last_nonleaf_level. 3941 * If it is clear, there are no large pages at this level, so clear 3942 * PT_PAGE_SIZE_MASK in gpte if that is the case. 3943 */ 3944 gpte &= level - mmu->last_nonleaf_level; 3945 3946 /* 3947 * PG_LEVEL_4K always terminates. The RHS has bit 7 set 3948 * iff level <= PG_LEVEL_4K, which for our purpose means 3949 * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then. 3950 */ 3951 gpte |= level - PG_LEVEL_4K - 1; 3952 3953 return gpte & PT_PAGE_SIZE_MASK; 3954 } 3955 3956 #define PTTYPE_EPT 18 /* arbitrary */ 3957 #define PTTYPE PTTYPE_EPT 3958 #include "paging_tmpl.h" 3959 #undef PTTYPE 3960 3961 #define PTTYPE 64 3962 #include "paging_tmpl.h" 3963 #undef PTTYPE 3964 3965 #define PTTYPE 32 3966 #include "paging_tmpl.h" 3967 #undef PTTYPE 3968 3969 static void 3970 __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, 3971 struct rsvd_bits_validate *rsvd_check, 3972 int maxphyaddr, int level, bool nx, bool gbpages, 3973 bool pse, bool amd) 3974 { 3975 u64 exb_bit_rsvd = 0; 3976 u64 gbpages_bit_rsvd = 0; 3977 u64 nonleaf_bit8_rsvd = 0; 3978 3979 rsvd_check->bad_mt_xwr = 0; 3980 3981 if (!nx) 3982 exb_bit_rsvd = rsvd_bits(63, 63); 3983 if (!gbpages) 3984 gbpages_bit_rsvd = rsvd_bits(7, 7); 3985 3986 /* 3987 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for 3988 * leaf entries) on AMD CPUs only. 3989 */ 3990 if (amd) 3991 nonleaf_bit8_rsvd = rsvd_bits(8, 8); 3992 3993 switch (level) { 3994 case PT32_ROOT_LEVEL: 3995 /* no rsvd bits for 2 level 4K page table entries */ 3996 rsvd_check->rsvd_bits_mask[0][1] = 0; 3997 rsvd_check->rsvd_bits_mask[0][0] = 0; 3998 rsvd_check->rsvd_bits_mask[1][0] = 3999 rsvd_check->rsvd_bits_mask[0][0]; 4000 4001 if (!pse) { 4002 rsvd_check->rsvd_bits_mask[1][1] = 0; 4003 break; 4004 } 4005 4006 if (is_cpuid_PSE36()) 4007 /* 36bits PSE 4MB page */ 4008 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21); 4009 else 4010 /* 32 bits PSE 4MB page */ 4011 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21); 4012 break; 4013 case PT32E_ROOT_LEVEL: 4014 rsvd_check->rsvd_bits_mask[0][2] = 4015 rsvd_bits(maxphyaddr, 63) | 4016 rsvd_bits(5, 8) | rsvd_bits(1, 2); /* PDPTE */ 4017 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd | 4018 rsvd_bits(maxphyaddr, 62); /* PDE */ 4019 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd | 4020 rsvd_bits(maxphyaddr, 62); /* PTE */ 4021 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd | 4022 rsvd_bits(maxphyaddr, 62) | 4023 rsvd_bits(13, 20); /* large page */ 4024 rsvd_check->rsvd_bits_mask[1][0] = 4025 rsvd_check->rsvd_bits_mask[0][0]; 4026 break; 4027 case PT64_ROOT_5LEVEL: 4028 rsvd_check->rsvd_bits_mask[0][4] = exb_bit_rsvd | 4029 nonleaf_bit8_rsvd | rsvd_bits(7, 7) | 4030 rsvd_bits(maxphyaddr, 51); 4031 rsvd_check->rsvd_bits_mask[1][4] = 4032 rsvd_check->rsvd_bits_mask[0][4]; 4033 fallthrough; 4034 case PT64_ROOT_4LEVEL: 4035 rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd | 4036 nonleaf_bit8_rsvd | rsvd_bits(7, 7) | 4037 rsvd_bits(maxphyaddr, 51); 4038 rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd | 4039 gbpages_bit_rsvd | 4040 rsvd_bits(maxphyaddr, 51); 4041 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd | 4042 rsvd_bits(maxphyaddr, 51); 4043 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd | 4044 rsvd_bits(maxphyaddr, 51); 4045 rsvd_check->rsvd_bits_mask[1][3] = 4046 rsvd_check->rsvd_bits_mask[0][3]; 4047 rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd | 4048 gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) | 4049 rsvd_bits(13, 29); 4050 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd | 4051 rsvd_bits(maxphyaddr, 51) | 4052 rsvd_bits(13, 20); /* large page */ 4053 rsvd_check->rsvd_bits_mask[1][0] = 4054 rsvd_check->rsvd_bits_mask[0][0]; 4055 break; 4056 } 4057 } 4058 4059 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, 4060 struct kvm_mmu *context) 4061 { 4062 __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check, 4063 cpuid_maxphyaddr(vcpu), context->root_level, 4064 context->nx, 4065 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), 4066 is_pse(vcpu), 4067 guest_cpuid_is_amd_or_hygon(vcpu)); 4068 } 4069 4070 static void 4071 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check, 4072 int maxphyaddr, bool execonly) 4073 { 4074 u64 bad_mt_xwr; 4075 4076 rsvd_check->rsvd_bits_mask[0][4] = 4077 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7); 4078 rsvd_check->rsvd_bits_mask[0][3] = 4079 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7); 4080 rsvd_check->rsvd_bits_mask[0][2] = 4081 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6); 4082 rsvd_check->rsvd_bits_mask[0][1] = 4083 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6); 4084 rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51); 4085 4086 /* large page */ 4087 rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4]; 4088 rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3]; 4089 rsvd_check->rsvd_bits_mask[1][2] = 4090 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29); 4091 rsvd_check->rsvd_bits_mask[1][1] = 4092 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20); 4093 rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0]; 4094 4095 bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */ 4096 bad_mt_xwr |= 0xFFull << (3 * 8); /* bits 3..5 must not be 3 */ 4097 bad_mt_xwr |= 0xFFull << (7 * 8); /* bits 3..5 must not be 7 */ 4098 bad_mt_xwr |= REPEAT_BYTE(1ull << 2); /* bits 0..2 must not be 010 */ 4099 bad_mt_xwr |= REPEAT_BYTE(1ull << 6); /* bits 0..2 must not be 110 */ 4100 if (!execonly) { 4101 /* bits 0..2 must not be 100 unless VMX capabilities allow it */ 4102 bad_mt_xwr |= REPEAT_BYTE(1ull << 4); 4103 } 4104 rsvd_check->bad_mt_xwr = bad_mt_xwr; 4105 } 4106 4107 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu, 4108 struct kvm_mmu *context, bool execonly) 4109 { 4110 __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check, 4111 cpuid_maxphyaddr(vcpu), execonly); 4112 } 4113 4114 /* 4115 * the page table on host is the shadow page table for the page 4116 * table in guest or amd nested guest, its mmu features completely 4117 * follow the features in guest. 4118 */ 4119 void 4120 reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) 4121 { 4122 bool uses_nx = context->nx || 4123 context->mmu_role.base.smep_andnot_wp; 4124 struct rsvd_bits_validate *shadow_zero_check; 4125 int i; 4126 4127 /* 4128 * Passing "true" to the last argument is okay; it adds a check 4129 * on bit 8 of the SPTEs which KVM doesn't use anyway. 4130 */ 4131 shadow_zero_check = &context->shadow_zero_check; 4132 __reset_rsvds_bits_mask(vcpu, shadow_zero_check, 4133 shadow_phys_bits, 4134 context->shadow_root_level, uses_nx, 4135 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), 4136 is_pse(vcpu), true); 4137 4138 if (!shadow_me_mask) 4139 return; 4140 4141 for (i = context->shadow_root_level; --i >= 0;) { 4142 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; 4143 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; 4144 } 4145 4146 } 4147 EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask); 4148 4149 static inline bool boot_cpu_is_amd(void) 4150 { 4151 WARN_ON_ONCE(!tdp_enabled); 4152 return shadow_x_mask == 0; 4153 } 4154 4155 /* 4156 * the direct page table on host, use as much mmu features as 4157 * possible, however, kvm currently does not do execution-protection. 4158 */ 4159 static void 4160 reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, 4161 struct kvm_mmu *context) 4162 { 4163 struct rsvd_bits_validate *shadow_zero_check; 4164 int i; 4165 4166 shadow_zero_check = &context->shadow_zero_check; 4167 4168 if (boot_cpu_is_amd()) 4169 __reset_rsvds_bits_mask(vcpu, shadow_zero_check, 4170 shadow_phys_bits, 4171 context->shadow_root_level, false, 4172 boot_cpu_has(X86_FEATURE_GBPAGES), 4173 true, true); 4174 else 4175 __reset_rsvds_bits_mask_ept(shadow_zero_check, 4176 shadow_phys_bits, 4177 false); 4178 4179 if (!shadow_me_mask) 4180 return; 4181 4182 for (i = context->shadow_root_level; --i >= 0;) { 4183 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; 4184 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; 4185 } 4186 } 4187 4188 /* 4189 * as the comments in reset_shadow_zero_bits_mask() except it 4190 * is the shadow page table for intel nested guest. 4191 */ 4192 static void 4193 reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, 4194 struct kvm_mmu *context, bool execonly) 4195 { 4196 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check, 4197 shadow_phys_bits, execonly); 4198 } 4199 4200 #define BYTE_MASK(access) \ 4201 ((1 & (access) ? 2 : 0) | \ 4202 (2 & (access) ? 4 : 0) | \ 4203 (3 & (access) ? 8 : 0) | \ 4204 (4 & (access) ? 16 : 0) | \ 4205 (5 & (access) ? 32 : 0) | \ 4206 (6 & (access) ? 64 : 0) | \ 4207 (7 & (access) ? 128 : 0)) 4208 4209 4210 static void update_permission_bitmask(struct kvm_vcpu *vcpu, 4211 struct kvm_mmu *mmu, bool ept) 4212 { 4213 unsigned byte; 4214 4215 const u8 x = BYTE_MASK(ACC_EXEC_MASK); 4216 const u8 w = BYTE_MASK(ACC_WRITE_MASK); 4217 const u8 u = BYTE_MASK(ACC_USER_MASK); 4218 4219 bool cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP) != 0; 4220 bool cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP) != 0; 4221 bool cr0_wp = is_write_protection(vcpu); 4222 4223 for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) { 4224 unsigned pfec = byte << 1; 4225 4226 /* 4227 * Each "*f" variable has a 1 bit for each UWX value 4228 * that causes a fault with the given PFEC. 4229 */ 4230 4231 /* Faults from writes to non-writable pages */ 4232 u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0; 4233 /* Faults from user mode accesses to supervisor pages */ 4234 u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0; 4235 /* Faults from fetches of non-executable pages*/ 4236 u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0; 4237 /* Faults from kernel mode fetches of user pages */ 4238 u8 smepf = 0; 4239 /* Faults from kernel mode accesses of user pages */ 4240 u8 smapf = 0; 4241 4242 if (!ept) { 4243 /* Faults from kernel mode accesses to user pages */ 4244 u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u; 4245 4246 /* Not really needed: !nx will cause pte.nx to fault */ 4247 if (!mmu->nx) 4248 ff = 0; 4249 4250 /* Allow supervisor writes if !cr0.wp */ 4251 if (!cr0_wp) 4252 wf = (pfec & PFERR_USER_MASK) ? wf : 0; 4253 4254 /* Disallow supervisor fetches of user code if cr4.smep */ 4255 if (cr4_smep) 4256 smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0; 4257 4258 /* 4259 * SMAP:kernel-mode data accesses from user-mode 4260 * mappings should fault. A fault is considered 4261 * as a SMAP violation if all of the following 4262 * conditions are true: 4263 * - X86_CR4_SMAP is set in CR4 4264 * - A user page is accessed 4265 * - The access is not a fetch 4266 * - Page fault in kernel mode 4267 * - if CPL = 3 or X86_EFLAGS_AC is clear 4268 * 4269 * Here, we cover the first three conditions. 4270 * The fourth is computed dynamically in permission_fault(); 4271 * PFERR_RSVD_MASK bit will be set in PFEC if the access is 4272 * *not* subject to SMAP restrictions. 4273 */ 4274 if (cr4_smap) 4275 smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf; 4276 } 4277 4278 mmu->permissions[byte] = ff | uf | wf | smepf | smapf; 4279 } 4280 } 4281 4282 /* 4283 * PKU is an additional mechanism by which the paging controls access to 4284 * user-mode addresses based on the value in the PKRU register. Protection 4285 * key violations are reported through a bit in the page fault error code. 4286 * Unlike other bits of the error code, the PK bit is not known at the 4287 * call site of e.g. gva_to_gpa; it must be computed directly in 4288 * permission_fault based on two bits of PKRU, on some machine state (CR4, 4289 * CR0, EFER, CPL), and on other bits of the error code and the page tables. 4290 * 4291 * In particular the following conditions come from the error code, the 4292 * page tables and the machine state: 4293 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1 4294 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch) 4295 * - PK is always zero if U=0 in the page tables 4296 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access. 4297 * 4298 * The PKRU bitmask caches the result of these four conditions. The error 4299 * code (minus the P bit) and the page table's U bit form an index into the 4300 * PKRU bitmask. Two bits of the PKRU bitmask are then extracted and ANDed 4301 * with the two bits of the PKRU register corresponding to the protection key. 4302 * For the first three conditions above the bits will be 00, thus masking 4303 * away both AD and WD. For all reads or if the last condition holds, WD 4304 * only will be masked away. 4305 */ 4306 static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 4307 bool ept) 4308 { 4309 unsigned bit; 4310 bool wp; 4311 4312 if (ept) { 4313 mmu->pkru_mask = 0; 4314 return; 4315 } 4316 4317 /* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */ 4318 if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) { 4319 mmu->pkru_mask = 0; 4320 return; 4321 } 4322 4323 wp = is_write_protection(vcpu); 4324 4325 for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) { 4326 unsigned pfec, pkey_bits; 4327 bool check_pkey, check_write, ff, uf, wf, pte_user; 4328 4329 pfec = bit << 1; 4330 ff = pfec & PFERR_FETCH_MASK; 4331 uf = pfec & PFERR_USER_MASK; 4332 wf = pfec & PFERR_WRITE_MASK; 4333 4334 /* PFEC.RSVD is replaced by ACC_USER_MASK. */ 4335 pte_user = pfec & PFERR_RSVD_MASK; 4336 4337 /* 4338 * Only need to check the access which is not an 4339 * instruction fetch and is to a user page. 4340 */ 4341 check_pkey = (!ff && pte_user); 4342 /* 4343 * write access is controlled by PKRU if it is a 4344 * user access or CR0.WP = 1. 4345 */ 4346 check_write = check_pkey && wf && (uf || wp); 4347 4348 /* PKRU.AD stops both read and write access. */ 4349 pkey_bits = !!check_pkey; 4350 /* PKRU.WD stops write access. */ 4351 pkey_bits |= (!!check_write) << 1; 4352 4353 mmu->pkru_mask |= (pkey_bits & 3) << pfec; 4354 } 4355 } 4356 4357 static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) 4358 { 4359 unsigned root_level = mmu->root_level; 4360 4361 mmu->last_nonleaf_level = root_level; 4362 if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu)) 4363 mmu->last_nonleaf_level++; 4364 } 4365 4366 static void paging64_init_context_common(struct kvm_vcpu *vcpu, 4367 struct kvm_mmu *context, 4368 int level) 4369 { 4370 context->nx = is_nx(vcpu); 4371 context->root_level = level; 4372 4373 reset_rsvds_bits_mask(vcpu, context); 4374 update_permission_bitmask(vcpu, context, false); 4375 update_pkru_bitmask(vcpu, context, false); 4376 update_last_nonleaf_level(vcpu, context); 4377 4378 MMU_WARN_ON(!is_pae(vcpu)); 4379 context->page_fault = paging64_page_fault; 4380 context->gva_to_gpa = paging64_gva_to_gpa; 4381 context->sync_page = paging64_sync_page; 4382 context->invlpg = paging64_invlpg; 4383 context->update_pte = paging64_update_pte; 4384 context->shadow_root_level = level; 4385 context->direct_map = false; 4386 } 4387 4388 static void paging64_init_context(struct kvm_vcpu *vcpu, 4389 struct kvm_mmu *context) 4390 { 4391 int root_level = is_la57_mode(vcpu) ? 4392 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4393 4394 paging64_init_context_common(vcpu, context, root_level); 4395 } 4396 4397 static void paging32_init_context(struct kvm_vcpu *vcpu, 4398 struct kvm_mmu *context) 4399 { 4400 context->nx = false; 4401 context->root_level = PT32_ROOT_LEVEL; 4402 4403 reset_rsvds_bits_mask(vcpu, context); 4404 update_permission_bitmask(vcpu, context, false); 4405 update_pkru_bitmask(vcpu, context, false); 4406 update_last_nonleaf_level(vcpu, context); 4407 4408 context->page_fault = paging32_page_fault; 4409 context->gva_to_gpa = paging32_gva_to_gpa; 4410 context->sync_page = paging32_sync_page; 4411 context->invlpg = paging32_invlpg; 4412 context->update_pte = paging32_update_pte; 4413 context->shadow_root_level = PT32E_ROOT_LEVEL; 4414 context->direct_map = false; 4415 } 4416 4417 static void paging32E_init_context(struct kvm_vcpu *vcpu, 4418 struct kvm_mmu *context) 4419 { 4420 paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL); 4421 } 4422 4423 static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu) 4424 { 4425 union kvm_mmu_extended_role ext = {0}; 4426 4427 ext.cr0_pg = !!is_paging(vcpu); 4428 ext.cr4_pae = !!is_pae(vcpu); 4429 ext.cr4_smep = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMEP); 4430 ext.cr4_smap = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMAP); 4431 ext.cr4_pse = !!is_pse(vcpu); 4432 ext.cr4_pke = !!kvm_read_cr4_bits(vcpu, X86_CR4_PKE); 4433 ext.maxphyaddr = cpuid_maxphyaddr(vcpu); 4434 4435 ext.valid = 1; 4436 4437 return ext; 4438 } 4439 4440 static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, 4441 bool base_only) 4442 { 4443 union kvm_mmu_role role = {0}; 4444 4445 role.base.access = ACC_ALL; 4446 role.base.nxe = !!is_nx(vcpu); 4447 role.base.cr0_wp = is_write_protection(vcpu); 4448 role.base.smm = is_smm(vcpu); 4449 role.base.guest_mode = is_guest_mode(vcpu); 4450 4451 if (base_only) 4452 return role; 4453 4454 role.ext = kvm_calc_mmu_role_ext(vcpu); 4455 4456 return role; 4457 } 4458 4459 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) 4460 { 4461 /* Use 5-level TDP if and only if it's useful/necessary. */ 4462 if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) 4463 return 4; 4464 4465 return max_tdp_level; 4466 } 4467 4468 static union kvm_mmu_role 4469 kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) 4470 { 4471 union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); 4472 4473 role.base.ad_disabled = (shadow_accessed_mask == 0); 4474 role.base.level = kvm_mmu_get_tdp_level(vcpu); 4475 role.base.direct = true; 4476 role.base.gpte_is_8_bytes = true; 4477 4478 return role; 4479 } 4480 4481 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) 4482 { 4483 struct kvm_mmu *context = &vcpu->arch.root_mmu; 4484 union kvm_mmu_role new_role = 4485 kvm_calc_tdp_mmu_root_page_role(vcpu, false); 4486 4487 if (new_role.as_u64 == context->mmu_role.as_u64) 4488 return; 4489 4490 context->mmu_role.as_u64 = new_role.as_u64; 4491 context->page_fault = kvm_tdp_page_fault; 4492 context->sync_page = nonpaging_sync_page; 4493 context->invlpg = NULL; 4494 context->update_pte = nonpaging_update_pte; 4495 context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu); 4496 context->direct_map = true; 4497 context->get_guest_pgd = get_cr3; 4498 context->get_pdptr = kvm_pdptr_read; 4499 context->inject_page_fault = kvm_inject_page_fault; 4500 4501 if (!is_paging(vcpu)) { 4502 context->nx = false; 4503 context->gva_to_gpa = nonpaging_gva_to_gpa; 4504 context->root_level = 0; 4505 } else if (is_long_mode(vcpu)) { 4506 context->nx = is_nx(vcpu); 4507 context->root_level = is_la57_mode(vcpu) ? 4508 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4509 reset_rsvds_bits_mask(vcpu, context); 4510 context->gva_to_gpa = paging64_gva_to_gpa; 4511 } else if (is_pae(vcpu)) { 4512 context->nx = is_nx(vcpu); 4513 context->root_level = PT32E_ROOT_LEVEL; 4514 reset_rsvds_bits_mask(vcpu, context); 4515 context->gva_to_gpa = paging64_gva_to_gpa; 4516 } else { 4517 context->nx = false; 4518 context->root_level = PT32_ROOT_LEVEL; 4519 reset_rsvds_bits_mask(vcpu, context); 4520 context->gva_to_gpa = paging32_gva_to_gpa; 4521 } 4522 4523 update_permission_bitmask(vcpu, context, false); 4524 update_pkru_bitmask(vcpu, context, false); 4525 update_last_nonleaf_level(vcpu, context); 4526 reset_tdp_shadow_zero_bits_mask(vcpu, context); 4527 } 4528 4529 static union kvm_mmu_role 4530 kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, bool base_only) 4531 { 4532 union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); 4533 4534 role.base.smep_andnot_wp = role.ext.cr4_smep && 4535 !is_write_protection(vcpu); 4536 role.base.smap_andnot_wp = role.ext.cr4_smap && 4537 !is_write_protection(vcpu); 4538 role.base.gpte_is_8_bytes = !!is_pae(vcpu); 4539 4540 return role; 4541 } 4542 4543 static union kvm_mmu_role 4544 kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) 4545 { 4546 union kvm_mmu_role role = 4547 kvm_calc_shadow_root_page_role_common(vcpu, base_only); 4548 4549 role.base.direct = !is_paging(vcpu); 4550 4551 if (!is_long_mode(vcpu)) 4552 role.base.level = PT32E_ROOT_LEVEL; 4553 else if (is_la57_mode(vcpu)) 4554 role.base.level = PT64_ROOT_5LEVEL; 4555 else 4556 role.base.level = PT64_ROOT_4LEVEL; 4557 4558 return role; 4559 } 4560 4561 static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, 4562 u32 cr0, u32 cr4, u32 efer, 4563 union kvm_mmu_role new_role) 4564 { 4565 if (!(cr0 & X86_CR0_PG)) 4566 nonpaging_init_context(vcpu, context); 4567 else if (efer & EFER_LMA) 4568 paging64_init_context(vcpu, context); 4569 else if (cr4 & X86_CR4_PAE) 4570 paging32E_init_context(vcpu, context); 4571 else 4572 paging32_init_context(vcpu, context); 4573 4574 context->mmu_role.as_u64 = new_role.as_u64; 4575 reset_shadow_zero_bits_mask(vcpu, context); 4576 } 4577 4578 static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer) 4579 { 4580 struct kvm_mmu *context = &vcpu->arch.root_mmu; 4581 union kvm_mmu_role new_role = 4582 kvm_calc_shadow_mmu_root_page_role(vcpu, false); 4583 4584 if (new_role.as_u64 != context->mmu_role.as_u64) 4585 shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); 4586 } 4587 4588 static union kvm_mmu_role 4589 kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu) 4590 { 4591 union kvm_mmu_role role = 4592 kvm_calc_shadow_root_page_role_common(vcpu, false); 4593 4594 role.base.direct = false; 4595 role.base.level = kvm_mmu_get_tdp_level(vcpu); 4596 4597 return role; 4598 } 4599 4600 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer, 4601 gpa_t nested_cr3) 4602 { 4603 struct kvm_mmu *context = &vcpu->arch.guest_mmu; 4604 union kvm_mmu_role new_role = kvm_calc_shadow_npt_root_page_role(vcpu); 4605 4606 context->shadow_root_level = new_role.base.level; 4607 4608 __kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base, false, false); 4609 4610 if (new_role.as_u64 != context->mmu_role.as_u64) 4611 shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); 4612 } 4613 EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); 4614 4615 static union kvm_mmu_role 4616 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, 4617 bool execonly, u8 level) 4618 { 4619 union kvm_mmu_role role = {0}; 4620 4621 /* SMM flag is inherited from root_mmu */ 4622 role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm; 4623 4624 role.base.level = level; 4625 role.base.gpte_is_8_bytes = true; 4626 role.base.direct = false; 4627 role.base.ad_disabled = !accessed_dirty; 4628 role.base.guest_mode = true; 4629 role.base.access = ACC_ALL; 4630 4631 /* 4632 * WP=1 and NOT_WP=1 is an impossible combination, use WP and the 4633 * SMAP variation to denote shadow EPT entries. 4634 */ 4635 role.base.cr0_wp = true; 4636 role.base.smap_andnot_wp = true; 4637 4638 role.ext = kvm_calc_mmu_role_ext(vcpu); 4639 role.ext.execonly = execonly; 4640 4641 return role; 4642 } 4643 4644 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, 4645 bool accessed_dirty, gpa_t new_eptp) 4646 { 4647 struct kvm_mmu *context = &vcpu->arch.guest_mmu; 4648 u8 level = vmx_eptp_page_walk_level(new_eptp); 4649 union kvm_mmu_role new_role = 4650 kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, 4651 execonly, level); 4652 4653 __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base, true, true); 4654 4655 if (new_role.as_u64 == context->mmu_role.as_u64) 4656 return; 4657 4658 context->shadow_root_level = level; 4659 4660 context->nx = true; 4661 context->ept_ad = accessed_dirty; 4662 context->page_fault = ept_page_fault; 4663 context->gva_to_gpa = ept_gva_to_gpa; 4664 context->sync_page = ept_sync_page; 4665 context->invlpg = ept_invlpg; 4666 context->update_pte = ept_update_pte; 4667 context->root_level = level; 4668 context->direct_map = false; 4669 context->mmu_role.as_u64 = new_role.as_u64; 4670 4671 update_permission_bitmask(vcpu, context, true); 4672 update_pkru_bitmask(vcpu, context, true); 4673 update_last_nonleaf_level(vcpu, context); 4674 reset_rsvds_bits_mask_ept(vcpu, context, execonly); 4675 reset_ept_shadow_zero_bits_mask(vcpu, context, execonly); 4676 } 4677 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); 4678 4679 static void init_kvm_softmmu(struct kvm_vcpu *vcpu) 4680 { 4681 struct kvm_mmu *context = &vcpu->arch.root_mmu; 4682 4683 kvm_init_shadow_mmu(vcpu, 4684 kvm_read_cr0_bits(vcpu, X86_CR0_PG), 4685 kvm_read_cr4_bits(vcpu, X86_CR4_PAE), 4686 vcpu->arch.efer); 4687 4688 context->get_guest_pgd = get_cr3; 4689 context->get_pdptr = kvm_pdptr_read; 4690 context->inject_page_fault = kvm_inject_page_fault; 4691 } 4692 4693 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) 4694 { 4695 union kvm_mmu_role new_role = kvm_calc_mmu_role_common(vcpu, false); 4696 struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; 4697 4698 if (new_role.as_u64 == g_context->mmu_role.as_u64) 4699 return; 4700 4701 g_context->mmu_role.as_u64 = new_role.as_u64; 4702 g_context->get_guest_pgd = get_cr3; 4703 g_context->get_pdptr = kvm_pdptr_read; 4704 g_context->inject_page_fault = kvm_inject_page_fault; 4705 4706 /* 4707 * L2 page tables are never shadowed, so there is no need to sync 4708 * SPTEs. 4709 */ 4710 g_context->invlpg = NULL; 4711 4712 /* 4713 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using 4714 * L1's nested page tables (e.g. EPT12). The nested translation 4715 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using 4716 * L2's page tables as the first level of translation and L1's 4717 * nested page tables as the second level of translation. Basically 4718 * the gva_to_gpa functions between mmu and nested_mmu are swapped. 4719 */ 4720 if (!is_paging(vcpu)) { 4721 g_context->nx = false; 4722 g_context->root_level = 0; 4723 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested; 4724 } else if (is_long_mode(vcpu)) { 4725 g_context->nx = is_nx(vcpu); 4726 g_context->root_level = is_la57_mode(vcpu) ? 4727 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4728 reset_rsvds_bits_mask(vcpu, g_context); 4729 g_context->gva_to_gpa = paging64_gva_to_gpa_nested; 4730 } else if (is_pae(vcpu)) { 4731 g_context->nx = is_nx(vcpu); 4732 g_context->root_level = PT32E_ROOT_LEVEL; 4733 reset_rsvds_bits_mask(vcpu, g_context); 4734 g_context->gva_to_gpa = paging64_gva_to_gpa_nested; 4735 } else { 4736 g_context->nx = false; 4737 g_context->root_level = PT32_ROOT_LEVEL; 4738 reset_rsvds_bits_mask(vcpu, g_context); 4739 g_context->gva_to_gpa = paging32_gva_to_gpa_nested; 4740 } 4741 4742 update_permission_bitmask(vcpu, g_context, false); 4743 update_pkru_bitmask(vcpu, g_context, false); 4744 update_last_nonleaf_level(vcpu, g_context); 4745 } 4746 4747 void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots) 4748 { 4749 if (reset_roots) { 4750 uint i; 4751 4752 vcpu->arch.mmu->root_hpa = INVALID_PAGE; 4753 4754 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 4755 vcpu->arch.mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 4756 } 4757 4758 if (mmu_is_nested(vcpu)) 4759 init_kvm_nested_mmu(vcpu); 4760 else if (tdp_enabled) 4761 init_kvm_tdp_mmu(vcpu); 4762 else 4763 init_kvm_softmmu(vcpu); 4764 } 4765 EXPORT_SYMBOL_GPL(kvm_init_mmu); 4766 4767 static union kvm_mmu_page_role 4768 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu) 4769 { 4770 union kvm_mmu_role role; 4771 4772 if (tdp_enabled) 4773 role = kvm_calc_tdp_mmu_root_page_role(vcpu, true); 4774 else 4775 role = kvm_calc_shadow_mmu_root_page_role(vcpu, true); 4776 4777 return role.base; 4778 } 4779 4780 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu) 4781 { 4782 kvm_mmu_unload(vcpu); 4783 kvm_init_mmu(vcpu, true); 4784 } 4785 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context); 4786 4787 int kvm_mmu_load(struct kvm_vcpu *vcpu) 4788 { 4789 int r; 4790 4791 r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->direct_map); 4792 if (r) 4793 goto out; 4794 r = mmu_alloc_roots(vcpu); 4795 kvm_mmu_sync_roots(vcpu); 4796 if (r) 4797 goto out; 4798 kvm_mmu_load_pgd(vcpu); 4799 kvm_x86_ops.tlb_flush_current(vcpu); 4800 out: 4801 return r; 4802 } 4803 EXPORT_SYMBOL_GPL(kvm_mmu_load); 4804 4805 void kvm_mmu_unload(struct kvm_vcpu *vcpu) 4806 { 4807 kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL); 4808 WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa)); 4809 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4810 WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa)); 4811 } 4812 EXPORT_SYMBOL_GPL(kvm_mmu_unload); 4813 4814 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, 4815 struct kvm_mmu_page *sp, u64 *spte, 4816 const void *new) 4817 { 4818 if (sp->role.level != PG_LEVEL_4K) { 4819 ++vcpu->kvm->stat.mmu_pde_zapped; 4820 return; 4821 } 4822 4823 ++vcpu->kvm->stat.mmu_pte_updated; 4824 vcpu->arch.mmu->update_pte(vcpu, sp, spte, new); 4825 } 4826 4827 static bool need_remote_flush(u64 old, u64 new) 4828 { 4829 if (!is_shadow_present_pte(old)) 4830 return false; 4831 if (!is_shadow_present_pte(new)) 4832 return true; 4833 if ((old ^ new) & PT64_BASE_ADDR_MASK) 4834 return true; 4835 old ^= shadow_nx_mask; 4836 new ^= shadow_nx_mask; 4837 return (old & ~new & PT64_PERM_MASK) != 0; 4838 } 4839 4840 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa, 4841 int *bytes) 4842 { 4843 u64 gentry = 0; 4844 int r; 4845 4846 /* 4847 * Assume that the pte write on a page table of the same type 4848 * as the current vcpu paging mode since we update the sptes only 4849 * when they have the same mode. 4850 */ 4851 if (is_pae(vcpu) && *bytes == 4) { 4852 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */ 4853 *gpa &= ~(gpa_t)7; 4854 *bytes = 8; 4855 } 4856 4857 if (*bytes == 4 || *bytes == 8) { 4858 r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes); 4859 if (r) 4860 gentry = 0; 4861 } 4862 4863 return gentry; 4864 } 4865 4866 /* 4867 * If we're seeing too many writes to a page, it may no longer be a page table, 4868 * or we may be forking, in which case it is better to unmap the page. 4869 */ 4870 static bool detect_write_flooding(struct kvm_mmu_page *sp) 4871 { 4872 /* 4873 * Skip write-flooding detected for the sp whose level is 1, because 4874 * it can become unsync, then the guest page is not write-protected. 4875 */ 4876 if (sp->role.level == PG_LEVEL_4K) 4877 return false; 4878 4879 atomic_inc(&sp->write_flooding_count); 4880 return atomic_read(&sp->write_flooding_count) >= 3; 4881 } 4882 4883 /* 4884 * Misaligned accesses are too much trouble to fix up; also, they usually 4885 * indicate a page is not used as a page table. 4886 */ 4887 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa, 4888 int bytes) 4889 { 4890 unsigned offset, pte_size, misaligned; 4891 4892 pgprintk("misaligned: gpa %llx bytes %d role %x\n", 4893 gpa, bytes, sp->role.word); 4894 4895 offset = offset_in_page(gpa); 4896 pte_size = sp->role.gpte_is_8_bytes ? 8 : 4; 4897 4898 /* 4899 * Sometimes, the OS only writes the last one bytes to update status 4900 * bits, for example, in linux, andb instruction is used in clear_bit(). 4901 */ 4902 if (!(offset & (pte_size - 1)) && bytes == 1) 4903 return false; 4904 4905 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); 4906 misaligned |= bytes < 4; 4907 4908 return misaligned; 4909 } 4910 4911 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte) 4912 { 4913 unsigned page_offset, quadrant; 4914 u64 *spte; 4915 int level; 4916 4917 page_offset = offset_in_page(gpa); 4918 level = sp->role.level; 4919 *nspte = 1; 4920 if (!sp->role.gpte_is_8_bytes) { 4921 page_offset <<= 1; /* 32->64 */ 4922 /* 4923 * A 32-bit pde maps 4MB while the shadow pdes map 4924 * only 2MB. So we need to double the offset again 4925 * and zap two pdes instead of one. 4926 */ 4927 if (level == PT32_ROOT_LEVEL) { 4928 page_offset &= ~7; /* kill rounding error */ 4929 page_offset <<= 1; 4930 *nspte = 2; 4931 } 4932 quadrant = page_offset >> PAGE_SHIFT; 4933 page_offset &= ~PAGE_MASK; 4934 if (quadrant != sp->role.quadrant) 4935 return NULL; 4936 } 4937 4938 spte = &sp->spt[page_offset / sizeof(*spte)]; 4939 return spte; 4940 } 4941 4942 /* 4943 * Ignore various flags when determining if a SPTE can be immediately 4944 * overwritten for the current MMU. 4945 * - level: explicitly checked in mmu_pte_write_new_pte(), and will never 4946 * match the current MMU role, as MMU's level tracks the root level. 4947 * - access: updated based on the new guest PTE 4948 * - quadrant: handled by get_written_sptes() 4949 * - invalid: always false (loop only walks valid shadow pages) 4950 */ 4951 static const union kvm_mmu_page_role role_ign = { 4952 .level = 0xf, 4953 .access = 0x7, 4954 .quadrant = 0x3, 4955 .invalid = 0x1, 4956 }; 4957 4958 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 4959 const u8 *new, int bytes, 4960 struct kvm_page_track_notifier_node *node) 4961 { 4962 gfn_t gfn = gpa >> PAGE_SHIFT; 4963 struct kvm_mmu_page *sp; 4964 LIST_HEAD(invalid_list); 4965 u64 entry, gentry, *spte; 4966 int npte; 4967 bool remote_flush, local_flush; 4968 4969 /* 4970 * If we don't have indirect shadow pages, it means no page is 4971 * write-protected, so we can exit simply. 4972 */ 4973 if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages)) 4974 return; 4975 4976 remote_flush = local_flush = false; 4977 4978 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes); 4979 4980 /* 4981 * No need to care whether allocation memory is successful 4982 * or not since pte prefetch is skiped if it does not have 4983 * enough objects in the cache. 4984 */ 4985 mmu_topup_memory_caches(vcpu, true); 4986 4987 spin_lock(&vcpu->kvm->mmu_lock); 4988 4989 gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes); 4990 4991 ++vcpu->kvm->stat.mmu_pte_write; 4992 kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE); 4993 4994 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { 4995 if (detect_write_misaligned(sp, gpa, bytes) || 4996 detect_write_flooding(sp)) { 4997 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list); 4998 ++vcpu->kvm->stat.mmu_flooded; 4999 continue; 5000 } 5001 5002 spte = get_written_sptes(sp, gpa, &npte); 5003 if (!spte) 5004 continue; 5005 5006 local_flush = true; 5007 while (npte--) { 5008 u32 base_role = vcpu->arch.mmu->mmu_role.base.word; 5009 5010 entry = *spte; 5011 mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL); 5012 if (gentry && 5013 !((sp->role.word ^ base_role) & ~role_ign.word) && 5014 rmap_can_add(vcpu)) 5015 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry); 5016 if (need_remote_flush(entry, *spte)) 5017 remote_flush = true; 5018 ++spte; 5019 } 5020 } 5021 kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush); 5022 kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE); 5023 spin_unlock(&vcpu->kvm->mmu_lock); 5024 } 5025 5026 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) 5027 { 5028 gpa_t gpa; 5029 int r; 5030 5031 if (vcpu->arch.mmu->direct_map) 5032 return 0; 5033 5034 gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); 5035 5036 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); 5037 5038 return r; 5039 } 5040 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt); 5041 5042 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, 5043 void *insn, int insn_len) 5044 { 5045 int r, emulation_type = EMULTYPE_PF; 5046 bool direct = vcpu->arch.mmu->direct_map; 5047 5048 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 5049 return RET_PF_RETRY; 5050 5051 r = RET_PF_INVALID; 5052 if (unlikely(error_code & PFERR_RSVD_MASK)) { 5053 r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); 5054 if (r == RET_PF_EMULATE) 5055 goto emulate; 5056 } 5057 5058 if (r == RET_PF_INVALID) { 5059 r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa, 5060 lower_32_bits(error_code), false); 5061 if (WARN_ON_ONCE(r == RET_PF_INVALID)) 5062 return -EIO; 5063 } 5064 5065 if (r < 0) 5066 return r; 5067 if (r != RET_PF_EMULATE) 5068 return 1; 5069 5070 /* 5071 * Before emulating the instruction, check if the error code 5072 * was due to a RO violation while translating the guest page. 5073 * This can occur when using nested virtualization with nested 5074 * paging in both guests. If true, we simply unprotect the page 5075 * and resume the guest. 5076 */ 5077 if (vcpu->arch.mmu->direct_map && 5078 (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) { 5079 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)); 5080 return 1; 5081 } 5082 5083 /* 5084 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still 5085 * optimistically try to just unprotect the page and let the processor 5086 * re-execute the instruction that caused the page fault. Do not allow 5087 * retrying MMIO emulation, as it's not only pointless but could also 5088 * cause us to enter an infinite loop because the processor will keep 5089 * faulting on the non-existent MMIO address. Retrying an instruction 5090 * from a nested guest is also pointless and dangerous as we are only 5091 * explicitly shadowing L1's page tables, i.e. unprotecting something 5092 * for L1 isn't going to magically fix whatever issue cause L2 to fail. 5093 */ 5094 if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu)) 5095 emulation_type |= EMULTYPE_ALLOW_RETRY_PF; 5096 emulate: 5097 return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, 5098 insn_len); 5099 } 5100 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); 5101 5102 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 5103 gva_t gva, hpa_t root_hpa) 5104 { 5105 int i; 5106 5107 /* It's actually a GPA for vcpu->arch.guest_mmu. */ 5108 if (mmu != &vcpu->arch.guest_mmu) { 5109 /* INVLPG on a non-canonical address is a NOP according to the SDM. */ 5110 if (is_noncanonical_address(gva, vcpu)) 5111 return; 5112 5113 kvm_x86_ops.tlb_flush_gva(vcpu, gva); 5114 } 5115 5116 if (!mmu->invlpg) 5117 return; 5118 5119 if (root_hpa == INVALID_PAGE) { 5120 mmu->invlpg(vcpu, gva, mmu->root_hpa); 5121 5122 /* 5123 * INVLPG is required to invalidate any global mappings for the VA, 5124 * irrespective of PCID. Since it would take us roughly similar amount 5125 * of work to determine whether any of the prev_root mappings of the VA 5126 * is marked global, or to just sync it blindly, so we might as well 5127 * just always sync it. 5128 * 5129 * Mappings not reachable via the current cr3 or the prev_roots will be 5130 * synced when switching to that cr3, so nothing needs to be done here 5131 * for them. 5132 */ 5133 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5134 if (VALID_PAGE(mmu->prev_roots[i].hpa)) 5135 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); 5136 } else { 5137 mmu->invlpg(vcpu, gva, root_hpa); 5138 } 5139 } 5140 EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva); 5141 5142 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) 5143 { 5144 kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, gva, INVALID_PAGE); 5145 ++vcpu->stat.invlpg; 5146 } 5147 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); 5148 5149 5150 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) 5151 { 5152 struct kvm_mmu *mmu = vcpu->arch.mmu; 5153 bool tlb_flush = false; 5154 uint i; 5155 5156 if (pcid == kvm_get_active_pcid(vcpu)) { 5157 mmu->invlpg(vcpu, gva, mmu->root_hpa); 5158 tlb_flush = true; 5159 } 5160 5161 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5162 if (VALID_PAGE(mmu->prev_roots[i].hpa) && 5163 pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) { 5164 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); 5165 tlb_flush = true; 5166 } 5167 } 5168 5169 if (tlb_flush) 5170 kvm_x86_ops.tlb_flush_gva(vcpu, gva); 5171 5172 ++vcpu->stat.invlpg; 5173 5174 /* 5175 * Mappings not reachable via the current cr3 or the prev_roots will be 5176 * synced when switching to that cr3, so nothing needs to be done here 5177 * for them. 5178 */ 5179 } 5180 EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); 5181 5182 void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, 5183 int tdp_huge_page_level) 5184 { 5185 tdp_enabled = enable_tdp; 5186 max_tdp_level = tdp_max_root_level; 5187 5188 /* 5189 * max_huge_page_level reflects KVM's MMU capabilities irrespective 5190 * of kernel support, e.g. KVM may be capable of using 1GB pages when 5191 * the kernel is not. But, KVM never creates a page size greater than 5192 * what is used by the kernel for any given HVA, i.e. the kernel's 5193 * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust(). 5194 */ 5195 if (tdp_enabled) 5196 max_huge_page_level = tdp_huge_page_level; 5197 else if (boot_cpu_has(X86_FEATURE_GBPAGES)) 5198 max_huge_page_level = PG_LEVEL_1G; 5199 else 5200 max_huge_page_level = PG_LEVEL_2M; 5201 } 5202 EXPORT_SYMBOL_GPL(kvm_configure_mmu); 5203 5204 /* The return value indicates if tlb flush on all vcpus is needed. */ 5205 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); 5206 5207 /* The caller should hold mmu-lock before calling this function. */ 5208 static __always_inline bool 5209 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, 5210 slot_level_handler fn, int start_level, int end_level, 5211 gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb) 5212 { 5213 struct slot_rmap_walk_iterator iterator; 5214 bool flush = false; 5215 5216 for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, 5217 end_gfn, &iterator) { 5218 if (iterator.rmap) 5219 flush |= fn(kvm, iterator.rmap); 5220 5221 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { 5222 if (flush && lock_flush_tlb) { 5223 kvm_flush_remote_tlbs_with_address(kvm, 5224 start_gfn, 5225 iterator.gfn - start_gfn + 1); 5226 flush = false; 5227 } 5228 cond_resched_lock(&kvm->mmu_lock); 5229 } 5230 } 5231 5232 if (flush && lock_flush_tlb) { 5233 kvm_flush_remote_tlbs_with_address(kvm, start_gfn, 5234 end_gfn - start_gfn + 1); 5235 flush = false; 5236 } 5237 5238 return flush; 5239 } 5240 5241 static __always_inline bool 5242 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5243 slot_level_handler fn, int start_level, int end_level, 5244 bool lock_flush_tlb) 5245 { 5246 return slot_handle_level_range(kvm, memslot, fn, start_level, 5247 end_level, memslot->base_gfn, 5248 memslot->base_gfn + memslot->npages - 1, 5249 lock_flush_tlb); 5250 } 5251 5252 static __always_inline bool 5253 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5254 slot_level_handler fn, bool lock_flush_tlb) 5255 { 5256 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, 5257 KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); 5258 } 5259 5260 static __always_inline bool 5261 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5262 slot_level_handler fn, bool lock_flush_tlb) 5263 { 5264 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K + 1, 5265 KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); 5266 } 5267 5268 static __always_inline bool 5269 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, 5270 slot_level_handler fn, bool lock_flush_tlb) 5271 { 5272 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, 5273 PG_LEVEL_4K, lock_flush_tlb); 5274 } 5275 5276 static void free_mmu_pages(struct kvm_mmu *mmu) 5277 { 5278 free_page((unsigned long)mmu->pae_root); 5279 free_page((unsigned long)mmu->lm_root); 5280 } 5281 5282 static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) 5283 { 5284 struct page *page; 5285 int i; 5286 5287 mmu->root_hpa = INVALID_PAGE; 5288 mmu->root_pgd = 0; 5289 mmu->translate_gpa = translate_gpa; 5290 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5291 mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 5292 5293 /* 5294 * When using PAE paging, the four PDPTEs are treated as 'root' pages, 5295 * while the PDP table is a per-vCPU construct that's allocated at MMU 5296 * creation. When emulating 32-bit mode, cr3 is only 32 bits even on 5297 * x86_64. Therefore we need to allocate the PDP table in the first 5298 * 4GB of memory, which happens to fit the DMA32 zone. Except for 5299 * SVM's 32-bit NPT support, TDP paging doesn't use PAE paging and can 5300 * skip allocating the PDP table. 5301 */ 5302 if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) 5303 return 0; 5304 5305 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); 5306 if (!page) 5307 return -ENOMEM; 5308 5309 mmu->pae_root = page_address(page); 5310 for (i = 0; i < 4; ++i) 5311 mmu->pae_root[i] = INVALID_PAGE; 5312 5313 return 0; 5314 } 5315 5316 int kvm_mmu_create(struct kvm_vcpu *vcpu) 5317 { 5318 int ret; 5319 5320 vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache; 5321 vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO; 5322 5323 vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache; 5324 vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO; 5325 5326 vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO; 5327 5328 vcpu->arch.mmu = &vcpu->arch.root_mmu; 5329 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 5330 5331 vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa; 5332 5333 ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu); 5334 if (ret) 5335 return ret; 5336 5337 ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu); 5338 if (ret) 5339 goto fail_allocate_root; 5340 5341 return ret; 5342 fail_allocate_root: 5343 free_mmu_pages(&vcpu->arch.guest_mmu); 5344 return ret; 5345 } 5346 5347 #define BATCH_ZAP_PAGES 10 5348 static void kvm_zap_obsolete_pages(struct kvm *kvm) 5349 { 5350 struct kvm_mmu_page *sp, *node; 5351 int nr_zapped, batch = 0; 5352 5353 restart: 5354 list_for_each_entry_safe_reverse(sp, node, 5355 &kvm->arch.active_mmu_pages, link) { 5356 /* 5357 * No obsolete valid page exists before a newly created page 5358 * since active_mmu_pages is a FIFO list. 5359 */ 5360 if (!is_obsolete_sp(kvm, sp)) 5361 break; 5362 5363 /* 5364 * Invalid pages should never land back on the list of active 5365 * pages. Skip the bogus page, otherwise we'll get stuck in an 5366 * infinite loop if the page gets put back on the list (again). 5367 */ 5368 if (WARN_ON(sp->role.invalid)) 5369 continue; 5370 5371 /* 5372 * No need to flush the TLB since we're only zapping shadow 5373 * pages with an obsolete generation number and all vCPUS have 5374 * loaded a new root, i.e. the shadow pages being zapped cannot 5375 * be in active use by the guest. 5376 */ 5377 if (batch >= BATCH_ZAP_PAGES && 5378 cond_resched_lock(&kvm->mmu_lock)) { 5379 batch = 0; 5380 goto restart; 5381 } 5382 5383 if (__kvm_mmu_prepare_zap_page(kvm, sp, 5384 &kvm->arch.zapped_obsolete_pages, &nr_zapped)) { 5385 batch += nr_zapped; 5386 goto restart; 5387 } 5388 } 5389 5390 /* 5391 * Trigger a remote TLB flush before freeing the page tables to ensure 5392 * KVM is not in the middle of a lockless shadow page table walk, which 5393 * may reference the pages. 5394 */ 5395 kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages); 5396 } 5397 5398 /* 5399 * Fast invalidate all shadow pages and use lock-break technique 5400 * to zap obsolete pages. 5401 * 5402 * It's required when memslot is being deleted or VM is being 5403 * destroyed, in these cases, we should ensure that KVM MMU does 5404 * not use any resource of the being-deleted slot or all slots 5405 * after calling the function. 5406 */ 5407 static void kvm_mmu_zap_all_fast(struct kvm *kvm) 5408 { 5409 lockdep_assert_held(&kvm->slots_lock); 5410 5411 spin_lock(&kvm->mmu_lock); 5412 trace_kvm_mmu_zap_all_fast(kvm); 5413 5414 /* 5415 * Toggle mmu_valid_gen between '0' and '1'. Because slots_lock is 5416 * held for the entire duration of zapping obsolete pages, it's 5417 * impossible for there to be multiple invalid generations associated 5418 * with *valid* shadow pages at any given time, i.e. there is exactly 5419 * one valid generation and (at most) one invalid generation. 5420 */ 5421 kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1; 5422 5423 /* 5424 * Notify all vcpus to reload its shadow page table and flush TLB. 5425 * Then all vcpus will switch to new shadow page table with the new 5426 * mmu_valid_gen. 5427 * 5428 * Note: we need to do this under the protection of mmu_lock, 5429 * otherwise, vcpu would purge shadow page but miss tlb flush. 5430 */ 5431 kvm_reload_remote_mmus(kvm); 5432 5433 kvm_zap_obsolete_pages(kvm); 5434 5435 if (kvm->arch.tdp_mmu_enabled) 5436 kvm_tdp_mmu_zap_all(kvm); 5437 5438 spin_unlock(&kvm->mmu_lock); 5439 } 5440 5441 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) 5442 { 5443 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages)); 5444 } 5445 5446 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm, 5447 struct kvm_memory_slot *slot, 5448 struct kvm_page_track_notifier_node *node) 5449 { 5450 kvm_mmu_zap_all_fast(kvm); 5451 } 5452 5453 void kvm_mmu_init_vm(struct kvm *kvm) 5454 { 5455 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 5456 5457 kvm_mmu_init_tdp_mmu(kvm); 5458 5459 node->track_write = kvm_mmu_pte_write; 5460 node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; 5461 kvm_page_track_register_notifier(kvm, node); 5462 } 5463 5464 void kvm_mmu_uninit_vm(struct kvm *kvm) 5465 { 5466 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 5467 5468 kvm_page_track_unregister_notifier(kvm, node); 5469 5470 kvm_mmu_uninit_tdp_mmu(kvm); 5471 } 5472 5473 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) 5474 { 5475 struct kvm_memslots *slots; 5476 struct kvm_memory_slot *memslot; 5477 int i; 5478 bool flush; 5479 5480 spin_lock(&kvm->mmu_lock); 5481 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 5482 slots = __kvm_memslots(kvm, i); 5483 kvm_for_each_memslot(memslot, slots) { 5484 gfn_t start, end; 5485 5486 start = max(gfn_start, memslot->base_gfn); 5487 end = min(gfn_end, memslot->base_gfn + memslot->npages); 5488 if (start >= end) 5489 continue; 5490 5491 slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, 5492 PG_LEVEL_4K, 5493 KVM_MAX_HUGEPAGE_LEVEL, 5494 start, end - 1, true); 5495 } 5496 } 5497 5498 if (kvm->arch.tdp_mmu_enabled) { 5499 flush = kvm_tdp_mmu_zap_gfn_range(kvm, gfn_start, gfn_end); 5500 if (flush) 5501 kvm_flush_remote_tlbs(kvm); 5502 } 5503 5504 spin_unlock(&kvm->mmu_lock); 5505 } 5506 5507 static bool slot_rmap_write_protect(struct kvm *kvm, 5508 struct kvm_rmap_head *rmap_head) 5509 { 5510 return __rmap_write_protect(kvm, rmap_head, false); 5511 } 5512 5513 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, 5514 struct kvm_memory_slot *memslot, 5515 int start_level) 5516 { 5517 bool flush; 5518 5519 spin_lock(&kvm->mmu_lock); 5520 flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, 5521 start_level, KVM_MAX_HUGEPAGE_LEVEL, false); 5522 if (kvm->arch.tdp_mmu_enabled) 5523 flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_4K); 5524 spin_unlock(&kvm->mmu_lock); 5525 5526 /* 5527 * We can flush all the TLBs out of the mmu lock without TLB 5528 * corruption since we just change the spte from writable to 5529 * readonly so that we only need to care the case of changing 5530 * spte from present to present (changing the spte from present 5531 * to nonpresent will flush all the TLBs immediately), in other 5532 * words, the only case we care is mmu_spte_update() where we 5533 * have checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE 5534 * instead of PT_WRITABLE_MASK, that means it does not depend 5535 * on PT_WRITABLE_MASK anymore. 5536 */ 5537 if (flush) 5538 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5539 } 5540 5541 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, 5542 struct kvm_rmap_head *rmap_head) 5543 { 5544 u64 *sptep; 5545 struct rmap_iterator iter; 5546 int need_tlb_flush = 0; 5547 kvm_pfn_t pfn; 5548 struct kvm_mmu_page *sp; 5549 5550 restart: 5551 for_each_rmap_spte(rmap_head, &iter, sptep) { 5552 sp = sptep_to_sp(sptep); 5553 pfn = spte_to_pfn(*sptep); 5554 5555 /* 5556 * We cannot do huge page mapping for indirect shadow pages, 5557 * which are found on the last rmap (level = 1) when not using 5558 * tdp; such shadow pages are synced with the page table in 5559 * the guest, and the guest page table is using 4K page size 5560 * mapping if the indirect sp has level = 1. 5561 */ 5562 if (sp->role.direct && !kvm_is_reserved_pfn(pfn) && 5563 (kvm_is_zone_device_pfn(pfn) || 5564 PageCompound(pfn_to_page(pfn)))) { 5565 pte_list_remove(rmap_head, sptep); 5566 5567 if (kvm_available_flush_tlb_with_range()) 5568 kvm_flush_remote_tlbs_with_address(kvm, sp->gfn, 5569 KVM_PAGES_PER_HPAGE(sp->role.level)); 5570 else 5571 need_tlb_flush = 1; 5572 5573 goto restart; 5574 } 5575 } 5576 5577 return need_tlb_flush; 5578 } 5579 5580 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, 5581 const struct kvm_memory_slot *memslot) 5582 { 5583 /* FIXME: const-ify all uses of struct kvm_memory_slot. */ 5584 spin_lock(&kvm->mmu_lock); 5585 slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot, 5586 kvm_mmu_zap_collapsible_spte, true); 5587 5588 if (kvm->arch.tdp_mmu_enabled) 5589 kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot); 5590 spin_unlock(&kvm->mmu_lock); 5591 } 5592 5593 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 5594 struct kvm_memory_slot *memslot) 5595 { 5596 /* 5597 * All current use cases for flushing the TLBs for a specific memslot 5598 * are related to dirty logging, and do the TLB flush out of mmu_lock. 5599 * The interaction between the various operations on memslot must be 5600 * serialized by slots_locks to ensure the TLB flush from one operation 5601 * is observed by any other operation on the same memslot. 5602 */ 5603 lockdep_assert_held(&kvm->slots_lock); 5604 kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, 5605 memslot->npages); 5606 } 5607 5608 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 5609 struct kvm_memory_slot *memslot) 5610 { 5611 bool flush; 5612 5613 spin_lock(&kvm->mmu_lock); 5614 flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false); 5615 if (kvm->arch.tdp_mmu_enabled) 5616 flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot); 5617 spin_unlock(&kvm->mmu_lock); 5618 5619 /* 5620 * It's also safe to flush TLBs out of mmu lock here as currently this 5621 * function is only used for dirty logging, in which case flushing TLB 5622 * out of mmu lock also guarantees no dirty pages will be lost in 5623 * dirty_bitmap. 5624 */ 5625 if (flush) 5626 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5627 } 5628 EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty); 5629 5630 void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, 5631 struct kvm_memory_slot *memslot) 5632 { 5633 bool flush; 5634 5635 spin_lock(&kvm->mmu_lock); 5636 flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect, 5637 false); 5638 if (kvm->arch.tdp_mmu_enabled) 5639 flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M); 5640 spin_unlock(&kvm->mmu_lock); 5641 5642 if (flush) 5643 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5644 } 5645 EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access); 5646 5647 void kvm_mmu_slot_set_dirty(struct kvm *kvm, 5648 struct kvm_memory_slot *memslot) 5649 { 5650 bool flush; 5651 5652 spin_lock(&kvm->mmu_lock); 5653 flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false); 5654 if (kvm->arch.tdp_mmu_enabled) 5655 flush |= kvm_tdp_mmu_slot_set_dirty(kvm, memslot); 5656 spin_unlock(&kvm->mmu_lock); 5657 5658 if (flush) 5659 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5660 } 5661 EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty); 5662 5663 void kvm_mmu_zap_all(struct kvm *kvm) 5664 { 5665 struct kvm_mmu_page *sp, *node; 5666 LIST_HEAD(invalid_list); 5667 int ign; 5668 5669 spin_lock(&kvm->mmu_lock); 5670 restart: 5671 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) { 5672 if (WARN_ON(sp->role.invalid)) 5673 continue; 5674 if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign)) 5675 goto restart; 5676 if (cond_resched_lock(&kvm->mmu_lock)) 5677 goto restart; 5678 } 5679 5680 kvm_mmu_commit_zap_page(kvm, &invalid_list); 5681 5682 if (kvm->arch.tdp_mmu_enabled) 5683 kvm_tdp_mmu_zap_all(kvm); 5684 5685 spin_unlock(&kvm->mmu_lock); 5686 } 5687 5688 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) 5689 { 5690 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); 5691 5692 gen &= MMIO_SPTE_GEN_MASK; 5693 5694 /* 5695 * Generation numbers are incremented in multiples of the number of 5696 * address spaces in order to provide unique generations across all 5697 * address spaces. Strip what is effectively the address space 5698 * modifier prior to checking for a wrap of the MMIO generation so 5699 * that a wrap in any address space is detected. 5700 */ 5701 gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1); 5702 5703 /* 5704 * The very rare case: if the MMIO generation number has wrapped, 5705 * zap all shadow pages. 5706 */ 5707 if (unlikely(gen == 0)) { 5708 kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n"); 5709 kvm_mmu_zap_all_fast(kvm); 5710 } 5711 } 5712 5713 static unsigned long 5714 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 5715 { 5716 struct kvm *kvm; 5717 int nr_to_scan = sc->nr_to_scan; 5718 unsigned long freed = 0; 5719 5720 mutex_lock(&kvm_lock); 5721 5722 list_for_each_entry(kvm, &vm_list, vm_list) { 5723 int idx; 5724 LIST_HEAD(invalid_list); 5725 5726 /* 5727 * Never scan more than sc->nr_to_scan VM instances. 5728 * Will not hit this condition practically since we do not try 5729 * to shrink more than one VM and it is very unlikely to see 5730 * !n_used_mmu_pages so many times. 5731 */ 5732 if (!nr_to_scan--) 5733 break; 5734 /* 5735 * n_used_mmu_pages is accessed without holding kvm->mmu_lock 5736 * here. We may skip a VM instance errorneosly, but we do not 5737 * want to shrink a VM that only started to populate its MMU 5738 * anyway. 5739 */ 5740 if (!kvm->arch.n_used_mmu_pages && 5741 !kvm_has_zapped_obsolete_pages(kvm)) 5742 continue; 5743 5744 idx = srcu_read_lock(&kvm->srcu); 5745 spin_lock(&kvm->mmu_lock); 5746 5747 if (kvm_has_zapped_obsolete_pages(kvm)) { 5748 kvm_mmu_commit_zap_page(kvm, 5749 &kvm->arch.zapped_obsolete_pages); 5750 goto unlock; 5751 } 5752 5753 freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan); 5754 5755 unlock: 5756 spin_unlock(&kvm->mmu_lock); 5757 srcu_read_unlock(&kvm->srcu, idx); 5758 5759 /* 5760 * unfair on small ones 5761 * per-vm shrinkers cry out 5762 * sadness comes quickly 5763 */ 5764 list_move_tail(&kvm->vm_list, &vm_list); 5765 break; 5766 } 5767 5768 mutex_unlock(&kvm_lock); 5769 return freed; 5770 } 5771 5772 static unsigned long 5773 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 5774 { 5775 return percpu_counter_read_positive(&kvm_total_used_mmu_pages); 5776 } 5777 5778 static struct shrinker mmu_shrinker = { 5779 .count_objects = mmu_shrink_count, 5780 .scan_objects = mmu_shrink_scan, 5781 .seeks = DEFAULT_SEEKS * 10, 5782 }; 5783 5784 static void mmu_destroy_caches(void) 5785 { 5786 kmem_cache_destroy(pte_list_desc_cache); 5787 kmem_cache_destroy(mmu_page_header_cache); 5788 } 5789 5790 static void kvm_set_mmio_spte_mask(void) 5791 { 5792 u64 mask; 5793 5794 /* 5795 * Set a reserved PA bit in MMIO SPTEs to generate page faults with 5796 * PFEC.RSVD=1 on MMIO accesses. 64-bit PTEs (PAE, x86-64, and EPT 5797 * paging) support a maximum of 52 bits of PA, i.e. if the CPU supports 5798 * 52-bit physical addresses then there are no reserved PA bits in the 5799 * PTEs and so the reserved PA approach must be disabled. 5800 */ 5801 if (shadow_phys_bits < 52) 5802 mask = BIT_ULL(51) | PT_PRESENT_MASK; 5803 else 5804 mask = 0; 5805 5806 kvm_mmu_set_mmio_spte_mask(mask, ACC_WRITE_MASK | ACC_USER_MASK); 5807 } 5808 5809 static bool get_nx_auto_mode(void) 5810 { 5811 /* Return true when CPU has the bug, and mitigations are ON */ 5812 return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off(); 5813 } 5814 5815 static void __set_nx_huge_pages(bool val) 5816 { 5817 nx_huge_pages = itlb_multihit_kvm_mitigation = val; 5818 } 5819 5820 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp) 5821 { 5822 bool old_val = nx_huge_pages; 5823 bool new_val; 5824 5825 /* In "auto" mode deploy workaround only if CPU has the bug. */ 5826 if (sysfs_streq(val, "off")) 5827 new_val = 0; 5828 else if (sysfs_streq(val, "force")) 5829 new_val = 1; 5830 else if (sysfs_streq(val, "auto")) 5831 new_val = get_nx_auto_mode(); 5832 else if (strtobool(val, &new_val) < 0) 5833 return -EINVAL; 5834 5835 __set_nx_huge_pages(new_val); 5836 5837 if (new_val != old_val) { 5838 struct kvm *kvm; 5839 5840 mutex_lock(&kvm_lock); 5841 5842 list_for_each_entry(kvm, &vm_list, vm_list) { 5843 mutex_lock(&kvm->slots_lock); 5844 kvm_mmu_zap_all_fast(kvm); 5845 mutex_unlock(&kvm->slots_lock); 5846 5847 wake_up_process(kvm->arch.nx_lpage_recovery_thread); 5848 } 5849 mutex_unlock(&kvm_lock); 5850 } 5851 5852 return 0; 5853 } 5854 5855 int kvm_mmu_module_init(void) 5856 { 5857 int ret = -ENOMEM; 5858 5859 if (nx_huge_pages == -1) 5860 __set_nx_huge_pages(get_nx_auto_mode()); 5861 5862 /* 5863 * MMU roles use union aliasing which is, generally speaking, an 5864 * undefined behavior. However, we supposedly know how compilers behave 5865 * and the current status quo is unlikely to change. Guardians below are 5866 * supposed to let us know if the assumption becomes false. 5867 */ 5868 BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32)); 5869 BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32)); 5870 BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64)); 5871 5872 kvm_mmu_reset_all_pte_masks(); 5873 5874 kvm_set_mmio_spte_mask(); 5875 5876 pte_list_desc_cache = kmem_cache_create("pte_list_desc", 5877 sizeof(struct pte_list_desc), 5878 0, SLAB_ACCOUNT, NULL); 5879 if (!pte_list_desc_cache) 5880 goto out; 5881 5882 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header", 5883 sizeof(struct kvm_mmu_page), 5884 0, SLAB_ACCOUNT, NULL); 5885 if (!mmu_page_header_cache) 5886 goto out; 5887 5888 if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL)) 5889 goto out; 5890 5891 ret = register_shrinker(&mmu_shrinker); 5892 if (ret) 5893 goto out; 5894 5895 return 0; 5896 5897 out: 5898 mmu_destroy_caches(); 5899 return ret; 5900 } 5901 5902 /* 5903 * Calculate mmu pages needed for kvm. 5904 */ 5905 unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm) 5906 { 5907 unsigned long nr_mmu_pages; 5908 unsigned long nr_pages = 0; 5909 struct kvm_memslots *slots; 5910 struct kvm_memory_slot *memslot; 5911 int i; 5912 5913 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 5914 slots = __kvm_memslots(kvm, i); 5915 5916 kvm_for_each_memslot(memslot, slots) 5917 nr_pages += memslot->npages; 5918 } 5919 5920 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; 5921 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); 5922 5923 return nr_mmu_pages; 5924 } 5925 5926 void kvm_mmu_destroy(struct kvm_vcpu *vcpu) 5927 { 5928 kvm_mmu_unload(vcpu); 5929 free_mmu_pages(&vcpu->arch.root_mmu); 5930 free_mmu_pages(&vcpu->arch.guest_mmu); 5931 mmu_free_memory_caches(vcpu); 5932 } 5933 5934 void kvm_mmu_module_exit(void) 5935 { 5936 mmu_destroy_caches(); 5937 percpu_counter_destroy(&kvm_total_used_mmu_pages); 5938 unregister_shrinker(&mmu_shrinker); 5939 mmu_audit_disable(); 5940 } 5941 5942 static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp) 5943 { 5944 unsigned int old_val; 5945 int err; 5946 5947 old_val = nx_huge_pages_recovery_ratio; 5948 err = param_set_uint(val, kp); 5949 if (err) 5950 return err; 5951 5952 if (READ_ONCE(nx_huge_pages) && 5953 !old_val && nx_huge_pages_recovery_ratio) { 5954 struct kvm *kvm; 5955 5956 mutex_lock(&kvm_lock); 5957 5958 list_for_each_entry(kvm, &vm_list, vm_list) 5959 wake_up_process(kvm->arch.nx_lpage_recovery_thread); 5960 5961 mutex_unlock(&kvm_lock); 5962 } 5963 5964 return err; 5965 } 5966 5967 static void kvm_recover_nx_lpages(struct kvm *kvm) 5968 { 5969 int rcu_idx; 5970 struct kvm_mmu_page *sp; 5971 unsigned int ratio; 5972 LIST_HEAD(invalid_list); 5973 ulong to_zap; 5974 5975 rcu_idx = srcu_read_lock(&kvm->srcu); 5976 spin_lock(&kvm->mmu_lock); 5977 5978 ratio = READ_ONCE(nx_huge_pages_recovery_ratio); 5979 to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0; 5980 for ( ; to_zap; --to_zap) { 5981 if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages)) 5982 break; 5983 5984 /* 5985 * We use a separate list instead of just using active_mmu_pages 5986 * because the number of lpage_disallowed pages is expected to 5987 * be relatively small compared to the total. 5988 */ 5989 sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages, 5990 struct kvm_mmu_page, 5991 lpage_disallowed_link); 5992 WARN_ON_ONCE(!sp->lpage_disallowed); 5993 if (sp->tdp_mmu_page) 5994 kvm_tdp_mmu_zap_gfn_range(kvm, sp->gfn, 5995 sp->gfn + KVM_PAGES_PER_HPAGE(sp->role.level)); 5996 else { 5997 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); 5998 WARN_ON_ONCE(sp->lpage_disallowed); 5999 } 6000 6001 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { 6002 kvm_mmu_commit_zap_page(kvm, &invalid_list); 6003 cond_resched_lock(&kvm->mmu_lock); 6004 } 6005 } 6006 kvm_mmu_commit_zap_page(kvm, &invalid_list); 6007 6008 spin_unlock(&kvm->mmu_lock); 6009 srcu_read_unlock(&kvm->srcu, rcu_idx); 6010 } 6011 6012 static long get_nx_lpage_recovery_timeout(u64 start_time) 6013 { 6014 return READ_ONCE(nx_huge_pages) && READ_ONCE(nx_huge_pages_recovery_ratio) 6015 ? start_time + 60 * HZ - get_jiffies_64() 6016 : MAX_SCHEDULE_TIMEOUT; 6017 } 6018 6019 static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data) 6020 { 6021 u64 start_time; 6022 long remaining_time; 6023 6024 while (true) { 6025 start_time = get_jiffies_64(); 6026 remaining_time = get_nx_lpage_recovery_timeout(start_time); 6027 6028 set_current_state(TASK_INTERRUPTIBLE); 6029 while (!kthread_should_stop() && remaining_time > 0) { 6030 schedule_timeout(remaining_time); 6031 remaining_time = get_nx_lpage_recovery_timeout(start_time); 6032 set_current_state(TASK_INTERRUPTIBLE); 6033 } 6034 6035 set_current_state(TASK_RUNNING); 6036 6037 if (kthread_should_stop()) 6038 return 0; 6039 6040 kvm_recover_nx_lpages(kvm); 6041 } 6042 } 6043 6044 int kvm_mmu_post_init_vm(struct kvm *kvm) 6045 { 6046 int err; 6047 6048 err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0, 6049 "kvm-nx-lpage-recovery", 6050 &kvm->arch.nx_lpage_recovery_thread); 6051 if (!err) 6052 kthread_unpark(kvm->arch.nx_lpage_recovery_thread); 6053 6054 return err; 6055 } 6056 6057 void kvm_mmu_pre_destroy_vm(struct kvm *kvm) 6058 { 6059 if (kvm->arch.nx_lpage_recovery_thread) 6060 kthread_stop(kvm->arch.nx_lpage_recovery_thread); 6061 } 6062