1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * MMU support 9 * 10 * Copyright (C) 2006 Qumranet, Inc. 11 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 12 * 13 * Authors: 14 * Yaniv Kamay <yaniv@qumranet.com> 15 * Avi Kivity <avi@qumranet.com> 16 */ 17 18 #include "irq.h" 19 #include "ioapic.h" 20 #include "mmu.h" 21 #include "mmu_internal.h" 22 #include "tdp_mmu.h" 23 #include "x86.h" 24 #include "kvm_cache_regs.h" 25 #include "kvm_emulate.h" 26 #include "cpuid.h" 27 #include "spte.h" 28 29 #include <linux/kvm_host.h> 30 #include <linux/types.h> 31 #include <linux/string.h> 32 #include <linux/mm.h> 33 #include <linux/highmem.h> 34 #include <linux/moduleparam.h> 35 #include <linux/export.h> 36 #include <linux/swap.h> 37 #include <linux/hugetlb.h> 38 #include <linux/compiler.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/sched/signal.h> 42 #include <linux/uaccess.h> 43 #include <linux/hash.h> 44 #include <linux/kern_levels.h> 45 #include <linux/kthread.h> 46 47 #include <asm/page.h> 48 #include <asm/memtype.h> 49 #include <asm/cmpxchg.h> 50 #include <asm/io.h> 51 #include <asm/set_memory.h> 52 #include <asm/vmx.h> 53 #include <asm/kvm_page_track.h> 54 #include "trace.h" 55 56 extern bool itlb_multihit_kvm_mitigation; 57 58 static int __read_mostly nx_huge_pages = -1; 59 #ifdef CONFIG_PREEMPT_RT 60 /* Recovery can cause latency spikes, disable it for PREEMPT_RT. */ 61 static uint __read_mostly nx_huge_pages_recovery_ratio = 0; 62 #else 63 static uint __read_mostly nx_huge_pages_recovery_ratio = 60; 64 #endif 65 66 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp); 67 static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp); 68 69 static const struct kernel_param_ops nx_huge_pages_ops = { 70 .set = set_nx_huge_pages, 71 .get = param_get_bool, 72 }; 73 74 static const struct kernel_param_ops nx_huge_pages_recovery_ratio_ops = { 75 .set = set_nx_huge_pages_recovery_ratio, 76 .get = param_get_uint, 77 }; 78 79 module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644); 80 __MODULE_PARM_TYPE(nx_huge_pages, "bool"); 81 module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_ratio_ops, 82 &nx_huge_pages_recovery_ratio, 0644); 83 __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint"); 84 85 static bool __read_mostly force_flush_and_sync_on_reuse; 86 module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); 87 88 /* 89 * When setting this variable to true it enables Two-Dimensional-Paging 90 * where the hardware walks 2 page tables: 91 * 1. the guest-virtual to guest-physical 92 * 2. while doing 1. it walks guest-physical to host-physical 93 * If the hardware supports that we don't need to do shadow paging. 94 */ 95 bool tdp_enabled = false; 96 97 static int max_huge_page_level __read_mostly; 98 static int max_tdp_level __read_mostly; 99 100 enum { 101 AUDIT_PRE_PAGE_FAULT, 102 AUDIT_POST_PAGE_FAULT, 103 AUDIT_PRE_PTE_WRITE, 104 AUDIT_POST_PTE_WRITE, 105 AUDIT_PRE_SYNC, 106 AUDIT_POST_SYNC 107 }; 108 109 #ifdef MMU_DEBUG 110 bool dbg = 0; 111 module_param(dbg, bool, 0644); 112 #endif 113 114 #define PTE_PREFETCH_NUM 8 115 116 #define PT32_LEVEL_BITS 10 117 118 #define PT32_LEVEL_SHIFT(level) \ 119 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) 120 121 #define PT32_LVL_OFFSET_MASK(level) \ 122 (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \ 123 * PT32_LEVEL_BITS))) - 1)) 124 125 #define PT32_INDEX(address, level)\ 126 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1)) 127 128 129 #define PT32_BASE_ADDR_MASK PAGE_MASK 130 #define PT32_DIR_BASE_ADDR_MASK \ 131 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) 132 #define PT32_LVL_ADDR_MASK(level) \ 133 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \ 134 * PT32_LEVEL_BITS))) - 1)) 135 136 #include <trace/events/kvm.h> 137 138 /* make pte_list_desc fit well in cache line */ 139 #define PTE_LIST_EXT 3 140 141 struct pte_list_desc { 142 u64 *sptes[PTE_LIST_EXT]; 143 struct pte_list_desc *more; 144 }; 145 146 struct kvm_shadow_walk_iterator { 147 u64 addr; 148 hpa_t shadow_addr; 149 u64 *sptep; 150 int level; 151 unsigned index; 152 }; 153 154 #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker) \ 155 for (shadow_walk_init_using_root(&(_walker), (_vcpu), \ 156 (_root), (_addr)); \ 157 shadow_walk_okay(&(_walker)); \ 158 shadow_walk_next(&(_walker))) 159 160 #define for_each_shadow_entry(_vcpu, _addr, _walker) \ 161 for (shadow_walk_init(&(_walker), _vcpu, _addr); \ 162 shadow_walk_okay(&(_walker)); \ 163 shadow_walk_next(&(_walker))) 164 165 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \ 166 for (shadow_walk_init(&(_walker), _vcpu, _addr); \ 167 shadow_walk_okay(&(_walker)) && \ 168 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \ 169 __shadow_walk_next(&(_walker), spte)) 170 171 static struct kmem_cache *pte_list_desc_cache; 172 struct kmem_cache *mmu_page_header_cache; 173 static struct percpu_counter kvm_total_used_mmu_pages; 174 175 static void mmu_spte_set(u64 *sptep, u64 spte); 176 static union kvm_mmu_page_role 177 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu); 178 179 #define CREATE_TRACE_POINTS 180 #include "mmutrace.h" 181 182 183 static inline bool kvm_available_flush_tlb_with_range(void) 184 { 185 return kvm_x86_ops.tlb_remote_flush_with_range; 186 } 187 188 static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, 189 struct kvm_tlb_range *range) 190 { 191 int ret = -ENOTSUPP; 192 193 if (range && kvm_x86_ops.tlb_remote_flush_with_range) 194 ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range); 195 196 if (ret) 197 kvm_flush_remote_tlbs(kvm); 198 } 199 200 void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, 201 u64 start_gfn, u64 pages) 202 { 203 struct kvm_tlb_range range; 204 205 range.start_gfn = start_gfn; 206 range.pages = pages; 207 208 kvm_flush_remote_tlbs_with_range(kvm, &range); 209 } 210 211 bool is_nx_huge_page_enabled(void) 212 { 213 return READ_ONCE(nx_huge_pages); 214 } 215 216 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, 217 unsigned int access) 218 { 219 u64 spte = make_mmio_spte(vcpu, gfn, access); 220 221 trace_mark_mmio_spte(sptep, gfn, spte); 222 mmu_spte_set(sptep, spte); 223 } 224 225 static gfn_t get_mmio_spte_gfn(u64 spte) 226 { 227 u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask; 228 229 gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN) 230 & shadow_nonpresent_or_rsvd_mask; 231 232 return gpa >> PAGE_SHIFT; 233 } 234 235 static unsigned get_mmio_spte_access(u64 spte) 236 { 237 return spte & shadow_mmio_access_mask; 238 } 239 240 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte) 241 { 242 u64 kvm_gen, spte_gen, gen; 243 244 gen = kvm_vcpu_memslots(vcpu)->generation; 245 if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS)) 246 return false; 247 248 kvm_gen = gen & MMIO_SPTE_GEN_MASK; 249 spte_gen = get_mmio_spte_generation(spte); 250 251 trace_check_mmio_spte(spte, kvm_gen, spte_gen); 252 return likely(kvm_gen == spte_gen); 253 } 254 255 static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, 256 struct x86_exception *exception) 257 { 258 /* Check if guest physical address doesn't exceed guest maximum */ 259 if (kvm_vcpu_is_illegal_gpa(vcpu, gpa)) { 260 exception->error_code |= PFERR_RSVD_MASK; 261 return UNMAPPED_GVA; 262 } 263 264 return gpa; 265 } 266 267 static int is_cpuid_PSE36(void) 268 { 269 return 1; 270 } 271 272 static int is_nx(struct kvm_vcpu *vcpu) 273 { 274 return vcpu->arch.efer & EFER_NX; 275 } 276 277 static gfn_t pse36_gfn_delta(u32 gpte) 278 { 279 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; 280 281 return (gpte & PT32_DIR_PSE36_MASK) << shift; 282 } 283 284 #ifdef CONFIG_X86_64 285 static void __set_spte(u64 *sptep, u64 spte) 286 { 287 WRITE_ONCE(*sptep, spte); 288 } 289 290 static void __update_clear_spte_fast(u64 *sptep, u64 spte) 291 { 292 WRITE_ONCE(*sptep, spte); 293 } 294 295 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) 296 { 297 return xchg(sptep, spte); 298 } 299 300 static u64 __get_spte_lockless(u64 *sptep) 301 { 302 return READ_ONCE(*sptep); 303 } 304 #else 305 union split_spte { 306 struct { 307 u32 spte_low; 308 u32 spte_high; 309 }; 310 u64 spte; 311 }; 312 313 static void count_spte_clear(u64 *sptep, u64 spte) 314 { 315 struct kvm_mmu_page *sp = sptep_to_sp(sptep); 316 317 if (is_shadow_present_pte(spte)) 318 return; 319 320 /* Ensure the spte is completely set before we increase the count */ 321 smp_wmb(); 322 sp->clear_spte_count++; 323 } 324 325 static void __set_spte(u64 *sptep, u64 spte) 326 { 327 union split_spte *ssptep, sspte; 328 329 ssptep = (union split_spte *)sptep; 330 sspte = (union split_spte)spte; 331 332 ssptep->spte_high = sspte.spte_high; 333 334 /* 335 * If we map the spte from nonpresent to present, We should store 336 * the high bits firstly, then set present bit, so cpu can not 337 * fetch this spte while we are setting the spte. 338 */ 339 smp_wmb(); 340 341 WRITE_ONCE(ssptep->spte_low, sspte.spte_low); 342 } 343 344 static void __update_clear_spte_fast(u64 *sptep, u64 spte) 345 { 346 union split_spte *ssptep, sspte; 347 348 ssptep = (union split_spte *)sptep; 349 sspte = (union split_spte)spte; 350 351 WRITE_ONCE(ssptep->spte_low, sspte.spte_low); 352 353 /* 354 * If we map the spte from present to nonpresent, we should clear 355 * present bit firstly to avoid vcpu fetch the old high bits. 356 */ 357 smp_wmb(); 358 359 ssptep->spte_high = sspte.spte_high; 360 count_spte_clear(sptep, spte); 361 } 362 363 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) 364 { 365 union split_spte *ssptep, sspte, orig; 366 367 ssptep = (union split_spte *)sptep; 368 sspte = (union split_spte)spte; 369 370 /* xchg acts as a barrier before the setting of the high bits */ 371 orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low); 372 orig.spte_high = ssptep->spte_high; 373 ssptep->spte_high = sspte.spte_high; 374 count_spte_clear(sptep, spte); 375 376 return orig.spte; 377 } 378 379 /* 380 * The idea using the light way get the spte on x86_32 guest is from 381 * gup_get_pte (mm/gup.c). 382 * 383 * An spte tlb flush may be pending, because kvm_set_pte_rmapp 384 * coalesces them and we are running out of the MMU lock. Therefore 385 * we need to protect against in-progress updates of the spte. 386 * 387 * Reading the spte while an update is in progress may get the old value 388 * for the high part of the spte. The race is fine for a present->non-present 389 * change (because the high part of the spte is ignored for non-present spte), 390 * but for a present->present change we must reread the spte. 391 * 392 * All such changes are done in two steps (present->non-present and 393 * non-present->present), hence it is enough to count the number of 394 * present->non-present updates: if it changed while reading the spte, 395 * we might have hit the race. This is done using clear_spte_count. 396 */ 397 static u64 __get_spte_lockless(u64 *sptep) 398 { 399 struct kvm_mmu_page *sp = sptep_to_sp(sptep); 400 union split_spte spte, *orig = (union split_spte *)sptep; 401 int count; 402 403 retry: 404 count = sp->clear_spte_count; 405 smp_rmb(); 406 407 spte.spte_low = orig->spte_low; 408 smp_rmb(); 409 410 spte.spte_high = orig->spte_high; 411 smp_rmb(); 412 413 if (unlikely(spte.spte_low != orig->spte_low || 414 count != sp->clear_spte_count)) 415 goto retry; 416 417 return spte.spte; 418 } 419 #endif 420 421 static bool spte_has_volatile_bits(u64 spte) 422 { 423 if (!is_shadow_present_pte(spte)) 424 return false; 425 426 /* 427 * Always atomically update spte if it can be updated 428 * out of mmu-lock, it can ensure dirty bit is not lost, 429 * also, it can help us to get a stable is_writable_pte() 430 * to ensure tlb flush is not missed. 431 */ 432 if (spte_can_locklessly_be_made_writable(spte) || 433 is_access_track_spte(spte)) 434 return true; 435 436 if (spte_ad_enabled(spte)) { 437 if ((spte & shadow_accessed_mask) == 0 || 438 (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0)) 439 return true; 440 } 441 442 return false; 443 } 444 445 /* Rules for using mmu_spte_set: 446 * Set the sptep from nonpresent to present. 447 * Note: the sptep being assigned *must* be either not present 448 * or in a state where the hardware will not attempt to update 449 * the spte. 450 */ 451 static void mmu_spte_set(u64 *sptep, u64 new_spte) 452 { 453 WARN_ON(is_shadow_present_pte(*sptep)); 454 __set_spte(sptep, new_spte); 455 } 456 457 /* 458 * Update the SPTE (excluding the PFN), but do not track changes in its 459 * accessed/dirty status. 460 */ 461 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte) 462 { 463 u64 old_spte = *sptep; 464 465 WARN_ON(!is_shadow_present_pte(new_spte)); 466 467 if (!is_shadow_present_pte(old_spte)) { 468 mmu_spte_set(sptep, new_spte); 469 return old_spte; 470 } 471 472 if (!spte_has_volatile_bits(old_spte)) 473 __update_clear_spte_fast(sptep, new_spte); 474 else 475 old_spte = __update_clear_spte_slow(sptep, new_spte); 476 477 WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte)); 478 479 return old_spte; 480 } 481 482 /* Rules for using mmu_spte_update: 483 * Update the state bits, it means the mapped pfn is not changed. 484 * 485 * Whenever we overwrite a writable spte with a read-only one we 486 * should flush remote TLBs. Otherwise rmap_write_protect 487 * will find a read-only spte, even though the writable spte 488 * might be cached on a CPU's TLB, the return value indicates this 489 * case. 490 * 491 * Returns true if the TLB needs to be flushed 492 */ 493 static bool mmu_spte_update(u64 *sptep, u64 new_spte) 494 { 495 bool flush = false; 496 u64 old_spte = mmu_spte_update_no_track(sptep, new_spte); 497 498 if (!is_shadow_present_pte(old_spte)) 499 return false; 500 501 /* 502 * For the spte updated out of mmu-lock is safe, since 503 * we always atomically update it, see the comments in 504 * spte_has_volatile_bits(). 505 */ 506 if (spte_can_locklessly_be_made_writable(old_spte) && 507 !is_writable_pte(new_spte)) 508 flush = true; 509 510 /* 511 * Flush TLB when accessed/dirty states are changed in the page tables, 512 * to guarantee consistency between TLB and page tables. 513 */ 514 515 if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) { 516 flush = true; 517 kvm_set_pfn_accessed(spte_to_pfn(old_spte)); 518 } 519 520 if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) { 521 flush = true; 522 kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 523 } 524 525 return flush; 526 } 527 528 /* 529 * Rules for using mmu_spte_clear_track_bits: 530 * It sets the sptep from present to nonpresent, and track the 531 * state bits, it is used to clear the last level sptep. 532 * Returns non-zero if the PTE was previously valid. 533 */ 534 static int mmu_spte_clear_track_bits(u64 *sptep) 535 { 536 kvm_pfn_t pfn; 537 u64 old_spte = *sptep; 538 539 if (!spte_has_volatile_bits(old_spte)) 540 __update_clear_spte_fast(sptep, 0ull); 541 else 542 old_spte = __update_clear_spte_slow(sptep, 0ull); 543 544 if (!is_shadow_present_pte(old_spte)) 545 return 0; 546 547 pfn = spte_to_pfn(old_spte); 548 549 /* 550 * KVM does not hold the refcount of the page used by 551 * kvm mmu, before reclaiming the page, we should 552 * unmap it from mmu first. 553 */ 554 WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn))); 555 556 if (is_accessed_spte(old_spte)) 557 kvm_set_pfn_accessed(pfn); 558 559 if (is_dirty_spte(old_spte)) 560 kvm_set_pfn_dirty(pfn); 561 562 return 1; 563 } 564 565 /* 566 * Rules for using mmu_spte_clear_no_track: 567 * Directly clear spte without caring the state bits of sptep, 568 * it is used to set the upper level spte. 569 */ 570 static void mmu_spte_clear_no_track(u64 *sptep) 571 { 572 __update_clear_spte_fast(sptep, 0ull); 573 } 574 575 static u64 mmu_spte_get_lockless(u64 *sptep) 576 { 577 return __get_spte_lockless(sptep); 578 } 579 580 /* Restore an acc-track PTE back to a regular PTE */ 581 static u64 restore_acc_track_spte(u64 spte) 582 { 583 u64 new_spte = spte; 584 u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT) 585 & SHADOW_ACC_TRACK_SAVED_BITS_MASK; 586 587 WARN_ON_ONCE(spte_ad_enabled(spte)); 588 WARN_ON_ONCE(!is_access_track_spte(spte)); 589 590 new_spte &= ~shadow_acc_track_mask; 591 new_spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK << 592 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT); 593 new_spte |= saved_bits; 594 595 return new_spte; 596 } 597 598 /* Returns the Accessed status of the PTE and resets it at the same time. */ 599 static bool mmu_spte_age(u64 *sptep) 600 { 601 u64 spte = mmu_spte_get_lockless(sptep); 602 603 if (!is_accessed_spte(spte)) 604 return false; 605 606 if (spte_ad_enabled(spte)) { 607 clear_bit((ffs(shadow_accessed_mask) - 1), 608 (unsigned long *)sptep); 609 } else { 610 /* 611 * Capture the dirty status of the page, so that it doesn't get 612 * lost when the SPTE is marked for access tracking. 613 */ 614 if (is_writable_pte(spte)) 615 kvm_set_pfn_dirty(spte_to_pfn(spte)); 616 617 spte = mark_spte_for_access_track(spte); 618 mmu_spte_update_no_track(sptep, spte); 619 } 620 621 return true; 622 } 623 624 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu) 625 { 626 /* 627 * Prevent page table teardown by making any free-er wait during 628 * kvm_flush_remote_tlbs() IPI to all active vcpus. 629 */ 630 local_irq_disable(); 631 632 /* 633 * Make sure a following spte read is not reordered ahead of the write 634 * to vcpu->mode. 635 */ 636 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES); 637 } 638 639 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu) 640 { 641 /* 642 * Make sure the write to vcpu->mode is not reordered in front of 643 * reads to sptes. If it does, kvm_mmu_commit_zap_page() can see us 644 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table. 645 */ 646 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); 647 local_irq_enable(); 648 } 649 650 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect) 651 { 652 int r; 653 654 /* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */ 655 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache, 656 1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM); 657 if (r) 658 return r; 659 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache, 660 PT64_ROOT_MAX_LEVEL); 661 if (r) 662 return r; 663 if (maybe_indirect) { 664 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_gfn_array_cache, 665 PT64_ROOT_MAX_LEVEL); 666 if (r) 667 return r; 668 } 669 return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, 670 PT64_ROOT_MAX_LEVEL); 671 } 672 673 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) 674 { 675 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache); 676 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache); 677 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_gfn_array_cache); 678 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache); 679 } 680 681 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu) 682 { 683 return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache); 684 } 685 686 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc) 687 { 688 kmem_cache_free(pte_list_desc_cache, pte_list_desc); 689 } 690 691 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index) 692 { 693 if (!sp->role.direct) 694 return sp->gfns[index]; 695 696 return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS)); 697 } 698 699 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) 700 { 701 if (!sp->role.direct) { 702 sp->gfns[index] = gfn; 703 return; 704 } 705 706 if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index))) 707 pr_err_ratelimited("gfn mismatch under direct page %llx " 708 "(expected %llx, got %llx)\n", 709 sp->gfn, 710 kvm_mmu_page_get_gfn(sp, index), gfn); 711 } 712 713 /* 714 * Return the pointer to the large page information for a given gfn, 715 * handling slots that are not large page aligned. 716 */ 717 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, 718 const struct kvm_memory_slot *slot, int level) 719 { 720 unsigned long idx; 721 722 idx = gfn_to_index(gfn, slot->base_gfn, level); 723 return &slot->arch.lpage_info[level - 2][idx]; 724 } 725 726 static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot, 727 gfn_t gfn, int count) 728 { 729 struct kvm_lpage_info *linfo; 730 int i; 731 732 for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { 733 linfo = lpage_info_slot(gfn, slot, i); 734 linfo->disallow_lpage += count; 735 WARN_ON(linfo->disallow_lpage < 0); 736 } 737 } 738 739 void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn) 740 { 741 update_gfn_disallow_lpage_count(slot, gfn, 1); 742 } 743 744 void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn) 745 { 746 update_gfn_disallow_lpage_count(slot, gfn, -1); 747 } 748 749 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) 750 { 751 struct kvm_memslots *slots; 752 struct kvm_memory_slot *slot; 753 gfn_t gfn; 754 755 kvm->arch.indirect_shadow_pages++; 756 gfn = sp->gfn; 757 slots = kvm_memslots_for_spte_role(kvm, sp->role); 758 slot = __gfn_to_memslot(slots, gfn); 759 760 /* the non-leaf shadow pages are keeping readonly. */ 761 if (sp->role.level > PG_LEVEL_4K) 762 return kvm_slot_page_track_add_page(kvm, slot, gfn, 763 KVM_PAGE_TRACK_WRITE); 764 765 kvm_mmu_gfn_disallow_lpage(slot, gfn); 766 } 767 768 void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp) 769 { 770 if (sp->lpage_disallowed) 771 return; 772 773 ++kvm->stat.nx_lpage_splits; 774 list_add_tail(&sp->lpage_disallowed_link, 775 &kvm->arch.lpage_disallowed_mmu_pages); 776 sp->lpage_disallowed = true; 777 } 778 779 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) 780 { 781 struct kvm_memslots *slots; 782 struct kvm_memory_slot *slot; 783 gfn_t gfn; 784 785 kvm->arch.indirect_shadow_pages--; 786 gfn = sp->gfn; 787 slots = kvm_memslots_for_spte_role(kvm, sp->role); 788 slot = __gfn_to_memslot(slots, gfn); 789 if (sp->role.level > PG_LEVEL_4K) 790 return kvm_slot_page_track_remove_page(kvm, slot, gfn, 791 KVM_PAGE_TRACK_WRITE); 792 793 kvm_mmu_gfn_allow_lpage(slot, gfn); 794 } 795 796 void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp) 797 { 798 --kvm->stat.nx_lpage_splits; 799 sp->lpage_disallowed = false; 800 list_del(&sp->lpage_disallowed_link); 801 } 802 803 static struct kvm_memory_slot * 804 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn, 805 bool no_dirty_log) 806 { 807 struct kvm_memory_slot *slot; 808 809 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 810 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 811 return NULL; 812 if (no_dirty_log && kvm_slot_dirty_track_enabled(slot)) 813 return NULL; 814 815 return slot; 816 } 817 818 /* 819 * About rmap_head encoding: 820 * 821 * If the bit zero of rmap_head->val is clear, then it points to the only spte 822 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct 823 * pte_list_desc containing more mappings. 824 */ 825 826 /* 827 * Returns the number of pointers in the rmap chain, not counting the new one. 828 */ 829 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte, 830 struct kvm_rmap_head *rmap_head) 831 { 832 struct pte_list_desc *desc; 833 int i, count = 0; 834 835 if (!rmap_head->val) { 836 rmap_printk("%p %llx 0->1\n", spte, *spte); 837 rmap_head->val = (unsigned long)spte; 838 } else if (!(rmap_head->val & 1)) { 839 rmap_printk("%p %llx 1->many\n", spte, *spte); 840 desc = mmu_alloc_pte_list_desc(vcpu); 841 desc->sptes[0] = (u64 *)rmap_head->val; 842 desc->sptes[1] = spte; 843 rmap_head->val = (unsigned long)desc | 1; 844 ++count; 845 } else { 846 rmap_printk("%p %llx many->many\n", spte, *spte); 847 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 848 while (desc->sptes[PTE_LIST_EXT-1]) { 849 count += PTE_LIST_EXT; 850 851 if (!desc->more) { 852 desc->more = mmu_alloc_pte_list_desc(vcpu); 853 desc = desc->more; 854 break; 855 } 856 desc = desc->more; 857 } 858 for (i = 0; desc->sptes[i]; ++i) 859 ++count; 860 desc->sptes[i] = spte; 861 } 862 return count; 863 } 864 865 static void 866 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head, 867 struct pte_list_desc *desc, int i, 868 struct pte_list_desc *prev_desc) 869 { 870 int j; 871 872 for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j) 873 ; 874 desc->sptes[i] = desc->sptes[j]; 875 desc->sptes[j] = NULL; 876 if (j != 0) 877 return; 878 if (!prev_desc && !desc->more) 879 rmap_head->val = 0; 880 else 881 if (prev_desc) 882 prev_desc->more = desc->more; 883 else 884 rmap_head->val = (unsigned long)desc->more | 1; 885 mmu_free_pte_list_desc(desc); 886 } 887 888 static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head) 889 { 890 struct pte_list_desc *desc; 891 struct pte_list_desc *prev_desc; 892 int i; 893 894 if (!rmap_head->val) { 895 pr_err("%s: %p 0->BUG\n", __func__, spte); 896 BUG(); 897 } else if (!(rmap_head->val & 1)) { 898 rmap_printk("%p 1->0\n", spte); 899 if ((u64 *)rmap_head->val != spte) { 900 pr_err("%s: %p 1->BUG\n", __func__, spte); 901 BUG(); 902 } 903 rmap_head->val = 0; 904 } else { 905 rmap_printk("%p many->many\n", spte); 906 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 907 prev_desc = NULL; 908 while (desc) { 909 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) { 910 if (desc->sptes[i] == spte) { 911 pte_list_desc_remove_entry(rmap_head, 912 desc, i, prev_desc); 913 return; 914 } 915 } 916 prev_desc = desc; 917 desc = desc->more; 918 } 919 pr_err("%s: %p many->many\n", __func__, spte); 920 BUG(); 921 } 922 } 923 924 static void pte_list_remove(struct kvm_rmap_head *rmap_head, u64 *sptep) 925 { 926 mmu_spte_clear_track_bits(sptep); 927 __pte_list_remove(sptep, rmap_head); 928 } 929 930 static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level, 931 struct kvm_memory_slot *slot) 932 { 933 unsigned long idx; 934 935 idx = gfn_to_index(gfn, slot->base_gfn, level); 936 return &slot->arch.rmap[level - PG_LEVEL_4K][idx]; 937 } 938 939 static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, 940 struct kvm_mmu_page *sp) 941 { 942 struct kvm_memslots *slots; 943 struct kvm_memory_slot *slot; 944 945 slots = kvm_memslots_for_spte_role(kvm, sp->role); 946 slot = __gfn_to_memslot(slots, gfn); 947 return __gfn_to_rmap(gfn, sp->role.level, slot); 948 } 949 950 static bool rmap_can_add(struct kvm_vcpu *vcpu) 951 { 952 struct kvm_mmu_memory_cache *mc; 953 954 mc = &vcpu->arch.mmu_pte_list_desc_cache; 955 return kvm_mmu_memory_cache_nr_free_objects(mc); 956 } 957 958 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) 959 { 960 struct kvm_mmu_page *sp; 961 struct kvm_rmap_head *rmap_head; 962 963 sp = sptep_to_sp(spte); 964 kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn); 965 rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp); 966 return pte_list_add(vcpu, spte, rmap_head); 967 } 968 969 static void rmap_remove(struct kvm *kvm, u64 *spte) 970 { 971 struct kvm_mmu_page *sp; 972 gfn_t gfn; 973 struct kvm_rmap_head *rmap_head; 974 975 sp = sptep_to_sp(spte); 976 gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt); 977 rmap_head = gfn_to_rmap(kvm, gfn, sp); 978 __pte_list_remove(spte, rmap_head); 979 } 980 981 /* 982 * Used by the following functions to iterate through the sptes linked by a 983 * rmap. All fields are private and not assumed to be used outside. 984 */ 985 struct rmap_iterator { 986 /* private fields */ 987 struct pte_list_desc *desc; /* holds the sptep if not NULL */ 988 int pos; /* index of the sptep */ 989 }; 990 991 /* 992 * Iteration must be started by this function. This should also be used after 993 * removing/dropping sptes from the rmap link because in such cases the 994 * information in the iterator may not be valid. 995 * 996 * Returns sptep if found, NULL otherwise. 997 */ 998 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, 999 struct rmap_iterator *iter) 1000 { 1001 u64 *sptep; 1002 1003 if (!rmap_head->val) 1004 return NULL; 1005 1006 if (!(rmap_head->val & 1)) { 1007 iter->desc = NULL; 1008 sptep = (u64 *)rmap_head->val; 1009 goto out; 1010 } 1011 1012 iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 1013 iter->pos = 0; 1014 sptep = iter->desc->sptes[iter->pos]; 1015 out: 1016 BUG_ON(!is_shadow_present_pte(*sptep)); 1017 return sptep; 1018 } 1019 1020 /* 1021 * Must be used with a valid iterator: e.g. after rmap_get_first(). 1022 * 1023 * Returns sptep if found, NULL otherwise. 1024 */ 1025 static u64 *rmap_get_next(struct rmap_iterator *iter) 1026 { 1027 u64 *sptep; 1028 1029 if (iter->desc) { 1030 if (iter->pos < PTE_LIST_EXT - 1) { 1031 ++iter->pos; 1032 sptep = iter->desc->sptes[iter->pos]; 1033 if (sptep) 1034 goto out; 1035 } 1036 1037 iter->desc = iter->desc->more; 1038 1039 if (iter->desc) { 1040 iter->pos = 0; 1041 /* desc->sptes[0] cannot be NULL */ 1042 sptep = iter->desc->sptes[iter->pos]; 1043 goto out; 1044 } 1045 } 1046 1047 return NULL; 1048 out: 1049 BUG_ON(!is_shadow_present_pte(*sptep)); 1050 return sptep; 1051 } 1052 1053 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_) \ 1054 for (_spte_ = rmap_get_first(_rmap_head_, _iter_); \ 1055 _spte_; _spte_ = rmap_get_next(_iter_)) 1056 1057 static void drop_spte(struct kvm *kvm, u64 *sptep) 1058 { 1059 if (mmu_spte_clear_track_bits(sptep)) 1060 rmap_remove(kvm, sptep); 1061 } 1062 1063 1064 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep) 1065 { 1066 if (is_large_pte(*sptep)) { 1067 WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K); 1068 drop_spte(kvm, sptep); 1069 --kvm->stat.lpages; 1070 return true; 1071 } 1072 1073 return false; 1074 } 1075 1076 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) 1077 { 1078 if (__drop_large_spte(vcpu->kvm, sptep)) { 1079 struct kvm_mmu_page *sp = sptep_to_sp(sptep); 1080 1081 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, 1082 KVM_PAGES_PER_HPAGE(sp->role.level)); 1083 } 1084 } 1085 1086 /* 1087 * Write-protect on the specified @sptep, @pt_protect indicates whether 1088 * spte write-protection is caused by protecting shadow page table. 1089 * 1090 * Note: write protection is difference between dirty logging and spte 1091 * protection: 1092 * - for dirty logging, the spte can be set to writable at anytime if 1093 * its dirty bitmap is properly set. 1094 * - for spte protection, the spte can be writable only after unsync-ing 1095 * shadow page. 1096 * 1097 * Return true if tlb need be flushed. 1098 */ 1099 static bool spte_write_protect(u64 *sptep, bool pt_protect) 1100 { 1101 u64 spte = *sptep; 1102 1103 if (!is_writable_pte(spte) && 1104 !(pt_protect && spte_can_locklessly_be_made_writable(spte))) 1105 return false; 1106 1107 rmap_printk("spte %p %llx\n", sptep, *sptep); 1108 1109 if (pt_protect) 1110 spte &= ~shadow_mmu_writable_mask; 1111 spte = spte & ~PT_WRITABLE_MASK; 1112 1113 return mmu_spte_update(sptep, spte); 1114 } 1115 1116 static bool __rmap_write_protect(struct kvm *kvm, 1117 struct kvm_rmap_head *rmap_head, 1118 bool pt_protect) 1119 { 1120 u64 *sptep; 1121 struct rmap_iterator iter; 1122 bool flush = false; 1123 1124 for_each_rmap_spte(rmap_head, &iter, sptep) 1125 flush |= spte_write_protect(sptep, pt_protect); 1126 1127 return flush; 1128 } 1129 1130 static bool spte_clear_dirty(u64 *sptep) 1131 { 1132 u64 spte = *sptep; 1133 1134 rmap_printk("spte %p %llx\n", sptep, *sptep); 1135 1136 MMU_WARN_ON(!spte_ad_enabled(spte)); 1137 spte &= ~shadow_dirty_mask; 1138 return mmu_spte_update(sptep, spte); 1139 } 1140 1141 static bool spte_wrprot_for_clear_dirty(u64 *sptep) 1142 { 1143 bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT, 1144 (unsigned long *)sptep); 1145 if (was_writable && !spte_ad_enabled(*sptep)) 1146 kvm_set_pfn_dirty(spte_to_pfn(*sptep)); 1147 1148 return was_writable; 1149 } 1150 1151 /* 1152 * Gets the GFN ready for another round of dirty logging by clearing the 1153 * - D bit on ad-enabled SPTEs, and 1154 * - W bit on ad-disabled SPTEs. 1155 * Returns true iff any D or W bits were cleared. 1156 */ 1157 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1158 struct kvm_memory_slot *slot) 1159 { 1160 u64 *sptep; 1161 struct rmap_iterator iter; 1162 bool flush = false; 1163 1164 for_each_rmap_spte(rmap_head, &iter, sptep) 1165 if (spte_ad_need_write_protect(*sptep)) 1166 flush |= spte_wrprot_for_clear_dirty(sptep); 1167 else 1168 flush |= spte_clear_dirty(sptep); 1169 1170 return flush; 1171 } 1172 1173 /** 1174 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages 1175 * @kvm: kvm instance 1176 * @slot: slot to protect 1177 * @gfn_offset: start of the BITS_PER_LONG pages we care about 1178 * @mask: indicates which pages we should protect 1179 * 1180 * Used when we do not need to care about huge page mappings: e.g. during dirty 1181 * logging we do not have any such mappings. 1182 */ 1183 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, 1184 struct kvm_memory_slot *slot, 1185 gfn_t gfn_offset, unsigned long mask) 1186 { 1187 struct kvm_rmap_head *rmap_head; 1188 1189 if (is_tdp_mmu_enabled(kvm)) 1190 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot, 1191 slot->base_gfn + gfn_offset, mask, true); 1192 while (mask) { 1193 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), 1194 PG_LEVEL_4K, slot); 1195 __rmap_write_protect(kvm, rmap_head, false); 1196 1197 /* clear the first set bit */ 1198 mask &= mask - 1; 1199 } 1200 } 1201 1202 /** 1203 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write 1204 * protect the page if the D-bit isn't supported. 1205 * @kvm: kvm instance 1206 * @slot: slot to clear D-bit 1207 * @gfn_offset: start of the BITS_PER_LONG pages we care about 1208 * @mask: indicates which pages we should clear D-bit 1209 * 1210 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap. 1211 */ 1212 static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1213 struct kvm_memory_slot *slot, 1214 gfn_t gfn_offset, unsigned long mask) 1215 { 1216 struct kvm_rmap_head *rmap_head; 1217 1218 if (is_tdp_mmu_enabled(kvm)) 1219 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot, 1220 slot->base_gfn + gfn_offset, mask, false); 1221 while (mask) { 1222 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), 1223 PG_LEVEL_4K, slot); 1224 __rmap_clear_dirty(kvm, rmap_head, slot); 1225 1226 /* clear the first set bit */ 1227 mask &= mask - 1; 1228 } 1229 } 1230 1231 /** 1232 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected 1233 * PT level pages. 1234 * 1235 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to 1236 * enable dirty logging for them. 1237 * 1238 * Used when we do not need to care about huge page mappings: e.g. during dirty 1239 * logging we do not have any such mappings. 1240 */ 1241 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 1242 struct kvm_memory_slot *slot, 1243 gfn_t gfn_offset, unsigned long mask) 1244 { 1245 if (kvm_x86_ops.cpu_dirty_log_size) 1246 kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask); 1247 else 1248 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); 1249 } 1250 1251 int kvm_cpu_dirty_log_size(void) 1252 { 1253 return kvm_x86_ops.cpu_dirty_log_size; 1254 } 1255 1256 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, 1257 struct kvm_memory_slot *slot, u64 gfn) 1258 { 1259 struct kvm_rmap_head *rmap_head; 1260 int i; 1261 bool write_protected = false; 1262 1263 for (i = PG_LEVEL_4K; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { 1264 rmap_head = __gfn_to_rmap(gfn, i, slot); 1265 write_protected |= __rmap_write_protect(kvm, rmap_head, true); 1266 } 1267 1268 if (is_tdp_mmu_enabled(kvm)) 1269 write_protected |= 1270 kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn); 1271 1272 return write_protected; 1273 } 1274 1275 static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) 1276 { 1277 struct kvm_memory_slot *slot; 1278 1279 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1280 return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn); 1281 } 1282 1283 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1284 struct kvm_memory_slot *slot) 1285 { 1286 u64 *sptep; 1287 struct rmap_iterator iter; 1288 bool flush = false; 1289 1290 while ((sptep = rmap_get_first(rmap_head, &iter))) { 1291 rmap_printk("spte %p %llx.\n", sptep, *sptep); 1292 1293 pte_list_remove(rmap_head, sptep); 1294 flush = true; 1295 } 1296 1297 return flush; 1298 } 1299 1300 static bool kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1301 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1302 pte_t unused) 1303 { 1304 return kvm_zap_rmapp(kvm, rmap_head, slot); 1305 } 1306 1307 static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1308 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1309 pte_t pte) 1310 { 1311 u64 *sptep; 1312 struct rmap_iterator iter; 1313 int need_flush = 0; 1314 u64 new_spte; 1315 kvm_pfn_t new_pfn; 1316 1317 WARN_ON(pte_huge(pte)); 1318 new_pfn = pte_pfn(pte); 1319 1320 restart: 1321 for_each_rmap_spte(rmap_head, &iter, sptep) { 1322 rmap_printk("spte %p %llx gfn %llx (%d)\n", 1323 sptep, *sptep, gfn, level); 1324 1325 need_flush = 1; 1326 1327 if (pte_write(pte)) { 1328 pte_list_remove(rmap_head, sptep); 1329 goto restart; 1330 } else { 1331 new_spte = kvm_mmu_changed_pte_notifier_make_spte( 1332 *sptep, new_pfn); 1333 1334 mmu_spte_clear_track_bits(sptep); 1335 mmu_spte_set(sptep, new_spte); 1336 } 1337 } 1338 1339 if (need_flush && kvm_available_flush_tlb_with_range()) { 1340 kvm_flush_remote_tlbs_with_address(kvm, gfn, 1); 1341 return 0; 1342 } 1343 1344 return need_flush; 1345 } 1346 1347 struct slot_rmap_walk_iterator { 1348 /* input fields. */ 1349 struct kvm_memory_slot *slot; 1350 gfn_t start_gfn; 1351 gfn_t end_gfn; 1352 int start_level; 1353 int end_level; 1354 1355 /* output fields. */ 1356 gfn_t gfn; 1357 struct kvm_rmap_head *rmap; 1358 int level; 1359 1360 /* private field. */ 1361 struct kvm_rmap_head *end_rmap; 1362 }; 1363 1364 static void 1365 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level) 1366 { 1367 iterator->level = level; 1368 iterator->gfn = iterator->start_gfn; 1369 iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot); 1370 iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level, 1371 iterator->slot); 1372 } 1373 1374 static void 1375 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator, 1376 struct kvm_memory_slot *slot, int start_level, 1377 int end_level, gfn_t start_gfn, gfn_t end_gfn) 1378 { 1379 iterator->slot = slot; 1380 iterator->start_level = start_level; 1381 iterator->end_level = end_level; 1382 iterator->start_gfn = start_gfn; 1383 iterator->end_gfn = end_gfn; 1384 1385 rmap_walk_init_level(iterator, iterator->start_level); 1386 } 1387 1388 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator) 1389 { 1390 return !!iterator->rmap; 1391 } 1392 1393 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator) 1394 { 1395 if (++iterator->rmap <= iterator->end_rmap) { 1396 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level)); 1397 return; 1398 } 1399 1400 if (++iterator->level > iterator->end_level) { 1401 iterator->rmap = NULL; 1402 return; 1403 } 1404 1405 rmap_walk_init_level(iterator, iterator->level); 1406 } 1407 1408 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_, \ 1409 _start_gfn, _end_gfn, _iter_) \ 1410 for (slot_rmap_walk_init(_iter_, _slot_, _start_level_, \ 1411 _end_level_, _start_gfn, _end_gfn); \ 1412 slot_rmap_walk_okay(_iter_); \ 1413 slot_rmap_walk_next(_iter_)) 1414 1415 typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1416 struct kvm_memory_slot *slot, gfn_t gfn, 1417 int level, pte_t pte); 1418 1419 static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm, 1420 struct kvm_gfn_range *range, 1421 rmap_handler_t handler) 1422 { 1423 struct slot_rmap_walk_iterator iterator; 1424 bool ret = false; 1425 1426 for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL, 1427 range->start, range->end - 1, &iterator) 1428 ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn, 1429 iterator.level, range->pte); 1430 1431 return ret; 1432 } 1433 1434 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) 1435 { 1436 bool flush; 1437 1438 flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp); 1439 1440 if (is_tdp_mmu_enabled(kvm)) 1441 flush |= kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush); 1442 1443 return flush; 1444 } 1445 1446 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 1447 { 1448 bool flush; 1449 1450 flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp); 1451 1452 if (is_tdp_mmu_enabled(kvm)) 1453 flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range); 1454 1455 return flush; 1456 } 1457 1458 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1459 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1460 pte_t unused) 1461 { 1462 u64 *sptep; 1463 struct rmap_iterator iter; 1464 int young = 0; 1465 1466 for_each_rmap_spte(rmap_head, &iter, sptep) 1467 young |= mmu_spte_age(sptep); 1468 1469 return young; 1470 } 1471 1472 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1473 struct kvm_memory_slot *slot, gfn_t gfn, 1474 int level, pte_t unused) 1475 { 1476 u64 *sptep; 1477 struct rmap_iterator iter; 1478 1479 for_each_rmap_spte(rmap_head, &iter, sptep) 1480 if (is_accessed_spte(*sptep)) 1481 return 1; 1482 return 0; 1483 } 1484 1485 #define RMAP_RECYCLE_THRESHOLD 1000 1486 1487 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) 1488 { 1489 struct kvm_rmap_head *rmap_head; 1490 struct kvm_mmu_page *sp; 1491 1492 sp = sptep_to_sp(spte); 1493 1494 rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp); 1495 1496 kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0)); 1497 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, 1498 KVM_PAGES_PER_HPAGE(sp->role.level)); 1499 } 1500 1501 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 1502 { 1503 bool young; 1504 1505 young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp); 1506 1507 if (is_tdp_mmu_enabled(kvm)) 1508 young |= kvm_tdp_mmu_age_gfn_range(kvm, range); 1509 1510 return young; 1511 } 1512 1513 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 1514 { 1515 bool young; 1516 1517 young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp); 1518 1519 if (is_tdp_mmu_enabled(kvm)) 1520 young |= kvm_tdp_mmu_test_age_gfn(kvm, range); 1521 1522 return young; 1523 } 1524 1525 #ifdef MMU_DEBUG 1526 static int is_empty_shadow_page(u64 *spt) 1527 { 1528 u64 *pos; 1529 u64 *end; 1530 1531 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) 1532 if (is_shadow_present_pte(*pos)) { 1533 printk(KERN_ERR "%s: %p %llx\n", __func__, 1534 pos, *pos); 1535 return 0; 1536 } 1537 return 1; 1538 } 1539 #endif 1540 1541 /* 1542 * This value is the sum of all of the kvm instances's 1543 * kvm->arch.n_used_mmu_pages values. We need a global, 1544 * aggregate version in order to make the slab shrinker 1545 * faster 1546 */ 1547 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, unsigned long nr) 1548 { 1549 kvm->arch.n_used_mmu_pages += nr; 1550 percpu_counter_add(&kvm_total_used_mmu_pages, nr); 1551 } 1552 1553 static void kvm_mmu_free_page(struct kvm_mmu_page *sp) 1554 { 1555 MMU_WARN_ON(!is_empty_shadow_page(sp->spt)); 1556 hlist_del(&sp->hash_link); 1557 list_del(&sp->link); 1558 free_page((unsigned long)sp->spt); 1559 if (!sp->role.direct) 1560 free_page((unsigned long)sp->gfns); 1561 kmem_cache_free(mmu_page_header_cache, sp); 1562 } 1563 1564 static unsigned kvm_page_table_hashfn(gfn_t gfn) 1565 { 1566 return hash_64(gfn, KVM_MMU_HASH_SHIFT); 1567 } 1568 1569 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, 1570 struct kvm_mmu_page *sp, u64 *parent_pte) 1571 { 1572 if (!parent_pte) 1573 return; 1574 1575 pte_list_add(vcpu, parent_pte, &sp->parent_ptes); 1576 } 1577 1578 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, 1579 u64 *parent_pte) 1580 { 1581 __pte_list_remove(parent_pte, &sp->parent_ptes); 1582 } 1583 1584 static void drop_parent_pte(struct kvm_mmu_page *sp, 1585 u64 *parent_pte) 1586 { 1587 mmu_page_remove_parent_pte(sp, parent_pte); 1588 mmu_spte_clear_no_track(parent_pte); 1589 } 1590 1591 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct) 1592 { 1593 struct kvm_mmu_page *sp; 1594 1595 sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 1596 sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 1597 if (!direct) 1598 sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache); 1599 set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 1600 1601 /* 1602 * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages() 1603 * depends on valid pages being added to the head of the list. See 1604 * comments in kvm_zap_obsolete_pages(). 1605 */ 1606 sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen; 1607 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); 1608 kvm_mod_used_mmu_pages(vcpu->kvm, +1); 1609 return sp; 1610 } 1611 1612 static void mark_unsync(u64 *spte); 1613 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp) 1614 { 1615 u64 *sptep; 1616 struct rmap_iterator iter; 1617 1618 for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) { 1619 mark_unsync(sptep); 1620 } 1621 } 1622 1623 static void mark_unsync(u64 *spte) 1624 { 1625 struct kvm_mmu_page *sp; 1626 unsigned int index; 1627 1628 sp = sptep_to_sp(spte); 1629 index = spte - sp->spt; 1630 if (__test_and_set_bit(index, sp->unsync_child_bitmap)) 1631 return; 1632 if (sp->unsync_children++) 1633 return; 1634 kvm_mmu_mark_parents_unsync(sp); 1635 } 1636 1637 static int nonpaging_sync_page(struct kvm_vcpu *vcpu, 1638 struct kvm_mmu_page *sp) 1639 { 1640 return 0; 1641 } 1642 1643 #define KVM_PAGE_ARRAY_NR 16 1644 1645 struct kvm_mmu_pages { 1646 struct mmu_page_and_offset { 1647 struct kvm_mmu_page *sp; 1648 unsigned int idx; 1649 } page[KVM_PAGE_ARRAY_NR]; 1650 unsigned int nr; 1651 }; 1652 1653 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp, 1654 int idx) 1655 { 1656 int i; 1657 1658 if (sp->unsync) 1659 for (i=0; i < pvec->nr; i++) 1660 if (pvec->page[i].sp == sp) 1661 return 0; 1662 1663 pvec->page[pvec->nr].sp = sp; 1664 pvec->page[pvec->nr].idx = idx; 1665 pvec->nr++; 1666 return (pvec->nr == KVM_PAGE_ARRAY_NR); 1667 } 1668 1669 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx) 1670 { 1671 --sp->unsync_children; 1672 WARN_ON((int)sp->unsync_children < 0); 1673 __clear_bit(idx, sp->unsync_child_bitmap); 1674 } 1675 1676 static int __mmu_unsync_walk(struct kvm_mmu_page *sp, 1677 struct kvm_mmu_pages *pvec) 1678 { 1679 int i, ret, nr_unsync_leaf = 0; 1680 1681 for_each_set_bit(i, sp->unsync_child_bitmap, 512) { 1682 struct kvm_mmu_page *child; 1683 u64 ent = sp->spt[i]; 1684 1685 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) { 1686 clear_unsync_child_bit(sp, i); 1687 continue; 1688 } 1689 1690 child = to_shadow_page(ent & PT64_BASE_ADDR_MASK); 1691 1692 if (child->unsync_children) { 1693 if (mmu_pages_add(pvec, child, i)) 1694 return -ENOSPC; 1695 1696 ret = __mmu_unsync_walk(child, pvec); 1697 if (!ret) { 1698 clear_unsync_child_bit(sp, i); 1699 continue; 1700 } else if (ret > 0) { 1701 nr_unsync_leaf += ret; 1702 } else 1703 return ret; 1704 } else if (child->unsync) { 1705 nr_unsync_leaf++; 1706 if (mmu_pages_add(pvec, child, i)) 1707 return -ENOSPC; 1708 } else 1709 clear_unsync_child_bit(sp, i); 1710 } 1711 1712 return nr_unsync_leaf; 1713 } 1714 1715 #define INVALID_INDEX (-1) 1716 1717 static int mmu_unsync_walk(struct kvm_mmu_page *sp, 1718 struct kvm_mmu_pages *pvec) 1719 { 1720 pvec->nr = 0; 1721 if (!sp->unsync_children) 1722 return 0; 1723 1724 mmu_pages_add(pvec, sp, INVALID_INDEX); 1725 return __mmu_unsync_walk(sp, pvec); 1726 } 1727 1728 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp) 1729 { 1730 WARN_ON(!sp->unsync); 1731 trace_kvm_mmu_sync_page(sp); 1732 sp->unsync = 0; 1733 --kvm->stat.mmu_unsync; 1734 } 1735 1736 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, 1737 struct list_head *invalid_list); 1738 static void kvm_mmu_commit_zap_page(struct kvm *kvm, 1739 struct list_head *invalid_list); 1740 1741 #define for_each_valid_sp(_kvm, _sp, _list) \ 1742 hlist_for_each_entry(_sp, _list, hash_link) \ 1743 if (is_obsolete_sp((_kvm), (_sp))) { \ 1744 } else 1745 1746 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \ 1747 for_each_valid_sp(_kvm, _sp, \ 1748 &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)]) \ 1749 if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else 1750 1751 static inline bool is_ept_sp(struct kvm_mmu_page *sp) 1752 { 1753 return sp->role.cr0_wp && sp->role.smap_andnot_wp; 1754 } 1755 1756 /* @sp->gfn should be write-protected at the call site */ 1757 static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 1758 struct list_head *invalid_list) 1759 { 1760 if ((!is_ept_sp(sp) && sp->role.gpte_is_8_bytes != !!is_pae(vcpu)) || 1761 vcpu->arch.mmu->sync_page(vcpu, sp) == 0) { 1762 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); 1763 return false; 1764 } 1765 1766 return true; 1767 } 1768 1769 static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, 1770 struct list_head *invalid_list, 1771 bool remote_flush) 1772 { 1773 if (!remote_flush && list_empty(invalid_list)) 1774 return false; 1775 1776 if (!list_empty(invalid_list)) 1777 kvm_mmu_commit_zap_page(kvm, invalid_list); 1778 else 1779 kvm_flush_remote_tlbs(kvm); 1780 return true; 1781 } 1782 1783 static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu, 1784 struct list_head *invalid_list, 1785 bool remote_flush, bool local_flush) 1786 { 1787 if (kvm_mmu_remote_flush_or_zap(vcpu->kvm, invalid_list, remote_flush)) 1788 return; 1789 1790 if (local_flush) 1791 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1792 } 1793 1794 #ifdef CONFIG_KVM_MMU_AUDIT 1795 #include "mmu_audit.c" 1796 #else 1797 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { } 1798 static void mmu_audit_disable(void) { } 1799 #endif 1800 1801 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 1802 { 1803 return sp->role.invalid || 1804 unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); 1805 } 1806 1807 static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 1808 struct list_head *invalid_list) 1809 { 1810 kvm_unlink_unsync_page(vcpu->kvm, sp); 1811 return __kvm_sync_page(vcpu, sp, invalid_list); 1812 } 1813 1814 /* @gfn should be write-protected at the call site */ 1815 static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, 1816 struct list_head *invalid_list) 1817 { 1818 struct kvm_mmu_page *s; 1819 bool ret = false; 1820 1821 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) { 1822 if (!s->unsync) 1823 continue; 1824 1825 WARN_ON(s->role.level != PG_LEVEL_4K); 1826 ret |= kvm_sync_page(vcpu, s, invalid_list); 1827 } 1828 1829 return ret; 1830 } 1831 1832 struct mmu_page_path { 1833 struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL]; 1834 unsigned int idx[PT64_ROOT_MAX_LEVEL]; 1835 }; 1836 1837 #define for_each_sp(pvec, sp, parents, i) \ 1838 for (i = mmu_pages_first(&pvec, &parents); \ 1839 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \ 1840 i = mmu_pages_next(&pvec, &parents, i)) 1841 1842 static int mmu_pages_next(struct kvm_mmu_pages *pvec, 1843 struct mmu_page_path *parents, 1844 int i) 1845 { 1846 int n; 1847 1848 for (n = i+1; n < pvec->nr; n++) { 1849 struct kvm_mmu_page *sp = pvec->page[n].sp; 1850 unsigned idx = pvec->page[n].idx; 1851 int level = sp->role.level; 1852 1853 parents->idx[level-1] = idx; 1854 if (level == PG_LEVEL_4K) 1855 break; 1856 1857 parents->parent[level-2] = sp; 1858 } 1859 1860 return n; 1861 } 1862 1863 static int mmu_pages_first(struct kvm_mmu_pages *pvec, 1864 struct mmu_page_path *parents) 1865 { 1866 struct kvm_mmu_page *sp; 1867 int level; 1868 1869 if (pvec->nr == 0) 1870 return 0; 1871 1872 WARN_ON(pvec->page[0].idx != INVALID_INDEX); 1873 1874 sp = pvec->page[0].sp; 1875 level = sp->role.level; 1876 WARN_ON(level == PG_LEVEL_4K); 1877 1878 parents->parent[level-2] = sp; 1879 1880 /* Also set up a sentinel. Further entries in pvec are all 1881 * children of sp, so this element is never overwritten. 1882 */ 1883 parents->parent[level-1] = NULL; 1884 return mmu_pages_next(pvec, parents, 0); 1885 } 1886 1887 static void mmu_pages_clear_parents(struct mmu_page_path *parents) 1888 { 1889 struct kvm_mmu_page *sp; 1890 unsigned int level = 0; 1891 1892 do { 1893 unsigned int idx = parents->idx[level]; 1894 sp = parents->parent[level]; 1895 if (!sp) 1896 return; 1897 1898 WARN_ON(idx == INVALID_INDEX); 1899 clear_unsync_child_bit(sp, idx); 1900 level++; 1901 } while (!sp->unsync_children); 1902 } 1903 1904 static void mmu_sync_children(struct kvm_vcpu *vcpu, 1905 struct kvm_mmu_page *parent) 1906 { 1907 int i; 1908 struct kvm_mmu_page *sp; 1909 struct mmu_page_path parents; 1910 struct kvm_mmu_pages pages; 1911 LIST_HEAD(invalid_list); 1912 bool flush = false; 1913 1914 while (mmu_unsync_walk(parent, &pages)) { 1915 bool protected = false; 1916 1917 for_each_sp(pages, sp, parents, i) 1918 protected |= rmap_write_protect(vcpu, sp->gfn); 1919 1920 if (protected) { 1921 kvm_flush_remote_tlbs(vcpu->kvm); 1922 flush = false; 1923 } 1924 1925 for_each_sp(pages, sp, parents, i) { 1926 flush |= kvm_sync_page(vcpu, sp, &invalid_list); 1927 mmu_pages_clear_parents(&parents); 1928 } 1929 if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) { 1930 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 1931 cond_resched_rwlock_write(&vcpu->kvm->mmu_lock); 1932 flush = false; 1933 } 1934 } 1935 1936 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 1937 } 1938 1939 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp) 1940 { 1941 atomic_set(&sp->write_flooding_count, 0); 1942 } 1943 1944 static void clear_sp_write_flooding_count(u64 *spte) 1945 { 1946 __clear_sp_write_flooding_count(sptep_to_sp(spte)); 1947 } 1948 1949 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, 1950 gfn_t gfn, 1951 gva_t gaddr, 1952 unsigned level, 1953 int direct, 1954 unsigned int access) 1955 { 1956 bool direct_mmu = vcpu->arch.mmu->direct_map; 1957 union kvm_mmu_page_role role; 1958 struct hlist_head *sp_list; 1959 unsigned quadrant; 1960 struct kvm_mmu_page *sp; 1961 bool need_sync = false; 1962 bool flush = false; 1963 int collisions = 0; 1964 LIST_HEAD(invalid_list); 1965 1966 role = vcpu->arch.mmu->mmu_role.base; 1967 role.level = level; 1968 role.direct = direct; 1969 if (role.direct) 1970 role.gpte_is_8_bytes = true; 1971 role.access = access; 1972 if (!direct_mmu && vcpu->arch.mmu->root_level <= PT32_ROOT_LEVEL) { 1973 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); 1974 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; 1975 role.quadrant = quadrant; 1976 } 1977 1978 sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]; 1979 for_each_valid_sp(vcpu->kvm, sp, sp_list) { 1980 if (sp->gfn != gfn) { 1981 collisions++; 1982 continue; 1983 } 1984 1985 if (!need_sync && sp->unsync) 1986 need_sync = true; 1987 1988 if (sp->role.word != role.word) 1989 continue; 1990 1991 if (direct_mmu) 1992 goto trace_get_page; 1993 1994 if (sp->unsync) { 1995 /* The page is good, but __kvm_sync_page might still end 1996 * up zapping it. If so, break in order to rebuild it. 1997 */ 1998 if (!__kvm_sync_page(vcpu, sp, &invalid_list)) 1999 break; 2000 2001 WARN_ON(!list_empty(&invalid_list)); 2002 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2003 } 2004 2005 if (sp->unsync_children) 2006 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 2007 2008 __clear_sp_write_flooding_count(sp); 2009 2010 trace_get_page: 2011 trace_kvm_mmu_get_page(sp, false); 2012 goto out; 2013 } 2014 2015 ++vcpu->kvm->stat.mmu_cache_miss; 2016 2017 sp = kvm_mmu_alloc_page(vcpu, direct); 2018 2019 sp->gfn = gfn; 2020 sp->role = role; 2021 hlist_add_head(&sp->hash_link, sp_list); 2022 if (!direct) { 2023 /* 2024 * we should do write protection before syncing pages 2025 * otherwise the content of the synced shadow page may 2026 * be inconsistent with guest page table. 2027 */ 2028 account_shadowed(vcpu->kvm, sp); 2029 if (level == PG_LEVEL_4K && rmap_write_protect(vcpu, gfn)) 2030 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1); 2031 2032 if (level > PG_LEVEL_4K && need_sync) 2033 flush |= kvm_sync_pages(vcpu, gfn, &invalid_list); 2034 } 2035 trace_kvm_mmu_get_page(sp, true); 2036 2037 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2038 out: 2039 if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions) 2040 vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions; 2041 return sp; 2042 } 2043 2044 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator, 2045 struct kvm_vcpu *vcpu, hpa_t root, 2046 u64 addr) 2047 { 2048 iterator->addr = addr; 2049 iterator->shadow_addr = root; 2050 iterator->level = vcpu->arch.mmu->shadow_root_level; 2051 2052 if (iterator->level == PT64_ROOT_4LEVEL && 2053 vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL && 2054 !vcpu->arch.mmu->direct_map) 2055 --iterator->level; 2056 2057 if (iterator->level == PT32E_ROOT_LEVEL) { 2058 /* 2059 * prev_root is currently only used for 64-bit hosts. So only 2060 * the active root_hpa is valid here. 2061 */ 2062 BUG_ON(root != vcpu->arch.mmu->root_hpa); 2063 2064 iterator->shadow_addr 2065 = vcpu->arch.mmu->pae_root[(addr >> 30) & 3]; 2066 iterator->shadow_addr &= PT64_BASE_ADDR_MASK; 2067 --iterator->level; 2068 if (!iterator->shadow_addr) 2069 iterator->level = 0; 2070 } 2071 } 2072 2073 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator, 2074 struct kvm_vcpu *vcpu, u64 addr) 2075 { 2076 shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa, 2077 addr); 2078 } 2079 2080 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator) 2081 { 2082 if (iterator->level < PG_LEVEL_4K) 2083 return false; 2084 2085 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level); 2086 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index; 2087 return true; 2088 } 2089 2090 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator, 2091 u64 spte) 2092 { 2093 if (is_last_spte(spte, iterator->level)) { 2094 iterator->level = 0; 2095 return; 2096 } 2097 2098 iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK; 2099 --iterator->level; 2100 } 2101 2102 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator) 2103 { 2104 __shadow_walk_next(iterator, *iterator->sptep); 2105 } 2106 2107 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep, 2108 struct kvm_mmu_page *sp) 2109 { 2110 u64 spte; 2111 2112 BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK); 2113 2114 spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp)); 2115 2116 mmu_spte_set(sptep, spte); 2117 2118 mmu_page_add_parent_pte(vcpu, sp, sptep); 2119 2120 if (sp->unsync_children || sp->unsync) 2121 mark_unsync(sptep); 2122 } 2123 2124 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2125 unsigned direct_access) 2126 { 2127 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) { 2128 struct kvm_mmu_page *child; 2129 2130 /* 2131 * For the direct sp, if the guest pte's dirty bit 2132 * changed form clean to dirty, it will corrupt the 2133 * sp's access: allow writable in the read-only sp, 2134 * so we should update the spte at this point to get 2135 * a new sp with the correct access. 2136 */ 2137 child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK); 2138 if (child->role.access == direct_access) 2139 return; 2140 2141 drop_parent_pte(child, sptep); 2142 kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1); 2143 } 2144 } 2145 2146 /* Returns the number of zapped non-leaf child shadow pages. */ 2147 static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, 2148 u64 *spte, struct list_head *invalid_list) 2149 { 2150 u64 pte; 2151 struct kvm_mmu_page *child; 2152 2153 pte = *spte; 2154 if (is_shadow_present_pte(pte)) { 2155 if (is_last_spte(pte, sp->role.level)) { 2156 drop_spte(kvm, spte); 2157 if (is_large_pte(pte)) 2158 --kvm->stat.lpages; 2159 } else { 2160 child = to_shadow_page(pte & PT64_BASE_ADDR_MASK); 2161 drop_parent_pte(child, spte); 2162 2163 /* 2164 * Recursively zap nested TDP SPs, parentless SPs are 2165 * unlikely to be used again in the near future. This 2166 * avoids retaining a large number of stale nested SPs. 2167 */ 2168 if (tdp_enabled && invalid_list && 2169 child->role.guest_mode && !child->parent_ptes.val) 2170 return kvm_mmu_prepare_zap_page(kvm, child, 2171 invalid_list); 2172 } 2173 } else if (is_mmio_spte(pte)) { 2174 mmu_spte_clear_no_track(spte); 2175 } 2176 return 0; 2177 } 2178 2179 static int kvm_mmu_page_unlink_children(struct kvm *kvm, 2180 struct kvm_mmu_page *sp, 2181 struct list_head *invalid_list) 2182 { 2183 int zapped = 0; 2184 unsigned i; 2185 2186 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) 2187 zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list); 2188 2189 return zapped; 2190 } 2191 2192 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp) 2193 { 2194 u64 *sptep; 2195 struct rmap_iterator iter; 2196 2197 while ((sptep = rmap_get_first(&sp->parent_ptes, &iter))) 2198 drop_parent_pte(sp, sptep); 2199 } 2200 2201 static int mmu_zap_unsync_children(struct kvm *kvm, 2202 struct kvm_mmu_page *parent, 2203 struct list_head *invalid_list) 2204 { 2205 int i, zapped = 0; 2206 struct mmu_page_path parents; 2207 struct kvm_mmu_pages pages; 2208 2209 if (parent->role.level == PG_LEVEL_4K) 2210 return 0; 2211 2212 while (mmu_unsync_walk(parent, &pages)) { 2213 struct kvm_mmu_page *sp; 2214 2215 for_each_sp(pages, sp, parents, i) { 2216 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 2217 mmu_pages_clear_parents(&parents); 2218 zapped++; 2219 } 2220 } 2221 2222 return zapped; 2223 } 2224 2225 static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, 2226 struct kvm_mmu_page *sp, 2227 struct list_head *invalid_list, 2228 int *nr_zapped) 2229 { 2230 bool list_unstable; 2231 2232 trace_kvm_mmu_prepare_zap_page(sp); 2233 ++kvm->stat.mmu_shadow_zapped; 2234 *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list); 2235 *nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list); 2236 kvm_mmu_unlink_parents(kvm, sp); 2237 2238 /* Zapping children means active_mmu_pages has become unstable. */ 2239 list_unstable = *nr_zapped; 2240 2241 if (!sp->role.invalid && !sp->role.direct) 2242 unaccount_shadowed(kvm, sp); 2243 2244 if (sp->unsync) 2245 kvm_unlink_unsync_page(kvm, sp); 2246 if (!sp->root_count) { 2247 /* Count self */ 2248 (*nr_zapped)++; 2249 2250 /* 2251 * Already invalid pages (previously active roots) are not on 2252 * the active page list. See list_del() in the "else" case of 2253 * !sp->root_count. 2254 */ 2255 if (sp->role.invalid) 2256 list_add(&sp->link, invalid_list); 2257 else 2258 list_move(&sp->link, invalid_list); 2259 kvm_mod_used_mmu_pages(kvm, -1); 2260 } else { 2261 /* 2262 * Remove the active root from the active page list, the root 2263 * will be explicitly freed when the root_count hits zero. 2264 */ 2265 list_del(&sp->link); 2266 2267 /* 2268 * Obsolete pages cannot be used on any vCPUs, see the comment 2269 * in kvm_mmu_zap_all_fast(). Note, is_obsolete_sp() also 2270 * treats invalid shadow pages as being obsolete. 2271 */ 2272 if (!is_obsolete_sp(kvm, sp)) 2273 kvm_reload_remote_mmus(kvm); 2274 } 2275 2276 if (sp->lpage_disallowed) 2277 unaccount_huge_nx_page(kvm, sp); 2278 2279 sp->role.invalid = 1; 2280 return list_unstable; 2281 } 2282 2283 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, 2284 struct list_head *invalid_list) 2285 { 2286 int nr_zapped; 2287 2288 __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped); 2289 return nr_zapped; 2290 } 2291 2292 static void kvm_mmu_commit_zap_page(struct kvm *kvm, 2293 struct list_head *invalid_list) 2294 { 2295 struct kvm_mmu_page *sp, *nsp; 2296 2297 if (list_empty(invalid_list)) 2298 return; 2299 2300 /* 2301 * We need to make sure everyone sees our modifications to 2302 * the page tables and see changes to vcpu->mode here. The barrier 2303 * in the kvm_flush_remote_tlbs() achieves this. This pairs 2304 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end. 2305 * 2306 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit 2307 * guest mode and/or lockless shadow page table walks. 2308 */ 2309 kvm_flush_remote_tlbs(kvm); 2310 2311 list_for_each_entry_safe(sp, nsp, invalid_list, link) { 2312 WARN_ON(!sp->role.invalid || sp->root_count); 2313 kvm_mmu_free_page(sp); 2314 } 2315 } 2316 2317 static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm, 2318 unsigned long nr_to_zap) 2319 { 2320 unsigned long total_zapped = 0; 2321 struct kvm_mmu_page *sp, *tmp; 2322 LIST_HEAD(invalid_list); 2323 bool unstable; 2324 int nr_zapped; 2325 2326 if (list_empty(&kvm->arch.active_mmu_pages)) 2327 return 0; 2328 2329 restart: 2330 list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) { 2331 /* 2332 * Don't zap active root pages, the page itself can't be freed 2333 * and zapping it will just force vCPUs to realloc and reload. 2334 */ 2335 if (sp->root_count) 2336 continue; 2337 2338 unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, 2339 &nr_zapped); 2340 total_zapped += nr_zapped; 2341 if (total_zapped >= nr_to_zap) 2342 break; 2343 2344 if (unstable) 2345 goto restart; 2346 } 2347 2348 kvm_mmu_commit_zap_page(kvm, &invalid_list); 2349 2350 kvm->stat.mmu_recycled += total_zapped; 2351 return total_zapped; 2352 } 2353 2354 static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm) 2355 { 2356 if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages) 2357 return kvm->arch.n_max_mmu_pages - 2358 kvm->arch.n_used_mmu_pages; 2359 2360 return 0; 2361 } 2362 2363 static int make_mmu_pages_available(struct kvm_vcpu *vcpu) 2364 { 2365 unsigned long avail = kvm_mmu_available_pages(vcpu->kvm); 2366 2367 if (likely(avail >= KVM_MIN_FREE_MMU_PAGES)) 2368 return 0; 2369 2370 kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail); 2371 2372 /* 2373 * Note, this check is intentionally soft, it only guarantees that one 2374 * page is available, while the caller may end up allocating as many as 2375 * four pages, e.g. for PAE roots or for 5-level paging. Temporarily 2376 * exceeding the (arbitrary by default) limit will not harm the host, 2377 * being too agressive may unnecessarily kill the guest, and getting an 2378 * exact count is far more trouble than it's worth, especially in the 2379 * page fault paths. 2380 */ 2381 if (!kvm_mmu_available_pages(vcpu->kvm)) 2382 return -ENOSPC; 2383 return 0; 2384 } 2385 2386 /* 2387 * Changing the number of mmu pages allocated to the vm 2388 * Note: if goal_nr_mmu_pages is too small, you will get dead lock 2389 */ 2390 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) 2391 { 2392 write_lock(&kvm->mmu_lock); 2393 2394 if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) { 2395 kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages - 2396 goal_nr_mmu_pages); 2397 2398 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages; 2399 } 2400 2401 kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages; 2402 2403 write_unlock(&kvm->mmu_lock); 2404 } 2405 2406 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) 2407 { 2408 struct kvm_mmu_page *sp; 2409 LIST_HEAD(invalid_list); 2410 int r; 2411 2412 pgprintk("%s: looking for gfn %llx\n", __func__, gfn); 2413 r = 0; 2414 write_lock(&kvm->mmu_lock); 2415 for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { 2416 pgprintk("%s: gfn %llx role %x\n", __func__, gfn, 2417 sp->role.word); 2418 r = 1; 2419 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); 2420 } 2421 kvm_mmu_commit_zap_page(kvm, &invalid_list); 2422 write_unlock(&kvm->mmu_lock); 2423 2424 return r; 2425 } 2426 2427 static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) 2428 { 2429 gpa_t gpa; 2430 int r; 2431 2432 if (vcpu->arch.mmu->direct_map) 2433 return 0; 2434 2435 gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); 2436 2437 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); 2438 2439 return r; 2440 } 2441 2442 static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) 2443 { 2444 trace_kvm_mmu_unsync_page(sp); 2445 ++vcpu->kvm->stat.mmu_unsync; 2446 sp->unsync = 1; 2447 2448 kvm_mmu_mark_parents_unsync(sp); 2449 } 2450 2451 bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, 2452 bool can_unsync) 2453 { 2454 struct kvm_mmu_page *sp; 2455 2456 if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) 2457 return true; 2458 2459 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { 2460 if (!can_unsync) 2461 return true; 2462 2463 if (sp->unsync) 2464 continue; 2465 2466 WARN_ON(sp->role.level != PG_LEVEL_4K); 2467 kvm_unsync_page(vcpu, sp); 2468 } 2469 2470 /* 2471 * We need to ensure that the marking of unsync pages is visible 2472 * before the SPTE is updated to allow writes because 2473 * kvm_mmu_sync_roots() checks the unsync flags without holding 2474 * the MMU lock and so can race with this. If the SPTE was updated 2475 * before the page had been marked as unsync-ed, something like the 2476 * following could happen: 2477 * 2478 * CPU 1 CPU 2 2479 * --------------------------------------------------------------------- 2480 * 1.2 Host updates SPTE 2481 * to be writable 2482 * 2.1 Guest writes a GPTE for GVA X. 2483 * (GPTE being in the guest page table shadowed 2484 * by the SP from CPU 1.) 2485 * This reads SPTE during the page table walk. 2486 * Since SPTE.W is read as 1, there is no 2487 * fault. 2488 * 2489 * 2.2 Guest issues TLB flush. 2490 * That causes a VM Exit. 2491 * 2492 * 2.3 kvm_mmu_sync_pages() reads sp->unsync. 2493 * Since it is false, so it just returns. 2494 * 2495 * 2.4 Guest accesses GVA X. 2496 * Since the mapping in the SP was not updated, 2497 * so the old mapping for GVA X incorrectly 2498 * gets used. 2499 * 1.1 Host marks SP 2500 * as unsync 2501 * (sp->unsync = true) 2502 * 2503 * The write barrier below ensures that 1.1 happens before 1.2 and thus 2504 * the situation in 2.4 does not arise. The implicit barrier in 2.2 2505 * pairs with this write barrier. 2506 */ 2507 smp_wmb(); 2508 2509 return false; 2510 } 2511 2512 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2513 unsigned int pte_access, int level, 2514 gfn_t gfn, kvm_pfn_t pfn, bool speculative, 2515 bool can_unsync, bool host_writable) 2516 { 2517 u64 spte; 2518 struct kvm_mmu_page *sp; 2519 int ret; 2520 2521 sp = sptep_to_sp(sptep); 2522 2523 ret = make_spte(vcpu, pte_access, level, gfn, pfn, *sptep, speculative, 2524 can_unsync, host_writable, sp_ad_disabled(sp), &spte); 2525 2526 if (spte & PT_WRITABLE_MASK) 2527 kvm_vcpu_mark_page_dirty(vcpu, gfn); 2528 2529 if (*sptep == spte) 2530 ret |= SET_SPTE_SPURIOUS; 2531 else if (mmu_spte_update(sptep, spte)) 2532 ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH; 2533 return ret; 2534 } 2535 2536 static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2537 unsigned int pte_access, bool write_fault, int level, 2538 gfn_t gfn, kvm_pfn_t pfn, bool speculative, 2539 bool host_writable) 2540 { 2541 int was_rmapped = 0; 2542 int rmap_count; 2543 int set_spte_ret; 2544 int ret = RET_PF_FIXED; 2545 bool flush = false; 2546 2547 pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__, 2548 *sptep, write_fault, gfn); 2549 2550 if (unlikely(is_noslot_pfn(pfn))) { 2551 mark_mmio_spte(vcpu, sptep, gfn, pte_access); 2552 return RET_PF_EMULATE; 2553 } 2554 2555 if (is_shadow_present_pte(*sptep)) { 2556 /* 2557 * If we overwrite a PTE page pointer with a 2MB PMD, unlink 2558 * the parent of the now unreachable PTE. 2559 */ 2560 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) { 2561 struct kvm_mmu_page *child; 2562 u64 pte = *sptep; 2563 2564 child = to_shadow_page(pte & PT64_BASE_ADDR_MASK); 2565 drop_parent_pte(child, sptep); 2566 flush = true; 2567 } else if (pfn != spte_to_pfn(*sptep)) { 2568 pgprintk("hfn old %llx new %llx\n", 2569 spte_to_pfn(*sptep), pfn); 2570 drop_spte(vcpu->kvm, sptep); 2571 flush = true; 2572 } else 2573 was_rmapped = 1; 2574 } 2575 2576 set_spte_ret = set_spte(vcpu, sptep, pte_access, level, gfn, pfn, 2577 speculative, true, host_writable); 2578 if (set_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { 2579 if (write_fault) 2580 ret = RET_PF_EMULATE; 2581 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2582 } 2583 2584 if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH || flush) 2585 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 2586 KVM_PAGES_PER_HPAGE(level)); 2587 2588 /* 2589 * The fault is fully spurious if and only if the new SPTE and old SPTE 2590 * are identical, and emulation is not required. 2591 */ 2592 if ((set_spte_ret & SET_SPTE_SPURIOUS) && ret == RET_PF_FIXED) { 2593 WARN_ON_ONCE(!was_rmapped); 2594 return RET_PF_SPURIOUS; 2595 } 2596 2597 pgprintk("%s: setting spte %llx\n", __func__, *sptep); 2598 trace_kvm_mmu_set_spte(level, gfn, sptep); 2599 if (!was_rmapped && is_large_pte(*sptep)) 2600 ++vcpu->kvm->stat.lpages; 2601 2602 if (is_shadow_present_pte(*sptep)) { 2603 if (!was_rmapped) { 2604 rmap_count = rmap_add(vcpu, sptep, gfn); 2605 if (rmap_count > RMAP_RECYCLE_THRESHOLD) 2606 rmap_recycle(vcpu, sptep, gfn); 2607 } 2608 } 2609 2610 return ret; 2611 } 2612 2613 static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, 2614 bool no_dirty_log) 2615 { 2616 struct kvm_memory_slot *slot; 2617 2618 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); 2619 if (!slot) 2620 return KVM_PFN_ERR_FAULT; 2621 2622 return gfn_to_pfn_memslot_atomic(slot, gfn); 2623 } 2624 2625 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, 2626 struct kvm_mmu_page *sp, 2627 u64 *start, u64 *end) 2628 { 2629 struct page *pages[PTE_PREFETCH_NUM]; 2630 struct kvm_memory_slot *slot; 2631 unsigned int access = sp->role.access; 2632 int i, ret; 2633 gfn_t gfn; 2634 2635 gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt); 2636 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK); 2637 if (!slot) 2638 return -1; 2639 2640 ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start); 2641 if (ret <= 0) 2642 return -1; 2643 2644 for (i = 0; i < ret; i++, gfn++, start++) { 2645 mmu_set_spte(vcpu, start, access, false, sp->role.level, gfn, 2646 page_to_pfn(pages[i]), true, true); 2647 put_page(pages[i]); 2648 } 2649 2650 return 0; 2651 } 2652 2653 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu, 2654 struct kvm_mmu_page *sp, u64 *sptep) 2655 { 2656 u64 *spte, *start = NULL; 2657 int i; 2658 2659 WARN_ON(!sp->role.direct); 2660 2661 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1); 2662 spte = sp->spt + i; 2663 2664 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) { 2665 if (is_shadow_present_pte(*spte) || spte == sptep) { 2666 if (!start) 2667 continue; 2668 if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0) 2669 break; 2670 start = NULL; 2671 } else if (!start) 2672 start = spte; 2673 } 2674 } 2675 2676 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) 2677 { 2678 struct kvm_mmu_page *sp; 2679 2680 sp = sptep_to_sp(sptep); 2681 2682 /* 2683 * Without accessed bits, there's no way to distinguish between 2684 * actually accessed translations and prefetched, so disable pte 2685 * prefetch if accessed bits aren't available. 2686 */ 2687 if (sp_ad_disabled(sp)) 2688 return; 2689 2690 if (sp->role.level > PG_LEVEL_4K) 2691 return; 2692 2693 /* 2694 * If addresses are being invalidated, skip prefetching to avoid 2695 * accidentally prefetching those addresses. 2696 */ 2697 if (unlikely(vcpu->kvm->mmu_notifier_count)) 2698 return; 2699 2700 __direct_pte_prefetch(vcpu, sp, sptep); 2701 } 2702 2703 static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 2704 const struct kvm_memory_slot *slot) 2705 { 2706 unsigned long hva; 2707 pte_t *pte; 2708 int level; 2709 2710 if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn)) 2711 return PG_LEVEL_4K; 2712 2713 /* 2714 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot() 2715 * is not solely for performance, it's also necessary to avoid the 2716 * "writable" check in __gfn_to_hva_many(), which will always fail on 2717 * read-only memslots due to gfn_to_hva() assuming writes. Earlier 2718 * page fault steps have already verified the guest isn't writing a 2719 * read-only memslot. 2720 */ 2721 hva = __gfn_to_hva_memslot(slot, gfn); 2722 2723 pte = lookup_address_in_mm(kvm->mm, hva, &level); 2724 if (unlikely(!pte)) 2725 return PG_LEVEL_4K; 2726 2727 return level; 2728 } 2729 2730 int kvm_mmu_max_mapping_level(struct kvm *kvm, 2731 const struct kvm_memory_slot *slot, gfn_t gfn, 2732 kvm_pfn_t pfn, int max_level) 2733 { 2734 struct kvm_lpage_info *linfo; 2735 2736 max_level = min(max_level, max_huge_page_level); 2737 for ( ; max_level > PG_LEVEL_4K; max_level--) { 2738 linfo = lpage_info_slot(gfn, slot, max_level); 2739 if (!linfo->disallow_lpage) 2740 break; 2741 } 2742 2743 if (max_level == PG_LEVEL_4K) 2744 return PG_LEVEL_4K; 2745 2746 return host_pfn_mapping_level(kvm, gfn, pfn, slot); 2747 } 2748 2749 int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, 2750 int max_level, kvm_pfn_t *pfnp, 2751 bool huge_page_disallowed, int *req_level) 2752 { 2753 struct kvm_memory_slot *slot; 2754 kvm_pfn_t pfn = *pfnp; 2755 kvm_pfn_t mask; 2756 int level; 2757 2758 *req_level = PG_LEVEL_4K; 2759 2760 if (unlikely(max_level == PG_LEVEL_4K)) 2761 return PG_LEVEL_4K; 2762 2763 if (is_error_noslot_pfn(pfn) || kvm_is_reserved_pfn(pfn)) 2764 return PG_LEVEL_4K; 2765 2766 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, true); 2767 if (!slot) 2768 return PG_LEVEL_4K; 2769 2770 level = kvm_mmu_max_mapping_level(vcpu->kvm, slot, gfn, pfn, max_level); 2771 if (level == PG_LEVEL_4K) 2772 return level; 2773 2774 *req_level = level = min(level, max_level); 2775 2776 /* 2777 * Enforce the iTLB multihit workaround after capturing the requested 2778 * level, which will be used to do precise, accurate accounting. 2779 */ 2780 if (huge_page_disallowed) 2781 return PG_LEVEL_4K; 2782 2783 /* 2784 * mmu_notifier_retry() was successful and mmu_lock is held, so 2785 * the pmd can't be split from under us. 2786 */ 2787 mask = KVM_PAGES_PER_HPAGE(level) - 1; 2788 VM_BUG_ON((gfn & mask) != (pfn & mask)); 2789 *pfnp = pfn & ~mask; 2790 2791 return level; 2792 } 2793 2794 void disallowed_hugepage_adjust(u64 spte, gfn_t gfn, int cur_level, 2795 kvm_pfn_t *pfnp, int *goal_levelp) 2796 { 2797 int level = *goal_levelp; 2798 2799 if (cur_level == level && level > PG_LEVEL_4K && 2800 is_shadow_present_pte(spte) && 2801 !is_large_pte(spte)) { 2802 /* 2803 * A small SPTE exists for this pfn, but FNAME(fetch) 2804 * and __direct_map would like to create a large PTE 2805 * instead: just force them to go down another level, 2806 * patching back for them into pfn the next 9 bits of 2807 * the address. 2808 */ 2809 u64 page_mask = KVM_PAGES_PER_HPAGE(level) - 2810 KVM_PAGES_PER_HPAGE(level - 1); 2811 *pfnp |= gfn & page_mask; 2812 (*goal_levelp)--; 2813 } 2814 } 2815 2816 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 2817 int map_writable, int max_level, kvm_pfn_t pfn, 2818 bool prefault, bool is_tdp) 2819 { 2820 bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(); 2821 bool write = error_code & PFERR_WRITE_MASK; 2822 bool exec = error_code & PFERR_FETCH_MASK; 2823 bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled; 2824 struct kvm_shadow_walk_iterator it; 2825 struct kvm_mmu_page *sp; 2826 int level, req_level, ret; 2827 gfn_t gfn = gpa >> PAGE_SHIFT; 2828 gfn_t base_gfn = gfn; 2829 2830 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 2831 return RET_PF_RETRY; 2832 2833 level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, 2834 huge_page_disallowed, &req_level); 2835 2836 trace_kvm_mmu_spte_requested(gpa, level, pfn); 2837 for_each_shadow_entry(vcpu, gpa, it) { 2838 /* 2839 * We cannot overwrite existing page tables with an NX 2840 * large page, as the leaf could be executable. 2841 */ 2842 if (nx_huge_page_workaround_enabled) 2843 disallowed_hugepage_adjust(*it.sptep, gfn, it.level, 2844 &pfn, &level); 2845 2846 base_gfn = gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); 2847 if (it.level == level) 2848 break; 2849 2850 drop_large_spte(vcpu, it.sptep); 2851 if (!is_shadow_present_pte(*it.sptep)) { 2852 sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr, 2853 it.level - 1, true, ACC_ALL); 2854 2855 link_shadow_page(vcpu, it.sptep, sp); 2856 if (is_tdp && huge_page_disallowed && 2857 req_level >= it.level) 2858 account_huge_nx_page(vcpu->kvm, sp); 2859 } 2860 } 2861 2862 ret = mmu_set_spte(vcpu, it.sptep, ACC_ALL, 2863 write, level, base_gfn, pfn, prefault, 2864 map_writable); 2865 if (ret == RET_PF_SPURIOUS) 2866 return ret; 2867 2868 direct_pte_prefetch(vcpu, it.sptep); 2869 ++vcpu->stat.pf_fixed; 2870 return ret; 2871 } 2872 2873 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk) 2874 { 2875 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk); 2876 } 2877 2878 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) 2879 { 2880 /* 2881 * Do not cache the mmio info caused by writing the readonly gfn 2882 * into the spte otherwise read access on readonly gfn also can 2883 * caused mmio page fault and treat it as mmio access. 2884 */ 2885 if (pfn == KVM_PFN_ERR_RO_FAULT) 2886 return RET_PF_EMULATE; 2887 2888 if (pfn == KVM_PFN_ERR_HWPOISON) { 2889 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current); 2890 return RET_PF_RETRY; 2891 } 2892 2893 return -EFAULT; 2894 } 2895 2896 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, 2897 kvm_pfn_t pfn, unsigned int access, 2898 int *ret_val) 2899 { 2900 /* The pfn is invalid, report the error! */ 2901 if (unlikely(is_error_pfn(pfn))) { 2902 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn); 2903 return true; 2904 } 2905 2906 if (unlikely(is_noslot_pfn(pfn))) { 2907 vcpu_cache_mmio_info(vcpu, gva, gfn, 2908 access & shadow_mmio_access_mask); 2909 /* 2910 * If MMIO caching is disabled, emulate immediately without 2911 * touching the shadow page tables as attempting to install an 2912 * MMIO SPTE will just be an expensive nop. 2913 */ 2914 if (unlikely(!shadow_mmio_value)) { 2915 *ret_val = RET_PF_EMULATE; 2916 return true; 2917 } 2918 } 2919 2920 return false; 2921 } 2922 2923 static bool page_fault_can_be_fast(u32 error_code) 2924 { 2925 /* 2926 * Do not fix the mmio spte with invalid generation number which 2927 * need to be updated by slow page fault path. 2928 */ 2929 if (unlikely(error_code & PFERR_RSVD_MASK)) 2930 return false; 2931 2932 /* See if the page fault is due to an NX violation */ 2933 if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)) 2934 == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)))) 2935 return false; 2936 2937 /* 2938 * #PF can be fast if: 2939 * 1. The shadow page table entry is not present, which could mean that 2940 * the fault is potentially caused by access tracking (if enabled). 2941 * 2. The shadow page table entry is present and the fault 2942 * is caused by write-protect, that means we just need change the W 2943 * bit of the spte which can be done out of mmu-lock. 2944 * 2945 * However, if access tracking is disabled we know that a non-present 2946 * page must be a genuine page fault where we have to create a new SPTE. 2947 * So, if access tracking is disabled, we return true only for write 2948 * accesses to a present page. 2949 */ 2950 2951 return shadow_acc_track_mask != 0 || 2952 ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)) 2953 == (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)); 2954 } 2955 2956 /* 2957 * Returns true if the SPTE was fixed successfully. Otherwise, 2958 * someone else modified the SPTE from its original value. 2959 */ 2960 static bool 2961 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 2962 u64 *sptep, u64 old_spte, u64 new_spte) 2963 { 2964 gfn_t gfn; 2965 2966 WARN_ON(!sp->role.direct); 2967 2968 /* 2969 * Theoretically we could also set dirty bit (and flush TLB) here in 2970 * order to eliminate unnecessary PML logging. See comments in 2971 * set_spte. But fast_page_fault is very unlikely to happen with PML 2972 * enabled, so we do not do this. This might result in the same GPA 2973 * to be logged in PML buffer again when the write really happens, and 2974 * eventually to be called by mark_page_dirty twice. But it's also no 2975 * harm. This also avoids the TLB flush needed after setting dirty bit 2976 * so non-PML cases won't be impacted. 2977 * 2978 * Compare with set_spte where instead shadow_dirty_mask is set. 2979 */ 2980 if (cmpxchg64(sptep, old_spte, new_spte) != old_spte) 2981 return false; 2982 2983 if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) { 2984 /* 2985 * The gfn of direct spte is stable since it is 2986 * calculated by sp->gfn. 2987 */ 2988 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); 2989 kvm_vcpu_mark_page_dirty(vcpu, gfn); 2990 } 2991 2992 return true; 2993 } 2994 2995 static bool is_access_allowed(u32 fault_err_code, u64 spte) 2996 { 2997 if (fault_err_code & PFERR_FETCH_MASK) 2998 return is_executable_pte(spte); 2999 3000 if (fault_err_code & PFERR_WRITE_MASK) 3001 return is_writable_pte(spte); 3002 3003 /* Fault was on Read access */ 3004 return spte & PT_PRESENT_MASK; 3005 } 3006 3007 /* 3008 * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS. 3009 */ 3010 static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 3011 u32 error_code) 3012 { 3013 struct kvm_shadow_walk_iterator iterator; 3014 struct kvm_mmu_page *sp; 3015 int ret = RET_PF_INVALID; 3016 u64 spte = 0ull; 3017 uint retry_count = 0; 3018 3019 if (!page_fault_can_be_fast(error_code)) 3020 return ret; 3021 3022 walk_shadow_page_lockless_begin(vcpu); 3023 3024 do { 3025 u64 new_spte; 3026 3027 for_each_shadow_entry_lockless(vcpu, cr2_or_gpa, iterator, spte) 3028 if (!is_shadow_present_pte(spte)) 3029 break; 3030 3031 if (!is_shadow_present_pte(spte)) 3032 break; 3033 3034 sp = sptep_to_sp(iterator.sptep); 3035 if (!is_last_spte(spte, sp->role.level)) 3036 break; 3037 3038 /* 3039 * Check whether the memory access that caused the fault would 3040 * still cause it if it were to be performed right now. If not, 3041 * then this is a spurious fault caused by TLB lazily flushed, 3042 * or some other CPU has already fixed the PTE after the 3043 * current CPU took the fault. 3044 * 3045 * Need not check the access of upper level table entries since 3046 * they are always ACC_ALL. 3047 */ 3048 if (is_access_allowed(error_code, spte)) { 3049 ret = RET_PF_SPURIOUS; 3050 break; 3051 } 3052 3053 new_spte = spte; 3054 3055 if (is_access_track_spte(spte)) 3056 new_spte = restore_acc_track_spte(new_spte); 3057 3058 /* 3059 * Currently, to simplify the code, write-protection can 3060 * be removed in the fast path only if the SPTE was 3061 * write-protected for dirty-logging or access tracking. 3062 */ 3063 if ((error_code & PFERR_WRITE_MASK) && 3064 spte_can_locklessly_be_made_writable(spte)) { 3065 new_spte |= PT_WRITABLE_MASK; 3066 3067 /* 3068 * Do not fix write-permission on the large spte. Since 3069 * we only dirty the first page into the dirty-bitmap in 3070 * fast_pf_fix_direct_spte(), other pages are missed 3071 * if its slot has dirty logging enabled. 3072 * 3073 * Instead, we let the slow page fault path create a 3074 * normal spte to fix the access. 3075 * 3076 * See the comments in kvm_arch_commit_memory_region(). 3077 */ 3078 if (sp->role.level > PG_LEVEL_4K) 3079 break; 3080 } 3081 3082 /* Verify that the fault can be handled in the fast path */ 3083 if (new_spte == spte || 3084 !is_access_allowed(error_code, new_spte)) 3085 break; 3086 3087 /* 3088 * Currently, fast page fault only works for direct mapping 3089 * since the gfn is not stable for indirect shadow page. See 3090 * Documentation/virt/kvm/locking.rst to get more detail. 3091 */ 3092 if (fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte, 3093 new_spte)) { 3094 ret = RET_PF_FIXED; 3095 break; 3096 } 3097 3098 if (++retry_count > 4) { 3099 printk_once(KERN_WARNING 3100 "kvm: Fast #PF retrying more than 4 times.\n"); 3101 break; 3102 } 3103 3104 } while (true); 3105 3106 trace_fast_page_fault(vcpu, cr2_or_gpa, error_code, iterator.sptep, 3107 spte, ret); 3108 walk_shadow_page_lockless_end(vcpu); 3109 3110 return ret; 3111 } 3112 3113 static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa, 3114 struct list_head *invalid_list) 3115 { 3116 struct kvm_mmu_page *sp; 3117 3118 if (!VALID_PAGE(*root_hpa)) 3119 return; 3120 3121 sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK); 3122 3123 if (is_tdp_mmu_page(sp)) 3124 kvm_tdp_mmu_put_root(kvm, sp, false); 3125 else if (!--sp->root_count && sp->role.invalid) 3126 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 3127 3128 *root_hpa = INVALID_PAGE; 3129 } 3130 3131 /* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */ 3132 void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 3133 ulong roots_to_free) 3134 { 3135 struct kvm *kvm = vcpu->kvm; 3136 int i; 3137 LIST_HEAD(invalid_list); 3138 bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT; 3139 3140 BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG); 3141 3142 /* Before acquiring the MMU lock, see if we need to do any real work. */ 3143 if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) { 3144 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 3145 if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) && 3146 VALID_PAGE(mmu->prev_roots[i].hpa)) 3147 break; 3148 3149 if (i == KVM_MMU_NUM_PREV_ROOTS) 3150 return; 3151 } 3152 3153 write_lock(&kvm->mmu_lock); 3154 3155 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 3156 if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) 3157 mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa, 3158 &invalid_list); 3159 3160 if (free_active_root) { 3161 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && 3162 (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) { 3163 mmu_free_root_page(kvm, &mmu->root_hpa, &invalid_list); 3164 } else if (mmu->pae_root) { 3165 for (i = 0; i < 4; ++i) { 3166 if (!IS_VALID_PAE_ROOT(mmu->pae_root[i])) 3167 continue; 3168 3169 mmu_free_root_page(kvm, &mmu->pae_root[i], 3170 &invalid_list); 3171 mmu->pae_root[i] = INVALID_PAE_ROOT; 3172 } 3173 } 3174 mmu->root_hpa = INVALID_PAGE; 3175 mmu->root_pgd = 0; 3176 } 3177 3178 kvm_mmu_commit_zap_page(kvm, &invalid_list); 3179 write_unlock(&kvm->mmu_lock); 3180 } 3181 EXPORT_SYMBOL_GPL(kvm_mmu_free_roots); 3182 3183 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn) 3184 { 3185 int ret = 0; 3186 3187 if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) { 3188 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3189 ret = 1; 3190 } 3191 3192 return ret; 3193 } 3194 3195 static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, 3196 u8 level, bool direct) 3197 { 3198 struct kvm_mmu_page *sp; 3199 3200 sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL); 3201 ++sp->root_count; 3202 3203 return __pa(sp->spt); 3204 } 3205 3206 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) 3207 { 3208 struct kvm_mmu *mmu = vcpu->arch.mmu; 3209 u8 shadow_root_level = mmu->shadow_root_level; 3210 hpa_t root; 3211 unsigned i; 3212 int r; 3213 3214 write_lock(&vcpu->kvm->mmu_lock); 3215 r = make_mmu_pages_available(vcpu); 3216 if (r < 0) 3217 goto out_unlock; 3218 3219 if (is_tdp_mmu_enabled(vcpu->kvm)) { 3220 root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu); 3221 mmu->root_hpa = root; 3222 } else if (shadow_root_level >= PT64_ROOT_4LEVEL) { 3223 root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true); 3224 mmu->root_hpa = root; 3225 } else if (shadow_root_level == PT32E_ROOT_LEVEL) { 3226 if (WARN_ON_ONCE(!mmu->pae_root)) { 3227 r = -EIO; 3228 goto out_unlock; 3229 } 3230 3231 for (i = 0; i < 4; ++i) { 3232 WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i])); 3233 3234 root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), 3235 i << 30, PT32_ROOT_LEVEL, true); 3236 mmu->pae_root[i] = root | PT_PRESENT_MASK | 3237 shadow_me_mask; 3238 } 3239 mmu->root_hpa = __pa(mmu->pae_root); 3240 } else { 3241 WARN_ONCE(1, "Bad TDP root level = %d\n", shadow_root_level); 3242 r = -EIO; 3243 goto out_unlock; 3244 } 3245 3246 /* root_pgd is ignored for direct MMUs. */ 3247 mmu->root_pgd = 0; 3248 out_unlock: 3249 write_unlock(&vcpu->kvm->mmu_lock); 3250 return r; 3251 } 3252 3253 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) 3254 { 3255 struct kvm_mmu *mmu = vcpu->arch.mmu; 3256 u64 pdptrs[4], pm_mask; 3257 gfn_t root_gfn, root_pgd; 3258 hpa_t root; 3259 unsigned i; 3260 int r; 3261 3262 root_pgd = mmu->get_guest_pgd(vcpu); 3263 root_gfn = root_pgd >> PAGE_SHIFT; 3264 3265 if (mmu_check_root(vcpu, root_gfn)) 3266 return 1; 3267 3268 /* 3269 * On SVM, reading PDPTRs might access guest memory, which might fault 3270 * and thus might sleep. Grab the PDPTRs before acquiring mmu_lock. 3271 */ 3272 if (mmu->root_level == PT32E_ROOT_LEVEL) { 3273 for (i = 0; i < 4; ++i) { 3274 pdptrs[i] = mmu->get_pdptr(vcpu, i); 3275 if (!(pdptrs[i] & PT_PRESENT_MASK)) 3276 continue; 3277 3278 if (mmu_check_root(vcpu, pdptrs[i] >> PAGE_SHIFT)) 3279 return 1; 3280 } 3281 } 3282 3283 write_lock(&vcpu->kvm->mmu_lock); 3284 r = make_mmu_pages_available(vcpu); 3285 if (r < 0) 3286 goto out_unlock; 3287 3288 /* 3289 * Do we shadow a long mode page table? If so we need to 3290 * write-protect the guests page table root. 3291 */ 3292 if (mmu->root_level >= PT64_ROOT_4LEVEL) { 3293 root = mmu_alloc_root(vcpu, root_gfn, 0, 3294 mmu->shadow_root_level, false); 3295 mmu->root_hpa = root; 3296 goto set_root_pgd; 3297 } 3298 3299 if (WARN_ON_ONCE(!mmu->pae_root)) { 3300 r = -EIO; 3301 goto out_unlock; 3302 } 3303 3304 /* 3305 * We shadow a 32 bit page table. This may be a legacy 2-level 3306 * or a PAE 3-level page table. In either case we need to be aware that 3307 * the shadow page table may be a PAE or a long mode page table. 3308 */ 3309 pm_mask = PT_PRESENT_MASK | shadow_me_mask; 3310 if (mmu->shadow_root_level == PT64_ROOT_4LEVEL) { 3311 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 3312 3313 if (WARN_ON_ONCE(!mmu->pml4_root)) { 3314 r = -EIO; 3315 goto out_unlock; 3316 } 3317 3318 mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask; 3319 } 3320 3321 for (i = 0; i < 4; ++i) { 3322 WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i])); 3323 3324 if (mmu->root_level == PT32E_ROOT_LEVEL) { 3325 if (!(pdptrs[i] & PT_PRESENT_MASK)) { 3326 mmu->pae_root[i] = INVALID_PAE_ROOT; 3327 continue; 3328 } 3329 root_gfn = pdptrs[i] >> PAGE_SHIFT; 3330 } 3331 3332 root = mmu_alloc_root(vcpu, root_gfn, i << 30, 3333 PT32_ROOT_LEVEL, false); 3334 mmu->pae_root[i] = root | pm_mask; 3335 } 3336 3337 if (mmu->shadow_root_level == PT64_ROOT_4LEVEL) 3338 mmu->root_hpa = __pa(mmu->pml4_root); 3339 else 3340 mmu->root_hpa = __pa(mmu->pae_root); 3341 3342 set_root_pgd: 3343 mmu->root_pgd = root_pgd; 3344 out_unlock: 3345 write_unlock(&vcpu->kvm->mmu_lock); 3346 3347 return 0; 3348 } 3349 3350 static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) 3351 { 3352 struct kvm_mmu *mmu = vcpu->arch.mmu; 3353 u64 *pml4_root, *pae_root; 3354 3355 /* 3356 * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP 3357 * tables are allocated and initialized at root creation as there is no 3358 * equivalent level in the guest's NPT to shadow. Allocate the tables 3359 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare. 3360 */ 3361 if (mmu->direct_map || mmu->root_level >= PT64_ROOT_4LEVEL || 3362 mmu->shadow_root_level < PT64_ROOT_4LEVEL) 3363 return 0; 3364 3365 /* 3366 * This mess only works with 4-level paging and needs to be updated to 3367 * work with 5-level paging. 3368 */ 3369 if (WARN_ON_ONCE(mmu->shadow_root_level != PT64_ROOT_4LEVEL)) 3370 return -EIO; 3371 3372 if (mmu->pae_root && mmu->pml4_root) 3373 return 0; 3374 3375 /* 3376 * The special roots should always be allocated in concert. Yell and 3377 * bail if KVM ends up in a state where only one of the roots is valid. 3378 */ 3379 if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root)) 3380 return -EIO; 3381 3382 /* 3383 * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and 3384 * doesn't need to be decrypted. 3385 */ 3386 pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 3387 if (!pae_root) 3388 return -ENOMEM; 3389 3390 pml4_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 3391 if (!pml4_root) { 3392 free_page((unsigned long)pae_root); 3393 return -ENOMEM; 3394 } 3395 3396 mmu->pae_root = pae_root; 3397 mmu->pml4_root = pml4_root; 3398 3399 return 0; 3400 } 3401 3402 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) 3403 { 3404 int i; 3405 struct kvm_mmu_page *sp; 3406 3407 if (vcpu->arch.mmu->direct_map) 3408 return; 3409 3410 if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) 3411 return; 3412 3413 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); 3414 3415 if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { 3416 hpa_t root = vcpu->arch.mmu->root_hpa; 3417 sp = to_shadow_page(root); 3418 3419 /* 3420 * Even if another CPU was marking the SP as unsync-ed 3421 * simultaneously, any guest page table changes are not 3422 * guaranteed to be visible anyway until this VCPU issues a TLB 3423 * flush strictly after those changes are made. We only need to 3424 * ensure that the other CPU sets these flags before any actual 3425 * changes to the page tables are made. The comments in 3426 * mmu_need_write_protect() describe what could go wrong if this 3427 * requirement isn't satisfied. 3428 */ 3429 if (!smp_load_acquire(&sp->unsync) && 3430 !smp_load_acquire(&sp->unsync_children)) 3431 return; 3432 3433 write_lock(&vcpu->kvm->mmu_lock); 3434 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); 3435 3436 mmu_sync_children(vcpu, sp); 3437 3438 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); 3439 write_unlock(&vcpu->kvm->mmu_lock); 3440 return; 3441 } 3442 3443 write_lock(&vcpu->kvm->mmu_lock); 3444 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); 3445 3446 for (i = 0; i < 4; ++i) { 3447 hpa_t root = vcpu->arch.mmu->pae_root[i]; 3448 3449 if (IS_VALID_PAE_ROOT(root)) { 3450 root &= PT64_BASE_ADDR_MASK; 3451 sp = to_shadow_page(root); 3452 mmu_sync_children(vcpu, sp); 3453 } 3454 } 3455 3456 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); 3457 write_unlock(&vcpu->kvm->mmu_lock); 3458 } 3459 3460 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr, 3461 u32 access, struct x86_exception *exception) 3462 { 3463 if (exception) 3464 exception->error_code = 0; 3465 return vaddr; 3466 } 3467 3468 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gpa_t vaddr, 3469 u32 access, 3470 struct x86_exception *exception) 3471 { 3472 if (exception) 3473 exception->error_code = 0; 3474 return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception); 3475 } 3476 3477 static bool 3478 __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level) 3479 { 3480 int bit7 = (pte >> 7) & 1; 3481 3482 return pte & rsvd_check->rsvd_bits_mask[bit7][level-1]; 3483 } 3484 3485 static bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check, u64 pte) 3486 { 3487 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f); 3488 } 3489 3490 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct) 3491 { 3492 /* 3493 * A nested guest cannot use the MMIO cache if it is using nested 3494 * page tables, because cr2 is a nGPA while the cache stores GPAs. 3495 */ 3496 if (mmu_is_nested(vcpu)) 3497 return false; 3498 3499 if (direct) 3500 return vcpu_match_mmio_gpa(vcpu, addr); 3501 3502 return vcpu_match_mmio_gva(vcpu, addr); 3503 } 3504 3505 /* 3506 * Return the level of the lowest level SPTE added to sptes. 3507 * That SPTE may be non-present. 3508 */ 3509 static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level) 3510 { 3511 struct kvm_shadow_walk_iterator iterator; 3512 int leaf = -1; 3513 u64 spte; 3514 3515 walk_shadow_page_lockless_begin(vcpu); 3516 3517 for (shadow_walk_init(&iterator, vcpu, addr), 3518 *root_level = iterator.level; 3519 shadow_walk_okay(&iterator); 3520 __shadow_walk_next(&iterator, spte)) { 3521 leaf = iterator.level; 3522 spte = mmu_spte_get_lockless(iterator.sptep); 3523 3524 sptes[leaf] = spte; 3525 3526 if (!is_shadow_present_pte(spte)) 3527 break; 3528 } 3529 3530 walk_shadow_page_lockless_end(vcpu); 3531 3532 return leaf; 3533 } 3534 3535 /* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */ 3536 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) 3537 { 3538 u64 sptes[PT64_ROOT_MAX_LEVEL + 1]; 3539 struct rsvd_bits_validate *rsvd_check; 3540 int root, leaf, level; 3541 bool reserved = false; 3542 3543 if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) { 3544 *sptep = 0ull; 3545 return reserved; 3546 } 3547 3548 if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) 3549 leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root); 3550 else 3551 leaf = get_walk(vcpu, addr, sptes, &root); 3552 3553 if (unlikely(leaf < 0)) { 3554 *sptep = 0ull; 3555 return reserved; 3556 } 3557 3558 *sptep = sptes[leaf]; 3559 3560 /* 3561 * Skip reserved bits checks on the terminal leaf if it's not a valid 3562 * SPTE. Note, this also (intentionally) skips MMIO SPTEs, which, by 3563 * design, always have reserved bits set. The purpose of the checks is 3564 * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs. 3565 */ 3566 if (!is_shadow_present_pte(sptes[leaf])) 3567 leaf++; 3568 3569 rsvd_check = &vcpu->arch.mmu->shadow_zero_check; 3570 3571 for (level = root; level >= leaf; level--) 3572 /* 3573 * Use a bitwise-OR instead of a logical-OR to aggregate the 3574 * reserved bit and EPT's invalid memtype/XWR checks to avoid 3575 * adding a Jcc in the loop. 3576 */ 3577 reserved |= __is_bad_mt_xwr(rsvd_check, sptes[level]) | 3578 __is_rsvd_bits_set(rsvd_check, sptes[level], level); 3579 3580 if (reserved) { 3581 pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n", 3582 __func__, addr); 3583 for (level = root; level >= leaf; level--) 3584 pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx", 3585 sptes[level], level, 3586 rsvd_check->rsvd_bits_mask[(sptes[level] >> 7) & 1][level-1]); 3587 } 3588 3589 return reserved; 3590 } 3591 3592 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) 3593 { 3594 u64 spte; 3595 bool reserved; 3596 3597 if (mmio_info_in_cache(vcpu, addr, direct)) 3598 return RET_PF_EMULATE; 3599 3600 reserved = get_mmio_spte(vcpu, addr, &spte); 3601 if (WARN_ON(reserved)) 3602 return -EINVAL; 3603 3604 if (is_mmio_spte(spte)) { 3605 gfn_t gfn = get_mmio_spte_gfn(spte); 3606 unsigned int access = get_mmio_spte_access(spte); 3607 3608 if (!check_mmio_spte(vcpu, spte)) 3609 return RET_PF_INVALID; 3610 3611 if (direct) 3612 addr = 0; 3613 3614 trace_handle_mmio_page_fault(addr, gfn, access); 3615 vcpu_cache_mmio_info(vcpu, addr, gfn, access); 3616 return RET_PF_EMULATE; 3617 } 3618 3619 /* 3620 * If the page table is zapped by other cpus, let CPU fault again on 3621 * the address. 3622 */ 3623 return RET_PF_RETRY; 3624 } 3625 3626 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, 3627 u32 error_code, gfn_t gfn) 3628 { 3629 if (unlikely(error_code & PFERR_RSVD_MASK)) 3630 return false; 3631 3632 if (!(error_code & PFERR_PRESENT_MASK) || 3633 !(error_code & PFERR_WRITE_MASK)) 3634 return false; 3635 3636 /* 3637 * guest is writing the page which is write tracked which can 3638 * not be fixed by page fault handler. 3639 */ 3640 if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) 3641 return true; 3642 3643 return false; 3644 } 3645 3646 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr) 3647 { 3648 struct kvm_shadow_walk_iterator iterator; 3649 u64 spte; 3650 3651 walk_shadow_page_lockless_begin(vcpu); 3652 for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) { 3653 clear_sp_write_flooding_count(iterator.sptep); 3654 if (!is_shadow_present_pte(spte)) 3655 break; 3656 } 3657 walk_shadow_page_lockless_end(vcpu); 3658 } 3659 3660 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 3661 gfn_t gfn) 3662 { 3663 struct kvm_arch_async_pf arch; 3664 3665 arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; 3666 arch.gfn = gfn; 3667 arch.direct_map = vcpu->arch.mmu->direct_map; 3668 arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); 3669 3670 return kvm_setup_async_pf(vcpu, cr2_or_gpa, 3671 kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); 3672 } 3673 3674 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, 3675 gpa_t cr2_or_gpa, kvm_pfn_t *pfn, hva_t *hva, 3676 bool write, bool *writable) 3677 { 3678 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 3679 bool async; 3680 3681 /* 3682 * Retry the page fault if the gfn hit a memslot that is being deleted 3683 * or moved. This ensures any existing SPTEs for the old memslot will 3684 * be zapped before KVM inserts a new MMIO SPTE for the gfn. 3685 */ 3686 if (slot && (slot->flags & KVM_MEMSLOT_INVALID)) 3687 return true; 3688 3689 /* Don't expose private memslots to L2. */ 3690 if (is_guest_mode(vcpu) && !kvm_is_visible_memslot(slot)) { 3691 *pfn = KVM_PFN_NOSLOT; 3692 *writable = false; 3693 return false; 3694 } 3695 3696 async = false; 3697 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, 3698 write, writable, hva); 3699 if (!async) 3700 return false; /* *pfn has correct page already */ 3701 3702 if (!prefault && kvm_can_do_async_pf(vcpu)) { 3703 trace_kvm_try_async_get_page(cr2_or_gpa, gfn); 3704 if (kvm_find_async_pf_gfn(vcpu, gfn)) { 3705 trace_kvm_async_pf_doublefault(cr2_or_gpa, gfn); 3706 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 3707 return true; 3708 } else if (kvm_arch_setup_async_pf(vcpu, cr2_or_gpa, gfn)) 3709 return true; 3710 } 3711 3712 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, 3713 write, writable, hva); 3714 return false; 3715 } 3716 3717 static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 3718 bool prefault, int max_level, bool is_tdp) 3719 { 3720 bool write = error_code & PFERR_WRITE_MASK; 3721 bool map_writable; 3722 3723 gfn_t gfn = gpa >> PAGE_SHIFT; 3724 unsigned long mmu_seq; 3725 kvm_pfn_t pfn; 3726 hva_t hva; 3727 int r; 3728 3729 if (page_fault_handle_page_track(vcpu, error_code, gfn)) 3730 return RET_PF_EMULATE; 3731 3732 if (!is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) { 3733 r = fast_page_fault(vcpu, gpa, error_code); 3734 if (r != RET_PF_INVALID) 3735 return r; 3736 } 3737 3738 r = mmu_topup_memory_caches(vcpu, false); 3739 if (r) 3740 return r; 3741 3742 mmu_seq = vcpu->kvm->mmu_notifier_seq; 3743 smp_rmb(); 3744 3745 if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, &hva, 3746 write, &map_writable)) 3747 return RET_PF_RETRY; 3748 3749 if (handle_abnormal_pfn(vcpu, is_tdp ? 0 : gpa, gfn, pfn, ACC_ALL, &r)) 3750 return r; 3751 3752 r = RET_PF_RETRY; 3753 3754 if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) 3755 read_lock(&vcpu->kvm->mmu_lock); 3756 else 3757 write_lock(&vcpu->kvm->mmu_lock); 3758 3759 if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) 3760 goto out_unlock; 3761 r = make_mmu_pages_available(vcpu); 3762 if (r) 3763 goto out_unlock; 3764 3765 if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) 3766 r = kvm_tdp_mmu_map(vcpu, gpa, error_code, map_writable, max_level, 3767 pfn, prefault); 3768 else 3769 r = __direct_map(vcpu, gpa, error_code, map_writable, max_level, pfn, 3770 prefault, is_tdp); 3771 3772 out_unlock: 3773 if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) 3774 read_unlock(&vcpu->kvm->mmu_lock); 3775 else 3776 write_unlock(&vcpu->kvm->mmu_lock); 3777 kvm_release_pfn_clean(pfn); 3778 return r; 3779 } 3780 3781 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, 3782 u32 error_code, bool prefault) 3783 { 3784 pgprintk("%s: gva %lx error %x\n", __func__, gpa, error_code); 3785 3786 /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */ 3787 return direct_page_fault(vcpu, gpa & PAGE_MASK, error_code, prefault, 3788 PG_LEVEL_2M, false); 3789 } 3790 3791 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, 3792 u64 fault_address, char *insn, int insn_len) 3793 { 3794 int r = 1; 3795 u32 flags = vcpu->arch.apf.host_apf_flags; 3796 3797 #ifndef CONFIG_X86_64 3798 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ 3799 if (WARN_ON_ONCE(fault_address >> 32)) 3800 return -EFAULT; 3801 #endif 3802 3803 vcpu->arch.l1tf_flush_l1d = true; 3804 if (!flags) { 3805 trace_kvm_page_fault(fault_address, error_code); 3806 3807 if (kvm_event_needs_reinjection(vcpu)) 3808 kvm_mmu_unprotect_page_virt(vcpu, fault_address); 3809 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn, 3810 insn_len); 3811 } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) { 3812 vcpu->arch.apf.host_apf_flags = 0; 3813 local_irq_disable(); 3814 kvm_async_pf_task_wait_schedule(fault_address); 3815 local_irq_enable(); 3816 } else { 3817 WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags); 3818 } 3819 3820 return r; 3821 } 3822 EXPORT_SYMBOL_GPL(kvm_handle_page_fault); 3823 3824 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 3825 bool prefault) 3826 { 3827 int max_level; 3828 3829 for (max_level = KVM_MAX_HUGEPAGE_LEVEL; 3830 max_level > PG_LEVEL_4K; 3831 max_level--) { 3832 int page_num = KVM_PAGES_PER_HPAGE(max_level); 3833 gfn_t base = (gpa >> PAGE_SHIFT) & ~(page_num - 1); 3834 3835 if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num)) 3836 break; 3837 } 3838 3839 return direct_page_fault(vcpu, gpa, error_code, prefault, 3840 max_level, true); 3841 } 3842 3843 static void nonpaging_init_context(struct kvm_vcpu *vcpu, 3844 struct kvm_mmu *context) 3845 { 3846 context->page_fault = nonpaging_page_fault; 3847 context->gva_to_gpa = nonpaging_gva_to_gpa; 3848 context->sync_page = nonpaging_sync_page; 3849 context->invlpg = NULL; 3850 context->root_level = 0; 3851 context->shadow_root_level = PT32E_ROOT_LEVEL; 3852 context->direct_map = true; 3853 context->nx = false; 3854 } 3855 3856 static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd, 3857 union kvm_mmu_page_role role) 3858 { 3859 return (role.direct || pgd == root->pgd) && 3860 VALID_PAGE(root->hpa) && to_shadow_page(root->hpa) && 3861 role.word == to_shadow_page(root->hpa)->role.word; 3862 } 3863 3864 /* 3865 * Find out if a previously cached root matching the new pgd/role is available. 3866 * The current root is also inserted into the cache. 3867 * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is 3868 * returned. 3869 * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and 3870 * false is returned. This root should now be freed by the caller. 3871 */ 3872 static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_pgd, 3873 union kvm_mmu_page_role new_role) 3874 { 3875 uint i; 3876 struct kvm_mmu_root_info root; 3877 struct kvm_mmu *mmu = vcpu->arch.mmu; 3878 3879 root.pgd = mmu->root_pgd; 3880 root.hpa = mmu->root_hpa; 3881 3882 if (is_root_usable(&root, new_pgd, new_role)) 3883 return true; 3884 3885 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 3886 swap(root, mmu->prev_roots[i]); 3887 3888 if (is_root_usable(&root, new_pgd, new_role)) 3889 break; 3890 } 3891 3892 mmu->root_hpa = root.hpa; 3893 mmu->root_pgd = root.pgd; 3894 3895 return i < KVM_MMU_NUM_PREV_ROOTS; 3896 } 3897 3898 static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd, 3899 union kvm_mmu_page_role new_role) 3900 { 3901 struct kvm_mmu *mmu = vcpu->arch.mmu; 3902 3903 /* 3904 * For now, limit the fast switch to 64-bit hosts+VMs in order to avoid 3905 * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs 3906 * later if necessary. 3907 */ 3908 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && 3909 mmu->root_level >= PT64_ROOT_4LEVEL) 3910 return cached_root_available(vcpu, new_pgd, new_role); 3911 3912 return false; 3913 } 3914 3915 static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, 3916 union kvm_mmu_page_role new_role, 3917 bool skip_tlb_flush, bool skip_mmu_sync) 3918 { 3919 if (!fast_pgd_switch(vcpu, new_pgd, new_role)) { 3920 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); 3921 return; 3922 } 3923 3924 /* 3925 * It's possible that the cached previous root page is obsolete because 3926 * of a change in the MMU generation number. However, changing the 3927 * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will 3928 * free the root set here and allocate a new one. 3929 */ 3930 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); 3931 3932 if (!skip_mmu_sync || force_flush_and_sync_on_reuse) 3933 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 3934 if (!skip_tlb_flush || force_flush_and_sync_on_reuse) 3935 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 3936 3937 /* 3938 * The last MMIO access's GVA and GPA are cached in the VCPU. When 3939 * switching to a new CR3, that GVA->GPA mapping may no longer be 3940 * valid. So clear any cached MMIO info even when we don't need to sync 3941 * the shadow page tables. 3942 */ 3943 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); 3944 3945 /* 3946 * If this is a direct root page, it doesn't have a write flooding 3947 * count. Otherwise, clear the write flooding count. 3948 */ 3949 if (!new_role.direct) 3950 __clear_sp_write_flooding_count( 3951 to_shadow_page(vcpu->arch.mmu->root_hpa)); 3952 } 3953 3954 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, 3955 bool skip_mmu_sync) 3956 { 3957 __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu), 3958 skip_tlb_flush, skip_mmu_sync); 3959 } 3960 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd); 3961 3962 static unsigned long get_cr3(struct kvm_vcpu *vcpu) 3963 { 3964 return kvm_read_cr3(vcpu); 3965 } 3966 3967 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, 3968 unsigned int access, int *nr_present) 3969 { 3970 if (unlikely(is_mmio_spte(*sptep))) { 3971 if (gfn != get_mmio_spte_gfn(*sptep)) { 3972 mmu_spte_clear_no_track(sptep); 3973 return true; 3974 } 3975 3976 (*nr_present)++; 3977 mark_mmio_spte(vcpu, sptep, gfn, access); 3978 return true; 3979 } 3980 3981 return false; 3982 } 3983 3984 static inline bool is_last_gpte(struct kvm_mmu *mmu, 3985 unsigned level, unsigned gpte) 3986 { 3987 /* 3988 * The RHS has bit 7 set iff level < mmu->last_nonleaf_level. 3989 * If it is clear, there are no large pages at this level, so clear 3990 * PT_PAGE_SIZE_MASK in gpte if that is the case. 3991 */ 3992 gpte &= level - mmu->last_nonleaf_level; 3993 3994 /* 3995 * PG_LEVEL_4K always terminates. The RHS has bit 7 set 3996 * iff level <= PG_LEVEL_4K, which for our purpose means 3997 * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then. 3998 */ 3999 gpte |= level - PG_LEVEL_4K - 1; 4000 4001 return gpte & PT_PAGE_SIZE_MASK; 4002 } 4003 4004 #define PTTYPE_EPT 18 /* arbitrary */ 4005 #define PTTYPE PTTYPE_EPT 4006 #include "paging_tmpl.h" 4007 #undef PTTYPE 4008 4009 #define PTTYPE 64 4010 #include "paging_tmpl.h" 4011 #undef PTTYPE 4012 4013 #define PTTYPE 32 4014 #include "paging_tmpl.h" 4015 #undef PTTYPE 4016 4017 static void 4018 __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, 4019 struct rsvd_bits_validate *rsvd_check, 4020 u64 pa_bits_rsvd, int level, bool nx, bool gbpages, 4021 bool pse, bool amd) 4022 { 4023 u64 gbpages_bit_rsvd = 0; 4024 u64 nonleaf_bit8_rsvd = 0; 4025 u64 high_bits_rsvd; 4026 4027 rsvd_check->bad_mt_xwr = 0; 4028 4029 if (!gbpages) 4030 gbpages_bit_rsvd = rsvd_bits(7, 7); 4031 4032 if (level == PT32E_ROOT_LEVEL) 4033 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62); 4034 else 4035 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51); 4036 4037 /* Note, NX doesn't exist in PDPTEs, this is handled below. */ 4038 if (!nx) 4039 high_bits_rsvd |= rsvd_bits(63, 63); 4040 4041 /* 4042 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for 4043 * leaf entries) on AMD CPUs only. 4044 */ 4045 if (amd) 4046 nonleaf_bit8_rsvd = rsvd_bits(8, 8); 4047 4048 switch (level) { 4049 case PT32_ROOT_LEVEL: 4050 /* no rsvd bits for 2 level 4K page table entries */ 4051 rsvd_check->rsvd_bits_mask[0][1] = 0; 4052 rsvd_check->rsvd_bits_mask[0][0] = 0; 4053 rsvd_check->rsvd_bits_mask[1][0] = 4054 rsvd_check->rsvd_bits_mask[0][0]; 4055 4056 if (!pse) { 4057 rsvd_check->rsvd_bits_mask[1][1] = 0; 4058 break; 4059 } 4060 4061 if (is_cpuid_PSE36()) 4062 /* 36bits PSE 4MB page */ 4063 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21); 4064 else 4065 /* 32 bits PSE 4MB page */ 4066 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21); 4067 break; 4068 case PT32E_ROOT_LEVEL: 4069 rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) | 4070 high_bits_rsvd | 4071 rsvd_bits(5, 8) | 4072 rsvd_bits(1, 2); /* PDPTE */ 4073 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd; /* PDE */ 4074 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; /* PTE */ 4075 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | 4076 rsvd_bits(13, 20); /* large page */ 4077 rsvd_check->rsvd_bits_mask[1][0] = 4078 rsvd_check->rsvd_bits_mask[0][0]; 4079 break; 4080 case PT64_ROOT_5LEVEL: 4081 rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | 4082 nonleaf_bit8_rsvd | 4083 rsvd_bits(7, 7); 4084 rsvd_check->rsvd_bits_mask[1][4] = 4085 rsvd_check->rsvd_bits_mask[0][4]; 4086 fallthrough; 4087 case PT64_ROOT_4LEVEL: 4088 rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | 4089 nonleaf_bit8_rsvd | 4090 rsvd_bits(7, 7); 4091 rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | 4092 gbpages_bit_rsvd; 4093 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd; 4094 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; 4095 rsvd_check->rsvd_bits_mask[1][3] = 4096 rsvd_check->rsvd_bits_mask[0][3]; 4097 rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | 4098 gbpages_bit_rsvd | 4099 rsvd_bits(13, 29); 4100 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | 4101 rsvd_bits(13, 20); /* large page */ 4102 rsvd_check->rsvd_bits_mask[1][0] = 4103 rsvd_check->rsvd_bits_mask[0][0]; 4104 break; 4105 } 4106 } 4107 4108 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, 4109 struct kvm_mmu *context) 4110 { 4111 __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check, 4112 vcpu->arch.reserved_gpa_bits, 4113 context->root_level, context->nx, 4114 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), 4115 is_pse(vcpu), 4116 guest_cpuid_is_amd_or_hygon(vcpu)); 4117 } 4118 4119 static void 4120 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check, 4121 u64 pa_bits_rsvd, bool execonly) 4122 { 4123 u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51); 4124 u64 bad_mt_xwr; 4125 4126 rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7); 4127 rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7); 4128 rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6); 4129 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6); 4130 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; 4131 4132 /* large page */ 4133 rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4]; 4134 rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3]; 4135 rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29); 4136 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20); 4137 rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0]; 4138 4139 bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */ 4140 bad_mt_xwr |= 0xFFull << (3 * 8); /* bits 3..5 must not be 3 */ 4141 bad_mt_xwr |= 0xFFull << (7 * 8); /* bits 3..5 must not be 7 */ 4142 bad_mt_xwr |= REPEAT_BYTE(1ull << 2); /* bits 0..2 must not be 010 */ 4143 bad_mt_xwr |= REPEAT_BYTE(1ull << 6); /* bits 0..2 must not be 110 */ 4144 if (!execonly) { 4145 /* bits 0..2 must not be 100 unless VMX capabilities allow it */ 4146 bad_mt_xwr |= REPEAT_BYTE(1ull << 4); 4147 } 4148 rsvd_check->bad_mt_xwr = bad_mt_xwr; 4149 } 4150 4151 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu, 4152 struct kvm_mmu *context, bool execonly) 4153 { 4154 __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check, 4155 vcpu->arch.reserved_gpa_bits, execonly); 4156 } 4157 4158 static inline u64 reserved_hpa_bits(void) 4159 { 4160 return rsvd_bits(shadow_phys_bits, 63); 4161 } 4162 4163 /* 4164 * the page table on host is the shadow page table for the page 4165 * table in guest or amd nested guest, its mmu features completely 4166 * follow the features in guest. 4167 */ 4168 void 4169 reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) 4170 { 4171 bool uses_nx = context->nx || 4172 context->mmu_role.base.smep_andnot_wp; 4173 struct rsvd_bits_validate *shadow_zero_check; 4174 int i; 4175 4176 /* 4177 * Passing "true" to the last argument is okay; it adds a check 4178 * on bit 8 of the SPTEs which KVM doesn't use anyway. 4179 */ 4180 shadow_zero_check = &context->shadow_zero_check; 4181 __reset_rsvds_bits_mask(vcpu, shadow_zero_check, 4182 reserved_hpa_bits(), 4183 context->shadow_root_level, uses_nx, 4184 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), 4185 is_pse(vcpu), true); 4186 4187 if (!shadow_me_mask) 4188 return; 4189 4190 for (i = context->shadow_root_level; --i >= 0;) { 4191 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; 4192 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; 4193 } 4194 4195 } 4196 EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask); 4197 4198 static inline bool boot_cpu_is_amd(void) 4199 { 4200 WARN_ON_ONCE(!tdp_enabled); 4201 return shadow_x_mask == 0; 4202 } 4203 4204 /* 4205 * the direct page table on host, use as much mmu features as 4206 * possible, however, kvm currently does not do execution-protection. 4207 */ 4208 static void 4209 reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, 4210 struct kvm_mmu *context) 4211 { 4212 struct rsvd_bits_validate *shadow_zero_check; 4213 int i; 4214 4215 shadow_zero_check = &context->shadow_zero_check; 4216 4217 if (boot_cpu_is_amd()) 4218 __reset_rsvds_bits_mask(vcpu, shadow_zero_check, 4219 reserved_hpa_bits(), 4220 context->shadow_root_level, false, 4221 boot_cpu_has(X86_FEATURE_GBPAGES), 4222 true, true); 4223 else 4224 __reset_rsvds_bits_mask_ept(shadow_zero_check, 4225 reserved_hpa_bits(), false); 4226 4227 if (!shadow_me_mask) 4228 return; 4229 4230 for (i = context->shadow_root_level; --i >= 0;) { 4231 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; 4232 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; 4233 } 4234 } 4235 4236 /* 4237 * as the comments in reset_shadow_zero_bits_mask() except it 4238 * is the shadow page table for intel nested guest. 4239 */ 4240 static void 4241 reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, 4242 struct kvm_mmu *context, bool execonly) 4243 { 4244 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check, 4245 reserved_hpa_bits(), execonly); 4246 } 4247 4248 #define BYTE_MASK(access) \ 4249 ((1 & (access) ? 2 : 0) | \ 4250 (2 & (access) ? 4 : 0) | \ 4251 (3 & (access) ? 8 : 0) | \ 4252 (4 & (access) ? 16 : 0) | \ 4253 (5 & (access) ? 32 : 0) | \ 4254 (6 & (access) ? 64 : 0) | \ 4255 (7 & (access) ? 128 : 0)) 4256 4257 4258 static void update_permission_bitmask(struct kvm_vcpu *vcpu, 4259 struct kvm_mmu *mmu, bool ept) 4260 { 4261 unsigned byte; 4262 4263 const u8 x = BYTE_MASK(ACC_EXEC_MASK); 4264 const u8 w = BYTE_MASK(ACC_WRITE_MASK); 4265 const u8 u = BYTE_MASK(ACC_USER_MASK); 4266 4267 bool cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP) != 0; 4268 bool cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP) != 0; 4269 bool cr0_wp = is_write_protection(vcpu); 4270 4271 for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) { 4272 unsigned pfec = byte << 1; 4273 4274 /* 4275 * Each "*f" variable has a 1 bit for each UWX value 4276 * that causes a fault with the given PFEC. 4277 */ 4278 4279 /* Faults from writes to non-writable pages */ 4280 u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0; 4281 /* Faults from user mode accesses to supervisor pages */ 4282 u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0; 4283 /* Faults from fetches of non-executable pages*/ 4284 u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0; 4285 /* Faults from kernel mode fetches of user pages */ 4286 u8 smepf = 0; 4287 /* Faults from kernel mode accesses of user pages */ 4288 u8 smapf = 0; 4289 4290 if (!ept) { 4291 /* Faults from kernel mode accesses to user pages */ 4292 u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u; 4293 4294 /* Not really needed: !nx will cause pte.nx to fault */ 4295 if (!mmu->nx) 4296 ff = 0; 4297 4298 /* Allow supervisor writes if !cr0.wp */ 4299 if (!cr0_wp) 4300 wf = (pfec & PFERR_USER_MASK) ? wf : 0; 4301 4302 /* Disallow supervisor fetches of user code if cr4.smep */ 4303 if (cr4_smep) 4304 smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0; 4305 4306 /* 4307 * SMAP:kernel-mode data accesses from user-mode 4308 * mappings should fault. A fault is considered 4309 * as a SMAP violation if all of the following 4310 * conditions are true: 4311 * - X86_CR4_SMAP is set in CR4 4312 * - A user page is accessed 4313 * - The access is not a fetch 4314 * - Page fault in kernel mode 4315 * - if CPL = 3 or X86_EFLAGS_AC is clear 4316 * 4317 * Here, we cover the first three conditions. 4318 * The fourth is computed dynamically in permission_fault(); 4319 * PFERR_RSVD_MASK bit will be set in PFEC if the access is 4320 * *not* subject to SMAP restrictions. 4321 */ 4322 if (cr4_smap) 4323 smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf; 4324 } 4325 4326 mmu->permissions[byte] = ff | uf | wf | smepf | smapf; 4327 } 4328 } 4329 4330 /* 4331 * PKU is an additional mechanism by which the paging controls access to 4332 * user-mode addresses based on the value in the PKRU register. Protection 4333 * key violations are reported through a bit in the page fault error code. 4334 * Unlike other bits of the error code, the PK bit is not known at the 4335 * call site of e.g. gva_to_gpa; it must be computed directly in 4336 * permission_fault based on two bits of PKRU, on some machine state (CR4, 4337 * CR0, EFER, CPL), and on other bits of the error code and the page tables. 4338 * 4339 * In particular the following conditions come from the error code, the 4340 * page tables and the machine state: 4341 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1 4342 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch) 4343 * - PK is always zero if U=0 in the page tables 4344 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access. 4345 * 4346 * The PKRU bitmask caches the result of these four conditions. The error 4347 * code (minus the P bit) and the page table's U bit form an index into the 4348 * PKRU bitmask. Two bits of the PKRU bitmask are then extracted and ANDed 4349 * with the two bits of the PKRU register corresponding to the protection key. 4350 * For the first three conditions above the bits will be 00, thus masking 4351 * away both AD and WD. For all reads or if the last condition holds, WD 4352 * only will be masked away. 4353 */ 4354 static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 4355 bool ept) 4356 { 4357 unsigned bit; 4358 bool wp; 4359 4360 if (ept) { 4361 mmu->pkru_mask = 0; 4362 return; 4363 } 4364 4365 /* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */ 4366 if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) { 4367 mmu->pkru_mask = 0; 4368 return; 4369 } 4370 4371 wp = is_write_protection(vcpu); 4372 4373 for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) { 4374 unsigned pfec, pkey_bits; 4375 bool check_pkey, check_write, ff, uf, wf, pte_user; 4376 4377 pfec = bit << 1; 4378 ff = pfec & PFERR_FETCH_MASK; 4379 uf = pfec & PFERR_USER_MASK; 4380 wf = pfec & PFERR_WRITE_MASK; 4381 4382 /* PFEC.RSVD is replaced by ACC_USER_MASK. */ 4383 pte_user = pfec & PFERR_RSVD_MASK; 4384 4385 /* 4386 * Only need to check the access which is not an 4387 * instruction fetch and is to a user page. 4388 */ 4389 check_pkey = (!ff && pte_user); 4390 /* 4391 * write access is controlled by PKRU if it is a 4392 * user access or CR0.WP = 1. 4393 */ 4394 check_write = check_pkey && wf && (uf || wp); 4395 4396 /* PKRU.AD stops both read and write access. */ 4397 pkey_bits = !!check_pkey; 4398 /* PKRU.WD stops write access. */ 4399 pkey_bits |= (!!check_write) << 1; 4400 4401 mmu->pkru_mask |= (pkey_bits & 3) << pfec; 4402 } 4403 } 4404 4405 static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) 4406 { 4407 unsigned root_level = mmu->root_level; 4408 4409 mmu->last_nonleaf_level = root_level; 4410 if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu)) 4411 mmu->last_nonleaf_level++; 4412 } 4413 4414 static void paging64_init_context_common(struct kvm_vcpu *vcpu, 4415 struct kvm_mmu *context, 4416 int level) 4417 { 4418 context->nx = is_nx(vcpu); 4419 context->root_level = level; 4420 4421 reset_rsvds_bits_mask(vcpu, context); 4422 update_permission_bitmask(vcpu, context, false); 4423 update_pkru_bitmask(vcpu, context, false); 4424 update_last_nonleaf_level(vcpu, context); 4425 4426 MMU_WARN_ON(!is_pae(vcpu)); 4427 context->page_fault = paging64_page_fault; 4428 context->gva_to_gpa = paging64_gva_to_gpa; 4429 context->sync_page = paging64_sync_page; 4430 context->invlpg = paging64_invlpg; 4431 context->shadow_root_level = level; 4432 context->direct_map = false; 4433 } 4434 4435 static void paging64_init_context(struct kvm_vcpu *vcpu, 4436 struct kvm_mmu *context) 4437 { 4438 int root_level = is_la57_mode(vcpu) ? 4439 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4440 4441 paging64_init_context_common(vcpu, context, root_level); 4442 } 4443 4444 static void paging32_init_context(struct kvm_vcpu *vcpu, 4445 struct kvm_mmu *context) 4446 { 4447 context->nx = false; 4448 context->root_level = PT32_ROOT_LEVEL; 4449 4450 reset_rsvds_bits_mask(vcpu, context); 4451 update_permission_bitmask(vcpu, context, false); 4452 update_pkru_bitmask(vcpu, context, false); 4453 update_last_nonleaf_level(vcpu, context); 4454 4455 context->page_fault = paging32_page_fault; 4456 context->gva_to_gpa = paging32_gva_to_gpa; 4457 context->sync_page = paging32_sync_page; 4458 context->invlpg = paging32_invlpg; 4459 context->shadow_root_level = PT32E_ROOT_LEVEL; 4460 context->direct_map = false; 4461 } 4462 4463 static void paging32E_init_context(struct kvm_vcpu *vcpu, 4464 struct kvm_mmu *context) 4465 { 4466 paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL); 4467 } 4468 4469 static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu) 4470 { 4471 union kvm_mmu_extended_role ext = {0}; 4472 4473 ext.cr0_pg = !!is_paging(vcpu); 4474 ext.cr4_pae = !!is_pae(vcpu); 4475 ext.cr4_smep = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMEP); 4476 ext.cr4_smap = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMAP); 4477 ext.cr4_pse = !!is_pse(vcpu); 4478 ext.cr4_pke = !!kvm_read_cr4_bits(vcpu, X86_CR4_PKE); 4479 ext.maxphyaddr = cpuid_maxphyaddr(vcpu); 4480 4481 ext.valid = 1; 4482 4483 return ext; 4484 } 4485 4486 static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, 4487 bool base_only) 4488 { 4489 union kvm_mmu_role role = {0}; 4490 4491 role.base.access = ACC_ALL; 4492 role.base.nxe = !!is_nx(vcpu); 4493 role.base.cr0_wp = is_write_protection(vcpu); 4494 role.base.smm = is_smm(vcpu); 4495 role.base.guest_mode = is_guest_mode(vcpu); 4496 4497 if (base_only) 4498 return role; 4499 4500 role.ext = kvm_calc_mmu_role_ext(vcpu); 4501 4502 return role; 4503 } 4504 4505 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) 4506 { 4507 /* Use 5-level TDP if and only if it's useful/necessary. */ 4508 if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) 4509 return 4; 4510 4511 return max_tdp_level; 4512 } 4513 4514 static union kvm_mmu_role 4515 kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) 4516 { 4517 union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); 4518 4519 role.base.ad_disabled = (shadow_accessed_mask == 0); 4520 role.base.level = kvm_mmu_get_tdp_level(vcpu); 4521 role.base.direct = true; 4522 role.base.gpte_is_8_bytes = true; 4523 4524 return role; 4525 } 4526 4527 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) 4528 { 4529 struct kvm_mmu *context = &vcpu->arch.root_mmu; 4530 union kvm_mmu_role new_role = 4531 kvm_calc_tdp_mmu_root_page_role(vcpu, false); 4532 4533 if (new_role.as_u64 == context->mmu_role.as_u64) 4534 return; 4535 4536 context->mmu_role.as_u64 = new_role.as_u64; 4537 context->page_fault = kvm_tdp_page_fault; 4538 context->sync_page = nonpaging_sync_page; 4539 context->invlpg = NULL; 4540 context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu); 4541 context->direct_map = true; 4542 context->get_guest_pgd = get_cr3; 4543 context->get_pdptr = kvm_pdptr_read; 4544 context->inject_page_fault = kvm_inject_page_fault; 4545 4546 if (!is_paging(vcpu)) { 4547 context->nx = false; 4548 context->gva_to_gpa = nonpaging_gva_to_gpa; 4549 context->root_level = 0; 4550 } else if (is_long_mode(vcpu)) { 4551 context->nx = is_nx(vcpu); 4552 context->root_level = is_la57_mode(vcpu) ? 4553 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4554 reset_rsvds_bits_mask(vcpu, context); 4555 context->gva_to_gpa = paging64_gva_to_gpa; 4556 } else if (is_pae(vcpu)) { 4557 context->nx = is_nx(vcpu); 4558 context->root_level = PT32E_ROOT_LEVEL; 4559 reset_rsvds_bits_mask(vcpu, context); 4560 context->gva_to_gpa = paging64_gva_to_gpa; 4561 } else { 4562 context->nx = false; 4563 context->root_level = PT32_ROOT_LEVEL; 4564 reset_rsvds_bits_mask(vcpu, context); 4565 context->gva_to_gpa = paging32_gva_to_gpa; 4566 } 4567 4568 update_permission_bitmask(vcpu, context, false); 4569 update_pkru_bitmask(vcpu, context, false); 4570 update_last_nonleaf_level(vcpu, context); 4571 reset_tdp_shadow_zero_bits_mask(vcpu, context); 4572 } 4573 4574 static union kvm_mmu_role 4575 kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, bool base_only) 4576 { 4577 union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); 4578 4579 role.base.smep_andnot_wp = role.ext.cr4_smep && 4580 !is_write_protection(vcpu); 4581 role.base.smap_andnot_wp = role.ext.cr4_smap && 4582 !is_write_protection(vcpu); 4583 role.base.gpte_is_8_bytes = !!is_pae(vcpu); 4584 4585 return role; 4586 } 4587 4588 static union kvm_mmu_role 4589 kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) 4590 { 4591 union kvm_mmu_role role = 4592 kvm_calc_shadow_root_page_role_common(vcpu, base_only); 4593 4594 role.base.direct = !is_paging(vcpu); 4595 4596 if (!is_long_mode(vcpu)) 4597 role.base.level = PT32E_ROOT_LEVEL; 4598 else if (is_la57_mode(vcpu)) 4599 role.base.level = PT64_ROOT_5LEVEL; 4600 else 4601 role.base.level = PT64_ROOT_4LEVEL; 4602 4603 return role; 4604 } 4605 4606 static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, 4607 u32 cr0, u32 cr4, u32 efer, 4608 union kvm_mmu_role new_role) 4609 { 4610 if (!(cr0 & X86_CR0_PG)) 4611 nonpaging_init_context(vcpu, context); 4612 else if (efer & EFER_LMA) 4613 paging64_init_context(vcpu, context); 4614 else if (cr4 & X86_CR4_PAE) 4615 paging32E_init_context(vcpu, context); 4616 else 4617 paging32_init_context(vcpu, context); 4618 4619 context->mmu_role.as_u64 = new_role.as_u64; 4620 reset_shadow_zero_bits_mask(vcpu, context); 4621 } 4622 4623 static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer) 4624 { 4625 struct kvm_mmu *context = &vcpu->arch.root_mmu; 4626 union kvm_mmu_role new_role = 4627 kvm_calc_shadow_mmu_root_page_role(vcpu, false); 4628 4629 if (new_role.as_u64 != context->mmu_role.as_u64) 4630 shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); 4631 } 4632 4633 static union kvm_mmu_role 4634 kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu) 4635 { 4636 union kvm_mmu_role role = 4637 kvm_calc_shadow_root_page_role_common(vcpu, false); 4638 4639 role.base.direct = false; 4640 role.base.level = kvm_mmu_get_tdp_level(vcpu); 4641 4642 return role; 4643 } 4644 4645 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer, 4646 gpa_t nested_cr3) 4647 { 4648 struct kvm_mmu *context = &vcpu->arch.guest_mmu; 4649 union kvm_mmu_role new_role = kvm_calc_shadow_npt_root_page_role(vcpu); 4650 4651 __kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base, false, false); 4652 4653 if (new_role.as_u64 != context->mmu_role.as_u64) { 4654 shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); 4655 4656 /* 4657 * Override the level set by the common init helper, nested TDP 4658 * always uses the host's TDP configuration. 4659 */ 4660 context->shadow_root_level = new_role.base.level; 4661 } 4662 } 4663 EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); 4664 4665 static union kvm_mmu_role 4666 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, 4667 bool execonly, u8 level) 4668 { 4669 union kvm_mmu_role role = {0}; 4670 4671 /* SMM flag is inherited from root_mmu */ 4672 role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm; 4673 4674 role.base.level = level; 4675 role.base.gpte_is_8_bytes = true; 4676 role.base.direct = false; 4677 role.base.ad_disabled = !accessed_dirty; 4678 role.base.guest_mode = true; 4679 role.base.access = ACC_ALL; 4680 4681 /* 4682 * WP=1 and NOT_WP=1 is an impossible combination, use WP and the 4683 * SMAP variation to denote shadow EPT entries. 4684 */ 4685 role.base.cr0_wp = true; 4686 role.base.smap_andnot_wp = true; 4687 4688 role.ext = kvm_calc_mmu_role_ext(vcpu); 4689 role.ext.execonly = execonly; 4690 4691 return role; 4692 } 4693 4694 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, 4695 bool accessed_dirty, gpa_t new_eptp) 4696 { 4697 struct kvm_mmu *context = &vcpu->arch.guest_mmu; 4698 u8 level = vmx_eptp_page_walk_level(new_eptp); 4699 union kvm_mmu_role new_role = 4700 kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, 4701 execonly, level); 4702 4703 __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base, true, true); 4704 4705 if (new_role.as_u64 == context->mmu_role.as_u64) 4706 return; 4707 4708 context->shadow_root_level = level; 4709 4710 context->nx = true; 4711 context->ept_ad = accessed_dirty; 4712 context->page_fault = ept_page_fault; 4713 context->gva_to_gpa = ept_gva_to_gpa; 4714 context->sync_page = ept_sync_page; 4715 context->invlpg = ept_invlpg; 4716 context->root_level = level; 4717 context->direct_map = false; 4718 context->mmu_role.as_u64 = new_role.as_u64; 4719 4720 update_permission_bitmask(vcpu, context, true); 4721 update_pkru_bitmask(vcpu, context, true); 4722 update_last_nonleaf_level(vcpu, context); 4723 reset_rsvds_bits_mask_ept(vcpu, context, execonly); 4724 reset_ept_shadow_zero_bits_mask(vcpu, context, execonly); 4725 } 4726 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); 4727 4728 static void init_kvm_softmmu(struct kvm_vcpu *vcpu) 4729 { 4730 struct kvm_mmu *context = &vcpu->arch.root_mmu; 4731 4732 kvm_init_shadow_mmu(vcpu, 4733 kvm_read_cr0_bits(vcpu, X86_CR0_PG), 4734 kvm_read_cr4_bits(vcpu, X86_CR4_PAE), 4735 vcpu->arch.efer); 4736 4737 context->get_guest_pgd = get_cr3; 4738 context->get_pdptr = kvm_pdptr_read; 4739 context->inject_page_fault = kvm_inject_page_fault; 4740 } 4741 4742 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) 4743 { 4744 union kvm_mmu_role new_role = kvm_calc_mmu_role_common(vcpu, false); 4745 struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; 4746 4747 if (new_role.as_u64 == g_context->mmu_role.as_u64) 4748 return; 4749 4750 g_context->mmu_role.as_u64 = new_role.as_u64; 4751 g_context->get_guest_pgd = get_cr3; 4752 g_context->get_pdptr = kvm_pdptr_read; 4753 g_context->inject_page_fault = kvm_inject_page_fault; 4754 4755 /* 4756 * L2 page tables are never shadowed, so there is no need to sync 4757 * SPTEs. 4758 */ 4759 g_context->invlpg = NULL; 4760 4761 /* 4762 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using 4763 * L1's nested page tables (e.g. EPT12). The nested translation 4764 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using 4765 * L2's page tables as the first level of translation and L1's 4766 * nested page tables as the second level of translation. Basically 4767 * the gva_to_gpa functions between mmu and nested_mmu are swapped. 4768 */ 4769 if (!is_paging(vcpu)) { 4770 g_context->nx = false; 4771 g_context->root_level = 0; 4772 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested; 4773 } else if (is_long_mode(vcpu)) { 4774 g_context->nx = is_nx(vcpu); 4775 g_context->root_level = is_la57_mode(vcpu) ? 4776 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4777 reset_rsvds_bits_mask(vcpu, g_context); 4778 g_context->gva_to_gpa = paging64_gva_to_gpa_nested; 4779 } else if (is_pae(vcpu)) { 4780 g_context->nx = is_nx(vcpu); 4781 g_context->root_level = PT32E_ROOT_LEVEL; 4782 reset_rsvds_bits_mask(vcpu, g_context); 4783 g_context->gva_to_gpa = paging64_gva_to_gpa_nested; 4784 } else { 4785 g_context->nx = false; 4786 g_context->root_level = PT32_ROOT_LEVEL; 4787 reset_rsvds_bits_mask(vcpu, g_context); 4788 g_context->gva_to_gpa = paging32_gva_to_gpa_nested; 4789 } 4790 4791 update_permission_bitmask(vcpu, g_context, false); 4792 update_pkru_bitmask(vcpu, g_context, false); 4793 update_last_nonleaf_level(vcpu, g_context); 4794 } 4795 4796 void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots) 4797 { 4798 if (reset_roots) { 4799 uint i; 4800 4801 vcpu->arch.mmu->root_hpa = INVALID_PAGE; 4802 4803 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 4804 vcpu->arch.mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 4805 } 4806 4807 if (mmu_is_nested(vcpu)) 4808 init_kvm_nested_mmu(vcpu); 4809 else if (tdp_enabled) 4810 init_kvm_tdp_mmu(vcpu); 4811 else 4812 init_kvm_softmmu(vcpu); 4813 } 4814 EXPORT_SYMBOL_GPL(kvm_init_mmu); 4815 4816 static union kvm_mmu_page_role 4817 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu) 4818 { 4819 union kvm_mmu_role role; 4820 4821 if (tdp_enabled) 4822 role = kvm_calc_tdp_mmu_root_page_role(vcpu, true); 4823 else 4824 role = kvm_calc_shadow_mmu_root_page_role(vcpu, true); 4825 4826 return role.base; 4827 } 4828 4829 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu) 4830 { 4831 kvm_mmu_unload(vcpu); 4832 kvm_init_mmu(vcpu, true); 4833 } 4834 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context); 4835 4836 int kvm_mmu_load(struct kvm_vcpu *vcpu) 4837 { 4838 int r; 4839 4840 r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->direct_map); 4841 if (r) 4842 goto out; 4843 r = mmu_alloc_special_roots(vcpu); 4844 if (r) 4845 goto out; 4846 if (vcpu->arch.mmu->direct_map) 4847 r = mmu_alloc_direct_roots(vcpu); 4848 else 4849 r = mmu_alloc_shadow_roots(vcpu); 4850 if (r) 4851 goto out; 4852 4853 kvm_mmu_sync_roots(vcpu); 4854 4855 kvm_mmu_load_pgd(vcpu); 4856 static_call(kvm_x86_tlb_flush_current)(vcpu); 4857 out: 4858 return r; 4859 } 4860 4861 void kvm_mmu_unload(struct kvm_vcpu *vcpu) 4862 { 4863 kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL); 4864 WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa)); 4865 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4866 WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa)); 4867 } 4868 4869 static bool need_remote_flush(u64 old, u64 new) 4870 { 4871 if (!is_shadow_present_pte(old)) 4872 return false; 4873 if (!is_shadow_present_pte(new)) 4874 return true; 4875 if ((old ^ new) & PT64_BASE_ADDR_MASK) 4876 return true; 4877 old ^= shadow_nx_mask; 4878 new ^= shadow_nx_mask; 4879 return (old & ~new & PT64_PERM_MASK) != 0; 4880 } 4881 4882 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa, 4883 int *bytes) 4884 { 4885 u64 gentry = 0; 4886 int r; 4887 4888 /* 4889 * Assume that the pte write on a page table of the same type 4890 * as the current vcpu paging mode since we update the sptes only 4891 * when they have the same mode. 4892 */ 4893 if (is_pae(vcpu) && *bytes == 4) { 4894 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */ 4895 *gpa &= ~(gpa_t)7; 4896 *bytes = 8; 4897 } 4898 4899 if (*bytes == 4 || *bytes == 8) { 4900 r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes); 4901 if (r) 4902 gentry = 0; 4903 } 4904 4905 return gentry; 4906 } 4907 4908 /* 4909 * If we're seeing too many writes to a page, it may no longer be a page table, 4910 * or we may be forking, in which case it is better to unmap the page. 4911 */ 4912 static bool detect_write_flooding(struct kvm_mmu_page *sp) 4913 { 4914 /* 4915 * Skip write-flooding detected for the sp whose level is 1, because 4916 * it can become unsync, then the guest page is not write-protected. 4917 */ 4918 if (sp->role.level == PG_LEVEL_4K) 4919 return false; 4920 4921 atomic_inc(&sp->write_flooding_count); 4922 return atomic_read(&sp->write_flooding_count) >= 3; 4923 } 4924 4925 /* 4926 * Misaligned accesses are too much trouble to fix up; also, they usually 4927 * indicate a page is not used as a page table. 4928 */ 4929 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa, 4930 int bytes) 4931 { 4932 unsigned offset, pte_size, misaligned; 4933 4934 pgprintk("misaligned: gpa %llx bytes %d role %x\n", 4935 gpa, bytes, sp->role.word); 4936 4937 offset = offset_in_page(gpa); 4938 pte_size = sp->role.gpte_is_8_bytes ? 8 : 4; 4939 4940 /* 4941 * Sometimes, the OS only writes the last one bytes to update status 4942 * bits, for example, in linux, andb instruction is used in clear_bit(). 4943 */ 4944 if (!(offset & (pte_size - 1)) && bytes == 1) 4945 return false; 4946 4947 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); 4948 misaligned |= bytes < 4; 4949 4950 return misaligned; 4951 } 4952 4953 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte) 4954 { 4955 unsigned page_offset, quadrant; 4956 u64 *spte; 4957 int level; 4958 4959 page_offset = offset_in_page(gpa); 4960 level = sp->role.level; 4961 *nspte = 1; 4962 if (!sp->role.gpte_is_8_bytes) { 4963 page_offset <<= 1; /* 32->64 */ 4964 /* 4965 * A 32-bit pde maps 4MB while the shadow pdes map 4966 * only 2MB. So we need to double the offset again 4967 * and zap two pdes instead of one. 4968 */ 4969 if (level == PT32_ROOT_LEVEL) { 4970 page_offset &= ~7; /* kill rounding error */ 4971 page_offset <<= 1; 4972 *nspte = 2; 4973 } 4974 quadrant = page_offset >> PAGE_SHIFT; 4975 page_offset &= ~PAGE_MASK; 4976 if (quadrant != sp->role.quadrant) 4977 return NULL; 4978 } 4979 4980 spte = &sp->spt[page_offset / sizeof(*spte)]; 4981 return spte; 4982 } 4983 4984 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 4985 const u8 *new, int bytes, 4986 struct kvm_page_track_notifier_node *node) 4987 { 4988 gfn_t gfn = gpa >> PAGE_SHIFT; 4989 struct kvm_mmu_page *sp; 4990 LIST_HEAD(invalid_list); 4991 u64 entry, gentry, *spte; 4992 int npte; 4993 bool remote_flush, local_flush; 4994 4995 /* 4996 * If we don't have indirect shadow pages, it means no page is 4997 * write-protected, so we can exit simply. 4998 */ 4999 if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages)) 5000 return; 5001 5002 remote_flush = local_flush = false; 5003 5004 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes); 5005 5006 /* 5007 * No need to care whether allocation memory is successful 5008 * or not since pte prefetch is skipped if it does not have 5009 * enough objects in the cache. 5010 */ 5011 mmu_topup_memory_caches(vcpu, true); 5012 5013 write_lock(&vcpu->kvm->mmu_lock); 5014 5015 gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes); 5016 5017 ++vcpu->kvm->stat.mmu_pte_write; 5018 kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE); 5019 5020 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { 5021 if (detect_write_misaligned(sp, gpa, bytes) || 5022 detect_write_flooding(sp)) { 5023 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list); 5024 ++vcpu->kvm->stat.mmu_flooded; 5025 continue; 5026 } 5027 5028 spte = get_written_sptes(sp, gpa, &npte); 5029 if (!spte) 5030 continue; 5031 5032 local_flush = true; 5033 while (npte--) { 5034 entry = *spte; 5035 mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL); 5036 if (gentry && sp->role.level != PG_LEVEL_4K) 5037 ++vcpu->kvm->stat.mmu_pde_zapped; 5038 if (need_remote_flush(entry, *spte)) 5039 remote_flush = true; 5040 ++spte; 5041 } 5042 } 5043 kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush); 5044 kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE); 5045 write_unlock(&vcpu->kvm->mmu_lock); 5046 } 5047 5048 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, 5049 void *insn, int insn_len) 5050 { 5051 int r, emulation_type = EMULTYPE_PF; 5052 bool direct = vcpu->arch.mmu->direct_map; 5053 5054 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 5055 return RET_PF_RETRY; 5056 5057 r = RET_PF_INVALID; 5058 if (unlikely(error_code & PFERR_RSVD_MASK)) { 5059 r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); 5060 if (r == RET_PF_EMULATE) 5061 goto emulate; 5062 } 5063 5064 if (r == RET_PF_INVALID) { 5065 r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa, 5066 lower_32_bits(error_code), false); 5067 if (WARN_ON_ONCE(r == RET_PF_INVALID)) 5068 return -EIO; 5069 } 5070 5071 if (r < 0) 5072 return r; 5073 if (r != RET_PF_EMULATE) 5074 return 1; 5075 5076 /* 5077 * Before emulating the instruction, check if the error code 5078 * was due to a RO violation while translating the guest page. 5079 * This can occur when using nested virtualization with nested 5080 * paging in both guests. If true, we simply unprotect the page 5081 * and resume the guest. 5082 */ 5083 if (vcpu->arch.mmu->direct_map && 5084 (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) { 5085 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)); 5086 return 1; 5087 } 5088 5089 /* 5090 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still 5091 * optimistically try to just unprotect the page and let the processor 5092 * re-execute the instruction that caused the page fault. Do not allow 5093 * retrying MMIO emulation, as it's not only pointless but could also 5094 * cause us to enter an infinite loop because the processor will keep 5095 * faulting on the non-existent MMIO address. Retrying an instruction 5096 * from a nested guest is also pointless and dangerous as we are only 5097 * explicitly shadowing L1's page tables, i.e. unprotecting something 5098 * for L1 isn't going to magically fix whatever issue cause L2 to fail. 5099 */ 5100 if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu)) 5101 emulation_type |= EMULTYPE_ALLOW_RETRY_PF; 5102 emulate: 5103 return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, 5104 insn_len); 5105 } 5106 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); 5107 5108 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 5109 gva_t gva, hpa_t root_hpa) 5110 { 5111 int i; 5112 5113 /* It's actually a GPA for vcpu->arch.guest_mmu. */ 5114 if (mmu != &vcpu->arch.guest_mmu) { 5115 /* INVLPG on a non-canonical address is a NOP according to the SDM. */ 5116 if (is_noncanonical_address(gva, vcpu)) 5117 return; 5118 5119 static_call(kvm_x86_tlb_flush_gva)(vcpu, gva); 5120 } 5121 5122 if (!mmu->invlpg) 5123 return; 5124 5125 if (root_hpa == INVALID_PAGE) { 5126 mmu->invlpg(vcpu, gva, mmu->root_hpa); 5127 5128 /* 5129 * INVLPG is required to invalidate any global mappings for the VA, 5130 * irrespective of PCID. Since it would take us roughly similar amount 5131 * of work to determine whether any of the prev_root mappings of the VA 5132 * is marked global, or to just sync it blindly, so we might as well 5133 * just always sync it. 5134 * 5135 * Mappings not reachable via the current cr3 or the prev_roots will be 5136 * synced when switching to that cr3, so nothing needs to be done here 5137 * for them. 5138 */ 5139 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5140 if (VALID_PAGE(mmu->prev_roots[i].hpa)) 5141 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); 5142 } else { 5143 mmu->invlpg(vcpu, gva, root_hpa); 5144 } 5145 } 5146 5147 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) 5148 { 5149 kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, gva, INVALID_PAGE); 5150 ++vcpu->stat.invlpg; 5151 } 5152 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); 5153 5154 5155 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) 5156 { 5157 struct kvm_mmu *mmu = vcpu->arch.mmu; 5158 bool tlb_flush = false; 5159 uint i; 5160 5161 if (pcid == kvm_get_active_pcid(vcpu)) { 5162 mmu->invlpg(vcpu, gva, mmu->root_hpa); 5163 tlb_flush = true; 5164 } 5165 5166 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5167 if (VALID_PAGE(mmu->prev_roots[i].hpa) && 5168 pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) { 5169 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); 5170 tlb_flush = true; 5171 } 5172 } 5173 5174 if (tlb_flush) 5175 static_call(kvm_x86_tlb_flush_gva)(vcpu, gva); 5176 5177 ++vcpu->stat.invlpg; 5178 5179 /* 5180 * Mappings not reachable via the current cr3 or the prev_roots will be 5181 * synced when switching to that cr3, so nothing needs to be done here 5182 * for them. 5183 */ 5184 } 5185 5186 void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, 5187 int tdp_huge_page_level) 5188 { 5189 tdp_enabled = enable_tdp; 5190 max_tdp_level = tdp_max_root_level; 5191 5192 /* 5193 * max_huge_page_level reflects KVM's MMU capabilities irrespective 5194 * of kernel support, e.g. KVM may be capable of using 1GB pages when 5195 * the kernel is not. But, KVM never creates a page size greater than 5196 * what is used by the kernel for any given HVA, i.e. the kernel's 5197 * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust(). 5198 */ 5199 if (tdp_enabled) 5200 max_huge_page_level = tdp_huge_page_level; 5201 else if (boot_cpu_has(X86_FEATURE_GBPAGES)) 5202 max_huge_page_level = PG_LEVEL_1G; 5203 else 5204 max_huge_page_level = PG_LEVEL_2M; 5205 } 5206 EXPORT_SYMBOL_GPL(kvm_configure_mmu); 5207 5208 /* The return value indicates if tlb flush on all vcpus is needed. */ 5209 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head, 5210 struct kvm_memory_slot *slot); 5211 5212 /* The caller should hold mmu-lock before calling this function. */ 5213 static __always_inline bool 5214 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, 5215 slot_level_handler fn, int start_level, int end_level, 5216 gfn_t start_gfn, gfn_t end_gfn, bool flush_on_yield, 5217 bool flush) 5218 { 5219 struct slot_rmap_walk_iterator iterator; 5220 5221 for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, 5222 end_gfn, &iterator) { 5223 if (iterator.rmap) 5224 flush |= fn(kvm, iterator.rmap, memslot); 5225 5226 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { 5227 if (flush && flush_on_yield) { 5228 kvm_flush_remote_tlbs_with_address(kvm, 5229 start_gfn, 5230 iterator.gfn - start_gfn + 1); 5231 flush = false; 5232 } 5233 cond_resched_rwlock_write(&kvm->mmu_lock); 5234 } 5235 } 5236 5237 return flush; 5238 } 5239 5240 static __always_inline bool 5241 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5242 slot_level_handler fn, int start_level, int end_level, 5243 bool flush_on_yield) 5244 { 5245 return slot_handle_level_range(kvm, memslot, fn, start_level, 5246 end_level, memslot->base_gfn, 5247 memslot->base_gfn + memslot->npages - 1, 5248 flush_on_yield, false); 5249 } 5250 5251 static __always_inline bool 5252 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, 5253 slot_level_handler fn, bool flush_on_yield) 5254 { 5255 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, 5256 PG_LEVEL_4K, flush_on_yield); 5257 } 5258 5259 static void free_mmu_pages(struct kvm_mmu *mmu) 5260 { 5261 if (!tdp_enabled && mmu->pae_root) 5262 set_memory_encrypted((unsigned long)mmu->pae_root, 1); 5263 free_page((unsigned long)mmu->pae_root); 5264 free_page((unsigned long)mmu->pml4_root); 5265 } 5266 5267 static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) 5268 { 5269 struct page *page; 5270 int i; 5271 5272 mmu->root_hpa = INVALID_PAGE; 5273 mmu->root_pgd = 0; 5274 mmu->translate_gpa = translate_gpa; 5275 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5276 mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 5277 5278 /* 5279 * When using PAE paging, the four PDPTEs are treated as 'root' pages, 5280 * while the PDP table is a per-vCPU construct that's allocated at MMU 5281 * creation. When emulating 32-bit mode, cr3 is only 32 bits even on 5282 * x86_64. Therefore we need to allocate the PDP table in the first 5283 * 4GB of memory, which happens to fit the DMA32 zone. TDP paging 5284 * generally doesn't use PAE paging and can skip allocating the PDP 5285 * table. The main exception, handled here, is SVM's 32-bit NPT. The 5286 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit 5287 * KVM; that horror is handled on-demand by mmu_alloc_shadow_roots(). 5288 */ 5289 if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) 5290 return 0; 5291 5292 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); 5293 if (!page) 5294 return -ENOMEM; 5295 5296 mmu->pae_root = page_address(page); 5297 5298 /* 5299 * CR3 is only 32 bits when PAE paging is used, thus it's impossible to 5300 * get the CPU to treat the PDPTEs as encrypted. Decrypt the page so 5301 * that KVM's writes and the CPU's reads get along. Note, this is 5302 * only necessary when using shadow paging, as 64-bit NPT can get at 5303 * the C-bit even when shadowing 32-bit NPT, and SME isn't supported 5304 * by 32-bit kernels (when KVM itself uses 32-bit NPT). 5305 */ 5306 if (!tdp_enabled) 5307 set_memory_decrypted((unsigned long)mmu->pae_root, 1); 5308 else 5309 WARN_ON_ONCE(shadow_me_mask); 5310 5311 for (i = 0; i < 4; ++i) 5312 mmu->pae_root[i] = INVALID_PAE_ROOT; 5313 5314 return 0; 5315 } 5316 5317 int kvm_mmu_create(struct kvm_vcpu *vcpu) 5318 { 5319 int ret; 5320 5321 vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache; 5322 vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO; 5323 5324 vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache; 5325 vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO; 5326 5327 vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO; 5328 5329 vcpu->arch.mmu = &vcpu->arch.root_mmu; 5330 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 5331 5332 vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa; 5333 5334 ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu); 5335 if (ret) 5336 return ret; 5337 5338 ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu); 5339 if (ret) 5340 goto fail_allocate_root; 5341 5342 return ret; 5343 fail_allocate_root: 5344 free_mmu_pages(&vcpu->arch.guest_mmu); 5345 return ret; 5346 } 5347 5348 #define BATCH_ZAP_PAGES 10 5349 static void kvm_zap_obsolete_pages(struct kvm *kvm) 5350 { 5351 struct kvm_mmu_page *sp, *node; 5352 int nr_zapped, batch = 0; 5353 5354 restart: 5355 list_for_each_entry_safe_reverse(sp, node, 5356 &kvm->arch.active_mmu_pages, link) { 5357 /* 5358 * No obsolete valid page exists before a newly created page 5359 * since active_mmu_pages is a FIFO list. 5360 */ 5361 if (!is_obsolete_sp(kvm, sp)) 5362 break; 5363 5364 /* 5365 * Invalid pages should never land back on the list of active 5366 * pages. Skip the bogus page, otherwise we'll get stuck in an 5367 * infinite loop if the page gets put back on the list (again). 5368 */ 5369 if (WARN_ON(sp->role.invalid)) 5370 continue; 5371 5372 /* 5373 * No need to flush the TLB since we're only zapping shadow 5374 * pages with an obsolete generation number and all vCPUS have 5375 * loaded a new root, i.e. the shadow pages being zapped cannot 5376 * be in active use by the guest. 5377 */ 5378 if (batch >= BATCH_ZAP_PAGES && 5379 cond_resched_rwlock_write(&kvm->mmu_lock)) { 5380 batch = 0; 5381 goto restart; 5382 } 5383 5384 if (__kvm_mmu_prepare_zap_page(kvm, sp, 5385 &kvm->arch.zapped_obsolete_pages, &nr_zapped)) { 5386 batch += nr_zapped; 5387 goto restart; 5388 } 5389 } 5390 5391 /* 5392 * Trigger a remote TLB flush before freeing the page tables to ensure 5393 * KVM is not in the middle of a lockless shadow page table walk, which 5394 * may reference the pages. 5395 */ 5396 kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages); 5397 } 5398 5399 /* 5400 * Fast invalidate all shadow pages and use lock-break technique 5401 * to zap obsolete pages. 5402 * 5403 * It's required when memslot is being deleted or VM is being 5404 * destroyed, in these cases, we should ensure that KVM MMU does 5405 * not use any resource of the being-deleted slot or all slots 5406 * after calling the function. 5407 */ 5408 static void kvm_mmu_zap_all_fast(struct kvm *kvm) 5409 { 5410 lockdep_assert_held(&kvm->slots_lock); 5411 5412 write_lock(&kvm->mmu_lock); 5413 trace_kvm_mmu_zap_all_fast(kvm); 5414 5415 /* 5416 * Toggle mmu_valid_gen between '0' and '1'. Because slots_lock is 5417 * held for the entire duration of zapping obsolete pages, it's 5418 * impossible for there to be multiple invalid generations associated 5419 * with *valid* shadow pages at any given time, i.e. there is exactly 5420 * one valid generation and (at most) one invalid generation. 5421 */ 5422 kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1; 5423 5424 /* In order to ensure all threads see this change when 5425 * handling the MMU reload signal, this must happen in the 5426 * same critical section as kvm_reload_remote_mmus, and 5427 * before kvm_zap_obsolete_pages as kvm_zap_obsolete_pages 5428 * could drop the MMU lock and yield. 5429 */ 5430 if (is_tdp_mmu_enabled(kvm)) 5431 kvm_tdp_mmu_invalidate_all_roots(kvm); 5432 5433 /* 5434 * Notify all vcpus to reload its shadow page table and flush TLB. 5435 * Then all vcpus will switch to new shadow page table with the new 5436 * mmu_valid_gen. 5437 * 5438 * Note: we need to do this under the protection of mmu_lock, 5439 * otherwise, vcpu would purge shadow page but miss tlb flush. 5440 */ 5441 kvm_reload_remote_mmus(kvm); 5442 5443 kvm_zap_obsolete_pages(kvm); 5444 5445 write_unlock(&kvm->mmu_lock); 5446 5447 if (is_tdp_mmu_enabled(kvm)) { 5448 read_lock(&kvm->mmu_lock); 5449 kvm_tdp_mmu_zap_invalidated_roots(kvm); 5450 read_unlock(&kvm->mmu_lock); 5451 } 5452 } 5453 5454 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) 5455 { 5456 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages)); 5457 } 5458 5459 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm, 5460 struct kvm_memory_slot *slot, 5461 struct kvm_page_track_notifier_node *node) 5462 { 5463 kvm_mmu_zap_all_fast(kvm); 5464 } 5465 5466 void kvm_mmu_init_vm(struct kvm *kvm) 5467 { 5468 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 5469 5470 kvm_mmu_init_tdp_mmu(kvm); 5471 5472 node->track_write = kvm_mmu_pte_write; 5473 node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; 5474 kvm_page_track_register_notifier(kvm, node); 5475 } 5476 5477 void kvm_mmu_uninit_vm(struct kvm *kvm) 5478 { 5479 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 5480 5481 kvm_page_track_unregister_notifier(kvm, node); 5482 5483 kvm_mmu_uninit_tdp_mmu(kvm); 5484 } 5485 5486 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) 5487 { 5488 struct kvm_memslots *slots; 5489 struct kvm_memory_slot *memslot; 5490 int i; 5491 bool flush = false; 5492 5493 write_lock(&kvm->mmu_lock); 5494 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 5495 slots = __kvm_memslots(kvm, i); 5496 kvm_for_each_memslot(memslot, slots) { 5497 gfn_t start, end; 5498 5499 start = max(gfn_start, memslot->base_gfn); 5500 end = min(gfn_end, memslot->base_gfn + memslot->npages); 5501 if (start >= end) 5502 continue; 5503 5504 flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, 5505 PG_LEVEL_4K, 5506 KVM_MAX_HUGEPAGE_LEVEL, 5507 start, end - 1, true, flush); 5508 } 5509 } 5510 5511 if (flush) 5512 kvm_flush_remote_tlbs_with_address(kvm, gfn_start, gfn_end); 5513 5514 write_unlock(&kvm->mmu_lock); 5515 5516 if (is_tdp_mmu_enabled(kvm)) { 5517 flush = false; 5518 5519 read_lock(&kvm->mmu_lock); 5520 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 5521 flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, gfn_start, 5522 gfn_end, flush, true); 5523 if (flush) 5524 kvm_flush_remote_tlbs_with_address(kvm, gfn_start, 5525 gfn_end); 5526 5527 read_unlock(&kvm->mmu_lock); 5528 } 5529 } 5530 5531 static bool slot_rmap_write_protect(struct kvm *kvm, 5532 struct kvm_rmap_head *rmap_head, 5533 struct kvm_memory_slot *slot) 5534 { 5535 return __rmap_write_protect(kvm, rmap_head, false); 5536 } 5537 5538 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, 5539 struct kvm_memory_slot *memslot, 5540 int start_level) 5541 { 5542 bool flush; 5543 5544 write_lock(&kvm->mmu_lock); 5545 flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, 5546 start_level, KVM_MAX_HUGEPAGE_LEVEL, false); 5547 write_unlock(&kvm->mmu_lock); 5548 5549 if (is_tdp_mmu_enabled(kvm)) { 5550 read_lock(&kvm->mmu_lock); 5551 flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level); 5552 read_unlock(&kvm->mmu_lock); 5553 } 5554 5555 /* 5556 * We can flush all the TLBs out of the mmu lock without TLB 5557 * corruption since we just change the spte from writable to 5558 * readonly so that we only need to care the case of changing 5559 * spte from present to present (changing the spte from present 5560 * to nonpresent will flush all the TLBs immediately), in other 5561 * words, the only case we care is mmu_spte_update() where we 5562 * have checked Host-writable | MMU-writable instead of 5563 * PT_WRITABLE_MASK, that means it does not depend on PT_WRITABLE_MASK 5564 * anymore. 5565 */ 5566 if (flush) 5567 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5568 } 5569 5570 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, 5571 struct kvm_rmap_head *rmap_head, 5572 struct kvm_memory_slot *slot) 5573 { 5574 u64 *sptep; 5575 struct rmap_iterator iter; 5576 int need_tlb_flush = 0; 5577 kvm_pfn_t pfn; 5578 struct kvm_mmu_page *sp; 5579 5580 restart: 5581 for_each_rmap_spte(rmap_head, &iter, sptep) { 5582 sp = sptep_to_sp(sptep); 5583 pfn = spte_to_pfn(*sptep); 5584 5585 /* 5586 * We cannot do huge page mapping for indirect shadow pages, 5587 * which are found on the last rmap (level = 1) when not using 5588 * tdp; such shadow pages are synced with the page table in 5589 * the guest, and the guest page table is using 4K page size 5590 * mapping if the indirect sp has level = 1. 5591 */ 5592 if (sp->role.direct && !kvm_is_reserved_pfn(pfn) && 5593 sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn, 5594 pfn, PG_LEVEL_NUM)) { 5595 pte_list_remove(rmap_head, sptep); 5596 5597 if (kvm_available_flush_tlb_with_range()) 5598 kvm_flush_remote_tlbs_with_address(kvm, sp->gfn, 5599 KVM_PAGES_PER_HPAGE(sp->role.level)); 5600 else 5601 need_tlb_flush = 1; 5602 5603 goto restart; 5604 } 5605 } 5606 5607 return need_tlb_flush; 5608 } 5609 5610 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, 5611 const struct kvm_memory_slot *memslot) 5612 { 5613 /* FIXME: const-ify all uses of struct kvm_memory_slot. */ 5614 struct kvm_memory_slot *slot = (struct kvm_memory_slot *)memslot; 5615 bool flush; 5616 5617 write_lock(&kvm->mmu_lock); 5618 flush = slot_handle_leaf(kvm, slot, kvm_mmu_zap_collapsible_spte, true); 5619 5620 if (flush) 5621 kvm_arch_flush_remote_tlbs_memslot(kvm, slot); 5622 write_unlock(&kvm->mmu_lock); 5623 5624 if (is_tdp_mmu_enabled(kvm)) { 5625 flush = false; 5626 5627 read_lock(&kvm->mmu_lock); 5628 flush = kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot, flush); 5629 if (flush) 5630 kvm_arch_flush_remote_tlbs_memslot(kvm, slot); 5631 read_unlock(&kvm->mmu_lock); 5632 } 5633 } 5634 5635 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 5636 const struct kvm_memory_slot *memslot) 5637 { 5638 /* 5639 * All current use cases for flushing the TLBs for a specific memslot 5640 * related to dirty logging, and many do the TLB flush out of mmu_lock. 5641 * The interaction between the various operations on memslot must be 5642 * serialized by slots_locks to ensure the TLB flush from one operation 5643 * is observed by any other operation on the same memslot. 5644 */ 5645 lockdep_assert_held(&kvm->slots_lock); 5646 kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, 5647 memslot->npages); 5648 } 5649 5650 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 5651 struct kvm_memory_slot *memslot) 5652 { 5653 bool flush; 5654 5655 write_lock(&kvm->mmu_lock); 5656 flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false); 5657 write_unlock(&kvm->mmu_lock); 5658 5659 if (is_tdp_mmu_enabled(kvm)) { 5660 read_lock(&kvm->mmu_lock); 5661 flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot); 5662 read_unlock(&kvm->mmu_lock); 5663 } 5664 5665 /* 5666 * It's also safe to flush TLBs out of mmu lock here as currently this 5667 * function is only used for dirty logging, in which case flushing TLB 5668 * out of mmu lock also guarantees no dirty pages will be lost in 5669 * dirty_bitmap. 5670 */ 5671 if (flush) 5672 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5673 } 5674 5675 void kvm_mmu_zap_all(struct kvm *kvm) 5676 { 5677 struct kvm_mmu_page *sp, *node; 5678 LIST_HEAD(invalid_list); 5679 int ign; 5680 5681 write_lock(&kvm->mmu_lock); 5682 restart: 5683 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) { 5684 if (WARN_ON(sp->role.invalid)) 5685 continue; 5686 if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign)) 5687 goto restart; 5688 if (cond_resched_rwlock_write(&kvm->mmu_lock)) 5689 goto restart; 5690 } 5691 5692 kvm_mmu_commit_zap_page(kvm, &invalid_list); 5693 5694 if (is_tdp_mmu_enabled(kvm)) 5695 kvm_tdp_mmu_zap_all(kvm); 5696 5697 write_unlock(&kvm->mmu_lock); 5698 } 5699 5700 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) 5701 { 5702 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); 5703 5704 gen &= MMIO_SPTE_GEN_MASK; 5705 5706 /* 5707 * Generation numbers are incremented in multiples of the number of 5708 * address spaces in order to provide unique generations across all 5709 * address spaces. Strip what is effectively the address space 5710 * modifier prior to checking for a wrap of the MMIO generation so 5711 * that a wrap in any address space is detected. 5712 */ 5713 gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1); 5714 5715 /* 5716 * The very rare case: if the MMIO generation number has wrapped, 5717 * zap all shadow pages. 5718 */ 5719 if (unlikely(gen == 0)) { 5720 kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n"); 5721 kvm_mmu_zap_all_fast(kvm); 5722 } 5723 } 5724 5725 static unsigned long 5726 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 5727 { 5728 struct kvm *kvm; 5729 int nr_to_scan = sc->nr_to_scan; 5730 unsigned long freed = 0; 5731 5732 mutex_lock(&kvm_lock); 5733 5734 list_for_each_entry(kvm, &vm_list, vm_list) { 5735 int idx; 5736 LIST_HEAD(invalid_list); 5737 5738 /* 5739 * Never scan more than sc->nr_to_scan VM instances. 5740 * Will not hit this condition practically since we do not try 5741 * to shrink more than one VM and it is very unlikely to see 5742 * !n_used_mmu_pages so many times. 5743 */ 5744 if (!nr_to_scan--) 5745 break; 5746 /* 5747 * n_used_mmu_pages is accessed without holding kvm->mmu_lock 5748 * here. We may skip a VM instance errorneosly, but we do not 5749 * want to shrink a VM that only started to populate its MMU 5750 * anyway. 5751 */ 5752 if (!kvm->arch.n_used_mmu_pages && 5753 !kvm_has_zapped_obsolete_pages(kvm)) 5754 continue; 5755 5756 idx = srcu_read_lock(&kvm->srcu); 5757 write_lock(&kvm->mmu_lock); 5758 5759 if (kvm_has_zapped_obsolete_pages(kvm)) { 5760 kvm_mmu_commit_zap_page(kvm, 5761 &kvm->arch.zapped_obsolete_pages); 5762 goto unlock; 5763 } 5764 5765 freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan); 5766 5767 unlock: 5768 write_unlock(&kvm->mmu_lock); 5769 srcu_read_unlock(&kvm->srcu, idx); 5770 5771 /* 5772 * unfair on small ones 5773 * per-vm shrinkers cry out 5774 * sadness comes quickly 5775 */ 5776 list_move_tail(&kvm->vm_list, &vm_list); 5777 break; 5778 } 5779 5780 mutex_unlock(&kvm_lock); 5781 return freed; 5782 } 5783 5784 static unsigned long 5785 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 5786 { 5787 return percpu_counter_read_positive(&kvm_total_used_mmu_pages); 5788 } 5789 5790 static struct shrinker mmu_shrinker = { 5791 .count_objects = mmu_shrink_count, 5792 .scan_objects = mmu_shrink_scan, 5793 .seeks = DEFAULT_SEEKS * 10, 5794 }; 5795 5796 static void mmu_destroy_caches(void) 5797 { 5798 kmem_cache_destroy(pte_list_desc_cache); 5799 kmem_cache_destroy(mmu_page_header_cache); 5800 } 5801 5802 static bool get_nx_auto_mode(void) 5803 { 5804 /* Return true when CPU has the bug, and mitigations are ON */ 5805 return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off(); 5806 } 5807 5808 static void __set_nx_huge_pages(bool val) 5809 { 5810 nx_huge_pages = itlb_multihit_kvm_mitigation = val; 5811 } 5812 5813 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp) 5814 { 5815 bool old_val = nx_huge_pages; 5816 bool new_val; 5817 5818 /* In "auto" mode deploy workaround only if CPU has the bug. */ 5819 if (sysfs_streq(val, "off")) 5820 new_val = 0; 5821 else if (sysfs_streq(val, "force")) 5822 new_val = 1; 5823 else if (sysfs_streq(val, "auto")) 5824 new_val = get_nx_auto_mode(); 5825 else if (strtobool(val, &new_val) < 0) 5826 return -EINVAL; 5827 5828 __set_nx_huge_pages(new_val); 5829 5830 if (new_val != old_val) { 5831 struct kvm *kvm; 5832 5833 mutex_lock(&kvm_lock); 5834 5835 list_for_each_entry(kvm, &vm_list, vm_list) { 5836 mutex_lock(&kvm->slots_lock); 5837 kvm_mmu_zap_all_fast(kvm); 5838 mutex_unlock(&kvm->slots_lock); 5839 5840 wake_up_process(kvm->arch.nx_lpage_recovery_thread); 5841 } 5842 mutex_unlock(&kvm_lock); 5843 } 5844 5845 return 0; 5846 } 5847 5848 int kvm_mmu_module_init(void) 5849 { 5850 int ret = -ENOMEM; 5851 5852 if (nx_huge_pages == -1) 5853 __set_nx_huge_pages(get_nx_auto_mode()); 5854 5855 /* 5856 * MMU roles use union aliasing which is, generally speaking, an 5857 * undefined behavior. However, we supposedly know how compilers behave 5858 * and the current status quo is unlikely to change. Guardians below are 5859 * supposed to let us know if the assumption becomes false. 5860 */ 5861 BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32)); 5862 BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32)); 5863 BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64)); 5864 5865 kvm_mmu_reset_all_pte_masks(); 5866 5867 pte_list_desc_cache = kmem_cache_create("pte_list_desc", 5868 sizeof(struct pte_list_desc), 5869 0, SLAB_ACCOUNT, NULL); 5870 if (!pte_list_desc_cache) 5871 goto out; 5872 5873 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header", 5874 sizeof(struct kvm_mmu_page), 5875 0, SLAB_ACCOUNT, NULL); 5876 if (!mmu_page_header_cache) 5877 goto out; 5878 5879 if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL)) 5880 goto out; 5881 5882 ret = register_shrinker(&mmu_shrinker); 5883 if (ret) 5884 goto out; 5885 5886 return 0; 5887 5888 out: 5889 mmu_destroy_caches(); 5890 return ret; 5891 } 5892 5893 /* 5894 * Calculate mmu pages needed for kvm. 5895 */ 5896 unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm) 5897 { 5898 unsigned long nr_mmu_pages; 5899 unsigned long nr_pages = 0; 5900 struct kvm_memslots *slots; 5901 struct kvm_memory_slot *memslot; 5902 int i; 5903 5904 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 5905 slots = __kvm_memslots(kvm, i); 5906 5907 kvm_for_each_memslot(memslot, slots) 5908 nr_pages += memslot->npages; 5909 } 5910 5911 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; 5912 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); 5913 5914 return nr_mmu_pages; 5915 } 5916 5917 void kvm_mmu_destroy(struct kvm_vcpu *vcpu) 5918 { 5919 kvm_mmu_unload(vcpu); 5920 free_mmu_pages(&vcpu->arch.root_mmu); 5921 free_mmu_pages(&vcpu->arch.guest_mmu); 5922 mmu_free_memory_caches(vcpu); 5923 } 5924 5925 void kvm_mmu_module_exit(void) 5926 { 5927 mmu_destroy_caches(); 5928 percpu_counter_destroy(&kvm_total_used_mmu_pages); 5929 unregister_shrinker(&mmu_shrinker); 5930 mmu_audit_disable(); 5931 } 5932 5933 static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp) 5934 { 5935 unsigned int old_val; 5936 int err; 5937 5938 old_val = nx_huge_pages_recovery_ratio; 5939 err = param_set_uint(val, kp); 5940 if (err) 5941 return err; 5942 5943 if (READ_ONCE(nx_huge_pages) && 5944 !old_val && nx_huge_pages_recovery_ratio) { 5945 struct kvm *kvm; 5946 5947 mutex_lock(&kvm_lock); 5948 5949 list_for_each_entry(kvm, &vm_list, vm_list) 5950 wake_up_process(kvm->arch.nx_lpage_recovery_thread); 5951 5952 mutex_unlock(&kvm_lock); 5953 } 5954 5955 return err; 5956 } 5957 5958 static void kvm_recover_nx_lpages(struct kvm *kvm) 5959 { 5960 int rcu_idx; 5961 struct kvm_mmu_page *sp; 5962 unsigned int ratio; 5963 LIST_HEAD(invalid_list); 5964 bool flush = false; 5965 ulong to_zap; 5966 5967 rcu_idx = srcu_read_lock(&kvm->srcu); 5968 write_lock(&kvm->mmu_lock); 5969 5970 ratio = READ_ONCE(nx_huge_pages_recovery_ratio); 5971 to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0; 5972 for ( ; to_zap; --to_zap) { 5973 if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages)) 5974 break; 5975 5976 /* 5977 * We use a separate list instead of just using active_mmu_pages 5978 * because the number of lpage_disallowed pages is expected to 5979 * be relatively small compared to the total. 5980 */ 5981 sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages, 5982 struct kvm_mmu_page, 5983 lpage_disallowed_link); 5984 WARN_ON_ONCE(!sp->lpage_disallowed); 5985 if (is_tdp_mmu_page(sp)) { 5986 flush |= kvm_tdp_mmu_zap_sp(kvm, sp); 5987 } else { 5988 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); 5989 WARN_ON_ONCE(sp->lpage_disallowed); 5990 } 5991 5992 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { 5993 kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); 5994 cond_resched_rwlock_write(&kvm->mmu_lock); 5995 flush = false; 5996 } 5997 } 5998 kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); 5999 6000 write_unlock(&kvm->mmu_lock); 6001 srcu_read_unlock(&kvm->srcu, rcu_idx); 6002 } 6003 6004 static long get_nx_lpage_recovery_timeout(u64 start_time) 6005 { 6006 return READ_ONCE(nx_huge_pages) && READ_ONCE(nx_huge_pages_recovery_ratio) 6007 ? start_time + 60 * HZ - get_jiffies_64() 6008 : MAX_SCHEDULE_TIMEOUT; 6009 } 6010 6011 static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data) 6012 { 6013 u64 start_time; 6014 long remaining_time; 6015 6016 while (true) { 6017 start_time = get_jiffies_64(); 6018 remaining_time = get_nx_lpage_recovery_timeout(start_time); 6019 6020 set_current_state(TASK_INTERRUPTIBLE); 6021 while (!kthread_should_stop() && remaining_time > 0) { 6022 schedule_timeout(remaining_time); 6023 remaining_time = get_nx_lpage_recovery_timeout(start_time); 6024 set_current_state(TASK_INTERRUPTIBLE); 6025 } 6026 6027 set_current_state(TASK_RUNNING); 6028 6029 if (kthread_should_stop()) 6030 return 0; 6031 6032 kvm_recover_nx_lpages(kvm); 6033 } 6034 } 6035 6036 int kvm_mmu_post_init_vm(struct kvm *kvm) 6037 { 6038 int err; 6039 6040 err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0, 6041 "kvm-nx-lpage-recovery", 6042 &kvm->arch.nx_lpage_recovery_thread); 6043 if (!err) 6044 kthread_unpark(kvm->arch.nx_lpage_recovery_thread); 6045 6046 return err; 6047 } 6048 6049 void kvm_mmu_pre_destroy_vm(struct kvm *kvm) 6050 { 6051 if (kvm->arch.nx_lpage_recovery_thread) 6052 kthread_stop(kvm->arch.nx_lpage_recovery_thread); 6053 } 6054