1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * MMU support 9 * 10 * Copyright (C) 2006 Qumranet, Inc. 11 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 12 * 13 * Authors: 14 * Yaniv Kamay <yaniv@qumranet.com> 15 * Avi Kivity <avi@qumranet.com> 16 */ 17 18 #include "irq.h" 19 #include "ioapic.h" 20 #include "mmu.h" 21 #include "x86.h" 22 #include "kvm_cache_regs.h" 23 #include "kvm_emulate.h" 24 #include "cpuid.h" 25 26 #include <linux/kvm_host.h> 27 #include <linux/types.h> 28 #include <linux/string.h> 29 #include <linux/mm.h> 30 #include <linux/highmem.h> 31 #include <linux/moduleparam.h> 32 #include <linux/export.h> 33 #include <linux/swap.h> 34 #include <linux/hugetlb.h> 35 #include <linux/compiler.h> 36 #include <linux/srcu.h> 37 #include <linux/slab.h> 38 #include <linux/sched/signal.h> 39 #include <linux/uaccess.h> 40 #include <linux/hash.h> 41 #include <linux/kern_levels.h> 42 #include <linux/kthread.h> 43 44 #include <asm/page.h> 45 #include <asm/memtype.h> 46 #include <asm/cmpxchg.h> 47 #include <asm/e820/api.h> 48 #include <asm/io.h> 49 #include <asm/vmx.h> 50 #include <asm/kvm_page_track.h> 51 #include "trace.h" 52 53 extern bool itlb_multihit_kvm_mitigation; 54 55 static int __read_mostly nx_huge_pages = -1; 56 #ifdef CONFIG_PREEMPT_RT 57 /* Recovery can cause latency spikes, disable it for PREEMPT_RT. */ 58 static uint __read_mostly nx_huge_pages_recovery_ratio = 0; 59 #else 60 static uint __read_mostly nx_huge_pages_recovery_ratio = 60; 61 #endif 62 63 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp); 64 static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp); 65 66 static struct kernel_param_ops nx_huge_pages_ops = { 67 .set = set_nx_huge_pages, 68 .get = param_get_bool, 69 }; 70 71 static struct kernel_param_ops nx_huge_pages_recovery_ratio_ops = { 72 .set = set_nx_huge_pages_recovery_ratio, 73 .get = param_get_uint, 74 }; 75 76 module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644); 77 __MODULE_PARM_TYPE(nx_huge_pages, "bool"); 78 module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_ratio_ops, 79 &nx_huge_pages_recovery_ratio, 0644); 80 __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint"); 81 82 static bool __read_mostly force_flush_and_sync_on_reuse; 83 module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); 84 85 /* 86 * When setting this variable to true it enables Two-Dimensional-Paging 87 * where the hardware walks 2 page tables: 88 * 1. the guest-virtual to guest-physical 89 * 2. while doing 1. it walks guest-physical to host-physical 90 * If the hardware supports that we don't need to do shadow paging. 91 */ 92 bool tdp_enabled = false; 93 94 static int max_page_level __read_mostly; 95 96 enum { 97 AUDIT_PRE_PAGE_FAULT, 98 AUDIT_POST_PAGE_FAULT, 99 AUDIT_PRE_PTE_WRITE, 100 AUDIT_POST_PTE_WRITE, 101 AUDIT_PRE_SYNC, 102 AUDIT_POST_SYNC 103 }; 104 105 #undef MMU_DEBUG 106 107 #ifdef MMU_DEBUG 108 static bool dbg = 0; 109 module_param(dbg, bool, 0644); 110 111 #define pgprintk(x...) do { if (dbg) printk(x); } while (0) 112 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0) 113 #define MMU_WARN_ON(x) WARN_ON(x) 114 #else 115 #define pgprintk(x...) do { } while (0) 116 #define rmap_printk(x...) do { } while (0) 117 #define MMU_WARN_ON(x) do { } while (0) 118 #endif 119 120 #define PTE_PREFETCH_NUM 8 121 122 #define PT_FIRST_AVAIL_BITS_SHIFT 10 123 #define PT64_SECOND_AVAIL_BITS_SHIFT 54 124 125 /* 126 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or 127 * Access Tracking SPTEs. 128 */ 129 #define SPTE_SPECIAL_MASK (3ULL << 52) 130 #define SPTE_AD_ENABLED_MASK (0ULL << 52) 131 #define SPTE_AD_DISABLED_MASK (1ULL << 52) 132 #define SPTE_AD_WRPROT_ONLY_MASK (2ULL << 52) 133 #define SPTE_MMIO_MASK (3ULL << 52) 134 135 #define PT64_LEVEL_BITS 9 136 137 #define PT64_LEVEL_SHIFT(level) \ 138 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS) 139 140 #define PT64_INDEX(address, level)\ 141 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1)) 142 143 144 #define PT32_LEVEL_BITS 10 145 146 #define PT32_LEVEL_SHIFT(level) \ 147 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) 148 149 #define PT32_LVL_OFFSET_MASK(level) \ 150 (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \ 151 * PT32_LEVEL_BITS))) - 1)) 152 153 #define PT32_INDEX(address, level)\ 154 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1)) 155 156 157 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK 158 #define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1)) 159 #else 160 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)) 161 #endif 162 #define PT64_LVL_ADDR_MASK(level) \ 163 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \ 164 * PT64_LEVEL_BITS))) - 1)) 165 #define PT64_LVL_OFFSET_MASK(level) \ 166 (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \ 167 * PT64_LEVEL_BITS))) - 1)) 168 169 #define PT32_BASE_ADDR_MASK PAGE_MASK 170 #define PT32_DIR_BASE_ADDR_MASK \ 171 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) 172 #define PT32_LVL_ADDR_MASK(level) \ 173 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \ 174 * PT32_LEVEL_BITS))) - 1)) 175 176 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \ 177 | shadow_x_mask | shadow_nx_mask | shadow_me_mask) 178 179 #define ACC_EXEC_MASK 1 180 #define ACC_WRITE_MASK PT_WRITABLE_MASK 181 #define ACC_USER_MASK PT_USER_MASK 182 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) 183 184 /* The mask for the R/X bits in EPT PTEs */ 185 #define PT64_EPT_READABLE_MASK 0x1ull 186 #define PT64_EPT_EXECUTABLE_MASK 0x4ull 187 188 #include <trace/events/kvm.h> 189 190 #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT) 191 #define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1)) 192 193 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) 194 195 /* make pte_list_desc fit well in cache line */ 196 #define PTE_LIST_EXT 3 197 198 /* 199 * Return values of handle_mmio_page_fault and mmu.page_fault: 200 * RET_PF_RETRY: let CPU fault again on the address. 201 * RET_PF_EMULATE: mmio page fault, emulate the instruction directly. 202 * 203 * For handle_mmio_page_fault only: 204 * RET_PF_INVALID: the spte is invalid, let the real page fault path update it. 205 */ 206 enum { 207 RET_PF_RETRY = 0, 208 RET_PF_EMULATE = 1, 209 RET_PF_INVALID = 2, 210 }; 211 212 struct pte_list_desc { 213 u64 *sptes[PTE_LIST_EXT]; 214 struct pte_list_desc *more; 215 }; 216 217 struct kvm_shadow_walk_iterator { 218 u64 addr; 219 hpa_t shadow_addr; 220 u64 *sptep; 221 int level; 222 unsigned index; 223 }; 224 225 #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker) \ 226 for (shadow_walk_init_using_root(&(_walker), (_vcpu), \ 227 (_root), (_addr)); \ 228 shadow_walk_okay(&(_walker)); \ 229 shadow_walk_next(&(_walker))) 230 231 #define for_each_shadow_entry(_vcpu, _addr, _walker) \ 232 for (shadow_walk_init(&(_walker), _vcpu, _addr); \ 233 shadow_walk_okay(&(_walker)); \ 234 shadow_walk_next(&(_walker))) 235 236 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \ 237 for (shadow_walk_init(&(_walker), _vcpu, _addr); \ 238 shadow_walk_okay(&(_walker)) && \ 239 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \ 240 __shadow_walk_next(&(_walker), spte)) 241 242 static struct kmem_cache *pte_list_desc_cache; 243 static struct kmem_cache *mmu_page_header_cache; 244 static struct percpu_counter kvm_total_used_mmu_pages; 245 246 static u64 __read_mostly shadow_nx_mask; 247 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ 248 static u64 __read_mostly shadow_user_mask; 249 static u64 __read_mostly shadow_accessed_mask; 250 static u64 __read_mostly shadow_dirty_mask; 251 static u64 __read_mostly shadow_mmio_value; 252 static u64 __read_mostly shadow_mmio_access_mask; 253 static u64 __read_mostly shadow_present_mask; 254 static u64 __read_mostly shadow_me_mask; 255 256 /* 257 * SPTEs used by MMUs without A/D bits are marked with SPTE_AD_DISABLED_MASK; 258 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed 259 * pages. 260 */ 261 static u64 __read_mostly shadow_acc_track_mask; 262 263 /* 264 * The mask/shift to use for saving the original R/X bits when marking the PTE 265 * as not-present for access tracking purposes. We do not save the W bit as the 266 * PTEs being access tracked also need to be dirty tracked, so the W bit will be 267 * restored only when a write is attempted to the page. 268 */ 269 static const u64 shadow_acc_track_saved_bits_mask = PT64_EPT_READABLE_MASK | 270 PT64_EPT_EXECUTABLE_MASK; 271 static const u64 shadow_acc_track_saved_bits_shift = PT64_SECOND_AVAIL_BITS_SHIFT; 272 273 /* 274 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order 275 * to guard against L1TF attacks. 276 */ 277 static u64 __read_mostly shadow_nonpresent_or_rsvd_mask; 278 279 /* 280 * The number of high-order 1 bits to use in the mask above. 281 */ 282 static const u64 shadow_nonpresent_or_rsvd_mask_len = 5; 283 284 /* 285 * In some cases, we need to preserve the GFN of a non-present or reserved 286 * SPTE when we usurp the upper five bits of the physical address space to 287 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll 288 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask 289 * left into the reserved bits, i.e. the GFN in the SPTE will be split into 290 * high and low parts. This mask covers the lower bits of the GFN. 291 */ 292 static u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask; 293 294 /* 295 * The number of non-reserved physical address bits irrespective of features 296 * that repurpose legal bits, e.g. MKTME. 297 */ 298 static u8 __read_mostly shadow_phys_bits; 299 300 static void mmu_spte_set(u64 *sptep, u64 spte); 301 static bool is_executable_pte(u64 spte); 302 static union kvm_mmu_page_role 303 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu); 304 305 #define CREATE_TRACE_POINTS 306 #include "mmutrace.h" 307 308 309 static inline bool kvm_available_flush_tlb_with_range(void) 310 { 311 return kvm_x86_ops.tlb_remote_flush_with_range; 312 } 313 314 static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, 315 struct kvm_tlb_range *range) 316 { 317 int ret = -ENOTSUPP; 318 319 if (range && kvm_x86_ops.tlb_remote_flush_with_range) 320 ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range); 321 322 if (ret) 323 kvm_flush_remote_tlbs(kvm); 324 } 325 326 static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, 327 u64 start_gfn, u64 pages) 328 { 329 struct kvm_tlb_range range; 330 331 range.start_gfn = start_gfn; 332 range.pages = pages; 333 334 kvm_flush_remote_tlbs_with_range(kvm, &range); 335 } 336 337 void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 access_mask) 338 { 339 BUG_ON((u64)(unsigned)access_mask != access_mask); 340 WARN_ON(mmio_value & (shadow_nonpresent_or_rsvd_mask << shadow_nonpresent_or_rsvd_mask_len)); 341 WARN_ON(mmio_value & shadow_nonpresent_or_rsvd_lower_gfn_mask); 342 shadow_mmio_value = mmio_value | SPTE_MMIO_MASK; 343 shadow_mmio_access_mask = access_mask; 344 } 345 EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask); 346 347 static bool is_mmio_spte(u64 spte) 348 { 349 return (spte & SPTE_SPECIAL_MASK) == SPTE_MMIO_MASK; 350 } 351 352 static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) 353 { 354 return sp->role.ad_disabled; 355 } 356 357 static inline bool kvm_vcpu_ad_need_write_protect(struct kvm_vcpu *vcpu) 358 { 359 /* 360 * When using the EPT page-modification log, the GPAs in the log 361 * would come from L2 rather than L1. Therefore, we need to rely 362 * on write protection to record dirty pages. This also bypasses 363 * PML, since writes now result in a vmexit. 364 */ 365 return vcpu->arch.mmu == &vcpu->arch.guest_mmu; 366 } 367 368 static inline bool spte_ad_enabled(u64 spte) 369 { 370 MMU_WARN_ON(is_mmio_spte(spte)); 371 return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_DISABLED_MASK; 372 } 373 374 static inline bool spte_ad_need_write_protect(u64 spte) 375 { 376 MMU_WARN_ON(is_mmio_spte(spte)); 377 return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_ENABLED_MASK; 378 } 379 380 static bool is_nx_huge_page_enabled(void) 381 { 382 return READ_ONCE(nx_huge_pages); 383 } 384 385 static inline u64 spte_shadow_accessed_mask(u64 spte) 386 { 387 MMU_WARN_ON(is_mmio_spte(spte)); 388 return spte_ad_enabled(spte) ? shadow_accessed_mask : 0; 389 } 390 391 static inline u64 spte_shadow_dirty_mask(u64 spte) 392 { 393 MMU_WARN_ON(is_mmio_spte(spte)); 394 return spte_ad_enabled(spte) ? shadow_dirty_mask : 0; 395 } 396 397 static inline bool is_access_track_spte(u64 spte) 398 { 399 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0; 400 } 401 402 /* 403 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of 404 * the memslots generation and is derived as follows: 405 * 406 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11 407 * Bits 9-18 of the MMIO generation are propagated to spte bits 52-61 408 * 409 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in 410 * the MMIO generation number, as doing so would require stealing a bit from 411 * the "real" generation number and thus effectively halve the maximum number 412 * of MMIO generations that can be handled before encountering a wrap (which 413 * requires a full MMU zap). The flag is instead explicitly queried when 414 * checking for MMIO spte cache hits. 415 */ 416 #define MMIO_SPTE_GEN_MASK GENMASK_ULL(17, 0) 417 418 #define MMIO_SPTE_GEN_LOW_START 3 419 #define MMIO_SPTE_GEN_LOW_END 11 420 #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \ 421 MMIO_SPTE_GEN_LOW_START) 422 423 #define MMIO_SPTE_GEN_HIGH_START PT64_SECOND_AVAIL_BITS_SHIFT 424 #define MMIO_SPTE_GEN_HIGH_END 62 425 #define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \ 426 MMIO_SPTE_GEN_HIGH_START) 427 428 static u64 generation_mmio_spte_mask(u64 gen) 429 { 430 u64 mask; 431 432 WARN_ON(gen & ~MMIO_SPTE_GEN_MASK); 433 BUILD_BUG_ON((MMIO_SPTE_GEN_HIGH_MASK | MMIO_SPTE_GEN_LOW_MASK) & SPTE_SPECIAL_MASK); 434 435 mask = (gen << MMIO_SPTE_GEN_LOW_START) & MMIO_SPTE_GEN_LOW_MASK; 436 mask |= (gen << MMIO_SPTE_GEN_HIGH_START) & MMIO_SPTE_GEN_HIGH_MASK; 437 return mask; 438 } 439 440 static u64 get_mmio_spte_generation(u64 spte) 441 { 442 u64 gen; 443 444 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_START; 445 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_START; 446 return gen; 447 } 448 449 static u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access) 450 { 451 452 u64 gen = kvm_vcpu_memslots(vcpu)->generation & MMIO_SPTE_GEN_MASK; 453 u64 mask = generation_mmio_spte_mask(gen); 454 u64 gpa = gfn << PAGE_SHIFT; 455 456 access &= shadow_mmio_access_mask; 457 mask |= shadow_mmio_value | access; 458 mask |= gpa | shadow_nonpresent_or_rsvd_mask; 459 mask |= (gpa & shadow_nonpresent_or_rsvd_mask) 460 << shadow_nonpresent_or_rsvd_mask_len; 461 462 return mask; 463 } 464 465 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, 466 unsigned int access) 467 { 468 u64 mask = make_mmio_spte(vcpu, gfn, access); 469 unsigned int gen = get_mmio_spte_generation(mask); 470 471 access = mask & ACC_ALL; 472 473 trace_mark_mmio_spte(sptep, gfn, access, gen); 474 mmu_spte_set(sptep, mask); 475 } 476 477 static gfn_t get_mmio_spte_gfn(u64 spte) 478 { 479 u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask; 480 481 gpa |= (spte >> shadow_nonpresent_or_rsvd_mask_len) 482 & shadow_nonpresent_or_rsvd_mask; 483 484 return gpa >> PAGE_SHIFT; 485 } 486 487 static unsigned get_mmio_spte_access(u64 spte) 488 { 489 return spte & shadow_mmio_access_mask; 490 } 491 492 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, 493 kvm_pfn_t pfn, unsigned int access) 494 { 495 if (unlikely(is_noslot_pfn(pfn))) { 496 mark_mmio_spte(vcpu, sptep, gfn, access); 497 return true; 498 } 499 500 return false; 501 } 502 503 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte) 504 { 505 u64 kvm_gen, spte_gen, gen; 506 507 gen = kvm_vcpu_memslots(vcpu)->generation; 508 if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS)) 509 return false; 510 511 kvm_gen = gen & MMIO_SPTE_GEN_MASK; 512 spte_gen = get_mmio_spte_generation(spte); 513 514 trace_check_mmio_spte(spte, kvm_gen, spte_gen); 515 return likely(kvm_gen == spte_gen); 516 } 517 518 /* 519 * Sets the shadow PTE masks used by the MMU. 520 * 521 * Assumptions: 522 * - Setting either @accessed_mask or @dirty_mask requires setting both 523 * - At least one of @accessed_mask or @acc_track_mask must be set 524 */ 525 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, 526 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask, 527 u64 acc_track_mask, u64 me_mask) 528 { 529 BUG_ON(!dirty_mask != !accessed_mask); 530 BUG_ON(!accessed_mask && !acc_track_mask); 531 BUG_ON(acc_track_mask & SPTE_SPECIAL_MASK); 532 533 shadow_user_mask = user_mask; 534 shadow_accessed_mask = accessed_mask; 535 shadow_dirty_mask = dirty_mask; 536 shadow_nx_mask = nx_mask; 537 shadow_x_mask = x_mask; 538 shadow_present_mask = p_mask; 539 shadow_acc_track_mask = acc_track_mask; 540 shadow_me_mask = me_mask; 541 } 542 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes); 543 544 static u8 kvm_get_shadow_phys_bits(void) 545 { 546 /* 547 * boot_cpu_data.x86_phys_bits is reduced when MKTME or SME are detected 548 * in CPU detection code, but the processor treats those reduced bits as 549 * 'keyID' thus they are not reserved bits. Therefore KVM needs to look at 550 * the physical address bits reported by CPUID. 551 */ 552 if (likely(boot_cpu_data.extended_cpuid_level >= 0x80000008)) 553 return cpuid_eax(0x80000008) & 0xff; 554 555 /* 556 * Quite weird to have VMX or SVM but not MAXPHYADDR; probably a VM with 557 * custom CPUID. Proceed with whatever the kernel found since these features 558 * aren't virtualizable (SME/SEV also require CPUIDs higher than 0x80000008). 559 */ 560 return boot_cpu_data.x86_phys_bits; 561 } 562 563 static void kvm_mmu_reset_all_pte_masks(void) 564 { 565 u8 low_phys_bits; 566 567 shadow_user_mask = 0; 568 shadow_accessed_mask = 0; 569 shadow_dirty_mask = 0; 570 shadow_nx_mask = 0; 571 shadow_x_mask = 0; 572 shadow_present_mask = 0; 573 shadow_acc_track_mask = 0; 574 575 shadow_phys_bits = kvm_get_shadow_phys_bits(); 576 577 /* 578 * If the CPU has 46 or less physical address bits, then set an 579 * appropriate mask to guard against L1TF attacks. Otherwise, it is 580 * assumed that the CPU is not vulnerable to L1TF. 581 * 582 * Some Intel CPUs address the L1 cache using more PA bits than are 583 * reported by CPUID. Use the PA width of the L1 cache when possible 584 * to achieve more effective mitigation, e.g. if system RAM overlaps 585 * the most significant bits of legal physical address space. 586 */ 587 shadow_nonpresent_or_rsvd_mask = 0; 588 low_phys_bits = boot_cpu_data.x86_phys_bits; 589 if (boot_cpu_has_bug(X86_BUG_L1TF) && 590 !WARN_ON_ONCE(boot_cpu_data.x86_cache_bits >= 591 52 - shadow_nonpresent_or_rsvd_mask_len)) { 592 low_phys_bits = boot_cpu_data.x86_cache_bits 593 - shadow_nonpresent_or_rsvd_mask_len; 594 shadow_nonpresent_or_rsvd_mask = 595 rsvd_bits(low_phys_bits, boot_cpu_data.x86_cache_bits - 1); 596 } 597 598 shadow_nonpresent_or_rsvd_lower_gfn_mask = 599 GENMASK_ULL(low_phys_bits - 1, PAGE_SHIFT); 600 } 601 602 static int is_cpuid_PSE36(void) 603 { 604 return 1; 605 } 606 607 static int is_nx(struct kvm_vcpu *vcpu) 608 { 609 return vcpu->arch.efer & EFER_NX; 610 } 611 612 static int is_shadow_present_pte(u64 pte) 613 { 614 return (pte != 0) && !is_mmio_spte(pte); 615 } 616 617 static int is_large_pte(u64 pte) 618 { 619 return pte & PT_PAGE_SIZE_MASK; 620 } 621 622 static int is_last_spte(u64 pte, int level) 623 { 624 if (level == PG_LEVEL_4K) 625 return 1; 626 if (is_large_pte(pte)) 627 return 1; 628 return 0; 629 } 630 631 static bool is_executable_pte(u64 spte) 632 { 633 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask; 634 } 635 636 static kvm_pfn_t spte_to_pfn(u64 pte) 637 { 638 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; 639 } 640 641 static gfn_t pse36_gfn_delta(u32 gpte) 642 { 643 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; 644 645 return (gpte & PT32_DIR_PSE36_MASK) << shift; 646 } 647 648 #ifdef CONFIG_X86_64 649 static void __set_spte(u64 *sptep, u64 spte) 650 { 651 WRITE_ONCE(*sptep, spte); 652 } 653 654 static void __update_clear_spte_fast(u64 *sptep, u64 spte) 655 { 656 WRITE_ONCE(*sptep, spte); 657 } 658 659 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) 660 { 661 return xchg(sptep, spte); 662 } 663 664 static u64 __get_spte_lockless(u64 *sptep) 665 { 666 return READ_ONCE(*sptep); 667 } 668 #else 669 union split_spte { 670 struct { 671 u32 spte_low; 672 u32 spte_high; 673 }; 674 u64 spte; 675 }; 676 677 static void count_spte_clear(u64 *sptep, u64 spte) 678 { 679 struct kvm_mmu_page *sp = page_header(__pa(sptep)); 680 681 if (is_shadow_present_pte(spte)) 682 return; 683 684 /* Ensure the spte is completely set before we increase the count */ 685 smp_wmb(); 686 sp->clear_spte_count++; 687 } 688 689 static void __set_spte(u64 *sptep, u64 spte) 690 { 691 union split_spte *ssptep, sspte; 692 693 ssptep = (union split_spte *)sptep; 694 sspte = (union split_spte)spte; 695 696 ssptep->spte_high = sspte.spte_high; 697 698 /* 699 * If we map the spte from nonpresent to present, We should store 700 * the high bits firstly, then set present bit, so cpu can not 701 * fetch this spte while we are setting the spte. 702 */ 703 smp_wmb(); 704 705 WRITE_ONCE(ssptep->spte_low, sspte.spte_low); 706 } 707 708 static void __update_clear_spte_fast(u64 *sptep, u64 spte) 709 { 710 union split_spte *ssptep, sspte; 711 712 ssptep = (union split_spte *)sptep; 713 sspte = (union split_spte)spte; 714 715 WRITE_ONCE(ssptep->spte_low, sspte.spte_low); 716 717 /* 718 * If we map the spte from present to nonpresent, we should clear 719 * present bit firstly to avoid vcpu fetch the old high bits. 720 */ 721 smp_wmb(); 722 723 ssptep->spte_high = sspte.spte_high; 724 count_spte_clear(sptep, spte); 725 } 726 727 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) 728 { 729 union split_spte *ssptep, sspte, orig; 730 731 ssptep = (union split_spte *)sptep; 732 sspte = (union split_spte)spte; 733 734 /* xchg acts as a barrier before the setting of the high bits */ 735 orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low); 736 orig.spte_high = ssptep->spte_high; 737 ssptep->spte_high = sspte.spte_high; 738 count_spte_clear(sptep, spte); 739 740 return orig.spte; 741 } 742 743 /* 744 * The idea using the light way get the spte on x86_32 guest is from 745 * gup_get_pte (mm/gup.c). 746 * 747 * An spte tlb flush may be pending, because kvm_set_pte_rmapp 748 * coalesces them and we are running out of the MMU lock. Therefore 749 * we need to protect against in-progress updates of the spte. 750 * 751 * Reading the spte while an update is in progress may get the old value 752 * for the high part of the spte. The race is fine for a present->non-present 753 * change (because the high part of the spte is ignored for non-present spte), 754 * but for a present->present change we must reread the spte. 755 * 756 * All such changes are done in two steps (present->non-present and 757 * non-present->present), hence it is enough to count the number of 758 * present->non-present updates: if it changed while reading the spte, 759 * we might have hit the race. This is done using clear_spte_count. 760 */ 761 static u64 __get_spte_lockless(u64 *sptep) 762 { 763 struct kvm_mmu_page *sp = page_header(__pa(sptep)); 764 union split_spte spte, *orig = (union split_spte *)sptep; 765 int count; 766 767 retry: 768 count = sp->clear_spte_count; 769 smp_rmb(); 770 771 spte.spte_low = orig->spte_low; 772 smp_rmb(); 773 774 spte.spte_high = orig->spte_high; 775 smp_rmb(); 776 777 if (unlikely(spte.spte_low != orig->spte_low || 778 count != sp->clear_spte_count)) 779 goto retry; 780 781 return spte.spte; 782 } 783 #endif 784 785 static bool spte_can_locklessly_be_made_writable(u64 spte) 786 { 787 return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) == 788 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE); 789 } 790 791 static bool spte_has_volatile_bits(u64 spte) 792 { 793 if (!is_shadow_present_pte(spte)) 794 return false; 795 796 /* 797 * Always atomically update spte if it can be updated 798 * out of mmu-lock, it can ensure dirty bit is not lost, 799 * also, it can help us to get a stable is_writable_pte() 800 * to ensure tlb flush is not missed. 801 */ 802 if (spte_can_locklessly_be_made_writable(spte) || 803 is_access_track_spte(spte)) 804 return true; 805 806 if (spte_ad_enabled(spte)) { 807 if ((spte & shadow_accessed_mask) == 0 || 808 (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0)) 809 return true; 810 } 811 812 return false; 813 } 814 815 static bool is_accessed_spte(u64 spte) 816 { 817 u64 accessed_mask = spte_shadow_accessed_mask(spte); 818 819 return accessed_mask ? spte & accessed_mask 820 : !is_access_track_spte(spte); 821 } 822 823 static bool is_dirty_spte(u64 spte) 824 { 825 u64 dirty_mask = spte_shadow_dirty_mask(spte); 826 827 return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK; 828 } 829 830 /* Rules for using mmu_spte_set: 831 * Set the sptep from nonpresent to present. 832 * Note: the sptep being assigned *must* be either not present 833 * or in a state where the hardware will not attempt to update 834 * the spte. 835 */ 836 static void mmu_spte_set(u64 *sptep, u64 new_spte) 837 { 838 WARN_ON(is_shadow_present_pte(*sptep)); 839 __set_spte(sptep, new_spte); 840 } 841 842 /* 843 * Update the SPTE (excluding the PFN), but do not track changes in its 844 * accessed/dirty status. 845 */ 846 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte) 847 { 848 u64 old_spte = *sptep; 849 850 WARN_ON(!is_shadow_present_pte(new_spte)); 851 852 if (!is_shadow_present_pte(old_spte)) { 853 mmu_spte_set(sptep, new_spte); 854 return old_spte; 855 } 856 857 if (!spte_has_volatile_bits(old_spte)) 858 __update_clear_spte_fast(sptep, new_spte); 859 else 860 old_spte = __update_clear_spte_slow(sptep, new_spte); 861 862 WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte)); 863 864 return old_spte; 865 } 866 867 /* Rules for using mmu_spte_update: 868 * Update the state bits, it means the mapped pfn is not changed. 869 * 870 * Whenever we overwrite a writable spte with a read-only one we 871 * should flush remote TLBs. Otherwise rmap_write_protect 872 * will find a read-only spte, even though the writable spte 873 * might be cached on a CPU's TLB, the return value indicates this 874 * case. 875 * 876 * Returns true if the TLB needs to be flushed 877 */ 878 static bool mmu_spte_update(u64 *sptep, u64 new_spte) 879 { 880 bool flush = false; 881 u64 old_spte = mmu_spte_update_no_track(sptep, new_spte); 882 883 if (!is_shadow_present_pte(old_spte)) 884 return false; 885 886 /* 887 * For the spte updated out of mmu-lock is safe, since 888 * we always atomically update it, see the comments in 889 * spte_has_volatile_bits(). 890 */ 891 if (spte_can_locklessly_be_made_writable(old_spte) && 892 !is_writable_pte(new_spte)) 893 flush = true; 894 895 /* 896 * Flush TLB when accessed/dirty states are changed in the page tables, 897 * to guarantee consistency between TLB and page tables. 898 */ 899 900 if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) { 901 flush = true; 902 kvm_set_pfn_accessed(spte_to_pfn(old_spte)); 903 } 904 905 if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) { 906 flush = true; 907 kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 908 } 909 910 return flush; 911 } 912 913 /* 914 * Rules for using mmu_spte_clear_track_bits: 915 * It sets the sptep from present to nonpresent, and track the 916 * state bits, it is used to clear the last level sptep. 917 * Returns non-zero if the PTE was previously valid. 918 */ 919 static int mmu_spte_clear_track_bits(u64 *sptep) 920 { 921 kvm_pfn_t pfn; 922 u64 old_spte = *sptep; 923 924 if (!spte_has_volatile_bits(old_spte)) 925 __update_clear_spte_fast(sptep, 0ull); 926 else 927 old_spte = __update_clear_spte_slow(sptep, 0ull); 928 929 if (!is_shadow_present_pte(old_spte)) 930 return 0; 931 932 pfn = spte_to_pfn(old_spte); 933 934 /* 935 * KVM does not hold the refcount of the page used by 936 * kvm mmu, before reclaiming the page, we should 937 * unmap it from mmu first. 938 */ 939 WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn))); 940 941 if (is_accessed_spte(old_spte)) 942 kvm_set_pfn_accessed(pfn); 943 944 if (is_dirty_spte(old_spte)) 945 kvm_set_pfn_dirty(pfn); 946 947 return 1; 948 } 949 950 /* 951 * Rules for using mmu_spte_clear_no_track: 952 * Directly clear spte without caring the state bits of sptep, 953 * it is used to set the upper level spte. 954 */ 955 static void mmu_spte_clear_no_track(u64 *sptep) 956 { 957 __update_clear_spte_fast(sptep, 0ull); 958 } 959 960 static u64 mmu_spte_get_lockless(u64 *sptep) 961 { 962 return __get_spte_lockless(sptep); 963 } 964 965 static u64 mark_spte_for_access_track(u64 spte) 966 { 967 if (spte_ad_enabled(spte)) 968 return spte & ~shadow_accessed_mask; 969 970 if (is_access_track_spte(spte)) 971 return spte; 972 973 /* 974 * Making an Access Tracking PTE will result in removal of write access 975 * from the PTE. So, verify that we will be able to restore the write 976 * access in the fast page fault path later on. 977 */ 978 WARN_ONCE((spte & PT_WRITABLE_MASK) && 979 !spte_can_locklessly_be_made_writable(spte), 980 "kvm: Writable SPTE is not locklessly dirty-trackable\n"); 981 982 WARN_ONCE(spte & (shadow_acc_track_saved_bits_mask << 983 shadow_acc_track_saved_bits_shift), 984 "kvm: Access Tracking saved bit locations are not zero\n"); 985 986 spte |= (spte & shadow_acc_track_saved_bits_mask) << 987 shadow_acc_track_saved_bits_shift; 988 spte &= ~shadow_acc_track_mask; 989 990 return spte; 991 } 992 993 /* Restore an acc-track PTE back to a regular PTE */ 994 static u64 restore_acc_track_spte(u64 spte) 995 { 996 u64 new_spte = spte; 997 u64 saved_bits = (spte >> shadow_acc_track_saved_bits_shift) 998 & shadow_acc_track_saved_bits_mask; 999 1000 WARN_ON_ONCE(spte_ad_enabled(spte)); 1001 WARN_ON_ONCE(!is_access_track_spte(spte)); 1002 1003 new_spte &= ~shadow_acc_track_mask; 1004 new_spte &= ~(shadow_acc_track_saved_bits_mask << 1005 shadow_acc_track_saved_bits_shift); 1006 new_spte |= saved_bits; 1007 1008 return new_spte; 1009 } 1010 1011 /* Returns the Accessed status of the PTE and resets it at the same time. */ 1012 static bool mmu_spte_age(u64 *sptep) 1013 { 1014 u64 spte = mmu_spte_get_lockless(sptep); 1015 1016 if (!is_accessed_spte(spte)) 1017 return false; 1018 1019 if (spte_ad_enabled(spte)) { 1020 clear_bit((ffs(shadow_accessed_mask) - 1), 1021 (unsigned long *)sptep); 1022 } else { 1023 /* 1024 * Capture the dirty status of the page, so that it doesn't get 1025 * lost when the SPTE is marked for access tracking. 1026 */ 1027 if (is_writable_pte(spte)) 1028 kvm_set_pfn_dirty(spte_to_pfn(spte)); 1029 1030 spte = mark_spte_for_access_track(spte); 1031 mmu_spte_update_no_track(sptep, spte); 1032 } 1033 1034 return true; 1035 } 1036 1037 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu) 1038 { 1039 /* 1040 * Prevent page table teardown by making any free-er wait during 1041 * kvm_flush_remote_tlbs() IPI to all active vcpus. 1042 */ 1043 local_irq_disable(); 1044 1045 /* 1046 * Make sure a following spte read is not reordered ahead of the write 1047 * to vcpu->mode. 1048 */ 1049 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES); 1050 } 1051 1052 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu) 1053 { 1054 /* 1055 * Make sure the write to vcpu->mode is not reordered in front of 1056 * reads to sptes. If it does, kvm_mmu_commit_zap_page() can see us 1057 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table. 1058 */ 1059 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); 1060 local_irq_enable(); 1061 } 1062 1063 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, 1064 struct kmem_cache *base_cache, int min) 1065 { 1066 void *obj; 1067 1068 if (cache->nobjs >= min) 1069 return 0; 1070 while (cache->nobjs < ARRAY_SIZE(cache->objects)) { 1071 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL_ACCOUNT); 1072 if (!obj) 1073 return cache->nobjs >= min ? 0 : -ENOMEM; 1074 cache->objects[cache->nobjs++] = obj; 1075 } 1076 return 0; 1077 } 1078 1079 static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache) 1080 { 1081 return cache->nobjs; 1082 } 1083 1084 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc, 1085 struct kmem_cache *cache) 1086 { 1087 while (mc->nobjs) 1088 kmem_cache_free(cache, mc->objects[--mc->nobjs]); 1089 } 1090 1091 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache, 1092 int min) 1093 { 1094 void *page; 1095 1096 if (cache->nobjs >= min) 1097 return 0; 1098 while (cache->nobjs < ARRAY_SIZE(cache->objects)) { 1099 page = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); 1100 if (!page) 1101 return cache->nobjs >= min ? 0 : -ENOMEM; 1102 cache->objects[cache->nobjs++] = page; 1103 } 1104 return 0; 1105 } 1106 1107 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc) 1108 { 1109 while (mc->nobjs) 1110 free_page((unsigned long)mc->objects[--mc->nobjs]); 1111 } 1112 1113 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) 1114 { 1115 int r; 1116 1117 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache, 1118 pte_list_desc_cache, 8 + PTE_PREFETCH_NUM); 1119 if (r) 1120 goto out; 1121 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8); 1122 if (r) 1123 goto out; 1124 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, 1125 mmu_page_header_cache, 4); 1126 out: 1127 return r; 1128 } 1129 1130 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) 1131 { 1132 mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache, 1133 pte_list_desc_cache); 1134 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache); 1135 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache, 1136 mmu_page_header_cache); 1137 } 1138 1139 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) 1140 { 1141 void *p; 1142 1143 BUG_ON(!mc->nobjs); 1144 p = mc->objects[--mc->nobjs]; 1145 return p; 1146 } 1147 1148 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu) 1149 { 1150 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache); 1151 } 1152 1153 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc) 1154 { 1155 kmem_cache_free(pte_list_desc_cache, pte_list_desc); 1156 } 1157 1158 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index) 1159 { 1160 if (!sp->role.direct) 1161 return sp->gfns[index]; 1162 1163 return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS)); 1164 } 1165 1166 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) 1167 { 1168 if (!sp->role.direct) { 1169 sp->gfns[index] = gfn; 1170 return; 1171 } 1172 1173 if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index))) 1174 pr_err_ratelimited("gfn mismatch under direct page %llx " 1175 "(expected %llx, got %llx)\n", 1176 sp->gfn, 1177 kvm_mmu_page_get_gfn(sp, index), gfn); 1178 } 1179 1180 /* 1181 * Return the pointer to the large page information for a given gfn, 1182 * handling slots that are not large page aligned. 1183 */ 1184 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, 1185 struct kvm_memory_slot *slot, 1186 int level) 1187 { 1188 unsigned long idx; 1189 1190 idx = gfn_to_index(gfn, slot->base_gfn, level); 1191 return &slot->arch.lpage_info[level - 2][idx]; 1192 } 1193 1194 static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot, 1195 gfn_t gfn, int count) 1196 { 1197 struct kvm_lpage_info *linfo; 1198 int i; 1199 1200 for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { 1201 linfo = lpage_info_slot(gfn, slot, i); 1202 linfo->disallow_lpage += count; 1203 WARN_ON(linfo->disallow_lpage < 0); 1204 } 1205 } 1206 1207 void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn) 1208 { 1209 update_gfn_disallow_lpage_count(slot, gfn, 1); 1210 } 1211 1212 void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn) 1213 { 1214 update_gfn_disallow_lpage_count(slot, gfn, -1); 1215 } 1216 1217 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) 1218 { 1219 struct kvm_memslots *slots; 1220 struct kvm_memory_slot *slot; 1221 gfn_t gfn; 1222 1223 kvm->arch.indirect_shadow_pages++; 1224 gfn = sp->gfn; 1225 slots = kvm_memslots_for_spte_role(kvm, sp->role); 1226 slot = __gfn_to_memslot(slots, gfn); 1227 1228 /* the non-leaf shadow pages are keeping readonly. */ 1229 if (sp->role.level > PG_LEVEL_4K) 1230 return kvm_slot_page_track_add_page(kvm, slot, gfn, 1231 KVM_PAGE_TRACK_WRITE); 1232 1233 kvm_mmu_gfn_disallow_lpage(slot, gfn); 1234 } 1235 1236 static void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp) 1237 { 1238 if (sp->lpage_disallowed) 1239 return; 1240 1241 ++kvm->stat.nx_lpage_splits; 1242 list_add_tail(&sp->lpage_disallowed_link, 1243 &kvm->arch.lpage_disallowed_mmu_pages); 1244 sp->lpage_disallowed = true; 1245 } 1246 1247 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) 1248 { 1249 struct kvm_memslots *slots; 1250 struct kvm_memory_slot *slot; 1251 gfn_t gfn; 1252 1253 kvm->arch.indirect_shadow_pages--; 1254 gfn = sp->gfn; 1255 slots = kvm_memslots_for_spte_role(kvm, sp->role); 1256 slot = __gfn_to_memslot(slots, gfn); 1257 if (sp->role.level > PG_LEVEL_4K) 1258 return kvm_slot_page_track_remove_page(kvm, slot, gfn, 1259 KVM_PAGE_TRACK_WRITE); 1260 1261 kvm_mmu_gfn_allow_lpage(slot, gfn); 1262 } 1263 1264 static void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp) 1265 { 1266 --kvm->stat.nx_lpage_splits; 1267 sp->lpage_disallowed = false; 1268 list_del(&sp->lpage_disallowed_link); 1269 } 1270 1271 static struct kvm_memory_slot * 1272 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn, 1273 bool no_dirty_log) 1274 { 1275 struct kvm_memory_slot *slot; 1276 1277 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1278 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1279 return NULL; 1280 if (no_dirty_log && slot->dirty_bitmap) 1281 return NULL; 1282 1283 return slot; 1284 } 1285 1286 /* 1287 * About rmap_head encoding: 1288 * 1289 * If the bit zero of rmap_head->val is clear, then it points to the only spte 1290 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct 1291 * pte_list_desc containing more mappings. 1292 */ 1293 1294 /* 1295 * Returns the number of pointers in the rmap chain, not counting the new one. 1296 */ 1297 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte, 1298 struct kvm_rmap_head *rmap_head) 1299 { 1300 struct pte_list_desc *desc; 1301 int i, count = 0; 1302 1303 if (!rmap_head->val) { 1304 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte); 1305 rmap_head->val = (unsigned long)spte; 1306 } else if (!(rmap_head->val & 1)) { 1307 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte); 1308 desc = mmu_alloc_pte_list_desc(vcpu); 1309 desc->sptes[0] = (u64 *)rmap_head->val; 1310 desc->sptes[1] = spte; 1311 rmap_head->val = (unsigned long)desc | 1; 1312 ++count; 1313 } else { 1314 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte); 1315 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 1316 while (desc->sptes[PTE_LIST_EXT-1] && desc->more) { 1317 desc = desc->more; 1318 count += PTE_LIST_EXT; 1319 } 1320 if (desc->sptes[PTE_LIST_EXT-1]) { 1321 desc->more = mmu_alloc_pte_list_desc(vcpu); 1322 desc = desc->more; 1323 } 1324 for (i = 0; desc->sptes[i]; ++i) 1325 ++count; 1326 desc->sptes[i] = spte; 1327 } 1328 return count; 1329 } 1330 1331 static void 1332 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head, 1333 struct pte_list_desc *desc, int i, 1334 struct pte_list_desc *prev_desc) 1335 { 1336 int j; 1337 1338 for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j) 1339 ; 1340 desc->sptes[i] = desc->sptes[j]; 1341 desc->sptes[j] = NULL; 1342 if (j != 0) 1343 return; 1344 if (!prev_desc && !desc->more) 1345 rmap_head->val = 0; 1346 else 1347 if (prev_desc) 1348 prev_desc->more = desc->more; 1349 else 1350 rmap_head->val = (unsigned long)desc->more | 1; 1351 mmu_free_pte_list_desc(desc); 1352 } 1353 1354 static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head) 1355 { 1356 struct pte_list_desc *desc; 1357 struct pte_list_desc *prev_desc; 1358 int i; 1359 1360 if (!rmap_head->val) { 1361 pr_err("%s: %p 0->BUG\n", __func__, spte); 1362 BUG(); 1363 } else if (!(rmap_head->val & 1)) { 1364 rmap_printk("%s: %p 1->0\n", __func__, spte); 1365 if ((u64 *)rmap_head->val != spte) { 1366 pr_err("%s: %p 1->BUG\n", __func__, spte); 1367 BUG(); 1368 } 1369 rmap_head->val = 0; 1370 } else { 1371 rmap_printk("%s: %p many->many\n", __func__, spte); 1372 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 1373 prev_desc = NULL; 1374 while (desc) { 1375 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) { 1376 if (desc->sptes[i] == spte) { 1377 pte_list_desc_remove_entry(rmap_head, 1378 desc, i, prev_desc); 1379 return; 1380 } 1381 } 1382 prev_desc = desc; 1383 desc = desc->more; 1384 } 1385 pr_err("%s: %p many->many\n", __func__, spte); 1386 BUG(); 1387 } 1388 } 1389 1390 static void pte_list_remove(struct kvm_rmap_head *rmap_head, u64 *sptep) 1391 { 1392 mmu_spte_clear_track_bits(sptep); 1393 __pte_list_remove(sptep, rmap_head); 1394 } 1395 1396 static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level, 1397 struct kvm_memory_slot *slot) 1398 { 1399 unsigned long idx; 1400 1401 idx = gfn_to_index(gfn, slot->base_gfn, level); 1402 return &slot->arch.rmap[level - PG_LEVEL_4K][idx]; 1403 } 1404 1405 static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, 1406 struct kvm_mmu_page *sp) 1407 { 1408 struct kvm_memslots *slots; 1409 struct kvm_memory_slot *slot; 1410 1411 slots = kvm_memslots_for_spte_role(kvm, sp->role); 1412 slot = __gfn_to_memslot(slots, gfn); 1413 return __gfn_to_rmap(gfn, sp->role.level, slot); 1414 } 1415 1416 static bool rmap_can_add(struct kvm_vcpu *vcpu) 1417 { 1418 struct kvm_mmu_memory_cache *cache; 1419 1420 cache = &vcpu->arch.mmu_pte_list_desc_cache; 1421 return mmu_memory_cache_free_objects(cache); 1422 } 1423 1424 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) 1425 { 1426 struct kvm_mmu_page *sp; 1427 struct kvm_rmap_head *rmap_head; 1428 1429 sp = page_header(__pa(spte)); 1430 kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn); 1431 rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp); 1432 return pte_list_add(vcpu, spte, rmap_head); 1433 } 1434 1435 static void rmap_remove(struct kvm *kvm, u64 *spte) 1436 { 1437 struct kvm_mmu_page *sp; 1438 gfn_t gfn; 1439 struct kvm_rmap_head *rmap_head; 1440 1441 sp = page_header(__pa(spte)); 1442 gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt); 1443 rmap_head = gfn_to_rmap(kvm, gfn, sp); 1444 __pte_list_remove(spte, rmap_head); 1445 } 1446 1447 /* 1448 * Used by the following functions to iterate through the sptes linked by a 1449 * rmap. All fields are private and not assumed to be used outside. 1450 */ 1451 struct rmap_iterator { 1452 /* private fields */ 1453 struct pte_list_desc *desc; /* holds the sptep if not NULL */ 1454 int pos; /* index of the sptep */ 1455 }; 1456 1457 /* 1458 * Iteration must be started by this function. This should also be used after 1459 * removing/dropping sptes from the rmap link because in such cases the 1460 * information in the iterator may not be valid. 1461 * 1462 * Returns sptep if found, NULL otherwise. 1463 */ 1464 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, 1465 struct rmap_iterator *iter) 1466 { 1467 u64 *sptep; 1468 1469 if (!rmap_head->val) 1470 return NULL; 1471 1472 if (!(rmap_head->val & 1)) { 1473 iter->desc = NULL; 1474 sptep = (u64 *)rmap_head->val; 1475 goto out; 1476 } 1477 1478 iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul); 1479 iter->pos = 0; 1480 sptep = iter->desc->sptes[iter->pos]; 1481 out: 1482 BUG_ON(!is_shadow_present_pte(*sptep)); 1483 return sptep; 1484 } 1485 1486 /* 1487 * Must be used with a valid iterator: e.g. after rmap_get_first(). 1488 * 1489 * Returns sptep if found, NULL otherwise. 1490 */ 1491 static u64 *rmap_get_next(struct rmap_iterator *iter) 1492 { 1493 u64 *sptep; 1494 1495 if (iter->desc) { 1496 if (iter->pos < PTE_LIST_EXT - 1) { 1497 ++iter->pos; 1498 sptep = iter->desc->sptes[iter->pos]; 1499 if (sptep) 1500 goto out; 1501 } 1502 1503 iter->desc = iter->desc->more; 1504 1505 if (iter->desc) { 1506 iter->pos = 0; 1507 /* desc->sptes[0] cannot be NULL */ 1508 sptep = iter->desc->sptes[iter->pos]; 1509 goto out; 1510 } 1511 } 1512 1513 return NULL; 1514 out: 1515 BUG_ON(!is_shadow_present_pte(*sptep)); 1516 return sptep; 1517 } 1518 1519 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_) \ 1520 for (_spte_ = rmap_get_first(_rmap_head_, _iter_); \ 1521 _spte_; _spte_ = rmap_get_next(_iter_)) 1522 1523 static void drop_spte(struct kvm *kvm, u64 *sptep) 1524 { 1525 if (mmu_spte_clear_track_bits(sptep)) 1526 rmap_remove(kvm, sptep); 1527 } 1528 1529 1530 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep) 1531 { 1532 if (is_large_pte(*sptep)) { 1533 WARN_ON(page_header(__pa(sptep))->role.level == PG_LEVEL_4K); 1534 drop_spte(kvm, sptep); 1535 --kvm->stat.lpages; 1536 return true; 1537 } 1538 1539 return false; 1540 } 1541 1542 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) 1543 { 1544 if (__drop_large_spte(vcpu->kvm, sptep)) { 1545 struct kvm_mmu_page *sp = page_header(__pa(sptep)); 1546 1547 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, 1548 KVM_PAGES_PER_HPAGE(sp->role.level)); 1549 } 1550 } 1551 1552 /* 1553 * Write-protect on the specified @sptep, @pt_protect indicates whether 1554 * spte write-protection is caused by protecting shadow page table. 1555 * 1556 * Note: write protection is difference between dirty logging and spte 1557 * protection: 1558 * - for dirty logging, the spte can be set to writable at anytime if 1559 * its dirty bitmap is properly set. 1560 * - for spte protection, the spte can be writable only after unsync-ing 1561 * shadow page. 1562 * 1563 * Return true if tlb need be flushed. 1564 */ 1565 static bool spte_write_protect(u64 *sptep, bool pt_protect) 1566 { 1567 u64 spte = *sptep; 1568 1569 if (!is_writable_pte(spte) && 1570 !(pt_protect && spte_can_locklessly_be_made_writable(spte))) 1571 return false; 1572 1573 rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep); 1574 1575 if (pt_protect) 1576 spte &= ~SPTE_MMU_WRITEABLE; 1577 spte = spte & ~PT_WRITABLE_MASK; 1578 1579 return mmu_spte_update(sptep, spte); 1580 } 1581 1582 static bool __rmap_write_protect(struct kvm *kvm, 1583 struct kvm_rmap_head *rmap_head, 1584 bool pt_protect) 1585 { 1586 u64 *sptep; 1587 struct rmap_iterator iter; 1588 bool flush = false; 1589 1590 for_each_rmap_spte(rmap_head, &iter, sptep) 1591 flush |= spte_write_protect(sptep, pt_protect); 1592 1593 return flush; 1594 } 1595 1596 static bool spte_clear_dirty(u64 *sptep) 1597 { 1598 u64 spte = *sptep; 1599 1600 rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep); 1601 1602 MMU_WARN_ON(!spte_ad_enabled(spte)); 1603 spte &= ~shadow_dirty_mask; 1604 return mmu_spte_update(sptep, spte); 1605 } 1606 1607 static bool spte_wrprot_for_clear_dirty(u64 *sptep) 1608 { 1609 bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT, 1610 (unsigned long *)sptep); 1611 if (was_writable && !spte_ad_enabled(*sptep)) 1612 kvm_set_pfn_dirty(spte_to_pfn(*sptep)); 1613 1614 return was_writable; 1615 } 1616 1617 /* 1618 * Gets the GFN ready for another round of dirty logging by clearing the 1619 * - D bit on ad-enabled SPTEs, and 1620 * - W bit on ad-disabled SPTEs. 1621 * Returns true iff any D or W bits were cleared. 1622 */ 1623 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) 1624 { 1625 u64 *sptep; 1626 struct rmap_iterator iter; 1627 bool flush = false; 1628 1629 for_each_rmap_spte(rmap_head, &iter, sptep) 1630 if (spte_ad_need_write_protect(*sptep)) 1631 flush |= spte_wrprot_for_clear_dirty(sptep); 1632 else 1633 flush |= spte_clear_dirty(sptep); 1634 1635 return flush; 1636 } 1637 1638 static bool spte_set_dirty(u64 *sptep) 1639 { 1640 u64 spte = *sptep; 1641 1642 rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep); 1643 1644 /* 1645 * Similar to the !kvm_x86_ops.slot_disable_log_dirty case, 1646 * do not bother adding back write access to pages marked 1647 * SPTE_AD_WRPROT_ONLY_MASK. 1648 */ 1649 spte |= shadow_dirty_mask; 1650 1651 return mmu_spte_update(sptep, spte); 1652 } 1653 1654 static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) 1655 { 1656 u64 *sptep; 1657 struct rmap_iterator iter; 1658 bool flush = false; 1659 1660 for_each_rmap_spte(rmap_head, &iter, sptep) 1661 if (spte_ad_enabled(*sptep)) 1662 flush |= spte_set_dirty(sptep); 1663 1664 return flush; 1665 } 1666 1667 /** 1668 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages 1669 * @kvm: kvm instance 1670 * @slot: slot to protect 1671 * @gfn_offset: start of the BITS_PER_LONG pages we care about 1672 * @mask: indicates which pages we should protect 1673 * 1674 * Used when we do not need to care about huge page mappings: e.g. during dirty 1675 * logging we do not have any such mappings. 1676 */ 1677 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, 1678 struct kvm_memory_slot *slot, 1679 gfn_t gfn_offset, unsigned long mask) 1680 { 1681 struct kvm_rmap_head *rmap_head; 1682 1683 while (mask) { 1684 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), 1685 PG_LEVEL_4K, slot); 1686 __rmap_write_protect(kvm, rmap_head, false); 1687 1688 /* clear the first set bit */ 1689 mask &= mask - 1; 1690 } 1691 } 1692 1693 /** 1694 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write 1695 * protect the page if the D-bit isn't supported. 1696 * @kvm: kvm instance 1697 * @slot: slot to clear D-bit 1698 * @gfn_offset: start of the BITS_PER_LONG pages we care about 1699 * @mask: indicates which pages we should clear D-bit 1700 * 1701 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap. 1702 */ 1703 void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1704 struct kvm_memory_slot *slot, 1705 gfn_t gfn_offset, unsigned long mask) 1706 { 1707 struct kvm_rmap_head *rmap_head; 1708 1709 while (mask) { 1710 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), 1711 PG_LEVEL_4K, slot); 1712 __rmap_clear_dirty(kvm, rmap_head); 1713 1714 /* clear the first set bit */ 1715 mask &= mask - 1; 1716 } 1717 } 1718 EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked); 1719 1720 /** 1721 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected 1722 * PT level pages. 1723 * 1724 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to 1725 * enable dirty logging for them. 1726 * 1727 * Used when we do not need to care about huge page mappings: e.g. during dirty 1728 * logging we do not have any such mappings. 1729 */ 1730 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 1731 struct kvm_memory_slot *slot, 1732 gfn_t gfn_offset, unsigned long mask) 1733 { 1734 if (kvm_x86_ops.enable_log_dirty_pt_masked) 1735 kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset, 1736 mask); 1737 else 1738 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); 1739 } 1740 1741 /** 1742 * kvm_arch_write_log_dirty - emulate dirty page logging 1743 * @vcpu: Guest mode vcpu 1744 * 1745 * Emulate arch specific page modification logging for the 1746 * nested hypervisor 1747 */ 1748 int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu, gpa_t l2_gpa) 1749 { 1750 if (kvm_x86_ops.write_log_dirty) 1751 return kvm_x86_ops.write_log_dirty(vcpu, l2_gpa); 1752 1753 return 0; 1754 } 1755 1756 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, 1757 struct kvm_memory_slot *slot, u64 gfn) 1758 { 1759 struct kvm_rmap_head *rmap_head; 1760 int i; 1761 bool write_protected = false; 1762 1763 for (i = PG_LEVEL_4K; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { 1764 rmap_head = __gfn_to_rmap(gfn, i, slot); 1765 write_protected |= __rmap_write_protect(kvm, rmap_head, true); 1766 } 1767 1768 return write_protected; 1769 } 1770 1771 static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) 1772 { 1773 struct kvm_memory_slot *slot; 1774 1775 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1776 return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn); 1777 } 1778 1779 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head) 1780 { 1781 u64 *sptep; 1782 struct rmap_iterator iter; 1783 bool flush = false; 1784 1785 while ((sptep = rmap_get_first(rmap_head, &iter))) { 1786 rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep); 1787 1788 pte_list_remove(rmap_head, sptep); 1789 flush = true; 1790 } 1791 1792 return flush; 1793 } 1794 1795 static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1796 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1797 unsigned long data) 1798 { 1799 return kvm_zap_rmapp(kvm, rmap_head); 1800 } 1801 1802 static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1803 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1804 unsigned long data) 1805 { 1806 u64 *sptep; 1807 struct rmap_iterator iter; 1808 int need_flush = 0; 1809 u64 new_spte; 1810 pte_t *ptep = (pte_t *)data; 1811 kvm_pfn_t new_pfn; 1812 1813 WARN_ON(pte_huge(*ptep)); 1814 new_pfn = pte_pfn(*ptep); 1815 1816 restart: 1817 for_each_rmap_spte(rmap_head, &iter, sptep) { 1818 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n", 1819 sptep, *sptep, gfn, level); 1820 1821 need_flush = 1; 1822 1823 if (pte_write(*ptep)) { 1824 pte_list_remove(rmap_head, sptep); 1825 goto restart; 1826 } else { 1827 new_spte = *sptep & ~PT64_BASE_ADDR_MASK; 1828 new_spte |= (u64)new_pfn << PAGE_SHIFT; 1829 1830 new_spte &= ~PT_WRITABLE_MASK; 1831 new_spte &= ~SPTE_HOST_WRITEABLE; 1832 1833 new_spte = mark_spte_for_access_track(new_spte); 1834 1835 mmu_spte_clear_track_bits(sptep); 1836 mmu_spte_set(sptep, new_spte); 1837 } 1838 } 1839 1840 if (need_flush && kvm_available_flush_tlb_with_range()) { 1841 kvm_flush_remote_tlbs_with_address(kvm, gfn, 1); 1842 return 0; 1843 } 1844 1845 return need_flush; 1846 } 1847 1848 struct slot_rmap_walk_iterator { 1849 /* input fields. */ 1850 struct kvm_memory_slot *slot; 1851 gfn_t start_gfn; 1852 gfn_t end_gfn; 1853 int start_level; 1854 int end_level; 1855 1856 /* output fields. */ 1857 gfn_t gfn; 1858 struct kvm_rmap_head *rmap; 1859 int level; 1860 1861 /* private field. */ 1862 struct kvm_rmap_head *end_rmap; 1863 }; 1864 1865 static void 1866 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level) 1867 { 1868 iterator->level = level; 1869 iterator->gfn = iterator->start_gfn; 1870 iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot); 1871 iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level, 1872 iterator->slot); 1873 } 1874 1875 static void 1876 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator, 1877 struct kvm_memory_slot *slot, int start_level, 1878 int end_level, gfn_t start_gfn, gfn_t end_gfn) 1879 { 1880 iterator->slot = slot; 1881 iterator->start_level = start_level; 1882 iterator->end_level = end_level; 1883 iterator->start_gfn = start_gfn; 1884 iterator->end_gfn = end_gfn; 1885 1886 rmap_walk_init_level(iterator, iterator->start_level); 1887 } 1888 1889 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator) 1890 { 1891 return !!iterator->rmap; 1892 } 1893 1894 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator) 1895 { 1896 if (++iterator->rmap <= iterator->end_rmap) { 1897 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level)); 1898 return; 1899 } 1900 1901 if (++iterator->level > iterator->end_level) { 1902 iterator->rmap = NULL; 1903 return; 1904 } 1905 1906 rmap_walk_init_level(iterator, iterator->level); 1907 } 1908 1909 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_, \ 1910 _start_gfn, _end_gfn, _iter_) \ 1911 for (slot_rmap_walk_init(_iter_, _slot_, _start_level_, \ 1912 _end_level_, _start_gfn, _end_gfn); \ 1913 slot_rmap_walk_okay(_iter_); \ 1914 slot_rmap_walk_next(_iter_)) 1915 1916 static int kvm_handle_hva_range(struct kvm *kvm, 1917 unsigned long start, 1918 unsigned long end, 1919 unsigned long data, 1920 int (*handler)(struct kvm *kvm, 1921 struct kvm_rmap_head *rmap_head, 1922 struct kvm_memory_slot *slot, 1923 gfn_t gfn, 1924 int level, 1925 unsigned long data)) 1926 { 1927 struct kvm_memslots *slots; 1928 struct kvm_memory_slot *memslot; 1929 struct slot_rmap_walk_iterator iterator; 1930 int ret = 0; 1931 int i; 1932 1933 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 1934 slots = __kvm_memslots(kvm, i); 1935 kvm_for_each_memslot(memslot, slots) { 1936 unsigned long hva_start, hva_end; 1937 gfn_t gfn_start, gfn_end; 1938 1939 hva_start = max(start, memslot->userspace_addr); 1940 hva_end = min(end, memslot->userspace_addr + 1941 (memslot->npages << PAGE_SHIFT)); 1942 if (hva_start >= hva_end) 1943 continue; 1944 /* 1945 * {gfn(page) | page intersects with [hva_start, hva_end)} = 1946 * {gfn_start, gfn_start+1, ..., gfn_end-1}. 1947 */ 1948 gfn_start = hva_to_gfn_memslot(hva_start, memslot); 1949 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); 1950 1951 for_each_slot_rmap_range(memslot, PG_LEVEL_4K, 1952 KVM_MAX_HUGEPAGE_LEVEL, 1953 gfn_start, gfn_end - 1, 1954 &iterator) 1955 ret |= handler(kvm, iterator.rmap, memslot, 1956 iterator.gfn, iterator.level, data); 1957 } 1958 } 1959 1960 return ret; 1961 } 1962 1963 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, 1964 unsigned long data, 1965 int (*handler)(struct kvm *kvm, 1966 struct kvm_rmap_head *rmap_head, 1967 struct kvm_memory_slot *slot, 1968 gfn_t gfn, int level, 1969 unsigned long data)) 1970 { 1971 return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler); 1972 } 1973 1974 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end) 1975 { 1976 return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp); 1977 } 1978 1979 int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) 1980 { 1981 return kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp); 1982 } 1983 1984 static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 1985 struct kvm_memory_slot *slot, gfn_t gfn, int level, 1986 unsigned long data) 1987 { 1988 u64 *sptep; 1989 struct rmap_iterator uninitialized_var(iter); 1990 int young = 0; 1991 1992 for_each_rmap_spte(rmap_head, &iter, sptep) 1993 young |= mmu_spte_age(sptep); 1994 1995 trace_kvm_age_page(gfn, level, slot, young); 1996 return young; 1997 } 1998 1999 static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 2000 struct kvm_memory_slot *slot, gfn_t gfn, 2001 int level, unsigned long data) 2002 { 2003 u64 *sptep; 2004 struct rmap_iterator iter; 2005 2006 for_each_rmap_spte(rmap_head, &iter, sptep) 2007 if (is_accessed_spte(*sptep)) 2008 return 1; 2009 return 0; 2010 } 2011 2012 #define RMAP_RECYCLE_THRESHOLD 1000 2013 2014 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) 2015 { 2016 struct kvm_rmap_head *rmap_head; 2017 struct kvm_mmu_page *sp; 2018 2019 sp = page_header(__pa(spte)); 2020 2021 rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp); 2022 2023 kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0); 2024 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, 2025 KVM_PAGES_PER_HPAGE(sp->role.level)); 2026 } 2027 2028 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) 2029 { 2030 return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp); 2031 } 2032 2033 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) 2034 { 2035 return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp); 2036 } 2037 2038 #ifdef MMU_DEBUG 2039 static int is_empty_shadow_page(u64 *spt) 2040 { 2041 u64 *pos; 2042 u64 *end; 2043 2044 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) 2045 if (is_shadow_present_pte(*pos)) { 2046 printk(KERN_ERR "%s: %p %llx\n", __func__, 2047 pos, *pos); 2048 return 0; 2049 } 2050 return 1; 2051 } 2052 #endif 2053 2054 /* 2055 * This value is the sum of all of the kvm instances's 2056 * kvm->arch.n_used_mmu_pages values. We need a global, 2057 * aggregate version in order to make the slab shrinker 2058 * faster 2059 */ 2060 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, unsigned long nr) 2061 { 2062 kvm->arch.n_used_mmu_pages += nr; 2063 percpu_counter_add(&kvm_total_used_mmu_pages, nr); 2064 } 2065 2066 static void kvm_mmu_free_page(struct kvm_mmu_page *sp) 2067 { 2068 MMU_WARN_ON(!is_empty_shadow_page(sp->spt)); 2069 hlist_del(&sp->hash_link); 2070 list_del(&sp->link); 2071 free_page((unsigned long)sp->spt); 2072 if (!sp->role.direct) 2073 free_page((unsigned long)sp->gfns); 2074 kmem_cache_free(mmu_page_header_cache, sp); 2075 } 2076 2077 static unsigned kvm_page_table_hashfn(gfn_t gfn) 2078 { 2079 return hash_64(gfn, KVM_MMU_HASH_SHIFT); 2080 } 2081 2082 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, 2083 struct kvm_mmu_page *sp, u64 *parent_pte) 2084 { 2085 if (!parent_pte) 2086 return; 2087 2088 pte_list_add(vcpu, parent_pte, &sp->parent_ptes); 2089 } 2090 2091 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, 2092 u64 *parent_pte) 2093 { 2094 __pte_list_remove(parent_pte, &sp->parent_ptes); 2095 } 2096 2097 static void drop_parent_pte(struct kvm_mmu_page *sp, 2098 u64 *parent_pte) 2099 { 2100 mmu_page_remove_parent_pte(sp, parent_pte); 2101 mmu_spte_clear_no_track(parent_pte); 2102 } 2103 2104 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct) 2105 { 2106 struct kvm_mmu_page *sp; 2107 2108 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 2109 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache); 2110 if (!direct) 2111 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache); 2112 set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 2113 2114 /* 2115 * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages() 2116 * depends on valid pages being added to the head of the list. See 2117 * comments in kvm_zap_obsolete_pages(). 2118 */ 2119 sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen; 2120 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); 2121 kvm_mod_used_mmu_pages(vcpu->kvm, +1); 2122 return sp; 2123 } 2124 2125 static void mark_unsync(u64 *spte); 2126 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp) 2127 { 2128 u64 *sptep; 2129 struct rmap_iterator iter; 2130 2131 for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) { 2132 mark_unsync(sptep); 2133 } 2134 } 2135 2136 static void mark_unsync(u64 *spte) 2137 { 2138 struct kvm_mmu_page *sp; 2139 unsigned int index; 2140 2141 sp = page_header(__pa(spte)); 2142 index = spte - sp->spt; 2143 if (__test_and_set_bit(index, sp->unsync_child_bitmap)) 2144 return; 2145 if (sp->unsync_children++) 2146 return; 2147 kvm_mmu_mark_parents_unsync(sp); 2148 } 2149 2150 static int nonpaging_sync_page(struct kvm_vcpu *vcpu, 2151 struct kvm_mmu_page *sp) 2152 { 2153 return 0; 2154 } 2155 2156 static void nonpaging_update_pte(struct kvm_vcpu *vcpu, 2157 struct kvm_mmu_page *sp, u64 *spte, 2158 const void *pte) 2159 { 2160 WARN_ON(1); 2161 } 2162 2163 #define KVM_PAGE_ARRAY_NR 16 2164 2165 struct kvm_mmu_pages { 2166 struct mmu_page_and_offset { 2167 struct kvm_mmu_page *sp; 2168 unsigned int idx; 2169 } page[KVM_PAGE_ARRAY_NR]; 2170 unsigned int nr; 2171 }; 2172 2173 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp, 2174 int idx) 2175 { 2176 int i; 2177 2178 if (sp->unsync) 2179 for (i=0; i < pvec->nr; i++) 2180 if (pvec->page[i].sp == sp) 2181 return 0; 2182 2183 pvec->page[pvec->nr].sp = sp; 2184 pvec->page[pvec->nr].idx = idx; 2185 pvec->nr++; 2186 return (pvec->nr == KVM_PAGE_ARRAY_NR); 2187 } 2188 2189 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx) 2190 { 2191 --sp->unsync_children; 2192 WARN_ON((int)sp->unsync_children < 0); 2193 __clear_bit(idx, sp->unsync_child_bitmap); 2194 } 2195 2196 static int __mmu_unsync_walk(struct kvm_mmu_page *sp, 2197 struct kvm_mmu_pages *pvec) 2198 { 2199 int i, ret, nr_unsync_leaf = 0; 2200 2201 for_each_set_bit(i, sp->unsync_child_bitmap, 512) { 2202 struct kvm_mmu_page *child; 2203 u64 ent = sp->spt[i]; 2204 2205 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) { 2206 clear_unsync_child_bit(sp, i); 2207 continue; 2208 } 2209 2210 child = page_header(ent & PT64_BASE_ADDR_MASK); 2211 2212 if (child->unsync_children) { 2213 if (mmu_pages_add(pvec, child, i)) 2214 return -ENOSPC; 2215 2216 ret = __mmu_unsync_walk(child, pvec); 2217 if (!ret) { 2218 clear_unsync_child_bit(sp, i); 2219 continue; 2220 } else if (ret > 0) { 2221 nr_unsync_leaf += ret; 2222 } else 2223 return ret; 2224 } else if (child->unsync) { 2225 nr_unsync_leaf++; 2226 if (mmu_pages_add(pvec, child, i)) 2227 return -ENOSPC; 2228 } else 2229 clear_unsync_child_bit(sp, i); 2230 } 2231 2232 return nr_unsync_leaf; 2233 } 2234 2235 #define INVALID_INDEX (-1) 2236 2237 static int mmu_unsync_walk(struct kvm_mmu_page *sp, 2238 struct kvm_mmu_pages *pvec) 2239 { 2240 pvec->nr = 0; 2241 if (!sp->unsync_children) 2242 return 0; 2243 2244 mmu_pages_add(pvec, sp, INVALID_INDEX); 2245 return __mmu_unsync_walk(sp, pvec); 2246 } 2247 2248 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp) 2249 { 2250 WARN_ON(!sp->unsync); 2251 trace_kvm_mmu_sync_page(sp); 2252 sp->unsync = 0; 2253 --kvm->stat.mmu_unsync; 2254 } 2255 2256 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, 2257 struct list_head *invalid_list); 2258 static void kvm_mmu_commit_zap_page(struct kvm *kvm, 2259 struct list_head *invalid_list); 2260 2261 2262 #define for_each_valid_sp(_kvm, _sp, _gfn) \ 2263 hlist_for_each_entry(_sp, \ 2264 &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \ 2265 if (is_obsolete_sp((_kvm), (_sp))) { \ 2266 } else 2267 2268 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \ 2269 for_each_valid_sp(_kvm, _sp, _gfn) \ 2270 if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else 2271 2272 static inline bool is_ept_sp(struct kvm_mmu_page *sp) 2273 { 2274 return sp->role.cr0_wp && sp->role.smap_andnot_wp; 2275 } 2276 2277 /* @sp->gfn should be write-protected at the call site */ 2278 static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 2279 struct list_head *invalid_list) 2280 { 2281 if ((!is_ept_sp(sp) && sp->role.gpte_is_8_bytes != !!is_pae(vcpu)) || 2282 vcpu->arch.mmu->sync_page(vcpu, sp) == 0) { 2283 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); 2284 return false; 2285 } 2286 2287 return true; 2288 } 2289 2290 static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, 2291 struct list_head *invalid_list, 2292 bool remote_flush) 2293 { 2294 if (!remote_flush && list_empty(invalid_list)) 2295 return false; 2296 2297 if (!list_empty(invalid_list)) 2298 kvm_mmu_commit_zap_page(kvm, invalid_list); 2299 else 2300 kvm_flush_remote_tlbs(kvm); 2301 return true; 2302 } 2303 2304 static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu, 2305 struct list_head *invalid_list, 2306 bool remote_flush, bool local_flush) 2307 { 2308 if (kvm_mmu_remote_flush_or_zap(vcpu->kvm, invalid_list, remote_flush)) 2309 return; 2310 2311 if (local_flush) 2312 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2313 } 2314 2315 #ifdef CONFIG_KVM_MMU_AUDIT 2316 #include "mmu_audit.c" 2317 #else 2318 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { } 2319 static void mmu_audit_disable(void) { } 2320 #endif 2321 2322 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 2323 { 2324 return sp->role.invalid || 2325 unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); 2326 } 2327 2328 static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 2329 struct list_head *invalid_list) 2330 { 2331 kvm_unlink_unsync_page(vcpu->kvm, sp); 2332 return __kvm_sync_page(vcpu, sp, invalid_list); 2333 } 2334 2335 /* @gfn should be write-protected at the call site */ 2336 static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, 2337 struct list_head *invalid_list) 2338 { 2339 struct kvm_mmu_page *s; 2340 bool ret = false; 2341 2342 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) { 2343 if (!s->unsync) 2344 continue; 2345 2346 WARN_ON(s->role.level != PG_LEVEL_4K); 2347 ret |= kvm_sync_page(vcpu, s, invalid_list); 2348 } 2349 2350 return ret; 2351 } 2352 2353 struct mmu_page_path { 2354 struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL]; 2355 unsigned int idx[PT64_ROOT_MAX_LEVEL]; 2356 }; 2357 2358 #define for_each_sp(pvec, sp, parents, i) \ 2359 for (i = mmu_pages_first(&pvec, &parents); \ 2360 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \ 2361 i = mmu_pages_next(&pvec, &parents, i)) 2362 2363 static int mmu_pages_next(struct kvm_mmu_pages *pvec, 2364 struct mmu_page_path *parents, 2365 int i) 2366 { 2367 int n; 2368 2369 for (n = i+1; n < pvec->nr; n++) { 2370 struct kvm_mmu_page *sp = pvec->page[n].sp; 2371 unsigned idx = pvec->page[n].idx; 2372 int level = sp->role.level; 2373 2374 parents->idx[level-1] = idx; 2375 if (level == PG_LEVEL_4K) 2376 break; 2377 2378 parents->parent[level-2] = sp; 2379 } 2380 2381 return n; 2382 } 2383 2384 static int mmu_pages_first(struct kvm_mmu_pages *pvec, 2385 struct mmu_page_path *parents) 2386 { 2387 struct kvm_mmu_page *sp; 2388 int level; 2389 2390 if (pvec->nr == 0) 2391 return 0; 2392 2393 WARN_ON(pvec->page[0].idx != INVALID_INDEX); 2394 2395 sp = pvec->page[0].sp; 2396 level = sp->role.level; 2397 WARN_ON(level == PG_LEVEL_4K); 2398 2399 parents->parent[level-2] = sp; 2400 2401 /* Also set up a sentinel. Further entries in pvec are all 2402 * children of sp, so this element is never overwritten. 2403 */ 2404 parents->parent[level-1] = NULL; 2405 return mmu_pages_next(pvec, parents, 0); 2406 } 2407 2408 static void mmu_pages_clear_parents(struct mmu_page_path *parents) 2409 { 2410 struct kvm_mmu_page *sp; 2411 unsigned int level = 0; 2412 2413 do { 2414 unsigned int idx = parents->idx[level]; 2415 sp = parents->parent[level]; 2416 if (!sp) 2417 return; 2418 2419 WARN_ON(idx == INVALID_INDEX); 2420 clear_unsync_child_bit(sp, idx); 2421 level++; 2422 } while (!sp->unsync_children); 2423 } 2424 2425 static void mmu_sync_children(struct kvm_vcpu *vcpu, 2426 struct kvm_mmu_page *parent) 2427 { 2428 int i; 2429 struct kvm_mmu_page *sp; 2430 struct mmu_page_path parents; 2431 struct kvm_mmu_pages pages; 2432 LIST_HEAD(invalid_list); 2433 bool flush = false; 2434 2435 while (mmu_unsync_walk(parent, &pages)) { 2436 bool protected = false; 2437 2438 for_each_sp(pages, sp, parents, i) 2439 protected |= rmap_write_protect(vcpu, sp->gfn); 2440 2441 if (protected) { 2442 kvm_flush_remote_tlbs(vcpu->kvm); 2443 flush = false; 2444 } 2445 2446 for_each_sp(pages, sp, parents, i) { 2447 flush |= kvm_sync_page(vcpu, sp, &invalid_list); 2448 mmu_pages_clear_parents(&parents); 2449 } 2450 if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) { 2451 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2452 cond_resched_lock(&vcpu->kvm->mmu_lock); 2453 flush = false; 2454 } 2455 } 2456 2457 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2458 } 2459 2460 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp) 2461 { 2462 atomic_set(&sp->write_flooding_count, 0); 2463 } 2464 2465 static void clear_sp_write_flooding_count(u64 *spte) 2466 { 2467 struct kvm_mmu_page *sp = page_header(__pa(spte)); 2468 2469 __clear_sp_write_flooding_count(sp); 2470 } 2471 2472 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, 2473 gfn_t gfn, 2474 gva_t gaddr, 2475 unsigned level, 2476 int direct, 2477 unsigned int access) 2478 { 2479 union kvm_mmu_page_role role; 2480 unsigned quadrant; 2481 struct kvm_mmu_page *sp; 2482 bool need_sync = false; 2483 bool flush = false; 2484 int collisions = 0; 2485 LIST_HEAD(invalid_list); 2486 2487 role = vcpu->arch.mmu->mmu_role.base; 2488 role.level = level; 2489 role.direct = direct; 2490 if (role.direct) 2491 role.gpte_is_8_bytes = true; 2492 role.access = access; 2493 if (!vcpu->arch.mmu->direct_map 2494 && vcpu->arch.mmu->root_level <= PT32_ROOT_LEVEL) { 2495 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); 2496 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; 2497 role.quadrant = quadrant; 2498 } 2499 for_each_valid_sp(vcpu->kvm, sp, gfn) { 2500 if (sp->gfn != gfn) { 2501 collisions++; 2502 continue; 2503 } 2504 2505 if (!need_sync && sp->unsync) 2506 need_sync = true; 2507 2508 if (sp->role.word != role.word) 2509 continue; 2510 2511 if (sp->unsync) { 2512 /* The page is good, but __kvm_sync_page might still end 2513 * up zapping it. If so, break in order to rebuild it. 2514 */ 2515 if (!__kvm_sync_page(vcpu, sp, &invalid_list)) 2516 break; 2517 2518 WARN_ON(!list_empty(&invalid_list)); 2519 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2520 } 2521 2522 if (sp->unsync_children) 2523 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2524 2525 __clear_sp_write_flooding_count(sp); 2526 trace_kvm_mmu_get_page(sp, false); 2527 goto out; 2528 } 2529 2530 ++vcpu->kvm->stat.mmu_cache_miss; 2531 2532 sp = kvm_mmu_alloc_page(vcpu, direct); 2533 2534 sp->gfn = gfn; 2535 sp->role = role; 2536 hlist_add_head(&sp->hash_link, 2537 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]); 2538 if (!direct) { 2539 /* 2540 * we should do write protection before syncing pages 2541 * otherwise the content of the synced shadow page may 2542 * be inconsistent with guest page table. 2543 */ 2544 account_shadowed(vcpu->kvm, sp); 2545 if (level == PG_LEVEL_4K && rmap_write_protect(vcpu, gfn)) 2546 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1); 2547 2548 if (level > PG_LEVEL_4K && need_sync) 2549 flush |= kvm_sync_pages(vcpu, gfn, &invalid_list); 2550 } 2551 clear_page(sp->spt); 2552 trace_kvm_mmu_get_page(sp, true); 2553 2554 kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); 2555 out: 2556 if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions) 2557 vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions; 2558 return sp; 2559 } 2560 2561 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator, 2562 struct kvm_vcpu *vcpu, hpa_t root, 2563 u64 addr) 2564 { 2565 iterator->addr = addr; 2566 iterator->shadow_addr = root; 2567 iterator->level = vcpu->arch.mmu->shadow_root_level; 2568 2569 if (iterator->level == PT64_ROOT_4LEVEL && 2570 vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL && 2571 !vcpu->arch.mmu->direct_map) 2572 --iterator->level; 2573 2574 if (iterator->level == PT32E_ROOT_LEVEL) { 2575 /* 2576 * prev_root is currently only used for 64-bit hosts. So only 2577 * the active root_hpa is valid here. 2578 */ 2579 BUG_ON(root != vcpu->arch.mmu->root_hpa); 2580 2581 iterator->shadow_addr 2582 = vcpu->arch.mmu->pae_root[(addr >> 30) & 3]; 2583 iterator->shadow_addr &= PT64_BASE_ADDR_MASK; 2584 --iterator->level; 2585 if (!iterator->shadow_addr) 2586 iterator->level = 0; 2587 } 2588 } 2589 2590 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator, 2591 struct kvm_vcpu *vcpu, u64 addr) 2592 { 2593 shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa, 2594 addr); 2595 } 2596 2597 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator) 2598 { 2599 if (iterator->level < PG_LEVEL_4K) 2600 return false; 2601 2602 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level); 2603 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index; 2604 return true; 2605 } 2606 2607 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator, 2608 u64 spte) 2609 { 2610 if (is_last_spte(spte, iterator->level)) { 2611 iterator->level = 0; 2612 return; 2613 } 2614 2615 iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK; 2616 --iterator->level; 2617 } 2618 2619 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator) 2620 { 2621 __shadow_walk_next(iterator, *iterator->sptep); 2622 } 2623 2624 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep, 2625 struct kvm_mmu_page *sp) 2626 { 2627 u64 spte; 2628 2629 BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK); 2630 2631 spte = __pa(sp->spt) | shadow_present_mask | PT_WRITABLE_MASK | 2632 shadow_user_mask | shadow_x_mask | shadow_me_mask; 2633 2634 if (sp_ad_disabled(sp)) 2635 spte |= SPTE_AD_DISABLED_MASK; 2636 else 2637 spte |= shadow_accessed_mask; 2638 2639 mmu_spte_set(sptep, spte); 2640 2641 mmu_page_add_parent_pte(vcpu, sp, sptep); 2642 2643 if (sp->unsync_children || sp->unsync) 2644 mark_unsync(sptep); 2645 } 2646 2647 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2648 unsigned direct_access) 2649 { 2650 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) { 2651 struct kvm_mmu_page *child; 2652 2653 /* 2654 * For the direct sp, if the guest pte's dirty bit 2655 * changed form clean to dirty, it will corrupt the 2656 * sp's access: allow writable in the read-only sp, 2657 * so we should update the spte at this point to get 2658 * a new sp with the correct access. 2659 */ 2660 child = page_header(*sptep & PT64_BASE_ADDR_MASK); 2661 if (child->role.access == direct_access) 2662 return; 2663 2664 drop_parent_pte(child, sptep); 2665 kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1); 2666 } 2667 } 2668 2669 static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, 2670 u64 *spte) 2671 { 2672 u64 pte; 2673 struct kvm_mmu_page *child; 2674 2675 pte = *spte; 2676 if (is_shadow_present_pte(pte)) { 2677 if (is_last_spte(pte, sp->role.level)) { 2678 drop_spte(kvm, spte); 2679 if (is_large_pte(pte)) 2680 --kvm->stat.lpages; 2681 } else { 2682 child = page_header(pte & PT64_BASE_ADDR_MASK); 2683 drop_parent_pte(child, spte); 2684 } 2685 return true; 2686 } 2687 2688 if (is_mmio_spte(pte)) 2689 mmu_spte_clear_no_track(spte); 2690 2691 return false; 2692 } 2693 2694 static void kvm_mmu_page_unlink_children(struct kvm *kvm, 2695 struct kvm_mmu_page *sp) 2696 { 2697 unsigned i; 2698 2699 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) 2700 mmu_page_zap_pte(kvm, sp, sp->spt + i); 2701 } 2702 2703 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp) 2704 { 2705 u64 *sptep; 2706 struct rmap_iterator iter; 2707 2708 while ((sptep = rmap_get_first(&sp->parent_ptes, &iter))) 2709 drop_parent_pte(sp, sptep); 2710 } 2711 2712 static int mmu_zap_unsync_children(struct kvm *kvm, 2713 struct kvm_mmu_page *parent, 2714 struct list_head *invalid_list) 2715 { 2716 int i, zapped = 0; 2717 struct mmu_page_path parents; 2718 struct kvm_mmu_pages pages; 2719 2720 if (parent->role.level == PG_LEVEL_4K) 2721 return 0; 2722 2723 while (mmu_unsync_walk(parent, &pages)) { 2724 struct kvm_mmu_page *sp; 2725 2726 for_each_sp(pages, sp, parents, i) { 2727 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 2728 mmu_pages_clear_parents(&parents); 2729 zapped++; 2730 } 2731 } 2732 2733 return zapped; 2734 } 2735 2736 static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, 2737 struct kvm_mmu_page *sp, 2738 struct list_head *invalid_list, 2739 int *nr_zapped) 2740 { 2741 bool list_unstable; 2742 2743 trace_kvm_mmu_prepare_zap_page(sp); 2744 ++kvm->stat.mmu_shadow_zapped; 2745 *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list); 2746 kvm_mmu_page_unlink_children(kvm, sp); 2747 kvm_mmu_unlink_parents(kvm, sp); 2748 2749 /* Zapping children means active_mmu_pages has become unstable. */ 2750 list_unstable = *nr_zapped; 2751 2752 if (!sp->role.invalid && !sp->role.direct) 2753 unaccount_shadowed(kvm, sp); 2754 2755 if (sp->unsync) 2756 kvm_unlink_unsync_page(kvm, sp); 2757 if (!sp->root_count) { 2758 /* Count self */ 2759 (*nr_zapped)++; 2760 list_move(&sp->link, invalid_list); 2761 kvm_mod_used_mmu_pages(kvm, -1); 2762 } else { 2763 list_move(&sp->link, &kvm->arch.active_mmu_pages); 2764 2765 /* 2766 * Obsolete pages cannot be used on any vCPUs, see the comment 2767 * in kvm_mmu_zap_all_fast(). Note, is_obsolete_sp() also 2768 * treats invalid shadow pages as being obsolete. 2769 */ 2770 if (!is_obsolete_sp(kvm, sp)) 2771 kvm_reload_remote_mmus(kvm); 2772 } 2773 2774 if (sp->lpage_disallowed) 2775 unaccount_huge_nx_page(kvm, sp); 2776 2777 sp->role.invalid = 1; 2778 return list_unstable; 2779 } 2780 2781 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, 2782 struct list_head *invalid_list) 2783 { 2784 int nr_zapped; 2785 2786 __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped); 2787 return nr_zapped; 2788 } 2789 2790 static void kvm_mmu_commit_zap_page(struct kvm *kvm, 2791 struct list_head *invalid_list) 2792 { 2793 struct kvm_mmu_page *sp, *nsp; 2794 2795 if (list_empty(invalid_list)) 2796 return; 2797 2798 /* 2799 * We need to make sure everyone sees our modifications to 2800 * the page tables and see changes to vcpu->mode here. The barrier 2801 * in the kvm_flush_remote_tlbs() achieves this. This pairs 2802 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end. 2803 * 2804 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit 2805 * guest mode and/or lockless shadow page table walks. 2806 */ 2807 kvm_flush_remote_tlbs(kvm); 2808 2809 list_for_each_entry_safe(sp, nsp, invalid_list, link) { 2810 WARN_ON(!sp->role.invalid || sp->root_count); 2811 kvm_mmu_free_page(sp); 2812 } 2813 } 2814 2815 static bool prepare_zap_oldest_mmu_page(struct kvm *kvm, 2816 struct list_head *invalid_list) 2817 { 2818 struct kvm_mmu_page *sp; 2819 2820 if (list_empty(&kvm->arch.active_mmu_pages)) 2821 return false; 2822 2823 sp = list_last_entry(&kvm->arch.active_mmu_pages, 2824 struct kvm_mmu_page, link); 2825 return kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 2826 } 2827 2828 static int make_mmu_pages_available(struct kvm_vcpu *vcpu) 2829 { 2830 LIST_HEAD(invalid_list); 2831 2832 if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES)) 2833 return 0; 2834 2835 while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) { 2836 if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list)) 2837 break; 2838 2839 ++vcpu->kvm->stat.mmu_recycled; 2840 } 2841 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list); 2842 2843 if (!kvm_mmu_available_pages(vcpu->kvm)) 2844 return -ENOSPC; 2845 return 0; 2846 } 2847 2848 /* 2849 * Changing the number of mmu pages allocated to the vm 2850 * Note: if goal_nr_mmu_pages is too small, you will get dead lock 2851 */ 2852 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) 2853 { 2854 LIST_HEAD(invalid_list); 2855 2856 spin_lock(&kvm->mmu_lock); 2857 2858 if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) { 2859 /* Need to free some mmu pages to achieve the goal. */ 2860 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) 2861 if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list)) 2862 break; 2863 2864 kvm_mmu_commit_zap_page(kvm, &invalid_list); 2865 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages; 2866 } 2867 2868 kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages; 2869 2870 spin_unlock(&kvm->mmu_lock); 2871 } 2872 2873 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) 2874 { 2875 struct kvm_mmu_page *sp; 2876 LIST_HEAD(invalid_list); 2877 int r; 2878 2879 pgprintk("%s: looking for gfn %llx\n", __func__, gfn); 2880 r = 0; 2881 spin_lock(&kvm->mmu_lock); 2882 for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { 2883 pgprintk("%s: gfn %llx role %x\n", __func__, gfn, 2884 sp->role.word); 2885 r = 1; 2886 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); 2887 } 2888 kvm_mmu_commit_zap_page(kvm, &invalid_list); 2889 spin_unlock(&kvm->mmu_lock); 2890 2891 return r; 2892 } 2893 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page); 2894 2895 static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) 2896 { 2897 trace_kvm_mmu_unsync_page(sp); 2898 ++vcpu->kvm->stat.mmu_unsync; 2899 sp->unsync = 1; 2900 2901 kvm_mmu_mark_parents_unsync(sp); 2902 } 2903 2904 static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, 2905 bool can_unsync) 2906 { 2907 struct kvm_mmu_page *sp; 2908 2909 if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) 2910 return true; 2911 2912 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { 2913 if (!can_unsync) 2914 return true; 2915 2916 if (sp->unsync) 2917 continue; 2918 2919 WARN_ON(sp->role.level != PG_LEVEL_4K); 2920 kvm_unsync_page(vcpu, sp); 2921 } 2922 2923 /* 2924 * We need to ensure that the marking of unsync pages is visible 2925 * before the SPTE is updated to allow writes because 2926 * kvm_mmu_sync_roots() checks the unsync flags without holding 2927 * the MMU lock and so can race with this. If the SPTE was updated 2928 * before the page had been marked as unsync-ed, something like the 2929 * following could happen: 2930 * 2931 * CPU 1 CPU 2 2932 * --------------------------------------------------------------------- 2933 * 1.2 Host updates SPTE 2934 * to be writable 2935 * 2.1 Guest writes a GPTE for GVA X. 2936 * (GPTE being in the guest page table shadowed 2937 * by the SP from CPU 1.) 2938 * This reads SPTE during the page table walk. 2939 * Since SPTE.W is read as 1, there is no 2940 * fault. 2941 * 2942 * 2.2 Guest issues TLB flush. 2943 * That causes a VM Exit. 2944 * 2945 * 2.3 kvm_mmu_sync_pages() reads sp->unsync. 2946 * Since it is false, so it just returns. 2947 * 2948 * 2.4 Guest accesses GVA X. 2949 * Since the mapping in the SP was not updated, 2950 * so the old mapping for GVA X incorrectly 2951 * gets used. 2952 * 1.1 Host marks SP 2953 * as unsync 2954 * (sp->unsync = true) 2955 * 2956 * The write barrier below ensures that 1.1 happens before 1.2 and thus 2957 * the situation in 2.4 does not arise. The implicit barrier in 2.2 2958 * pairs with this write barrier. 2959 */ 2960 smp_wmb(); 2961 2962 return false; 2963 } 2964 2965 static bool kvm_is_mmio_pfn(kvm_pfn_t pfn) 2966 { 2967 if (pfn_valid(pfn)) 2968 return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn)) && 2969 /* 2970 * Some reserved pages, such as those from NVDIMM 2971 * DAX devices, are not for MMIO, and can be mapped 2972 * with cached memory type for better performance. 2973 * However, the above check misconceives those pages 2974 * as MMIO, and results in KVM mapping them with UC 2975 * memory type, which would hurt the performance. 2976 * Therefore, we check the host memory type in addition 2977 * and only treat UC/UC-/WC pages as MMIO. 2978 */ 2979 (!pat_enabled() || pat_pfn_immune_to_uc_mtrr(pfn)); 2980 2981 return !e820__mapped_raw_any(pfn_to_hpa(pfn), 2982 pfn_to_hpa(pfn + 1) - 1, 2983 E820_TYPE_RAM); 2984 } 2985 2986 /* Bits which may be returned by set_spte() */ 2987 #define SET_SPTE_WRITE_PROTECTED_PT BIT(0) 2988 #define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1) 2989 2990 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 2991 unsigned int pte_access, int level, 2992 gfn_t gfn, kvm_pfn_t pfn, bool speculative, 2993 bool can_unsync, bool host_writable) 2994 { 2995 u64 spte = 0; 2996 int ret = 0; 2997 struct kvm_mmu_page *sp; 2998 2999 if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access)) 3000 return 0; 3001 3002 sp = page_header(__pa(sptep)); 3003 if (sp_ad_disabled(sp)) 3004 spte |= SPTE_AD_DISABLED_MASK; 3005 else if (kvm_vcpu_ad_need_write_protect(vcpu)) 3006 spte |= SPTE_AD_WRPROT_ONLY_MASK; 3007 3008 /* 3009 * For the EPT case, shadow_present_mask is 0 if hardware 3010 * supports exec-only page table entries. In that case, 3011 * ACC_USER_MASK and shadow_user_mask are used to represent 3012 * read access. See FNAME(gpte_access) in paging_tmpl.h. 3013 */ 3014 spte |= shadow_present_mask; 3015 if (!speculative) 3016 spte |= spte_shadow_accessed_mask(spte); 3017 3018 if (level > PG_LEVEL_4K && (pte_access & ACC_EXEC_MASK) && 3019 is_nx_huge_page_enabled()) { 3020 pte_access &= ~ACC_EXEC_MASK; 3021 } 3022 3023 if (pte_access & ACC_EXEC_MASK) 3024 spte |= shadow_x_mask; 3025 else 3026 spte |= shadow_nx_mask; 3027 3028 if (pte_access & ACC_USER_MASK) 3029 spte |= shadow_user_mask; 3030 3031 if (level > PG_LEVEL_4K) 3032 spte |= PT_PAGE_SIZE_MASK; 3033 if (tdp_enabled) 3034 spte |= kvm_x86_ops.get_mt_mask(vcpu, gfn, 3035 kvm_is_mmio_pfn(pfn)); 3036 3037 if (host_writable) 3038 spte |= SPTE_HOST_WRITEABLE; 3039 else 3040 pte_access &= ~ACC_WRITE_MASK; 3041 3042 if (!kvm_is_mmio_pfn(pfn)) 3043 spte |= shadow_me_mask; 3044 3045 spte |= (u64)pfn << PAGE_SHIFT; 3046 3047 if (pte_access & ACC_WRITE_MASK) { 3048 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE; 3049 3050 /* 3051 * Optimization: for pte sync, if spte was writable the hash 3052 * lookup is unnecessary (and expensive). Write protection 3053 * is responsibility of mmu_get_page / kvm_sync_page. 3054 * Same reasoning can be applied to dirty page accounting. 3055 */ 3056 if (!can_unsync && is_writable_pte(*sptep)) 3057 goto set_pte; 3058 3059 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) { 3060 pgprintk("%s: found shadow page for %llx, marking ro\n", 3061 __func__, gfn); 3062 ret |= SET_SPTE_WRITE_PROTECTED_PT; 3063 pte_access &= ~ACC_WRITE_MASK; 3064 spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE); 3065 } 3066 } 3067 3068 if (pte_access & ACC_WRITE_MASK) { 3069 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3070 spte |= spte_shadow_dirty_mask(spte); 3071 } 3072 3073 if (speculative) 3074 spte = mark_spte_for_access_track(spte); 3075 3076 set_pte: 3077 if (mmu_spte_update(sptep, spte)) 3078 ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH; 3079 return ret; 3080 } 3081 3082 static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 3083 unsigned int pte_access, int write_fault, int level, 3084 gfn_t gfn, kvm_pfn_t pfn, bool speculative, 3085 bool host_writable) 3086 { 3087 int was_rmapped = 0; 3088 int rmap_count; 3089 int set_spte_ret; 3090 int ret = RET_PF_RETRY; 3091 bool flush = false; 3092 3093 pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__, 3094 *sptep, write_fault, gfn); 3095 3096 if (is_shadow_present_pte(*sptep)) { 3097 /* 3098 * If we overwrite a PTE page pointer with a 2MB PMD, unlink 3099 * the parent of the now unreachable PTE. 3100 */ 3101 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) { 3102 struct kvm_mmu_page *child; 3103 u64 pte = *sptep; 3104 3105 child = page_header(pte & PT64_BASE_ADDR_MASK); 3106 drop_parent_pte(child, sptep); 3107 flush = true; 3108 } else if (pfn != spte_to_pfn(*sptep)) { 3109 pgprintk("hfn old %llx new %llx\n", 3110 spte_to_pfn(*sptep), pfn); 3111 drop_spte(vcpu->kvm, sptep); 3112 flush = true; 3113 } else 3114 was_rmapped = 1; 3115 } 3116 3117 set_spte_ret = set_spte(vcpu, sptep, pte_access, level, gfn, pfn, 3118 speculative, true, host_writable); 3119 if (set_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { 3120 if (write_fault) 3121 ret = RET_PF_EMULATE; 3122 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 3123 } 3124 3125 if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH || flush) 3126 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 3127 KVM_PAGES_PER_HPAGE(level)); 3128 3129 if (unlikely(is_mmio_spte(*sptep))) 3130 ret = RET_PF_EMULATE; 3131 3132 pgprintk("%s: setting spte %llx\n", __func__, *sptep); 3133 trace_kvm_mmu_set_spte(level, gfn, sptep); 3134 if (!was_rmapped && is_large_pte(*sptep)) 3135 ++vcpu->kvm->stat.lpages; 3136 3137 if (is_shadow_present_pte(*sptep)) { 3138 if (!was_rmapped) { 3139 rmap_count = rmap_add(vcpu, sptep, gfn); 3140 if (rmap_count > RMAP_RECYCLE_THRESHOLD) 3141 rmap_recycle(vcpu, sptep, gfn); 3142 } 3143 } 3144 3145 return ret; 3146 } 3147 3148 static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, 3149 bool no_dirty_log) 3150 { 3151 struct kvm_memory_slot *slot; 3152 3153 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); 3154 if (!slot) 3155 return KVM_PFN_ERR_FAULT; 3156 3157 return gfn_to_pfn_memslot_atomic(slot, gfn); 3158 } 3159 3160 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, 3161 struct kvm_mmu_page *sp, 3162 u64 *start, u64 *end) 3163 { 3164 struct page *pages[PTE_PREFETCH_NUM]; 3165 struct kvm_memory_slot *slot; 3166 unsigned int access = sp->role.access; 3167 int i, ret; 3168 gfn_t gfn; 3169 3170 gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt); 3171 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK); 3172 if (!slot) 3173 return -1; 3174 3175 ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start); 3176 if (ret <= 0) 3177 return -1; 3178 3179 for (i = 0; i < ret; i++, gfn++, start++) { 3180 mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn, 3181 page_to_pfn(pages[i]), true, true); 3182 put_page(pages[i]); 3183 } 3184 3185 return 0; 3186 } 3187 3188 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu, 3189 struct kvm_mmu_page *sp, u64 *sptep) 3190 { 3191 u64 *spte, *start = NULL; 3192 int i; 3193 3194 WARN_ON(!sp->role.direct); 3195 3196 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1); 3197 spte = sp->spt + i; 3198 3199 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) { 3200 if (is_shadow_present_pte(*spte) || spte == sptep) { 3201 if (!start) 3202 continue; 3203 if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0) 3204 break; 3205 start = NULL; 3206 } else if (!start) 3207 start = spte; 3208 } 3209 } 3210 3211 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) 3212 { 3213 struct kvm_mmu_page *sp; 3214 3215 sp = page_header(__pa(sptep)); 3216 3217 /* 3218 * Without accessed bits, there's no way to distinguish between 3219 * actually accessed translations and prefetched, so disable pte 3220 * prefetch if accessed bits aren't available. 3221 */ 3222 if (sp_ad_disabled(sp)) 3223 return; 3224 3225 if (sp->role.level > PG_LEVEL_4K) 3226 return; 3227 3228 __direct_pte_prefetch(vcpu, sp, sptep); 3229 } 3230 3231 static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, 3232 kvm_pfn_t pfn, struct kvm_memory_slot *slot) 3233 { 3234 unsigned long hva; 3235 pte_t *pte; 3236 int level; 3237 3238 if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn)) 3239 return PG_LEVEL_4K; 3240 3241 /* 3242 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot() 3243 * is not solely for performance, it's also necessary to avoid the 3244 * "writable" check in __gfn_to_hva_many(), which will always fail on 3245 * read-only memslots due to gfn_to_hva() assuming writes. Earlier 3246 * page fault steps have already verified the guest isn't writing a 3247 * read-only memslot. 3248 */ 3249 hva = __gfn_to_hva_memslot(slot, gfn); 3250 3251 pte = lookup_address_in_mm(vcpu->kvm->mm, hva, &level); 3252 if (unlikely(!pte)) 3253 return PG_LEVEL_4K; 3254 3255 return level; 3256 } 3257 3258 static int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, 3259 int max_level, kvm_pfn_t *pfnp) 3260 { 3261 struct kvm_memory_slot *slot; 3262 struct kvm_lpage_info *linfo; 3263 kvm_pfn_t pfn = *pfnp; 3264 kvm_pfn_t mask; 3265 int level; 3266 3267 if (unlikely(max_level == PG_LEVEL_4K)) 3268 return PG_LEVEL_4K; 3269 3270 if (is_error_noslot_pfn(pfn) || kvm_is_reserved_pfn(pfn)) 3271 return PG_LEVEL_4K; 3272 3273 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, true); 3274 if (!slot) 3275 return PG_LEVEL_4K; 3276 3277 max_level = min(max_level, max_page_level); 3278 for ( ; max_level > PG_LEVEL_4K; max_level--) { 3279 linfo = lpage_info_slot(gfn, slot, max_level); 3280 if (!linfo->disallow_lpage) 3281 break; 3282 } 3283 3284 if (max_level == PG_LEVEL_4K) 3285 return PG_LEVEL_4K; 3286 3287 level = host_pfn_mapping_level(vcpu, gfn, pfn, slot); 3288 if (level == PG_LEVEL_4K) 3289 return level; 3290 3291 level = min(level, max_level); 3292 3293 /* 3294 * mmu_notifier_retry() was successful and mmu_lock is held, so 3295 * the pmd can't be split from under us. 3296 */ 3297 mask = KVM_PAGES_PER_HPAGE(level) - 1; 3298 VM_BUG_ON((gfn & mask) != (pfn & mask)); 3299 *pfnp = pfn & ~mask; 3300 3301 return level; 3302 } 3303 3304 static void disallowed_hugepage_adjust(struct kvm_shadow_walk_iterator it, 3305 gfn_t gfn, kvm_pfn_t *pfnp, int *levelp) 3306 { 3307 int level = *levelp; 3308 u64 spte = *it.sptep; 3309 3310 if (it.level == level && level > PG_LEVEL_4K && 3311 is_nx_huge_page_enabled() && 3312 is_shadow_present_pte(spte) && 3313 !is_large_pte(spte)) { 3314 /* 3315 * A small SPTE exists for this pfn, but FNAME(fetch) 3316 * and __direct_map would like to create a large PTE 3317 * instead: just force them to go down another level, 3318 * patching back for them into pfn the next 9 bits of 3319 * the address. 3320 */ 3321 u64 page_mask = KVM_PAGES_PER_HPAGE(level) - KVM_PAGES_PER_HPAGE(level - 1); 3322 *pfnp |= gfn & page_mask; 3323 (*levelp)--; 3324 } 3325 } 3326 3327 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t gpa, int write, 3328 int map_writable, int max_level, kvm_pfn_t pfn, 3329 bool prefault, bool account_disallowed_nx_lpage) 3330 { 3331 struct kvm_shadow_walk_iterator it; 3332 struct kvm_mmu_page *sp; 3333 int level, ret; 3334 gfn_t gfn = gpa >> PAGE_SHIFT; 3335 gfn_t base_gfn = gfn; 3336 3337 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 3338 return RET_PF_RETRY; 3339 3340 level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn); 3341 3342 trace_kvm_mmu_spte_requested(gpa, level, pfn); 3343 for_each_shadow_entry(vcpu, gpa, it) { 3344 /* 3345 * We cannot overwrite existing page tables with an NX 3346 * large page, as the leaf could be executable. 3347 */ 3348 disallowed_hugepage_adjust(it, gfn, &pfn, &level); 3349 3350 base_gfn = gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); 3351 if (it.level == level) 3352 break; 3353 3354 drop_large_spte(vcpu, it.sptep); 3355 if (!is_shadow_present_pte(*it.sptep)) { 3356 sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr, 3357 it.level - 1, true, ACC_ALL); 3358 3359 link_shadow_page(vcpu, it.sptep, sp); 3360 if (account_disallowed_nx_lpage) 3361 account_huge_nx_page(vcpu->kvm, sp); 3362 } 3363 } 3364 3365 ret = mmu_set_spte(vcpu, it.sptep, ACC_ALL, 3366 write, level, base_gfn, pfn, prefault, 3367 map_writable); 3368 direct_pte_prefetch(vcpu, it.sptep); 3369 ++vcpu->stat.pf_fixed; 3370 return ret; 3371 } 3372 3373 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk) 3374 { 3375 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk); 3376 } 3377 3378 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) 3379 { 3380 /* 3381 * Do not cache the mmio info caused by writing the readonly gfn 3382 * into the spte otherwise read access on readonly gfn also can 3383 * caused mmio page fault and treat it as mmio access. 3384 */ 3385 if (pfn == KVM_PFN_ERR_RO_FAULT) 3386 return RET_PF_EMULATE; 3387 3388 if (pfn == KVM_PFN_ERR_HWPOISON) { 3389 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current); 3390 return RET_PF_RETRY; 3391 } 3392 3393 return -EFAULT; 3394 } 3395 3396 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, 3397 kvm_pfn_t pfn, unsigned int access, 3398 int *ret_val) 3399 { 3400 /* The pfn is invalid, report the error! */ 3401 if (unlikely(is_error_pfn(pfn))) { 3402 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn); 3403 return true; 3404 } 3405 3406 if (unlikely(is_noslot_pfn(pfn))) 3407 vcpu_cache_mmio_info(vcpu, gva, gfn, 3408 access & shadow_mmio_access_mask); 3409 3410 return false; 3411 } 3412 3413 static bool page_fault_can_be_fast(u32 error_code) 3414 { 3415 /* 3416 * Do not fix the mmio spte with invalid generation number which 3417 * need to be updated by slow page fault path. 3418 */ 3419 if (unlikely(error_code & PFERR_RSVD_MASK)) 3420 return false; 3421 3422 /* See if the page fault is due to an NX violation */ 3423 if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)) 3424 == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)))) 3425 return false; 3426 3427 /* 3428 * #PF can be fast if: 3429 * 1. The shadow page table entry is not present, which could mean that 3430 * the fault is potentially caused by access tracking (if enabled). 3431 * 2. The shadow page table entry is present and the fault 3432 * is caused by write-protect, that means we just need change the W 3433 * bit of the spte which can be done out of mmu-lock. 3434 * 3435 * However, if access tracking is disabled we know that a non-present 3436 * page must be a genuine page fault where we have to create a new SPTE. 3437 * So, if access tracking is disabled, we return true only for write 3438 * accesses to a present page. 3439 */ 3440 3441 return shadow_acc_track_mask != 0 || 3442 ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)) 3443 == (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)); 3444 } 3445 3446 /* 3447 * Returns true if the SPTE was fixed successfully. Otherwise, 3448 * someone else modified the SPTE from its original value. 3449 */ 3450 static bool 3451 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 3452 u64 *sptep, u64 old_spte, u64 new_spte) 3453 { 3454 gfn_t gfn; 3455 3456 WARN_ON(!sp->role.direct); 3457 3458 /* 3459 * Theoretically we could also set dirty bit (and flush TLB) here in 3460 * order to eliminate unnecessary PML logging. See comments in 3461 * set_spte. But fast_page_fault is very unlikely to happen with PML 3462 * enabled, so we do not do this. This might result in the same GPA 3463 * to be logged in PML buffer again when the write really happens, and 3464 * eventually to be called by mark_page_dirty twice. But it's also no 3465 * harm. This also avoids the TLB flush needed after setting dirty bit 3466 * so non-PML cases won't be impacted. 3467 * 3468 * Compare with set_spte where instead shadow_dirty_mask is set. 3469 */ 3470 if (cmpxchg64(sptep, old_spte, new_spte) != old_spte) 3471 return false; 3472 3473 if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) { 3474 /* 3475 * The gfn of direct spte is stable since it is 3476 * calculated by sp->gfn. 3477 */ 3478 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); 3479 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3480 } 3481 3482 return true; 3483 } 3484 3485 static bool is_access_allowed(u32 fault_err_code, u64 spte) 3486 { 3487 if (fault_err_code & PFERR_FETCH_MASK) 3488 return is_executable_pte(spte); 3489 3490 if (fault_err_code & PFERR_WRITE_MASK) 3491 return is_writable_pte(spte); 3492 3493 /* Fault was on Read access */ 3494 return spte & PT_PRESENT_MASK; 3495 } 3496 3497 /* 3498 * Return value: 3499 * - true: let the vcpu to access on the same address again. 3500 * - false: let the real page fault path to fix it. 3501 */ 3502 static bool fast_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 3503 u32 error_code) 3504 { 3505 struct kvm_shadow_walk_iterator iterator; 3506 struct kvm_mmu_page *sp; 3507 bool fault_handled = false; 3508 u64 spte = 0ull; 3509 uint retry_count = 0; 3510 3511 if (!page_fault_can_be_fast(error_code)) 3512 return false; 3513 3514 walk_shadow_page_lockless_begin(vcpu); 3515 3516 do { 3517 u64 new_spte; 3518 3519 for_each_shadow_entry_lockless(vcpu, cr2_or_gpa, iterator, spte) 3520 if (!is_shadow_present_pte(spte)) 3521 break; 3522 3523 sp = page_header(__pa(iterator.sptep)); 3524 if (!is_last_spte(spte, sp->role.level)) 3525 break; 3526 3527 /* 3528 * Check whether the memory access that caused the fault would 3529 * still cause it if it were to be performed right now. If not, 3530 * then this is a spurious fault caused by TLB lazily flushed, 3531 * or some other CPU has already fixed the PTE after the 3532 * current CPU took the fault. 3533 * 3534 * Need not check the access of upper level table entries since 3535 * they are always ACC_ALL. 3536 */ 3537 if (is_access_allowed(error_code, spte)) { 3538 fault_handled = true; 3539 break; 3540 } 3541 3542 new_spte = spte; 3543 3544 if (is_access_track_spte(spte)) 3545 new_spte = restore_acc_track_spte(new_spte); 3546 3547 /* 3548 * Currently, to simplify the code, write-protection can 3549 * be removed in the fast path only if the SPTE was 3550 * write-protected for dirty-logging or access tracking. 3551 */ 3552 if ((error_code & PFERR_WRITE_MASK) && 3553 spte_can_locklessly_be_made_writable(spte)) { 3554 new_spte |= PT_WRITABLE_MASK; 3555 3556 /* 3557 * Do not fix write-permission on the large spte. Since 3558 * we only dirty the first page into the dirty-bitmap in 3559 * fast_pf_fix_direct_spte(), other pages are missed 3560 * if its slot has dirty logging enabled. 3561 * 3562 * Instead, we let the slow page fault path create a 3563 * normal spte to fix the access. 3564 * 3565 * See the comments in kvm_arch_commit_memory_region(). 3566 */ 3567 if (sp->role.level > PG_LEVEL_4K) 3568 break; 3569 } 3570 3571 /* Verify that the fault can be handled in the fast path */ 3572 if (new_spte == spte || 3573 !is_access_allowed(error_code, new_spte)) 3574 break; 3575 3576 /* 3577 * Currently, fast page fault only works for direct mapping 3578 * since the gfn is not stable for indirect shadow page. See 3579 * Documentation/virt/kvm/locking.rst to get more detail. 3580 */ 3581 fault_handled = fast_pf_fix_direct_spte(vcpu, sp, 3582 iterator.sptep, spte, 3583 new_spte); 3584 if (fault_handled) 3585 break; 3586 3587 if (++retry_count > 4) { 3588 printk_once(KERN_WARNING 3589 "kvm: Fast #PF retrying more than 4 times.\n"); 3590 break; 3591 } 3592 3593 } while (true); 3594 3595 trace_fast_page_fault(vcpu, cr2_or_gpa, error_code, iterator.sptep, 3596 spte, fault_handled); 3597 walk_shadow_page_lockless_end(vcpu); 3598 3599 return fault_handled; 3600 } 3601 3602 static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa, 3603 struct list_head *invalid_list) 3604 { 3605 struct kvm_mmu_page *sp; 3606 3607 if (!VALID_PAGE(*root_hpa)) 3608 return; 3609 3610 sp = page_header(*root_hpa & PT64_BASE_ADDR_MASK); 3611 --sp->root_count; 3612 if (!sp->root_count && sp->role.invalid) 3613 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); 3614 3615 *root_hpa = INVALID_PAGE; 3616 } 3617 3618 /* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */ 3619 void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 3620 ulong roots_to_free) 3621 { 3622 int i; 3623 LIST_HEAD(invalid_list); 3624 bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT; 3625 3626 BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG); 3627 3628 /* Before acquiring the MMU lock, see if we need to do any real work. */ 3629 if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) { 3630 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 3631 if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) && 3632 VALID_PAGE(mmu->prev_roots[i].hpa)) 3633 break; 3634 3635 if (i == KVM_MMU_NUM_PREV_ROOTS) 3636 return; 3637 } 3638 3639 spin_lock(&vcpu->kvm->mmu_lock); 3640 3641 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 3642 if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) 3643 mmu_free_root_page(vcpu->kvm, &mmu->prev_roots[i].hpa, 3644 &invalid_list); 3645 3646 if (free_active_root) { 3647 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && 3648 (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) { 3649 mmu_free_root_page(vcpu->kvm, &mmu->root_hpa, 3650 &invalid_list); 3651 } else { 3652 for (i = 0; i < 4; ++i) 3653 if (mmu->pae_root[i] != 0) 3654 mmu_free_root_page(vcpu->kvm, 3655 &mmu->pae_root[i], 3656 &invalid_list); 3657 mmu->root_hpa = INVALID_PAGE; 3658 } 3659 mmu->root_pgd = 0; 3660 } 3661 3662 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list); 3663 spin_unlock(&vcpu->kvm->mmu_lock); 3664 } 3665 EXPORT_SYMBOL_GPL(kvm_mmu_free_roots); 3666 3667 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn) 3668 { 3669 int ret = 0; 3670 3671 if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) { 3672 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3673 ret = 1; 3674 } 3675 3676 return ret; 3677 } 3678 3679 static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, 3680 u8 level, bool direct) 3681 { 3682 struct kvm_mmu_page *sp; 3683 3684 spin_lock(&vcpu->kvm->mmu_lock); 3685 3686 if (make_mmu_pages_available(vcpu)) { 3687 spin_unlock(&vcpu->kvm->mmu_lock); 3688 return INVALID_PAGE; 3689 } 3690 sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL); 3691 ++sp->root_count; 3692 3693 spin_unlock(&vcpu->kvm->mmu_lock); 3694 return __pa(sp->spt); 3695 } 3696 3697 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) 3698 { 3699 u8 shadow_root_level = vcpu->arch.mmu->shadow_root_level; 3700 hpa_t root; 3701 unsigned i; 3702 3703 if (shadow_root_level >= PT64_ROOT_4LEVEL) { 3704 root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true); 3705 if (!VALID_PAGE(root)) 3706 return -ENOSPC; 3707 vcpu->arch.mmu->root_hpa = root; 3708 } else if (shadow_root_level == PT32E_ROOT_LEVEL) { 3709 for (i = 0; i < 4; ++i) { 3710 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); 3711 3712 root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), 3713 i << 30, PT32_ROOT_LEVEL, true); 3714 if (!VALID_PAGE(root)) 3715 return -ENOSPC; 3716 vcpu->arch.mmu->pae_root[i] = root | PT_PRESENT_MASK; 3717 } 3718 vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); 3719 } else 3720 BUG(); 3721 3722 /* root_pgd is ignored for direct MMUs. */ 3723 vcpu->arch.mmu->root_pgd = 0; 3724 3725 return 0; 3726 } 3727 3728 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) 3729 { 3730 u64 pdptr, pm_mask; 3731 gfn_t root_gfn, root_pgd; 3732 hpa_t root; 3733 int i; 3734 3735 root_pgd = vcpu->arch.mmu->get_guest_pgd(vcpu); 3736 root_gfn = root_pgd >> PAGE_SHIFT; 3737 3738 if (mmu_check_root(vcpu, root_gfn)) 3739 return 1; 3740 3741 /* 3742 * Do we shadow a long mode page table? If so we need to 3743 * write-protect the guests page table root. 3744 */ 3745 if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { 3746 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->root_hpa)); 3747 3748 root = mmu_alloc_root(vcpu, root_gfn, 0, 3749 vcpu->arch.mmu->shadow_root_level, false); 3750 if (!VALID_PAGE(root)) 3751 return -ENOSPC; 3752 vcpu->arch.mmu->root_hpa = root; 3753 goto set_root_pgd; 3754 } 3755 3756 /* 3757 * We shadow a 32 bit page table. This may be a legacy 2-level 3758 * or a PAE 3-level page table. In either case we need to be aware that 3759 * the shadow page table may be a PAE or a long mode page table. 3760 */ 3761 pm_mask = PT_PRESENT_MASK; 3762 if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) 3763 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 3764 3765 for (i = 0; i < 4; ++i) { 3766 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); 3767 if (vcpu->arch.mmu->root_level == PT32E_ROOT_LEVEL) { 3768 pdptr = vcpu->arch.mmu->get_pdptr(vcpu, i); 3769 if (!(pdptr & PT_PRESENT_MASK)) { 3770 vcpu->arch.mmu->pae_root[i] = 0; 3771 continue; 3772 } 3773 root_gfn = pdptr >> PAGE_SHIFT; 3774 if (mmu_check_root(vcpu, root_gfn)) 3775 return 1; 3776 } 3777 3778 root = mmu_alloc_root(vcpu, root_gfn, i << 30, 3779 PT32_ROOT_LEVEL, false); 3780 if (!VALID_PAGE(root)) 3781 return -ENOSPC; 3782 vcpu->arch.mmu->pae_root[i] = root | pm_mask; 3783 } 3784 vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); 3785 3786 /* 3787 * If we shadow a 32 bit page table with a long mode page 3788 * table we enter this path. 3789 */ 3790 if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { 3791 if (vcpu->arch.mmu->lm_root == NULL) { 3792 /* 3793 * The additional page necessary for this is only 3794 * allocated on demand. 3795 */ 3796 3797 u64 *lm_root; 3798 3799 lm_root = (void*)get_zeroed_page(GFP_KERNEL_ACCOUNT); 3800 if (lm_root == NULL) 3801 return 1; 3802 3803 lm_root[0] = __pa(vcpu->arch.mmu->pae_root) | pm_mask; 3804 3805 vcpu->arch.mmu->lm_root = lm_root; 3806 } 3807 3808 vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); 3809 } 3810 3811 set_root_pgd: 3812 vcpu->arch.mmu->root_pgd = root_pgd; 3813 3814 return 0; 3815 } 3816 3817 static int mmu_alloc_roots(struct kvm_vcpu *vcpu) 3818 { 3819 if (vcpu->arch.mmu->direct_map) 3820 return mmu_alloc_direct_roots(vcpu); 3821 else 3822 return mmu_alloc_shadow_roots(vcpu); 3823 } 3824 3825 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) 3826 { 3827 int i; 3828 struct kvm_mmu_page *sp; 3829 3830 if (vcpu->arch.mmu->direct_map) 3831 return; 3832 3833 if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) 3834 return; 3835 3836 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); 3837 3838 if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { 3839 hpa_t root = vcpu->arch.mmu->root_hpa; 3840 sp = page_header(root); 3841 3842 /* 3843 * Even if another CPU was marking the SP as unsync-ed 3844 * simultaneously, any guest page table changes are not 3845 * guaranteed to be visible anyway until this VCPU issues a TLB 3846 * flush strictly after those changes are made. We only need to 3847 * ensure that the other CPU sets these flags before any actual 3848 * changes to the page tables are made. The comments in 3849 * mmu_need_write_protect() describe what could go wrong if this 3850 * requirement isn't satisfied. 3851 */ 3852 if (!smp_load_acquire(&sp->unsync) && 3853 !smp_load_acquire(&sp->unsync_children)) 3854 return; 3855 3856 spin_lock(&vcpu->kvm->mmu_lock); 3857 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); 3858 3859 mmu_sync_children(vcpu, sp); 3860 3861 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); 3862 spin_unlock(&vcpu->kvm->mmu_lock); 3863 return; 3864 } 3865 3866 spin_lock(&vcpu->kvm->mmu_lock); 3867 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); 3868 3869 for (i = 0; i < 4; ++i) { 3870 hpa_t root = vcpu->arch.mmu->pae_root[i]; 3871 3872 if (root && VALID_PAGE(root)) { 3873 root &= PT64_BASE_ADDR_MASK; 3874 sp = page_header(root); 3875 mmu_sync_children(vcpu, sp); 3876 } 3877 } 3878 3879 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); 3880 spin_unlock(&vcpu->kvm->mmu_lock); 3881 } 3882 EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots); 3883 3884 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr, 3885 u32 access, struct x86_exception *exception) 3886 { 3887 if (exception) 3888 exception->error_code = 0; 3889 return vaddr; 3890 } 3891 3892 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gpa_t vaddr, 3893 u32 access, 3894 struct x86_exception *exception) 3895 { 3896 if (exception) 3897 exception->error_code = 0; 3898 return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception); 3899 } 3900 3901 static bool 3902 __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level) 3903 { 3904 int bit7 = (pte >> 7) & 1; 3905 3906 return pte & rsvd_check->rsvd_bits_mask[bit7][level-1]; 3907 } 3908 3909 static bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check, u64 pte) 3910 { 3911 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f); 3912 } 3913 3914 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct) 3915 { 3916 /* 3917 * A nested guest cannot use the MMIO cache if it is using nested 3918 * page tables, because cr2 is a nGPA while the cache stores GPAs. 3919 */ 3920 if (mmu_is_nested(vcpu)) 3921 return false; 3922 3923 if (direct) 3924 return vcpu_match_mmio_gpa(vcpu, addr); 3925 3926 return vcpu_match_mmio_gva(vcpu, addr); 3927 } 3928 3929 /* return true if reserved bit is detected on spte. */ 3930 static bool 3931 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) 3932 { 3933 struct kvm_shadow_walk_iterator iterator; 3934 u64 sptes[PT64_ROOT_MAX_LEVEL], spte = 0ull; 3935 struct rsvd_bits_validate *rsvd_check; 3936 int root, leaf; 3937 bool reserved = false; 3938 3939 rsvd_check = &vcpu->arch.mmu->shadow_zero_check; 3940 3941 walk_shadow_page_lockless_begin(vcpu); 3942 3943 for (shadow_walk_init(&iterator, vcpu, addr), 3944 leaf = root = iterator.level; 3945 shadow_walk_okay(&iterator); 3946 __shadow_walk_next(&iterator, spte)) { 3947 spte = mmu_spte_get_lockless(iterator.sptep); 3948 3949 sptes[leaf - 1] = spte; 3950 leaf--; 3951 3952 if (!is_shadow_present_pte(spte)) 3953 break; 3954 3955 /* 3956 * Use a bitwise-OR instead of a logical-OR to aggregate the 3957 * reserved bit and EPT's invalid memtype/XWR checks to avoid 3958 * adding a Jcc in the loop. 3959 */ 3960 reserved |= __is_bad_mt_xwr(rsvd_check, spte) | 3961 __is_rsvd_bits_set(rsvd_check, spte, iterator.level); 3962 } 3963 3964 walk_shadow_page_lockless_end(vcpu); 3965 3966 if (reserved) { 3967 pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n", 3968 __func__, addr); 3969 while (root > leaf) { 3970 pr_err("------ spte 0x%llx level %d.\n", 3971 sptes[root - 1], root); 3972 root--; 3973 } 3974 } 3975 3976 *sptep = spte; 3977 return reserved; 3978 } 3979 3980 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) 3981 { 3982 u64 spte; 3983 bool reserved; 3984 3985 if (mmio_info_in_cache(vcpu, addr, direct)) 3986 return RET_PF_EMULATE; 3987 3988 reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte); 3989 if (WARN_ON(reserved)) 3990 return -EINVAL; 3991 3992 if (is_mmio_spte(spte)) { 3993 gfn_t gfn = get_mmio_spte_gfn(spte); 3994 unsigned int access = get_mmio_spte_access(spte); 3995 3996 if (!check_mmio_spte(vcpu, spte)) 3997 return RET_PF_INVALID; 3998 3999 if (direct) 4000 addr = 0; 4001 4002 trace_handle_mmio_page_fault(addr, gfn, access); 4003 vcpu_cache_mmio_info(vcpu, addr, gfn, access); 4004 return RET_PF_EMULATE; 4005 } 4006 4007 /* 4008 * If the page table is zapped by other cpus, let CPU fault again on 4009 * the address. 4010 */ 4011 return RET_PF_RETRY; 4012 } 4013 4014 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, 4015 u32 error_code, gfn_t gfn) 4016 { 4017 if (unlikely(error_code & PFERR_RSVD_MASK)) 4018 return false; 4019 4020 if (!(error_code & PFERR_PRESENT_MASK) || 4021 !(error_code & PFERR_WRITE_MASK)) 4022 return false; 4023 4024 /* 4025 * guest is writing the page which is write tracked which can 4026 * not be fixed by page fault handler. 4027 */ 4028 if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) 4029 return true; 4030 4031 return false; 4032 } 4033 4034 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr) 4035 { 4036 struct kvm_shadow_walk_iterator iterator; 4037 u64 spte; 4038 4039 walk_shadow_page_lockless_begin(vcpu); 4040 for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) { 4041 clear_sp_write_flooding_count(iterator.sptep); 4042 if (!is_shadow_present_pte(spte)) 4043 break; 4044 } 4045 walk_shadow_page_lockless_end(vcpu); 4046 } 4047 4048 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 4049 gfn_t gfn) 4050 { 4051 struct kvm_arch_async_pf arch; 4052 4053 arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; 4054 arch.gfn = gfn; 4055 arch.direct_map = vcpu->arch.mmu->direct_map; 4056 arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); 4057 4058 return kvm_setup_async_pf(vcpu, cr2_or_gpa, 4059 kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); 4060 } 4061 4062 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, 4063 gpa_t cr2_or_gpa, kvm_pfn_t *pfn, bool write, 4064 bool *writable) 4065 { 4066 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 4067 bool async; 4068 4069 /* Don't expose private memslots to L2. */ 4070 if (is_guest_mode(vcpu) && !kvm_is_visible_memslot(slot)) { 4071 *pfn = KVM_PFN_NOSLOT; 4072 *writable = false; 4073 return false; 4074 } 4075 4076 async = false; 4077 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable); 4078 if (!async) 4079 return false; /* *pfn has correct page already */ 4080 4081 if (!prefault && kvm_can_do_async_pf(vcpu)) { 4082 trace_kvm_try_async_get_page(cr2_or_gpa, gfn); 4083 if (kvm_find_async_pf_gfn(vcpu, gfn)) { 4084 trace_kvm_async_pf_doublefault(cr2_or_gpa, gfn); 4085 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 4086 return true; 4087 } else if (kvm_arch_setup_async_pf(vcpu, cr2_or_gpa, gfn)) 4088 return true; 4089 } 4090 4091 *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable); 4092 return false; 4093 } 4094 4095 static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 4096 bool prefault, int max_level, bool is_tdp) 4097 { 4098 bool write = error_code & PFERR_WRITE_MASK; 4099 bool exec = error_code & PFERR_FETCH_MASK; 4100 bool lpage_disallowed = exec && is_nx_huge_page_enabled(); 4101 bool map_writable; 4102 4103 gfn_t gfn = gpa >> PAGE_SHIFT; 4104 unsigned long mmu_seq; 4105 kvm_pfn_t pfn; 4106 int r; 4107 4108 if (page_fault_handle_page_track(vcpu, error_code, gfn)) 4109 return RET_PF_EMULATE; 4110 4111 r = mmu_topup_memory_caches(vcpu); 4112 if (r) 4113 return r; 4114 4115 if (lpage_disallowed) 4116 max_level = PG_LEVEL_4K; 4117 4118 if (fast_page_fault(vcpu, gpa, error_code)) 4119 return RET_PF_RETRY; 4120 4121 mmu_seq = vcpu->kvm->mmu_notifier_seq; 4122 smp_rmb(); 4123 4124 if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable)) 4125 return RET_PF_RETRY; 4126 4127 if (handle_abnormal_pfn(vcpu, is_tdp ? 0 : gpa, gfn, pfn, ACC_ALL, &r)) 4128 return r; 4129 4130 r = RET_PF_RETRY; 4131 spin_lock(&vcpu->kvm->mmu_lock); 4132 if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) 4133 goto out_unlock; 4134 if (make_mmu_pages_available(vcpu) < 0) 4135 goto out_unlock; 4136 r = __direct_map(vcpu, gpa, write, map_writable, max_level, pfn, 4137 prefault, is_tdp && lpage_disallowed); 4138 4139 out_unlock: 4140 spin_unlock(&vcpu->kvm->mmu_lock); 4141 kvm_release_pfn_clean(pfn); 4142 return r; 4143 } 4144 4145 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, 4146 u32 error_code, bool prefault) 4147 { 4148 pgprintk("%s: gva %lx error %x\n", __func__, gpa, error_code); 4149 4150 /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */ 4151 return direct_page_fault(vcpu, gpa & PAGE_MASK, error_code, prefault, 4152 PG_LEVEL_2M, false); 4153 } 4154 4155 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, 4156 u64 fault_address, char *insn, int insn_len) 4157 { 4158 int r = 1; 4159 4160 #ifndef CONFIG_X86_64 4161 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ 4162 if (WARN_ON_ONCE(fault_address >> 32)) 4163 return -EFAULT; 4164 #endif 4165 4166 vcpu->arch.l1tf_flush_l1d = true; 4167 switch (vcpu->arch.apf.host_apf_flags) { 4168 default: 4169 trace_kvm_page_fault(fault_address, error_code); 4170 4171 if (kvm_event_needs_reinjection(vcpu)) 4172 kvm_mmu_unprotect_page_virt(vcpu, fault_address); 4173 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn, 4174 insn_len); 4175 break; 4176 case KVM_PV_REASON_PAGE_NOT_PRESENT: 4177 vcpu->arch.apf.host_apf_flags = 0; 4178 local_irq_disable(); 4179 kvm_async_pf_task_wait_schedule(fault_address); 4180 local_irq_enable(); 4181 break; 4182 case KVM_PV_REASON_PAGE_READY: 4183 vcpu->arch.apf.host_apf_flags = 0; 4184 local_irq_disable(); 4185 kvm_async_pf_task_wake(fault_address); 4186 local_irq_enable(); 4187 break; 4188 } 4189 return r; 4190 } 4191 EXPORT_SYMBOL_GPL(kvm_handle_page_fault); 4192 4193 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 4194 bool prefault) 4195 { 4196 int max_level; 4197 4198 for (max_level = KVM_MAX_HUGEPAGE_LEVEL; 4199 max_level > PG_LEVEL_4K; 4200 max_level--) { 4201 int page_num = KVM_PAGES_PER_HPAGE(max_level); 4202 gfn_t base = (gpa >> PAGE_SHIFT) & ~(page_num - 1); 4203 4204 if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num)) 4205 break; 4206 } 4207 4208 return direct_page_fault(vcpu, gpa, error_code, prefault, 4209 max_level, true); 4210 } 4211 4212 static void nonpaging_init_context(struct kvm_vcpu *vcpu, 4213 struct kvm_mmu *context) 4214 { 4215 context->page_fault = nonpaging_page_fault; 4216 context->gva_to_gpa = nonpaging_gva_to_gpa; 4217 context->sync_page = nonpaging_sync_page; 4218 context->invlpg = NULL; 4219 context->update_pte = nonpaging_update_pte; 4220 context->root_level = 0; 4221 context->shadow_root_level = PT32E_ROOT_LEVEL; 4222 context->direct_map = true; 4223 context->nx = false; 4224 } 4225 4226 static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd, 4227 union kvm_mmu_page_role role) 4228 { 4229 return (role.direct || pgd == root->pgd) && 4230 VALID_PAGE(root->hpa) && page_header(root->hpa) && 4231 role.word == page_header(root->hpa)->role.word; 4232 } 4233 4234 /* 4235 * Find out if a previously cached root matching the new pgd/role is available. 4236 * The current root is also inserted into the cache. 4237 * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is 4238 * returned. 4239 * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and 4240 * false is returned. This root should now be freed by the caller. 4241 */ 4242 static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_pgd, 4243 union kvm_mmu_page_role new_role) 4244 { 4245 uint i; 4246 struct kvm_mmu_root_info root; 4247 struct kvm_mmu *mmu = vcpu->arch.mmu; 4248 4249 root.pgd = mmu->root_pgd; 4250 root.hpa = mmu->root_hpa; 4251 4252 if (is_root_usable(&root, new_pgd, new_role)) 4253 return true; 4254 4255 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 4256 swap(root, mmu->prev_roots[i]); 4257 4258 if (is_root_usable(&root, new_pgd, new_role)) 4259 break; 4260 } 4261 4262 mmu->root_hpa = root.hpa; 4263 mmu->root_pgd = root.pgd; 4264 4265 return i < KVM_MMU_NUM_PREV_ROOTS; 4266 } 4267 4268 static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd, 4269 union kvm_mmu_page_role new_role) 4270 { 4271 struct kvm_mmu *mmu = vcpu->arch.mmu; 4272 4273 /* 4274 * For now, limit the fast switch to 64-bit hosts+VMs in order to avoid 4275 * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs 4276 * later if necessary. 4277 */ 4278 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && 4279 mmu->root_level >= PT64_ROOT_4LEVEL) 4280 return !mmu_check_root(vcpu, new_pgd >> PAGE_SHIFT) && 4281 cached_root_available(vcpu, new_pgd, new_role); 4282 4283 return false; 4284 } 4285 4286 static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, 4287 union kvm_mmu_page_role new_role, 4288 bool skip_tlb_flush, bool skip_mmu_sync) 4289 { 4290 if (!fast_pgd_switch(vcpu, new_pgd, new_role)) { 4291 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); 4292 return; 4293 } 4294 4295 /* 4296 * It's possible that the cached previous root page is obsolete because 4297 * of a change in the MMU generation number. However, changing the 4298 * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will 4299 * free the root set here and allocate a new one. 4300 */ 4301 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); 4302 4303 if (!skip_mmu_sync || force_flush_and_sync_on_reuse) 4304 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 4305 if (!skip_tlb_flush || force_flush_and_sync_on_reuse) 4306 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 4307 4308 /* 4309 * The last MMIO access's GVA and GPA are cached in the VCPU. When 4310 * switching to a new CR3, that GVA->GPA mapping may no longer be 4311 * valid. So clear any cached MMIO info even when we don't need to sync 4312 * the shadow page tables. 4313 */ 4314 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); 4315 4316 __clear_sp_write_flooding_count(page_header(vcpu->arch.mmu->root_hpa)); 4317 } 4318 4319 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, 4320 bool skip_mmu_sync) 4321 { 4322 __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu), 4323 skip_tlb_flush, skip_mmu_sync); 4324 } 4325 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd); 4326 4327 static unsigned long get_cr3(struct kvm_vcpu *vcpu) 4328 { 4329 return kvm_read_cr3(vcpu); 4330 } 4331 4332 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, 4333 unsigned int access, int *nr_present) 4334 { 4335 if (unlikely(is_mmio_spte(*sptep))) { 4336 if (gfn != get_mmio_spte_gfn(*sptep)) { 4337 mmu_spte_clear_no_track(sptep); 4338 return true; 4339 } 4340 4341 (*nr_present)++; 4342 mark_mmio_spte(vcpu, sptep, gfn, access); 4343 return true; 4344 } 4345 4346 return false; 4347 } 4348 4349 static inline bool is_last_gpte(struct kvm_mmu *mmu, 4350 unsigned level, unsigned gpte) 4351 { 4352 /* 4353 * The RHS has bit 7 set iff level < mmu->last_nonleaf_level. 4354 * If it is clear, there are no large pages at this level, so clear 4355 * PT_PAGE_SIZE_MASK in gpte if that is the case. 4356 */ 4357 gpte &= level - mmu->last_nonleaf_level; 4358 4359 /* 4360 * PG_LEVEL_4K always terminates. The RHS has bit 7 set 4361 * iff level <= PG_LEVEL_4K, which for our purpose means 4362 * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then. 4363 */ 4364 gpte |= level - PG_LEVEL_4K - 1; 4365 4366 return gpte & PT_PAGE_SIZE_MASK; 4367 } 4368 4369 #define PTTYPE_EPT 18 /* arbitrary */ 4370 #define PTTYPE PTTYPE_EPT 4371 #include "paging_tmpl.h" 4372 #undef PTTYPE 4373 4374 #define PTTYPE 64 4375 #include "paging_tmpl.h" 4376 #undef PTTYPE 4377 4378 #define PTTYPE 32 4379 #include "paging_tmpl.h" 4380 #undef PTTYPE 4381 4382 static void 4383 __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, 4384 struct rsvd_bits_validate *rsvd_check, 4385 int maxphyaddr, int level, bool nx, bool gbpages, 4386 bool pse, bool amd) 4387 { 4388 u64 exb_bit_rsvd = 0; 4389 u64 gbpages_bit_rsvd = 0; 4390 u64 nonleaf_bit8_rsvd = 0; 4391 4392 rsvd_check->bad_mt_xwr = 0; 4393 4394 if (!nx) 4395 exb_bit_rsvd = rsvd_bits(63, 63); 4396 if (!gbpages) 4397 gbpages_bit_rsvd = rsvd_bits(7, 7); 4398 4399 /* 4400 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for 4401 * leaf entries) on AMD CPUs only. 4402 */ 4403 if (amd) 4404 nonleaf_bit8_rsvd = rsvd_bits(8, 8); 4405 4406 switch (level) { 4407 case PT32_ROOT_LEVEL: 4408 /* no rsvd bits for 2 level 4K page table entries */ 4409 rsvd_check->rsvd_bits_mask[0][1] = 0; 4410 rsvd_check->rsvd_bits_mask[0][0] = 0; 4411 rsvd_check->rsvd_bits_mask[1][0] = 4412 rsvd_check->rsvd_bits_mask[0][0]; 4413 4414 if (!pse) { 4415 rsvd_check->rsvd_bits_mask[1][1] = 0; 4416 break; 4417 } 4418 4419 if (is_cpuid_PSE36()) 4420 /* 36bits PSE 4MB page */ 4421 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21); 4422 else 4423 /* 32 bits PSE 4MB page */ 4424 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21); 4425 break; 4426 case PT32E_ROOT_LEVEL: 4427 rsvd_check->rsvd_bits_mask[0][2] = 4428 rsvd_bits(maxphyaddr, 63) | 4429 rsvd_bits(5, 8) | rsvd_bits(1, 2); /* PDPTE */ 4430 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd | 4431 rsvd_bits(maxphyaddr, 62); /* PDE */ 4432 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd | 4433 rsvd_bits(maxphyaddr, 62); /* PTE */ 4434 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd | 4435 rsvd_bits(maxphyaddr, 62) | 4436 rsvd_bits(13, 20); /* large page */ 4437 rsvd_check->rsvd_bits_mask[1][0] = 4438 rsvd_check->rsvd_bits_mask[0][0]; 4439 break; 4440 case PT64_ROOT_5LEVEL: 4441 rsvd_check->rsvd_bits_mask[0][4] = exb_bit_rsvd | 4442 nonleaf_bit8_rsvd | rsvd_bits(7, 7) | 4443 rsvd_bits(maxphyaddr, 51); 4444 rsvd_check->rsvd_bits_mask[1][4] = 4445 rsvd_check->rsvd_bits_mask[0][4]; 4446 /* fall through */ 4447 case PT64_ROOT_4LEVEL: 4448 rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd | 4449 nonleaf_bit8_rsvd | rsvd_bits(7, 7) | 4450 rsvd_bits(maxphyaddr, 51); 4451 rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd | 4452 nonleaf_bit8_rsvd | gbpages_bit_rsvd | 4453 rsvd_bits(maxphyaddr, 51); 4454 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd | 4455 rsvd_bits(maxphyaddr, 51); 4456 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd | 4457 rsvd_bits(maxphyaddr, 51); 4458 rsvd_check->rsvd_bits_mask[1][3] = 4459 rsvd_check->rsvd_bits_mask[0][3]; 4460 rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd | 4461 gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) | 4462 rsvd_bits(13, 29); 4463 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd | 4464 rsvd_bits(maxphyaddr, 51) | 4465 rsvd_bits(13, 20); /* large page */ 4466 rsvd_check->rsvd_bits_mask[1][0] = 4467 rsvd_check->rsvd_bits_mask[0][0]; 4468 break; 4469 } 4470 } 4471 4472 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, 4473 struct kvm_mmu *context) 4474 { 4475 __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check, 4476 cpuid_maxphyaddr(vcpu), context->root_level, 4477 context->nx, 4478 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), 4479 is_pse(vcpu), 4480 guest_cpuid_is_amd_or_hygon(vcpu)); 4481 } 4482 4483 static void 4484 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check, 4485 int maxphyaddr, bool execonly) 4486 { 4487 u64 bad_mt_xwr; 4488 4489 rsvd_check->rsvd_bits_mask[0][4] = 4490 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7); 4491 rsvd_check->rsvd_bits_mask[0][3] = 4492 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7); 4493 rsvd_check->rsvd_bits_mask[0][2] = 4494 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6); 4495 rsvd_check->rsvd_bits_mask[0][1] = 4496 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6); 4497 rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51); 4498 4499 /* large page */ 4500 rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4]; 4501 rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3]; 4502 rsvd_check->rsvd_bits_mask[1][2] = 4503 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29); 4504 rsvd_check->rsvd_bits_mask[1][1] = 4505 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20); 4506 rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0]; 4507 4508 bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */ 4509 bad_mt_xwr |= 0xFFull << (3 * 8); /* bits 3..5 must not be 3 */ 4510 bad_mt_xwr |= 0xFFull << (7 * 8); /* bits 3..5 must not be 7 */ 4511 bad_mt_xwr |= REPEAT_BYTE(1ull << 2); /* bits 0..2 must not be 010 */ 4512 bad_mt_xwr |= REPEAT_BYTE(1ull << 6); /* bits 0..2 must not be 110 */ 4513 if (!execonly) { 4514 /* bits 0..2 must not be 100 unless VMX capabilities allow it */ 4515 bad_mt_xwr |= REPEAT_BYTE(1ull << 4); 4516 } 4517 rsvd_check->bad_mt_xwr = bad_mt_xwr; 4518 } 4519 4520 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu, 4521 struct kvm_mmu *context, bool execonly) 4522 { 4523 __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check, 4524 cpuid_maxphyaddr(vcpu), execonly); 4525 } 4526 4527 /* 4528 * the page table on host is the shadow page table for the page 4529 * table in guest or amd nested guest, its mmu features completely 4530 * follow the features in guest. 4531 */ 4532 void 4533 reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) 4534 { 4535 bool uses_nx = context->nx || 4536 context->mmu_role.base.smep_andnot_wp; 4537 struct rsvd_bits_validate *shadow_zero_check; 4538 int i; 4539 4540 /* 4541 * Passing "true" to the last argument is okay; it adds a check 4542 * on bit 8 of the SPTEs which KVM doesn't use anyway. 4543 */ 4544 shadow_zero_check = &context->shadow_zero_check; 4545 __reset_rsvds_bits_mask(vcpu, shadow_zero_check, 4546 shadow_phys_bits, 4547 context->shadow_root_level, uses_nx, 4548 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), 4549 is_pse(vcpu), true); 4550 4551 if (!shadow_me_mask) 4552 return; 4553 4554 for (i = context->shadow_root_level; --i >= 0;) { 4555 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; 4556 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; 4557 } 4558 4559 } 4560 EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask); 4561 4562 static inline bool boot_cpu_is_amd(void) 4563 { 4564 WARN_ON_ONCE(!tdp_enabled); 4565 return shadow_x_mask == 0; 4566 } 4567 4568 /* 4569 * the direct page table on host, use as much mmu features as 4570 * possible, however, kvm currently does not do execution-protection. 4571 */ 4572 static void 4573 reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, 4574 struct kvm_mmu *context) 4575 { 4576 struct rsvd_bits_validate *shadow_zero_check; 4577 int i; 4578 4579 shadow_zero_check = &context->shadow_zero_check; 4580 4581 if (boot_cpu_is_amd()) 4582 __reset_rsvds_bits_mask(vcpu, shadow_zero_check, 4583 shadow_phys_bits, 4584 context->shadow_root_level, false, 4585 boot_cpu_has(X86_FEATURE_GBPAGES), 4586 true, true); 4587 else 4588 __reset_rsvds_bits_mask_ept(shadow_zero_check, 4589 shadow_phys_bits, 4590 false); 4591 4592 if (!shadow_me_mask) 4593 return; 4594 4595 for (i = context->shadow_root_level; --i >= 0;) { 4596 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; 4597 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; 4598 } 4599 } 4600 4601 /* 4602 * as the comments in reset_shadow_zero_bits_mask() except it 4603 * is the shadow page table for intel nested guest. 4604 */ 4605 static void 4606 reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, 4607 struct kvm_mmu *context, bool execonly) 4608 { 4609 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check, 4610 shadow_phys_bits, execonly); 4611 } 4612 4613 #define BYTE_MASK(access) \ 4614 ((1 & (access) ? 2 : 0) | \ 4615 (2 & (access) ? 4 : 0) | \ 4616 (3 & (access) ? 8 : 0) | \ 4617 (4 & (access) ? 16 : 0) | \ 4618 (5 & (access) ? 32 : 0) | \ 4619 (6 & (access) ? 64 : 0) | \ 4620 (7 & (access) ? 128 : 0)) 4621 4622 4623 static void update_permission_bitmask(struct kvm_vcpu *vcpu, 4624 struct kvm_mmu *mmu, bool ept) 4625 { 4626 unsigned byte; 4627 4628 const u8 x = BYTE_MASK(ACC_EXEC_MASK); 4629 const u8 w = BYTE_MASK(ACC_WRITE_MASK); 4630 const u8 u = BYTE_MASK(ACC_USER_MASK); 4631 4632 bool cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP) != 0; 4633 bool cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP) != 0; 4634 bool cr0_wp = is_write_protection(vcpu); 4635 4636 for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) { 4637 unsigned pfec = byte << 1; 4638 4639 /* 4640 * Each "*f" variable has a 1 bit for each UWX value 4641 * that causes a fault with the given PFEC. 4642 */ 4643 4644 /* Faults from writes to non-writable pages */ 4645 u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0; 4646 /* Faults from user mode accesses to supervisor pages */ 4647 u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0; 4648 /* Faults from fetches of non-executable pages*/ 4649 u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0; 4650 /* Faults from kernel mode fetches of user pages */ 4651 u8 smepf = 0; 4652 /* Faults from kernel mode accesses of user pages */ 4653 u8 smapf = 0; 4654 4655 if (!ept) { 4656 /* Faults from kernel mode accesses to user pages */ 4657 u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u; 4658 4659 /* Not really needed: !nx will cause pte.nx to fault */ 4660 if (!mmu->nx) 4661 ff = 0; 4662 4663 /* Allow supervisor writes if !cr0.wp */ 4664 if (!cr0_wp) 4665 wf = (pfec & PFERR_USER_MASK) ? wf : 0; 4666 4667 /* Disallow supervisor fetches of user code if cr4.smep */ 4668 if (cr4_smep) 4669 smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0; 4670 4671 /* 4672 * SMAP:kernel-mode data accesses from user-mode 4673 * mappings should fault. A fault is considered 4674 * as a SMAP violation if all of the following 4675 * conditions are true: 4676 * - X86_CR4_SMAP is set in CR4 4677 * - A user page is accessed 4678 * - The access is not a fetch 4679 * - Page fault in kernel mode 4680 * - if CPL = 3 or X86_EFLAGS_AC is clear 4681 * 4682 * Here, we cover the first three conditions. 4683 * The fourth is computed dynamically in permission_fault(); 4684 * PFERR_RSVD_MASK bit will be set in PFEC if the access is 4685 * *not* subject to SMAP restrictions. 4686 */ 4687 if (cr4_smap) 4688 smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf; 4689 } 4690 4691 mmu->permissions[byte] = ff | uf | wf | smepf | smapf; 4692 } 4693 } 4694 4695 /* 4696 * PKU is an additional mechanism by which the paging controls access to 4697 * user-mode addresses based on the value in the PKRU register. Protection 4698 * key violations are reported through a bit in the page fault error code. 4699 * Unlike other bits of the error code, the PK bit is not known at the 4700 * call site of e.g. gva_to_gpa; it must be computed directly in 4701 * permission_fault based on two bits of PKRU, on some machine state (CR4, 4702 * CR0, EFER, CPL), and on other bits of the error code and the page tables. 4703 * 4704 * In particular the following conditions come from the error code, the 4705 * page tables and the machine state: 4706 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1 4707 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch) 4708 * - PK is always zero if U=0 in the page tables 4709 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access. 4710 * 4711 * The PKRU bitmask caches the result of these four conditions. The error 4712 * code (minus the P bit) and the page table's U bit form an index into the 4713 * PKRU bitmask. Two bits of the PKRU bitmask are then extracted and ANDed 4714 * with the two bits of the PKRU register corresponding to the protection key. 4715 * For the first three conditions above the bits will be 00, thus masking 4716 * away both AD and WD. For all reads or if the last condition holds, WD 4717 * only will be masked away. 4718 */ 4719 static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 4720 bool ept) 4721 { 4722 unsigned bit; 4723 bool wp; 4724 4725 if (ept) { 4726 mmu->pkru_mask = 0; 4727 return; 4728 } 4729 4730 /* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */ 4731 if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) { 4732 mmu->pkru_mask = 0; 4733 return; 4734 } 4735 4736 wp = is_write_protection(vcpu); 4737 4738 for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) { 4739 unsigned pfec, pkey_bits; 4740 bool check_pkey, check_write, ff, uf, wf, pte_user; 4741 4742 pfec = bit << 1; 4743 ff = pfec & PFERR_FETCH_MASK; 4744 uf = pfec & PFERR_USER_MASK; 4745 wf = pfec & PFERR_WRITE_MASK; 4746 4747 /* PFEC.RSVD is replaced by ACC_USER_MASK. */ 4748 pte_user = pfec & PFERR_RSVD_MASK; 4749 4750 /* 4751 * Only need to check the access which is not an 4752 * instruction fetch and is to a user page. 4753 */ 4754 check_pkey = (!ff && pte_user); 4755 /* 4756 * write access is controlled by PKRU if it is a 4757 * user access or CR0.WP = 1. 4758 */ 4759 check_write = check_pkey && wf && (uf || wp); 4760 4761 /* PKRU.AD stops both read and write access. */ 4762 pkey_bits = !!check_pkey; 4763 /* PKRU.WD stops write access. */ 4764 pkey_bits |= (!!check_write) << 1; 4765 4766 mmu->pkru_mask |= (pkey_bits & 3) << pfec; 4767 } 4768 } 4769 4770 static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) 4771 { 4772 unsigned root_level = mmu->root_level; 4773 4774 mmu->last_nonleaf_level = root_level; 4775 if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu)) 4776 mmu->last_nonleaf_level++; 4777 } 4778 4779 static void paging64_init_context_common(struct kvm_vcpu *vcpu, 4780 struct kvm_mmu *context, 4781 int level) 4782 { 4783 context->nx = is_nx(vcpu); 4784 context->root_level = level; 4785 4786 reset_rsvds_bits_mask(vcpu, context); 4787 update_permission_bitmask(vcpu, context, false); 4788 update_pkru_bitmask(vcpu, context, false); 4789 update_last_nonleaf_level(vcpu, context); 4790 4791 MMU_WARN_ON(!is_pae(vcpu)); 4792 context->page_fault = paging64_page_fault; 4793 context->gva_to_gpa = paging64_gva_to_gpa; 4794 context->sync_page = paging64_sync_page; 4795 context->invlpg = paging64_invlpg; 4796 context->update_pte = paging64_update_pte; 4797 context->shadow_root_level = level; 4798 context->direct_map = false; 4799 } 4800 4801 static void paging64_init_context(struct kvm_vcpu *vcpu, 4802 struct kvm_mmu *context) 4803 { 4804 int root_level = is_la57_mode(vcpu) ? 4805 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4806 4807 paging64_init_context_common(vcpu, context, root_level); 4808 } 4809 4810 static void paging32_init_context(struct kvm_vcpu *vcpu, 4811 struct kvm_mmu *context) 4812 { 4813 context->nx = false; 4814 context->root_level = PT32_ROOT_LEVEL; 4815 4816 reset_rsvds_bits_mask(vcpu, context); 4817 update_permission_bitmask(vcpu, context, false); 4818 update_pkru_bitmask(vcpu, context, false); 4819 update_last_nonleaf_level(vcpu, context); 4820 4821 context->page_fault = paging32_page_fault; 4822 context->gva_to_gpa = paging32_gva_to_gpa; 4823 context->sync_page = paging32_sync_page; 4824 context->invlpg = paging32_invlpg; 4825 context->update_pte = paging32_update_pte; 4826 context->shadow_root_level = PT32E_ROOT_LEVEL; 4827 context->direct_map = false; 4828 } 4829 4830 static void paging32E_init_context(struct kvm_vcpu *vcpu, 4831 struct kvm_mmu *context) 4832 { 4833 paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL); 4834 } 4835 4836 static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu) 4837 { 4838 union kvm_mmu_extended_role ext = {0}; 4839 4840 ext.cr0_pg = !!is_paging(vcpu); 4841 ext.cr4_pae = !!is_pae(vcpu); 4842 ext.cr4_smep = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMEP); 4843 ext.cr4_smap = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMAP); 4844 ext.cr4_pse = !!is_pse(vcpu); 4845 ext.cr4_pke = !!kvm_read_cr4_bits(vcpu, X86_CR4_PKE); 4846 ext.maxphyaddr = cpuid_maxphyaddr(vcpu); 4847 4848 ext.valid = 1; 4849 4850 return ext; 4851 } 4852 4853 static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, 4854 bool base_only) 4855 { 4856 union kvm_mmu_role role = {0}; 4857 4858 role.base.access = ACC_ALL; 4859 role.base.nxe = !!is_nx(vcpu); 4860 role.base.cr0_wp = is_write_protection(vcpu); 4861 role.base.smm = is_smm(vcpu); 4862 role.base.guest_mode = is_guest_mode(vcpu); 4863 4864 if (base_only) 4865 return role; 4866 4867 role.ext = kvm_calc_mmu_role_ext(vcpu); 4868 4869 return role; 4870 } 4871 4872 static union kvm_mmu_role 4873 kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) 4874 { 4875 union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); 4876 4877 role.base.ad_disabled = (shadow_accessed_mask == 0); 4878 role.base.level = vcpu->arch.tdp_level; 4879 role.base.direct = true; 4880 role.base.gpte_is_8_bytes = true; 4881 4882 return role; 4883 } 4884 4885 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) 4886 { 4887 struct kvm_mmu *context = vcpu->arch.mmu; 4888 union kvm_mmu_role new_role = 4889 kvm_calc_tdp_mmu_root_page_role(vcpu, false); 4890 4891 if (new_role.as_u64 == context->mmu_role.as_u64) 4892 return; 4893 4894 context->mmu_role.as_u64 = new_role.as_u64; 4895 context->page_fault = kvm_tdp_page_fault; 4896 context->sync_page = nonpaging_sync_page; 4897 context->invlpg = NULL; 4898 context->update_pte = nonpaging_update_pte; 4899 context->shadow_root_level = vcpu->arch.tdp_level; 4900 context->direct_map = true; 4901 context->get_guest_pgd = get_cr3; 4902 context->get_pdptr = kvm_pdptr_read; 4903 context->inject_page_fault = kvm_inject_page_fault; 4904 4905 if (!is_paging(vcpu)) { 4906 context->nx = false; 4907 context->gva_to_gpa = nonpaging_gva_to_gpa; 4908 context->root_level = 0; 4909 } else if (is_long_mode(vcpu)) { 4910 context->nx = is_nx(vcpu); 4911 context->root_level = is_la57_mode(vcpu) ? 4912 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 4913 reset_rsvds_bits_mask(vcpu, context); 4914 context->gva_to_gpa = paging64_gva_to_gpa; 4915 } else if (is_pae(vcpu)) { 4916 context->nx = is_nx(vcpu); 4917 context->root_level = PT32E_ROOT_LEVEL; 4918 reset_rsvds_bits_mask(vcpu, context); 4919 context->gva_to_gpa = paging64_gva_to_gpa; 4920 } else { 4921 context->nx = false; 4922 context->root_level = PT32_ROOT_LEVEL; 4923 reset_rsvds_bits_mask(vcpu, context); 4924 context->gva_to_gpa = paging32_gva_to_gpa; 4925 } 4926 4927 update_permission_bitmask(vcpu, context, false); 4928 update_pkru_bitmask(vcpu, context, false); 4929 update_last_nonleaf_level(vcpu, context); 4930 reset_tdp_shadow_zero_bits_mask(vcpu, context); 4931 } 4932 4933 static union kvm_mmu_role 4934 kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) 4935 { 4936 union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); 4937 4938 role.base.smep_andnot_wp = role.ext.cr4_smep && 4939 !is_write_protection(vcpu); 4940 role.base.smap_andnot_wp = role.ext.cr4_smap && 4941 !is_write_protection(vcpu); 4942 role.base.direct = !is_paging(vcpu); 4943 role.base.gpte_is_8_bytes = !!is_pae(vcpu); 4944 4945 if (!is_long_mode(vcpu)) 4946 role.base.level = PT32E_ROOT_LEVEL; 4947 else if (is_la57_mode(vcpu)) 4948 role.base.level = PT64_ROOT_5LEVEL; 4949 else 4950 role.base.level = PT64_ROOT_4LEVEL; 4951 4952 return role; 4953 } 4954 4955 void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer) 4956 { 4957 struct kvm_mmu *context = vcpu->arch.mmu; 4958 union kvm_mmu_role new_role = 4959 kvm_calc_shadow_mmu_root_page_role(vcpu, false); 4960 4961 if (new_role.as_u64 == context->mmu_role.as_u64) 4962 return; 4963 4964 if (!(cr0 & X86_CR0_PG)) 4965 nonpaging_init_context(vcpu, context); 4966 else if (efer & EFER_LMA) 4967 paging64_init_context(vcpu, context); 4968 else if (cr4 & X86_CR4_PAE) 4969 paging32E_init_context(vcpu, context); 4970 else 4971 paging32_init_context(vcpu, context); 4972 4973 context->mmu_role.as_u64 = new_role.as_u64; 4974 reset_shadow_zero_bits_mask(vcpu, context); 4975 } 4976 EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu); 4977 4978 static union kvm_mmu_role 4979 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, 4980 bool execonly, u8 level) 4981 { 4982 union kvm_mmu_role role = {0}; 4983 4984 /* SMM flag is inherited from root_mmu */ 4985 role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm; 4986 4987 role.base.level = level; 4988 role.base.gpte_is_8_bytes = true; 4989 role.base.direct = false; 4990 role.base.ad_disabled = !accessed_dirty; 4991 role.base.guest_mode = true; 4992 role.base.access = ACC_ALL; 4993 4994 /* 4995 * WP=1 and NOT_WP=1 is an impossible combination, use WP and the 4996 * SMAP variation to denote shadow EPT entries. 4997 */ 4998 role.base.cr0_wp = true; 4999 role.base.smap_andnot_wp = true; 5000 5001 role.ext = kvm_calc_mmu_role_ext(vcpu); 5002 role.ext.execonly = execonly; 5003 5004 return role; 5005 } 5006 5007 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, 5008 bool accessed_dirty, gpa_t new_eptp) 5009 { 5010 struct kvm_mmu *context = vcpu->arch.mmu; 5011 u8 level = vmx_eptp_page_walk_level(new_eptp); 5012 union kvm_mmu_role new_role = 5013 kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, 5014 execonly, level); 5015 5016 __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base, true, true); 5017 5018 if (new_role.as_u64 == context->mmu_role.as_u64) 5019 return; 5020 5021 context->shadow_root_level = level; 5022 5023 context->nx = true; 5024 context->ept_ad = accessed_dirty; 5025 context->page_fault = ept_page_fault; 5026 context->gva_to_gpa = ept_gva_to_gpa; 5027 context->sync_page = ept_sync_page; 5028 context->invlpg = ept_invlpg; 5029 context->update_pte = ept_update_pte; 5030 context->root_level = level; 5031 context->direct_map = false; 5032 context->mmu_role.as_u64 = new_role.as_u64; 5033 5034 update_permission_bitmask(vcpu, context, true); 5035 update_pkru_bitmask(vcpu, context, true); 5036 update_last_nonleaf_level(vcpu, context); 5037 reset_rsvds_bits_mask_ept(vcpu, context, execonly); 5038 reset_ept_shadow_zero_bits_mask(vcpu, context, execonly); 5039 } 5040 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); 5041 5042 static void init_kvm_softmmu(struct kvm_vcpu *vcpu) 5043 { 5044 struct kvm_mmu *context = vcpu->arch.mmu; 5045 5046 kvm_init_shadow_mmu(vcpu, 5047 kvm_read_cr0_bits(vcpu, X86_CR0_PG), 5048 kvm_read_cr4_bits(vcpu, X86_CR4_PAE), 5049 vcpu->arch.efer); 5050 5051 context->get_guest_pgd = get_cr3; 5052 context->get_pdptr = kvm_pdptr_read; 5053 context->inject_page_fault = kvm_inject_page_fault; 5054 } 5055 5056 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) 5057 { 5058 union kvm_mmu_role new_role = kvm_calc_mmu_role_common(vcpu, false); 5059 struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; 5060 5061 if (new_role.as_u64 == g_context->mmu_role.as_u64) 5062 return; 5063 5064 g_context->mmu_role.as_u64 = new_role.as_u64; 5065 g_context->get_guest_pgd = get_cr3; 5066 g_context->get_pdptr = kvm_pdptr_read; 5067 g_context->inject_page_fault = kvm_inject_page_fault; 5068 5069 /* 5070 * L2 page tables are never shadowed, so there is no need to sync 5071 * SPTEs. 5072 */ 5073 g_context->invlpg = NULL; 5074 5075 /* 5076 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using 5077 * L1's nested page tables (e.g. EPT12). The nested translation 5078 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using 5079 * L2's page tables as the first level of translation and L1's 5080 * nested page tables as the second level of translation. Basically 5081 * the gva_to_gpa functions between mmu and nested_mmu are swapped. 5082 */ 5083 if (!is_paging(vcpu)) { 5084 g_context->nx = false; 5085 g_context->root_level = 0; 5086 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested; 5087 } else if (is_long_mode(vcpu)) { 5088 g_context->nx = is_nx(vcpu); 5089 g_context->root_level = is_la57_mode(vcpu) ? 5090 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 5091 reset_rsvds_bits_mask(vcpu, g_context); 5092 g_context->gva_to_gpa = paging64_gva_to_gpa_nested; 5093 } else if (is_pae(vcpu)) { 5094 g_context->nx = is_nx(vcpu); 5095 g_context->root_level = PT32E_ROOT_LEVEL; 5096 reset_rsvds_bits_mask(vcpu, g_context); 5097 g_context->gva_to_gpa = paging64_gva_to_gpa_nested; 5098 } else { 5099 g_context->nx = false; 5100 g_context->root_level = PT32_ROOT_LEVEL; 5101 reset_rsvds_bits_mask(vcpu, g_context); 5102 g_context->gva_to_gpa = paging32_gva_to_gpa_nested; 5103 } 5104 5105 update_permission_bitmask(vcpu, g_context, false); 5106 update_pkru_bitmask(vcpu, g_context, false); 5107 update_last_nonleaf_level(vcpu, g_context); 5108 } 5109 5110 void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots) 5111 { 5112 if (reset_roots) { 5113 uint i; 5114 5115 vcpu->arch.mmu->root_hpa = INVALID_PAGE; 5116 5117 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5118 vcpu->arch.mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 5119 } 5120 5121 if (mmu_is_nested(vcpu)) 5122 init_kvm_nested_mmu(vcpu); 5123 else if (tdp_enabled) 5124 init_kvm_tdp_mmu(vcpu); 5125 else 5126 init_kvm_softmmu(vcpu); 5127 } 5128 EXPORT_SYMBOL_GPL(kvm_init_mmu); 5129 5130 static union kvm_mmu_page_role 5131 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu) 5132 { 5133 union kvm_mmu_role role; 5134 5135 if (tdp_enabled) 5136 role = kvm_calc_tdp_mmu_root_page_role(vcpu, true); 5137 else 5138 role = kvm_calc_shadow_mmu_root_page_role(vcpu, true); 5139 5140 return role.base; 5141 } 5142 5143 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu) 5144 { 5145 kvm_mmu_unload(vcpu); 5146 kvm_init_mmu(vcpu, true); 5147 } 5148 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context); 5149 5150 int kvm_mmu_load(struct kvm_vcpu *vcpu) 5151 { 5152 int r; 5153 5154 r = mmu_topup_memory_caches(vcpu); 5155 if (r) 5156 goto out; 5157 r = mmu_alloc_roots(vcpu); 5158 kvm_mmu_sync_roots(vcpu); 5159 if (r) 5160 goto out; 5161 kvm_mmu_load_pgd(vcpu); 5162 kvm_x86_ops.tlb_flush_current(vcpu); 5163 out: 5164 return r; 5165 } 5166 EXPORT_SYMBOL_GPL(kvm_mmu_load); 5167 5168 void kvm_mmu_unload(struct kvm_vcpu *vcpu) 5169 { 5170 kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL); 5171 WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa)); 5172 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 5173 WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa)); 5174 } 5175 EXPORT_SYMBOL_GPL(kvm_mmu_unload); 5176 5177 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, 5178 struct kvm_mmu_page *sp, u64 *spte, 5179 const void *new) 5180 { 5181 if (sp->role.level != PG_LEVEL_4K) { 5182 ++vcpu->kvm->stat.mmu_pde_zapped; 5183 return; 5184 } 5185 5186 ++vcpu->kvm->stat.mmu_pte_updated; 5187 vcpu->arch.mmu->update_pte(vcpu, sp, spte, new); 5188 } 5189 5190 static bool need_remote_flush(u64 old, u64 new) 5191 { 5192 if (!is_shadow_present_pte(old)) 5193 return false; 5194 if (!is_shadow_present_pte(new)) 5195 return true; 5196 if ((old ^ new) & PT64_BASE_ADDR_MASK) 5197 return true; 5198 old ^= shadow_nx_mask; 5199 new ^= shadow_nx_mask; 5200 return (old & ~new & PT64_PERM_MASK) != 0; 5201 } 5202 5203 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa, 5204 int *bytes) 5205 { 5206 u64 gentry = 0; 5207 int r; 5208 5209 /* 5210 * Assume that the pte write on a page table of the same type 5211 * as the current vcpu paging mode since we update the sptes only 5212 * when they have the same mode. 5213 */ 5214 if (is_pae(vcpu) && *bytes == 4) { 5215 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */ 5216 *gpa &= ~(gpa_t)7; 5217 *bytes = 8; 5218 } 5219 5220 if (*bytes == 4 || *bytes == 8) { 5221 r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes); 5222 if (r) 5223 gentry = 0; 5224 } 5225 5226 return gentry; 5227 } 5228 5229 /* 5230 * If we're seeing too many writes to a page, it may no longer be a page table, 5231 * or we may be forking, in which case it is better to unmap the page. 5232 */ 5233 static bool detect_write_flooding(struct kvm_mmu_page *sp) 5234 { 5235 /* 5236 * Skip write-flooding detected for the sp whose level is 1, because 5237 * it can become unsync, then the guest page is not write-protected. 5238 */ 5239 if (sp->role.level == PG_LEVEL_4K) 5240 return false; 5241 5242 atomic_inc(&sp->write_flooding_count); 5243 return atomic_read(&sp->write_flooding_count) >= 3; 5244 } 5245 5246 /* 5247 * Misaligned accesses are too much trouble to fix up; also, they usually 5248 * indicate a page is not used as a page table. 5249 */ 5250 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa, 5251 int bytes) 5252 { 5253 unsigned offset, pte_size, misaligned; 5254 5255 pgprintk("misaligned: gpa %llx bytes %d role %x\n", 5256 gpa, bytes, sp->role.word); 5257 5258 offset = offset_in_page(gpa); 5259 pte_size = sp->role.gpte_is_8_bytes ? 8 : 4; 5260 5261 /* 5262 * Sometimes, the OS only writes the last one bytes to update status 5263 * bits, for example, in linux, andb instruction is used in clear_bit(). 5264 */ 5265 if (!(offset & (pte_size - 1)) && bytes == 1) 5266 return false; 5267 5268 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); 5269 misaligned |= bytes < 4; 5270 5271 return misaligned; 5272 } 5273 5274 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte) 5275 { 5276 unsigned page_offset, quadrant; 5277 u64 *spte; 5278 int level; 5279 5280 page_offset = offset_in_page(gpa); 5281 level = sp->role.level; 5282 *nspte = 1; 5283 if (!sp->role.gpte_is_8_bytes) { 5284 page_offset <<= 1; /* 32->64 */ 5285 /* 5286 * A 32-bit pde maps 4MB while the shadow pdes map 5287 * only 2MB. So we need to double the offset again 5288 * and zap two pdes instead of one. 5289 */ 5290 if (level == PT32_ROOT_LEVEL) { 5291 page_offset &= ~7; /* kill rounding error */ 5292 page_offset <<= 1; 5293 *nspte = 2; 5294 } 5295 quadrant = page_offset >> PAGE_SHIFT; 5296 page_offset &= ~PAGE_MASK; 5297 if (quadrant != sp->role.quadrant) 5298 return NULL; 5299 } 5300 5301 spte = &sp->spt[page_offset / sizeof(*spte)]; 5302 return spte; 5303 } 5304 5305 /* 5306 * Ignore various flags when determining if a SPTE can be immediately 5307 * overwritten for the current MMU. 5308 * - level: explicitly checked in mmu_pte_write_new_pte(), and will never 5309 * match the current MMU role, as MMU's level tracks the root level. 5310 * - access: updated based on the new guest PTE 5311 * - quadrant: handled by get_written_sptes() 5312 * - invalid: always false (loop only walks valid shadow pages) 5313 */ 5314 static const union kvm_mmu_page_role role_ign = { 5315 .level = 0xf, 5316 .access = 0x7, 5317 .quadrant = 0x3, 5318 .invalid = 0x1, 5319 }; 5320 5321 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 5322 const u8 *new, int bytes, 5323 struct kvm_page_track_notifier_node *node) 5324 { 5325 gfn_t gfn = gpa >> PAGE_SHIFT; 5326 struct kvm_mmu_page *sp; 5327 LIST_HEAD(invalid_list); 5328 u64 entry, gentry, *spte; 5329 int npte; 5330 bool remote_flush, local_flush; 5331 5332 /* 5333 * If we don't have indirect shadow pages, it means no page is 5334 * write-protected, so we can exit simply. 5335 */ 5336 if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages)) 5337 return; 5338 5339 remote_flush = local_flush = false; 5340 5341 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes); 5342 5343 /* 5344 * No need to care whether allocation memory is successful 5345 * or not since pte prefetch is skiped if it does not have 5346 * enough objects in the cache. 5347 */ 5348 mmu_topup_memory_caches(vcpu); 5349 5350 spin_lock(&vcpu->kvm->mmu_lock); 5351 5352 gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes); 5353 5354 ++vcpu->kvm->stat.mmu_pte_write; 5355 kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE); 5356 5357 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { 5358 if (detect_write_misaligned(sp, gpa, bytes) || 5359 detect_write_flooding(sp)) { 5360 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list); 5361 ++vcpu->kvm->stat.mmu_flooded; 5362 continue; 5363 } 5364 5365 spte = get_written_sptes(sp, gpa, &npte); 5366 if (!spte) 5367 continue; 5368 5369 local_flush = true; 5370 while (npte--) { 5371 u32 base_role = vcpu->arch.mmu->mmu_role.base.word; 5372 5373 entry = *spte; 5374 mmu_page_zap_pte(vcpu->kvm, sp, spte); 5375 if (gentry && 5376 !((sp->role.word ^ base_role) & ~role_ign.word) && 5377 rmap_can_add(vcpu)) 5378 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry); 5379 if (need_remote_flush(entry, *spte)) 5380 remote_flush = true; 5381 ++spte; 5382 } 5383 } 5384 kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush); 5385 kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE); 5386 spin_unlock(&vcpu->kvm->mmu_lock); 5387 } 5388 5389 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) 5390 { 5391 gpa_t gpa; 5392 int r; 5393 5394 if (vcpu->arch.mmu->direct_map) 5395 return 0; 5396 5397 gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); 5398 5399 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); 5400 5401 return r; 5402 } 5403 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt); 5404 5405 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, 5406 void *insn, int insn_len) 5407 { 5408 int r, emulation_type = EMULTYPE_PF; 5409 bool direct = vcpu->arch.mmu->direct_map; 5410 5411 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 5412 return RET_PF_RETRY; 5413 5414 r = RET_PF_INVALID; 5415 if (unlikely(error_code & PFERR_RSVD_MASK)) { 5416 r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); 5417 if (r == RET_PF_EMULATE) 5418 goto emulate; 5419 } 5420 5421 if (r == RET_PF_INVALID) { 5422 r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa, 5423 lower_32_bits(error_code), false); 5424 WARN_ON(r == RET_PF_INVALID); 5425 } 5426 5427 if (r == RET_PF_RETRY) 5428 return 1; 5429 if (r < 0) 5430 return r; 5431 5432 /* 5433 * Before emulating the instruction, check if the error code 5434 * was due to a RO violation while translating the guest page. 5435 * This can occur when using nested virtualization with nested 5436 * paging in both guests. If true, we simply unprotect the page 5437 * and resume the guest. 5438 */ 5439 if (vcpu->arch.mmu->direct_map && 5440 (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) { 5441 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)); 5442 return 1; 5443 } 5444 5445 /* 5446 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still 5447 * optimistically try to just unprotect the page and let the processor 5448 * re-execute the instruction that caused the page fault. Do not allow 5449 * retrying MMIO emulation, as it's not only pointless but could also 5450 * cause us to enter an infinite loop because the processor will keep 5451 * faulting on the non-existent MMIO address. Retrying an instruction 5452 * from a nested guest is also pointless and dangerous as we are only 5453 * explicitly shadowing L1's page tables, i.e. unprotecting something 5454 * for L1 isn't going to magically fix whatever issue cause L2 to fail. 5455 */ 5456 if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu)) 5457 emulation_type |= EMULTYPE_ALLOW_RETRY_PF; 5458 emulate: 5459 /* 5460 * On AMD platforms, under certain conditions insn_len may be zero on #NPF. 5461 * This can happen if a guest gets a page-fault on data access but the HW 5462 * table walker is not able to read the instruction page (e.g instruction 5463 * page is not present in memory). In those cases we simply restart the 5464 * guest, with the exception of AMD Erratum 1096 which is unrecoverable. 5465 */ 5466 if (unlikely(insn && !insn_len)) { 5467 if (!kvm_x86_ops.need_emulation_on_page_fault(vcpu)) 5468 return 1; 5469 } 5470 5471 return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, 5472 insn_len); 5473 } 5474 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); 5475 5476 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 5477 gva_t gva, hpa_t root_hpa) 5478 { 5479 int i; 5480 5481 /* It's actually a GPA for vcpu->arch.guest_mmu. */ 5482 if (mmu != &vcpu->arch.guest_mmu) { 5483 /* INVLPG on a non-canonical address is a NOP according to the SDM. */ 5484 if (is_noncanonical_address(gva, vcpu)) 5485 return; 5486 5487 kvm_x86_ops.tlb_flush_gva(vcpu, gva); 5488 } 5489 5490 if (!mmu->invlpg) 5491 return; 5492 5493 if (root_hpa == INVALID_PAGE) { 5494 mmu->invlpg(vcpu, gva, mmu->root_hpa); 5495 5496 /* 5497 * INVLPG is required to invalidate any global mappings for the VA, 5498 * irrespective of PCID. Since it would take us roughly similar amount 5499 * of work to determine whether any of the prev_root mappings of the VA 5500 * is marked global, or to just sync it blindly, so we might as well 5501 * just always sync it. 5502 * 5503 * Mappings not reachable via the current cr3 or the prev_roots will be 5504 * synced when switching to that cr3, so nothing needs to be done here 5505 * for them. 5506 */ 5507 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5508 if (VALID_PAGE(mmu->prev_roots[i].hpa)) 5509 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); 5510 } else { 5511 mmu->invlpg(vcpu, gva, root_hpa); 5512 } 5513 } 5514 EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva); 5515 5516 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) 5517 { 5518 kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, gva, INVALID_PAGE); 5519 ++vcpu->stat.invlpg; 5520 } 5521 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); 5522 5523 5524 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) 5525 { 5526 struct kvm_mmu *mmu = vcpu->arch.mmu; 5527 bool tlb_flush = false; 5528 uint i; 5529 5530 if (pcid == kvm_get_active_pcid(vcpu)) { 5531 mmu->invlpg(vcpu, gva, mmu->root_hpa); 5532 tlb_flush = true; 5533 } 5534 5535 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5536 if (VALID_PAGE(mmu->prev_roots[i].hpa) && 5537 pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) { 5538 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); 5539 tlb_flush = true; 5540 } 5541 } 5542 5543 if (tlb_flush) 5544 kvm_x86_ops.tlb_flush_gva(vcpu, gva); 5545 5546 ++vcpu->stat.invlpg; 5547 5548 /* 5549 * Mappings not reachable via the current cr3 or the prev_roots will be 5550 * synced when switching to that cr3, so nothing needs to be done here 5551 * for them. 5552 */ 5553 } 5554 EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); 5555 5556 void kvm_configure_mmu(bool enable_tdp, int tdp_page_level) 5557 { 5558 tdp_enabled = enable_tdp; 5559 5560 /* 5561 * max_page_level reflects the capabilities of KVM's MMU irrespective 5562 * of kernel support, e.g. KVM may be capable of using 1GB pages when 5563 * the kernel is not. But, KVM never creates a page size greater than 5564 * what is used by the kernel for any given HVA, i.e. the kernel's 5565 * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust(). 5566 */ 5567 if (tdp_enabled) 5568 max_page_level = tdp_page_level; 5569 else if (boot_cpu_has(X86_FEATURE_GBPAGES)) 5570 max_page_level = PG_LEVEL_1G; 5571 else 5572 max_page_level = PG_LEVEL_2M; 5573 } 5574 EXPORT_SYMBOL_GPL(kvm_configure_mmu); 5575 5576 /* The return value indicates if tlb flush on all vcpus is needed. */ 5577 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); 5578 5579 /* The caller should hold mmu-lock before calling this function. */ 5580 static __always_inline bool 5581 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, 5582 slot_level_handler fn, int start_level, int end_level, 5583 gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb) 5584 { 5585 struct slot_rmap_walk_iterator iterator; 5586 bool flush = false; 5587 5588 for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, 5589 end_gfn, &iterator) { 5590 if (iterator.rmap) 5591 flush |= fn(kvm, iterator.rmap); 5592 5593 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { 5594 if (flush && lock_flush_tlb) { 5595 kvm_flush_remote_tlbs_with_address(kvm, 5596 start_gfn, 5597 iterator.gfn - start_gfn + 1); 5598 flush = false; 5599 } 5600 cond_resched_lock(&kvm->mmu_lock); 5601 } 5602 } 5603 5604 if (flush && lock_flush_tlb) { 5605 kvm_flush_remote_tlbs_with_address(kvm, start_gfn, 5606 end_gfn - start_gfn + 1); 5607 flush = false; 5608 } 5609 5610 return flush; 5611 } 5612 5613 static __always_inline bool 5614 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5615 slot_level_handler fn, int start_level, int end_level, 5616 bool lock_flush_tlb) 5617 { 5618 return slot_handle_level_range(kvm, memslot, fn, start_level, 5619 end_level, memslot->base_gfn, 5620 memslot->base_gfn + memslot->npages - 1, 5621 lock_flush_tlb); 5622 } 5623 5624 static __always_inline bool 5625 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5626 slot_level_handler fn, bool lock_flush_tlb) 5627 { 5628 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, 5629 KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); 5630 } 5631 5632 static __always_inline bool 5633 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 5634 slot_level_handler fn, bool lock_flush_tlb) 5635 { 5636 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K + 1, 5637 KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); 5638 } 5639 5640 static __always_inline bool 5641 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, 5642 slot_level_handler fn, bool lock_flush_tlb) 5643 { 5644 return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, 5645 PG_LEVEL_4K, lock_flush_tlb); 5646 } 5647 5648 static void free_mmu_pages(struct kvm_mmu *mmu) 5649 { 5650 free_page((unsigned long)mmu->pae_root); 5651 free_page((unsigned long)mmu->lm_root); 5652 } 5653 5654 static int alloc_mmu_pages(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) 5655 { 5656 struct page *page; 5657 int i; 5658 5659 /* 5660 * When using PAE paging, the four PDPTEs are treated as 'root' pages, 5661 * while the PDP table is a per-vCPU construct that's allocated at MMU 5662 * creation. When emulating 32-bit mode, cr3 is only 32 bits even on 5663 * x86_64. Therefore we need to allocate the PDP table in the first 5664 * 4GB of memory, which happens to fit the DMA32 zone. Except for 5665 * SVM's 32-bit NPT support, TDP paging doesn't use PAE paging and can 5666 * skip allocating the PDP table. 5667 */ 5668 if (tdp_enabled && vcpu->arch.tdp_level > PT32E_ROOT_LEVEL) 5669 return 0; 5670 5671 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); 5672 if (!page) 5673 return -ENOMEM; 5674 5675 mmu->pae_root = page_address(page); 5676 for (i = 0; i < 4; ++i) 5677 mmu->pae_root[i] = INVALID_PAGE; 5678 5679 return 0; 5680 } 5681 5682 int kvm_mmu_create(struct kvm_vcpu *vcpu) 5683 { 5684 uint i; 5685 int ret; 5686 5687 vcpu->arch.mmu = &vcpu->arch.root_mmu; 5688 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 5689 5690 vcpu->arch.root_mmu.root_hpa = INVALID_PAGE; 5691 vcpu->arch.root_mmu.root_pgd = 0; 5692 vcpu->arch.root_mmu.translate_gpa = translate_gpa; 5693 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5694 vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 5695 5696 vcpu->arch.guest_mmu.root_hpa = INVALID_PAGE; 5697 vcpu->arch.guest_mmu.root_pgd = 0; 5698 vcpu->arch.guest_mmu.translate_gpa = translate_gpa; 5699 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5700 vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; 5701 5702 vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa; 5703 5704 ret = alloc_mmu_pages(vcpu, &vcpu->arch.guest_mmu); 5705 if (ret) 5706 return ret; 5707 5708 ret = alloc_mmu_pages(vcpu, &vcpu->arch.root_mmu); 5709 if (ret) 5710 goto fail_allocate_root; 5711 5712 return ret; 5713 fail_allocate_root: 5714 free_mmu_pages(&vcpu->arch.guest_mmu); 5715 return ret; 5716 } 5717 5718 #define BATCH_ZAP_PAGES 10 5719 static void kvm_zap_obsolete_pages(struct kvm *kvm) 5720 { 5721 struct kvm_mmu_page *sp, *node; 5722 int nr_zapped, batch = 0; 5723 5724 restart: 5725 list_for_each_entry_safe_reverse(sp, node, 5726 &kvm->arch.active_mmu_pages, link) { 5727 /* 5728 * No obsolete valid page exists before a newly created page 5729 * since active_mmu_pages is a FIFO list. 5730 */ 5731 if (!is_obsolete_sp(kvm, sp)) 5732 break; 5733 5734 /* 5735 * Skip invalid pages with a non-zero root count, zapping pages 5736 * with a non-zero root count will never succeed, i.e. the page 5737 * will get thrown back on active_mmu_pages and we'll get stuck 5738 * in an infinite loop. 5739 */ 5740 if (sp->role.invalid && sp->root_count) 5741 continue; 5742 5743 /* 5744 * No need to flush the TLB since we're only zapping shadow 5745 * pages with an obsolete generation number and all vCPUS have 5746 * loaded a new root, i.e. the shadow pages being zapped cannot 5747 * be in active use by the guest. 5748 */ 5749 if (batch >= BATCH_ZAP_PAGES && 5750 cond_resched_lock(&kvm->mmu_lock)) { 5751 batch = 0; 5752 goto restart; 5753 } 5754 5755 if (__kvm_mmu_prepare_zap_page(kvm, sp, 5756 &kvm->arch.zapped_obsolete_pages, &nr_zapped)) { 5757 batch += nr_zapped; 5758 goto restart; 5759 } 5760 } 5761 5762 /* 5763 * Trigger a remote TLB flush before freeing the page tables to ensure 5764 * KVM is not in the middle of a lockless shadow page table walk, which 5765 * may reference the pages. 5766 */ 5767 kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages); 5768 } 5769 5770 /* 5771 * Fast invalidate all shadow pages and use lock-break technique 5772 * to zap obsolete pages. 5773 * 5774 * It's required when memslot is being deleted or VM is being 5775 * destroyed, in these cases, we should ensure that KVM MMU does 5776 * not use any resource of the being-deleted slot or all slots 5777 * after calling the function. 5778 */ 5779 static void kvm_mmu_zap_all_fast(struct kvm *kvm) 5780 { 5781 lockdep_assert_held(&kvm->slots_lock); 5782 5783 spin_lock(&kvm->mmu_lock); 5784 trace_kvm_mmu_zap_all_fast(kvm); 5785 5786 /* 5787 * Toggle mmu_valid_gen between '0' and '1'. Because slots_lock is 5788 * held for the entire duration of zapping obsolete pages, it's 5789 * impossible for there to be multiple invalid generations associated 5790 * with *valid* shadow pages at any given time, i.e. there is exactly 5791 * one valid generation and (at most) one invalid generation. 5792 */ 5793 kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1; 5794 5795 /* 5796 * Notify all vcpus to reload its shadow page table and flush TLB. 5797 * Then all vcpus will switch to new shadow page table with the new 5798 * mmu_valid_gen. 5799 * 5800 * Note: we need to do this under the protection of mmu_lock, 5801 * otherwise, vcpu would purge shadow page but miss tlb flush. 5802 */ 5803 kvm_reload_remote_mmus(kvm); 5804 5805 kvm_zap_obsolete_pages(kvm); 5806 spin_unlock(&kvm->mmu_lock); 5807 } 5808 5809 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) 5810 { 5811 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages)); 5812 } 5813 5814 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm, 5815 struct kvm_memory_slot *slot, 5816 struct kvm_page_track_notifier_node *node) 5817 { 5818 kvm_mmu_zap_all_fast(kvm); 5819 } 5820 5821 void kvm_mmu_init_vm(struct kvm *kvm) 5822 { 5823 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 5824 5825 node->track_write = kvm_mmu_pte_write; 5826 node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; 5827 kvm_page_track_register_notifier(kvm, node); 5828 } 5829 5830 void kvm_mmu_uninit_vm(struct kvm *kvm) 5831 { 5832 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 5833 5834 kvm_page_track_unregister_notifier(kvm, node); 5835 } 5836 5837 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) 5838 { 5839 struct kvm_memslots *slots; 5840 struct kvm_memory_slot *memslot; 5841 int i; 5842 5843 spin_lock(&kvm->mmu_lock); 5844 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 5845 slots = __kvm_memslots(kvm, i); 5846 kvm_for_each_memslot(memslot, slots) { 5847 gfn_t start, end; 5848 5849 start = max(gfn_start, memslot->base_gfn); 5850 end = min(gfn_end, memslot->base_gfn + memslot->npages); 5851 if (start >= end) 5852 continue; 5853 5854 slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, 5855 PG_LEVEL_4K, 5856 KVM_MAX_HUGEPAGE_LEVEL, 5857 start, end - 1, true); 5858 } 5859 } 5860 5861 spin_unlock(&kvm->mmu_lock); 5862 } 5863 5864 static bool slot_rmap_write_protect(struct kvm *kvm, 5865 struct kvm_rmap_head *rmap_head) 5866 { 5867 return __rmap_write_protect(kvm, rmap_head, false); 5868 } 5869 5870 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, 5871 struct kvm_memory_slot *memslot, 5872 int start_level) 5873 { 5874 bool flush; 5875 5876 spin_lock(&kvm->mmu_lock); 5877 flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, 5878 start_level, KVM_MAX_HUGEPAGE_LEVEL, false); 5879 spin_unlock(&kvm->mmu_lock); 5880 5881 /* 5882 * We can flush all the TLBs out of the mmu lock without TLB 5883 * corruption since we just change the spte from writable to 5884 * readonly so that we only need to care the case of changing 5885 * spte from present to present (changing the spte from present 5886 * to nonpresent will flush all the TLBs immediately), in other 5887 * words, the only case we care is mmu_spte_update() where we 5888 * have checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE 5889 * instead of PT_WRITABLE_MASK, that means it does not depend 5890 * on PT_WRITABLE_MASK anymore. 5891 */ 5892 if (flush) 5893 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5894 } 5895 5896 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, 5897 struct kvm_rmap_head *rmap_head) 5898 { 5899 u64 *sptep; 5900 struct rmap_iterator iter; 5901 int need_tlb_flush = 0; 5902 kvm_pfn_t pfn; 5903 struct kvm_mmu_page *sp; 5904 5905 restart: 5906 for_each_rmap_spte(rmap_head, &iter, sptep) { 5907 sp = page_header(__pa(sptep)); 5908 pfn = spte_to_pfn(*sptep); 5909 5910 /* 5911 * We cannot do huge page mapping for indirect shadow pages, 5912 * which are found on the last rmap (level = 1) when not using 5913 * tdp; such shadow pages are synced with the page table in 5914 * the guest, and the guest page table is using 4K page size 5915 * mapping if the indirect sp has level = 1. 5916 */ 5917 if (sp->role.direct && !kvm_is_reserved_pfn(pfn) && 5918 (kvm_is_zone_device_pfn(pfn) || 5919 PageCompound(pfn_to_page(pfn)))) { 5920 pte_list_remove(rmap_head, sptep); 5921 5922 if (kvm_available_flush_tlb_with_range()) 5923 kvm_flush_remote_tlbs_with_address(kvm, sp->gfn, 5924 KVM_PAGES_PER_HPAGE(sp->role.level)); 5925 else 5926 need_tlb_flush = 1; 5927 5928 goto restart; 5929 } 5930 } 5931 5932 return need_tlb_flush; 5933 } 5934 5935 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, 5936 const struct kvm_memory_slot *memslot) 5937 { 5938 /* FIXME: const-ify all uses of struct kvm_memory_slot. */ 5939 spin_lock(&kvm->mmu_lock); 5940 slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot, 5941 kvm_mmu_zap_collapsible_spte, true); 5942 spin_unlock(&kvm->mmu_lock); 5943 } 5944 5945 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 5946 struct kvm_memory_slot *memslot) 5947 { 5948 /* 5949 * All current use cases for flushing the TLBs for a specific memslot 5950 * are related to dirty logging, and do the TLB flush out of mmu_lock. 5951 * The interaction between the various operations on memslot must be 5952 * serialized by slots_locks to ensure the TLB flush from one operation 5953 * is observed by any other operation on the same memslot. 5954 */ 5955 lockdep_assert_held(&kvm->slots_lock); 5956 kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, 5957 memslot->npages); 5958 } 5959 5960 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 5961 struct kvm_memory_slot *memslot) 5962 { 5963 bool flush; 5964 5965 spin_lock(&kvm->mmu_lock); 5966 flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false); 5967 spin_unlock(&kvm->mmu_lock); 5968 5969 /* 5970 * It's also safe to flush TLBs out of mmu lock here as currently this 5971 * function is only used for dirty logging, in which case flushing TLB 5972 * out of mmu lock also guarantees no dirty pages will be lost in 5973 * dirty_bitmap. 5974 */ 5975 if (flush) 5976 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5977 } 5978 EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty); 5979 5980 void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, 5981 struct kvm_memory_slot *memslot) 5982 { 5983 bool flush; 5984 5985 spin_lock(&kvm->mmu_lock); 5986 flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect, 5987 false); 5988 spin_unlock(&kvm->mmu_lock); 5989 5990 if (flush) 5991 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 5992 } 5993 EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access); 5994 5995 void kvm_mmu_slot_set_dirty(struct kvm *kvm, 5996 struct kvm_memory_slot *memslot) 5997 { 5998 bool flush; 5999 6000 spin_lock(&kvm->mmu_lock); 6001 flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false); 6002 spin_unlock(&kvm->mmu_lock); 6003 6004 if (flush) 6005 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); 6006 } 6007 EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty); 6008 6009 void kvm_mmu_zap_all(struct kvm *kvm) 6010 { 6011 struct kvm_mmu_page *sp, *node; 6012 LIST_HEAD(invalid_list); 6013 int ign; 6014 6015 spin_lock(&kvm->mmu_lock); 6016 restart: 6017 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) { 6018 if (sp->role.invalid && sp->root_count) 6019 continue; 6020 if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign)) 6021 goto restart; 6022 if (cond_resched_lock(&kvm->mmu_lock)) 6023 goto restart; 6024 } 6025 6026 kvm_mmu_commit_zap_page(kvm, &invalid_list); 6027 spin_unlock(&kvm->mmu_lock); 6028 } 6029 6030 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) 6031 { 6032 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); 6033 6034 gen &= MMIO_SPTE_GEN_MASK; 6035 6036 /* 6037 * Generation numbers are incremented in multiples of the number of 6038 * address spaces in order to provide unique generations across all 6039 * address spaces. Strip what is effectively the address space 6040 * modifier prior to checking for a wrap of the MMIO generation so 6041 * that a wrap in any address space is detected. 6042 */ 6043 gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1); 6044 6045 /* 6046 * The very rare case: if the MMIO generation number has wrapped, 6047 * zap all shadow pages. 6048 */ 6049 if (unlikely(gen == 0)) { 6050 kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n"); 6051 kvm_mmu_zap_all_fast(kvm); 6052 } 6053 } 6054 6055 static unsigned long 6056 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 6057 { 6058 struct kvm *kvm; 6059 int nr_to_scan = sc->nr_to_scan; 6060 unsigned long freed = 0; 6061 6062 mutex_lock(&kvm_lock); 6063 6064 list_for_each_entry(kvm, &vm_list, vm_list) { 6065 int idx; 6066 LIST_HEAD(invalid_list); 6067 6068 /* 6069 * Never scan more than sc->nr_to_scan VM instances. 6070 * Will not hit this condition practically since we do not try 6071 * to shrink more than one VM and it is very unlikely to see 6072 * !n_used_mmu_pages so many times. 6073 */ 6074 if (!nr_to_scan--) 6075 break; 6076 /* 6077 * n_used_mmu_pages is accessed without holding kvm->mmu_lock 6078 * here. We may skip a VM instance errorneosly, but we do not 6079 * want to shrink a VM that only started to populate its MMU 6080 * anyway. 6081 */ 6082 if (!kvm->arch.n_used_mmu_pages && 6083 !kvm_has_zapped_obsolete_pages(kvm)) 6084 continue; 6085 6086 idx = srcu_read_lock(&kvm->srcu); 6087 spin_lock(&kvm->mmu_lock); 6088 6089 if (kvm_has_zapped_obsolete_pages(kvm)) { 6090 kvm_mmu_commit_zap_page(kvm, 6091 &kvm->arch.zapped_obsolete_pages); 6092 goto unlock; 6093 } 6094 6095 if (prepare_zap_oldest_mmu_page(kvm, &invalid_list)) 6096 freed++; 6097 kvm_mmu_commit_zap_page(kvm, &invalid_list); 6098 6099 unlock: 6100 spin_unlock(&kvm->mmu_lock); 6101 srcu_read_unlock(&kvm->srcu, idx); 6102 6103 /* 6104 * unfair on small ones 6105 * per-vm shrinkers cry out 6106 * sadness comes quickly 6107 */ 6108 list_move_tail(&kvm->vm_list, &vm_list); 6109 break; 6110 } 6111 6112 mutex_unlock(&kvm_lock); 6113 return freed; 6114 } 6115 6116 static unsigned long 6117 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 6118 { 6119 return percpu_counter_read_positive(&kvm_total_used_mmu_pages); 6120 } 6121 6122 static struct shrinker mmu_shrinker = { 6123 .count_objects = mmu_shrink_count, 6124 .scan_objects = mmu_shrink_scan, 6125 .seeks = DEFAULT_SEEKS * 10, 6126 }; 6127 6128 static void mmu_destroy_caches(void) 6129 { 6130 kmem_cache_destroy(pte_list_desc_cache); 6131 kmem_cache_destroy(mmu_page_header_cache); 6132 } 6133 6134 static void kvm_set_mmio_spte_mask(void) 6135 { 6136 u64 mask; 6137 6138 /* 6139 * Set a reserved PA bit in MMIO SPTEs to generate page faults with 6140 * PFEC.RSVD=1 on MMIO accesses. 64-bit PTEs (PAE, x86-64, and EPT 6141 * paging) support a maximum of 52 bits of PA, i.e. if the CPU supports 6142 * 52-bit physical addresses then there are no reserved PA bits in the 6143 * PTEs and so the reserved PA approach must be disabled. 6144 */ 6145 if (shadow_phys_bits < 52) 6146 mask = BIT_ULL(51) | PT_PRESENT_MASK; 6147 else 6148 mask = 0; 6149 6150 kvm_mmu_set_mmio_spte_mask(mask, ACC_WRITE_MASK | ACC_USER_MASK); 6151 } 6152 6153 static bool get_nx_auto_mode(void) 6154 { 6155 /* Return true when CPU has the bug, and mitigations are ON */ 6156 return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off(); 6157 } 6158 6159 static void __set_nx_huge_pages(bool val) 6160 { 6161 nx_huge_pages = itlb_multihit_kvm_mitigation = val; 6162 } 6163 6164 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp) 6165 { 6166 bool old_val = nx_huge_pages; 6167 bool new_val; 6168 6169 /* In "auto" mode deploy workaround only if CPU has the bug. */ 6170 if (sysfs_streq(val, "off")) 6171 new_val = 0; 6172 else if (sysfs_streq(val, "force")) 6173 new_val = 1; 6174 else if (sysfs_streq(val, "auto")) 6175 new_val = get_nx_auto_mode(); 6176 else if (strtobool(val, &new_val) < 0) 6177 return -EINVAL; 6178 6179 __set_nx_huge_pages(new_val); 6180 6181 if (new_val != old_val) { 6182 struct kvm *kvm; 6183 6184 mutex_lock(&kvm_lock); 6185 6186 list_for_each_entry(kvm, &vm_list, vm_list) { 6187 mutex_lock(&kvm->slots_lock); 6188 kvm_mmu_zap_all_fast(kvm); 6189 mutex_unlock(&kvm->slots_lock); 6190 6191 wake_up_process(kvm->arch.nx_lpage_recovery_thread); 6192 } 6193 mutex_unlock(&kvm_lock); 6194 } 6195 6196 return 0; 6197 } 6198 6199 int kvm_mmu_module_init(void) 6200 { 6201 int ret = -ENOMEM; 6202 6203 if (nx_huge_pages == -1) 6204 __set_nx_huge_pages(get_nx_auto_mode()); 6205 6206 /* 6207 * MMU roles use union aliasing which is, generally speaking, an 6208 * undefined behavior. However, we supposedly know how compilers behave 6209 * and the current status quo is unlikely to change. Guardians below are 6210 * supposed to let us know if the assumption becomes false. 6211 */ 6212 BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32)); 6213 BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32)); 6214 BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64)); 6215 6216 kvm_mmu_reset_all_pte_masks(); 6217 6218 kvm_set_mmio_spte_mask(); 6219 6220 pte_list_desc_cache = kmem_cache_create("pte_list_desc", 6221 sizeof(struct pte_list_desc), 6222 0, SLAB_ACCOUNT, NULL); 6223 if (!pte_list_desc_cache) 6224 goto out; 6225 6226 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header", 6227 sizeof(struct kvm_mmu_page), 6228 0, SLAB_ACCOUNT, NULL); 6229 if (!mmu_page_header_cache) 6230 goto out; 6231 6232 if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL)) 6233 goto out; 6234 6235 ret = register_shrinker(&mmu_shrinker); 6236 if (ret) 6237 goto out; 6238 6239 return 0; 6240 6241 out: 6242 mmu_destroy_caches(); 6243 return ret; 6244 } 6245 6246 /* 6247 * Calculate mmu pages needed for kvm. 6248 */ 6249 unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm) 6250 { 6251 unsigned long nr_mmu_pages; 6252 unsigned long nr_pages = 0; 6253 struct kvm_memslots *slots; 6254 struct kvm_memory_slot *memslot; 6255 int i; 6256 6257 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 6258 slots = __kvm_memslots(kvm, i); 6259 6260 kvm_for_each_memslot(memslot, slots) 6261 nr_pages += memslot->npages; 6262 } 6263 6264 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; 6265 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); 6266 6267 return nr_mmu_pages; 6268 } 6269 6270 void kvm_mmu_destroy(struct kvm_vcpu *vcpu) 6271 { 6272 kvm_mmu_unload(vcpu); 6273 free_mmu_pages(&vcpu->arch.root_mmu); 6274 free_mmu_pages(&vcpu->arch.guest_mmu); 6275 mmu_free_memory_caches(vcpu); 6276 } 6277 6278 void kvm_mmu_module_exit(void) 6279 { 6280 mmu_destroy_caches(); 6281 percpu_counter_destroy(&kvm_total_used_mmu_pages); 6282 unregister_shrinker(&mmu_shrinker); 6283 mmu_audit_disable(); 6284 } 6285 6286 static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp) 6287 { 6288 unsigned int old_val; 6289 int err; 6290 6291 old_val = nx_huge_pages_recovery_ratio; 6292 err = param_set_uint(val, kp); 6293 if (err) 6294 return err; 6295 6296 if (READ_ONCE(nx_huge_pages) && 6297 !old_val && nx_huge_pages_recovery_ratio) { 6298 struct kvm *kvm; 6299 6300 mutex_lock(&kvm_lock); 6301 6302 list_for_each_entry(kvm, &vm_list, vm_list) 6303 wake_up_process(kvm->arch.nx_lpage_recovery_thread); 6304 6305 mutex_unlock(&kvm_lock); 6306 } 6307 6308 return err; 6309 } 6310 6311 static void kvm_recover_nx_lpages(struct kvm *kvm) 6312 { 6313 int rcu_idx; 6314 struct kvm_mmu_page *sp; 6315 unsigned int ratio; 6316 LIST_HEAD(invalid_list); 6317 ulong to_zap; 6318 6319 rcu_idx = srcu_read_lock(&kvm->srcu); 6320 spin_lock(&kvm->mmu_lock); 6321 6322 ratio = READ_ONCE(nx_huge_pages_recovery_ratio); 6323 to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0; 6324 while (to_zap && !list_empty(&kvm->arch.lpage_disallowed_mmu_pages)) { 6325 /* 6326 * We use a separate list instead of just using active_mmu_pages 6327 * because the number of lpage_disallowed pages is expected to 6328 * be relatively small compared to the total. 6329 */ 6330 sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages, 6331 struct kvm_mmu_page, 6332 lpage_disallowed_link); 6333 WARN_ON_ONCE(!sp->lpage_disallowed); 6334 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); 6335 WARN_ON_ONCE(sp->lpage_disallowed); 6336 6337 if (!--to_zap || need_resched() || spin_needbreak(&kvm->mmu_lock)) { 6338 kvm_mmu_commit_zap_page(kvm, &invalid_list); 6339 if (to_zap) 6340 cond_resched_lock(&kvm->mmu_lock); 6341 } 6342 } 6343 6344 spin_unlock(&kvm->mmu_lock); 6345 srcu_read_unlock(&kvm->srcu, rcu_idx); 6346 } 6347 6348 static long get_nx_lpage_recovery_timeout(u64 start_time) 6349 { 6350 return READ_ONCE(nx_huge_pages) && READ_ONCE(nx_huge_pages_recovery_ratio) 6351 ? start_time + 60 * HZ - get_jiffies_64() 6352 : MAX_SCHEDULE_TIMEOUT; 6353 } 6354 6355 static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data) 6356 { 6357 u64 start_time; 6358 long remaining_time; 6359 6360 while (true) { 6361 start_time = get_jiffies_64(); 6362 remaining_time = get_nx_lpage_recovery_timeout(start_time); 6363 6364 set_current_state(TASK_INTERRUPTIBLE); 6365 while (!kthread_should_stop() && remaining_time > 0) { 6366 schedule_timeout(remaining_time); 6367 remaining_time = get_nx_lpage_recovery_timeout(start_time); 6368 set_current_state(TASK_INTERRUPTIBLE); 6369 } 6370 6371 set_current_state(TASK_RUNNING); 6372 6373 if (kthread_should_stop()) 6374 return 0; 6375 6376 kvm_recover_nx_lpages(kvm); 6377 } 6378 } 6379 6380 int kvm_mmu_post_init_vm(struct kvm *kvm) 6381 { 6382 int err; 6383 6384 err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0, 6385 "kvm-nx-lpage-recovery", 6386 &kvm->arch.nx_lpage_recovery_thread); 6387 if (!err) 6388 kthread_unpark(kvm->arch.nx_lpage_recovery_thread); 6389 6390 return err; 6391 } 6392 6393 void kvm_mmu_pre_destroy_vm(struct kvm *kvm) 6394 { 6395 if (kvm->arch.nx_lpage_recovery_thread) 6396 kthread_stop(kvm->arch.nx_lpage_recovery_thread); 6397 } 6398