15a9624afSPaolo Bonzini // SPDX-License-Identifier: GPL-2.0-only 25a9624afSPaolo Bonzini /* 35a9624afSPaolo Bonzini * Kernel-based Virtual Machine driver for Linux 45a9624afSPaolo Bonzini * 55a9624afSPaolo Bonzini * Macros and functions to access KVM PTEs (also known as SPTEs) 65a9624afSPaolo Bonzini * 75a9624afSPaolo Bonzini * Copyright (C) 2006 Qumranet, Inc. 85a9624afSPaolo Bonzini * Copyright 2020 Red Hat, Inc. and/or its affiliates. 95a9624afSPaolo Bonzini */ 105a9624afSPaolo Bonzini 115a9624afSPaolo Bonzini 125a9624afSPaolo Bonzini #include <linux/kvm_host.h> 135a9624afSPaolo Bonzini #include "mmu.h" 145a9624afSPaolo Bonzini #include "mmu_internal.h" 155a9624afSPaolo Bonzini #include "x86.h" 165a9624afSPaolo Bonzini #include "spte.h" 175a9624afSPaolo Bonzini 185a9624afSPaolo Bonzini #include <asm/e820/api.h> 194d5cff69SChristoph Hellwig #include <asm/memtype.h> 20e7b7bdeaSSean Christopherson #include <asm/vmx.h> 215a9624afSPaolo Bonzini 228b9e74bfSSean Christopherson bool __read_mostly enable_mmio_caching = true; 23b09763daSSean Christopherson module_param_named(mmio_caching, enable_mmio_caching, bool, 0444); 24b09763daSSean Christopherson 255fc3424fSSean Christopherson u64 __read_mostly shadow_host_writable_mask; 265fc3424fSSean Christopherson u64 __read_mostly shadow_mmu_writable_mask; 275a9624afSPaolo Bonzini u64 __read_mostly shadow_nx_mask; 285a9624afSPaolo Bonzini u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ 295a9624afSPaolo Bonzini u64 __read_mostly shadow_user_mask; 305a9624afSPaolo Bonzini u64 __read_mostly shadow_accessed_mask; 315a9624afSPaolo Bonzini u64 __read_mostly shadow_dirty_mask; 325a9624afSPaolo Bonzini u64 __read_mostly shadow_mmio_value; 338120337aSSean Christopherson u64 __read_mostly shadow_mmio_mask; 345a9624afSPaolo Bonzini u64 __read_mostly shadow_mmio_access_mask; 355a9624afSPaolo Bonzini u64 __read_mostly shadow_present_mask; 36e54f1ff2SKai Huang u64 __read_mostly shadow_me_value; 375a9624afSPaolo Bonzini u64 __read_mostly shadow_me_mask; 385a9624afSPaolo Bonzini u64 __read_mostly shadow_acc_track_mask; 395a9624afSPaolo Bonzini 405a9624afSPaolo Bonzini u64 __read_mostly shadow_nonpresent_or_rsvd_mask; 415a9624afSPaolo Bonzini u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask; 425a9624afSPaolo Bonzini 435a9624afSPaolo Bonzini u8 __read_mostly shadow_phys_bits; 445a9624afSPaolo Bonzini 455a9624afSPaolo Bonzini static u64 generation_mmio_spte_mask(u64 gen) 465a9624afSPaolo Bonzini { 475a9624afSPaolo Bonzini u64 mask; 485a9624afSPaolo Bonzini 495a9624afSPaolo Bonzini WARN_ON(gen & ~MMIO_SPTE_GEN_MASK); 505a9624afSPaolo Bonzini 5134c0f6f2SMaciej S. Szmigiero mask = (gen << MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_SPTE_GEN_LOW_MASK; 5234c0f6f2SMaciej S. Szmigiero mask |= (gen << MMIO_SPTE_GEN_HIGH_SHIFT) & MMIO_SPTE_GEN_HIGH_MASK; 535a9624afSPaolo Bonzini return mask; 545a9624afSPaolo Bonzini } 555a9624afSPaolo Bonzini 565a9624afSPaolo Bonzini u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access) 575a9624afSPaolo Bonzini { 585a9624afSPaolo Bonzini u64 gen = kvm_vcpu_memslots(vcpu)->generation & MMIO_SPTE_GEN_MASK; 59c236d962SSean Christopherson u64 spte = generation_mmio_spte_mask(gen); 605a9624afSPaolo Bonzini u64 gpa = gfn << PAGE_SHIFT; 615a9624afSPaolo Bonzini 6230ab5901SSean Christopherson WARN_ON_ONCE(!shadow_mmio_value); 6330ab5901SSean Christopherson 645a9624afSPaolo Bonzini access &= shadow_mmio_access_mask; 65c236d962SSean Christopherson spte |= shadow_mmio_value | access; 66c236d962SSean Christopherson spte |= gpa | shadow_nonpresent_or_rsvd_mask; 67c236d962SSean Christopherson spte |= (gpa & shadow_nonpresent_or_rsvd_mask) 688a967d65SPaolo Bonzini << SHADOW_NONPRESENT_OR_RSVD_MASK_LEN; 695a9624afSPaolo Bonzini 70c236d962SSean Christopherson return spte; 715a9624afSPaolo Bonzini } 725a9624afSPaolo Bonzini 735a9624afSPaolo Bonzini static bool kvm_is_mmio_pfn(kvm_pfn_t pfn) 745a9624afSPaolo Bonzini { 755a9624afSPaolo Bonzini if (pfn_valid(pfn)) 765a9624afSPaolo Bonzini return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn)) && 775a9624afSPaolo Bonzini /* 785a9624afSPaolo Bonzini * Some reserved pages, such as those from NVDIMM 795a9624afSPaolo Bonzini * DAX devices, are not for MMIO, and can be mapped 805a9624afSPaolo Bonzini * with cached memory type for better performance. 815a9624afSPaolo Bonzini * However, the above check misconceives those pages 825a9624afSPaolo Bonzini * as MMIO, and results in KVM mapping them with UC 835a9624afSPaolo Bonzini * memory type, which would hurt the performance. 845a9624afSPaolo Bonzini * Therefore, we check the host memory type in addition 855a9624afSPaolo Bonzini * and only treat UC/UC-/WC pages as MMIO. 865a9624afSPaolo Bonzini */ 875a9624afSPaolo Bonzini (!pat_enabled() || pat_pfn_immune_to_uc_mtrr(pfn)); 885a9624afSPaolo Bonzini 895a9624afSPaolo Bonzini return !e820__mapped_raw_any(pfn_to_hpa(pfn), 905a9624afSPaolo Bonzini pfn_to_hpa(pfn + 1) - 1, 915a9624afSPaolo Bonzini E820_TYPE_RAM); 925a9624afSPaolo Bonzini } 935a9624afSPaolo Bonzini 9454eb3ef5SSean Christopherson /* 9554eb3ef5SSean Christopherson * Returns true if the SPTE has bits that may be set without holding mmu_lock. 9654eb3ef5SSean Christopherson * The caller is responsible for checking if the SPTE is shadow-present, and 9754eb3ef5SSean Christopherson * for determining whether or not the caller cares about non-leaf SPTEs. 9854eb3ef5SSean Christopherson */ 9954eb3ef5SSean Christopherson bool spte_has_volatile_bits(u64 spte) 10054eb3ef5SSean Christopherson { 10154eb3ef5SSean Christopherson /* 10254eb3ef5SSean Christopherson * Always atomically update spte if it can be updated 10354eb3ef5SSean Christopherson * out of mmu-lock, it can ensure dirty bit is not lost, 10454eb3ef5SSean Christopherson * also, it can help us to get a stable is_writable_pte() 10554eb3ef5SSean Christopherson * to ensure tlb flush is not missed. 10654eb3ef5SSean Christopherson */ 10754eb3ef5SSean Christopherson if (!is_writable_pte(spte) && is_mmu_writable_spte(spte)) 10854eb3ef5SSean Christopherson return true; 10954eb3ef5SSean Christopherson 11054eb3ef5SSean Christopherson if (is_access_track_spte(spte)) 11154eb3ef5SSean Christopherson return true; 11254eb3ef5SSean Christopherson 11354eb3ef5SSean Christopherson if (spte_ad_enabled(spte)) { 11454eb3ef5SSean Christopherson if (!(spte & shadow_accessed_mask) || 11554eb3ef5SSean Christopherson (is_writable_pte(spte) && !(spte & shadow_dirty_mask))) 11654eb3ef5SSean Christopherson return true; 11754eb3ef5SSean Christopherson } 11854eb3ef5SSean Christopherson 11954eb3ef5SSean Christopherson return false; 12054eb3ef5SSean Christopherson } 12154eb3ef5SSean Christopherson 1227158bee4SPaolo Bonzini bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 1238283e36aSBen Gardon const struct kvm_memory_slot *slot, 1247158bee4SPaolo Bonzini unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn, 1252839180cSPaolo Bonzini u64 old_spte, bool prefetch, bool can_unsync, 1267158bee4SPaolo Bonzini bool host_writable, u64 *new_spte) 1275a9624afSPaolo Bonzini { 1287158bee4SPaolo Bonzini int level = sp->role.level; 129edea7c4fSSean Christopherson u64 spte = SPTE_MMU_PRESENT_MASK; 130ad67e480SPaolo Bonzini bool wrprot = false; 1315a9624afSPaolo Bonzini 1329fb35657SSean Christopherson WARN_ON_ONCE(!pte_access && !shadow_present_mask); 1339fb35657SSean Christopherson 1347158bee4SPaolo Bonzini if (sp->role.ad_disabled) 1358a406c89SSean Christopherson spte |= SPTE_TDP_AD_DISABLED_MASK; 136ce92ef76SSean Christopherson else if (kvm_mmu_page_ad_need_write_protect(sp)) 1378a406c89SSean Christopherson spte |= SPTE_TDP_AD_WRPROT_ONLY_MASK; 1388a406c89SSean Christopherson 1398a406c89SSean Christopherson /* 1405a9624afSPaolo Bonzini * For the EPT case, shadow_present_mask is 0 if hardware 1415a9624afSPaolo Bonzini * supports exec-only page table entries. In that case, 1425a9624afSPaolo Bonzini * ACC_USER_MASK and shadow_user_mask are used to represent 1435a9624afSPaolo Bonzini * read access. See FNAME(gpte_access) in paging_tmpl.h. 1445a9624afSPaolo Bonzini */ 1455a9624afSPaolo Bonzini spte |= shadow_present_mask; 1462839180cSPaolo Bonzini if (!prefetch) 1475a9624afSPaolo Bonzini spte |= spte_shadow_accessed_mask(spte); 1485a9624afSPaolo Bonzini 1495a9624afSPaolo Bonzini if (level > PG_LEVEL_4K && (pte_access & ACC_EXEC_MASK) && 150*084cc29fSBen Gardon is_nx_huge_page_enabled(vcpu->kvm)) { 1515a9624afSPaolo Bonzini pte_access &= ~ACC_EXEC_MASK; 1525a9624afSPaolo Bonzini } 1535a9624afSPaolo Bonzini 1545a9624afSPaolo Bonzini if (pte_access & ACC_EXEC_MASK) 1555a9624afSPaolo Bonzini spte |= shadow_x_mask; 1565a9624afSPaolo Bonzini else 1575a9624afSPaolo Bonzini spte |= shadow_nx_mask; 1585a9624afSPaolo Bonzini 1595a9624afSPaolo Bonzini if (pte_access & ACC_USER_MASK) 1605a9624afSPaolo Bonzini spte |= shadow_user_mask; 1615a9624afSPaolo Bonzini 1625a9624afSPaolo Bonzini if (level > PG_LEVEL_4K) 1635a9624afSPaolo Bonzini spte |= PT_PAGE_SIZE_MASK; 1645a9624afSPaolo Bonzini if (tdp_enabled) 165b3646477SJason Baron spte |= static_call(kvm_x86_get_mt_mask)(vcpu, gfn, 1665a9624afSPaolo Bonzini kvm_is_mmio_pfn(pfn)); 1675a9624afSPaolo Bonzini 1685a9624afSPaolo Bonzini if (host_writable) 1695fc3424fSSean Christopherson spte |= shadow_host_writable_mask; 1705a9624afSPaolo Bonzini else 1715a9624afSPaolo Bonzini pte_access &= ~ACC_WRITE_MASK; 1725a9624afSPaolo Bonzini 173e54f1ff2SKai Huang if (shadow_me_value && !kvm_is_mmio_pfn(pfn)) 174e54f1ff2SKai Huang spte |= shadow_me_value; 1755a9624afSPaolo Bonzini 1765a9624afSPaolo Bonzini spte |= (u64)pfn << PAGE_SHIFT; 1775a9624afSPaolo Bonzini 1785a9624afSPaolo Bonzini if (pte_access & ACC_WRITE_MASK) { 1795fc3424fSSean Christopherson spte |= PT_WRITABLE_MASK | shadow_mmu_writable_mask; 1805a9624afSPaolo Bonzini 1815a9624afSPaolo Bonzini /* 1825a9624afSPaolo Bonzini * Optimization: for pte sync, if spte was writable the hash 1835a9624afSPaolo Bonzini * lookup is unnecessary (and expensive). Write protection 1840337f585SSean Christopherson * is responsibility of kvm_mmu_get_page / kvm_mmu_sync_roots. 1855a9624afSPaolo Bonzini * Same reasoning can be applied to dirty page accounting. 1865a9624afSPaolo Bonzini */ 1878b8f9d75SLai Jiangshan if (is_writable_pte(old_spte)) 1885a9624afSPaolo Bonzini goto out; 1895a9624afSPaolo Bonzini 1900337f585SSean Christopherson /* 1910337f585SSean Christopherson * Unsync shadow pages that are reachable by the new, writable 1920337f585SSean Christopherson * SPTE. Write-protect the SPTE if the page can't be unsync'd, 1930337f585SSean Christopherson * e.g. it's write-tracked (upper-level SPs) or has one or more 1940337f585SSean Christopherson * shadow pages and unsync'ing pages is not allowed. 1950337f585SSean Christopherson */ 1964d78d0b3SBen Gardon if (mmu_try_to_unsync_pages(vcpu->kvm, slot, gfn, can_unsync, prefetch)) { 1975a9624afSPaolo Bonzini pgprintk("%s: found shadow page for %llx, marking ro\n", 1985a9624afSPaolo Bonzini __func__, gfn); 199ad67e480SPaolo Bonzini wrprot = true; 2005a9624afSPaolo Bonzini pte_access &= ~ACC_WRITE_MASK; 2015fc3424fSSean Christopherson spte &= ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask); 2025a9624afSPaolo Bonzini } 2035a9624afSPaolo Bonzini } 2045a9624afSPaolo Bonzini 2055a9624afSPaolo Bonzini if (pte_access & ACC_WRITE_MASK) 2065a9624afSPaolo Bonzini spte |= spte_shadow_dirty_mask(spte); 2075a9624afSPaolo Bonzini 2088b8f9d75SLai Jiangshan out: 2092839180cSPaolo Bonzini if (prefetch) 2105a9624afSPaolo Bonzini spte = mark_spte_for_access_track(spte); 2115a9624afSPaolo Bonzini 2123b77daa5SSean Christopherson WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level), 2133b77daa5SSean Christopherson "spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level, 2143b77daa5SSean Christopherson get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level)); 2153b77daa5SSean Christopherson 21653597858SDavid Matlack if ((spte & PT_WRITABLE_MASK) && kvm_slot_dirty_track_enabled(slot)) { 21753597858SDavid Matlack /* Enforced by kvm_mmu_hugepage_adjust. */ 21853597858SDavid Matlack WARN_ON(level > PG_LEVEL_4K); 21953597858SDavid Matlack mark_page_dirty_in_slot(vcpu->kvm, slot, gfn); 22053597858SDavid Matlack } 221bcc4f2bcSPaolo Bonzini 2225a9624afSPaolo Bonzini *new_spte = spte; 223ad67e480SPaolo Bonzini return wrprot; 2245a9624afSPaolo Bonzini } 2255a9624afSPaolo Bonzini 226a3fe5dbdSDavid Matlack static u64 make_spte_executable(u64 spte) 227a3fe5dbdSDavid Matlack { 228a3fe5dbdSDavid Matlack bool is_access_track = is_access_track_spte(spte); 229a3fe5dbdSDavid Matlack 230a3fe5dbdSDavid Matlack if (is_access_track) 231a3fe5dbdSDavid Matlack spte = restore_acc_track_spte(spte); 232a3fe5dbdSDavid Matlack 233a3fe5dbdSDavid Matlack spte &= ~shadow_nx_mask; 234a3fe5dbdSDavid Matlack spte |= shadow_x_mask; 235a3fe5dbdSDavid Matlack 236a3fe5dbdSDavid Matlack if (is_access_track) 237a3fe5dbdSDavid Matlack spte = mark_spte_for_access_track(spte); 238a3fe5dbdSDavid Matlack 239a3fe5dbdSDavid Matlack return spte; 240a3fe5dbdSDavid Matlack } 241a3fe5dbdSDavid Matlack 242a3fe5dbdSDavid Matlack /* 243a3fe5dbdSDavid Matlack * Construct an SPTE that maps a sub-page of the given huge page SPTE where 244a3fe5dbdSDavid Matlack * `index` identifies which sub-page. 245a3fe5dbdSDavid Matlack * 246a3fe5dbdSDavid Matlack * This is used during huge page splitting to build the SPTEs that make up the 247a3fe5dbdSDavid Matlack * new page table. 248a3fe5dbdSDavid Matlack */ 249*084cc29fSBen Gardon u64 make_huge_page_split_spte(struct kvm *kvm, u64 huge_spte, int huge_level, 250*084cc29fSBen Gardon int index) 251a3fe5dbdSDavid Matlack { 252a3fe5dbdSDavid Matlack u64 child_spte; 253a3fe5dbdSDavid Matlack int child_level; 254a3fe5dbdSDavid Matlack 255a3fe5dbdSDavid Matlack if (WARN_ON_ONCE(!is_shadow_present_pte(huge_spte))) 256a3fe5dbdSDavid Matlack return 0; 257a3fe5dbdSDavid Matlack 258a3fe5dbdSDavid Matlack if (WARN_ON_ONCE(!is_large_pte(huge_spte))) 259a3fe5dbdSDavid Matlack return 0; 260a3fe5dbdSDavid Matlack 261a3fe5dbdSDavid Matlack child_spte = huge_spte; 262a3fe5dbdSDavid Matlack child_level = huge_level - 1; 263a3fe5dbdSDavid Matlack 264a3fe5dbdSDavid Matlack /* 265a3fe5dbdSDavid Matlack * The child_spte already has the base address of the huge page being 266a3fe5dbdSDavid Matlack * split. So we just have to OR in the offset to the page at the next 267a3fe5dbdSDavid Matlack * lower level for the given index. 268a3fe5dbdSDavid Matlack */ 269a3fe5dbdSDavid Matlack child_spte |= (index * KVM_PAGES_PER_HPAGE(child_level)) << PAGE_SHIFT; 270a3fe5dbdSDavid Matlack 271a3fe5dbdSDavid Matlack if (child_level == PG_LEVEL_4K) { 272a3fe5dbdSDavid Matlack child_spte &= ~PT_PAGE_SIZE_MASK; 273a3fe5dbdSDavid Matlack 274a3fe5dbdSDavid Matlack /* 275a3fe5dbdSDavid Matlack * When splitting to a 4K page, mark the page executable as the 276a3fe5dbdSDavid Matlack * NX hugepage mitigation no longer applies. 277a3fe5dbdSDavid Matlack */ 278*084cc29fSBen Gardon if (is_nx_huge_page_enabled(kvm)) 279a3fe5dbdSDavid Matlack child_spte = make_spte_executable(child_spte); 280a3fe5dbdSDavid Matlack } 281a3fe5dbdSDavid Matlack 282a3fe5dbdSDavid Matlack return child_spte; 283a3fe5dbdSDavid Matlack } 284a3fe5dbdSDavid Matlack 285a3fe5dbdSDavid Matlack 2865a9624afSPaolo Bonzini u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled) 2875a9624afSPaolo Bonzini { 288edea7c4fSSean Christopherson u64 spte = SPTE_MMU_PRESENT_MASK; 2895a9624afSPaolo Bonzini 290edea7c4fSSean Christopherson spte |= __pa(child_pt) | shadow_present_mask | PT_WRITABLE_MASK | 291e54f1ff2SKai Huang shadow_user_mask | shadow_x_mask | shadow_me_value; 2925a9624afSPaolo Bonzini 2935a9624afSPaolo Bonzini if (ad_disabled) 2948a406c89SSean Christopherson spte |= SPTE_TDP_AD_DISABLED_MASK; 2955a9624afSPaolo Bonzini else 2965a9624afSPaolo Bonzini spte |= shadow_accessed_mask; 2975a9624afSPaolo Bonzini 2985a9624afSPaolo Bonzini return spte; 2995a9624afSPaolo Bonzini } 3005a9624afSPaolo Bonzini 3015a9624afSPaolo Bonzini u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn) 3025a9624afSPaolo Bonzini { 3035a9624afSPaolo Bonzini u64 new_spte; 3045a9624afSPaolo Bonzini 3052ca3129eSSean Christopherson new_spte = old_spte & ~SPTE_BASE_ADDR_MASK; 3065a9624afSPaolo Bonzini new_spte |= (u64)new_pfn << PAGE_SHIFT; 3075a9624afSPaolo Bonzini 3085a9624afSPaolo Bonzini new_spte &= ~PT_WRITABLE_MASK; 3095fc3424fSSean Christopherson new_spte &= ~shadow_host_writable_mask; 310f082d86eSDavid Matlack new_spte &= ~shadow_mmu_writable_mask; 3115a9624afSPaolo Bonzini 3125a9624afSPaolo Bonzini new_spte = mark_spte_for_access_track(new_spte); 3135a9624afSPaolo Bonzini 3145a9624afSPaolo Bonzini return new_spte; 3155a9624afSPaolo Bonzini } 3165a9624afSPaolo Bonzini 3175a9624afSPaolo Bonzini u64 mark_spte_for_access_track(u64 spte) 3185a9624afSPaolo Bonzini { 3195a9624afSPaolo Bonzini if (spte_ad_enabled(spte)) 3205a9624afSPaolo Bonzini return spte & ~shadow_accessed_mask; 3215a9624afSPaolo Bonzini 3225a9624afSPaolo Bonzini if (is_access_track_spte(spte)) 3235a9624afSPaolo Bonzini return spte; 3245a9624afSPaolo Bonzini 325115111efSDavid Matlack check_spte_writable_invariants(spte); 3265a9624afSPaolo Bonzini 3278a967d65SPaolo Bonzini WARN_ONCE(spte & (SHADOW_ACC_TRACK_SAVED_BITS_MASK << 3288a967d65SPaolo Bonzini SHADOW_ACC_TRACK_SAVED_BITS_SHIFT), 3295a9624afSPaolo Bonzini "kvm: Access Tracking saved bit locations are not zero\n"); 3305a9624afSPaolo Bonzini 3318a967d65SPaolo Bonzini spte |= (spte & SHADOW_ACC_TRACK_SAVED_BITS_MASK) << 3328a967d65SPaolo Bonzini SHADOW_ACC_TRACK_SAVED_BITS_SHIFT; 3335a9624afSPaolo Bonzini spte &= ~shadow_acc_track_mask; 3345a9624afSPaolo Bonzini 3355a9624afSPaolo Bonzini return spte; 3365a9624afSPaolo Bonzini } 3375a9624afSPaolo Bonzini 3388120337aSSean Christopherson void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask) 3395a9624afSPaolo Bonzini { 3405a9624afSPaolo Bonzini BUG_ON((u64)(unsigned)access_mask != access_mask); 3415a9624afSPaolo Bonzini WARN_ON(mmio_value & shadow_nonpresent_or_rsvd_lower_gfn_mask); 34244aaa015SSean Christopherson 343b09763daSSean Christopherson if (!enable_mmio_caching) 344b09763daSSean Christopherson mmio_value = 0; 345b09763daSSean Christopherson 34644aaa015SSean Christopherson /* 34744aaa015SSean Christopherson * Disable MMIO caching if the MMIO value collides with the bits that 34844aaa015SSean Christopherson * are used to hold the relocated GFN when the L1TF mitigation is 34944aaa015SSean Christopherson * enabled. This should never fire as there is no known hardware that 35044aaa015SSean Christopherson * can trigger this condition, e.g. SME/SEV CPUs that require a custom 35144aaa015SSean Christopherson * MMIO value are not susceptible to L1TF. 35244aaa015SSean Christopherson */ 35344aaa015SSean Christopherson if (WARN_ON(mmio_value & (shadow_nonpresent_or_rsvd_mask << 35444aaa015SSean Christopherson SHADOW_NONPRESENT_OR_RSVD_MASK_LEN))) 35544aaa015SSean Christopherson mmio_value = 0; 35644aaa015SSean Christopherson 357715f1079SSean Christopherson /* 358715f1079SSean Christopherson * The masked MMIO value must obviously match itself and a removed SPTE 359715f1079SSean Christopherson * must not get a false positive. Removed SPTEs and MMIO SPTEs should 360715f1079SSean Christopherson * never collide as MMIO must set some RWX bits, and removed SPTEs must 361715f1079SSean Christopherson * not set any RWX bits. 362715f1079SSean Christopherson */ 363715f1079SSean Christopherson if (WARN_ON((mmio_value & mmio_mask) != mmio_value) || 364715f1079SSean Christopherson WARN_ON(mmio_value && (REMOVED_SPTE & mmio_mask) == mmio_value)) 365715f1079SSean Christopherson mmio_value = 0; 366715f1079SSean Christopherson 3678b9e74bfSSean Christopherson if (!mmio_value) 3688b9e74bfSSean Christopherson enable_mmio_caching = false; 3698b9e74bfSSean Christopherson 3708120337aSSean Christopherson shadow_mmio_value = mmio_value; 3718120337aSSean Christopherson shadow_mmio_mask = mmio_mask; 3725a9624afSPaolo Bonzini shadow_mmio_access_mask = access_mask; 3735a9624afSPaolo Bonzini } 3745a9624afSPaolo Bonzini EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask); 3755a9624afSPaolo Bonzini 376e54f1ff2SKai Huang void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask) 377e54f1ff2SKai Huang { 378e54f1ff2SKai Huang /* shadow_me_value must be a subset of shadow_me_mask */ 379e54f1ff2SKai Huang if (WARN_ON(me_value & ~me_mask)) 380e54f1ff2SKai Huang me_value = me_mask = 0; 381e54f1ff2SKai Huang 382e54f1ff2SKai Huang shadow_me_value = me_value; 383e54f1ff2SKai Huang shadow_me_mask = me_mask; 384e54f1ff2SKai Huang } 385e54f1ff2SKai Huang EXPORT_SYMBOL_GPL(kvm_mmu_set_me_spte_mask); 386e54f1ff2SKai Huang 387e7b7bdeaSSean Christopherson void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only) 3885a9624afSPaolo Bonzini { 389e7b7bdeaSSean Christopherson shadow_user_mask = VMX_EPT_READABLE_MASK; 390e7b7bdeaSSean Christopherson shadow_accessed_mask = has_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull; 391e7b7bdeaSSean Christopherson shadow_dirty_mask = has_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull; 392e7b7bdeaSSean Christopherson shadow_nx_mask = 0ull; 393e7b7bdeaSSean Christopherson shadow_x_mask = VMX_EPT_EXECUTABLE_MASK; 394e7b7bdeaSSean Christopherson shadow_present_mask = has_exec_only ? 0ull : VMX_EPT_READABLE_MASK; 395e7b7bdeaSSean Christopherson shadow_acc_track_mask = VMX_EPT_RWX_MASK; 396613a3f37SSean Christopherson shadow_host_writable_mask = EPT_SPTE_HOST_WRITABLE; 397613a3f37SSean Christopherson shadow_mmu_writable_mask = EPT_SPTE_MMU_WRITABLE; 398613a3f37SSean Christopherson 399e7b7bdeaSSean Christopherson /* 400e7b7bdeaSSean Christopherson * EPT Misconfigurations are generated if the value of bits 2:0 401e7b7bdeaSSean Christopherson * of an EPT paging-structure entry is 110b (write/execute). 402e7b7bdeaSSean Christopherson */ 403e7b7bdeaSSean Christopherson kvm_mmu_set_mmio_spte_mask(VMX_EPT_MISCONFIG_WX_VALUE, 404e7b7bdeaSSean Christopherson VMX_EPT_RWX_MASK, 0); 4055a9624afSPaolo Bonzini } 406e7b7bdeaSSean Christopherson EXPORT_SYMBOL_GPL(kvm_mmu_set_ept_masks); 4075a9624afSPaolo Bonzini 4085a9624afSPaolo Bonzini void kvm_mmu_reset_all_pte_masks(void) 4095a9624afSPaolo Bonzini { 4105a9624afSPaolo Bonzini u8 low_phys_bits; 411d6b87f25SSean Christopherson u64 mask; 4125a9624afSPaolo Bonzini 4135a9624afSPaolo Bonzini shadow_phys_bits = kvm_get_shadow_phys_bits(); 4145a9624afSPaolo Bonzini 4155a9624afSPaolo Bonzini /* 4165a9624afSPaolo Bonzini * If the CPU has 46 or less physical address bits, then set an 4175a9624afSPaolo Bonzini * appropriate mask to guard against L1TF attacks. Otherwise, it is 4185a9624afSPaolo Bonzini * assumed that the CPU is not vulnerable to L1TF. 4195a9624afSPaolo Bonzini * 4205a9624afSPaolo Bonzini * Some Intel CPUs address the L1 cache using more PA bits than are 4215a9624afSPaolo Bonzini * reported by CPUID. Use the PA width of the L1 cache when possible 4225a9624afSPaolo Bonzini * to achieve more effective mitigation, e.g. if system RAM overlaps 4235a9624afSPaolo Bonzini * the most significant bits of legal physical address space. 4245a9624afSPaolo Bonzini */ 4255a9624afSPaolo Bonzini shadow_nonpresent_or_rsvd_mask = 0; 4265a9624afSPaolo Bonzini low_phys_bits = boot_cpu_data.x86_phys_bits; 4275a9624afSPaolo Bonzini if (boot_cpu_has_bug(X86_BUG_L1TF) && 4285a9624afSPaolo Bonzini !WARN_ON_ONCE(boot_cpu_data.x86_cache_bits >= 4298a967d65SPaolo Bonzini 52 - SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)) { 4305a9624afSPaolo Bonzini low_phys_bits = boot_cpu_data.x86_cache_bits 4318a967d65SPaolo Bonzini - SHADOW_NONPRESENT_OR_RSVD_MASK_LEN; 4325a9624afSPaolo Bonzini shadow_nonpresent_or_rsvd_mask = 4335a9624afSPaolo Bonzini rsvd_bits(low_phys_bits, boot_cpu_data.x86_cache_bits - 1); 4345a9624afSPaolo Bonzini } 4355a9624afSPaolo Bonzini 4365a9624afSPaolo Bonzini shadow_nonpresent_or_rsvd_lower_gfn_mask = 4375a9624afSPaolo Bonzini GENMASK_ULL(low_phys_bits - 1, PAGE_SHIFT); 438d6b87f25SSean Christopherson 439e7b7bdeaSSean Christopherson shadow_user_mask = PT_USER_MASK; 440e7b7bdeaSSean Christopherson shadow_accessed_mask = PT_ACCESSED_MASK; 441e7b7bdeaSSean Christopherson shadow_dirty_mask = PT_DIRTY_MASK; 442e7b7bdeaSSean Christopherson shadow_nx_mask = PT64_NX_MASK; 443e7b7bdeaSSean Christopherson shadow_x_mask = 0; 444e7b7bdeaSSean Christopherson shadow_present_mask = PT_PRESENT_MASK; 445e7b7bdeaSSean Christopherson shadow_acc_track_mask = 0; 446e54f1ff2SKai Huang shadow_me_mask = 0; 447e54f1ff2SKai Huang shadow_me_value = 0; 448e7b7bdeaSSean Christopherson 4491ca87e01SDavid Matlack shadow_host_writable_mask = DEFAULT_SPTE_HOST_WRITABLE; 4501ca87e01SDavid Matlack shadow_mmu_writable_mask = DEFAULT_SPTE_MMU_WRITABLE; 4515fc3424fSSean Christopherson 452d6b87f25SSean Christopherson /* 453d6b87f25SSean Christopherson * Set a reserved PA bit in MMIO SPTEs to generate page faults with 454d6b87f25SSean Christopherson * PFEC.RSVD=1 on MMIO accesses. 64-bit PTEs (PAE, x86-64, and EPT 455d6b87f25SSean Christopherson * paging) support a maximum of 52 bits of PA, i.e. if the CPU supports 456d6b87f25SSean Christopherson * 52-bit physical addresses then there are no reserved PA bits in the 457d6b87f25SSean Christopherson * PTEs and so the reserved PA approach must be disabled. 458d6b87f25SSean Christopherson */ 459d6b87f25SSean Christopherson if (shadow_phys_bits < 52) 460d6b87f25SSean Christopherson mask = BIT_ULL(51) | PT_PRESENT_MASK; 461d6b87f25SSean Christopherson else 462d6b87f25SSean Christopherson mask = 0; 463d6b87f25SSean Christopherson 464d6b87f25SSean Christopherson kvm_mmu_set_mmio_spte_mask(mask, mask, ACC_WRITE_MASK | ACC_USER_MASK); 4655a9624afSPaolo Bonzini } 466