1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #ifndef KVM_X86_MMU_SPTE_H 4 #define KVM_X86_MMU_SPTE_H 5 6 #include "mmu_internal.h" 7 8 extern bool __read_mostly enable_mmio_caching; 9 10 /* 11 * A MMU present SPTE is backed by actual memory and may or may not be present 12 * in hardware. E.g. MMIO SPTEs are not considered present. Use bit 11, as it 13 * is ignored by all flavors of SPTEs and checking a low bit often generates 14 * better code than for a high bit, e.g. 56+. MMU present checks are pervasive 15 * enough that the improved code generation is noticeable in KVM's footprint. 16 */ 17 #define SPTE_MMU_PRESENT_MASK BIT_ULL(11) 18 19 /* 20 * TDP SPTES (more specifically, EPT SPTEs) may not have A/D bits, and may also 21 * be restricted to using write-protection (for L2 when CPU dirty logging, i.e. 22 * PML, is enabled). Use bits 52 and 53 to hold the type of A/D tracking that 23 * is must be employed for a given TDP SPTE. 24 * 25 * Note, the "enabled" mask must be '0', as bits 62:52 are _reserved_ for PAE 26 * paging, including NPT PAE. This scheme works because legacy shadow paging 27 * is guaranteed to have A/D bits and write-protection is forced only for 28 * TDP with CPU dirty logging (PML). If NPT ever gains PML-like support, it 29 * must be restricted to 64-bit KVM. 30 */ 31 #define SPTE_TDP_AD_SHIFT 52 32 #define SPTE_TDP_AD_MASK (3ULL << SPTE_TDP_AD_SHIFT) 33 #define SPTE_TDP_AD_ENABLED_MASK (0ULL << SPTE_TDP_AD_SHIFT) 34 #define SPTE_TDP_AD_DISABLED_MASK (1ULL << SPTE_TDP_AD_SHIFT) 35 #define SPTE_TDP_AD_WRPROT_ONLY_MASK (2ULL << SPTE_TDP_AD_SHIFT) 36 static_assert(SPTE_TDP_AD_ENABLED_MASK == 0); 37 38 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK 39 #define SPTE_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1)) 40 #else 41 #define SPTE_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)) 42 #endif 43 44 #define SPTE_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \ 45 | shadow_x_mask | shadow_nx_mask | shadow_me_mask) 46 47 #define ACC_EXEC_MASK 1 48 #define ACC_WRITE_MASK PT_WRITABLE_MASK 49 #define ACC_USER_MASK PT_USER_MASK 50 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) 51 52 /* The mask for the R/X bits in EPT PTEs */ 53 #define SPTE_EPT_READABLE_MASK 0x1ull 54 #define SPTE_EPT_EXECUTABLE_MASK 0x4ull 55 56 #define SPTE_LEVEL_BITS 9 57 #define SPTE_LEVEL_SHIFT(level) __PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS) 58 #define SPTE_INDEX(address, level) __PT_INDEX(address, level, SPTE_LEVEL_BITS) 59 #define SPTE_ENT_PER_PAGE __PT_ENT_PER_PAGE(SPTE_LEVEL_BITS) 60 61 /* 62 * The mask/shift to use for saving the original R/X bits when marking the PTE 63 * as not-present for access tracking purposes. We do not save the W bit as the 64 * PTEs being access tracked also need to be dirty tracked, so the W bit will be 65 * restored only when a write is attempted to the page. This mask obviously 66 * must not overlap the A/D type mask. 67 */ 68 #define SHADOW_ACC_TRACK_SAVED_BITS_MASK (SPTE_EPT_READABLE_MASK | \ 69 SPTE_EPT_EXECUTABLE_MASK) 70 #define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 54 71 #define SHADOW_ACC_TRACK_SAVED_MASK (SHADOW_ACC_TRACK_SAVED_BITS_MASK << \ 72 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT) 73 static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK)); 74 75 /* 76 * {DEFAULT,EPT}_SPTE_{HOST,MMU}_WRITABLE are used to keep track of why a given 77 * SPTE is write-protected. See is_writable_pte() for details. 78 */ 79 80 /* Bits 9 and 10 are ignored by all non-EPT PTEs. */ 81 #define DEFAULT_SPTE_HOST_WRITABLE BIT_ULL(9) 82 #define DEFAULT_SPTE_MMU_WRITABLE BIT_ULL(10) 83 84 /* 85 * Low ignored bits are at a premium for EPT, use high ignored bits, taking care 86 * to not overlap the A/D type mask or the saved access bits of access-tracked 87 * SPTEs when A/D bits are disabled. 88 */ 89 #define EPT_SPTE_HOST_WRITABLE BIT_ULL(57) 90 #define EPT_SPTE_MMU_WRITABLE BIT_ULL(58) 91 92 static_assert(!(EPT_SPTE_HOST_WRITABLE & SPTE_TDP_AD_MASK)); 93 static_assert(!(EPT_SPTE_MMU_WRITABLE & SPTE_TDP_AD_MASK)); 94 static_assert(!(EPT_SPTE_HOST_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK)); 95 static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK)); 96 97 /* Defined only to keep the above static asserts readable. */ 98 #undef SHADOW_ACC_TRACK_SAVED_MASK 99 100 /* 101 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of 102 * the memslots generation and is derived as follows: 103 * 104 * Bits 0-7 of the MMIO generation are propagated to spte bits 3-10 105 * Bits 8-18 of the MMIO generation are propagated to spte bits 52-62 106 * 107 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in 108 * the MMIO generation number, as doing so would require stealing a bit from 109 * the "real" generation number and thus effectively halve the maximum number 110 * of MMIO generations that can be handled before encountering a wrap (which 111 * requires a full MMU zap). The flag is instead explicitly queried when 112 * checking for MMIO spte cache hits. 113 */ 114 115 #define MMIO_SPTE_GEN_LOW_START 3 116 #define MMIO_SPTE_GEN_LOW_END 10 117 118 #define MMIO_SPTE_GEN_HIGH_START 52 119 #define MMIO_SPTE_GEN_HIGH_END 62 120 121 #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \ 122 MMIO_SPTE_GEN_LOW_START) 123 #define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \ 124 MMIO_SPTE_GEN_HIGH_START) 125 static_assert(!(SPTE_MMU_PRESENT_MASK & 126 (MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK))); 127 128 #define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1) 129 #define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1) 130 131 /* remember to adjust the comment above as well if you change these */ 132 static_assert(MMIO_SPTE_GEN_LOW_BITS == 8 && MMIO_SPTE_GEN_HIGH_BITS == 11); 133 134 #define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0) 135 #define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS) 136 137 #define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0) 138 139 extern u64 __read_mostly shadow_host_writable_mask; 140 extern u64 __read_mostly shadow_mmu_writable_mask; 141 extern u64 __read_mostly shadow_nx_mask; 142 extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ 143 extern u64 __read_mostly shadow_user_mask; 144 extern u64 __read_mostly shadow_accessed_mask; 145 extern u64 __read_mostly shadow_dirty_mask; 146 extern u64 __read_mostly shadow_mmio_value; 147 extern u64 __read_mostly shadow_mmio_mask; 148 extern u64 __read_mostly shadow_mmio_access_mask; 149 extern u64 __read_mostly shadow_present_mask; 150 extern u64 __read_mostly shadow_memtype_mask; 151 extern u64 __read_mostly shadow_me_value; 152 extern u64 __read_mostly shadow_me_mask; 153 154 /* 155 * SPTEs in MMUs without A/D bits are marked with SPTE_TDP_AD_DISABLED_MASK; 156 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed 157 * pages. 158 */ 159 extern u64 __read_mostly shadow_acc_track_mask; 160 161 /* 162 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order 163 * to guard against L1TF attacks. 164 */ 165 extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask; 166 167 /* 168 * The number of high-order 1 bits to use in the mask above. 169 */ 170 #define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5 171 172 /* 173 * If a thread running without exclusive control of the MMU lock must perform a 174 * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a 175 * non-present intermediate value. Other threads which encounter this value 176 * should not modify the SPTE. 177 * 178 * Use a semi-arbitrary value that doesn't set RWX bits, i.e. is not-present on 179 * bot AMD and Intel CPUs, and doesn't set PFN bits, i.e. doesn't create a L1TF 180 * vulnerability. Use only low bits to avoid 64-bit immediates. 181 * 182 * Only used by the TDP MMU. 183 */ 184 #define REMOVED_SPTE 0x5a0ULL 185 186 /* Removed SPTEs must not be misconstrued as shadow present PTEs. */ 187 static_assert(!(REMOVED_SPTE & SPTE_MMU_PRESENT_MASK)); 188 189 static inline bool is_removed_spte(u64 spte) 190 { 191 return spte == REMOVED_SPTE; 192 } 193 194 /* Get an SPTE's index into its parent's page table (and the spt array). */ 195 static inline int spte_index(u64 *sptep) 196 { 197 return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1); 198 } 199 200 /* 201 * In some cases, we need to preserve the GFN of a non-present or reserved 202 * SPTE when we usurp the upper five bits of the physical address space to 203 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll 204 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask 205 * left into the reserved bits, i.e. the GFN in the SPTE will be split into 206 * high and low parts. This mask covers the lower bits of the GFN. 207 */ 208 extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask; 209 210 static inline bool is_mmio_spte(u64 spte) 211 { 212 return (spte & shadow_mmio_mask) == shadow_mmio_value && 213 likely(enable_mmio_caching); 214 } 215 216 static inline bool is_shadow_present_pte(u64 pte) 217 { 218 return !!(pte & SPTE_MMU_PRESENT_MASK); 219 } 220 221 /* 222 * Returns true if A/D bits are supported in hardware and are enabled by KVM. 223 * When enabled, KVM uses A/D bits for all non-nested MMUs. Because L1 can 224 * disable A/D bits in EPTP12, SP and SPTE variants are needed to handle the 225 * scenario where KVM is using A/D bits for L1, but not L2. 226 */ 227 static inline bool kvm_ad_enabled(void) 228 { 229 return !!shadow_accessed_mask; 230 } 231 232 static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) 233 { 234 return sp->role.ad_disabled; 235 } 236 237 static inline bool spte_ad_enabled(u64 spte) 238 { 239 MMU_WARN_ON(!is_shadow_present_pte(spte)); 240 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_DISABLED_MASK; 241 } 242 243 static inline bool spte_ad_need_write_protect(u64 spte) 244 { 245 MMU_WARN_ON(!is_shadow_present_pte(spte)); 246 /* 247 * This is benign for non-TDP SPTEs as SPTE_TDP_AD_ENABLED_MASK is '0', 248 * and non-TDP SPTEs will never set these bits. Optimize for 64-bit 249 * TDP and do the A/D type check unconditionally. 250 */ 251 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_ENABLED_MASK; 252 } 253 254 static inline u64 spte_shadow_accessed_mask(u64 spte) 255 { 256 MMU_WARN_ON(!is_shadow_present_pte(spte)); 257 return spte_ad_enabled(spte) ? shadow_accessed_mask : 0; 258 } 259 260 static inline u64 spte_shadow_dirty_mask(u64 spte) 261 { 262 MMU_WARN_ON(!is_shadow_present_pte(spte)); 263 return spte_ad_enabled(spte) ? shadow_dirty_mask : 0; 264 } 265 266 static inline bool is_access_track_spte(u64 spte) 267 { 268 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0; 269 } 270 271 static inline bool is_large_pte(u64 pte) 272 { 273 return pte & PT_PAGE_SIZE_MASK; 274 } 275 276 static inline bool is_last_spte(u64 pte, int level) 277 { 278 return (level == PG_LEVEL_4K) || is_large_pte(pte); 279 } 280 281 static inline bool is_executable_pte(u64 spte) 282 { 283 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask; 284 } 285 286 static inline kvm_pfn_t spte_to_pfn(u64 pte) 287 { 288 return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT; 289 } 290 291 static inline bool is_accessed_spte(u64 spte) 292 { 293 u64 accessed_mask = spte_shadow_accessed_mask(spte); 294 295 return accessed_mask ? spte & accessed_mask 296 : !is_access_track_spte(spte); 297 } 298 299 static inline bool is_dirty_spte(u64 spte) 300 { 301 u64 dirty_mask = spte_shadow_dirty_mask(spte); 302 303 return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK; 304 } 305 306 static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte, 307 int level) 308 { 309 int bit7 = (pte >> 7) & 1; 310 311 return rsvd_check->rsvd_bits_mask[bit7][level-1]; 312 } 313 314 static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, 315 u64 pte, int level) 316 { 317 return pte & get_rsvd_bits(rsvd_check, pte, level); 318 } 319 320 static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check, 321 u64 pte) 322 { 323 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f); 324 } 325 326 static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check, 327 u64 spte, int level) 328 { 329 return __is_bad_mt_xwr(rsvd_check, spte) || 330 __is_rsvd_bits_set(rsvd_check, spte, level); 331 } 332 333 /* 334 * An shadow-present leaf SPTE may be non-writable for 3 possible reasons: 335 * 336 * 1. To intercept writes for dirty logging. KVM write-protects huge pages 337 * so that they can be split be split down into the dirty logging 338 * granularity (4KiB) whenever the guest writes to them. KVM also 339 * write-protects 4KiB pages so that writes can be recorded in the dirty log 340 * (e.g. if not using PML). SPTEs are write-protected for dirty logging 341 * during the VM-iotcls that enable dirty logging. 342 * 343 * 2. To intercept writes to guest page tables that KVM is shadowing. When a 344 * guest writes to its page table the corresponding shadow page table will 345 * be marked "unsync". That way KVM knows which shadow page tables need to 346 * be updated on the next TLB flush, INVLPG, etc. and which do not. 347 * 348 * 3. To prevent guest writes to read-only memory, such as for memory in a 349 * read-only memslot or guest memory backed by a read-only VMA. Writes to 350 * such pages are disallowed entirely. 351 * 352 * To keep track of why a given SPTE is write-protected, KVM uses 2 353 * software-only bits in the SPTE: 354 * 355 * shadow_mmu_writable_mask, aka MMU-writable - 356 * Cleared on SPTEs that KVM is currently write-protecting for shadow paging 357 * purposes (case 2 above). 358 * 359 * shadow_host_writable_mask, aka Host-writable - 360 * Cleared on SPTEs that are not host-writable (case 3 above) 361 * 362 * Note, not all possible combinations of PT_WRITABLE_MASK, 363 * shadow_mmu_writable_mask, and shadow_host_writable_mask are valid. A given 364 * SPTE can be in only one of the following states, which map to the 365 * aforementioned 3 cases: 366 * 367 * shadow_host_writable_mask | shadow_mmu_writable_mask | PT_WRITABLE_MASK 368 * ------------------------- | ------------------------ | ---------------- 369 * 1 | 1 | 1 (writable) 370 * 1 | 1 | 0 (case 1) 371 * 1 | 0 | 0 (case 2) 372 * 0 | 0 | 0 (case 3) 373 * 374 * The valid combinations of these bits are checked by 375 * check_spte_writable_invariants() whenever an SPTE is modified. 376 * 377 * Clearing the MMU-writable bit is always done under the MMU lock and always 378 * accompanied by a TLB flush before dropping the lock to avoid corrupting the 379 * shadow page tables between vCPUs. Write-protecting an SPTE for dirty logging 380 * (which does not clear the MMU-writable bit), does not flush TLBs before 381 * dropping the lock, as it only needs to synchronize guest writes with the 382 * dirty bitmap. 383 * 384 * So, there is the problem: clearing the MMU-writable bit can encounter a 385 * write-protected SPTE while CPUs still have writable mappings for that SPTE 386 * cached in their TLB. To address this, KVM always flushes TLBs when 387 * write-protecting SPTEs if the MMU-writable bit is set on the old SPTE. 388 * 389 * The Host-writable bit is not modified on present SPTEs, it is only set or 390 * cleared when an SPTE is first faulted in from non-present and then remains 391 * immutable. 392 */ 393 static inline bool is_writable_pte(unsigned long pte) 394 { 395 return pte & PT_WRITABLE_MASK; 396 } 397 398 /* Note: spte must be a shadow-present leaf SPTE. */ 399 static inline void check_spte_writable_invariants(u64 spte) 400 { 401 if (spte & shadow_mmu_writable_mask) 402 WARN_ONCE(!(spte & shadow_host_writable_mask), 403 "kvm: MMU-writable SPTE is not Host-writable: %llx", 404 spte); 405 else 406 WARN_ONCE(is_writable_pte(spte), 407 "kvm: Writable SPTE is not MMU-writable: %llx", spte); 408 } 409 410 static inline bool is_mmu_writable_spte(u64 spte) 411 { 412 return spte & shadow_mmu_writable_mask; 413 } 414 415 static inline u64 get_mmio_spte_generation(u64 spte) 416 { 417 u64 gen; 418 419 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT; 420 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT; 421 return gen; 422 } 423 424 bool spte_has_volatile_bits(u64 spte); 425 426 bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 427 const struct kvm_memory_slot *slot, 428 unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn, 429 u64 old_spte, bool prefetch, bool can_unsync, 430 bool host_writable, u64 *new_spte); 431 u64 make_huge_page_split_spte(struct kvm *kvm, u64 huge_spte, 432 union kvm_mmu_page_role role, int index); 433 u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled); 434 u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access); 435 u64 mark_spte_for_access_track(u64 spte); 436 437 /* Restore an acc-track PTE back to a regular PTE */ 438 static inline u64 restore_acc_track_spte(u64 spte) 439 { 440 u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT) 441 & SHADOW_ACC_TRACK_SAVED_BITS_MASK; 442 443 spte &= ~shadow_acc_track_mask; 444 spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK << 445 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT); 446 spte |= saved_bits; 447 448 return spte; 449 } 450 451 u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn); 452 453 void kvm_mmu_reset_all_pte_masks(void); 454 455 #endif 456