1c0ef6326SSuzuki K Poulose /* 2c0ef6326SSuzuki K Poulose * Copyright (C) 2016 - ARM Ltd 3c0ef6326SSuzuki K Poulose * 4c0ef6326SSuzuki K Poulose * stage2 page table helpers 5c0ef6326SSuzuki K Poulose * 6c0ef6326SSuzuki K Poulose * This program is free software; you can redistribute it and/or modify 7c0ef6326SSuzuki K Poulose * it under the terms of the GNU General Public License version 2 as 8c0ef6326SSuzuki K Poulose * published by the Free Software Foundation. 9c0ef6326SSuzuki K Poulose * 10c0ef6326SSuzuki K Poulose * This program is distributed in the hope that it will be useful, 11c0ef6326SSuzuki K Poulose * but WITHOUT ANY WARRANTY; without even the implied warranty of 12c0ef6326SSuzuki K Poulose * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13c0ef6326SSuzuki K Poulose * GNU General Public License for more details. 14c0ef6326SSuzuki K Poulose * 15c0ef6326SSuzuki K Poulose * You should have received a copy of the GNU General Public License 16c0ef6326SSuzuki K Poulose * along with this program. If not, see <http://www.gnu.org/licenses/>. 17c0ef6326SSuzuki K Poulose */ 18c0ef6326SSuzuki K Poulose 19c0ef6326SSuzuki K Poulose #ifndef __ARM64_S2_PGTABLE_H_ 20c0ef6326SSuzuki K Poulose #define __ARM64_S2_PGTABLE_H_ 21c0ef6326SSuzuki K Poulose 22865b30cdSSuzuki K Poulose #include <linux/hugetlb.h> 23c0ef6326SSuzuki K Poulose #include <asm/pgtable.h> 24c0ef6326SSuzuki K Poulose 25c0ef6326SSuzuki K Poulose /* 2661fa5a86SSuzuki K Poulose * PGDIR_SHIFT determines the size a top-level page table entry can map 2761fa5a86SSuzuki K Poulose * and depends on the number of levels in the page table. Compute the 2861fa5a86SSuzuki K Poulose * PGDIR_SHIFT for a given number of levels. 2961fa5a86SSuzuki K Poulose */ 3061fa5a86SSuzuki K Poulose #define pt_levels_pgdir_shift(lvls) ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - (lvls)) 3161fa5a86SSuzuki K Poulose 3261fa5a86SSuzuki K Poulose /* 33da04fa04SSuzuki K Poulose * The hardware supports concatenation of up to 16 tables at stage2 entry level 34da04fa04SSuzuki K Poulose * and we use the feature whenever possible. 35da04fa04SSuzuki K Poulose * 36da04fa04SSuzuki K Poulose * Now, the minimum number of bits resolved at any level is (PAGE_SHIFT - 3). 37da04fa04SSuzuki K Poulose * On arm64, the smallest PAGE_SIZE supported is 4k, which means 38da04fa04SSuzuki K Poulose * (PAGE_SHIFT - 3) > 4 holds for all page sizes. 39da04fa04SSuzuki K Poulose * This implies, the total number of page table levels at stage2 expected 4061fa5a86SSuzuki K Poulose * by the hardware is actually the number of levels required for (IPA_SHIFT - 4) 41da04fa04SSuzuki K Poulose * in normal translations(e.g, stage1), since we cannot have another level in 4261fa5a86SSuzuki K Poulose * the range (IPA_SHIFT, IPA_SHIFT - 4). 43c0ef6326SSuzuki K Poulose */ 4461fa5a86SSuzuki K Poulose #define stage2_pgtable_levels(ipa) ARM64_HW_PGTABLE_LEVELS((ipa) - 4) 4561fa5a86SSuzuki K Poulose #define STAGE2_PGTABLE_LEVELS stage2_pgtable_levels(KVM_PHYS_SHIFT) 4661fa5a86SSuzuki K Poulose #define kvm_stage2_levels(kvm) stage2_pgtable_levels(kvm_phys_shift(kvm)) 47c0ef6326SSuzuki K Poulose 48c0ef6326SSuzuki K Poulose /* 49da04fa04SSuzuki K Poulose * With all the supported VA_BITs and 40bit guest IPA, the following condition 50da04fa04SSuzuki K Poulose * is always true: 51da04fa04SSuzuki K Poulose * 52da04fa04SSuzuki K Poulose * STAGE2_PGTABLE_LEVELS <= CONFIG_PGTABLE_LEVELS 53da04fa04SSuzuki K Poulose * 54da04fa04SSuzuki K Poulose * We base our stage-2 page table walker helpers on this assumption and 55da04fa04SSuzuki K Poulose * fall back to using the host version of the helper wherever possible. 56da04fa04SSuzuki K Poulose * i.e, if a particular level is not folded (e.g, PUD) at stage2, we fall back 57da04fa04SSuzuki K Poulose * to using the host version, since it is guaranteed it is not folded at host. 58da04fa04SSuzuki K Poulose * 59da04fa04SSuzuki K Poulose * If the condition breaks in the future, we can rearrange the host level 60da04fa04SSuzuki K Poulose * definitions and reuse them for stage2. Till then... 61c0ef6326SSuzuki K Poulose */ 62da04fa04SSuzuki K Poulose #if STAGE2_PGTABLE_LEVELS > CONFIG_PGTABLE_LEVELS 63da04fa04SSuzuki K Poulose #error "Unsupported combination of guest IPA and host VA_BITS." 64c0ef6326SSuzuki K Poulose #endif 65c0ef6326SSuzuki K Poulose 6661fa5a86SSuzuki K Poulose 6761fa5a86SSuzuki K Poulose /* stage2_pgdir_shift() is the size mapped by top-level stage2 entry for the VM */ 6861fa5a86SSuzuki K Poulose #define stage2_pgdir_shift(kvm) pt_levels_pgdir_shift(kvm_stage2_levels(kvm)) 6961fa5a86SSuzuki K Poulose #define stage2_pgdir_size(kvm) (1ULL << stage2_pgdir_shift(kvm)) 7061fa5a86SSuzuki K Poulose #define stage2_pgdir_mask(kvm) ~(stage2_pgdir_size(kvm) - 1) 71da04fa04SSuzuki K Poulose 72da04fa04SSuzuki K Poulose /* 73da04fa04SSuzuki K Poulose * The number of PTRS across all concatenated stage2 tables given by the 74da04fa04SSuzuki K Poulose * number of bits resolved at the initial level. 75da04fa04SSuzuki K Poulose */ 7661fa5a86SSuzuki K Poulose #define __s2_pgd_ptrs(ipa, lvls) (1 << ((ipa) - pt_levels_pgdir_shift((lvls)))) 7761fa5a86SSuzuki K Poulose #define __s2_pgd_size(ipa, lvls) (__s2_pgd_ptrs((ipa), (lvls)) * sizeof(pgd_t)) 7861fa5a86SSuzuki K Poulose 7961fa5a86SSuzuki K Poulose #define stage2_pgd_ptrs(kvm) __s2_pgd_ptrs(kvm_phys_shift(kvm), kvm_stage2_levels(kvm)) 8061fa5a86SSuzuki K Poulose #define stage2_pgd_size(kvm) __s2_pgd_size(kvm_phys_shift(kvm), kvm_stage2_levels(kvm)) 81da04fa04SSuzuki K Poulose 82da04fa04SSuzuki K Poulose /* 83e55cac5bSSuzuki K Poulose * kvm_mmmu_cache_min_pages() is the number of pages required to install 84e55cac5bSSuzuki K Poulose * a stage-2 translation. We pre-allocate the entry level page table at 85e55cac5bSSuzuki K Poulose * the VM creation. 86da04fa04SSuzuki K Poulose */ 8761fa5a86SSuzuki K Poulose #define kvm_mmu_cache_min_pages(kvm) (kvm_stage2_levels(kvm) - 1) 88da04fa04SSuzuki K Poulose 89865b30cdSSuzuki K Poulose /* Stage2 PUD definitions when the level is present */ 9061fa5a86SSuzuki K Poulose static inline bool kvm_stage2_has_pud(struct kvm *kvm) 9161fa5a86SSuzuki K Poulose { 9261fa5a86SSuzuki K Poulose return (CONFIG_PGTABLE_LEVELS > 3) && (kvm_stage2_levels(kvm) > 3); 9361fa5a86SSuzuki K Poulose } 9461fa5a86SSuzuki K Poulose 95da04fa04SSuzuki K Poulose #define S2_PUD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(1) 96e55cac5bSSuzuki K Poulose #define S2_PUD_SIZE (1UL << S2_PUD_SHIFT) 97da04fa04SSuzuki K Poulose #define S2_PUD_MASK (~(S2_PUD_SIZE - 1)) 98da04fa04SSuzuki K Poulose 99865b30cdSSuzuki K Poulose static inline bool stage2_pgd_none(struct kvm *kvm, pgd_t pgd) 100865b30cdSSuzuki K Poulose { 10161fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 102865b30cdSSuzuki K Poulose return pgd_none(pgd); 103865b30cdSSuzuki K Poulose else 104865b30cdSSuzuki K Poulose return 0; 105865b30cdSSuzuki K Poulose } 106c0ef6326SSuzuki K Poulose 107865b30cdSSuzuki K Poulose static inline void stage2_pgd_clear(struct kvm *kvm, pgd_t *pgdp) 108865b30cdSSuzuki K Poulose { 10961fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 110865b30cdSSuzuki K Poulose pgd_clear(pgdp); 111865b30cdSSuzuki K Poulose } 112865b30cdSSuzuki K Poulose 113865b30cdSSuzuki K Poulose static inline bool stage2_pgd_present(struct kvm *kvm, pgd_t pgd) 114865b30cdSSuzuki K Poulose { 11561fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 116865b30cdSSuzuki K Poulose return pgd_present(pgd); 117865b30cdSSuzuki K Poulose else 118865b30cdSSuzuki K Poulose return 1; 119865b30cdSSuzuki K Poulose } 120865b30cdSSuzuki K Poulose 121865b30cdSSuzuki K Poulose static inline void stage2_pgd_populate(struct kvm *kvm, pgd_t *pgd, pud_t *pud) 122865b30cdSSuzuki K Poulose { 12361fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 124865b30cdSSuzuki K Poulose pgd_populate(NULL, pgd, pud); 125865b30cdSSuzuki K Poulose } 126865b30cdSSuzuki K Poulose 127865b30cdSSuzuki K Poulose static inline pud_t *stage2_pud_offset(struct kvm *kvm, 128865b30cdSSuzuki K Poulose pgd_t *pgd, unsigned long address) 129865b30cdSSuzuki K Poulose { 13061fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 131865b30cdSSuzuki K Poulose return pud_offset(pgd, address); 132865b30cdSSuzuki K Poulose else 133865b30cdSSuzuki K Poulose return (pud_t *)pgd; 134865b30cdSSuzuki K Poulose } 135865b30cdSSuzuki K Poulose 136865b30cdSSuzuki K Poulose static inline void stage2_pud_free(struct kvm *kvm, pud_t *pud) 137865b30cdSSuzuki K Poulose { 13861fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 139865b30cdSSuzuki K Poulose pud_free(NULL, pud); 140865b30cdSSuzuki K Poulose } 141865b30cdSSuzuki K Poulose 142865b30cdSSuzuki K Poulose static inline bool stage2_pud_table_empty(struct kvm *kvm, pud_t *pudp) 143865b30cdSSuzuki K Poulose { 14461fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) 145865b30cdSSuzuki K Poulose return kvm_page_empty(pudp); 146865b30cdSSuzuki K Poulose else 147865b30cdSSuzuki K Poulose return false; 148865b30cdSSuzuki K Poulose } 149da04fa04SSuzuki K Poulose 150e55cac5bSSuzuki K Poulose static inline phys_addr_t 151e55cac5bSSuzuki K Poulose stage2_pud_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) 152da04fa04SSuzuki K Poulose { 15361fa5a86SSuzuki K Poulose if (kvm_stage2_has_pud(kvm)) { 154da04fa04SSuzuki K Poulose phys_addr_t boundary = (addr + S2_PUD_SIZE) & S2_PUD_MASK; 155da04fa04SSuzuki K Poulose 156da04fa04SSuzuki K Poulose return (boundary - 1 < end - 1) ? boundary : end; 157865b30cdSSuzuki K Poulose } else { 158865b30cdSSuzuki K Poulose return end; 159865b30cdSSuzuki K Poulose } 160da04fa04SSuzuki K Poulose } 161da04fa04SSuzuki K Poulose 162865b30cdSSuzuki K Poulose /* Stage2 PMD definitions when the level is present */ 16361fa5a86SSuzuki K Poulose static inline bool kvm_stage2_has_pmd(struct kvm *kvm) 16461fa5a86SSuzuki K Poulose { 16561fa5a86SSuzuki K Poulose return (CONFIG_PGTABLE_LEVELS > 2) && (kvm_stage2_levels(kvm) > 2); 16661fa5a86SSuzuki K Poulose } 16761fa5a86SSuzuki K Poulose 168da04fa04SSuzuki K Poulose #define S2_PMD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(2) 169e55cac5bSSuzuki K Poulose #define S2_PMD_SIZE (1UL << S2_PMD_SHIFT) 170da04fa04SSuzuki K Poulose #define S2_PMD_MASK (~(S2_PMD_SIZE - 1)) 171da04fa04SSuzuki K Poulose 172865b30cdSSuzuki K Poulose static inline bool stage2_pud_none(struct kvm *kvm, pud_t pud) 173865b30cdSSuzuki K Poulose { 17461fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 175865b30cdSSuzuki K Poulose return pud_none(pud); 176865b30cdSSuzuki K Poulose else 177865b30cdSSuzuki K Poulose return 0; 178865b30cdSSuzuki K Poulose } 179c0ef6326SSuzuki K Poulose 180865b30cdSSuzuki K Poulose static inline void stage2_pud_clear(struct kvm *kvm, pud_t *pud) 181865b30cdSSuzuki K Poulose { 18261fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 183865b30cdSSuzuki K Poulose pud_clear(pud); 184865b30cdSSuzuki K Poulose } 185865b30cdSSuzuki K Poulose 186865b30cdSSuzuki K Poulose static inline bool stage2_pud_present(struct kvm *kvm, pud_t pud) 187865b30cdSSuzuki K Poulose { 18861fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 189865b30cdSSuzuki K Poulose return pud_present(pud); 190865b30cdSSuzuki K Poulose else 191865b30cdSSuzuki K Poulose return 1; 192865b30cdSSuzuki K Poulose } 193865b30cdSSuzuki K Poulose 194865b30cdSSuzuki K Poulose static inline void stage2_pud_populate(struct kvm *kvm, pud_t *pud, pmd_t *pmd) 195865b30cdSSuzuki K Poulose { 19661fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 197865b30cdSSuzuki K Poulose pud_populate(NULL, pud, pmd); 198865b30cdSSuzuki K Poulose } 199865b30cdSSuzuki K Poulose 200865b30cdSSuzuki K Poulose static inline pmd_t *stage2_pmd_offset(struct kvm *kvm, 201865b30cdSSuzuki K Poulose pud_t *pud, unsigned long address) 202865b30cdSSuzuki K Poulose { 20361fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 204865b30cdSSuzuki K Poulose return pmd_offset(pud, address); 205865b30cdSSuzuki K Poulose else 206865b30cdSSuzuki K Poulose return (pmd_t *)pud; 207865b30cdSSuzuki K Poulose } 208865b30cdSSuzuki K Poulose 209865b30cdSSuzuki K Poulose static inline void stage2_pmd_free(struct kvm *kvm, pmd_t *pmd) 210865b30cdSSuzuki K Poulose { 21161fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 212865b30cdSSuzuki K Poulose pmd_free(NULL, pmd); 213865b30cdSSuzuki K Poulose } 214865b30cdSSuzuki K Poulose 215865b30cdSSuzuki K Poulose static inline bool stage2_pud_huge(struct kvm *kvm, pud_t pud) 216865b30cdSSuzuki K Poulose { 21761fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 218865b30cdSSuzuki K Poulose return pud_huge(pud); 219865b30cdSSuzuki K Poulose else 220865b30cdSSuzuki K Poulose return 0; 221865b30cdSSuzuki K Poulose } 222865b30cdSSuzuki K Poulose 223865b30cdSSuzuki K Poulose static inline bool stage2_pmd_table_empty(struct kvm *kvm, pmd_t *pmdp) 224865b30cdSSuzuki K Poulose { 22561fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) 226865b30cdSSuzuki K Poulose return kvm_page_empty(pmdp); 227865b30cdSSuzuki K Poulose else 228865b30cdSSuzuki K Poulose return 0; 229865b30cdSSuzuki K Poulose } 230c0ef6326SSuzuki K Poulose 231e55cac5bSSuzuki K Poulose static inline phys_addr_t 232e55cac5bSSuzuki K Poulose stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) 233da04fa04SSuzuki K Poulose { 23461fa5a86SSuzuki K Poulose if (kvm_stage2_has_pmd(kvm)) { 235da04fa04SSuzuki K Poulose phys_addr_t boundary = (addr + S2_PMD_SIZE) & S2_PMD_MASK; 236da04fa04SSuzuki K Poulose 237da04fa04SSuzuki K Poulose return (boundary - 1 < end - 1) ? boundary : end; 238865b30cdSSuzuki K Poulose } else { 239865b30cdSSuzuki K Poulose return end; 240865b30cdSSuzuki K Poulose } 241da04fa04SSuzuki K Poulose } 242da04fa04SSuzuki K Poulose 243865b30cdSSuzuki K Poulose static inline bool stage2_pte_table_empty(struct kvm *kvm, pte_t *ptep) 244865b30cdSSuzuki K Poulose { 245865b30cdSSuzuki K Poulose return kvm_page_empty(ptep); 246865b30cdSSuzuki K Poulose } 247c0ef6326SSuzuki K Poulose 248865b30cdSSuzuki K Poulose static inline unsigned long stage2_pgd_index(struct kvm *kvm, phys_addr_t addr) 249865b30cdSSuzuki K Poulose { 25061fa5a86SSuzuki K Poulose return (((addr) >> stage2_pgdir_shift(kvm)) & (stage2_pgd_ptrs(kvm) - 1)); 251865b30cdSSuzuki K Poulose } 252da04fa04SSuzuki K Poulose 253e55cac5bSSuzuki K Poulose static inline phys_addr_t 254e55cac5bSSuzuki K Poulose stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) 255da04fa04SSuzuki K Poulose { 25661fa5a86SSuzuki K Poulose phys_addr_t boundary = (addr + stage2_pgdir_size(kvm)) & stage2_pgdir_mask(kvm); 257da04fa04SSuzuki K Poulose 258da04fa04SSuzuki K Poulose return (boundary - 1 < end - 1) ? boundary : end; 259da04fa04SSuzuki K Poulose } 260c0ef6326SSuzuki K Poulose 261c0ef6326SSuzuki K Poulose #endif /* __ARM64_S2_PGTABLE_H_ */ 262