1c0ef6326SSuzuki K Poulose /* 2c0ef6326SSuzuki K Poulose * Copyright (C) 2016 - ARM Ltd 3c0ef6326SSuzuki K Poulose * 4c0ef6326SSuzuki K Poulose * stage2 page table helpers 5c0ef6326SSuzuki K Poulose * 6c0ef6326SSuzuki K Poulose * This program is free software; you can redistribute it and/or modify 7c0ef6326SSuzuki K Poulose * it under the terms of the GNU General Public License version 2 as 8c0ef6326SSuzuki K Poulose * published by the Free Software Foundation. 9c0ef6326SSuzuki K Poulose * 10c0ef6326SSuzuki K Poulose * This program is distributed in the hope that it will be useful, 11c0ef6326SSuzuki K Poulose * but WITHOUT ANY WARRANTY; without even the implied warranty of 12c0ef6326SSuzuki K Poulose * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13c0ef6326SSuzuki K Poulose * GNU General Public License for more details. 14c0ef6326SSuzuki K Poulose * 15c0ef6326SSuzuki K Poulose * You should have received a copy of the GNU General Public License 16c0ef6326SSuzuki K Poulose * along with this program. If not, see <http://www.gnu.org/licenses/>. 17c0ef6326SSuzuki K Poulose */ 18c0ef6326SSuzuki K Poulose 19c0ef6326SSuzuki K Poulose #ifndef __ARM64_S2_PGTABLE_H_ 20c0ef6326SSuzuki K Poulose #define __ARM64_S2_PGTABLE_H_ 21c0ef6326SSuzuki K Poulose 22865b30cdSSuzuki K Poulose #include <linux/hugetlb.h> 23c0ef6326SSuzuki K Poulose #include <asm/pgtable.h> 24c0ef6326SSuzuki K Poulose 25c0ef6326SSuzuki K Poulose /* 26da04fa04SSuzuki K Poulose * The hardware supports concatenation of up to 16 tables at stage2 entry level 27da04fa04SSuzuki K Poulose * and we use the feature whenever possible. 28da04fa04SSuzuki K Poulose * 29da04fa04SSuzuki K Poulose * Now, the minimum number of bits resolved at any level is (PAGE_SHIFT - 3). 30da04fa04SSuzuki K Poulose * On arm64, the smallest PAGE_SIZE supported is 4k, which means 31da04fa04SSuzuki K Poulose * (PAGE_SHIFT - 3) > 4 holds for all page sizes. 32da04fa04SSuzuki K Poulose * This implies, the total number of page table levels at stage2 expected 33da04fa04SSuzuki K Poulose * by the hardware is actually the number of levels required for (KVM_PHYS_SHIFT - 4) 34da04fa04SSuzuki K Poulose * in normal translations(e.g, stage1), since we cannot have another level in 35da04fa04SSuzuki K Poulose * the range (KVM_PHYS_SHIFT, KVM_PHYS_SHIFT - 4). 36c0ef6326SSuzuki K Poulose */ 37da04fa04SSuzuki K Poulose #define STAGE2_PGTABLE_LEVELS ARM64_HW_PGTABLE_LEVELS(KVM_PHYS_SHIFT - 4) 38c0ef6326SSuzuki K Poulose 39c0ef6326SSuzuki K Poulose /* 40da04fa04SSuzuki K Poulose * With all the supported VA_BITs and 40bit guest IPA, the following condition 41da04fa04SSuzuki K Poulose * is always true: 42da04fa04SSuzuki K Poulose * 43da04fa04SSuzuki K Poulose * STAGE2_PGTABLE_LEVELS <= CONFIG_PGTABLE_LEVELS 44da04fa04SSuzuki K Poulose * 45da04fa04SSuzuki K Poulose * We base our stage-2 page table walker helpers on this assumption and 46da04fa04SSuzuki K Poulose * fall back to using the host version of the helper wherever possible. 47da04fa04SSuzuki K Poulose * i.e, if a particular level is not folded (e.g, PUD) at stage2, we fall back 48da04fa04SSuzuki K Poulose * to using the host version, since it is guaranteed it is not folded at host. 49da04fa04SSuzuki K Poulose * 50da04fa04SSuzuki K Poulose * If the condition breaks in the future, we can rearrange the host level 51da04fa04SSuzuki K Poulose * definitions and reuse them for stage2. Till then... 52c0ef6326SSuzuki K Poulose */ 53da04fa04SSuzuki K Poulose #if STAGE2_PGTABLE_LEVELS > CONFIG_PGTABLE_LEVELS 54da04fa04SSuzuki K Poulose #error "Unsupported combination of guest IPA and host VA_BITS." 55c0ef6326SSuzuki K Poulose #endif 56c0ef6326SSuzuki K Poulose 57da04fa04SSuzuki K Poulose /* S2_PGDIR_SHIFT is the size mapped by top-level stage2 entry */ 58da04fa04SSuzuki K Poulose #define S2_PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - STAGE2_PGTABLE_LEVELS) 59e55cac5bSSuzuki K Poulose #define S2_PGDIR_SIZE (1UL << S2_PGDIR_SHIFT) 60da04fa04SSuzuki K Poulose #define S2_PGDIR_MASK (~(S2_PGDIR_SIZE - 1)) 61da04fa04SSuzuki K Poulose 62da04fa04SSuzuki K Poulose /* 63da04fa04SSuzuki K Poulose * The number of PTRS across all concatenated stage2 tables given by the 64da04fa04SSuzuki K Poulose * number of bits resolved at the initial level. 65da04fa04SSuzuki K Poulose */ 66da04fa04SSuzuki K Poulose #define PTRS_PER_S2_PGD (1 << (KVM_PHYS_SHIFT - S2_PGDIR_SHIFT)) 67da04fa04SSuzuki K Poulose 68da04fa04SSuzuki K Poulose /* 69e55cac5bSSuzuki K Poulose * kvm_mmmu_cache_min_pages() is the number of pages required to install 70e55cac5bSSuzuki K Poulose * a stage-2 translation. We pre-allocate the entry level page table at 71e55cac5bSSuzuki K Poulose * the VM creation. 72da04fa04SSuzuki K Poulose */ 73e55cac5bSSuzuki K Poulose #define kvm_mmu_cache_min_pages(kvm) (STAGE2_PGTABLE_LEVELS - 1) 74da04fa04SSuzuki K Poulose 75865b30cdSSuzuki K Poulose /* Stage2 PUD definitions when the level is present */ 76865b30cdSSuzuki K Poulose #define STAGE2_PGTABLE_HAS_PUD (STAGE2_PGTABLE_LEVELS > 3) 77da04fa04SSuzuki K Poulose #define S2_PUD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(1) 78e55cac5bSSuzuki K Poulose #define S2_PUD_SIZE (1UL << S2_PUD_SHIFT) 79da04fa04SSuzuki K Poulose #define S2_PUD_MASK (~(S2_PUD_SIZE - 1)) 80da04fa04SSuzuki K Poulose 81865b30cdSSuzuki K Poulose static inline bool stage2_pgd_none(struct kvm *kvm, pgd_t pgd) 82865b30cdSSuzuki K Poulose { 83865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 84865b30cdSSuzuki K Poulose return pgd_none(pgd); 85865b30cdSSuzuki K Poulose else 86865b30cdSSuzuki K Poulose return 0; 87865b30cdSSuzuki K Poulose } 88c0ef6326SSuzuki K Poulose 89865b30cdSSuzuki K Poulose static inline void stage2_pgd_clear(struct kvm *kvm, pgd_t *pgdp) 90865b30cdSSuzuki K Poulose { 91865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 92865b30cdSSuzuki K Poulose pgd_clear(pgdp); 93865b30cdSSuzuki K Poulose } 94865b30cdSSuzuki K Poulose 95865b30cdSSuzuki K Poulose static inline bool stage2_pgd_present(struct kvm *kvm, pgd_t pgd) 96865b30cdSSuzuki K Poulose { 97865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 98865b30cdSSuzuki K Poulose return pgd_present(pgd); 99865b30cdSSuzuki K Poulose else 100865b30cdSSuzuki K Poulose return 1; 101865b30cdSSuzuki K Poulose } 102865b30cdSSuzuki K Poulose 103865b30cdSSuzuki K Poulose static inline void stage2_pgd_populate(struct kvm *kvm, pgd_t *pgd, pud_t *pud) 104865b30cdSSuzuki K Poulose { 105865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 106865b30cdSSuzuki K Poulose pgd_populate(NULL, pgd, pud); 107865b30cdSSuzuki K Poulose } 108865b30cdSSuzuki K Poulose 109865b30cdSSuzuki K Poulose static inline pud_t *stage2_pud_offset(struct kvm *kvm, 110865b30cdSSuzuki K Poulose pgd_t *pgd, unsigned long address) 111865b30cdSSuzuki K Poulose { 112865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 113865b30cdSSuzuki K Poulose return pud_offset(pgd, address); 114865b30cdSSuzuki K Poulose else 115865b30cdSSuzuki K Poulose return (pud_t *)pgd; 116865b30cdSSuzuki K Poulose } 117865b30cdSSuzuki K Poulose 118865b30cdSSuzuki K Poulose static inline void stage2_pud_free(struct kvm *kvm, pud_t *pud) 119865b30cdSSuzuki K Poulose { 120865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 121865b30cdSSuzuki K Poulose pud_free(NULL, pud); 122865b30cdSSuzuki K Poulose } 123865b30cdSSuzuki K Poulose 124865b30cdSSuzuki K Poulose static inline bool stage2_pud_table_empty(struct kvm *kvm, pud_t *pudp) 125865b30cdSSuzuki K Poulose { 126865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) 127865b30cdSSuzuki K Poulose return kvm_page_empty(pudp); 128865b30cdSSuzuki K Poulose else 129865b30cdSSuzuki K Poulose return false; 130865b30cdSSuzuki K Poulose } 131da04fa04SSuzuki K Poulose 132e55cac5bSSuzuki K Poulose static inline phys_addr_t 133e55cac5bSSuzuki K Poulose stage2_pud_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) 134da04fa04SSuzuki K Poulose { 135865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PUD) { 136da04fa04SSuzuki K Poulose phys_addr_t boundary = (addr + S2_PUD_SIZE) & S2_PUD_MASK; 137da04fa04SSuzuki K Poulose 138da04fa04SSuzuki K Poulose return (boundary - 1 < end - 1) ? boundary : end; 139865b30cdSSuzuki K Poulose } else { 140865b30cdSSuzuki K Poulose return end; 141865b30cdSSuzuki K Poulose } 142da04fa04SSuzuki K Poulose } 143da04fa04SSuzuki K Poulose 144865b30cdSSuzuki K Poulose /* Stage2 PMD definitions when the level is present */ 145865b30cdSSuzuki K Poulose #define STAGE2_PGTABLE_HAS_PMD (STAGE2_PGTABLE_LEVELS > 2) 146da04fa04SSuzuki K Poulose #define S2_PMD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(2) 147e55cac5bSSuzuki K Poulose #define S2_PMD_SIZE (1UL << S2_PMD_SHIFT) 148da04fa04SSuzuki K Poulose #define S2_PMD_MASK (~(S2_PMD_SIZE - 1)) 149da04fa04SSuzuki K Poulose 150865b30cdSSuzuki K Poulose static inline bool stage2_pud_none(struct kvm *kvm, pud_t pud) 151865b30cdSSuzuki K Poulose { 152865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 153865b30cdSSuzuki K Poulose return pud_none(pud); 154865b30cdSSuzuki K Poulose else 155865b30cdSSuzuki K Poulose return 0; 156865b30cdSSuzuki K Poulose } 157c0ef6326SSuzuki K Poulose 158865b30cdSSuzuki K Poulose static inline void stage2_pud_clear(struct kvm *kvm, pud_t *pud) 159865b30cdSSuzuki K Poulose { 160865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 161865b30cdSSuzuki K Poulose pud_clear(pud); 162865b30cdSSuzuki K Poulose } 163865b30cdSSuzuki K Poulose 164865b30cdSSuzuki K Poulose static inline bool stage2_pud_present(struct kvm *kvm, pud_t pud) 165865b30cdSSuzuki K Poulose { 166865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 167865b30cdSSuzuki K Poulose return pud_present(pud); 168865b30cdSSuzuki K Poulose else 169865b30cdSSuzuki K Poulose return 1; 170865b30cdSSuzuki K Poulose } 171865b30cdSSuzuki K Poulose 172865b30cdSSuzuki K Poulose static inline void stage2_pud_populate(struct kvm *kvm, pud_t *pud, pmd_t *pmd) 173865b30cdSSuzuki K Poulose { 174865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 175865b30cdSSuzuki K Poulose pud_populate(NULL, pud, pmd); 176865b30cdSSuzuki K Poulose } 177865b30cdSSuzuki K Poulose 178865b30cdSSuzuki K Poulose static inline pmd_t *stage2_pmd_offset(struct kvm *kvm, 179865b30cdSSuzuki K Poulose pud_t *pud, unsigned long address) 180865b30cdSSuzuki K Poulose { 181865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 182865b30cdSSuzuki K Poulose return pmd_offset(pud, address); 183865b30cdSSuzuki K Poulose else 184865b30cdSSuzuki K Poulose return (pmd_t *)pud; 185865b30cdSSuzuki K Poulose } 186865b30cdSSuzuki K Poulose 187865b30cdSSuzuki K Poulose static inline void stage2_pmd_free(struct kvm *kvm, pmd_t *pmd) 188865b30cdSSuzuki K Poulose { 189865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 190865b30cdSSuzuki K Poulose pmd_free(NULL, pmd); 191865b30cdSSuzuki K Poulose } 192865b30cdSSuzuki K Poulose 193865b30cdSSuzuki K Poulose static inline bool stage2_pud_huge(struct kvm *kvm, pud_t pud) 194865b30cdSSuzuki K Poulose { 195865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 196865b30cdSSuzuki K Poulose return pud_huge(pud); 197865b30cdSSuzuki K Poulose else 198865b30cdSSuzuki K Poulose return 0; 199865b30cdSSuzuki K Poulose } 200865b30cdSSuzuki K Poulose 201865b30cdSSuzuki K Poulose static inline bool stage2_pmd_table_empty(struct kvm *kvm, pmd_t *pmdp) 202865b30cdSSuzuki K Poulose { 203865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) 204865b30cdSSuzuki K Poulose return kvm_page_empty(pmdp); 205865b30cdSSuzuki K Poulose else 206865b30cdSSuzuki K Poulose return 0; 207865b30cdSSuzuki K Poulose } 208c0ef6326SSuzuki K Poulose 209e55cac5bSSuzuki K Poulose static inline phys_addr_t 210e55cac5bSSuzuki K Poulose stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) 211da04fa04SSuzuki K Poulose { 212865b30cdSSuzuki K Poulose if (STAGE2_PGTABLE_HAS_PMD) { 213da04fa04SSuzuki K Poulose phys_addr_t boundary = (addr + S2_PMD_SIZE) & S2_PMD_MASK; 214da04fa04SSuzuki K Poulose 215da04fa04SSuzuki K Poulose return (boundary - 1 < end - 1) ? boundary : end; 216865b30cdSSuzuki K Poulose } else { 217865b30cdSSuzuki K Poulose return end; 218865b30cdSSuzuki K Poulose } 219da04fa04SSuzuki K Poulose } 220da04fa04SSuzuki K Poulose 221865b30cdSSuzuki K Poulose static inline bool stage2_pte_table_empty(struct kvm *kvm, pte_t *ptep) 222865b30cdSSuzuki K Poulose { 223865b30cdSSuzuki K Poulose return kvm_page_empty(ptep); 224865b30cdSSuzuki K Poulose } 225c0ef6326SSuzuki K Poulose 226e55cac5bSSuzuki K Poulose #define stage2_pgd_size(kvm) (PTRS_PER_S2_PGD * sizeof(pgd_t)) 227c0ef6326SSuzuki K Poulose 228865b30cdSSuzuki K Poulose static inline unsigned long stage2_pgd_index(struct kvm *kvm, phys_addr_t addr) 229865b30cdSSuzuki K Poulose { 230865b30cdSSuzuki K Poulose return (((addr) >> S2_PGDIR_SHIFT) & (PTRS_PER_S2_PGD - 1)); 231865b30cdSSuzuki K Poulose } 232da04fa04SSuzuki K Poulose 233e55cac5bSSuzuki K Poulose static inline phys_addr_t 234e55cac5bSSuzuki K Poulose stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) 235da04fa04SSuzuki K Poulose { 236da04fa04SSuzuki K Poulose phys_addr_t boundary = (addr + S2_PGDIR_SIZE) & S2_PGDIR_MASK; 237da04fa04SSuzuki K Poulose 238da04fa04SSuzuki K Poulose return (boundary - 1 < end - 1) ? boundary : end; 239da04fa04SSuzuki K Poulose } 240c0ef6326SSuzuki K Poulose 241c0ef6326SSuzuki K Poulose #endif /* __ARM64_S2_PGTABLE_H_ */ 242