1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_PGTABLE_64_H 7 #define _ASM_RISCV_PGTABLE_64_H 8 9 #include <linux/const.h> 10 11 extern bool pgtable_l4_enabled; 12 extern bool pgtable_l5_enabled; 13 14 #define PGDIR_SHIFT_L3 30 15 #define PGDIR_SHIFT_L4 39 16 #define PGDIR_SHIFT_L5 48 17 #define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) 18 19 #define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ 20 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) 21 /* Size of region mapped by a page global directory */ 22 #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) 23 #define PGDIR_MASK (~(PGDIR_SIZE - 1)) 24 25 /* p4d is folded into pgd in case of 4-level page table */ 26 #define P4D_SHIFT 39 27 #define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) 28 #define P4D_MASK (~(P4D_SIZE - 1)) 29 30 /* pud is folded into pgd in case of 3-level page table */ 31 #define PUD_SHIFT 30 32 #define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) 33 #define PUD_MASK (~(PUD_SIZE - 1)) 34 35 #define PMD_SHIFT 21 36 /* Size of region mapped by a page middle directory */ 37 #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) 38 #define PMD_MASK (~(PMD_SIZE - 1)) 39 40 /* Page 4th Directory entry */ 41 typedef struct { 42 unsigned long p4d; 43 } p4d_t; 44 45 #define p4d_val(x) ((x).p4d) 46 #define __p4d(x) ((p4d_t) { (x) }) 47 #define PTRS_PER_P4D (PAGE_SIZE / sizeof(p4d_t)) 48 49 /* Page Upper Directory entry */ 50 typedef struct { 51 unsigned long pud; 52 } pud_t; 53 54 #define pud_val(x) ((x).pud) 55 #define __pud(x) ((pud_t) { (x) }) 56 #define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t)) 57 58 /* Page Middle Directory entry */ 59 typedef struct { 60 unsigned long pmd; 61 } pmd_t; 62 63 #define pmd_val(x) ((x).pmd) 64 #define __pmd(x) ((pmd_t) { (x) }) 65 66 #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) 67 68 static inline int pud_present(pud_t pud) 69 { 70 return (pud_val(pud) & _PAGE_PRESENT); 71 } 72 73 static inline int pud_none(pud_t pud) 74 { 75 return (pud_val(pud) == 0); 76 } 77 78 static inline int pud_bad(pud_t pud) 79 { 80 return !pud_present(pud); 81 } 82 83 #define pud_leaf pud_leaf 84 static inline int pud_leaf(pud_t pud) 85 { 86 return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); 87 } 88 89 static inline void set_pud(pud_t *pudp, pud_t pud) 90 { 91 *pudp = pud; 92 } 93 94 static inline void pud_clear(pud_t *pudp) 95 { 96 set_pud(pudp, __pud(0)); 97 } 98 99 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) 100 { 101 return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 102 } 103 104 static inline unsigned long _pud_pfn(pud_t pud) 105 { 106 return pud_val(pud) >> _PAGE_PFN_SHIFT; 107 } 108 109 static inline pmd_t *pud_pgtable(pud_t pud) 110 { 111 return (pmd_t *)pfn_to_virt(pud_val(pud) >> _PAGE_PFN_SHIFT); 112 } 113 114 static inline struct page *pud_page(pud_t pud) 115 { 116 return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); 117 } 118 119 #define mm_p4d_folded mm_p4d_folded 120 static inline bool mm_p4d_folded(struct mm_struct *mm) 121 { 122 if (pgtable_l5_enabled) 123 return false; 124 125 return true; 126 } 127 128 #define mm_pud_folded mm_pud_folded 129 static inline bool mm_pud_folded(struct mm_struct *mm) 130 { 131 if (pgtable_l4_enabled) 132 return false; 133 134 return true; 135 } 136 137 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) 138 139 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) 140 { 141 return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 142 } 143 144 static inline unsigned long _pmd_pfn(pmd_t pmd) 145 { 146 return pmd_val(pmd) >> _PAGE_PFN_SHIFT; 147 } 148 149 #define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) 150 151 #define pmd_ERROR(e) \ 152 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) 153 154 #define pud_ERROR(e) \ 155 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) 156 157 #define p4d_ERROR(e) \ 158 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) 159 160 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 161 { 162 if (pgtable_l4_enabled) 163 *p4dp = p4d; 164 else 165 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); 166 } 167 168 static inline int p4d_none(p4d_t p4d) 169 { 170 if (pgtable_l4_enabled) 171 return (p4d_val(p4d) == 0); 172 173 return 0; 174 } 175 176 static inline int p4d_present(p4d_t p4d) 177 { 178 if (pgtable_l4_enabled) 179 return (p4d_val(p4d) & _PAGE_PRESENT); 180 181 return 1; 182 } 183 184 static inline int p4d_bad(p4d_t p4d) 185 { 186 if (pgtable_l4_enabled) 187 return !p4d_present(p4d); 188 189 return 0; 190 } 191 192 static inline void p4d_clear(p4d_t *p4d) 193 { 194 if (pgtable_l4_enabled) 195 set_p4d(p4d, __p4d(0)); 196 } 197 198 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot) 199 { 200 return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 201 } 202 203 static inline unsigned long _p4d_pfn(p4d_t p4d) 204 { 205 return p4d_val(p4d) >> _PAGE_PFN_SHIFT; 206 } 207 208 static inline pud_t *p4d_pgtable(p4d_t p4d) 209 { 210 if (pgtable_l4_enabled) 211 return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PAGE_PFN_SHIFT); 212 213 return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); 214 } 215 #define p4d_page_vaddr(p4d) ((unsigned long)p4d_pgtable(p4d)) 216 217 static inline struct page *p4d_page(p4d_t p4d) 218 { 219 return pfn_to_page(p4d_val(p4d) >> _PAGE_PFN_SHIFT); 220 } 221 222 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 223 224 #define pud_offset pud_offset 225 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 226 { 227 if (pgtable_l4_enabled) 228 return p4d_pgtable(*p4d) + pud_index(address); 229 230 return (pud_t *)p4d; 231 } 232 233 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 234 { 235 if (pgtable_l5_enabled) 236 *pgdp = pgd; 237 else 238 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); 239 } 240 241 static inline int pgd_none(pgd_t pgd) 242 { 243 if (pgtable_l5_enabled) 244 return (pgd_val(pgd) == 0); 245 246 return 0; 247 } 248 249 static inline int pgd_present(pgd_t pgd) 250 { 251 if (pgtable_l5_enabled) 252 return (pgd_val(pgd) & _PAGE_PRESENT); 253 254 return 1; 255 } 256 257 static inline int pgd_bad(pgd_t pgd) 258 { 259 if (pgtable_l5_enabled) 260 return !pgd_present(pgd); 261 262 return 0; 263 } 264 265 static inline void pgd_clear(pgd_t *pgd) 266 { 267 if (pgtable_l5_enabled) 268 set_pgd(pgd, __pgd(0)); 269 } 270 271 static inline p4d_t *pgd_pgtable(pgd_t pgd) 272 { 273 if (pgtable_l5_enabled) 274 return (p4d_t *)pfn_to_virt(pgd_val(pgd) >> _PAGE_PFN_SHIFT); 275 276 return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); 277 } 278 #define pgd_page_vaddr(pgd) ((unsigned long)pgd_pgtable(pgd)) 279 280 static inline struct page *pgd_page(pgd_t pgd) 281 { 282 return pfn_to_page(pgd_val(pgd) >> _PAGE_PFN_SHIFT); 283 } 284 #define pgd_page(pgd) pgd_page(pgd) 285 286 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) 287 288 #define p4d_offset p4d_offset 289 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) 290 { 291 if (pgtable_l5_enabled) 292 return pgd_pgtable(*pgd) + p4d_index(address); 293 294 return (p4d_t *)pgd; 295 } 296 297 #endif /* _ASM_RISCV_PGTABLE_64_H */ 298