1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_PGTABLE_64_H 7 #define _ASM_RISCV_PGTABLE_64_H 8 9 #include <linux/const.h> 10 11 extern bool pgtable_l4_enabled; 12 extern bool pgtable_l5_enabled; 13 14 #define PGDIR_SHIFT_L3 30 15 #define PGDIR_SHIFT_L4 39 16 #define PGDIR_SHIFT_L5 48 17 #define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) 18 19 #define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ 20 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) 21 /* Size of region mapped by a page global directory */ 22 #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) 23 #define PGDIR_MASK (~(PGDIR_SIZE - 1)) 24 25 /* p4d is folded into pgd in case of 4-level page table */ 26 #define P4D_SHIFT 39 27 #define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) 28 #define P4D_MASK (~(P4D_SIZE - 1)) 29 30 /* pud is folded into pgd in case of 3-level page table */ 31 #define PUD_SHIFT 30 32 #define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) 33 #define PUD_MASK (~(PUD_SIZE - 1)) 34 35 #define PMD_SHIFT 21 36 /* Size of region mapped by a page middle directory */ 37 #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) 38 #define PMD_MASK (~(PMD_SIZE - 1)) 39 40 /* Page 4th Directory entry */ 41 typedef struct { 42 unsigned long p4d; 43 } p4d_t; 44 45 #define p4d_val(x) ((x).p4d) 46 #define __p4d(x) ((p4d_t) { (x) }) 47 #define PTRS_PER_P4D (PAGE_SIZE / sizeof(p4d_t)) 48 49 /* Page Upper Directory entry */ 50 typedef struct { 51 unsigned long pud; 52 } pud_t; 53 54 #define pud_val(x) ((x).pud) 55 #define __pud(x) ((pud_t) { (x) }) 56 #define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t)) 57 58 /* Page Middle Directory entry */ 59 typedef struct { 60 unsigned long pmd; 61 } pmd_t; 62 63 #define pmd_val(x) ((x).pmd) 64 #define __pmd(x) ((pmd_t) { (x) }) 65 66 #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) 67 68 static inline int pud_present(pud_t pud) 69 { 70 return (pud_val(pud) & _PAGE_PRESENT); 71 } 72 73 static inline int pud_none(pud_t pud) 74 { 75 return (pud_val(pud) == 0); 76 } 77 78 static inline int pud_bad(pud_t pud) 79 { 80 return !pud_present(pud); 81 } 82 83 #define pud_leaf pud_leaf 84 static inline int pud_leaf(pud_t pud) 85 { 86 return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); 87 } 88 89 static inline int pud_user(pud_t pud) 90 { 91 return pud_val(pud) & _PAGE_USER; 92 } 93 94 static inline void set_pud(pud_t *pudp, pud_t pud) 95 { 96 *pudp = pud; 97 } 98 99 static inline void pud_clear(pud_t *pudp) 100 { 101 set_pud(pudp, __pud(0)); 102 } 103 104 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) 105 { 106 return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 107 } 108 109 static inline unsigned long _pud_pfn(pud_t pud) 110 { 111 return pud_val(pud) >> _PAGE_PFN_SHIFT; 112 } 113 114 static inline pmd_t *pud_pgtable(pud_t pud) 115 { 116 return (pmd_t *)pfn_to_virt(pud_val(pud) >> _PAGE_PFN_SHIFT); 117 } 118 119 static inline struct page *pud_page(pud_t pud) 120 { 121 return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); 122 } 123 124 #define mm_p4d_folded mm_p4d_folded 125 static inline bool mm_p4d_folded(struct mm_struct *mm) 126 { 127 if (pgtable_l5_enabled) 128 return false; 129 130 return true; 131 } 132 133 #define mm_pud_folded mm_pud_folded 134 static inline bool mm_pud_folded(struct mm_struct *mm) 135 { 136 if (pgtable_l4_enabled) 137 return false; 138 139 return true; 140 } 141 142 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) 143 144 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) 145 { 146 return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 147 } 148 149 static inline unsigned long _pmd_pfn(pmd_t pmd) 150 { 151 return pmd_val(pmd) >> _PAGE_PFN_SHIFT; 152 } 153 154 #define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) 155 156 #define pmd_ERROR(e) \ 157 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) 158 159 #define pud_ERROR(e) \ 160 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) 161 162 #define p4d_ERROR(e) \ 163 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) 164 165 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 166 { 167 if (pgtable_l4_enabled) 168 *p4dp = p4d; 169 else 170 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); 171 } 172 173 static inline int p4d_none(p4d_t p4d) 174 { 175 if (pgtable_l4_enabled) 176 return (p4d_val(p4d) == 0); 177 178 return 0; 179 } 180 181 static inline int p4d_present(p4d_t p4d) 182 { 183 if (pgtable_l4_enabled) 184 return (p4d_val(p4d) & _PAGE_PRESENT); 185 186 return 1; 187 } 188 189 static inline int p4d_bad(p4d_t p4d) 190 { 191 if (pgtable_l4_enabled) 192 return !p4d_present(p4d); 193 194 return 0; 195 } 196 197 static inline void p4d_clear(p4d_t *p4d) 198 { 199 if (pgtable_l4_enabled) 200 set_p4d(p4d, __p4d(0)); 201 } 202 203 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot) 204 { 205 return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 206 } 207 208 static inline unsigned long _p4d_pfn(p4d_t p4d) 209 { 210 return p4d_val(p4d) >> _PAGE_PFN_SHIFT; 211 } 212 213 static inline pud_t *p4d_pgtable(p4d_t p4d) 214 { 215 if (pgtable_l4_enabled) 216 return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PAGE_PFN_SHIFT); 217 218 return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); 219 } 220 #define p4d_page_vaddr(p4d) ((unsigned long)p4d_pgtable(p4d)) 221 222 static inline struct page *p4d_page(p4d_t p4d) 223 { 224 return pfn_to_page(p4d_val(p4d) >> _PAGE_PFN_SHIFT); 225 } 226 227 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 228 229 #define pud_offset pud_offset 230 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 231 { 232 if (pgtable_l4_enabled) 233 return p4d_pgtable(*p4d) + pud_index(address); 234 235 return (pud_t *)p4d; 236 } 237 238 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 239 { 240 if (pgtable_l5_enabled) 241 *pgdp = pgd; 242 else 243 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); 244 } 245 246 static inline int pgd_none(pgd_t pgd) 247 { 248 if (pgtable_l5_enabled) 249 return (pgd_val(pgd) == 0); 250 251 return 0; 252 } 253 254 static inline int pgd_present(pgd_t pgd) 255 { 256 if (pgtable_l5_enabled) 257 return (pgd_val(pgd) & _PAGE_PRESENT); 258 259 return 1; 260 } 261 262 static inline int pgd_bad(pgd_t pgd) 263 { 264 if (pgtable_l5_enabled) 265 return !pgd_present(pgd); 266 267 return 0; 268 } 269 270 static inline void pgd_clear(pgd_t *pgd) 271 { 272 if (pgtable_l5_enabled) 273 set_pgd(pgd, __pgd(0)); 274 } 275 276 static inline p4d_t *pgd_pgtable(pgd_t pgd) 277 { 278 if (pgtable_l5_enabled) 279 return (p4d_t *)pfn_to_virt(pgd_val(pgd) >> _PAGE_PFN_SHIFT); 280 281 return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); 282 } 283 #define pgd_page_vaddr(pgd) ((unsigned long)pgd_pgtable(pgd)) 284 285 static inline struct page *pgd_page(pgd_t pgd) 286 { 287 return pfn_to_page(pgd_val(pgd) >> _PAGE_PFN_SHIFT); 288 } 289 #define pgd_page(pgd) pgd_page(pgd) 290 291 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) 292 293 #define p4d_offset p4d_offset 294 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) 295 { 296 if (pgtable_l5_enabled) 297 return pgd_pgtable(*pgd) + p4d_index(address); 298 299 return (p4d_t *)pgd; 300 } 301 302 #endif /* _ASM_RISCV_PGTABLE_64_H */ 303