1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle 7 * Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc. 8 */ 9 #ifndef _ASM_PGTABLE_64_H 10 #define _ASM_PGTABLE_64_H 11 12 #include <linux/compiler.h> 13 #include <linux/linkage.h> 14 15 #include <asm/addrspace.h> 16 #include <asm/page.h> 17 #include <asm/cachectl.h> 18 #include <asm/fixmap.h> 19 20 #define __ARCH_USE_5LEVEL_HACK 21 #if defined(CONFIG_PAGE_SIZE_64KB) && !defined(CONFIG_MIPS_VA_BITS_48) 22 #include <asm-generic/pgtable-nopmd.h> 23 #elif !(defined(CONFIG_PAGE_SIZE_4KB) && defined(CONFIG_MIPS_VA_BITS_48)) 24 #include <asm-generic/pgtable-nopud.h> 25 #endif 26 27 /* 28 * Each address space has 2 4K pages as its page directory, giving 1024 29 * (== PTRS_PER_PGD) 8 byte pointers to pmd tables. Each pmd table is a 30 * single 4K page, giving 512 (== PTRS_PER_PMD) 8 byte pointers to page 31 * tables. Each page table is also a single 4K page, giving 512 (== 32 * PTRS_PER_PTE) 8 byte ptes. Each pud entry is initialized to point to 33 * invalid_pmd_table, each pmd entry is initialized to point to 34 * invalid_pte_table, each pte is initialized to 0. 35 * 36 * Kernel mappings: kernel mappings are held in the swapper_pg_table. 37 * The layout is identical to userspace except it's indexed with the 38 * fault address - VMALLOC_START. 39 */ 40 41 42 /* PGDIR_SHIFT determines what a third-level page table entry can map */ 43 #ifdef __PAGETABLE_PMD_FOLDED 44 #define PGDIR_SHIFT (PAGE_SHIFT + PAGE_SHIFT + PTE_ORDER - 3) 45 #else 46 47 /* PMD_SHIFT determines the size of the area a second-level page table can map */ 48 #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) 49 #define PMD_SIZE (1UL << PMD_SHIFT) 50 #define PMD_MASK (~(PMD_SIZE-1)) 51 52 # ifdef __PAGETABLE_PUD_FOLDED 53 # define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) 54 # endif 55 #endif 56 57 #ifndef __PAGETABLE_PUD_FOLDED 58 #define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) 59 #define PUD_SIZE (1UL << PUD_SHIFT) 60 #define PUD_MASK (~(PUD_SIZE-1)) 61 #define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_ORDER - 3)) 62 #endif 63 64 #define PGDIR_SIZE (1UL << PGDIR_SHIFT) 65 #define PGDIR_MASK (~(PGDIR_SIZE-1)) 66 67 /* 68 * For 4kB page size we use a 3 level page tree and an 8kB pud, which 69 * permits us mapping 40 bits of virtual address space. 70 * 71 * We used to implement 41 bits by having an order 1 pmd level but that seemed 72 * rather pointless. 73 * 74 * For 8kB page size we use a 3 level page tree which permits a total of 75 * 8TB of address space. Alternatively a 33-bit / 8GB organization using 76 * two levels would be easy to implement. 77 * 78 * For 16kB page size we use a 2 level page tree which permits a total of 79 * 36 bits of virtual address space. We could add a third level but it seems 80 * like at the moment there's no need for this. 81 * 82 * For 64kB page size we use a 2 level page table tree for a total of 42 bits 83 * of virtual address space. 84 */ 85 #ifdef CONFIG_PAGE_SIZE_4KB 86 # ifdef CONFIG_MIPS_VA_BITS_48 87 # define PGD_ORDER 0 88 # define PUD_ORDER 0 89 # else 90 # define PGD_ORDER 1 91 # define PUD_ORDER aieeee_attempt_to_allocate_pud 92 # endif 93 #define PMD_ORDER 0 94 #define PTE_ORDER 0 95 #endif 96 #ifdef CONFIG_PAGE_SIZE_8KB 97 #define PGD_ORDER 0 98 #define PUD_ORDER aieeee_attempt_to_allocate_pud 99 #define PMD_ORDER 0 100 #define PTE_ORDER 0 101 #endif 102 #ifdef CONFIG_PAGE_SIZE_16KB 103 #ifdef CONFIG_MIPS_VA_BITS_48 104 #define PGD_ORDER 1 105 #else 106 #define PGD_ORDER 0 107 #endif 108 #define PUD_ORDER aieeee_attempt_to_allocate_pud 109 #define PMD_ORDER 0 110 #define PTE_ORDER 0 111 #endif 112 #ifdef CONFIG_PAGE_SIZE_32KB 113 #define PGD_ORDER 0 114 #define PUD_ORDER aieeee_attempt_to_allocate_pud 115 #define PMD_ORDER 0 116 #define PTE_ORDER 0 117 #endif 118 #ifdef CONFIG_PAGE_SIZE_64KB 119 #define PGD_ORDER 0 120 #define PUD_ORDER aieeee_attempt_to_allocate_pud 121 #ifdef CONFIG_MIPS_VA_BITS_48 122 #define PMD_ORDER 0 123 #else 124 #define PMD_ORDER aieeee_attempt_to_allocate_pmd 125 #endif 126 #define PTE_ORDER 0 127 #endif 128 129 #define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) / sizeof(pgd_t)) 130 #ifndef __PAGETABLE_PUD_FOLDED 131 #define PTRS_PER_PUD ((PAGE_SIZE << PUD_ORDER) / sizeof(pud_t)) 132 #endif 133 #ifndef __PAGETABLE_PMD_FOLDED 134 #define PTRS_PER_PMD ((PAGE_SIZE << PMD_ORDER) / sizeof(pmd_t)) 135 #endif 136 #define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) / sizeof(pte_t)) 137 138 #define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) 139 #define FIRST_USER_ADDRESS 0UL 140 141 /* 142 * TLB refill handlers also map the vmalloc area into xuseg. Avoid 143 * the first couple of pages so NULL pointer dereferences will still 144 * reliably trap. 145 */ 146 #define VMALLOC_START (MAP_BASE + (2 * PAGE_SIZE)) 147 #define VMALLOC_END \ 148 (MAP_BASE + \ 149 min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, \ 150 (1UL << cpu_vmbits)) - (1UL << 32)) 151 152 #if defined(CONFIG_MODULES) && defined(KBUILD_64BIT_SYM32) && \ 153 VMALLOC_START != CKSSEG 154 /* Load modules into 32bit-compatible segment. */ 155 #define MODULE_START CKSSEG 156 #define MODULE_END (FIXADDR_START-2*PAGE_SIZE) 157 #endif 158 159 #define pte_ERROR(e) \ 160 printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) 161 #ifndef __PAGETABLE_PMD_FOLDED 162 #define pmd_ERROR(e) \ 163 printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) 164 #endif 165 #ifndef __PAGETABLE_PUD_FOLDED 166 #define pud_ERROR(e) \ 167 printk("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) 168 #endif 169 #define pgd_ERROR(e) \ 170 printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) 171 172 extern pte_t invalid_pte_table[PTRS_PER_PTE]; 173 174 #ifndef __PAGETABLE_PUD_FOLDED 175 /* 176 * For 4-level pagetables we defines these ourselves, for 3-level the 177 * definitions are below, for 2-level the 178 * definitions are supplied by <asm-generic/pgtable-nopmd.h>. 179 */ 180 typedef struct { unsigned long pud; } pud_t; 181 #define pud_val(x) ((x).pud) 182 #define __pud(x) ((pud_t) { (x) }) 183 184 extern pud_t invalid_pud_table[PTRS_PER_PUD]; 185 186 /* 187 * Empty pgd entries point to the invalid_pud_table. 188 */ 189 static inline int pgd_none(pgd_t pgd) 190 { 191 return pgd_val(pgd) == (unsigned long)invalid_pud_table; 192 } 193 194 static inline int pgd_bad(pgd_t pgd) 195 { 196 if (unlikely(pgd_val(pgd) & ~PAGE_MASK)) 197 return 1; 198 199 return 0; 200 } 201 202 static inline int pgd_present(pgd_t pgd) 203 { 204 return pgd_val(pgd) != (unsigned long)invalid_pud_table; 205 } 206 207 static inline void pgd_clear(pgd_t *pgdp) 208 { 209 pgd_val(*pgdp) = (unsigned long)invalid_pud_table; 210 } 211 212 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 213 214 static inline unsigned long pgd_page_vaddr(pgd_t pgd) 215 { 216 return pgd_val(pgd); 217 } 218 219 static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) 220 { 221 return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address); 222 } 223 224 static inline void set_pgd(pgd_t *pgd, pgd_t pgdval) 225 { 226 *pgd = pgdval; 227 } 228 229 #endif 230 231 #ifndef __PAGETABLE_PMD_FOLDED 232 /* 233 * For 3-level pagetables we defines these ourselves, for 2-level the 234 * definitions are supplied by <asm-generic/pgtable-nopmd.h>. 235 */ 236 typedef struct { unsigned long pmd; } pmd_t; 237 #define pmd_val(x) ((x).pmd) 238 #define __pmd(x) ((pmd_t) { (x) } ) 239 240 241 extern pmd_t invalid_pmd_table[PTRS_PER_PMD]; 242 #endif 243 244 /* 245 * Empty pgd/pmd entries point to the invalid_pte_table. 246 */ 247 static inline int pmd_none(pmd_t pmd) 248 { 249 return pmd_val(pmd) == (unsigned long) invalid_pte_table; 250 } 251 252 static inline int pmd_bad(pmd_t pmd) 253 { 254 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 255 /* pmd_huge(pmd) but inline */ 256 if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) 257 return 0; 258 #endif 259 260 if (unlikely(pmd_val(pmd) & ~PAGE_MASK)) 261 return 1; 262 263 return 0; 264 } 265 266 static inline int pmd_present(pmd_t pmd) 267 { 268 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 269 if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) 270 return pmd_val(pmd) & _PAGE_PRESENT; 271 #endif 272 273 return pmd_val(pmd) != (unsigned long) invalid_pte_table; 274 } 275 276 static inline void pmd_clear(pmd_t *pmdp) 277 { 278 pmd_val(*pmdp) = ((unsigned long) invalid_pte_table); 279 } 280 #ifndef __PAGETABLE_PMD_FOLDED 281 282 /* 283 * Empty pud entries point to the invalid_pmd_table. 284 */ 285 static inline int pud_none(pud_t pud) 286 { 287 return pud_val(pud) == (unsigned long) invalid_pmd_table; 288 } 289 290 static inline int pud_bad(pud_t pud) 291 { 292 return pud_val(pud) & ~PAGE_MASK; 293 } 294 295 static inline int pud_present(pud_t pud) 296 { 297 return pud_val(pud) != (unsigned long) invalid_pmd_table; 298 } 299 300 static inline void pud_clear(pud_t *pudp) 301 { 302 pud_val(*pudp) = ((unsigned long) invalid_pmd_table); 303 } 304 #endif 305 306 #define pte_page(x) pfn_to_page(pte_pfn(x)) 307 308 #ifdef CONFIG_CPU_VR41XX 309 #define pte_pfn(x) ((unsigned long)((x).pte >> (PAGE_SHIFT + 2))) 310 #define pfn_pte(pfn, prot) __pte(((pfn) << (PAGE_SHIFT + 2)) | pgprot_val(prot)) 311 #else 312 #define pte_pfn(x) ((unsigned long)((x).pte >> _PFN_SHIFT)) 313 #define pfn_pte(pfn, prot) __pte(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) 314 #define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) 315 #endif 316 317 #define __pgd_offset(address) pgd_index(address) 318 #define __pud_offset(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) 319 #define __pmd_offset(address) pmd_index(address) 320 321 /* to find an entry in a kernel page-table-directory */ 322 #define pgd_offset_k(address) pgd_offset(&init_mm, address) 323 324 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) 325 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) 326 327 /* to find an entry in a page-table-directory */ 328 #define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) 329 330 #ifndef __PAGETABLE_PMD_FOLDED 331 static inline unsigned long pud_page_vaddr(pud_t pud) 332 { 333 return pud_val(pud); 334 } 335 #define pud_phys(pud) virt_to_phys((void *)pud_val(pud)) 336 #define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) 337 338 /* Find an entry in the second-level page table.. */ 339 static inline pmd_t *pmd_offset(pud_t * pud, unsigned long address) 340 { 341 return (pmd_t *) pud_page_vaddr(*pud) + pmd_index(address); 342 } 343 #endif 344 345 /* Find an entry in the third-level page table.. */ 346 #define __pte_offset(address) \ 347 (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) 348 #define pte_offset(dir, address) \ 349 ((pte_t *) pmd_page_vaddr(*(dir)) + __pte_offset(address)) 350 #define pte_offset_kernel(dir, address) \ 351 ((pte_t *) pmd_page_vaddr(*(dir)) + __pte_offset(address)) 352 #define pte_offset_map(dir, address) \ 353 ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset(address)) 354 #define pte_unmap(pte) ((void)(pte)) 355 356 /* 357 * Initialize a new pgd / pmd table with invalid pointers. 358 */ 359 extern void pgd_init(unsigned long page); 360 extern void pud_init(unsigned long page, unsigned long pagetable); 361 extern void pmd_init(unsigned long page, unsigned long pagetable); 362 363 /* 364 * Non-present pages: high 40 bits are offset, next 8 bits type, 365 * low 16 bits zero. 366 */ 367 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) 368 { pte_t pte; pte_val(pte) = (type << 16) | (offset << 24); return pte; } 369 370 #define __swp_type(x) (((x).val >> 16) & 0xff) 371 #define __swp_offset(x) ((x).val >> 24) 372 #define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) 373 #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) 374 #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) 375 376 #endif /* _ASM_PGTABLE_64_H */ 377