1 #ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H 2 #define _ASM_POWERPC_BOOK3S_64_HASH_64K_H 3 4 #include <asm-generic/pgtable-nopud.h> 5 6 #define PTE_INDEX_SIZE 8 7 #define PMD_INDEX_SIZE 10 8 #define PUD_INDEX_SIZE 0 9 #define PGD_INDEX_SIZE 12 10 11 #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) 12 #define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) 13 #define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) 14 15 /* With 4k base page size, hugepage PTEs go at the PMD level */ 16 #define MIN_HUGEPTE_SHIFT PAGE_SHIFT 17 18 /* PMD_SHIFT determines what a second-level page table entry can map */ 19 #define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE) 20 #define PMD_SIZE (1UL << PMD_SHIFT) 21 #define PMD_MASK (~(PMD_SIZE-1)) 22 23 /* PGDIR_SHIFT determines what a third-level page table entry can map */ 24 #define PGDIR_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) 25 #define PGDIR_SIZE (1UL << PGDIR_SHIFT) 26 #define PGDIR_MASK (~(PGDIR_SIZE-1)) 27 28 #define _PAGE_COMBO 0x00040000 /* this is a combo 4k page */ 29 #define _PAGE_4K_PFN 0x00080000 /* PFN is for a single 4k page */ 30 /* 31 * Used to track subpage group valid if _PAGE_COMBO is set 32 * This overloads _PAGE_F_GIX and _PAGE_F_SECOND 33 */ 34 #define _PAGE_COMBO_VALID (_PAGE_F_GIX | _PAGE_F_SECOND) 35 36 /* PTE flags to conserve for HPTE identification */ 37 #define _PAGE_HPTEFLAGS (_PAGE_BUSY | _PAGE_F_SECOND | \ 38 _PAGE_F_GIX | _PAGE_HASHPTE | _PAGE_COMBO) 39 40 /* Shift to put page number into pte. 41 * 42 * That gives us a max RPN of 34 bits, which means a max of 50 bits 43 * of addressable physical space, or 46 bits for the special 4k PFNs. 44 */ 45 #define PTE_RPN_SHIFT (30) 46 /* 47 * we support 16 fragments per PTE page of 64K size. 48 */ 49 #define PTE_FRAG_NR 16 50 /* 51 * We use a 2K PTE page fragment and another 2K for storing 52 * real_pte_t hash index 53 */ 54 #define PTE_FRAG_SIZE_SHIFT 12 55 #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) 56 57 /* 58 * Bits to mask out from a PMD to get to the PTE page 59 * PMDs point to PTE table fragments which are PTE_FRAG_SIZE aligned. 60 */ 61 #define PMD_MASKED_BITS (PTE_FRAG_SIZE - 1) 62 /* Bits to mask out from a PGD/PUD to get to the PMD page */ 63 #define PUD_MASKED_BITS 0x1ff 64 65 #ifndef __ASSEMBLY__ 66 67 /* 68 * With 64K pages on hash table, we have a special PTE format that 69 * uses a second "half" of the page table to encode sub-page information 70 * in order to deal with 64K made of 4K HW pages. Thus we override the 71 * generic accessors and iterators here 72 */ 73 #define __real_pte __real_pte 74 static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep) 75 { 76 real_pte_t rpte; 77 unsigned long *hidxp; 78 79 rpte.pte = pte; 80 rpte.hidx = 0; 81 if (pte_val(pte) & _PAGE_COMBO) { 82 /* 83 * Make sure we order the hidx load against the _PAGE_COMBO 84 * check. The store side ordering is done in __hash_page_4K 85 */ 86 smp_rmb(); 87 hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); 88 rpte.hidx = *hidxp; 89 } 90 return rpte; 91 } 92 93 static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) 94 { 95 if ((pte_val(rpte.pte) & _PAGE_COMBO)) 96 return (rpte.hidx >> (index<<2)) & 0xf; 97 return (pte_val(rpte.pte) >> _PAGE_F_GIX_SHIFT) & 0xf; 98 } 99 100 #define __rpte_to_pte(r) ((r).pte) 101 extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); 102 /* 103 * Trick: we set __end to va + 64k, which happens works for 104 * a 16M page as well as we want only one iteration 105 */ 106 #define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \ 107 do { \ 108 unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \ 109 unsigned __split = (psize == MMU_PAGE_4K || \ 110 psize == MMU_PAGE_64K_AP); \ 111 shift = mmu_psize_defs[psize].shift; \ 112 for (index = 0; vpn < __end; index++, \ 113 vpn += (1L << (shift - VPN_SHIFT))) { \ 114 if (!__split || __rpte_sub_valid(rpte, index)) \ 115 do { 116 117 #define pte_iterate_hashed_end() } while(0); } } while(0) 118 119 #define pte_pagesize_index(mm, addr, pte) \ 120 (((pte) & _PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) 121 122 #define remap_4k_pfn(vma, addr, pfn, prot) \ 123 (WARN_ON(((pfn) >= (1UL << (64 - PTE_RPN_SHIFT)))) ? -EINVAL : \ 124 remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, \ 125 __pgprot(pgprot_val((prot)) | _PAGE_4K_PFN))) 126 127 #define PTE_TABLE_SIZE PTE_FRAG_SIZE 128 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 129 #define PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + (sizeof(unsigned long) << PMD_INDEX_SIZE)) 130 #else 131 #define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) 132 #endif 133 #define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) 134 135 #define pgd_pte(pgd) (pud_pte(((pud_t){ pgd }))) 136 #define pte_pgd(pte) ((pgd_t)pte_pud(pte)) 137 138 #ifdef CONFIG_HUGETLB_PAGE 139 /* 140 * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have 141 * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD; 142 * 143 * Defined in such a way that we can optimize away code block at build time 144 * if CONFIG_HUGETLB_PAGE=n. 145 */ 146 static inline int pmd_huge(pmd_t pmd) 147 { 148 /* 149 * leaf pte for huge page 150 */ 151 return !!(pmd_val(pmd) & _PAGE_PTE); 152 } 153 154 static inline int pud_huge(pud_t pud) 155 { 156 /* 157 * leaf pte for huge page 158 */ 159 return !!(pud_val(pud) & _PAGE_PTE); 160 } 161 162 static inline int pgd_huge(pgd_t pgd) 163 { 164 /* 165 * leaf pte for huge page 166 */ 167 return !!(pgd_val(pgd) & _PAGE_PTE); 168 } 169 #define pgd_huge pgd_huge 170 171 #ifdef CONFIG_DEBUG_VM 172 extern int hugepd_ok(hugepd_t hpd); 173 #define is_hugepd(hpd) (hugepd_ok(hpd)) 174 #else 175 /* 176 * With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't 177 * need to setup hugepage directory for them. Our pte and page directory format 178 * enable us to have this enabled. 179 */ 180 static inline int hugepd_ok(hugepd_t hpd) 181 { 182 return 0; 183 } 184 #define is_hugepd(pdep) 0 185 #endif /* CONFIG_DEBUG_VM */ 186 187 #endif /* CONFIG_HUGETLB_PAGE */ 188 189 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 190 extern unsigned long pmd_hugepage_update(struct mm_struct *mm, 191 unsigned long addr, 192 pmd_t *pmdp, 193 unsigned long clr, 194 unsigned long set); 195 static inline char *get_hpte_slot_array(pmd_t *pmdp) 196 { 197 /* 198 * The hpte hindex is stored in the pgtable whose address is in the 199 * second half of the PMD 200 * 201 * Order this load with the test for pmd_trans_huge in the caller 202 */ 203 smp_rmb(); 204 return *(char **)(pmdp + PTRS_PER_PMD); 205 206 207 } 208 /* 209 * The linux hugepage PMD now include the pmd entries followed by the address 210 * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. 211 * [ 1 bit secondary | 3 bit hidx | 1 bit valid | 000]. We use one byte per 212 * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and 213 * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. 214 * 215 * The last three bits are intentionally left to zero. This memory location 216 * are also used as normal page PTE pointers. So if we have any pointers 217 * left around while we collapse a hugepage, we need to make sure 218 * _PAGE_PRESENT bit of that is zero when we look at them 219 */ 220 static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) 221 { 222 return (hpte_slot_array[index] >> 3) & 0x1; 223 } 224 225 static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, 226 int index) 227 { 228 return hpte_slot_array[index] >> 4; 229 } 230 231 static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, 232 unsigned int index, unsigned int hidx) 233 { 234 hpte_slot_array[index] = hidx << 4 | 0x1 << 3; 235 } 236 237 /* 238 * 239 * For core kernel code by design pmd_trans_huge is never run on any hugetlbfs 240 * page. The hugetlbfs page table walking and mangling paths are totally 241 * separated form the core VM paths and they're differentiated by 242 * VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run. 243 * 244 * pmd_trans_huge() is defined as false at build time if 245 * CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build 246 * time in such case. 247 * 248 * For ppc64 we need to differntiate from explicit hugepages from THP, because 249 * for THP we also track the subpage details at the pmd level. We don't do 250 * that for explicit huge pages. 251 * 252 */ 253 static inline int pmd_trans_huge(pmd_t pmd) 254 { 255 return !!((pmd_val(pmd) & (_PAGE_PTE | _PAGE_THP_HUGE)) == 256 (_PAGE_PTE | _PAGE_THP_HUGE)); 257 } 258 259 static inline int pmd_large(pmd_t pmd) 260 { 261 return !!(pmd_val(pmd) & _PAGE_PTE); 262 } 263 264 static inline pmd_t pmd_mknotpresent(pmd_t pmd) 265 { 266 return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT); 267 } 268 269 #define __HAVE_ARCH_PMD_SAME 270 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) 271 { 272 return (((pmd_val(pmd_a) ^ pmd_val(pmd_b)) & ~_PAGE_HPTEFLAGS) == 0); 273 } 274 275 static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, 276 unsigned long addr, pmd_t *pmdp) 277 { 278 unsigned long old; 279 280 if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) 281 return 0; 282 old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0); 283 return ((old & _PAGE_ACCESSED) != 0); 284 } 285 286 #define __HAVE_ARCH_PMDP_SET_WRPROTECT 287 static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, 288 pmd_t *pmdp) 289 { 290 291 if ((pmd_val(*pmdp) & _PAGE_RW) == 0) 292 return; 293 294 pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW, 0); 295 } 296 297 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 298 #endif /* __ASSEMBLY__ */ 299 300 #endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */ 301