1 #ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H 2 #define _ASM_POWERPC_BOOK3S_64_HASH_64K_H 3 4 #define H_PTE_INDEX_SIZE 8 5 #define H_PMD_INDEX_SIZE 5 6 #define H_PUD_INDEX_SIZE 5 7 #define H_PGD_INDEX_SIZE 12 8 9 #define H_PAGE_COMBO 0x00001000 /* this is a combo 4k page */ 10 #define H_PAGE_4K_PFN 0x00002000 /* PFN is for a single 4k page */ 11 /* 12 * We need to differentiate between explicit huge page and THP huge 13 * page, since THP huge page also need to track real subpage details 14 */ 15 #define H_PAGE_THP_HUGE H_PAGE_4K_PFN 16 17 /* 18 * Used to track subpage group valid if H_PAGE_COMBO is set 19 * This overloads H_PAGE_F_GIX and H_PAGE_F_SECOND 20 */ 21 #define H_PAGE_COMBO_VALID (H_PAGE_F_GIX | H_PAGE_F_SECOND) 22 23 /* PTE flags to conserve for HPTE identification */ 24 #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \ 25 H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO) 26 /* 27 * we support 16 fragments per PTE page of 64K size. 28 */ 29 #define H_PTE_FRAG_NR 16 30 /* 31 * We use a 2K PTE page fragment and another 2K for storing 32 * real_pte_t hash index 33 */ 34 #define H_PTE_FRAG_SIZE_SHIFT 12 35 #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) 36 37 #ifndef __ASSEMBLY__ 38 #include <asm/errno.h> 39 40 /* 41 * With 64K pages on hash table, we have a special PTE format that 42 * uses a second "half" of the page table to encode sub-page information 43 * in order to deal with 64K made of 4K HW pages. Thus we override the 44 * generic accessors and iterators here 45 */ 46 #define __real_pte __real_pte 47 static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep) 48 { 49 real_pte_t rpte; 50 unsigned long *hidxp; 51 52 rpte.pte = pte; 53 rpte.hidx = 0; 54 if (pte_val(pte) & H_PAGE_COMBO) { 55 /* 56 * Make sure we order the hidx load against the H_PAGE_COMBO 57 * check. The store side ordering is done in __hash_page_4K 58 */ 59 smp_rmb(); 60 hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); 61 rpte.hidx = *hidxp; 62 } 63 return rpte; 64 } 65 66 static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) 67 { 68 if ((pte_val(rpte.pte) & H_PAGE_COMBO)) 69 return (rpte.hidx >> (index<<2)) & 0xf; 70 return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf; 71 } 72 73 #define __rpte_to_pte(r) ((r).pte) 74 extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); 75 /* 76 * Trick: we set __end to va + 64k, which happens works for 77 * a 16M page as well as we want only one iteration 78 */ 79 #define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \ 80 do { \ 81 unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \ 82 unsigned __split = (psize == MMU_PAGE_4K || \ 83 psize == MMU_PAGE_64K_AP); \ 84 shift = mmu_psize_defs[psize].shift; \ 85 for (index = 0; vpn < __end; index++, \ 86 vpn += (1L << (shift - VPN_SHIFT))) { \ 87 if (!__split || __rpte_sub_valid(rpte, index)) \ 88 do { 89 90 #define pte_iterate_hashed_end() } while(0); } } while(0) 91 92 #define pte_pagesize_index(mm, addr, pte) \ 93 (((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) 94 95 extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr, 96 unsigned long pfn, unsigned long size, pgprot_t); 97 static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr, 98 unsigned long pfn, pgprot_t prot) 99 { 100 if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) { 101 WARN(1, "remap_4k_pfn called with wrong pfn value\n"); 102 return -EINVAL; 103 } 104 return remap_pfn_range(vma, addr, pfn, PAGE_SIZE, 105 __pgprot(pgprot_val(prot) | H_PAGE_4K_PFN)); 106 } 107 108 #define H_PTE_TABLE_SIZE PTE_FRAG_SIZE 109 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 110 #define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \ 111 (sizeof(unsigned long) << PMD_INDEX_SIZE)) 112 #else 113 #define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) 114 #endif 115 #define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE) 116 #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) 117 118 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 119 static inline char *get_hpte_slot_array(pmd_t *pmdp) 120 { 121 /* 122 * The hpte hindex is stored in the pgtable whose address is in the 123 * second half of the PMD 124 * 125 * Order this load with the test for pmd_trans_huge in the caller 126 */ 127 smp_rmb(); 128 return *(char **)(pmdp + PTRS_PER_PMD); 129 130 131 } 132 /* 133 * The linux hugepage PMD now include the pmd entries followed by the address 134 * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. 135 * [ 000 | 1 bit secondary | 3 bit hidx | 1 bit valid]. We use one byte per 136 * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and 137 * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. 138 * 139 * The top three bits are intentionally left as zero. This memory location 140 * are also used as normal page PTE pointers. So if we have any pointers 141 * left around while we collapse a hugepage, we need to make sure 142 * _PAGE_PRESENT bit of that is zero when we look at them 143 */ 144 static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) 145 { 146 return hpte_slot_array[index] & 0x1; 147 } 148 149 static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, 150 int index) 151 { 152 return hpte_slot_array[index] >> 1; 153 } 154 155 static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, 156 unsigned int index, unsigned int hidx) 157 { 158 hpte_slot_array[index] = (hidx << 1) | 0x1; 159 } 160 161 /* 162 * 163 * For core kernel code by design pmd_trans_huge is never run on any hugetlbfs 164 * page. The hugetlbfs page table walking and mangling paths are totally 165 * separated form the core VM paths and they're differentiated by 166 * VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run. 167 * 168 * pmd_trans_huge() is defined as false at build time if 169 * CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build 170 * time in such case. 171 * 172 * For ppc64 we need to differntiate from explicit hugepages from THP, because 173 * for THP we also track the subpage details at the pmd level. We don't do 174 * that for explicit huge pages. 175 * 176 */ 177 static inline int hash__pmd_trans_huge(pmd_t pmd) 178 { 179 return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) == 180 (_PAGE_PTE | H_PAGE_THP_HUGE)); 181 } 182 183 static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b) 184 { 185 return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); 186 } 187 188 static inline pmd_t hash__pmd_mkhuge(pmd_t pmd) 189 { 190 return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE)); 191 } 192 193 extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, 194 unsigned long addr, pmd_t *pmdp, 195 unsigned long clr, unsigned long set); 196 extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, 197 unsigned long address, pmd_t *pmdp); 198 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 199 pgtable_t pgtable); 200 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 201 extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma, 202 unsigned long address, pmd_t *pmdp); 203 extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, 204 unsigned long addr, pmd_t *pmdp); 205 extern int hash__has_transparent_hugepage(void); 206 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 207 #endif /* __ASSEMBLY__ */ 208 209 #endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */ 210