1 /* 2 * IBM System z Huge TLB Page Support for Kernel. 3 * 4 * Copyright IBM Corp. 2007 5 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/hugetlb.h> 10 11 static inline pmd_t __pte_to_pmd(pte_t pte) 12 { 13 int none, young, prot; 14 pmd_t pmd; 15 16 /* 17 * Convert encoding pte bits pmd bits 18 * .IR...wrdytp ..R...I...y. 19 * empty .10...000000 -> ..0...1...0. 20 * prot-none, clean, old .11...000001 -> ..0...1...1. 21 * prot-none, clean, young .11...000101 -> ..1...1...1. 22 * prot-none, dirty, old .10...001001 -> ..0...1...1. 23 * prot-none, dirty, young .10...001101 -> ..1...1...1. 24 * read-only, clean, old .11...010001 -> ..1...1...0. 25 * read-only, clean, young .01...010101 -> ..1...0...1. 26 * read-only, dirty, old .11...011001 -> ..1...1...0. 27 * read-only, dirty, young .01...011101 -> ..1...0...1. 28 * read-write, clean, old .11...110001 -> ..0...1...0. 29 * read-write, clean, young .01...110101 -> ..0...0...1. 30 * read-write, dirty, old .10...111001 -> ..0...1...0. 31 * read-write, dirty, young .00...111101 -> ..0...0...1. 32 * Huge ptes are dirty by definition, a clean pte is made dirty 33 * by the conversion. 34 */ 35 if (pte_present(pte)) { 36 pmd_val(pmd) = pte_val(pte) & PAGE_MASK; 37 if (pte_val(pte) & _PAGE_INVALID) 38 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID; 39 none = (pte_val(pte) & _PAGE_PRESENT) && 40 !(pte_val(pte) & _PAGE_READ) && 41 !(pte_val(pte) & _PAGE_WRITE); 42 prot = (pte_val(pte) & _PAGE_PROTECT) && 43 !(pte_val(pte) & _PAGE_WRITE); 44 young = pte_val(pte) & _PAGE_YOUNG; 45 if (none || young) 46 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG; 47 if (prot || (none && young)) 48 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; 49 } else 50 pmd_val(pmd) = _SEGMENT_ENTRY_INVALID; 51 return pmd; 52 } 53 54 static inline pte_t __pmd_to_pte(pmd_t pmd) 55 { 56 pte_t pte; 57 58 /* 59 * Convert encoding pmd bits pte bits 60 * ..R...I...y. .IR...wrdytp 61 * empty ..0...1...0. -> .10...000000 62 * prot-none, old ..0...1...1. -> .10...001001 63 * prot-none, young ..1...1...1. -> .10...001101 64 * read-only, old ..1...1...0. -> .11...011001 65 * read-only, young ..1...0...1. -> .01...011101 66 * read-write, old ..0...1...0. -> .10...111001 67 * read-write, young ..0...0...1. -> .00...111101 68 * Huge ptes are dirty by definition 69 */ 70 if (pmd_present(pmd)) { 71 pte_val(pte) = _PAGE_PRESENT | _PAGE_LARGE | _PAGE_DIRTY | 72 (pmd_val(pmd) & PAGE_MASK); 73 if (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) 74 pte_val(pte) |= _PAGE_INVALID; 75 if (pmd_prot_none(pmd)) { 76 if (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) 77 pte_val(pte) |= _PAGE_YOUNG; 78 } else { 79 pte_val(pte) |= _PAGE_READ; 80 if (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) 81 pte_val(pte) |= _PAGE_PROTECT; 82 else 83 pte_val(pte) |= _PAGE_WRITE; 84 if (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) 85 pte_val(pte) |= _PAGE_YOUNG; 86 } 87 } else 88 pte_val(pte) = _PAGE_INVALID; 89 return pte; 90 } 91 92 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 93 pte_t *ptep, pte_t pte) 94 { 95 pmd_t pmd; 96 97 pmd = __pte_to_pmd(pte); 98 if (!MACHINE_HAS_HPAGE) { 99 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN; 100 pmd_val(pmd) |= pte_page(pte)[1].index; 101 } else 102 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO; 103 *(pmd_t *) ptep = pmd; 104 } 105 106 pte_t huge_ptep_get(pte_t *ptep) 107 { 108 unsigned long origin; 109 pmd_t pmd; 110 111 pmd = *(pmd_t *) ptep; 112 if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) { 113 origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN; 114 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN; 115 pmd_val(pmd) |= *(unsigned long *) origin; 116 } 117 return __pmd_to_pte(pmd); 118 } 119 120 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, 121 unsigned long addr, pte_t *ptep) 122 { 123 pmd_t *pmdp = (pmd_t *) ptep; 124 pte_t pte = huge_ptep_get(ptep); 125 126 pmdp_flush_direct(mm, addr, pmdp); 127 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; 128 return pte; 129 } 130 131 int arch_prepare_hugepage(struct page *page) 132 { 133 unsigned long addr = page_to_phys(page); 134 pte_t pte; 135 pte_t *ptep; 136 int i; 137 138 if (MACHINE_HAS_HPAGE) 139 return 0; 140 141 ptep = (pte_t *) pte_alloc_one(&init_mm, addr); 142 if (!ptep) 143 return -ENOMEM; 144 145 pte_val(pte) = addr; 146 for (i = 0; i < PTRS_PER_PTE; i++) { 147 set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte); 148 pte_val(pte) += PAGE_SIZE; 149 } 150 page[1].index = (unsigned long) ptep; 151 return 0; 152 } 153 154 void arch_release_hugepage(struct page *page) 155 { 156 pte_t *ptep; 157 158 if (MACHINE_HAS_HPAGE) 159 return; 160 161 ptep = (pte_t *) page[1].index; 162 if (!ptep) 163 return; 164 clear_table((unsigned long *) ptep, _PAGE_INVALID, 165 PTRS_PER_PTE * sizeof(pte_t)); 166 page_table_free(&init_mm, (unsigned long *) ptep); 167 page[1].index = 0; 168 } 169 170 pte_t *huge_pte_alloc(struct mm_struct *mm, 171 unsigned long addr, unsigned long sz) 172 { 173 pgd_t *pgdp; 174 pud_t *pudp; 175 pmd_t *pmdp = NULL; 176 177 pgdp = pgd_offset(mm, addr); 178 pudp = pud_alloc(mm, pgdp, addr); 179 if (pudp) 180 pmdp = pmd_alloc(mm, pudp, addr); 181 return (pte_t *) pmdp; 182 } 183 184 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 185 { 186 pgd_t *pgdp; 187 pud_t *pudp; 188 pmd_t *pmdp = NULL; 189 190 pgdp = pgd_offset(mm, addr); 191 if (pgd_present(*pgdp)) { 192 pudp = pud_offset(pgdp, addr); 193 if (pud_present(*pudp)) 194 pmdp = pmd_offset(pudp, addr); 195 } 196 return (pte_t *) pmdp; 197 } 198 199 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 200 { 201 return 0; 202 } 203 204 struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, 205 int write) 206 { 207 return ERR_PTR(-EINVAL); 208 } 209 210 int pmd_huge(pmd_t pmd) 211 { 212 if (!MACHINE_HAS_HPAGE) 213 return 0; 214 215 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE); 216 } 217 218 int pud_huge(pud_t pud) 219 { 220 return 0; 221 } 222 223 struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, 224 pmd_t *pmdp, int write) 225 { 226 struct page *page; 227 228 if (!MACHINE_HAS_HPAGE) 229 return NULL; 230 231 page = pmd_page(*pmdp); 232 if (page) 233 page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT); 234 return page; 235 } 236