1 /* 2 * IBM System z Huge TLB Page Support for Kernel. 3 * 4 * Copyright IBM Corp. 2007,2016 5 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> 6 */ 7 8 #define KMSG_COMPONENT "hugetlb" 9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 14 /* 15 * If the bit selected by single-bit bitmask "a" is set within "x", move 16 * it to the position indicated by single-bit bitmask "b". 17 */ 18 #define move_set_bit(x, a, b) (((x) & (a)) >> ilog2(a) << ilog2(b)) 19 20 static inline unsigned long __pte_to_rste(pte_t pte) 21 { 22 unsigned long rste; 23 24 /* 25 * Convert encoding pte bits pmd / pud bits 26 * lIR.uswrdy.p dy..R...I...wr 27 * empty 010.000000.0 -> 00..0...1...00 28 * prot-none, clean, old 111.000000.1 -> 00..1...1...00 29 * prot-none, clean, young 111.000001.1 -> 01..1...1...00 30 * prot-none, dirty, old 111.000010.1 -> 10..1...1...00 31 * prot-none, dirty, young 111.000011.1 -> 11..1...1...00 32 * read-only, clean, old 111.000100.1 -> 00..1...1...01 33 * read-only, clean, young 101.000101.1 -> 01..1...0...01 34 * read-only, dirty, old 111.000110.1 -> 10..1...1...01 35 * read-only, dirty, young 101.000111.1 -> 11..1...0...01 36 * read-write, clean, old 111.001100.1 -> 00..1...1...11 37 * read-write, clean, young 101.001101.1 -> 01..1...0...11 38 * read-write, dirty, old 110.001110.1 -> 10..0...1...11 39 * read-write, dirty, young 100.001111.1 -> 11..0...0...11 40 * HW-bits: R read-only, I invalid 41 * SW-bits: p present, y young, d dirty, r read, w write, s special, 42 * u unused, l large 43 */ 44 if (pte_present(pte)) { 45 rste = pte_val(pte) & PAGE_MASK; 46 rste |= move_set_bit(pte_val(pte), _PAGE_READ, 47 _SEGMENT_ENTRY_READ); 48 rste |= move_set_bit(pte_val(pte), _PAGE_WRITE, 49 _SEGMENT_ENTRY_WRITE); 50 rste |= move_set_bit(pte_val(pte), _PAGE_INVALID, 51 _SEGMENT_ENTRY_INVALID); 52 rste |= move_set_bit(pte_val(pte), _PAGE_PROTECT, 53 _SEGMENT_ENTRY_PROTECT); 54 rste |= move_set_bit(pte_val(pte), _PAGE_DIRTY, 55 _SEGMENT_ENTRY_DIRTY); 56 rste |= move_set_bit(pte_val(pte), _PAGE_YOUNG, 57 _SEGMENT_ENTRY_YOUNG); 58 #ifdef CONFIG_MEM_SOFT_DIRTY 59 rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY, 60 _SEGMENT_ENTRY_SOFT_DIRTY); 61 #endif 62 rste |= move_set_bit(pte_val(pte), _PAGE_NOEXEC, 63 _SEGMENT_ENTRY_NOEXEC); 64 } else 65 rste = _SEGMENT_ENTRY_EMPTY; 66 return rste; 67 } 68 69 static inline pte_t __rste_to_pte(unsigned long rste) 70 { 71 int present; 72 pte_t pte; 73 74 if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) 75 present = pud_present(__pud(rste)); 76 else 77 present = pmd_present(__pmd(rste)); 78 79 /* 80 * Convert encoding pmd / pud bits pte bits 81 * dy..R...I...wr lIR.uswrdy.p 82 * empty 00..0...1...00 -> 010.000000.0 83 * prot-none, clean, old 00..1...1...00 -> 111.000000.1 84 * prot-none, clean, young 01..1...1...00 -> 111.000001.1 85 * prot-none, dirty, old 10..1...1...00 -> 111.000010.1 86 * prot-none, dirty, young 11..1...1...00 -> 111.000011.1 87 * read-only, clean, old 00..1...1...01 -> 111.000100.1 88 * read-only, clean, young 01..1...0...01 -> 101.000101.1 89 * read-only, dirty, old 10..1...1...01 -> 111.000110.1 90 * read-only, dirty, young 11..1...0...01 -> 101.000111.1 91 * read-write, clean, old 00..1...1...11 -> 111.001100.1 92 * read-write, clean, young 01..1...0...11 -> 101.001101.1 93 * read-write, dirty, old 10..0...1...11 -> 110.001110.1 94 * read-write, dirty, young 11..0...0...11 -> 100.001111.1 95 * HW-bits: R read-only, I invalid 96 * SW-bits: p present, y young, d dirty, r read, w write, s special, 97 * u unused, l large 98 */ 99 if (present) { 100 pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE; 101 pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT; 102 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_READ, 103 _PAGE_READ); 104 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_WRITE, 105 _PAGE_WRITE); 106 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_INVALID, 107 _PAGE_INVALID); 108 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT, 109 _PAGE_PROTECT); 110 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY, 111 _PAGE_DIRTY); 112 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG, 113 _PAGE_YOUNG); 114 #ifdef CONFIG_MEM_SOFT_DIRTY 115 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY, 116 _PAGE_DIRTY); 117 #endif 118 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC, 119 _PAGE_NOEXEC); 120 } else 121 pte_val(pte) = _PAGE_INVALID; 122 return pte; 123 } 124 125 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 126 pte_t *ptep, pte_t pte) 127 { 128 unsigned long rste; 129 130 rste = __pte_to_rste(pte); 131 if (!MACHINE_HAS_NX) 132 rste &= ~_SEGMENT_ENTRY_NOEXEC; 133 134 /* Set correct table type for 2G hugepages */ 135 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) 136 rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE; 137 else 138 rste |= _SEGMENT_ENTRY_LARGE; 139 pte_val(*ptep) = rste; 140 } 141 142 pte_t huge_ptep_get(pte_t *ptep) 143 { 144 return __rste_to_pte(pte_val(*ptep)); 145 } 146 147 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, 148 unsigned long addr, pte_t *ptep) 149 { 150 pte_t pte = huge_ptep_get(ptep); 151 pmd_t *pmdp = (pmd_t *) ptep; 152 pud_t *pudp = (pud_t *) ptep; 153 154 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) 155 pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); 156 else 157 pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); 158 return pte; 159 } 160 161 pte_t *huge_pte_alloc(struct mm_struct *mm, 162 unsigned long addr, unsigned long sz) 163 { 164 pgd_t *pgdp; 165 p4d_t *p4dp; 166 pud_t *pudp; 167 pmd_t *pmdp = NULL; 168 169 pgdp = pgd_offset(mm, addr); 170 p4dp = p4d_alloc(mm, pgdp, addr); 171 if (p4dp) { 172 pudp = pud_alloc(mm, p4dp, addr); 173 if (pudp) { 174 if (sz == PUD_SIZE) 175 return (pte_t *) pudp; 176 else if (sz == PMD_SIZE) 177 pmdp = pmd_alloc(mm, pudp, addr); 178 } 179 } 180 return (pte_t *) pmdp; 181 } 182 183 pte_t *huge_pte_offset(struct mm_struct *mm, 184 unsigned long addr, unsigned long sz) 185 { 186 pgd_t *pgdp; 187 p4d_t *p4dp; 188 pud_t *pudp; 189 pmd_t *pmdp = NULL; 190 191 pgdp = pgd_offset(mm, addr); 192 if (pgd_present(*pgdp)) { 193 p4dp = p4d_offset(pgdp, addr); 194 if (p4d_present(*p4dp)) { 195 pudp = pud_offset(p4dp, addr); 196 if (pud_present(*pudp)) { 197 if (pud_large(*pudp)) 198 return (pte_t *) pudp; 199 pmdp = pmd_offset(pudp, addr); 200 } 201 } 202 } 203 return (pte_t *) pmdp; 204 } 205 206 int pmd_huge(pmd_t pmd) 207 { 208 return pmd_large(pmd); 209 } 210 211 int pud_huge(pud_t pud) 212 { 213 return pud_large(pud); 214 } 215 216 struct page * 217 follow_huge_pud(struct mm_struct *mm, unsigned long address, 218 pud_t *pud, int flags) 219 { 220 if (flags & FOLL_GET) 221 return NULL; 222 223 return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 224 } 225 226 static __init int setup_hugepagesz(char *opt) 227 { 228 unsigned long size; 229 char *string = opt; 230 231 size = memparse(opt, &opt); 232 if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) { 233 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); 234 } else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) { 235 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); 236 } else { 237 hugetlb_bad_size(); 238 pr_err("hugepagesz= specifies an unsupported page size %s\n", 239 string); 240 return 0; 241 } 242 return 1; 243 } 244 __setup("hugepagesz=", setup_hugepagesz); 245