1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/hugetlb.h> 3 #include <linux/err.h> 4 5 #ifdef CONFIG_RISCV_ISA_SVNAPOT 6 pte_t *huge_pte_alloc(struct mm_struct *mm, 7 struct vm_area_struct *vma, 8 unsigned long addr, 9 unsigned long sz) 10 { 11 unsigned long order; 12 pte_t *pte = NULL; 13 pgd_t *pgd; 14 p4d_t *p4d; 15 pud_t *pud; 16 pmd_t *pmd; 17 18 pgd = pgd_offset(mm, addr); 19 p4d = p4d_alloc(mm, pgd, addr); 20 if (!p4d) 21 return NULL; 22 23 pud = pud_alloc(mm, p4d, addr); 24 if (!pud) 25 return NULL; 26 27 if (sz == PUD_SIZE) { 28 pte = (pte_t *)pud; 29 goto out; 30 } 31 32 if (sz == PMD_SIZE) { 33 if (want_pmd_share(vma, addr) && pud_none(*pud)) 34 pte = huge_pmd_share(mm, vma, addr, pud); 35 else 36 pte = (pte_t *)pmd_alloc(mm, pud, addr); 37 goto out; 38 } 39 40 pmd = pmd_alloc(mm, pud, addr); 41 if (!pmd) 42 return NULL; 43 44 for_each_napot_order(order) { 45 if (napot_cont_size(order) == sz) { 46 pte = pte_alloc_map(mm, pmd, addr & napot_cont_mask(order)); 47 break; 48 } 49 } 50 51 out: 52 WARN_ON_ONCE(pte && pte_present(*pte) && !pte_huge(*pte)); 53 return pte; 54 } 55 56 pte_t *huge_pte_offset(struct mm_struct *mm, 57 unsigned long addr, 58 unsigned long sz) 59 { 60 unsigned long order; 61 pte_t *pte = NULL; 62 pgd_t *pgd; 63 p4d_t *p4d; 64 pud_t *pud; 65 pmd_t *pmd; 66 67 pgd = pgd_offset(mm, addr); 68 if (!pgd_present(*pgd)) 69 return NULL; 70 71 p4d = p4d_offset(pgd, addr); 72 if (!p4d_present(*p4d)) 73 return NULL; 74 75 pud = pud_offset(p4d, addr); 76 if (sz == PUD_SIZE) 77 /* must be pud huge, non-present or none */ 78 return (pte_t *)pud; 79 80 if (!pud_present(*pud)) 81 return NULL; 82 83 pmd = pmd_offset(pud, addr); 84 if (sz == PMD_SIZE) 85 /* must be pmd huge, non-present or none */ 86 return (pte_t *)pmd; 87 88 if (!pmd_present(*pmd)) 89 return NULL; 90 91 for_each_napot_order(order) { 92 if (napot_cont_size(order) == sz) { 93 pte = pte_offset_kernel(pmd, addr & napot_cont_mask(order)); 94 break; 95 } 96 } 97 return pte; 98 } 99 100 static pte_t get_clear_contig(struct mm_struct *mm, 101 unsigned long addr, 102 pte_t *ptep, 103 unsigned long pte_num) 104 { 105 pte_t orig_pte = ptep_get(ptep); 106 unsigned long i; 107 108 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) { 109 pte_t pte = ptep_get_and_clear(mm, addr, ptep); 110 111 if (pte_dirty(pte)) 112 orig_pte = pte_mkdirty(orig_pte); 113 114 if (pte_young(pte)) 115 orig_pte = pte_mkyoung(orig_pte); 116 } 117 118 return orig_pte; 119 } 120 121 static pte_t get_clear_contig_flush(struct mm_struct *mm, 122 unsigned long addr, 123 pte_t *ptep, 124 unsigned long pte_num) 125 { 126 pte_t orig_pte = get_clear_contig(mm, addr, ptep, pte_num); 127 struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); 128 bool valid = !pte_none(orig_pte); 129 130 if (valid) 131 flush_tlb_range(&vma, addr, addr + (PAGE_SIZE * pte_num)); 132 133 return orig_pte; 134 } 135 136 pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) 137 { 138 unsigned long order; 139 140 for_each_napot_order(order) { 141 if (shift == napot_cont_shift(order)) { 142 entry = pte_mknapot(entry, order); 143 break; 144 } 145 } 146 if (order == NAPOT_ORDER_MAX) 147 entry = pte_mkhuge(entry); 148 149 return entry; 150 } 151 152 void set_huge_pte_at(struct mm_struct *mm, 153 unsigned long addr, 154 pte_t *ptep, 155 pte_t pte) 156 { 157 int i, pte_num; 158 159 if (!pte_napot(pte)) { 160 set_pte_at(mm, addr, ptep, pte); 161 return; 162 } 163 164 pte_num = napot_pte_num(napot_cont_order(pte)); 165 for (i = 0; i < pte_num; i++, ptep++, addr += PAGE_SIZE) 166 set_pte_at(mm, addr, ptep, pte); 167 } 168 169 int huge_ptep_set_access_flags(struct vm_area_struct *vma, 170 unsigned long addr, 171 pte_t *ptep, 172 pte_t pte, 173 int dirty) 174 { 175 struct mm_struct *mm = vma->vm_mm; 176 unsigned long order; 177 pte_t orig_pte; 178 int i, pte_num; 179 180 if (!pte_napot(pte)) 181 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); 182 183 order = napot_cont_order(pte); 184 pte_num = napot_pte_num(order); 185 ptep = huge_pte_offset(mm, addr, napot_cont_size(order)); 186 orig_pte = get_clear_contig_flush(mm, addr, ptep, pte_num); 187 188 if (pte_dirty(orig_pte)) 189 pte = pte_mkdirty(pte); 190 191 if (pte_young(orig_pte)) 192 pte = pte_mkyoung(pte); 193 194 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) 195 set_pte_at(mm, addr, ptep, pte); 196 197 return true; 198 } 199 200 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, 201 unsigned long addr, 202 pte_t *ptep) 203 { 204 pte_t orig_pte = ptep_get(ptep); 205 int pte_num; 206 207 if (!pte_napot(orig_pte)) 208 return ptep_get_and_clear(mm, addr, ptep); 209 210 pte_num = napot_pte_num(napot_cont_order(orig_pte)); 211 212 return get_clear_contig(mm, addr, ptep, pte_num); 213 } 214 215 void huge_ptep_set_wrprotect(struct mm_struct *mm, 216 unsigned long addr, 217 pte_t *ptep) 218 { 219 pte_t pte = ptep_get(ptep); 220 unsigned long order; 221 int i, pte_num; 222 223 if (!pte_napot(pte)) { 224 ptep_set_wrprotect(mm, addr, ptep); 225 return; 226 } 227 228 order = napot_cont_order(pte); 229 pte_num = napot_pte_num(order); 230 ptep = huge_pte_offset(mm, addr, napot_cont_size(order)); 231 232 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) 233 ptep_set_wrprotect(mm, addr, ptep); 234 } 235 236 pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, 237 unsigned long addr, 238 pte_t *ptep) 239 { 240 pte_t pte = ptep_get(ptep); 241 int pte_num; 242 243 if (!pte_napot(pte)) 244 return ptep_clear_flush(vma, addr, ptep); 245 246 pte_num = napot_pte_num(napot_cont_order(pte)); 247 248 return get_clear_contig_flush(vma->vm_mm, addr, ptep, pte_num); 249 } 250 251 void huge_pte_clear(struct mm_struct *mm, 252 unsigned long addr, 253 pte_t *ptep, 254 unsigned long sz) 255 { 256 pte_t pte = READ_ONCE(*ptep); 257 int i, pte_num; 258 259 if (!pte_napot(pte)) { 260 pte_clear(mm, addr, ptep); 261 return; 262 } 263 264 pte_num = napot_pte_num(napot_cont_order(pte)); 265 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) 266 pte_clear(mm, addr, ptep); 267 } 268 269 static __init bool is_napot_size(unsigned long size) 270 { 271 unsigned long order; 272 273 if (!has_svnapot()) 274 return false; 275 276 for_each_napot_order(order) { 277 if (size == napot_cont_size(order)) 278 return true; 279 } 280 return false; 281 } 282 283 static __init int napot_hugetlbpages_init(void) 284 { 285 if (has_svnapot()) { 286 unsigned long order; 287 288 for_each_napot_order(order) 289 hugetlb_add_hstate(order); 290 } 291 return 0; 292 } 293 arch_initcall(napot_hugetlbpages_init); 294 295 #else 296 297 static __init bool is_napot_size(unsigned long size) 298 { 299 return false; 300 } 301 302 #endif /*CONFIG_RISCV_ISA_SVNAPOT*/ 303 304 int pud_huge(pud_t pud) 305 { 306 return pud_leaf(pud); 307 } 308 309 int pmd_huge(pmd_t pmd) 310 { 311 return pmd_leaf(pmd); 312 } 313 314 bool __init arch_hugetlb_valid_size(unsigned long size) 315 { 316 if (size == HPAGE_SIZE) 317 return true; 318 else if (IS_ENABLED(CONFIG_64BIT) && size == PUD_SIZE) 319 return true; 320 else if (is_napot_size(size)) 321 return true; 322 else 323 return false; 324 } 325 326 #ifdef CONFIG_CONTIG_ALLOC 327 static __init int gigantic_pages_init(void) 328 { 329 /* With CONTIG_ALLOC, we can allocate gigantic pages at runtime */ 330 if (IS_ENABLED(CONFIG_64BIT)) 331 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); 332 return 0; 333 } 334 arch_initcall(gigantic_pages_init); 335 #endif 336