1 /* 2 * arch/arm64/mm/hugetlbpage.c 3 * 4 * Copyright (C) 2013 Linaro Ltd. 5 * 6 * Based on arch/x86/mm/hugetlbpage.c. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 */ 17 18 #include <linux/init.h> 19 #include <linux/fs.h> 20 #include <linux/mm.h> 21 #include <linux/hugetlb.h> 22 #include <linux/pagemap.h> 23 #include <linux/err.h> 24 #include <linux/sysctl.h> 25 #include <asm/mman.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbflush.h> 28 #include <asm/pgalloc.h> 29 30 int pmd_huge(pmd_t pmd) 31 { 32 return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); 33 } 34 35 int pud_huge(pud_t pud) 36 { 37 #ifndef __PAGETABLE_PMD_FOLDED 38 return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); 39 #else 40 return 0; 41 #endif 42 } 43 44 static int find_num_contig(struct mm_struct *mm, unsigned long addr, 45 pte_t *ptep, pte_t pte, size_t *pgsize) 46 { 47 pgd_t *pgd = pgd_offset(mm, addr); 48 pud_t *pud; 49 pmd_t *pmd; 50 51 *pgsize = PAGE_SIZE; 52 if (!pte_cont(pte)) 53 return 1; 54 pud = pud_offset(pgd, addr); 55 pmd = pmd_offset(pud, addr); 56 if ((pte_t *)pmd == ptep) { 57 *pgsize = PMD_SIZE; 58 return CONT_PMDS; 59 } 60 return CONT_PTES; 61 } 62 63 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 64 pte_t *ptep, pte_t pte) 65 { 66 size_t pgsize; 67 int i; 68 int ncontig = find_num_contig(mm, addr, ptep, pte, &pgsize); 69 unsigned long pfn; 70 pgprot_t hugeprot; 71 72 if (ncontig == 1) { 73 set_pte_at(mm, addr, ptep, pte); 74 return; 75 } 76 77 pfn = pte_pfn(pte); 78 hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); 79 for (i = 0; i < ncontig; i++) { 80 pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, 81 pte_val(pfn_pte(pfn, hugeprot))); 82 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); 83 ptep++; 84 pfn += pgsize >> PAGE_SHIFT; 85 addr += pgsize; 86 } 87 } 88 89 pte_t *huge_pte_alloc(struct mm_struct *mm, 90 unsigned long addr, unsigned long sz) 91 { 92 pgd_t *pgd; 93 pud_t *pud; 94 pte_t *pte = NULL; 95 96 pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); 97 pgd = pgd_offset(mm, addr); 98 pud = pud_alloc(mm, pgd, addr); 99 if (!pud) 100 return NULL; 101 102 if (sz == PUD_SIZE) { 103 pte = (pte_t *)pud; 104 } else if (sz == (PAGE_SIZE * CONT_PTES)) { 105 pmd_t *pmd = pmd_alloc(mm, pud, addr); 106 107 WARN_ON(addr & (sz - 1)); 108 /* 109 * Note that if this code were ever ported to the 110 * 32-bit arm platform then it will cause trouble in 111 * the case where CONFIG_HIGHPTE is set, since there 112 * will be no pte_unmap() to correspond with this 113 * pte_alloc_map(). 114 */ 115 pte = pte_alloc_map(mm, pmd, addr); 116 } else if (sz == PMD_SIZE) { 117 if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && 118 pud_none(*pud)) 119 pte = huge_pmd_share(mm, addr, pud); 120 else 121 pte = (pte_t *)pmd_alloc(mm, pud, addr); 122 } else if (sz == (PMD_SIZE * CONT_PMDS)) { 123 pmd_t *pmd; 124 125 pmd = pmd_alloc(mm, pud, addr); 126 WARN_ON(addr & (sz - 1)); 127 return (pte_t *)pmd; 128 } 129 130 pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, 131 sz, pte, pte_val(*pte)); 132 return pte; 133 } 134 135 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 136 { 137 pgd_t *pgd; 138 pud_t *pud; 139 pmd_t *pmd = NULL; 140 pte_t *pte = NULL; 141 142 pgd = pgd_offset(mm, addr); 143 pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd); 144 if (!pgd_present(*pgd)) 145 return NULL; 146 pud = pud_offset(pgd, addr); 147 if (!pud_present(*pud)) 148 return NULL; 149 150 if (pud_huge(*pud)) 151 return (pte_t *)pud; 152 pmd = pmd_offset(pud, addr); 153 if (!pmd_present(*pmd)) 154 return NULL; 155 156 if (pte_cont(pmd_pte(*pmd))) { 157 pmd = pmd_offset( 158 pud, (addr & CONT_PMD_MASK)); 159 return (pte_t *)pmd; 160 } 161 if (pmd_huge(*pmd)) 162 return (pte_t *)pmd; 163 pte = pte_offset_kernel(pmd, addr); 164 if (pte_present(*pte) && pte_cont(*pte)) { 165 pte = pte_offset_kernel( 166 pmd, (addr & CONT_PTE_MASK)); 167 return pte; 168 } 169 return NULL; 170 } 171 172 pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, 173 struct page *page, int writable) 174 { 175 size_t pagesize = huge_page_size(hstate_vma(vma)); 176 177 if (pagesize == CONT_PTE_SIZE) { 178 entry = pte_mkcont(entry); 179 } else if (pagesize == CONT_PMD_SIZE) { 180 entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); 181 } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { 182 pr_warn("%s: unrecognized huge page size 0x%lx\n", 183 __func__, pagesize); 184 } 185 return entry; 186 } 187 188 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, 189 unsigned long addr, pte_t *ptep) 190 { 191 pte_t pte; 192 193 if (pte_cont(*ptep)) { 194 int ncontig, i; 195 size_t pgsize; 196 pte_t *cpte; 197 bool is_dirty = false; 198 199 cpte = huge_pte_offset(mm, addr); 200 ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); 201 /* save the 1st pte to return */ 202 pte = ptep_get_and_clear(mm, addr, cpte); 203 for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { 204 /* 205 * If HW_AFDBM is enabled, then the HW could 206 * turn on the dirty bit for any of the page 207 * in the set, so check them all. 208 */ 209 ++cpte; 210 if (pte_dirty(ptep_get_and_clear(mm, addr, cpte))) 211 is_dirty = true; 212 } 213 if (is_dirty) 214 return pte_mkdirty(pte); 215 else 216 return pte; 217 } else { 218 return ptep_get_and_clear(mm, addr, ptep); 219 } 220 } 221 222 int huge_ptep_set_access_flags(struct vm_area_struct *vma, 223 unsigned long addr, pte_t *ptep, 224 pte_t pte, int dirty) 225 { 226 pte_t *cpte; 227 228 if (pte_cont(pte)) { 229 int ncontig, i, changed = 0; 230 size_t pgsize = 0; 231 unsigned long pfn = pte_pfn(pte); 232 /* Select all bits except the pfn */ 233 pgprot_t hugeprot = 234 __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ 235 pte_val(pte)); 236 237 cpte = huge_pte_offset(vma->vm_mm, addr); 238 pfn = pte_pfn(*cpte); 239 ncontig = find_num_contig(vma->vm_mm, addr, cpte, 240 *cpte, &pgsize); 241 for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) { 242 changed |= ptep_set_access_flags(vma, addr, cpte, 243 pfn_pte(pfn, 244 hugeprot), 245 dirty); 246 pfn += pgsize >> PAGE_SHIFT; 247 } 248 return changed; 249 } else { 250 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); 251 } 252 } 253 254 void huge_ptep_set_wrprotect(struct mm_struct *mm, 255 unsigned long addr, pte_t *ptep) 256 { 257 if (pte_cont(*ptep)) { 258 int ncontig, i; 259 pte_t *cpte; 260 size_t pgsize = 0; 261 262 cpte = huge_pte_offset(mm, addr); 263 ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); 264 for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) 265 ptep_set_wrprotect(mm, addr, cpte); 266 } else { 267 ptep_set_wrprotect(mm, addr, ptep); 268 } 269 } 270 271 void huge_ptep_clear_flush(struct vm_area_struct *vma, 272 unsigned long addr, pte_t *ptep) 273 { 274 if (pte_cont(*ptep)) { 275 int ncontig, i; 276 pte_t *cpte; 277 size_t pgsize = 0; 278 279 cpte = huge_pte_offset(vma->vm_mm, addr); 280 ncontig = find_num_contig(vma->vm_mm, addr, cpte, 281 *cpte, &pgsize); 282 for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) 283 ptep_clear_flush(vma, addr, cpte); 284 } else { 285 ptep_clear_flush(vma, addr, ptep); 286 } 287 } 288 289 static __init int setup_hugepagesz(char *opt) 290 { 291 unsigned long ps = memparse(opt, &opt); 292 293 if (ps == PMD_SIZE) { 294 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); 295 } else if (ps == PUD_SIZE) { 296 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); 297 } else if (ps == (PAGE_SIZE * CONT_PTES)) { 298 hugetlb_add_hstate(CONT_PTE_SHIFT); 299 } else if (ps == (PMD_SIZE * CONT_PMDS)) { 300 hugetlb_add_hstate((PMD_SHIFT + CONT_PMD_SHIFT) - PAGE_SHIFT); 301 } else { 302 hugetlb_bad_size(); 303 pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); 304 return 0; 305 } 306 return 1; 307 } 308 __setup("hugepagesz=", setup_hugepagesz); 309 310 #ifdef CONFIG_ARM64_64K_PAGES 311 static __init int add_default_hugepagesz(void) 312 { 313 if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) 314 hugetlb_add_hstate(CONT_PTE_SHIFT); 315 return 0; 316 } 317 arch_initcall(add_default_hugepagesz); 318 #endif 319