1 /* 2 * arch/arm64/mm/hugetlbpage.c 3 * 4 * Copyright (C) 2013 Linaro Ltd. 5 * 6 * Based on arch/x86/mm/hugetlbpage.c. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 */ 17 18 #include <linux/init.h> 19 #include <linux/fs.h> 20 #include <linux/mm.h> 21 #include <linux/hugetlb.h> 22 #include <linux/pagemap.h> 23 #include <linux/err.h> 24 #include <linux/sysctl.h> 25 #include <asm/mman.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbflush.h> 28 #include <asm/pgalloc.h> 29 30 int pmd_huge(pmd_t pmd) 31 { 32 return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); 33 } 34 35 int pud_huge(pud_t pud) 36 { 37 #ifndef __PAGETABLE_PMD_FOLDED 38 return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); 39 #else 40 return 0; 41 #endif 42 } 43 44 static int find_num_contig(struct mm_struct *mm, unsigned long addr, 45 pte_t *ptep, size_t *pgsize) 46 { 47 pgd_t *pgd = pgd_offset(mm, addr); 48 pud_t *pud; 49 pmd_t *pmd; 50 51 *pgsize = PAGE_SIZE; 52 pud = pud_offset(pgd, addr); 53 pmd = pmd_offset(pud, addr); 54 if ((pte_t *)pmd == ptep) { 55 *pgsize = PMD_SIZE; 56 return CONT_PMDS; 57 } 58 return CONT_PTES; 59 } 60 61 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 62 pte_t *ptep, pte_t pte) 63 { 64 size_t pgsize; 65 int i; 66 int ncontig; 67 unsigned long pfn; 68 pgprot_t hugeprot; 69 70 if (!pte_cont(pte)) { 71 set_pte_at(mm, addr, ptep, pte); 72 return; 73 } 74 75 ncontig = find_num_contig(mm, addr, ptep, &pgsize); 76 pfn = pte_pfn(pte); 77 hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); 78 for (i = 0; i < ncontig; i++) { 79 pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, 80 pte_val(pfn_pte(pfn, hugeprot))); 81 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); 82 ptep++; 83 pfn += pgsize >> PAGE_SHIFT; 84 addr += pgsize; 85 } 86 } 87 88 pte_t *huge_pte_alloc(struct mm_struct *mm, 89 unsigned long addr, unsigned long sz) 90 { 91 pgd_t *pgd; 92 pud_t *pud; 93 pte_t *pte = NULL; 94 95 pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); 96 pgd = pgd_offset(mm, addr); 97 pud = pud_alloc(mm, pgd, addr); 98 if (!pud) 99 return NULL; 100 101 if (sz == PUD_SIZE) { 102 pte = (pte_t *)pud; 103 } else if (sz == (PAGE_SIZE * CONT_PTES)) { 104 pmd_t *pmd = pmd_alloc(mm, pud, addr); 105 106 WARN_ON(addr & (sz - 1)); 107 /* 108 * Note that if this code were ever ported to the 109 * 32-bit arm platform then it will cause trouble in 110 * the case where CONFIG_HIGHPTE is set, since there 111 * will be no pte_unmap() to correspond with this 112 * pte_alloc_map(). 113 */ 114 pte = pte_alloc_map(mm, pmd, addr); 115 } else if (sz == PMD_SIZE) { 116 if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && 117 pud_none(*pud)) 118 pte = huge_pmd_share(mm, addr, pud); 119 else 120 pte = (pte_t *)pmd_alloc(mm, pud, addr); 121 } else if (sz == (PMD_SIZE * CONT_PMDS)) { 122 pmd_t *pmd; 123 124 pmd = pmd_alloc(mm, pud, addr); 125 WARN_ON(addr & (sz - 1)); 126 return (pte_t *)pmd; 127 } 128 129 pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, 130 sz, pte, pte_val(*pte)); 131 return pte; 132 } 133 134 pte_t *huge_pte_offset(struct mm_struct *mm, 135 unsigned long addr, unsigned long sz) 136 { 137 pgd_t *pgd; 138 pud_t *pud; 139 pmd_t *pmd; 140 141 pgd = pgd_offset(mm, addr); 142 pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd); 143 if (!pgd_present(*pgd)) 144 return NULL; 145 146 pud = pud_offset(pgd, addr); 147 if (pud_none(*pud)) 148 return NULL; 149 /* swap or huge page */ 150 if (!pud_present(*pud) || pud_huge(*pud)) 151 return (pte_t *)pud; 152 /* table; check the next level */ 153 154 pmd = pmd_offset(pud, addr); 155 if (pmd_none(*pmd)) 156 return NULL; 157 if (!pmd_present(*pmd) || pmd_huge(*pmd)) 158 return (pte_t *)pmd; 159 160 return NULL; 161 } 162 163 pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, 164 struct page *page, int writable) 165 { 166 size_t pagesize = huge_page_size(hstate_vma(vma)); 167 168 if (pagesize == CONT_PTE_SIZE) { 169 entry = pte_mkcont(entry); 170 } else if (pagesize == CONT_PMD_SIZE) { 171 entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); 172 } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { 173 pr_warn("%s: unrecognized huge page size 0x%lx\n", 174 __func__, pagesize); 175 } 176 return entry; 177 } 178 179 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, 180 unsigned long addr, pte_t *ptep) 181 { 182 pte_t pte; 183 184 if (pte_cont(*ptep)) { 185 int ncontig, i; 186 size_t pgsize; 187 bool is_dirty = false; 188 189 ncontig = find_num_contig(mm, addr, ptep, &pgsize); 190 /* save the 1st pte to return */ 191 pte = ptep_get_and_clear(mm, addr, ptep); 192 for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { 193 /* 194 * If HW_AFDBM is enabled, then the HW could 195 * turn on the dirty bit for any of the page 196 * in the set, so check them all. 197 */ 198 ++ptep; 199 if (pte_dirty(ptep_get_and_clear(mm, addr, ptep))) 200 is_dirty = true; 201 } 202 if (is_dirty) 203 return pte_mkdirty(pte); 204 else 205 return pte; 206 } else { 207 return ptep_get_and_clear(mm, addr, ptep); 208 } 209 } 210 211 int huge_ptep_set_access_flags(struct vm_area_struct *vma, 212 unsigned long addr, pte_t *ptep, 213 pte_t pte, int dirty) 214 { 215 if (pte_cont(pte)) { 216 int ncontig, i, changed = 0; 217 size_t pgsize = 0; 218 unsigned long pfn = pte_pfn(pte); 219 /* Select all bits except the pfn */ 220 pgprot_t hugeprot = 221 __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ 222 pte_val(pte)); 223 224 pfn = pte_pfn(pte); 225 ncontig = find_num_contig(vma->vm_mm, addr, ptep, 226 &pgsize); 227 for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) { 228 changed |= ptep_set_access_flags(vma, addr, ptep, 229 pfn_pte(pfn, 230 hugeprot), 231 dirty); 232 pfn += pgsize >> PAGE_SHIFT; 233 } 234 return changed; 235 } else { 236 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); 237 } 238 } 239 240 void huge_ptep_set_wrprotect(struct mm_struct *mm, 241 unsigned long addr, pte_t *ptep) 242 { 243 if (pte_cont(*ptep)) { 244 int ncontig, i; 245 size_t pgsize = 0; 246 247 ncontig = find_num_contig(mm, addr, ptep, &pgsize); 248 for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) 249 ptep_set_wrprotect(mm, addr, ptep); 250 } else { 251 ptep_set_wrprotect(mm, addr, ptep); 252 } 253 } 254 255 void huge_ptep_clear_flush(struct vm_area_struct *vma, 256 unsigned long addr, pte_t *ptep) 257 { 258 if (pte_cont(*ptep)) { 259 int ncontig, i; 260 size_t pgsize = 0; 261 262 ncontig = find_num_contig(vma->vm_mm, addr, ptep, 263 &pgsize); 264 for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) 265 ptep_clear_flush(vma, addr, ptep); 266 } else { 267 ptep_clear_flush(vma, addr, ptep); 268 } 269 } 270 271 static __init int setup_hugepagesz(char *opt) 272 { 273 unsigned long ps = memparse(opt, &opt); 274 275 if (ps == PMD_SIZE) { 276 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); 277 } else if (ps == PUD_SIZE) { 278 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); 279 } else { 280 hugetlb_bad_size(); 281 pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); 282 return 0; 283 } 284 return 1; 285 } 286 __setup("hugepagesz=", setup_hugepagesz); 287