xref: /openbmc/linux/arch/arm64/mm/hugetlbpage.c (revision 6aa7de05)
1 /*
2  * arch/arm64/mm/hugetlbpage.c
3  *
4  * Copyright (C) 2013 Linaro Ltd.
5  *
6  * Based on arch/x86/mm/hugetlbpage.c.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  */
17 
18 #include <linux/init.h>
19 #include <linux/fs.h>
20 #include <linux/mm.h>
21 #include <linux/hugetlb.h>
22 #include <linux/pagemap.h>
23 #include <linux/err.h>
24 #include <linux/sysctl.h>
25 #include <asm/mman.h>
26 #include <asm/tlb.h>
27 #include <asm/tlbflush.h>
28 #include <asm/pgalloc.h>
29 
30 int pmd_huge(pmd_t pmd)
31 {
32 	return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
33 }
34 
35 int pud_huge(pud_t pud)
36 {
37 #ifndef __PAGETABLE_PMD_FOLDED
38 	return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
39 #else
40 	return 0;
41 #endif
42 }
43 
44 /*
45  * Select all bits except the pfn
46  */
47 static inline pgprot_t pte_pgprot(pte_t pte)
48 {
49 	unsigned long pfn = pte_pfn(pte);
50 
51 	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
52 }
53 
54 static int find_num_contig(struct mm_struct *mm, unsigned long addr,
55 			   pte_t *ptep, size_t *pgsize)
56 {
57 	pgd_t *pgd = pgd_offset(mm, addr);
58 	pud_t *pud;
59 	pmd_t *pmd;
60 
61 	*pgsize = PAGE_SIZE;
62 	pud = pud_offset(pgd, addr);
63 	pmd = pmd_offset(pud, addr);
64 	if ((pte_t *)pmd == ptep) {
65 		*pgsize = PMD_SIZE;
66 		return CONT_PMDS;
67 	}
68 	return CONT_PTES;
69 }
70 
71 static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
72 {
73 	int contig_ptes = 0;
74 
75 	*pgsize = size;
76 
77 	switch (size) {
78 #ifdef CONFIG_ARM64_4K_PAGES
79 	case PUD_SIZE:
80 #endif
81 	case PMD_SIZE:
82 		contig_ptes = 1;
83 		break;
84 	case CONT_PMD_SIZE:
85 		*pgsize = PMD_SIZE;
86 		contig_ptes = CONT_PMDS;
87 		break;
88 	case CONT_PTE_SIZE:
89 		*pgsize = PAGE_SIZE;
90 		contig_ptes = CONT_PTES;
91 		break;
92 	}
93 
94 	return contig_ptes;
95 }
96 
97 /*
98  * Changing some bits of contiguous entries requires us to follow a
99  * Break-Before-Make approach, breaking the whole contiguous set
100  * before we can change any entries. See ARM DDI 0487A.k_iss10775,
101  * "Misprogramming of the Contiguous bit", page D4-1762.
102  *
103  * This helper performs the break step.
104  */
105 static pte_t get_clear_flush(struct mm_struct *mm,
106 			     unsigned long addr,
107 			     pte_t *ptep,
108 			     unsigned long pgsize,
109 			     unsigned long ncontig)
110 {
111 	struct vm_area_struct vma = { .vm_mm = mm };
112 	pte_t orig_pte = huge_ptep_get(ptep);
113 	bool valid = pte_valid(orig_pte);
114 	unsigned long i, saddr = addr;
115 
116 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
117 		pte_t pte = ptep_get_and_clear(mm, addr, ptep);
118 
119 		/*
120 		 * If HW_AFDBM is enabled, then the HW could turn on
121 		 * the dirty bit for any page in the set, so check
122 		 * them all.  All hugetlb entries are already young.
123 		 */
124 		if (pte_dirty(pte))
125 			orig_pte = pte_mkdirty(orig_pte);
126 	}
127 
128 	if (valid)
129 		flush_tlb_range(&vma, saddr, addr);
130 	return orig_pte;
131 }
132 
133 /*
134  * Changing some bits of contiguous entries requires us to follow a
135  * Break-Before-Make approach, breaking the whole contiguous set
136  * before we can change any entries. See ARM DDI 0487A.k_iss10775,
137  * "Misprogramming of the Contiguous bit", page D4-1762.
138  *
139  * This helper performs the break step for use cases where the
140  * original pte is not needed.
141  */
142 static void clear_flush(struct mm_struct *mm,
143 			     unsigned long addr,
144 			     pte_t *ptep,
145 			     unsigned long pgsize,
146 			     unsigned long ncontig)
147 {
148 	struct vm_area_struct vma = { .vm_mm = mm };
149 	unsigned long i, saddr = addr;
150 
151 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
152 		pte_clear(mm, addr, ptep);
153 
154 	flush_tlb_range(&vma, saddr, addr);
155 }
156 
157 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
158 			    pte_t *ptep, pte_t pte)
159 {
160 	size_t pgsize;
161 	int i;
162 	int ncontig;
163 	unsigned long pfn, dpfn;
164 	pgprot_t hugeprot;
165 
166 	/*
167 	 * Code needs to be expanded to handle huge swap and migration
168 	 * entries. Needed for HUGETLB and MEMORY_FAILURE.
169 	 */
170 	WARN_ON(!pte_present(pte));
171 
172 	if (!pte_cont(pte)) {
173 		set_pte_at(mm, addr, ptep, pte);
174 		return;
175 	}
176 
177 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
178 	pfn = pte_pfn(pte);
179 	dpfn = pgsize >> PAGE_SHIFT;
180 	hugeprot = pte_pgprot(pte);
181 
182 	clear_flush(mm, addr, ptep, pgsize, ncontig);
183 
184 	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) {
185 		pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep,
186 			 pte_val(pfn_pte(pfn, hugeprot)));
187 		set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
188 	}
189 }
190 
191 void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
192 			  pte_t *ptep, pte_t pte, unsigned long sz)
193 {
194 	int i, ncontig;
195 	size_t pgsize;
196 
197 	ncontig = num_contig_ptes(sz, &pgsize);
198 
199 	for (i = 0; i < ncontig; i++, ptep++)
200 		set_pte(ptep, pte);
201 }
202 
203 pte_t *huge_pte_alloc(struct mm_struct *mm,
204 		      unsigned long addr, unsigned long sz)
205 {
206 	pgd_t *pgd;
207 	pud_t *pud;
208 	pte_t *pte = NULL;
209 
210 	pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz);
211 	pgd = pgd_offset(mm, addr);
212 	pud = pud_alloc(mm, pgd, addr);
213 	if (!pud)
214 		return NULL;
215 
216 	if (sz == PUD_SIZE) {
217 		pte = (pte_t *)pud;
218 	} else if (sz == (PAGE_SIZE * CONT_PTES)) {
219 		pmd_t *pmd = pmd_alloc(mm, pud, addr);
220 
221 		WARN_ON(addr & (sz - 1));
222 		/*
223 		 * Note that if this code were ever ported to the
224 		 * 32-bit arm platform then it will cause trouble in
225 		 * the case where CONFIG_HIGHPTE is set, since there
226 		 * will be no pte_unmap() to correspond with this
227 		 * pte_alloc_map().
228 		 */
229 		pte = pte_alloc_map(mm, pmd, addr);
230 	} else if (sz == PMD_SIZE) {
231 		if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
232 		    pud_none(*pud))
233 			pte = huge_pmd_share(mm, addr, pud);
234 		else
235 			pte = (pte_t *)pmd_alloc(mm, pud, addr);
236 	} else if (sz == (PMD_SIZE * CONT_PMDS)) {
237 		pmd_t *pmd;
238 
239 		pmd = pmd_alloc(mm, pud, addr);
240 		WARN_ON(addr & (sz - 1));
241 		return (pte_t *)pmd;
242 	}
243 
244 	pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr,
245 	       sz, pte, pte_val(*pte));
246 	return pte;
247 }
248 
249 pte_t *huge_pte_offset(struct mm_struct *mm,
250 		       unsigned long addr, unsigned long sz)
251 {
252 	pgd_t *pgd;
253 	pud_t *pud;
254 	pmd_t *pmd;
255 
256 	pgd = pgd_offset(mm, addr);
257 	pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd);
258 	if (!pgd_present(*pgd))
259 		return NULL;
260 
261 	pud = pud_offset(pgd, addr);
262 	if (sz != PUD_SIZE && pud_none(*pud))
263 		return NULL;
264 	/* hugepage or swap? */
265 	if (pud_huge(*pud) || !pud_present(*pud))
266 		return (pte_t *)pud;
267 	/* table; check the next level */
268 
269 	if (sz == CONT_PMD_SIZE)
270 		addr &= CONT_PMD_MASK;
271 
272 	pmd = pmd_offset(pud, addr);
273 	if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
274 	    pmd_none(*pmd))
275 		return NULL;
276 	if (pmd_huge(*pmd) || !pmd_present(*pmd))
277 		return (pte_t *)pmd;
278 
279 	if (sz == CONT_PTE_SIZE) {
280 		pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK));
281 		return pte;
282 	}
283 
284 	return NULL;
285 }
286 
287 pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
288 			 struct page *page, int writable)
289 {
290 	size_t pagesize = huge_page_size(hstate_vma(vma));
291 
292 	if (pagesize == CONT_PTE_SIZE) {
293 		entry = pte_mkcont(entry);
294 	} else if (pagesize == CONT_PMD_SIZE) {
295 		entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
296 	} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
297 		pr_warn("%s: unrecognized huge page size 0x%lx\n",
298 			__func__, pagesize);
299 	}
300 	return entry;
301 }
302 
303 void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
304 		    pte_t *ptep, unsigned long sz)
305 {
306 	int i, ncontig;
307 	size_t pgsize;
308 
309 	ncontig = num_contig_ptes(sz, &pgsize);
310 
311 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
312 		pte_clear(mm, addr, ptep);
313 }
314 
315 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
316 			      unsigned long addr, pte_t *ptep)
317 {
318 	int ncontig;
319 	size_t pgsize;
320 	pte_t orig_pte = huge_ptep_get(ptep);
321 
322 	if (!pte_cont(orig_pte))
323 		return ptep_get_and_clear(mm, addr, ptep);
324 
325 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
326 
327 	return get_clear_flush(mm, addr, ptep, pgsize, ncontig);
328 }
329 
330 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
331 			       unsigned long addr, pte_t *ptep,
332 			       pte_t pte, int dirty)
333 {
334 	int ncontig, i, changed = 0;
335 	size_t pgsize = 0;
336 	unsigned long pfn = pte_pfn(pte), dpfn;
337 	pgprot_t hugeprot;
338 	pte_t orig_pte;
339 
340 	if (!pte_cont(pte))
341 		return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
342 
343 	ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
344 	dpfn = pgsize >> PAGE_SHIFT;
345 
346 	orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
347 	if (!pte_same(orig_pte, pte))
348 		changed = 1;
349 
350 	/* Make sure we don't lose the dirty state */
351 	if (pte_dirty(orig_pte))
352 		pte = pte_mkdirty(pte);
353 
354 	hugeprot = pte_pgprot(pte);
355 	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
356 		set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
357 
358 	return changed;
359 }
360 
361 void huge_ptep_set_wrprotect(struct mm_struct *mm,
362 			     unsigned long addr, pte_t *ptep)
363 {
364 	unsigned long pfn, dpfn;
365 	pgprot_t hugeprot;
366 	int ncontig, i;
367 	size_t pgsize;
368 	pte_t pte;
369 
370 	if (!pte_cont(*ptep)) {
371 		ptep_set_wrprotect(mm, addr, ptep);
372 		return;
373 	}
374 
375 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
376 	dpfn = pgsize >> PAGE_SHIFT;
377 
378 	pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig);
379 	pte = pte_wrprotect(pte);
380 
381 	hugeprot = pte_pgprot(pte);
382 	pfn = pte_pfn(pte);
383 
384 	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
385 		set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
386 }
387 
388 void huge_ptep_clear_flush(struct vm_area_struct *vma,
389 			   unsigned long addr, pte_t *ptep)
390 {
391 	size_t pgsize;
392 	int ncontig;
393 
394 	if (!pte_cont(*ptep)) {
395 		ptep_clear_flush(vma, addr, ptep);
396 		return;
397 	}
398 
399 	ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
400 	clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
401 }
402 
403 static __init int setup_hugepagesz(char *opt)
404 {
405 	unsigned long ps = memparse(opt, &opt);
406 
407 	switch (ps) {
408 #ifdef CONFIG_ARM64_4K_PAGES
409 	case PUD_SIZE:
410 #endif
411 	case PMD_SIZE * CONT_PMDS:
412 	case PMD_SIZE:
413 	case PAGE_SIZE * CONT_PTES:
414 		hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT);
415 		return 1;
416 	}
417 
418 	hugetlb_bad_size();
419 	pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
420 	return 0;
421 }
422 __setup("hugepagesz=", setup_hugepagesz);
423 
424 #ifdef CONFIG_ARM64_64K_PAGES
425 static __init int add_default_hugepagesz(void)
426 {
427 	if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL)
428 		hugetlb_add_hstate(CONT_PTE_SHIFT);
429 	return 0;
430 }
431 arch_initcall(add_default_hugepagesz);
432 #endif
433