xref: /openbmc/linux/arch/arm64/mm/hugetlbpage.c (revision 4da722ca)
1 /*
2  * arch/arm64/mm/hugetlbpage.c
3  *
4  * Copyright (C) 2013 Linaro Ltd.
5  *
6  * Based on arch/x86/mm/hugetlbpage.c.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  */
17 
18 #include <linux/init.h>
19 #include <linux/fs.h>
20 #include <linux/mm.h>
21 #include <linux/hugetlb.h>
22 #include <linux/pagemap.h>
23 #include <linux/err.h>
24 #include <linux/sysctl.h>
25 #include <asm/mman.h>
26 #include <asm/tlb.h>
27 #include <asm/tlbflush.h>
28 #include <asm/pgalloc.h>
29 
30 int pmd_huge(pmd_t pmd)
31 {
32 	return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
33 }
34 
35 int pud_huge(pud_t pud)
36 {
37 #ifndef __PAGETABLE_PMD_FOLDED
38 	return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
39 #else
40 	return 0;
41 #endif
42 }
43 
44 static int find_num_contig(struct mm_struct *mm, unsigned long addr,
45 			   pte_t *ptep, size_t *pgsize)
46 {
47 	pgd_t *pgd = pgd_offset(mm, addr);
48 	pud_t *pud;
49 	pmd_t *pmd;
50 
51 	*pgsize = PAGE_SIZE;
52 	pud = pud_offset(pgd, addr);
53 	pmd = pmd_offset(pud, addr);
54 	if ((pte_t *)pmd == ptep) {
55 		*pgsize = PMD_SIZE;
56 		return CONT_PMDS;
57 	}
58 	return CONT_PTES;
59 }
60 
61 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
62 			    pte_t *ptep, pte_t pte)
63 {
64 	size_t pgsize;
65 	int i;
66 	int ncontig;
67 	unsigned long pfn;
68 	pgprot_t hugeprot;
69 
70 	if (!pte_cont(pte)) {
71 		set_pte_at(mm, addr, ptep, pte);
72 		return;
73 	}
74 
75 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
76 	pfn = pte_pfn(pte);
77 	hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
78 	for (i = 0; i < ncontig; i++) {
79 		pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep,
80 			 pte_val(pfn_pte(pfn, hugeprot)));
81 		set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
82 		ptep++;
83 		pfn += pgsize >> PAGE_SHIFT;
84 		addr += pgsize;
85 	}
86 }
87 
88 pte_t *huge_pte_alloc(struct mm_struct *mm,
89 		      unsigned long addr, unsigned long sz)
90 {
91 	pgd_t *pgd;
92 	pud_t *pud;
93 	pte_t *pte = NULL;
94 
95 	pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz);
96 	pgd = pgd_offset(mm, addr);
97 	pud = pud_alloc(mm, pgd, addr);
98 	if (!pud)
99 		return NULL;
100 
101 	if (sz == PUD_SIZE) {
102 		pte = (pte_t *)pud;
103 	} else if (sz == (PAGE_SIZE * CONT_PTES)) {
104 		pmd_t *pmd = pmd_alloc(mm, pud, addr);
105 
106 		WARN_ON(addr & (sz - 1));
107 		/*
108 		 * Note that if this code were ever ported to the
109 		 * 32-bit arm platform then it will cause trouble in
110 		 * the case where CONFIG_HIGHPTE is set, since there
111 		 * will be no pte_unmap() to correspond with this
112 		 * pte_alloc_map().
113 		 */
114 		pte = pte_alloc_map(mm, pmd, addr);
115 	} else if (sz == PMD_SIZE) {
116 		if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
117 		    pud_none(*pud))
118 			pte = huge_pmd_share(mm, addr, pud);
119 		else
120 			pte = (pte_t *)pmd_alloc(mm, pud, addr);
121 	} else if (sz == (PMD_SIZE * CONT_PMDS)) {
122 		pmd_t *pmd;
123 
124 		pmd = pmd_alloc(mm, pud, addr);
125 		WARN_ON(addr & (sz - 1));
126 		return (pte_t *)pmd;
127 	}
128 
129 	pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr,
130 	       sz, pte, pte_val(*pte));
131 	return pte;
132 }
133 
134 pte_t *huge_pte_offset(struct mm_struct *mm,
135 		       unsigned long addr, unsigned long sz)
136 {
137 	pgd_t *pgd;
138 	pud_t *pud;
139 	pmd_t *pmd;
140 
141 	pgd = pgd_offset(mm, addr);
142 	pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd);
143 	if (!pgd_present(*pgd))
144 		return NULL;
145 
146 	pud = pud_offset(pgd, addr);
147 	if (pud_none(*pud))
148 		return NULL;
149 	/* swap or huge page */
150 	if (!pud_present(*pud) || pud_huge(*pud))
151 		return (pte_t *)pud;
152 	/* table; check the next level */
153 
154 	pmd = pmd_offset(pud, addr);
155 	if (pmd_none(*pmd))
156 		return NULL;
157 	if (!pmd_present(*pmd) || pmd_huge(*pmd))
158 		return (pte_t *)pmd;
159 
160 	return NULL;
161 }
162 
163 pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
164 			 struct page *page, int writable)
165 {
166 	size_t pagesize = huge_page_size(hstate_vma(vma));
167 
168 	if (pagesize == CONT_PTE_SIZE) {
169 		entry = pte_mkcont(entry);
170 	} else if (pagesize == CONT_PMD_SIZE) {
171 		entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
172 	} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
173 		pr_warn("%s: unrecognized huge page size 0x%lx\n",
174 			__func__, pagesize);
175 	}
176 	return entry;
177 }
178 
179 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
180 			      unsigned long addr, pte_t *ptep)
181 {
182 	pte_t pte;
183 
184 	if (pte_cont(*ptep)) {
185 		int ncontig, i;
186 		size_t pgsize;
187 		bool is_dirty = false;
188 
189 		ncontig = find_num_contig(mm, addr, ptep, &pgsize);
190 		/* save the 1st pte to return */
191 		pte = ptep_get_and_clear(mm, addr, ptep);
192 		for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) {
193 			/*
194 			 * If HW_AFDBM is enabled, then the HW could
195 			 * turn on the dirty bit for any of the page
196 			 * in the set, so check them all.
197 			 */
198 			++ptep;
199 			if (pte_dirty(ptep_get_and_clear(mm, addr, ptep)))
200 				is_dirty = true;
201 		}
202 		if (is_dirty)
203 			return pte_mkdirty(pte);
204 		else
205 			return pte;
206 	} else {
207 		return ptep_get_and_clear(mm, addr, ptep);
208 	}
209 }
210 
211 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
212 			       unsigned long addr, pte_t *ptep,
213 			       pte_t pte, int dirty)
214 {
215 	if (pte_cont(pte)) {
216 		int ncontig, i, changed = 0;
217 		size_t pgsize = 0;
218 		unsigned long pfn = pte_pfn(pte);
219 		/* Select all bits except the pfn */
220 		pgprot_t hugeprot =
221 			__pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^
222 				 pte_val(pte));
223 
224 		pfn = pte_pfn(pte);
225 		ncontig = find_num_contig(vma->vm_mm, addr, ptep,
226 					  &pgsize);
227 		for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) {
228 			changed |= ptep_set_access_flags(vma, addr, ptep,
229 							pfn_pte(pfn,
230 								hugeprot),
231 							dirty);
232 			pfn += pgsize >> PAGE_SHIFT;
233 		}
234 		return changed;
235 	} else {
236 		return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
237 	}
238 }
239 
240 void huge_ptep_set_wrprotect(struct mm_struct *mm,
241 			     unsigned long addr, pte_t *ptep)
242 {
243 	if (pte_cont(*ptep)) {
244 		int ncontig, i;
245 		size_t pgsize = 0;
246 
247 		ncontig = find_num_contig(mm, addr, ptep, &pgsize);
248 		for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize)
249 			ptep_set_wrprotect(mm, addr, ptep);
250 	} else {
251 		ptep_set_wrprotect(mm, addr, ptep);
252 	}
253 }
254 
255 void huge_ptep_clear_flush(struct vm_area_struct *vma,
256 			   unsigned long addr, pte_t *ptep)
257 {
258 	if (pte_cont(*ptep)) {
259 		int ncontig, i;
260 		size_t pgsize = 0;
261 
262 		ncontig = find_num_contig(vma->vm_mm, addr, ptep,
263 					  &pgsize);
264 		for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize)
265 			ptep_clear_flush(vma, addr, ptep);
266 	} else {
267 		ptep_clear_flush(vma, addr, ptep);
268 	}
269 }
270 
271 static __init int setup_hugepagesz(char *opt)
272 {
273 	unsigned long ps = memparse(opt, &opt);
274 
275 	if (ps == PMD_SIZE) {
276 		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
277 	} else if (ps == PUD_SIZE) {
278 		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
279 	} else {
280 		hugetlb_bad_size();
281 		pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
282 		return 0;
283 	}
284 	return 1;
285 }
286 __setup("hugepagesz=", setup_hugepagesz);
287