xref: /openbmc/linux/mm/mprotect.c (revision d7a3d85e)
1 /*
2  *  mm/mprotect.c
3  *
4  *  (C) Copyright 1994 Linus Torvalds
5  *  (C) Copyright 2002 Christoph Hellwig
6  *
7  *  Address space accounting code	<alan@lxorguk.ukuu.org.uk>
8  *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
9  */
10 
11 #include <linux/mm.h>
12 #include <linux/hugetlb.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
15 #include <linux/fs.h>
16 #include <linux/highmem.h>
17 #include <linux/security.h>
18 #include <linux/mempolicy.h>
19 #include <linux/personality.h>
20 #include <linux/syscalls.h>
21 #include <linux/swap.h>
22 #include <linux/swapops.h>
23 #include <linux/mmu_notifier.h>
24 #include <linux/migrate.h>
25 #include <linux/perf_event.h>
26 #include <linux/ksm.h>
27 #include <asm/uaccess.h>
28 #include <asm/pgtable.h>
29 #include <asm/cacheflush.h>
30 #include <asm/tlbflush.h>
31 
32 /*
33  * For a prot_numa update we only hold mmap_sem for read so there is a
34  * potential race with faulting where a pmd was temporarily none. This
35  * function checks for a transhuge pmd under the appropriate lock. It
36  * returns a pte if it was successfully locked or NULL if it raced with
37  * a transhuge insertion.
38  */
39 static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd,
40 			unsigned long addr, int prot_numa, spinlock_t **ptl)
41 {
42 	pte_t *pte;
43 	spinlock_t *pmdl;
44 
45 	/* !prot_numa is protected by mmap_sem held for write */
46 	if (!prot_numa)
47 		return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl);
48 
49 	pmdl = pmd_lock(vma->vm_mm, pmd);
50 	if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) {
51 		spin_unlock(pmdl);
52 		return NULL;
53 	}
54 
55 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl);
56 	spin_unlock(pmdl);
57 	return pte;
58 }
59 
60 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
61 		unsigned long addr, unsigned long end, pgprot_t newprot,
62 		int dirty_accountable, int prot_numa)
63 {
64 	struct mm_struct *mm = vma->vm_mm;
65 	pte_t *pte, oldpte;
66 	spinlock_t *ptl;
67 	unsigned long pages = 0;
68 
69 	pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl);
70 	if (!pte)
71 		return 0;
72 
73 	arch_enter_lazy_mmu_mode();
74 	do {
75 		oldpte = *pte;
76 		if (pte_present(oldpte)) {
77 			pte_t ptent;
78 			bool preserve_write = prot_numa && pte_write(oldpte);
79 
80 			/*
81 			 * Avoid trapping faults against the zero or KSM
82 			 * pages. See similar comment in change_huge_pmd.
83 			 */
84 			if (prot_numa) {
85 				struct page *page;
86 
87 				page = vm_normal_page(vma, addr, oldpte);
88 				if (!page || PageKsm(page))
89 					continue;
90 
91 				/* Avoid TLB flush if possible */
92 				if (pte_protnone(oldpte))
93 					continue;
94 			}
95 
96 			ptent = ptep_modify_prot_start(mm, addr, pte);
97 			ptent = pte_modify(ptent, newprot);
98 			if (preserve_write)
99 				ptent = pte_mkwrite(ptent);
100 
101 			/* Avoid taking write faults for known dirty pages */
102 			if (dirty_accountable && pte_dirty(ptent) &&
103 					(pte_soft_dirty(ptent) ||
104 					 !(vma->vm_flags & VM_SOFTDIRTY))) {
105 				ptent = pte_mkwrite(ptent);
106 			}
107 			ptep_modify_prot_commit(mm, addr, pte, ptent);
108 			pages++;
109 		} else if (IS_ENABLED(CONFIG_MIGRATION)) {
110 			swp_entry_t entry = pte_to_swp_entry(oldpte);
111 
112 			if (is_write_migration_entry(entry)) {
113 				pte_t newpte;
114 				/*
115 				 * A protection check is difficult so
116 				 * just be safe and disable write
117 				 */
118 				make_migration_entry_read(&entry);
119 				newpte = swp_entry_to_pte(entry);
120 				if (pte_swp_soft_dirty(oldpte))
121 					newpte = pte_swp_mksoft_dirty(newpte);
122 				set_pte_at(mm, addr, pte, newpte);
123 
124 				pages++;
125 			}
126 		}
127 	} while (pte++, addr += PAGE_SIZE, addr != end);
128 	arch_leave_lazy_mmu_mode();
129 	pte_unmap_unlock(pte - 1, ptl);
130 
131 	return pages;
132 }
133 
134 static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
135 		pud_t *pud, unsigned long addr, unsigned long end,
136 		pgprot_t newprot, int dirty_accountable, int prot_numa)
137 {
138 	pmd_t *pmd;
139 	struct mm_struct *mm = vma->vm_mm;
140 	unsigned long next;
141 	unsigned long pages = 0;
142 	unsigned long nr_huge_updates = 0;
143 	unsigned long mni_start = 0;
144 
145 	pmd = pmd_offset(pud, addr);
146 	do {
147 		unsigned long this_pages;
148 
149 		next = pmd_addr_end(addr, end);
150 		if (!pmd_trans_huge(*pmd) && pmd_none_or_clear_bad(pmd))
151 			continue;
152 
153 		/* invoke the mmu notifier if the pmd is populated */
154 		if (!mni_start) {
155 			mni_start = addr;
156 			mmu_notifier_invalidate_range_start(mm, mni_start, end);
157 		}
158 
159 		if (pmd_trans_huge(*pmd)) {
160 			if (next - addr != HPAGE_PMD_SIZE)
161 				split_huge_page_pmd(vma, addr, pmd);
162 			else {
163 				int nr_ptes = change_huge_pmd(vma, pmd, addr,
164 						newprot, prot_numa);
165 
166 				if (nr_ptes) {
167 					if (nr_ptes == HPAGE_PMD_NR) {
168 						pages += HPAGE_PMD_NR;
169 						nr_huge_updates++;
170 					}
171 
172 					/* huge pmd was handled */
173 					continue;
174 				}
175 			}
176 			/* fall through, the trans huge pmd just split */
177 		}
178 		this_pages = change_pte_range(vma, pmd, addr, next, newprot,
179 				 dirty_accountable, prot_numa);
180 		pages += this_pages;
181 	} while (pmd++, addr = next, addr != end);
182 
183 	if (mni_start)
184 		mmu_notifier_invalidate_range_end(mm, mni_start, end);
185 
186 	if (nr_huge_updates)
187 		count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
188 	return pages;
189 }
190 
191 static inline unsigned long change_pud_range(struct vm_area_struct *vma,
192 		pgd_t *pgd, unsigned long addr, unsigned long end,
193 		pgprot_t newprot, int dirty_accountable, int prot_numa)
194 {
195 	pud_t *pud;
196 	unsigned long next;
197 	unsigned long pages = 0;
198 
199 	pud = pud_offset(pgd, addr);
200 	do {
201 		next = pud_addr_end(addr, end);
202 		if (pud_none_or_clear_bad(pud))
203 			continue;
204 		pages += change_pmd_range(vma, pud, addr, next, newprot,
205 				 dirty_accountable, prot_numa);
206 	} while (pud++, addr = next, addr != end);
207 
208 	return pages;
209 }
210 
211 static unsigned long change_protection_range(struct vm_area_struct *vma,
212 		unsigned long addr, unsigned long end, pgprot_t newprot,
213 		int dirty_accountable, int prot_numa)
214 {
215 	struct mm_struct *mm = vma->vm_mm;
216 	pgd_t *pgd;
217 	unsigned long next;
218 	unsigned long start = addr;
219 	unsigned long pages = 0;
220 
221 	BUG_ON(addr >= end);
222 	pgd = pgd_offset(mm, addr);
223 	flush_cache_range(vma, addr, end);
224 	set_tlb_flush_pending(mm);
225 	do {
226 		next = pgd_addr_end(addr, end);
227 		if (pgd_none_or_clear_bad(pgd))
228 			continue;
229 		pages += change_pud_range(vma, pgd, addr, next, newprot,
230 				 dirty_accountable, prot_numa);
231 	} while (pgd++, addr = next, addr != end);
232 
233 	/* Only flush the TLB if we actually modified any entries: */
234 	if (pages)
235 		flush_tlb_range(vma, start, end);
236 	clear_tlb_flush_pending(mm);
237 
238 	return pages;
239 }
240 
241 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
242 		       unsigned long end, pgprot_t newprot,
243 		       int dirty_accountable, int prot_numa)
244 {
245 	unsigned long pages;
246 
247 	if (is_vm_hugetlb_page(vma))
248 		pages = hugetlb_change_protection(vma, start, end, newprot);
249 	else
250 		pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
251 
252 	return pages;
253 }
254 
255 int
256 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
257 	unsigned long start, unsigned long end, unsigned long newflags)
258 {
259 	struct mm_struct *mm = vma->vm_mm;
260 	unsigned long oldflags = vma->vm_flags;
261 	long nrpages = (end - start) >> PAGE_SHIFT;
262 	unsigned long charged = 0;
263 	pgoff_t pgoff;
264 	int error;
265 	int dirty_accountable = 0;
266 
267 	if (newflags == oldflags) {
268 		*pprev = vma;
269 		return 0;
270 	}
271 
272 	/*
273 	 * If we make a private mapping writable we increase our commit;
274 	 * but (without finer accounting) cannot reduce our commit if we
275 	 * make it unwritable again. hugetlb mapping were accounted for
276 	 * even if read-only so there is no need to account for them here
277 	 */
278 	if (newflags & VM_WRITE) {
279 		if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
280 						VM_SHARED|VM_NORESERVE))) {
281 			charged = nrpages;
282 			if (security_vm_enough_memory_mm(mm, charged))
283 				return -ENOMEM;
284 			newflags |= VM_ACCOUNT;
285 		}
286 	}
287 
288 	/*
289 	 * First try to merge with previous and/or next vma.
290 	 */
291 	pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
292 	*pprev = vma_merge(mm, *pprev, start, end, newflags,
293 			vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
294 	if (*pprev) {
295 		vma = *pprev;
296 		goto success;
297 	}
298 
299 	*pprev = vma;
300 
301 	if (start != vma->vm_start) {
302 		error = split_vma(mm, vma, start, 1);
303 		if (error)
304 			goto fail;
305 	}
306 
307 	if (end != vma->vm_end) {
308 		error = split_vma(mm, vma, end, 0);
309 		if (error)
310 			goto fail;
311 	}
312 
313 success:
314 	/*
315 	 * vm_flags and vm_page_prot are protected by the mmap_sem
316 	 * held in write mode.
317 	 */
318 	vma->vm_flags = newflags;
319 	dirty_accountable = vma_wants_writenotify(vma);
320 	vma_set_page_prot(vma);
321 
322 	change_protection(vma, start, end, vma->vm_page_prot,
323 			  dirty_accountable, 0);
324 
325 	vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
326 	vm_stat_account(mm, newflags, vma->vm_file, nrpages);
327 	perf_event_mmap(vma);
328 	return 0;
329 
330 fail:
331 	vm_unacct_memory(charged);
332 	return error;
333 }
334 
335 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
336 		unsigned long, prot)
337 {
338 	unsigned long vm_flags, nstart, end, tmp, reqprot;
339 	struct vm_area_struct *vma, *prev;
340 	int error = -EINVAL;
341 	const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
342 	prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
343 	if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
344 		return -EINVAL;
345 
346 	if (start & ~PAGE_MASK)
347 		return -EINVAL;
348 	if (!len)
349 		return 0;
350 	len = PAGE_ALIGN(len);
351 	end = start + len;
352 	if (end <= start)
353 		return -ENOMEM;
354 	if (!arch_validate_prot(prot))
355 		return -EINVAL;
356 
357 	reqprot = prot;
358 	/*
359 	 * Does the application expect PROT_READ to imply PROT_EXEC:
360 	 */
361 	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
362 		prot |= PROT_EXEC;
363 
364 	vm_flags = calc_vm_prot_bits(prot);
365 
366 	down_write(&current->mm->mmap_sem);
367 
368 	vma = find_vma(current->mm, start);
369 	error = -ENOMEM;
370 	if (!vma)
371 		goto out;
372 	prev = vma->vm_prev;
373 	if (unlikely(grows & PROT_GROWSDOWN)) {
374 		if (vma->vm_start >= end)
375 			goto out;
376 		start = vma->vm_start;
377 		error = -EINVAL;
378 		if (!(vma->vm_flags & VM_GROWSDOWN))
379 			goto out;
380 	} else {
381 		if (vma->vm_start > start)
382 			goto out;
383 		if (unlikely(grows & PROT_GROWSUP)) {
384 			end = vma->vm_end;
385 			error = -EINVAL;
386 			if (!(vma->vm_flags & VM_GROWSUP))
387 				goto out;
388 		}
389 	}
390 	if (start > vma->vm_start)
391 		prev = vma;
392 
393 	for (nstart = start ; ; ) {
394 		unsigned long newflags;
395 
396 		/* Here we know that vma->vm_start <= nstart < vma->vm_end. */
397 
398 		newflags = vm_flags;
399 		newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC));
400 
401 		/* newflags >> 4 shift VM_MAY% in place of VM_% */
402 		if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
403 			error = -EACCES;
404 			goto out;
405 		}
406 
407 		error = security_file_mprotect(vma, reqprot, prot);
408 		if (error)
409 			goto out;
410 
411 		tmp = vma->vm_end;
412 		if (tmp > end)
413 			tmp = end;
414 		error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
415 		if (error)
416 			goto out;
417 		nstart = tmp;
418 
419 		if (nstart < prev->vm_end)
420 			nstart = prev->vm_end;
421 		if (nstart >= end)
422 			goto out;
423 
424 		vma = prev->vm_next;
425 		if (!vma || vma->vm_start != nstart) {
426 			error = -ENOMEM;
427 			goto out;
428 		}
429 	}
430 out:
431 	up_write(&current->mm->mmap_sem);
432 	return error;
433 }
434