xref: /openbmc/linux/mm/mremap.c (revision d5cb9783536a41df9f9cba5b0a1d78047ed787f7)
1 /*
2  *	mm/mremap.c
3  *
4  *	(C) Copyright 1996 Linus Torvalds
5  *
6  *	Address space accounting code	<alan@redhat.com>
7  *	(C) Copyright 2002 Red Hat Inc, All Rights Reserved
8  */
9 
10 #include <linux/mm.h>
11 #include <linux/hugetlb.h>
12 #include <linux/slab.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
15 #include <linux/swap.h>
16 #include <linux/fs.h>
17 #include <linux/highmem.h>
18 #include <linux/security.h>
19 #include <linux/syscalls.h>
20 
21 #include <asm/uaccess.h>
22 #include <asm/cacheflush.h>
23 #include <asm/tlbflush.h>
24 
25 static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
26 {
27 	pgd_t *pgd;
28 	pud_t *pud;
29 	pmd_t *pmd;
30 
31 	pgd = pgd_offset(mm, addr);
32 	if (pgd_none_or_clear_bad(pgd))
33 		return NULL;
34 
35 	pud = pud_offset(pgd, addr);
36 	if (pud_none_or_clear_bad(pud))
37 		return NULL;
38 
39 	pmd = pmd_offset(pud, addr);
40 	if (pmd_none_or_clear_bad(pmd))
41 		return NULL;
42 
43 	return pmd;
44 }
45 
46 static pmd_t *alloc_new_pmd(struct mm_struct *mm, unsigned long addr)
47 {
48 	pgd_t *pgd;
49 	pud_t *pud;
50 	pmd_t *pmd;
51 
52 	pgd = pgd_offset(mm, addr);
53 	pud = pud_alloc(mm, pgd, addr);
54 	if (!pud)
55 		return NULL;
56 
57 	pmd = pmd_alloc(mm, pud, addr);
58 	if (!pmd)
59 		return NULL;
60 
61 	if (!pmd_present(*pmd) && __pte_alloc(mm, pmd, addr))
62 		return NULL;
63 
64 	return pmd;
65 }
66 
67 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
68 		unsigned long old_addr, unsigned long old_end,
69 		struct vm_area_struct *new_vma, pmd_t *new_pmd,
70 		unsigned long new_addr)
71 {
72 	struct address_space *mapping = NULL;
73 	struct mm_struct *mm = vma->vm_mm;
74 	pte_t *old_pte, *new_pte, pte;
75 	spinlock_t *old_ptl, *new_ptl;
76 
77 	if (vma->vm_file) {
78 		/*
79 		 * Subtle point from Rajesh Venkatasubramanian: before
80 		 * moving file-based ptes, we must lock vmtruncate out,
81 		 * since it might clean the dst vma before the src vma,
82 		 * and we propagate stale pages into the dst afterward.
83 		 */
84 		mapping = vma->vm_file->f_mapping;
85 		spin_lock(&mapping->i_mmap_lock);
86 		if (new_vma->vm_truncate_count &&
87 		    new_vma->vm_truncate_count != vma->vm_truncate_count)
88 			new_vma->vm_truncate_count = 0;
89 	}
90 
91 	/*
92 	 * We don't have to worry about the ordering of src and dst
93 	 * pte locks because exclusive mmap_sem prevents deadlock.
94 	 */
95 	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
96  	new_pte = pte_offset_map_nested(new_pmd, new_addr);
97 	new_ptl = pte_lockptr(mm, new_pmd);
98 	if (new_ptl != old_ptl)
99 		spin_lock(new_ptl);
100 
101 	for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
102 				   new_pte++, new_addr += PAGE_SIZE) {
103 		if (pte_none(*old_pte))
104 			continue;
105 		pte = ptep_clear_flush(vma, old_addr, old_pte);
106 		/* ZERO_PAGE can be dependant on virtual addr */
107 		pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
108 		set_pte_at(mm, new_addr, new_pte, pte);
109 	}
110 
111 	if (new_ptl != old_ptl)
112 		spin_unlock(new_ptl);
113 	pte_unmap_nested(new_pte - 1);
114 	pte_unmap_unlock(old_pte - 1, old_ptl);
115 	if (mapping)
116 		spin_unlock(&mapping->i_mmap_lock);
117 }
118 
119 #define LATENCY_LIMIT	(64 * PAGE_SIZE)
120 
121 static unsigned long move_page_tables(struct vm_area_struct *vma,
122 		unsigned long old_addr, struct vm_area_struct *new_vma,
123 		unsigned long new_addr, unsigned long len)
124 {
125 	unsigned long extent, next, old_end;
126 	pmd_t *old_pmd, *new_pmd;
127 
128 	old_end = old_addr + len;
129 	flush_cache_range(vma, old_addr, old_end);
130 
131 	for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
132 		cond_resched();
133 		next = (old_addr + PMD_SIZE) & PMD_MASK;
134 		if (next - 1 > old_end)
135 			next = old_end;
136 		extent = next - old_addr;
137 		old_pmd = get_old_pmd(vma->vm_mm, old_addr);
138 		if (!old_pmd)
139 			continue;
140 		new_pmd = alloc_new_pmd(vma->vm_mm, new_addr);
141 		if (!new_pmd)
142 			break;
143 		next = (new_addr + PMD_SIZE) & PMD_MASK;
144 		if (extent > next - new_addr)
145 			extent = next - new_addr;
146 		if (extent > LATENCY_LIMIT)
147 			extent = LATENCY_LIMIT;
148 		move_ptes(vma, old_pmd, old_addr, old_addr + extent,
149 				new_vma, new_pmd, new_addr);
150 	}
151 
152 	return len + old_addr - old_end;	/* how much done */
153 }
154 
155 static unsigned long move_vma(struct vm_area_struct *vma,
156 		unsigned long old_addr, unsigned long old_len,
157 		unsigned long new_len, unsigned long new_addr)
158 {
159 	struct mm_struct *mm = vma->vm_mm;
160 	struct vm_area_struct *new_vma;
161 	unsigned long vm_flags = vma->vm_flags;
162 	unsigned long new_pgoff;
163 	unsigned long moved_len;
164 	unsigned long excess = 0;
165 	unsigned long hiwater_vm;
166 	int split = 0;
167 
168 	/*
169 	 * We'd prefer to avoid failure later on in do_munmap:
170 	 * which may split one vma into three before unmapping.
171 	 */
172 	if (mm->map_count >= sysctl_max_map_count - 3)
173 		return -ENOMEM;
174 
175 	new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
176 	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
177 	if (!new_vma)
178 		return -ENOMEM;
179 
180 	moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
181 	if (moved_len < old_len) {
182 		/*
183 		 * On error, move entries back from new area to old,
184 		 * which will succeed since page tables still there,
185 		 * and then proceed to unmap new area instead of old.
186 		 */
187 		move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
188 		vma = new_vma;
189 		old_len = new_len;
190 		old_addr = new_addr;
191 		new_addr = -ENOMEM;
192 	}
193 
194 	/* Conceal VM_ACCOUNT so old reservation is not undone */
195 	if (vm_flags & VM_ACCOUNT) {
196 		vma->vm_flags &= ~VM_ACCOUNT;
197 		excess = vma->vm_end - vma->vm_start - old_len;
198 		if (old_addr > vma->vm_start &&
199 		    old_addr + old_len < vma->vm_end)
200 			split = 1;
201 	}
202 
203 	/*
204 	 * If we failed to move page tables we still do total_vm increment
205 	 * since do_munmap() will decrement it by old_len == new_len.
206 	 *
207 	 * Since total_vm is about to be raised artificially high for a
208 	 * moment, we need to restore high watermark afterwards: if stats
209 	 * are taken meanwhile, total_vm and hiwater_vm appear too high.
210 	 * If this were a serious issue, we'd add a flag to do_munmap().
211 	 */
212 	hiwater_vm = mm->hiwater_vm;
213 	mm->total_vm += new_len >> PAGE_SHIFT;
214 	vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
215 
216 	if (do_munmap(mm, old_addr, old_len) < 0) {
217 		/* OOM: unable to split vma, just get accounts right */
218 		vm_unacct_memory(excess >> PAGE_SHIFT);
219 		excess = 0;
220 	}
221 	mm->hiwater_vm = hiwater_vm;
222 
223 	/* Restore VM_ACCOUNT if one or two pieces of vma left */
224 	if (excess) {
225 		vma->vm_flags |= VM_ACCOUNT;
226 		if (split)
227 			vma->vm_next->vm_flags |= VM_ACCOUNT;
228 	}
229 
230 	if (vm_flags & VM_LOCKED) {
231 		mm->locked_vm += new_len >> PAGE_SHIFT;
232 		if (new_len > old_len)
233 			make_pages_present(new_addr + old_len,
234 					   new_addr + new_len);
235 	}
236 
237 	return new_addr;
238 }
239 
240 /*
241  * Expand (or shrink) an existing mapping, potentially moving it at the
242  * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
243  *
244  * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
245  * This option implies MREMAP_MAYMOVE.
246  */
247 unsigned long do_mremap(unsigned long addr,
248 	unsigned long old_len, unsigned long new_len,
249 	unsigned long flags, unsigned long new_addr)
250 {
251 	struct mm_struct *mm = current->mm;
252 	struct vm_area_struct *vma;
253 	unsigned long ret = -EINVAL;
254 	unsigned long charged = 0;
255 
256 	if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
257 		goto out;
258 
259 	if (addr & ~PAGE_MASK)
260 		goto out;
261 
262 	old_len = PAGE_ALIGN(old_len);
263 	new_len = PAGE_ALIGN(new_len);
264 
265 	/*
266 	 * We allow a zero old-len as a special case
267 	 * for DOS-emu "duplicate shm area" thing. But
268 	 * a zero new-len is nonsensical.
269 	 */
270 	if (!new_len)
271 		goto out;
272 
273 	/* new_addr is only valid if MREMAP_FIXED is specified */
274 	if (flags & MREMAP_FIXED) {
275 		if (new_addr & ~PAGE_MASK)
276 			goto out;
277 		if (!(flags & MREMAP_MAYMOVE))
278 			goto out;
279 
280 		if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
281 			goto out;
282 
283 		/* Check if the location we're moving into overlaps the
284 		 * old location at all, and fail if it does.
285 		 */
286 		if ((new_addr <= addr) && (new_addr+new_len) > addr)
287 			goto out;
288 
289 		if ((addr <= new_addr) && (addr+old_len) > new_addr)
290 			goto out;
291 
292 		ret = do_munmap(mm, new_addr, new_len);
293 		if (ret)
294 			goto out;
295 	}
296 
297 	/*
298 	 * Always allow a shrinking remap: that just unmaps
299 	 * the unnecessary pages..
300 	 * do_munmap does all the needed commit accounting
301 	 */
302 	if (old_len >= new_len) {
303 		ret = do_munmap(mm, addr+new_len, old_len - new_len);
304 		if (ret && old_len != new_len)
305 			goto out;
306 		ret = addr;
307 		if (!(flags & MREMAP_FIXED) || (new_addr == addr))
308 			goto out;
309 		old_len = new_len;
310 	}
311 
312 	/*
313 	 * Ok, we need to grow..  or relocate.
314 	 */
315 	ret = -EFAULT;
316 	vma = find_vma(mm, addr);
317 	if (!vma || vma->vm_start > addr)
318 		goto out;
319 	if (is_vm_hugetlb_page(vma)) {
320 		ret = -EINVAL;
321 		goto out;
322 	}
323 	/* We can't remap across vm area boundaries */
324 	if (old_len > vma->vm_end - addr)
325 		goto out;
326 	if (vma->vm_flags & VM_DONTEXPAND) {
327 		if (new_len > old_len)
328 			goto out;
329 	}
330 	if (vma->vm_flags & VM_LOCKED) {
331 		unsigned long locked, lock_limit;
332 		locked = mm->locked_vm << PAGE_SHIFT;
333 		lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
334 		locked += new_len - old_len;
335 		ret = -EAGAIN;
336 		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
337 			goto out;
338 	}
339 	if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) {
340 		ret = -ENOMEM;
341 		goto out;
342 	}
343 
344 	if (vma->vm_flags & VM_ACCOUNT) {
345 		charged = (new_len - old_len) >> PAGE_SHIFT;
346 		if (security_vm_enough_memory(charged))
347 			goto out_nc;
348 	}
349 
350 	/* old_len exactly to the end of the area..
351 	 * And we're not relocating the area.
352 	 */
353 	if (old_len == vma->vm_end - addr &&
354 	    !((flags & MREMAP_FIXED) && (addr != new_addr)) &&
355 	    (old_len != new_len || !(flags & MREMAP_MAYMOVE))) {
356 		unsigned long max_addr = TASK_SIZE;
357 		if (vma->vm_next)
358 			max_addr = vma->vm_next->vm_start;
359 		/* can we just expand the current mapping? */
360 		if (max_addr - addr >= new_len) {
361 			int pages = (new_len - old_len) >> PAGE_SHIFT;
362 
363 			vma_adjust(vma, vma->vm_start,
364 				addr + new_len, vma->vm_pgoff, NULL);
365 
366 			mm->total_vm += pages;
367 			vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
368 			if (vma->vm_flags & VM_LOCKED) {
369 				mm->locked_vm += pages;
370 				make_pages_present(addr + old_len,
371 						   addr + new_len);
372 			}
373 			ret = addr;
374 			goto out;
375 		}
376 	}
377 
378 	/*
379 	 * We weren't able to just expand or shrink the area,
380 	 * we need to create a new one and move it..
381 	 */
382 	ret = -ENOMEM;
383 	if (flags & MREMAP_MAYMOVE) {
384 		if (!(flags & MREMAP_FIXED)) {
385 			unsigned long map_flags = 0;
386 			if (vma->vm_flags & VM_MAYSHARE)
387 				map_flags |= MAP_SHARED;
388 
389 			new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
390 						vma->vm_pgoff, map_flags);
391 			ret = new_addr;
392 			if (new_addr & ~PAGE_MASK)
393 				goto out;
394 		}
395 		ret = move_vma(vma, addr, old_len, new_len, new_addr);
396 	}
397 out:
398 	if (ret & ~PAGE_MASK)
399 		vm_unacct_memory(charged);
400 out_nc:
401 	return ret;
402 }
403 
404 asmlinkage unsigned long sys_mremap(unsigned long addr,
405 	unsigned long old_len, unsigned long new_len,
406 	unsigned long flags, unsigned long new_addr)
407 {
408 	unsigned long ret;
409 
410 	down_write(&current->mm->mmap_sem);
411 	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
412 	up_write(&current->mm->mmap_sem);
413 	return ret;
414 }
415