1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm/mmap.c 4 * 5 * Written by obz. 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 */ 9 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 #include <linux/backing-dev.h> 15 #include <linux/mm.h> 16 #include <linux/vmacache.h> 17 #include <linux/shm.h> 18 #include <linux/mman.h> 19 #include <linux/pagemap.h> 20 #include <linux/swap.h> 21 #include <linux/syscalls.h> 22 #include <linux/capability.h> 23 #include <linux/init.h> 24 #include <linux/file.h> 25 #include <linux/fs.h> 26 #include <linux/personality.h> 27 #include <linux/security.h> 28 #include <linux/hugetlb.h> 29 #include <linux/shmem_fs.h> 30 #include <linux/profile.h> 31 #include <linux/export.h> 32 #include <linux/mount.h> 33 #include <linux/mempolicy.h> 34 #include <linux/rmap.h> 35 #include <linux/mmu_notifier.h> 36 #include <linux/mmdebug.h> 37 #include <linux/perf_event.h> 38 #include <linux/audit.h> 39 #include <linux/khugepaged.h> 40 #include <linux/uprobes.h> 41 #include <linux/rbtree_augmented.h> 42 #include <linux/notifier.h> 43 #include <linux/memory.h> 44 #include <linux/printk.h> 45 #include <linux/userfaultfd_k.h> 46 #include <linux/moduleparam.h> 47 #include <linux/pkeys.h> 48 #include <linux/oom.h> 49 #include <linux/sched/mm.h> 50 51 #include <linux/uaccess.h> 52 #include <asm/cacheflush.h> 53 #include <asm/tlb.h> 54 #include <asm/mmu_context.h> 55 56 #include "internal.h" 57 58 #ifndef arch_mmap_check 59 #define arch_mmap_check(addr, len, flags) (0) 60 #endif 61 62 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 63 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; 64 const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX; 65 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; 66 #endif 67 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 68 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; 69 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; 70 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; 71 #endif 72 73 static bool ignore_rlimit_data; 74 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); 75 76 static void unmap_region(struct mm_struct *mm, 77 struct vm_area_struct *vma, struct vm_area_struct *prev, 78 unsigned long start, unsigned long end); 79 80 /* description of effects of mapping type and prot in current implementation. 81 * this is due to the limited x86 page protection hardware. The expected 82 * behavior is in parens: 83 * 84 * map_type prot 85 * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC 86 * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes 87 * w: (no) no w: (no) no w: (yes) yes w: (no) no 88 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes 89 * 90 * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes 91 * w: (no) no w: (no) no w: (copy) copy w: (no) no 92 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes 93 * 94 * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and 95 * MAP_PRIVATE: 96 * r: (no) no 97 * w: (no) no 98 * x: (yes) yes 99 */ 100 pgprot_t protection_map[16] __ro_after_init = { 101 __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111, 102 __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111 103 }; 104 105 #ifndef CONFIG_ARCH_HAS_FILTER_PGPROT 106 static inline pgprot_t arch_filter_pgprot(pgprot_t prot) 107 { 108 return prot; 109 } 110 #endif 111 112 pgprot_t vm_get_page_prot(unsigned long vm_flags) 113 { 114 pgprot_t ret = __pgprot(pgprot_val(protection_map[vm_flags & 115 (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) | 116 pgprot_val(arch_vm_get_page_prot(vm_flags))); 117 118 return arch_filter_pgprot(ret); 119 } 120 EXPORT_SYMBOL(vm_get_page_prot); 121 122 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) 123 { 124 return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); 125 } 126 127 /* Update vma->vm_page_prot to reflect vma->vm_flags. */ 128 void vma_set_page_prot(struct vm_area_struct *vma) 129 { 130 unsigned long vm_flags = vma->vm_flags; 131 pgprot_t vm_page_prot; 132 133 vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); 134 if (vma_wants_writenotify(vma, vm_page_prot)) { 135 vm_flags &= ~VM_SHARED; 136 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); 137 } 138 /* remove_protection_ptes reads vma->vm_page_prot without mmap_sem */ 139 WRITE_ONCE(vma->vm_page_prot, vm_page_prot); 140 } 141 142 /* 143 * Requires inode->i_mapping->i_mmap_rwsem 144 */ 145 static void __remove_shared_vm_struct(struct vm_area_struct *vma, 146 struct file *file, struct address_space *mapping) 147 { 148 if (vma->vm_flags & VM_DENYWRITE) 149 atomic_inc(&file_inode(file)->i_writecount); 150 if (vma->vm_flags & VM_SHARED) 151 mapping_unmap_writable(mapping); 152 153 flush_dcache_mmap_lock(mapping); 154 vma_interval_tree_remove(vma, &mapping->i_mmap); 155 flush_dcache_mmap_unlock(mapping); 156 } 157 158 /* 159 * Unlink a file-based vm structure from its interval tree, to hide 160 * vma from rmap and vmtruncate before freeing its page tables. 161 */ 162 void unlink_file_vma(struct vm_area_struct *vma) 163 { 164 struct file *file = vma->vm_file; 165 166 if (file) { 167 struct address_space *mapping = file->f_mapping; 168 i_mmap_lock_write(mapping); 169 __remove_shared_vm_struct(vma, file, mapping); 170 i_mmap_unlock_write(mapping); 171 } 172 } 173 174 /* 175 * Close a vm structure and free it, returning the next. 176 */ 177 static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) 178 { 179 struct vm_area_struct *next = vma->vm_next; 180 181 might_sleep(); 182 if (vma->vm_ops && vma->vm_ops->close) 183 vma->vm_ops->close(vma); 184 if (vma->vm_file) 185 fput(vma->vm_file); 186 mpol_put(vma_policy(vma)); 187 vm_area_free(vma); 188 return next; 189 } 190 191 static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, 192 struct list_head *uf); 193 SYSCALL_DEFINE1(brk, unsigned long, brk) 194 { 195 unsigned long retval; 196 unsigned long newbrk, oldbrk, origbrk; 197 struct mm_struct *mm = current->mm; 198 struct vm_area_struct *next; 199 unsigned long min_brk; 200 bool populate; 201 bool downgraded = false; 202 LIST_HEAD(uf); 203 204 brk = untagged_addr(brk); 205 206 if (down_write_killable(&mm->mmap_sem)) 207 return -EINTR; 208 209 origbrk = mm->brk; 210 211 #ifdef CONFIG_COMPAT_BRK 212 /* 213 * CONFIG_COMPAT_BRK can still be overridden by setting 214 * randomize_va_space to 2, which will still cause mm->start_brk 215 * to be arbitrarily shifted 216 */ 217 if (current->brk_randomized) 218 min_brk = mm->start_brk; 219 else 220 min_brk = mm->end_data; 221 #else 222 min_brk = mm->start_brk; 223 #endif 224 if (brk < min_brk) 225 goto out; 226 227 /* 228 * Check against rlimit here. If this check is done later after the test 229 * of oldbrk with newbrk then it can escape the test and let the data 230 * segment grow beyond its set limit the in case where the limit is 231 * not page aligned -Ram Gupta 232 */ 233 if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk, 234 mm->end_data, mm->start_data)) 235 goto out; 236 237 newbrk = PAGE_ALIGN(brk); 238 oldbrk = PAGE_ALIGN(mm->brk); 239 if (oldbrk == newbrk) { 240 mm->brk = brk; 241 goto success; 242 } 243 244 /* 245 * Always allow shrinking brk. 246 * __do_munmap() may downgrade mmap_sem to read. 247 */ 248 if (brk <= mm->brk) { 249 int ret; 250 251 /* 252 * mm->brk must to be protected by write mmap_sem so update it 253 * before downgrading mmap_sem. When __do_munmap() fails, 254 * mm->brk will be restored from origbrk. 255 */ 256 mm->brk = brk; 257 ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true); 258 if (ret < 0) { 259 mm->brk = origbrk; 260 goto out; 261 } else if (ret == 1) { 262 downgraded = true; 263 } 264 goto success; 265 } 266 267 /* Check against existing mmap mappings. */ 268 next = find_vma(mm, oldbrk); 269 if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) 270 goto out; 271 272 /* Ok, looks good - let it rip. */ 273 if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) 274 goto out; 275 mm->brk = brk; 276 277 success: 278 populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0; 279 if (downgraded) 280 up_read(&mm->mmap_sem); 281 else 282 up_write(&mm->mmap_sem); 283 userfaultfd_unmap_complete(mm, &uf); 284 if (populate) 285 mm_populate(oldbrk, newbrk - oldbrk); 286 return brk; 287 288 out: 289 retval = origbrk; 290 up_write(&mm->mmap_sem); 291 return retval; 292 } 293 294 static inline unsigned long vma_compute_gap(struct vm_area_struct *vma) 295 { 296 unsigned long gap, prev_end; 297 298 /* 299 * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we 300 * allow two stack_guard_gaps between them here, and when choosing 301 * an unmapped area; whereas when expanding we only require one. 302 * That's a little inconsistent, but keeps the code here simpler. 303 */ 304 gap = vm_start_gap(vma); 305 if (vma->vm_prev) { 306 prev_end = vm_end_gap(vma->vm_prev); 307 if (gap > prev_end) 308 gap -= prev_end; 309 else 310 gap = 0; 311 } 312 return gap; 313 } 314 315 #ifdef CONFIG_DEBUG_VM_RB 316 static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma) 317 { 318 unsigned long max = vma_compute_gap(vma), subtree_gap; 319 if (vma->vm_rb.rb_left) { 320 subtree_gap = rb_entry(vma->vm_rb.rb_left, 321 struct vm_area_struct, vm_rb)->rb_subtree_gap; 322 if (subtree_gap > max) 323 max = subtree_gap; 324 } 325 if (vma->vm_rb.rb_right) { 326 subtree_gap = rb_entry(vma->vm_rb.rb_right, 327 struct vm_area_struct, vm_rb)->rb_subtree_gap; 328 if (subtree_gap > max) 329 max = subtree_gap; 330 } 331 return max; 332 } 333 334 static int browse_rb(struct mm_struct *mm) 335 { 336 struct rb_root *root = &mm->mm_rb; 337 int i = 0, j, bug = 0; 338 struct rb_node *nd, *pn = NULL; 339 unsigned long prev = 0, pend = 0; 340 341 for (nd = rb_first(root); nd; nd = rb_next(nd)) { 342 struct vm_area_struct *vma; 343 vma = rb_entry(nd, struct vm_area_struct, vm_rb); 344 if (vma->vm_start < prev) { 345 pr_emerg("vm_start %lx < prev %lx\n", 346 vma->vm_start, prev); 347 bug = 1; 348 } 349 if (vma->vm_start < pend) { 350 pr_emerg("vm_start %lx < pend %lx\n", 351 vma->vm_start, pend); 352 bug = 1; 353 } 354 if (vma->vm_start > vma->vm_end) { 355 pr_emerg("vm_start %lx > vm_end %lx\n", 356 vma->vm_start, vma->vm_end); 357 bug = 1; 358 } 359 spin_lock(&mm->page_table_lock); 360 if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) { 361 pr_emerg("free gap %lx, correct %lx\n", 362 vma->rb_subtree_gap, 363 vma_compute_subtree_gap(vma)); 364 bug = 1; 365 } 366 spin_unlock(&mm->page_table_lock); 367 i++; 368 pn = nd; 369 prev = vma->vm_start; 370 pend = vma->vm_end; 371 } 372 j = 0; 373 for (nd = pn; nd; nd = rb_prev(nd)) 374 j++; 375 if (i != j) { 376 pr_emerg("backwards %d, forwards %d\n", j, i); 377 bug = 1; 378 } 379 return bug ? -1 : i; 380 } 381 382 static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore) 383 { 384 struct rb_node *nd; 385 386 for (nd = rb_first(root); nd; nd = rb_next(nd)) { 387 struct vm_area_struct *vma; 388 vma = rb_entry(nd, struct vm_area_struct, vm_rb); 389 VM_BUG_ON_VMA(vma != ignore && 390 vma->rb_subtree_gap != vma_compute_subtree_gap(vma), 391 vma); 392 } 393 } 394 395 static void validate_mm(struct mm_struct *mm) 396 { 397 int bug = 0; 398 int i = 0; 399 unsigned long highest_address = 0; 400 struct vm_area_struct *vma = mm->mmap; 401 402 while (vma) { 403 struct anon_vma *anon_vma = vma->anon_vma; 404 struct anon_vma_chain *avc; 405 406 if (anon_vma) { 407 anon_vma_lock_read(anon_vma); 408 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 409 anon_vma_interval_tree_verify(avc); 410 anon_vma_unlock_read(anon_vma); 411 } 412 413 highest_address = vm_end_gap(vma); 414 vma = vma->vm_next; 415 i++; 416 } 417 if (i != mm->map_count) { 418 pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); 419 bug = 1; 420 } 421 if (highest_address != mm->highest_vm_end) { 422 pr_emerg("mm->highest_vm_end %lx, found %lx\n", 423 mm->highest_vm_end, highest_address); 424 bug = 1; 425 } 426 i = browse_rb(mm); 427 if (i != mm->map_count) { 428 if (i != -1) 429 pr_emerg("map_count %d rb %d\n", mm->map_count, i); 430 bug = 1; 431 } 432 VM_BUG_ON_MM(bug, mm); 433 } 434 #else 435 #define validate_mm_rb(root, ignore) do { } while (0) 436 #define validate_mm(mm) do { } while (0) 437 #endif 438 439 RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks, 440 struct vm_area_struct, vm_rb, 441 unsigned long, rb_subtree_gap, vma_compute_gap) 442 443 /* 444 * Update augmented rbtree rb_subtree_gap values after vma->vm_start or 445 * vma->vm_prev->vm_end values changed, without modifying the vma's position 446 * in the rbtree. 447 */ 448 static void vma_gap_update(struct vm_area_struct *vma) 449 { 450 /* 451 * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created 452 * a callback function that does exactly what we want. 453 */ 454 vma_gap_callbacks_propagate(&vma->vm_rb, NULL); 455 } 456 457 static inline void vma_rb_insert(struct vm_area_struct *vma, 458 struct rb_root *root) 459 { 460 /* All rb_subtree_gap values must be consistent prior to insertion */ 461 validate_mm_rb(root, NULL); 462 463 rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks); 464 } 465 466 static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) 467 { 468 /* 469 * Note rb_erase_augmented is a fairly large inline function, 470 * so make sure we instantiate it only once with our desired 471 * augmented rbtree callbacks. 472 */ 473 rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); 474 } 475 476 static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, 477 struct rb_root *root, 478 struct vm_area_struct *ignore) 479 { 480 /* 481 * All rb_subtree_gap values must be consistent prior to erase, 482 * with the possible exception of the "next" vma being erased if 483 * next->vm_start was reduced. 484 */ 485 validate_mm_rb(root, ignore); 486 487 __vma_rb_erase(vma, root); 488 } 489 490 static __always_inline void vma_rb_erase(struct vm_area_struct *vma, 491 struct rb_root *root) 492 { 493 /* 494 * All rb_subtree_gap values must be consistent prior to erase, 495 * with the possible exception of the vma being erased. 496 */ 497 validate_mm_rb(root, vma); 498 499 __vma_rb_erase(vma, root); 500 } 501 502 /* 503 * vma has some anon_vma assigned, and is already inserted on that 504 * anon_vma's interval trees. 505 * 506 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 507 * vma must be removed from the anon_vma's interval trees using 508 * anon_vma_interval_tree_pre_update_vma(). 509 * 510 * After the update, the vma will be reinserted using 511 * anon_vma_interval_tree_post_update_vma(). 512 * 513 * The entire update must be protected by exclusive mmap_sem and by 514 * the root anon_vma's mutex. 515 */ 516 static inline void 517 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 518 { 519 struct anon_vma_chain *avc; 520 521 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 522 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 523 } 524 525 static inline void 526 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 527 { 528 struct anon_vma_chain *avc; 529 530 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 531 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 532 } 533 534 static int find_vma_links(struct mm_struct *mm, unsigned long addr, 535 unsigned long end, struct vm_area_struct **pprev, 536 struct rb_node ***rb_link, struct rb_node **rb_parent) 537 { 538 struct rb_node **__rb_link, *__rb_parent, *rb_prev; 539 540 __rb_link = &mm->mm_rb.rb_node; 541 rb_prev = __rb_parent = NULL; 542 543 while (*__rb_link) { 544 struct vm_area_struct *vma_tmp; 545 546 __rb_parent = *__rb_link; 547 vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb); 548 549 if (vma_tmp->vm_end > addr) { 550 /* Fail if an existing vma overlaps the area */ 551 if (vma_tmp->vm_start < end) 552 return -ENOMEM; 553 __rb_link = &__rb_parent->rb_left; 554 } else { 555 rb_prev = __rb_parent; 556 __rb_link = &__rb_parent->rb_right; 557 } 558 } 559 560 *pprev = NULL; 561 if (rb_prev) 562 *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); 563 *rb_link = __rb_link; 564 *rb_parent = __rb_parent; 565 return 0; 566 } 567 568 static unsigned long count_vma_pages_range(struct mm_struct *mm, 569 unsigned long addr, unsigned long end) 570 { 571 unsigned long nr_pages = 0; 572 struct vm_area_struct *vma; 573 574 /* Find first overlaping mapping */ 575 vma = find_vma_intersection(mm, addr, end); 576 if (!vma) 577 return 0; 578 579 nr_pages = (min(end, vma->vm_end) - 580 max(addr, vma->vm_start)) >> PAGE_SHIFT; 581 582 /* Iterate over the rest of the overlaps */ 583 for (vma = vma->vm_next; vma; vma = vma->vm_next) { 584 unsigned long overlap_len; 585 586 if (vma->vm_start > end) 587 break; 588 589 overlap_len = min(end, vma->vm_end) - vma->vm_start; 590 nr_pages += overlap_len >> PAGE_SHIFT; 591 } 592 593 return nr_pages; 594 } 595 596 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, 597 struct rb_node **rb_link, struct rb_node *rb_parent) 598 { 599 /* Update tracking information for the gap following the new vma. */ 600 if (vma->vm_next) 601 vma_gap_update(vma->vm_next); 602 else 603 mm->highest_vm_end = vm_end_gap(vma); 604 605 /* 606 * vma->vm_prev wasn't known when we followed the rbtree to find the 607 * correct insertion point for that vma. As a result, we could not 608 * update the vma vm_rb parents rb_subtree_gap values on the way down. 609 * So, we first insert the vma with a zero rb_subtree_gap value 610 * (to be consistent with what we did on the way down), and then 611 * immediately update the gap to the correct value. Finally we 612 * rebalance the rbtree after all augmented values have been set. 613 */ 614 rb_link_node(&vma->vm_rb, rb_parent, rb_link); 615 vma->rb_subtree_gap = 0; 616 vma_gap_update(vma); 617 vma_rb_insert(vma, &mm->mm_rb); 618 } 619 620 static void __vma_link_file(struct vm_area_struct *vma) 621 { 622 struct file *file; 623 624 file = vma->vm_file; 625 if (file) { 626 struct address_space *mapping = file->f_mapping; 627 628 if (vma->vm_flags & VM_DENYWRITE) 629 atomic_dec(&file_inode(file)->i_writecount); 630 if (vma->vm_flags & VM_SHARED) 631 atomic_inc(&mapping->i_mmap_writable); 632 633 flush_dcache_mmap_lock(mapping); 634 vma_interval_tree_insert(vma, &mapping->i_mmap); 635 flush_dcache_mmap_unlock(mapping); 636 } 637 } 638 639 static void 640 __vma_link(struct mm_struct *mm, struct vm_area_struct *vma, 641 struct vm_area_struct *prev, struct rb_node **rb_link, 642 struct rb_node *rb_parent) 643 { 644 __vma_link_list(mm, vma, prev); 645 __vma_link_rb(mm, vma, rb_link, rb_parent); 646 } 647 648 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, 649 struct vm_area_struct *prev, struct rb_node **rb_link, 650 struct rb_node *rb_parent) 651 { 652 struct address_space *mapping = NULL; 653 654 if (vma->vm_file) { 655 mapping = vma->vm_file->f_mapping; 656 i_mmap_lock_write(mapping); 657 } 658 659 __vma_link(mm, vma, prev, rb_link, rb_parent); 660 __vma_link_file(vma); 661 662 if (mapping) 663 i_mmap_unlock_write(mapping); 664 665 mm->map_count++; 666 validate_mm(mm); 667 } 668 669 /* 670 * Helper for vma_adjust() in the split_vma insert case: insert a vma into the 671 * mm's list and rbtree. It has already been inserted into the interval tree. 672 */ 673 static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) 674 { 675 struct vm_area_struct *prev; 676 struct rb_node **rb_link, *rb_parent; 677 678 if (find_vma_links(mm, vma->vm_start, vma->vm_end, 679 &prev, &rb_link, &rb_parent)) 680 BUG(); 681 __vma_link(mm, vma, prev, rb_link, rb_parent); 682 mm->map_count++; 683 } 684 685 static __always_inline void __vma_unlink_common(struct mm_struct *mm, 686 struct vm_area_struct *vma, 687 struct vm_area_struct *ignore) 688 { 689 vma_rb_erase_ignore(vma, &mm->mm_rb, ignore); 690 __vma_unlink_list(mm, vma); 691 /* Kill the cache */ 692 vmacache_invalidate(mm); 693 } 694 695 /* 696 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that 697 * is already present in an i_mmap tree without adjusting the tree. 698 * The following helper function should be used when such adjustments 699 * are necessary. The "insert" vma (if any) is to be inserted 700 * before we drop the necessary locks. 701 */ 702 int __vma_adjust(struct vm_area_struct *vma, unsigned long start, 703 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert, 704 struct vm_area_struct *expand) 705 { 706 struct mm_struct *mm = vma->vm_mm; 707 struct vm_area_struct *next = vma->vm_next, *orig_vma = vma; 708 struct address_space *mapping = NULL; 709 struct rb_root_cached *root = NULL; 710 struct anon_vma *anon_vma = NULL; 711 struct file *file = vma->vm_file; 712 bool start_changed = false, end_changed = false; 713 long adjust_next = 0; 714 int remove_next = 0; 715 716 if (next && !insert) { 717 struct vm_area_struct *exporter = NULL, *importer = NULL; 718 719 if (end >= next->vm_end) { 720 /* 721 * vma expands, overlapping all the next, and 722 * perhaps the one after too (mprotect case 6). 723 * The only other cases that gets here are 724 * case 1, case 7 and case 8. 725 */ 726 if (next == expand) { 727 /* 728 * The only case where we don't expand "vma" 729 * and we expand "next" instead is case 8. 730 */ 731 VM_WARN_ON(end != next->vm_end); 732 /* 733 * remove_next == 3 means we're 734 * removing "vma" and that to do so we 735 * swapped "vma" and "next". 736 */ 737 remove_next = 3; 738 VM_WARN_ON(file != next->vm_file); 739 swap(vma, next); 740 } else { 741 VM_WARN_ON(expand != vma); 742 /* 743 * case 1, 6, 7, remove_next == 2 is case 6, 744 * remove_next == 1 is case 1 or 7. 745 */ 746 remove_next = 1 + (end > next->vm_end); 747 VM_WARN_ON(remove_next == 2 && 748 end != next->vm_next->vm_end); 749 /* trim end to next, for case 6 first pass */ 750 end = next->vm_end; 751 } 752 753 exporter = next; 754 importer = vma; 755 756 /* 757 * If next doesn't have anon_vma, import from vma after 758 * next, if the vma overlaps with it. 759 */ 760 if (remove_next == 2 && !next->anon_vma) 761 exporter = next->vm_next; 762 763 } else if (end > next->vm_start) { 764 /* 765 * vma expands, overlapping part of the next: 766 * mprotect case 5 shifting the boundary up. 767 */ 768 adjust_next = (end - next->vm_start) >> PAGE_SHIFT; 769 exporter = next; 770 importer = vma; 771 VM_WARN_ON(expand != importer); 772 } else if (end < vma->vm_end) { 773 /* 774 * vma shrinks, and !insert tells it's not 775 * split_vma inserting another: so it must be 776 * mprotect case 4 shifting the boundary down. 777 */ 778 adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT); 779 exporter = vma; 780 importer = next; 781 VM_WARN_ON(expand != importer); 782 } 783 784 /* 785 * Easily overlooked: when mprotect shifts the boundary, 786 * make sure the expanding vma has anon_vma set if the 787 * shrinking vma had, to cover any anon pages imported. 788 */ 789 if (exporter && exporter->anon_vma && !importer->anon_vma) { 790 int error; 791 792 importer->anon_vma = exporter->anon_vma; 793 error = anon_vma_clone(importer, exporter); 794 if (error) 795 return error; 796 } 797 } 798 again: 799 vma_adjust_trans_huge(orig_vma, start, end, adjust_next); 800 801 if (file) { 802 mapping = file->f_mapping; 803 root = &mapping->i_mmap; 804 uprobe_munmap(vma, vma->vm_start, vma->vm_end); 805 806 if (adjust_next) 807 uprobe_munmap(next, next->vm_start, next->vm_end); 808 809 i_mmap_lock_write(mapping); 810 if (insert) { 811 /* 812 * Put into interval tree now, so instantiated pages 813 * are visible to arm/parisc __flush_dcache_page 814 * throughout; but we cannot insert into address 815 * space until vma start or end is updated. 816 */ 817 __vma_link_file(insert); 818 } 819 } 820 821 anon_vma = vma->anon_vma; 822 if (!anon_vma && adjust_next) 823 anon_vma = next->anon_vma; 824 if (anon_vma) { 825 VM_WARN_ON(adjust_next && next->anon_vma && 826 anon_vma != next->anon_vma); 827 anon_vma_lock_write(anon_vma); 828 anon_vma_interval_tree_pre_update_vma(vma); 829 if (adjust_next) 830 anon_vma_interval_tree_pre_update_vma(next); 831 } 832 833 if (root) { 834 flush_dcache_mmap_lock(mapping); 835 vma_interval_tree_remove(vma, root); 836 if (adjust_next) 837 vma_interval_tree_remove(next, root); 838 } 839 840 if (start != vma->vm_start) { 841 vma->vm_start = start; 842 start_changed = true; 843 } 844 if (end != vma->vm_end) { 845 vma->vm_end = end; 846 end_changed = true; 847 } 848 vma->vm_pgoff = pgoff; 849 if (adjust_next) { 850 next->vm_start += adjust_next << PAGE_SHIFT; 851 next->vm_pgoff += adjust_next; 852 } 853 854 if (root) { 855 if (adjust_next) 856 vma_interval_tree_insert(next, root); 857 vma_interval_tree_insert(vma, root); 858 flush_dcache_mmap_unlock(mapping); 859 } 860 861 if (remove_next) { 862 /* 863 * vma_merge has merged next into vma, and needs 864 * us to remove next before dropping the locks. 865 */ 866 if (remove_next != 3) 867 __vma_unlink_common(mm, next, next); 868 else 869 /* 870 * vma is not before next if they've been 871 * swapped. 872 * 873 * pre-swap() next->vm_start was reduced so 874 * tell validate_mm_rb to ignore pre-swap() 875 * "next" (which is stored in post-swap() 876 * "vma"). 877 */ 878 __vma_unlink_common(mm, next, vma); 879 if (file) 880 __remove_shared_vm_struct(next, file, mapping); 881 } else if (insert) { 882 /* 883 * split_vma has split insert from vma, and needs 884 * us to insert it before dropping the locks 885 * (it may either follow vma or precede it). 886 */ 887 __insert_vm_struct(mm, insert); 888 } else { 889 if (start_changed) 890 vma_gap_update(vma); 891 if (end_changed) { 892 if (!next) 893 mm->highest_vm_end = vm_end_gap(vma); 894 else if (!adjust_next) 895 vma_gap_update(next); 896 } 897 } 898 899 if (anon_vma) { 900 anon_vma_interval_tree_post_update_vma(vma); 901 if (adjust_next) 902 anon_vma_interval_tree_post_update_vma(next); 903 anon_vma_unlock_write(anon_vma); 904 } 905 if (mapping) 906 i_mmap_unlock_write(mapping); 907 908 if (root) { 909 uprobe_mmap(vma); 910 911 if (adjust_next) 912 uprobe_mmap(next); 913 } 914 915 if (remove_next) { 916 if (file) { 917 uprobe_munmap(next, next->vm_start, next->vm_end); 918 fput(file); 919 } 920 if (next->anon_vma) 921 anon_vma_merge(vma, next); 922 mm->map_count--; 923 mpol_put(vma_policy(next)); 924 vm_area_free(next); 925 /* 926 * In mprotect's case 6 (see comments on vma_merge), 927 * we must remove another next too. It would clutter 928 * up the code too much to do both in one go. 929 */ 930 if (remove_next != 3) { 931 /* 932 * If "next" was removed and vma->vm_end was 933 * expanded (up) over it, in turn 934 * "next->vm_prev->vm_end" changed and the 935 * "vma->vm_next" gap must be updated. 936 */ 937 next = vma->vm_next; 938 } else { 939 /* 940 * For the scope of the comment "next" and 941 * "vma" considered pre-swap(): if "vma" was 942 * removed, next->vm_start was expanded (down) 943 * over it and the "next" gap must be updated. 944 * Because of the swap() the post-swap() "vma" 945 * actually points to pre-swap() "next" 946 * (post-swap() "next" as opposed is now a 947 * dangling pointer). 948 */ 949 next = vma; 950 } 951 if (remove_next == 2) { 952 remove_next = 1; 953 end = next->vm_end; 954 goto again; 955 } 956 else if (next) 957 vma_gap_update(next); 958 else { 959 /* 960 * If remove_next == 2 we obviously can't 961 * reach this path. 962 * 963 * If remove_next == 3 we can't reach this 964 * path because pre-swap() next is always not 965 * NULL. pre-swap() "next" is not being 966 * removed and its next->vm_end is not altered 967 * (and furthermore "end" already matches 968 * next->vm_end in remove_next == 3). 969 * 970 * We reach this only in the remove_next == 1 971 * case if the "next" vma that was removed was 972 * the highest vma of the mm. However in such 973 * case next->vm_end == "end" and the extended 974 * "vma" has vma->vm_end == next->vm_end so 975 * mm->highest_vm_end doesn't need any update 976 * in remove_next == 1 case. 977 */ 978 VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); 979 } 980 } 981 if (insert && file) 982 uprobe_mmap(insert); 983 984 validate_mm(mm); 985 986 return 0; 987 } 988 989 /* 990 * If the vma has a ->close operation then the driver probably needs to release 991 * per-vma resources, so we don't attempt to merge those. 992 */ 993 static inline int is_mergeable_vma(struct vm_area_struct *vma, 994 struct file *file, unsigned long vm_flags, 995 struct vm_userfaultfd_ctx vm_userfaultfd_ctx) 996 { 997 /* 998 * VM_SOFTDIRTY should not prevent from VMA merging, if we 999 * match the flags but dirty bit -- the caller should mark 1000 * merged VMA as dirty. If dirty bit won't be excluded from 1001 * comparison, we increase pressure on the memory system forcing 1002 * the kernel to generate new VMAs when old one could be 1003 * extended instead. 1004 */ 1005 if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 1006 return 0; 1007 if (vma->vm_file != file) 1008 return 0; 1009 if (vma->vm_ops && vma->vm_ops->close) 1010 return 0; 1011 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 1012 return 0; 1013 return 1; 1014 } 1015 1016 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1, 1017 struct anon_vma *anon_vma2, 1018 struct vm_area_struct *vma) 1019 { 1020 /* 1021 * The list_is_singular() test is to avoid merging VMA cloned from 1022 * parents. This can improve scalability caused by anon_vma lock. 1023 */ 1024 if ((!anon_vma1 || !anon_vma2) && (!vma || 1025 list_is_singular(&vma->anon_vma_chain))) 1026 return 1; 1027 return anon_vma1 == anon_vma2; 1028 } 1029 1030 /* 1031 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 1032 * in front of (at a lower virtual address and file offset than) the vma. 1033 * 1034 * We cannot merge two vmas if they have differently assigned (non-NULL) 1035 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 1036 * 1037 * We don't check here for the merged mmap wrapping around the end of pagecache 1038 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which 1039 * wrap, nor mmaps which cover the final page at index -1UL. 1040 */ 1041 static int 1042 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 1043 struct anon_vma *anon_vma, struct file *file, 1044 pgoff_t vm_pgoff, 1045 struct vm_userfaultfd_ctx vm_userfaultfd_ctx) 1046 { 1047 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) && 1048 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 1049 if (vma->vm_pgoff == vm_pgoff) 1050 return 1; 1051 } 1052 return 0; 1053 } 1054 1055 /* 1056 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 1057 * beyond (at a higher virtual address and file offset than) the vma. 1058 * 1059 * We cannot merge two vmas if they have differently assigned (non-NULL) 1060 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 1061 */ 1062 static int 1063 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 1064 struct anon_vma *anon_vma, struct file *file, 1065 pgoff_t vm_pgoff, 1066 struct vm_userfaultfd_ctx vm_userfaultfd_ctx) 1067 { 1068 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) && 1069 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 1070 pgoff_t vm_pglen; 1071 vm_pglen = vma_pages(vma); 1072 if (vma->vm_pgoff + vm_pglen == vm_pgoff) 1073 return 1; 1074 } 1075 return 0; 1076 } 1077 1078 /* 1079 * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out 1080 * whether that can be merged with its predecessor or its successor. 1081 * Or both (it neatly fills a hole). 1082 * 1083 * In most cases - when called for mmap, brk or mremap - [addr,end) is 1084 * certain not to be mapped by the time vma_merge is called; but when 1085 * called for mprotect, it is certain to be already mapped (either at 1086 * an offset within prev, or at the start of next), and the flags of 1087 * this area are about to be changed to vm_flags - and the no-change 1088 * case has already been eliminated. 1089 * 1090 * The following mprotect cases have to be considered, where AAAA is 1091 * the area passed down from mprotect_fixup, never extending beyond one 1092 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after: 1093 * 1094 * AAAA AAAA AAAA 1095 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN 1096 * cannot merge might become might become 1097 * PPNNNNNNNNNN PPPPPPPPPPNN 1098 * mmap, brk or case 4 below case 5 below 1099 * mremap move: 1100 * AAAA AAAA 1101 * PPPP NNNN PPPPNNNNXXXX 1102 * might become might become 1103 * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 1104 * PPPPPPPPNNNN 2 or PPPPPPPPXXXX 7 or 1105 * PPPPNNNNNNNN 3 PPPPXXXXXXXX 8 1106 * 1107 * It is important for case 8 that the vma NNNN overlapping the 1108 * region AAAA is never going to extended over XXXX. Instead XXXX must 1109 * be extended in region AAAA and NNNN must be removed. This way in 1110 * all cases where vma_merge succeeds, the moment vma_adjust drops the 1111 * rmap_locks, the properties of the merged vma will be already 1112 * correct for the whole merged range. Some of those properties like 1113 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 1114 * be correct for the whole merged range immediately after the 1115 * rmap_locks are released. Otherwise if XXXX would be removed and 1116 * NNNN would be extended over the XXXX range, remove_migration_ptes 1117 * or other rmap walkers (if working on addresses beyond the "end" 1118 * parameter) may establish ptes with the wrong permissions of NNNN 1119 * instead of the right permissions of XXXX. 1120 */ 1121 struct vm_area_struct *vma_merge(struct mm_struct *mm, 1122 struct vm_area_struct *prev, unsigned long addr, 1123 unsigned long end, unsigned long vm_flags, 1124 struct anon_vma *anon_vma, struct file *file, 1125 pgoff_t pgoff, struct mempolicy *policy, 1126 struct vm_userfaultfd_ctx vm_userfaultfd_ctx) 1127 { 1128 pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 1129 struct vm_area_struct *area, *next; 1130 int err; 1131 1132 /* 1133 * We later require that vma->vm_flags == vm_flags, 1134 * so this tests vma->vm_flags & VM_SPECIAL, too. 1135 */ 1136 if (vm_flags & VM_SPECIAL) 1137 return NULL; 1138 1139 if (prev) 1140 next = prev->vm_next; 1141 else 1142 next = mm->mmap; 1143 area = next; 1144 if (area && area->vm_end == end) /* cases 6, 7, 8 */ 1145 next = next->vm_next; 1146 1147 /* verify some invariant that must be enforced by the caller */ 1148 VM_WARN_ON(prev && addr <= prev->vm_start); 1149 VM_WARN_ON(area && end > area->vm_end); 1150 VM_WARN_ON(addr >= end); 1151 1152 /* 1153 * Can it merge with the predecessor? 1154 */ 1155 if (prev && prev->vm_end == addr && 1156 mpol_equal(vma_policy(prev), policy) && 1157 can_vma_merge_after(prev, vm_flags, 1158 anon_vma, file, pgoff, 1159 vm_userfaultfd_ctx)) { 1160 /* 1161 * OK, it can. Can we now merge in the successor as well? 1162 */ 1163 if (next && end == next->vm_start && 1164 mpol_equal(policy, vma_policy(next)) && 1165 can_vma_merge_before(next, vm_flags, 1166 anon_vma, file, 1167 pgoff+pglen, 1168 vm_userfaultfd_ctx) && 1169 is_mergeable_anon_vma(prev->anon_vma, 1170 next->anon_vma, NULL)) { 1171 /* cases 1, 6 */ 1172 err = __vma_adjust(prev, prev->vm_start, 1173 next->vm_end, prev->vm_pgoff, NULL, 1174 prev); 1175 } else /* cases 2, 5, 7 */ 1176 err = __vma_adjust(prev, prev->vm_start, 1177 end, prev->vm_pgoff, NULL, prev); 1178 if (err) 1179 return NULL; 1180 khugepaged_enter_vma_merge(prev, vm_flags); 1181 return prev; 1182 } 1183 1184 /* 1185 * Can this new request be merged in front of next? 1186 */ 1187 if (next && end == next->vm_start && 1188 mpol_equal(policy, vma_policy(next)) && 1189 can_vma_merge_before(next, vm_flags, 1190 anon_vma, file, pgoff+pglen, 1191 vm_userfaultfd_ctx)) { 1192 if (prev && addr < prev->vm_end) /* case 4 */ 1193 err = __vma_adjust(prev, prev->vm_start, 1194 addr, prev->vm_pgoff, NULL, next); 1195 else { /* cases 3, 8 */ 1196 err = __vma_adjust(area, addr, next->vm_end, 1197 next->vm_pgoff - pglen, NULL, next); 1198 /* 1199 * In case 3 area is already equal to next and 1200 * this is a noop, but in case 8 "area" has 1201 * been removed and next was expanded over it. 1202 */ 1203 area = next; 1204 } 1205 if (err) 1206 return NULL; 1207 khugepaged_enter_vma_merge(area, vm_flags); 1208 return area; 1209 } 1210 1211 return NULL; 1212 } 1213 1214 /* 1215 * Rough compatbility check to quickly see if it's even worth looking 1216 * at sharing an anon_vma. 1217 * 1218 * They need to have the same vm_file, and the flags can only differ 1219 * in things that mprotect may change. 1220 * 1221 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1222 * we can merge the two vma's. For example, we refuse to merge a vma if 1223 * there is a vm_ops->close() function, because that indicates that the 1224 * driver is doing some kind of reference counting. But that doesn't 1225 * really matter for the anon_vma sharing case. 1226 */ 1227 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1228 { 1229 return a->vm_end == b->vm_start && 1230 mpol_equal(vma_policy(a), vma_policy(b)) && 1231 a->vm_file == b->vm_file && 1232 !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) && 1233 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1234 } 1235 1236 /* 1237 * Do some basic sanity checking to see if we can re-use the anon_vma 1238 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1239 * the same as 'old', the other will be the new one that is trying 1240 * to share the anon_vma. 1241 * 1242 * NOTE! This runs with mm_sem held for reading, so it is possible that 1243 * the anon_vma of 'old' is concurrently in the process of being set up 1244 * by another page fault trying to merge _that_. But that's ok: if it 1245 * is being set up, that automatically means that it will be a singleton 1246 * acceptable for merging, so we can do all of this optimistically. But 1247 * we do that READ_ONCE() to make sure that we never re-load the pointer. 1248 * 1249 * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1250 * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1251 * is to return an anon_vma that is "complex" due to having gone through 1252 * a fork). 1253 * 1254 * We also make sure that the two vma's are compatible (adjacent, 1255 * and with the same memory policies). That's all stable, even with just 1256 * a read lock on the mm_sem. 1257 */ 1258 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b) 1259 { 1260 if (anon_vma_compatible(a, b)) { 1261 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1262 1263 if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1264 return anon_vma; 1265 } 1266 return NULL; 1267 } 1268 1269 /* 1270 * find_mergeable_anon_vma is used by anon_vma_prepare, to check 1271 * neighbouring vmas for a suitable anon_vma, before it goes off 1272 * to allocate a new anon_vma. It checks because a repetitive 1273 * sequence of mprotects and faults may otherwise lead to distinct 1274 * anon_vmas being allocated, preventing vma merge in subsequent 1275 * mprotect. 1276 */ 1277 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 1278 { 1279 struct anon_vma *anon_vma; 1280 struct vm_area_struct *near; 1281 1282 near = vma->vm_next; 1283 if (!near) 1284 goto try_prev; 1285 1286 anon_vma = reusable_anon_vma(near, vma, near); 1287 if (anon_vma) 1288 return anon_vma; 1289 try_prev: 1290 near = vma->vm_prev; 1291 if (!near) 1292 goto none; 1293 1294 anon_vma = reusable_anon_vma(near, near, vma); 1295 if (anon_vma) 1296 return anon_vma; 1297 none: 1298 /* 1299 * There's no absolute need to look only at touching neighbours: 1300 * we could search further afield for "compatible" anon_vmas. 1301 * But it would probably just be a waste of time searching, 1302 * or lead to too many vmas hanging off the same anon_vma. 1303 * We're trying to allow mprotect remerging later on, 1304 * not trying to minimize memory used for anon_vmas. 1305 */ 1306 return NULL; 1307 } 1308 1309 /* 1310 * If a hint addr is less than mmap_min_addr change hint to be as 1311 * low as possible but still greater than mmap_min_addr 1312 */ 1313 static inline unsigned long round_hint_to_min(unsigned long hint) 1314 { 1315 hint &= PAGE_MASK; 1316 if (((void *)hint != NULL) && 1317 (hint < mmap_min_addr)) 1318 return PAGE_ALIGN(mmap_min_addr); 1319 return hint; 1320 } 1321 1322 static inline int mlock_future_check(struct mm_struct *mm, 1323 unsigned long flags, 1324 unsigned long len) 1325 { 1326 unsigned long locked, lock_limit; 1327 1328 /* mlock MCL_FUTURE? */ 1329 if (flags & VM_LOCKED) { 1330 locked = len >> PAGE_SHIFT; 1331 locked += mm->locked_vm; 1332 lock_limit = rlimit(RLIMIT_MEMLOCK); 1333 lock_limit >>= PAGE_SHIFT; 1334 if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 1335 return -EAGAIN; 1336 } 1337 return 0; 1338 } 1339 1340 static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) 1341 { 1342 if (S_ISREG(inode->i_mode)) 1343 return MAX_LFS_FILESIZE; 1344 1345 if (S_ISBLK(inode->i_mode)) 1346 return MAX_LFS_FILESIZE; 1347 1348 if (S_ISSOCK(inode->i_mode)) 1349 return MAX_LFS_FILESIZE; 1350 1351 /* Special "we do even unsigned file positions" case */ 1352 if (file->f_mode & FMODE_UNSIGNED_OFFSET) 1353 return 0; 1354 1355 /* Yes, random drivers might want more. But I'm tired of buggy drivers */ 1356 return ULONG_MAX; 1357 } 1358 1359 static inline bool file_mmap_ok(struct file *file, struct inode *inode, 1360 unsigned long pgoff, unsigned long len) 1361 { 1362 u64 maxsize = file_mmap_size_max(file, inode); 1363 1364 if (maxsize && len > maxsize) 1365 return false; 1366 maxsize -= len; 1367 if (pgoff > maxsize >> PAGE_SHIFT) 1368 return false; 1369 return true; 1370 } 1371 1372 /* 1373 * The caller must hold down_write(¤t->mm->mmap_sem). 1374 */ 1375 unsigned long do_mmap(struct file *file, unsigned long addr, 1376 unsigned long len, unsigned long prot, 1377 unsigned long flags, vm_flags_t vm_flags, 1378 unsigned long pgoff, unsigned long *populate, 1379 struct list_head *uf) 1380 { 1381 struct mm_struct *mm = current->mm; 1382 int pkey = 0; 1383 1384 *populate = 0; 1385 1386 if (!len) 1387 return -EINVAL; 1388 1389 /* 1390 * Does the application expect PROT_READ to imply PROT_EXEC? 1391 * 1392 * (the exception is when the underlying filesystem is noexec 1393 * mounted, in which case we dont add PROT_EXEC.) 1394 */ 1395 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 1396 if (!(file && path_noexec(&file->f_path))) 1397 prot |= PROT_EXEC; 1398 1399 /* force arch specific MAP_FIXED handling in get_unmapped_area */ 1400 if (flags & MAP_FIXED_NOREPLACE) 1401 flags |= MAP_FIXED; 1402 1403 if (!(flags & MAP_FIXED)) 1404 addr = round_hint_to_min(addr); 1405 1406 /* Careful about overflows.. */ 1407 len = PAGE_ALIGN(len); 1408 if (!len) 1409 return -ENOMEM; 1410 1411 /* offset overflow? */ 1412 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 1413 return -EOVERFLOW; 1414 1415 /* Too many mappings? */ 1416 if (mm->map_count > sysctl_max_map_count) 1417 return -ENOMEM; 1418 1419 /* Obtain the address to map to. we verify (or select) it and ensure 1420 * that it represents a valid section of the address space. 1421 */ 1422 addr = get_unmapped_area(file, addr, len, pgoff, flags); 1423 if (IS_ERR_VALUE(addr)) 1424 return addr; 1425 1426 if (flags & MAP_FIXED_NOREPLACE) { 1427 struct vm_area_struct *vma = find_vma(mm, addr); 1428 1429 if (vma && vma->vm_start < addr + len) 1430 return -EEXIST; 1431 } 1432 1433 if (prot == PROT_EXEC) { 1434 pkey = execute_only_pkey(mm); 1435 if (pkey < 0) 1436 pkey = 0; 1437 } 1438 1439 /* Do simple checking here so the lower-level routines won't have 1440 * to. we assume access permissions have been handled by the open 1441 * of the memory object, so we don't do any here. 1442 */ 1443 vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) | 1444 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 1445 1446 if (flags & MAP_LOCKED) 1447 if (!can_do_mlock()) 1448 return -EPERM; 1449 1450 if (mlock_future_check(mm, vm_flags, len)) 1451 return -EAGAIN; 1452 1453 if (file) { 1454 struct inode *inode = file_inode(file); 1455 unsigned long flags_mask; 1456 1457 if (!file_mmap_ok(file, inode, pgoff, len)) 1458 return -EOVERFLOW; 1459 1460 flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags; 1461 1462 switch (flags & MAP_TYPE) { 1463 case MAP_SHARED: 1464 /* 1465 * Force use of MAP_SHARED_VALIDATE with non-legacy 1466 * flags. E.g. MAP_SYNC is dangerous to use with 1467 * MAP_SHARED as you don't know which consistency model 1468 * you will get. We silently ignore unsupported flags 1469 * with MAP_SHARED to preserve backward compatibility. 1470 */ 1471 flags &= LEGACY_MAP_MASK; 1472 /* fall through */ 1473 case MAP_SHARED_VALIDATE: 1474 if (flags & ~flags_mask) 1475 return -EOPNOTSUPP; 1476 if (prot & PROT_WRITE) { 1477 if (!(file->f_mode & FMODE_WRITE)) 1478 return -EACCES; 1479 if (IS_SWAPFILE(file->f_mapping->host)) 1480 return -ETXTBSY; 1481 } 1482 1483 /* 1484 * Make sure we don't allow writing to an append-only 1485 * file.. 1486 */ 1487 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) 1488 return -EACCES; 1489 1490 /* 1491 * Make sure there are no mandatory locks on the file. 1492 */ 1493 if (locks_verify_locked(file)) 1494 return -EAGAIN; 1495 1496 vm_flags |= VM_SHARED | VM_MAYSHARE; 1497 if (!(file->f_mode & FMODE_WRITE)) 1498 vm_flags &= ~(VM_MAYWRITE | VM_SHARED); 1499 1500 /* fall through */ 1501 case MAP_PRIVATE: 1502 if (!(file->f_mode & FMODE_READ)) 1503 return -EACCES; 1504 if (path_noexec(&file->f_path)) { 1505 if (vm_flags & VM_EXEC) 1506 return -EPERM; 1507 vm_flags &= ~VM_MAYEXEC; 1508 } 1509 1510 if (!file->f_op->mmap) 1511 return -ENODEV; 1512 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1513 return -EINVAL; 1514 break; 1515 1516 default: 1517 return -EINVAL; 1518 } 1519 } else { 1520 switch (flags & MAP_TYPE) { 1521 case MAP_SHARED: 1522 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1523 return -EINVAL; 1524 /* 1525 * Ignore pgoff. 1526 */ 1527 pgoff = 0; 1528 vm_flags |= VM_SHARED | VM_MAYSHARE; 1529 break; 1530 case MAP_PRIVATE: 1531 /* 1532 * Set pgoff according to addr for anon_vma. 1533 */ 1534 pgoff = addr >> PAGE_SHIFT; 1535 break; 1536 default: 1537 return -EINVAL; 1538 } 1539 } 1540 1541 /* 1542 * Set 'VM_NORESERVE' if we should not account for the 1543 * memory use of this mapping. 1544 */ 1545 if (flags & MAP_NORESERVE) { 1546 /* We honor MAP_NORESERVE if allowed to overcommit */ 1547 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) 1548 vm_flags |= VM_NORESERVE; 1549 1550 /* hugetlb applies strict overcommit unless MAP_NORESERVE */ 1551 if (file && is_file_hugepages(file)) 1552 vm_flags |= VM_NORESERVE; 1553 } 1554 1555 addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); 1556 if (!IS_ERR_VALUE(addr) && 1557 ((vm_flags & VM_LOCKED) || 1558 (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) 1559 *populate = len; 1560 return addr; 1561 } 1562 1563 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, 1564 unsigned long prot, unsigned long flags, 1565 unsigned long fd, unsigned long pgoff) 1566 { 1567 struct file *file = NULL; 1568 unsigned long retval; 1569 1570 addr = untagged_addr(addr); 1571 1572 if (!(flags & MAP_ANONYMOUS)) { 1573 audit_mmap_fd(fd, flags); 1574 file = fget(fd); 1575 if (!file) 1576 return -EBADF; 1577 if (is_file_hugepages(file)) 1578 len = ALIGN(len, huge_page_size(hstate_file(file))); 1579 retval = -EINVAL; 1580 if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file))) 1581 goto out_fput; 1582 } else if (flags & MAP_HUGETLB) { 1583 struct user_struct *user = NULL; 1584 struct hstate *hs; 1585 1586 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1587 if (!hs) 1588 return -EINVAL; 1589 1590 len = ALIGN(len, huge_page_size(hs)); 1591 /* 1592 * VM_NORESERVE is used because the reservations will be 1593 * taken when vm_ops->mmap() is called 1594 * A dummy user value is used because we are not locking 1595 * memory so no accounting is necessary 1596 */ 1597 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, 1598 VM_NORESERVE, 1599 &user, HUGETLB_ANONHUGE_INODE, 1600 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1601 if (IS_ERR(file)) 1602 return PTR_ERR(file); 1603 } 1604 1605 flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE); 1606 1607 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 1608 out_fput: 1609 if (file) 1610 fput(file); 1611 return retval; 1612 } 1613 1614 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 1615 unsigned long, prot, unsigned long, flags, 1616 unsigned long, fd, unsigned long, pgoff) 1617 { 1618 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); 1619 } 1620 1621 #ifdef __ARCH_WANT_SYS_OLD_MMAP 1622 struct mmap_arg_struct { 1623 unsigned long addr; 1624 unsigned long len; 1625 unsigned long prot; 1626 unsigned long flags; 1627 unsigned long fd; 1628 unsigned long offset; 1629 }; 1630 1631 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1632 { 1633 struct mmap_arg_struct a; 1634 1635 if (copy_from_user(&a, arg, sizeof(a))) 1636 return -EFAULT; 1637 if (offset_in_page(a.offset)) 1638 return -EINVAL; 1639 1640 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1641 a.offset >> PAGE_SHIFT); 1642 } 1643 #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1644 1645 /* 1646 * Some shared mappings will want the pages marked read-only 1647 * to track write events. If so, we'll downgrade vm_page_prot 1648 * to the private version (using protection_map[] without the 1649 * VM_SHARED bit). 1650 */ 1651 int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 1652 { 1653 vm_flags_t vm_flags = vma->vm_flags; 1654 const struct vm_operations_struct *vm_ops = vma->vm_ops; 1655 1656 /* If it was private or non-writable, the write bit is already clear */ 1657 if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED))) 1658 return 0; 1659 1660 /* The backer wishes to know when pages are first written to? */ 1661 if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite)) 1662 return 1; 1663 1664 /* The open routine did something to the protections that pgprot_modify 1665 * won't preserve? */ 1666 if (pgprot_val(vm_page_prot) != 1667 pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags))) 1668 return 0; 1669 1670 /* Do we need to track softdirty? */ 1671 if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY)) 1672 return 1; 1673 1674 /* Specialty mapping? */ 1675 if (vm_flags & VM_PFNMAP) 1676 return 0; 1677 1678 /* Can the mapping track the dirty pages? */ 1679 return vma->vm_file && vma->vm_file->f_mapping && 1680 mapping_cap_account_dirty(vma->vm_file->f_mapping); 1681 } 1682 1683 /* 1684 * We account for memory if it's a private writeable mapping, 1685 * not hugepages and VM_NORESERVE wasn't set. 1686 */ 1687 static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) 1688 { 1689 /* 1690 * hugetlb has its own accounting separate from the core VM 1691 * VM_HUGETLB may not be set yet so we cannot check for that flag. 1692 */ 1693 if (file && is_file_hugepages(file)) 1694 return 0; 1695 1696 return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; 1697 } 1698 1699 unsigned long mmap_region(struct file *file, unsigned long addr, 1700 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, 1701 struct list_head *uf) 1702 { 1703 struct mm_struct *mm = current->mm; 1704 struct vm_area_struct *vma, *prev; 1705 int error; 1706 struct rb_node **rb_link, *rb_parent; 1707 unsigned long charged = 0; 1708 1709 /* Check against address space limit. */ 1710 if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { 1711 unsigned long nr_pages; 1712 1713 /* 1714 * MAP_FIXED may remove pages of mappings that intersects with 1715 * requested mapping. Account for the pages it would unmap. 1716 */ 1717 nr_pages = count_vma_pages_range(mm, addr, addr + len); 1718 1719 if (!may_expand_vm(mm, vm_flags, 1720 (len >> PAGE_SHIFT) - nr_pages)) 1721 return -ENOMEM; 1722 } 1723 1724 /* Clear old maps */ 1725 while (find_vma_links(mm, addr, addr + len, &prev, &rb_link, 1726 &rb_parent)) { 1727 if (do_munmap(mm, addr, len, uf)) 1728 return -ENOMEM; 1729 } 1730 1731 /* 1732 * Private writable mapping: check memory availability 1733 */ 1734 if (accountable_mapping(file, vm_flags)) { 1735 charged = len >> PAGE_SHIFT; 1736 if (security_vm_enough_memory_mm(mm, charged)) 1737 return -ENOMEM; 1738 vm_flags |= VM_ACCOUNT; 1739 } 1740 1741 /* 1742 * Can we just expand an old mapping? 1743 */ 1744 vma = vma_merge(mm, prev, addr, addr + len, vm_flags, 1745 NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX); 1746 if (vma) 1747 goto out; 1748 1749 /* 1750 * Determine the object being mapped and call the appropriate 1751 * specific mapper. the address has already been validated, but 1752 * not unmapped, but the maps are removed from the list. 1753 */ 1754 vma = vm_area_alloc(mm); 1755 if (!vma) { 1756 error = -ENOMEM; 1757 goto unacct_error; 1758 } 1759 1760 vma->vm_start = addr; 1761 vma->vm_end = addr + len; 1762 vma->vm_flags = vm_flags; 1763 vma->vm_page_prot = vm_get_page_prot(vm_flags); 1764 vma->vm_pgoff = pgoff; 1765 1766 if (file) { 1767 if (vm_flags & VM_DENYWRITE) { 1768 error = deny_write_access(file); 1769 if (error) 1770 goto free_vma; 1771 } 1772 if (vm_flags & VM_SHARED) { 1773 error = mapping_map_writable(file->f_mapping); 1774 if (error) 1775 goto allow_write_and_free_vma; 1776 } 1777 1778 /* ->mmap() can change vma->vm_file, but must guarantee that 1779 * vma_link() below can deny write-access if VM_DENYWRITE is set 1780 * and map writably if VM_SHARED is set. This usually means the 1781 * new file must not have been exposed to user-space, yet. 1782 */ 1783 vma->vm_file = get_file(file); 1784 error = call_mmap(file, vma); 1785 if (error) 1786 goto unmap_and_free_vma; 1787 1788 /* Can addr have changed?? 1789 * 1790 * Answer: Yes, several device drivers can do it in their 1791 * f_op->mmap method. -DaveM 1792 * Bug: If addr is changed, prev, rb_link, rb_parent should 1793 * be updated for vma_link() 1794 */ 1795 WARN_ON_ONCE(addr != vma->vm_start); 1796 1797 addr = vma->vm_start; 1798 vm_flags = vma->vm_flags; 1799 } else if (vm_flags & VM_SHARED) { 1800 error = shmem_zero_setup(vma); 1801 if (error) 1802 goto free_vma; 1803 } else { 1804 vma_set_anonymous(vma); 1805 } 1806 1807 vma_link(mm, vma, prev, rb_link, rb_parent); 1808 /* Once vma denies write, undo our temporary denial count */ 1809 if (file) { 1810 if (vm_flags & VM_SHARED) 1811 mapping_unmap_writable(file->f_mapping); 1812 if (vm_flags & VM_DENYWRITE) 1813 allow_write_access(file); 1814 } 1815 file = vma->vm_file; 1816 out: 1817 perf_event_mmap(vma); 1818 1819 vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); 1820 if (vm_flags & VM_LOCKED) { 1821 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || 1822 is_vm_hugetlb_page(vma) || 1823 vma == get_gate_vma(current->mm)) 1824 vma->vm_flags &= VM_LOCKED_CLEAR_MASK; 1825 else 1826 mm->locked_vm += (len >> PAGE_SHIFT); 1827 } 1828 1829 if (file) 1830 uprobe_mmap(vma); 1831 1832 /* 1833 * New (or expanded) vma always get soft dirty status. 1834 * Otherwise user-space soft-dirty page tracker won't 1835 * be able to distinguish situation when vma area unmapped, 1836 * then new mapped in-place (which must be aimed as 1837 * a completely new data area). 1838 */ 1839 vma->vm_flags |= VM_SOFTDIRTY; 1840 1841 vma_set_page_prot(vma); 1842 1843 return addr; 1844 1845 unmap_and_free_vma: 1846 vma->vm_file = NULL; 1847 fput(file); 1848 1849 /* Undo any partial mapping done by a device driver. */ 1850 unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); 1851 charged = 0; 1852 if (vm_flags & VM_SHARED) 1853 mapping_unmap_writable(file->f_mapping); 1854 allow_write_and_free_vma: 1855 if (vm_flags & VM_DENYWRITE) 1856 allow_write_access(file); 1857 free_vma: 1858 vm_area_free(vma); 1859 unacct_error: 1860 if (charged) 1861 vm_unacct_memory(charged); 1862 return error; 1863 } 1864 1865 unsigned long unmapped_area(struct vm_unmapped_area_info *info) 1866 { 1867 /* 1868 * We implement the search by looking for an rbtree node that 1869 * immediately follows a suitable gap. That is, 1870 * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length; 1871 * - gap_end = vma->vm_start >= info->low_limit + length; 1872 * - gap_end - gap_start >= length 1873 */ 1874 1875 struct mm_struct *mm = current->mm; 1876 struct vm_area_struct *vma; 1877 unsigned long length, low_limit, high_limit, gap_start, gap_end; 1878 1879 /* Adjust search length to account for worst case alignment overhead */ 1880 length = info->length + info->align_mask; 1881 if (length < info->length) 1882 return -ENOMEM; 1883 1884 /* Adjust search limits by the desired length */ 1885 if (info->high_limit < length) 1886 return -ENOMEM; 1887 high_limit = info->high_limit - length; 1888 1889 if (info->low_limit > high_limit) 1890 return -ENOMEM; 1891 low_limit = info->low_limit + length; 1892 1893 /* Check if rbtree root looks promising */ 1894 if (RB_EMPTY_ROOT(&mm->mm_rb)) 1895 goto check_highest; 1896 vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); 1897 if (vma->rb_subtree_gap < length) 1898 goto check_highest; 1899 1900 while (true) { 1901 /* Visit left subtree if it looks promising */ 1902 gap_end = vm_start_gap(vma); 1903 if (gap_end >= low_limit && vma->vm_rb.rb_left) { 1904 struct vm_area_struct *left = 1905 rb_entry(vma->vm_rb.rb_left, 1906 struct vm_area_struct, vm_rb); 1907 if (left->rb_subtree_gap >= length) { 1908 vma = left; 1909 continue; 1910 } 1911 } 1912 1913 gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; 1914 check_current: 1915 /* Check if current node has a suitable gap */ 1916 if (gap_start > high_limit) 1917 return -ENOMEM; 1918 if (gap_end >= low_limit && 1919 gap_end > gap_start && gap_end - gap_start >= length) 1920 goto found; 1921 1922 /* Visit right subtree if it looks promising */ 1923 if (vma->vm_rb.rb_right) { 1924 struct vm_area_struct *right = 1925 rb_entry(vma->vm_rb.rb_right, 1926 struct vm_area_struct, vm_rb); 1927 if (right->rb_subtree_gap >= length) { 1928 vma = right; 1929 continue; 1930 } 1931 } 1932 1933 /* Go back up the rbtree to find next candidate node */ 1934 while (true) { 1935 struct rb_node *prev = &vma->vm_rb; 1936 if (!rb_parent(prev)) 1937 goto check_highest; 1938 vma = rb_entry(rb_parent(prev), 1939 struct vm_area_struct, vm_rb); 1940 if (prev == vma->vm_rb.rb_left) { 1941 gap_start = vm_end_gap(vma->vm_prev); 1942 gap_end = vm_start_gap(vma); 1943 goto check_current; 1944 } 1945 } 1946 } 1947 1948 check_highest: 1949 /* Check highest gap, which does not precede any rbtree node */ 1950 gap_start = mm->highest_vm_end; 1951 gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */ 1952 if (gap_start > high_limit) 1953 return -ENOMEM; 1954 1955 found: 1956 /* We found a suitable gap. Clip it with the original low_limit. */ 1957 if (gap_start < info->low_limit) 1958 gap_start = info->low_limit; 1959 1960 /* Adjust gap address to the desired alignment */ 1961 gap_start += (info->align_offset - gap_start) & info->align_mask; 1962 1963 VM_BUG_ON(gap_start + info->length > info->high_limit); 1964 VM_BUG_ON(gap_start + info->length > gap_end); 1965 return gap_start; 1966 } 1967 1968 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) 1969 { 1970 struct mm_struct *mm = current->mm; 1971 struct vm_area_struct *vma; 1972 unsigned long length, low_limit, high_limit, gap_start, gap_end; 1973 1974 /* Adjust search length to account for worst case alignment overhead */ 1975 length = info->length + info->align_mask; 1976 if (length < info->length) 1977 return -ENOMEM; 1978 1979 /* 1980 * Adjust search limits by the desired length. 1981 * See implementation comment at top of unmapped_area(). 1982 */ 1983 gap_end = info->high_limit; 1984 if (gap_end < length) 1985 return -ENOMEM; 1986 high_limit = gap_end - length; 1987 1988 if (info->low_limit > high_limit) 1989 return -ENOMEM; 1990 low_limit = info->low_limit + length; 1991 1992 /* Check highest gap, which does not precede any rbtree node */ 1993 gap_start = mm->highest_vm_end; 1994 if (gap_start <= high_limit) 1995 goto found_highest; 1996 1997 /* Check if rbtree root looks promising */ 1998 if (RB_EMPTY_ROOT(&mm->mm_rb)) 1999 return -ENOMEM; 2000 vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); 2001 if (vma->rb_subtree_gap < length) 2002 return -ENOMEM; 2003 2004 while (true) { 2005 /* Visit right subtree if it looks promising */ 2006 gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; 2007 if (gap_start <= high_limit && vma->vm_rb.rb_right) { 2008 struct vm_area_struct *right = 2009 rb_entry(vma->vm_rb.rb_right, 2010 struct vm_area_struct, vm_rb); 2011 if (right->rb_subtree_gap >= length) { 2012 vma = right; 2013 continue; 2014 } 2015 } 2016 2017 check_current: 2018 /* Check if current node has a suitable gap */ 2019 gap_end = vm_start_gap(vma); 2020 if (gap_end < low_limit) 2021 return -ENOMEM; 2022 if (gap_start <= high_limit && 2023 gap_end > gap_start && gap_end - gap_start >= length) 2024 goto found; 2025 2026 /* Visit left subtree if it looks promising */ 2027 if (vma->vm_rb.rb_left) { 2028 struct vm_area_struct *left = 2029 rb_entry(vma->vm_rb.rb_left, 2030 struct vm_area_struct, vm_rb); 2031 if (left->rb_subtree_gap >= length) { 2032 vma = left; 2033 continue; 2034 } 2035 } 2036 2037 /* Go back up the rbtree to find next candidate node */ 2038 while (true) { 2039 struct rb_node *prev = &vma->vm_rb; 2040 if (!rb_parent(prev)) 2041 return -ENOMEM; 2042 vma = rb_entry(rb_parent(prev), 2043 struct vm_area_struct, vm_rb); 2044 if (prev == vma->vm_rb.rb_right) { 2045 gap_start = vma->vm_prev ? 2046 vm_end_gap(vma->vm_prev) : 0; 2047 goto check_current; 2048 } 2049 } 2050 } 2051 2052 found: 2053 /* We found a suitable gap. Clip it with the original high_limit. */ 2054 if (gap_end > info->high_limit) 2055 gap_end = info->high_limit; 2056 2057 found_highest: 2058 /* Compute highest gap address at the desired alignment */ 2059 gap_end -= info->length; 2060 gap_end -= (gap_end - info->align_offset) & info->align_mask; 2061 2062 VM_BUG_ON(gap_end < info->low_limit); 2063 VM_BUG_ON(gap_end < gap_start); 2064 return gap_end; 2065 } 2066 2067 2068 #ifndef arch_get_mmap_end 2069 #define arch_get_mmap_end(addr) (TASK_SIZE) 2070 #endif 2071 2072 #ifndef arch_get_mmap_base 2073 #define arch_get_mmap_base(addr, base) (base) 2074 #endif 2075 2076 /* Get an address range which is currently unmapped. 2077 * For shmat() with addr=0. 2078 * 2079 * Ugly calling convention alert: 2080 * Return value with the low bits set means error value, 2081 * ie 2082 * if (ret & ~PAGE_MASK) 2083 * error = ret; 2084 * 2085 * This function "knows" that -ENOMEM has the bits set. 2086 */ 2087 #ifndef HAVE_ARCH_UNMAPPED_AREA 2088 unsigned long 2089 arch_get_unmapped_area(struct file *filp, unsigned long addr, 2090 unsigned long len, unsigned long pgoff, unsigned long flags) 2091 { 2092 struct mm_struct *mm = current->mm; 2093 struct vm_area_struct *vma, *prev; 2094 struct vm_unmapped_area_info info; 2095 const unsigned long mmap_end = arch_get_mmap_end(addr); 2096 2097 if (len > mmap_end - mmap_min_addr) 2098 return -ENOMEM; 2099 2100 if (flags & MAP_FIXED) 2101 return addr; 2102 2103 if (addr) { 2104 addr = PAGE_ALIGN(addr); 2105 vma = find_vma_prev(mm, addr, &prev); 2106 if (mmap_end - len >= addr && addr >= mmap_min_addr && 2107 (!vma || addr + len <= vm_start_gap(vma)) && 2108 (!prev || addr >= vm_end_gap(prev))) 2109 return addr; 2110 } 2111 2112 info.flags = 0; 2113 info.length = len; 2114 info.low_limit = mm->mmap_base; 2115 info.high_limit = mmap_end; 2116 info.align_mask = 0; 2117 return vm_unmapped_area(&info); 2118 } 2119 #endif 2120 2121 /* 2122 * This mmap-allocator allocates new areas top-down from below the 2123 * stack's low limit (the base): 2124 */ 2125 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 2126 unsigned long 2127 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 2128 unsigned long len, unsigned long pgoff, 2129 unsigned long flags) 2130 { 2131 struct vm_area_struct *vma, *prev; 2132 struct mm_struct *mm = current->mm; 2133 struct vm_unmapped_area_info info; 2134 const unsigned long mmap_end = arch_get_mmap_end(addr); 2135 2136 /* requested length too big for entire address space */ 2137 if (len > mmap_end - mmap_min_addr) 2138 return -ENOMEM; 2139 2140 if (flags & MAP_FIXED) 2141 return addr; 2142 2143 /* requesting a specific address */ 2144 if (addr) { 2145 addr = PAGE_ALIGN(addr); 2146 vma = find_vma_prev(mm, addr, &prev); 2147 if (mmap_end - len >= addr && addr >= mmap_min_addr && 2148 (!vma || addr + len <= vm_start_gap(vma)) && 2149 (!prev || addr >= vm_end_gap(prev))) 2150 return addr; 2151 } 2152 2153 info.flags = VM_UNMAPPED_AREA_TOPDOWN; 2154 info.length = len; 2155 info.low_limit = max(PAGE_SIZE, mmap_min_addr); 2156 info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); 2157 info.align_mask = 0; 2158 addr = vm_unmapped_area(&info); 2159 2160 /* 2161 * A failed mmap() very likely causes application failure, 2162 * so fall back to the bottom-up function here. This scenario 2163 * can happen with large stack limits and large mmap() 2164 * allocations. 2165 */ 2166 if (offset_in_page(addr)) { 2167 VM_BUG_ON(addr != -ENOMEM); 2168 info.flags = 0; 2169 info.low_limit = TASK_UNMAPPED_BASE; 2170 info.high_limit = mmap_end; 2171 addr = vm_unmapped_area(&info); 2172 } 2173 2174 return addr; 2175 } 2176 #endif 2177 2178 unsigned long 2179 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 2180 unsigned long pgoff, unsigned long flags) 2181 { 2182 unsigned long (*get_area)(struct file *, unsigned long, 2183 unsigned long, unsigned long, unsigned long); 2184 2185 unsigned long error = arch_mmap_check(addr, len, flags); 2186 if (error) 2187 return error; 2188 2189 /* Careful about overflows.. */ 2190 if (len > TASK_SIZE) 2191 return -ENOMEM; 2192 2193 get_area = current->mm->get_unmapped_area; 2194 if (file) { 2195 if (file->f_op->get_unmapped_area) 2196 get_area = file->f_op->get_unmapped_area; 2197 } else if (flags & MAP_SHARED) { 2198 /* 2199 * mmap_region() will call shmem_zero_setup() to create a file, 2200 * so use shmem's get_unmapped_area in case it can be huge. 2201 * do_mmap_pgoff() will clear pgoff, so match alignment. 2202 */ 2203 pgoff = 0; 2204 get_area = shmem_get_unmapped_area; 2205 } 2206 2207 addr = get_area(file, addr, len, pgoff, flags); 2208 if (IS_ERR_VALUE(addr)) 2209 return addr; 2210 2211 if (addr > TASK_SIZE - len) 2212 return -ENOMEM; 2213 if (offset_in_page(addr)) 2214 return -EINVAL; 2215 2216 error = security_mmap_addr(addr); 2217 return error ? error : addr; 2218 } 2219 2220 EXPORT_SYMBOL(get_unmapped_area); 2221 2222 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 2223 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 2224 { 2225 struct rb_node *rb_node; 2226 struct vm_area_struct *vma; 2227 2228 /* Check the cache first. */ 2229 vma = vmacache_find(mm, addr); 2230 if (likely(vma)) 2231 return vma; 2232 2233 rb_node = mm->mm_rb.rb_node; 2234 2235 while (rb_node) { 2236 struct vm_area_struct *tmp; 2237 2238 tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb); 2239 2240 if (tmp->vm_end > addr) { 2241 vma = tmp; 2242 if (tmp->vm_start <= addr) 2243 break; 2244 rb_node = rb_node->rb_left; 2245 } else 2246 rb_node = rb_node->rb_right; 2247 } 2248 2249 if (vma) 2250 vmacache_update(addr, vma); 2251 return vma; 2252 } 2253 2254 EXPORT_SYMBOL(find_vma); 2255 2256 /* 2257 * Same as find_vma, but also return a pointer to the previous VMA in *pprev. 2258 */ 2259 struct vm_area_struct * 2260 find_vma_prev(struct mm_struct *mm, unsigned long addr, 2261 struct vm_area_struct **pprev) 2262 { 2263 struct vm_area_struct *vma; 2264 2265 vma = find_vma(mm, addr); 2266 if (vma) { 2267 *pprev = vma->vm_prev; 2268 } else { 2269 struct rb_node *rb_node = rb_last(&mm->mm_rb); 2270 2271 *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL; 2272 } 2273 return vma; 2274 } 2275 2276 /* 2277 * Verify that the stack growth is acceptable and 2278 * update accounting. This is shared with both the 2279 * grow-up and grow-down cases. 2280 */ 2281 static int acct_stack_growth(struct vm_area_struct *vma, 2282 unsigned long size, unsigned long grow) 2283 { 2284 struct mm_struct *mm = vma->vm_mm; 2285 unsigned long new_start; 2286 2287 /* address space limit tests */ 2288 if (!may_expand_vm(mm, vma->vm_flags, grow)) 2289 return -ENOMEM; 2290 2291 /* Stack limit test */ 2292 if (size > rlimit(RLIMIT_STACK)) 2293 return -ENOMEM; 2294 2295 /* mlock limit tests */ 2296 if (vma->vm_flags & VM_LOCKED) { 2297 unsigned long locked; 2298 unsigned long limit; 2299 locked = mm->locked_vm + grow; 2300 limit = rlimit(RLIMIT_MEMLOCK); 2301 limit >>= PAGE_SHIFT; 2302 if (locked > limit && !capable(CAP_IPC_LOCK)) 2303 return -ENOMEM; 2304 } 2305 2306 /* Check to ensure the stack will not grow into a hugetlb-only region */ 2307 new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start : 2308 vma->vm_end - size; 2309 if (is_hugepage_only_range(vma->vm_mm, new_start, size)) 2310 return -EFAULT; 2311 2312 /* 2313 * Overcommit.. This must be the final test, as it will 2314 * update security statistics. 2315 */ 2316 if (security_vm_enough_memory_mm(mm, grow)) 2317 return -ENOMEM; 2318 2319 return 0; 2320 } 2321 2322 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64) 2323 /* 2324 * PA-RISC uses this for its stack; IA64 for its Register Backing Store. 2325 * vma is the last one with address > vma->vm_end. Have to extend vma. 2326 */ 2327 int expand_upwards(struct vm_area_struct *vma, unsigned long address) 2328 { 2329 struct mm_struct *mm = vma->vm_mm; 2330 struct vm_area_struct *next; 2331 unsigned long gap_addr; 2332 int error = 0; 2333 2334 if (!(vma->vm_flags & VM_GROWSUP)) 2335 return -EFAULT; 2336 2337 /* Guard against exceeding limits of the address space. */ 2338 address &= PAGE_MASK; 2339 if (address >= (TASK_SIZE & PAGE_MASK)) 2340 return -ENOMEM; 2341 address += PAGE_SIZE; 2342 2343 /* Enforce stack_guard_gap */ 2344 gap_addr = address + stack_guard_gap; 2345 2346 /* Guard against overflow */ 2347 if (gap_addr < address || gap_addr > TASK_SIZE) 2348 gap_addr = TASK_SIZE; 2349 2350 next = vma->vm_next; 2351 if (next && next->vm_start < gap_addr && 2352 (next->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) { 2353 if (!(next->vm_flags & VM_GROWSUP)) 2354 return -ENOMEM; 2355 /* Check that both stack segments have the same anon_vma? */ 2356 } 2357 2358 /* We must make sure the anon_vma is allocated. */ 2359 if (unlikely(anon_vma_prepare(vma))) 2360 return -ENOMEM; 2361 2362 /* 2363 * vma->vm_start/vm_end cannot change under us because the caller 2364 * is required to hold the mmap_sem in read mode. We need the 2365 * anon_vma lock to serialize against concurrent expand_stacks. 2366 */ 2367 anon_vma_lock_write(vma->anon_vma); 2368 2369 /* Somebody else might have raced and expanded it already */ 2370 if (address > vma->vm_end) { 2371 unsigned long size, grow; 2372 2373 size = address - vma->vm_start; 2374 grow = (address - vma->vm_end) >> PAGE_SHIFT; 2375 2376 error = -ENOMEM; 2377 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { 2378 error = acct_stack_growth(vma, size, grow); 2379 if (!error) { 2380 /* 2381 * vma_gap_update() doesn't support concurrent 2382 * updates, but we only hold a shared mmap_sem 2383 * lock here, so we need to protect against 2384 * concurrent vma expansions. 2385 * anon_vma_lock_write() doesn't help here, as 2386 * we don't guarantee that all growable vmas 2387 * in a mm share the same root anon vma. 2388 * So, we reuse mm->page_table_lock to guard 2389 * against concurrent vma expansions. 2390 */ 2391 spin_lock(&mm->page_table_lock); 2392 if (vma->vm_flags & VM_LOCKED) 2393 mm->locked_vm += grow; 2394 vm_stat_account(mm, vma->vm_flags, grow); 2395 anon_vma_interval_tree_pre_update_vma(vma); 2396 vma->vm_end = address; 2397 anon_vma_interval_tree_post_update_vma(vma); 2398 if (vma->vm_next) 2399 vma_gap_update(vma->vm_next); 2400 else 2401 mm->highest_vm_end = vm_end_gap(vma); 2402 spin_unlock(&mm->page_table_lock); 2403 2404 perf_event_mmap(vma); 2405 } 2406 } 2407 } 2408 anon_vma_unlock_write(vma->anon_vma); 2409 khugepaged_enter_vma_merge(vma, vma->vm_flags); 2410 validate_mm(mm); 2411 return error; 2412 } 2413 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ 2414 2415 /* 2416 * vma is the first one with address < vma->vm_start. Have to extend vma. 2417 */ 2418 int expand_downwards(struct vm_area_struct *vma, 2419 unsigned long address) 2420 { 2421 struct mm_struct *mm = vma->vm_mm; 2422 struct vm_area_struct *prev; 2423 int error = 0; 2424 2425 address &= PAGE_MASK; 2426 if (address < mmap_min_addr) 2427 return -EPERM; 2428 2429 /* Enforce stack_guard_gap */ 2430 prev = vma->vm_prev; 2431 /* Check that both stack segments have the same anon_vma? */ 2432 if (prev && !(prev->vm_flags & VM_GROWSDOWN) && 2433 (prev->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) { 2434 if (address - prev->vm_end < stack_guard_gap) 2435 return -ENOMEM; 2436 } 2437 2438 /* We must make sure the anon_vma is allocated. */ 2439 if (unlikely(anon_vma_prepare(vma))) 2440 return -ENOMEM; 2441 2442 /* 2443 * vma->vm_start/vm_end cannot change under us because the caller 2444 * is required to hold the mmap_sem in read mode. We need the 2445 * anon_vma lock to serialize against concurrent expand_stacks. 2446 */ 2447 anon_vma_lock_write(vma->anon_vma); 2448 2449 /* Somebody else might have raced and expanded it already */ 2450 if (address < vma->vm_start) { 2451 unsigned long size, grow; 2452 2453 size = vma->vm_end - address; 2454 grow = (vma->vm_start - address) >> PAGE_SHIFT; 2455 2456 error = -ENOMEM; 2457 if (grow <= vma->vm_pgoff) { 2458 error = acct_stack_growth(vma, size, grow); 2459 if (!error) { 2460 /* 2461 * vma_gap_update() doesn't support concurrent 2462 * updates, but we only hold a shared mmap_sem 2463 * lock here, so we need to protect against 2464 * concurrent vma expansions. 2465 * anon_vma_lock_write() doesn't help here, as 2466 * we don't guarantee that all growable vmas 2467 * in a mm share the same root anon vma. 2468 * So, we reuse mm->page_table_lock to guard 2469 * against concurrent vma expansions. 2470 */ 2471 spin_lock(&mm->page_table_lock); 2472 if (vma->vm_flags & VM_LOCKED) 2473 mm->locked_vm += grow; 2474 vm_stat_account(mm, vma->vm_flags, grow); 2475 anon_vma_interval_tree_pre_update_vma(vma); 2476 vma->vm_start = address; 2477 vma->vm_pgoff -= grow; 2478 anon_vma_interval_tree_post_update_vma(vma); 2479 vma_gap_update(vma); 2480 spin_unlock(&mm->page_table_lock); 2481 2482 perf_event_mmap(vma); 2483 } 2484 } 2485 } 2486 anon_vma_unlock_write(vma->anon_vma); 2487 khugepaged_enter_vma_merge(vma, vma->vm_flags); 2488 validate_mm(mm); 2489 return error; 2490 } 2491 2492 /* enforced gap between the expanding stack and other mappings. */ 2493 unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT; 2494 2495 static int __init cmdline_parse_stack_guard_gap(char *p) 2496 { 2497 unsigned long val; 2498 char *endptr; 2499 2500 val = simple_strtoul(p, &endptr, 10); 2501 if (!*endptr) 2502 stack_guard_gap = val << PAGE_SHIFT; 2503 2504 return 0; 2505 } 2506 __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap); 2507 2508 #ifdef CONFIG_STACK_GROWSUP 2509 int expand_stack(struct vm_area_struct *vma, unsigned long address) 2510 { 2511 return expand_upwards(vma, address); 2512 } 2513 2514 struct vm_area_struct * 2515 find_extend_vma(struct mm_struct *mm, unsigned long addr) 2516 { 2517 struct vm_area_struct *vma, *prev; 2518 2519 addr &= PAGE_MASK; 2520 vma = find_vma_prev(mm, addr, &prev); 2521 if (vma && (vma->vm_start <= addr)) 2522 return vma; 2523 /* don't alter vm_end if the coredump is running */ 2524 if (!prev || !mmget_still_valid(mm) || expand_stack(prev, addr)) 2525 return NULL; 2526 if (prev->vm_flags & VM_LOCKED) 2527 populate_vma_page_range(prev, addr, prev->vm_end, NULL); 2528 return prev; 2529 } 2530 #else 2531 int expand_stack(struct vm_area_struct *vma, unsigned long address) 2532 { 2533 return expand_downwards(vma, address); 2534 } 2535 2536 struct vm_area_struct * 2537 find_extend_vma(struct mm_struct *mm, unsigned long addr) 2538 { 2539 struct vm_area_struct *vma; 2540 unsigned long start; 2541 2542 addr &= PAGE_MASK; 2543 vma = find_vma(mm, addr); 2544 if (!vma) 2545 return NULL; 2546 if (vma->vm_start <= addr) 2547 return vma; 2548 if (!(vma->vm_flags & VM_GROWSDOWN)) 2549 return NULL; 2550 /* don't alter vm_start if the coredump is running */ 2551 if (!mmget_still_valid(mm)) 2552 return NULL; 2553 start = vma->vm_start; 2554 if (expand_stack(vma, addr)) 2555 return NULL; 2556 if (vma->vm_flags & VM_LOCKED) 2557 populate_vma_page_range(vma, addr, start, NULL); 2558 return vma; 2559 } 2560 #endif 2561 2562 EXPORT_SYMBOL_GPL(find_extend_vma); 2563 2564 /* 2565 * Ok - we have the memory areas we should free on the vma list, 2566 * so release them, and do the vma updates. 2567 * 2568 * Called with the mm semaphore held. 2569 */ 2570 static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) 2571 { 2572 unsigned long nr_accounted = 0; 2573 2574 /* Update high watermark before we lower total_vm */ 2575 update_hiwater_vm(mm); 2576 do { 2577 long nrpages = vma_pages(vma); 2578 2579 if (vma->vm_flags & VM_ACCOUNT) 2580 nr_accounted += nrpages; 2581 vm_stat_account(mm, vma->vm_flags, -nrpages); 2582 vma = remove_vma(vma); 2583 } while (vma); 2584 vm_unacct_memory(nr_accounted); 2585 validate_mm(mm); 2586 } 2587 2588 /* 2589 * Get rid of page table information in the indicated region. 2590 * 2591 * Called with the mm semaphore held. 2592 */ 2593 static void unmap_region(struct mm_struct *mm, 2594 struct vm_area_struct *vma, struct vm_area_struct *prev, 2595 unsigned long start, unsigned long end) 2596 { 2597 struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap; 2598 struct mmu_gather tlb; 2599 2600 lru_add_drain(); 2601 tlb_gather_mmu(&tlb, mm, start, end); 2602 update_hiwater_rss(mm); 2603 unmap_vmas(&tlb, vma, start, end); 2604 free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 2605 next ? next->vm_start : USER_PGTABLES_CEILING); 2606 tlb_finish_mmu(&tlb, start, end); 2607 } 2608 2609 /* 2610 * Create a list of vma's touched by the unmap, removing them from the mm's 2611 * vma list as we go.. 2612 */ 2613 static void 2614 detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, 2615 struct vm_area_struct *prev, unsigned long end) 2616 { 2617 struct vm_area_struct **insertion_point; 2618 struct vm_area_struct *tail_vma = NULL; 2619 2620 insertion_point = (prev ? &prev->vm_next : &mm->mmap); 2621 vma->vm_prev = NULL; 2622 do { 2623 vma_rb_erase(vma, &mm->mm_rb); 2624 mm->map_count--; 2625 tail_vma = vma; 2626 vma = vma->vm_next; 2627 } while (vma && vma->vm_start < end); 2628 *insertion_point = vma; 2629 if (vma) { 2630 vma->vm_prev = prev; 2631 vma_gap_update(vma); 2632 } else 2633 mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; 2634 tail_vma->vm_next = NULL; 2635 2636 /* Kill the cache */ 2637 vmacache_invalidate(mm); 2638 } 2639 2640 /* 2641 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 2642 * has already been checked or doesn't make sense to fail. 2643 */ 2644 int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 2645 unsigned long addr, int new_below) 2646 { 2647 struct vm_area_struct *new; 2648 int err; 2649 2650 if (vma->vm_ops && vma->vm_ops->split) { 2651 err = vma->vm_ops->split(vma, addr); 2652 if (err) 2653 return err; 2654 } 2655 2656 new = vm_area_dup(vma); 2657 if (!new) 2658 return -ENOMEM; 2659 2660 if (new_below) 2661 new->vm_end = addr; 2662 else { 2663 new->vm_start = addr; 2664 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 2665 } 2666 2667 err = vma_dup_policy(vma, new); 2668 if (err) 2669 goto out_free_vma; 2670 2671 err = anon_vma_clone(new, vma); 2672 if (err) 2673 goto out_free_mpol; 2674 2675 if (new->vm_file) 2676 get_file(new->vm_file); 2677 2678 if (new->vm_ops && new->vm_ops->open) 2679 new->vm_ops->open(new); 2680 2681 if (new_below) 2682 err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff + 2683 ((addr - new->vm_start) >> PAGE_SHIFT), new); 2684 else 2685 err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); 2686 2687 /* Success. */ 2688 if (!err) 2689 return 0; 2690 2691 /* Clean everything up if vma_adjust failed. */ 2692 if (new->vm_ops && new->vm_ops->close) 2693 new->vm_ops->close(new); 2694 if (new->vm_file) 2695 fput(new->vm_file); 2696 unlink_anon_vmas(new); 2697 out_free_mpol: 2698 mpol_put(vma_policy(new)); 2699 out_free_vma: 2700 vm_area_free(new); 2701 return err; 2702 } 2703 2704 /* 2705 * Split a vma into two pieces at address 'addr', a new vma is allocated 2706 * either for the first part or the tail. 2707 */ 2708 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 2709 unsigned long addr, int new_below) 2710 { 2711 if (mm->map_count >= sysctl_max_map_count) 2712 return -ENOMEM; 2713 2714 return __split_vma(mm, vma, addr, new_below); 2715 } 2716 2717 /* Munmap is split into 2 main parts -- this part which finds 2718 * what needs doing, and the areas themselves, which do the 2719 * work. This now handles partial unmappings. 2720 * Jeremy Fitzhardinge <jeremy@goop.org> 2721 */ 2722 int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 2723 struct list_head *uf, bool downgrade) 2724 { 2725 unsigned long end; 2726 struct vm_area_struct *vma, *prev, *last; 2727 2728 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 2729 return -EINVAL; 2730 2731 len = PAGE_ALIGN(len); 2732 end = start + len; 2733 if (len == 0) 2734 return -EINVAL; 2735 2736 /* 2737 * arch_unmap() might do unmaps itself. It must be called 2738 * and finish any rbtree manipulation before this code 2739 * runs and also starts to manipulate the rbtree. 2740 */ 2741 arch_unmap(mm, start, end); 2742 2743 /* Find the first overlapping VMA */ 2744 vma = find_vma(mm, start); 2745 if (!vma) 2746 return 0; 2747 prev = vma->vm_prev; 2748 /* we have start < vma->vm_end */ 2749 2750 /* if it doesn't overlap, we have nothing.. */ 2751 if (vma->vm_start >= end) 2752 return 0; 2753 2754 /* 2755 * If we need to split any vma, do it now to save pain later. 2756 * 2757 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 2758 * unmapped vm_area_struct will remain in use: so lower split_vma 2759 * places tmp vma above, and higher split_vma places tmp vma below. 2760 */ 2761 if (start > vma->vm_start) { 2762 int error; 2763 2764 /* 2765 * Make sure that map_count on return from munmap() will 2766 * not exceed its limit; but let map_count go just above 2767 * its limit temporarily, to help free resources as expected. 2768 */ 2769 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) 2770 return -ENOMEM; 2771 2772 error = __split_vma(mm, vma, start, 0); 2773 if (error) 2774 return error; 2775 prev = vma; 2776 } 2777 2778 /* Does it split the last one? */ 2779 last = find_vma(mm, end); 2780 if (last && end > last->vm_start) { 2781 int error = __split_vma(mm, last, end, 1); 2782 if (error) 2783 return error; 2784 } 2785 vma = prev ? prev->vm_next : mm->mmap; 2786 2787 if (unlikely(uf)) { 2788 /* 2789 * If userfaultfd_unmap_prep returns an error the vmas 2790 * will remain splitted, but userland will get a 2791 * highly unexpected error anyway. This is no 2792 * different than the case where the first of the two 2793 * __split_vma fails, but we don't undo the first 2794 * split, despite we could. This is unlikely enough 2795 * failure that it's not worth optimizing it for. 2796 */ 2797 int error = userfaultfd_unmap_prep(vma, start, end, uf); 2798 if (error) 2799 return error; 2800 } 2801 2802 /* 2803 * unlock any mlock()ed ranges before detaching vmas 2804 */ 2805 if (mm->locked_vm) { 2806 struct vm_area_struct *tmp = vma; 2807 while (tmp && tmp->vm_start < end) { 2808 if (tmp->vm_flags & VM_LOCKED) { 2809 mm->locked_vm -= vma_pages(tmp); 2810 munlock_vma_pages_all(tmp); 2811 } 2812 2813 tmp = tmp->vm_next; 2814 } 2815 } 2816 2817 /* Detach vmas from rbtree */ 2818 detach_vmas_to_be_unmapped(mm, vma, prev, end); 2819 2820 if (downgrade) 2821 downgrade_write(&mm->mmap_sem); 2822 2823 unmap_region(mm, vma, prev, start, end); 2824 2825 /* Fix up all other VM information */ 2826 remove_vma_list(mm, vma); 2827 2828 return downgrade ? 1 : 0; 2829 } 2830 2831 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 2832 struct list_head *uf) 2833 { 2834 return __do_munmap(mm, start, len, uf, false); 2835 } 2836 2837 static int __vm_munmap(unsigned long start, size_t len, bool downgrade) 2838 { 2839 int ret; 2840 struct mm_struct *mm = current->mm; 2841 LIST_HEAD(uf); 2842 2843 if (down_write_killable(&mm->mmap_sem)) 2844 return -EINTR; 2845 2846 ret = __do_munmap(mm, start, len, &uf, downgrade); 2847 /* 2848 * Returning 1 indicates mmap_sem is downgraded. 2849 * But 1 is not legal return value of vm_munmap() and munmap(), reset 2850 * it to 0 before return. 2851 */ 2852 if (ret == 1) { 2853 up_read(&mm->mmap_sem); 2854 ret = 0; 2855 } else 2856 up_write(&mm->mmap_sem); 2857 2858 userfaultfd_unmap_complete(mm, &uf); 2859 return ret; 2860 } 2861 2862 int vm_munmap(unsigned long start, size_t len) 2863 { 2864 return __vm_munmap(start, len, false); 2865 } 2866 EXPORT_SYMBOL(vm_munmap); 2867 2868 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 2869 { 2870 addr = untagged_addr(addr); 2871 profile_munmap(addr); 2872 return __vm_munmap(addr, len, true); 2873 } 2874 2875 2876 /* 2877 * Emulation of deprecated remap_file_pages() syscall. 2878 */ 2879 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, 2880 unsigned long, prot, unsigned long, pgoff, unsigned long, flags) 2881 { 2882 2883 struct mm_struct *mm = current->mm; 2884 struct vm_area_struct *vma; 2885 unsigned long populate = 0; 2886 unsigned long ret = -EINVAL; 2887 struct file *file; 2888 2889 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.rst.\n", 2890 current->comm, current->pid); 2891 2892 if (prot) 2893 return ret; 2894 start = start & PAGE_MASK; 2895 size = size & PAGE_MASK; 2896 2897 if (start + size <= start) 2898 return ret; 2899 2900 /* Does pgoff wrap? */ 2901 if (pgoff + (size >> PAGE_SHIFT) < pgoff) 2902 return ret; 2903 2904 if (down_write_killable(&mm->mmap_sem)) 2905 return -EINTR; 2906 2907 vma = find_vma(mm, start); 2908 2909 if (!vma || !(vma->vm_flags & VM_SHARED)) 2910 goto out; 2911 2912 if (start < vma->vm_start) 2913 goto out; 2914 2915 if (start + size > vma->vm_end) { 2916 struct vm_area_struct *next; 2917 2918 for (next = vma->vm_next; next; next = next->vm_next) { 2919 /* hole between vmas ? */ 2920 if (next->vm_start != next->vm_prev->vm_end) 2921 goto out; 2922 2923 if (next->vm_file != vma->vm_file) 2924 goto out; 2925 2926 if (next->vm_flags != vma->vm_flags) 2927 goto out; 2928 2929 if (start + size <= next->vm_end) 2930 break; 2931 } 2932 2933 if (!next) 2934 goto out; 2935 } 2936 2937 prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; 2938 prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; 2939 prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; 2940 2941 flags &= MAP_NONBLOCK; 2942 flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; 2943 if (vma->vm_flags & VM_LOCKED) { 2944 struct vm_area_struct *tmp; 2945 flags |= MAP_LOCKED; 2946 2947 /* drop PG_Mlocked flag for over-mapped range */ 2948 for (tmp = vma; tmp->vm_start >= start + size; 2949 tmp = tmp->vm_next) { 2950 /* 2951 * Split pmd and munlock page on the border 2952 * of the range. 2953 */ 2954 vma_adjust_trans_huge(tmp, start, start + size, 0); 2955 2956 munlock_vma_pages_range(tmp, 2957 max(tmp->vm_start, start), 2958 min(tmp->vm_end, start + size)); 2959 } 2960 } 2961 2962 file = get_file(vma->vm_file); 2963 ret = do_mmap_pgoff(vma->vm_file, start, size, 2964 prot, flags, pgoff, &populate, NULL); 2965 fput(file); 2966 out: 2967 up_write(&mm->mmap_sem); 2968 if (populate) 2969 mm_populate(ret, populate); 2970 if (!IS_ERR_VALUE(ret)) 2971 ret = 0; 2972 return ret; 2973 } 2974 2975 /* 2976 * this is really a simplified "do_mmap". it only handles 2977 * anonymous maps. eventually we may be able to do some 2978 * brk-specific accounting here. 2979 */ 2980 static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf) 2981 { 2982 struct mm_struct *mm = current->mm; 2983 struct vm_area_struct *vma, *prev; 2984 struct rb_node **rb_link, *rb_parent; 2985 pgoff_t pgoff = addr >> PAGE_SHIFT; 2986 int error; 2987 unsigned long mapped_addr; 2988 2989 /* Until we need other flags, refuse anything except VM_EXEC. */ 2990 if ((flags & (~VM_EXEC)) != 0) 2991 return -EINVAL; 2992 flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; 2993 2994 mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); 2995 if (IS_ERR_VALUE(mapped_addr)) 2996 return mapped_addr; 2997 2998 error = mlock_future_check(mm, mm->def_flags, len); 2999 if (error) 3000 return error; 3001 3002 /* 3003 * Clear old maps. this also does some error checking for us 3004 */ 3005 while (find_vma_links(mm, addr, addr + len, &prev, &rb_link, 3006 &rb_parent)) { 3007 if (do_munmap(mm, addr, len, uf)) 3008 return -ENOMEM; 3009 } 3010 3011 /* Check against address space limits *after* clearing old maps... */ 3012 if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) 3013 return -ENOMEM; 3014 3015 if (mm->map_count > sysctl_max_map_count) 3016 return -ENOMEM; 3017 3018 if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) 3019 return -ENOMEM; 3020 3021 /* Can we just expand an old private anonymous mapping? */ 3022 vma = vma_merge(mm, prev, addr, addr + len, flags, 3023 NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX); 3024 if (vma) 3025 goto out; 3026 3027 /* 3028 * create a vma struct for an anonymous mapping 3029 */ 3030 vma = vm_area_alloc(mm); 3031 if (!vma) { 3032 vm_unacct_memory(len >> PAGE_SHIFT); 3033 return -ENOMEM; 3034 } 3035 3036 vma_set_anonymous(vma); 3037 vma->vm_start = addr; 3038 vma->vm_end = addr + len; 3039 vma->vm_pgoff = pgoff; 3040 vma->vm_flags = flags; 3041 vma->vm_page_prot = vm_get_page_prot(flags); 3042 vma_link(mm, vma, prev, rb_link, rb_parent); 3043 out: 3044 perf_event_mmap(vma); 3045 mm->total_vm += len >> PAGE_SHIFT; 3046 mm->data_vm += len >> PAGE_SHIFT; 3047 if (flags & VM_LOCKED) 3048 mm->locked_vm += (len >> PAGE_SHIFT); 3049 vma->vm_flags |= VM_SOFTDIRTY; 3050 return 0; 3051 } 3052 3053 int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) 3054 { 3055 struct mm_struct *mm = current->mm; 3056 unsigned long len; 3057 int ret; 3058 bool populate; 3059 LIST_HEAD(uf); 3060 3061 len = PAGE_ALIGN(request); 3062 if (len < request) 3063 return -ENOMEM; 3064 if (!len) 3065 return 0; 3066 3067 if (down_write_killable(&mm->mmap_sem)) 3068 return -EINTR; 3069 3070 ret = do_brk_flags(addr, len, flags, &uf); 3071 populate = ((mm->def_flags & VM_LOCKED) != 0); 3072 up_write(&mm->mmap_sem); 3073 userfaultfd_unmap_complete(mm, &uf); 3074 if (populate && !ret) 3075 mm_populate(addr, len); 3076 return ret; 3077 } 3078 EXPORT_SYMBOL(vm_brk_flags); 3079 3080 int vm_brk(unsigned long addr, unsigned long len) 3081 { 3082 return vm_brk_flags(addr, len, 0); 3083 } 3084 EXPORT_SYMBOL(vm_brk); 3085 3086 /* Release all mmaps. */ 3087 void exit_mmap(struct mm_struct *mm) 3088 { 3089 struct mmu_gather tlb; 3090 struct vm_area_struct *vma; 3091 unsigned long nr_accounted = 0; 3092 3093 /* mm's last user has gone, and its about to be pulled down */ 3094 mmu_notifier_release(mm); 3095 3096 if (unlikely(mm_is_oom_victim(mm))) { 3097 /* 3098 * Manually reap the mm to free as much memory as possible. 3099 * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard 3100 * this mm from further consideration. Taking mm->mmap_sem for 3101 * write after setting MMF_OOM_SKIP will guarantee that the oom 3102 * reaper will not run on this mm again after mmap_sem is 3103 * dropped. 3104 * 3105 * Nothing can be holding mm->mmap_sem here and the above call 3106 * to mmu_notifier_release(mm) ensures mmu notifier callbacks in 3107 * __oom_reap_task_mm() will not block. 3108 * 3109 * This needs to be done before calling munlock_vma_pages_all(), 3110 * which clears VM_LOCKED, otherwise the oom reaper cannot 3111 * reliably test it. 3112 */ 3113 (void)__oom_reap_task_mm(mm); 3114 3115 set_bit(MMF_OOM_SKIP, &mm->flags); 3116 down_write(&mm->mmap_sem); 3117 up_write(&mm->mmap_sem); 3118 } 3119 3120 if (mm->locked_vm) { 3121 vma = mm->mmap; 3122 while (vma) { 3123 if (vma->vm_flags & VM_LOCKED) 3124 munlock_vma_pages_all(vma); 3125 vma = vma->vm_next; 3126 } 3127 } 3128 3129 arch_exit_mmap(mm); 3130 3131 vma = mm->mmap; 3132 if (!vma) /* Can happen if dup_mmap() received an OOM */ 3133 return; 3134 3135 lru_add_drain(); 3136 flush_cache_mm(mm); 3137 tlb_gather_mmu(&tlb, mm, 0, -1); 3138 /* update_hiwater_rss(mm) here? but nobody should be looking */ 3139 /* Use -1 here to ensure all VMAs in the mm are unmapped */ 3140 unmap_vmas(&tlb, vma, 0, -1); 3141 free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); 3142 tlb_finish_mmu(&tlb, 0, -1); 3143 3144 /* 3145 * Walk the list again, actually closing and freeing it, 3146 * with preemption enabled, without holding any MM locks. 3147 */ 3148 while (vma) { 3149 if (vma->vm_flags & VM_ACCOUNT) 3150 nr_accounted += vma_pages(vma); 3151 vma = remove_vma(vma); 3152 } 3153 vm_unacct_memory(nr_accounted); 3154 } 3155 3156 /* Insert vm structure into process list sorted by address 3157 * and into the inode's i_mmap tree. If vm_file is non-NULL 3158 * then i_mmap_rwsem is taken here. 3159 */ 3160 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) 3161 { 3162 struct vm_area_struct *prev; 3163 struct rb_node **rb_link, *rb_parent; 3164 3165 if (find_vma_links(mm, vma->vm_start, vma->vm_end, 3166 &prev, &rb_link, &rb_parent)) 3167 return -ENOMEM; 3168 if ((vma->vm_flags & VM_ACCOUNT) && 3169 security_vm_enough_memory_mm(mm, vma_pages(vma))) 3170 return -ENOMEM; 3171 3172 /* 3173 * The vm_pgoff of a purely anonymous vma should be irrelevant 3174 * until its first write fault, when page's anon_vma and index 3175 * are set. But now set the vm_pgoff it will almost certainly 3176 * end up with (unless mremap moves it elsewhere before that 3177 * first wfault), so /proc/pid/maps tells a consistent story. 3178 * 3179 * By setting it to reflect the virtual start address of the 3180 * vma, merges and splits can happen in a seamless way, just 3181 * using the existing file pgoff checks and manipulations. 3182 * Similarly in do_mmap_pgoff and in do_brk. 3183 */ 3184 if (vma_is_anonymous(vma)) { 3185 BUG_ON(vma->anon_vma); 3186 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; 3187 } 3188 3189 vma_link(mm, vma, prev, rb_link, rb_parent); 3190 return 0; 3191 } 3192 3193 /* 3194 * Copy the vma structure to a new location in the same mm, 3195 * prior to moving page table entries, to effect an mremap move. 3196 */ 3197 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 3198 unsigned long addr, unsigned long len, pgoff_t pgoff, 3199 bool *need_rmap_locks) 3200 { 3201 struct vm_area_struct *vma = *vmap; 3202 unsigned long vma_start = vma->vm_start; 3203 struct mm_struct *mm = vma->vm_mm; 3204 struct vm_area_struct *new_vma, *prev; 3205 struct rb_node **rb_link, *rb_parent; 3206 bool faulted_in_anon_vma = true; 3207 3208 /* 3209 * If anonymous vma has not yet been faulted, update new pgoff 3210 * to match new location, to increase its chance of merging. 3211 */ 3212 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 3213 pgoff = addr >> PAGE_SHIFT; 3214 faulted_in_anon_vma = false; 3215 } 3216 3217 if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) 3218 return NULL; /* should never get here */ 3219 new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, 3220 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 3221 vma->vm_userfaultfd_ctx); 3222 if (new_vma) { 3223 /* 3224 * Source vma may have been merged into new_vma 3225 */ 3226 if (unlikely(vma_start >= new_vma->vm_start && 3227 vma_start < new_vma->vm_end)) { 3228 /* 3229 * The only way we can get a vma_merge with 3230 * self during an mremap is if the vma hasn't 3231 * been faulted in yet and we were allowed to 3232 * reset the dst vma->vm_pgoff to the 3233 * destination address of the mremap to allow 3234 * the merge to happen. mremap must change the 3235 * vm_pgoff linearity between src and dst vmas 3236 * (in turn preventing a vma_merge) to be 3237 * safe. It is only safe to keep the vm_pgoff 3238 * linear if there are no pages mapped yet. 3239 */ 3240 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 3241 *vmap = vma = new_vma; 3242 } 3243 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 3244 } else { 3245 new_vma = vm_area_dup(vma); 3246 if (!new_vma) 3247 goto out; 3248 new_vma->vm_start = addr; 3249 new_vma->vm_end = addr + len; 3250 new_vma->vm_pgoff = pgoff; 3251 if (vma_dup_policy(vma, new_vma)) 3252 goto out_free_vma; 3253 if (anon_vma_clone(new_vma, vma)) 3254 goto out_free_mempol; 3255 if (new_vma->vm_file) 3256 get_file(new_vma->vm_file); 3257 if (new_vma->vm_ops && new_vma->vm_ops->open) 3258 new_vma->vm_ops->open(new_vma); 3259 vma_link(mm, new_vma, prev, rb_link, rb_parent); 3260 *need_rmap_locks = false; 3261 } 3262 return new_vma; 3263 3264 out_free_mempol: 3265 mpol_put(vma_policy(new_vma)); 3266 out_free_vma: 3267 vm_area_free(new_vma); 3268 out: 3269 return NULL; 3270 } 3271 3272 /* 3273 * Return true if the calling process may expand its vm space by the passed 3274 * number of pages 3275 */ 3276 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) 3277 { 3278 if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) 3279 return false; 3280 3281 if (is_data_mapping(flags) && 3282 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { 3283 /* Workaround for Valgrind */ 3284 if (rlimit(RLIMIT_DATA) == 0 && 3285 mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) 3286 return true; 3287 3288 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", 3289 current->comm, current->pid, 3290 (mm->data_vm + npages) << PAGE_SHIFT, 3291 rlimit(RLIMIT_DATA), 3292 ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); 3293 3294 if (!ignore_rlimit_data) 3295 return false; 3296 } 3297 3298 return true; 3299 } 3300 3301 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) 3302 { 3303 mm->total_vm += npages; 3304 3305 if (is_exec_mapping(flags)) 3306 mm->exec_vm += npages; 3307 else if (is_stack_mapping(flags)) 3308 mm->stack_vm += npages; 3309 else if (is_data_mapping(flags)) 3310 mm->data_vm += npages; 3311 } 3312 3313 static vm_fault_t special_mapping_fault(struct vm_fault *vmf); 3314 3315 /* 3316 * Having a close hook prevents vma merging regardless of flags. 3317 */ 3318 static void special_mapping_close(struct vm_area_struct *vma) 3319 { 3320 } 3321 3322 static const char *special_mapping_name(struct vm_area_struct *vma) 3323 { 3324 return ((struct vm_special_mapping *)vma->vm_private_data)->name; 3325 } 3326 3327 static int special_mapping_mremap(struct vm_area_struct *new_vma) 3328 { 3329 struct vm_special_mapping *sm = new_vma->vm_private_data; 3330 3331 if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) 3332 return -EFAULT; 3333 3334 if (sm->mremap) 3335 return sm->mremap(sm, new_vma); 3336 3337 return 0; 3338 } 3339 3340 static const struct vm_operations_struct special_mapping_vmops = { 3341 .close = special_mapping_close, 3342 .fault = special_mapping_fault, 3343 .mremap = special_mapping_mremap, 3344 .name = special_mapping_name, 3345 }; 3346 3347 static const struct vm_operations_struct legacy_special_mapping_vmops = { 3348 .close = special_mapping_close, 3349 .fault = special_mapping_fault, 3350 }; 3351 3352 static vm_fault_t special_mapping_fault(struct vm_fault *vmf) 3353 { 3354 struct vm_area_struct *vma = vmf->vma; 3355 pgoff_t pgoff; 3356 struct page **pages; 3357 3358 if (vma->vm_ops == &legacy_special_mapping_vmops) { 3359 pages = vma->vm_private_data; 3360 } else { 3361 struct vm_special_mapping *sm = vma->vm_private_data; 3362 3363 if (sm->fault) 3364 return sm->fault(sm, vmf->vma, vmf); 3365 3366 pages = sm->pages; 3367 } 3368 3369 for (pgoff = vmf->pgoff; pgoff && *pages; ++pages) 3370 pgoff--; 3371 3372 if (*pages) { 3373 struct page *page = *pages; 3374 get_page(page); 3375 vmf->page = page; 3376 return 0; 3377 } 3378 3379 return VM_FAULT_SIGBUS; 3380 } 3381 3382 static struct vm_area_struct *__install_special_mapping( 3383 struct mm_struct *mm, 3384 unsigned long addr, unsigned long len, 3385 unsigned long vm_flags, void *priv, 3386 const struct vm_operations_struct *ops) 3387 { 3388 int ret; 3389 struct vm_area_struct *vma; 3390 3391 vma = vm_area_alloc(mm); 3392 if (unlikely(vma == NULL)) 3393 return ERR_PTR(-ENOMEM); 3394 3395 vma->vm_start = addr; 3396 vma->vm_end = addr + len; 3397 3398 vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY; 3399 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 3400 3401 vma->vm_ops = ops; 3402 vma->vm_private_data = priv; 3403 3404 ret = insert_vm_struct(mm, vma); 3405 if (ret) 3406 goto out; 3407 3408 vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); 3409 3410 perf_event_mmap(vma); 3411 3412 return vma; 3413 3414 out: 3415 vm_area_free(vma); 3416 return ERR_PTR(ret); 3417 } 3418 3419 bool vma_is_special_mapping(const struct vm_area_struct *vma, 3420 const struct vm_special_mapping *sm) 3421 { 3422 return vma->vm_private_data == sm && 3423 (vma->vm_ops == &special_mapping_vmops || 3424 vma->vm_ops == &legacy_special_mapping_vmops); 3425 } 3426 3427 /* 3428 * Called with mm->mmap_sem held for writing. 3429 * Insert a new vma covering the given region, with the given flags. 3430 * Its pages are supplied by the given array of struct page *. 3431 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. 3432 * The region past the last page supplied will always produce SIGBUS. 3433 * The array pointer and the pages it points to are assumed to stay alive 3434 * for as long as this mapping might exist. 3435 */ 3436 struct vm_area_struct *_install_special_mapping( 3437 struct mm_struct *mm, 3438 unsigned long addr, unsigned long len, 3439 unsigned long vm_flags, const struct vm_special_mapping *spec) 3440 { 3441 return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec, 3442 &special_mapping_vmops); 3443 } 3444 3445 int install_special_mapping(struct mm_struct *mm, 3446 unsigned long addr, unsigned long len, 3447 unsigned long vm_flags, struct page **pages) 3448 { 3449 struct vm_area_struct *vma = __install_special_mapping( 3450 mm, addr, len, vm_flags, (void *)pages, 3451 &legacy_special_mapping_vmops); 3452 3453 return PTR_ERR_OR_ZERO(vma); 3454 } 3455 3456 static DEFINE_MUTEX(mm_all_locks_mutex); 3457 3458 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 3459 { 3460 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 3461 /* 3462 * The LSB of head.next can't change from under us 3463 * because we hold the mm_all_locks_mutex. 3464 */ 3465 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem); 3466 /* 3467 * We can safely modify head.next after taking the 3468 * anon_vma->root->rwsem. If some other vma in this mm shares 3469 * the same anon_vma we won't take it again. 3470 * 3471 * No need of atomic instructions here, head.next 3472 * can't change from under us thanks to the 3473 * anon_vma->root->rwsem. 3474 */ 3475 if (__test_and_set_bit(0, (unsigned long *) 3476 &anon_vma->root->rb_root.rb_root.rb_node)) 3477 BUG(); 3478 } 3479 } 3480 3481 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 3482 { 3483 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 3484 /* 3485 * AS_MM_ALL_LOCKS can't change from under us because 3486 * we hold the mm_all_locks_mutex. 3487 * 3488 * Operations on ->flags have to be atomic because 3489 * even if AS_MM_ALL_LOCKS is stable thanks to the 3490 * mm_all_locks_mutex, there may be other cpus 3491 * changing other bitflags in parallel to us. 3492 */ 3493 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 3494 BUG(); 3495 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem); 3496 } 3497 } 3498 3499 /* 3500 * This operation locks against the VM for all pte/vma/mm related 3501 * operations that could ever happen on a certain mm. This includes 3502 * vmtruncate, try_to_unmap, and all page faults. 3503 * 3504 * The caller must take the mmap_sem in write mode before calling 3505 * mm_take_all_locks(). The caller isn't allowed to release the 3506 * mmap_sem until mm_drop_all_locks() returns. 3507 * 3508 * mmap_sem in write mode is required in order to block all operations 3509 * that could modify pagetables and free pages without need of 3510 * altering the vma layout. It's also needed in write mode to avoid new 3511 * anon_vmas to be associated with existing vmas. 3512 * 3513 * A single task can't take more than one mm_take_all_locks() in a row 3514 * or it would deadlock. 3515 * 3516 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 3517 * mapping->flags avoid to take the same lock twice, if more than one 3518 * vma in this mm is backed by the same anon_vma or address_space. 3519 * 3520 * We take locks in following order, accordingly to comment at beginning 3521 * of mm/rmap.c: 3522 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 3523 * hugetlb mapping); 3524 * - all i_mmap_rwsem locks; 3525 * - all anon_vma->rwseml 3526 * 3527 * We can take all locks within these types randomly because the VM code 3528 * doesn't nest them and we protected from parallel mm_take_all_locks() by 3529 * mm_all_locks_mutex. 3530 * 3531 * mm_take_all_locks() and mm_drop_all_locks are expensive operations 3532 * that may have to take thousand of locks. 3533 * 3534 * mm_take_all_locks() can fail if it's interrupted by signals. 3535 */ 3536 int mm_take_all_locks(struct mm_struct *mm) 3537 { 3538 struct vm_area_struct *vma; 3539 struct anon_vma_chain *avc; 3540 3541 BUG_ON(down_read_trylock(&mm->mmap_sem)); 3542 3543 mutex_lock(&mm_all_locks_mutex); 3544 3545 for (vma = mm->mmap; vma; vma = vma->vm_next) { 3546 if (signal_pending(current)) 3547 goto out_unlock; 3548 if (vma->vm_file && vma->vm_file->f_mapping && 3549 is_vm_hugetlb_page(vma)) 3550 vm_lock_mapping(mm, vma->vm_file->f_mapping); 3551 } 3552 3553 for (vma = mm->mmap; vma; vma = vma->vm_next) { 3554 if (signal_pending(current)) 3555 goto out_unlock; 3556 if (vma->vm_file && vma->vm_file->f_mapping && 3557 !is_vm_hugetlb_page(vma)) 3558 vm_lock_mapping(mm, vma->vm_file->f_mapping); 3559 } 3560 3561 for (vma = mm->mmap; vma; vma = vma->vm_next) { 3562 if (signal_pending(current)) 3563 goto out_unlock; 3564 if (vma->anon_vma) 3565 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 3566 vm_lock_anon_vma(mm, avc->anon_vma); 3567 } 3568 3569 return 0; 3570 3571 out_unlock: 3572 mm_drop_all_locks(mm); 3573 return -EINTR; 3574 } 3575 3576 static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 3577 { 3578 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 3579 /* 3580 * The LSB of head.next can't change to 0 from under 3581 * us because we hold the mm_all_locks_mutex. 3582 * 3583 * We must however clear the bitflag before unlocking 3584 * the vma so the users using the anon_vma->rb_root will 3585 * never see our bitflag. 3586 * 3587 * No need of atomic instructions here, head.next 3588 * can't change from under us until we release the 3589 * anon_vma->root->rwsem. 3590 */ 3591 if (!__test_and_clear_bit(0, (unsigned long *) 3592 &anon_vma->root->rb_root.rb_root.rb_node)) 3593 BUG(); 3594 anon_vma_unlock_write(anon_vma); 3595 } 3596 } 3597 3598 static void vm_unlock_mapping(struct address_space *mapping) 3599 { 3600 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 3601 /* 3602 * AS_MM_ALL_LOCKS can't change to 0 from under us 3603 * because we hold the mm_all_locks_mutex. 3604 */ 3605 i_mmap_unlock_write(mapping); 3606 if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 3607 &mapping->flags)) 3608 BUG(); 3609 } 3610 } 3611 3612 /* 3613 * The mmap_sem cannot be released by the caller until 3614 * mm_drop_all_locks() returns. 3615 */ 3616 void mm_drop_all_locks(struct mm_struct *mm) 3617 { 3618 struct vm_area_struct *vma; 3619 struct anon_vma_chain *avc; 3620 3621 BUG_ON(down_read_trylock(&mm->mmap_sem)); 3622 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 3623 3624 for (vma = mm->mmap; vma; vma = vma->vm_next) { 3625 if (vma->anon_vma) 3626 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 3627 vm_unlock_anon_vma(avc->anon_vma); 3628 if (vma->vm_file && vma->vm_file->f_mapping) 3629 vm_unlock_mapping(vma->vm_file->f_mapping); 3630 } 3631 3632 mutex_unlock(&mm_all_locks_mutex); 3633 } 3634 3635 /* 3636 * initialise the percpu counter for VM 3637 */ 3638 void __init mmap_init(void) 3639 { 3640 int ret; 3641 3642 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); 3643 VM_BUG_ON(ret); 3644 } 3645 3646 /* 3647 * Initialise sysctl_user_reserve_kbytes. 3648 * 3649 * This is intended to prevent a user from starting a single memory hogging 3650 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER 3651 * mode. 3652 * 3653 * The default value is min(3% of free memory, 128MB) 3654 * 128MB is enough to recover with sshd/login, bash, and top/kill. 3655 */ 3656 static int init_user_reserve(void) 3657 { 3658 unsigned long free_kbytes; 3659 3660 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3661 3662 sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17); 3663 return 0; 3664 } 3665 subsys_initcall(init_user_reserve); 3666 3667 /* 3668 * Initialise sysctl_admin_reserve_kbytes. 3669 * 3670 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin 3671 * to log in and kill a memory hogging process. 3672 * 3673 * Systems with more than 256MB will reserve 8MB, enough to recover 3674 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will 3675 * only reserve 3% of free pages by default. 3676 */ 3677 static int init_admin_reserve(void) 3678 { 3679 unsigned long free_kbytes; 3680 3681 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3682 3683 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13); 3684 return 0; 3685 } 3686 subsys_initcall(init_admin_reserve); 3687 3688 /* 3689 * Reinititalise user and admin reserves if memory is added or removed. 3690 * 3691 * The default user reserve max is 128MB, and the default max for the 3692 * admin reserve is 8MB. These are usually, but not always, enough to 3693 * enable recovery from a memory hogging process using login/sshd, a shell, 3694 * and tools like top. It may make sense to increase or even disable the 3695 * reserve depending on the existence of swap or variations in the recovery 3696 * tools. So, the admin may have changed them. 3697 * 3698 * If memory is added and the reserves have been eliminated or increased above 3699 * the default max, then we'll trust the admin. 3700 * 3701 * If memory is removed and there isn't enough free memory, then we 3702 * need to reset the reserves. 3703 * 3704 * Otherwise keep the reserve set by the admin. 3705 */ 3706 static int reserve_mem_notifier(struct notifier_block *nb, 3707 unsigned long action, void *data) 3708 { 3709 unsigned long tmp, free_kbytes; 3710 3711 switch (action) { 3712 case MEM_ONLINE: 3713 /* Default max is 128MB. Leave alone if modified by operator. */ 3714 tmp = sysctl_user_reserve_kbytes; 3715 if (0 < tmp && tmp < (1UL << 17)) 3716 init_user_reserve(); 3717 3718 /* Default max is 8MB. Leave alone if modified by operator. */ 3719 tmp = sysctl_admin_reserve_kbytes; 3720 if (0 < tmp && tmp < (1UL << 13)) 3721 init_admin_reserve(); 3722 3723 break; 3724 case MEM_OFFLINE: 3725 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3726 3727 if (sysctl_user_reserve_kbytes > free_kbytes) { 3728 init_user_reserve(); 3729 pr_info("vm.user_reserve_kbytes reset to %lu\n", 3730 sysctl_user_reserve_kbytes); 3731 } 3732 3733 if (sysctl_admin_reserve_kbytes > free_kbytes) { 3734 init_admin_reserve(); 3735 pr_info("vm.admin_reserve_kbytes reset to %lu\n", 3736 sysctl_admin_reserve_kbytes); 3737 } 3738 break; 3739 default: 3740 break; 3741 } 3742 return NOTIFY_OK; 3743 } 3744 3745 static struct notifier_block reserve_mem_nb = { 3746 .notifier_call = reserve_mem_notifier, 3747 }; 3748 3749 static int __meminit init_reserve_notifier(void) 3750 { 3751 if (register_hotmemory_notifier(&reserve_mem_nb)) 3752 pr_err("Failed registering memory add/remove notifier for admin reserve\n"); 3753 3754 return 0; 3755 } 3756 subsys_initcall(init_reserve_notifier); 3757