1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm/mmap.c 4 * 5 * Written by obz. 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 */ 9 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 #include <linux/backing-dev.h> 15 #include <linux/mm.h> 16 #include <linux/mm_inline.h> 17 #include <linux/shm.h> 18 #include <linux/mman.h> 19 #include <linux/pagemap.h> 20 #include <linux/swap.h> 21 #include <linux/syscalls.h> 22 #include <linux/capability.h> 23 #include <linux/init.h> 24 #include <linux/file.h> 25 #include <linux/fs.h> 26 #include <linux/personality.h> 27 #include <linux/security.h> 28 #include <linux/hugetlb.h> 29 #include <linux/shmem_fs.h> 30 #include <linux/profile.h> 31 #include <linux/export.h> 32 #include <linux/mount.h> 33 #include <linux/mempolicy.h> 34 #include <linux/rmap.h> 35 #include <linux/mmu_notifier.h> 36 #include <linux/mmdebug.h> 37 #include <linux/perf_event.h> 38 #include <linux/audit.h> 39 #include <linux/khugepaged.h> 40 #include <linux/uprobes.h> 41 #include <linux/notifier.h> 42 #include <linux/memory.h> 43 #include <linux/printk.h> 44 #include <linux/userfaultfd_k.h> 45 #include <linux/moduleparam.h> 46 #include <linux/pkeys.h> 47 #include <linux/oom.h> 48 #include <linux/sched/mm.h> 49 50 #include <linux/uaccess.h> 51 #include <asm/cacheflush.h> 52 #include <asm/tlb.h> 53 #include <asm/mmu_context.h> 54 55 #define CREATE_TRACE_POINTS 56 #include <trace/events/mmap.h> 57 58 #include "internal.h" 59 60 #ifndef arch_mmap_check 61 #define arch_mmap_check(addr, len, flags) (0) 62 #endif 63 64 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 65 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; 66 const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX; 67 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; 68 #endif 69 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 70 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; 71 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; 72 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; 73 #endif 74 75 static bool ignore_rlimit_data; 76 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); 77 78 static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, 79 struct vm_area_struct *vma, struct vm_area_struct *prev, 80 struct vm_area_struct *next, unsigned long start, 81 unsigned long end); 82 83 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) 84 { 85 return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); 86 } 87 88 /* Update vma->vm_page_prot to reflect vma->vm_flags. */ 89 void vma_set_page_prot(struct vm_area_struct *vma) 90 { 91 unsigned long vm_flags = vma->vm_flags; 92 pgprot_t vm_page_prot; 93 94 vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); 95 if (vma_wants_writenotify(vma, vm_page_prot)) { 96 vm_flags &= ~VM_SHARED; 97 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); 98 } 99 /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ 100 WRITE_ONCE(vma->vm_page_prot, vm_page_prot); 101 } 102 103 /* 104 * Requires inode->i_mapping->i_mmap_rwsem 105 */ 106 static void __remove_shared_vm_struct(struct vm_area_struct *vma, 107 struct file *file, struct address_space *mapping) 108 { 109 if (vma->vm_flags & VM_SHARED) 110 mapping_unmap_writable(mapping); 111 112 flush_dcache_mmap_lock(mapping); 113 vma_interval_tree_remove(vma, &mapping->i_mmap); 114 flush_dcache_mmap_unlock(mapping); 115 } 116 117 /* 118 * Unlink a file-based vm structure from its interval tree, to hide 119 * vma from rmap and vmtruncate before freeing its page tables. 120 */ 121 void unlink_file_vma(struct vm_area_struct *vma) 122 { 123 struct file *file = vma->vm_file; 124 125 if (file) { 126 struct address_space *mapping = file->f_mapping; 127 i_mmap_lock_write(mapping); 128 __remove_shared_vm_struct(vma, file, mapping); 129 i_mmap_unlock_write(mapping); 130 } 131 } 132 133 /* 134 * Close a vm structure and free it. 135 */ 136 static void remove_vma(struct vm_area_struct *vma) 137 { 138 might_sleep(); 139 if (vma->vm_ops && vma->vm_ops->close) 140 vma->vm_ops->close(vma); 141 if (vma->vm_file) 142 fput(vma->vm_file); 143 mpol_put(vma_policy(vma)); 144 vm_area_free(vma); 145 } 146 147 /* 148 * check_brk_limits() - Use platform specific check of range & verify mlock 149 * limits. 150 * @addr: The address to check 151 * @len: The size of increase. 152 * 153 * Return: 0 on success. 154 */ 155 static int check_brk_limits(unsigned long addr, unsigned long len) 156 { 157 unsigned long mapped_addr; 158 159 mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); 160 if (IS_ERR_VALUE(mapped_addr)) 161 return mapped_addr; 162 163 return mlock_future_check(current->mm, current->mm->def_flags, len); 164 } 165 static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, 166 unsigned long newbrk, unsigned long oldbrk, 167 struct list_head *uf); 168 static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma, 169 unsigned long addr, unsigned long request, unsigned long flags); 170 SYSCALL_DEFINE1(brk, unsigned long, brk) 171 { 172 unsigned long newbrk, oldbrk, origbrk; 173 struct mm_struct *mm = current->mm; 174 struct vm_area_struct *brkvma, *next = NULL; 175 unsigned long min_brk; 176 bool populate; 177 bool downgraded = false; 178 LIST_HEAD(uf); 179 MA_STATE(mas, &mm->mm_mt, 0, 0); 180 181 if (mmap_write_lock_killable(mm)) 182 return -EINTR; 183 184 origbrk = mm->brk; 185 186 #ifdef CONFIG_COMPAT_BRK 187 /* 188 * CONFIG_COMPAT_BRK can still be overridden by setting 189 * randomize_va_space to 2, which will still cause mm->start_brk 190 * to be arbitrarily shifted 191 */ 192 if (current->brk_randomized) 193 min_brk = mm->start_brk; 194 else 195 min_brk = mm->end_data; 196 #else 197 min_brk = mm->start_brk; 198 #endif 199 if (brk < min_brk) 200 goto out; 201 202 /* 203 * Check against rlimit here. If this check is done later after the test 204 * of oldbrk with newbrk then it can escape the test and let the data 205 * segment grow beyond its set limit the in case where the limit is 206 * not page aligned -Ram Gupta 207 */ 208 if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk, 209 mm->end_data, mm->start_data)) 210 goto out; 211 212 newbrk = PAGE_ALIGN(brk); 213 oldbrk = PAGE_ALIGN(mm->brk); 214 if (oldbrk == newbrk) { 215 mm->brk = brk; 216 goto success; 217 } 218 219 /* 220 * Always allow shrinking brk. 221 * do_brk_munmap() may downgrade mmap_lock to read. 222 */ 223 if (brk <= mm->brk) { 224 int ret; 225 226 /* Search one past newbrk */ 227 mas_set(&mas, newbrk); 228 brkvma = mas_find(&mas, oldbrk); 229 BUG_ON(brkvma == NULL); 230 if (brkvma->vm_start >= oldbrk) 231 goto out; /* mapping intersects with an existing non-brk vma. */ 232 /* 233 * mm->brk must be protected by write mmap_lock. 234 * do_brk_munmap() may downgrade the lock, so update it 235 * before calling do_brk_munmap(). 236 */ 237 mm->brk = brk; 238 ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf); 239 if (ret == 1) { 240 downgraded = true; 241 goto success; 242 } else if (!ret) 243 goto success; 244 245 mm->brk = origbrk; 246 goto out; 247 } 248 249 if (check_brk_limits(oldbrk, newbrk - oldbrk)) 250 goto out; 251 252 /* 253 * Only check if the next VMA is within the stack_guard_gap of the 254 * expansion area 255 */ 256 mas_set(&mas, oldbrk); 257 next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap); 258 if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) 259 goto out; 260 261 brkvma = mas_prev(&mas, mm->start_brk); 262 /* Ok, looks good - let it rip. */ 263 if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) 264 goto out; 265 266 mm->brk = brk; 267 268 success: 269 populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0; 270 if (downgraded) 271 mmap_read_unlock(mm); 272 else 273 mmap_write_unlock(mm); 274 userfaultfd_unmap_complete(mm, &uf); 275 if (populate) 276 mm_populate(oldbrk, newbrk - oldbrk); 277 return brk; 278 279 out: 280 mmap_write_unlock(mm); 281 return origbrk; 282 } 283 284 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 285 extern void mt_validate(struct maple_tree *mt); 286 extern void mt_dump(const struct maple_tree *mt); 287 288 /* Validate the maple tree */ 289 static void validate_mm_mt(struct mm_struct *mm) 290 { 291 struct maple_tree *mt = &mm->mm_mt; 292 struct vm_area_struct *vma_mt; 293 294 MA_STATE(mas, mt, 0, 0); 295 296 mt_validate(&mm->mm_mt); 297 mas_for_each(&mas, vma_mt, ULONG_MAX) { 298 if ((vma_mt->vm_start != mas.index) || 299 (vma_mt->vm_end - 1 != mas.last)) { 300 pr_emerg("issue in %s\n", current->comm); 301 dump_stack(); 302 dump_vma(vma_mt); 303 pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, 304 mas.index, mas.last); 305 pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, 306 vma_mt->vm_start, vma_mt->vm_end); 307 308 mt_dump(mas.tree); 309 if (vma_mt->vm_end != mas.last + 1) { 310 pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", 311 mm, vma_mt->vm_start, vma_mt->vm_end, 312 mas.index, mas.last); 313 mt_dump(mas.tree); 314 } 315 VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); 316 if (vma_mt->vm_start != mas.index) { 317 pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", 318 mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); 319 mt_dump(mas.tree); 320 } 321 VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); 322 } 323 } 324 } 325 326 static void validate_mm(struct mm_struct *mm) 327 { 328 int bug = 0; 329 int i = 0; 330 struct vm_area_struct *vma; 331 MA_STATE(mas, &mm->mm_mt, 0, 0); 332 333 validate_mm_mt(mm); 334 335 mas_for_each(&mas, vma, ULONG_MAX) { 336 #ifdef CONFIG_DEBUG_VM_RB 337 struct anon_vma *anon_vma = vma->anon_vma; 338 struct anon_vma_chain *avc; 339 340 if (anon_vma) { 341 anon_vma_lock_read(anon_vma); 342 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 343 anon_vma_interval_tree_verify(avc); 344 anon_vma_unlock_read(anon_vma); 345 } 346 #endif 347 i++; 348 } 349 if (i != mm->map_count) { 350 pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i); 351 bug = 1; 352 } 353 VM_BUG_ON_MM(bug, mm); 354 } 355 356 #else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ 357 #define validate_mm_mt(root) do { } while (0) 358 #define validate_mm(mm) do { } while (0) 359 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 360 361 /* 362 * vma has some anon_vma assigned, and is already inserted on that 363 * anon_vma's interval trees. 364 * 365 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 366 * vma must be removed from the anon_vma's interval trees using 367 * anon_vma_interval_tree_pre_update_vma(). 368 * 369 * After the update, the vma will be reinserted using 370 * anon_vma_interval_tree_post_update_vma(). 371 * 372 * The entire update must be protected by exclusive mmap_lock and by 373 * the root anon_vma's mutex. 374 */ 375 static inline void 376 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 377 { 378 struct anon_vma_chain *avc; 379 380 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 381 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 382 } 383 384 static inline void 385 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 386 { 387 struct anon_vma_chain *avc; 388 389 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 390 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 391 } 392 393 static unsigned long count_vma_pages_range(struct mm_struct *mm, 394 unsigned long addr, unsigned long end) 395 { 396 VMA_ITERATOR(vmi, mm, addr); 397 struct vm_area_struct *vma; 398 unsigned long nr_pages = 0; 399 400 for_each_vma_range(vmi, vma, end) { 401 unsigned long vm_start = max(addr, vma->vm_start); 402 unsigned long vm_end = min(end, vma->vm_end); 403 404 nr_pages += PHYS_PFN(vm_end - vm_start); 405 } 406 407 return nr_pages; 408 } 409 410 static void __vma_link_file(struct vm_area_struct *vma, 411 struct address_space *mapping) 412 { 413 if (vma->vm_flags & VM_SHARED) 414 mapping_allow_writable(mapping); 415 416 flush_dcache_mmap_lock(mapping); 417 vma_interval_tree_insert(vma, &mapping->i_mmap); 418 flush_dcache_mmap_unlock(mapping); 419 } 420 421 /* 422 * vma_mas_store() - Store a VMA in the maple tree. 423 * @vma: The vm_area_struct 424 * @mas: The maple state 425 * 426 * Efficient way to store a VMA in the maple tree when the @mas has already 427 * walked to the correct location. 428 * 429 * Note: the end address is inclusive in the maple tree. 430 */ 431 void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) 432 { 433 trace_vma_store(mas->tree, vma); 434 mas_set_range(mas, vma->vm_start, vma->vm_end - 1); 435 mas_store_prealloc(mas, vma); 436 } 437 438 /* 439 * vma_mas_remove() - Remove a VMA from the maple tree. 440 * @vma: The vm_area_struct 441 * @mas: The maple state 442 * 443 * Efficient way to remove a VMA from the maple tree when the @mas has already 444 * been established and points to the correct location. 445 * Note: the end address is inclusive in the maple tree. 446 */ 447 void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) 448 { 449 trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); 450 mas->index = vma->vm_start; 451 mas->last = vma->vm_end - 1; 452 mas_store_prealloc(mas, NULL); 453 } 454 455 /* 456 * vma_mas_szero() - Set a given range to zero. Used when modifying a 457 * vm_area_struct start or end. 458 * 459 * @mas: The maple tree ma_state 460 * @start: The start address to zero 461 * @end: The end address to zero. 462 */ 463 static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, 464 unsigned long end) 465 { 466 trace_vma_mas_szero(mas->tree, start, end - 1); 467 mas_set_range(mas, start, end - 1); 468 mas_store_prealloc(mas, NULL); 469 } 470 471 static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) 472 { 473 MA_STATE(mas, &mm->mm_mt, 0, 0); 474 struct address_space *mapping = NULL; 475 476 if (mas_preallocate(&mas, vma, GFP_KERNEL)) 477 return -ENOMEM; 478 479 if (vma->vm_file) { 480 mapping = vma->vm_file->f_mapping; 481 i_mmap_lock_write(mapping); 482 } 483 484 vma_mas_store(vma, &mas); 485 486 if (mapping) { 487 __vma_link_file(vma, mapping); 488 i_mmap_unlock_write(mapping); 489 } 490 491 mm->map_count++; 492 validate_mm(mm); 493 return 0; 494 } 495 496 /* 497 * vma_expand - Expand an existing VMA 498 * 499 * @mas: The maple state 500 * @vma: The vma to expand 501 * @start: The start of the vma 502 * @end: The exclusive end of the vma 503 * @pgoff: The page offset of vma 504 * @next: The current of next vma. 505 * 506 * Expand @vma to @start and @end. Can expand off the start and end. Will 507 * expand over @next if it's different from @vma and @end == @next->vm_end. 508 * Checking if the @vma can expand and merge with @next needs to be handled by 509 * the caller. 510 * 511 * Returns: 0 on success 512 */ 513 inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, 514 unsigned long start, unsigned long end, pgoff_t pgoff, 515 struct vm_area_struct *next) 516 { 517 struct mm_struct *mm = vma->vm_mm; 518 struct address_space *mapping = NULL; 519 struct rb_root_cached *root = NULL; 520 struct anon_vma *anon_vma = vma->anon_vma; 521 struct file *file = vma->vm_file; 522 bool remove_next = false; 523 524 if (next && (vma != next) && (end == next->vm_end)) { 525 remove_next = true; 526 if (next->anon_vma && !vma->anon_vma) { 527 int error; 528 529 anon_vma = next->anon_vma; 530 vma->anon_vma = anon_vma; 531 error = anon_vma_clone(vma, next); 532 if (error) 533 return error; 534 } 535 } 536 537 /* Not merging but overwriting any part of next is not handled. */ 538 VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start); 539 /* Only handles expanding */ 540 VM_BUG_ON(vma->vm_start < start || vma->vm_end > end); 541 542 if (mas_preallocate(mas, vma, GFP_KERNEL)) 543 goto nomem; 544 545 vma_adjust_trans_huge(vma, start, end, 0); 546 547 if (file) { 548 mapping = file->f_mapping; 549 root = &mapping->i_mmap; 550 uprobe_munmap(vma, vma->vm_start, vma->vm_end); 551 i_mmap_lock_write(mapping); 552 } 553 554 if (anon_vma) { 555 anon_vma_lock_write(anon_vma); 556 anon_vma_interval_tree_pre_update_vma(vma); 557 } 558 559 if (file) { 560 flush_dcache_mmap_lock(mapping); 561 vma_interval_tree_remove(vma, root); 562 } 563 564 vma->vm_start = start; 565 vma->vm_end = end; 566 vma->vm_pgoff = pgoff; 567 /* Note: mas must be pointing to the expanding VMA */ 568 vma_mas_store(vma, mas); 569 570 if (file) { 571 vma_interval_tree_insert(vma, root); 572 flush_dcache_mmap_unlock(mapping); 573 } 574 575 /* Expanding over the next vma */ 576 if (remove_next && file) { 577 __remove_shared_vm_struct(next, file, mapping); 578 } 579 580 if (anon_vma) { 581 anon_vma_interval_tree_post_update_vma(vma); 582 anon_vma_unlock_write(anon_vma); 583 } 584 585 if (file) { 586 i_mmap_unlock_write(mapping); 587 uprobe_mmap(vma); 588 } 589 590 if (remove_next) { 591 if (file) { 592 uprobe_munmap(next, next->vm_start, next->vm_end); 593 fput(file); 594 } 595 if (next->anon_vma) 596 anon_vma_merge(vma, next); 597 mm->map_count--; 598 mpol_put(vma_policy(next)); 599 vm_area_free(next); 600 } 601 602 validate_mm(mm); 603 return 0; 604 605 nomem: 606 return -ENOMEM; 607 } 608 609 /* 610 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that 611 * is already present in an i_mmap tree without adjusting the tree. 612 * The following helper function should be used when such adjustments 613 * are necessary. The "insert" vma (if any) is to be inserted 614 * before we drop the necessary locks. 615 */ 616 int __vma_adjust(struct vm_area_struct *vma, unsigned long start, 617 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert, 618 struct vm_area_struct *expand) 619 { 620 struct mm_struct *mm = vma->vm_mm; 621 struct vm_area_struct *next_next = NULL; /* uninit var warning */ 622 struct vm_area_struct *next = find_vma(mm, vma->vm_end); 623 struct vm_area_struct *orig_vma = vma; 624 struct address_space *mapping = NULL; 625 struct rb_root_cached *root = NULL; 626 struct anon_vma *anon_vma = NULL; 627 struct file *file = vma->vm_file; 628 bool vma_changed = false; 629 long adjust_next = 0; 630 int remove_next = 0; 631 MA_STATE(mas, &mm->mm_mt, 0, 0); 632 struct vm_area_struct *exporter = NULL, *importer = NULL; 633 634 if (next && !insert) { 635 if (end >= next->vm_end) { 636 /* 637 * vma expands, overlapping all the next, and 638 * perhaps the one after too (mprotect case 6). 639 * The only other cases that gets here are 640 * case 1, case 7 and case 8. 641 */ 642 if (next == expand) { 643 /* 644 * The only case where we don't expand "vma" 645 * and we expand "next" instead is case 8. 646 */ 647 VM_WARN_ON(end != next->vm_end); 648 /* 649 * remove_next == 3 means we're 650 * removing "vma" and that to do so we 651 * swapped "vma" and "next". 652 */ 653 remove_next = 3; 654 VM_WARN_ON(file != next->vm_file); 655 swap(vma, next); 656 } else { 657 VM_WARN_ON(expand != vma); 658 /* 659 * case 1, 6, 7, remove_next == 2 is case 6, 660 * remove_next == 1 is case 1 or 7. 661 */ 662 remove_next = 1 + (end > next->vm_end); 663 if (remove_next == 2) 664 next_next = find_vma(mm, next->vm_end); 665 666 VM_WARN_ON(remove_next == 2 && 667 end != next_next->vm_end); 668 } 669 670 exporter = next; 671 importer = vma; 672 673 /* 674 * If next doesn't have anon_vma, import from vma after 675 * next, if the vma overlaps with it. 676 */ 677 if (remove_next == 2 && !next->anon_vma) 678 exporter = next_next; 679 680 } else if (end > next->vm_start) { 681 /* 682 * vma expands, overlapping part of the next: 683 * mprotect case 5 shifting the boundary up. 684 */ 685 adjust_next = (end - next->vm_start); 686 exporter = next; 687 importer = vma; 688 VM_WARN_ON(expand != importer); 689 } else if (end < vma->vm_end) { 690 /* 691 * vma shrinks, and !insert tells it's not 692 * split_vma inserting another: so it must be 693 * mprotect case 4 shifting the boundary down. 694 */ 695 adjust_next = -(vma->vm_end - end); 696 exporter = vma; 697 importer = next; 698 VM_WARN_ON(expand != importer); 699 } 700 701 /* 702 * Easily overlooked: when mprotect shifts the boundary, 703 * make sure the expanding vma has anon_vma set if the 704 * shrinking vma had, to cover any anon pages imported. 705 */ 706 if (exporter && exporter->anon_vma && !importer->anon_vma) { 707 int error; 708 709 importer->anon_vma = exporter->anon_vma; 710 error = anon_vma_clone(importer, exporter); 711 if (error) 712 return error; 713 } 714 } 715 716 if (mas_preallocate(&mas, vma, GFP_KERNEL)) 717 return -ENOMEM; 718 719 vma_adjust_trans_huge(orig_vma, start, end, adjust_next); 720 if (file) { 721 mapping = file->f_mapping; 722 root = &mapping->i_mmap; 723 uprobe_munmap(vma, vma->vm_start, vma->vm_end); 724 725 if (adjust_next) 726 uprobe_munmap(next, next->vm_start, next->vm_end); 727 728 i_mmap_lock_write(mapping); 729 if (insert && insert->vm_file) { 730 /* 731 * Put into interval tree now, so instantiated pages 732 * are visible to arm/parisc __flush_dcache_page 733 * throughout; but we cannot insert into address 734 * space until vma start or end is updated. 735 */ 736 __vma_link_file(insert, insert->vm_file->f_mapping); 737 } 738 } 739 740 anon_vma = vma->anon_vma; 741 if (!anon_vma && adjust_next) 742 anon_vma = next->anon_vma; 743 if (anon_vma) { 744 VM_WARN_ON(adjust_next && next->anon_vma && 745 anon_vma != next->anon_vma); 746 anon_vma_lock_write(anon_vma); 747 anon_vma_interval_tree_pre_update_vma(vma); 748 if (adjust_next) 749 anon_vma_interval_tree_pre_update_vma(next); 750 } 751 752 if (file) { 753 flush_dcache_mmap_lock(mapping); 754 vma_interval_tree_remove(vma, root); 755 if (adjust_next) 756 vma_interval_tree_remove(next, root); 757 } 758 759 if (start != vma->vm_start) { 760 if ((vma->vm_start < start) && 761 (!insert || (insert->vm_end != start))) { 762 vma_mas_szero(&mas, vma->vm_start, start); 763 VM_WARN_ON(insert && insert->vm_start > vma->vm_start); 764 } else { 765 vma_changed = true; 766 } 767 vma->vm_start = start; 768 } 769 if (end != vma->vm_end) { 770 if (vma->vm_end > end) { 771 if (!insert || (insert->vm_start != end)) { 772 vma_mas_szero(&mas, end, vma->vm_end); 773 mas_reset(&mas); 774 VM_WARN_ON(insert && 775 insert->vm_end < vma->vm_end); 776 } 777 } else { 778 vma_changed = true; 779 } 780 vma->vm_end = end; 781 } 782 783 if (vma_changed) 784 vma_mas_store(vma, &mas); 785 786 vma->vm_pgoff = pgoff; 787 if (adjust_next) { 788 next->vm_start += adjust_next; 789 next->vm_pgoff += adjust_next >> PAGE_SHIFT; 790 vma_mas_store(next, &mas); 791 } 792 793 if (file) { 794 if (adjust_next) 795 vma_interval_tree_insert(next, root); 796 vma_interval_tree_insert(vma, root); 797 flush_dcache_mmap_unlock(mapping); 798 } 799 800 if (remove_next && file) { 801 __remove_shared_vm_struct(next, file, mapping); 802 if (remove_next == 2) 803 __remove_shared_vm_struct(next_next, file, mapping); 804 } else if (insert) { 805 /* 806 * split_vma has split insert from vma, and needs 807 * us to insert it before dropping the locks 808 * (it may either follow vma or precede it). 809 */ 810 mas_reset(&mas); 811 vma_mas_store(insert, &mas); 812 mm->map_count++; 813 } 814 815 if (anon_vma) { 816 anon_vma_interval_tree_post_update_vma(vma); 817 if (adjust_next) 818 anon_vma_interval_tree_post_update_vma(next); 819 anon_vma_unlock_write(anon_vma); 820 } 821 822 if (file) { 823 i_mmap_unlock_write(mapping); 824 uprobe_mmap(vma); 825 826 if (adjust_next) 827 uprobe_mmap(next); 828 } 829 830 if (remove_next) { 831 again: 832 if (file) { 833 uprobe_munmap(next, next->vm_start, next->vm_end); 834 fput(file); 835 } 836 if (next->anon_vma) 837 anon_vma_merge(vma, next); 838 mm->map_count--; 839 mpol_put(vma_policy(next)); 840 if (remove_next != 2) 841 BUG_ON(vma->vm_end < next->vm_end); 842 vm_area_free(next); 843 844 /* 845 * In mprotect's case 6 (see comments on vma_merge), 846 * we must remove next_next too. 847 */ 848 if (remove_next == 2) { 849 remove_next = 1; 850 next = next_next; 851 goto again; 852 } 853 } 854 if (insert && file) 855 uprobe_mmap(insert); 856 857 mas_destroy(&mas); 858 validate_mm(mm); 859 860 return 0; 861 } 862 863 /* 864 * If the vma has a ->close operation then the driver probably needs to release 865 * per-vma resources, so we don't attempt to merge those. 866 */ 867 static inline int is_mergeable_vma(struct vm_area_struct *vma, 868 struct file *file, unsigned long vm_flags, 869 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 870 struct anon_vma_name *anon_name) 871 { 872 /* 873 * VM_SOFTDIRTY should not prevent from VMA merging, if we 874 * match the flags but dirty bit -- the caller should mark 875 * merged VMA as dirty. If dirty bit won't be excluded from 876 * comparison, we increase pressure on the memory system forcing 877 * the kernel to generate new VMAs when old one could be 878 * extended instead. 879 */ 880 if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 881 return 0; 882 if (vma->vm_file != file) 883 return 0; 884 if (vma->vm_ops && vma->vm_ops->close) 885 return 0; 886 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 887 return 0; 888 if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) 889 return 0; 890 return 1; 891 } 892 893 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1, 894 struct anon_vma *anon_vma2, 895 struct vm_area_struct *vma) 896 { 897 /* 898 * The list_is_singular() test is to avoid merging VMA cloned from 899 * parents. This can improve scalability caused by anon_vma lock. 900 */ 901 if ((!anon_vma1 || !anon_vma2) && (!vma || 902 list_is_singular(&vma->anon_vma_chain))) 903 return 1; 904 return anon_vma1 == anon_vma2; 905 } 906 907 /* 908 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 909 * in front of (at a lower virtual address and file offset than) the vma. 910 * 911 * We cannot merge two vmas if they have differently assigned (non-NULL) 912 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 913 * 914 * We don't check here for the merged mmap wrapping around the end of pagecache 915 * indices (16TB on ia32) because do_mmap() does not permit mmap's which 916 * wrap, nor mmaps which cover the final page at index -1UL. 917 */ 918 static int 919 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 920 struct anon_vma *anon_vma, struct file *file, 921 pgoff_t vm_pgoff, 922 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 923 struct anon_vma_name *anon_name) 924 { 925 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && 926 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 927 if (vma->vm_pgoff == vm_pgoff) 928 return 1; 929 } 930 return 0; 931 } 932 933 /* 934 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 935 * beyond (at a higher virtual address and file offset than) the vma. 936 * 937 * We cannot merge two vmas if they have differently assigned (non-NULL) 938 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 939 */ 940 static int 941 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 942 struct anon_vma *anon_vma, struct file *file, 943 pgoff_t vm_pgoff, 944 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 945 struct anon_vma_name *anon_name) 946 { 947 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && 948 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 949 pgoff_t vm_pglen; 950 vm_pglen = vma_pages(vma); 951 if (vma->vm_pgoff + vm_pglen == vm_pgoff) 952 return 1; 953 } 954 return 0; 955 } 956 957 /* 958 * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), 959 * figure out whether that can be merged with its predecessor or its 960 * successor. Or both (it neatly fills a hole). 961 * 962 * In most cases - when called for mmap, brk or mremap - [addr,end) is 963 * certain not to be mapped by the time vma_merge is called; but when 964 * called for mprotect, it is certain to be already mapped (either at 965 * an offset within prev, or at the start of next), and the flags of 966 * this area are about to be changed to vm_flags - and the no-change 967 * case has already been eliminated. 968 * 969 * The following mprotect cases have to be considered, where AAAA is 970 * the area passed down from mprotect_fixup, never extending beyond one 971 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after: 972 * 973 * AAAA AAAA AAAA 974 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN 975 * cannot merge might become might become 976 * PPNNNNNNNNNN PPPPPPPPPPNN 977 * mmap, brk or case 4 below case 5 below 978 * mremap move: 979 * AAAA AAAA 980 * PPPP NNNN PPPPNNNNXXXX 981 * might become might become 982 * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 983 * PPPPPPPPNNNN 2 or PPPPPPPPXXXX 7 or 984 * PPPPNNNNNNNN 3 PPPPXXXXXXXX 8 985 * 986 * It is important for case 8 that the vma NNNN overlapping the 987 * region AAAA is never going to extended over XXXX. Instead XXXX must 988 * be extended in region AAAA and NNNN must be removed. This way in 989 * all cases where vma_merge succeeds, the moment vma_adjust drops the 990 * rmap_locks, the properties of the merged vma will be already 991 * correct for the whole merged range. Some of those properties like 992 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 993 * be correct for the whole merged range immediately after the 994 * rmap_locks are released. Otherwise if XXXX would be removed and 995 * NNNN would be extended over the XXXX range, remove_migration_ptes 996 * or other rmap walkers (if working on addresses beyond the "end" 997 * parameter) may establish ptes with the wrong permissions of NNNN 998 * instead of the right permissions of XXXX. 999 */ 1000 struct vm_area_struct *vma_merge(struct mm_struct *mm, 1001 struct vm_area_struct *prev, unsigned long addr, 1002 unsigned long end, unsigned long vm_flags, 1003 struct anon_vma *anon_vma, struct file *file, 1004 pgoff_t pgoff, struct mempolicy *policy, 1005 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 1006 struct anon_vma_name *anon_name) 1007 { 1008 pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 1009 struct vm_area_struct *mid, *next, *res; 1010 int err = -1; 1011 bool merge_prev = false; 1012 bool merge_next = false; 1013 1014 /* 1015 * We later require that vma->vm_flags == vm_flags, 1016 * so this tests vma->vm_flags & VM_SPECIAL, too. 1017 */ 1018 if (vm_flags & VM_SPECIAL) 1019 return NULL; 1020 1021 next = find_vma(mm, prev ? prev->vm_end : 0); 1022 mid = next; 1023 if (next && next->vm_end == end) /* cases 6, 7, 8 */ 1024 next = find_vma(mm, next->vm_end); 1025 1026 /* verify some invariant that must be enforced by the caller */ 1027 VM_WARN_ON(prev && addr <= prev->vm_start); 1028 VM_WARN_ON(mid && end > mid->vm_end); 1029 VM_WARN_ON(addr >= end); 1030 1031 /* Can we merge the predecessor? */ 1032 if (prev && prev->vm_end == addr && 1033 mpol_equal(vma_policy(prev), policy) && 1034 can_vma_merge_after(prev, vm_flags, 1035 anon_vma, file, pgoff, 1036 vm_userfaultfd_ctx, anon_name)) { 1037 merge_prev = true; 1038 } 1039 /* Can we merge the successor? */ 1040 if (next && end == next->vm_start && 1041 mpol_equal(policy, vma_policy(next)) && 1042 can_vma_merge_before(next, vm_flags, 1043 anon_vma, file, pgoff+pglen, 1044 vm_userfaultfd_ctx, anon_name)) { 1045 merge_next = true; 1046 } 1047 /* Can we merge both the predecessor and the successor? */ 1048 if (merge_prev && merge_next && 1049 is_mergeable_anon_vma(prev->anon_vma, 1050 next->anon_vma, NULL)) { /* cases 1, 6 */ 1051 err = __vma_adjust(prev, prev->vm_start, 1052 next->vm_end, prev->vm_pgoff, NULL, 1053 prev); 1054 res = prev; 1055 } else if (merge_prev) { /* cases 2, 5, 7 */ 1056 err = __vma_adjust(prev, prev->vm_start, 1057 end, prev->vm_pgoff, NULL, prev); 1058 res = prev; 1059 } else if (merge_next) { 1060 if (prev && addr < prev->vm_end) /* case 4 */ 1061 err = __vma_adjust(prev, prev->vm_start, 1062 addr, prev->vm_pgoff, NULL, next); 1063 else /* cases 3, 8 */ 1064 err = __vma_adjust(mid, addr, next->vm_end, 1065 next->vm_pgoff - pglen, NULL, next); 1066 res = next; 1067 } 1068 1069 /* 1070 * Cannot merge with predecessor or successor or error in __vma_adjust? 1071 */ 1072 if (err) 1073 return NULL; 1074 khugepaged_enter_vma(res, vm_flags); 1075 return res; 1076 } 1077 1078 /* 1079 * Rough compatibility check to quickly see if it's even worth looking 1080 * at sharing an anon_vma. 1081 * 1082 * They need to have the same vm_file, and the flags can only differ 1083 * in things that mprotect may change. 1084 * 1085 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1086 * we can merge the two vma's. For example, we refuse to merge a vma if 1087 * there is a vm_ops->close() function, because that indicates that the 1088 * driver is doing some kind of reference counting. But that doesn't 1089 * really matter for the anon_vma sharing case. 1090 */ 1091 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1092 { 1093 return a->vm_end == b->vm_start && 1094 mpol_equal(vma_policy(a), vma_policy(b)) && 1095 a->vm_file == b->vm_file && 1096 !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && 1097 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1098 } 1099 1100 /* 1101 * Do some basic sanity checking to see if we can re-use the anon_vma 1102 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1103 * the same as 'old', the other will be the new one that is trying 1104 * to share the anon_vma. 1105 * 1106 * NOTE! This runs with mmap_lock held for reading, so it is possible that 1107 * the anon_vma of 'old' is concurrently in the process of being set up 1108 * by another page fault trying to merge _that_. But that's ok: if it 1109 * is being set up, that automatically means that it will be a singleton 1110 * acceptable for merging, so we can do all of this optimistically. But 1111 * we do that READ_ONCE() to make sure that we never re-load the pointer. 1112 * 1113 * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1114 * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1115 * is to return an anon_vma that is "complex" due to having gone through 1116 * a fork). 1117 * 1118 * We also make sure that the two vma's are compatible (adjacent, 1119 * and with the same memory policies). That's all stable, even with just 1120 * a read lock on the mmap_lock. 1121 */ 1122 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b) 1123 { 1124 if (anon_vma_compatible(a, b)) { 1125 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1126 1127 if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1128 return anon_vma; 1129 } 1130 return NULL; 1131 } 1132 1133 /* 1134 * find_mergeable_anon_vma is used by anon_vma_prepare, to check 1135 * neighbouring vmas for a suitable anon_vma, before it goes off 1136 * to allocate a new anon_vma. It checks because a repetitive 1137 * sequence of mprotects and faults may otherwise lead to distinct 1138 * anon_vmas being allocated, preventing vma merge in subsequent 1139 * mprotect. 1140 */ 1141 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 1142 { 1143 MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); 1144 struct anon_vma *anon_vma = NULL; 1145 struct vm_area_struct *prev, *next; 1146 1147 /* Try next first. */ 1148 next = mas_walk(&mas); 1149 if (next) { 1150 anon_vma = reusable_anon_vma(next, vma, next); 1151 if (anon_vma) 1152 return anon_vma; 1153 } 1154 1155 prev = mas_prev(&mas, 0); 1156 VM_BUG_ON_VMA(prev != vma, vma); 1157 prev = mas_prev(&mas, 0); 1158 /* Try prev next. */ 1159 if (prev) 1160 anon_vma = reusable_anon_vma(prev, prev, vma); 1161 1162 /* 1163 * We might reach here with anon_vma == NULL if we can't find 1164 * any reusable anon_vma. 1165 * There's no absolute need to look only at touching neighbours: 1166 * we could search further afield for "compatible" anon_vmas. 1167 * But it would probably just be a waste of time searching, 1168 * or lead to too many vmas hanging off the same anon_vma. 1169 * We're trying to allow mprotect remerging later on, 1170 * not trying to minimize memory used for anon_vmas. 1171 */ 1172 return anon_vma; 1173 } 1174 1175 /* 1176 * If a hint addr is less than mmap_min_addr change hint to be as 1177 * low as possible but still greater than mmap_min_addr 1178 */ 1179 static inline unsigned long round_hint_to_min(unsigned long hint) 1180 { 1181 hint &= PAGE_MASK; 1182 if (((void *)hint != NULL) && 1183 (hint < mmap_min_addr)) 1184 return PAGE_ALIGN(mmap_min_addr); 1185 return hint; 1186 } 1187 1188 int mlock_future_check(struct mm_struct *mm, unsigned long flags, 1189 unsigned long len) 1190 { 1191 unsigned long locked, lock_limit; 1192 1193 /* mlock MCL_FUTURE? */ 1194 if (flags & VM_LOCKED) { 1195 locked = len >> PAGE_SHIFT; 1196 locked += mm->locked_vm; 1197 lock_limit = rlimit(RLIMIT_MEMLOCK); 1198 lock_limit >>= PAGE_SHIFT; 1199 if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 1200 return -EAGAIN; 1201 } 1202 return 0; 1203 } 1204 1205 static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) 1206 { 1207 if (S_ISREG(inode->i_mode)) 1208 return MAX_LFS_FILESIZE; 1209 1210 if (S_ISBLK(inode->i_mode)) 1211 return MAX_LFS_FILESIZE; 1212 1213 if (S_ISSOCK(inode->i_mode)) 1214 return MAX_LFS_FILESIZE; 1215 1216 /* Special "we do even unsigned file positions" case */ 1217 if (file->f_mode & FMODE_UNSIGNED_OFFSET) 1218 return 0; 1219 1220 /* Yes, random drivers might want more. But I'm tired of buggy drivers */ 1221 return ULONG_MAX; 1222 } 1223 1224 static inline bool file_mmap_ok(struct file *file, struct inode *inode, 1225 unsigned long pgoff, unsigned long len) 1226 { 1227 u64 maxsize = file_mmap_size_max(file, inode); 1228 1229 if (maxsize && len > maxsize) 1230 return false; 1231 maxsize -= len; 1232 if (pgoff > maxsize >> PAGE_SHIFT) 1233 return false; 1234 return true; 1235 } 1236 1237 /* 1238 * The caller must write-lock current->mm->mmap_lock. 1239 */ 1240 unsigned long do_mmap(struct file *file, unsigned long addr, 1241 unsigned long len, unsigned long prot, 1242 unsigned long flags, unsigned long pgoff, 1243 unsigned long *populate, struct list_head *uf) 1244 { 1245 struct mm_struct *mm = current->mm; 1246 vm_flags_t vm_flags; 1247 int pkey = 0; 1248 1249 validate_mm(mm); 1250 *populate = 0; 1251 1252 if (!len) 1253 return -EINVAL; 1254 1255 /* 1256 * Does the application expect PROT_READ to imply PROT_EXEC? 1257 * 1258 * (the exception is when the underlying filesystem is noexec 1259 * mounted, in which case we dont add PROT_EXEC.) 1260 */ 1261 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 1262 if (!(file && path_noexec(&file->f_path))) 1263 prot |= PROT_EXEC; 1264 1265 /* force arch specific MAP_FIXED handling in get_unmapped_area */ 1266 if (flags & MAP_FIXED_NOREPLACE) 1267 flags |= MAP_FIXED; 1268 1269 if (!(flags & MAP_FIXED)) 1270 addr = round_hint_to_min(addr); 1271 1272 /* Careful about overflows.. */ 1273 len = PAGE_ALIGN(len); 1274 if (!len) 1275 return -ENOMEM; 1276 1277 /* offset overflow? */ 1278 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 1279 return -EOVERFLOW; 1280 1281 /* Too many mappings? */ 1282 if (mm->map_count > sysctl_max_map_count) 1283 return -ENOMEM; 1284 1285 /* Obtain the address to map to. we verify (or select) it and ensure 1286 * that it represents a valid section of the address space. 1287 */ 1288 addr = get_unmapped_area(file, addr, len, pgoff, flags); 1289 if (IS_ERR_VALUE(addr)) 1290 return addr; 1291 1292 if (flags & MAP_FIXED_NOREPLACE) { 1293 if (find_vma_intersection(mm, addr, addr + len)) 1294 return -EEXIST; 1295 } 1296 1297 if (prot == PROT_EXEC) { 1298 pkey = execute_only_pkey(mm); 1299 if (pkey < 0) 1300 pkey = 0; 1301 } 1302 1303 /* Do simple checking here so the lower-level routines won't have 1304 * to. we assume access permissions have been handled by the open 1305 * of the memory object, so we don't do any here. 1306 */ 1307 vm_flags = calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) | 1308 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 1309 1310 if (flags & MAP_LOCKED) 1311 if (!can_do_mlock()) 1312 return -EPERM; 1313 1314 if (mlock_future_check(mm, vm_flags, len)) 1315 return -EAGAIN; 1316 1317 if (file) { 1318 struct inode *inode = file_inode(file); 1319 unsigned long flags_mask; 1320 1321 if (!file_mmap_ok(file, inode, pgoff, len)) 1322 return -EOVERFLOW; 1323 1324 flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags; 1325 1326 switch (flags & MAP_TYPE) { 1327 case MAP_SHARED: 1328 /* 1329 * Force use of MAP_SHARED_VALIDATE with non-legacy 1330 * flags. E.g. MAP_SYNC is dangerous to use with 1331 * MAP_SHARED as you don't know which consistency model 1332 * you will get. We silently ignore unsupported flags 1333 * with MAP_SHARED to preserve backward compatibility. 1334 */ 1335 flags &= LEGACY_MAP_MASK; 1336 fallthrough; 1337 case MAP_SHARED_VALIDATE: 1338 if (flags & ~flags_mask) 1339 return -EOPNOTSUPP; 1340 if (prot & PROT_WRITE) { 1341 if (!(file->f_mode & FMODE_WRITE)) 1342 return -EACCES; 1343 if (IS_SWAPFILE(file->f_mapping->host)) 1344 return -ETXTBSY; 1345 } 1346 1347 /* 1348 * Make sure we don't allow writing to an append-only 1349 * file.. 1350 */ 1351 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) 1352 return -EACCES; 1353 1354 vm_flags |= VM_SHARED | VM_MAYSHARE; 1355 if (!(file->f_mode & FMODE_WRITE)) 1356 vm_flags &= ~(VM_MAYWRITE | VM_SHARED); 1357 fallthrough; 1358 case MAP_PRIVATE: 1359 if (!(file->f_mode & FMODE_READ)) 1360 return -EACCES; 1361 if (path_noexec(&file->f_path)) { 1362 if (vm_flags & VM_EXEC) 1363 return -EPERM; 1364 vm_flags &= ~VM_MAYEXEC; 1365 } 1366 1367 if (!file->f_op->mmap) 1368 return -ENODEV; 1369 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1370 return -EINVAL; 1371 break; 1372 1373 default: 1374 return -EINVAL; 1375 } 1376 } else { 1377 switch (flags & MAP_TYPE) { 1378 case MAP_SHARED: 1379 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1380 return -EINVAL; 1381 /* 1382 * Ignore pgoff. 1383 */ 1384 pgoff = 0; 1385 vm_flags |= VM_SHARED | VM_MAYSHARE; 1386 break; 1387 case MAP_PRIVATE: 1388 /* 1389 * Set pgoff according to addr for anon_vma. 1390 */ 1391 pgoff = addr >> PAGE_SHIFT; 1392 break; 1393 default: 1394 return -EINVAL; 1395 } 1396 } 1397 1398 /* 1399 * Set 'VM_NORESERVE' if we should not account for the 1400 * memory use of this mapping. 1401 */ 1402 if (flags & MAP_NORESERVE) { 1403 /* We honor MAP_NORESERVE if allowed to overcommit */ 1404 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) 1405 vm_flags |= VM_NORESERVE; 1406 1407 /* hugetlb applies strict overcommit unless MAP_NORESERVE */ 1408 if (file && is_file_hugepages(file)) 1409 vm_flags |= VM_NORESERVE; 1410 } 1411 1412 addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); 1413 if (!IS_ERR_VALUE(addr) && 1414 ((vm_flags & VM_LOCKED) || 1415 (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) 1416 *populate = len; 1417 return addr; 1418 } 1419 1420 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, 1421 unsigned long prot, unsigned long flags, 1422 unsigned long fd, unsigned long pgoff) 1423 { 1424 struct file *file = NULL; 1425 unsigned long retval; 1426 1427 if (!(flags & MAP_ANONYMOUS)) { 1428 audit_mmap_fd(fd, flags); 1429 file = fget(fd); 1430 if (!file) 1431 return -EBADF; 1432 if (is_file_hugepages(file)) { 1433 len = ALIGN(len, huge_page_size(hstate_file(file))); 1434 } else if (unlikely(flags & MAP_HUGETLB)) { 1435 retval = -EINVAL; 1436 goto out_fput; 1437 } 1438 } else if (flags & MAP_HUGETLB) { 1439 struct hstate *hs; 1440 1441 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1442 if (!hs) 1443 return -EINVAL; 1444 1445 len = ALIGN(len, huge_page_size(hs)); 1446 /* 1447 * VM_NORESERVE is used because the reservations will be 1448 * taken when vm_ops->mmap() is called 1449 */ 1450 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, 1451 VM_NORESERVE, 1452 HUGETLB_ANONHUGE_INODE, 1453 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1454 if (IS_ERR(file)) 1455 return PTR_ERR(file); 1456 } 1457 1458 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 1459 out_fput: 1460 if (file) 1461 fput(file); 1462 return retval; 1463 } 1464 1465 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 1466 unsigned long, prot, unsigned long, flags, 1467 unsigned long, fd, unsigned long, pgoff) 1468 { 1469 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); 1470 } 1471 1472 #ifdef __ARCH_WANT_SYS_OLD_MMAP 1473 struct mmap_arg_struct { 1474 unsigned long addr; 1475 unsigned long len; 1476 unsigned long prot; 1477 unsigned long flags; 1478 unsigned long fd; 1479 unsigned long offset; 1480 }; 1481 1482 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1483 { 1484 struct mmap_arg_struct a; 1485 1486 if (copy_from_user(&a, arg, sizeof(a))) 1487 return -EFAULT; 1488 if (offset_in_page(a.offset)) 1489 return -EINVAL; 1490 1491 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1492 a.offset >> PAGE_SHIFT); 1493 } 1494 #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1495 1496 /* 1497 * Some shared mappings will want the pages marked read-only 1498 * to track write events. If so, we'll downgrade vm_page_prot 1499 * to the private version (using protection_map[] without the 1500 * VM_SHARED bit). 1501 */ 1502 int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 1503 { 1504 vm_flags_t vm_flags = vma->vm_flags; 1505 const struct vm_operations_struct *vm_ops = vma->vm_ops; 1506 1507 /* If it was private or non-writable, the write bit is already clear */ 1508 if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED))) 1509 return 0; 1510 1511 /* The backer wishes to know when pages are first written to? */ 1512 if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite)) 1513 return 1; 1514 1515 /* The open routine did something to the protections that pgprot_modify 1516 * won't preserve? */ 1517 if (pgprot_val(vm_page_prot) != 1518 pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags))) 1519 return 0; 1520 1521 /* 1522 * Do we need to track softdirty? hugetlb does not support softdirty 1523 * tracking yet. 1524 */ 1525 if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) 1526 return 1; 1527 1528 /* Specialty mapping? */ 1529 if (vm_flags & VM_PFNMAP) 1530 return 0; 1531 1532 /* Can the mapping track the dirty pages? */ 1533 return vma->vm_file && vma->vm_file->f_mapping && 1534 mapping_can_writeback(vma->vm_file->f_mapping); 1535 } 1536 1537 /* 1538 * We account for memory if it's a private writeable mapping, 1539 * not hugepages and VM_NORESERVE wasn't set. 1540 */ 1541 static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) 1542 { 1543 /* 1544 * hugetlb has its own accounting separate from the core VM 1545 * VM_HUGETLB may not be set yet so we cannot check for that flag. 1546 */ 1547 if (file && is_file_hugepages(file)) 1548 return 0; 1549 1550 return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; 1551 } 1552 1553 /** 1554 * unmapped_area() - Find an area between the low_limit and the high_limit with 1555 * the correct alignment and offset, all from @info. Note: current->mm is used 1556 * for the search. 1557 * 1558 * @info: The unmapped area information including the range (low_limit - 1559 * hight_limit), the alignment offset and mask. 1560 * 1561 * Return: A memory address or -ENOMEM. 1562 */ 1563 static unsigned long unmapped_area(struct vm_unmapped_area_info *info) 1564 { 1565 unsigned long length, gap; 1566 1567 MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); 1568 1569 /* Adjust search length to account for worst case alignment overhead */ 1570 length = info->length + info->align_mask; 1571 if (length < info->length) 1572 return -ENOMEM; 1573 1574 if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, 1575 length)) 1576 return -ENOMEM; 1577 1578 gap = mas.index; 1579 gap += (info->align_offset - gap) & info->align_mask; 1580 return gap; 1581 } 1582 1583 /** 1584 * unmapped_area_topdown() - Find an area between the low_limit and the 1585 * high_limit with * the correct alignment and offset at the highest available 1586 * address, all from @info. Note: current->mm is used for the search. 1587 * 1588 * @info: The unmapped area information including the range (low_limit - 1589 * hight_limit), the alignment offset and mask. 1590 * 1591 * Return: A memory address or -ENOMEM. 1592 */ 1593 static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) 1594 { 1595 unsigned long length, gap; 1596 1597 MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); 1598 /* Adjust search length to account for worst case alignment overhead */ 1599 length = info->length + info->align_mask; 1600 if (length < info->length) 1601 return -ENOMEM; 1602 1603 if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, 1604 length)) 1605 return -ENOMEM; 1606 1607 gap = mas.last + 1 - info->length; 1608 gap -= (gap - info->align_offset) & info->align_mask; 1609 return gap; 1610 } 1611 1612 /* 1613 * Search for an unmapped address range. 1614 * 1615 * We are looking for a range that: 1616 * - does not intersect with any VMA; 1617 * - is contained within the [low_limit, high_limit) interval; 1618 * - is at least the desired size. 1619 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) 1620 */ 1621 unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) 1622 { 1623 unsigned long addr; 1624 1625 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) 1626 addr = unmapped_area_topdown(info); 1627 else 1628 addr = unmapped_area(info); 1629 1630 trace_vm_unmapped_area(addr, info); 1631 return addr; 1632 } 1633 1634 /* Get an address range which is currently unmapped. 1635 * For shmat() with addr=0. 1636 * 1637 * Ugly calling convention alert: 1638 * Return value with the low bits set means error value, 1639 * ie 1640 * if (ret & ~PAGE_MASK) 1641 * error = ret; 1642 * 1643 * This function "knows" that -ENOMEM has the bits set. 1644 */ 1645 unsigned long 1646 generic_get_unmapped_area(struct file *filp, unsigned long addr, 1647 unsigned long len, unsigned long pgoff, 1648 unsigned long flags) 1649 { 1650 struct mm_struct *mm = current->mm; 1651 struct vm_area_struct *vma, *prev; 1652 struct vm_unmapped_area_info info; 1653 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 1654 1655 if (len > mmap_end - mmap_min_addr) 1656 return -ENOMEM; 1657 1658 if (flags & MAP_FIXED) 1659 return addr; 1660 1661 if (addr) { 1662 addr = PAGE_ALIGN(addr); 1663 vma = find_vma_prev(mm, addr, &prev); 1664 if (mmap_end - len >= addr && addr >= mmap_min_addr && 1665 (!vma || addr + len <= vm_start_gap(vma)) && 1666 (!prev || addr >= vm_end_gap(prev))) 1667 return addr; 1668 } 1669 1670 info.flags = 0; 1671 info.length = len; 1672 info.low_limit = mm->mmap_base; 1673 info.high_limit = mmap_end; 1674 info.align_mask = 0; 1675 info.align_offset = 0; 1676 return vm_unmapped_area(&info); 1677 } 1678 1679 #ifndef HAVE_ARCH_UNMAPPED_AREA 1680 unsigned long 1681 arch_get_unmapped_area(struct file *filp, unsigned long addr, 1682 unsigned long len, unsigned long pgoff, 1683 unsigned long flags) 1684 { 1685 return generic_get_unmapped_area(filp, addr, len, pgoff, flags); 1686 } 1687 #endif 1688 1689 /* 1690 * This mmap-allocator allocates new areas top-down from below the 1691 * stack's low limit (the base): 1692 */ 1693 unsigned long 1694 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 1695 unsigned long len, unsigned long pgoff, 1696 unsigned long flags) 1697 { 1698 struct vm_area_struct *vma, *prev; 1699 struct mm_struct *mm = current->mm; 1700 struct vm_unmapped_area_info info; 1701 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 1702 1703 /* requested length too big for entire address space */ 1704 if (len > mmap_end - mmap_min_addr) 1705 return -ENOMEM; 1706 1707 if (flags & MAP_FIXED) 1708 return addr; 1709 1710 /* requesting a specific address */ 1711 if (addr) { 1712 addr = PAGE_ALIGN(addr); 1713 vma = find_vma_prev(mm, addr, &prev); 1714 if (mmap_end - len >= addr && addr >= mmap_min_addr && 1715 (!vma || addr + len <= vm_start_gap(vma)) && 1716 (!prev || addr >= vm_end_gap(prev))) 1717 return addr; 1718 } 1719 1720 info.flags = VM_UNMAPPED_AREA_TOPDOWN; 1721 info.length = len; 1722 info.low_limit = max(PAGE_SIZE, mmap_min_addr); 1723 info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); 1724 info.align_mask = 0; 1725 info.align_offset = 0; 1726 addr = vm_unmapped_area(&info); 1727 1728 /* 1729 * A failed mmap() very likely causes application failure, 1730 * so fall back to the bottom-up function here. This scenario 1731 * can happen with large stack limits and large mmap() 1732 * allocations. 1733 */ 1734 if (offset_in_page(addr)) { 1735 VM_BUG_ON(addr != -ENOMEM); 1736 info.flags = 0; 1737 info.low_limit = TASK_UNMAPPED_BASE; 1738 info.high_limit = mmap_end; 1739 addr = vm_unmapped_area(&info); 1740 } 1741 1742 return addr; 1743 } 1744 1745 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 1746 unsigned long 1747 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 1748 unsigned long len, unsigned long pgoff, 1749 unsigned long flags) 1750 { 1751 return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags); 1752 } 1753 #endif 1754 1755 unsigned long 1756 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 1757 unsigned long pgoff, unsigned long flags) 1758 { 1759 unsigned long (*get_area)(struct file *, unsigned long, 1760 unsigned long, unsigned long, unsigned long); 1761 1762 unsigned long error = arch_mmap_check(addr, len, flags); 1763 if (error) 1764 return error; 1765 1766 /* Careful about overflows.. */ 1767 if (len > TASK_SIZE) 1768 return -ENOMEM; 1769 1770 get_area = current->mm->get_unmapped_area; 1771 if (file) { 1772 if (file->f_op->get_unmapped_area) 1773 get_area = file->f_op->get_unmapped_area; 1774 } else if (flags & MAP_SHARED) { 1775 /* 1776 * mmap_region() will call shmem_zero_setup() to create a file, 1777 * so use shmem's get_unmapped_area in case it can be huge. 1778 * do_mmap() will clear pgoff, so match alignment. 1779 */ 1780 pgoff = 0; 1781 get_area = shmem_get_unmapped_area; 1782 } 1783 1784 addr = get_area(file, addr, len, pgoff, flags); 1785 if (IS_ERR_VALUE(addr)) 1786 return addr; 1787 1788 if (addr > TASK_SIZE - len) 1789 return -ENOMEM; 1790 if (offset_in_page(addr)) 1791 return -EINVAL; 1792 1793 error = security_mmap_addr(addr); 1794 return error ? error : addr; 1795 } 1796 1797 EXPORT_SYMBOL(get_unmapped_area); 1798 1799 /** 1800 * find_vma_intersection() - Look up the first VMA which intersects the interval 1801 * @mm: The process address space. 1802 * @start_addr: The inclusive start user address. 1803 * @end_addr: The exclusive end user address. 1804 * 1805 * Returns: The first VMA within the provided range, %NULL otherwise. Assumes 1806 * start_addr < end_addr. 1807 */ 1808 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 1809 unsigned long start_addr, 1810 unsigned long end_addr) 1811 { 1812 unsigned long index = start_addr; 1813 1814 mmap_assert_locked(mm); 1815 return mt_find(&mm->mm_mt, &index, end_addr - 1); 1816 } 1817 EXPORT_SYMBOL(find_vma_intersection); 1818 1819 /** 1820 * find_vma() - Find the VMA for a given address, or the next VMA. 1821 * @mm: The mm_struct to check 1822 * @addr: The address 1823 * 1824 * Returns: The VMA associated with addr, or the next VMA. 1825 * May return %NULL in the case of no VMA at addr or above. 1826 */ 1827 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 1828 { 1829 unsigned long index = addr; 1830 1831 mmap_assert_locked(mm); 1832 return mt_find(&mm->mm_mt, &index, ULONG_MAX); 1833 } 1834 EXPORT_SYMBOL(find_vma); 1835 1836 /** 1837 * find_vma_prev() - Find the VMA for a given address, or the next vma and 1838 * set %pprev to the previous VMA, if any. 1839 * @mm: The mm_struct to check 1840 * @addr: The address 1841 * @pprev: The pointer to set to the previous VMA 1842 * 1843 * Note that RCU lock is missing here since the external mmap_lock() is used 1844 * instead. 1845 * 1846 * Returns: The VMA associated with @addr, or the next vma. 1847 * May return %NULL in the case of no vma at addr or above. 1848 */ 1849 struct vm_area_struct * 1850 find_vma_prev(struct mm_struct *mm, unsigned long addr, 1851 struct vm_area_struct **pprev) 1852 { 1853 struct vm_area_struct *vma; 1854 MA_STATE(mas, &mm->mm_mt, addr, addr); 1855 1856 vma = mas_walk(&mas); 1857 *pprev = mas_prev(&mas, 0); 1858 if (!vma) 1859 vma = mas_next(&mas, ULONG_MAX); 1860 return vma; 1861 } 1862 1863 /* 1864 * Verify that the stack growth is acceptable and 1865 * update accounting. This is shared with both the 1866 * grow-up and grow-down cases. 1867 */ 1868 static int acct_stack_growth(struct vm_area_struct *vma, 1869 unsigned long size, unsigned long grow) 1870 { 1871 struct mm_struct *mm = vma->vm_mm; 1872 unsigned long new_start; 1873 1874 /* address space limit tests */ 1875 if (!may_expand_vm(mm, vma->vm_flags, grow)) 1876 return -ENOMEM; 1877 1878 /* Stack limit test */ 1879 if (size > rlimit(RLIMIT_STACK)) 1880 return -ENOMEM; 1881 1882 /* mlock limit tests */ 1883 if (mlock_future_check(mm, vma->vm_flags, grow << PAGE_SHIFT)) 1884 return -ENOMEM; 1885 1886 /* Check to ensure the stack will not grow into a hugetlb-only region */ 1887 new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start : 1888 vma->vm_end - size; 1889 if (is_hugepage_only_range(vma->vm_mm, new_start, size)) 1890 return -EFAULT; 1891 1892 /* 1893 * Overcommit.. This must be the final test, as it will 1894 * update security statistics. 1895 */ 1896 if (security_vm_enough_memory_mm(mm, grow)) 1897 return -ENOMEM; 1898 1899 return 0; 1900 } 1901 1902 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64) 1903 /* 1904 * PA-RISC uses this for its stack; IA64 for its Register Backing Store. 1905 * vma is the last one with address > vma->vm_end. Have to extend vma. 1906 */ 1907 int expand_upwards(struct vm_area_struct *vma, unsigned long address) 1908 { 1909 struct mm_struct *mm = vma->vm_mm; 1910 struct vm_area_struct *next; 1911 unsigned long gap_addr; 1912 int error = 0; 1913 MA_STATE(mas, &mm->mm_mt, 0, 0); 1914 1915 if (!(vma->vm_flags & VM_GROWSUP)) 1916 return -EFAULT; 1917 1918 /* Guard against exceeding limits of the address space. */ 1919 address &= PAGE_MASK; 1920 if (address >= (TASK_SIZE & PAGE_MASK)) 1921 return -ENOMEM; 1922 address += PAGE_SIZE; 1923 1924 /* Enforce stack_guard_gap */ 1925 gap_addr = address + stack_guard_gap; 1926 1927 /* Guard against overflow */ 1928 if (gap_addr < address || gap_addr > TASK_SIZE) 1929 gap_addr = TASK_SIZE; 1930 1931 next = find_vma_intersection(mm, vma->vm_end, gap_addr); 1932 if (next && vma_is_accessible(next)) { 1933 if (!(next->vm_flags & VM_GROWSUP)) 1934 return -ENOMEM; 1935 /* Check that both stack segments have the same anon_vma? */ 1936 } 1937 1938 if (mas_preallocate(&mas, vma, GFP_KERNEL)) 1939 return -ENOMEM; 1940 1941 /* We must make sure the anon_vma is allocated. */ 1942 if (unlikely(anon_vma_prepare(vma))) { 1943 mas_destroy(&mas); 1944 return -ENOMEM; 1945 } 1946 1947 /* 1948 * vma->vm_start/vm_end cannot change under us because the caller 1949 * is required to hold the mmap_lock in read mode. We need the 1950 * anon_vma lock to serialize against concurrent expand_stacks. 1951 */ 1952 anon_vma_lock_write(vma->anon_vma); 1953 1954 /* Somebody else might have raced and expanded it already */ 1955 if (address > vma->vm_end) { 1956 unsigned long size, grow; 1957 1958 size = address - vma->vm_start; 1959 grow = (address - vma->vm_end) >> PAGE_SHIFT; 1960 1961 error = -ENOMEM; 1962 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { 1963 error = acct_stack_growth(vma, size, grow); 1964 if (!error) { 1965 /* 1966 * We only hold a shared mmap_lock lock here, so 1967 * we need to protect against concurrent vma 1968 * expansions. anon_vma_lock_write() doesn't 1969 * help here, as we don't guarantee that all 1970 * growable vmas in a mm share the same root 1971 * anon vma. So, we reuse mm->page_table_lock 1972 * to guard against concurrent vma expansions. 1973 */ 1974 spin_lock(&mm->page_table_lock); 1975 if (vma->vm_flags & VM_LOCKED) 1976 mm->locked_vm += grow; 1977 vm_stat_account(mm, vma->vm_flags, grow); 1978 anon_vma_interval_tree_pre_update_vma(vma); 1979 vma->vm_end = address; 1980 /* Overwrite old entry in mtree. */ 1981 vma_mas_store(vma, &mas); 1982 anon_vma_interval_tree_post_update_vma(vma); 1983 spin_unlock(&mm->page_table_lock); 1984 1985 perf_event_mmap(vma); 1986 } 1987 } 1988 } 1989 anon_vma_unlock_write(vma->anon_vma); 1990 khugepaged_enter_vma(vma, vma->vm_flags); 1991 mas_destroy(&mas); 1992 return error; 1993 } 1994 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ 1995 1996 /* 1997 * vma is the first one with address < vma->vm_start. Have to extend vma. 1998 */ 1999 int expand_downwards(struct vm_area_struct *vma, unsigned long address) 2000 { 2001 struct mm_struct *mm = vma->vm_mm; 2002 MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); 2003 struct vm_area_struct *prev; 2004 int error = 0; 2005 2006 address &= PAGE_MASK; 2007 if (address < mmap_min_addr) 2008 return -EPERM; 2009 2010 /* Enforce stack_guard_gap */ 2011 prev = mas_prev(&mas, 0); 2012 /* Check that both stack segments have the same anon_vma? */ 2013 if (prev && !(prev->vm_flags & VM_GROWSDOWN) && 2014 vma_is_accessible(prev)) { 2015 if (address - prev->vm_end < stack_guard_gap) 2016 return -ENOMEM; 2017 } 2018 2019 if (mas_preallocate(&mas, vma, GFP_KERNEL)) 2020 return -ENOMEM; 2021 2022 /* We must make sure the anon_vma is allocated. */ 2023 if (unlikely(anon_vma_prepare(vma))) { 2024 mas_destroy(&mas); 2025 return -ENOMEM; 2026 } 2027 2028 /* 2029 * vma->vm_start/vm_end cannot change under us because the caller 2030 * is required to hold the mmap_lock in read mode. We need the 2031 * anon_vma lock to serialize against concurrent expand_stacks. 2032 */ 2033 anon_vma_lock_write(vma->anon_vma); 2034 2035 /* Somebody else might have raced and expanded it already */ 2036 if (address < vma->vm_start) { 2037 unsigned long size, grow; 2038 2039 size = vma->vm_end - address; 2040 grow = (vma->vm_start - address) >> PAGE_SHIFT; 2041 2042 error = -ENOMEM; 2043 if (grow <= vma->vm_pgoff) { 2044 error = acct_stack_growth(vma, size, grow); 2045 if (!error) { 2046 /* 2047 * We only hold a shared mmap_lock lock here, so 2048 * we need to protect against concurrent vma 2049 * expansions. anon_vma_lock_write() doesn't 2050 * help here, as we don't guarantee that all 2051 * growable vmas in a mm share the same root 2052 * anon vma. So, we reuse mm->page_table_lock 2053 * to guard against concurrent vma expansions. 2054 */ 2055 spin_lock(&mm->page_table_lock); 2056 if (vma->vm_flags & VM_LOCKED) 2057 mm->locked_vm += grow; 2058 vm_stat_account(mm, vma->vm_flags, grow); 2059 anon_vma_interval_tree_pre_update_vma(vma); 2060 vma->vm_start = address; 2061 vma->vm_pgoff -= grow; 2062 /* Overwrite old entry in mtree. */ 2063 vma_mas_store(vma, &mas); 2064 anon_vma_interval_tree_post_update_vma(vma); 2065 spin_unlock(&mm->page_table_lock); 2066 2067 perf_event_mmap(vma); 2068 } 2069 } 2070 } 2071 anon_vma_unlock_write(vma->anon_vma); 2072 khugepaged_enter_vma(vma, vma->vm_flags); 2073 mas_destroy(&mas); 2074 return error; 2075 } 2076 2077 /* enforced gap between the expanding stack and other mappings. */ 2078 unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT; 2079 2080 static int __init cmdline_parse_stack_guard_gap(char *p) 2081 { 2082 unsigned long val; 2083 char *endptr; 2084 2085 val = simple_strtoul(p, &endptr, 10); 2086 if (!*endptr) 2087 stack_guard_gap = val << PAGE_SHIFT; 2088 2089 return 1; 2090 } 2091 __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap); 2092 2093 #ifdef CONFIG_STACK_GROWSUP 2094 int expand_stack(struct vm_area_struct *vma, unsigned long address) 2095 { 2096 return expand_upwards(vma, address); 2097 } 2098 2099 struct vm_area_struct * 2100 find_extend_vma(struct mm_struct *mm, unsigned long addr) 2101 { 2102 struct vm_area_struct *vma, *prev; 2103 2104 addr &= PAGE_MASK; 2105 vma = find_vma_prev(mm, addr, &prev); 2106 if (vma && (vma->vm_start <= addr)) 2107 return vma; 2108 if (!prev || expand_stack(prev, addr)) 2109 return NULL; 2110 if (prev->vm_flags & VM_LOCKED) 2111 populate_vma_page_range(prev, addr, prev->vm_end, NULL); 2112 return prev; 2113 } 2114 #else 2115 int expand_stack(struct vm_area_struct *vma, unsigned long address) 2116 { 2117 return expand_downwards(vma, address); 2118 } 2119 2120 struct vm_area_struct * 2121 find_extend_vma(struct mm_struct *mm, unsigned long addr) 2122 { 2123 struct vm_area_struct *vma; 2124 unsigned long start; 2125 2126 addr &= PAGE_MASK; 2127 vma = find_vma(mm, addr); 2128 if (!vma) 2129 return NULL; 2130 if (vma->vm_start <= addr) 2131 return vma; 2132 if (!(vma->vm_flags & VM_GROWSDOWN)) 2133 return NULL; 2134 start = vma->vm_start; 2135 if (expand_stack(vma, addr)) 2136 return NULL; 2137 if (vma->vm_flags & VM_LOCKED) 2138 populate_vma_page_range(vma, addr, start, NULL); 2139 return vma; 2140 } 2141 #endif 2142 2143 EXPORT_SYMBOL_GPL(find_extend_vma); 2144 2145 /* 2146 * Ok - we have the memory areas we should free on a maple tree so release them, 2147 * and do the vma updates. 2148 * 2149 * Called with the mm semaphore held. 2150 */ 2151 static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) 2152 { 2153 unsigned long nr_accounted = 0; 2154 struct vm_area_struct *vma; 2155 2156 /* Update high watermark before we lower total_vm */ 2157 update_hiwater_vm(mm); 2158 mas_for_each(mas, vma, ULONG_MAX) { 2159 long nrpages = vma_pages(vma); 2160 2161 if (vma->vm_flags & VM_ACCOUNT) 2162 nr_accounted += nrpages; 2163 vm_stat_account(mm, vma->vm_flags, -nrpages); 2164 remove_vma(vma); 2165 } 2166 vm_unacct_memory(nr_accounted); 2167 validate_mm(mm); 2168 } 2169 2170 /* 2171 * Get rid of page table information in the indicated region. 2172 * 2173 * Called with the mm semaphore held. 2174 */ 2175 static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, 2176 struct vm_area_struct *vma, struct vm_area_struct *prev, 2177 struct vm_area_struct *next, 2178 unsigned long start, unsigned long end) 2179 { 2180 struct mmu_gather tlb; 2181 2182 lru_add_drain(); 2183 tlb_gather_mmu(&tlb, mm); 2184 update_hiwater_rss(mm); 2185 unmap_vmas(&tlb, mt, vma, start, end); 2186 free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 2187 next ? next->vm_start : USER_PGTABLES_CEILING); 2188 tlb_finish_mmu(&tlb); 2189 } 2190 2191 /* 2192 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 2193 * has already been checked or doesn't make sense to fail. 2194 */ 2195 int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 2196 unsigned long addr, int new_below) 2197 { 2198 struct vm_area_struct *new; 2199 int err; 2200 validate_mm_mt(mm); 2201 2202 if (vma->vm_ops && vma->vm_ops->may_split) { 2203 err = vma->vm_ops->may_split(vma, addr); 2204 if (err) 2205 return err; 2206 } 2207 2208 new = vm_area_dup(vma); 2209 if (!new) 2210 return -ENOMEM; 2211 2212 if (new_below) 2213 new->vm_end = addr; 2214 else { 2215 new->vm_start = addr; 2216 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 2217 } 2218 2219 err = vma_dup_policy(vma, new); 2220 if (err) 2221 goto out_free_vma; 2222 2223 err = anon_vma_clone(new, vma); 2224 if (err) 2225 goto out_free_mpol; 2226 2227 if (new->vm_file) 2228 get_file(new->vm_file); 2229 2230 if (new->vm_ops && new->vm_ops->open) 2231 new->vm_ops->open(new); 2232 2233 if (new_below) 2234 err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff + 2235 ((addr - new->vm_start) >> PAGE_SHIFT), new); 2236 else 2237 err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); 2238 2239 /* Success. */ 2240 if (!err) 2241 return 0; 2242 2243 /* Avoid vm accounting in close() operation */ 2244 new->vm_start = new->vm_end; 2245 new->vm_pgoff = 0; 2246 /* Clean everything up if vma_adjust failed. */ 2247 if (new->vm_ops && new->vm_ops->close) 2248 new->vm_ops->close(new); 2249 if (new->vm_file) 2250 fput(new->vm_file); 2251 unlink_anon_vmas(new); 2252 out_free_mpol: 2253 mpol_put(vma_policy(new)); 2254 out_free_vma: 2255 vm_area_free(new); 2256 validate_mm_mt(mm); 2257 return err; 2258 } 2259 2260 /* 2261 * Split a vma into two pieces at address 'addr', a new vma is allocated 2262 * either for the first part or the tail. 2263 */ 2264 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 2265 unsigned long addr, int new_below) 2266 { 2267 if (mm->map_count >= sysctl_max_map_count) 2268 return -ENOMEM; 2269 2270 return __split_vma(mm, vma, addr, new_below); 2271 } 2272 2273 static inline int munmap_sidetree(struct vm_area_struct *vma, 2274 struct ma_state *mas_detach) 2275 { 2276 mas_set_range(mas_detach, vma->vm_start, vma->vm_end - 1); 2277 if (mas_store_gfp(mas_detach, vma, GFP_KERNEL)) 2278 return -ENOMEM; 2279 2280 if (vma->vm_flags & VM_LOCKED) 2281 vma->vm_mm->locked_vm -= vma_pages(vma); 2282 2283 return 0; 2284 } 2285 2286 /* 2287 * do_mas_align_munmap() - munmap the aligned region from @start to @end. 2288 * @mas: The maple_state, ideally set up to alter the correct tree location. 2289 * @vma: The starting vm_area_struct 2290 * @mm: The mm_struct 2291 * @start: The aligned start address to munmap. 2292 * @end: The aligned end address to munmap. 2293 * @uf: The userfaultfd list_head 2294 * @downgrade: Set to true to attempt a write downgrade of the mmap_sem 2295 * 2296 * If @downgrade is true, check return code for potential release of the lock. 2297 */ 2298 static int 2299 do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, 2300 struct mm_struct *mm, unsigned long start, 2301 unsigned long end, struct list_head *uf, bool downgrade) 2302 { 2303 struct vm_area_struct *prev, *next = NULL; 2304 struct maple_tree mt_detach; 2305 int count = 0; 2306 int error = -ENOMEM; 2307 MA_STATE(mas_detach, &mt_detach, 0, 0); 2308 mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN); 2309 mt_set_external_lock(&mt_detach, &mm->mmap_lock); 2310 2311 if (mas_preallocate(mas, vma, GFP_KERNEL)) 2312 return -ENOMEM; 2313 2314 mas->last = end - 1; 2315 /* 2316 * If we need to split any vma, do it now to save pain later. 2317 * 2318 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 2319 * unmapped vm_area_struct will remain in use: so lower split_vma 2320 * places tmp vma above, and higher split_vma places tmp vma below. 2321 */ 2322 2323 /* Does it split the first one? */ 2324 if (start > vma->vm_start) { 2325 2326 /* 2327 * Make sure that map_count on return from munmap() will 2328 * not exceed its limit; but let map_count go just above 2329 * its limit temporarily, to help free resources as expected. 2330 */ 2331 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) 2332 goto map_count_exceeded; 2333 2334 /* 2335 * mas_pause() is not needed since mas->index needs to be set 2336 * differently than vma->vm_end anyways. 2337 */ 2338 error = __split_vma(mm, vma, start, 0); 2339 if (error) 2340 goto start_split_failed; 2341 2342 mas_set(mas, start); 2343 vma = mas_walk(mas); 2344 } 2345 2346 prev = mas_prev(mas, 0); 2347 if (unlikely((!prev))) 2348 mas_set(mas, start); 2349 2350 /* 2351 * Detach a range of VMAs from the mm. Using next as a temp variable as 2352 * it is always overwritten. 2353 */ 2354 mas_for_each(mas, next, end - 1) { 2355 /* Does it split the end? */ 2356 if (next->vm_end > end) { 2357 struct vm_area_struct *split; 2358 2359 error = __split_vma(mm, next, end, 1); 2360 if (error) 2361 goto end_split_failed; 2362 2363 mas_set(mas, end); 2364 split = mas_prev(mas, 0); 2365 error = munmap_sidetree(split, &mas_detach); 2366 if (error) 2367 goto munmap_sidetree_failed; 2368 2369 count++; 2370 if (vma == next) 2371 vma = split; 2372 break; 2373 } 2374 error = munmap_sidetree(next, &mas_detach); 2375 if (error) 2376 goto munmap_sidetree_failed; 2377 2378 count++; 2379 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 2380 BUG_ON(next->vm_start < start); 2381 BUG_ON(next->vm_start > end); 2382 #endif 2383 } 2384 2385 if (!next) 2386 next = mas_next(mas, ULONG_MAX); 2387 2388 if (unlikely(uf)) { 2389 /* 2390 * If userfaultfd_unmap_prep returns an error the vmas 2391 * will remain split, but userland will get a 2392 * highly unexpected error anyway. This is no 2393 * different than the case where the first of the two 2394 * __split_vma fails, but we don't undo the first 2395 * split, despite we could. This is unlikely enough 2396 * failure that it's not worth optimizing it for. 2397 */ 2398 error = userfaultfd_unmap_prep(mm, start, end, uf); 2399 2400 if (error) 2401 goto userfaultfd_error; 2402 } 2403 2404 /* Point of no return */ 2405 mas_set_range(mas, start, end - 1); 2406 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 2407 /* Make sure no VMAs are about to be lost. */ 2408 { 2409 MA_STATE(test, &mt_detach, start, end - 1); 2410 struct vm_area_struct *vma_mas, *vma_test; 2411 int test_count = 0; 2412 2413 rcu_read_lock(); 2414 vma_test = mas_find(&test, end - 1); 2415 mas_for_each(mas, vma_mas, end - 1) { 2416 BUG_ON(vma_mas != vma_test); 2417 test_count++; 2418 vma_test = mas_next(&test, end - 1); 2419 } 2420 rcu_read_unlock(); 2421 BUG_ON(count != test_count); 2422 mas_set_range(mas, start, end - 1); 2423 } 2424 #endif 2425 mas_store_prealloc(mas, NULL); 2426 mm->map_count -= count; 2427 /* 2428 * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or 2429 * VM_GROWSUP VMA. Such VMAs can change their size under 2430 * down_read(mmap_lock) and collide with the VMA we are about to unmap. 2431 */ 2432 if (downgrade) { 2433 if (next && (next->vm_flags & VM_GROWSDOWN)) 2434 downgrade = false; 2435 else if (prev && (prev->vm_flags & VM_GROWSUP)) 2436 downgrade = false; 2437 else 2438 mmap_write_downgrade(mm); 2439 } 2440 2441 unmap_region(mm, &mt_detach, vma, prev, next, start, end); 2442 /* Statistics and freeing VMAs */ 2443 mas_set(&mas_detach, start); 2444 remove_mt(mm, &mas_detach); 2445 __mt_destroy(&mt_detach); 2446 2447 2448 validate_mm(mm); 2449 return downgrade ? 1 : 0; 2450 2451 userfaultfd_error: 2452 munmap_sidetree_failed: 2453 end_split_failed: 2454 __mt_destroy(&mt_detach); 2455 start_split_failed: 2456 map_count_exceeded: 2457 mas_destroy(mas); 2458 return error; 2459 } 2460 2461 /* 2462 * do_mas_munmap() - munmap a given range. 2463 * @mas: The maple state 2464 * @mm: The mm_struct 2465 * @start: The start address to munmap 2466 * @len: The length of the range to munmap 2467 * @uf: The userfaultfd list_head 2468 * @downgrade: set to true if the user wants to attempt to write_downgrade the 2469 * mmap_sem 2470 * 2471 * This function takes a @mas that is either pointing to the previous VMA or set 2472 * to MA_START and sets it up to remove the mapping(s). The @len will be 2473 * aligned and any arch_unmap work will be preformed. 2474 * 2475 * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise. 2476 */ 2477 int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, 2478 unsigned long start, size_t len, struct list_head *uf, 2479 bool downgrade) 2480 { 2481 unsigned long end; 2482 struct vm_area_struct *vma; 2483 2484 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 2485 return -EINVAL; 2486 2487 end = start + PAGE_ALIGN(len); 2488 if (end == start) 2489 return -EINVAL; 2490 2491 /* arch_unmap() might do unmaps itself. */ 2492 arch_unmap(mm, start, end); 2493 2494 /* Find the first overlapping VMA */ 2495 vma = mas_find(mas, end - 1); 2496 if (!vma) 2497 return 0; 2498 2499 return do_mas_align_munmap(mas, vma, mm, start, end, uf, downgrade); 2500 } 2501 2502 /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. 2503 * @mm: The mm_struct 2504 * @start: The start address to munmap 2505 * @len: The length to be munmapped. 2506 * @uf: The userfaultfd list_head 2507 */ 2508 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 2509 struct list_head *uf) 2510 { 2511 MA_STATE(mas, &mm->mm_mt, start, start); 2512 2513 return do_mas_munmap(&mas, mm, start, len, uf, false); 2514 } 2515 2516 unsigned long mmap_region(struct file *file, unsigned long addr, 2517 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, 2518 struct list_head *uf) 2519 { 2520 struct mm_struct *mm = current->mm; 2521 struct vm_area_struct *vma = NULL; 2522 struct vm_area_struct *next, *prev, *merge; 2523 pgoff_t pglen = len >> PAGE_SHIFT; 2524 unsigned long charged = 0; 2525 unsigned long end = addr + len; 2526 unsigned long merge_start = addr, merge_end = end; 2527 pgoff_t vm_pgoff; 2528 int error; 2529 MA_STATE(mas, &mm->mm_mt, addr, end - 1); 2530 2531 /* Check against address space limit. */ 2532 if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { 2533 unsigned long nr_pages; 2534 2535 /* 2536 * MAP_FIXED may remove pages of mappings that intersects with 2537 * requested mapping. Account for the pages it would unmap. 2538 */ 2539 nr_pages = count_vma_pages_range(mm, addr, end); 2540 2541 if (!may_expand_vm(mm, vm_flags, 2542 (len >> PAGE_SHIFT) - nr_pages)) 2543 return -ENOMEM; 2544 } 2545 2546 /* Unmap any existing mapping in the area */ 2547 if (do_mas_munmap(&mas, mm, addr, len, uf, false)) 2548 return -ENOMEM; 2549 2550 /* 2551 * Private writable mapping: check memory availability 2552 */ 2553 if (accountable_mapping(file, vm_flags)) { 2554 charged = len >> PAGE_SHIFT; 2555 if (security_vm_enough_memory_mm(mm, charged)) 2556 return -ENOMEM; 2557 vm_flags |= VM_ACCOUNT; 2558 } 2559 2560 next = mas_next(&mas, ULONG_MAX); 2561 prev = mas_prev(&mas, 0); 2562 if (vm_flags & VM_SPECIAL) 2563 goto cannot_expand; 2564 2565 /* Attempt to expand an old mapping */ 2566 /* Check next */ 2567 if (next && next->vm_start == end && !vma_policy(next) && 2568 can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, 2569 NULL_VM_UFFD_CTX, NULL)) { 2570 merge_end = next->vm_end; 2571 vma = next; 2572 vm_pgoff = next->vm_pgoff - pglen; 2573 } 2574 2575 /* Check prev */ 2576 if (prev && prev->vm_end == addr && !vma_policy(prev) && 2577 (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, 2578 pgoff, vma->vm_userfaultfd_ctx, NULL) : 2579 can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, 2580 NULL_VM_UFFD_CTX, NULL))) { 2581 merge_start = prev->vm_start; 2582 vma = prev; 2583 vm_pgoff = prev->vm_pgoff; 2584 } 2585 2586 2587 /* Actually expand, if possible */ 2588 if (vma && 2589 !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) { 2590 khugepaged_enter_vma(vma, vm_flags); 2591 goto expanded; 2592 } 2593 2594 mas.index = addr; 2595 mas.last = end - 1; 2596 cannot_expand: 2597 /* 2598 * Determine the object being mapped and call the appropriate 2599 * specific mapper. the address has already been validated, but 2600 * not unmapped, but the maps are removed from the list. 2601 */ 2602 vma = vm_area_alloc(mm); 2603 if (!vma) { 2604 error = -ENOMEM; 2605 goto unacct_error; 2606 } 2607 2608 vma->vm_start = addr; 2609 vma->vm_end = end; 2610 vma->vm_flags = vm_flags; 2611 vma->vm_page_prot = vm_get_page_prot(vm_flags); 2612 vma->vm_pgoff = pgoff; 2613 2614 if (file) { 2615 if (vm_flags & VM_SHARED) { 2616 error = mapping_map_writable(file->f_mapping); 2617 if (error) 2618 goto free_vma; 2619 } 2620 2621 vma->vm_file = get_file(file); 2622 error = call_mmap(file, vma); 2623 if (error) 2624 goto unmap_and_free_vma; 2625 2626 /* 2627 * Expansion is handled above, merging is handled below. 2628 * Drivers should not alter the address of the VMA. 2629 */ 2630 if (WARN_ON((addr != vma->vm_start))) { 2631 error = -EINVAL; 2632 goto close_and_free_vma; 2633 } 2634 mas_reset(&mas); 2635 2636 /* 2637 * If vm_flags changed after call_mmap(), we should try merge 2638 * vma again as we may succeed this time. 2639 */ 2640 if (unlikely(vm_flags != vma->vm_flags && prev)) { 2641 merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, 2642 NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); 2643 if (merge) { 2644 /* 2645 * ->mmap() can change vma->vm_file and fput 2646 * the original file. So fput the vma->vm_file 2647 * here or we would add an extra fput for file 2648 * and cause general protection fault 2649 * ultimately. 2650 */ 2651 fput(vma->vm_file); 2652 vm_area_free(vma); 2653 vma = merge; 2654 /* Update vm_flags to pick up the change. */ 2655 vm_flags = vma->vm_flags; 2656 goto unmap_writable; 2657 } 2658 } 2659 2660 vm_flags = vma->vm_flags; 2661 } else if (vm_flags & VM_SHARED) { 2662 error = shmem_zero_setup(vma); 2663 if (error) 2664 goto free_vma; 2665 } else { 2666 vma_set_anonymous(vma); 2667 } 2668 2669 /* Allow architectures to sanity-check the vm_flags */ 2670 if (!arch_validate_flags(vma->vm_flags)) { 2671 error = -EINVAL; 2672 if (file) 2673 goto close_and_free_vma; 2674 else if (vma->vm_file) 2675 goto unmap_and_free_vma; 2676 else 2677 goto free_vma; 2678 } 2679 2680 if (mas_preallocate(&mas, vma, GFP_KERNEL)) { 2681 error = -ENOMEM; 2682 if (file) 2683 goto close_and_free_vma; 2684 else if (vma->vm_file) 2685 goto unmap_and_free_vma; 2686 else 2687 goto free_vma; 2688 } 2689 2690 if (vma->vm_file) 2691 i_mmap_lock_write(vma->vm_file->f_mapping); 2692 2693 vma_mas_store(vma, &mas); 2694 mm->map_count++; 2695 if (vma->vm_file) { 2696 if (vma->vm_flags & VM_SHARED) 2697 mapping_allow_writable(vma->vm_file->f_mapping); 2698 2699 flush_dcache_mmap_lock(vma->vm_file->f_mapping); 2700 vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap); 2701 flush_dcache_mmap_unlock(vma->vm_file->f_mapping); 2702 i_mmap_unlock_write(vma->vm_file->f_mapping); 2703 } 2704 2705 /* 2706 * vma_merge() calls khugepaged_enter_vma() either, the below 2707 * call covers the non-merge case. 2708 */ 2709 khugepaged_enter_vma(vma, vma->vm_flags); 2710 2711 /* Once vma denies write, undo our temporary denial count */ 2712 unmap_writable: 2713 if (file && vm_flags & VM_SHARED) 2714 mapping_unmap_writable(file->f_mapping); 2715 file = vma->vm_file; 2716 expanded: 2717 perf_event_mmap(vma); 2718 2719 vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); 2720 if (vm_flags & VM_LOCKED) { 2721 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || 2722 is_vm_hugetlb_page(vma) || 2723 vma == get_gate_vma(current->mm)) 2724 vma->vm_flags &= VM_LOCKED_CLEAR_MASK; 2725 else 2726 mm->locked_vm += (len >> PAGE_SHIFT); 2727 } 2728 2729 if (file) 2730 uprobe_mmap(vma); 2731 2732 /* 2733 * New (or expanded) vma always get soft dirty status. 2734 * Otherwise user-space soft-dirty page tracker won't 2735 * be able to distinguish situation when vma area unmapped, 2736 * then new mapped in-place (which must be aimed as 2737 * a completely new data area). 2738 */ 2739 vma->vm_flags |= VM_SOFTDIRTY; 2740 2741 vma_set_page_prot(vma); 2742 2743 validate_mm(mm); 2744 return addr; 2745 2746 close_and_free_vma: 2747 if (vma->vm_ops && vma->vm_ops->close) 2748 vma->vm_ops->close(vma); 2749 unmap_and_free_vma: 2750 fput(vma->vm_file); 2751 vma->vm_file = NULL; 2752 2753 /* Undo any partial mapping done by a device driver. */ 2754 unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end); 2755 if (file && (vm_flags & VM_SHARED)) 2756 mapping_unmap_writable(file->f_mapping); 2757 free_vma: 2758 vm_area_free(vma); 2759 unacct_error: 2760 if (charged) 2761 vm_unacct_memory(charged); 2762 validate_mm(mm); 2763 return error; 2764 } 2765 2766 static int __vm_munmap(unsigned long start, size_t len, bool downgrade) 2767 { 2768 int ret; 2769 struct mm_struct *mm = current->mm; 2770 LIST_HEAD(uf); 2771 MA_STATE(mas, &mm->mm_mt, start, start); 2772 2773 if (mmap_write_lock_killable(mm)) 2774 return -EINTR; 2775 2776 ret = do_mas_munmap(&mas, mm, start, len, &uf, downgrade); 2777 /* 2778 * Returning 1 indicates mmap_lock is downgraded. 2779 * But 1 is not legal return value of vm_munmap() and munmap(), reset 2780 * it to 0 before return. 2781 */ 2782 if (ret == 1) { 2783 mmap_read_unlock(mm); 2784 ret = 0; 2785 } else 2786 mmap_write_unlock(mm); 2787 2788 userfaultfd_unmap_complete(mm, &uf); 2789 return ret; 2790 } 2791 2792 int vm_munmap(unsigned long start, size_t len) 2793 { 2794 return __vm_munmap(start, len, false); 2795 } 2796 EXPORT_SYMBOL(vm_munmap); 2797 2798 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 2799 { 2800 addr = untagged_addr(addr); 2801 return __vm_munmap(addr, len, true); 2802 } 2803 2804 2805 /* 2806 * Emulation of deprecated remap_file_pages() syscall. 2807 */ 2808 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, 2809 unsigned long, prot, unsigned long, pgoff, unsigned long, flags) 2810 { 2811 2812 struct mm_struct *mm = current->mm; 2813 struct vm_area_struct *vma; 2814 unsigned long populate = 0; 2815 unsigned long ret = -EINVAL; 2816 struct file *file; 2817 2818 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n", 2819 current->comm, current->pid); 2820 2821 if (prot) 2822 return ret; 2823 start = start & PAGE_MASK; 2824 size = size & PAGE_MASK; 2825 2826 if (start + size <= start) 2827 return ret; 2828 2829 /* Does pgoff wrap? */ 2830 if (pgoff + (size >> PAGE_SHIFT) < pgoff) 2831 return ret; 2832 2833 if (mmap_write_lock_killable(mm)) 2834 return -EINTR; 2835 2836 vma = vma_lookup(mm, start); 2837 2838 if (!vma || !(vma->vm_flags & VM_SHARED)) 2839 goto out; 2840 2841 if (start + size > vma->vm_end) { 2842 VMA_ITERATOR(vmi, mm, vma->vm_end); 2843 struct vm_area_struct *next, *prev = vma; 2844 2845 for_each_vma_range(vmi, next, start + size) { 2846 /* hole between vmas ? */ 2847 if (next->vm_start != prev->vm_end) 2848 goto out; 2849 2850 if (next->vm_file != vma->vm_file) 2851 goto out; 2852 2853 if (next->vm_flags != vma->vm_flags) 2854 goto out; 2855 2856 if (start + size <= next->vm_end) 2857 break; 2858 2859 prev = next; 2860 } 2861 2862 if (!next) 2863 goto out; 2864 } 2865 2866 prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; 2867 prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; 2868 prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; 2869 2870 flags &= MAP_NONBLOCK; 2871 flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; 2872 if (vma->vm_flags & VM_LOCKED) 2873 flags |= MAP_LOCKED; 2874 2875 file = get_file(vma->vm_file); 2876 ret = do_mmap(vma->vm_file, start, size, 2877 prot, flags, pgoff, &populate, NULL); 2878 fput(file); 2879 out: 2880 mmap_write_unlock(mm); 2881 if (populate) 2882 mm_populate(ret, populate); 2883 if (!IS_ERR_VALUE(ret)) 2884 ret = 0; 2885 return ret; 2886 } 2887 2888 /* 2889 * brk_munmap() - Unmap a parital vma. 2890 * @mas: The maple tree state. 2891 * @vma: The vma to be modified 2892 * @newbrk: the start of the address to unmap 2893 * @oldbrk: The end of the address to unmap 2894 * @uf: The userfaultfd list_head 2895 * 2896 * Returns: 1 on success. 2897 * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if 2898 * possible. 2899 */ 2900 static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, 2901 unsigned long newbrk, unsigned long oldbrk, 2902 struct list_head *uf) 2903 { 2904 struct mm_struct *mm = vma->vm_mm; 2905 int ret; 2906 2907 arch_unmap(mm, newbrk, oldbrk); 2908 ret = do_mas_align_munmap(mas, vma, mm, newbrk, oldbrk, uf, true); 2909 validate_mm_mt(mm); 2910 return ret; 2911 } 2912 2913 /* 2914 * do_brk_flags() - Increase the brk vma if the flags match. 2915 * @mas: The maple tree state. 2916 * @addr: The start address 2917 * @len: The length of the increase 2918 * @vma: The vma, 2919 * @flags: The VMA Flags 2920 * 2921 * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags 2922 * do not match then create a new anonymous VMA. Eventually we may be able to 2923 * do some brk-specific accounting here. 2924 */ 2925 static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma, 2926 unsigned long addr, unsigned long len, unsigned long flags) 2927 { 2928 struct mm_struct *mm = current->mm; 2929 2930 validate_mm_mt(mm); 2931 /* 2932 * Check against address space limits by the changed size 2933 * Note: This happens *after* clearing old mappings in some code paths. 2934 */ 2935 flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; 2936 if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) 2937 return -ENOMEM; 2938 2939 if (mm->map_count > sysctl_max_map_count) 2940 return -ENOMEM; 2941 2942 if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) 2943 return -ENOMEM; 2944 2945 /* 2946 * Expand the existing vma if possible; Note that singular lists do not 2947 * occur after forking, so the expand will only happen on new VMAs. 2948 */ 2949 if (vma && 2950 (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) && 2951 ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) { 2952 mas_set_range(mas, vma->vm_start, addr + len - 1); 2953 if (mas_preallocate(mas, vma, GFP_KERNEL)) 2954 return -ENOMEM; 2955 2956 vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0); 2957 if (vma->anon_vma) { 2958 anon_vma_lock_write(vma->anon_vma); 2959 anon_vma_interval_tree_pre_update_vma(vma); 2960 } 2961 vma->vm_end = addr + len; 2962 vma->vm_flags |= VM_SOFTDIRTY; 2963 mas_store_prealloc(mas, vma); 2964 2965 if (vma->anon_vma) { 2966 anon_vma_interval_tree_post_update_vma(vma); 2967 anon_vma_unlock_write(vma->anon_vma); 2968 } 2969 khugepaged_enter_vma(vma, flags); 2970 goto out; 2971 } 2972 2973 /* create a vma struct for an anonymous mapping */ 2974 vma = vm_area_alloc(mm); 2975 if (!vma) 2976 goto vma_alloc_fail; 2977 2978 vma_set_anonymous(vma); 2979 vma->vm_start = addr; 2980 vma->vm_end = addr + len; 2981 vma->vm_pgoff = addr >> PAGE_SHIFT; 2982 vma->vm_flags = flags; 2983 vma->vm_page_prot = vm_get_page_prot(flags); 2984 mas_set_range(mas, vma->vm_start, addr + len - 1); 2985 if (mas_store_gfp(mas, vma, GFP_KERNEL)) 2986 goto mas_store_fail; 2987 2988 mm->map_count++; 2989 out: 2990 perf_event_mmap(vma); 2991 mm->total_vm += len >> PAGE_SHIFT; 2992 mm->data_vm += len >> PAGE_SHIFT; 2993 if (flags & VM_LOCKED) 2994 mm->locked_vm += (len >> PAGE_SHIFT); 2995 vma->vm_flags |= VM_SOFTDIRTY; 2996 validate_mm(mm); 2997 return 0; 2998 2999 mas_store_fail: 3000 vm_area_free(vma); 3001 vma_alloc_fail: 3002 vm_unacct_memory(len >> PAGE_SHIFT); 3003 return -ENOMEM; 3004 } 3005 3006 int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) 3007 { 3008 struct mm_struct *mm = current->mm; 3009 struct vm_area_struct *vma = NULL; 3010 unsigned long len; 3011 int ret; 3012 bool populate; 3013 LIST_HEAD(uf); 3014 MA_STATE(mas, &mm->mm_mt, addr, addr); 3015 3016 len = PAGE_ALIGN(request); 3017 if (len < request) 3018 return -ENOMEM; 3019 if (!len) 3020 return 0; 3021 3022 if (mmap_write_lock_killable(mm)) 3023 return -EINTR; 3024 3025 /* Until we need other flags, refuse anything except VM_EXEC. */ 3026 if ((flags & (~VM_EXEC)) != 0) 3027 return -EINVAL; 3028 3029 ret = check_brk_limits(addr, len); 3030 if (ret) 3031 goto limits_failed; 3032 3033 ret = do_mas_munmap(&mas, mm, addr, len, &uf, 0); 3034 if (ret) 3035 goto munmap_failed; 3036 3037 vma = mas_prev(&mas, 0); 3038 if (!vma || vma->vm_end != addr || vma_policy(vma) || 3039 !can_vma_merge_after(vma, flags, NULL, NULL, 3040 addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) 3041 vma = NULL; 3042 3043 ret = do_brk_flags(&mas, vma, addr, len, flags); 3044 populate = ((mm->def_flags & VM_LOCKED) != 0); 3045 mmap_write_unlock(mm); 3046 userfaultfd_unmap_complete(mm, &uf); 3047 if (populate && !ret) 3048 mm_populate(addr, len); 3049 return ret; 3050 3051 munmap_failed: 3052 limits_failed: 3053 mmap_write_unlock(mm); 3054 return ret; 3055 } 3056 EXPORT_SYMBOL(vm_brk_flags); 3057 3058 int vm_brk(unsigned long addr, unsigned long len) 3059 { 3060 return vm_brk_flags(addr, len, 0); 3061 } 3062 EXPORT_SYMBOL(vm_brk); 3063 3064 /* Release all mmaps. */ 3065 void exit_mmap(struct mm_struct *mm) 3066 { 3067 struct mmu_gather tlb; 3068 struct vm_area_struct *vma; 3069 unsigned long nr_accounted = 0; 3070 MA_STATE(mas, &mm->mm_mt, 0, 0); 3071 int count = 0; 3072 3073 /* mm's last user has gone, and its about to be pulled down */ 3074 mmu_notifier_release(mm); 3075 3076 mmap_read_lock(mm); 3077 arch_exit_mmap(mm); 3078 3079 vma = mas_find(&mas, ULONG_MAX); 3080 if (!vma) { 3081 /* Can happen if dup_mmap() received an OOM */ 3082 mmap_read_unlock(mm); 3083 return; 3084 } 3085 3086 lru_add_drain(); 3087 flush_cache_mm(mm); 3088 tlb_gather_mmu_fullmm(&tlb, mm); 3089 /* update_hiwater_rss(mm) here? but nobody should be looking */ 3090 /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ 3091 unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX); 3092 mmap_read_unlock(mm); 3093 3094 /* 3095 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper 3096 * because the memory has been already freed. 3097 */ 3098 set_bit(MMF_OOM_SKIP, &mm->flags); 3099 mmap_write_lock(mm); 3100 free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS, 3101 USER_PGTABLES_CEILING); 3102 tlb_finish_mmu(&tlb); 3103 3104 /* 3105 * Walk the list again, actually closing and freeing it, with preemption 3106 * enabled, without holding any MM locks besides the unreachable 3107 * mmap_write_lock. 3108 */ 3109 do { 3110 if (vma->vm_flags & VM_ACCOUNT) 3111 nr_accounted += vma_pages(vma); 3112 remove_vma(vma); 3113 count++; 3114 cond_resched(); 3115 } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); 3116 3117 BUG_ON(count != mm->map_count); 3118 3119 trace_exit_mmap(mm); 3120 __mt_destroy(&mm->mm_mt); 3121 mmap_write_unlock(mm); 3122 vm_unacct_memory(nr_accounted); 3123 } 3124 3125 /* Insert vm structure into process list sorted by address 3126 * and into the inode's i_mmap tree. If vm_file is non-NULL 3127 * then i_mmap_rwsem is taken here. 3128 */ 3129 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) 3130 { 3131 unsigned long charged = vma_pages(vma); 3132 3133 3134 if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) 3135 return -ENOMEM; 3136 3137 if ((vma->vm_flags & VM_ACCOUNT) && 3138 security_vm_enough_memory_mm(mm, charged)) 3139 return -ENOMEM; 3140 3141 /* 3142 * The vm_pgoff of a purely anonymous vma should be irrelevant 3143 * until its first write fault, when page's anon_vma and index 3144 * are set. But now set the vm_pgoff it will almost certainly 3145 * end up with (unless mremap moves it elsewhere before that 3146 * first wfault), so /proc/pid/maps tells a consistent story. 3147 * 3148 * By setting it to reflect the virtual start address of the 3149 * vma, merges and splits can happen in a seamless way, just 3150 * using the existing file pgoff checks and manipulations. 3151 * Similarly in do_mmap and in do_brk_flags. 3152 */ 3153 if (vma_is_anonymous(vma)) { 3154 BUG_ON(vma->anon_vma); 3155 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; 3156 } 3157 3158 if (vma_link(mm, vma)) { 3159 vm_unacct_memory(charged); 3160 return -ENOMEM; 3161 } 3162 3163 return 0; 3164 } 3165 3166 /* 3167 * Copy the vma structure to a new location in the same mm, 3168 * prior to moving page table entries, to effect an mremap move. 3169 */ 3170 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 3171 unsigned long addr, unsigned long len, pgoff_t pgoff, 3172 bool *need_rmap_locks) 3173 { 3174 struct vm_area_struct *vma = *vmap; 3175 unsigned long vma_start = vma->vm_start; 3176 struct mm_struct *mm = vma->vm_mm; 3177 struct vm_area_struct *new_vma, *prev; 3178 bool faulted_in_anon_vma = true; 3179 3180 validate_mm_mt(mm); 3181 /* 3182 * If anonymous vma has not yet been faulted, update new pgoff 3183 * to match new location, to increase its chance of merging. 3184 */ 3185 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 3186 pgoff = addr >> PAGE_SHIFT; 3187 faulted_in_anon_vma = false; 3188 } 3189 3190 new_vma = find_vma_prev(mm, addr, &prev); 3191 if (new_vma && new_vma->vm_start < addr + len) 3192 return NULL; /* should never get here */ 3193 3194 new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, 3195 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 3196 vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 3197 if (new_vma) { 3198 /* 3199 * Source vma may have been merged into new_vma 3200 */ 3201 if (unlikely(vma_start >= new_vma->vm_start && 3202 vma_start < new_vma->vm_end)) { 3203 /* 3204 * The only way we can get a vma_merge with 3205 * self during an mremap is if the vma hasn't 3206 * been faulted in yet and we were allowed to 3207 * reset the dst vma->vm_pgoff to the 3208 * destination address of the mremap to allow 3209 * the merge to happen. mremap must change the 3210 * vm_pgoff linearity between src and dst vmas 3211 * (in turn preventing a vma_merge) to be 3212 * safe. It is only safe to keep the vm_pgoff 3213 * linear if there are no pages mapped yet. 3214 */ 3215 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 3216 *vmap = vma = new_vma; 3217 } 3218 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 3219 } else { 3220 new_vma = vm_area_dup(vma); 3221 if (!new_vma) 3222 goto out; 3223 new_vma->vm_start = addr; 3224 new_vma->vm_end = addr + len; 3225 new_vma->vm_pgoff = pgoff; 3226 if (vma_dup_policy(vma, new_vma)) 3227 goto out_free_vma; 3228 if (anon_vma_clone(new_vma, vma)) 3229 goto out_free_mempol; 3230 if (new_vma->vm_file) 3231 get_file(new_vma->vm_file); 3232 if (new_vma->vm_ops && new_vma->vm_ops->open) 3233 new_vma->vm_ops->open(new_vma); 3234 if (vma_link(mm, new_vma)) 3235 goto out_vma_link; 3236 *need_rmap_locks = false; 3237 } 3238 validate_mm_mt(mm); 3239 return new_vma; 3240 3241 out_vma_link: 3242 if (new_vma->vm_ops && new_vma->vm_ops->close) 3243 new_vma->vm_ops->close(new_vma); 3244 3245 if (new_vma->vm_file) 3246 fput(new_vma->vm_file); 3247 3248 unlink_anon_vmas(new_vma); 3249 out_free_mempol: 3250 mpol_put(vma_policy(new_vma)); 3251 out_free_vma: 3252 vm_area_free(new_vma); 3253 out: 3254 validate_mm_mt(mm); 3255 return NULL; 3256 } 3257 3258 /* 3259 * Return true if the calling process may expand its vm space by the passed 3260 * number of pages 3261 */ 3262 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) 3263 { 3264 if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) 3265 return false; 3266 3267 if (is_data_mapping(flags) && 3268 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { 3269 /* Workaround for Valgrind */ 3270 if (rlimit(RLIMIT_DATA) == 0 && 3271 mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) 3272 return true; 3273 3274 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", 3275 current->comm, current->pid, 3276 (mm->data_vm + npages) << PAGE_SHIFT, 3277 rlimit(RLIMIT_DATA), 3278 ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); 3279 3280 if (!ignore_rlimit_data) 3281 return false; 3282 } 3283 3284 return true; 3285 } 3286 3287 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) 3288 { 3289 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages); 3290 3291 if (is_exec_mapping(flags)) 3292 mm->exec_vm += npages; 3293 else if (is_stack_mapping(flags)) 3294 mm->stack_vm += npages; 3295 else if (is_data_mapping(flags)) 3296 mm->data_vm += npages; 3297 } 3298 3299 static vm_fault_t special_mapping_fault(struct vm_fault *vmf); 3300 3301 /* 3302 * Having a close hook prevents vma merging regardless of flags. 3303 */ 3304 static void special_mapping_close(struct vm_area_struct *vma) 3305 { 3306 } 3307 3308 static const char *special_mapping_name(struct vm_area_struct *vma) 3309 { 3310 return ((struct vm_special_mapping *)vma->vm_private_data)->name; 3311 } 3312 3313 static int special_mapping_mremap(struct vm_area_struct *new_vma) 3314 { 3315 struct vm_special_mapping *sm = new_vma->vm_private_data; 3316 3317 if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) 3318 return -EFAULT; 3319 3320 if (sm->mremap) 3321 return sm->mremap(sm, new_vma); 3322 3323 return 0; 3324 } 3325 3326 static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) 3327 { 3328 /* 3329 * Forbid splitting special mappings - kernel has expectations over 3330 * the number of pages in mapping. Together with VM_DONTEXPAND 3331 * the size of vma should stay the same over the special mapping's 3332 * lifetime. 3333 */ 3334 return -EINVAL; 3335 } 3336 3337 static const struct vm_operations_struct special_mapping_vmops = { 3338 .close = special_mapping_close, 3339 .fault = special_mapping_fault, 3340 .mremap = special_mapping_mremap, 3341 .name = special_mapping_name, 3342 /* vDSO code relies that VVAR can't be accessed remotely */ 3343 .access = NULL, 3344 .may_split = special_mapping_split, 3345 }; 3346 3347 static const struct vm_operations_struct legacy_special_mapping_vmops = { 3348 .close = special_mapping_close, 3349 .fault = special_mapping_fault, 3350 }; 3351 3352 static vm_fault_t special_mapping_fault(struct vm_fault *vmf) 3353 { 3354 struct vm_area_struct *vma = vmf->vma; 3355 pgoff_t pgoff; 3356 struct page **pages; 3357 3358 if (vma->vm_ops == &legacy_special_mapping_vmops) { 3359 pages = vma->vm_private_data; 3360 } else { 3361 struct vm_special_mapping *sm = vma->vm_private_data; 3362 3363 if (sm->fault) 3364 return sm->fault(sm, vmf->vma, vmf); 3365 3366 pages = sm->pages; 3367 } 3368 3369 for (pgoff = vmf->pgoff; pgoff && *pages; ++pages) 3370 pgoff--; 3371 3372 if (*pages) { 3373 struct page *page = *pages; 3374 get_page(page); 3375 vmf->page = page; 3376 return 0; 3377 } 3378 3379 return VM_FAULT_SIGBUS; 3380 } 3381 3382 static struct vm_area_struct *__install_special_mapping( 3383 struct mm_struct *mm, 3384 unsigned long addr, unsigned long len, 3385 unsigned long vm_flags, void *priv, 3386 const struct vm_operations_struct *ops) 3387 { 3388 int ret; 3389 struct vm_area_struct *vma; 3390 3391 validate_mm_mt(mm); 3392 vma = vm_area_alloc(mm); 3393 if (unlikely(vma == NULL)) 3394 return ERR_PTR(-ENOMEM); 3395 3396 vma->vm_start = addr; 3397 vma->vm_end = addr + len; 3398 3399 vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY; 3400 vma->vm_flags &= VM_LOCKED_CLEAR_MASK; 3401 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 3402 3403 vma->vm_ops = ops; 3404 vma->vm_private_data = priv; 3405 3406 ret = insert_vm_struct(mm, vma); 3407 if (ret) 3408 goto out; 3409 3410 vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); 3411 3412 perf_event_mmap(vma); 3413 3414 validate_mm_mt(mm); 3415 return vma; 3416 3417 out: 3418 vm_area_free(vma); 3419 validate_mm_mt(mm); 3420 return ERR_PTR(ret); 3421 } 3422 3423 bool vma_is_special_mapping(const struct vm_area_struct *vma, 3424 const struct vm_special_mapping *sm) 3425 { 3426 return vma->vm_private_data == sm && 3427 (vma->vm_ops == &special_mapping_vmops || 3428 vma->vm_ops == &legacy_special_mapping_vmops); 3429 } 3430 3431 /* 3432 * Called with mm->mmap_lock held for writing. 3433 * Insert a new vma covering the given region, with the given flags. 3434 * Its pages are supplied by the given array of struct page *. 3435 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. 3436 * The region past the last page supplied will always produce SIGBUS. 3437 * The array pointer and the pages it points to are assumed to stay alive 3438 * for as long as this mapping might exist. 3439 */ 3440 struct vm_area_struct *_install_special_mapping( 3441 struct mm_struct *mm, 3442 unsigned long addr, unsigned long len, 3443 unsigned long vm_flags, const struct vm_special_mapping *spec) 3444 { 3445 return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec, 3446 &special_mapping_vmops); 3447 } 3448 3449 int install_special_mapping(struct mm_struct *mm, 3450 unsigned long addr, unsigned long len, 3451 unsigned long vm_flags, struct page **pages) 3452 { 3453 struct vm_area_struct *vma = __install_special_mapping( 3454 mm, addr, len, vm_flags, (void *)pages, 3455 &legacy_special_mapping_vmops); 3456 3457 return PTR_ERR_OR_ZERO(vma); 3458 } 3459 3460 static DEFINE_MUTEX(mm_all_locks_mutex); 3461 3462 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 3463 { 3464 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 3465 /* 3466 * The LSB of head.next can't change from under us 3467 * because we hold the mm_all_locks_mutex. 3468 */ 3469 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); 3470 /* 3471 * We can safely modify head.next after taking the 3472 * anon_vma->root->rwsem. If some other vma in this mm shares 3473 * the same anon_vma we won't take it again. 3474 * 3475 * No need of atomic instructions here, head.next 3476 * can't change from under us thanks to the 3477 * anon_vma->root->rwsem. 3478 */ 3479 if (__test_and_set_bit(0, (unsigned long *) 3480 &anon_vma->root->rb_root.rb_root.rb_node)) 3481 BUG(); 3482 } 3483 } 3484 3485 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 3486 { 3487 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 3488 /* 3489 * AS_MM_ALL_LOCKS can't change from under us because 3490 * we hold the mm_all_locks_mutex. 3491 * 3492 * Operations on ->flags have to be atomic because 3493 * even if AS_MM_ALL_LOCKS is stable thanks to the 3494 * mm_all_locks_mutex, there may be other cpus 3495 * changing other bitflags in parallel to us. 3496 */ 3497 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 3498 BUG(); 3499 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); 3500 } 3501 } 3502 3503 /* 3504 * This operation locks against the VM for all pte/vma/mm related 3505 * operations that could ever happen on a certain mm. This includes 3506 * vmtruncate, try_to_unmap, and all page faults. 3507 * 3508 * The caller must take the mmap_lock in write mode before calling 3509 * mm_take_all_locks(). The caller isn't allowed to release the 3510 * mmap_lock until mm_drop_all_locks() returns. 3511 * 3512 * mmap_lock in write mode is required in order to block all operations 3513 * that could modify pagetables and free pages without need of 3514 * altering the vma layout. It's also needed in write mode to avoid new 3515 * anon_vmas to be associated with existing vmas. 3516 * 3517 * A single task can't take more than one mm_take_all_locks() in a row 3518 * or it would deadlock. 3519 * 3520 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 3521 * mapping->flags avoid to take the same lock twice, if more than one 3522 * vma in this mm is backed by the same anon_vma or address_space. 3523 * 3524 * We take locks in following order, accordingly to comment at beginning 3525 * of mm/rmap.c: 3526 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 3527 * hugetlb mapping); 3528 * - all i_mmap_rwsem locks; 3529 * - all anon_vma->rwseml 3530 * 3531 * We can take all locks within these types randomly because the VM code 3532 * doesn't nest them and we protected from parallel mm_take_all_locks() by 3533 * mm_all_locks_mutex. 3534 * 3535 * mm_take_all_locks() and mm_drop_all_locks are expensive operations 3536 * that may have to take thousand of locks. 3537 * 3538 * mm_take_all_locks() can fail if it's interrupted by signals. 3539 */ 3540 int mm_take_all_locks(struct mm_struct *mm) 3541 { 3542 struct vm_area_struct *vma; 3543 struct anon_vma_chain *avc; 3544 MA_STATE(mas, &mm->mm_mt, 0, 0); 3545 3546 mmap_assert_write_locked(mm); 3547 3548 mutex_lock(&mm_all_locks_mutex); 3549 3550 mas_for_each(&mas, vma, ULONG_MAX) { 3551 if (signal_pending(current)) 3552 goto out_unlock; 3553 if (vma->vm_file && vma->vm_file->f_mapping && 3554 is_vm_hugetlb_page(vma)) 3555 vm_lock_mapping(mm, vma->vm_file->f_mapping); 3556 } 3557 3558 mas_set(&mas, 0); 3559 mas_for_each(&mas, vma, ULONG_MAX) { 3560 if (signal_pending(current)) 3561 goto out_unlock; 3562 if (vma->vm_file && vma->vm_file->f_mapping && 3563 !is_vm_hugetlb_page(vma)) 3564 vm_lock_mapping(mm, vma->vm_file->f_mapping); 3565 } 3566 3567 mas_set(&mas, 0); 3568 mas_for_each(&mas, vma, ULONG_MAX) { 3569 if (signal_pending(current)) 3570 goto out_unlock; 3571 if (vma->anon_vma) 3572 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 3573 vm_lock_anon_vma(mm, avc->anon_vma); 3574 } 3575 3576 return 0; 3577 3578 out_unlock: 3579 mm_drop_all_locks(mm); 3580 return -EINTR; 3581 } 3582 3583 static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 3584 { 3585 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 3586 /* 3587 * The LSB of head.next can't change to 0 from under 3588 * us because we hold the mm_all_locks_mutex. 3589 * 3590 * We must however clear the bitflag before unlocking 3591 * the vma so the users using the anon_vma->rb_root will 3592 * never see our bitflag. 3593 * 3594 * No need of atomic instructions here, head.next 3595 * can't change from under us until we release the 3596 * anon_vma->root->rwsem. 3597 */ 3598 if (!__test_and_clear_bit(0, (unsigned long *) 3599 &anon_vma->root->rb_root.rb_root.rb_node)) 3600 BUG(); 3601 anon_vma_unlock_write(anon_vma); 3602 } 3603 } 3604 3605 static void vm_unlock_mapping(struct address_space *mapping) 3606 { 3607 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 3608 /* 3609 * AS_MM_ALL_LOCKS can't change to 0 from under us 3610 * because we hold the mm_all_locks_mutex. 3611 */ 3612 i_mmap_unlock_write(mapping); 3613 if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 3614 &mapping->flags)) 3615 BUG(); 3616 } 3617 } 3618 3619 /* 3620 * The mmap_lock cannot be released by the caller until 3621 * mm_drop_all_locks() returns. 3622 */ 3623 void mm_drop_all_locks(struct mm_struct *mm) 3624 { 3625 struct vm_area_struct *vma; 3626 struct anon_vma_chain *avc; 3627 MA_STATE(mas, &mm->mm_mt, 0, 0); 3628 3629 mmap_assert_write_locked(mm); 3630 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 3631 3632 mas_for_each(&mas, vma, ULONG_MAX) { 3633 if (vma->anon_vma) 3634 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 3635 vm_unlock_anon_vma(avc->anon_vma); 3636 if (vma->vm_file && vma->vm_file->f_mapping) 3637 vm_unlock_mapping(vma->vm_file->f_mapping); 3638 } 3639 3640 mutex_unlock(&mm_all_locks_mutex); 3641 } 3642 3643 /* 3644 * initialise the percpu counter for VM 3645 */ 3646 void __init mmap_init(void) 3647 { 3648 int ret; 3649 3650 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); 3651 VM_BUG_ON(ret); 3652 } 3653 3654 /* 3655 * Initialise sysctl_user_reserve_kbytes. 3656 * 3657 * This is intended to prevent a user from starting a single memory hogging 3658 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER 3659 * mode. 3660 * 3661 * The default value is min(3% of free memory, 128MB) 3662 * 128MB is enough to recover with sshd/login, bash, and top/kill. 3663 */ 3664 static int init_user_reserve(void) 3665 { 3666 unsigned long free_kbytes; 3667 3668 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3669 3670 sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17); 3671 return 0; 3672 } 3673 subsys_initcall(init_user_reserve); 3674 3675 /* 3676 * Initialise sysctl_admin_reserve_kbytes. 3677 * 3678 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin 3679 * to log in and kill a memory hogging process. 3680 * 3681 * Systems with more than 256MB will reserve 8MB, enough to recover 3682 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will 3683 * only reserve 3% of free pages by default. 3684 */ 3685 static int init_admin_reserve(void) 3686 { 3687 unsigned long free_kbytes; 3688 3689 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3690 3691 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13); 3692 return 0; 3693 } 3694 subsys_initcall(init_admin_reserve); 3695 3696 /* 3697 * Reinititalise user and admin reserves if memory is added or removed. 3698 * 3699 * The default user reserve max is 128MB, and the default max for the 3700 * admin reserve is 8MB. These are usually, but not always, enough to 3701 * enable recovery from a memory hogging process using login/sshd, a shell, 3702 * and tools like top. It may make sense to increase or even disable the 3703 * reserve depending on the existence of swap or variations in the recovery 3704 * tools. So, the admin may have changed them. 3705 * 3706 * If memory is added and the reserves have been eliminated or increased above 3707 * the default max, then we'll trust the admin. 3708 * 3709 * If memory is removed and there isn't enough free memory, then we 3710 * need to reset the reserves. 3711 * 3712 * Otherwise keep the reserve set by the admin. 3713 */ 3714 static int reserve_mem_notifier(struct notifier_block *nb, 3715 unsigned long action, void *data) 3716 { 3717 unsigned long tmp, free_kbytes; 3718 3719 switch (action) { 3720 case MEM_ONLINE: 3721 /* Default max is 128MB. Leave alone if modified by operator. */ 3722 tmp = sysctl_user_reserve_kbytes; 3723 if (0 < tmp && tmp < (1UL << 17)) 3724 init_user_reserve(); 3725 3726 /* Default max is 8MB. Leave alone if modified by operator. */ 3727 tmp = sysctl_admin_reserve_kbytes; 3728 if (0 < tmp && tmp < (1UL << 13)) 3729 init_admin_reserve(); 3730 3731 break; 3732 case MEM_OFFLINE: 3733 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3734 3735 if (sysctl_user_reserve_kbytes > free_kbytes) { 3736 init_user_reserve(); 3737 pr_info("vm.user_reserve_kbytes reset to %lu\n", 3738 sysctl_user_reserve_kbytes); 3739 } 3740 3741 if (sysctl_admin_reserve_kbytes > free_kbytes) { 3742 init_admin_reserve(); 3743 pr_info("vm.admin_reserve_kbytes reset to %lu\n", 3744 sysctl_admin_reserve_kbytes); 3745 } 3746 break; 3747 default: 3748 break; 3749 } 3750 return NOTIFY_OK; 3751 } 3752 3753 static struct notifier_block reserve_mem_nb = { 3754 .notifier_call = reserve_mem_notifier, 3755 }; 3756 3757 static int __meminit init_reserve_notifier(void) 3758 { 3759 if (register_hotmemory_notifier(&reserve_mem_nb)) 3760 pr_err("Failed registering memory add/remove notifier for admin reserve\n"); 3761 3762 return 0; 3763 } 3764 subsys_initcall(init_reserve_notifier); 3765