1 /* 2 * linux/mm/nommu.c 3 * 4 * Replacement code for mm functions to support CPU's that don't 5 * have any form of memory management unit (thus no virtual memory). 6 * 7 * See Documentation/nommu-mmap.txt 8 * 9 * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com> 10 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> 11 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> 12 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> 13 * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org> 14 */ 15 16 #include <linux/export.h> 17 #include <linux/mm.h> 18 #include <linux/mman.h> 19 #include <linux/swap.h> 20 #include <linux/file.h> 21 #include <linux/highmem.h> 22 #include <linux/pagemap.h> 23 #include <linux/slab.h> 24 #include <linux/vmalloc.h> 25 #include <linux/blkdev.h> 26 #include <linux/backing-dev.h> 27 #include <linux/mount.h> 28 #include <linux/personality.h> 29 #include <linux/security.h> 30 #include <linux/syscalls.h> 31 #include <linux/audit.h> 32 33 #include <asm/uaccess.h> 34 #include <asm/tlb.h> 35 #include <asm/tlbflush.h> 36 #include <asm/mmu_context.h> 37 #include "internal.h" 38 39 #if 0 40 #define kenter(FMT, ...) \ 41 printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__) 42 #define kleave(FMT, ...) \ 43 printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__) 44 #define kdebug(FMT, ...) \ 45 printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__) 46 #else 47 #define kenter(FMT, ...) \ 48 no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__) 49 #define kleave(FMT, ...) \ 50 no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__) 51 #define kdebug(FMT, ...) \ 52 no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__) 53 #endif 54 55 void *high_memory; 56 struct page *mem_map; 57 unsigned long max_mapnr; 58 unsigned long num_physpages; 59 unsigned long highest_memmap_pfn; 60 struct percpu_counter vm_committed_as; 61 int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 62 int sysctl_overcommit_ratio = 50; /* default is 50% */ 63 int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; 64 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS; 65 int heap_stack_gap = 0; 66 67 atomic_long_t mmap_pages_allocated; 68 69 EXPORT_SYMBOL(mem_map); 70 EXPORT_SYMBOL(num_physpages); 71 72 /* list of mapped, potentially shareable regions */ 73 static struct kmem_cache *vm_region_jar; 74 struct rb_root nommu_region_tree = RB_ROOT; 75 DECLARE_RWSEM(nommu_region_sem); 76 77 const struct vm_operations_struct generic_file_vm_ops = { 78 }; 79 80 /* 81 * Return the total memory allocated for this pointer, not 82 * just what the caller asked for. 83 * 84 * Doesn't have to be accurate, i.e. may have races. 85 */ 86 unsigned int kobjsize(const void *objp) 87 { 88 struct page *page; 89 90 /* 91 * If the object we have should not have ksize performed on it, 92 * return size of 0 93 */ 94 if (!objp || !virt_addr_valid(objp)) 95 return 0; 96 97 page = virt_to_head_page(objp); 98 99 /* 100 * If the allocator sets PageSlab, we know the pointer came from 101 * kmalloc(). 102 */ 103 if (PageSlab(page)) 104 return ksize(objp); 105 106 /* 107 * If it's not a compound page, see if we have a matching VMA 108 * region. This test is intentionally done in reverse order, 109 * so if there's no VMA, we still fall through and hand back 110 * PAGE_SIZE for 0-order pages. 111 */ 112 if (!PageCompound(page)) { 113 struct vm_area_struct *vma; 114 115 vma = find_vma(current->mm, (unsigned long)objp); 116 if (vma) 117 return vma->vm_end - vma->vm_start; 118 } 119 120 /* 121 * The ksize() function is only guaranteed to work for pointers 122 * returned by kmalloc(). So handle arbitrary pointers here. 123 */ 124 return PAGE_SIZE << compound_order(page); 125 } 126 127 int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 128 unsigned long start, int nr_pages, unsigned int foll_flags, 129 struct page **pages, struct vm_area_struct **vmas, 130 int *retry) 131 { 132 struct vm_area_struct *vma; 133 unsigned long vm_flags; 134 int i; 135 136 /* calculate required read or write permissions. 137 * If FOLL_FORCE is set, we only require the "MAY" flags. 138 */ 139 vm_flags = (foll_flags & FOLL_WRITE) ? 140 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 141 vm_flags &= (foll_flags & FOLL_FORCE) ? 142 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 143 144 for (i = 0; i < nr_pages; i++) { 145 vma = find_vma(mm, start); 146 if (!vma) 147 goto finish_or_fault; 148 149 /* protect what we can, including chardevs */ 150 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) || 151 !(vm_flags & vma->vm_flags)) 152 goto finish_or_fault; 153 154 if (pages) { 155 pages[i] = virt_to_page(start); 156 if (pages[i]) 157 page_cache_get(pages[i]); 158 } 159 if (vmas) 160 vmas[i] = vma; 161 start = (start + PAGE_SIZE) & PAGE_MASK; 162 } 163 164 return i; 165 166 finish_or_fault: 167 return i ? : -EFAULT; 168 } 169 170 /* 171 * get a list of pages in an address range belonging to the specified process 172 * and indicate the VMA that covers each page 173 * - this is potentially dodgy as we may end incrementing the page count of a 174 * slab page or a secondary page from a compound page 175 * - don't permit access to VMAs that don't support it, such as I/O mappings 176 */ 177 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 178 unsigned long start, int nr_pages, int write, int force, 179 struct page **pages, struct vm_area_struct **vmas) 180 { 181 int flags = 0; 182 183 if (write) 184 flags |= FOLL_WRITE; 185 if (force) 186 flags |= FOLL_FORCE; 187 188 return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas, 189 NULL); 190 } 191 EXPORT_SYMBOL(get_user_pages); 192 193 /** 194 * follow_pfn - look up PFN at a user virtual address 195 * @vma: memory mapping 196 * @address: user virtual address 197 * @pfn: location to store found PFN 198 * 199 * Only IO mappings and raw PFN mappings are allowed. 200 * 201 * Returns zero and the pfn at @pfn on success, -ve otherwise. 202 */ 203 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 204 unsigned long *pfn) 205 { 206 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 207 return -EINVAL; 208 209 *pfn = address >> PAGE_SHIFT; 210 return 0; 211 } 212 EXPORT_SYMBOL(follow_pfn); 213 214 DEFINE_RWLOCK(vmlist_lock); 215 struct vm_struct *vmlist; 216 217 void vfree(const void *addr) 218 { 219 kfree(addr); 220 } 221 EXPORT_SYMBOL(vfree); 222 223 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 224 { 225 /* 226 * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc() 227 * returns only a logical address. 228 */ 229 return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM); 230 } 231 EXPORT_SYMBOL(__vmalloc); 232 233 void *vmalloc_user(unsigned long size) 234 { 235 void *ret; 236 237 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 238 PAGE_KERNEL); 239 if (ret) { 240 struct vm_area_struct *vma; 241 242 down_write(¤t->mm->mmap_sem); 243 vma = find_vma(current->mm, (unsigned long)ret); 244 if (vma) 245 vma->vm_flags |= VM_USERMAP; 246 up_write(¤t->mm->mmap_sem); 247 } 248 249 return ret; 250 } 251 EXPORT_SYMBOL(vmalloc_user); 252 253 struct page *vmalloc_to_page(const void *addr) 254 { 255 return virt_to_page(addr); 256 } 257 EXPORT_SYMBOL(vmalloc_to_page); 258 259 unsigned long vmalloc_to_pfn(const void *addr) 260 { 261 return page_to_pfn(virt_to_page(addr)); 262 } 263 EXPORT_SYMBOL(vmalloc_to_pfn); 264 265 long vread(char *buf, char *addr, unsigned long count) 266 { 267 memcpy(buf, addr, count); 268 return count; 269 } 270 271 long vwrite(char *buf, char *addr, unsigned long count) 272 { 273 /* Don't allow overflow */ 274 if ((unsigned long) addr + count < count) 275 count = -(unsigned long) addr; 276 277 memcpy(addr, buf, count); 278 return(count); 279 } 280 281 /* 282 * vmalloc - allocate virtually continguos memory 283 * 284 * @size: allocation size 285 * 286 * Allocate enough pages to cover @size from the page level 287 * allocator and map them into continguos kernel virtual space. 288 * 289 * For tight control over page level allocator and protection flags 290 * use __vmalloc() instead. 291 */ 292 void *vmalloc(unsigned long size) 293 { 294 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL); 295 } 296 EXPORT_SYMBOL(vmalloc); 297 298 /* 299 * vzalloc - allocate virtually continguos memory with zero fill 300 * 301 * @size: allocation size 302 * 303 * Allocate enough pages to cover @size from the page level 304 * allocator and map them into continguos kernel virtual space. 305 * The memory allocated is set to zero. 306 * 307 * For tight control over page level allocator and protection flags 308 * use __vmalloc() instead. 309 */ 310 void *vzalloc(unsigned long size) 311 { 312 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 313 PAGE_KERNEL); 314 } 315 EXPORT_SYMBOL(vzalloc); 316 317 /** 318 * vmalloc_node - allocate memory on a specific node 319 * @size: allocation size 320 * @node: numa node 321 * 322 * Allocate enough pages to cover @size from the page level 323 * allocator and map them into contiguous kernel virtual space. 324 * 325 * For tight control over page level allocator and protection flags 326 * use __vmalloc() instead. 327 */ 328 void *vmalloc_node(unsigned long size, int node) 329 { 330 return vmalloc(size); 331 } 332 EXPORT_SYMBOL(vmalloc_node); 333 334 /** 335 * vzalloc_node - allocate memory on a specific node with zero fill 336 * @size: allocation size 337 * @node: numa node 338 * 339 * Allocate enough pages to cover @size from the page level 340 * allocator and map them into contiguous kernel virtual space. 341 * The memory allocated is set to zero. 342 * 343 * For tight control over page level allocator and protection flags 344 * use __vmalloc() instead. 345 */ 346 void *vzalloc_node(unsigned long size, int node) 347 { 348 return vzalloc(size); 349 } 350 EXPORT_SYMBOL(vzalloc_node); 351 352 #ifndef PAGE_KERNEL_EXEC 353 # define PAGE_KERNEL_EXEC PAGE_KERNEL 354 #endif 355 356 /** 357 * vmalloc_exec - allocate virtually contiguous, executable memory 358 * @size: allocation size 359 * 360 * Kernel-internal function to allocate enough pages to cover @size 361 * the page level allocator and map them into contiguous and 362 * executable kernel virtual space. 363 * 364 * For tight control over page level allocator and protection flags 365 * use __vmalloc() instead. 366 */ 367 368 void *vmalloc_exec(unsigned long size) 369 { 370 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC); 371 } 372 373 /** 374 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 375 * @size: allocation size 376 * 377 * Allocate enough 32bit PA addressable pages to cover @size from the 378 * page level allocator and map them into continguos kernel virtual space. 379 */ 380 void *vmalloc_32(unsigned long size) 381 { 382 return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); 383 } 384 EXPORT_SYMBOL(vmalloc_32); 385 386 /** 387 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 388 * @size: allocation size 389 * 390 * The resulting memory area is 32bit addressable and zeroed so it can be 391 * mapped to userspace without leaking data. 392 * 393 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to 394 * remap_vmalloc_range() are permissible. 395 */ 396 void *vmalloc_32_user(unsigned long size) 397 { 398 /* 399 * We'll have to sort out the ZONE_DMA bits for 64-bit, 400 * but for now this can simply use vmalloc_user() directly. 401 */ 402 return vmalloc_user(size); 403 } 404 EXPORT_SYMBOL(vmalloc_32_user); 405 406 void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) 407 { 408 BUG(); 409 return NULL; 410 } 411 EXPORT_SYMBOL(vmap); 412 413 void vunmap(const void *addr) 414 { 415 BUG(); 416 } 417 EXPORT_SYMBOL(vunmap); 418 419 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) 420 { 421 BUG(); 422 return NULL; 423 } 424 EXPORT_SYMBOL(vm_map_ram); 425 426 void vm_unmap_ram(const void *mem, unsigned int count) 427 { 428 BUG(); 429 } 430 EXPORT_SYMBOL(vm_unmap_ram); 431 432 void vm_unmap_aliases(void) 433 { 434 } 435 EXPORT_SYMBOL_GPL(vm_unmap_aliases); 436 437 /* 438 * Implement a stub for vmalloc_sync_all() if the architecture chose not to 439 * have one. 440 */ 441 void __attribute__((weak)) vmalloc_sync_all(void) 442 { 443 } 444 445 /** 446 * alloc_vm_area - allocate a range of kernel address space 447 * @size: size of the area 448 * 449 * Returns: NULL on failure, vm_struct on success 450 * 451 * This function reserves a range of kernel address space, and 452 * allocates pagetables to map that range. No actual mappings 453 * are created. If the kernel address space is not shared 454 * between processes, it syncs the pagetable across all 455 * processes. 456 */ 457 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) 458 { 459 BUG(); 460 return NULL; 461 } 462 EXPORT_SYMBOL_GPL(alloc_vm_area); 463 464 void free_vm_area(struct vm_struct *area) 465 { 466 BUG(); 467 } 468 EXPORT_SYMBOL_GPL(free_vm_area); 469 470 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 471 struct page *page) 472 { 473 return -EINVAL; 474 } 475 EXPORT_SYMBOL(vm_insert_page); 476 477 /* 478 * sys_brk() for the most part doesn't need the global kernel 479 * lock, except when an application is doing something nasty 480 * like trying to un-brk an area that has already been mapped 481 * to a regular file. in this case, the unmapping will need 482 * to invoke file system routines that need the global lock. 483 */ 484 SYSCALL_DEFINE1(brk, unsigned long, brk) 485 { 486 struct mm_struct *mm = current->mm; 487 488 if (brk < mm->start_brk || brk > mm->context.end_brk) 489 return mm->brk; 490 491 if (mm->brk == brk) 492 return mm->brk; 493 494 /* 495 * Always allow shrinking brk 496 */ 497 if (brk <= mm->brk) { 498 mm->brk = brk; 499 return brk; 500 } 501 502 /* 503 * Ok, looks good - let it rip. 504 */ 505 flush_icache_range(mm->brk, brk); 506 return mm->brk = brk; 507 } 508 509 /* 510 * initialise the VMA and region record slabs 511 */ 512 void __init mmap_init(void) 513 { 514 int ret; 515 516 ret = percpu_counter_init(&vm_committed_as, 0); 517 VM_BUG_ON(ret); 518 vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC); 519 } 520 521 /* 522 * validate the region tree 523 * - the caller must hold the region lock 524 */ 525 #ifdef CONFIG_DEBUG_NOMMU_REGIONS 526 static noinline void validate_nommu_regions(void) 527 { 528 struct vm_region *region, *last; 529 struct rb_node *p, *lastp; 530 531 lastp = rb_first(&nommu_region_tree); 532 if (!lastp) 533 return; 534 535 last = rb_entry(lastp, struct vm_region, vm_rb); 536 BUG_ON(unlikely(last->vm_end <= last->vm_start)); 537 BUG_ON(unlikely(last->vm_top < last->vm_end)); 538 539 while ((p = rb_next(lastp))) { 540 region = rb_entry(p, struct vm_region, vm_rb); 541 last = rb_entry(lastp, struct vm_region, vm_rb); 542 543 BUG_ON(unlikely(region->vm_end <= region->vm_start)); 544 BUG_ON(unlikely(region->vm_top < region->vm_end)); 545 BUG_ON(unlikely(region->vm_start < last->vm_top)); 546 547 lastp = p; 548 } 549 } 550 #else 551 static void validate_nommu_regions(void) 552 { 553 } 554 #endif 555 556 /* 557 * add a region into the global tree 558 */ 559 static void add_nommu_region(struct vm_region *region) 560 { 561 struct vm_region *pregion; 562 struct rb_node **p, *parent; 563 564 validate_nommu_regions(); 565 566 parent = NULL; 567 p = &nommu_region_tree.rb_node; 568 while (*p) { 569 parent = *p; 570 pregion = rb_entry(parent, struct vm_region, vm_rb); 571 if (region->vm_start < pregion->vm_start) 572 p = &(*p)->rb_left; 573 else if (region->vm_start > pregion->vm_start) 574 p = &(*p)->rb_right; 575 else if (pregion == region) 576 return; 577 else 578 BUG(); 579 } 580 581 rb_link_node(®ion->vm_rb, parent, p); 582 rb_insert_color(®ion->vm_rb, &nommu_region_tree); 583 584 validate_nommu_regions(); 585 } 586 587 /* 588 * delete a region from the global tree 589 */ 590 static void delete_nommu_region(struct vm_region *region) 591 { 592 BUG_ON(!nommu_region_tree.rb_node); 593 594 validate_nommu_regions(); 595 rb_erase(®ion->vm_rb, &nommu_region_tree); 596 validate_nommu_regions(); 597 } 598 599 /* 600 * free a contiguous series of pages 601 */ 602 static void free_page_series(unsigned long from, unsigned long to) 603 { 604 for (; from < to; from += PAGE_SIZE) { 605 struct page *page = virt_to_page(from); 606 607 kdebug("- free %lx", from); 608 atomic_long_dec(&mmap_pages_allocated); 609 if (page_count(page) != 1) 610 kdebug("free page %p: refcount not one: %d", 611 page, page_count(page)); 612 put_page(page); 613 } 614 } 615 616 /* 617 * release a reference to a region 618 * - the caller must hold the region semaphore for writing, which this releases 619 * - the region may not have been added to the tree yet, in which case vm_top 620 * will equal vm_start 621 */ 622 static void __put_nommu_region(struct vm_region *region) 623 __releases(nommu_region_sem) 624 { 625 kenter("%p{%d}", region, region->vm_usage); 626 627 BUG_ON(!nommu_region_tree.rb_node); 628 629 if (--region->vm_usage == 0) { 630 if (region->vm_top > region->vm_start) 631 delete_nommu_region(region); 632 up_write(&nommu_region_sem); 633 634 if (region->vm_file) 635 fput(region->vm_file); 636 637 /* IO memory and memory shared directly out of the pagecache 638 * from ramfs/tmpfs mustn't be released here */ 639 if (region->vm_flags & VM_MAPPED_COPY) { 640 kdebug("free series"); 641 free_page_series(region->vm_start, region->vm_top); 642 } 643 kmem_cache_free(vm_region_jar, region); 644 } else { 645 up_write(&nommu_region_sem); 646 } 647 } 648 649 /* 650 * release a reference to a region 651 */ 652 static void put_nommu_region(struct vm_region *region) 653 { 654 down_write(&nommu_region_sem); 655 __put_nommu_region(region); 656 } 657 658 /* 659 * update protection on a vma 660 */ 661 static void protect_vma(struct vm_area_struct *vma, unsigned long flags) 662 { 663 #ifdef CONFIG_MPU 664 struct mm_struct *mm = vma->vm_mm; 665 long start = vma->vm_start & PAGE_MASK; 666 while (start < vma->vm_end) { 667 protect_page(mm, start, flags); 668 start += PAGE_SIZE; 669 } 670 update_protections(mm); 671 #endif 672 } 673 674 /* 675 * add a VMA into a process's mm_struct in the appropriate place in the list 676 * and tree and add to the address space's page tree also if not an anonymous 677 * page 678 * - should be called with mm->mmap_sem held writelocked 679 */ 680 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) 681 { 682 struct vm_area_struct *pvma, *prev; 683 struct address_space *mapping; 684 struct rb_node **p, *parent, *rb_prev; 685 686 kenter(",%p", vma); 687 688 BUG_ON(!vma->vm_region); 689 690 mm->map_count++; 691 vma->vm_mm = mm; 692 693 protect_vma(vma, vma->vm_flags); 694 695 /* add the VMA to the mapping */ 696 if (vma->vm_file) { 697 mapping = vma->vm_file->f_mapping; 698 699 mutex_lock(&mapping->i_mmap_mutex); 700 flush_dcache_mmap_lock(mapping); 701 vma_interval_tree_insert(vma, &mapping->i_mmap); 702 flush_dcache_mmap_unlock(mapping); 703 mutex_unlock(&mapping->i_mmap_mutex); 704 } 705 706 /* add the VMA to the tree */ 707 parent = rb_prev = NULL; 708 p = &mm->mm_rb.rb_node; 709 while (*p) { 710 parent = *p; 711 pvma = rb_entry(parent, struct vm_area_struct, vm_rb); 712 713 /* sort by: start addr, end addr, VMA struct addr in that order 714 * (the latter is necessary as we may get identical VMAs) */ 715 if (vma->vm_start < pvma->vm_start) 716 p = &(*p)->rb_left; 717 else if (vma->vm_start > pvma->vm_start) { 718 rb_prev = parent; 719 p = &(*p)->rb_right; 720 } else if (vma->vm_end < pvma->vm_end) 721 p = &(*p)->rb_left; 722 else if (vma->vm_end > pvma->vm_end) { 723 rb_prev = parent; 724 p = &(*p)->rb_right; 725 } else if (vma < pvma) 726 p = &(*p)->rb_left; 727 else if (vma > pvma) { 728 rb_prev = parent; 729 p = &(*p)->rb_right; 730 } else 731 BUG(); 732 } 733 734 rb_link_node(&vma->vm_rb, parent, p); 735 rb_insert_color(&vma->vm_rb, &mm->mm_rb); 736 737 /* add VMA to the VMA list also */ 738 prev = NULL; 739 if (rb_prev) 740 prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); 741 742 __vma_link_list(mm, vma, prev, parent); 743 } 744 745 /* 746 * delete a VMA from its owning mm_struct and address space 747 */ 748 static void delete_vma_from_mm(struct vm_area_struct *vma) 749 { 750 struct address_space *mapping; 751 struct mm_struct *mm = vma->vm_mm; 752 753 kenter("%p", vma); 754 755 protect_vma(vma, 0); 756 757 mm->map_count--; 758 if (mm->mmap_cache == vma) 759 mm->mmap_cache = NULL; 760 761 /* remove the VMA from the mapping */ 762 if (vma->vm_file) { 763 mapping = vma->vm_file->f_mapping; 764 765 mutex_lock(&mapping->i_mmap_mutex); 766 flush_dcache_mmap_lock(mapping); 767 vma_interval_tree_remove(vma, &mapping->i_mmap); 768 flush_dcache_mmap_unlock(mapping); 769 mutex_unlock(&mapping->i_mmap_mutex); 770 } 771 772 /* remove from the MM's tree and list */ 773 rb_erase(&vma->vm_rb, &mm->mm_rb); 774 775 if (vma->vm_prev) 776 vma->vm_prev->vm_next = vma->vm_next; 777 else 778 mm->mmap = vma->vm_next; 779 780 if (vma->vm_next) 781 vma->vm_next->vm_prev = vma->vm_prev; 782 } 783 784 /* 785 * destroy a VMA record 786 */ 787 static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) 788 { 789 kenter("%p", vma); 790 if (vma->vm_ops && vma->vm_ops->close) 791 vma->vm_ops->close(vma); 792 if (vma->vm_file) 793 fput(vma->vm_file); 794 put_nommu_region(vma->vm_region); 795 kmem_cache_free(vm_area_cachep, vma); 796 } 797 798 /* 799 * look up the first VMA in which addr resides, NULL if none 800 * - should be called with mm->mmap_sem at least held readlocked 801 */ 802 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 803 { 804 struct vm_area_struct *vma; 805 806 /* check the cache first */ 807 vma = mm->mmap_cache; 808 if (vma && vma->vm_start <= addr && vma->vm_end > addr) 809 return vma; 810 811 /* trawl the list (there may be multiple mappings in which addr 812 * resides) */ 813 for (vma = mm->mmap; vma; vma = vma->vm_next) { 814 if (vma->vm_start > addr) 815 return NULL; 816 if (vma->vm_end > addr) { 817 mm->mmap_cache = vma; 818 return vma; 819 } 820 } 821 822 return NULL; 823 } 824 EXPORT_SYMBOL(find_vma); 825 826 /* 827 * find a VMA 828 * - we don't extend stack VMAs under NOMMU conditions 829 */ 830 struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) 831 { 832 return find_vma(mm, addr); 833 } 834 835 /* 836 * expand a stack to a given address 837 * - not supported under NOMMU conditions 838 */ 839 int expand_stack(struct vm_area_struct *vma, unsigned long address) 840 { 841 return -ENOMEM; 842 } 843 844 /* 845 * look up the first VMA exactly that exactly matches addr 846 * - should be called with mm->mmap_sem at least held readlocked 847 */ 848 static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, 849 unsigned long addr, 850 unsigned long len) 851 { 852 struct vm_area_struct *vma; 853 unsigned long end = addr + len; 854 855 /* check the cache first */ 856 vma = mm->mmap_cache; 857 if (vma && vma->vm_start == addr && vma->vm_end == end) 858 return vma; 859 860 /* trawl the list (there may be multiple mappings in which addr 861 * resides) */ 862 for (vma = mm->mmap; vma; vma = vma->vm_next) { 863 if (vma->vm_start < addr) 864 continue; 865 if (vma->vm_start > addr) 866 return NULL; 867 if (vma->vm_end == end) { 868 mm->mmap_cache = vma; 869 return vma; 870 } 871 } 872 873 return NULL; 874 } 875 876 /* 877 * determine whether a mapping should be permitted and, if so, what sort of 878 * mapping we're capable of supporting 879 */ 880 static int validate_mmap_request(struct file *file, 881 unsigned long addr, 882 unsigned long len, 883 unsigned long prot, 884 unsigned long flags, 885 unsigned long pgoff, 886 unsigned long *_capabilities) 887 { 888 unsigned long capabilities, rlen; 889 int ret; 890 891 /* do the simple checks first */ 892 if (flags & MAP_FIXED) { 893 printk(KERN_DEBUG 894 "%d: Can't do fixed-address/overlay mmap of RAM\n", 895 current->pid); 896 return -EINVAL; 897 } 898 899 if ((flags & MAP_TYPE) != MAP_PRIVATE && 900 (flags & MAP_TYPE) != MAP_SHARED) 901 return -EINVAL; 902 903 if (!len) 904 return -EINVAL; 905 906 /* Careful about overflows.. */ 907 rlen = PAGE_ALIGN(len); 908 if (!rlen || rlen > TASK_SIZE) 909 return -ENOMEM; 910 911 /* offset overflow? */ 912 if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff) 913 return -EOVERFLOW; 914 915 if (file) { 916 /* validate file mapping requests */ 917 struct address_space *mapping; 918 919 /* files must support mmap */ 920 if (!file->f_op || !file->f_op->mmap) 921 return -ENODEV; 922 923 /* work out if what we've got could possibly be shared 924 * - we support chardevs that provide their own "memory" 925 * - we support files/blockdevs that are memory backed 926 */ 927 mapping = file->f_mapping; 928 if (!mapping) 929 mapping = file->f_path.dentry->d_inode->i_mapping; 930 931 capabilities = 0; 932 if (mapping && mapping->backing_dev_info) 933 capabilities = mapping->backing_dev_info->capabilities; 934 935 if (!capabilities) { 936 /* no explicit capabilities set, so assume some 937 * defaults */ 938 switch (file->f_path.dentry->d_inode->i_mode & S_IFMT) { 939 case S_IFREG: 940 case S_IFBLK: 941 capabilities = BDI_CAP_MAP_COPY; 942 break; 943 944 case S_IFCHR: 945 capabilities = 946 BDI_CAP_MAP_DIRECT | 947 BDI_CAP_READ_MAP | 948 BDI_CAP_WRITE_MAP; 949 break; 950 951 default: 952 return -EINVAL; 953 } 954 } 955 956 /* eliminate any capabilities that we can't support on this 957 * device */ 958 if (!file->f_op->get_unmapped_area) 959 capabilities &= ~BDI_CAP_MAP_DIRECT; 960 if (!file->f_op->read) 961 capabilities &= ~BDI_CAP_MAP_COPY; 962 963 /* The file shall have been opened with read permission. */ 964 if (!(file->f_mode & FMODE_READ)) 965 return -EACCES; 966 967 if (flags & MAP_SHARED) { 968 /* do checks for writing, appending and locking */ 969 if ((prot & PROT_WRITE) && 970 !(file->f_mode & FMODE_WRITE)) 971 return -EACCES; 972 973 if (IS_APPEND(file->f_path.dentry->d_inode) && 974 (file->f_mode & FMODE_WRITE)) 975 return -EACCES; 976 977 if (locks_verify_locked(file->f_path.dentry->d_inode)) 978 return -EAGAIN; 979 980 if (!(capabilities & BDI_CAP_MAP_DIRECT)) 981 return -ENODEV; 982 983 /* we mustn't privatise shared mappings */ 984 capabilities &= ~BDI_CAP_MAP_COPY; 985 } 986 else { 987 /* we're going to read the file into private memory we 988 * allocate */ 989 if (!(capabilities & BDI_CAP_MAP_COPY)) 990 return -ENODEV; 991 992 /* we don't permit a private writable mapping to be 993 * shared with the backing device */ 994 if (prot & PROT_WRITE) 995 capabilities &= ~BDI_CAP_MAP_DIRECT; 996 } 997 998 if (capabilities & BDI_CAP_MAP_DIRECT) { 999 if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) || 1000 ((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) || 1001 ((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP)) 1002 ) { 1003 capabilities &= ~BDI_CAP_MAP_DIRECT; 1004 if (flags & MAP_SHARED) { 1005 printk(KERN_WARNING 1006 "MAP_SHARED not completely supported on !MMU\n"); 1007 return -EINVAL; 1008 } 1009 } 1010 } 1011 1012 /* handle executable mappings and implied executable 1013 * mappings */ 1014 if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) { 1015 if (prot & PROT_EXEC) 1016 return -EPERM; 1017 } 1018 else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) { 1019 /* handle implication of PROT_EXEC by PROT_READ */ 1020 if (current->personality & READ_IMPLIES_EXEC) { 1021 if (capabilities & BDI_CAP_EXEC_MAP) 1022 prot |= PROT_EXEC; 1023 } 1024 } 1025 else if ((prot & PROT_READ) && 1026 (prot & PROT_EXEC) && 1027 !(capabilities & BDI_CAP_EXEC_MAP) 1028 ) { 1029 /* backing file is not executable, try to copy */ 1030 capabilities &= ~BDI_CAP_MAP_DIRECT; 1031 } 1032 } 1033 else { 1034 /* anonymous mappings are always memory backed and can be 1035 * privately mapped 1036 */ 1037 capabilities = BDI_CAP_MAP_COPY; 1038 1039 /* handle PROT_EXEC implication by PROT_READ */ 1040 if ((prot & PROT_READ) && 1041 (current->personality & READ_IMPLIES_EXEC)) 1042 prot |= PROT_EXEC; 1043 } 1044 1045 /* allow the security API to have its say */ 1046 ret = security_mmap_addr(addr); 1047 if (ret < 0) 1048 return ret; 1049 1050 /* looks okay */ 1051 *_capabilities = capabilities; 1052 return 0; 1053 } 1054 1055 /* 1056 * we've determined that we can make the mapping, now translate what we 1057 * now know into VMA flags 1058 */ 1059 static unsigned long determine_vm_flags(struct file *file, 1060 unsigned long prot, 1061 unsigned long flags, 1062 unsigned long capabilities) 1063 { 1064 unsigned long vm_flags; 1065 1066 vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags); 1067 /* vm_flags |= mm->def_flags; */ 1068 1069 if (!(capabilities & BDI_CAP_MAP_DIRECT)) { 1070 /* attempt to share read-only copies of mapped file chunks */ 1071 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 1072 if (file && !(prot & PROT_WRITE)) 1073 vm_flags |= VM_MAYSHARE; 1074 } else { 1075 /* overlay a shareable mapping on the backing device or inode 1076 * if possible - used for chardevs, ramfs/tmpfs/shmfs and 1077 * romfs/cramfs */ 1078 vm_flags |= VM_MAYSHARE | (capabilities & BDI_CAP_VMFLAGS); 1079 if (flags & MAP_SHARED) 1080 vm_flags |= VM_SHARED; 1081 } 1082 1083 /* refuse to let anyone share private mappings with this process if 1084 * it's being traced - otherwise breakpoints set in it may interfere 1085 * with another untraced process 1086 */ 1087 if ((flags & MAP_PRIVATE) && current->ptrace) 1088 vm_flags &= ~VM_MAYSHARE; 1089 1090 return vm_flags; 1091 } 1092 1093 /* 1094 * set up a shared mapping on a file (the driver or filesystem provides and 1095 * pins the storage) 1096 */ 1097 static int do_mmap_shared_file(struct vm_area_struct *vma) 1098 { 1099 int ret; 1100 1101 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); 1102 if (ret == 0) { 1103 vma->vm_region->vm_top = vma->vm_region->vm_end; 1104 return 0; 1105 } 1106 if (ret != -ENOSYS) 1107 return ret; 1108 1109 /* getting -ENOSYS indicates that direct mmap isn't possible (as 1110 * opposed to tried but failed) so we can only give a suitable error as 1111 * it's not possible to make a private copy if MAP_SHARED was given */ 1112 return -ENODEV; 1113 } 1114 1115 /* 1116 * set up a private mapping or an anonymous shared mapping 1117 */ 1118 static int do_mmap_private(struct vm_area_struct *vma, 1119 struct vm_region *region, 1120 unsigned long len, 1121 unsigned long capabilities) 1122 { 1123 struct page *pages; 1124 unsigned long total, point, n; 1125 void *base; 1126 int ret, order; 1127 1128 /* invoke the file's mapping function so that it can keep track of 1129 * shared mappings on devices or memory 1130 * - VM_MAYSHARE will be set if it may attempt to share 1131 */ 1132 if (capabilities & BDI_CAP_MAP_DIRECT) { 1133 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); 1134 if (ret == 0) { 1135 /* shouldn't return success if we're not sharing */ 1136 BUG_ON(!(vma->vm_flags & VM_MAYSHARE)); 1137 vma->vm_region->vm_top = vma->vm_region->vm_end; 1138 return 0; 1139 } 1140 if (ret != -ENOSYS) 1141 return ret; 1142 1143 /* getting an ENOSYS error indicates that direct mmap isn't 1144 * possible (as opposed to tried but failed) so we'll try to 1145 * make a private copy of the data and map that instead */ 1146 } 1147 1148 1149 /* allocate some memory to hold the mapping 1150 * - note that this may not return a page-aligned address if the object 1151 * we're allocating is smaller than a page 1152 */ 1153 order = get_order(len); 1154 kdebug("alloc order %d for %lx", order, len); 1155 1156 pages = alloc_pages(GFP_KERNEL, order); 1157 if (!pages) 1158 goto enomem; 1159 1160 total = 1 << order; 1161 atomic_long_add(total, &mmap_pages_allocated); 1162 1163 point = len >> PAGE_SHIFT; 1164 1165 /* we allocated a power-of-2 sized page set, so we may want to trim off 1166 * the excess */ 1167 if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) { 1168 while (total > point) { 1169 order = ilog2(total - point); 1170 n = 1 << order; 1171 kdebug("shave %lu/%lu @%lu", n, total - point, total); 1172 atomic_long_sub(n, &mmap_pages_allocated); 1173 total -= n; 1174 set_page_refcounted(pages + total); 1175 __free_pages(pages + total, order); 1176 } 1177 } 1178 1179 for (point = 1; point < total; point++) 1180 set_page_refcounted(&pages[point]); 1181 1182 base = page_address(pages); 1183 region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY; 1184 region->vm_start = (unsigned long) base; 1185 region->vm_end = region->vm_start + len; 1186 region->vm_top = region->vm_start + (total << PAGE_SHIFT); 1187 1188 vma->vm_start = region->vm_start; 1189 vma->vm_end = region->vm_start + len; 1190 1191 if (vma->vm_file) { 1192 /* read the contents of a file into the copy */ 1193 mm_segment_t old_fs; 1194 loff_t fpos; 1195 1196 fpos = vma->vm_pgoff; 1197 fpos <<= PAGE_SHIFT; 1198 1199 old_fs = get_fs(); 1200 set_fs(KERNEL_DS); 1201 ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); 1202 set_fs(old_fs); 1203 1204 if (ret < 0) 1205 goto error_free; 1206 1207 /* clear the last little bit */ 1208 if (ret < len) 1209 memset(base + ret, 0, len - ret); 1210 1211 } 1212 1213 return 0; 1214 1215 error_free: 1216 free_page_series(region->vm_start, region->vm_top); 1217 region->vm_start = vma->vm_start = 0; 1218 region->vm_end = vma->vm_end = 0; 1219 region->vm_top = 0; 1220 return ret; 1221 1222 enomem: 1223 printk("Allocation of length %lu from process %d (%s) failed\n", 1224 len, current->pid, current->comm); 1225 show_free_areas(0); 1226 return -ENOMEM; 1227 } 1228 1229 /* 1230 * handle mapping creation for uClinux 1231 */ 1232 unsigned long do_mmap_pgoff(struct file *file, 1233 unsigned long addr, 1234 unsigned long len, 1235 unsigned long prot, 1236 unsigned long flags, 1237 unsigned long pgoff) 1238 { 1239 struct vm_area_struct *vma; 1240 struct vm_region *region; 1241 struct rb_node *rb; 1242 unsigned long capabilities, vm_flags, result; 1243 int ret; 1244 1245 kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff); 1246 1247 /* decide whether we should attempt the mapping, and if so what sort of 1248 * mapping */ 1249 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, 1250 &capabilities); 1251 if (ret < 0) { 1252 kleave(" = %d [val]", ret); 1253 return ret; 1254 } 1255 1256 /* we ignore the address hint */ 1257 addr = 0; 1258 len = PAGE_ALIGN(len); 1259 1260 /* we've determined that we can make the mapping, now translate what we 1261 * now know into VMA flags */ 1262 vm_flags = determine_vm_flags(file, prot, flags, capabilities); 1263 1264 /* we're going to need to record the mapping */ 1265 region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); 1266 if (!region) 1267 goto error_getting_region; 1268 1269 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); 1270 if (!vma) 1271 goto error_getting_vma; 1272 1273 region->vm_usage = 1; 1274 region->vm_flags = vm_flags; 1275 region->vm_pgoff = pgoff; 1276 1277 INIT_LIST_HEAD(&vma->anon_vma_chain); 1278 vma->vm_flags = vm_flags; 1279 vma->vm_pgoff = pgoff; 1280 1281 if (file) { 1282 region->vm_file = get_file(file); 1283 vma->vm_file = get_file(file); 1284 } 1285 1286 down_write(&nommu_region_sem); 1287 1288 /* if we want to share, we need to check for regions created by other 1289 * mmap() calls that overlap with our proposed mapping 1290 * - we can only share with a superset match on most regular files 1291 * - shared mappings on character devices and memory backed files are 1292 * permitted to overlap inexactly as far as we are concerned for in 1293 * these cases, sharing is handled in the driver or filesystem rather 1294 * than here 1295 */ 1296 if (vm_flags & VM_MAYSHARE) { 1297 struct vm_region *pregion; 1298 unsigned long pglen, rpglen, pgend, rpgend, start; 1299 1300 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1301 pgend = pgoff + pglen; 1302 1303 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) { 1304 pregion = rb_entry(rb, struct vm_region, vm_rb); 1305 1306 if (!(pregion->vm_flags & VM_MAYSHARE)) 1307 continue; 1308 1309 /* search for overlapping mappings on the same file */ 1310 if (pregion->vm_file->f_path.dentry->d_inode != 1311 file->f_path.dentry->d_inode) 1312 continue; 1313 1314 if (pregion->vm_pgoff >= pgend) 1315 continue; 1316 1317 rpglen = pregion->vm_end - pregion->vm_start; 1318 rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT; 1319 rpgend = pregion->vm_pgoff + rpglen; 1320 if (pgoff >= rpgend) 1321 continue; 1322 1323 /* handle inexactly overlapping matches between 1324 * mappings */ 1325 if ((pregion->vm_pgoff != pgoff || rpglen != pglen) && 1326 !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) { 1327 /* new mapping is not a subset of the region */ 1328 if (!(capabilities & BDI_CAP_MAP_DIRECT)) 1329 goto sharing_violation; 1330 continue; 1331 } 1332 1333 /* we've found a region we can share */ 1334 pregion->vm_usage++; 1335 vma->vm_region = pregion; 1336 start = pregion->vm_start; 1337 start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT; 1338 vma->vm_start = start; 1339 vma->vm_end = start + len; 1340 1341 if (pregion->vm_flags & VM_MAPPED_COPY) { 1342 kdebug("share copy"); 1343 vma->vm_flags |= VM_MAPPED_COPY; 1344 } else { 1345 kdebug("share mmap"); 1346 ret = do_mmap_shared_file(vma); 1347 if (ret < 0) { 1348 vma->vm_region = NULL; 1349 vma->vm_start = 0; 1350 vma->vm_end = 0; 1351 pregion->vm_usage--; 1352 pregion = NULL; 1353 goto error_just_free; 1354 } 1355 } 1356 fput(region->vm_file); 1357 kmem_cache_free(vm_region_jar, region); 1358 region = pregion; 1359 result = start; 1360 goto share; 1361 } 1362 1363 /* obtain the address at which to make a shared mapping 1364 * - this is the hook for quasi-memory character devices to 1365 * tell us the location of a shared mapping 1366 */ 1367 if (capabilities & BDI_CAP_MAP_DIRECT) { 1368 addr = file->f_op->get_unmapped_area(file, addr, len, 1369 pgoff, flags); 1370 if (IS_ERR_VALUE(addr)) { 1371 ret = addr; 1372 if (ret != -ENOSYS) 1373 goto error_just_free; 1374 1375 /* the driver refused to tell us where to site 1376 * the mapping so we'll have to attempt to copy 1377 * it */ 1378 ret = -ENODEV; 1379 if (!(capabilities & BDI_CAP_MAP_COPY)) 1380 goto error_just_free; 1381 1382 capabilities &= ~BDI_CAP_MAP_DIRECT; 1383 } else { 1384 vma->vm_start = region->vm_start = addr; 1385 vma->vm_end = region->vm_end = addr + len; 1386 } 1387 } 1388 } 1389 1390 vma->vm_region = region; 1391 1392 /* set up the mapping 1393 * - the region is filled in if BDI_CAP_MAP_DIRECT is still set 1394 */ 1395 if (file && vma->vm_flags & VM_SHARED) 1396 ret = do_mmap_shared_file(vma); 1397 else 1398 ret = do_mmap_private(vma, region, len, capabilities); 1399 if (ret < 0) 1400 goto error_just_free; 1401 add_nommu_region(region); 1402 1403 /* clear anonymous mappings that don't ask for uninitialized data */ 1404 if (!vma->vm_file && !(flags & MAP_UNINITIALIZED)) 1405 memset((void *)region->vm_start, 0, 1406 region->vm_end - region->vm_start); 1407 1408 /* okay... we have a mapping; now we have to register it */ 1409 result = vma->vm_start; 1410 1411 current->mm->total_vm += len >> PAGE_SHIFT; 1412 1413 share: 1414 add_vma_to_mm(current->mm, vma); 1415 1416 /* we flush the region from the icache only when the first executable 1417 * mapping of it is made */ 1418 if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) { 1419 flush_icache_range(region->vm_start, region->vm_end); 1420 region->vm_icache_flushed = true; 1421 } 1422 1423 up_write(&nommu_region_sem); 1424 1425 kleave(" = %lx", result); 1426 return result; 1427 1428 error_just_free: 1429 up_write(&nommu_region_sem); 1430 error: 1431 if (region->vm_file) 1432 fput(region->vm_file); 1433 kmem_cache_free(vm_region_jar, region); 1434 if (vma->vm_file) 1435 fput(vma->vm_file); 1436 kmem_cache_free(vm_area_cachep, vma); 1437 kleave(" = %d", ret); 1438 return ret; 1439 1440 sharing_violation: 1441 up_write(&nommu_region_sem); 1442 printk(KERN_WARNING "Attempt to share mismatched mappings\n"); 1443 ret = -EINVAL; 1444 goto error; 1445 1446 error_getting_vma: 1447 kmem_cache_free(vm_region_jar, region); 1448 printk(KERN_WARNING "Allocation of vma for %lu byte allocation" 1449 " from process %d failed\n", 1450 len, current->pid); 1451 show_free_areas(0); 1452 return -ENOMEM; 1453 1454 error_getting_region: 1455 printk(KERN_WARNING "Allocation of vm region for %lu byte allocation" 1456 " from process %d failed\n", 1457 len, current->pid); 1458 show_free_areas(0); 1459 return -ENOMEM; 1460 } 1461 1462 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 1463 unsigned long, prot, unsigned long, flags, 1464 unsigned long, fd, unsigned long, pgoff) 1465 { 1466 struct file *file = NULL; 1467 unsigned long retval = -EBADF; 1468 1469 audit_mmap_fd(fd, flags); 1470 if (!(flags & MAP_ANONYMOUS)) { 1471 file = fget(fd); 1472 if (!file) 1473 goto out; 1474 } 1475 1476 flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE); 1477 1478 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 1479 1480 if (file) 1481 fput(file); 1482 out: 1483 return retval; 1484 } 1485 1486 #ifdef __ARCH_WANT_SYS_OLD_MMAP 1487 struct mmap_arg_struct { 1488 unsigned long addr; 1489 unsigned long len; 1490 unsigned long prot; 1491 unsigned long flags; 1492 unsigned long fd; 1493 unsigned long offset; 1494 }; 1495 1496 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1497 { 1498 struct mmap_arg_struct a; 1499 1500 if (copy_from_user(&a, arg, sizeof(a))) 1501 return -EFAULT; 1502 if (a.offset & ~PAGE_MASK) 1503 return -EINVAL; 1504 1505 return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1506 a.offset >> PAGE_SHIFT); 1507 } 1508 #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1509 1510 /* 1511 * split a vma into two pieces at address 'addr', a new vma is allocated either 1512 * for the first part or the tail. 1513 */ 1514 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 1515 unsigned long addr, int new_below) 1516 { 1517 struct vm_area_struct *new; 1518 struct vm_region *region; 1519 unsigned long npages; 1520 1521 kenter(""); 1522 1523 /* we're only permitted to split anonymous regions (these should have 1524 * only a single usage on the region) */ 1525 if (vma->vm_file) 1526 return -ENOMEM; 1527 1528 if (mm->map_count >= sysctl_max_map_count) 1529 return -ENOMEM; 1530 1531 region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL); 1532 if (!region) 1533 return -ENOMEM; 1534 1535 new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 1536 if (!new) { 1537 kmem_cache_free(vm_region_jar, region); 1538 return -ENOMEM; 1539 } 1540 1541 /* most fields are the same, copy all, and then fixup */ 1542 *new = *vma; 1543 *region = *vma->vm_region; 1544 new->vm_region = region; 1545 1546 npages = (addr - vma->vm_start) >> PAGE_SHIFT; 1547 1548 if (new_below) { 1549 region->vm_top = region->vm_end = new->vm_end = addr; 1550 } else { 1551 region->vm_start = new->vm_start = addr; 1552 region->vm_pgoff = new->vm_pgoff += npages; 1553 } 1554 1555 if (new->vm_ops && new->vm_ops->open) 1556 new->vm_ops->open(new); 1557 1558 delete_vma_from_mm(vma); 1559 down_write(&nommu_region_sem); 1560 delete_nommu_region(vma->vm_region); 1561 if (new_below) { 1562 vma->vm_region->vm_start = vma->vm_start = addr; 1563 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; 1564 } else { 1565 vma->vm_region->vm_end = vma->vm_end = addr; 1566 vma->vm_region->vm_top = addr; 1567 } 1568 add_nommu_region(vma->vm_region); 1569 add_nommu_region(new->vm_region); 1570 up_write(&nommu_region_sem); 1571 add_vma_to_mm(mm, vma); 1572 add_vma_to_mm(mm, new); 1573 return 0; 1574 } 1575 1576 /* 1577 * shrink a VMA by removing the specified chunk from either the beginning or 1578 * the end 1579 */ 1580 static int shrink_vma(struct mm_struct *mm, 1581 struct vm_area_struct *vma, 1582 unsigned long from, unsigned long to) 1583 { 1584 struct vm_region *region; 1585 1586 kenter(""); 1587 1588 /* adjust the VMA's pointers, which may reposition it in the MM's tree 1589 * and list */ 1590 delete_vma_from_mm(vma); 1591 if (from > vma->vm_start) 1592 vma->vm_end = from; 1593 else 1594 vma->vm_start = to; 1595 add_vma_to_mm(mm, vma); 1596 1597 /* cut the backing region down to size */ 1598 region = vma->vm_region; 1599 BUG_ON(region->vm_usage != 1); 1600 1601 down_write(&nommu_region_sem); 1602 delete_nommu_region(region); 1603 if (from > region->vm_start) { 1604 to = region->vm_top; 1605 region->vm_top = region->vm_end = from; 1606 } else { 1607 region->vm_start = to; 1608 } 1609 add_nommu_region(region); 1610 up_write(&nommu_region_sem); 1611 1612 free_page_series(from, to); 1613 return 0; 1614 } 1615 1616 /* 1617 * release a mapping 1618 * - under NOMMU conditions the chunk to be unmapped must be backed by a single 1619 * VMA, though it need not cover the whole VMA 1620 */ 1621 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len) 1622 { 1623 struct vm_area_struct *vma; 1624 unsigned long end; 1625 int ret; 1626 1627 kenter(",%lx,%zx", start, len); 1628 1629 len = PAGE_ALIGN(len); 1630 if (len == 0) 1631 return -EINVAL; 1632 1633 end = start + len; 1634 1635 /* find the first potentially overlapping VMA */ 1636 vma = find_vma(mm, start); 1637 if (!vma) { 1638 static int limit = 0; 1639 if (limit < 5) { 1640 printk(KERN_WARNING 1641 "munmap of memory not mmapped by process %d" 1642 " (%s): 0x%lx-0x%lx\n", 1643 current->pid, current->comm, 1644 start, start + len - 1); 1645 limit++; 1646 } 1647 return -EINVAL; 1648 } 1649 1650 /* we're allowed to split an anonymous VMA but not a file-backed one */ 1651 if (vma->vm_file) { 1652 do { 1653 if (start > vma->vm_start) { 1654 kleave(" = -EINVAL [miss]"); 1655 return -EINVAL; 1656 } 1657 if (end == vma->vm_end) 1658 goto erase_whole_vma; 1659 vma = vma->vm_next; 1660 } while (vma); 1661 kleave(" = -EINVAL [split file]"); 1662 return -EINVAL; 1663 } else { 1664 /* the chunk must be a subset of the VMA found */ 1665 if (start == vma->vm_start && end == vma->vm_end) 1666 goto erase_whole_vma; 1667 if (start < vma->vm_start || end > vma->vm_end) { 1668 kleave(" = -EINVAL [superset]"); 1669 return -EINVAL; 1670 } 1671 if (start & ~PAGE_MASK) { 1672 kleave(" = -EINVAL [unaligned start]"); 1673 return -EINVAL; 1674 } 1675 if (end != vma->vm_end && end & ~PAGE_MASK) { 1676 kleave(" = -EINVAL [unaligned split]"); 1677 return -EINVAL; 1678 } 1679 if (start != vma->vm_start && end != vma->vm_end) { 1680 ret = split_vma(mm, vma, start, 1); 1681 if (ret < 0) { 1682 kleave(" = %d [split]", ret); 1683 return ret; 1684 } 1685 } 1686 return shrink_vma(mm, vma, start, end); 1687 } 1688 1689 erase_whole_vma: 1690 delete_vma_from_mm(vma); 1691 delete_vma(mm, vma); 1692 kleave(" = 0"); 1693 return 0; 1694 } 1695 EXPORT_SYMBOL(do_munmap); 1696 1697 int vm_munmap(unsigned long addr, size_t len) 1698 { 1699 struct mm_struct *mm = current->mm; 1700 int ret; 1701 1702 down_write(&mm->mmap_sem); 1703 ret = do_munmap(mm, addr, len); 1704 up_write(&mm->mmap_sem); 1705 return ret; 1706 } 1707 EXPORT_SYMBOL(vm_munmap); 1708 1709 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 1710 { 1711 return vm_munmap(addr, len); 1712 } 1713 1714 /* 1715 * release all the mappings made in a process's VM space 1716 */ 1717 void exit_mmap(struct mm_struct *mm) 1718 { 1719 struct vm_area_struct *vma; 1720 1721 if (!mm) 1722 return; 1723 1724 kenter(""); 1725 1726 mm->total_vm = 0; 1727 1728 while ((vma = mm->mmap)) { 1729 mm->mmap = vma->vm_next; 1730 delete_vma_from_mm(vma); 1731 delete_vma(mm, vma); 1732 cond_resched(); 1733 } 1734 1735 kleave(""); 1736 } 1737 1738 unsigned long vm_brk(unsigned long addr, unsigned long len) 1739 { 1740 return -ENOMEM; 1741 } 1742 1743 /* 1744 * expand (or shrink) an existing mapping, potentially moving it at the same 1745 * time (controlled by the MREMAP_MAYMOVE flag and available VM space) 1746 * 1747 * under NOMMU conditions, we only permit changing a mapping's size, and only 1748 * as long as it stays within the region allocated by do_mmap_private() and the 1749 * block is not shareable 1750 * 1751 * MREMAP_FIXED is not supported under NOMMU conditions 1752 */ 1753 unsigned long do_mremap(unsigned long addr, 1754 unsigned long old_len, unsigned long new_len, 1755 unsigned long flags, unsigned long new_addr) 1756 { 1757 struct vm_area_struct *vma; 1758 1759 /* insanity checks first */ 1760 old_len = PAGE_ALIGN(old_len); 1761 new_len = PAGE_ALIGN(new_len); 1762 if (old_len == 0 || new_len == 0) 1763 return (unsigned long) -EINVAL; 1764 1765 if (addr & ~PAGE_MASK) 1766 return -EINVAL; 1767 1768 if (flags & MREMAP_FIXED && new_addr != addr) 1769 return (unsigned long) -EINVAL; 1770 1771 vma = find_vma_exact(current->mm, addr, old_len); 1772 if (!vma) 1773 return (unsigned long) -EINVAL; 1774 1775 if (vma->vm_end != vma->vm_start + old_len) 1776 return (unsigned long) -EFAULT; 1777 1778 if (vma->vm_flags & VM_MAYSHARE) 1779 return (unsigned long) -EPERM; 1780 1781 if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start) 1782 return (unsigned long) -ENOMEM; 1783 1784 /* all checks complete - do it */ 1785 vma->vm_end = vma->vm_start + new_len; 1786 return vma->vm_start; 1787 } 1788 EXPORT_SYMBOL(do_mremap); 1789 1790 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, 1791 unsigned long, new_len, unsigned long, flags, 1792 unsigned long, new_addr) 1793 { 1794 unsigned long ret; 1795 1796 down_write(¤t->mm->mmap_sem); 1797 ret = do_mremap(addr, old_len, new_len, flags, new_addr); 1798 up_write(¤t->mm->mmap_sem); 1799 return ret; 1800 } 1801 1802 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 1803 unsigned int foll_flags) 1804 { 1805 return NULL; 1806 } 1807 1808 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 1809 unsigned long pfn, unsigned long size, pgprot_t prot) 1810 { 1811 if (addr != (pfn << PAGE_SHIFT)) 1812 return -EINVAL; 1813 1814 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1815 return 0; 1816 } 1817 EXPORT_SYMBOL(remap_pfn_range); 1818 1819 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 1820 unsigned long pgoff) 1821 { 1822 unsigned int size = vma->vm_end - vma->vm_start; 1823 1824 if (!(vma->vm_flags & VM_USERMAP)) 1825 return -EINVAL; 1826 1827 vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT)); 1828 vma->vm_end = vma->vm_start + size; 1829 1830 return 0; 1831 } 1832 EXPORT_SYMBOL(remap_vmalloc_range); 1833 1834 unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, 1835 unsigned long len, unsigned long pgoff, unsigned long flags) 1836 { 1837 return -ENOMEM; 1838 } 1839 1840 void arch_unmap_area(struct mm_struct *mm, unsigned long addr) 1841 { 1842 } 1843 1844 void unmap_mapping_range(struct address_space *mapping, 1845 loff_t const holebegin, loff_t const holelen, 1846 int even_cows) 1847 { 1848 } 1849 EXPORT_SYMBOL(unmap_mapping_range); 1850 1851 /* 1852 * Check that a process has enough memory to allocate a new virtual 1853 * mapping. 0 means there is enough memory for the allocation to 1854 * succeed and -ENOMEM implies there is not. 1855 * 1856 * We currently support three overcommit policies, which are set via the 1857 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting 1858 * 1859 * Strict overcommit modes added 2002 Feb 26 by Alan Cox. 1860 * Additional code 2002 Jul 20 by Robert Love. 1861 * 1862 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise. 1863 * 1864 * Note this is a helper function intended to be used by LSMs which 1865 * wish to use this logic. 1866 */ 1867 int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) 1868 { 1869 unsigned long free, allowed; 1870 1871 vm_acct_memory(pages); 1872 1873 /* 1874 * Sometimes we want to use more memory than we have 1875 */ 1876 if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS) 1877 return 0; 1878 1879 if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { 1880 free = global_page_state(NR_FREE_PAGES); 1881 free += global_page_state(NR_FILE_PAGES); 1882 1883 /* 1884 * shmem pages shouldn't be counted as free in this 1885 * case, they can't be purged, only swapped out, and 1886 * that won't affect the overall amount of available 1887 * memory in the system. 1888 */ 1889 free -= global_page_state(NR_SHMEM); 1890 1891 free += nr_swap_pages; 1892 1893 /* 1894 * Any slabs which are created with the 1895 * SLAB_RECLAIM_ACCOUNT flag claim to have contents 1896 * which are reclaimable, under pressure. The dentry 1897 * cache and most inode caches should fall into this 1898 */ 1899 free += global_page_state(NR_SLAB_RECLAIMABLE); 1900 1901 /* 1902 * Leave reserved pages. The pages are not for anonymous pages. 1903 */ 1904 if (free <= totalreserve_pages) 1905 goto error; 1906 else 1907 free -= totalreserve_pages; 1908 1909 /* 1910 * Leave the last 3% for root 1911 */ 1912 if (!cap_sys_admin) 1913 free -= free / 32; 1914 1915 if (free > pages) 1916 return 0; 1917 1918 goto error; 1919 } 1920 1921 allowed = totalram_pages * sysctl_overcommit_ratio / 100; 1922 /* 1923 * Leave the last 3% for root 1924 */ 1925 if (!cap_sys_admin) 1926 allowed -= allowed / 32; 1927 allowed += total_swap_pages; 1928 1929 /* Don't let a single process grow too big: 1930 leave 3% of the size of this process for other processes */ 1931 if (mm) 1932 allowed -= mm->total_vm / 32; 1933 1934 if (percpu_counter_read_positive(&vm_committed_as) < allowed) 1935 return 0; 1936 1937 error: 1938 vm_unacct_memory(pages); 1939 1940 return -ENOMEM; 1941 } 1942 1943 int in_gate_area_no_mm(unsigned long addr) 1944 { 1945 return 0; 1946 } 1947 1948 int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1949 { 1950 BUG(); 1951 return 0; 1952 } 1953 EXPORT_SYMBOL(filemap_fault); 1954 1955 int generic_file_remap_pages(struct vm_area_struct *vma, unsigned long addr, 1956 unsigned long size, pgoff_t pgoff) 1957 { 1958 BUG(); 1959 return 0; 1960 } 1961 EXPORT_SYMBOL(generic_file_remap_pages); 1962 1963 static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 1964 unsigned long addr, void *buf, int len, int write) 1965 { 1966 struct vm_area_struct *vma; 1967 1968 down_read(&mm->mmap_sem); 1969 1970 /* the access must start within one of the target process's mappings */ 1971 vma = find_vma(mm, addr); 1972 if (vma) { 1973 /* don't overrun this mapping */ 1974 if (addr + len >= vma->vm_end) 1975 len = vma->vm_end - addr; 1976 1977 /* only read or write mappings where it is permitted */ 1978 if (write && vma->vm_flags & VM_MAYWRITE) 1979 copy_to_user_page(vma, NULL, addr, 1980 (void *) addr, buf, len); 1981 else if (!write && vma->vm_flags & VM_MAYREAD) 1982 copy_from_user_page(vma, NULL, addr, 1983 buf, (void *) addr, len); 1984 else 1985 len = 0; 1986 } else { 1987 len = 0; 1988 } 1989 1990 up_read(&mm->mmap_sem); 1991 1992 return len; 1993 } 1994 1995 /** 1996 * @access_remote_vm - access another process' address space 1997 * @mm: the mm_struct of the target address space 1998 * @addr: start address to access 1999 * @buf: source or destination buffer 2000 * @len: number of bytes to transfer 2001 * @write: whether the access is a write 2002 * 2003 * The caller must hold a reference on @mm. 2004 */ 2005 int access_remote_vm(struct mm_struct *mm, unsigned long addr, 2006 void *buf, int len, int write) 2007 { 2008 return __access_remote_vm(NULL, mm, addr, buf, len, write); 2009 } 2010 2011 /* 2012 * Access another process' address space. 2013 * - source/target buffer must be kernel space 2014 */ 2015 int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) 2016 { 2017 struct mm_struct *mm; 2018 2019 if (addr + len < addr) 2020 return 0; 2021 2022 mm = get_task_mm(tsk); 2023 if (!mm) 2024 return 0; 2025 2026 len = __access_remote_vm(tsk, mm, addr, buf, len, write); 2027 2028 mmput(mm); 2029 return len; 2030 } 2031 2032 /** 2033 * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode 2034 * @inode: The inode to check 2035 * @size: The current filesize of the inode 2036 * @newsize: The proposed filesize of the inode 2037 * 2038 * Check the shared mappings on an inode on behalf of a shrinking truncate to 2039 * make sure that that any outstanding VMAs aren't broken and then shrink the 2040 * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't 2041 * automatically grant mappings that are too large. 2042 */ 2043 int nommu_shrink_inode_mappings(struct inode *inode, size_t size, 2044 size_t newsize) 2045 { 2046 struct vm_area_struct *vma; 2047 struct vm_region *region; 2048 pgoff_t low, high; 2049 size_t r_size, r_top; 2050 2051 low = newsize >> PAGE_SHIFT; 2052 high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2053 2054 down_write(&nommu_region_sem); 2055 mutex_lock(&inode->i_mapping->i_mmap_mutex); 2056 2057 /* search for VMAs that fall within the dead zone */ 2058 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) { 2059 /* found one - only interested if it's shared out of the page 2060 * cache */ 2061 if (vma->vm_flags & VM_SHARED) { 2062 mutex_unlock(&inode->i_mapping->i_mmap_mutex); 2063 up_write(&nommu_region_sem); 2064 return -ETXTBSY; /* not quite true, but near enough */ 2065 } 2066 } 2067 2068 /* reduce any regions that overlap the dead zone - if in existence, 2069 * these will be pointed to by VMAs that don't overlap the dead zone 2070 * 2071 * we don't check for any regions that start beyond the EOF as there 2072 * shouldn't be any 2073 */ 2074 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 2075 0, ULONG_MAX) { 2076 if (!(vma->vm_flags & VM_SHARED)) 2077 continue; 2078 2079 region = vma->vm_region; 2080 r_size = region->vm_top - region->vm_start; 2081 r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size; 2082 2083 if (r_top > newsize) { 2084 region->vm_top -= r_top - newsize; 2085 if (region->vm_end > region->vm_top) 2086 region->vm_end = region->vm_top; 2087 } 2088 } 2089 2090 mutex_unlock(&inode->i_mapping->i_mmap_mutex); 2091 up_write(&nommu_region_sem); 2092 return 0; 2093 } 2094