1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/export.h> 33 #include <linux/swap.h> 34 #include <linux/uio.h> 35 #include <linux/khugepaged.h> 36 37 static struct vfsmount *shm_mnt; 38 39 #ifdef CONFIG_SHMEM 40 /* 41 * This virtual memory filesystem is heavily based on the ramfs. It 42 * extends ramfs by the ability to use swap and honor resource limits 43 * which makes it a completely usable filesystem. 44 */ 45 46 #include <linux/xattr.h> 47 #include <linux/exportfs.h> 48 #include <linux/posix_acl.h> 49 #include <linux/posix_acl_xattr.h> 50 #include <linux/mman.h> 51 #include <linux/string.h> 52 #include <linux/slab.h> 53 #include <linux/backing-dev.h> 54 #include <linux/shmem_fs.h> 55 #include <linux/writeback.h> 56 #include <linux/blkdev.h> 57 #include <linux/pagevec.h> 58 #include <linux/percpu_counter.h> 59 #include <linux/falloc.h> 60 #include <linux/splice.h> 61 #include <linux/security.h> 62 #include <linux/swapops.h> 63 #include <linux/mempolicy.h> 64 #include <linux/namei.h> 65 #include <linux/ctype.h> 66 #include <linux/migrate.h> 67 #include <linux/highmem.h> 68 #include <linux/seq_file.h> 69 #include <linux/magic.h> 70 #include <linux/syscalls.h> 71 #include <linux/fcntl.h> 72 #include <uapi/linux/memfd.h> 73 74 #include <asm/uaccess.h> 75 #include <asm/pgtable.h> 76 77 #include "internal.h" 78 79 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 80 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 81 82 /* Pretend that each entry is of this size in directory's i_size */ 83 #define BOGO_DIRENT_SIZE 20 84 85 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 86 #define SHORT_SYMLINK_LEN 128 87 88 /* 89 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 90 * inode->i_private (with i_mutex making sure that it has only one user at 91 * a time): we would prefer not to enlarge the shmem inode just for that. 92 */ 93 struct shmem_falloc { 94 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 95 pgoff_t start; /* start of range currently being fallocated */ 96 pgoff_t next; /* the next page offset to be fallocated */ 97 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 98 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 99 }; 100 101 #ifdef CONFIG_TMPFS 102 static unsigned long shmem_default_max_blocks(void) 103 { 104 return totalram_pages / 2; 105 } 106 107 static unsigned long shmem_default_max_inodes(void) 108 { 109 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 110 } 111 #endif 112 113 static bool shmem_should_replace_page(struct page *page, gfp_t gfp); 114 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 115 struct shmem_inode_info *info, pgoff_t index); 116 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 117 struct page **pagep, enum sgp_type sgp, 118 gfp_t gfp, struct mm_struct *fault_mm, int *fault_type); 119 120 int shmem_getpage(struct inode *inode, pgoff_t index, 121 struct page **pagep, enum sgp_type sgp) 122 { 123 return shmem_getpage_gfp(inode, index, pagep, sgp, 124 mapping_gfp_mask(inode->i_mapping), NULL, NULL); 125 } 126 127 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 128 { 129 return sb->s_fs_info; 130 } 131 132 /* 133 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 134 * for shared memory and for shared anonymous (/dev/zero) mappings 135 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 136 * consistent with the pre-accounting of private mappings ... 137 */ 138 static inline int shmem_acct_size(unsigned long flags, loff_t size) 139 { 140 return (flags & VM_NORESERVE) ? 141 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 142 } 143 144 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 145 { 146 if (!(flags & VM_NORESERVE)) 147 vm_unacct_memory(VM_ACCT(size)); 148 } 149 150 static inline int shmem_reacct_size(unsigned long flags, 151 loff_t oldsize, loff_t newsize) 152 { 153 if (!(flags & VM_NORESERVE)) { 154 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 155 return security_vm_enough_memory_mm(current->mm, 156 VM_ACCT(newsize) - VM_ACCT(oldsize)); 157 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 158 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 159 } 160 return 0; 161 } 162 163 /* 164 * ... whereas tmpfs objects are accounted incrementally as 165 * pages are allocated, in order to allow large sparse files. 166 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 167 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 168 */ 169 static inline int shmem_acct_block(unsigned long flags, long pages) 170 { 171 if (!(flags & VM_NORESERVE)) 172 return 0; 173 174 return security_vm_enough_memory_mm(current->mm, 175 pages * VM_ACCT(PAGE_SIZE)); 176 } 177 178 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 179 { 180 if (flags & VM_NORESERVE) 181 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 182 } 183 184 static const struct super_operations shmem_ops; 185 static const struct address_space_operations shmem_aops; 186 static const struct file_operations shmem_file_operations; 187 static const struct inode_operations shmem_inode_operations; 188 static const struct inode_operations shmem_dir_inode_operations; 189 static const struct inode_operations shmem_special_inode_operations; 190 static const struct vm_operations_struct shmem_vm_ops; 191 static struct file_system_type shmem_fs_type; 192 193 static LIST_HEAD(shmem_swaplist); 194 static DEFINE_MUTEX(shmem_swaplist_mutex); 195 196 static int shmem_reserve_inode(struct super_block *sb) 197 { 198 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 199 if (sbinfo->max_inodes) { 200 spin_lock(&sbinfo->stat_lock); 201 if (!sbinfo->free_inodes) { 202 spin_unlock(&sbinfo->stat_lock); 203 return -ENOSPC; 204 } 205 sbinfo->free_inodes--; 206 spin_unlock(&sbinfo->stat_lock); 207 } 208 return 0; 209 } 210 211 static void shmem_free_inode(struct super_block *sb) 212 { 213 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 214 if (sbinfo->max_inodes) { 215 spin_lock(&sbinfo->stat_lock); 216 sbinfo->free_inodes++; 217 spin_unlock(&sbinfo->stat_lock); 218 } 219 } 220 221 /** 222 * shmem_recalc_inode - recalculate the block usage of an inode 223 * @inode: inode to recalc 224 * 225 * We have to calculate the free blocks since the mm can drop 226 * undirtied hole pages behind our back. 227 * 228 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 229 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 230 * 231 * It has to be called with the spinlock held. 232 */ 233 static void shmem_recalc_inode(struct inode *inode) 234 { 235 struct shmem_inode_info *info = SHMEM_I(inode); 236 long freed; 237 238 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 239 if (freed > 0) { 240 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 241 if (sbinfo->max_blocks) 242 percpu_counter_add(&sbinfo->used_blocks, -freed); 243 info->alloced -= freed; 244 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 245 shmem_unacct_blocks(info->flags, freed); 246 } 247 } 248 249 bool shmem_charge(struct inode *inode, long pages) 250 { 251 struct shmem_inode_info *info = SHMEM_I(inode); 252 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 253 unsigned long flags; 254 255 if (shmem_acct_block(info->flags, pages)) 256 return false; 257 spin_lock_irqsave(&info->lock, flags); 258 info->alloced += pages; 259 inode->i_blocks += pages * BLOCKS_PER_PAGE; 260 shmem_recalc_inode(inode); 261 spin_unlock_irqrestore(&info->lock, flags); 262 inode->i_mapping->nrpages += pages; 263 264 if (!sbinfo->max_blocks) 265 return true; 266 if (percpu_counter_compare(&sbinfo->used_blocks, 267 sbinfo->max_blocks - pages) > 0) { 268 inode->i_mapping->nrpages -= pages; 269 spin_lock_irqsave(&info->lock, flags); 270 info->alloced -= pages; 271 shmem_recalc_inode(inode); 272 spin_unlock_irqrestore(&info->lock, flags); 273 shmem_unacct_blocks(info->flags, pages); 274 return false; 275 } 276 percpu_counter_add(&sbinfo->used_blocks, pages); 277 return true; 278 } 279 280 void shmem_uncharge(struct inode *inode, long pages) 281 { 282 struct shmem_inode_info *info = SHMEM_I(inode); 283 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 284 unsigned long flags; 285 286 spin_lock_irqsave(&info->lock, flags); 287 info->alloced -= pages; 288 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 289 shmem_recalc_inode(inode); 290 spin_unlock_irqrestore(&info->lock, flags); 291 292 if (sbinfo->max_blocks) 293 percpu_counter_sub(&sbinfo->used_blocks, pages); 294 shmem_unacct_blocks(info->flags, pages); 295 } 296 297 /* 298 * Replace item expected in radix tree by a new item, while holding tree lock. 299 */ 300 static int shmem_radix_tree_replace(struct address_space *mapping, 301 pgoff_t index, void *expected, void *replacement) 302 { 303 void **pslot; 304 void *item; 305 306 VM_BUG_ON(!expected); 307 VM_BUG_ON(!replacement); 308 pslot = radix_tree_lookup_slot(&mapping->page_tree, index); 309 if (!pslot) 310 return -ENOENT; 311 item = radix_tree_deref_slot_protected(pslot, &mapping->tree_lock); 312 if (item != expected) 313 return -ENOENT; 314 radix_tree_replace_slot(pslot, replacement); 315 return 0; 316 } 317 318 /* 319 * Sometimes, before we decide whether to proceed or to fail, we must check 320 * that an entry was not already brought back from swap by a racing thread. 321 * 322 * Checking page is not enough: by the time a SwapCache page is locked, it 323 * might be reused, and again be SwapCache, using the same swap as before. 324 */ 325 static bool shmem_confirm_swap(struct address_space *mapping, 326 pgoff_t index, swp_entry_t swap) 327 { 328 void *item; 329 330 rcu_read_lock(); 331 item = radix_tree_lookup(&mapping->page_tree, index); 332 rcu_read_unlock(); 333 return item == swp_to_radix_entry(swap); 334 } 335 336 /* 337 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 338 * 339 * SHMEM_HUGE_NEVER: 340 * disables huge pages for the mount; 341 * SHMEM_HUGE_ALWAYS: 342 * enables huge pages for the mount; 343 * SHMEM_HUGE_WITHIN_SIZE: 344 * only allocate huge pages if the page will be fully within i_size, 345 * also respect fadvise()/madvise() hints; 346 * SHMEM_HUGE_ADVISE: 347 * only allocate huge pages if requested with fadvise()/madvise(); 348 */ 349 350 #define SHMEM_HUGE_NEVER 0 351 #define SHMEM_HUGE_ALWAYS 1 352 #define SHMEM_HUGE_WITHIN_SIZE 2 353 #define SHMEM_HUGE_ADVISE 3 354 355 /* 356 * Special values. 357 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 358 * 359 * SHMEM_HUGE_DENY: 360 * disables huge on shm_mnt and all mounts, for emergency use; 361 * SHMEM_HUGE_FORCE: 362 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 363 * 364 */ 365 #define SHMEM_HUGE_DENY (-1) 366 #define SHMEM_HUGE_FORCE (-2) 367 368 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 369 /* ifdef here to avoid bloating shmem.o when not necessary */ 370 371 int shmem_huge __read_mostly; 372 373 static int shmem_parse_huge(const char *str) 374 { 375 if (!strcmp(str, "never")) 376 return SHMEM_HUGE_NEVER; 377 if (!strcmp(str, "always")) 378 return SHMEM_HUGE_ALWAYS; 379 if (!strcmp(str, "within_size")) 380 return SHMEM_HUGE_WITHIN_SIZE; 381 if (!strcmp(str, "advise")) 382 return SHMEM_HUGE_ADVISE; 383 if (!strcmp(str, "deny")) 384 return SHMEM_HUGE_DENY; 385 if (!strcmp(str, "force")) 386 return SHMEM_HUGE_FORCE; 387 return -EINVAL; 388 } 389 390 static const char *shmem_format_huge(int huge) 391 { 392 switch (huge) { 393 case SHMEM_HUGE_NEVER: 394 return "never"; 395 case SHMEM_HUGE_ALWAYS: 396 return "always"; 397 case SHMEM_HUGE_WITHIN_SIZE: 398 return "within_size"; 399 case SHMEM_HUGE_ADVISE: 400 return "advise"; 401 case SHMEM_HUGE_DENY: 402 return "deny"; 403 case SHMEM_HUGE_FORCE: 404 return "force"; 405 default: 406 VM_BUG_ON(1); 407 return "bad_val"; 408 } 409 } 410 411 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 412 struct shrink_control *sc, unsigned long nr_to_split) 413 { 414 LIST_HEAD(list), *pos, *next; 415 struct inode *inode; 416 struct shmem_inode_info *info; 417 struct page *page; 418 unsigned long batch = sc ? sc->nr_to_scan : 128; 419 int removed = 0, split = 0; 420 421 if (list_empty(&sbinfo->shrinklist)) 422 return SHRINK_STOP; 423 424 spin_lock(&sbinfo->shrinklist_lock); 425 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 426 info = list_entry(pos, struct shmem_inode_info, shrinklist); 427 428 /* pin the inode */ 429 inode = igrab(&info->vfs_inode); 430 431 /* inode is about to be evicted */ 432 if (!inode) { 433 list_del_init(&info->shrinklist); 434 removed++; 435 goto next; 436 } 437 438 /* Check if there's anything to gain */ 439 if (round_up(inode->i_size, PAGE_SIZE) == 440 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 441 list_del_init(&info->shrinklist); 442 removed++; 443 iput(inode); 444 goto next; 445 } 446 447 list_move(&info->shrinklist, &list); 448 next: 449 if (!--batch) 450 break; 451 } 452 spin_unlock(&sbinfo->shrinklist_lock); 453 454 list_for_each_safe(pos, next, &list) { 455 int ret; 456 457 info = list_entry(pos, struct shmem_inode_info, shrinklist); 458 inode = &info->vfs_inode; 459 460 if (nr_to_split && split >= nr_to_split) { 461 iput(inode); 462 continue; 463 } 464 465 page = find_lock_page(inode->i_mapping, 466 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 467 if (!page) 468 goto drop; 469 470 if (!PageTransHuge(page)) { 471 unlock_page(page); 472 put_page(page); 473 goto drop; 474 } 475 476 ret = split_huge_page(page); 477 unlock_page(page); 478 put_page(page); 479 480 if (ret) { 481 /* split failed: leave it on the list */ 482 iput(inode); 483 continue; 484 } 485 486 split++; 487 drop: 488 list_del_init(&info->shrinklist); 489 removed++; 490 iput(inode); 491 } 492 493 spin_lock(&sbinfo->shrinklist_lock); 494 list_splice_tail(&list, &sbinfo->shrinklist); 495 sbinfo->shrinklist_len -= removed; 496 spin_unlock(&sbinfo->shrinklist_lock); 497 498 return split; 499 } 500 501 static long shmem_unused_huge_scan(struct super_block *sb, 502 struct shrink_control *sc) 503 { 504 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 505 506 if (!READ_ONCE(sbinfo->shrinklist_len)) 507 return SHRINK_STOP; 508 509 return shmem_unused_huge_shrink(sbinfo, sc, 0); 510 } 511 512 static long shmem_unused_huge_count(struct super_block *sb, 513 struct shrink_control *sc) 514 { 515 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 516 return READ_ONCE(sbinfo->shrinklist_len); 517 } 518 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 519 520 #define shmem_huge SHMEM_HUGE_DENY 521 522 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 523 struct shrink_control *sc, unsigned long nr_to_split) 524 { 525 return 0; 526 } 527 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 528 529 /* 530 * Like add_to_page_cache_locked, but error if expected item has gone. 531 */ 532 static int shmem_add_to_page_cache(struct page *page, 533 struct address_space *mapping, 534 pgoff_t index, void *expected) 535 { 536 int error, nr = hpage_nr_pages(page); 537 538 VM_BUG_ON_PAGE(PageTail(page), page); 539 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 540 VM_BUG_ON_PAGE(!PageLocked(page), page); 541 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 542 VM_BUG_ON(expected && PageTransHuge(page)); 543 544 page_ref_add(page, nr); 545 page->mapping = mapping; 546 page->index = index; 547 548 spin_lock_irq(&mapping->tree_lock); 549 if (PageTransHuge(page)) { 550 void __rcu **results; 551 pgoff_t idx; 552 int i; 553 554 error = 0; 555 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 556 &results, &idx, index, 1) && 557 idx < index + HPAGE_PMD_NR) { 558 error = -EEXIST; 559 } 560 561 if (!error) { 562 for (i = 0; i < HPAGE_PMD_NR; i++) { 563 error = radix_tree_insert(&mapping->page_tree, 564 index + i, page + i); 565 VM_BUG_ON(error); 566 } 567 count_vm_event(THP_FILE_ALLOC); 568 } 569 } else if (!expected) { 570 error = radix_tree_insert(&mapping->page_tree, index, page); 571 } else { 572 error = shmem_radix_tree_replace(mapping, index, expected, 573 page); 574 } 575 576 if (!error) { 577 mapping->nrpages += nr; 578 if (PageTransHuge(page)) 579 __inc_node_page_state(page, NR_SHMEM_THPS); 580 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 581 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 582 spin_unlock_irq(&mapping->tree_lock); 583 } else { 584 page->mapping = NULL; 585 spin_unlock_irq(&mapping->tree_lock); 586 page_ref_sub(page, nr); 587 } 588 return error; 589 } 590 591 /* 592 * Like delete_from_page_cache, but substitutes swap for page. 593 */ 594 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 595 { 596 struct address_space *mapping = page->mapping; 597 int error; 598 599 VM_BUG_ON_PAGE(PageCompound(page), page); 600 601 spin_lock_irq(&mapping->tree_lock); 602 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 603 page->mapping = NULL; 604 mapping->nrpages--; 605 __dec_node_page_state(page, NR_FILE_PAGES); 606 __dec_node_page_state(page, NR_SHMEM); 607 spin_unlock_irq(&mapping->tree_lock); 608 put_page(page); 609 BUG_ON(error); 610 } 611 612 /* 613 * Remove swap entry from radix tree, free the swap and its page cache. 614 */ 615 static int shmem_free_swap(struct address_space *mapping, 616 pgoff_t index, void *radswap) 617 { 618 void *old; 619 620 spin_lock_irq(&mapping->tree_lock); 621 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 622 spin_unlock_irq(&mapping->tree_lock); 623 if (old != radswap) 624 return -ENOENT; 625 free_swap_and_cache(radix_to_swp_entry(radswap)); 626 return 0; 627 } 628 629 /* 630 * Determine (in bytes) how many of the shmem object's pages mapped by the 631 * given offsets are swapped out. 632 * 633 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 634 * as long as the inode doesn't go away and racy results are not a problem. 635 */ 636 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 637 pgoff_t start, pgoff_t end) 638 { 639 struct radix_tree_iter iter; 640 void **slot; 641 struct page *page; 642 unsigned long swapped = 0; 643 644 rcu_read_lock(); 645 646 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 647 if (iter.index >= end) 648 break; 649 650 page = radix_tree_deref_slot(slot); 651 652 if (radix_tree_deref_retry(page)) { 653 slot = radix_tree_iter_retry(&iter); 654 continue; 655 } 656 657 if (radix_tree_exceptional_entry(page)) 658 swapped++; 659 660 if (need_resched()) { 661 cond_resched_rcu(); 662 slot = radix_tree_iter_next(&iter); 663 } 664 } 665 666 rcu_read_unlock(); 667 668 return swapped << PAGE_SHIFT; 669 } 670 671 /* 672 * Determine (in bytes) how many of the shmem object's pages mapped by the 673 * given vma is swapped out. 674 * 675 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 676 * as long as the inode doesn't go away and racy results are not a problem. 677 */ 678 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 679 { 680 struct inode *inode = file_inode(vma->vm_file); 681 struct shmem_inode_info *info = SHMEM_I(inode); 682 struct address_space *mapping = inode->i_mapping; 683 unsigned long swapped; 684 685 /* Be careful as we don't hold info->lock */ 686 swapped = READ_ONCE(info->swapped); 687 688 /* 689 * The easier cases are when the shmem object has nothing in swap, or 690 * the vma maps it whole. Then we can simply use the stats that we 691 * already track. 692 */ 693 if (!swapped) 694 return 0; 695 696 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 697 return swapped << PAGE_SHIFT; 698 699 /* Here comes the more involved part */ 700 return shmem_partial_swap_usage(mapping, 701 linear_page_index(vma, vma->vm_start), 702 linear_page_index(vma, vma->vm_end)); 703 } 704 705 /* 706 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 707 */ 708 void shmem_unlock_mapping(struct address_space *mapping) 709 { 710 struct pagevec pvec; 711 pgoff_t indices[PAGEVEC_SIZE]; 712 pgoff_t index = 0; 713 714 pagevec_init(&pvec, 0); 715 /* 716 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 717 */ 718 while (!mapping_unevictable(mapping)) { 719 /* 720 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it 721 * has finished, if it hits a row of PAGEVEC_SIZE swap entries. 722 */ 723 pvec.nr = find_get_entries(mapping, index, 724 PAGEVEC_SIZE, pvec.pages, indices); 725 if (!pvec.nr) 726 break; 727 index = indices[pvec.nr - 1] + 1; 728 pagevec_remove_exceptionals(&pvec); 729 check_move_unevictable_pages(pvec.pages, pvec.nr); 730 pagevec_release(&pvec); 731 cond_resched(); 732 } 733 } 734 735 /* 736 * Remove range of pages and swap entries from radix tree, and free them. 737 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 738 */ 739 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 740 bool unfalloc) 741 { 742 struct address_space *mapping = inode->i_mapping; 743 struct shmem_inode_info *info = SHMEM_I(inode); 744 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 745 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 746 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 747 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 748 struct pagevec pvec; 749 pgoff_t indices[PAGEVEC_SIZE]; 750 long nr_swaps_freed = 0; 751 pgoff_t index; 752 int i; 753 754 if (lend == -1) 755 end = -1; /* unsigned, so actually very big */ 756 757 pagevec_init(&pvec, 0); 758 index = start; 759 while (index < end) { 760 pvec.nr = find_get_entries(mapping, index, 761 min(end - index, (pgoff_t)PAGEVEC_SIZE), 762 pvec.pages, indices); 763 if (!pvec.nr) 764 break; 765 for (i = 0; i < pagevec_count(&pvec); i++) { 766 struct page *page = pvec.pages[i]; 767 768 index = indices[i]; 769 if (index >= end) 770 break; 771 772 if (radix_tree_exceptional_entry(page)) { 773 if (unfalloc) 774 continue; 775 nr_swaps_freed += !shmem_free_swap(mapping, 776 index, page); 777 continue; 778 } 779 780 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); 781 782 if (!trylock_page(page)) 783 continue; 784 785 if (PageTransTail(page)) { 786 /* Middle of THP: zero out the page */ 787 clear_highpage(page); 788 unlock_page(page); 789 continue; 790 } else if (PageTransHuge(page)) { 791 if (index == round_down(end, HPAGE_PMD_NR)) { 792 /* 793 * Range ends in the middle of THP: 794 * zero out the page 795 */ 796 clear_highpage(page); 797 unlock_page(page); 798 continue; 799 } 800 index += HPAGE_PMD_NR - 1; 801 i += HPAGE_PMD_NR - 1; 802 } 803 804 if (!unfalloc || !PageUptodate(page)) { 805 VM_BUG_ON_PAGE(PageTail(page), page); 806 if (page_mapping(page) == mapping) { 807 VM_BUG_ON_PAGE(PageWriteback(page), page); 808 truncate_inode_page(mapping, page); 809 } 810 } 811 unlock_page(page); 812 } 813 pagevec_remove_exceptionals(&pvec); 814 pagevec_release(&pvec); 815 cond_resched(); 816 index++; 817 } 818 819 if (partial_start) { 820 struct page *page = NULL; 821 shmem_getpage(inode, start - 1, &page, SGP_READ); 822 if (page) { 823 unsigned int top = PAGE_SIZE; 824 if (start > end) { 825 top = partial_end; 826 partial_end = 0; 827 } 828 zero_user_segment(page, partial_start, top); 829 set_page_dirty(page); 830 unlock_page(page); 831 put_page(page); 832 } 833 } 834 if (partial_end) { 835 struct page *page = NULL; 836 shmem_getpage(inode, end, &page, SGP_READ); 837 if (page) { 838 zero_user_segment(page, 0, partial_end); 839 set_page_dirty(page); 840 unlock_page(page); 841 put_page(page); 842 } 843 } 844 if (start >= end) 845 return; 846 847 index = start; 848 while (index < end) { 849 cond_resched(); 850 851 pvec.nr = find_get_entries(mapping, index, 852 min(end - index, (pgoff_t)PAGEVEC_SIZE), 853 pvec.pages, indices); 854 if (!pvec.nr) { 855 /* If all gone or hole-punch or unfalloc, we're done */ 856 if (index == start || end != -1) 857 break; 858 /* But if truncating, restart to make sure all gone */ 859 index = start; 860 continue; 861 } 862 for (i = 0; i < pagevec_count(&pvec); i++) { 863 struct page *page = pvec.pages[i]; 864 865 index = indices[i]; 866 if (index >= end) 867 break; 868 869 if (radix_tree_exceptional_entry(page)) { 870 if (unfalloc) 871 continue; 872 if (shmem_free_swap(mapping, index, page)) { 873 /* Swap was replaced by page: retry */ 874 index--; 875 break; 876 } 877 nr_swaps_freed++; 878 continue; 879 } 880 881 lock_page(page); 882 883 if (PageTransTail(page)) { 884 /* Middle of THP: zero out the page */ 885 clear_highpage(page); 886 unlock_page(page); 887 /* 888 * Partial thp truncate due 'start' in middle 889 * of THP: don't need to look on these pages 890 * again on !pvec.nr restart. 891 */ 892 if (index != round_down(end, HPAGE_PMD_NR)) 893 start++; 894 continue; 895 } else if (PageTransHuge(page)) { 896 if (index == round_down(end, HPAGE_PMD_NR)) { 897 /* 898 * Range ends in the middle of THP: 899 * zero out the page 900 */ 901 clear_highpage(page); 902 unlock_page(page); 903 continue; 904 } 905 index += HPAGE_PMD_NR - 1; 906 i += HPAGE_PMD_NR - 1; 907 } 908 909 if (!unfalloc || !PageUptodate(page)) { 910 VM_BUG_ON_PAGE(PageTail(page), page); 911 if (page_mapping(page) == mapping) { 912 VM_BUG_ON_PAGE(PageWriteback(page), page); 913 truncate_inode_page(mapping, page); 914 } else { 915 /* Page was replaced by swap: retry */ 916 unlock_page(page); 917 index--; 918 break; 919 } 920 } 921 unlock_page(page); 922 } 923 pagevec_remove_exceptionals(&pvec); 924 pagevec_release(&pvec); 925 index++; 926 } 927 928 spin_lock_irq(&info->lock); 929 info->swapped -= nr_swaps_freed; 930 shmem_recalc_inode(inode); 931 spin_unlock_irq(&info->lock); 932 } 933 934 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 935 { 936 shmem_undo_range(inode, lstart, lend, false); 937 inode->i_ctime = inode->i_mtime = current_time(inode); 938 } 939 EXPORT_SYMBOL_GPL(shmem_truncate_range); 940 941 static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry, 942 struct kstat *stat) 943 { 944 struct inode *inode = dentry->d_inode; 945 struct shmem_inode_info *info = SHMEM_I(inode); 946 947 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 948 spin_lock_irq(&info->lock); 949 shmem_recalc_inode(inode); 950 spin_unlock_irq(&info->lock); 951 } 952 generic_fillattr(inode, stat); 953 return 0; 954 } 955 956 static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 957 { 958 struct inode *inode = d_inode(dentry); 959 struct shmem_inode_info *info = SHMEM_I(inode); 960 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 961 int error; 962 963 error = setattr_prepare(dentry, attr); 964 if (error) 965 return error; 966 967 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 968 loff_t oldsize = inode->i_size; 969 loff_t newsize = attr->ia_size; 970 971 /* protected by i_mutex */ 972 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 973 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 974 return -EPERM; 975 976 if (newsize != oldsize) { 977 error = shmem_reacct_size(SHMEM_I(inode)->flags, 978 oldsize, newsize); 979 if (error) 980 return error; 981 i_size_write(inode, newsize); 982 inode->i_ctime = inode->i_mtime = current_time(inode); 983 } 984 if (newsize <= oldsize) { 985 loff_t holebegin = round_up(newsize, PAGE_SIZE); 986 if (oldsize > holebegin) 987 unmap_mapping_range(inode->i_mapping, 988 holebegin, 0, 1); 989 if (info->alloced) 990 shmem_truncate_range(inode, 991 newsize, (loff_t)-1); 992 /* unmap again to remove racily COWed private pages */ 993 if (oldsize > holebegin) 994 unmap_mapping_range(inode->i_mapping, 995 holebegin, 0, 1); 996 997 /* 998 * Part of the huge page can be beyond i_size: subject 999 * to shrink under memory pressure. 1000 */ 1001 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 1002 spin_lock(&sbinfo->shrinklist_lock); 1003 if (list_empty(&info->shrinklist)) { 1004 list_add_tail(&info->shrinklist, 1005 &sbinfo->shrinklist); 1006 sbinfo->shrinklist_len++; 1007 } 1008 spin_unlock(&sbinfo->shrinklist_lock); 1009 } 1010 } 1011 } 1012 1013 setattr_copy(inode, attr); 1014 if (attr->ia_valid & ATTR_MODE) 1015 error = posix_acl_chmod(inode, inode->i_mode); 1016 return error; 1017 } 1018 1019 static void shmem_evict_inode(struct inode *inode) 1020 { 1021 struct shmem_inode_info *info = SHMEM_I(inode); 1022 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1023 1024 if (inode->i_mapping->a_ops == &shmem_aops) { 1025 shmem_unacct_size(info->flags, inode->i_size); 1026 inode->i_size = 0; 1027 shmem_truncate_range(inode, 0, (loff_t)-1); 1028 if (!list_empty(&info->shrinklist)) { 1029 spin_lock(&sbinfo->shrinklist_lock); 1030 if (!list_empty(&info->shrinklist)) { 1031 list_del_init(&info->shrinklist); 1032 sbinfo->shrinklist_len--; 1033 } 1034 spin_unlock(&sbinfo->shrinklist_lock); 1035 } 1036 if (!list_empty(&info->swaplist)) { 1037 mutex_lock(&shmem_swaplist_mutex); 1038 list_del_init(&info->swaplist); 1039 mutex_unlock(&shmem_swaplist_mutex); 1040 } 1041 } 1042 1043 simple_xattrs_free(&info->xattrs); 1044 WARN_ON(inode->i_blocks); 1045 shmem_free_inode(inode->i_sb); 1046 clear_inode(inode); 1047 } 1048 1049 /* 1050 * If swap found in inode, free it and move page from swapcache to filecache. 1051 */ 1052 static int shmem_unuse_inode(struct shmem_inode_info *info, 1053 swp_entry_t swap, struct page **pagep) 1054 { 1055 struct address_space *mapping = info->vfs_inode.i_mapping; 1056 void *radswap; 1057 pgoff_t index; 1058 gfp_t gfp; 1059 int error = 0; 1060 1061 radswap = swp_to_radix_entry(swap); 1062 index = radix_tree_locate_item(&mapping->page_tree, radswap); 1063 if (index == -1) 1064 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1065 1066 /* 1067 * Move _head_ to start search for next from here. 1068 * But be careful: shmem_evict_inode checks list_empty without taking 1069 * mutex, and there's an instant in list_move_tail when info->swaplist 1070 * would appear empty, if it were the only one on shmem_swaplist. 1071 */ 1072 if (shmem_swaplist.next != &info->swaplist) 1073 list_move_tail(&shmem_swaplist, &info->swaplist); 1074 1075 gfp = mapping_gfp_mask(mapping); 1076 if (shmem_should_replace_page(*pagep, gfp)) { 1077 mutex_unlock(&shmem_swaplist_mutex); 1078 error = shmem_replace_page(pagep, gfp, info, index); 1079 mutex_lock(&shmem_swaplist_mutex); 1080 /* 1081 * We needed to drop mutex to make that restrictive page 1082 * allocation, but the inode might have been freed while we 1083 * dropped it: although a racing shmem_evict_inode() cannot 1084 * complete without emptying the radix_tree, our page lock 1085 * on this swapcache page is not enough to prevent that - 1086 * free_swap_and_cache() of our swap entry will only 1087 * trylock_page(), removing swap from radix_tree whatever. 1088 * 1089 * We must not proceed to shmem_add_to_page_cache() if the 1090 * inode has been freed, but of course we cannot rely on 1091 * inode or mapping or info to check that. However, we can 1092 * safely check if our swap entry is still in use (and here 1093 * it can't have got reused for another page): if it's still 1094 * in use, then the inode cannot have been freed yet, and we 1095 * can safely proceed (if it's no longer in use, that tells 1096 * nothing about the inode, but we don't need to unuse swap). 1097 */ 1098 if (!page_swapcount(*pagep)) 1099 error = -ENOENT; 1100 } 1101 1102 /* 1103 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 1104 * but also to hold up shmem_evict_inode(): so inode cannot be freed 1105 * beneath us (pagelock doesn't help until the page is in pagecache). 1106 */ 1107 if (!error) 1108 error = shmem_add_to_page_cache(*pagep, mapping, index, 1109 radswap); 1110 if (error != -ENOMEM) { 1111 /* 1112 * Truncation and eviction use free_swap_and_cache(), which 1113 * only does trylock page: if we raced, best clean up here. 1114 */ 1115 delete_from_swap_cache(*pagep); 1116 set_page_dirty(*pagep); 1117 if (!error) { 1118 spin_lock_irq(&info->lock); 1119 info->swapped--; 1120 spin_unlock_irq(&info->lock); 1121 swap_free(swap); 1122 } 1123 } 1124 return error; 1125 } 1126 1127 /* 1128 * Search through swapped inodes to find and replace swap by page. 1129 */ 1130 int shmem_unuse(swp_entry_t swap, struct page *page) 1131 { 1132 struct list_head *this, *next; 1133 struct shmem_inode_info *info; 1134 struct mem_cgroup *memcg; 1135 int error = 0; 1136 1137 /* 1138 * There's a faint possibility that swap page was replaced before 1139 * caller locked it: caller will come back later with the right page. 1140 */ 1141 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) 1142 goto out; 1143 1144 /* 1145 * Charge page using GFP_KERNEL while we can wait, before taking 1146 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1147 * Charged back to the user (not to caller) when swap account is used. 1148 */ 1149 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg, 1150 false); 1151 if (error) 1152 goto out; 1153 /* No radix_tree_preload: swap entry keeps a place for page in tree */ 1154 error = -EAGAIN; 1155 1156 mutex_lock(&shmem_swaplist_mutex); 1157 list_for_each_safe(this, next, &shmem_swaplist) { 1158 info = list_entry(this, struct shmem_inode_info, swaplist); 1159 if (info->swapped) 1160 error = shmem_unuse_inode(info, swap, &page); 1161 else 1162 list_del_init(&info->swaplist); 1163 cond_resched(); 1164 if (error != -EAGAIN) 1165 break; 1166 /* found nothing in this: move on to search the next */ 1167 } 1168 mutex_unlock(&shmem_swaplist_mutex); 1169 1170 if (error) { 1171 if (error != -ENOMEM) 1172 error = 0; 1173 mem_cgroup_cancel_charge(page, memcg, false); 1174 } else 1175 mem_cgroup_commit_charge(page, memcg, true, false); 1176 out: 1177 unlock_page(page); 1178 put_page(page); 1179 return error; 1180 } 1181 1182 /* 1183 * Move the page from the page cache to the swap cache. 1184 */ 1185 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1186 { 1187 struct shmem_inode_info *info; 1188 struct address_space *mapping; 1189 struct inode *inode; 1190 swp_entry_t swap; 1191 pgoff_t index; 1192 1193 VM_BUG_ON_PAGE(PageCompound(page), page); 1194 BUG_ON(!PageLocked(page)); 1195 mapping = page->mapping; 1196 index = page->index; 1197 inode = mapping->host; 1198 info = SHMEM_I(inode); 1199 if (info->flags & VM_LOCKED) 1200 goto redirty; 1201 if (!total_swap_pages) 1202 goto redirty; 1203 1204 /* 1205 * Our capabilities prevent regular writeback or sync from ever calling 1206 * shmem_writepage; but a stacking filesystem might use ->writepage of 1207 * its underlying filesystem, in which case tmpfs should write out to 1208 * swap only in response to memory pressure, and not for the writeback 1209 * threads or sync. 1210 */ 1211 if (!wbc->for_reclaim) { 1212 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1213 goto redirty; 1214 } 1215 1216 /* 1217 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1218 * value into swapfile.c, the only way we can correctly account for a 1219 * fallocated page arriving here is now to initialize it and write it. 1220 * 1221 * That's okay for a page already fallocated earlier, but if we have 1222 * not yet completed the fallocation, then (a) we want to keep track 1223 * of this page in case we have to undo it, and (b) it may not be a 1224 * good idea to continue anyway, once we're pushing into swap. So 1225 * reactivate the page, and let shmem_fallocate() quit when too many. 1226 */ 1227 if (!PageUptodate(page)) { 1228 if (inode->i_private) { 1229 struct shmem_falloc *shmem_falloc; 1230 spin_lock(&inode->i_lock); 1231 shmem_falloc = inode->i_private; 1232 if (shmem_falloc && 1233 !shmem_falloc->waitq && 1234 index >= shmem_falloc->start && 1235 index < shmem_falloc->next) 1236 shmem_falloc->nr_unswapped++; 1237 else 1238 shmem_falloc = NULL; 1239 spin_unlock(&inode->i_lock); 1240 if (shmem_falloc) 1241 goto redirty; 1242 } 1243 clear_highpage(page); 1244 flush_dcache_page(page); 1245 SetPageUptodate(page); 1246 } 1247 1248 swap = get_swap_page(); 1249 if (!swap.val) 1250 goto redirty; 1251 1252 if (mem_cgroup_try_charge_swap(page, swap)) 1253 goto free_swap; 1254 1255 /* 1256 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1257 * if it's not already there. Do it now before the page is 1258 * moved to swap cache, when its pagelock no longer protects 1259 * the inode from eviction. But don't unlock the mutex until 1260 * we've incremented swapped, because shmem_unuse_inode() will 1261 * prune a !swapped inode from the swaplist under this mutex. 1262 */ 1263 mutex_lock(&shmem_swaplist_mutex); 1264 if (list_empty(&info->swaplist)) 1265 list_add_tail(&info->swaplist, &shmem_swaplist); 1266 1267 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1268 spin_lock_irq(&info->lock); 1269 shmem_recalc_inode(inode); 1270 info->swapped++; 1271 spin_unlock_irq(&info->lock); 1272 1273 swap_shmem_alloc(swap); 1274 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1275 1276 mutex_unlock(&shmem_swaplist_mutex); 1277 BUG_ON(page_mapped(page)); 1278 swap_writepage(page, wbc); 1279 return 0; 1280 } 1281 1282 mutex_unlock(&shmem_swaplist_mutex); 1283 free_swap: 1284 swapcache_free(swap); 1285 redirty: 1286 set_page_dirty(page); 1287 if (wbc->for_reclaim) 1288 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1289 unlock_page(page); 1290 return 0; 1291 } 1292 1293 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1294 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1295 { 1296 char buffer[64]; 1297 1298 if (!mpol || mpol->mode == MPOL_DEFAULT) 1299 return; /* show nothing */ 1300 1301 mpol_to_str(buffer, sizeof(buffer), mpol); 1302 1303 seq_printf(seq, ",mpol=%s", buffer); 1304 } 1305 1306 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1307 { 1308 struct mempolicy *mpol = NULL; 1309 if (sbinfo->mpol) { 1310 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1311 mpol = sbinfo->mpol; 1312 mpol_get(mpol); 1313 spin_unlock(&sbinfo->stat_lock); 1314 } 1315 return mpol; 1316 } 1317 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1318 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1319 { 1320 } 1321 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1322 { 1323 return NULL; 1324 } 1325 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1326 #ifndef CONFIG_NUMA 1327 #define vm_policy vm_private_data 1328 #endif 1329 1330 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1331 struct shmem_inode_info *info, pgoff_t index) 1332 { 1333 /* Create a pseudo vma that just contains the policy */ 1334 vma->vm_start = 0; 1335 /* Bias interleave by inode number to distribute better across nodes */ 1336 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1337 vma->vm_ops = NULL; 1338 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1339 } 1340 1341 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1342 { 1343 /* Drop reference taken by mpol_shared_policy_lookup() */ 1344 mpol_cond_put(vma->vm_policy); 1345 } 1346 1347 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1348 struct shmem_inode_info *info, pgoff_t index) 1349 { 1350 struct vm_area_struct pvma; 1351 struct page *page; 1352 1353 shmem_pseudo_vma_init(&pvma, info, index); 1354 page = swapin_readahead(swap, gfp, &pvma, 0); 1355 shmem_pseudo_vma_destroy(&pvma); 1356 1357 return page; 1358 } 1359 1360 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1361 struct shmem_inode_info *info, pgoff_t index) 1362 { 1363 struct vm_area_struct pvma; 1364 struct inode *inode = &info->vfs_inode; 1365 struct address_space *mapping = inode->i_mapping; 1366 pgoff_t idx, hindex; 1367 void __rcu **results; 1368 struct page *page; 1369 1370 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1371 return NULL; 1372 1373 hindex = round_down(index, HPAGE_PMD_NR); 1374 rcu_read_lock(); 1375 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1376 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1377 rcu_read_unlock(); 1378 return NULL; 1379 } 1380 rcu_read_unlock(); 1381 1382 shmem_pseudo_vma_init(&pvma, info, hindex); 1383 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, 1384 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); 1385 shmem_pseudo_vma_destroy(&pvma); 1386 if (page) 1387 prep_transhuge_page(page); 1388 return page; 1389 } 1390 1391 static struct page *shmem_alloc_page(gfp_t gfp, 1392 struct shmem_inode_info *info, pgoff_t index) 1393 { 1394 struct vm_area_struct pvma; 1395 struct page *page; 1396 1397 shmem_pseudo_vma_init(&pvma, info, index); 1398 page = alloc_page_vma(gfp, &pvma, 0); 1399 shmem_pseudo_vma_destroy(&pvma); 1400 1401 return page; 1402 } 1403 1404 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1405 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo, 1406 pgoff_t index, bool huge) 1407 { 1408 struct page *page; 1409 int nr; 1410 int err = -ENOSPC; 1411 1412 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1413 huge = false; 1414 nr = huge ? HPAGE_PMD_NR : 1; 1415 1416 if (shmem_acct_block(info->flags, nr)) 1417 goto failed; 1418 if (sbinfo->max_blocks) { 1419 if (percpu_counter_compare(&sbinfo->used_blocks, 1420 sbinfo->max_blocks - nr) > 0) 1421 goto unacct; 1422 percpu_counter_add(&sbinfo->used_blocks, nr); 1423 } 1424 1425 if (huge) 1426 page = shmem_alloc_hugepage(gfp, info, index); 1427 else 1428 page = shmem_alloc_page(gfp, info, index); 1429 if (page) { 1430 __SetPageLocked(page); 1431 __SetPageSwapBacked(page); 1432 return page; 1433 } 1434 1435 err = -ENOMEM; 1436 if (sbinfo->max_blocks) 1437 percpu_counter_add(&sbinfo->used_blocks, -nr); 1438 unacct: 1439 shmem_unacct_blocks(info->flags, nr); 1440 failed: 1441 return ERR_PTR(err); 1442 } 1443 1444 /* 1445 * When a page is moved from swapcache to shmem filecache (either by the 1446 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1447 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1448 * ignorance of the mapping it belongs to. If that mapping has special 1449 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1450 * we may need to copy to a suitable page before moving to filecache. 1451 * 1452 * In a future release, this may well be extended to respect cpuset and 1453 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1454 * but for now it is a simple matter of zone. 1455 */ 1456 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1457 { 1458 return page_zonenum(page) > gfp_zone(gfp); 1459 } 1460 1461 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1462 struct shmem_inode_info *info, pgoff_t index) 1463 { 1464 struct page *oldpage, *newpage; 1465 struct address_space *swap_mapping; 1466 pgoff_t swap_index; 1467 int error; 1468 1469 oldpage = *pagep; 1470 swap_index = page_private(oldpage); 1471 swap_mapping = page_mapping(oldpage); 1472 1473 /* 1474 * We have arrived here because our zones are constrained, so don't 1475 * limit chance of success by further cpuset and node constraints. 1476 */ 1477 gfp &= ~GFP_CONSTRAINT_MASK; 1478 newpage = shmem_alloc_page(gfp, info, index); 1479 if (!newpage) 1480 return -ENOMEM; 1481 1482 get_page(newpage); 1483 copy_highpage(newpage, oldpage); 1484 flush_dcache_page(newpage); 1485 1486 __SetPageLocked(newpage); 1487 __SetPageSwapBacked(newpage); 1488 SetPageUptodate(newpage); 1489 set_page_private(newpage, swap_index); 1490 SetPageSwapCache(newpage); 1491 1492 /* 1493 * Our caller will very soon move newpage out of swapcache, but it's 1494 * a nice clean interface for us to replace oldpage by newpage there. 1495 */ 1496 spin_lock_irq(&swap_mapping->tree_lock); 1497 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1498 newpage); 1499 if (!error) { 1500 __inc_node_page_state(newpage, NR_FILE_PAGES); 1501 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1502 } 1503 spin_unlock_irq(&swap_mapping->tree_lock); 1504 1505 if (unlikely(error)) { 1506 /* 1507 * Is this possible? I think not, now that our callers check 1508 * both PageSwapCache and page_private after getting page lock; 1509 * but be defensive. Reverse old to newpage for clear and free. 1510 */ 1511 oldpage = newpage; 1512 } else { 1513 mem_cgroup_migrate(oldpage, newpage); 1514 lru_cache_add_anon(newpage); 1515 *pagep = newpage; 1516 } 1517 1518 ClearPageSwapCache(oldpage); 1519 set_page_private(oldpage, 0); 1520 1521 unlock_page(oldpage); 1522 put_page(oldpage); 1523 put_page(oldpage); 1524 return error; 1525 } 1526 1527 /* 1528 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1529 * 1530 * If we allocate a new one we do not mark it dirty. That's up to the 1531 * vm. If we swap it in we mark it dirty since we also free the swap 1532 * entry since a page cannot live in both the swap and page cache. 1533 * 1534 * fault_mm and fault_type are only supplied by shmem_fault: 1535 * otherwise they are NULL. 1536 */ 1537 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1538 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1539 struct mm_struct *fault_mm, int *fault_type) 1540 { 1541 struct address_space *mapping = inode->i_mapping; 1542 struct shmem_inode_info *info; 1543 struct shmem_sb_info *sbinfo; 1544 struct mm_struct *charge_mm; 1545 struct mem_cgroup *memcg; 1546 struct page *page; 1547 swp_entry_t swap; 1548 enum sgp_type sgp_huge = sgp; 1549 pgoff_t hindex = index; 1550 int error; 1551 int once = 0; 1552 int alloced = 0; 1553 1554 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1555 return -EFBIG; 1556 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) 1557 sgp = SGP_CACHE; 1558 repeat: 1559 swap.val = 0; 1560 page = find_lock_entry(mapping, index); 1561 if (radix_tree_exceptional_entry(page)) { 1562 swap = radix_to_swp_entry(page); 1563 page = NULL; 1564 } 1565 1566 if (sgp <= SGP_CACHE && 1567 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1568 error = -EINVAL; 1569 goto unlock; 1570 } 1571 1572 if (page && sgp == SGP_WRITE) 1573 mark_page_accessed(page); 1574 1575 /* fallocated page? */ 1576 if (page && !PageUptodate(page)) { 1577 if (sgp != SGP_READ) 1578 goto clear; 1579 unlock_page(page); 1580 put_page(page); 1581 page = NULL; 1582 } 1583 if (page || (sgp == SGP_READ && !swap.val)) { 1584 *pagep = page; 1585 return 0; 1586 } 1587 1588 /* 1589 * Fast cache lookup did not find it: 1590 * bring it back from swap or allocate. 1591 */ 1592 info = SHMEM_I(inode); 1593 sbinfo = SHMEM_SB(inode->i_sb); 1594 charge_mm = fault_mm ? : current->mm; 1595 1596 if (swap.val) { 1597 /* Look it up and read it in.. */ 1598 page = lookup_swap_cache(swap); 1599 if (!page) { 1600 /* Or update major stats only when swapin succeeds?? */ 1601 if (fault_type) { 1602 *fault_type |= VM_FAULT_MAJOR; 1603 count_vm_event(PGMAJFAULT); 1604 mem_cgroup_count_vm_event(fault_mm, PGMAJFAULT); 1605 } 1606 /* Here we actually start the io */ 1607 page = shmem_swapin(swap, gfp, info, index); 1608 if (!page) { 1609 error = -ENOMEM; 1610 goto failed; 1611 } 1612 } 1613 1614 /* We have to do this with page locked to prevent races */ 1615 lock_page(page); 1616 if (!PageSwapCache(page) || page_private(page) != swap.val || 1617 !shmem_confirm_swap(mapping, index, swap)) { 1618 error = -EEXIST; /* try again */ 1619 goto unlock; 1620 } 1621 if (!PageUptodate(page)) { 1622 error = -EIO; 1623 goto failed; 1624 } 1625 wait_on_page_writeback(page); 1626 1627 if (shmem_should_replace_page(page, gfp)) { 1628 error = shmem_replace_page(&page, gfp, info, index); 1629 if (error) 1630 goto failed; 1631 } 1632 1633 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1634 false); 1635 if (!error) { 1636 error = shmem_add_to_page_cache(page, mapping, index, 1637 swp_to_radix_entry(swap)); 1638 /* 1639 * We already confirmed swap under page lock, and make 1640 * no memory allocation here, so usually no possibility 1641 * of error; but free_swap_and_cache() only trylocks a 1642 * page, so it is just possible that the entry has been 1643 * truncated or holepunched since swap was confirmed. 1644 * shmem_undo_range() will have done some of the 1645 * unaccounting, now delete_from_swap_cache() will do 1646 * the rest. 1647 * Reset swap.val? No, leave it so "failed" goes back to 1648 * "repeat": reading a hole and writing should succeed. 1649 */ 1650 if (error) { 1651 mem_cgroup_cancel_charge(page, memcg, false); 1652 delete_from_swap_cache(page); 1653 } 1654 } 1655 if (error) 1656 goto failed; 1657 1658 mem_cgroup_commit_charge(page, memcg, true, false); 1659 1660 spin_lock_irq(&info->lock); 1661 info->swapped--; 1662 shmem_recalc_inode(inode); 1663 spin_unlock_irq(&info->lock); 1664 1665 if (sgp == SGP_WRITE) 1666 mark_page_accessed(page); 1667 1668 delete_from_swap_cache(page); 1669 set_page_dirty(page); 1670 swap_free(swap); 1671 1672 } else { 1673 /* shmem_symlink() */ 1674 if (mapping->a_ops != &shmem_aops) 1675 goto alloc_nohuge; 1676 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) 1677 goto alloc_nohuge; 1678 if (shmem_huge == SHMEM_HUGE_FORCE) 1679 goto alloc_huge; 1680 switch (sbinfo->huge) { 1681 loff_t i_size; 1682 pgoff_t off; 1683 case SHMEM_HUGE_NEVER: 1684 goto alloc_nohuge; 1685 case SHMEM_HUGE_WITHIN_SIZE: 1686 off = round_up(index, HPAGE_PMD_NR); 1687 i_size = round_up(i_size_read(inode), PAGE_SIZE); 1688 if (i_size >= HPAGE_PMD_SIZE && 1689 i_size >> PAGE_SHIFT >= off) 1690 goto alloc_huge; 1691 /* fallthrough */ 1692 case SHMEM_HUGE_ADVISE: 1693 if (sgp_huge == SGP_HUGE) 1694 goto alloc_huge; 1695 /* TODO: implement fadvise() hints */ 1696 goto alloc_nohuge; 1697 } 1698 1699 alloc_huge: 1700 page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1701 index, true); 1702 if (IS_ERR(page)) { 1703 alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1704 index, false); 1705 } 1706 if (IS_ERR(page)) { 1707 int retry = 5; 1708 error = PTR_ERR(page); 1709 page = NULL; 1710 if (error != -ENOSPC) 1711 goto failed; 1712 /* 1713 * Try to reclaim some spece by splitting a huge page 1714 * beyond i_size on the filesystem. 1715 */ 1716 while (retry--) { 1717 int ret; 1718 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1719 if (ret == SHRINK_STOP) 1720 break; 1721 if (ret) 1722 goto alloc_nohuge; 1723 } 1724 goto failed; 1725 } 1726 1727 if (PageTransHuge(page)) 1728 hindex = round_down(index, HPAGE_PMD_NR); 1729 else 1730 hindex = index; 1731 1732 if (sgp == SGP_WRITE) 1733 __SetPageReferenced(page); 1734 1735 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1736 PageTransHuge(page)); 1737 if (error) 1738 goto unacct; 1739 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, 1740 compound_order(page)); 1741 if (!error) { 1742 error = shmem_add_to_page_cache(page, mapping, hindex, 1743 NULL); 1744 radix_tree_preload_end(); 1745 } 1746 if (error) { 1747 mem_cgroup_cancel_charge(page, memcg, 1748 PageTransHuge(page)); 1749 goto unacct; 1750 } 1751 mem_cgroup_commit_charge(page, memcg, false, 1752 PageTransHuge(page)); 1753 lru_cache_add_anon(page); 1754 1755 spin_lock_irq(&info->lock); 1756 info->alloced += 1 << compound_order(page); 1757 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1758 shmem_recalc_inode(inode); 1759 spin_unlock_irq(&info->lock); 1760 alloced = true; 1761 1762 if (PageTransHuge(page) && 1763 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1764 hindex + HPAGE_PMD_NR - 1) { 1765 /* 1766 * Part of the huge page is beyond i_size: subject 1767 * to shrink under memory pressure. 1768 */ 1769 spin_lock(&sbinfo->shrinklist_lock); 1770 if (list_empty(&info->shrinklist)) { 1771 list_add_tail(&info->shrinklist, 1772 &sbinfo->shrinklist); 1773 sbinfo->shrinklist_len++; 1774 } 1775 spin_unlock(&sbinfo->shrinklist_lock); 1776 } 1777 1778 /* 1779 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1780 */ 1781 if (sgp == SGP_FALLOC) 1782 sgp = SGP_WRITE; 1783 clear: 1784 /* 1785 * Let SGP_WRITE caller clear ends if write does not fill page; 1786 * but SGP_FALLOC on a page fallocated earlier must initialize 1787 * it now, lest undo on failure cancel our earlier guarantee. 1788 */ 1789 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1790 struct page *head = compound_head(page); 1791 int i; 1792 1793 for (i = 0; i < (1 << compound_order(head)); i++) { 1794 clear_highpage(head + i); 1795 flush_dcache_page(head + i); 1796 } 1797 SetPageUptodate(head); 1798 } 1799 } 1800 1801 /* Perhaps the file has been truncated since we checked */ 1802 if (sgp <= SGP_CACHE && 1803 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1804 if (alloced) { 1805 ClearPageDirty(page); 1806 delete_from_page_cache(page); 1807 spin_lock_irq(&info->lock); 1808 shmem_recalc_inode(inode); 1809 spin_unlock_irq(&info->lock); 1810 } 1811 error = -EINVAL; 1812 goto unlock; 1813 } 1814 *pagep = page + index - hindex; 1815 return 0; 1816 1817 /* 1818 * Error recovery. 1819 */ 1820 unacct: 1821 if (sbinfo->max_blocks) 1822 percpu_counter_sub(&sbinfo->used_blocks, 1823 1 << compound_order(page)); 1824 shmem_unacct_blocks(info->flags, 1 << compound_order(page)); 1825 1826 if (PageTransHuge(page)) { 1827 unlock_page(page); 1828 put_page(page); 1829 goto alloc_nohuge; 1830 } 1831 failed: 1832 if (swap.val && !shmem_confirm_swap(mapping, index, swap)) 1833 error = -EEXIST; 1834 unlock: 1835 if (page) { 1836 unlock_page(page); 1837 put_page(page); 1838 } 1839 if (error == -ENOSPC && !once++) { 1840 info = SHMEM_I(inode); 1841 spin_lock_irq(&info->lock); 1842 shmem_recalc_inode(inode); 1843 spin_unlock_irq(&info->lock); 1844 goto repeat; 1845 } 1846 if (error == -EEXIST) /* from above or from radix_tree_insert */ 1847 goto repeat; 1848 return error; 1849 } 1850 1851 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1852 { 1853 struct inode *inode = file_inode(vma->vm_file); 1854 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 1855 enum sgp_type sgp; 1856 int error; 1857 int ret = VM_FAULT_LOCKED; 1858 1859 /* 1860 * Trinity finds that probing a hole which tmpfs is punching can 1861 * prevent the hole-punch from ever completing: which in turn 1862 * locks writers out with its hold on i_mutex. So refrain from 1863 * faulting pages into the hole while it's being punched. Although 1864 * shmem_undo_range() does remove the additions, it may be unable to 1865 * keep up, as each new page needs its own unmap_mapping_range() call, 1866 * and the i_mmap tree grows ever slower to scan if new vmas are added. 1867 * 1868 * It does not matter if we sometimes reach this check just before the 1869 * hole-punch begins, so that one fault then races with the punch: 1870 * we just need to make racing faults a rare case. 1871 * 1872 * The implementation below would be much simpler if we just used a 1873 * standard mutex or completion: but we cannot take i_mutex in fault, 1874 * and bloating every shmem inode for this unlikely case would be sad. 1875 */ 1876 if (unlikely(inode->i_private)) { 1877 struct shmem_falloc *shmem_falloc; 1878 1879 spin_lock(&inode->i_lock); 1880 shmem_falloc = inode->i_private; 1881 if (shmem_falloc && 1882 shmem_falloc->waitq && 1883 vmf->pgoff >= shmem_falloc->start && 1884 vmf->pgoff < shmem_falloc->next) { 1885 wait_queue_head_t *shmem_falloc_waitq; 1886 DEFINE_WAIT(shmem_fault_wait); 1887 1888 ret = VM_FAULT_NOPAGE; 1889 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && 1890 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 1891 /* It's polite to up mmap_sem if we can */ 1892 up_read(&vma->vm_mm->mmap_sem); 1893 ret = VM_FAULT_RETRY; 1894 } 1895 1896 shmem_falloc_waitq = shmem_falloc->waitq; 1897 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 1898 TASK_UNINTERRUPTIBLE); 1899 spin_unlock(&inode->i_lock); 1900 schedule(); 1901 1902 /* 1903 * shmem_falloc_waitq points into the shmem_fallocate() 1904 * stack of the hole-punching task: shmem_falloc_waitq 1905 * is usually invalid by the time we reach here, but 1906 * finish_wait() does not dereference it in that case; 1907 * though i_lock needed lest racing with wake_up_all(). 1908 */ 1909 spin_lock(&inode->i_lock); 1910 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 1911 spin_unlock(&inode->i_lock); 1912 return ret; 1913 } 1914 spin_unlock(&inode->i_lock); 1915 } 1916 1917 sgp = SGP_CACHE; 1918 if (vma->vm_flags & VM_HUGEPAGE) 1919 sgp = SGP_HUGE; 1920 else if (vma->vm_flags & VM_NOHUGEPAGE) 1921 sgp = SGP_NOHUGE; 1922 1923 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, 1924 gfp, vma->vm_mm, &ret); 1925 if (error) 1926 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1927 return ret; 1928 } 1929 1930 unsigned long shmem_get_unmapped_area(struct file *file, 1931 unsigned long uaddr, unsigned long len, 1932 unsigned long pgoff, unsigned long flags) 1933 { 1934 unsigned long (*get_area)(struct file *, 1935 unsigned long, unsigned long, unsigned long, unsigned long); 1936 unsigned long addr; 1937 unsigned long offset; 1938 unsigned long inflated_len; 1939 unsigned long inflated_addr; 1940 unsigned long inflated_offset; 1941 1942 if (len > TASK_SIZE) 1943 return -ENOMEM; 1944 1945 get_area = current->mm->get_unmapped_area; 1946 addr = get_area(file, uaddr, len, pgoff, flags); 1947 1948 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1949 return addr; 1950 if (IS_ERR_VALUE(addr)) 1951 return addr; 1952 if (addr & ~PAGE_MASK) 1953 return addr; 1954 if (addr > TASK_SIZE - len) 1955 return addr; 1956 1957 if (shmem_huge == SHMEM_HUGE_DENY) 1958 return addr; 1959 if (len < HPAGE_PMD_SIZE) 1960 return addr; 1961 if (flags & MAP_FIXED) 1962 return addr; 1963 /* 1964 * Our priority is to support MAP_SHARED mapped hugely; 1965 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 1966 * But if caller specified an address hint, respect that as before. 1967 */ 1968 if (uaddr) 1969 return addr; 1970 1971 if (shmem_huge != SHMEM_HUGE_FORCE) { 1972 struct super_block *sb; 1973 1974 if (file) { 1975 VM_BUG_ON(file->f_op != &shmem_file_operations); 1976 sb = file_inode(file)->i_sb; 1977 } else { 1978 /* 1979 * Called directly from mm/mmap.c, or drivers/char/mem.c 1980 * for "/dev/zero", to create a shared anonymous object. 1981 */ 1982 if (IS_ERR(shm_mnt)) 1983 return addr; 1984 sb = shm_mnt->mnt_sb; 1985 } 1986 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 1987 return addr; 1988 } 1989 1990 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 1991 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 1992 return addr; 1993 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 1994 return addr; 1995 1996 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 1997 if (inflated_len > TASK_SIZE) 1998 return addr; 1999 if (inflated_len < len) 2000 return addr; 2001 2002 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); 2003 if (IS_ERR_VALUE(inflated_addr)) 2004 return addr; 2005 if (inflated_addr & ~PAGE_MASK) 2006 return addr; 2007 2008 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2009 inflated_addr += offset - inflated_offset; 2010 if (inflated_offset > offset) 2011 inflated_addr += HPAGE_PMD_SIZE; 2012 2013 if (inflated_addr > TASK_SIZE - len) 2014 return addr; 2015 return inflated_addr; 2016 } 2017 2018 #ifdef CONFIG_NUMA 2019 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2020 { 2021 struct inode *inode = file_inode(vma->vm_file); 2022 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2023 } 2024 2025 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2026 unsigned long addr) 2027 { 2028 struct inode *inode = file_inode(vma->vm_file); 2029 pgoff_t index; 2030 2031 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2032 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2033 } 2034 #endif 2035 2036 int shmem_lock(struct file *file, int lock, struct user_struct *user) 2037 { 2038 struct inode *inode = file_inode(file); 2039 struct shmem_inode_info *info = SHMEM_I(inode); 2040 int retval = -ENOMEM; 2041 2042 spin_lock_irq(&info->lock); 2043 if (lock && !(info->flags & VM_LOCKED)) { 2044 if (!user_shm_lock(inode->i_size, user)) 2045 goto out_nomem; 2046 info->flags |= VM_LOCKED; 2047 mapping_set_unevictable(file->f_mapping); 2048 } 2049 if (!lock && (info->flags & VM_LOCKED) && user) { 2050 user_shm_unlock(inode->i_size, user); 2051 info->flags &= ~VM_LOCKED; 2052 mapping_clear_unevictable(file->f_mapping); 2053 } 2054 retval = 0; 2055 2056 out_nomem: 2057 spin_unlock_irq(&info->lock); 2058 return retval; 2059 } 2060 2061 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2062 { 2063 file_accessed(file); 2064 vma->vm_ops = &shmem_vm_ops; 2065 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 2066 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2067 (vma->vm_end & HPAGE_PMD_MASK)) { 2068 khugepaged_enter(vma, vma->vm_flags); 2069 } 2070 return 0; 2071 } 2072 2073 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2074 umode_t mode, dev_t dev, unsigned long flags) 2075 { 2076 struct inode *inode; 2077 struct shmem_inode_info *info; 2078 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2079 2080 if (shmem_reserve_inode(sb)) 2081 return NULL; 2082 2083 inode = new_inode(sb); 2084 if (inode) { 2085 inode->i_ino = get_next_ino(); 2086 inode_init_owner(inode, dir, mode); 2087 inode->i_blocks = 0; 2088 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2089 inode->i_generation = get_seconds(); 2090 info = SHMEM_I(inode); 2091 memset(info, 0, (char *)inode - (char *)info); 2092 spin_lock_init(&info->lock); 2093 info->seals = F_SEAL_SEAL; 2094 info->flags = flags & VM_NORESERVE; 2095 INIT_LIST_HEAD(&info->shrinklist); 2096 INIT_LIST_HEAD(&info->swaplist); 2097 simple_xattrs_init(&info->xattrs); 2098 cache_no_acl(inode); 2099 2100 switch (mode & S_IFMT) { 2101 default: 2102 inode->i_op = &shmem_special_inode_operations; 2103 init_special_inode(inode, mode, dev); 2104 break; 2105 case S_IFREG: 2106 inode->i_mapping->a_ops = &shmem_aops; 2107 inode->i_op = &shmem_inode_operations; 2108 inode->i_fop = &shmem_file_operations; 2109 mpol_shared_policy_init(&info->policy, 2110 shmem_get_sbmpol(sbinfo)); 2111 break; 2112 case S_IFDIR: 2113 inc_nlink(inode); 2114 /* Some things misbehave if size == 0 on a directory */ 2115 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2116 inode->i_op = &shmem_dir_inode_operations; 2117 inode->i_fop = &simple_dir_operations; 2118 break; 2119 case S_IFLNK: 2120 /* 2121 * Must not load anything in the rbtree, 2122 * mpol_free_shared_policy will not be called. 2123 */ 2124 mpol_shared_policy_init(&info->policy, NULL); 2125 break; 2126 } 2127 } else 2128 shmem_free_inode(sb); 2129 return inode; 2130 } 2131 2132 bool shmem_mapping(struct address_space *mapping) 2133 { 2134 if (!mapping->host) 2135 return false; 2136 2137 return mapping->host->i_sb->s_op == &shmem_ops; 2138 } 2139 2140 #ifdef CONFIG_TMPFS 2141 static const struct inode_operations shmem_symlink_inode_operations; 2142 static const struct inode_operations shmem_short_symlink_operations; 2143 2144 #ifdef CONFIG_TMPFS_XATTR 2145 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2146 #else 2147 #define shmem_initxattrs NULL 2148 #endif 2149 2150 static int 2151 shmem_write_begin(struct file *file, struct address_space *mapping, 2152 loff_t pos, unsigned len, unsigned flags, 2153 struct page **pagep, void **fsdata) 2154 { 2155 struct inode *inode = mapping->host; 2156 struct shmem_inode_info *info = SHMEM_I(inode); 2157 pgoff_t index = pos >> PAGE_SHIFT; 2158 2159 /* i_mutex is held by caller */ 2160 if (unlikely(info->seals)) { 2161 if (info->seals & F_SEAL_WRITE) 2162 return -EPERM; 2163 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2164 return -EPERM; 2165 } 2166 2167 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2168 } 2169 2170 static int 2171 shmem_write_end(struct file *file, struct address_space *mapping, 2172 loff_t pos, unsigned len, unsigned copied, 2173 struct page *page, void *fsdata) 2174 { 2175 struct inode *inode = mapping->host; 2176 2177 if (pos + copied > inode->i_size) 2178 i_size_write(inode, pos + copied); 2179 2180 if (!PageUptodate(page)) { 2181 struct page *head = compound_head(page); 2182 if (PageTransCompound(page)) { 2183 int i; 2184 2185 for (i = 0; i < HPAGE_PMD_NR; i++) { 2186 if (head + i == page) 2187 continue; 2188 clear_highpage(head + i); 2189 flush_dcache_page(head + i); 2190 } 2191 } 2192 if (copied < PAGE_SIZE) { 2193 unsigned from = pos & (PAGE_SIZE - 1); 2194 zero_user_segments(page, 0, from, 2195 from + copied, PAGE_SIZE); 2196 } 2197 SetPageUptodate(head); 2198 } 2199 set_page_dirty(page); 2200 unlock_page(page); 2201 put_page(page); 2202 2203 return copied; 2204 } 2205 2206 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2207 { 2208 struct file *file = iocb->ki_filp; 2209 struct inode *inode = file_inode(file); 2210 struct address_space *mapping = inode->i_mapping; 2211 pgoff_t index; 2212 unsigned long offset; 2213 enum sgp_type sgp = SGP_READ; 2214 int error = 0; 2215 ssize_t retval = 0; 2216 loff_t *ppos = &iocb->ki_pos; 2217 2218 /* 2219 * Might this read be for a stacking filesystem? Then when reading 2220 * holes of a sparse file, we actually need to allocate those pages, 2221 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2222 */ 2223 if (!iter_is_iovec(to)) 2224 sgp = SGP_CACHE; 2225 2226 index = *ppos >> PAGE_SHIFT; 2227 offset = *ppos & ~PAGE_MASK; 2228 2229 for (;;) { 2230 struct page *page = NULL; 2231 pgoff_t end_index; 2232 unsigned long nr, ret; 2233 loff_t i_size = i_size_read(inode); 2234 2235 end_index = i_size >> PAGE_SHIFT; 2236 if (index > end_index) 2237 break; 2238 if (index == end_index) { 2239 nr = i_size & ~PAGE_MASK; 2240 if (nr <= offset) 2241 break; 2242 } 2243 2244 error = shmem_getpage(inode, index, &page, sgp); 2245 if (error) { 2246 if (error == -EINVAL) 2247 error = 0; 2248 break; 2249 } 2250 if (page) { 2251 if (sgp == SGP_CACHE) 2252 set_page_dirty(page); 2253 unlock_page(page); 2254 } 2255 2256 /* 2257 * We must evaluate after, since reads (unlike writes) 2258 * are called without i_mutex protection against truncate 2259 */ 2260 nr = PAGE_SIZE; 2261 i_size = i_size_read(inode); 2262 end_index = i_size >> PAGE_SHIFT; 2263 if (index == end_index) { 2264 nr = i_size & ~PAGE_MASK; 2265 if (nr <= offset) { 2266 if (page) 2267 put_page(page); 2268 break; 2269 } 2270 } 2271 nr -= offset; 2272 2273 if (page) { 2274 /* 2275 * If users can be writing to this page using arbitrary 2276 * virtual addresses, take care about potential aliasing 2277 * before reading the page on the kernel side. 2278 */ 2279 if (mapping_writably_mapped(mapping)) 2280 flush_dcache_page(page); 2281 /* 2282 * Mark the page accessed if we read the beginning. 2283 */ 2284 if (!offset) 2285 mark_page_accessed(page); 2286 } else { 2287 page = ZERO_PAGE(0); 2288 get_page(page); 2289 } 2290 2291 /* 2292 * Ok, we have the page, and it's up-to-date, so 2293 * now we can copy it to user space... 2294 */ 2295 ret = copy_page_to_iter(page, offset, nr, to); 2296 retval += ret; 2297 offset += ret; 2298 index += offset >> PAGE_SHIFT; 2299 offset &= ~PAGE_MASK; 2300 2301 put_page(page); 2302 if (!iov_iter_count(to)) 2303 break; 2304 if (ret < nr) { 2305 error = -EFAULT; 2306 break; 2307 } 2308 cond_resched(); 2309 } 2310 2311 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2312 file_accessed(file); 2313 return retval ? retval : error; 2314 } 2315 2316 /* 2317 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree. 2318 */ 2319 static pgoff_t shmem_seek_hole_data(struct address_space *mapping, 2320 pgoff_t index, pgoff_t end, int whence) 2321 { 2322 struct page *page; 2323 struct pagevec pvec; 2324 pgoff_t indices[PAGEVEC_SIZE]; 2325 bool done = false; 2326 int i; 2327 2328 pagevec_init(&pvec, 0); 2329 pvec.nr = 1; /* start small: we may be there already */ 2330 while (!done) { 2331 pvec.nr = find_get_entries(mapping, index, 2332 pvec.nr, pvec.pages, indices); 2333 if (!pvec.nr) { 2334 if (whence == SEEK_DATA) 2335 index = end; 2336 break; 2337 } 2338 for (i = 0; i < pvec.nr; i++, index++) { 2339 if (index < indices[i]) { 2340 if (whence == SEEK_HOLE) { 2341 done = true; 2342 break; 2343 } 2344 index = indices[i]; 2345 } 2346 page = pvec.pages[i]; 2347 if (page && !radix_tree_exceptional_entry(page)) { 2348 if (!PageUptodate(page)) 2349 page = NULL; 2350 } 2351 if (index >= end || 2352 (page && whence == SEEK_DATA) || 2353 (!page && whence == SEEK_HOLE)) { 2354 done = true; 2355 break; 2356 } 2357 } 2358 pagevec_remove_exceptionals(&pvec); 2359 pagevec_release(&pvec); 2360 pvec.nr = PAGEVEC_SIZE; 2361 cond_resched(); 2362 } 2363 return index; 2364 } 2365 2366 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2367 { 2368 struct address_space *mapping = file->f_mapping; 2369 struct inode *inode = mapping->host; 2370 pgoff_t start, end; 2371 loff_t new_offset; 2372 2373 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2374 return generic_file_llseek_size(file, offset, whence, 2375 MAX_LFS_FILESIZE, i_size_read(inode)); 2376 inode_lock(inode); 2377 /* We're holding i_mutex so we can access i_size directly */ 2378 2379 if (offset < 0) 2380 offset = -EINVAL; 2381 else if (offset >= inode->i_size) 2382 offset = -ENXIO; 2383 else { 2384 start = offset >> PAGE_SHIFT; 2385 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2386 new_offset = shmem_seek_hole_data(mapping, start, end, whence); 2387 new_offset <<= PAGE_SHIFT; 2388 if (new_offset > offset) { 2389 if (new_offset < inode->i_size) 2390 offset = new_offset; 2391 else if (whence == SEEK_DATA) 2392 offset = -ENXIO; 2393 else 2394 offset = inode->i_size; 2395 } 2396 } 2397 2398 if (offset >= 0) 2399 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2400 inode_unlock(inode); 2401 return offset; 2402 } 2403 2404 /* 2405 * We need a tag: a new tag would expand every radix_tree_node by 8 bytes, 2406 * so reuse a tag which we firmly believe is never set or cleared on shmem. 2407 */ 2408 #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE 2409 #define LAST_SCAN 4 /* about 150ms max */ 2410 2411 static void shmem_tag_pins(struct address_space *mapping) 2412 { 2413 struct radix_tree_iter iter; 2414 void **slot; 2415 pgoff_t start; 2416 struct page *page; 2417 2418 lru_add_drain(); 2419 start = 0; 2420 rcu_read_lock(); 2421 2422 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2423 page = radix_tree_deref_slot(slot); 2424 if (!page || radix_tree_exception(page)) { 2425 if (radix_tree_deref_retry(page)) { 2426 slot = radix_tree_iter_retry(&iter); 2427 continue; 2428 } 2429 } else if (page_count(page) - page_mapcount(page) > 1) { 2430 spin_lock_irq(&mapping->tree_lock); 2431 radix_tree_tag_set(&mapping->page_tree, iter.index, 2432 SHMEM_TAG_PINNED); 2433 spin_unlock_irq(&mapping->tree_lock); 2434 } 2435 2436 if (need_resched()) { 2437 cond_resched_rcu(); 2438 slot = radix_tree_iter_next(&iter); 2439 } 2440 } 2441 rcu_read_unlock(); 2442 } 2443 2444 /* 2445 * Setting SEAL_WRITE requires us to verify there's no pending writer. However, 2446 * via get_user_pages(), drivers might have some pending I/O without any active 2447 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages 2448 * and see whether it has an elevated ref-count. If so, we tag them and wait for 2449 * them to be dropped. 2450 * The caller must guarantee that no new user will acquire writable references 2451 * to those pages to avoid races. 2452 */ 2453 static int shmem_wait_for_pins(struct address_space *mapping) 2454 { 2455 struct radix_tree_iter iter; 2456 void **slot; 2457 pgoff_t start; 2458 struct page *page; 2459 int error, scan; 2460 2461 shmem_tag_pins(mapping); 2462 2463 error = 0; 2464 for (scan = 0; scan <= LAST_SCAN; scan++) { 2465 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2466 break; 2467 2468 if (!scan) 2469 lru_add_drain_all(); 2470 else if (schedule_timeout_killable((HZ << scan) / 200)) 2471 scan = LAST_SCAN; 2472 2473 start = 0; 2474 rcu_read_lock(); 2475 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2476 start, SHMEM_TAG_PINNED) { 2477 2478 page = radix_tree_deref_slot(slot); 2479 if (radix_tree_exception(page)) { 2480 if (radix_tree_deref_retry(page)) { 2481 slot = radix_tree_iter_retry(&iter); 2482 continue; 2483 } 2484 2485 page = NULL; 2486 } 2487 2488 if (page && 2489 page_count(page) - page_mapcount(page) != 1) { 2490 if (scan < LAST_SCAN) 2491 goto continue_resched; 2492 2493 /* 2494 * On the last scan, we clean up all those tags 2495 * we inserted; but make a note that we still 2496 * found pages pinned. 2497 */ 2498 error = -EBUSY; 2499 } 2500 2501 spin_lock_irq(&mapping->tree_lock); 2502 radix_tree_tag_clear(&mapping->page_tree, 2503 iter.index, SHMEM_TAG_PINNED); 2504 spin_unlock_irq(&mapping->tree_lock); 2505 continue_resched: 2506 if (need_resched()) { 2507 cond_resched_rcu(); 2508 slot = radix_tree_iter_next(&iter); 2509 } 2510 } 2511 rcu_read_unlock(); 2512 } 2513 2514 return error; 2515 } 2516 2517 #define F_ALL_SEALS (F_SEAL_SEAL | \ 2518 F_SEAL_SHRINK | \ 2519 F_SEAL_GROW | \ 2520 F_SEAL_WRITE) 2521 2522 int shmem_add_seals(struct file *file, unsigned int seals) 2523 { 2524 struct inode *inode = file_inode(file); 2525 struct shmem_inode_info *info = SHMEM_I(inode); 2526 int error; 2527 2528 /* 2529 * SEALING 2530 * Sealing allows multiple parties to share a shmem-file but restrict 2531 * access to a specific subset of file operations. Seals can only be 2532 * added, but never removed. This way, mutually untrusted parties can 2533 * share common memory regions with a well-defined policy. A malicious 2534 * peer can thus never perform unwanted operations on a shared object. 2535 * 2536 * Seals are only supported on special shmem-files and always affect 2537 * the whole underlying inode. Once a seal is set, it may prevent some 2538 * kinds of access to the file. Currently, the following seals are 2539 * defined: 2540 * SEAL_SEAL: Prevent further seals from being set on this file 2541 * SEAL_SHRINK: Prevent the file from shrinking 2542 * SEAL_GROW: Prevent the file from growing 2543 * SEAL_WRITE: Prevent write access to the file 2544 * 2545 * As we don't require any trust relationship between two parties, we 2546 * must prevent seals from being removed. Therefore, sealing a file 2547 * only adds a given set of seals to the file, it never touches 2548 * existing seals. Furthermore, the "setting seals"-operation can be 2549 * sealed itself, which basically prevents any further seal from being 2550 * added. 2551 * 2552 * Semantics of sealing are only defined on volatile files. Only 2553 * anonymous shmem files support sealing. More importantly, seals are 2554 * never written to disk. Therefore, there's no plan to support it on 2555 * other file types. 2556 */ 2557 2558 if (file->f_op != &shmem_file_operations) 2559 return -EINVAL; 2560 if (!(file->f_mode & FMODE_WRITE)) 2561 return -EPERM; 2562 if (seals & ~(unsigned int)F_ALL_SEALS) 2563 return -EINVAL; 2564 2565 inode_lock(inode); 2566 2567 if (info->seals & F_SEAL_SEAL) { 2568 error = -EPERM; 2569 goto unlock; 2570 } 2571 2572 if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) { 2573 error = mapping_deny_writable(file->f_mapping); 2574 if (error) 2575 goto unlock; 2576 2577 error = shmem_wait_for_pins(file->f_mapping); 2578 if (error) { 2579 mapping_allow_writable(file->f_mapping); 2580 goto unlock; 2581 } 2582 } 2583 2584 info->seals |= seals; 2585 error = 0; 2586 2587 unlock: 2588 inode_unlock(inode); 2589 return error; 2590 } 2591 EXPORT_SYMBOL_GPL(shmem_add_seals); 2592 2593 int shmem_get_seals(struct file *file) 2594 { 2595 if (file->f_op != &shmem_file_operations) 2596 return -EINVAL; 2597 2598 return SHMEM_I(file_inode(file))->seals; 2599 } 2600 EXPORT_SYMBOL_GPL(shmem_get_seals); 2601 2602 long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 2603 { 2604 long error; 2605 2606 switch (cmd) { 2607 case F_ADD_SEALS: 2608 /* disallow upper 32bit */ 2609 if (arg > UINT_MAX) 2610 return -EINVAL; 2611 2612 error = shmem_add_seals(file, arg); 2613 break; 2614 case F_GET_SEALS: 2615 error = shmem_get_seals(file); 2616 break; 2617 default: 2618 error = -EINVAL; 2619 break; 2620 } 2621 2622 return error; 2623 } 2624 2625 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2626 loff_t len) 2627 { 2628 struct inode *inode = file_inode(file); 2629 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2630 struct shmem_inode_info *info = SHMEM_I(inode); 2631 struct shmem_falloc shmem_falloc; 2632 pgoff_t start, index, end; 2633 int error; 2634 2635 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2636 return -EOPNOTSUPP; 2637 2638 inode_lock(inode); 2639 2640 if (mode & FALLOC_FL_PUNCH_HOLE) { 2641 struct address_space *mapping = file->f_mapping; 2642 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2643 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2644 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2645 2646 /* protected by i_mutex */ 2647 if (info->seals & F_SEAL_WRITE) { 2648 error = -EPERM; 2649 goto out; 2650 } 2651 2652 shmem_falloc.waitq = &shmem_falloc_waitq; 2653 shmem_falloc.start = unmap_start >> PAGE_SHIFT; 2654 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2655 spin_lock(&inode->i_lock); 2656 inode->i_private = &shmem_falloc; 2657 spin_unlock(&inode->i_lock); 2658 2659 if ((u64)unmap_end > (u64)unmap_start) 2660 unmap_mapping_range(mapping, unmap_start, 2661 1 + unmap_end - unmap_start, 0); 2662 shmem_truncate_range(inode, offset, offset + len - 1); 2663 /* No need to unmap again: hole-punching leaves COWed pages */ 2664 2665 spin_lock(&inode->i_lock); 2666 inode->i_private = NULL; 2667 wake_up_all(&shmem_falloc_waitq); 2668 spin_unlock(&inode->i_lock); 2669 error = 0; 2670 goto out; 2671 } 2672 2673 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2674 error = inode_newsize_ok(inode, offset + len); 2675 if (error) 2676 goto out; 2677 2678 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2679 error = -EPERM; 2680 goto out; 2681 } 2682 2683 start = offset >> PAGE_SHIFT; 2684 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2685 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2686 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2687 error = -ENOSPC; 2688 goto out; 2689 } 2690 2691 shmem_falloc.waitq = NULL; 2692 shmem_falloc.start = start; 2693 shmem_falloc.next = start; 2694 shmem_falloc.nr_falloced = 0; 2695 shmem_falloc.nr_unswapped = 0; 2696 spin_lock(&inode->i_lock); 2697 inode->i_private = &shmem_falloc; 2698 spin_unlock(&inode->i_lock); 2699 2700 for (index = start; index < end; index++) { 2701 struct page *page; 2702 2703 /* 2704 * Good, the fallocate(2) manpage permits EINTR: we may have 2705 * been interrupted because we are using up too much memory. 2706 */ 2707 if (signal_pending(current)) 2708 error = -EINTR; 2709 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2710 error = -ENOMEM; 2711 else 2712 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2713 if (error) { 2714 /* Remove the !PageUptodate pages we added */ 2715 if (index > start) { 2716 shmem_undo_range(inode, 2717 (loff_t)start << PAGE_SHIFT, 2718 ((loff_t)index << PAGE_SHIFT) - 1, true); 2719 } 2720 goto undone; 2721 } 2722 2723 /* 2724 * Inform shmem_writepage() how far we have reached. 2725 * No need for lock or barrier: we have the page lock. 2726 */ 2727 shmem_falloc.next++; 2728 if (!PageUptodate(page)) 2729 shmem_falloc.nr_falloced++; 2730 2731 /* 2732 * If !PageUptodate, leave it that way so that freeable pages 2733 * can be recognized if we need to rollback on error later. 2734 * But set_page_dirty so that memory pressure will swap rather 2735 * than free the pages we are allocating (and SGP_CACHE pages 2736 * might still be clean: we now need to mark those dirty too). 2737 */ 2738 set_page_dirty(page); 2739 unlock_page(page); 2740 put_page(page); 2741 cond_resched(); 2742 } 2743 2744 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2745 i_size_write(inode, offset + len); 2746 inode->i_ctime = current_time(inode); 2747 undone: 2748 spin_lock(&inode->i_lock); 2749 inode->i_private = NULL; 2750 spin_unlock(&inode->i_lock); 2751 out: 2752 inode_unlock(inode); 2753 return error; 2754 } 2755 2756 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2757 { 2758 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2759 2760 buf->f_type = TMPFS_MAGIC; 2761 buf->f_bsize = PAGE_SIZE; 2762 buf->f_namelen = NAME_MAX; 2763 if (sbinfo->max_blocks) { 2764 buf->f_blocks = sbinfo->max_blocks; 2765 buf->f_bavail = 2766 buf->f_bfree = sbinfo->max_blocks - 2767 percpu_counter_sum(&sbinfo->used_blocks); 2768 } 2769 if (sbinfo->max_inodes) { 2770 buf->f_files = sbinfo->max_inodes; 2771 buf->f_ffree = sbinfo->free_inodes; 2772 } 2773 /* else leave those fields 0 like simple_statfs */ 2774 return 0; 2775 } 2776 2777 /* 2778 * File creation. Allocate an inode, and we're done.. 2779 */ 2780 static int 2781 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) 2782 { 2783 struct inode *inode; 2784 int error = -ENOSPC; 2785 2786 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2787 if (inode) { 2788 error = simple_acl_create(dir, inode); 2789 if (error) 2790 goto out_iput; 2791 error = security_inode_init_security(inode, dir, 2792 &dentry->d_name, 2793 shmem_initxattrs, NULL); 2794 if (error && error != -EOPNOTSUPP) 2795 goto out_iput; 2796 2797 error = 0; 2798 dir->i_size += BOGO_DIRENT_SIZE; 2799 dir->i_ctime = dir->i_mtime = current_time(dir); 2800 d_instantiate(dentry, inode); 2801 dget(dentry); /* Extra count - pin the dentry in core */ 2802 } 2803 return error; 2804 out_iput: 2805 iput(inode); 2806 return error; 2807 } 2808 2809 static int 2810 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 2811 { 2812 struct inode *inode; 2813 int error = -ENOSPC; 2814 2815 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2816 if (inode) { 2817 error = security_inode_init_security(inode, dir, 2818 NULL, 2819 shmem_initxattrs, NULL); 2820 if (error && error != -EOPNOTSUPP) 2821 goto out_iput; 2822 error = simple_acl_create(dir, inode); 2823 if (error) 2824 goto out_iput; 2825 d_tmpfile(dentry, inode); 2826 } 2827 return error; 2828 out_iput: 2829 iput(inode); 2830 return error; 2831 } 2832 2833 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 2834 { 2835 int error; 2836 2837 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 2838 return error; 2839 inc_nlink(dir); 2840 return 0; 2841 } 2842 2843 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, 2844 bool excl) 2845 { 2846 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 2847 } 2848 2849 /* 2850 * Link a file.. 2851 */ 2852 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2853 { 2854 struct inode *inode = d_inode(old_dentry); 2855 int ret; 2856 2857 /* 2858 * No ordinary (disk based) filesystem counts links as inodes; 2859 * but each new link needs a new dentry, pinning lowmem, and 2860 * tmpfs dentries cannot be pruned until they are unlinked. 2861 */ 2862 ret = shmem_reserve_inode(inode->i_sb); 2863 if (ret) 2864 goto out; 2865 2866 dir->i_size += BOGO_DIRENT_SIZE; 2867 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2868 inc_nlink(inode); 2869 ihold(inode); /* New dentry reference */ 2870 dget(dentry); /* Extra pinning count for the created dentry */ 2871 d_instantiate(dentry, inode); 2872 out: 2873 return ret; 2874 } 2875 2876 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2877 { 2878 struct inode *inode = d_inode(dentry); 2879 2880 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2881 shmem_free_inode(inode->i_sb); 2882 2883 dir->i_size -= BOGO_DIRENT_SIZE; 2884 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2885 drop_nlink(inode); 2886 dput(dentry); /* Undo the count from "create" - this does all the work */ 2887 return 0; 2888 } 2889 2890 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2891 { 2892 if (!simple_empty(dentry)) 2893 return -ENOTEMPTY; 2894 2895 drop_nlink(d_inode(dentry)); 2896 drop_nlink(dir); 2897 return shmem_unlink(dir, dentry); 2898 } 2899 2900 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 2901 { 2902 bool old_is_dir = d_is_dir(old_dentry); 2903 bool new_is_dir = d_is_dir(new_dentry); 2904 2905 if (old_dir != new_dir && old_is_dir != new_is_dir) { 2906 if (old_is_dir) { 2907 drop_nlink(old_dir); 2908 inc_nlink(new_dir); 2909 } else { 2910 drop_nlink(new_dir); 2911 inc_nlink(old_dir); 2912 } 2913 } 2914 old_dir->i_ctime = old_dir->i_mtime = 2915 new_dir->i_ctime = new_dir->i_mtime = 2916 d_inode(old_dentry)->i_ctime = 2917 d_inode(new_dentry)->i_ctime = current_time(old_dir); 2918 2919 return 0; 2920 } 2921 2922 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) 2923 { 2924 struct dentry *whiteout; 2925 int error; 2926 2927 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 2928 if (!whiteout) 2929 return -ENOMEM; 2930 2931 error = shmem_mknod(old_dir, whiteout, 2932 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 2933 dput(whiteout); 2934 if (error) 2935 return error; 2936 2937 /* 2938 * Cheat and hash the whiteout while the old dentry is still in 2939 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 2940 * 2941 * d_lookup() will consistently find one of them at this point, 2942 * not sure which one, but that isn't even important. 2943 */ 2944 d_rehash(whiteout); 2945 return 0; 2946 } 2947 2948 /* 2949 * The VFS layer already does all the dentry stuff for rename, 2950 * we just have to decrement the usage count for the target if 2951 * it exists so that the VFS layer correctly free's it when it 2952 * gets overwritten. 2953 */ 2954 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) 2955 { 2956 struct inode *inode = d_inode(old_dentry); 2957 int they_are_dirs = S_ISDIR(inode->i_mode); 2958 2959 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 2960 return -EINVAL; 2961 2962 if (flags & RENAME_EXCHANGE) 2963 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 2964 2965 if (!simple_empty(new_dentry)) 2966 return -ENOTEMPTY; 2967 2968 if (flags & RENAME_WHITEOUT) { 2969 int error; 2970 2971 error = shmem_whiteout(old_dir, old_dentry); 2972 if (error) 2973 return error; 2974 } 2975 2976 if (d_really_is_positive(new_dentry)) { 2977 (void) shmem_unlink(new_dir, new_dentry); 2978 if (they_are_dirs) { 2979 drop_nlink(d_inode(new_dentry)); 2980 drop_nlink(old_dir); 2981 } 2982 } else if (they_are_dirs) { 2983 drop_nlink(old_dir); 2984 inc_nlink(new_dir); 2985 } 2986 2987 old_dir->i_size -= BOGO_DIRENT_SIZE; 2988 new_dir->i_size += BOGO_DIRENT_SIZE; 2989 old_dir->i_ctime = old_dir->i_mtime = 2990 new_dir->i_ctime = new_dir->i_mtime = 2991 inode->i_ctime = current_time(old_dir); 2992 return 0; 2993 } 2994 2995 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 2996 { 2997 int error; 2998 int len; 2999 struct inode *inode; 3000 struct page *page; 3001 struct shmem_inode_info *info; 3002 3003 len = strlen(symname) + 1; 3004 if (len > PAGE_SIZE) 3005 return -ENAMETOOLONG; 3006 3007 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 3008 if (!inode) 3009 return -ENOSPC; 3010 3011 error = security_inode_init_security(inode, dir, &dentry->d_name, 3012 shmem_initxattrs, NULL); 3013 if (error) { 3014 if (error != -EOPNOTSUPP) { 3015 iput(inode); 3016 return error; 3017 } 3018 error = 0; 3019 } 3020 3021 info = SHMEM_I(inode); 3022 inode->i_size = len-1; 3023 if (len <= SHORT_SYMLINK_LEN) { 3024 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3025 if (!inode->i_link) { 3026 iput(inode); 3027 return -ENOMEM; 3028 } 3029 inode->i_op = &shmem_short_symlink_operations; 3030 } else { 3031 inode_nohighmem(inode); 3032 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3033 if (error) { 3034 iput(inode); 3035 return error; 3036 } 3037 inode->i_mapping->a_ops = &shmem_aops; 3038 inode->i_op = &shmem_symlink_inode_operations; 3039 memcpy(page_address(page), symname, len); 3040 SetPageUptodate(page); 3041 set_page_dirty(page); 3042 unlock_page(page); 3043 put_page(page); 3044 } 3045 dir->i_size += BOGO_DIRENT_SIZE; 3046 dir->i_ctime = dir->i_mtime = current_time(dir); 3047 d_instantiate(dentry, inode); 3048 dget(dentry); 3049 return 0; 3050 } 3051 3052 static void shmem_put_link(void *arg) 3053 { 3054 mark_page_accessed(arg); 3055 put_page(arg); 3056 } 3057 3058 static const char *shmem_get_link(struct dentry *dentry, 3059 struct inode *inode, 3060 struct delayed_call *done) 3061 { 3062 struct page *page = NULL; 3063 int error; 3064 if (!dentry) { 3065 page = find_get_page(inode->i_mapping, 0); 3066 if (!page) 3067 return ERR_PTR(-ECHILD); 3068 if (!PageUptodate(page)) { 3069 put_page(page); 3070 return ERR_PTR(-ECHILD); 3071 } 3072 } else { 3073 error = shmem_getpage(inode, 0, &page, SGP_READ); 3074 if (error) 3075 return ERR_PTR(error); 3076 unlock_page(page); 3077 } 3078 set_delayed_call(done, shmem_put_link, page); 3079 return page_address(page); 3080 } 3081 3082 #ifdef CONFIG_TMPFS_XATTR 3083 /* 3084 * Superblocks without xattr inode operations may get some security.* xattr 3085 * support from the LSM "for free". As soon as we have any other xattrs 3086 * like ACLs, we also need to implement the security.* handlers at 3087 * filesystem level, though. 3088 */ 3089 3090 /* 3091 * Callback for security_inode_init_security() for acquiring xattrs. 3092 */ 3093 static int shmem_initxattrs(struct inode *inode, 3094 const struct xattr *xattr_array, 3095 void *fs_info) 3096 { 3097 struct shmem_inode_info *info = SHMEM_I(inode); 3098 const struct xattr *xattr; 3099 struct simple_xattr *new_xattr; 3100 size_t len; 3101 3102 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3103 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3104 if (!new_xattr) 3105 return -ENOMEM; 3106 3107 len = strlen(xattr->name) + 1; 3108 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3109 GFP_KERNEL); 3110 if (!new_xattr->name) { 3111 kfree(new_xattr); 3112 return -ENOMEM; 3113 } 3114 3115 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3116 XATTR_SECURITY_PREFIX_LEN); 3117 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3118 xattr->name, len); 3119 3120 simple_xattr_list_add(&info->xattrs, new_xattr); 3121 } 3122 3123 return 0; 3124 } 3125 3126 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3127 struct dentry *unused, struct inode *inode, 3128 const char *name, void *buffer, size_t size) 3129 { 3130 struct shmem_inode_info *info = SHMEM_I(inode); 3131 3132 name = xattr_full_name(handler, name); 3133 return simple_xattr_get(&info->xattrs, name, buffer, size); 3134 } 3135 3136 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3137 struct dentry *unused, struct inode *inode, 3138 const char *name, const void *value, 3139 size_t size, int flags) 3140 { 3141 struct shmem_inode_info *info = SHMEM_I(inode); 3142 3143 name = xattr_full_name(handler, name); 3144 return simple_xattr_set(&info->xattrs, name, value, size, flags); 3145 } 3146 3147 static const struct xattr_handler shmem_security_xattr_handler = { 3148 .prefix = XATTR_SECURITY_PREFIX, 3149 .get = shmem_xattr_handler_get, 3150 .set = shmem_xattr_handler_set, 3151 }; 3152 3153 static const struct xattr_handler shmem_trusted_xattr_handler = { 3154 .prefix = XATTR_TRUSTED_PREFIX, 3155 .get = shmem_xattr_handler_get, 3156 .set = shmem_xattr_handler_set, 3157 }; 3158 3159 static const struct xattr_handler *shmem_xattr_handlers[] = { 3160 #ifdef CONFIG_TMPFS_POSIX_ACL 3161 &posix_acl_access_xattr_handler, 3162 &posix_acl_default_xattr_handler, 3163 #endif 3164 &shmem_security_xattr_handler, 3165 &shmem_trusted_xattr_handler, 3166 NULL 3167 }; 3168 3169 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3170 { 3171 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3172 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3173 } 3174 #endif /* CONFIG_TMPFS_XATTR */ 3175 3176 static const struct inode_operations shmem_short_symlink_operations = { 3177 .readlink = generic_readlink, 3178 .get_link = simple_get_link, 3179 #ifdef CONFIG_TMPFS_XATTR 3180 .listxattr = shmem_listxattr, 3181 #endif 3182 }; 3183 3184 static const struct inode_operations shmem_symlink_inode_operations = { 3185 .readlink = generic_readlink, 3186 .get_link = shmem_get_link, 3187 #ifdef CONFIG_TMPFS_XATTR 3188 .listxattr = shmem_listxattr, 3189 #endif 3190 }; 3191 3192 static struct dentry *shmem_get_parent(struct dentry *child) 3193 { 3194 return ERR_PTR(-ESTALE); 3195 } 3196 3197 static int shmem_match(struct inode *ino, void *vfh) 3198 { 3199 __u32 *fh = vfh; 3200 __u64 inum = fh[2]; 3201 inum = (inum << 32) | fh[1]; 3202 return ino->i_ino == inum && fh[0] == ino->i_generation; 3203 } 3204 3205 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3206 struct fid *fid, int fh_len, int fh_type) 3207 { 3208 struct inode *inode; 3209 struct dentry *dentry = NULL; 3210 u64 inum; 3211 3212 if (fh_len < 3) 3213 return NULL; 3214 3215 inum = fid->raw[2]; 3216 inum = (inum << 32) | fid->raw[1]; 3217 3218 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3219 shmem_match, fid->raw); 3220 if (inode) { 3221 dentry = d_find_alias(inode); 3222 iput(inode); 3223 } 3224 3225 return dentry; 3226 } 3227 3228 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3229 struct inode *parent) 3230 { 3231 if (*len < 3) { 3232 *len = 3; 3233 return FILEID_INVALID; 3234 } 3235 3236 if (inode_unhashed(inode)) { 3237 /* Unfortunately insert_inode_hash is not idempotent, 3238 * so as we hash inodes here rather than at creation 3239 * time, we need a lock to ensure we only try 3240 * to do it once 3241 */ 3242 static DEFINE_SPINLOCK(lock); 3243 spin_lock(&lock); 3244 if (inode_unhashed(inode)) 3245 __insert_inode_hash(inode, 3246 inode->i_ino + inode->i_generation); 3247 spin_unlock(&lock); 3248 } 3249 3250 fh[0] = inode->i_generation; 3251 fh[1] = inode->i_ino; 3252 fh[2] = ((__u64)inode->i_ino) >> 32; 3253 3254 *len = 3; 3255 return 1; 3256 } 3257 3258 static const struct export_operations shmem_export_ops = { 3259 .get_parent = shmem_get_parent, 3260 .encode_fh = shmem_encode_fh, 3261 .fh_to_dentry = shmem_fh_to_dentry, 3262 }; 3263 3264 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 3265 bool remount) 3266 { 3267 char *this_char, *value, *rest; 3268 struct mempolicy *mpol = NULL; 3269 uid_t uid; 3270 gid_t gid; 3271 3272 while (options != NULL) { 3273 this_char = options; 3274 for (;;) { 3275 /* 3276 * NUL-terminate this option: unfortunately, 3277 * mount options form a comma-separated list, 3278 * but mpol's nodelist may also contain commas. 3279 */ 3280 options = strchr(options, ','); 3281 if (options == NULL) 3282 break; 3283 options++; 3284 if (!isdigit(*options)) { 3285 options[-1] = '\0'; 3286 break; 3287 } 3288 } 3289 if (!*this_char) 3290 continue; 3291 if ((value = strchr(this_char,'=')) != NULL) { 3292 *value++ = 0; 3293 } else { 3294 pr_err("tmpfs: No value for mount option '%s'\n", 3295 this_char); 3296 goto error; 3297 } 3298 3299 if (!strcmp(this_char,"size")) { 3300 unsigned long long size; 3301 size = memparse(value,&rest); 3302 if (*rest == '%') { 3303 size <<= PAGE_SHIFT; 3304 size *= totalram_pages; 3305 do_div(size, 100); 3306 rest++; 3307 } 3308 if (*rest) 3309 goto bad_val; 3310 sbinfo->max_blocks = 3311 DIV_ROUND_UP(size, PAGE_SIZE); 3312 } else if (!strcmp(this_char,"nr_blocks")) { 3313 sbinfo->max_blocks = memparse(value, &rest); 3314 if (*rest) 3315 goto bad_val; 3316 } else if (!strcmp(this_char,"nr_inodes")) { 3317 sbinfo->max_inodes = memparse(value, &rest); 3318 if (*rest) 3319 goto bad_val; 3320 } else if (!strcmp(this_char,"mode")) { 3321 if (remount) 3322 continue; 3323 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 3324 if (*rest) 3325 goto bad_val; 3326 } else if (!strcmp(this_char,"uid")) { 3327 if (remount) 3328 continue; 3329 uid = simple_strtoul(value, &rest, 0); 3330 if (*rest) 3331 goto bad_val; 3332 sbinfo->uid = make_kuid(current_user_ns(), uid); 3333 if (!uid_valid(sbinfo->uid)) 3334 goto bad_val; 3335 } else if (!strcmp(this_char,"gid")) { 3336 if (remount) 3337 continue; 3338 gid = simple_strtoul(value, &rest, 0); 3339 if (*rest) 3340 goto bad_val; 3341 sbinfo->gid = make_kgid(current_user_ns(), gid); 3342 if (!gid_valid(sbinfo->gid)) 3343 goto bad_val; 3344 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3345 } else if (!strcmp(this_char, "huge")) { 3346 int huge; 3347 huge = shmem_parse_huge(value); 3348 if (huge < 0) 3349 goto bad_val; 3350 if (!has_transparent_hugepage() && 3351 huge != SHMEM_HUGE_NEVER) 3352 goto bad_val; 3353 sbinfo->huge = huge; 3354 #endif 3355 #ifdef CONFIG_NUMA 3356 } else if (!strcmp(this_char,"mpol")) { 3357 mpol_put(mpol); 3358 mpol = NULL; 3359 if (mpol_parse_str(value, &mpol)) 3360 goto bad_val; 3361 #endif 3362 } else { 3363 pr_err("tmpfs: Bad mount option %s\n", this_char); 3364 goto error; 3365 } 3366 } 3367 sbinfo->mpol = mpol; 3368 return 0; 3369 3370 bad_val: 3371 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n", 3372 value, this_char); 3373 error: 3374 mpol_put(mpol); 3375 return 1; 3376 3377 } 3378 3379 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 3380 { 3381 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3382 struct shmem_sb_info config = *sbinfo; 3383 unsigned long inodes; 3384 int error = -EINVAL; 3385 3386 config.mpol = NULL; 3387 if (shmem_parse_options(data, &config, true)) 3388 return error; 3389 3390 spin_lock(&sbinfo->stat_lock); 3391 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3392 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 3393 goto out; 3394 if (config.max_inodes < inodes) 3395 goto out; 3396 /* 3397 * Those tests disallow limited->unlimited while any are in use; 3398 * but we must separately disallow unlimited->limited, because 3399 * in that case we have no record of how much is already in use. 3400 */ 3401 if (config.max_blocks && !sbinfo->max_blocks) 3402 goto out; 3403 if (config.max_inodes && !sbinfo->max_inodes) 3404 goto out; 3405 3406 error = 0; 3407 sbinfo->huge = config.huge; 3408 sbinfo->max_blocks = config.max_blocks; 3409 sbinfo->max_inodes = config.max_inodes; 3410 sbinfo->free_inodes = config.max_inodes - inodes; 3411 3412 /* 3413 * Preserve previous mempolicy unless mpol remount option was specified. 3414 */ 3415 if (config.mpol) { 3416 mpol_put(sbinfo->mpol); 3417 sbinfo->mpol = config.mpol; /* transfers initial ref */ 3418 } 3419 out: 3420 spin_unlock(&sbinfo->stat_lock); 3421 return error; 3422 } 3423 3424 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3425 { 3426 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3427 3428 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3429 seq_printf(seq, ",size=%luk", 3430 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3431 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3432 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3433 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 3434 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3435 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3436 seq_printf(seq, ",uid=%u", 3437 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3438 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3439 seq_printf(seq, ",gid=%u", 3440 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3441 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3442 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3443 if (sbinfo->huge) 3444 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3445 #endif 3446 shmem_show_mpol(seq, sbinfo->mpol); 3447 return 0; 3448 } 3449 3450 #define MFD_NAME_PREFIX "memfd:" 3451 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) 3452 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) 3453 3454 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING) 3455 3456 SYSCALL_DEFINE2(memfd_create, 3457 const char __user *, uname, 3458 unsigned int, flags) 3459 { 3460 struct shmem_inode_info *info; 3461 struct file *file; 3462 int fd, error; 3463 char *name; 3464 long len; 3465 3466 if (flags & ~(unsigned int)MFD_ALL_FLAGS) 3467 return -EINVAL; 3468 3469 /* length includes terminating zero */ 3470 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); 3471 if (len <= 0) 3472 return -EFAULT; 3473 if (len > MFD_NAME_MAX_LEN + 1) 3474 return -EINVAL; 3475 3476 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY); 3477 if (!name) 3478 return -ENOMEM; 3479 3480 strcpy(name, MFD_NAME_PREFIX); 3481 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { 3482 error = -EFAULT; 3483 goto err_name; 3484 } 3485 3486 /* terminating-zero may have changed after strnlen_user() returned */ 3487 if (name[len + MFD_NAME_PREFIX_LEN - 1]) { 3488 error = -EFAULT; 3489 goto err_name; 3490 } 3491 3492 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); 3493 if (fd < 0) { 3494 error = fd; 3495 goto err_name; 3496 } 3497 3498 file = shmem_file_setup(name, 0, VM_NORESERVE); 3499 if (IS_ERR(file)) { 3500 error = PTR_ERR(file); 3501 goto err_fd; 3502 } 3503 info = SHMEM_I(file_inode(file)); 3504 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; 3505 file->f_flags |= O_RDWR | O_LARGEFILE; 3506 if (flags & MFD_ALLOW_SEALING) 3507 info->seals &= ~F_SEAL_SEAL; 3508 3509 fd_install(fd, file); 3510 kfree(name); 3511 return fd; 3512 3513 err_fd: 3514 put_unused_fd(fd); 3515 err_name: 3516 kfree(name); 3517 return error; 3518 } 3519 3520 #endif /* CONFIG_TMPFS */ 3521 3522 static void shmem_put_super(struct super_block *sb) 3523 { 3524 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3525 3526 percpu_counter_destroy(&sbinfo->used_blocks); 3527 mpol_put(sbinfo->mpol); 3528 kfree(sbinfo); 3529 sb->s_fs_info = NULL; 3530 } 3531 3532 int shmem_fill_super(struct super_block *sb, void *data, int silent) 3533 { 3534 struct inode *inode; 3535 struct shmem_sb_info *sbinfo; 3536 int err = -ENOMEM; 3537 3538 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3539 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3540 L1_CACHE_BYTES), GFP_KERNEL); 3541 if (!sbinfo) 3542 return -ENOMEM; 3543 3544 sbinfo->mode = S_IRWXUGO | S_ISVTX; 3545 sbinfo->uid = current_fsuid(); 3546 sbinfo->gid = current_fsgid(); 3547 sb->s_fs_info = sbinfo; 3548 3549 #ifdef CONFIG_TMPFS 3550 /* 3551 * Per default we only allow half of the physical ram per 3552 * tmpfs instance, limiting inodes to one per page of lowmem; 3553 * but the internal instance is left unlimited. 3554 */ 3555 if (!(sb->s_flags & MS_KERNMOUNT)) { 3556 sbinfo->max_blocks = shmem_default_max_blocks(); 3557 sbinfo->max_inodes = shmem_default_max_inodes(); 3558 if (shmem_parse_options(data, sbinfo, false)) { 3559 err = -EINVAL; 3560 goto failed; 3561 } 3562 } else { 3563 sb->s_flags |= MS_NOUSER; 3564 } 3565 sb->s_export_op = &shmem_export_ops; 3566 sb->s_flags |= MS_NOSEC; 3567 #else 3568 sb->s_flags |= MS_NOUSER; 3569 #endif 3570 3571 spin_lock_init(&sbinfo->stat_lock); 3572 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3573 goto failed; 3574 sbinfo->free_inodes = sbinfo->max_inodes; 3575 spin_lock_init(&sbinfo->shrinklist_lock); 3576 INIT_LIST_HEAD(&sbinfo->shrinklist); 3577 3578 sb->s_maxbytes = MAX_LFS_FILESIZE; 3579 sb->s_blocksize = PAGE_SIZE; 3580 sb->s_blocksize_bits = PAGE_SHIFT; 3581 sb->s_magic = TMPFS_MAGIC; 3582 sb->s_op = &shmem_ops; 3583 sb->s_time_gran = 1; 3584 #ifdef CONFIG_TMPFS_XATTR 3585 sb->s_xattr = shmem_xattr_handlers; 3586 #endif 3587 #ifdef CONFIG_TMPFS_POSIX_ACL 3588 sb->s_flags |= MS_POSIXACL; 3589 #endif 3590 3591 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3592 if (!inode) 3593 goto failed; 3594 inode->i_uid = sbinfo->uid; 3595 inode->i_gid = sbinfo->gid; 3596 sb->s_root = d_make_root(inode); 3597 if (!sb->s_root) 3598 goto failed; 3599 return 0; 3600 3601 failed: 3602 shmem_put_super(sb); 3603 return err; 3604 } 3605 3606 static struct kmem_cache *shmem_inode_cachep; 3607 3608 static struct inode *shmem_alloc_inode(struct super_block *sb) 3609 { 3610 struct shmem_inode_info *info; 3611 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3612 if (!info) 3613 return NULL; 3614 return &info->vfs_inode; 3615 } 3616 3617 static void shmem_destroy_callback(struct rcu_head *head) 3618 { 3619 struct inode *inode = container_of(head, struct inode, i_rcu); 3620 if (S_ISLNK(inode->i_mode)) 3621 kfree(inode->i_link); 3622 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3623 } 3624 3625 static void shmem_destroy_inode(struct inode *inode) 3626 { 3627 if (S_ISREG(inode->i_mode)) 3628 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3629 call_rcu(&inode->i_rcu, shmem_destroy_callback); 3630 } 3631 3632 static void shmem_init_inode(void *foo) 3633 { 3634 struct shmem_inode_info *info = foo; 3635 inode_init_once(&info->vfs_inode); 3636 } 3637 3638 static int shmem_init_inodecache(void) 3639 { 3640 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3641 sizeof(struct shmem_inode_info), 3642 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3643 return 0; 3644 } 3645 3646 static void shmem_destroy_inodecache(void) 3647 { 3648 kmem_cache_destroy(shmem_inode_cachep); 3649 } 3650 3651 static const struct address_space_operations shmem_aops = { 3652 .writepage = shmem_writepage, 3653 .set_page_dirty = __set_page_dirty_no_writeback, 3654 #ifdef CONFIG_TMPFS 3655 .write_begin = shmem_write_begin, 3656 .write_end = shmem_write_end, 3657 #endif 3658 #ifdef CONFIG_MIGRATION 3659 .migratepage = migrate_page, 3660 #endif 3661 .error_remove_page = generic_error_remove_page, 3662 }; 3663 3664 static const struct file_operations shmem_file_operations = { 3665 .mmap = shmem_mmap, 3666 .get_unmapped_area = shmem_get_unmapped_area, 3667 #ifdef CONFIG_TMPFS 3668 .llseek = shmem_file_llseek, 3669 .read_iter = shmem_file_read_iter, 3670 .write_iter = generic_file_write_iter, 3671 .fsync = noop_fsync, 3672 .splice_read = generic_file_splice_read, 3673 .splice_write = iter_file_splice_write, 3674 .fallocate = shmem_fallocate, 3675 #endif 3676 }; 3677 3678 static const struct inode_operations shmem_inode_operations = { 3679 .getattr = shmem_getattr, 3680 .setattr = shmem_setattr, 3681 #ifdef CONFIG_TMPFS_XATTR 3682 .listxattr = shmem_listxattr, 3683 .set_acl = simple_set_acl, 3684 #endif 3685 }; 3686 3687 static const struct inode_operations shmem_dir_inode_operations = { 3688 #ifdef CONFIG_TMPFS 3689 .create = shmem_create, 3690 .lookup = simple_lookup, 3691 .link = shmem_link, 3692 .unlink = shmem_unlink, 3693 .symlink = shmem_symlink, 3694 .mkdir = shmem_mkdir, 3695 .rmdir = shmem_rmdir, 3696 .mknod = shmem_mknod, 3697 .rename = shmem_rename2, 3698 .tmpfile = shmem_tmpfile, 3699 #endif 3700 #ifdef CONFIG_TMPFS_XATTR 3701 .listxattr = shmem_listxattr, 3702 #endif 3703 #ifdef CONFIG_TMPFS_POSIX_ACL 3704 .setattr = shmem_setattr, 3705 .set_acl = simple_set_acl, 3706 #endif 3707 }; 3708 3709 static const struct inode_operations shmem_special_inode_operations = { 3710 #ifdef CONFIG_TMPFS_XATTR 3711 .listxattr = shmem_listxattr, 3712 #endif 3713 #ifdef CONFIG_TMPFS_POSIX_ACL 3714 .setattr = shmem_setattr, 3715 .set_acl = simple_set_acl, 3716 #endif 3717 }; 3718 3719 static const struct super_operations shmem_ops = { 3720 .alloc_inode = shmem_alloc_inode, 3721 .destroy_inode = shmem_destroy_inode, 3722 #ifdef CONFIG_TMPFS 3723 .statfs = shmem_statfs, 3724 .remount_fs = shmem_remount_fs, 3725 .show_options = shmem_show_options, 3726 #endif 3727 .evict_inode = shmem_evict_inode, 3728 .drop_inode = generic_delete_inode, 3729 .put_super = shmem_put_super, 3730 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3731 .nr_cached_objects = shmem_unused_huge_count, 3732 .free_cached_objects = shmem_unused_huge_scan, 3733 #endif 3734 }; 3735 3736 static const struct vm_operations_struct shmem_vm_ops = { 3737 .fault = shmem_fault, 3738 .map_pages = filemap_map_pages, 3739 #ifdef CONFIG_NUMA 3740 .set_policy = shmem_set_policy, 3741 .get_policy = shmem_get_policy, 3742 #endif 3743 }; 3744 3745 static struct dentry *shmem_mount(struct file_system_type *fs_type, 3746 int flags, const char *dev_name, void *data) 3747 { 3748 return mount_nodev(fs_type, flags, data, shmem_fill_super); 3749 } 3750 3751 static struct file_system_type shmem_fs_type = { 3752 .owner = THIS_MODULE, 3753 .name = "tmpfs", 3754 .mount = shmem_mount, 3755 .kill_sb = kill_litter_super, 3756 .fs_flags = FS_USERNS_MOUNT, 3757 }; 3758 3759 int __init shmem_init(void) 3760 { 3761 int error; 3762 3763 /* If rootfs called this, don't re-init */ 3764 if (shmem_inode_cachep) 3765 return 0; 3766 3767 error = shmem_init_inodecache(); 3768 if (error) 3769 goto out3; 3770 3771 error = register_filesystem(&shmem_fs_type); 3772 if (error) { 3773 pr_err("Could not register tmpfs\n"); 3774 goto out2; 3775 } 3776 3777 shm_mnt = kern_mount(&shmem_fs_type); 3778 if (IS_ERR(shm_mnt)) { 3779 error = PTR_ERR(shm_mnt); 3780 pr_err("Could not kern_mount tmpfs\n"); 3781 goto out1; 3782 } 3783 3784 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3785 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY) 3786 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3787 else 3788 shmem_huge = 0; /* just in case it was patched */ 3789 #endif 3790 return 0; 3791 3792 out1: 3793 unregister_filesystem(&shmem_fs_type); 3794 out2: 3795 shmem_destroy_inodecache(); 3796 out3: 3797 shm_mnt = ERR_PTR(error); 3798 return error; 3799 } 3800 3801 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) 3802 static ssize_t shmem_enabled_show(struct kobject *kobj, 3803 struct kobj_attribute *attr, char *buf) 3804 { 3805 int values[] = { 3806 SHMEM_HUGE_ALWAYS, 3807 SHMEM_HUGE_WITHIN_SIZE, 3808 SHMEM_HUGE_ADVISE, 3809 SHMEM_HUGE_NEVER, 3810 SHMEM_HUGE_DENY, 3811 SHMEM_HUGE_FORCE, 3812 }; 3813 int i, count; 3814 3815 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { 3816 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; 3817 3818 count += sprintf(buf + count, fmt, 3819 shmem_format_huge(values[i])); 3820 } 3821 buf[count - 1] = '\n'; 3822 return count; 3823 } 3824 3825 static ssize_t shmem_enabled_store(struct kobject *kobj, 3826 struct kobj_attribute *attr, const char *buf, size_t count) 3827 { 3828 char tmp[16]; 3829 int huge; 3830 3831 if (count + 1 > sizeof(tmp)) 3832 return -EINVAL; 3833 memcpy(tmp, buf, count); 3834 tmp[count] = '\0'; 3835 if (count && tmp[count - 1] == '\n') 3836 tmp[count - 1] = '\0'; 3837 3838 huge = shmem_parse_huge(tmp); 3839 if (huge == -EINVAL) 3840 return -EINVAL; 3841 if (!has_transparent_hugepage() && 3842 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3843 return -EINVAL; 3844 3845 shmem_huge = huge; 3846 if (shmem_huge < SHMEM_HUGE_DENY) 3847 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3848 return count; 3849 } 3850 3851 struct kobj_attribute shmem_enabled_attr = 3852 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3853 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ 3854 3855 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3856 bool shmem_huge_enabled(struct vm_area_struct *vma) 3857 { 3858 struct inode *inode = file_inode(vma->vm_file); 3859 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3860 loff_t i_size; 3861 pgoff_t off; 3862 3863 if (shmem_huge == SHMEM_HUGE_FORCE) 3864 return true; 3865 if (shmem_huge == SHMEM_HUGE_DENY) 3866 return false; 3867 switch (sbinfo->huge) { 3868 case SHMEM_HUGE_NEVER: 3869 return false; 3870 case SHMEM_HUGE_ALWAYS: 3871 return true; 3872 case SHMEM_HUGE_WITHIN_SIZE: 3873 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); 3874 i_size = round_up(i_size_read(inode), PAGE_SIZE); 3875 if (i_size >= HPAGE_PMD_SIZE && 3876 i_size >> PAGE_SHIFT >= off) 3877 return true; 3878 case SHMEM_HUGE_ADVISE: 3879 /* TODO: implement fadvise() hints */ 3880 return (vma->vm_flags & VM_HUGEPAGE); 3881 default: 3882 VM_BUG_ON(1); 3883 return false; 3884 } 3885 } 3886 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 3887 3888 #else /* !CONFIG_SHMEM */ 3889 3890 /* 3891 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 3892 * 3893 * This is intended for small system where the benefits of the full 3894 * shmem code (swap-backed and resource-limited) are outweighed by 3895 * their complexity. On systems without swap this code should be 3896 * effectively equivalent, but much lighter weight. 3897 */ 3898 3899 static struct file_system_type shmem_fs_type = { 3900 .name = "tmpfs", 3901 .mount = ramfs_mount, 3902 .kill_sb = kill_litter_super, 3903 .fs_flags = FS_USERNS_MOUNT, 3904 }; 3905 3906 int __init shmem_init(void) 3907 { 3908 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 3909 3910 shm_mnt = kern_mount(&shmem_fs_type); 3911 BUG_ON(IS_ERR(shm_mnt)); 3912 3913 return 0; 3914 } 3915 3916 int shmem_unuse(swp_entry_t swap, struct page *page) 3917 { 3918 return 0; 3919 } 3920 3921 int shmem_lock(struct file *file, int lock, struct user_struct *user) 3922 { 3923 return 0; 3924 } 3925 3926 void shmem_unlock_mapping(struct address_space *mapping) 3927 { 3928 } 3929 3930 #ifdef CONFIG_MMU 3931 unsigned long shmem_get_unmapped_area(struct file *file, 3932 unsigned long addr, unsigned long len, 3933 unsigned long pgoff, unsigned long flags) 3934 { 3935 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 3936 } 3937 #endif 3938 3939 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 3940 { 3941 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 3942 } 3943 EXPORT_SYMBOL_GPL(shmem_truncate_range); 3944 3945 #define shmem_vm_ops generic_file_vm_ops 3946 #define shmem_file_operations ramfs_file_operations 3947 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 3948 #define shmem_acct_size(flags, size) 0 3949 #define shmem_unacct_size(flags, size) do {} while (0) 3950 3951 #endif /* CONFIG_SHMEM */ 3952 3953 /* common code */ 3954 3955 static const struct dentry_operations anon_ops = { 3956 .d_dname = simple_dname 3957 }; 3958 3959 static struct file *__shmem_file_setup(const char *name, loff_t size, 3960 unsigned long flags, unsigned int i_flags) 3961 { 3962 struct file *res; 3963 struct inode *inode; 3964 struct path path; 3965 struct super_block *sb; 3966 struct qstr this; 3967 3968 if (IS_ERR(shm_mnt)) 3969 return ERR_CAST(shm_mnt); 3970 3971 if (size < 0 || size > MAX_LFS_FILESIZE) 3972 return ERR_PTR(-EINVAL); 3973 3974 if (shmem_acct_size(flags, size)) 3975 return ERR_PTR(-ENOMEM); 3976 3977 res = ERR_PTR(-ENOMEM); 3978 this.name = name; 3979 this.len = strlen(name); 3980 this.hash = 0; /* will go */ 3981 sb = shm_mnt->mnt_sb; 3982 path.mnt = mntget(shm_mnt); 3983 path.dentry = d_alloc_pseudo(sb, &this); 3984 if (!path.dentry) 3985 goto put_memory; 3986 d_set_d_op(path.dentry, &anon_ops); 3987 3988 res = ERR_PTR(-ENOSPC); 3989 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 3990 if (!inode) 3991 goto put_memory; 3992 3993 inode->i_flags |= i_flags; 3994 d_instantiate(path.dentry, inode); 3995 inode->i_size = size; 3996 clear_nlink(inode); /* It is unlinked */ 3997 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 3998 if (IS_ERR(res)) 3999 goto put_path; 4000 4001 res = alloc_file(&path, FMODE_WRITE | FMODE_READ, 4002 &shmem_file_operations); 4003 if (IS_ERR(res)) 4004 goto put_path; 4005 4006 return res; 4007 4008 put_memory: 4009 shmem_unacct_size(flags, size); 4010 put_path: 4011 path_put(&path); 4012 return res; 4013 } 4014 4015 /** 4016 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4017 * kernel internal. There will be NO LSM permission checks against the 4018 * underlying inode. So users of this interface must do LSM checks at a 4019 * higher layer. The users are the big_key and shm implementations. LSM 4020 * checks are provided at the key or shm level rather than the inode. 4021 * @name: name for dentry (to be seen in /proc/<pid>/maps 4022 * @size: size to be set for the file 4023 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4024 */ 4025 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4026 { 4027 return __shmem_file_setup(name, size, flags, S_PRIVATE); 4028 } 4029 4030 /** 4031 * shmem_file_setup - get an unlinked file living in tmpfs 4032 * @name: name for dentry (to be seen in /proc/<pid>/maps 4033 * @size: size to be set for the file 4034 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4035 */ 4036 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4037 { 4038 return __shmem_file_setup(name, size, flags, 0); 4039 } 4040 EXPORT_SYMBOL_GPL(shmem_file_setup); 4041 4042 /** 4043 * shmem_zero_setup - setup a shared anonymous mapping 4044 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 4045 */ 4046 int shmem_zero_setup(struct vm_area_struct *vma) 4047 { 4048 struct file *file; 4049 loff_t size = vma->vm_end - vma->vm_start; 4050 4051 /* 4052 * Cloning a new file under mmap_sem leads to a lock ordering conflict 4053 * between XFS directory reading and selinux: since this file is only 4054 * accessible to the user through its mapping, use S_PRIVATE flag to 4055 * bypass file security, in the same way as shmem_kernel_file_setup(). 4056 */ 4057 file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE); 4058 if (IS_ERR(file)) 4059 return PTR_ERR(file); 4060 4061 if (vma->vm_file) 4062 fput(vma->vm_file); 4063 vma->vm_file = file; 4064 vma->vm_ops = &shmem_vm_ops; 4065 4066 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 4067 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4068 (vma->vm_end & HPAGE_PMD_MASK)) { 4069 khugepaged_enter(vma, vma->vm_flags); 4070 } 4071 4072 return 0; 4073 } 4074 4075 /** 4076 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4077 * @mapping: the page's address_space 4078 * @index: the page index 4079 * @gfp: the page allocator flags to use if allocating 4080 * 4081 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4082 * with any new page allocations done using the specified allocation flags. 4083 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4084 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4085 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4086 * 4087 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4088 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4089 */ 4090 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4091 pgoff_t index, gfp_t gfp) 4092 { 4093 #ifdef CONFIG_SHMEM 4094 struct inode *inode = mapping->host; 4095 struct page *page; 4096 int error; 4097 4098 BUG_ON(mapping->a_ops != &shmem_aops); 4099 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4100 gfp, NULL, NULL); 4101 if (error) 4102 page = ERR_PTR(error); 4103 else 4104 unlock_page(page); 4105 return page; 4106 #else 4107 /* 4108 * The tiny !SHMEM case uses ramfs without swap 4109 */ 4110 return read_cache_page_gfp(mapping, index, gfp); 4111 #endif 4112 } 4113 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4114