1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/export.h> 33 #include <linux/swap.h> 34 #include <linux/uio.h> 35 #include <linux/khugepaged.h> 36 37 static struct vfsmount *shm_mnt; 38 39 #ifdef CONFIG_SHMEM 40 /* 41 * This virtual memory filesystem is heavily based on the ramfs. It 42 * extends ramfs by the ability to use swap and honor resource limits 43 * which makes it a completely usable filesystem. 44 */ 45 46 #include <linux/xattr.h> 47 #include <linux/exportfs.h> 48 #include <linux/posix_acl.h> 49 #include <linux/posix_acl_xattr.h> 50 #include <linux/mman.h> 51 #include <linux/string.h> 52 #include <linux/slab.h> 53 #include <linux/backing-dev.h> 54 #include <linux/shmem_fs.h> 55 #include <linux/writeback.h> 56 #include <linux/blkdev.h> 57 #include <linux/pagevec.h> 58 #include <linux/percpu_counter.h> 59 #include <linux/falloc.h> 60 #include <linux/splice.h> 61 #include <linux/security.h> 62 #include <linux/swapops.h> 63 #include <linux/mempolicy.h> 64 #include <linux/namei.h> 65 #include <linux/ctype.h> 66 #include <linux/migrate.h> 67 #include <linux/highmem.h> 68 #include <linux/seq_file.h> 69 #include <linux/magic.h> 70 #include <linux/syscalls.h> 71 #include <linux/fcntl.h> 72 #include <uapi/linux/memfd.h> 73 74 #include <linux/uaccess.h> 75 #include <asm/pgtable.h> 76 77 #include "internal.h" 78 79 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 80 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 81 82 /* Pretend that each entry is of this size in directory's i_size */ 83 #define BOGO_DIRENT_SIZE 20 84 85 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 86 #define SHORT_SYMLINK_LEN 128 87 88 /* 89 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 90 * inode->i_private (with i_mutex making sure that it has only one user at 91 * a time): we would prefer not to enlarge the shmem inode just for that. 92 */ 93 struct shmem_falloc { 94 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 95 pgoff_t start; /* start of range currently being fallocated */ 96 pgoff_t next; /* the next page offset to be fallocated */ 97 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 98 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 99 }; 100 101 #ifdef CONFIG_TMPFS 102 static unsigned long shmem_default_max_blocks(void) 103 { 104 return totalram_pages / 2; 105 } 106 107 static unsigned long shmem_default_max_inodes(void) 108 { 109 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 110 } 111 #endif 112 113 static bool shmem_should_replace_page(struct page *page, gfp_t gfp); 114 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 115 struct shmem_inode_info *info, pgoff_t index); 116 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 117 struct page **pagep, enum sgp_type sgp, 118 gfp_t gfp, struct mm_struct *fault_mm, int *fault_type); 119 120 int shmem_getpage(struct inode *inode, pgoff_t index, 121 struct page **pagep, enum sgp_type sgp) 122 { 123 return shmem_getpage_gfp(inode, index, pagep, sgp, 124 mapping_gfp_mask(inode->i_mapping), NULL, NULL); 125 } 126 127 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 128 { 129 return sb->s_fs_info; 130 } 131 132 /* 133 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 134 * for shared memory and for shared anonymous (/dev/zero) mappings 135 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 136 * consistent with the pre-accounting of private mappings ... 137 */ 138 static inline int shmem_acct_size(unsigned long flags, loff_t size) 139 { 140 return (flags & VM_NORESERVE) ? 141 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 142 } 143 144 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 145 { 146 if (!(flags & VM_NORESERVE)) 147 vm_unacct_memory(VM_ACCT(size)); 148 } 149 150 static inline int shmem_reacct_size(unsigned long flags, 151 loff_t oldsize, loff_t newsize) 152 { 153 if (!(flags & VM_NORESERVE)) { 154 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 155 return security_vm_enough_memory_mm(current->mm, 156 VM_ACCT(newsize) - VM_ACCT(oldsize)); 157 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 158 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 159 } 160 return 0; 161 } 162 163 /* 164 * ... whereas tmpfs objects are accounted incrementally as 165 * pages are allocated, in order to allow large sparse files. 166 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 167 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 168 */ 169 static inline int shmem_acct_block(unsigned long flags, long pages) 170 { 171 if (!(flags & VM_NORESERVE)) 172 return 0; 173 174 return security_vm_enough_memory_mm(current->mm, 175 pages * VM_ACCT(PAGE_SIZE)); 176 } 177 178 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 179 { 180 if (flags & VM_NORESERVE) 181 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 182 } 183 184 static const struct super_operations shmem_ops; 185 static const struct address_space_operations shmem_aops; 186 static const struct file_operations shmem_file_operations; 187 static const struct inode_operations shmem_inode_operations; 188 static const struct inode_operations shmem_dir_inode_operations; 189 static const struct inode_operations shmem_special_inode_operations; 190 static const struct vm_operations_struct shmem_vm_ops; 191 static struct file_system_type shmem_fs_type; 192 193 static LIST_HEAD(shmem_swaplist); 194 static DEFINE_MUTEX(shmem_swaplist_mutex); 195 196 static int shmem_reserve_inode(struct super_block *sb) 197 { 198 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 199 if (sbinfo->max_inodes) { 200 spin_lock(&sbinfo->stat_lock); 201 if (!sbinfo->free_inodes) { 202 spin_unlock(&sbinfo->stat_lock); 203 return -ENOSPC; 204 } 205 sbinfo->free_inodes--; 206 spin_unlock(&sbinfo->stat_lock); 207 } 208 return 0; 209 } 210 211 static void shmem_free_inode(struct super_block *sb) 212 { 213 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 214 if (sbinfo->max_inodes) { 215 spin_lock(&sbinfo->stat_lock); 216 sbinfo->free_inodes++; 217 spin_unlock(&sbinfo->stat_lock); 218 } 219 } 220 221 /** 222 * shmem_recalc_inode - recalculate the block usage of an inode 223 * @inode: inode to recalc 224 * 225 * We have to calculate the free blocks since the mm can drop 226 * undirtied hole pages behind our back. 227 * 228 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 229 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 230 * 231 * It has to be called with the spinlock held. 232 */ 233 static void shmem_recalc_inode(struct inode *inode) 234 { 235 struct shmem_inode_info *info = SHMEM_I(inode); 236 long freed; 237 238 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 239 if (freed > 0) { 240 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 241 if (sbinfo->max_blocks) 242 percpu_counter_add(&sbinfo->used_blocks, -freed); 243 info->alloced -= freed; 244 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 245 shmem_unacct_blocks(info->flags, freed); 246 } 247 } 248 249 bool shmem_charge(struct inode *inode, long pages) 250 { 251 struct shmem_inode_info *info = SHMEM_I(inode); 252 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 253 unsigned long flags; 254 255 if (shmem_acct_block(info->flags, pages)) 256 return false; 257 spin_lock_irqsave(&info->lock, flags); 258 info->alloced += pages; 259 inode->i_blocks += pages * BLOCKS_PER_PAGE; 260 shmem_recalc_inode(inode); 261 spin_unlock_irqrestore(&info->lock, flags); 262 inode->i_mapping->nrpages += pages; 263 264 if (!sbinfo->max_blocks) 265 return true; 266 if (percpu_counter_compare(&sbinfo->used_blocks, 267 sbinfo->max_blocks - pages) > 0) { 268 inode->i_mapping->nrpages -= pages; 269 spin_lock_irqsave(&info->lock, flags); 270 info->alloced -= pages; 271 shmem_recalc_inode(inode); 272 spin_unlock_irqrestore(&info->lock, flags); 273 shmem_unacct_blocks(info->flags, pages); 274 return false; 275 } 276 percpu_counter_add(&sbinfo->used_blocks, pages); 277 return true; 278 } 279 280 void shmem_uncharge(struct inode *inode, long pages) 281 { 282 struct shmem_inode_info *info = SHMEM_I(inode); 283 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 284 unsigned long flags; 285 286 spin_lock_irqsave(&info->lock, flags); 287 info->alloced -= pages; 288 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 289 shmem_recalc_inode(inode); 290 spin_unlock_irqrestore(&info->lock, flags); 291 292 if (sbinfo->max_blocks) 293 percpu_counter_sub(&sbinfo->used_blocks, pages); 294 shmem_unacct_blocks(info->flags, pages); 295 } 296 297 /* 298 * Replace item expected in radix tree by a new item, while holding tree lock. 299 */ 300 static int shmem_radix_tree_replace(struct address_space *mapping, 301 pgoff_t index, void *expected, void *replacement) 302 { 303 struct radix_tree_node *node; 304 void **pslot; 305 void *item; 306 307 VM_BUG_ON(!expected); 308 VM_BUG_ON(!replacement); 309 item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); 310 if (!item) 311 return -ENOENT; 312 if (item != expected) 313 return -ENOENT; 314 __radix_tree_replace(&mapping->page_tree, node, pslot, 315 replacement, NULL, NULL); 316 return 0; 317 } 318 319 /* 320 * Sometimes, before we decide whether to proceed or to fail, we must check 321 * that an entry was not already brought back from swap by a racing thread. 322 * 323 * Checking page is not enough: by the time a SwapCache page is locked, it 324 * might be reused, and again be SwapCache, using the same swap as before. 325 */ 326 static bool shmem_confirm_swap(struct address_space *mapping, 327 pgoff_t index, swp_entry_t swap) 328 { 329 void *item; 330 331 rcu_read_lock(); 332 item = radix_tree_lookup(&mapping->page_tree, index); 333 rcu_read_unlock(); 334 return item == swp_to_radix_entry(swap); 335 } 336 337 /* 338 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 339 * 340 * SHMEM_HUGE_NEVER: 341 * disables huge pages for the mount; 342 * SHMEM_HUGE_ALWAYS: 343 * enables huge pages for the mount; 344 * SHMEM_HUGE_WITHIN_SIZE: 345 * only allocate huge pages if the page will be fully within i_size, 346 * also respect fadvise()/madvise() hints; 347 * SHMEM_HUGE_ADVISE: 348 * only allocate huge pages if requested with fadvise()/madvise(); 349 */ 350 351 #define SHMEM_HUGE_NEVER 0 352 #define SHMEM_HUGE_ALWAYS 1 353 #define SHMEM_HUGE_WITHIN_SIZE 2 354 #define SHMEM_HUGE_ADVISE 3 355 356 /* 357 * Special values. 358 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 359 * 360 * SHMEM_HUGE_DENY: 361 * disables huge on shm_mnt and all mounts, for emergency use; 362 * SHMEM_HUGE_FORCE: 363 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 364 * 365 */ 366 #define SHMEM_HUGE_DENY (-1) 367 #define SHMEM_HUGE_FORCE (-2) 368 369 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 370 /* ifdef here to avoid bloating shmem.o when not necessary */ 371 372 int shmem_huge __read_mostly; 373 374 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 375 static int shmem_parse_huge(const char *str) 376 { 377 if (!strcmp(str, "never")) 378 return SHMEM_HUGE_NEVER; 379 if (!strcmp(str, "always")) 380 return SHMEM_HUGE_ALWAYS; 381 if (!strcmp(str, "within_size")) 382 return SHMEM_HUGE_WITHIN_SIZE; 383 if (!strcmp(str, "advise")) 384 return SHMEM_HUGE_ADVISE; 385 if (!strcmp(str, "deny")) 386 return SHMEM_HUGE_DENY; 387 if (!strcmp(str, "force")) 388 return SHMEM_HUGE_FORCE; 389 return -EINVAL; 390 } 391 392 static const char *shmem_format_huge(int huge) 393 { 394 switch (huge) { 395 case SHMEM_HUGE_NEVER: 396 return "never"; 397 case SHMEM_HUGE_ALWAYS: 398 return "always"; 399 case SHMEM_HUGE_WITHIN_SIZE: 400 return "within_size"; 401 case SHMEM_HUGE_ADVISE: 402 return "advise"; 403 case SHMEM_HUGE_DENY: 404 return "deny"; 405 case SHMEM_HUGE_FORCE: 406 return "force"; 407 default: 408 VM_BUG_ON(1); 409 return "bad_val"; 410 } 411 } 412 #endif 413 414 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 415 struct shrink_control *sc, unsigned long nr_to_split) 416 { 417 LIST_HEAD(list), *pos, *next; 418 struct inode *inode; 419 struct shmem_inode_info *info; 420 struct page *page; 421 unsigned long batch = sc ? sc->nr_to_scan : 128; 422 int removed = 0, split = 0; 423 424 if (list_empty(&sbinfo->shrinklist)) 425 return SHRINK_STOP; 426 427 spin_lock(&sbinfo->shrinklist_lock); 428 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 429 info = list_entry(pos, struct shmem_inode_info, shrinklist); 430 431 /* pin the inode */ 432 inode = igrab(&info->vfs_inode); 433 434 /* inode is about to be evicted */ 435 if (!inode) { 436 list_del_init(&info->shrinklist); 437 removed++; 438 goto next; 439 } 440 441 /* Check if there's anything to gain */ 442 if (round_up(inode->i_size, PAGE_SIZE) == 443 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 444 list_del_init(&info->shrinklist); 445 removed++; 446 iput(inode); 447 goto next; 448 } 449 450 list_move(&info->shrinklist, &list); 451 next: 452 if (!--batch) 453 break; 454 } 455 spin_unlock(&sbinfo->shrinklist_lock); 456 457 list_for_each_safe(pos, next, &list) { 458 int ret; 459 460 info = list_entry(pos, struct shmem_inode_info, shrinklist); 461 inode = &info->vfs_inode; 462 463 if (nr_to_split && split >= nr_to_split) { 464 iput(inode); 465 continue; 466 } 467 468 page = find_lock_page(inode->i_mapping, 469 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 470 if (!page) 471 goto drop; 472 473 if (!PageTransHuge(page)) { 474 unlock_page(page); 475 put_page(page); 476 goto drop; 477 } 478 479 ret = split_huge_page(page); 480 unlock_page(page); 481 put_page(page); 482 483 if (ret) { 484 /* split failed: leave it on the list */ 485 iput(inode); 486 continue; 487 } 488 489 split++; 490 drop: 491 list_del_init(&info->shrinklist); 492 removed++; 493 iput(inode); 494 } 495 496 spin_lock(&sbinfo->shrinklist_lock); 497 list_splice_tail(&list, &sbinfo->shrinklist); 498 sbinfo->shrinklist_len -= removed; 499 spin_unlock(&sbinfo->shrinklist_lock); 500 501 return split; 502 } 503 504 static long shmem_unused_huge_scan(struct super_block *sb, 505 struct shrink_control *sc) 506 { 507 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 508 509 if (!READ_ONCE(sbinfo->shrinklist_len)) 510 return SHRINK_STOP; 511 512 return shmem_unused_huge_shrink(sbinfo, sc, 0); 513 } 514 515 static long shmem_unused_huge_count(struct super_block *sb, 516 struct shrink_control *sc) 517 { 518 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 519 return READ_ONCE(sbinfo->shrinklist_len); 520 } 521 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 522 523 #define shmem_huge SHMEM_HUGE_DENY 524 525 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 526 struct shrink_control *sc, unsigned long nr_to_split) 527 { 528 return 0; 529 } 530 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 531 532 /* 533 * Like add_to_page_cache_locked, but error if expected item has gone. 534 */ 535 static int shmem_add_to_page_cache(struct page *page, 536 struct address_space *mapping, 537 pgoff_t index, void *expected) 538 { 539 int error, nr = hpage_nr_pages(page); 540 541 VM_BUG_ON_PAGE(PageTail(page), page); 542 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 543 VM_BUG_ON_PAGE(!PageLocked(page), page); 544 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 545 VM_BUG_ON(expected && PageTransHuge(page)); 546 547 page_ref_add(page, nr); 548 page->mapping = mapping; 549 page->index = index; 550 551 spin_lock_irq(&mapping->tree_lock); 552 if (PageTransHuge(page)) { 553 void __rcu **results; 554 pgoff_t idx; 555 int i; 556 557 error = 0; 558 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 559 &results, &idx, index, 1) && 560 idx < index + HPAGE_PMD_NR) { 561 error = -EEXIST; 562 } 563 564 if (!error) { 565 for (i = 0; i < HPAGE_PMD_NR; i++) { 566 error = radix_tree_insert(&mapping->page_tree, 567 index + i, page + i); 568 VM_BUG_ON(error); 569 } 570 count_vm_event(THP_FILE_ALLOC); 571 } 572 } else if (!expected) { 573 error = radix_tree_insert(&mapping->page_tree, index, page); 574 } else { 575 error = shmem_radix_tree_replace(mapping, index, expected, 576 page); 577 } 578 579 if (!error) { 580 mapping->nrpages += nr; 581 if (PageTransHuge(page)) 582 __inc_node_page_state(page, NR_SHMEM_THPS); 583 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 584 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 585 spin_unlock_irq(&mapping->tree_lock); 586 } else { 587 page->mapping = NULL; 588 spin_unlock_irq(&mapping->tree_lock); 589 page_ref_sub(page, nr); 590 } 591 return error; 592 } 593 594 /* 595 * Like delete_from_page_cache, but substitutes swap for page. 596 */ 597 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 598 { 599 struct address_space *mapping = page->mapping; 600 int error; 601 602 VM_BUG_ON_PAGE(PageCompound(page), page); 603 604 spin_lock_irq(&mapping->tree_lock); 605 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 606 page->mapping = NULL; 607 mapping->nrpages--; 608 __dec_node_page_state(page, NR_FILE_PAGES); 609 __dec_node_page_state(page, NR_SHMEM); 610 spin_unlock_irq(&mapping->tree_lock); 611 put_page(page); 612 BUG_ON(error); 613 } 614 615 /* 616 * Remove swap entry from radix tree, free the swap and its page cache. 617 */ 618 static int shmem_free_swap(struct address_space *mapping, 619 pgoff_t index, void *radswap) 620 { 621 void *old; 622 623 spin_lock_irq(&mapping->tree_lock); 624 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 625 spin_unlock_irq(&mapping->tree_lock); 626 if (old != radswap) 627 return -ENOENT; 628 free_swap_and_cache(radix_to_swp_entry(radswap)); 629 return 0; 630 } 631 632 /* 633 * Determine (in bytes) how many of the shmem object's pages mapped by the 634 * given offsets are swapped out. 635 * 636 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 637 * as long as the inode doesn't go away and racy results are not a problem. 638 */ 639 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 640 pgoff_t start, pgoff_t end) 641 { 642 struct radix_tree_iter iter; 643 void **slot; 644 struct page *page; 645 unsigned long swapped = 0; 646 647 rcu_read_lock(); 648 649 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 650 if (iter.index >= end) 651 break; 652 653 page = radix_tree_deref_slot(slot); 654 655 if (radix_tree_deref_retry(page)) { 656 slot = radix_tree_iter_retry(&iter); 657 continue; 658 } 659 660 if (radix_tree_exceptional_entry(page)) 661 swapped++; 662 663 if (need_resched()) { 664 slot = radix_tree_iter_resume(slot, &iter); 665 cond_resched_rcu(); 666 } 667 } 668 669 rcu_read_unlock(); 670 671 return swapped << PAGE_SHIFT; 672 } 673 674 /* 675 * Determine (in bytes) how many of the shmem object's pages mapped by the 676 * given vma is swapped out. 677 * 678 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 679 * as long as the inode doesn't go away and racy results are not a problem. 680 */ 681 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 682 { 683 struct inode *inode = file_inode(vma->vm_file); 684 struct shmem_inode_info *info = SHMEM_I(inode); 685 struct address_space *mapping = inode->i_mapping; 686 unsigned long swapped; 687 688 /* Be careful as we don't hold info->lock */ 689 swapped = READ_ONCE(info->swapped); 690 691 /* 692 * The easier cases are when the shmem object has nothing in swap, or 693 * the vma maps it whole. Then we can simply use the stats that we 694 * already track. 695 */ 696 if (!swapped) 697 return 0; 698 699 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 700 return swapped << PAGE_SHIFT; 701 702 /* Here comes the more involved part */ 703 return shmem_partial_swap_usage(mapping, 704 linear_page_index(vma, vma->vm_start), 705 linear_page_index(vma, vma->vm_end)); 706 } 707 708 /* 709 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 710 */ 711 void shmem_unlock_mapping(struct address_space *mapping) 712 { 713 struct pagevec pvec; 714 pgoff_t indices[PAGEVEC_SIZE]; 715 pgoff_t index = 0; 716 717 pagevec_init(&pvec, 0); 718 /* 719 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 720 */ 721 while (!mapping_unevictable(mapping)) { 722 /* 723 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it 724 * has finished, if it hits a row of PAGEVEC_SIZE swap entries. 725 */ 726 pvec.nr = find_get_entries(mapping, index, 727 PAGEVEC_SIZE, pvec.pages, indices); 728 if (!pvec.nr) 729 break; 730 index = indices[pvec.nr - 1] + 1; 731 pagevec_remove_exceptionals(&pvec); 732 check_move_unevictable_pages(pvec.pages, pvec.nr); 733 pagevec_release(&pvec); 734 cond_resched(); 735 } 736 } 737 738 /* 739 * Remove range of pages and swap entries from radix tree, and free them. 740 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 741 */ 742 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 743 bool unfalloc) 744 { 745 struct address_space *mapping = inode->i_mapping; 746 struct shmem_inode_info *info = SHMEM_I(inode); 747 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 748 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 749 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 750 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 751 struct pagevec pvec; 752 pgoff_t indices[PAGEVEC_SIZE]; 753 long nr_swaps_freed = 0; 754 pgoff_t index; 755 int i; 756 757 if (lend == -1) 758 end = -1; /* unsigned, so actually very big */ 759 760 pagevec_init(&pvec, 0); 761 index = start; 762 while (index < end) { 763 pvec.nr = find_get_entries(mapping, index, 764 min(end - index, (pgoff_t)PAGEVEC_SIZE), 765 pvec.pages, indices); 766 if (!pvec.nr) 767 break; 768 for (i = 0; i < pagevec_count(&pvec); i++) { 769 struct page *page = pvec.pages[i]; 770 771 index = indices[i]; 772 if (index >= end) 773 break; 774 775 if (radix_tree_exceptional_entry(page)) { 776 if (unfalloc) 777 continue; 778 nr_swaps_freed += !shmem_free_swap(mapping, 779 index, page); 780 continue; 781 } 782 783 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); 784 785 if (!trylock_page(page)) 786 continue; 787 788 if (PageTransTail(page)) { 789 /* Middle of THP: zero out the page */ 790 clear_highpage(page); 791 unlock_page(page); 792 continue; 793 } else if (PageTransHuge(page)) { 794 if (index == round_down(end, HPAGE_PMD_NR)) { 795 /* 796 * Range ends in the middle of THP: 797 * zero out the page 798 */ 799 clear_highpage(page); 800 unlock_page(page); 801 continue; 802 } 803 index += HPAGE_PMD_NR - 1; 804 i += HPAGE_PMD_NR - 1; 805 } 806 807 if (!unfalloc || !PageUptodate(page)) { 808 VM_BUG_ON_PAGE(PageTail(page), page); 809 if (page_mapping(page) == mapping) { 810 VM_BUG_ON_PAGE(PageWriteback(page), page); 811 truncate_inode_page(mapping, page); 812 } 813 } 814 unlock_page(page); 815 } 816 pagevec_remove_exceptionals(&pvec); 817 pagevec_release(&pvec); 818 cond_resched(); 819 index++; 820 } 821 822 if (partial_start) { 823 struct page *page = NULL; 824 shmem_getpage(inode, start - 1, &page, SGP_READ); 825 if (page) { 826 unsigned int top = PAGE_SIZE; 827 if (start > end) { 828 top = partial_end; 829 partial_end = 0; 830 } 831 zero_user_segment(page, partial_start, top); 832 set_page_dirty(page); 833 unlock_page(page); 834 put_page(page); 835 } 836 } 837 if (partial_end) { 838 struct page *page = NULL; 839 shmem_getpage(inode, end, &page, SGP_READ); 840 if (page) { 841 zero_user_segment(page, 0, partial_end); 842 set_page_dirty(page); 843 unlock_page(page); 844 put_page(page); 845 } 846 } 847 if (start >= end) 848 return; 849 850 index = start; 851 while (index < end) { 852 cond_resched(); 853 854 pvec.nr = find_get_entries(mapping, index, 855 min(end - index, (pgoff_t)PAGEVEC_SIZE), 856 pvec.pages, indices); 857 if (!pvec.nr) { 858 /* If all gone or hole-punch or unfalloc, we're done */ 859 if (index == start || end != -1) 860 break; 861 /* But if truncating, restart to make sure all gone */ 862 index = start; 863 continue; 864 } 865 for (i = 0; i < pagevec_count(&pvec); i++) { 866 struct page *page = pvec.pages[i]; 867 868 index = indices[i]; 869 if (index >= end) 870 break; 871 872 if (radix_tree_exceptional_entry(page)) { 873 if (unfalloc) 874 continue; 875 if (shmem_free_swap(mapping, index, page)) { 876 /* Swap was replaced by page: retry */ 877 index--; 878 break; 879 } 880 nr_swaps_freed++; 881 continue; 882 } 883 884 lock_page(page); 885 886 if (PageTransTail(page)) { 887 /* Middle of THP: zero out the page */ 888 clear_highpage(page); 889 unlock_page(page); 890 /* 891 * Partial thp truncate due 'start' in middle 892 * of THP: don't need to look on these pages 893 * again on !pvec.nr restart. 894 */ 895 if (index != round_down(end, HPAGE_PMD_NR)) 896 start++; 897 continue; 898 } else if (PageTransHuge(page)) { 899 if (index == round_down(end, HPAGE_PMD_NR)) { 900 /* 901 * Range ends in the middle of THP: 902 * zero out the page 903 */ 904 clear_highpage(page); 905 unlock_page(page); 906 continue; 907 } 908 index += HPAGE_PMD_NR - 1; 909 i += HPAGE_PMD_NR - 1; 910 } 911 912 if (!unfalloc || !PageUptodate(page)) { 913 VM_BUG_ON_PAGE(PageTail(page), page); 914 if (page_mapping(page) == mapping) { 915 VM_BUG_ON_PAGE(PageWriteback(page), page); 916 truncate_inode_page(mapping, page); 917 } else { 918 /* Page was replaced by swap: retry */ 919 unlock_page(page); 920 index--; 921 break; 922 } 923 } 924 unlock_page(page); 925 } 926 pagevec_remove_exceptionals(&pvec); 927 pagevec_release(&pvec); 928 index++; 929 } 930 931 spin_lock_irq(&info->lock); 932 info->swapped -= nr_swaps_freed; 933 shmem_recalc_inode(inode); 934 spin_unlock_irq(&info->lock); 935 } 936 937 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 938 { 939 shmem_undo_range(inode, lstart, lend, false); 940 inode->i_ctime = inode->i_mtime = current_time(inode); 941 } 942 EXPORT_SYMBOL_GPL(shmem_truncate_range); 943 944 static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry, 945 struct kstat *stat) 946 { 947 struct inode *inode = dentry->d_inode; 948 struct shmem_inode_info *info = SHMEM_I(inode); 949 950 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 951 spin_lock_irq(&info->lock); 952 shmem_recalc_inode(inode); 953 spin_unlock_irq(&info->lock); 954 } 955 generic_fillattr(inode, stat); 956 return 0; 957 } 958 959 static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 960 { 961 struct inode *inode = d_inode(dentry); 962 struct shmem_inode_info *info = SHMEM_I(inode); 963 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 964 int error; 965 966 error = setattr_prepare(dentry, attr); 967 if (error) 968 return error; 969 970 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 971 loff_t oldsize = inode->i_size; 972 loff_t newsize = attr->ia_size; 973 974 /* protected by i_mutex */ 975 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 976 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 977 return -EPERM; 978 979 if (newsize != oldsize) { 980 error = shmem_reacct_size(SHMEM_I(inode)->flags, 981 oldsize, newsize); 982 if (error) 983 return error; 984 i_size_write(inode, newsize); 985 inode->i_ctime = inode->i_mtime = current_time(inode); 986 } 987 if (newsize <= oldsize) { 988 loff_t holebegin = round_up(newsize, PAGE_SIZE); 989 if (oldsize > holebegin) 990 unmap_mapping_range(inode->i_mapping, 991 holebegin, 0, 1); 992 if (info->alloced) 993 shmem_truncate_range(inode, 994 newsize, (loff_t)-1); 995 /* unmap again to remove racily COWed private pages */ 996 if (oldsize > holebegin) 997 unmap_mapping_range(inode->i_mapping, 998 holebegin, 0, 1); 999 1000 /* 1001 * Part of the huge page can be beyond i_size: subject 1002 * to shrink under memory pressure. 1003 */ 1004 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 1005 spin_lock(&sbinfo->shrinklist_lock); 1006 if (list_empty(&info->shrinklist)) { 1007 list_add_tail(&info->shrinklist, 1008 &sbinfo->shrinklist); 1009 sbinfo->shrinklist_len++; 1010 } 1011 spin_unlock(&sbinfo->shrinklist_lock); 1012 } 1013 } 1014 } 1015 1016 setattr_copy(inode, attr); 1017 if (attr->ia_valid & ATTR_MODE) 1018 error = posix_acl_chmod(inode, inode->i_mode); 1019 return error; 1020 } 1021 1022 static void shmem_evict_inode(struct inode *inode) 1023 { 1024 struct shmem_inode_info *info = SHMEM_I(inode); 1025 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1026 1027 if (inode->i_mapping->a_ops == &shmem_aops) { 1028 shmem_unacct_size(info->flags, inode->i_size); 1029 inode->i_size = 0; 1030 shmem_truncate_range(inode, 0, (loff_t)-1); 1031 if (!list_empty(&info->shrinklist)) { 1032 spin_lock(&sbinfo->shrinklist_lock); 1033 if (!list_empty(&info->shrinklist)) { 1034 list_del_init(&info->shrinklist); 1035 sbinfo->shrinklist_len--; 1036 } 1037 spin_unlock(&sbinfo->shrinklist_lock); 1038 } 1039 if (!list_empty(&info->swaplist)) { 1040 mutex_lock(&shmem_swaplist_mutex); 1041 list_del_init(&info->swaplist); 1042 mutex_unlock(&shmem_swaplist_mutex); 1043 } 1044 } 1045 1046 simple_xattrs_free(&info->xattrs); 1047 WARN_ON(inode->i_blocks); 1048 shmem_free_inode(inode->i_sb); 1049 clear_inode(inode); 1050 } 1051 1052 static unsigned long find_swap_entry(struct radix_tree_root *root, void *item) 1053 { 1054 struct radix_tree_iter iter; 1055 void **slot; 1056 unsigned long found = -1; 1057 unsigned int checked = 0; 1058 1059 rcu_read_lock(); 1060 radix_tree_for_each_slot(slot, root, &iter, 0) { 1061 if (*slot == item) { 1062 found = iter.index; 1063 break; 1064 } 1065 checked++; 1066 if ((checked % 4096) != 0) 1067 continue; 1068 slot = radix_tree_iter_resume(slot, &iter); 1069 cond_resched_rcu(); 1070 } 1071 1072 rcu_read_unlock(); 1073 return found; 1074 } 1075 1076 /* 1077 * If swap found in inode, free it and move page from swapcache to filecache. 1078 */ 1079 static int shmem_unuse_inode(struct shmem_inode_info *info, 1080 swp_entry_t swap, struct page **pagep) 1081 { 1082 struct address_space *mapping = info->vfs_inode.i_mapping; 1083 void *radswap; 1084 pgoff_t index; 1085 gfp_t gfp; 1086 int error = 0; 1087 1088 radswap = swp_to_radix_entry(swap); 1089 index = find_swap_entry(&mapping->page_tree, radswap); 1090 if (index == -1) 1091 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1092 1093 /* 1094 * Move _head_ to start search for next from here. 1095 * But be careful: shmem_evict_inode checks list_empty without taking 1096 * mutex, and there's an instant in list_move_tail when info->swaplist 1097 * would appear empty, if it were the only one on shmem_swaplist. 1098 */ 1099 if (shmem_swaplist.next != &info->swaplist) 1100 list_move_tail(&shmem_swaplist, &info->swaplist); 1101 1102 gfp = mapping_gfp_mask(mapping); 1103 if (shmem_should_replace_page(*pagep, gfp)) { 1104 mutex_unlock(&shmem_swaplist_mutex); 1105 error = shmem_replace_page(pagep, gfp, info, index); 1106 mutex_lock(&shmem_swaplist_mutex); 1107 /* 1108 * We needed to drop mutex to make that restrictive page 1109 * allocation, but the inode might have been freed while we 1110 * dropped it: although a racing shmem_evict_inode() cannot 1111 * complete without emptying the radix_tree, our page lock 1112 * on this swapcache page is not enough to prevent that - 1113 * free_swap_and_cache() of our swap entry will only 1114 * trylock_page(), removing swap from radix_tree whatever. 1115 * 1116 * We must not proceed to shmem_add_to_page_cache() if the 1117 * inode has been freed, but of course we cannot rely on 1118 * inode or mapping or info to check that. However, we can 1119 * safely check if our swap entry is still in use (and here 1120 * it can't have got reused for another page): if it's still 1121 * in use, then the inode cannot have been freed yet, and we 1122 * can safely proceed (if it's no longer in use, that tells 1123 * nothing about the inode, but we don't need to unuse swap). 1124 */ 1125 if (!page_swapcount(*pagep)) 1126 error = -ENOENT; 1127 } 1128 1129 /* 1130 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 1131 * but also to hold up shmem_evict_inode(): so inode cannot be freed 1132 * beneath us (pagelock doesn't help until the page is in pagecache). 1133 */ 1134 if (!error) 1135 error = shmem_add_to_page_cache(*pagep, mapping, index, 1136 radswap); 1137 if (error != -ENOMEM) { 1138 /* 1139 * Truncation and eviction use free_swap_and_cache(), which 1140 * only does trylock page: if we raced, best clean up here. 1141 */ 1142 delete_from_swap_cache(*pagep); 1143 set_page_dirty(*pagep); 1144 if (!error) { 1145 spin_lock_irq(&info->lock); 1146 info->swapped--; 1147 spin_unlock_irq(&info->lock); 1148 swap_free(swap); 1149 } 1150 } 1151 return error; 1152 } 1153 1154 /* 1155 * Search through swapped inodes to find and replace swap by page. 1156 */ 1157 int shmem_unuse(swp_entry_t swap, struct page *page) 1158 { 1159 struct list_head *this, *next; 1160 struct shmem_inode_info *info; 1161 struct mem_cgroup *memcg; 1162 int error = 0; 1163 1164 /* 1165 * There's a faint possibility that swap page was replaced before 1166 * caller locked it: caller will come back later with the right page. 1167 */ 1168 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) 1169 goto out; 1170 1171 /* 1172 * Charge page using GFP_KERNEL while we can wait, before taking 1173 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1174 * Charged back to the user (not to caller) when swap account is used. 1175 */ 1176 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg, 1177 false); 1178 if (error) 1179 goto out; 1180 /* No radix_tree_preload: swap entry keeps a place for page in tree */ 1181 error = -EAGAIN; 1182 1183 mutex_lock(&shmem_swaplist_mutex); 1184 list_for_each_safe(this, next, &shmem_swaplist) { 1185 info = list_entry(this, struct shmem_inode_info, swaplist); 1186 if (info->swapped) 1187 error = shmem_unuse_inode(info, swap, &page); 1188 else 1189 list_del_init(&info->swaplist); 1190 cond_resched(); 1191 if (error != -EAGAIN) 1192 break; 1193 /* found nothing in this: move on to search the next */ 1194 } 1195 mutex_unlock(&shmem_swaplist_mutex); 1196 1197 if (error) { 1198 if (error != -ENOMEM) 1199 error = 0; 1200 mem_cgroup_cancel_charge(page, memcg, false); 1201 } else 1202 mem_cgroup_commit_charge(page, memcg, true, false); 1203 out: 1204 unlock_page(page); 1205 put_page(page); 1206 return error; 1207 } 1208 1209 /* 1210 * Move the page from the page cache to the swap cache. 1211 */ 1212 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1213 { 1214 struct shmem_inode_info *info; 1215 struct address_space *mapping; 1216 struct inode *inode; 1217 swp_entry_t swap; 1218 pgoff_t index; 1219 1220 VM_BUG_ON_PAGE(PageCompound(page), page); 1221 BUG_ON(!PageLocked(page)); 1222 mapping = page->mapping; 1223 index = page->index; 1224 inode = mapping->host; 1225 info = SHMEM_I(inode); 1226 if (info->flags & VM_LOCKED) 1227 goto redirty; 1228 if (!total_swap_pages) 1229 goto redirty; 1230 1231 /* 1232 * Our capabilities prevent regular writeback or sync from ever calling 1233 * shmem_writepage; but a stacking filesystem might use ->writepage of 1234 * its underlying filesystem, in which case tmpfs should write out to 1235 * swap only in response to memory pressure, and not for the writeback 1236 * threads or sync. 1237 */ 1238 if (!wbc->for_reclaim) { 1239 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1240 goto redirty; 1241 } 1242 1243 /* 1244 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1245 * value into swapfile.c, the only way we can correctly account for a 1246 * fallocated page arriving here is now to initialize it and write it. 1247 * 1248 * That's okay for a page already fallocated earlier, but if we have 1249 * not yet completed the fallocation, then (a) we want to keep track 1250 * of this page in case we have to undo it, and (b) it may not be a 1251 * good idea to continue anyway, once we're pushing into swap. So 1252 * reactivate the page, and let shmem_fallocate() quit when too many. 1253 */ 1254 if (!PageUptodate(page)) { 1255 if (inode->i_private) { 1256 struct shmem_falloc *shmem_falloc; 1257 spin_lock(&inode->i_lock); 1258 shmem_falloc = inode->i_private; 1259 if (shmem_falloc && 1260 !shmem_falloc->waitq && 1261 index >= shmem_falloc->start && 1262 index < shmem_falloc->next) 1263 shmem_falloc->nr_unswapped++; 1264 else 1265 shmem_falloc = NULL; 1266 spin_unlock(&inode->i_lock); 1267 if (shmem_falloc) 1268 goto redirty; 1269 } 1270 clear_highpage(page); 1271 flush_dcache_page(page); 1272 SetPageUptodate(page); 1273 } 1274 1275 swap = get_swap_page(); 1276 if (!swap.val) 1277 goto redirty; 1278 1279 if (mem_cgroup_try_charge_swap(page, swap)) 1280 goto free_swap; 1281 1282 /* 1283 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1284 * if it's not already there. Do it now before the page is 1285 * moved to swap cache, when its pagelock no longer protects 1286 * the inode from eviction. But don't unlock the mutex until 1287 * we've incremented swapped, because shmem_unuse_inode() will 1288 * prune a !swapped inode from the swaplist under this mutex. 1289 */ 1290 mutex_lock(&shmem_swaplist_mutex); 1291 if (list_empty(&info->swaplist)) 1292 list_add_tail(&info->swaplist, &shmem_swaplist); 1293 1294 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1295 spin_lock_irq(&info->lock); 1296 shmem_recalc_inode(inode); 1297 info->swapped++; 1298 spin_unlock_irq(&info->lock); 1299 1300 swap_shmem_alloc(swap); 1301 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1302 1303 mutex_unlock(&shmem_swaplist_mutex); 1304 BUG_ON(page_mapped(page)); 1305 swap_writepage(page, wbc); 1306 return 0; 1307 } 1308 1309 mutex_unlock(&shmem_swaplist_mutex); 1310 free_swap: 1311 swapcache_free(swap); 1312 redirty: 1313 set_page_dirty(page); 1314 if (wbc->for_reclaim) 1315 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1316 unlock_page(page); 1317 return 0; 1318 } 1319 1320 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1321 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1322 { 1323 char buffer[64]; 1324 1325 if (!mpol || mpol->mode == MPOL_DEFAULT) 1326 return; /* show nothing */ 1327 1328 mpol_to_str(buffer, sizeof(buffer), mpol); 1329 1330 seq_printf(seq, ",mpol=%s", buffer); 1331 } 1332 1333 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1334 { 1335 struct mempolicy *mpol = NULL; 1336 if (sbinfo->mpol) { 1337 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1338 mpol = sbinfo->mpol; 1339 mpol_get(mpol); 1340 spin_unlock(&sbinfo->stat_lock); 1341 } 1342 return mpol; 1343 } 1344 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1345 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1346 { 1347 } 1348 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1349 { 1350 return NULL; 1351 } 1352 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1353 #ifndef CONFIG_NUMA 1354 #define vm_policy vm_private_data 1355 #endif 1356 1357 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1358 struct shmem_inode_info *info, pgoff_t index) 1359 { 1360 /* Create a pseudo vma that just contains the policy */ 1361 vma->vm_start = 0; 1362 /* Bias interleave by inode number to distribute better across nodes */ 1363 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1364 vma->vm_ops = NULL; 1365 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1366 } 1367 1368 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1369 { 1370 /* Drop reference taken by mpol_shared_policy_lookup() */ 1371 mpol_cond_put(vma->vm_policy); 1372 } 1373 1374 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1375 struct shmem_inode_info *info, pgoff_t index) 1376 { 1377 struct vm_area_struct pvma; 1378 struct page *page; 1379 1380 shmem_pseudo_vma_init(&pvma, info, index); 1381 page = swapin_readahead(swap, gfp, &pvma, 0); 1382 shmem_pseudo_vma_destroy(&pvma); 1383 1384 return page; 1385 } 1386 1387 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1388 struct shmem_inode_info *info, pgoff_t index) 1389 { 1390 struct vm_area_struct pvma; 1391 struct inode *inode = &info->vfs_inode; 1392 struct address_space *mapping = inode->i_mapping; 1393 pgoff_t idx, hindex; 1394 void __rcu **results; 1395 struct page *page; 1396 1397 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1398 return NULL; 1399 1400 hindex = round_down(index, HPAGE_PMD_NR); 1401 rcu_read_lock(); 1402 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1403 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1404 rcu_read_unlock(); 1405 return NULL; 1406 } 1407 rcu_read_unlock(); 1408 1409 shmem_pseudo_vma_init(&pvma, info, hindex); 1410 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, 1411 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); 1412 shmem_pseudo_vma_destroy(&pvma); 1413 if (page) 1414 prep_transhuge_page(page); 1415 return page; 1416 } 1417 1418 static struct page *shmem_alloc_page(gfp_t gfp, 1419 struct shmem_inode_info *info, pgoff_t index) 1420 { 1421 struct vm_area_struct pvma; 1422 struct page *page; 1423 1424 shmem_pseudo_vma_init(&pvma, info, index); 1425 page = alloc_page_vma(gfp, &pvma, 0); 1426 shmem_pseudo_vma_destroy(&pvma); 1427 1428 return page; 1429 } 1430 1431 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1432 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo, 1433 pgoff_t index, bool huge) 1434 { 1435 struct page *page; 1436 int nr; 1437 int err = -ENOSPC; 1438 1439 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1440 huge = false; 1441 nr = huge ? HPAGE_PMD_NR : 1; 1442 1443 if (shmem_acct_block(info->flags, nr)) 1444 goto failed; 1445 if (sbinfo->max_blocks) { 1446 if (percpu_counter_compare(&sbinfo->used_blocks, 1447 sbinfo->max_blocks - nr) > 0) 1448 goto unacct; 1449 percpu_counter_add(&sbinfo->used_blocks, nr); 1450 } 1451 1452 if (huge) 1453 page = shmem_alloc_hugepage(gfp, info, index); 1454 else 1455 page = shmem_alloc_page(gfp, info, index); 1456 if (page) { 1457 __SetPageLocked(page); 1458 __SetPageSwapBacked(page); 1459 return page; 1460 } 1461 1462 err = -ENOMEM; 1463 if (sbinfo->max_blocks) 1464 percpu_counter_add(&sbinfo->used_blocks, -nr); 1465 unacct: 1466 shmem_unacct_blocks(info->flags, nr); 1467 failed: 1468 return ERR_PTR(err); 1469 } 1470 1471 /* 1472 * When a page is moved from swapcache to shmem filecache (either by the 1473 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1474 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1475 * ignorance of the mapping it belongs to. If that mapping has special 1476 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1477 * we may need to copy to a suitable page before moving to filecache. 1478 * 1479 * In a future release, this may well be extended to respect cpuset and 1480 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1481 * but for now it is a simple matter of zone. 1482 */ 1483 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1484 { 1485 return page_zonenum(page) > gfp_zone(gfp); 1486 } 1487 1488 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1489 struct shmem_inode_info *info, pgoff_t index) 1490 { 1491 struct page *oldpage, *newpage; 1492 struct address_space *swap_mapping; 1493 pgoff_t swap_index; 1494 int error; 1495 1496 oldpage = *pagep; 1497 swap_index = page_private(oldpage); 1498 swap_mapping = page_mapping(oldpage); 1499 1500 /* 1501 * We have arrived here because our zones are constrained, so don't 1502 * limit chance of success by further cpuset and node constraints. 1503 */ 1504 gfp &= ~GFP_CONSTRAINT_MASK; 1505 newpage = shmem_alloc_page(gfp, info, index); 1506 if (!newpage) 1507 return -ENOMEM; 1508 1509 get_page(newpage); 1510 copy_highpage(newpage, oldpage); 1511 flush_dcache_page(newpage); 1512 1513 __SetPageLocked(newpage); 1514 __SetPageSwapBacked(newpage); 1515 SetPageUptodate(newpage); 1516 set_page_private(newpage, swap_index); 1517 SetPageSwapCache(newpage); 1518 1519 /* 1520 * Our caller will very soon move newpage out of swapcache, but it's 1521 * a nice clean interface for us to replace oldpage by newpage there. 1522 */ 1523 spin_lock_irq(&swap_mapping->tree_lock); 1524 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1525 newpage); 1526 if (!error) { 1527 __inc_node_page_state(newpage, NR_FILE_PAGES); 1528 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1529 } 1530 spin_unlock_irq(&swap_mapping->tree_lock); 1531 1532 if (unlikely(error)) { 1533 /* 1534 * Is this possible? I think not, now that our callers check 1535 * both PageSwapCache and page_private after getting page lock; 1536 * but be defensive. Reverse old to newpage for clear and free. 1537 */ 1538 oldpage = newpage; 1539 } else { 1540 mem_cgroup_migrate(oldpage, newpage); 1541 lru_cache_add_anon(newpage); 1542 *pagep = newpage; 1543 } 1544 1545 ClearPageSwapCache(oldpage); 1546 set_page_private(oldpage, 0); 1547 1548 unlock_page(oldpage); 1549 put_page(oldpage); 1550 put_page(oldpage); 1551 return error; 1552 } 1553 1554 /* 1555 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1556 * 1557 * If we allocate a new one we do not mark it dirty. That's up to the 1558 * vm. If we swap it in we mark it dirty since we also free the swap 1559 * entry since a page cannot live in both the swap and page cache. 1560 * 1561 * fault_mm and fault_type are only supplied by shmem_fault: 1562 * otherwise they are NULL. 1563 */ 1564 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1565 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1566 struct mm_struct *fault_mm, int *fault_type) 1567 { 1568 struct address_space *mapping = inode->i_mapping; 1569 struct shmem_inode_info *info = SHMEM_I(inode); 1570 struct shmem_sb_info *sbinfo; 1571 struct mm_struct *charge_mm; 1572 struct mem_cgroup *memcg; 1573 struct page *page; 1574 swp_entry_t swap; 1575 enum sgp_type sgp_huge = sgp; 1576 pgoff_t hindex = index; 1577 int error; 1578 int once = 0; 1579 int alloced = 0; 1580 1581 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1582 return -EFBIG; 1583 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) 1584 sgp = SGP_CACHE; 1585 repeat: 1586 swap.val = 0; 1587 page = find_lock_entry(mapping, index); 1588 if (radix_tree_exceptional_entry(page)) { 1589 swap = radix_to_swp_entry(page); 1590 page = NULL; 1591 } 1592 1593 if (sgp <= SGP_CACHE && 1594 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1595 error = -EINVAL; 1596 goto unlock; 1597 } 1598 1599 if (page && sgp == SGP_WRITE) 1600 mark_page_accessed(page); 1601 1602 /* fallocated page? */ 1603 if (page && !PageUptodate(page)) { 1604 if (sgp != SGP_READ) 1605 goto clear; 1606 unlock_page(page); 1607 put_page(page); 1608 page = NULL; 1609 } 1610 if (page || (sgp == SGP_READ && !swap.val)) { 1611 *pagep = page; 1612 return 0; 1613 } 1614 1615 /* 1616 * Fast cache lookup did not find it: 1617 * bring it back from swap or allocate. 1618 */ 1619 sbinfo = SHMEM_SB(inode->i_sb); 1620 charge_mm = fault_mm ? : current->mm; 1621 1622 if (swap.val) { 1623 /* Look it up and read it in.. */ 1624 page = lookup_swap_cache(swap); 1625 if (!page) { 1626 /* Or update major stats only when swapin succeeds?? */ 1627 if (fault_type) { 1628 *fault_type |= VM_FAULT_MAJOR; 1629 count_vm_event(PGMAJFAULT); 1630 mem_cgroup_count_vm_event(fault_mm, PGMAJFAULT); 1631 } 1632 /* Here we actually start the io */ 1633 page = shmem_swapin(swap, gfp, info, index); 1634 if (!page) { 1635 error = -ENOMEM; 1636 goto failed; 1637 } 1638 } 1639 1640 /* We have to do this with page locked to prevent races */ 1641 lock_page(page); 1642 if (!PageSwapCache(page) || page_private(page) != swap.val || 1643 !shmem_confirm_swap(mapping, index, swap)) { 1644 error = -EEXIST; /* try again */ 1645 goto unlock; 1646 } 1647 if (!PageUptodate(page)) { 1648 error = -EIO; 1649 goto failed; 1650 } 1651 wait_on_page_writeback(page); 1652 1653 if (shmem_should_replace_page(page, gfp)) { 1654 error = shmem_replace_page(&page, gfp, info, index); 1655 if (error) 1656 goto failed; 1657 } 1658 1659 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1660 false); 1661 if (!error) { 1662 error = shmem_add_to_page_cache(page, mapping, index, 1663 swp_to_radix_entry(swap)); 1664 /* 1665 * We already confirmed swap under page lock, and make 1666 * no memory allocation here, so usually no possibility 1667 * of error; but free_swap_and_cache() only trylocks a 1668 * page, so it is just possible that the entry has been 1669 * truncated or holepunched since swap was confirmed. 1670 * shmem_undo_range() will have done some of the 1671 * unaccounting, now delete_from_swap_cache() will do 1672 * the rest. 1673 * Reset swap.val? No, leave it so "failed" goes back to 1674 * "repeat": reading a hole and writing should succeed. 1675 */ 1676 if (error) { 1677 mem_cgroup_cancel_charge(page, memcg, false); 1678 delete_from_swap_cache(page); 1679 } 1680 } 1681 if (error) 1682 goto failed; 1683 1684 mem_cgroup_commit_charge(page, memcg, true, false); 1685 1686 spin_lock_irq(&info->lock); 1687 info->swapped--; 1688 shmem_recalc_inode(inode); 1689 spin_unlock_irq(&info->lock); 1690 1691 if (sgp == SGP_WRITE) 1692 mark_page_accessed(page); 1693 1694 delete_from_swap_cache(page); 1695 set_page_dirty(page); 1696 swap_free(swap); 1697 1698 } else { 1699 /* shmem_symlink() */ 1700 if (mapping->a_ops != &shmem_aops) 1701 goto alloc_nohuge; 1702 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) 1703 goto alloc_nohuge; 1704 if (shmem_huge == SHMEM_HUGE_FORCE) 1705 goto alloc_huge; 1706 switch (sbinfo->huge) { 1707 loff_t i_size; 1708 pgoff_t off; 1709 case SHMEM_HUGE_NEVER: 1710 goto alloc_nohuge; 1711 case SHMEM_HUGE_WITHIN_SIZE: 1712 off = round_up(index, HPAGE_PMD_NR); 1713 i_size = round_up(i_size_read(inode), PAGE_SIZE); 1714 if (i_size >= HPAGE_PMD_SIZE && 1715 i_size >> PAGE_SHIFT >= off) 1716 goto alloc_huge; 1717 /* fallthrough */ 1718 case SHMEM_HUGE_ADVISE: 1719 if (sgp_huge == SGP_HUGE) 1720 goto alloc_huge; 1721 /* TODO: implement fadvise() hints */ 1722 goto alloc_nohuge; 1723 } 1724 1725 alloc_huge: 1726 page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1727 index, true); 1728 if (IS_ERR(page)) { 1729 alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1730 index, false); 1731 } 1732 if (IS_ERR(page)) { 1733 int retry = 5; 1734 error = PTR_ERR(page); 1735 page = NULL; 1736 if (error != -ENOSPC) 1737 goto failed; 1738 /* 1739 * Try to reclaim some spece by splitting a huge page 1740 * beyond i_size on the filesystem. 1741 */ 1742 while (retry--) { 1743 int ret; 1744 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1745 if (ret == SHRINK_STOP) 1746 break; 1747 if (ret) 1748 goto alloc_nohuge; 1749 } 1750 goto failed; 1751 } 1752 1753 if (PageTransHuge(page)) 1754 hindex = round_down(index, HPAGE_PMD_NR); 1755 else 1756 hindex = index; 1757 1758 if (sgp == SGP_WRITE) 1759 __SetPageReferenced(page); 1760 1761 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1762 PageTransHuge(page)); 1763 if (error) 1764 goto unacct; 1765 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, 1766 compound_order(page)); 1767 if (!error) { 1768 error = shmem_add_to_page_cache(page, mapping, hindex, 1769 NULL); 1770 radix_tree_preload_end(); 1771 } 1772 if (error) { 1773 mem_cgroup_cancel_charge(page, memcg, 1774 PageTransHuge(page)); 1775 goto unacct; 1776 } 1777 mem_cgroup_commit_charge(page, memcg, false, 1778 PageTransHuge(page)); 1779 lru_cache_add_anon(page); 1780 1781 spin_lock_irq(&info->lock); 1782 info->alloced += 1 << compound_order(page); 1783 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1784 shmem_recalc_inode(inode); 1785 spin_unlock_irq(&info->lock); 1786 alloced = true; 1787 1788 if (PageTransHuge(page) && 1789 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1790 hindex + HPAGE_PMD_NR - 1) { 1791 /* 1792 * Part of the huge page is beyond i_size: subject 1793 * to shrink under memory pressure. 1794 */ 1795 spin_lock(&sbinfo->shrinklist_lock); 1796 if (list_empty(&info->shrinklist)) { 1797 list_add_tail(&info->shrinklist, 1798 &sbinfo->shrinklist); 1799 sbinfo->shrinklist_len++; 1800 } 1801 spin_unlock(&sbinfo->shrinklist_lock); 1802 } 1803 1804 /* 1805 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1806 */ 1807 if (sgp == SGP_FALLOC) 1808 sgp = SGP_WRITE; 1809 clear: 1810 /* 1811 * Let SGP_WRITE caller clear ends if write does not fill page; 1812 * but SGP_FALLOC on a page fallocated earlier must initialize 1813 * it now, lest undo on failure cancel our earlier guarantee. 1814 */ 1815 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1816 struct page *head = compound_head(page); 1817 int i; 1818 1819 for (i = 0; i < (1 << compound_order(head)); i++) { 1820 clear_highpage(head + i); 1821 flush_dcache_page(head + i); 1822 } 1823 SetPageUptodate(head); 1824 } 1825 } 1826 1827 /* Perhaps the file has been truncated since we checked */ 1828 if (sgp <= SGP_CACHE && 1829 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1830 if (alloced) { 1831 ClearPageDirty(page); 1832 delete_from_page_cache(page); 1833 spin_lock_irq(&info->lock); 1834 shmem_recalc_inode(inode); 1835 spin_unlock_irq(&info->lock); 1836 } 1837 error = -EINVAL; 1838 goto unlock; 1839 } 1840 *pagep = page + index - hindex; 1841 return 0; 1842 1843 /* 1844 * Error recovery. 1845 */ 1846 unacct: 1847 if (sbinfo->max_blocks) 1848 percpu_counter_sub(&sbinfo->used_blocks, 1849 1 << compound_order(page)); 1850 shmem_unacct_blocks(info->flags, 1 << compound_order(page)); 1851 1852 if (PageTransHuge(page)) { 1853 unlock_page(page); 1854 put_page(page); 1855 goto alloc_nohuge; 1856 } 1857 failed: 1858 if (swap.val && !shmem_confirm_swap(mapping, index, swap)) 1859 error = -EEXIST; 1860 unlock: 1861 if (page) { 1862 unlock_page(page); 1863 put_page(page); 1864 } 1865 if (error == -ENOSPC && !once++) { 1866 spin_lock_irq(&info->lock); 1867 shmem_recalc_inode(inode); 1868 spin_unlock_irq(&info->lock); 1869 goto repeat; 1870 } 1871 if (error == -EEXIST) /* from above or from radix_tree_insert */ 1872 goto repeat; 1873 return error; 1874 } 1875 1876 /* 1877 * This is like autoremove_wake_function, but it removes the wait queue 1878 * entry unconditionally - even if something else had already woken the 1879 * target. 1880 */ 1881 static int synchronous_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) 1882 { 1883 int ret = default_wake_function(wait, mode, sync, key); 1884 list_del_init(&wait->task_list); 1885 return ret; 1886 } 1887 1888 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1889 { 1890 struct inode *inode = file_inode(vma->vm_file); 1891 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 1892 enum sgp_type sgp; 1893 int error; 1894 int ret = VM_FAULT_LOCKED; 1895 1896 /* 1897 * Trinity finds that probing a hole which tmpfs is punching can 1898 * prevent the hole-punch from ever completing: which in turn 1899 * locks writers out with its hold on i_mutex. So refrain from 1900 * faulting pages into the hole while it's being punched. Although 1901 * shmem_undo_range() does remove the additions, it may be unable to 1902 * keep up, as each new page needs its own unmap_mapping_range() call, 1903 * and the i_mmap tree grows ever slower to scan if new vmas are added. 1904 * 1905 * It does not matter if we sometimes reach this check just before the 1906 * hole-punch begins, so that one fault then races with the punch: 1907 * we just need to make racing faults a rare case. 1908 * 1909 * The implementation below would be much simpler if we just used a 1910 * standard mutex or completion: but we cannot take i_mutex in fault, 1911 * and bloating every shmem inode for this unlikely case would be sad. 1912 */ 1913 if (unlikely(inode->i_private)) { 1914 struct shmem_falloc *shmem_falloc; 1915 1916 spin_lock(&inode->i_lock); 1917 shmem_falloc = inode->i_private; 1918 if (shmem_falloc && 1919 shmem_falloc->waitq && 1920 vmf->pgoff >= shmem_falloc->start && 1921 vmf->pgoff < shmem_falloc->next) { 1922 wait_queue_head_t *shmem_falloc_waitq; 1923 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 1924 1925 ret = VM_FAULT_NOPAGE; 1926 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && 1927 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 1928 /* It's polite to up mmap_sem if we can */ 1929 up_read(&vma->vm_mm->mmap_sem); 1930 ret = VM_FAULT_RETRY; 1931 } 1932 1933 shmem_falloc_waitq = shmem_falloc->waitq; 1934 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 1935 TASK_UNINTERRUPTIBLE); 1936 spin_unlock(&inode->i_lock); 1937 schedule(); 1938 1939 /* 1940 * shmem_falloc_waitq points into the shmem_fallocate() 1941 * stack of the hole-punching task: shmem_falloc_waitq 1942 * is usually invalid by the time we reach here, but 1943 * finish_wait() does not dereference it in that case; 1944 * though i_lock needed lest racing with wake_up_all(). 1945 */ 1946 spin_lock(&inode->i_lock); 1947 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 1948 spin_unlock(&inode->i_lock); 1949 return ret; 1950 } 1951 spin_unlock(&inode->i_lock); 1952 } 1953 1954 sgp = SGP_CACHE; 1955 if (vma->vm_flags & VM_HUGEPAGE) 1956 sgp = SGP_HUGE; 1957 else if (vma->vm_flags & VM_NOHUGEPAGE) 1958 sgp = SGP_NOHUGE; 1959 1960 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, 1961 gfp, vma->vm_mm, &ret); 1962 if (error) 1963 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1964 return ret; 1965 } 1966 1967 unsigned long shmem_get_unmapped_area(struct file *file, 1968 unsigned long uaddr, unsigned long len, 1969 unsigned long pgoff, unsigned long flags) 1970 { 1971 unsigned long (*get_area)(struct file *, 1972 unsigned long, unsigned long, unsigned long, unsigned long); 1973 unsigned long addr; 1974 unsigned long offset; 1975 unsigned long inflated_len; 1976 unsigned long inflated_addr; 1977 unsigned long inflated_offset; 1978 1979 if (len > TASK_SIZE) 1980 return -ENOMEM; 1981 1982 get_area = current->mm->get_unmapped_area; 1983 addr = get_area(file, uaddr, len, pgoff, flags); 1984 1985 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1986 return addr; 1987 if (IS_ERR_VALUE(addr)) 1988 return addr; 1989 if (addr & ~PAGE_MASK) 1990 return addr; 1991 if (addr > TASK_SIZE - len) 1992 return addr; 1993 1994 if (shmem_huge == SHMEM_HUGE_DENY) 1995 return addr; 1996 if (len < HPAGE_PMD_SIZE) 1997 return addr; 1998 if (flags & MAP_FIXED) 1999 return addr; 2000 /* 2001 * Our priority is to support MAP_SHARED mapped hugely; 2002 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2003 * But if caller specified an address hint, respect that as before. 2004 */ 2005 if (uaddr) 2006 return addr; 2007 2008 if (shmem_huge != SHMEM_HUGE_FORCE) { 2009 struct super_block *sb; 2010 2011 if (file) { 2012 VM_BUG_ON(file->f_op != &shmem_file_operations); 2013 sb = file_inode(file)->i_sb; 2014 } else { 2015 /* 2016 * Called directly from mm/mmap.c, or drivers/char/mem.c 2017 * for "/dev/zero", to create a shared anonymous object. 2018 */ 2019 if (IS_ERR(shm_mnt)) 2020 return addr; 2021 sb = shm_mnt->mnt_sb; 2022 } 2023 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2024 return addr; 2025 } 2026 2027 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2028 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2029 return addr; 2030 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2031 return addr; 2032 2033 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2034 if (inflated_len > TASK_SIZE) 2035 return addr; 2036 if (inflated_len < len) 2037 return addr; 2038 2039 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); 2040 if (IS_ERR_VALUE(inflated_addr)) 2041 return addr; 2042 if (inflated_addr & ~PAGE_MASK) 2043 return addr; 2044 2045 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2046 inflated_addr += offset - inflated_offset; 2047 if (inflated_offset > offset) 2048 inflated_addr += HPAGE_PMD_SIZE; 2049 2050 if (inflated_addr > TASK_SIZE - len) 2051 return addr; 2052 return inflated_addr; 2053 } 2054 2055 #ifdef CONFIG_NUMA 2056 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2057 { 2058 struct inode *inode = file_inode(vma->vm_file); 2059 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2060 } 2061 2062 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2063 unsigned long addr) 2064 { 2065 struct inode *inode = file_inode(vma->vm_file); 2066 pgoff_t index; 2067 2068 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2069 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2070 } 2071 #endif 2072 2073 int shmem_lock(struct file *file, int lock, struct user_struct *user) 2074 { 2075 struct inode *inode = file_inode(file); 2076 struct shmem_inode_info *info = SHMEM_I(inode); 2077 int retval = -ENOMEM; 2078 2079 spin_lock_irq(&info->lock); 2080 if (lock && !(info->flags & VM_LOCKED)) { 2081 if (!user_shm_lock(inode->i_size, user)) 2082 goto out_nomem; 2083 info->flags |= VM_LOCKED; 2084 mapping_set_unevictable(file->f_mapping); 2085 } 2086 if (!lock && (info->flags & VM_LOCKED) && user) { 2087 user_shm_unlock(inode->i_size, user); 2088 info->flags &= ~VM_LOCKED; 2089 mapping_clear_unevictable(file->f_mapping); 2090 } 2091 retval = 0; 2092 2093 out_nomem: 2094 spin_unlock_irq(&info->lock); 2095 return retval; 2096 } 2097 2098 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2099 { 2100 file_accessed(file); 2101 vma->vm_ops = &shmem_vm_ops; 2102 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 2103 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2104 (vma->vm_end & HPAGE_PMD_MASK)) { 2105 khugepaged_enter(vma, vma->vm_flags); 2106 } 2107 return 0; 2108 } 2109 2110 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2111 umode_t mode, dev_t dev, unsigned long flags) 2112 { 2113 struct inode *inode; 2114 struct shmem_inode_info *info; 2115 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2116 2117 if (shmem_reserve_inode(sb)) 2118 return NULL; 2119 2120 inode = new_inode(sb); 2121 if (inode) { 2122 inode->i_ino = get_next_ino(); 2123 inode_init_owner(inode, dir, mode); 2124 inode->i_blocks = 0; 2125 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2126 inode->i_generation = get_seconds(); 2127 info = SHMEM_I(inode); 2128 memset(info, 0, (char *)inode - (char *)info); 2129 spin_lock_init(&info->lock); 2130 info->seals = F_SEAL_SEAL; 2131 info->flags = flags & VM_NORESERVE; 2132 INIT_LIST_HEAD(&info->shrinklist); 2133 INIT_LIST_HEAD(&info->swaplist); 2134 simple_xattrs_init(&info->xattrs); 2135 cache_no_acl(inode); 2136 2137 switch (mode & S_IFMT) { 2138 default: 2139 inode->i_op = &shmem_special_inode_operations; 2140 init_special_inode(inode, mode, dev); 2141 break; 2142 case S_IFREG: 2143 inode->i_mapping->a_ops = &shmem_aops; 2144 inode->i_op = &shmem_inode_operations; 2145 inode->i_fop = &shmem_file_operations; 2146 mpol_shared_policy_init(&info->policy, 2147 shmem_get_sbmpol(sbinfo)); 2148 break; 2149 case S_IFDIR: 2150 inc_nlink(inode); 2151 /* Some things misbehave if size == 0 on a directory */ 2152 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2153 inode->i_op = &shmem_dir_inode_operations; 2154 inode->i_fop = &simple_dir_operations; 2155 break; 2156 case S_IFLNK: 2157 /* 2158 * Must not load anything in the rbtree, 2159 * mpol_free_shared_policy will not be called. 2160 */ 2161 mpol_shared_policy_init(&info->policy, NULL); 2162 break; 2163 } 2164 } else 2165 shmem_free_inode(sb); 2166 return inode; 2167 } 2168 2169 bool shmem_mapping(struct address_space *mapping) 2170 { 2171 if (!mapping->host) 2172 return false; 2173 2174 return mapping->host->i_sb->s_op == &shmem_ops; 2175 } 2176 2177 #ifdef CONFIG_TMPFS 2178 static const struct inode_operations shmem_symlink_inode_operations; 2179 static const struct inode_operations shmem_short_symlink_operations; 2180 2181 #ifdef CONFIG_TMPFS_XATTR 2182 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2183 #else 2184 #define shmem_initxattrs NULL 2185 #endif 2186 2187 static int 2188 shmem_write_begin(struct file *file, struct address_space *mapping, 2189 loff_t pos, unsigned len, unsigned flags, 2190 struct page **pagep, void **fsdata) 2191 { 2192 struct inode *inode = mapping->host; 2193 struct shmem_inode_info *info = SHMEM_I(inode); 2194 pgoff_t index = pos >> PAGE_SHIFT; 2195 2196 /* i_mutex is held by caller */ 2197 if (unlikely(info->seals)) { 2198 if (info->seals & F_SEAL_WRITE) 2199 return -EPERM; 2200 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2201 return -EPERM; 2202 } 2203 2204 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2205 } 2206 2207 static int 2208 shmem_write_end(struct file *file, struct address_space *mapping, 2209 loff_t pos, unsigned len, unsigned copied, 2210 struct page *page, void *fsdata) 2211 { 2212 struct inode *inode = mapping->host; 2213 2214 if (pos + copied > inode->i_size) 2215 i_size_write(inode, pos + copied); 2216 2217 if (!PageUptodate(page)) { 2218 struct page *head = compound_head(page); 2219 if (PageTransCompound(page)) { 2220 int i; 2221 2222 for (i = 0; i < HPAGE_PMD_NR; i++) { 2223 if (head + i == page) 2224 continue; 2225 clear_highpage(head + i); 2226 flush_dcache_page(head + i); 2227 } 2228 } 2229 if (copied < PAGE_SIZE) { 2230 unsigned from = pos & (PAGE_SIZE - 1); 2231 zero_user_segments(page, 0, from, 2232 from + copied, PAGE_SIZE); 2233 } 2234 SetPageUptodate(head); 2235 } 2236 set_page_dirty(page); 2237 unlock_page(page); 2238 put_page(page); 2239 2240 return copied; 2241 } 2242 2243 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2244 { 2245 struct file *file = iocb->ki_filp; 2246 struct inode *inode = file_inode(file); 2247 struct address_space *mapping = inode->i_mapping; 2248 pgoff_t index; 2249 unsigned long offset; 2250 enum sgp_type sgp = SGP_READ; 2251 int error = 0; 2252 ssize_t retval = 0; 2253 loff_t *ppos = &iocb->ki_pos; 2254 2255 /* 2256 * Might this read be for a stacking filesystem? Then when reading 2257 * holes of a sparse file, we actually need to allocate those pages, 2258 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2259 */ 2260 if (!iter_is_iovec(to)) 2261 sgp = SGP_CACHE; 2262 2263 index = *ppos >> PAGE_SHIFT; 2264 offset = *ppos & ~PAGE_MASK; 2265 2266 for (;;) { 2267 struct page *page = NULL; 2268 pgoff_t end_index; 2269 unsigned long nr, ret; 2270 loff_t i_size = i_size_read(inode); 2271 2272 end_index = i_size >> PAGE_SHIFT; 2273 if (index > end_index) 2274 break; 2275 if (index == end_index) { 2276 nr = i_size & ~PAGE_MASK; 2277 if (nr <= offset) 2278 break; 2279 } 2280 2281 error = shmem_getpage(inode, index, &page, sgp); 2282 if (error) { 2283 if (error == -EINVAL) 2284 error = 0; 2285 break; 2286 } 2287 if (page) { 2288 if (sgp == SGP_CACHE) 2289 set_page_dirty(page); 2290 unlock_page(page); 2291 } 2292 2293 /* 2294 * We must evaluate after, since reads (unlike writes) 2295 * are called without i_mutex protection against truncate 2296 */ 2297 nr = PAGE_SIZE; 2298 i_size = i_size_read(inode); 2299 end_index = i_size >> PAGE_SHIFT; 2300 if (index == end_index) { 2301 nr = i_size & ~PAGE_MASK; 2302 if (nr <= offset) { 2303 if (page) 2304 put_page(page); 2305 break; 2306 } 2307 } 2308 nr -= offset; 2309 2310 if (page) { 2311 /* 2312 * If users can be writing to this page using arbitrary 2313 * virtual addresses, take care about potential aliasing 2314 * before reading the page on the kernel side. 2315 */ 2316 if (mapping_writably_mapped(mapping)) 2317 flush_dcache_page(page); 2318 /* 2319 * Mark the page accessed if we read the beginning. 2320 */ 2321 if (!offset) 2322 mark_page_accessed(page); 2323 } else { 2324 page = ZERO_PAGE(0); 2325 get_page(page); 2326 } 2327 2328 /* 2329 * Ok, we have the page, and it's up-to-date, so 2330 * now we can copy it to user space... 2331 */ 2332 ret = copy_page_to_iter(page, offset, nr, to); 2333 retval += ret; 2334 offset += ret; 2335 index += offset >> PAGE_SHIFT; 2336 offset &= ~PAGE_MASK; 2337 2338 put_page(page); 2339 if (!iov_iter_count(to)) 2340 break; 2341 if (ret < nr) { 2342 error = -EFAULT; 2343 break; 2344 } 2345 cond_resched(); 2346 } 2347 2348 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2349 file_accessed(file); 2350 return retval ? retval : error; 2351 } 2352 2353 /* 2354 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree. 2355 */ 2356 static pgoff_t shmem_seek_hole_data(struct address_space *mapping, 2357 pgoff_t index, pgoff_t end, int whence) 2358 { 2359 struct page *page; 2360 struct pagevec pvec; 2361 pgoff_t indices[PAGEVEC_SIZE]; 2362 bool done = false; 2363 int i; 2364 2365 pagevec_init(&pvec, 0); 2366 pvec.nr = 1; /* start small: we may be there already */ 2367 while (!done) { 2368 pvec.nr = find_get_entries(mapping, index, 2369 pvec.nr, pvec.pages, indices); 2370 if (!pvec.nr) { 2371 if (whence == SEEK_DATA) 2372 index = end; 2373 break; 2374 } 2375 for (i = 0; i < pvec.nr; i++, index++) { 2376 if (index < indices[i]) { 2377 if (whence == SEEK_HOLE) { 2378 done = true; 2379 break; 2380 } 2381 index = indices[i]; 2382 } 2383 page = pvec.pages[i]; 2384 if (page && !radix_tree_exceptional_entry(page)) { 2385 if (!PageUptodate(page)) 2386 page = NULL; 2387 } 2388 if (index >= end || 2389 (page && whence == SEEK_DATA) || 2390 (!page && whence == SEEK_HOLE)) { 2391 done = true; 2392 break; 2393 } 2394 } 2395 pagevec_remove_exceptionals(&pvec); 2396 pagevec_release(&pvec); 2397 pvec.nr = PAGEVEC_SIZE; 2398 cond_resched(); 2399 } 2400 return index; 2401 } 2402 2403 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2404 { 2405 struct address_space *mapping = file->f_mapping; 2406 struct inode *inode = mapping->host; 2407 pgoff_t start, end; 2408 loff_t new_offset; 2409 2410 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2411 return generic_file_llseek_size(file, offset, whence, 2412 MAX_LFS_FILESIZE, i_size_read(inode)); 2413 inode_lock(inode); 2414 /* We're holding i_mutex so we can access i_size directly */ 2415 2416 if (offset < 0) 2417 offset = -EINVAL; 2418 else if (offset >= inode->i_size) 2419 offset = -ENXIO; 2420 else { 2421 start = offset >> PAGE_SHIFT; 2422 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2423 new_offset = shmem_seek_hole_data(mapping, start, end, whence); 2424 new_offset <<= PAGE_SHIFT; 2425 if (new_offset > offset) { 2426 if (new_offset < inode->i_size) 2427 offset = new_offset; 2428 else if (whence == SEEK_DATA) 2429 offset = -ENXIO; 2430 else 2431 offset = inode->i_size; 2432 } 2433 } 2434 2435 if (offset >= 0) 2436 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2437 inode_unlock(inode); 2438 return offset; 2439 } 2440 2441 /* 2442 * We need a tag: a new tag would expand every radix_tree_node by 8 bytes, 2443 * so reuse a tag which we firmly believe is never set or cleared on shmem. 2444 */ 2445 #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE 2446 #define LAST_SCAN 4 /* about 150ms max */ 2447 2448 static void shmem_tag_pins(struct address_space *mapping) 2449 { 2450 struct radix_tree_iter iter; 2451 void **slot; 2452 pgoff_t start; 2453 struct page *page; 2454 2455 lru_add_drain(); 2456 start = 0; 2457 rcu_read_lock(); 2458 2459 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2460 page = radix_tree_deref_slot(slot); 2461 if (!page || radix_tree_exception(page)) { 2462 if (radix_tree_deref_retry(page)) { 2463 slot = radix_tree_iter_retry(&iter); 2464 continue; 2465 } 2466 } else if (page_count(page) - page_mapcount(page) > 1) { 2467 spin_lock_irq(&mapping->tree_lock); 2468 radix_tree_tag_set(&mapping->page_tree, iter.index, 2469 SHMEM_TAG_PINNED); 2470 spin_unlock_irq(&mapping->tree_lock); 2471 } 2472 2473 if (need_resched()) { 2474 slot = radix_tree_iter_resume(slot, &iter); 2475 cond_resched_rcu(); 2476 } 2477 } 2478 rcu_read_unlock(); 2479 } 2480 2481 /* 2482 * Setting SEAL_WRITE requires us to verify there's no pending writer. However, 2483 * via get_user_pages(), drivers might have some pending I/O without any active 2484 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages 2485 * and see whether it has an elevated ref-count. If so, we tag them and wait for 2486 * them to be dropped. 2487 * The caller must guarantee that no new user will acquire writable references 2488 * to those pages to avoid races. 2489 */ 2490 static int shmem_wait_for_pins(struct address_space *mapping) 2491 { 2492 struct radix_tree_iter iter; 2493 void **slot; 2494 pgoff_t start; 2495 struct page *page; 2496 int error, scan; 2497 2498 shmem_tag_pins(mapping); 2499 2500 error = 0; 2501 for (scan = 0; scan <= LAST_SCAN; scan++) { 2502 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2503 break; 2504 2505 if (!scan) 2506 lru_add_drain_all(); 2507 else if (schedule_timeout_killable((HZ << scan) / 200)) 2508 scan = LAST_SCAN; 2509 2510 start = 0; 2511 rcu_read_lock(); 2512 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2513 start, SHMEM_TAG_PINNED) { 2514 2515 page = radix_tree_deref_slot(slot); 2516 if (radix_tree_exception(page)) { 2517 if (radix_tree_deref_retry(page)) { 2518 slot = radix_tree_iter_retry(&iter); 2519 continue; 2520 } 2521 2522 page = NULL; 2523 } 2524 2525 if (page && 2526 page_count(page) - page_mapcount(page) != 1) { 2527 if (scan < LAST_SCAN) 2528 goto continue_resched; 2529 2530 /* 2531 * On the last scan, we clean up all those tags 2532 * we inserted; but make a note that we still 2533 * found pages pinned. 2534 */ 2535 error = -EBUSY; 2536 } 2537 2538 spin_lock_irq(&mapping->tree_lock); 2539 radix_tree_tag_clear(&mapping->page_tree, 2540 iter.index, SHMEM_TAG_PINNED); 2541 spin_unlock_irq(&mapping->tree_lock); 2542 continue_resched: 2543 if (need_resched()) { 2544 slot = radix_tree_iter_resume(slot, &iter); 2545 cond_resched_rcu(); 2546 } 2547 } 2548 rcu_read_unlock(); 2549 } 2550 2551 return error; 2552 } 2553 2554 #define F_ALL_SEALS (F_SEAL_SEAL | \ 2555 F_SEAL_SHRINK | \ 2556 F_SEAL_GROW | \ 2557 F_SEAL_WRITE) 2558 2559 int shmem_add_seals(struct file *file, unsigned int seals) 2560 { 2561 struct inode *inode = file_inode(file); 2562 struct shmem_inode_info *info = SHMEM_I(inode); 2563 int error; 2564 2565 /* 2566 * SEALING 2567 * Sealing allows multiple parties to share a shmem-file but restrict 2568 * access to a specific subset of file operations. Seals can only be 2569 * added, but never removed. This way, mutually untrusted parties can 2570 * share common memory regions with a well-defined policy. A malicious 2571 * peer can thus never perform unwanted operations on a shared object. 2572 * 2573 * Seals are only supported on special shmem-files and always affect 2574 * the whole underlying inode. Once a seal is set, it may prevent some 2575 * kinds of access to the file. Currently, the following seals are 2576 * defined: 2577 * SEAL_SEAL: Prevent further seals from being set on this file 2578 * SEAL_SHRINK: Prevent the file from shrinking 2579 * SEAL_GROW: Prevent the file from growing 2580 * SEAL_WRITE: Prevent write access to the file 2581 * 2582 * As we don't require any trust relationship between two parties, we 2583 * must prevent seals from being removed. Therefore, sealing a file 2584 * only adds a given set of seals to the file, it never touches 2585 * existing seals. Furthermore, the "setting seals"-operation can be 2586 * sealed itself, which basically prevents any further seal from being 2587 * added. 2588 * 2589 * Semantics of sealing are only defined on volatile files. Only 2590 * anonymous shmem files support sealing. More importantly, seals are 2591 * never written to disk. Therefore, there's no plan to support it on 2592 * other file types. 2593 */ 2594 2595 if (file->f_op != &shmem_file_operations) 2596 return -EINVAL; 2597 if (!(file->f_mode & FMODE_WRITE)) 2598 return -EPERM; 2599 if (seals & ~(unsigned int)F_ALL_SEALS) 2600 return -EINVAL; 2601 2602 inode_lock(inode); 2603 2604 if (info->seals & F_SEAL_SEAL) { 2605 error = -EPERM; 2606 goto unlock; 2607 } 2608 2609 if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) { 2610 error = mapping_deny_writable(file->f_mapping); 2611 if (error) 2612 goto unlock; 2613 2614 error = shmem_wait_for_pins(file->f_mapping); 2615 if (error) { 2616 mapping_allow_writable(file->f_mapping); 2617 goto unlock; 2618 } 2619 } 2620 2621 info->seals |= seals; 2622 error = 0; 2623 2624 unlock: 2625 inode_unlock(inode); 2626 return error; 2627 } 2628 EXPORT_SYMBOL_GPL(shmem_add_seals); 2629 2630 int shmem_get_seals(struct file *file) 2631 { 2632 if (file->f_op != &shmem_file_operations) 2633 return -EINVAL; 2634 2635 return SHMEM_I(file_inode(file))->seals; 2636 } 2637 EXPORT_SYMBOL_GPL(shmem_get_seals); 2638 2639 long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 2640 { 2641 long error; 2642 2643 switch (cmd) { 2644 case F_ADD_SEALS: 2645 /* disallow upper 32bit */ 2646 if (arg > UINT_MAX) 2647 return -EINVAL; 2648 2649 error = shmem_add_seals(file, arg); 2650 break; 2651 case F_GET_SEALS: 2652 error = shmem_get_seals(file); 2653 break; 2654 default: 2655 error = -EINVAL; 2656 break; 2657 } 2658 2659 return error; 2660 } 2661 2662 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2663 loff_t len) 2664 { 2665 struct inode *inode = file_inode(file); 2666 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2667 struct shmem_inode_info *info = SHMEM_I(inode); 2668 struct shmem_falloc shmem_falloc; 2669 pgoff_t start, index, end; 2670 int error; 2671 2672 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2673 return -EOPNOTSUPP; 2674 2675 inode_lock(inode); 2676 2677 if (mode & FALLOC_FL_PUNCH_HOLE) { 2678 struct address_space *mapping = file->f_mapping; 2679 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2680 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2681 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2682 2683 /* protected by i_mutex */ 2684 if (info->seals & F_SEAL_WRITE) { 2685 error = -EPERM; 2686 goto out; 2687 } 2688 2689 shmem_falloc.waitq = &shmem_falloc_waitq; 2690 shmem_falloc.start = unmap_start >> PAGE_SHIFT; 2691 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2692 spin_lock(&inode->i_lock); 2693 inode->i_private = &shmem_falloc; 2694 spin_unlock(&inode->i_lock); 2695 2696 if ((u64)unmap_end > (u64)unmap_start) 2697 unmap_mapping_range(mapping, unmap_start, 2698 1 + unmap_end - unmap_start, 0); 2699 shmem_truncate_range(inode, offset, offset + len - 1); 2700 /* No need to unmap again: hole-punching leaves COWed pages */ 2701 2702 spin_lock(&inode->i_lock); 2703 inode->i_private = NULL; 2704 wake_up_all(&shmem_falloc_waitq); 2705 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.task_list)); 2706 spin_unlock(&inode->i_lock); 2707 error = 0; 2708 goto out; 2709 } 2710 2711 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2712 error = inode_newsize_ok(inode, offset + len); 2713 if (error) 2714 goto out; 2715 2716 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2717 error = -EPERM; 2718 goto out; 2719 } 2720 2721 start = offset >> PAGE_SHIFT; 2722 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2723 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2724 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2725 error = -ENOSPC; 2726 goto out; 2727 } 2728 2729 shmem_falloc.waitq = NULL; 2730 shmem_falloc.start = start; 2731 shmem_falloc.next = start; 2732 shmem_falloc.nr_falloced = 0; 2733 shmem_falloc.nr_unswapped = 0; 2734 spin_lock(&inode->i_lock); 2735 inode->i_private = &shmem_falloc; 2736 spin_unlock(&inode->i_lock); 2737 2738 for (index = start; index < end; index++) { 2739 struct page *page; 2740 2741 /* 2742 * Good, the fallocate(2) manpage permits EINTR: we may have 2743 * been interrupted because we are using up too much memory. 2744 */ 2745 if (signal_pending(current)) 2746 error = -EINTR; 2747 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2748 error = -ENOMEM; 2749 else 2750 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2751 if (error) { 2752 /* Remove the !PageUptodate pages we added */ 2753 if (index > start) { 2754 shmem_undo_range(inode, 2755 (loff_t)start << PAGE_SHIFT, 2756 ((loff_t)index << PAGE_SHIFT) - 1, true); 2757 } 2758 goto undone; 2759 } 2760 2761 /* 2762 * Inform shmem_writepage() how far we have reached. 2763 * No need for lock or barrier: we have the page lock. 2764 */ 2765 shmem_falloc.next++; 2766 if (!PageUptodate(page)) 2767 shmem_falloc.nr_falloced++; 2768 2769 /* 2770 * If !PageUptodate, leave it that way so that freeable pages 2771 * can be recognized if we need to rollback on error later. 2772 * But set_page_dirty so that memory pressure will swap rather 2773 * than free the pages we are allocating (and SGP_CACHE pages 2774 * might still be clean: we now need to mark those dirty too). 2775 */ 2776 set_page_dirty(page); 2777 unlock_page(page); 2778 put_page(page); 2779 cond_resched(); 2780 } 2781 2782 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2783 i_size_write(inode, offset + len); 2784 inode->i_ctime = current_time(inode); 2785 undone: 2786 spin_lock(&inode->i_lock); 2787 inode->i_private = NULL; 2788 spin_unlock(&inode->i_lock); 2789 out: 2790 inode_unlock(inode); 2791 return error; 2792 } 2793 2794 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2795 { 2796 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2797 2798 buf->f_type = TMPFS_MAGIC; 2799 buf->f_bsize = PAGE_SIZE; 2800 buf->f_namelen = NAME_MAX; 2801 if (sbinfo->max_blocks) { 2802 buf->f_blocks = sbinfo->max_blocks; 2803 buf->f_bavail = 2804 buf->f_bfree = sbinfo->max_blocks - 2805 percpu_counter_sum(&sbinfo->used_blocks); 2806 } 2807 if (sbinfo->max_inodes) { 2808 buf->f_files = sbinfo->max_inodes; 2809 buf->f_ffree = sbinfo->free_inodes; 2810 } 2811 /* else leave those fields 0 like simple_statfs */ 2812 return 0; 2813 } 2814 2815 /* 2816 * File creation. Allocate an inode, and we're done.. 2817 */ 2818 static int 2819 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) 2820 { 2821 struct inode *inode; 2822 int error = -ENOSPC; 2823 2824 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2825 if (inode) { 2826 error = simple_acl_create(dir, inode); 2827 if (error) 2828 goto out_iput; 2829 error = security_inode_init_security(inode, dir, 2830 &dentry->d_name, 2831 shmem_initxattrs, NULL); 2832 if (error && error != -EOPNOTSUPP) 2833 goto out_iput; 2834 2835 error = 0; 2836 dir->i_size += BOGO_DIRENT_SIZE; 2837 dir->i_ctime = dir->i_mtime = current_time(dir); 2838 d_instantiate(dentry, inode); 2839 dget(dentry); /* Extra count - pin the dentry in core */ 2840 } 2841 return error; 2842 out_iput: 2843 iput(inode); 2844 return error; 2845 } 2846 2847 static int 2848 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 2849 { 2850 struct inode *inode; 2851 int error = -ENOSPC; 2852 2853 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2854 if (inode) { 2855 error = security_inode_init_security(inode, dir, 2856 NULL, 2857 shmem_initxattrs, NULL); 2858 if (error && error != -EOPNOTSUPP) 2859 goto out_iput; 2860 error = simple_acl_create(dir, inode); 2861 if (error) 2862 goto out_iput; 2863 d_tmpfile(dentry, inode); 2864 } 2865 return error; 2866 out_iput: 2867 iput(inode); 2868 return error; 2869 } 2870 2871 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 2872 { 2873 int error; 2874 2875 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 2876 return error; 2877 inc_nlink(dir); 2878 return 0; 2879 } 2880 2881 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, 2882 bool excl) 2883 { 2884 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 2885 } 2886 2887 /* 2888 * Link a file.. 2889 */ 2890 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2891 { 2892 struct inode *inode = d_inode(old_dentry); 2893 int ret; 2894 2895 /* 2896 * No ordinary (disk based) filesystem counts links as inodes; 2897 * but each new link needs a new dentry, pinning lowmem, and 2898 * tmpfs dentries cannot be pruned until they are unlinked. 2899 */ 2900 ret = shmem_reserve_inode(inode->i_sb); 2901 if (ret) 2902 goto out; 2903 2904 dir->i_size += BOGO_DIRENT_SIZE; 2905 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2906 inc_nlink(inode); 2907 ihold(inode); /* New dentry reference */ 2908 dget(dentry); /* Extra pinning count for the created dentry */ 2909 d_instantiate(dentry, inode); 2910 out: 2911 return ret; 2912 } 2913 2914 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2915 { 2916 struct inode *inode = d_inode(dentry); 2917 2918 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2919 shmem_free_inode(inode->i_sb); 2920 2921 dir->i_size -= BOGO_DIRENT_SIZE; 2922 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2923 drop_nlink(inode); 2924 dput(dentry); /* Undo the count from "create" - this does all the work */ 2925 return 0; 2926 } 2927 2928 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2929 { 2930 if (!simple_empty(dentry)) 2931 return -ENOTEMPTY; 2932 2933 drop_nlink(d_inode(dentry)); 2934 drop_nlink(dir); 2935 return shmem_unlink(dir, dentry); 2936 } 2937 2938 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 2939 { 2940 bool old_is_dir = d_is_dir(old_dentry); 2941 bool new_is_dir = d_is_dir(new_dentry); 2942 2943 if (old_dir != new_dir && old_is_dir != new_is_dir) { 2944 if (old_is_dir) { 2945 drop_nlink(old_dir); 2946 inc_nlink(new_dir); 2947 } else { 2948 drop_nlink(new_dir); 2949 inc_nlink(old_dir); 2950 } 2951 } 2952 old_dir->i_ctime = old_dir->i_mtime = 2953 new_dir->i_ctime = new_dir->i_mtime = 2954 d_inode(old_dentry)->i_ctime = 2955 d_inode(new_dentry)->i_ctime = current_time(old_dir); 2956 2957 return 0; 2958 } 2959 2960 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) 2961 { 2962 struct dentry *whiteout; 2963 int error; 2964 2965 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 2966 if (!whiteout) 2967 return -ENOMEM; 2968 2969 error = shmem_mknod(old_dir, whiteout, 2970 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 2971 dput(whiteout); 2972 if (error) 2973 return error; 2974 2975 /* 2976 * Cheat and hash the whiteout while the old dentry is still in 2977 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 2978 * 2979 * d_lookup() will consistently find one of them at this point, 2980 * not sure which one, but that isn't even important. 2981 */ 2982 d_rehash(whiteout); 2983 return 0; 2984 } 2985 2986 /* 2987 * The VFS layer already does all the dentry stuff for rename, 2988 * we just have to decrement the usage count for the target if 2989 * it exists so that the VFS layer correctly free's it when it 2990 * gets overwritten. 2991 */ 2992 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) 2993 { 2994 struct inode *inode = d_inode(old_dentry); 2995 int they_are_dirs = S_ISDIR(inode->i_mode); 2996 2997 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 2998 return -EINVAL; 2999 3000 if (flags & RENAME_EXCHANGE) 3001 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 3002 3003 if (!simple_empty(new_dentry)) 3004 return -ENOTEMPTY; 3005 3006 if (flags & RENAME_WHITEOUT) { 3007 int error; 3008 3009 error = shmem_whiteout(old_dir, old_dentry); 3010 if (error) 3011 return error; 3012 } 3013 3014 if (d_really_is_positive(new_dentry)) { 3015 (void) shmem_unlink(new_dir, new_dentry); 3016 if (they_are_dirs) { 3017 drop_nlink(d_inode(new_dentry)); 3018 drop_nlink(old_dir); 3019 } 3020 } else if (they_are_dirs) { 3021 drop_nlink(old_dir); 3022 inc_nlink(new_dir); 3023 } 3024 3025 old_dir->i_size -= BOGO_DIRENT_SIZE; 3026 new_dir->i_size += BOGO_DIRENT_SIZE; 3027 old_dir->i_ctime = old_dir->i_mtime = 3028 new_dir->i_ctime = new_dir->i_mtime = 3029 inode->i_ctime = current_time(old_dir); 3030 return 0; 3031 } 3032 3033 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 3034 { 3035 int error; 3036 int len; 3037 struct inode *inode; 3038 struct page *page; 3039 struct shmem_inode_info *info; 3040 3041 len = strlen(symname) + 1; 3042 if (len > PAGE_SIZE) 3043 return -ENAMETOOLONG; 3044 3045 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 3046 if (!inode) 3047 return -ENOSPC; 3048 3049 error = security_inode_init_security(inode, dir, &dentry->d_name, 3050 shmem_initxattrs, NULL); 3051 if (error) { 3052 if (error != -EOPNOTSUPP) { 3053 iput(inode); 3054 return error; 3055 } 3056 error = 0; 3057 } 3058 3059 info = SHMEM_I(inode); 3060 inode->i_size = len-1; 3061 if (len <= SHORT_SYMLINK_LEN) { 3062 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3063 if (!inode->i_link) { 3064 iput(inode); 3065 return -ENOMEM; 3066 } 3067 inode->i_op = &shmem_short_symlink_operations; 3068 } else { 3069 inode_nohighmem(inode); 3070 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3071 if (error) { 3072 iput(inode); 3073 return error; 3074 } 3075 inode->i_mapping->a_ops = &shmem_aops; 3076 inode->i_op = &shmem_symlink_inode_operations; 3077 memcpy(page_address(page), symname, len); 3078 SetPageUptodate(page); 3079 set_page_dirty(page); 3080 unlock_page(page); 3081 put_page(page); 3082 } 3083 dir->i_size += BOGO_DIRENT_SIZE; 3084 dir->i_ctime = dir->i_mtime = current_time(dir); 3085 d_instantiate(dentry, inode); 3086 dget(dentry); 3087 return 0; 3088 } 3089 3090 static void shmem_put_link(void *arg) 3091 { 3092 mark_page_accessed(arg); 3093 put_page(arg); 3094 } 3095 3096 static const char *shmem_get_link(struct dentry *dentry, 3097 struct inode *inode, 3098 struct delayed_call *done) 3099 { 3100 struct page *page = NULL; 3101 int error; 3102 if (!dentry) { 3103 page = find_get_page(inode->i_mapping, 0); 3104 if (!page) 3105 return ERR_PTR(-ECHILD); 3106 if (!PageUptodate(page)) { 3107 put_page(page); 3108 return ERR_PTR(-ECHILD); 3109 } 3110 } else { 3111 error = shmem_getpage(inode, 0, &page, SGP_READ); 3112 if (error) 3113 return ERR_PTR(error); 3114 unlock_page(page); 3115 } 3116 set_delayed_call(done, shmem_put_link, page); 3117 return page_address(page); 3118 } 3119 3120 #ifdef CONFIG_TMPFS_XATTR 3121 /* 3122 * Superblocks without xattr inode operations may get some security.* xattr 3123 * support from the LSM "for free". As soon as we have any other xattrs 3124 * like ACLs, we also need to implement the security.* handlers at 3125 * filesystem level, though. 3126 */ 3127 3128 /* 3129 * Callback for security_inode_init_security() for acquiring xattrs. 3130 */ 3131 static int shmem_initxattrs(struct inode *inode, 3132 const struct xattr *xattr_array, 3133 void *fs_info) 3134 { 3135 struct shmem_inode_info *info = SHMEM_I(inode); 3136 const struct xattr *xattr; 3137 struct simple_xattr *new_xattr; 3138 size_t len; 3139 3140 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3141 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3142 if (!new_xattr) 3143 return -ENOMEM; 3144 3145 len = strlen(xattr->name) + 1; 3146 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3147 GFP_KERNEL); 3148 if (!new_xattr->name) { 3149 kfree(new_xattr); 3150 return -ENOMEM; 3151 } 3152 3153 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3154 XATTR_SECURITY_PREFIX_LEN); 3155 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3156 xattr->name, len); 3157 3158 simple_xattr_list_add(&info->xattrs, new_xattr); 3159 } 3160 3161 return 0; 3162 } 3163 3164 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3165 struct dentry *unused, struct inode *inode, 3166 const char *name, void *buffer, size_t size) 3167 { 3168 struct shmem_inode_info *info = SHMEM_I(inode); 3169 3170 name = xattr_full_name(handler, name); 3171 return simple_xattr_get(&info->xattrs, name, buffer, size); 3172 } 3173 3174 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3175 struct dentry *unused, struct inode *inode, 3176 const char *name, const void *value, 3177 size_t size, int flags) 3178 { 3179 struct shmem_inode_info *info = SHMEM_I(inode); 3180 3181 name = xattr_full_name(handler, name); 3182 return simple_xattr_set(&info->xattrs, name, value, size, flags); 3183 } 3184 3185 static const struct xattr_handler shmem_security_xattr_handler = { 3186 .prefix = XATTR_SECURITY_PREFIX, 3187 .get = shmem_xattr_handler_get, 3188 .set = shmem_xattr_handler_set, 3189 }; 3190 3191 static const struct xattr_handler shmem_trusted_xattr_handler = { 3192 .prefix = XATTR_TRUSTED_PREFIX, 3193 .get = shmem_xattr_handler_get, 3194 .set = shmem_xattr_handler_set, 3195 }; 3196 3197 static const struct xattr_handler *shmem_xattr_handlers[] = { 3198 #ifdef CONFIG_TMPFS_POSIX_ACL 3199 &posix_acl_access_xattr_handler, 3200 &posix_acl_default_xattr_handler, 3201 #endif 3202 &shmem_security_xattr_handler, 3203 &shmem_trusted_xattr_handler, 3204 NULL 3205 }; 3206 3207 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3208 { 3209 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3210 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3211 } 3212 #endif /* CONFIG_TMPFS_XATTR */ 3213 3214 static const struct inode_operations shmem_short_symlink_operations = { 3215 .get_link = simple_get_link, 3216 #ifdef CONFIG_TMPFS_XATTR 3217 .listxattr = shmem_listxattr, 3218 #endif 3219 }; 3220 3221 static const struct inode_operations shmem_symlink_inode_operations = { 3222 .get_link = shmem_get_link, 3223 #ifdef CONFIG_TMPFS_XATTR 3224 .listxattr = shmem_listxattr, 3225 #endif 3226 }; 3227 3228 static struct dentry *shmem_get_parent(struct dentry *child) 3229 { 3230 return ERR_PTR(-ESTALE); 3231 } 3232 3233 static int shmem_match(struct inode *ino, void *vfh) 3234 { 3235 __u32 *fh = vfh; 3236 __u64 inum = fh[2]; 3237 inum = (inum << 32) | fh[1]; 3238 return ino->i_ino == inum && fh[0] == ino->i_generation; 3239 } 3240 3241 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3242 struct fid *fid, int fh_len, int fh_type) 3243 { 3244 struct inode *inode; 3245 struct dentry *dentry = NULL; 3246 u64 inum; 3247 3248 if (fh_len < 3) 3249 return NULL; 3250 3251 inum = fid->raw[2]; 3252 inum = (inum << 32) | fid->raw[1]; 3253 3254 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3255 shmem_match, fid->raw); 3256 if (inode) { 3257 dentry = d_find_alias(inode); 3258 iput(inode); 3259 } 3260 3261 return dentry; 3262 } 3263 3264 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3265 struct inode *parent) 3266 { 3267 if (*len < 3) { 3268 *len = 3; 3269 return FILEID_INVALID; 3270 } 3271 3272 if (inode_unhashed(inode)) { 3273 /* Unfortunately insert_inode_hash is not idempotent, 3274 * so as we hash inodes here rather than at creation 3275 * time, we need a lock to ensure we only try 3276 * to do it once 3277 */ 3278 static DEFINE_SPINLOCK(lock); 3279 spin_lock(&lock); 3280 if (inode_unhashed(inode)) 3281 __insert_inode_hash(inode, 3282 inode->i_ino + inode->i_generation); 3283 spin_unlock(&lock); 3284 } 3285 3286 fh[0] = inode->i_generation; 3287 fh[1] = inode->i_ino; 3288 fh[2] = ((__u64)inode->i_ino) >> 32; 3289 3290 *len = 3; 3291 return 1; 3292 } 3293 3294 static const struct export_operations shmem_export_ops = { 3295 .get_parent = shmem_get_parent, 3296 .encode_fh = shmem_encode_fh, 3297 .fh_to_dentry = shmem_fh_to_dentry, 3298 }; 3299 3300 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 3301 bool remount) 3302 { 3303 char *this_char, *value, *rest; 3304 struct mempolicy *mpol = NULL; 3305 uid_t uid; 3306 gid_t gid; 3307 3308 while (options != NULL) { 3309 this_char = options; 3310 for (;;) { 3311 /* 3312 * NUL-terminate this option: unfortunately, 3313 * mount options form a comma-separated list, 3314 * but mpol's nodelist may also contain commas. 3315 */ 3316 options = strchr(options, ','); 3317 if (options == NULL) 3318 break; 3319 options++; 3320 if (!isdigit(*options)) { 3321 options[-1] = '\0'; 3322 break; 3323 } 3324 } 3325 if (!*this_char) 3326 continue; 3327 if ((value = strchr(this_char,'=')) != NULL) { 3328 *value++ = 0; 3329 } else { 3330 pr_err("tmpfs: No value for mount option '%s'\n", 3331 this_char); 3332 goto error; 3333 } 3334 3335 if (!strcmp(this_char,"size")) { 3336 unsigned long long size; 3337 size = memparse(value,&rest); 3338 if (*rest == '%') { 3339 size <<= PAGE_SHIFT; 3340 size *= totalram_pages; 3341 do_div(size, 100); 3342 rest++; 3343 } 3344 if (*rest) 3345 goto bad_val; 3346 sbinfo->max_blocks = 3347 DIV_ROUND_UP(size, PAGE_SIZE); 3348 } else if (!strcmp(this_char,"nr_blocks")) { 3349 sbinfo->max_blocks = memparse(value, &rest); 3350 if (*rest) 3351 goto bad_val; 3352 } else if (!strcmp(this_char,"nr_inodes")) { 3353 sbinfo->max_inodes = memparse(value, &rest); 3354 if (*rest) 3355 goto bad_val; 3356 } else if (!strcmp(this_char,"mode")) { 3357 if (remount) 3358 continue; 3359 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 3360 if (*rest) 3361 goto bad_val; 3362 } else if (!strcmp(this_char,"uid")) { 3363 if (remount) 3364 continue; 3365 uid = simple_strtoul(value, &rest, 0); 3366 if (*rest) 3367 goto bad_val; 3368 sbinfo->uid = make_kuid(current_user_ns(), uid); 3369 if (!uid_valid(sbinfo->uid)) 3370 goto bad_val; 3371 } else if (!strcmp(this_char,"gid")) { 3372 if (remount) 3373 continue; 3374 gid = simple_strtoul(value, &rest, 0); 3375 if (*rest) 3376 goto bad_val; 3377 sbinfo->gid = make_kgid(current_user_ns(), gid); 3378 if (!gid_valid(sbinfo->gid)) 3379 goto bad_val; 3380 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3381 } else if (!strcmp(this_char, "huge")) { 3382 int huge; 3383 huge = shmem_parse_huge(value); 3384 if (huge < 0) 3385 goto bad_val; 3386 if (!has_transparent_hugepage() && 3387 huge != SHMEM_HUGE_NEVER) 3388 goto bad_val; 3389 sbinfo->huge = huge; 3390 #endif 3391 #ifdef CONFIG_NUMA 3392 } else if (!strcmp(this_char,"mpol")) { 3393 mpol_put(mpol); 3394 mpol = NULL; 3395 if (mpol_parse_str(value, &mpol)) 3396 goto bad_val; 3397 #endif 3398 } else { 3399 pr_err("tmpfs: Bad mount option %s\n", this_char); 3400 goto error; 3401 } 3402 } 3403 sbinfo->mpol = mpol; 3404 return 0; 3405 3406 bad_val: 3407 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n", 3408 value, this_char); 3409 error: 3410 mpol_put(mpol); 3411 return 1; 3412 3413 } 3414 3415 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 3416 { 3417 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3418 struct shmem_sb_info config = *sbinfo; 3419 unsigned long inodes; 3420 int error = -EINVAL; 3421 3422 config.mpol = NULL; 3423 if (shmem_parse_options(data, &config, true)) 3424 return error; 3425 3426 spin_lock(&sbinfo->stat_lock); 3427 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3428 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 3429 goto out; 3430 if (config.max_inodes < inodes) 3431 goto out; 3432 /* 3433 * Those tests disallow limited->unlimited while any are in use; 3434 * but we must separately disallow unlimited->limited, because 3435 * in that case we have no record of how much is already in use. 3436 */ 3437 if (config.max_blocks && !sbinfo->max_blocks) 3438 goto out; 3439 if (config.max_inodes && !sbinfo->max_inodes) 3440 goto out; 3441 3442 error = 0; 3443 sbinfo->huge = config.huge; 3444 sbinfo->max_blocks = config.max_blocks; 3445 sbinfo->max_inodes = config.max_inodes; 3446 sbinfo->free_inodes = config.max_inodes - inodes; 3447 3448 /* 3449 * Preserve previous mempolicy unless mpol remount option was specified. 3450 */ 3451 if (config.mpol) { 3452 mpol_put(sbinfo->mpol); 3453 sbinfo->mpol = config.mpol; /* transfers initial ref */ 3454 } 3455 out: 3456 spin_unlock(&sbinfo->stat_lock); 3457 return error; 3458 } 3459 3460 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3461 { 3462 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3463 3464 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3465 seq_printf(seq, ",size=%luk", 3466 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3467 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3468 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3469 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 3470 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3471 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3472 seq_printf(seq, ",uid=%u", 3473 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3474 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3475 seq_printf(seq, ",gid=%u", 3476 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3477 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3478 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3479 if (sbinfo->huge) 3480 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3481 #endif 3482 shmem_show_mpol(seq, sbinfo->mpol); 3483 return 0; 3484 } 3485 3486 #define MFD_NAME_PREFIX "memfd:" 3487 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) 3488 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) 3489 3490 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING) 3491 3492 SYSCALL_DEFINE2(memfd_create, 3493 const char __user *, uname, 3494 unsigned int, flags) 3495 { 3496 struct shmem_inode_info *info; 3497 struct file *file; 3498 int fd, error; 3499 char *name; 3500 long len; 3501 3502 if (flags & ~(unsigned int)MFD_ALL_FLAGS) 3503 return -EINVAL; 3504 3505 /* length includes terminating zero */ 3506 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); 3507 if (len <= 0) 3508 return -EFAULT; 3509 if (len > MFD_NAME_MAX_LEN + 1) 3510 return -EINVAL; 3511 3512 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY); 3513 if (!name) 3514 return -ENOMEM; 3515 3516 strcpy(name, MFD_NAME_PREFIX); 3517 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { 3518 error = -EFAULT; 3519 goto err_name; 3520 } 3521 3522 /* terminating-zero may have changed after strnlen_user() returned */ 3523 if (name[len + MFD_NAME_PREFIX_LEN - 1]) { 3524 error = -EFAULT; 3525 goto err_name; 3526 } 3527 3528 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); 3529 if (fd < 0) { 3530 error = fd; 3531 goto err_name; 3532 } 3533 3534 file = shmem_file_setup(name, 0, VM_NORESERVE); 3535 if (IS_ERR(file)) { 3536 error = PTR_ERR(file); 3537 goto err_fd; 3538 } 3539 info = SHMEM_I(file_inode(file)); 3540 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; 3541 file->f_flags |= O_RDWR | O_LARGEFILE; 3542 if (flags & MFD_ALLOW_SEALING) 3543 info->seals &= ~F_SEAL_SEAL; 3544 3545 fd_install(fd, file); 3546 kfree(name); 3547 return fd; 3548 3549 err_fd: 3550 put_unused_fd(fd); 3551 err_name: 3552 kfree(name); 3553 return error; 3554 } 3555 3556 #endif /* CONFIG_TMPFS */ 3557 3558 static void shmem_put_super(struct super_block *sb) 3559 { 3560 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3561 3562 percpu_counter_destroy(&sbinfo->used_blocks); 3563 mpol_put(sbinfo->mpol); 3564 kfree(sbinfo); 3565 sb->s_fs_info = NULL; 3566 } 3567 3568 int shmem_fill_super(struct super_block *sb, void *data, int silent) 3569 { 3570 struct inode *inode; 3571 struct shmem_sb_info *sbinfo; 3572 int err = -ENOMEM; 3573 3574 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3575 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3576 L1_CACHE_BYTES), GFP_KERNEL); 3577 if (!sbinfo) 3578 return -ENOMEM; 3579 3580 sbinfo->mode = S_IRWXUGO | S_ISVTX; 3581 sbinfo->uid = current_fsuid(); 3582 sbinfo->gid = current_fsgid(); 3583 sb->s_fs_info = sbinfo; 3584 3585 #ifdef CONFIG_TMPFS 3586 /* 3587 * Per default we only allow half of the physical ram per 3588 * tmpfs instance, limiting inodes to one per page of lowmem; 3589 * but the internal instance is left unlimited. 3590 */ 3591 if (!(sb->s_flags & MS_KERNMOUNT)) { 3592 sbinfo->max_blocks = shmem_default_max_blocks(); 3593 sbinfo->max_inodes = shmem_default_max_inodes(); 3594 if (shmem_parse_options(data, sbinfo, false)) { 3595 err = -EINVAL; 3596 goto failed; 3597 } 3598 } else { 3599 sb->s_flags |= MS_NOUSER; 3600 } 3601 sb->s_export_op = &shmem_export_ops; 3602 sb->s_flags |= MS_NOSEC; 3603 #else 3604 sb->s_flags |= MS_NOUSER; 3605 #endif 3606 3607 spin_lock_init(&sbinfo->stat_lock); 3608 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3609 goto failed; 3610 sbinfo->free_inodes = sbinfo->max_inodes; 3611 spin_lock_init(&sbinfo->shrinklist_lock); 3612 INIT_LIST_HEAD(&sbinfo->shrinklist); 3613 3614 sb->s_maxbytes = MAX_LFS_FILESIZE; 3615 sb->s_blocksize = PAGE_SIZE; 3616 sb->s_blocksize_bits = PAGE_SHIFT; 3617 sb->s_magic = TMPFS_MAGIC; 3618 sb->s_op = &shmem_ops; 3619 sb->s_time_gran = 1; 3620 #ifdef CONFIG_TMPFS_XATTR 3621 sb->s_xattr = shmem_xattr_handlers; 3622 #endif 3623 #ifdef CONFIG_TMPFS_POSIX_ACL 3624 sb->s_flags |= MS_POSIXACL; 3625 #endif 3626 3627 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3628 if (!inode) 3629 goto failed; 3630 inode->i_uid = sbinfo->uid; 3631 inode->i_gid = sbinfo->gid; 3632 sb->s_root = d_make_root(inode); 3633 if (!sb->s_root) 3634 goto failed; 3635 return 0; 3636 3637 failed: 3638 shmem_put_super(sb); 3639 return err; 3640 } 3641 3642 static struct kmem_cache *shmem_inode_cachep; 3643 3644 static struct inode *shmem_alloc_inode(struct super_block *sb) 3645 { 3646 struct shmem_inode_info *info; 3647 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3648 if (!info) 3649 return NULL; 3650 return &info->vfs_inode; 3651 } 3652 3653 static void shmem_destroy_callback(struct rcu_head *head) 3654 { 3655 struct inode *inode = container_of(head, struct inode, i_rcu); 3656 if (S_ISLNK(inode->i_mode)) 3657 kfree(inode->i_link); 3658 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3659 } 3660 3661 static void shmem_destroy_inode(struct inode *inode) 3662 { 3663 if (S_ISREG(inode->i_mode)) 3664 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3665 call_rcu(&inode->i_rcu, shmem_destroy_callback); 3666 } 3667 3668 static void shmem_init_inode(void *foo) 3669 { 3670 struct shmem_inode_info *info = foo; 3671 inode_init_once(&info->vfs_inode); 3672 } 3673 3674 static int shmem_init_inodecache(void) 3675 { 3676 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3677 sizeof(struct shmem_inode_info), 3678 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3679 return 0; 3680 } 3681 3682 static void shmem_destroy_inodecache(void) 3683 { 3684 kmem_cache_destroy(shmem_inode_cachep); 3685 } 3686 3687 static const struct address_space_operations shmem_aops = { 3688 .writepage = shmem_writepage, 3689 .set_page_dirty = __set_page_dirty_no_writeback, 3690 #ifdef CONFIG_TMPFS 3691 .write_begin = shmem_write_begin, 3692 .write_end = shmem_write_end, 3693 #endif 3694 #ifdef CONFIG_MIGRATION 3695 .migratepage = migrate_page, 3696 #endif 3697 .error_remove_page = generic_error_remove_page, 3698 }; 3699 3700 static const struct file_operations shmem_file_operations = { 3701 .mmap = shmem_mmap, 3702 .get_unmapped_area = shmem_get_unmapped_area, 3703 #ifdef CONFIG_TMPFS 3704 .llseek = shmem_file_llseek, 3705 .read_iter = shmem_file_read_iter, 3706 .write_iter = generic_file_write_iter, 3707 .fsync = noop_fsync, 3708 .splice_read = generic_file_splice_read, 3709 .splice_write = iter_file_splice_write, 3710 .fallocate = shmem_fallocate, 3711 #endif 3712 }; 3713 3714 static const struct inode_operations shmem_inode_operations = { 3715 .getattr = shmem_getattr, 3716 .setattr = shmem_setattr, 3717 #ifdef CONFIG_TMPFS_XATTR 3718 .listxattr = shmem_listxattr, 3719 .set_acl = simple_set_acl, 3720 #endif 3721 }; 3722 3723 static const struct inode_operations shmem_dir_inode_operations = { 3724 #ifdef CONFIG_TMPFS 3725 .create = shmem_create, 3726 .lookup = simple_lookup, 3727 .link = shmem_link, 3728 .unlink = shmem_unlink, 3729 .symlink = shmem_symlink, 3730 .mkdir = shmem_mkdir, 3731 .rmdir = shmem_rmdir, 3732 .mknod = shmem_mknod, 3733 .rename = shmem_rename2, 3734 .tmpfile = shmem_tmpfile, 3735 #endif 3736 #ifdef CONFIG_TMPFS_XATTR 3737 .listxattr = shmem_listxattr, 3738 #endif 3739 #ifdef CONFIG_TMPFS_POSIX_ACL 3740 .setattr = shmem_setattr, 3741 .set_acl = simple_set_acl, 3742 #endif 3743 }; 3744 3745 static const struct inode_operations shmem_special_inode_operations = { 3746 #ifdef CONFIG_TMPFS_XATTR 3747 .listxattr = shmem_listxattr, 3748 #endif 3749 #ifdef CONFIG_TMPFS_POSIX_ACL 3750 .setattr = shmem_setattr, 3751 .set_acl = simple_set_acl, 3752 #endif 3753 }; 3754 3755 static const struct super_operations shmem_ops = { 3756 .alloc_inode = shmem_alloc_inode, 3757 .destroy_inode = shmem_destroy_inode, 3758 #ifdef CONFIG_TMPFS 3759 .statfs = shmem_statfs, 3760 .remount_fs = shmem_remount_fs, 3761 .show_options = shmem_show_options, 3762 #endif 3763 .evict_inode = shmem_evict_inode, 3764 .drop_inode = generic_delete_inode, 3765 .put_super = shmem_put_super, 3766 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3767 .nr_cached_objects = shmem_unused_huge_count, 3768 .free_cached_objects = shmem_unused_huge_scan, 3769 #endif 3770 }; 3771 3772 static const struct vm_operations_struct shmem_vm_ops = { 3773 .fault = shmem_fault, 3774 .map_pages = filemap_map_pages, 3775 #ifdef CONFIG_NUMA 3776 .set_policy = shmem_set_policy, 3777 .get_policy = shmem_get_policy, 3778 #endif 3779 }; 3780 3781 static struct dentry *shmem_mount(struct file_system_type *fs_type, 3782 int flags, const char *dev_name, void *data) 3783 { 3784 return mount_nodev(fs_type, flags, data, shmem_fill_super); 3785 } 3786 3787 static struct file_system_type shmem_fs_type = { 3788 .owner = THIS_MODULE, 3789 .name = "tmpfs", 3790 .mount = shmem_mount, 3791 .kill_sb = kill_litter_super, 3792 .fs_flags = FS_USERNS_MOUNT, 3793 }; 3794 3795 int __init shmem_init(void) 3796 { 3797 int error; 3798 3799 /* If rootfs called this, don't re-init */ 3800 if (shmem_inode_cachep) 3801 return 0; 3802 3803 error = shmem_init_inodecache(); 3804 if (error) 3805 goto out3; 3806 3807 error = register_filesystem(&shmem_fs_type); 3808 if (error) { 3809 pr_err("Could not register tmpfs\n"); 3810 goto out2; 3811 } 3812 3813 shm_mnt = kern_mount(&shmem_fs_type); 3814 if (IS_ERR(shm_mnt)) { 3815 error = PTR_ERR(shm_mnt); 3816 pr_err("Could not kern_mount tmpfs\n"); 3817 goto out1; 3818 } 3819 3820 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3821 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY) 3822 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3823 else 3824 shmem_huge = 0; /* just in case it was patched */ 3825 #endif 3826 return 0; 3827 3828 out1: 3829 unregister_filesystem(&shmem_fs_type); 3830 out2: 3831 shmem_destroy_inodecache(); 3832 out3: 3833 shm_mnt = ERR_PTR(error); 3834 return error; 3835 } 3836 3837 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) 3838 static ssize_t shmem_enabled_show(struct kobject *kobj, 3839 struct kobj_attribute *attr, char *buf) 3840 { 3841 int values[] = { 3842 SHMEM_HUGE_ALWAYS, 3843 SHMEM_HUGE_WITHIN_SIZE, 3844 SHMEM_HUGE_ADVISE, 3845 SHMEM_HUGE_NEVER, 3846 SHMEM_HUGE_DENY, 3847 SHMEM_HUGE_FORCE, 3848 }; 3849 int i, count; 3850 3851 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { 3852 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; 3853 3854 count += sprintf(buf + count, fmt, 3855 shmem_format_huge(values[i])); 3856 } 3857 buf[count - 1] = '\n'; 3858 return count; 3859 } 3860 3861 static ssize_t shmem_enabled_store(struct kobject *kobj, 3862 struct kobj_attribute *attr, const char *buf, size_t count) 3863 { 3864 char tmp[16]; 3865 int huge; 3866 3867 if (count + 1 > sizeof(tmp)) 3868 return -EINVAL; 3869 memcpy(tmp, buf, count); 3870 tmp[count] = '\0'; 3871 if (count && tmp[count - 1] == '\n') 3872 tmp[count - 1] = '\0'; 3873 3874 huge = shmem_parse_huge(tmp); 3875 if (huge == -EINVAL) 3876 return -EINVAL; 3877 if (!has_transparent_hugepage() && 3878 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3879 return -EINVAL; 3880 3881 shmem_huge = huge; 3882 if (shmem_huge < SHMEM_HUGE_DENY) 3883 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3884 return count; 3885 } 3886 3887 struct kobj_attribute shmem_enabled_attr = 3888 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3889 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ 3890 3891 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3892 bool shmem_huge_enabled(struct vm_area_struct *vma) 3893 { 3894 struct inode *inode = file_inode(vma->vm_file); 3895 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3896 loff_t i_size; 3897 pgoff_t off; 3898 3899 if (shmem_huge == SHMEM_HUGE_FORCE) 3900 return true; 3901 if (shmem_huge == SHMEM_HUGE_DENY) 3902 return false; 3903 switch (sbinfo->huge) { 3904 case SHMEM_HUGE_NEVER: 3905 return false; 3906 case SHMEM_HUGE_ALWAYS: 3907 return true; 3908 case SHMEM_HUGE_WITHIN_SIZE: 3909 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); 3910 i_size = round_up(i_size_read(inode), PAGE_SIZE); 3911 if (i_size >= HPAGE_PMD_SIZE && 3912 i_size >> PAGE_SHIFT >= off) 3913 return true; 3914 case SHMEM_HUGE_ADVISE: 3915 /* TODO: implement fadvise() hints */ 3916 return (vma->vm_flags & VM_HUGEPAGE); 3917 default: 3918 VM_BUG_ON(1); 3919 return false; 3920 } 3921 } 3922 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 3923 3924 #else /* !CONFIG_SHMEM */ 3925 3926 /* 3927 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 3928 * 3929 * This is intended for small system where the benefits of the full 3930 * shmem code (swap-backed and resource-limited) are outweighed by 3931 * their complexity. On systems without swap this code should be 3932 * effectively equivalent, but much lighter weight. 3933 */ 3934 3935 static struct file_system_type shmem_fs_type = { 3936 .name = "tmpfs", 3937 .mount = ramfs_mount, 3938 .kill_sb = kill_litter_super, 3939 .fs_flags = FS_USERNS_MOUNT, 3940 }; 3941 3942 int __init shmem_init(void) 3943 { 3944 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 3945 3946 shm_mnt = kern_mount(&shmem_fs_type); 3947 BUG_ON(IS_ERR(shm_mnt)); 3948 3949 return 0; 3950 } 3951 3952 int shmem_unuse(swp_entry_t swap, struct page *page) 3953 { 3954 return 0; 3955 } 3956 3957 int shmem_lock(struct file *file, int lock, struct user_struct *user) 3958 { 3959 return 0; 3960 } 3961 3962 void shmem_unlock_mapping(struct address_space *mapping) 3963 { 3964 } 3965 3966 #ifdef CONFIG_MMU 3967 unsigned long shmem_get_unmapped_area(struct file *file, 3968 unsigned long addr, unsigned long len, 3969 unsigned long pgoff, unsigned long flags) 3970 { 3971 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 3972 } 3973 #endif 3974 3975 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 3976 { 3977 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 3978 } 3979 EXPORT_SYMBOL_GPL(shmem_truncate_range); 3980 3981 #define shmem_vm_ops generic_file_vm_ops 3982 #define shmem_file_operations ramfs_file_operations 3983 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 3984 #define shmem_acct_size(flags, size) 0 3985 #define shmem_unacct_size(flags, size) do {} while (0) 3986 3987 #endif /* CONFIG_SHMEM */ 3988 3989 /* common code */ 3990 3991 static const struct dentry_operations anon_ops = { 3992 .d_dname = simple_dname 3993 }; 3994 3995 static struct file *__shmem_file_setup(const char *name, loff_t size, 3996 unsigned long flags, unsigned int i_flags) 3997 { 3998 struct file *res; 3999 struct inode *inode; 4000 struct path path; 4001 struct super_block *sb; 4002 struct qstr this; 4003 4004 if (IS_ERR(shm_mnt)) 4005 return ERR_CAST(shm_mnt); 4006 4007 if (size < 0 || size > MAX_LFS_FILESIZE) 4008 return ERR_PTR(-EINVAL); 4009 4010 if (shmem_acct_size(flags, size)) 4011 return ERR_PTR(-ENOMEM); 4012 4013 res = ERR_PTR(-ENOMEM); 4014 this.name = name; 4015 this.len = strlen(name); 4016 this.hash = 0; /* will go */ 4017 sb = shm_mnt->mnt_sb; 4018 path.mnt = mntget(shm_mnt); 4019 path.dentry = d_alloc_pseudo(sb, &this); 4020 if (!path.dentry) 4021 goto put_memory; 4022 d_set_d_op(path.dentry, &anon_ops); 4023 4024 res = ERR_PTR(-ENOSPC); 4025 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 4026 if (!inode) 4027 goto put_memory; 4028 4029 inode->i_flags |= i_flags; 4030 d_instantiate(path.dentry, inode); 4031 inode->i_size = size; 4032 clear_nlink(inode); /* It is unlinked */ 4033 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4034 if (IS_ERR(res)) 4035 goto put_path; 4036 4037 res = alloc_file(&path, FMODE_WRITE | FMODE_READ, 4038 &shmem_file_operations); 4039 if (IS_ERR(res)) 4040 goto put_path; 4041 4042 return res; 4043 4044 put_memory: 4045 shmem_unacct_size(flags, size); 4046 put_path: 4047 path_put(&path); 4048 return res; 4049 } 4050 4051 /** 4052 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4053 * kernel internal. There will be NO LSM permission checks against the 4054 * underlying inode. So users of this interface must do LSM checks at a 4055 * higher layer. The users are the big_key and shm implementations. LSM 4056 * checks are provided at the key or shm level rather than the inode. 4057 * @name: name for dentry (to be seen in /proc/<pid>/maps 4058 * @size: size to be set for the file 4059 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4060 */ 4061 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4062 { 4063 return __shmem_file_setup(name, size, flags, S_PRIVATE); 4064 } 4065 4066 /** 4067 * shmem_file_setup - get an unlinked file living in tmpfs 4068 * @name: name for dentry (to be seen in /proc/<pid>/maps 4069 * @size: size to be set for the file 4070 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4071 */ 4072 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4073 { 4074 return __shmem_file_setup(name, size, flags, 0); 4075 } 4076 EXPORT_SYMBOL_GPL(shmem_file_setup); 4077 4078 /** 4079 * shmem_zero_setup - setup a shared anonymous mapping 4080 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 4081 */ 4082 int shmem_zero_setup(struct vm_area_struct *vma) 4083 { 4084 struct file *file; 4085 loff_t size = vma->vm_end - vma->vm_start; 4086 4087 /* 4088 * Cloning a new file under mmap_sem leads to a lock ordering conflict 4089 * between XFS directory reading and selinux: since this file is only 4090 * accessible to the user through its mapping, use S_PRIVATE flag to 4091 * bypass file security, in the same way as shmem_kernel_file_setup(). 4092 */ 4093 file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE); 4094 if (IS_ERR(file)) 4095 return PTR_ERR(file); 4096 4097 if (vma->vm_file) 4098 fput(vma->vm_file); 4099 vma->vm_file = file; 4100 vma->vm_ops = &shmem_vm_ops; 4101 4102 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 4103 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4104 (vma->vm_end & HPAGE_PMD_MASK)) { 4105 khugepaged_enter(vma, vma->vm_flags); 4106 } 4107 4108 return 0; 4109 } 4110 4111 /** 4112 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4113 * @mapping: the page's address_space 4114 * @index: the page index 4115 * @gfp: the page allocator flags to use if allocating 4116 * 4117 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4118 * with any new page allocations done using the specified allocation flags. 4119 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4120 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4121 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4122 * 4123 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4124 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4125 */ 4126 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4127 pgoff_t index, gfp_t gfp) 4128 { 4129 #ifdef CONFIG_SHMEM 4130 struct inode *inode = mapping->host; 4131 struct page *page; 4132 int error; 4133 4134 BUG_ON(mapping->a_ops != &shmem_aops); 4135 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4136 gfp, NULL, NULL); 4137 if (error) 4138 page = ERR_PTR(error); 4139 else 4140 unlock_page(page); 4141 return page; 4142 #else 4143 /* 4144 * The tiny !SHMEM case uses ramfs without swap 4145 */ 4146 return read_cache_page_gfp(mapping, index, gfp); 4147 #endif 4148 } 4149 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4150