1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/export.h> 33 #include <linux/swap.h> 34 #include <linux/uio.h> 35 #include <linux/khugepaged.h> 36 37 static struct vfsmount *shm_mnt; 38 39 #ifdef CONFIG_SHMEM 40 /* 41 * This virtual memory filesystem is heavily based on the ramfs. It 42 * extends ramfs by the ability to use swap and honor resource limits 43 * which makes it a completely usable filesystem. 44 */ 45 46 #include <linux/xattr.h> 47 #include <linux/exportfs.h> 48 #include <linux/posix_acl.h> 49 #include <linux/posix_acl_xattr.h> 50 #include <linux/mman.h> 51 #include <linux/string.h> 52 #include <linux/slab.h> 53 #include <linux/backing-dev.h> 54 #include <linux/shmem_fs.h> 55 #include <linux/writeback.h> 56 #include <linux/blkdev.h> 57 #include <linux/pagevec.h> 58 #include <linux/percpu_counter.h> 59 #include <linux/falloc.h> 60 #include <linux/splice.h> 61 #include <linux/security.h> 62 #include <linux/swapops.h> 63 #include <linux/mempolicy.h> 64 #include <linux/namei.h> 65 #include <linux/ctype.h> 66 #include <linux/migrate.h> 67 #include <linux/highmem.h> 68 #include <linux/seq_file.h> 69 #include <linux/magic.h> 70 #include <linux/syscalls.h> 71 #include <linux/fcntl.h> 72 #include <uapi/linux/memfd.h> 73 74 #include <linux/uaccess.h> 75 #include <asm/pgtable.h> 76 77 #include "internal.h" 78 79 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 80 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 81 82 /* Pretend that each entry is of this size in directory's i_size */ 83 #define BOGO_DIRENT_SIZE 20 84 85 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 86 #define SHORT_SYMLINK_LEN 128 87 88 /* 89 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 90 * inode->i_private (with i_mutex making sure that it has only one user at 91 * a time): we would prefer not to enlarge the shmem inode just for that. 92 */ 93 struct shmem_falloc { 94 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 95 pgoff_t start; /* start of range currently being fallocated */ 96 pgoff_t next; /* the next page offset to be fallocated */ 97 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 98 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 99 }; 100 101 #ifdef CONFIG_TMPFS 102 static unsigned long shmem_default_max_blocks(void) 103 { 104 return totalram_pages / 2; 105 } 106 107 static unsigned long shmem_default_max_inodes(void) 108 { 109 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 110 } 111 #endif 112 113 static bool shmem_should_replace_page(struct page *page, gfp_t gfp); 114 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 115 struct shmem_inode_info *info, pgoff_t index); 116 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 117 struct page **pagep, enum sgp_type sgp, 118 gfp_t gfp, struct mm_struct *fault_mm, int *fault_type); 119 120 int shmem_getpage(struct inode *inode, pgoff_t index, 121 struct page **pagep, enum sgp_type sgp) 122 { 123 return shmem_getpage_gfp(inode, index, pagep, sgp, 124 mapping_gfp_mask(inode->i_mapping), NULL, NULL); 125 } 126 127 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 128 { 129 return sb->s_fs_info; 130 } 131 132 /* 133 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 134 * for shared memory and for shared anonymous (/dev/zero) mappings 135 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 136 * consistent with the pre-accounting of private mappings ... 137 */ 138 static inline int shmem_acct_size(unsigned long flags, loff_t size) 139 { 140 return (flags & VM_NORESERVE) ? 141 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 142 } 143 144 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 145 { 146 if (!(flags & VM_NORESERVE)) 147 vm_unacct_memory(VM_ACCT(size)); 148 } 149 150 static inline int shmem_reacct_size(unsigned long flags, 151 loff_t oldsize, loff_t newsize) 152 { 153 if (!(flags & VM_NORESERVE)) { 154 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 155 return security_vm_enough_memory_mm(current->mm, 156 VM_ACCT(newsize) - VM_ACCT(oldsize)); 157 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 158 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 159 } 160 return 0; 161 } 162 163 /* 164 * ... whereas tmpfs objects are accounted incrementally as 165 * pages are allocated, in order to allow large sparse files. 166 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 167 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 168 */ 169 static inline int shmem_acct_block(unsigned long flags, long pages) 170 { 171 if (!(flags & VM_NORESERVE)) 172 return 0; 173 174 return security_vm_enough_memory_mm(current->mm, 175 pages * VM_ACCT(PAGE_SIZE)); 176 } 177 178 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 179 { 180 if (flags & VM_NORESERVE) 181 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 182 } 183 184 static const struct super_operations shmem_ops; 185 static const struct address_space_operations shmem_aops; 186 static const struct file_operations shmem_file_operations; 187 static const struct inode_operations shmem_inode_operations; 188 static const struct inode_operations shmem_dir_inode_operations; 189 static const struct inode_operations shmem_special_inode_operations; 190 static const struct vm_operations_struct shmem_vm_ops; 191 static struct file_system_type shmem_fs_type; 192 193 static LIST_HEAD(shmem_swaplist); 194 static DEFINE_MUTEX(shmem_swaplist_mutex); 195 196 static int shmem_reserve_inode(struct super_block *sb) 197 { 198 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 199 if (sbinfo->max_inodes) { 200 spin_lock(&sbinfo->stat_lock); 201 if (!sbinfo->free_inodes) { 202 spin_unlock(&sbinfo->stat_lock); 203 return -ENOSPC; 204 } 205 sbinfo->free_inodes--; 206 spin_unlock(&sbinfo->stat_lock); 207 } 208 return 0; 209 } 210 211 static void shmem_free_inode(struct super_block *sb) 212 { 213 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 214 if (sbinfo->max_inodes) { 215 spin_lock(&sbinfo->stat_lock); 216 sbinfo->free_inodes++; 217 spin_unlock(&sbinfo->stat_lock); 218 } 219 } 220 221 /** 222 * shmem_recalc_inode - recalculate the block usage of an inode 223 * @inode: inode to recalc 224 * 225 * We have to calculate the free blocks since the mm can drop 226 * undirtied hole pages behind our back. 227 * 228 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 229 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 230 * 231 * It has to be called with the spinlock held. 232 */ 233 static void shmem_recalc_inode(struct inode *inode) 234 { 235 struct shmem_inode_info *info = SHMEM_I(inode); 236 long freed; 237 238 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 239 if (freed > 0) { 240 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 241 if (sbinfo->max_blocks) 242 percpu_counter_add(&sbinfo->used_blocks, -freed); 243 info->alloced -= freed; 244 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 245 shmem_unacct_blocks(info->flags, freed); 246 } 247 } 248 249 bool shmem_charge(struct inode *inode, long pages) 250 { 251 struct shmem_inode_info *info = SHMEM_I(inode); 252 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 253 unsigned long flags; 254 255 if (shmem_acct_block(info->flags, pages)) 256 return false; 257 spin_lock_irqsave(&info->lock, flags); 258 info->alloced += pages; 259 inode->i_blocks += pages * BLOCKS_PER_PAGE; 260 shmem_recalc_inode(inode); 261 spin_unlock_irqrestore(&info->lock, flags); 262 inode->i_mapping->nrpages += pages; 263 264 if (!sbinfo->max_blocks) 265 return true; 266 if (percpu_counter_compare(&sbinfo->used_blocks, 267 sbinfo->max_blocks - pages) > 0) { 268 inode->i_mapping->nrpages -= pages; 269 spin_lock_irqsave(&info->lock, flags); 270 info->alloced -= pages; 271 shmem_recalc_inode(inode); 272 spin_unlock_irqrestore(&info->lock, flags); 273 shmem_unacct_blocks(info->flags, pages); 274 return false; 275 } 276 percpu_counter_add(&sbinfo->used_blocks, pages); 277 return true; 278 } 279 280 void shmem_uncharge(struct inode *inode, long pages) 281 { 282 struct shmem_inode_info *info = SHMEM_I(inode); 283 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 284 unsigned long flags; 285 286 spin_lock_irqsave(&info->lock, flags); 287 info->alloced -= pages; 288 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 289 shmem_recalc_inode(inode); 290 spin_unlock_irqrestore(&info->lock, flags); 291 292 if (sbinfo->max_blocks) 293 percpu_counter_sub(&sbinfo->used_blocks, pages); 294 shmem_unacct_blocks(info->flags, pages); 295 } 296 297 /* 298 * Replace item expected in radix tree by a new item, while holding tree lock. 299 */ 300 static int shmem_radix_tree_replace(struct address_space *mapping, 301 pgoff_t index, void *expected, void *replacement) 302 { 303 struct radix_tree_node *node; 304 void **pslot; 305 void *item; 306 307 VM_BUG_ON(!expected); 308 VM_BUG_ON(!replacement); 309 item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); 310 if (!item) 311 return -ENOENT; 312 if (item != expected) 313 return -ENOENT; 314 __radix_tree_replace(&mapping->page_tree, node, pslot, 315 replacement, NULL, NULL); 316 return 0; 317 } 318 319 /* 320 * Sometimes, before we decide whether to proceed or to fail, we must check 321 * that an entry was not already brought back from swap by a racing thread. 322 * 323 * Checking page is not enough: by the time a SwapCache page is locked, it 324 * might be reused, and again be SwapCache, using the same swap as before. 325 */ 326 static bool shmem_confirm_swap(struct address_space *mapping, 327 pgoff_t index, swp_entry_t swap) 328 { 329 void *item; 330 331 rcu_read_lock(); 332 item = radix_tree_lookup(&mapping->page_tree, index); 333 rcu_read_unlock(); 334 return item == swp_to_radix_entry(swap); 335 } 336 337 /* 338 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 339 * 340 * SHMEM_HUGE_NEVER: 341 * disables huge pages for the mount; 342 * SHMEM_HUGE_ALWAYS: 343 * enables huge pages for the mount; 344 * SHMEM_HUGE_WITHIN_SIZE: 345 * only allocate huge pages if the page will be fully within i_size, 346 * also respect fadvise()/madvise() hints; 347 * SHMEM_HUGE_ADVISE: 348 * only allocate huge pages if requested with fadvise()/madvise(); 349 */ 350 351 #define SHMEM_HUGE_NEVER 0 352 #define SHMEM_HUGE_ALWAYS 1 353 #define SHMEM_HUGE_WITHIN_SIZE 2 354 #define SHMEM_HUGE_ADVISE 3 355 356 /* 357 * Special values. 358 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 359 * 360 * SHMEM_HUGE_DENY: 361 * disables huge on shm_mnt and all mounts, for emergency use; 362 * SHMEM_HUGE_FORCE: 363 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 364 * 365 */ 366 #define SHMEM_HUGE_DENY (-1) 367 #define SHMEM_HUGE_FORCE (-2) 368 369 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 370 /* ifdef here to avoid bloating shmem.o when not necessary */ 371 372 int shmem_huge __read_mostly; 373 374 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 375 static int shmem_parse_huge(const char *str) 376 { 377 if (!strcmp(str, "never")) 378 return SHMEM_HUGE_NEVER; 379 if (!strcmp(str, "always")) 380 return SHMEM_HUGE_ALWAYS; 381 if (!strcmp(str, "within_size")) 382 return SHMEM_HUGE_WITHIN_SIZE; 383 if (!strcmp(str, "advise")) 384 return SHMEM_HUGE_ADVISE; 385 if (!strcmp(str, "deny")) 386 return SHMEM_HUGE_DENY; 387 if (!strcmp(str, "force")) 388 return SHMEM_HUGE_FORCE; 389 return -EINVAL; 390 } 391 392 static const char *shmem_format_huge(int huge) 393 { 394 switch (huge) { 395 case SHMEM_HUGE_NEVER: 396 return "never"; 397 case SHMEM_HUGE_ALWAYS: 398 return "always"; 399 case SHMEM_HUGE_WITHIN_SIZE: 400 return "within_size"; 401 case SHMEM_HUGE_ADVISE: 402 return "advise"; 403 case SHMEM_HUGE_DENY: 404 return "deny"; 405 case SHMEM_HUGE_FORCE: 406 return "force"; 407 default: 408 VM_BUG_ON(1); 409 return "bad_val"; 410 } 411 } 412 #endif 413 414 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 415 struct shrink_control *sc, unsigned long nr_to_split) 416 { 417 LIST_HEAD(list), *pos, *next; 418 LIST_HEAD(to_remove); 419 struct inode *inode; 420 struct shmem_inode_info *info; 421 struct page *page; 422 unsigned long batch = sc ? sc->nr_to_scan : 128; 423 int removed = 0, split = 0; 424 425 if (list_empty(&sbinfo->shrinklist)) 426 return SHRINK_STOP; 427 428 spin_lock(&sbinfo->shrinklist_lock); 429 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 430 info = list_entry(pos, struct shmem_inode_info, shrinklist); 431 432 /* pin the inode */ 433 inode = igrab(&info->vfs_inode); 434 435 /* inode is about to be evicted */ 436 if (!inode) { 437 list_del_init(&info->shrinklist); 438 removed++; 439 goto next; 440 } 441 442 /* Check if there's anything to gain */ 443 if (round_up(inode->i_size, PAGE_SIZE) == 444 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 445 list_move(&info->shrinklist, &to_remove); 446 removed++; 447 goto next; 448 } 449 450 list_move(&info->shrinklist, &list); 451 next: 452 if (!--batch) 453 break; 454 } 455 spin_unlock(&sbinfo->shrinklist_lock); 456 457 list_for_each_safe(pos, next, &to_remove) { 458 info = list_entry(pos, struct shmem_inode_info, shrinklist); 459 inode = &info->vfs_inode; 460 list_del_init(&info->shrinklist); 461 iput(inode); 462 } 463 464 list_for_each_safe(pos, next, &list) { 465 int ret; 466 467 info = list_entry(pos, struct shmem_inode_info, shrinklist); 468 inode = &info->vfs_inode; 469 470 if (nr_to_split && split >= nr_to_split) { 471 iput(inode); 472 continue; 473 } 474 475 page = find_lock_page(inode->i_mapping, 476 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 477 if (!page) 478 goto drop; 479 480 if (!PageTransHuge(page)) { 481 unlock_page(page); 482 put_page(page); 483 goto drop; 484 } 485 486 ret = split_huge_page(page); 487 unlock_page(page); 488 put_page(page); 489 490 if (ret) { 491 /* split failed: leave it on the list */ 492 iput(inode); 493 continue; 494 } 495 496 split++; 497 drop: 498 list_del_init(&info->shrinklist); 499 removed++; 500 iput(inode); 501 } 502 503 spin_lock(&sbinfo->shrinklist_lock); 504 list_splice_tail(&list, &sbinfo->shrinklist); 505 sbinfo->shrinklist_len -= removed; 506 spin_unlock(&sbinfo->shrinklist_lock); 507 508 return split; 509 } 510 511 static long shmem_unused_huge_scan(struct super_block *sb, 512 struct shrink_control *sc) 513 { 514 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 515 516 if (!READ_ONCE(sbinfo->shrinklist_len)) 517 return SHRINK_STOP; 518 519 return shmem_unused_huge_shrink(sbinfo, sc, 0); 520 } 521 522 static long shmem_unused_huge_count(struct super_block *sb, 523 struct shrink_control *sc) 524 { 525 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 526 return READ_ONCE(sbinfo->shrinklist_len); 527 } 528 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 529 530 #define shmem_huge SHMEM_HUGE_DENY 531 532 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 533 struct shrink_control *sc, unsigned long nr_to_split) 534 { 535 return 0; 536 } 537 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 538 539 /* 540 * Like add_to_page_cache_locked, but error if expected item has gone. 541 */ 542 static int shmem_add_to_page_cache(struct page *page, 543 struct address_space *mapping, 544 pgoff_t index, void *expected) 545 { 546 int error, nr = hpage_nr_pages(page); 547 548 VM_BUG_ON_PAGE(PageTail(page), page); 549 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 550 VM_BUG_ON_PAGE(!PageLocked(page), page); 551 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 552 VM_BUG_ON(expected && PageTransHuge(page)); 553 554 page_ref_add(page, nr); 555 page->mapping = mapping; 556 page->index = index; 557 558 spin_lock_irq(&mapping->tree_lock); 559 if (PageTransHuge(page)) { 560 void __rcu **results; 561 pgoff_t idx; 562 int i; 563 564 error = 0; 565 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 566 &results, &idx, index, 1) && 567 idx < index + HPAGE_PMD_NR) { 568 error = -EEXIST; 569 } 570 571 if (!error) { 572 for (i = 0; i < HPAGE_PMD_NR; i++) { 573 error = radix_tree_insert(&mapping->page_tree, 574 index + i, page + i); 575 VM_BUG_ON(error); 576 } 577 count_vm_event(THP_FILE_ALLOC); 578 } 579 } else if (!expected) { 580 error = radix_tree_insert(&mapping->page_tree, index, page); 581 } else { 582 error = shmem_radix_tree_replace(mapping, index, expected, 583 page); 584 } 585 586 if (!error) { 587 mapping->nrpages += nr; 588 if (PageTransHuge(page)) 589 __inc_node_page_state(page, NR_SHMEM_THPS); 590 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 591 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 592 spin_unlock_irq(&mapping->tree_lock); 593 } else { 594 page->mapping = NULL; 595 spin_unlock_irq(&mapping->tree_lock); 596 page_ref_sub(page, nr); 597 } 598 return error; 599 } 600 601 /* 602 * Like delete_from_page_cache, but substitutes swap for page. 603 */ 604 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 605 { 606 struct address_space *mapping = page->mapping; 607 int error; 608 609 VM_BUG_ON_PAGE(PageCompound(page), page); 610 611 spin_lock_irq(&mapping->tree_lock); 612 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 613 page->mapping = NULL; 614 mapping->nrpages--; 615 __dec_node_page_state(page, NR_FILE_PAGES); 616 __dec_node_page_state(page, NR_SHMEM); 617 spin_unlock_irq(&mapping->tree_lock); 618 put_page(page); 619 BUG_ON(error); 620 } 621 622 /* 623 * Remove swap entry from radix tree, free the swap and its page cache. 624 */ 625 static int shmem_free_swap(struct address_space *mapping, 626 pgoff_t index, void *radswap) 627 { 628 void *old; 629 630 spin_lock_irq(&mapping->tree_lock); 631 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 632 spin_unlock_irq(&mapping->tree_lock); 633 if (old != radswap) 634 return -ENOENT; 635 free_swap_and_cache(radix_to_swp_entry(radswap)); 636 return 0; 637 } 638 639 /* 640 * Determine (in bytes) how many of the shmem object's pages mapped by the 641 * given offsets are swapped out. 642 * 643 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 644 * as long as the inode doesn't go away and racy results are not a problem. 645 */ 646 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 647 pgoff_t start, pgoff_t end) 648 { 649 struct radix_tree_iter iter; 650 void **slot; 651 struct page *page; 652 unsigned long swapped = 0; 653 654 rcu_read_lock(); 655 656 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 657 if (iter.index >= end) 658 break; 659 660 page = radix_tree_deref_slot(slot); 661 662 if (radix_tree_deref_retry(page)) { 663 slot = radix_tree_iter_retry(&iter); 664 continue; 665 } 666 667 if (radix_tree_exceptional_entry(page)) 668 swapped++; 669 670 if (need_resched()) { 671 slot = radix_tree_iter_resume(slot, &iter); 672 cond_resched_rcu(); 673 } 674 } 675 676 rcu_read_unlock(); 677 678 return swapped << PAGE_SHIFT; 679 } 680 681 /* 682 * Determine (in bytes) how many of the shmem object's pages mapped by the 683 * given vma is swapped out. 684 * 685 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 686 * as long as the inode doesn't go away and racy results are not a problem. 687 */ 688 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 689 { 690 struct inode *inode = file_inode(vma->vm_file); 691 struct shmem_inode_info *info = SHMEM_I(inode); 692 struct address_space *mapping = inode->i_mapping; 693 unsigned long swapped; 694 695 /* Be careful as we don't hold info->lock */ 696 swapped = READ_ONCE(info->swapped); 697 698 /* 699 * The easier cases are when the shmem object has nothing in swap, or 700 * the vma maps it whole. Then we can simply use the stats that we 701 * already track. 702 */ 703 if (!swapped) 704 return 0; 705 706 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 707 return swapped << PAGE_SHIFT; 708 709 /* Here comes the more involved part */ 710 return shmem_partial_swap_usage(mapping, 711 linear_page_index(vma, vma->vm_start), 712 linear_page_index(vma, vma->vm_end)); 713 } 714 715 /* 716 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 717 */ 718 void shmem_unlock_mapping(struct address_space *mapping) 719 { 720 struct pagevec pvec; 721 pgoff_t indices[PAGEVEC_SIZE]; 722 pgoff_t index = 0; 723 724 pagevec_init(&pvec, 0); 725 /* 726 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 727 */ 728 while (!mapping_unevictable(mapping)) { 729 /* 730 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it 731 * has finished, if it hits a row of PAGEVEC_SIZE swap entries. 732 */ 733 pvec.nr = find_get_entries(mapping, index, 734 PAGEVEC_SIZE, pvec.pages, indices); 735 if (!pvec.nr) 736 break; 737 index = indices[pvec.nr - 1] + 1; 738 pagevec_remove_exceptionals(&pvec); 739 check_move_unevictable_pages(pvec.pages, pvec.nr); 740 pagevec_release(&pvec); 741 cond_resched(); 742 } 743 } 744 745 /* 746 * Remove range of pages and swap entries from radix tree, and free them. 747 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 748 */ 749 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 750 bool unfalloc) 751 { 752 struct address_space *mapping = inode->i_mapping; 753 struct shmem_inode_info *info = SHMEM_I(inode); 754 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 755 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 756 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 757 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 758 struct pagevec pvec; 759 pgoff_t indices[PAGEVEC_SIZE]; 760 long nr_swaps_freed = 0; 761 pgoff_t index; 762 int i; 763 764 if (lend == -1) 765 end = -1; /* unsigned, so actually very big */ 766 767 pagevec_init(&pvec, 0); 768 index = start; 769 while (index < end) { 770 pvec.nr = find_get_entries(mapping, index, 771 min(end - index, (pgoff_t)PAGEVEC_SIZE), 772 pvec.pages, indices); 773 if (!pvec.nr) 774 break; 775 for (i = 0; i < pagevec_count(&pvec); i++) { 776 struct page *page = pvec.pages[i]; 777 778 index = indices[i]; 779 if (index >= end) 780 break; 781 782 if (radix_tree_exceptional_entry(page)) { 783 if (unfalloc) 784 continue; 785 nr_swaps_freed += !shmem_free_swap(mapping, 786 index, page); 787 continue; 788 } 789 790 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); 791 792 if (!trylock_page(page)) 793 continue; 794 795 if (PageTransTail(page)) { 796 /* Middle of THP: zero out the page */ 797 clear_highpage(page); 798 unlock_page(page); 799 continue; 800 } else if (PageTransHuge(page)) { 801 if (index == round_down(end, HPAGE_PMD_NR)) { 802 /* 803 * Range ends in the middle of THP: 804 * zero out the page 805 */ 806 clear_highpage(page); 807 unlock_page(page); 808 continue; 809 } 810 index += HPAGE_PMD_NR - 1; 811 i += HPAGE_PMD_NR - 1; 812 } 813 814 if (!unfalloc || !PageUptodate(page)) { 815 VM_BUG_ON_PAGE(PageTail(page), page); 816 if (page_mapping(page) == mapping) { 817 VM_BUG_ON_PAGE(PageWriteback(page), page); 818 truncate_inode_page(mapping, page); 819 } 820 } 821 unlock_page(page); 822 } 823 pagevec_remove_exceptionals(&pvec); 824 pagevec_release(&pvec); 825 cond_resched(); 826 index++; 827 } 828 829 if (partial_start) { 830 struct page *page = NULL; 831 shmem_getpage(inode, start - 1, &page, SGP_READ); 832 if (page) { 833 unsigned int top = PAGE_SIZE; 834 if (start > end) { 835 top = partial_end; 836 partial_end = 0; 837 } 838 zero_user_segment(page, partial_start, top); 839 set_page_dirty(page); 840 unlock_page(page); 841 put_page(page); 842 } 843 } 844 if (partial_end) { 845 struct page *page = NULL; 846 shmem_getpage(inode, end, &page, SGP_READ); 847 if (page) { 848 zero_user_segment(page, 0, partial_end); 849 set_page_dirty(page); 850 unlock_page(page); 851 put_page(page); 852 } 853 } 854 if (start >= end) 855 return; 856 857 index = start; 858 while (index < end) { 859 cond_resched(); 860 861 pvec.nr = find_get_entries(mapping, index, 862 min(end - index, (pgoff_t)PAGEVEC_SIZE), 863 pvec.pages, indices); 864 if (!pvec.nr) { 865 /* If all gone or hole-punch or unfalloc, we're done */ 866 if (index == start || end != -1) 867 break; 868 /* But if truncating, restart to make sure all gone */ 869 index = start; 870 continue; 871 } 872 for (i = 0; i < pagevec_count(&pvec); i++) { 873 struct page *page = pvec.pages[i]; 874 875 index = indices[i]; 876 if (index >= end) 877 break; 878 879 if (radix_tree_exceptional_entry(page)) { 880 if (unfalloc) 881 continue; 882 if (shmem_free_swap(mapping, index, page)) { 883 /* Swap was replaced by page: retry */ 884 index--; 885 break; 886 } 887 nr_swaps_freed++; 888 continue; 889 } 890 891 lock_page(page); 892 893 if (PageTransTail(page)) { 894 /* Middle of THP: zero out the page */ 895 clear_highpage(page); 896 unlock_page(page); 897 /* 898 * Partial thp truncate due 'start' in middle 899 * of THP: don't need to look on these pages 900 * again on !pvec.nr restart. 901 */ 902 if (index != round_down(end, HPAGE_PMD_NR)) 903 start++; 904 continue; 905 } else if (PageTransHuge(page)) { 906 if (index == round_down(end, HPAGE_PMD_NR)) { 907 /* 908 * Range ends in the middle of THP: 909 * zero out the page 910 */ 911 clear_highpage(page); 912 unlock_page(page); 913 continue; 914 } 915 index += HPAGE_PMD_NR - 1; 916 i += HPAGE_PMD_NR - 1; 917 } 918 919 if (!unfalloc || !PageUptodate(page)) { 920 VM_BUG_ON_PAGE(PageTail(page), page); 921 if (page_mapping(page) == mapping) { 922 VM_BUG_ON_PAGE(PageWriteback(page), page); 923 truncate_inode_page(mapping, page); 924 } else { 925 /* Page was replaced by swap: retry */ 926 unlock_page(page); 927 index--; 928 break; 929 } 930 } 931 unlock_page(page); 932 } 933 pagevec_remove_exceptionals(&pvec); 934 pagevec_release(&pvec); 935 index++; 936 } 937 938 spin_lock_irq(&info->lock); 939 info->swapped -= nr_swaps_freed; 940 shmem_recalc_inode(inode); 941 spin_unlock_irq(&info->lock); 942 } 943 944 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 945 { 946 shmem_undo_range(inode, lstart, lend, false); 947 inode->i_ctime = inode->i_mtime = current_time(inode); 948 } 949 EXPORT_SYMBOL_GPL(shmem_truncate_range); 950 951 static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry, 952 struct kstat *stat) 953 { 954 struct inode *inode = dentry->d_inode; 955 struct shmem_inode_info *info = SHMEM_I(inode); 956 957 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 958 spin_lock_irq(&info->lock); 959 shmem_recalc_inode(inode); 960 spin_unlock_irq(&info->lock); 961 } 962 generic_fillattr(inode, stat); 963 return 0; 964 } 965 966 static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 967 { 968 struct inode *inode = d_inode(dentry); 969 struct shmem_inode_info *info = SHMEM_I(inode); 970 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 971 int error; 972 973 error = setattr_prepare(dentry, attr); 974 if (error) 975 return error; 976 977 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 978 loff_t oldsize = inode->i_size; 979 loff_t newsize = attr->ia_size; 980 981 /* protected by i_mutex */ 982 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 983 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 984 return -EPERM; 985 986 if (newsize != oldsize) { 987 error = shmem_reacct_size(SHMEM_I(inode)->flags, 988 oldsize, newsize); 989 if (error) 990 return error; 991 i_size_write(inode, newsize); 992 inode->i_ctime = inode->i_mtime = current_time(inode); 993 } 994 if (newsize <= oldsize) { 995 loff_t holebegin = round_up(newsize, PAGE_SIZE); 996 if (oldsize > holebegin) 997 unmap_mapping_range(inode->i_mapping, 998 holebegin, 0, 1); 999 if (info->alloced) 1000 shmem_truncate_range(inode, 1001 newsize, (loff_t)-1); 1002 /* unmap again to remove racily COWed private pages */ 1003 if (oldsize > holebegin) 1004 unmap_mapping_range(inode->i_mapping, 1005 holebegin, 0, 1); 1006 1007 /* 1008 * Part of the huge page can be beyond i_size: subject 1009 * to shrink under memory pressure. 1010 */ 1011 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 1012 spin_lock(&sbinfo->shrinklist_lock); 1013 if (list_empty(&info->shrinklist)) { 1014 list_add_tail(&info->shrinklist, 1015 &sbinfo->shrinklist); 1016 sbinfo->shrinklist_len++; 1017 } 1018 spin_unlock(&sbinfo->shrinklist_lock); 1019 } 1020 } 1021 } 1022 1023 setattr_copy(inode, attr); 1024 if (attr->ia_valid & ATTR_MODE) 1025 error = posix_acl_chmod(inode, inode->i_mode); 1026 return error; 1027 } 1028 1029 static void shmem_evict_inode(struct inode *inode) 1030 { 1031 struct shmem_inode_info *info = SHMEM_I(inode); 1032 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1033 1034 if (inode->i_mapping->a_ops == &shmem_aops) { 1035 shmem_unacct_size(info->flags, inode->i_size); 1036 inode->i_size = 0; 1037 shmem_truncate_range(inode, 0, (loff_t)-1); 1038 if (!list_empty(&info->shrinklist)) { 1039 spin_lock(&sbinfo->shrinklist_lock); 1040 if (!list_empty(&info->shrinklist)) { 1041 list_del_init(&info->shrinklist); 1042 sbinfo->shrinklist_len--; 1043 } 1044 spin_unlock(&sbinfo->shrinklist_lock); 1045 } 1046 if (!list_empty(&info->swaplist)) { 1047 mutex_lock(&shmem_swaplist_mutex); 1048 list_del_init(&info->swaplist); 1049 mutex_unlock(&shmem_swaplist_mutex); 1050 } 1051 } 1052 1053 simple_xattrs_free(&info->xattrs); 1054 WARN_ON(inode->i_blocks); 1055 shmem_free_inode(inode->i_sb); 1056 clear_inode(inode); 1057 } 1058 1059 static unsigned long find_swap_entry(struct radix_tree_root *root, void *item) 1060 { 1061 struct radix_tree_iter iter; 1062 void **slot; 1063 unsigned long found = -1; 1064 unsigned int checked = 0; 1065 1066 rcu_read_lock(); 1067 radix_tree_for_each_slot(slot, root, &iter, 0) { 1068 if (*slot == item) { 1069 found = iter.index; 1070 break; 1071 } 1072 checked++; 1073 if ((checked % 4096) != 0) 1074 continue; 1075 slot = radix_tree_iter_resume(slot, &iter); 1076 cond_resched_rcu(); 1077 } 1078 1079 rcu_read_unlock(); 1080 return found; 1081 } 1082 1083 /* 1084 * If swap found in inode, free it and move page from swapcache to filecache. 1085 */ 1086 static int shmem_unuse_inode(struct shmem_inode_info *info, 1087 swp_entry_t swap, struct page **pagep) 1088 { 1089 struct address_space *mapping = info->vfs_inode.i_mapping; 1090 void *radswap; 1091 pgoff_t index; 1092 gfp_t gfp; 1093 int error = 0; 1094 1095 radswap = swp_to_radix_entry(swap); 1096 index = find_swap_entry(&mapping->page_tree, radswap); 1097 if (index == -1) 1098 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1099 1100 /* 1101 * Move _head_ to start search for next from here. 1102 * But be careful: shmem_evict_inode checks list_empty without taking 1103 * mutex, and there's an instant in list_move_tail when info->swaplist 1104 * would appear empty, if it were the only one on shmem_swaplist. 1105 */ 1106 if (shmem_swaplist.next != &info->swaplist) 1107 list_move_tail(&shmem_swaplist, &info->swaplist); 1108 1109 gfp = mapping_gfp_mask(mapping); 1110 if (shmem_should_replace_page(*pagep, gfp)) { 1111 mutex_unlock(&shmem_swaplist_mutex); 1112 error = shmem_replace_page(pagep, gfp, info, index); 1113 mutex_lock(&shmem_swaplist_mutex); 1114 /* 1115 * We needed to drop mutex to make that restrictive page 1116 * allocation, but the inode might have been freed while we 1117 * dropped it: although a racing shmem_evict_inode() cannot 1118 * complete without emptying the radix_tree, our page lock 1119 * on this swapcache page is not enough to prevent that - 1120 * free_swap_and_cache() of our swap entry will only 1121 * trylock_page(), removing swap from radix_tree whatever. 1122 * 1123 * We must not proceed to shmem_add_to_page_cache() if the 1124 * inode has been freed, but of course we cannot rely on 1125 * inode or mapping or info to check that. However, we can 1126 * safely check if our swap entry is still in use (and here 1127 * it can't have got reused for another page): if it's still 1128 * in use, then the inode cannot have been freed yet, and we 1129 * can safely proceed (if it's no longer in use, that tells 1130 * nothing about the inode, but we don't need to unuse swap). 1131 */ 1132 if (!page_swapcount(*pagep)) 1133 error = -ENOENT; 1134 } 1135 1136 /* 1137 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 1138 * but also to hold up shmem_evict_inode(): so inode cannot be freed 1139 * beneath us (pagelock doesn't help until the page is in pagecache). 1140 */ 1141 if (!error) 1142 error = shmem_add_to_page_cache(*pagep, mapping, index, 1143 radswap); 1144 if (error != -ENOMEM) { 1145 /* 1146 * Truncation and eviction use free_swap_and_cache(), which 1147 * only does trylock page: if we raced, best clean up here. 1148 */ 1149 delete_from_swap_cache(*pagep); 1150 set_page_dirty(*pagep); 1151 if (!error) { 1152 spin_lock_irq(&info->lock); 1153 info->swapped--; 1154 spin_unlock_irq(&info->lock); 1155 swap_free(swap); 1156 } 1157 } 1158 return error; 1159 } 1160 1161 /* 1162 * Search through swapped inodes to find and replace swap by page. 1163 */ 1164 int shmem_unuse(swp_entry_t swap, struct page *page) 1165 { 1166 struct list_head *this, *next; 1167 struct shmem_inode_info *info; 1168 struct mem_cgroup *memcg; 1169 int error = 0; 1170 1171 /* 1172 * There's a faint possibility that swap page was replaced before 1173 * caller locked it: caller will come back later with the right page. 1174 */ 1175 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) 1176 goto out; 1177 1178 /* 1179 * Charge page using GFP_KERNEL while we can wait, before taking 1180 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1181 * Charged back to the user (not to caller) when swap account is used. 1182 */ 1183 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg, 1184 false); 1185 if (error) 1186 goto out; 1187 /* No radix_tree_preload: swap entry keeps a place for page in tree */ 1188 error = -EAGAIN; 1189 1190 mutex_lock(&shmem_swaplist_mutex); 1191 list_for_each_safe(this, next, &shmem_swaplist) { 1192 info = list_entry(this, struct shmem_inode_info, swaplist); 1193 if (info->swapped) 1194 error = shmem_unuse_inode(info, swap, &page); 1195 else 1196 list_del_init(&info->swaplist); 1197 cond_resched(); 1198 if (error != -EAGAIN) 1199 break; 1200 /* found nothing in this: move on to search the next */ 1201 } 1202 mutex_unlock(&shmem_swaplist_mutex); 1203 1204 if (error) { 1205 if (error != -ENOMEM) 1206 error = 0; 1207 mem_cgroup_cancel_charge(page, memcg, false); 1208 } else 1209 mem_cgroup_commit_charge(page, memcg, true, false); 1210 out: 1211 unlock_page(page); 1212 put_page(page); 1213 return error; 1214 } 1215 1216 /* 1217 * Move the page from the page cache to the swap cache. 1218 */ 1219 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1220 { 1221 struct shmem_inode_info *info; 1222 struct address_space *mapping; 1223 struct inode *inode; 1224 swp_entry_t swap; 1225 pgoff_t index; 1226 1227 VM_BUG_ON_PAGE(PageCompound(page), page); 1228 BUG_ON(!PageLocked(page)); 1229 mapping = page->mapping; 1230 index = page->index; 1231 inode = mapping->host; 1232 info = SHMEM_I(inode); 1233 if (info->flags & VM_LOCKED) 1234 goto redirty; 1235 if (!total_swap_pages) 1236 goto redirty; 1237 1238 /* 1239 * Our capabilities prevent regular writeback or sync from ever calling 1240 * shmem_writepage; but a stacking filesystem might use ->writepage of 1241 * its underlying filesystem, in which case tmpfs should write out to 1242 * swap only in response to memory pressure, and not for the writeback 1243 * threads or sync. 1244 */ 1245 if (!wbc->for_reclaim) { 1246 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1247 goto redirty; 1248 } 1249 1250 /* 1251 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1252 * value into swapfile.c, the only way we can correctly account for a 1253 * fallocated page arriving here is now to initialize it and write it. 1254 * 1255 * That's okay for a page already fallocated earlier, but if we have 1256 * not yet completed the fallocation, then (a) we want to keep track 1257 * of this page in case we have to undo it, and (b) it may not be a 1258 * good idea to continue anyway, once we're pushing into swap. So 1259 * reactivate the page, and let shmem_fallocate() quit when too many. 1260 */ 1261 if (!PageUptodate(page)) { 1262 if (inode->i_private) { 1263 struct shmem_falloc *shmem_falloc; 1264 spin_lock(&inode->i_lock); 1265 shmem_falloc = inode->i_private; 1266 if (shmem_falloc && 1267 !shmem_falloc->waitq && 1268 index >= shmem_falloc->start && 1269 index < shmem_falloc->next) 1270 shmem_falloc->nr_unswapped++; 1271 else 1272 shmem_falloc = NULL; 1273 spin_unlock(&inode->i_lock); 1274 if (shmem_falloc) 1275 goto redirty; 1276 } 1277 clear_highpage(page); 1278 flush_dcache_page(page); 1279 SetPageUptodate(page); 1280 } 1281 1282 swap = get_swap_page(); 1283 if (!swap.val) 1284 goto redirty; 1285 1286 if (mem_cgroup_try_charge_swap(page, swap)) 1287 goto free_swap; 1288 1289 /* 1290 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1291 * if it's not already there. Do it now before the page is 1292 * moved to swap cache, when its pagelock no longer protects 1293 * the inode from eviction. But don't unlock the mutex until 1294 * we've incremented swapped, because shmem_unuse_inode() will 1295 * prune a !swapped inode from the swaplist under this mutex. 1296 */ 1297 mutex_lock(&shmem_swaplist_mutex); 1298 if (list_empty(&info->swaplist)) 1299 list_add_tail(&info->swaplist, &shmem_swaplist); 1300 1301 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1302 spin_lock_irq(&info->lock); 1303 shmem_recalc_inode(inode); 1304 info->swapped++; 1305 spin_unlock_irq(&info->lock); 1306 1307 swap_shmem_alloc(swap); 1308 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1309 1310 mutex_unlock(&shmem_swaplist_mutex); 1311 BUG_ON(page_mapped(page)); 1312 swap_writepage(page, wbc); 1313 return 0; 1314 } 1315 1316 mutex_unlock(&shmem_swaplist_mutex); 1317 free_swap: 1318 swapcache_free(swap); 1319 redirty: 1320 set_page_dirty(page); 1321 if (wbc->for_reclaim) 1322 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1323 unlock_page(page); 1324 return 0; 1325 } 1326 1327 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1328 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1329 { 1330 char buffer[64]; 1331 1332 if (!mpol || mpol->mode == MPOL_DEFAULT) 1333 return; /* show nothing */ 1334 1335 mpol_to_str(buffer, sizeof(buffer), mpol); 1336 1337 seq_printf(seq, ",mpol=%s", buffer); 1338 } 1339 1340 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1341 { 1342 struct mempolicy *mpol = NULL; 1343 if (sbinfo->mpol) { 1344 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1345 mpol = sbinfo->mpol; 1346 mpol_get(mpol); 1347 spin_unlock(&sbinfo->stat_lock); 1348 } 1349 return mpol; 1350 } 1351 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1352 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1353 { 1354 } 1355 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1356 { 1357 return NULL; 1358 } 1359 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1360 #ifndef CONFIG_NUMA 1361 #define vm_policy vm_private_data 1362 #endif 1363 1364 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1365 struct shmem_inode_info *info, pgoff_t index) 1366 { 1367 /* Create a pseudo vma that just contains the policy */ 1368 vma->vm_start = 0; 1369 /* Bias interleave by inode number to distribute better across nodes */ 1370 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1371 vma->vm_ops = NULL; 1372 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1373 } 1374 1375 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1376 { 1377 /* Drop reference taken by mpol_shared_policy_lookup() */ 1378 mpol_cond_put(vma->vm_policy); 1379 } 1380 1381 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1382 struct shmem_inode_info *info, pgoff_t index) 1383 { 1384 struct vm_area_struct pvma; 1385 struct page *page; 1386 1387 shmem_pseudo_vma_init(&pvma, info, index); 1388 page = swapin_readahead(swap, gfp, &pvma, 0); 1389 shmem_pseudo_vma_destroy(&pvma); 1390 1391 return page; 1392 } 1393 1394 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1395 struct shmem_inode_info *info, pgoff_t index) 1396 { 1397 struct vm_area_struct pvma; 1398 struct inode *inode = &info->vfs_inode; 1399 struct address_space *mapping = inode->i_mapping; 1400 pgoff_t idx, hindex; 1401 void __rcu **results; 1402 struct page *page; 1403 1404 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1405 return NULL; 1406 1407 hindex = round_down(index, HPAGE_PMD_NR); 1408 rcu_read_lock(); 1409 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1410 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1411 rcu_read_unlock(); 1412 return NULL; 1413 } 1414 rcu_read_unlock(); 1415 1416 shmem_pseudo_vma_init(&pvma, info, hindex); 1417 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, 1418 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); 1419 shmem_pseudo_vma_destroy(&pvma); 1420 if (page) 1421 prep_transhuge_page(page); 1422 return page; 1423 } 1424 1425 static struct page *shmem_alloc_page(gfp_t gfp, 1426 struct shmem_inode_info *info, pgoff_t index) 1427 { 1428 struct vm_area_struct pvma; 1429 struct page *page; 1430 1431 shmem_pseudo_vma_init(&pvma, info, index); 1432 page = alloc_page_vma(gfp, &pvma, 0); 1433 shmem_pseudo_vma_destroy(&pvma); 1434 1435 return page; 1436 } 1437 1438 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1439 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo, 1440 pgoff_t index, bool huge) 1441 { 1442 struct page *page; 1443 int nr; 1444 int err = -ENOSPC; 1445 1446 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1447 huge = false; 1448 nr = huge ? HPAGE_PMD_NR : 1; 1449 1450 if (shmem_acct_block(info->flags, nr)) 1451 goto failed; 1452 if (sbinfo->max_blocks) { 1453 if (percpu_counter_compare(&sbinfo->used_blocks, 1454 sbinfo->max_blocks - nr) > 0) 1455 goto unacct; 1456 percpu_counter_add(&sbinfo->used_blocks, nr); 1457 } 1458 1459 if (huge) 1460 page = shmem_alloc_hugepage(gfp, info, index); 1461 else 1462 page = shmem_alloc_page(gfp, info, index); 1463 if (page) { 1464 __SetPageLocked(page); 1465 __SetPageSwapBacked(page); 1466 return page; 1467 } 1468 1469 err = -ENOMEM; 1470 if (sbinfo->max_blocks) 1471 percpu_counter_add(&sbinfo->used_blocks, -nr); 1472 unacct: 1473 shmem_unacct_blocks(info->flags, nr); 1474 failed: 1475 return ERR_PTR(err); 1476 } 1477 1478 /* 1479 * When a page is moved from swapcache to shmem filecache (either by the 1480 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1481 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1482 * ignorance of the mapping it belongs to. If that mapping has special 1483 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1484 * we may need to copy to a suitable page before moving to filecache. 1485 * 1486 * In a future release, this may well be extended to respect cpuset and 1487 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1488 * but for now it is a simple matter of zone. 1489 */ 1490 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1491 { 1492 return page_zonenum(page) > gfp_zone(gfp); 1493 } 1494 1495 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1496 struct shmem_inode_info *info, pgoff_t index) 1497 { 1498 struct page *oldpage, *newpage; 1499 struct address_space *swap_mapping; 1500 pgoff_t swap_index; 1501 int error; 1502 1503 oldpage = *pagep; 1504 swap_index = page_private(oldpage); 1505 swap_mapping = page_mapping(oldpage); 1506 1507 /* 1508 * We have arrived here because our zones are constrained, so don't 1509 * limit chance of success by further cpuset and node constraints. 1510 */ 1511 gfp &= ~GFP_CONSTRAINT_MASK; 1512 newpage = shmem_alloc_page(gfp, info, index); 1513 if (!newpage) 1514 return -ENOMEM; 1515 1516 get_page(newpage); 1517 copy_highpage(newpage, oldpage); 1518 flush_dcache_page(newpage); 1519 1520 __SetPageLocked(newpage); 1521 __SetPageSwapBacked(newpage); 1522 SetPageUptodate(newpage); 1523 set_page_private(newpage, swap_index); 1524 SetPageSwapCache(newpage); 1525 1526 /* 1527 * Our caller will very soon move newpage out of swapcache, but it's 1528 * a nice clean interface for us to replace oldpage by newpage there. 1529 */ 1530 spin_lock_irq(&swap_mapping->tree_lock); 1531 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1532 newpage); 1533 if (!error) { 1534 __inc_node_page_state(newpage, NR_FILE_PAGES); 1535 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1536 } 1537 spin_unlock_irq(&swap_mapping->tree_lock); 1538 1539 if (unlikely(error)) { 1540 /* 1541 * Is this possible? I think not, now that our callers check 1542 * both PageSwapCache and page_private after getting page lock; 1543 * but be defensive. Reverse old to newpage for clear and free. 1544 */ 1545 oldpage = newpage; 1546 } else { 1547 mem_cgroup_migrate(oldpage, newpage); 1548 lru_cache_add_anon(newpage); 1549 *pagep = newpage; 1550 } 1551 1552 ClearPageSwapCache(oldpage); 1553 set_page_private(oldpage, 0); 1554 1555 unlock_page(oldpage); 1556 put_page(oldpage); 1557 put_page(oldpage); 1558 return error; 1559 } 1560 1561 /* 1562 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1563 * 1564 * If we allocate a new one we do not mark it dirty. That's up to the 1565 * vm. If we swap it in we mark it dirty since we also free the swap 1566 * entry since a page cannot live in both the swap and page cache. 1567 * 1568 * fault_mm and fault_type are only supplied by shmem_fault: 1569 * otherwise they are NULL. 1570 */ 1571 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1572 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1573 struct mm_struct *fault_mm, int *fault_type) 1574 { 1575 struct address_space *mapping = inode->i_mapping; 1576 struct shmem_inode_info *info = SHMEM_I(inode); 1577 struct shmem_sb_info *sbinfo; 1578 struct mm_struct *charge_mm; 1579 struct mem_cgroup *memcg; 1580 struct page *page; 1581 swp_entry_t swap; 1582 enum sgp_type sgp_huge = sgp; 1583 pgoff_t hindex = index; 1584 int error; 1585 int once = 0; 1586 int alloced = 0; 1587 1588 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1589 return -EFBIG; 1590 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) 1591 sgp = SGP_CACHE; 1592 repeat: 1593 swap.val = 0; 1594 page = find_lock_entry(mapping, index); 1595 if (radix_tree_exceptional_entry(page)) { 1596 swap = radix_to_swp_entry(page); 1597 page = NULL; 1598 } 1599 1600 if (sgp <= SGP_CACHE && 1601 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1602 error = -EINVAL; 1603 goto unlock; 1604 } 1605 1606 if (page && sgp == SGP_WRITE) 1607 mark_page_accessed(page); 1608 1609 /* fallocated page? */ 1610 if (page && !PageUptodate(page)) { 1611 if (sgp != SGP_READ) 1612 goto clear; 1613 unlock_page(page); 1614 put_page(page); 1615 page = NULL; 1616 } 1617 if (page || (sgp == SGP_READ && !swap.val)) { 1618 *pagep = page; 1619 return 0; 1620 } 1621 1622 /* 1623 * Fast cache lookup did not find it: 1624 * bring it back from swap or allocate. 1625 */ 1626 sbinfo = SHMEM_SB(inode->i_sb); 1627 charge_mm = fault_mm ? : current->mm; 1628 1629 if (swap.val) { 1630 /* Look it up and read it in.. */ 1631 page = lookup_swap_cache(swap); 1632 if (!page) { 1633 /* Or update major stats only when swapin succeeds?? */ 1634 if (fault_type) { 1635 *fault_type |= VM_FAULT_MAJOR; 1636 count_vm_event(PGMAJFAULT); 1637 mem_cgroup_count_vm_event(fault_mm, PGMAJFAULT); 1638 } 1639 /* Here we actually start the io */ 1640 page = shmem_swapin(swap, gfp, info, index); 1641 if (!page) { 1642 error = -ENOMEM; 1643 goto failed; 1644 } 1645 } 1646 1647 /* We have to do this with page locked to prevent races */ 1648 lock_page(page); 1649 if (!PageSwapCache(page) || page_private(page) != swap.val || 1650 !shmem_confirm_swap(mapping, index, swap)) { 1651 error = -EEXIST; /* try again */ 1652 goto unlock; 1653 } 1654 if (!PageUptodate(page)) { 1655 error = -EIO; 1656 goto failed; 1657 } 1658 wait_on_page_writeback(page); 1659 1660 if (shmem_should_replace_page(page, gfp)) { 1661 error = shmem_replace_page(&page, gfp, info, index); 1662 if (error) 1663 goto failed; 1664 } 1665 1666 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1667 false); 1668 if (!error) { 1669 error = shmem_add_to_page_cache(page, mapping, index, 1670 swp_to_radix_entry(swap)); 1671 /* 1672 * We already confirmed swap under page lock, and make 1673 * no memory allocation here, so usually no possibility 1674 * of error; but free_swap_and_cache() only trylocks a 1675 * page, so it is just possible that the entry has been 1676 * truncated or holepunched since swap was confirmed. 1677 * shmem_undo_range() will have done some of the 1678 * unaccounting, now delete_from_swap_cache() will do 1679 * the rest. 1680 * Reset swap.val? No, leave it so "failed" goes back to 1681 * "repeat": reading a hole and writing should succeed. 1682 */ 1683 if (error) { 1684 mem_cgroup_cancel_charge(page, memcg, false); 1685 delete_from_swap_cache(page); 1686 } 1687 } 1688 if (error) 1689 goto failed; 1690 1691 mem_cgroup_commit_charge(page, memcg, true, false); 1692 1693 spin_lock_irq(&info->lock); 1694 info->swapped--; 1695 shmem_recalc_inode(inode); 1696 spin_unlock_irq(&info->lock); 1697 1698 if (sgp == SGP_WRITE) 1699 mark_page_accessed(page); 1700 1701 delete_from_swap_cache(page); 1702 set_page_dirty(page); 1703 swap_free(swap); 1704 1705 } else { 1706 /* shmem_symlink() */ 1707 if (mapping->a_ops != &shmem_aops) 1708 goto alloc_nohuge; 1709 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) 1710 goto alloc_nohuge; 1711 if (shmem_huge == SHMEM_HUGE_FORCE) 1712 goto alloc_huge; 1713 switch (sbinfo->huge) { 1714 loff_t i_size; 1715 pgoff_t off; 1716 case SHMEM_HUGE_NEVER: 1717 goto alloc_nohuge; 1718 case SHMEM_HUGE_WITHIN_SIZE: 1719 off = round_up(index, HPAGE_PMD_NR); 1720 i_size = round_up(i_size_read(inode), PAGE_SIZE); 1721 if (i_size >= HPAGE_PMD_SIZE && 1722 i_size >> PAGE_SHIFT >= off) 1723 goto alloc_huge; 1724 /* fallthrough */ 1725 case SHMEM_HUGE_ADVISE: 1726 if (sgp_huge == SGP_HUGE) 1727 goto alloc_huge; 1728 /* TODO: implement fadvise() hints */ 1729 goto alloc_nohuge; 1730 } 1731 1732 alloc_huge: 1733 page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1734 index, true); 1735 if (IS_ERR(page)) { 1736 alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1737 index, false); 1738 } 1739 if (IS_ERR(page)) { 1740 int retry = 5; 1741 error = PTR_ERR(page); 1742 page = NULL; 1743 if (error != -ENOSPC) 1744 goto failed; 1745 /* 1746 * Try to reclaim some spece by splitting a huge page 1747 * beyond i_size on the filesystem. 1748 */ 1749 while (retry--) { 1750 int ret; 1751 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1752 if (ret == SHRINK_STOP) 1753 break; 1754 if (ret) 1755 goto alloc_nohuge; 1756 } 1757 goto failed; 1758 } 1759 1760 if (PageTransHuge(page)) 1761 hindex = round_down(index, HPAGE_PMD_NR); 1762 else 1763 hindex = index; 1764 1765 if (sgp == SGP_WRITE) 1766 __SetPageReferenced(page); 1767 1768 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1769 PageTransHuge(page)); 1770 if (error) 1771 goto unacct; 1772 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, 1773 compound_order(page)); 1774 if (!error) { 1775 error = shmem_add_to_page_cache(page, mapping, hindex, 1776 NULL); 1777 radix_tree_preload_end(); 1778 } 1779 if (error) { 1780 mem_cgroup_cancel_charge(page, memcg, 1781 PageTransHuge(page)); 1782 goto unacct; 1783 } 1784 mem_cgroup_commit_charge(page, memcg, false, 1785 PageTransHuge(page)); 1786 lru_cache_add_anon(page); 1787 1788 spin_lock_irq(&info->lock); 1789 info->alloced += 1 << compound_order(page); 1790 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1791 shmem_recalc_inode(inode); 1792 spin_unlock_irq(&info->lock); 1793 alloced = true; 1794 1795 if (PageTransHuge(page) && 1796 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1797 hindex + HPAGE_PMD_NR - 1) { 1798 /* 1799 * Part of the huge page is beyond i_size: subject 1800 * to shrink under memory pressure. 1801 */ 1802 spin_lock(&sbinfo->shrinklist_lock); 1803 if (list_empty(&info->shrinklist)) { 1804 list_add_tail(&info->shrinklist, 1805 &sbinfo->shrinklist); 1806 sbinfo->shrinklist_len++; 1807 } 1808 spin_unlock(&sbinfo->shrinklist_lock); 1809 } 1810 1811 /* 1812 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1813 */ 1814 if (sgp == SGP_FALLOC) 1815 sgp = SGP_WRITE; 1816 clear: 1817 /* 1818 * Let SGP_WRITE caller clear ends if write does not fill page; 1819 * but SGP_FALLOC on a page fallocated earlier must initialize 1820 * it now, lest undo on failure cancel our earlier guarantee. 1821 */ 1822 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1823 struct page *head = compound_head(page); 1824 int i; 1825 1826 for (i = 0; i < (1 << compound_order(head)); i++) { 1827 clear_highpage(head + i); 1828 flush_dcache_page(head + i); 1829 } 1830 SetPageUptodate(head); 1831 } 1832 } 1833 1834 /* Perhaps the file has been truncated since we checked */ 1835 if (sgp <= SGP_CACHE && 1836 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1837 if (alloced) { 1838 ClearPageDirty(page); 1839 delete_from_page_cache(page); 1840 spin_lock_irq(&info->lock); 1841 shmem_recalc_inode(inode); 1842 spin_unlock_irq(&info->lock); 1843 } 1844 error = -EINVAL; 1845 goto unlock; 1846 } 1847 *pagep = page + index - hindex; 1848 return 0; 1849 1850 /* 1851 * Error recovery. 1852 */ 1853 unacct: 1854 if (sbinfo->max_blocks) 1855 percpu_counter_sub(&sbinfo->used_blocks, 1856 1 << compound_order(page)); 1857 shmem_unacct_blocks(info->flags, 1 << compound_order(page)); 1858 1859 if (PageTransHuge(page)) { 1860 unlock_page(page); 1861 put_page(page); 1862 goto alloc_nohuge; 1863 } 1864 failed: 1865 if (swap.val && !shmem_confirm_swap(mapping, index, swap)) 1866 error = -EEXIST; 1867 unlock: 1868 if (page) { 1869 unlock_page(page); 1870 put_page(page); 1871 } 1872 if (error == -ENOSPC && !once++) { 1873 spin_lock_irq(&info->lock); 1874 shmem_recalc_inode(inode); 1875 spin_unlock_irq(&info->lock); 1876 goto repeat; 1877 } 1878 if (error == -EEXIST) /* from above or from radix_tree_insert */ 1879 goto repeat; 1880 return error; 1881 } 1882 1883 /* 1884 * This is like autoremove_wake_function, but it removes the wait queue 1885 * entry unconditionally - even if something else had already woken the 1886 * target. 1887 */ 1888 static int synchronous_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) 1889 { 1890 int ret = default_wake_function(wait, mode, sync, key); 1891 list_del_init(&wait->task_list); 1892 return ret; 1893 } 1894 1895 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1896 { 1897 struct inode *inode = file_inode(vma->vm_file); 1898 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 1899 enum sgp_type sgp; 1900 int error; 1901 int ret = VM_FAULT_LOCKED; 1902 1903 /* 1904 * Trinity finds that probing a hole which tmpfs is punching can 1905 * prevent the hole-punch from ever completing: which in turn 1906 * locks writers out with its hold on i_mutex. So refrain from 1907 * faulting pages into the hole while it's being punched. Although 1908 * shmem_undo_range() does remove the additions, it may be unable to 1909 * keep up, as each new page needs its own unmap_mapping_range() call, 1910 * and the i_mmap tree grows ever slower to scan if new vmas are added. 1911 * 1912 * It does not matter if we sometimes reach this check just before the 1913 * hole-punch begins, so that one fault then races with the punch: 1914 * we just need to make racing faults a rare case. 1915 * 1916 * The implementation below would be much simpler if we just used a 1917 * standard mutex or completion: but we cannot take i_mutex in fault, 1918 * and bloating every shmem inode for this unlikely case would be sad. 1919 */ 1920 if (unlikely(inode->i_private)) { 1921 struct shmem_falloc *shmem_falloc; 1922 1923 spin_lock(&inode->i_lock); 1924 shmem_falloc = inode->i_private; 1925 if (shmem_falloc && 1926 shmem_falloc->waitq && 1927 vmf->pgoff >= shmem_falloc->start && 1928 vmf->pgoff < shmem_falloc->next) { 1929 wait_queue_head_t *shmem_falloc_waitq; 1930 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 1931 1932 ret = VM_FAULT_NOPAGE; 1933 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && 1934 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 1935 /* It's polite to up mmap_sem if we can */ 1936 up_read(&vma->vm_mm->mmap_sem); 1937 ret = VM_FAULT_RETRY; 1938 } 1939 1940 shmem_falloc_waitq = shmem_falloc->waitq; 1941 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 1942 TASK_UNINTERRUPTIBLE); 1943 spin_unlock(&inode->i_lock); 1944 schedule(); 1945 1946 /* 1947 * shmem_falloc_waitq points into the shmem_fallocate() 1948 * stack of the hole-punching task: shmem_falloc_waitq 1949 * is usually invalid by the time we reach here, but 1950 * finish_wait() does not dereference it in that case; 1951 * though i_lock needed lest racing with wake_up_all(). 1952 */ 1953 spin_lock(&inode->i_lock); 1954 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 1955 spin_unlock(&inode->i_lock); 1956 return ret; 1957 } 1958 spin_unlock(&inode->i_lock); 1959 } 1960 1961 sgp = SGP_CACHE; 1962 if (vma->vm_flags & VM_HUGEPAGE) 1963 sgp = SGP_HUGE; 1964 else if (vma->vm_flags & VM_NOHUGEPAGE) 1965 sgp = SGP_NOHUGE; 1966 1967 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, 1968 gfp, vma->vm_mm, &ret); 1969 if (error) 1970 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1971 return ret; 1972 } 1973 1974 unsigned long shmem_get_unmapped_area(struct file *file, 1975 unsigned long uaddr, unsigned long len, 1976 unsigned long pgoff, unsigned long flags) 1977 { 1978 unsigned long (*get_area)(struct file *, 1979 unsigned long, unsigned long, unsigned long, unsigned long); 1980 unsigned long addr; 1981 unsigned long offset; 1982 unsigned long inflated_len; 1983 unsigned long inflated_addr; 1984 unsigned long inflated_offset; 1985 1986 if (len > TASK_SIZE) 1987 return -ENOMEM; 1988 1989 get_area = current->mm->get_unmapped_area; 1990 addr = get_area(file, uaddr, len, pgoff, flags); 1991 1992 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1993 return addr; 1994 if (IS_ERR_VALUE(addr)) 1995 return addr; 1996 if (addr & ~PAGE_MASK) 1997 return addr; 1998 if (addr > TASK_SIZE - len) 1999 return addr; 2000 2001 if (shmem_huge == SHMEM_HUGE_DENY) 2002 return addr; 2003 if (len < HPAGE_PMD_SIZE) 2004 return addr; 2005 if (flags & MAP_FIXED) 2006 return addr; 2007 /* 2008 * Our priority is to support MAP_SHARED mapped hugely; 2009 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2010 * But if caller specified an address hint, respect that as before. 2011 */ 2012 if (uaddr) 2013 return addr; 2014 2015 if (shmem_huge != SHMEM_HUGE_FORCE) { 2016 struct super_block *sb; 2017 2018 if (file) { 2019 VM_BUG_ON(file->f_op != &shmem_file_operations); 2020 sb = file_inode(file)->i_sb; 2021 } else { 2022 /* 2023 * Called directly from mm/mmap.c, or drivers/char/mem.c 2024 * for "/dev/zero", to create a shared anonymous object. 2025 */ 2026 if (IS_ERR(shm_mnt)) 2027 return addr; 2028 sb = shm_mnt->mnt_sb; 2029 } 2030 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2031 return addr; 2032 } 2033 2034 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2035 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2036 return addr; 2037 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2038 return addr; 2039 2040 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2041 if (inflated_len > TASK_SIZE) 2042 return addr; 2043 if (inflated_len < len) 2044 return addr; 2045 2046 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); 2047 if (IS_ERR_VALUE(inflated_addr)) 2048 return addr; 2049 if (inflated_addr & ~PAGE_MASK) 2050 return addr; 2051 2052 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2053 inflated_addr += offset - inflated_offset; 2054 if (inflated_offset > offset) 2055 inflated_addr += HPAGE_PMD_SIZE; 2056 2057 if (inflated_addr > TASK_SIZE - len) 2058 return addr; 2059 return inflated_addr; 2060 } 2061 2062 #ifdef CONFIG_NUMA 2063 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2064 { 2065 struct inode *inode = file_inode(vma->vm_file); 2066 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2067 } 2068 2069 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2070 unsigned long addr) 2071 { 2072 struct inode *inode = file_inode(vma->vm_file); 2073 pgoff_t index; 2074 2075 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2076 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2077 } 2078 #endif 2079 2080 int shmem_lock(struct file *file, int lock, struct user_struct *user) 2081 { 2082 struct inode *inode = file_inode(file); 2083 struct shmem_inode_info *info = SHMEM_I(inode); 2084 int retval = -ENOMEM; 2085 2086 spin_lock_irq(&info->lock); 2087 if (lock && !(info->flags & VM_LOCKED)) { 2088 if (!user_shm_lock(inode->i_size, user)) 2089 goto out_nomem; 2090 info->flags |= VM_LOCKED; 2091 mapping_set_unevictable(file->f_mapping); 2092 } 2093 if (!lock && (info->flags & VM_LOCKED) && user) { 2094 user_shm_unlock(inode->i_size, user); 2095 info->flags &= ~VM_LOCKED; 2096 mapping_clear_unevictable(file->f_mapping); 2097 } 2098 retval = 0; 2099 2100 out_nomem: 2101 spin_unlock_irq(&info->lock); 2102 return retval; 2103 } 2104 2105 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2106 { 2107 file_accessed(file); 2108 vma->vm_ops = &shmem_vm_ops; 2109 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 2110 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2111 (vma->vm_end & HPAGE_PMD_MASK)) { 2112 khugepaged_enter(vma, vma->vm_flags); 2113 } 2114 return 0; 2115 } 2116 2117 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2118 umode_t mode, dev_t dev, unsigned long flags) 2119 { 2120 struct inode *inode; 2121 struct shmem_inode_info *info; 2122 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2123 2124 if (shmem_reserve_inode(sb)) 2125 return NULL; 2126 2127 inode = new_inode(sb); 2128 if (inode) { 2129 inode->i_ino = get_next_ino(); 2130 inode_init_owner(inode, dir, mode); 2131 inode->i_blocks = 0; 2132 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2133 inode->i_generation = get_seconds(); 2134 info = SHMEM_I(inode); 2135 memset(info, 0, (char *)inode - (char *)info); 2136 spin_lock_init(&info->lock); 2137 info->seals = F_SEAL_SEAL; 2138 info->flags = flags & VM_NORESERVE; 2139 INIT_LIST_HEAD(&info->shrinklist); 2140 INIT_LIST_HEAD(&info->swaplist); 2141 simple_xattrs_init(&info->xattrs); 2142 cache_no_acl(inode); 2143 2144 switch (mode & S_IFMT) { 2145 default: 2146 inode->i_op = &shmem_special_inode_operations; 2147 init_special_inode(inode, mode, dev); 2148 break; 2149 case S_IFREG: 2150 inode->i_mapping->a_ops = &shmem_aops; 2151 inode->i_op = &shmem_inode_operations; 2152 inode->i_fop = &shmem_file_operations; 2153 mpol_shared_policy_init(&info->policy, 2154 shmem_get_sbmpol(sbinfo)); 2155 break; 2156 case S_IFDIR: 2157 inc_nlink(inode); 2158 /* Some things misbehave if size == 0 on a directory */ 2159 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2160 inode->i_op = &shmem_dir_inode_operations; 2161 inode->i_fop = &simple_dir_operations; 2162 break; 2163 case S_IFLNK: 2164 /* 2165 * Must not load anything in the rbtree, 2166 * mpol_free_shared_policy will not be called. 2167 */ 2168 mpol_shared_policy_init(&info->policy, NULL); 2169 break; 2170 } 2171 } else 2172 shmem_free_inode(sb); 2173 return inode; 2174 } 2175 2176 bool shmem_mapping(struct address_space *mapping) 2177 { 2178 if (!mapping->host) 2179 return false; 2180 2181 return mapping->host->i_sb->s_op == &shmem_ops; 2182 } 2183 2184 #ifdef CONFIG_TMPFS 2185 static const struct inode_operations shmem_symlink_inode_operations; 2186 static const struct inode_operations shmem_short_symlink_operations; 2187 2188 #ifdef CONFIG_TMPFS_XATTR 2189 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2190 #else 2191 #define shmem_initxattrs NULL 2192 #endif 2193 2194 static int 2195 shmem_write_begin(struct file *file, struct address_space *mapping, 2196 loff_t pos, unsigned len, unsigned flags, 2197 struct page **pagep, void **fsdata) 2198 { 2199 struct inode *inode = mapping->host; 2200 struct shmem_inode_info *info = SHMEM_I(inode); 2201 pgoff_t index = pos >> PAGE_SHIFT; 2202 2203 /* i_mutex is held by caller */ 2204 if (unlikely(info->seals)) { 2205 if (info->seals & F_SEAL_WRITE) 2206 return -EPERM; 2207 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2208 return -EPERM; 2209 } 2210 2211 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2212 } 2213 2214 static int 2215 shmem_write_end(struct file *file, struct address_space *mapping, 2216 loff_t pos, unsigned len, unsigned copied, 2217 struct page *page, void *fsdata) 2218 { 2219 struct inode *inode = mapping->host; 2220 2221 if (pos + copied > inode->i_size) 2222 i_size_write(inode, pos + copied); 2223 2224 if (!PageUptodate(page)) { 2225 struct page *head = compound_head(page); 2226 if (PageTransCompound(page)) { 2227 int i; 2228 2229 for (i = 0; i < HPAGE_PMD_NR; i++) { 2230 if (head + i == page) 2231 continue; 2232 clear_highpage(head + i); 2233 flush_dcache_page(head + i); 2234 } 2235 } 2236 if (copied < PAGE_SIZE) { 2237 unsigned from = pos & (PAGE_SIZE - 1); 2238 zero_user_segments(page, 0, from, 2239 from + copied, PAGE_SIZE); 2240 } 2241 SetPageUptodate(head); 2242 } 2243 set_page_dirty(page); 2244 unlock_page(page); 2245 put_page(page); 2246 2247 return copied; 2248 } 2249 2250 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2251 { 2252 struct file *file = iocb->ki_filp; 2253 struct inode *inode = file_inode(file); 2254 struct address_space *mapping = inode->i_mapping; 2255 pgoff_t index; 2256 unsigned long offset; 2257 enum sgp_type sgp = SGP_READ; 2258 int error = 0; 2259 ssize_t retval = 0; 2260 loff_t *ppos = &iocb->ki_pos; 2261 2262 /* 2263 * Might this read be for a stacking filesystem? Then when reading 2264 * holes of a sparse file, we actually need to allocate those pages, 2265 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2266 */ 2267 if (!iter_is_iovec(to)) 2268 sgp = SGP_CACHE; 2269 2270 index = *ppos >> PAGE_SHIFT; 2271 offset = *ppos & ~PAGE_MASK; 2272 2273 for (;;) { 2274 struct page *page = NULL; 2275 pgoff_t end_index; 2276 unsigned long nr, ret; 2277 loff_t i_size = i_size_read(inode); 2278 2279 end_index = i_size >> PAGE_SHIFT; 2280 if (index > end_index) 2281 break; 2282 if (index == end_index) { 2283 nr = i_size & ~PAGE_MASK; 2284 if (nr <= offset) 2285 break; 2286 } 2287 2288 error = shmem_getpage(inode, index, &page, sgp); 2289 if (error) { 2290 if (error == -EINVAL) 2291 error = 0; 2292 break; 2293 } 2294 if (page) { 2295 if (sgp == SGP_CACHE) 2296 set_page_dirty(page); 2297 unlock_page(page); 2298 } 2299 2300 /* 2301 * We must evaluate after, since reads (unlike writes) 2302 * are called without i_mutex protection against truncate 2303 */ 2304 nr = PAGE_SIZE; 2305 i_size = i_size_read(inode); 2306 end_index = i_size >> PAGE_SHIFT; 2307 if (index == end_index) { 2308 nr = i_size & ~PAGE_MASK; 2309 if (nr <= offset) { 2310 if (page) 2311 put_page(page); 2312 break; 2313 } 2314 } 2315 nr -= offset; 2316 2317 if (page) { 2318 /* 2319 * If users can be writing to this page using arbitrary 2320 * virtual addresses, take care about potential aliasing 2321 * before reading the page on the kernel side. 2322 */ 2323 if (mapping_writably_mapped(mapping)) 2324 flush_dcache_page(page); 2325 /* 2326 * Mark the page accessed if we read the beginning. 2327 */ 2328 if (!offset) 2329 mark_page_accessed(page); 2330 } else { 2331 page = ZERO_PAGE(0); 2332 get_page(page); 2333 } 2334 2335 /* 2336 * Ok, we have the page, and it's up-to-date, so 2337 * now we can copy it to user space... 2338 */ 2339 ret = copy_page_to_iter(page, offset, nr, to); 2340 retval += ret; 2341 offset += ret; 2342 index += offset >> PAGE_SHIFT; 2343 offset &= ~PAGE_MASK; 2344 2345 put_page(page); 2346 if (!iov_iter_count(to)) 2347 break; 2348 if (ret < nr) { 2349 error = -EFAULT; 2350 break; 2351 } 2352 cond_resched(); 2353 } 2354 2355 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2356 file_accessed(file); 2357 return retval ? retval : error; 2358 } 2359 2360 /* 2361 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree. 2362 */ 2363 static pgoff_t shmem_seek_hole_data(struct address_space *mapping, 2364 pgoff_t index, pgoff_t end, int whence) 2365 { 2366 struct page *page; 2367 struct pagevec pvec; 2368 pgoff_t indices[PAGEVEC_SIZE]; 2369 bool done = false; 2370 int i; 2371 2372 pagevec_init(&pvec, 0); 2373 pvec.nr = 1; /* start small: we may be there already */ 2374 while (!done) { 2375 pvec.nr = find_get_entries(mapping, index, 2376 pvec.nr, pvec.pages, indices); 2377 if (!pvec.nr) { 2378 if (whence == SEEK_DATA) 2379 index = end; 2380 break; 2381 } 2382 for (i = 0; i < pvec.nr; i++, index++) { 2383 if (index < indices[i]) { 2384 if (whence == SEEK_HOLE) { 2385 done = true; 2386 break; 2387 } 2388 index = indices[i]; 2389 } 2390 page = pvec.pages[i]; 2391 if (page && !radix_tree_exceptional_entry(page)) { 2392 if (!PageUptodate(page)) 2393 page = NULL; 2394 } 2395 if (index >= end || 2396 (page && whence == SEEK_DATA) || 2397 (!page && whence == SEEK_HOLE)) { 2398 done = true; 2399 break; 2400 } 2401 } 2402 pagevec_remove_exceptionals(&pvec); 2403 pagevec_release(&pvec); 2404 pvec.nr = PAGEVEC_SIZE; 2405 cond_resched(); 2406 } 2407 return index; 2408 } 2409 2410 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2411 { 2412 struct address_space *mapping = file->f_mapping; 2413 struct inode *inode = mapping->host; 2414 pgoff_t start, end; 2415 loff_t new_offset; 2416 2417 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2418 return generic_file_llseek_size(file, offset, whence, 2419 MAX_LFS_FILESIZE, i_size_read(inode)); 2420 inode_lock(inode); 2421 /* We're holding i_mutex so we can access i_size directly */ 2422 2423 if (offset < 0) 2424 offset = -EINVAL; 2425 else if (offset >= inode->i_size) 2426 offset = -ENXIO; 2427 else { 2428 start = offset >> PAGE_SHIFT; 2429 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2430 new_offset = shmem_seek_hole_data(mapping, start, end, whence); 2431 new_offset <<= PAGE_SHIFT; 2432 if (new_offset > offset) { 2433 if (new_offset < inode->i_size) 2434 offset = new_offset; 2435 else if (whence == SEEK_DATA) 2436 offset = -ENXIO; 2437 else 2438 offset = inode->i_size; 2439 } 2440 } 2441 2442 if (offset >= 0) 2443 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2444 inode_unlock(inode); 2445 return offset; 2446 } 2447 2448 /* 2449 * We need a tag: a new tag would expand every radix_tree_node by 8 bytes, 2450 * so reuse a tag which we firmly believe is never set or cleared on shmem. 2451 */ 2452 #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE 2453 #define LAST_SCAN 4 /* about 150ms max */ 2454 2455 static void shmem_tag_pins(struct address_space *mapping) 2456 { 2457 struct radix_tree_iter iter; 2458 void **slot; 2459 pgoff_t start; 2460 struct page *page; 2461 2462 lru_add_drain(); 2463 start = 0; 2464 rcu_read_lock(); 2465 2466 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2467 page = radix_tree_deref_slot(slot); 2468 if (!page || radix_tree_exception(page)) { 2469 if (radix_tree_deref_retry(page)) { 2470 slot = radix_tree_iter_retry(&iter); 2471 continue; 2472 } 2473 } else if (page_count(page) - page_mapcount(page) > 1) { 2474 spin_lock_irq(&mapping->tree_lock); 2475 radix_tree_tag_set(&mapping->page_tree, iter.index, 2476 SHMEM_TAG_PINNED); 2477 spin_unlock_irq(&mapping->tree_lock); 2478 } 2479 2480 if (need_resched()) { 2481 slot = radix_tree_iter_resume(slot, &iter); 2482 cond_resched_rcu(); 2483 } 2484 } 2485 rcu_read_unlock(); 2486 } 2487 2488 /* 2489 * Setting SEAL_WRITE requires us to verify there's no pending writer. However, 2490 * via get_user_pages(), drivers might have some pending I/O without any active 2491 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages 2492 * and see whether it has an elevated ref-count. If so, we tag them and wait for 2493 * them to be dropped. 2494 * The caller must guarantee that no new user will acquire writable references 2495 * to those pages to avoid races. 2496 */ 2497 static int shmem_wait_for_pins(struct address_space *mapping) 2498 { 2499 struct radix_tree_iter iter; 2500 void **slot; 2501 pgoff_t start; 2502 struct page *page; 2503 int error, scan; 2504 2505 shmem_tag_pins(mapping); 2506 2507 error = 0; 2508 for (scan = 0; scan <= LAST_SCAN; scan++) { 2509 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2510 break; 2511 2512 if (!scan) 2513 lru_add_drain_all(); 2514 else if (schedule_timeout_killable((HZ << scan) / 200)) 2515 scan = LAST_SCAN; 2516 2517 start = 0; 2518 rcu_read_lock(); 2519 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2520 start, SHMEM_TAG_PINNED) { 2521 2522 page = radix_tree_deref_slot(slot); 2523 if (radix_tree_exception(page)) { 2524 if (radix_tree_deref_retry(page)) { 2525 slot = radix_tree_iter_retry(&iter); 2526 continue; 2527 } 2528 2529 page = NULL; 2530 } 2531 2532 if (page && 2533 page_count(page) - page_mapcount(page) != 1) { 2534 if (scan < LAST_SCAN) 2535 goto continue_resched; 2536 2537 /* 2538 * On the last scan, we clean up all those tags 2539 * we inserted; but make a note that we still 2540 * found pages pinned. 2541 */ 2542 error = -EBUSY; 2543 } 2544 2545 spin_lock_irq(&mapping->tree_lock); 2546 radix_tree_tag_clear(&mapping->page_tree, 2547 iter.index, SHMEM_TAG_PINNED); 2548 spin_unlock_irq(&mapping->tree_lock); 2549 continue_resched: 2550 if (need_resched()) { 2551 slot = radix_tree_iter_resume(slot, &iter); 2552 cond_resched_rcu(); 2553 } 2554 } 2555 rcu_read_unlock(); 2556 } 2557 2558 return error; 2559 } 2560 2561 #define F_ALL_SEALS (F_SEAL_SEAL | \ 2562 F_SEAL_SHRINK | \ 2563 F_SEAL_GROW | \ 2564 F_SEAL_WRITE) 2565 2566 int shmem_add_seals(struct file *file, unsigned int seals) 2567 { 2568 struct inode *inode = file_inode(file); 2569 struct shmem_inode_info *info = SHMEM_I(inode); 2570 int error; 2571 2572 /* 2573 * SEALING 2574 * Sealing allows multiple parties to share a shmem-file but restrict 2575 * access to a specific subset of file operations. Seals can only be 2576 * added, but never removed. This way, mutually untrusted parties can 2577 * share common memory regions with a well-defined policy. A malicious 2578 * peer can thus never perform unwanted operations on a shared object. 2579 * 2580 * Seals are only supported on special shmem-files and always affect 2581 * the whole underlying inode. Once a seal is set, it may prevent some 2582 * kinds of access to the file. Currently, the following seals are 2583 * defined: 2584 * SEAL_SEAL: Prevent further seals from being set on this file 2585 * SEAL_SHRINK: Prevent the file from shrinking 2586 * SEAL_GROW: Prevent the file from growing 2587 * SEAL_WRITE: Prevent write access to the file 2588 * 2589 * As we don't require any trust relationship between two parties, we 2590 * must prevent seals from being removed. Therefore, sealing a file 2591 * only adds a given set of seals to the file, it never touches 2592 * existing seals. Furthermore, the "setting seals"-operation can be 2593 * sealed itself, which basically prevents any further seal from being 2594 * added. 2595 * 2596 * Semantics of sealing are only defined on volatile files. Only 2597 * anonymous shmem files support sealing. More importantly, seals are 2598 * never written to disk. Therefore, there's no plan to support it on 2599 * other file types. 2600 */ 2601 2602 if (file->f_op != &shmem_file_operations) 2603 return -EINVAL; 2604 if (!(file->f_mode & FMODE_WRITE)) 2605 return -EPERM; 2606 if (seals & ~(unsigned int)F_ALL_SEALS) 2607 return -EINVAL; 2608 2609 inode_lock(inode); 2610 2611 if (info->seals & F_SEAL_SEAL) { 2612 error = -EPERM; 2613 goto unlock; 2614 } 2615 2616 if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) { 2617 error = mapping_deny_writable(file->f_mapping); 2618 if (error) 2619 goto unlock; 2620 2621 error = shmem_wait_for_pins(file->f_mapping); 2622 if (error) { 2623 mapping_allow_writable(file->f_mapping); 2624 goto unlock; 2625 } 2626 } 2627 2628 info->seals |= seals; 2629 error = 0; 2630 2631 unlock: 2632 inode_unlock(inode); 2633 return error; 2634 } 2635 EXPORT_SYMBOL_GPL(shmem_add_seals); 2636 2637 int shmem_get_seals(struct file *file) 2638 { 2639 if (file->f_op != &shmem_file_operations) 2640 return -EINVAL; 2641 2642 return SHMEM_I(file_inode(file))->seals; 2643 } 2644 EXPORT_SYMBOL_GPL(shmem_get_seals); 2645 2646 long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 2647 { 2648 long error; 2649 2650 switch (cmd) { 2651 case F_ADD_SEALS: 2652 /* disallow upper 32bit */ 2653 if (arg > UINT_MAX) 2654 return -EINVAL; 2655 2656 error = shmem_add_seals(file, arg); 2657 break; 2658 case F_GET_SEALS: 2659 error = shmem_get_seals(file); 2660 break; 2661 default: 2662 error = -EINVAL; 2663 break; 2664 } 2665 2666 return error; 2667 } 2668 2669 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2670 loff_t len) 2671 { 2672 struct inode *inode = file_inode(file); 2673 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2674 struct shmem_inode_info *info = SHMEM_I(inode); 2675 struct shmem_falloc shmem_falloc; 2676 pgoff_t start, index, end; 2677 int error; 2678 2679 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2680 return -EOPNOTSUPP; 2681 2682 inode_lock(inode); 2683 2684 if (mode & FALLOC_FL_PUNCH_HOLE) { 2685 struct address_space *mapping = file->f_mapping; 2686 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2687 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2688 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2689 2690 /* protected by i_mutex */ 2691 if (info->seals & F_SEAL_WRITE) { 2692 error = -EPERM; 2693 goto out; 2694 } 2695 2696 shmem_falloc.waitq = &shmem_falloc_waitq; 2697 shmem_falloc.start = unmap_start >> PAGE_SHIFT; 2698 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2699 spin_lock(&inode->i_lock); 2700 inode->i_private = &shmem_falloc; 2701 spin_unlock(&inode->i_lock); 2702 2703 if ((u64)unmap_end > (u64)unmap_start) 2704 unmap_mapping_range(mapping, unmap_start, 2705 1 + unmap_end - unmap_start, 0); 2706 shmem_truncate_range(inode, offset, offset + len - 1); 2707 /* No need to unmap again: hole-punching leaves COWed pages */ 2708 2709 spin_lock(&inode->i_lock); 2710 inode->i_private = NULL; 2711 wake_up_all(&shmem_falloc_waitq); 2712 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.task_list)); 2713 spin_unlock(&inode->i_lock); 2714 error = 0; 2715 goto out; 2716 } 2717 2718 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2719 error = inode_newsize_ok(inode, offset + len); 2720 if (error) 2721 goto out; 2722 2723 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2724 error = -EPERM; 2725 goto out; 2726 } 2727 2728 start = offset >> PAGE_SHIFT; 2729 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2730 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2731 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2732 error = -ENOSPC; 2733 goto out; 2734 } 2735 2736 shmem_falloc.waitq = NULL; 2737 shmem_falloc.start = start; 2738 shmem_falloc.next = start; 2739 shmem_falloc.nr_falloced = 0; 2740 shmem_falloc.nr_unswapped = 0; 2741 spin_lock(&inode->i_lock); 2742 inode->i_private = &shmem_falloc; 2743 spin_unlock(&inode->i_lock); 2744 2745 for (index = start; index < end; index++) { 2746 struct page *page; 2747 2748 /* 2749 * Good, the fallocate(2) manpage permits EINTR: we may have 2750 * been interrupted because we are using up too much memory. 2751 */ 2752 if (signal_pending(current)) 2753 error = -EINTR; 2754 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2755 error = -ENOMEM; 2756 else 2757 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2758 if (error) { 2759 /* Remove the !PageUptodate pages we added */ 2760 if (index > start) { 2761 shmem_undo_range(inode, 2762 (loff_t)start << PAGE_SHIFT, 2763 ((loff_t)index << PAGE_SHIFT) - 1, true); 2764 } 2765 goto undone; 2766 } 2767 2768 /* 2769 * Inform shmem_writepage() how far we have reached. 2770 * No need for lock or barrier: we have the page lock. 2771 */ 2772 shmem_falloc.next++; 2773 if (!PageUptodate(page)) 2774 shmem_falloc.nr_falloced++; 2775 2776 /* 2777 * If !PageUptodate, leave it that way so that freeable pages 2778 * can be recognized if we need to rollback on error later. 2779 * But set_page_dirty so that memory pressure will swap rather 2780 * than free the pages we are allocating (and SGP_CACHE pages 2781 * might still be clean: we now need to mark those dirty too). 2782 */ 2783 set_page_dirty(page); 2784 unlock_page(page); 2785 put_page(page); 2786 cond_resched(); 2787 } 2788 2789 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2790 i_size_write(inode, offset + len); 2791 inode->i_ctime = current_time(inode); 2792 undone: 2793 spin_lock(&inode->i_lock); 2794 inode->i_private = NULL; 2795 spin_unlock(&inode->i_lock); 2796 out: 2797 inode_unlock(inode); 2798 return error; 2799 } 2800 2801 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2802 { 2803 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2804 2805 buf->f_type = TMPFS_MAGIC; 2806 buf->f_bsize = PAGE_SIZE; 2807 buf->f_namelen = NAME_MAX; 2808 if (sbinfo->max_blocks) { 2809 buf->f_blocks = sbinfo->max_blocks; 2810 buf->f_bavail = 2811 buf->f_bfree = sbinfo->max_blocks - 2812 percpu_counter_sum(&sbinfo->used_blocks); 2813 } 2814 if (sbinfo->max_inodes) { 2815 buf->f_files = sbinfo->max_inodes; 2816 buf->f_ffree = sbinfo->free_inodes; 2817 } 2818 /* else leave those fields 0 like simple_statfs */ 2819 return 0; 2820 } 2821 2822 /* 2823 * File creation. Allocate an inode, and we're done.. 2824 */ 2825 static int 2826 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) 2827 { 2828 struct inode *inode; 2829 int error = -ENOSPC; 2830 2831 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2832 if (inode) { 2833 error = simple_acl_create(dir, inode); 2834 if (error) 2835 goto out_iput; 2836 error = security_inode_init_security(inode, dir, 2837 &dentry->d_name, 2838 shmem_initxattrs, NULL); 2839 if (error && error != -EOPNOTSUPP) 2840 goto out_iput; 2841 2842 error = 0; 2843 dir->i_size += BOGO_DIRENT_SIZE; 2844 dir->i_ctime = dir->i_mtime = current_time(dir); 2845 d_instantiate(dentry, inode); 2846 dget(dentry); /* Extra count - pin the dentry in core */ 2847 } 2848 return error; 2849 out_iput: 2850 iput(inode); 2851 return error; 2852 } 2853 2854 static int 2855 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 2856 { 2857 struct inode *inode; 2858 int error = -ENOSPC; 2859 2860 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2861 if (inode) { 2862 error = security_inode_init_security(inode, dir, 2863 NULL, 2864 shmem_initxattrs, NULL); 2865 if (error && error != -EOPNOTSUPP) 2866 goto out_iput; 2867 error = simple_acl_create(dir, inode); 2868 if (error) 2869 goto out_iput; 2870 d_tmpfile(dentry, inode); 2871 } 2872 return error; 2873 out_iput: 2874 iput(inode); 2875 return error; 2876 } 2877 2878 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 2879 { 2880 int error; 2881 2882 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 2883 return error; 2884 inc_nlink(dir); 2885 return 0; 2886 } 2887 2888 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, 2889 bool excl) 2890 { 2891 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 2892 } 2893 2894 /* 2895 * Link a file.. 2896 */ 2897 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2898 { 2899 struct inode *inode = d_inode(old_dentry); 2900 int ret; 2901 2902 /* 2903 * No ordinary (disk based) filesystem counts links as inodes; 2904 * but each new link needs a new dentry, pinning lowmem, and 2905 * tmpfs dentries cannot be pruned until they are unlinked. 2906 */ 2907 ret = shmem_reserve_inode(inode->i_sb); 2908 if (ret) 2909 goto out; 2910 2911 dir->i_size += BOGO_DIRENT_SIZE; 2912 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2913 inc_nlink(inode); 2914 ihold(inode); /* New dentry reference */ 2915 dget(dentry); /* Extra pinning count for the created dentry */ 2916 d_instantiate(dentry, inode); 2917 out: 2918 return ret; 2919 } 2920 2921 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2922 { 2923 struct inode *inode = d_inode(dentry); 2924 2925 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2926 shmem_free_inode(inode->i_sb); 2927 2928 dir->i_size -= BOGO_DIRENT_SIZE; 2929 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2930 drop_nlink(inode); 2931 dput(dentry); /* Undo the count from "create" - this does all the work */ 2932 return 0; 2933 } 2934 2935 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2936 { 2937 if (!simple_empty(dentry)) 2938 return -ENOTEMPTY; 2939 2940 drop_nlink(d_inode(dentry)); 2941 drop_nlink(dir); 2942 return shmem_unlink(dir, dentry); 2943 } 2944 2945 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 2946 { 2947 bool old_is_dir = d_is_dir(old_dentry); 2948 bool new_is_dir = d_is_dir(new_dentry); 2949 2950 if (old_dir != new_dir && old_is_dir != new_is_dir) { 2951 if (old_is_dir) { 2952 drop_nlink(old_dir); 2953 inc_nlink(new_dir); 2954 } else { 2955 drop_nlink(new_dir); 2956 inc_nlink(old_dir); 2957 } 2958 } 2959 old_dir->i_ctime = old_dir->i_mtime = 2960 new_dir->i_ctime = new_dir->i_mtime = 2961 d_inode(old_dentry)->i_ctime = 2962 d_inode(new_dentry)->i_ctime = current_time(old_dir); 2963 2964 return 0; 2965 } 2966 2967 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) 2968 { 2969 struct dentry *whiteout; 2970 int error; 2971 2972 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 2973 if (!whiteout) 2974 return -ENOMEM; 2975 2976 error = shmem_mknod(old_dir, whiteout, 2977 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 2978 dput(whiteout); 2979 if (error) 2980 return error; 2981 2982 /* 2983 * Cheat and hash the whiteout while the old dentry is still in 2984 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 2985 * 2986 * d_lookup() will consistently find one of them at this point, 2987 * not sure which one, but that isn't even important. 2988 */ 2989 d_rehash(whiteout); 2990 return 0; 2991 } 2992 2993 /* 2994 * The VFS layer already does all the dentry stuff for rename, 2995 * we just have to decrement the usage count for the target if 2996 * it exists so that the VFS layer correctly free's it when it 2997 * gets overwritten. 2998 */ 2999 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) 3000 { 3001 struct inode *inode = d_inode(old_dentry); 3002 int they_are_dirs = S_ISDIR(inode->i_mode); 3003 3004 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3005 return -EINVAL; 3006 3007 if (flags & RENAME_EXCHANGE) 3008 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 3009 3010 if (!simple_empty(new_dentry)) 3011 return -ENOTEMPTY; 3012 3013 if (flags & RENAME_WHITEOUT) { 3014 int error; 3015 3016 error = shmem_whiteout(old_dir, old_dentry); 3017 if (error) 3018 return error; 3019 } 3020 3021 if (d_really_is_positive(new_dentry)) { 3022 (void) shmem_unlink(new_dir, new_dentry); 3023 if (they_are_dirs) { 3024 drop_nlink(d_inode(new_dentry)); 3025 drop_nlink(old_dir); 3026 } 3027 } else if (they_are_dirs) { 3028 drop_nlink(old_dir); 3029 inc_nlink(new_dir); 3030 } 3031 3032 old_dir->i_size -= BOGO_DIRENT_SIZE; 3033 new_dir->i_size += BOGO_DIRENT_SIZE; 3034 old_dir->i_ctime = old_dir->i_mtime = 3035 new_dir->i_ctime = new_dir->i_mtime = 3036 inode->i_ctime = current_time(old_dir); 3037 return 0; 3038 } 3039 3040 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 3041 { 3042 int error; 3043 int len; 3044 struct inode *inode; 3045 struct page *page; 3046 struct shmem_inode_info *info; 3047 3048 len = strlen(symname) + 1; 3049 if (len > PAGE_SIZE) 3050 return -ENAMETOOLONG; 3051 3052 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 3053 if (!inode) 3054 return -ENOSPC; 3055 3056 error = security_inode_init_security(inode, dir, &dentry->d_name, 3057 shmem_initxattrs, NULL); 3058 if (error) { 3059 if (error != -EOPNOTSUPP) { 3060 iput(inode); 3061 return error; 3062 } 3063 error = 0; 3064 } 3065 3066 info = SHMEM_I(inode); 3067 inode->i_size = len-1; 3068 if (len <= SHORT_SYMLINK_LEN) { 3069 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3070 if (!inode->i_link) { 3071 iput(inode); 3072 return -ENOMEM; 3073 } 3074 inode->i_op = &shmem_short_symlink_operations; 3075 } else { 3076 inode_nohighmem(inode); 3077 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3078 if (error) { 3079 iput(inode); 3080 return error; 3081 } 3082 inode->i_mapping->a_ops = &shmem_aops; 3083 inode->i_op = &shmem_symlink_inode_operations; 3084 memcpy(page_address(page), symname, len); 3085 SetPageUptodate(page); 3086 set_page_dirty(page); 3087 unlock_page(page); 3088 put_page(page); 3089 } 3090 dir->i_size += BOGO_DIRENT_SIZE; 3091 dir->i_ctime = dir->i_mtime = current_time(dir); 3092 d_instantiate(dentry, inode); 3093 dget(dentry); 3094 return 0; 3095 } 3096 3097 static void shmem_put_link(void *arg) 3098 { 3099 mark_page_accessed(arg); 3100 put_page(arg); 3101 } 3102 3103 static const char *shmem_get_link(struct dentry *dentry, 3104 struct inode *inode, 3105 struct delayed_call *done) 3106 { 3107 struct page *page = NULL; 3108 int error; 3109 if (!dentry) { 3110 page = find_get_page(inode->i_mapping, 0); 3111 if (!page) 3112 return ERR_PTR(-ECHILD); 3113 if (!PageUptodate(page)) { 3114 put_page(page); 3115 return ERR_PTR(-ECHILD); 3116 } 3117 } else { 3118 error = shmem_getpage(inode, 0, &page, SGP_READ); 3119 if (error) 3120 return ERR_PTR(error); 3121 unlock_page(page); 3122 } 3123 set_delayed_call(done, shmem_put_link, page); 3124 return page_address(page); 3125 } 3126 3127 #ifdef CONFIG_TMPFS_XATTR 3128 /* 3129 * Superblocks without xattr inode operations may get some security.* xattr 3130 * support from the LSM "for free". As soon as we have any other xattrs 3131 * like ACLs, we also need to implement the security.* handlers at 3132 * filesystem level, though. 3133 */ 3134 3135 /* 3136 * Callback for security_inode_init_security() for acquiring xattrs. 3137 */ 3138 static int shmem_initxattrs(struct inode *inode, 3139 const struct xattr *xattr_array, 3140 void *fs_info) 3141 { 3142 struct shmem_inode_info *info = SHMEM_I(inode); 3143 const struct xattr *xattr; 3144 struct simple_xattr *new_xattr; 3145 size_t len; 3146 3147 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3148 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3149 if (!new_xattr) 3150 return -ENOMEM; 3151 3152 len = strlen(xattr->name) + 1; 3153 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3154 GFP_KERNEL); 3155 if (!new_xattr->name) { 3156 kfree(new_xattr); 3157 return -ENOMEM; 3158 } 3159 3160 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3161 XATTR_SECURITY_PREFIX_LEN); 3162 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3163 xattr->name, len); 3164 3165 simple_xattr_list_add(&info->xattrs, new_xattr); 3166 } 3167 3168 return 0; 3169 } 3170 3171 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3172 struct dentry *unused, struct inode *inode, 3173 const char *name, void *buffer, size_t size) 3174 { 3175 struct shmem_inode_info *info = SHMEM_I(inode); 3176 3177 name = xattr_full_name(handler, name); 3178 return simple_xattr_get(&info->xattrs, name, buffer, size); 3179 } 3180 3181 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3182 struct dentry *unused, struct inode *inode, 3183 const char *name, const void *value, 3184 size_t size, int flags) 3185 { 3186 struct shmem_inode_info *info = SHMEM_I(inode); 3187 3188 name = xattr_full_name(handler, name); 3189 return simple_xattr_set(&info->xattrs, name, value, size, flags); 3190 } 3191 3192 static const struct xattr_handler shmem_security_xattr_handler = { 3193 .prefix = XATTR_SECURITY_PREFIX, 3194 .get = shmem_xattr_handler_get, 3195 .set = shmem_xattr_handler_set, 3196 }; 3197 3198 static const struct xattr_handler shmem_trusted_xattr_handler = { 3199 .prefix = XATTR_TRUSTED_PREFIX, 3200 .get = shmem_xattr_handler_get, 3201 .set = shmem_xattr_handler_set, 3202 }; 3203 3204 static const struct xattr_handler *shmem_xattr_handlers[] = { 3205 #ifdef CONFIG_TMPFS_POSIX_ACL 3206 &posix_acl_access_xattr_handler, 3207 &posix_acl_default_xattr_handler, 3208 #endif 3209 &shmem_security_xattr_handler, 3210 &shmem_trusted_xattr_handler, 3211 NULL 3212 }; 3213 3214 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3215 { 3216 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3217 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3218 } 3219 #endif /* CONFIG_TMPFS_XATTR */ 3220 3221 static const struct inode_operations shmem_short_symlink_operations = { 3222 .get_link = simple_get_link, 3223 #ifdef CONFIG_TMPFS_XATTR 3224 .listxattr = shmem_listxattr, 3225 #endif 3226 }; 3227 3228 static const struct inode_operations shmem_symlink_inode_operations = { 3229 .get_link = shmem_get_link, 3230 #ifdef CONFIG_TMPFS_XATTR 3231 .listxattr = shmem_listxattr, 3232 #endif 3233 }; 3234 3235 static struct dentry *shmem_get_parent(struct dentry *child) 3236 { 3237 return ERR_PTR(-ESTALE); 3238 } 3239 3240 static int shmem_match(struct inode *ino, void *vfh) 3241 { 3242 __u32 *fh = vfh; 3243 __u64 inum = fh[2]; 3244 inum = (inum << 32) | fh[1]; 3245 return ino->i_ino == inum && fh[0] == ino->i_generation; 3246 } 3247 3248 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3249 struct fid *fid, int fh_len, int fh_type) 3250 { 3251 struct inode *inode; 3252 struct dentry *dentry = NULL; 3253 u64 inum; 3254 3255 if (fh_len < 3) 3256 return NULL; 3257 3258 inum = fid->raw[2]; 3259 inum = (inum << 32) | fid->raw[1]; 3260 3261 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3262 shmem_match, fid->raw); 3263 if (inode) { 3264 dentry = d_find_alias(inode); 3265 iput(inode); 3266 } 3267 3268 return dentry; 3269 } 3270 3271 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3272 struct inode *parent) 3273 { 3274 if (*len < 3) { 3275 *len = 3; 3276 return FILEID_INVALID; 3277 } 3278 3279 if (inode_unhashed(inode)) { 3280 /* Unfortunately insert_inode_hash is not idempotent, 3281 * so as we hash inodes here rather than at creation 3282 * time, we need a lock to ensure we only try 3283 * to do it once 3284 */ 3285 static DEFINE_SPINLOCK(lock); 3286 spin_lock(&lock); 3287 if (inode_unhashed(inode)) 3288 __insert_inode_hash(inode, 3289 inode->i_ino + inode->i_generation); 3290 spin_unlock(&lock); 3291 } 3292 3293 fh[0] = inode->i_generation; 3294 fh[1] = inode->i_ino; 3295 fh[2] = ((__u64)inode->i_ino) >> 32; 3296 3297 *len = 3; 3298 return 1; 3299 } 3300 3301 static const struct export_operations shmem_export_ops = { 3302 .get_parent = shmem_get_parent, 3303 .encode_fh = shmem_encode_fh, 3304 .fh_to_dentry = shmem_fh_to_dentry, 3305 }; 3306 3307 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 3308 bool remount) 3309 { 3310 char *this_char, *value, *rest; 3311 struct mempolicy *mpol = NULL; 3312 uid_t uid; 3313 gid_t gid; 3314 3315 while (options != NULL) { 3316 this_char = options; 3317 for (;;) { 3318 /* 3319 * NUL-terminate this option: unfortunately, 3320 * mount options form a comma-separated list, 3321 * but mpol's nodelist may also contain commas. 3322 */ 3323 options = strchr(options, ','); 3324 if (options == NULL) 3325 break; 3326 options++; 3327 if (!isdigit(*options)) { 3328 options[-1] = '\0'; 3329 break; 3330 } 3331 } 3332 if (!*this_char) 3333 continue; 3334 if ((value = strchr(this_char,'=')) != NULL) { 3335 *value++ = 0; 3336 } else { 3337 pr_err("tmpfs: No value for mount option '%s'\n", 3338 this_char); 3339 goto error; 3340 } 3341 3342 if (!strcmp(this_char,"size")) { 3343 unsigned long long size; 3344 size = memparse(value,&rest); 3345 if (*rest == '%') { 3346 size <<= PAGE_SHIFT; 3347 size *= totalram_pages; 3348 do_div(size, 100); 3349 rest++; 3350 } 3351 if (*rest) 3352 goto bad_val; 3353 sbinfo->max_blocks = 3354 DIV_ROUND_UP(size, PAGE_SIZE); 3355 } else if (!strcmp(this_char,"nr_blocks")) { 3356 sbinfo->max_blocks = memparse(value, &rest); 3357 if (*rest) 3358 goto bad_val; 3359 } else if (!strcmp(this_char,"nr_inodes")) { 3360 sbinfo->max_inodes = memparse(value, &rest); 3361 if (*rest) 3362 goto bad_val; 3363 } else if (!strcmp(this_char,"mode")) { 3364 if (remount) 3365 continue; 3366 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 3367 if (*rest) 3368 goto bad_val; 3369 } else if (!strcmp(this_char,"uid")) { 3370 if (remount) 3371 continue; 3372 uid = simple_strtoul(value, &rest, 0); 3373 if (*rest) 3374 goto bad_val; 3375 sbinfo->uid = make_kuid(current_user_ns(), uid); 3376 if (!uid_valid(sbinfo->uid)) 3377 goto bad_val; 3378 } else if (!strcmp(this_char,"gid")) { 3379 if (remount) 3380 continue; 3381 gid = simple_strtoul(value, &rest, 0); 3382 if (*rest) 3383 goto bad_val; 3384 sbinfo->gid = make_kgid(current_user_ns(), gid); 3385 if (!gid_valid(sbinfo->gid)) 3386 goto bad_val; 3387 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3388 } else if (!strcmp(this_char, "huge")) { 3389 int huge; 3390 huge = shmem_parse_huge(value); 3391 if (huge < 0) 3392 goto bad_val; 3393 if (!has_transparent_hugepage() && 3394 huge != SHMEM_HUGE_NEVER) 3395 goto bad_val; 3396 sbinfo->huge = huge; 3397 #endif 3398 #ifdef CONFIG_NUMA 3399 } else if (!strcmp(this_char,"mpol")) { 3400 mpol_put(mpol); 3401 mpol = NULL; 3402 if (mpol_parse_str(value, &mpol)) 3403 goto bad_val; 3404 #endif 3405 } else { 3406 pr_err("tmpfs: Bad mount option %s\n", this_char); 3407 goto error; 3408 } 3409 } 3410 sbinfo->mpol = mpol; 3411 return 0; 3412 3413 bad_val: 3414 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n", 3415 value, this_char); 3416 error: 3417 mpol_put(mpol); 3418 return 1; 3419 3420 } 3421 3422 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 3423 { 3424 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3425 struct shmem_sb_info config = *sbinfo; 3426 unsigned long inodes; 3427 int error = -EINVAL; 3428 3429 config.mpol = NULL; 3430 if (shmem_parse_options(data, &config, true)) 3431 return error; 3432 3433 spin_lock(&sbinfo->stat_lock); 3434 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3435 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 3436 goto out; 3437 if (config.max_inodes < inodes) 3438 goto out; 3439 /* 3440 * Those tests disallow limited->unlimited while any are in use; 3441 * but we must separately disallow unlimited->limited, because 3442 * in that case we have no record of how much is already in use. 3443 */ 3444 if (config.max_blocks && !sbinfo->max_blocks) 3445 goto out; 3446 if (config.max_inodes && !sbinfo->max_inodes) 3447 goto out; 3448 3449 error = 0; 3450 sbinfo->huge = config.huge; 3451 sbinfo->max_blocks = config.max_blocks; 3452 sbinfo->max_inodes = config.max_inodes; 3453 sbinfo->free_inodes = config.max_inodes - inodes; 3454 3455 /* 3456 * Preserve previous mempolicy unless mpol remount option was specified. 3457 */ 3458 if (config.mpol) { 3459 mpol_put(sbinfo->mpol); 3460 sbinfo->mpol = config.mpol; /* transfers initial ref */ 3461 } 3462 out: 3463 spin_unlock(&sbinfo->stat_lock); 3464 return error; 3465 } 3466 3467 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3468 { 3469 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3470 3471 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3472 seq_printf(seq, ",size=%luk", 3473 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3474 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3475 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3476 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 3477 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3478 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3479 seq_printf(seq, ",uid=%u", 3480 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3481 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3482 seq_printf(seq, ",gid=%u", 3483 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3484 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3485 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3486 if (sbinfo->huge) 3487 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3488 #endif 3489 shmem_show_mpol(seq, sbinfo->mpol); 3490 return 0; 3491 } 3492 3493 #define MFD_NAME_PREFIX "memfd:" 3494 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) 3495 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) 3496 3497 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING) 3498 3499 SYSCALL_DEFINE2(memfd_create, 3500 const char __user *, uname, 3501 unsigned int, flags) 3502 { 3503 struct shmem_inode_info *info; 3504 struct file *file; 3505 int fd, error; 3506 char *name; 3507 long len; 3508 3509 if (flags & ~(unsigned int)MFD_ALL_FLAGS) 3510 return -EINVAL; 3511 3512 /* length includes terminating zero */ 3513 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); 3514 if (len <= 0) 3515 return -EFAULT; 3516 if (len > MFD_NAME_MAX_LEN + 1) 3517 return -EINVAL; 3518 3519 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY); 3520 if (!name) 3521 return -ENOMEM; 3522 3523 strcpy(name, MFD_NAME_PREFIX); 3524 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { 3525 error = -EFAULT; 3526 goto err_name; 3527 } 3528 3529 /* terminating-zero may have changed after strnlen_user() returned */ 3530 if (name[len + MFD_NAME_PREFIX_LEN - 1]) { 3531 error = -EFAULT; 3532 goto err_name; 3533 } 3534 3535 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); 3536 if (fd < 0) { 3537 error = fd; 3538 goto err_name; 3539 } 3540 3541 file = shmem_file_setup(name, 0, VM_NORESERVE); 3542 if (IS_ERR(file)) { 3543 error = PTR_ERR(file); 3544 goto err_fd; 3545 } 3546 info = SHMEM_I(file_inode(file)); 3547 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; 3548 file->f_flags |= O_RDWR | O_LARGEFILE; 3549 if (flags & MFD_ALLOW_SEALING) 3550 info->seals &= ~F_SEAL_SEAL; 3551 3552 fd_install(fd, file); 3553 kfree(name); 3554 return fd; 3555 3556 err_fd: 3557 put_unused_fd(fd); 3558 err_name: 3559 kfree(name); 3560 return error; 3561 } 3562 3563 #endif /* CONFIG_TMPFS */ 3564 3565 static void shmem_put_super(struct super_block *sb) 3566 { 3567 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3568 3569 percpu_counter_destroy(&sbinfo->used_blocks); 3570 mpol_put(sbinfo->mpol); 3571 kfree(sbinfo); 3572 sb->s_fs_info = NULL; 3573 } 3574 3575 int shmem_fill_super(struct super_block *sb, void *data, int silent) 3576 { 3577 struct inode *inode; 3578 struct shmem_sb_info *sbinfo; 3579 int err = -ENOMEM; 3580 3581 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3582 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3583 L1_CACHE_BYTES), GFP_KERNEL); 3584 if (!sbinfo) 3585 return -ENOMEM; 3586 3587 sbinfo->mode = S_IRWXUGO | S_ISVTX; 3588 sbinfo->uid = current_fsuid(); 3589 sbinfo->gid = current_fsgid(); 3590 sb->s_fs_info = sbinfo; 3591 3592 #ifdef CONFIG_TMPFS 3593 /* 3594 * Per default we only allow half of the physical ram per 3595 * tmpfs instance, limiting inodes to one per page of lowmem; 3596 * but the internal instance is left unlimited. 3597 */ 3598 if (!(sb->s_flags & MS_KERNMOUNT)) { 3599 sbinfo->max_blocks = shmem_default_max_blocks(); 3600 sbinfo->max_inodes = shmem_default_max_inodes(); 3601 if (shmem_parse_options(data, sbinfo, false)) { 3602 err = -EINVAL; 3603 goto failed; 3604 } 3605 } else { 3606 sb->s_flags |= MS_NOUSER; 3607 } 3608 sb->s_export_op = &shmem_export_ops; 3609 sb->s_flags |= MS_NOSEC; 3610 #else 3611 sb->s_flags |= MS_NOUSER; 3612 #endif 3613 3614 spin_lock_init(&sbinfo->stat_lock); 3615 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3616 goto failed; 3617 sbinfo->free_inodes = sbinfo->max_inodes; 3618 spin_lock_init(&sbinfo->shrinklist_lock); 3619 INIT_LIST_HEAD(&sbinfo->shrinklist); 3620 3621 sb->s_maxbytes = MAX_LFS_FILESIZE; 3622 sb->s_blocksize = PAGE_SIZE; 3623 sb->s_blocksize_bits = PAGE_SHIFT; 3624 sb->s_magic = TMPFS_MAGIC; 3625 sb->s_op = &shmem_ops; 3626 sb->s_time_gran = 1; 3627 #ifdef CONFIG_TMPFS_XATTR 3628 sb->s_xattr = shmem_xattr_handlers; 3629 #endif 3630 #ifdef CONFIG_TMPFS_POSIX_ACL 3631 sb->s_flags |= MS_POSIXACL; 3632 #endif 3633 3634 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3635 if (!inode) 3636 goto failed; 3637 inode->i_uid = sbinfo->uid; 3638 inode->i_gid = sbinfo->gid; 3639 sb->s_root = d_make_root(inode); 3640 if (!sb->s_root) 3641 goto failed; 3642 return 0; 3643 3644 failed: 3645 shmem_put_super(sb); 3646 return err; 3647 } 3648 3649 static struct kmem_cache *shmem_inode_cachep; 3650 3651 static struct inode *shmem_alloc_inode(struct super_block *sb) 3652 { 3653 struct shmem_inode_info *info; 3654 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3655 if (!info) 3656 return NULL; 3657 return &info->vfs_inode; 3658 } 3659 3660 static void shmem_destroy_callback(struct rcu_head *head) 3661 { 3662 struct inode *inode = container_of(head, struct inode, i_rcu); 3663 if (S_ISLNK(inode->i_mode)) 3664 kfree(inode->i_link); 3665 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3666 } 3667 3668 static void shmem_destroy_inode(struct inode *inode) 3669 { 3670 if (S_ISREG(inode->i_mode)) 3671 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3672 call_rcu(&inode->i_rcu, shmem_destroy_callback); 3673 } 3674 3675 static void shmem_init_inode(void *foo) 3676 { 3677 struct shmem_inode_info *info = foo; 3678 inode_init_once(&info->vfs_inode); 3679 } 3680 3681 static int shmem_init_inodecache(void) 3682 { 3683 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3684 sizeof(struct shmem_inode_info), 3685 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3686 return 0; 3687 } 3688 3689 static void shmem_destroy_inodecache(void) 3690 { 3691 kmem_cache_destroy(shmem_inode_cachep); 3692 } 3693 3694 static const struct address_space_operations shmem_aops = { 3695 .writepage = shmem_writepage, 3696 .set_page_dirty = __set_page_dirty_no_writeback, 3697 #ifdef CONFIG_TMPFS 3698 .write_begin = shmem_write_begin, 3699 .write_end = shmem_write_end, 3700 #endif 3701 #ifdef CONFIG_MIGRATION 3702 .migratepage = migrate_page, 3703 #endif 3704 .error_remove_page = generic_error_remove_page, 3705 }; 3706 3707 static const struct file_operations shmem_file_operations = { 3708 .mmap = shmem_mmap, 3709 .get_unmapped_area = shmem_get_unmapped_area, 3710 #ifdef CONFIG_TMPFS 3711 .llseek = shmem_file_llseek, 3712 .read_iter = shmem_file_read_iter, 3713 .write_iter = generic_file_write_iter, 3714 .fsync = noop_fsync, 3715 .splice_read = generic_file_splice_read, 3716 .splice_write = iter_file_splice_write, 3717 .fallocate = shmem_fallocate, 3718 #endif 3719 }; 3720 3721 static const struct inode_operations shmem_inode_operations = { 3722 .getattr = shmem_getattr, 3723 .setattr = shmem_setattr, 3724 #ifdef CONFIG_TMPFS_XATTR 3725 .listxattr = shmem_listxattr, 3726 .set_acl = simple_set_acl, 3727 #endif 3728 }; 3729 3730 static const struct inode_operations shmem_dir_inode_operations = { 3731 #ifdef CONFIG_TMPFS 3732 .create = shmem_create, 3733 .lookup = simple_lookup, 3734 .link = shmem_link, 3735 .unlink = shmem_unlink, 3736 .symlink = shmem_symlink, 3737 .mkdir = shmem_mkdir, 3738 .rmdir = shmem_rmdir, 3739 .mknod = shmem_mknod, 3740 .rename = shmem_rename2, 3741 .tmpfile = shmem_tmpfile, 3742 #endif 3743 #ifdef CONFIG_TMPFS_XATTR 3744 .listxattr = shmem_listxattr, 3745 #endif 3746 #ifdef CONFIG_TMPFS_POSIX_ACL 3747 .setattr = shmem_setattr, 3748 .set_acl = simple_set_acl, 3749 #endif 3750 }; 3751 3752 static const struct inode_operations shmem_special_inode_operations = { 3753 #ifdef CONFIG_TMPFS_XATTR 3754 .listxattr = shmem_listxattr, 3755 #endif 3756 #ifdef CONFIG_TMPFS_POSIX_ACL 3757 .setattr = shmem_setattr, 3758 .set_acl = simple_set_acl, 3759 #endif 3760 }; 3761 3762 static const struct super_operations shmem_ops = { 3763 .alloc_inode = shmem_alloc_inode, 3764 .destroy_inode = shmem_destroy_inode, 3765 #ifdef CONFIG_TMPFS 3766 .statfs = shmem_statfs, 3767 .remount_fs = shmem_remount_fs, 3768 .show_options = shmem_show_options, 3769 #endif 3770 .evict_inode = shmem_evict_inode, 3771 .drop_inode = generic_delete_inode, 3772 .put_super = shmem_put_super, 3773 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3774 .nr_cached_objects = shmem_unused_huge_count, 3775 .free_cached_objects = shmem_unused_huge_scan, 3776 #endif 3777 }; 3778 3779 static const struct vm_operations_struct shmem_vm_ops = { 3780 .fault = shmem_fault, 3781 .map_pages = filemap_map_pages, 3782 #ifdef CONFIG_NUMA 3783 .set_policy = shmem_set_policy, 3784 .get_policy = shmem_get_policy, 3785 #endif 3786 }; 3787 3788 static struct dentry *shmem_mount(struct file_system_type *fs_type, 3789 int flags, const char *dev_name, void *data) 3790 { 3791 return mount_nodev(fs_type, flags, data, shmem_fill_super); 3792 } 3793 3794 static struct file_system_type shmem_fs_type = { 3795 .owner = THIS_MODULE, 3796 .name = "tmpfs", 3797 .mount = shmem_mount, 3798 .kill_sb = kill_litter_super, 3799 .fs_flags = FS_USERNS_MOUNT, 3800 }; 3801 3802 int __init shmem_init(void) 3803 { 3804 int error; 3805 3806 /* If rootfs called this, don't re-init */ 3807 if (shmem_inode_cachep) 3808 return 0; 3809 3810 error = shmem_init_inodecache(); 3811 if (error) 3812 goto out3; 3813 3814 error = register_filesystem(&shmem_fs_type); 3815 if (error) { 3816 pr_err("Could not register tmpfs\n"); 3817 goto out2; 3818 } 3819 3820 shm_mnt = kern_mount(&shmem_fs_type); 3821 if (IS_ERR(shm_mnt)) { 3822 error = PTR_ERR(shm_mnt); 3823 pr_err("Could not kern_mount tmpfs\n"); 3824 goto out1; 3825 } 3826 3827 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3828 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY) 3829 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3830 else 3831 shmem_huge = 0; /* just in case it was patched */ 3832 #endif 3833 return 0; 3834 3835 out1: 3836 unregister_filesystem(&shmem_fs_type); 3837 out2: 3838 shmem_destroy_inodecache(); 3839 out3: 3840 shm_mnt = ERR_PTR(error); 3841 return error; 3842 } 3843 3844 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) 3845 static ssize_t shmem_enabled_show(struct kobject *kobj, 3846 struct kobj_attribute *attr, char *buf) 3847 { 3848 int values[] = { 3849 SHMEM_HUGE_ALWAYS, 3850 SHMEM_HUGE_WITHIN_SIZE, 3851 SHMEM_HUGE_ADVISE, 3852 SHMEM_HUGE_NEVER, 3853 SHMEM_HUGE_DENY, 3854 SHMEM_HUGE_FORCE, 3855 }; 3856 int i, count; 3857 3858 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { 3859 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; 3860 3861 count += sprintf(buf + count, fmt, 3862 shmem_format_huge(values[i])); 3863 } 3864 buf[count - 1] = '\n'; 3865 return count; 3866 } 3867 3868 static ssize_t shmem_enabled_store(struct kobject *kobj, 3869 struct kobj_attribute *attr, const char *buf, size_t count) 3870 { 3871 char tmp[16]; 3872 int huge; 3873 3874 if (count + 1 > sizeof(tmp)) 3875 return -EINVAL; 3876 memcpy(tmp, buf, count); 3877 tmp[count] = '\0'; 3878 if (count && tmp[count - 1] == '\n') 3879 tmp[count - 1] = '\0'; 3880 3881 huge = shmem_parse_huge(tmp); 3882 if (huge == -EINVAL) 3883 return -EINVAL; 3884 if (!has_transparent_hugepage() && 3885 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3886 return -EINVAL; 3887 3888 shmem_huge = huge; 3889 if (shmem_huge < SHMEM_HUGE_DENY) 3890 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3891 return count; 3892 } 3893 3894 struct kobj_attribute shmem_enabled_attr = 3895 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3896 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ 3897 3898 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3899 bool shmem_huge_enabled(struct vm_area_struct *vma) 3900 { 3901 struct inode *inode = file_inode(vma->vm_file); 3902 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3903 loff_t i_size; 3904 pgoff_t off; 3905 3906 if (shmem_huge == SHMEM_HUGE_FORCE) 3907 return true; 3908 if (shmem_huge == SHMEM_HUGE_DENY) 3909 return false; 3910 switch (sbinfo->huge) { 3911 case SHMEM_HUGE_NEVER: 3912 return false; 3913 case SHMEM_HUGE_ALWAYS: 3914 return true; 3915 case SHMEM_HUGE_WITHIN_SIZE: 3916 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); 3917 i_size = round_up(i_size_read(inode), PAGE_SIZE); 3918 if (i_size >= HPAGE_PMD_SIZE && 3919 i_size >> PAGE_SHIFT >= off) 3920 return true; 3921 case SHMEM_HUGE_ADVISE: 3922 /* TODO: implement fadvise() hints */ 3923 return (vma->vm_flags & VM_HUGEPAGE); 3924 default: 3925 VM_BUG_ON(1); 3926 return false; 3927 } 3928 } 3929 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 3930 3931 #else /* !CONFIG_SHMEM */ 3932 3933 /* 3934 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 3935 * 3936 * This is intended for small system where the benefits of the full 3937 * shmem code (swap-backed and resource-limited) are outweighed by 3938 * their complexity. On systems without swap this code should be 3939 * effectively equivalent, but much lighter weight. 3940 */ 3941 3942 static struct file_system_type shmem_fs_type = { 3943 .name = "tmpfs", 3944 .mount = ramfs_mount, 3945 .kill_sb = kill_litter_super, 3946 .fs_flags = FS_USERNS_MOUNT, 3947 }; 3948 3949 int __init shmem_init(void) 3950 { 3951 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 3952 3953 shm_mnt = kern_mount(&shmem_fs_type); 3954 BUG_ON(IS_ERR(shm_mnt)); 3955 3956 return 0; 3957 } 3958 3959 int shmem_unuse(swp_entry_t swap, struct page *page) 3960 { 3961 return 0; 3962 } 3963 3964 int shmem_lock(struct file *file, int lock, struct user_struct *user) 3965 { 3966 return 0; 3967 } 3968 3969 void shmem_unlock_mapping(struct address_space *mapping) 3970 { 3971 } 3972 3973 #ifdef CONFIG_MMU 3974 unsigned long shmem_get_unmapped_area(struct file *file, 3975 unsigned long addr, unsigned long len, 3976 unsigned long pgoff, unsigned long flags) 3977 { 3978 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 3979 } 3980 #endif 3981 3982 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 3983 { 3984 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 3985 } 3986 EXPORT_SYMBOL_GPL(shmem_truncate_range); 3987 3988 #define shmem_vm_ops generic_file_vm_ops 3989 #define shmem_file_operations ramfs_file_operations 3990 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 3991 #define shmem_acct_size(flags, size) 0 3992 #define shmem_unacct_size(flags, size) do {} while (0) 3993 3994 #endif /* CONFIG_SHMEM */ 3995 3996 /* common code */ 3997 3998 static const struct dentry_operations anon_ops = { 3999 .d_dname = simple_dname 4000 }; 4001 4002 static struct file *__shmem_file_setup(const char *name, loff_t size, 4003 unsigned long flags, unsigned int i_flags) 4004 { 4005 struct file *res; 4006 struct inode *inode; 4007 struct path path; 4008 struct super_block *sb; 4009 struct qstr this; 4010 4011 if (IS_ERR(shm_mnt)) 4012 return ERR_CAST(shm_mnt); 4013 4014 if (size < 0 || size > MAX_LFS_FILESIZE) 4015 return ERR_PTR(-EINVAL); 4016 4017 if (shmem_acct_size(flags, size)) 4018 return ERR_PTR(-ENOMEM); 4019 4020 res = ERR_PTR(-ENOMEM); 4021 this.name = name; 4022 this.len = strlen(name); 4023 this.hash = 0; /* will go */ 4024 sb = shm_mnt->mnt_sb; 4025 path.mnt = mntget(shm_mnt); 4026 path.dentry = d_alloc_pseudo(sb, &this); 4027 if (!path.dentry) 4028 goto put_memory; 4029 d_set_d_op(path.dentry, &anon_ops); 4030 4031 res = ERR_PTR(-ENOSPC); 4032 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 4033 if (!inode) 4034 goto put_memory; 4035 4036 inode->i_flags |= i_flags; 4037 d_instantiate(path.dentry, inode); 4038 inode->i_size = size; 4039 clear_nlink(inode); /* It is unlinked */ 4040 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4041 if (IS_ERR(res)) 4042 goto put_path; 4043 4044 res = alloc_file(&path, FMODE_WRITE | FMODE_READ, 4045 &shmem_file_operations); 4046 if (IS_ERR(res)) 4047 goto put_path; 4048 4049 return res; 4050 4051 put_memory: 4052 shmem_unacct_size(flags, size); 4053 put_path: 4054 path_put(&path); 4055 return res; 4056 } 4057 4058 /** 4059 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4060 * kernel internal. There will be NO LSM permission checks against the 4061 * underlying inode. So users of this interface must do LSM checks at a 4062 * higher layer. The users are the big_key and shm implementations. LSM 4063 * checks are provided at the key or shm level rather than the inode. 4064 * @name: name for dentry (to be seen in /proc/<pid>/maps 4065 * @size: size to be set for the file 4066 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4067 */ 4068 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4069 { 4070 return __shmem_file_setup(name, size, flags, S_PRIVATE); 4071 } 4072 4073 /** 4074 * shmem_file_setup - get an unlinked file living in tmpfs 4075 * @name: name for dentry (to be seen in /proc/<pid>/maps 4076 * @size: size to be set for the file 4077 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4078 */ 4079 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4080 { 4081 return __shmem_file_setup(name, size, flags, 0); 4082 } 4083 EXPORT_SYMBOL_GPL(shmem_file_setup); 4084 4085 /** 4086 * shmem_zero_setup - setup a shared anonymous mapping 4087 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 4088 */ 4089 int shmem_zero_setup(struct vm_area_struct *vma) 4090 { 4091 struct file *file; 4092 loff_t size = vma->vm_end - vma->vm_start; 4093 4094 /* 4095 * Cloning a new file under mmap_sem leads to a lock ordering conflict 4096 * between XFS directory reading and selinux: since this file is only 4097 * accessible to the user through its mapping, use S_PRIVATE flag to 4098 * bypass file security, in the same way as shmem_kernel_file_setup(). 4099 */ 4100 file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE); 4101 if (IS_ERR(file)) 4102 return PTR_ERR(file); 4103 4104 if (vma->vm_file) 4105 fput(vma->vm_file); 4106 vma->vm_file = file; 4107 vma->vm_ops = &shmem_vm_ops; 4108 4109 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 4110 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4111 (vma->vm_end & HPAGE_PMD_MASK)) { 4112 khugepaged_enter(vma, vma->vm_flags); 4113 } 4114 4115 return 0; 4116 } 4117 4118 /** 4119 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4120 * @mapping: the page's address_space 4121 * @index: the page index 4122 * @gfp: the page allocator flags to use if allocating 4123 * 4124 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4125 * with any new page allocations done using the specified allocation flags. 4126 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4127 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4128 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4129 * 4130 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4131 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4132 */ 4133 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4134 pgoff_t index, gfp_t gfp) 4135 { 4136 #ifdef CONFIG_SHMEM 4137 struct inode *inode = mapping->host; 4138 struct page *page; 4139 int error; 4140 4141 BUG_ON(mapping->a_ops != &shmem_aops); 4142 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4143 gfp, NULL, NULL); 4144 if (error) 4145 page = ERR_PTR(error); 4146 else 4147 unlock_page(page); 4148 return page; 4149 #else 4150 /* 4151 * The tiny !SHMEM case uses ramfs without swap 4152 */ 4153 return read_cache_page_gfp(mapping, index, gfp); 4154 #endif 4155 } 4156 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4157