1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/sched/signal.h> 33 #include <linux/export.h> 34 #include <linux/swap.h> 35 #include <linux/uio.h> 36 #include <linux/khugepaged.h> 37 38 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */ 39 40 static struct vfsmount *shm_mnt; 41 42 #ifdef CONFIG_SHMEM 43 /* 44 * This virtual memory filesystem is heavily based on the ramfs. It 45 * extends ramfs by the ability to use swap and honor resource limits 46 * which makes it a completely usable filesystem. 47 */ 48 49 #include <linux/xattr.h> 50 #include <linux/exportfs.h> 51 #include <linux/posix_acl.h> 52 #include <linux/posix_acl_xattr.h> 53 #include <linux/mman.h> 54 #include <linux/string.h> 55 #include <linux/slab.h> 56 #include <linux/backing-dev.h> 57 #include <linux/shmem_fs.h> 58 #include <linux/writeback.h> 59 #include <linux/blkdev.h> 60 #include <linux/pagevec.h> 61 #include <linux/percpu_counter.h> 62 #include <linux/falloc.h> 63 #include <linux/splice.h> 64 #include <linux/security.h> 65 #include <linux/swapops.h> 66 #include <linux/mempolicy.h> 67 #include <linux/namei.h> 68 #include <linux/ctype.h> 69 #include <linux/migrate.h> 70 #include <linux/highmem.h> 71 #include <linux/seq_file.h> 72 #include <linux/magic.h> 73 #include <linux/syscalls.h> 74 #include <linux/fcntl.h> 75 #include <uapi/linux/memfd.h> 76 #include <linux/userfaultfd_k.h> 77 #include <linux/rmap.h> 78 #include <linux/uuid.h> 79 80 #include <linux/uaccess.h> 81 #include <asm/pgtable.h> 82 83 #include "internal.h" 84 85 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 86 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 87 88 /* Pretend that each entry is of this size in directory's i_size */ 89 #define BOGO_DIRENT_SIZE 20 90 91 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 92 #define SHORT_SYMLINK_LEN 128 93 94 /* 95 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 96 * inode->i_private (with i_mutex making sure that it has only one user at 97 * a time): we would prefer not to enlarge the shmem inode just for that. 98 */ 99 struct shmem_falloc { 100 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 101 pgoff_t start; /* start of range currently being fallocated */ 102 pgoff_t next; /* the next page offset to be fallocated */ 103 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 104 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 105 }; 106 107 #ifdef CONFIG_TMPFS 108 static unsigned long shmem_default_max_blocks(void) 109 { 110 return totalram_pages / 2; 111 } 112 113 static unsigned long shmem_default_max_inodes(void) 114 { 115 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 116 } 117 #endif 118 119 static bool shmem_should_replace_page(struct page *page, gfp_t gfp); 120 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 121 struct shmem_inode_info *info, pgoff_t index); 122 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 123 struct page **pagep, enum sgp_type sgp, 124 gfp_t gfp, struct vm_area_struct *vma, 125 struct vm_fault *vmf, int *fault_type); 126 127 int shmem_getpage(struct inode *inode, pgoff_t index, 128 struct page **pagep, enum sgp_type sgp) 129 { 130 return shmem_getpage_gfp(inode, index, pagep, sgp, 131 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); 132 } 133 134 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 135 { 136 return sb->s_fs_info; 137 } 138 139 /* 140 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 141 * for shared memory and for shared anonymous (/dev/zero) mappings 142 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 143 * consistent with the pre-accounting of private mappings ... 144 */ 145 static inline int shmem_acct_size(unsigned long flags, loff_t size) 146 { 147 return (flags & VM_NORESERVE) ? 148 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 149 } 150 151 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 152 { 153 if (!(flags & VM_NORESERVE)) 154 vm_unacct_memory(VM_ACCT(size)); 155 } 156 157 static inline int shmem_reacct_size(unsigned long flags, 158 loff_t oldsize, loff_t newsize) 159 { 160 if (!(flags & VM_NORESERVE)) { 161 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 162 return security_vm_enough_memory_mm(current->mm, 163 VM_ACCT(newsize) - VM_ACCT(oldsize)); 164 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 165 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 166 } 167 return 0; 168 } 169 170 /* 171 * ... whereas tmpfs objects are accounted incrementally as 172 * pages are allocated, in order to allow large sparse files. 173 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 174 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 175 */ 176 static inline int shmem_acct_block(unsigned long flags, long pages) 177 { 178 if (!(flags & VM_NORESERVE)) 179 return 0; 180 181 return security_vm_enough_memory_mm(current->mm, 182 pages * VM_ACCT(PAGE_SIZE)); 183 } 184 185 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 186 { 187 if (flags & VM_NORESERVE) 188 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 189 } 190 191 static const struct super_operations shmem_ops; 192 static const struct address_space_operations shmem_aops; 193 static const struct file_operations shmem_file_operations; 194 static const struct inode_operations shmem_inode_operations; 195 static const struct inode_operations shmem_dir_inode_operations; 196 static const struct inode_operations shmem_special_inode_operations; 197 static const struct vm_operations_struct shmem_vm_ops; 198 static struct file_system_type shmem_fs_type; 199 200 bool vma_is_shmem(struct vm_area_struct *vma) 201 { 202 return vma->vm_ops == &shmem_vm_ops; 203 } 204 205 static LIST_HEAD(shmem_swaplist); 206 static DEFINE_MUTEX(shmem_swaplist_mutex); 207 208 static int shmem_reserve_inode(struct super_block *sb) 209 { 210 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 211 if (sbinfo->max_inodes) { 212 spin_lock(&sbinfo->stat_lock); 213 if (!sbinfo->free_inodes) { 214 spin_unlock(&sbinfo->stat_lock); 215 return -ENOSPC; 216 } 217 sbinfo->free_inodes--; 218 spin_unlock(&sbinfo->stat_lock); 219 } 220 return 0; 221 } 222 223 static void shmem_free_inode(struct super_block *sb) 224 { 225 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 226 if (sbinfo->max_inodes) { 227 spin_lock(&sbinfo->stat_lock); 228 sbinfo->free_inodes++; 229 spin_unlock(&sbinfo->stat_lock); 230 } 231 } 232 233 /** 234 * shmem_recalc_inode - recalculate the block usage of an inode 235 * @inode: inode to recalc 236 * 237 * We have to calculate the free blocks since the mm can drop 238 * undirtied hole pages behind our back. 239 * 240 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 241 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 242 * 243 * It has to be called with the spinlock held. 244 */ 245 static void shmem_recalc_inode(struct inode *inode) 246 { 247 struct shmem_inode_info *info = SHMEM_I(inode); 248 long freed; 249 250 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 251 if (freed > 0) { 252 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 253 if (sbinfo->max_blocks) 254 percpu_counter_add(&sbinfo->used_blocks, -freed); 255 info->alloced -= freed; 256 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 257 shmem_unacct_blocks(info->flags, freed); 258 } 259 } 260 261 bool shmem_charge(struct inode *inode, long pages) 262 { 263 struct shmem_inode_info *info = SHMEM_I(inode); 264 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 265 unsigned long flags; 266 267 if (shmem_acct_block(info->flags, pages)) 268 return false; 269 spin_lock_irqsave(&info->lock, flags); 270 info->alloced += pages; 271 inode->i_blocks += pages * BLOCKS_PER_PAGE; 272 shmem_recalc_inode(inode); 273 spin_unlock_irqrestore(&info->lock, flags); 274 inode->i_mapping->nrpages += pages; 275 276 if (!sbinfo->max_blocks) 277 return true; 278 if (percpu_counter_compare(&sbinfo->used_blocks, 279 sbinfo->max_blocks - pages) > 0) { 280 inode->i_mapping->nrpages -= pages; 281 spin_lock_irqsave(&info->lock, flags); 282 info->alloced -= pages; 283 shmem_recalc_inode(inode); 284 spin_unlock_irqrestore(&info->lock, flags); 285 shmem_unacct_blocks(info->flags, pages); 286 return false; 287 } 288 percpu_counter_add(&sbinfo->used_blocks, pages); 289 return true; 290 } 291 292 void shmem_uncharge(struct inode *inode, long pages) 293 { 294 struct shmem_inode_info *info = SHMEM_I(inode); 295 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 296 unsigned long flags; 297 298 spin_lock_irqsave(&info->lock, flags); 299 info->alloced -= pages; 300 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 301 shmem_recalc_inode(inode); 302 spin_unlock_irqrestore(&info->lock, flags); 303 304 if (sbinfo->max_blocks) 305 percpu_counter_sub(&sbinfo->used_blocks, pages); 306 shmem_unacct_blocks(info->flags, pages); 307 } 308 309 /* 310 * Replace item expected in radix tree by a new item, while holding tree lock. 311 */ 312 static int shmem_radix_tree_replace(struct address_space *mapping, 313 pgoff_t index, void *expected, void *replacement) 314 { 315 struct radix_tree_node *node; 316 void **pslot; 317 void *item; 318 319 VM_BUG_ON(!expected); 320 VM_BUG_ON(!replacement); 321 item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); 322 if (!item) 323 return -ENOENT; 324 if (item != expected) 325 return -ENOENT; 326 __radix_tree_replace(&mapping->page_tree, node, pslot, 327 replacement, NULL, NULL); 328 return 0; 329 } 330 331 /* 332 * Sometimes, before we decide whether to proceed or to fail, we must check 333 * that an entry was not already brought back from swap by a racing thread. 334 * 335 * Checking page is not enough: by the time a SwapCache page is locked, it 336 * might be reused, and again be SwapCache, using the same swap as before. 337 */ 338 static bool shmem_confirm_swap(struct address_space *mapping, 339 pgoff_t index, swp_entry_t swap) 340 { 341 void *item; 342 343 rcu_read_lock(); 344 item = radix_tree_lookup(&mapping->page_tree, index); 345 rcu_read_unlock(); 346 return item == swp_to_radix_entry(swap); 347 } 348 349 /* 350 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 351 * 352 * SHMEM_HUGE_NEVER: 353 * disables huge pages for the mount; 354 * SHMEM_HUGE_ALWAYS: 355 * enables huge pages for the mount; 356 * SHMEM_HUGE_WITHIN_SIZE: 357 * only allocate huge pages if the page will be fully within i_size, 358 * also respect fadvise()/madvise() hints; 359 * SHMEM_HUGE_ADVISE: 360 * only allocate huge pages if requested with fadvise()/madvise(); 361 */ 362 363 #define SHMEM_HUGE_NEVER 0 364 #define SHMEM_HUGE_ALWAYS 1 365 #define SHMEM_HUGE_WITHIN_SIZE 2 366 #define SHMEM_HUGE_ADVISE 3 367 368 /* 369 * Special values. 370 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 371 * 372 * SHMEM_HUGE_DENY: 373 * disables huge on shm_mnt and all mounts, for emergency use; 374 * SHMEM_HUGE_FORCE: 375 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 376 * 377 */ 378 #define SHMEM_HUGE_DENY (-1) 379 #define SHMEM_HUGE_FORCE (-2) 380 381 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 382 /* ifdef here to avoid bloating shmem.o when not necessary */ 383 384 int shmem_huge __read_mostly; 385 386 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 387 static int shmem_parse_huge(const char *str) 388 { 389 if (!strcmp(str, "never")) 390 return SHMEM_HUGE_NEVER; 391 if (!strcmp(str, "always")) 392 return SHMEM_HUGE_ALWAYS; 393 if (!strcmp(str, "within_size")) 394 return SHMEM_HUGE_WITHIN_SIZE; 395 if (!strcmp(str, "advise")) 396 return SHMEM_HUGE_ADVISE; 397 if (!strcmp(str, "deny")) 398 return SHMEM_HUGE_DENY; 399 if (!strcmp(str, "force")) 400 return SHMEM_HUGE_FORCE; 401 return -EINVAL; 402 } 403 404 static const char *shmem_format_huge(int huge) 405 { 406 switch (huge) { 407 case SHMEM_HUGE_NEVER: 408 return "never"; 409 case SHMEM_HUGE_ALWAYS: 410 return "always"; 411 case SHMEM_HUGE_WITHIN_SIZE: 412 return "within_size"; 413 case SHMEM_HUGE_ADVISE: 414 return "advise"; 415 case SHMEM_HUGE_DENY: 416 return "deny"; 417 case SHMEM_HUGE_FORCE: 418 return "force"; 419 default: 420 VM_BUG_ON(1); 421 return "bad_val"; 422 } 423 } 424 #endif 425 426 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 427 struct shrink_control *sc, unsigned long nr_to_split) 428 { 429 LIST_HEAD(list), *pos, *next; 430 LIST_HEAD(to_remove); 431 struct inode *inode; 432 struct shmem_inode_info *info; 433 struct page *page; 434 unsigned long batch = sc ? sc->nr_to_scan : 128; 435 int removed = 0, split = 0; 436 437 if (list_empty(&sbinfo->shrinklist)) 438 return SHRINK_STOP; 439 440 spin_lock(&sbinfo->shrinklist_lock); 441 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 442 info = list_entry(pos, struct shmem_inode_info, shrinklist); 443 444 /* pin the inode */ 445 inode = igrab(&info->vfs_inode); 446 447 /* inode is about to be evicted */ 448 if (!inode) { 449 list_del_init(&info->shrinklist); 450 removed++; 451 goto next; 452 } 453 454 /* Check if there's anything to gain */ 455 if (round_up(inode->i_size, PAGE_SIZE) == 456 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 457 list_move(&info->shrinklist, &to_remove); 458 removed++; 459 goto next; 460 } 461 462 list_move(&info->shrinklist, &list); 463 next: 464 if (!--batch) 465 break; 466 } 467 spin_unlock(&sbinfo->shrinklist_lock); 468 469 list_for_each_safe(pos, next, &to_remove) { 470 info = list_entry(pos, struct shmem_inode_info, shrinklist); 471 inode = &info->vfs_inode; 472 list_del_init(&info->shrinklist); 473 iput(inode); 474 } 475 476 list_for_each_safe(pos, next, &list) { 477 int ret; 478 479 info = list_entry(pos, struct shmem_inode_info, shrinklist); 480 inode = &info->vfs_inode; 481 482 if (nr_to_split && split >= nr_to_split) { 483 iput(inode); 484 continue; 485 } 486 487 page = find_lock_page(inode->i_mapping, 488 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 489 if (!page) 490 goto drop; 491 492 if (!PageTransHuge(page)) { 493 unlock_page(page); 494 put_page(page); 495 goto drop; 496 } 497 498 ret = split_huge_page(page); 499 unlock_page(page); 500 put_page(page); 501 502 if (ret) { 503 /* split failed: leave it on the list */ 504 iput(inode); 505 continue; 506 } 507 508 split++; 509 drop: 510 list_del_init(&info->shrinklist); 511 removed++; 512 iput(inode); 513 } 514 515 spin_lock(&sbinfo->shrinklist_lock); 516 list_splice_tail(&list, &sbinfo->shrinklist); 517 sbinfo->shrinklist_len -= removed; 518 spin_unlock(&sbinfo->shrinklist_lock); 519 520 return split; 521 } 522 523 static long shmem_unused_huge_scan(struct super_block *sb, 524 struct shrink_control *sc) 525 { 526 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 527 528 if (!READ_ONCE(sbinfo->shrinklist_len)) 529 return SHRINK_STOP; 530 531 return shmem_unused_huge_shrink(sbinfo, sc, 0); 532 } 533 534 static long shmem_unused_huge_count(struct super_block *sb, 535 struct shrink_control *sc) 536 { 537 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 538 return READ_ONCE(sbinfo->shrinklist_len); 539 } 540 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 541 542 #define shmem_huge SHMEM_HUGE_DENY 543 544 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 545 struct shrink_control *sc, unsigned long nr_to_split) 546 { 547 return 0; 548 } 549 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 550 551 /* 552 * Like add_to_page_cache_locked, but error if expected item has gone. 553 */ 554 static int shmem_add_to_page_cache(struct page *page, 555 struct address_space *mapping, 556 pgoff_t index, void *expected) 557 { 558 int error, nr = hpage_nr_pages(page); 559 560 VM_BUG_ON_PAGE(PageTail(page), page); 561 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 562 VM_BUG_ON_PAGE(!PageLocked(page), page); 563 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 564 VM_BUG_ON(expected && PageTransHuge(page)); 565 566 page_ref_add(page, nr); 567 page->mapping = mapping; 568 page->index = index; 569 570 spin_lock_irq(&mapping->tree_lock); 571 if (PageTransHuge(page)) { 572 void __rcu **results; 573 pgoff_t idx; 574 int i; 575 576 error = 0; 577 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 578 &results, &idx, index, 1) && 579 idx < index + HPAGE_PMD_NR) { 580 error = -EEXIST; 581 } 582 583 if (!error) { 584 for (i = 0; i < HPAGE_PMD_NR; i++) { 585 error = radix_tree_insert(&mapping->page_tree, 586 index + i, page + i); 587 VM_BUG_ON(error); 588 } 589 count_vm_event(THP_FILE_ALLOC); 590 } 591 } else if (!expected) { 592 error = radix_tree_insert(&mapping->page_tree, index, page); 593 } else { 594 error = shmem_radix_tree_replace(mapping, index, expected, 595 page); 596 } 597 598 if (!error) { 599 mapping->nrpages += nr; 600 if (PageTransHuge(page)) 601 __inc_node_page_state(page, NR_SHMEM_THPS); 602 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 603 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 604 spin_unlock_irq(&mapping->tree_lock); 605 } else { 606 page->mapping = NULL; 607 spin_unlock_irq(&mapping->tree_lock); 608 page_ref_sub(page, nr); 609 } 610 return error; 611 } 612 613 /* 614 * Like delete_from_page_cache, but substitutes swap for page. 615 */ 616 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 617 { 618 struct address_space *mapping = page->mapping; 619 int error; 620 621 VM_BUG_ON_PAGE(PageCompound(page), page); 622 623 spin_lock_irq(&mapping->tree_lock); 624 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 625 page->mapping = NULL; 626 mapping->nrpages--; 627 __dec_node_page_state(page, NR_FILE_PAGES); 628 __dec_node_page_state(page, NR_SHMEM); 629 spin_unlock_irq(&mapping->tree_lock); 630 put_page(page); 631 BUG_ON(error); 632 } 633 634 /* 635 * Remove swap entry from radix tree, free the swap and its page cache. 636 */ 637 static int shmem_free_swap(struct address_space *mapping, 638 pgoff_t index, void *radswap) 639 { 640 void *old; 641 642 spin_lock_irq(&mapping->tree_lock); 643 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 644 spin_unlock_irq(&mapping->tree_lock); 645 if (old != radswap) 646 return -ENOENT; 647 free_swap_and_cache(radix_to_swp_entry(radswap)); 648 return 0; 649 } 650 651 /* 652 * Determine (in bytes) how many of the shmem object's pages mapped by the 653 * given offsets are swapped out. 654 * 655 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 656 * as long as the inode doesn't go away and racy results are not a problem. 657 */ 658 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 659 pgoff_t start, pgoff_t end) 660 { 661 struct radix_tree_iter iter; 662 void **slot; 663 struct page *page; 664 unsigned long swapped = 0; 665 666 rcu_read_lock(); 667 668 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 669 if (iter.index >= end) 670 break; 671 672 page = radix_tree_deref_slot(slot); 673 674 if (radix_tree_deref_retry(page)) { 675 slot = radix_tree_iter_retry(&iter); 676 continue; 677 } 678 679 if (radix_tree_exceptional_entry(page)) 680 swapped++; 681 682 if (need_resched()) { 683 slot = radix_tree_iter_resume(slot, &iter); 684 cond_resched_rcu(); 685 } 686 } 687 688 rcu_read_unlock(); 689 690 return swapped << PAGE_SHIFT; 691 } 692 693 /* 694 * Determine (in bytes) how many of the shmem object's pages mapped by the 695 * given vma is swapped out. 696 * 697 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 698 * as long as the inode doesn't go away and racy results are not a problem. 699 */ 700 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 701 { 702 struct inode *inode = file_inode(vma->vm_file); 703 struct shmem_inode_info *info = SHMEM_I(inode); 704 struct address_space *mapping = inode->i_mapping; 705 unsigned long swapped; 706 707 /* Be careful as we don't hold info->lock */ 708 swapped = READ_ONCE(info->swapped); 709 710 /* 711 * The easier cases are when the shmem object has nothing in swap, or 712 * the vma maps it whole. Then we can simply use the stats that we 713 * already track. 714 */ 715 if (!swapped) 716 return 0; 717 718 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 719 return swapped << PAGE_SHIFT; 720 721 /* Here comes the more involved part */ 722 return shmem_partial_swap_usage(mapping, 723 linear_page_index(vma, vma->vm_start), 724 linear_page_index(vma, vma->vm_end)); 725 } 726 727 /* 728 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 729 */ 730 void shmem_unlock_mapping(struct address_space *mapping) 731 { 732 struct pagevec pvec; 733 pgoff_t indices[PAGEVEC_SIZE]; 734 pgoff_t index = 0; 735 736 pagevec_init(&pvec, 0); 737 /* 738 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 739 */ 740 while (!mapping_unevictable(mapping)) { 741 /* 742 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it 743 * has finished, if it hits a row of PAGEVEC_SIZE swap entries. 744 */ 745 pvec.nr = find_get_entries(mapping, index, 746 PAGEVEC_SIZE, pvec.pages, indices); 747 if (!pvec.nr) 748 break; 749 index = indices[pvec.nr - 1] + 1; 750 pagevec_remove_exceptionals(&pvec); 751 check_move_unevictable_pages(pvec.pages, pvec.nr); 752 pagevec_release(&pvec); 753 cond_resched(); 754 } 755 } 756 757 /* 758 * Remove range of pages and swap entries from radix tree, and free them. 759 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 760 */ 761 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 762 bool unfalloc) 763 { 764 struct address_space *mapping = inode->i_mapping; 765 struct shmem_inode_info *info = SHMEM_I(inode); 766 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 767 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 768 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 769 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 770 struct pagevec pvec; 771 pgoff_t indices[PAGEVEC_SIZE]; 772 long nr_swaps_freed = 0; 773 pgoff_t index; 774 int i; 775 776 if (lend == -1) 777 end = -1; /* unsigned, so actually very big */ 778 779 pagevec_init(&pvec, 0); 780 index = start; 781 while (index < end) { 782 pvec.nr = find_get_entries(mapping, index, 783 min(end - index, (pgoff_t)PAGEVEC_SIZE), 784 pvec.pages, indices); 785 if (!pvec.nr) 786 break; 787 for (i = 0; i < pagevec_count(&pvec); i++) { 788 struct page *page = pvec.pages[i]; 789 790 index = indices[i]; 791 if (index >= end) 792 break; 793 794 if (radix_tree_exceptional_entry(page)) { 795 if (unfalloc) 796 continue; 797 nr_swaps_freed += !shmem_free_swap(mapping, 798 index, page); 799 continue; 800 } 801 802 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); 803 804 if (!trylock_page(page)) 805 continue; 806 807 if (PageTransTail(page)) { 808 /* Middle of THP: zero out the page */ 809 clear_highpage(page); 810 unlock_page(page); 811 continue; 812 } else if (PageTransHuge(page)) { 813 if (index == round_down(end, HPAGE_PMD_NR)) { 814 /* 815 * Range ends in the middle of THP: 816 * zero out the page 817 */ 818 clear_highpage(page); 819 unlock_page(page); 820 continue; 821 } 822 index += HPAGE_PMD_NR - 1; 823 i += HPAGE_PMD_NR - 1; 824 } 825 826 if (!unfalloc || !PageUptodate(page)) { 827 VM_BUG_ON_PAGE(PageTail(page), page); 828 if (page_mapping(page) == mapping) { 829 VM_BUG_ON_PAGE(PageWriteback(page), page); 830 truncate_inode_page(mapping, page); 831 } 832 } 833 unlock_page(page); 834 } 835 pagevec_remove_exceptionals(&pvec); 836 pagevec_release(&pvec); 837 cond_resched(); 838 index++; 839 } 840 841 if (partial_start) { 842 struct page *page = NULL; 843 shmem_getpage(inode, start - 1, &page, SGP_READ); 844 if (page) { 845 unsigned int top = PAGE_SIZE; 846 if (start > end) { 847 top = partial_end; 848 partial_end = 0; 849 } 850 zero_user_segment(page, partial_start, top); 851 set_page_dirty(page); 852 unlock_page(page); 853 put_page(page); 854 } 855 } 856 if (partial_end) { 857 struct page *page = NULL; 858 shmem_getpage(inode, end, &page, SGP_READ); 859 if (page) { 860 zero_user_segment(page, 0, partial_end); 861 set_page_dirty(page); 862 unlock_page(page); 863 put_page(page); 864 } 865 } 866 if (start >= end) 867 return; 868 869 index = start; 870 while (index < end) { 871 cond_resched(); 872 873 pvec.nr = find_get_entries(mapping, index, 874 min(end - index, (pgoff_t)PAGEVEC_SIZE), 875 pvec.pages, indices); 876 if (!pvec.nr) { 877 /* If all gone or hole-punch or unfalloc, we're done */ 878 if (index == start || end != -1) 879 break; 880 /* But if truncating, restart to make sure all gone */ 881 index = start; 882 continue; 883 } 884 for (i = 0; i < pagevec_count(&pvec); i++) { 885 struct page *page = pvec.pages[i]; 886 887 index = indices[i]; 888 if (index >= end) 889 break; 890 891 if (radix_tree_exceptional_entry(page)) { 892 if (unfalloc) 893 continue; 894 if (shmem_free_swap(mapping, index, page)) { 895 /* Swap was replaced by page: retry */ 896 index--; 897 break; 898 } 899 nr_swaps_freed++; 900 continue; 901 } 902 903 lock_page(page); 904 905 if (PageTransTail(page)) { 906 /* Middle of THP: zero out the page */ 907 clear_highpage(page); 908 unlock_page(page); 909 /* 910 * Partial thp truncate due 'start' in middle 911 * of THP: don't need to look on these pages 912 * again on !pvec.nr restart. 913 */ 914 if (index != round_down(end, HPAGE_PMD_NR)) 915 start++; 916 continue; 917 } else if (PageTransHuge(page)) { 918 if (index == round_down(end, HPAGE_PMD_NR)) { 919 /* 920 * Range ends in the middle of THP: 921 * zero out the page 922 */ 923 clear_highpage(page); 924 unlock_page(page); 925 continue; 926 } 927 index += HPAGE_PMD_NR - 1; 928 i += HPAGE_PMD_NR - 1; 929 } 930 931 if (!unfalloc || !PageUptodate(page)) { 932 VM_BUG_ON_PAGE(PageTail(page), page); 933 if (page_mapping(page) == mapping) { 934 VM_BUG_ON_PAGE(PageWriteback(page), page); 935 truncate_inode_page(mapping, page); 936 } else { 937 /* Page was replaced by swap: retry */ 938 unlock_page(page); 939 index--; 940 break; 941 } 942 } 943 unlock_page(page); 944 } 945 pagevec_remove_exceptionals(&pvec); 946 pagevec_release(&pvec); 947 index++; 948 } 949 950 spin_lock_irq(&info->lock); 951 info->swapped -= nr_swaps_freed; 952 shmem_recalc_inode(inode); 953 spin_unlock_irq(&info->lock); 954 } 955 956 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 957 { 958 shmem_undo_range(inode, lstart, lend, false); 959 inode->i_ctime = inode->i_mtime = current_time(inode); 960 } 961 EXPORT_SYMBOL_GPL(shmem_truncate_range); 962 963 static int shmem_getattr(const struct path *path, struct kstat *stat, 964 u32 request_mask, unsigned int query_flags) 965 { 966 struct inode *inode = path->dentry->d_inode; 967 struct shmem_inode_info *info = SHMEM_I(inode); 968 969 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 970 spin_lock_irq(&info->lock); 971 shmem_recalc_inode(inode); 972 spin_unlock_irq(&info->lock); 973 } 974 generic_fillattr(inode, stat); 975 return 0; 976 } 977 978 static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 979 { 980 struct inode *inode = d_inode(dentry); 981 struct shmem_inode_info *info = SHMEM_I(inode); 982 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 983 int error; 984 985 error = setattr_prepare(dentry, attr); 986 if (error) 987 return error; 988 989 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 990 loff_t oldsize = inode->i_size; 991 loff_t newsize = attr->ia_size; 992 993 /* protected by i_mutex */ 994 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 995 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 996 return -EPERM; 997 998 if (newsize != oldsize) { 999 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1000 oldsize, newsize); 1001 if (error) 1002 return error; 1003 i_size_write(inode, newsize); 1004 inode->i_ctime = inode->i_mtime = current_time(inode); 1005 } 1006 if (newsize <= oldsize) { 1007 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1008 if (oldsize > holebegin) 1009 unmap_mapping_range(inode->i_mapping, 1010 holebegin, 0, 1); 1011 if (info->alloced) 1012 shmem_truncate_range(inode, 1013 newsize, (loff_t)-1); 1014 /* unmap again to remove racily COWed private pages */ 1015 if (oldsize > holebegin) 1016 unmap_mapping_range(inode->i_mapping, 1017 holebegin, 0, 1); 1018 1019 /* 1020 * Part of the huge page can be beyond i_size: subject 1021 * to shrink under memory pressure. 1022 */ 1023 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 1024 spin_lock(&sbinfo->shrinklist_lock); 1025 if (list_empty(&info->shrinklist)) { 1026 list_add_tail(&info->shrinklist, 1027 &sbinfo->shrinklist); 1028 sbinfo->shrinklist_len++; 1029 } 1030 spin_unlock(&sbinfo->shrinklist_lock); 1031 } 1032 } 1033 } 1034 1035 setattr_copy(inode, attr); 1036 if (attr->ia_valid & ATTR_MODE) 1037 error = posix_acl_chmod(inode, inode->i_mode); 1038 return error; 1039 } 1040 1041 static void shmem_evict_inode(struct inode *inode) 1042 { 1043 struct shmem_inode_info *info = SHMEM_I(inode); 1044 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1045 1046 if (inode->i_mapping->a_ops == &shmem_aops) { 1047 shmem_unacct_size(info->flags, inode->i_size); 1048 inode->i_size = 0; 1049 shmem_truncate_range(inode, 0, (loff_t)-1); 1050 if (!list_empty(&info->shrinklist)) { 1051 spin_lock(&sbinfo->shrinklist_lock); 1052 if (!list_empty(&info->shrinklist)) { 1053 list_del_init(&info->shrinklist); 1054 sbinfo->shrinklist_len--; 1055 } 1056 spin_unlock(&sbinfo->shrinklist_lock); 1057 } 1058 if (!list_empty(&info->swaplist)) { 1059 mutex_lock(&shmem_swaplist_mutex); 1060 list_del_init(&info->swaplist); 1061 mutex_unlock(&shmem_swaplist_mutex); 1062 } 1063 } 1064 1065 simple_xattrs_free(&info->xattrs); 1066 WARN_ON(inode->i_blocks); 1067 shmem_free_inode(inode->i_sb); 1068 clear_inode(inode); 1069 } 1070 1071 static unsigned long find_swap_entry(struct radix_tree_root *root, void *item) 1072 { 1073 struct radix_tree_iter iter; 1074 void **slot; 1075 unsigned long found = -1; 1076 unsigned int checked = 0; 1077 1078 rcu_read_lock(); 1079 radix_tree_for_each_slot(slot, root, &iter, 0) { 1080 if (*slot == item) { 1081 found = iter.index; 1082 break; 1083 } 1084 checked++; 1085 if ((checked % 4096) != 0) 1086 continue; 1087 slot = radix_tree_iter_resume(slot, &iter); 1088 cond_resched_rcu(); 1089 } 1090 1091 rcu_read_unlock(); 1092 return found; 1093 } 1094 1095 /* 1096 * If swap found in inode, free it and move page from swapcache to filecache. 1097 */ 1098 static int shmem_unuse_inode(struct shmem_inode_info *info, 1099 swp_entry_t swap, struct page **pagep) 1100 { 1101 struct address_space *mapping = info->vfs_inode.i_mapping; 1102 void *radswap; 1103 pgoff_t index; 1104 gfp_t gfp; 1105 int error = 0; 1106 1107 radswap = swp_to_radix_entry(swap); 1108 index = find_swap_entry(&mapping->page_tree, radswap); 1109 if (index == -1) 1110 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1111 1112 /* 1113 * Move _head_ to start search for next from here. 1114 * But be careful: shmem_evict_inode checks list_empty without taking 1115 * mutex, and there's an instant in list_move_tail when info->swaplist 1116 * would appear empty, if it were the only one on shmem_swaplist. 1117 */ 1118 if (shmem_swaplist.next != &info->swaplist) 1119 list_move_tail(&shmem_swaplist, &info->swaplist); 1120 1121 gfp = mapping_gfp_mask(mapping); 1122 if (shmem_should_replace_page(*pagep, gfp)) { 1123 mutex_unlock(&shmem_swaplist_mutex); 1124 error = shmem_replace_page(pagep, gfp, info, index); 1125 mutex_lock(&shmem_swaplist_mutex); 1126 /* 1127 * We needed to drop mutex to make that restrictive page 1128 * allocation, but the inode might have been freed while we 1129 * dropped it: although a racing shmem_evict_inode() cannot 1130 * complete without emptying the radix_tree, our page lock 1131 * on this swapcache page is not enough to prevent that - 1132 * free_swap_and_cache() of our swap entry will only 1133 * trylock_page(), removing swap from radix_tree whatever. 1134 * 1135 * We must not proceed to shmem_add_to_page_cache() if the 1136 * inode has been freed, but of course we cannot rely on 1137 * inode or mapping or info to check that. However, we can 1138 * safely check if our swap entry is still in use (and here 1139 * it can't have got reused for another page): if it's still 1140 * in use, then the inode cannot have been freed yet, and we 1141 * can safely proceed (if it's no longer in use, that tells 1142 * nothing about the inode, but we don't need to unuse swap). 1143 */ 1144 if (!page_swapcount(*pagep)) 1145 error = -ENOENT; 1146 } 1147 1148 /* 1149 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 1150 * but also to hold up shmem_evict_inode(): so inode cannot be freed 1151 * beneath us (pagelock doesn't help until the page is in pagecache). 1152 */ 1153 if (!error) 1154 error = shmem_add_to_page_cache(*pagep, mapping, index, 1155 radswap); 1156 if (error != -ENOMEM) { 1157 /* 1158 * Truncation and eviction use free_swap_and_cache(), which 1159 * only does trylock page: if we raced, best clean up here. 1160 */ 1161 delete_from_swap_cache(*pagep); 1162 set_page_dirty(*pagep); 1163 if (!error) { 1164 spin_lock_irq(&info->lock); 1165 info->swapped--; 1166 spin_unlock_irq(&info->lock); 1167 swap_free(swap); 1168 } 1169 } 1170 return error; 1171 } 1172 1173 /* 1174 * Search through swapped inodes to find and replace swap by page. 1175 */ 1176 int shmem_unuse(swp_entry_t swap, struct page *page) 1177 { 1178 struct list_head *this, *next; 1179 struct shmem_inode_info *info; 1180 struct mem_cgroup *memcg; 1181 int error = 0; 1182 1183 /* 1184 * There's a faint possibility that swap page was replaced before 1185 * caller locked it: caller will come back later with the right page. 1186 */ 1187 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) 1188 goto out; 1189 1190 /* 1191 * Charge page using GFP_KERNEL while we can wait, before taking 1192 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1193 * Charged back to the user (not to caller) when swap account is used. 1194 */ 1195 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg, 1196 false); 1197 if (error) 1198 goto out; 1199 /* No radix_tree_preload: swap entry keeps a place for page in tree */ 1200 error = -EAGAIN; 1201 1202 mutex_lock(&shmem_swaplist_mutex); 1203 list_for_each_safe(this, next, &shmem_swaplist) { 1204 info = list_entry(this, struct shmem_inode_info, swaplist); 1205 if (info->swapped) 1206 error = shmem_unuse_inode(info, swap, &page); 1207 else 1208 list_del_init(&info->swaplist); 1209 cond_resched(); 1210 if (error != -EAGAIN) 1211 break; 1212 /* found nothing in this: move on to search the next */ 1213 } 1214 mutex_unlock(&shmem_swaplist_mutex); 1215 1216 if (error) { 1217 if (error != -ENOMEM) 1218 error = 0; 1219 mem_cgroup_cancel_charge(page, memcg, false); 1220 } else 1221 mem_cgroup_commit_charge(page, memcg, true, false); 1222 out: 1223 unlock_page(page); 1224 put_page(page); 1225 return error; 1226 } 1227 1228 /* 1229 * Move the page from the page cache to the swap cache. 1230 */ 1231 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1232 { 1233 struct shmem_inode_info *info; 1234 struct address_space *mapping; 1235 struct inode *inode; 1236 swp_entry_t swap; 1237 pgoff_t index; 1238 1239 VM_BUG_ON_PAGE(PageCompound(page), page); 1240 BUG_ON(!PageLocked(page)); 1241 mapping = page->mapping; 1242 index = page->index; 1243 inode = mapping->host; 1244 info = SHMEM_I(inode); 1245 if (info->flags & VM_LOCKED) 1246 goto redirty; 1247 if (!total_swap_pages) 1248 goto redirty; 1249 1250 /* 1251 * Our capabilities prevent regular writeback or sync from ever calling 1252 * shmem_writepage; but a stacking filesystem might use ->writepage of 1253 * its underlying filesystem, in which case tmpfs should write out to 1254 * swap only in response to memory pressure, and not for the writeback 1255 * threads or sync. 1256 */ 1257 if (!wbc->for_reclaim) { 1258 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1259 goto redirty; 1260 } 1261 1262 /* 1263 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1264 * value into swapfile.c, the only way we can correctly account for a 1265 * fallocated page arriving here is now to initialize it and write it. 1266 * 1267 * That's okay for a page already fallocated earlier, but if we have 1268 * not yet completed the fallocation, then (a) we want to keep track 1269 * of this page in case we have to undo it, and (b) it may not be a 1270 * good idea to continue anyway, once we're pushing into swap. So 1271 * reactivate the page, and let shmem_fallocate() quit when too many. 1272 */ 1273 if (!PageUptodate(page)) { 1274 if (inode->i_private) { 1275 struct shmem_falloc *shmem_falloc; 1276 spin_lock(&inode->i_lock); 1277 shmem_falloc = inode->i_private; 1278 if (shmem_falloc && 1279 !shmem_falloc->waitq && 1280 index >= shmem_falloc->start && 1281 index < shmem_falloc->next) 1282 shmem_falloc->nr_unswapped++; 1283 else 1284 shmem_falloc = NULL; 1285 spin_unlock(&inode->i_lock); 1286 if (shmem_falloc) 1287 goto redirty; 1288 } 1289 clear_highpage(page); 1290 flush_dcache_page(page); 1291 SetPageUptodate(page); 1292 } 1293 1294 swap = get_swap_page(page); 1295 if (!swap.val) 1296 goto redirty; 1297 1298 if (mem_cgroup_try_charge_swap(page, swap)) 1299 goto free_swap; 1300 1301 /* 1302 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1303 * if it's not already there. Do it now before the page is 1304 * moved to swap cache, when its pagelock no longer protects 1305 * the inode from eviction. But don't unlock the mutex until 1306 * we've incremented swapped, because shmem_unuse_inode() will 1307 * prune a !swapped inode from the swaplist under this mutex. 1308 */ 1309 mutex_lock(&shmem_swaplist_mutex); 1310 if (list_empty(&info->swaplist)) 1311 list_add_tail(&info->swaplist, &shmem_swaplist); 1312 1313 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1314 spin_lock_irq(&info->lock); 1315 shmem_recalc_inode(inode); 1316 info->swapped++; 1317 spin_unlock_irq(&info->lock); 1318 1319 swap_shmem_alloc(swap); 1320 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1321 1322 mutex_unlock(&shmem_swaplist_mutex); 1323 BUG_ON(page_mapped(page)); 1324 swap_writepage(page, wbc); 1325 return 0; 1326 } 1327 1328 mutex_unlock(&shmem_swaplist_mutex); 1329 free_swap: 1330 put_swap_page(page, swap); 1331 redirty: 1332 set_page_dirty(page); 1333 if (wbc->for_reclaim) 1334 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1335 unlock_page(page); 1336 return 0; 1337 } 1338 1339 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1340 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1341 { 1342 char buffer[64]; 1343 1344 if (!mpol || mpol->mode == MPOL_DEFAULT) 1345 return; /* show nothing */ 1346 1347 mpol_to_str(buffer, sizeof(buffer), mpol); 1348 1349 seq_printf(seq, ",mpol=%s", buffer); 1350 } 1351 1352 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1353 { 1354 struct mempolicy *mpol = NULL; 1355 if (sbinfo->mpol) { 1356 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1357 mpol = sbinfo->mpol; 1358 mpol_get(mpol); 1359 spin_unlock(&sbinfo->stat_lock); 1360 } 1361 return mpol; 1362 } 1363 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1364 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1365 { 1366 } 1367 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1368 { 1369 return NULL; 1370 } 1371 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1372 #ifndef CONFIG_NUMA 1373 #define vm_policy vm_private_data 1374 #endif 1375 1376 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1377 struct shmem_inode_info *info, pgoff_t index) 1378 { 1379 /* Create a pseudo vma that just contains the policy */ 1380 vma->vm_start = 0; 1381 /* Bias interleave by inode number to distribute better across nodes */ 1382 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1383 vma->vm_ops = NULL; 1384 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1385 } 1386 1387 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1388 { 1389 /* Drop reference taken by mpol_shared_policy_lookup() */ 1390 mpol_cond_put(vma->vm_policy); 1391 } 1392 1393 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1394 struct shmem_inode_info *info, pgoff_t index) 1395 { 1396 struct vm_area_struct pvma; 1397 struct page *page; 1398 1399 shmem_pseudo_vma_init(&pvma, info, index); 1400 page = swapin_readahead(swap, gfp, &pvma, 0); 1401 shmem_pseudo_vma_destroy(&pvma); 1402 1403 return page; 1404 } 1405 1406 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1407 struct shmem_inode_info *info, pgoff_t index) 1408 { 1409 struct vm_area_struct pvma; 1410 struct inode *inode = &info->vfs_inode; 1411 struct address_space *mapping = inode->i_mapping; 1412 pgoff_t idx, hindex; 1413 void __rcu **results; 1414 struct page *page; 1415 1416 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1417 return NULL; 1418 1419 hindex = round_down(index, HPAGE_PMD_NR); 1420 rcu_read_lock(); 1421 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1422 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1423 rcu_read_unlock(); 1424 return NULL; 1425 } 1426 rcu_read_unlock(); 1427 1428 shmem_pseudo_vma_init(&pvma, info, hindex); 1429 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, 1430 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); 1431 shmem_pseudo_vma_destroy(&pvma); 1432 if (page) 1433 prep_transhuge_page(page); 1434 return page; 1435 } 1436 1437 static struct page *shmem_alloc_page(gfp_t gfp, 1438 struct shmem_inode_info *info, pgoff_t index) 1439 { 1440 struct vm_area_struct pvma; 1441 struct page *page; 1442 1443 shmem_pseudo_vma_init(&pvma, info, index); 1444 page = alloc_page_vma(gfp, &pvma, 0); 1445 shmem_pseudo_vma_destroy(&pvma); 1446 1447 return page; 1448 } 1449 1450 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1451 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo, 1452 pgoff_t index, bool huge) 1453 { 1454 struct page *page; 1455 int nr; 1456 int err = -ENOSPC; 1457 1458 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1459 huge = false; 1460 nr = huge ? HPAGE_PMD_NR : 1; 1461 1462 if (shmem_acct_block(info->flags, nr)) 1463 goto failed; 1464 if (sbinfo->max_blocks) { 1465 if (percpu_counter_compare(&sbinfo->used_blocks, 1466 sbinfo->max_blocks - nr) > 0) 1467 goto unacct; 1468 percpu_counter_add(&sbinfo->used_blocks, nr); 1469 } 1470 1471 if (huge) 1472 page = shmem_alloc_hugepage(gfp, info, index); 1473 else 1474 page = shmem_alloc_page(gfp, info, index); 1475 if (page) { 1476 __SetPageLocked(page); 1477 __SetPageSwapBacked(page); 1478 return page; 1479 } 1480 1481 err = -ENOMEM; 1482 if (sbinfo->max_blocks) 1483 percpu_counter_add(&sbinfo->used_blocks, -nr); 1484 unacct: 1485 shmem_unacct_blocks(info->flags, nr); 1486 failed: 1487 return ERR_PTR(err); 1488 } 1489 1490 /* 1491 * When a page is moved from swapcache to shmem filecache (either by the 1492 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1493 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1494 * ignorance of the mapping it belongs to. If that mapping has special 1495 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1496 * we may need to copy to a suitable page before moving to filecache. 1497 * 1498 * In a future release, this may well be extended to respect cpuset and 1499 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1500 * but for now it is a simple matter of zone. 1501 */ 1502 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1503 { 1504 return page_zonenum(page) > gfp_zone(gfp); 1505 } 1506 1507 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1508 struct shmem_inode_info *info, pgoff_t index) 1509 { 1510 struct page *oldpage, *newpage; 1511 struct address_space *swap_mapping; 1512 pgoff_t swap_index; 1513 int error; 1514 1515 oldpage = *pagep; 1516 swap_index = page_private(oldpage); 1517 swap_mapping = page_mapping(oldpage); 1518 1519 /* 1520 * We have arrived here because our zones are constrained, so don't 1521 * limit chance of success by further cpuset and node constraints. 1522 */ 1523 gfp &= ~GFP_CONSTRAINT_MASK; 1524 newpage = shmem_alloc_page(gfp, info, index); 1525 if (!newpage) 1526 return -ENOMEM; 1527 1528 get_page(newpage); 1529 copy_highpage(newpage, oldpage); 1530 flush_dcache_page(newpage); 1531 1532 __SetPageLocked(newpage); 1533 __SetPageSwapBacked(newpage); 1534 SetPageUptodate(newpage); 1535 set_page_private(newpage, swap_index); 1536 SetPageSwapCache(newpage); 1537 1538 /* 1539 * Our caller will very soon move newpage out of swapcache, but it's 1540 * a nice clean interface for us to replace oldpage by newpage there. 1541 */ 1542 spin_lock_irq(&swap_mapping->tree_lock); 1543 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1544 newpage); 1545 if (!error) { 1546 __inc_node_page_state(newpage, NR_FILE_PAGES); 1547 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1548 } 1549 spin_unlock_irq(&swap_mapping->tree_lock); 1550 1551 if (unlikely(error)) { 1552 /* 1553 * Is this possible? I think not, now that our callers check 1554 * both PageSwapCache and page_private after getting page lock; 1555 * but be defensive. Reverse old to newpage for clear and free. 1556 */ 1557 oldpage = newpage; 1558 } else { 1559 mem_cgroup_migrate(oldpage, newpage); 1560 lru_cache_add_anon(newpage); 1561 *pagep = newpage; 1562 } 1563 1564 ClearPageSwapCache(oldpage); 1565 set_page_private(oldpage, 0); 1566 1567 unlock_page(oldpage); 1568 put_page(oldpage); 1569 put_page(oldpage); 1570 return error; 1571 } 1572 1573 /* 1574 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1575 * 1576 * If we allocate a new one we do not mark it dirty. That's up to the 1577 * vm. If we swap it in we mark it dirty since we also free the swap 1578 * entry since a page cannot live in both the swap and page cache. 1579 * 1580 * fault_mm and fault_type are only supplied by shmem_fault: 1581 * otherwise they are NULL. 1582 */ 1583 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1584 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1585 struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type) 1586 { 1587 struct address_space *mapping = inode->i_mapping; 1588 struct shmem_inode_info *info = SHMEM_I(inode); 1589 struct shmem_sb_info *sbinfo; 1590 struct mm_struct *charge_mm; 1591 struct mem_cgroup *memcg; 1592 struct page *page; 1593 swp_entry_t swap; 1594 enum sgp_type sgp_huge = sgp; 1595 pgoff_t hindex = index; 1596 int error; 1597 int once = 0; 1598 int alloced = 0; 1599 1600 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1601 return -EFBIG; 1602 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) 1603 sgp = SGP_CACHE; 1604 repeat: 1605 swap.val = 0; 1606 page = find_lock_entry(mapping, index); 1607 if (radix_tree_exceptional_entry(page)) { 1608 swap = radix_to_swp_entry(page); 1609 page = NULL; 1610 } 1611 1612 if (sgp <= SGP_CACHE && 1613 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1614 error = -EINVAL; 1615 goto unlock; 1616 } 1617 1618 if (page && sgp == SGP_WRITE) 1619 mark_page_accessed(page); 1620 1621 /* fallocated page? */ 1622 if (page && !PageUptodate(page)) { 1623 if (sgp != SGP_READ) 1624 goto clear; 1625 unlock_page(page); 1626 put_page(page); 1627 page = NULL; 1628 } 1629 if (page || (sgp == SGP_READ && !swap.val)) { 1630 *pagep = page; 1631 return 0; 1632 } 1633 1634 /* 1635 * Fast cache lookup did not find it: 1636 * bring it back from swap or allocate. 1637 */ 1638 sbinfo = SHMEM_SB(inode->i_sb); 1639 charge_mm = vma ? vma->vm_mm : current->mm; 1640 1641 if (swap.val) { 1642 /* Look it up and read it in.. */ 1643 page = lookup_swap_cache(swap); 1644 if (!page) { 1645 /* Or update major stats only when swapin succeeds?? */ 1646 if (fault_type) { 1647 *fault_type |= VM_FAULT_MAJOR; 1648 count_vm_event(PGMAJFAULT); 1649 count_memcg_event_mm(charge_mm, PGMAJFAULT); 1650 } 1651 /* Here we actually start the io */ 1652 page = shmem_swapin(swap, gfp, info, index); 1653 if (!page) { 1654 error = -ENOMEM; 1655 goto failed; 1656 } 1657 } 1658 1659 /* We have to do this with page locked to prevent races */ 1660 lock_page(page); 1661 if (!PageSwapCache(page) || page_private(page) != swap.val || 1662 !shmem_confirm_swap(mapping, index, swap)) { 1663 error = -EEXIST; /* try again */ 1664 goto unlock; 1665 } 1666 if (!PageUptodate(page)) { 1667 error = -EIO; 1668 goto failed; 1669 } 1670 wait_on_page_writeback(page); 1671 1672 if (shmem_should_replace_page(page, gfp)) { 1673 error = shmem_replace_page(&page, gfp, info, index); 1674 if (error) 1675 goto failed; 1676 } 1677 1678 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1679 false); 1680 if (!error) { 1681 error = shmem_add_to_page_cache(page, mapping, index, 1682 swp_to_radix_entry(swap)); 1683 /* 1684 * We already confirmed swap under page lock, and make 1685 * no memory allocation here, so usually no possibility 1686 * of error; but free_swap_and_cache() only trylocks a 1687 * page, so it is just possible that the entry has been 1688 * truncated or holepunched since swap was confirmed. 1689 * shmem_undo_range() will have done some of the 1690 * unaccounting, now delete_from_swap_cache() will do 1691 * the rest. 1692 * Reset swap.val? No, leave it so "failed" goes back to 1693 * "repeat": reading a hole and writing should succeed. 1694 */ 1695 if (error) { 1696 mem_cgroup_cancel_charge(page, memcg, false); 1697 delete_from_swap_cache(page); 1698 } 1699 } 1700 if (error) 1701 goto failed; 1702 1703 mem_cgroup_commit_charge(page, memcg, true, false); 1704 1705 spin_lock_irq(&info->lock); 1706 info->swapped--; 1707 shmem_recalc_inode(inode); 1708 spin_unlock_irq(&info->lock); 1709 1710 if (sgp == SGP_WRITE) 1711 mark_page_accessed(page); 1712 1713 delete_from_swap_cache(page); 1714 set_page_dirty(page); 1715 swap_free(swap); 1716 1717 } else { 1718 if (vma && userfaultfd_missing(vma)) { 1719 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 1720 return 0; 1721 } 1722 1723 /* shmem_symlink() */ 1724 if (mapping->a_ops != &shmem_aops) 1725 goto alloc_nohuge; 1726 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) 1727 goto alloc_nohuge; 1728 if (shmem_huge == SHMEM_HUGE_FORCE) 1729 goto alloc_huge; 1730 switch (sbinfo->huge) { 1731 loff_t i_size; 1732 pgoff_t off; 1733 case SHMEM_HUGE_NEVER: 1734 goto alloc_nohuge; 1735 case SHMEM_HUGE_WITHIN_SIZE: 1736 off = round_up(index, HPAGE_PMD_NR); 1737 i_size = round_up(i_size_read(inode), PAGE_SIZE); 1738 if (i_size >= HPAGE_PMD_SIZE && 1739 i_size >> PAGE_SHIFT >= off) 1740 goto alloc_huge; 1741 /* fallthrough */ 1742 case SHMEM_HUGE_ADVISE: 1743 if (sgp_huge == SGP_HUGE) 1744 goto alloc_huge; 1745 /* TODO: implement fadvise() hints */ 1746 goto alloc_nohuge; 1747 } 1748 1749 alloc_huge: 1750 page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1751 index, true); 1752 if (IS_ERR(page)) { 1753 alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1754 index, false); 1755 } 1756 if (IS_ERR(page)) { 1757 int retry = 5; 1758 error = PTR_ERR(page); 1759 page = NULL; 1760 if (error != -ENOSPC) 1761 goto failed; 1762 /* 1763 * Try to reclaim some spece by splitting a huge page 1764 * beyond i_size on the filesystem. 1765 */ 1766 while (retry--) { 1767 int ret; 1768 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1769 if (ret == SHRINK_STOP) 1770 break; 1771 if (ret) 1772 goto alloc_nohuge; 1773 } 1774 goto failed; 1775 } 1776 1777 if (PageTransHuge(page)) 1778 hindex = round_down(index, HPAGE_PMD_NR); 1779 else 1780 hindex = index; 1781 1782 if (sgp == SGP_WRITE) 1783 __SetPageReferenced(page); 1784 1785 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1786 PageTransHuge(page)); 1787 if (error) 1788 goto unacct; 1789 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, 1790 compound_order(page)); 1791 if (!error) { 1792 error = shmem_add_to_page_cache(page, mapping, hindex, 1793 NULL); 1794 radix_tree_preload_end(); 1795 } 1796 if (error) { 1797 mem_cgroup_cancel_charge(page, memcg, 1798 PageTransHuge(page)); 1799 goto unacct; 1800 } 1801 mem_cgroup_commit_charge(page, memcg, false, 1802 PageTransHuge(page)); 1803 lru_cache_add_anon(page); 1804 1805 spin_lock_irq(&info->lock); 1806 info->alloced += 1 << compound_order(page); 1807 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1808 shmem_recalc_inode(inode); 1809 spin_unlock_irq(&info->lock); 1810 alloced = true; 1811 1812 if (PageTransHuge(page) && 1813 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1814 hindex + HPAGE_PMD_NR - 1) { 1815 /* 1816 * Part of the huge page is beyond i_size: subject 1817 * to shrink under memory pressure. 1818 */ 1819 spin_lock(&sbinfo->shrinklist_lock); 1820 if (list_empty(&info->shrinklist)) { 1821 list_add_tail(&info->shrinklist, 1822 &sbinfo->shrinklist); 1823 sbinfo->shrinklist_len++; 1824 } 1825 spin_unlock(&sbinfo->shrinklist_lock); 1826 } 1827 1828 /* 1829 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1830 */ 1831 if (sgp == SGP_FALLOC) 1832 sgp = SGP_WRITE; 1833 clear: 1834 /* 1835 * Let SGP_WRITE caller clear ends if write does not fill page; 1836 * but SGP_FALLOC on a page fallocated earlier must initialize 1837 * it now, lest undo on failure cancel our earlier guarantee. 1838 */ 1839 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1840 struct page *head = compound_head(page); 1841 int i; 1842 1843 for (i = 0; i < (1 << compound_order(head)); i++) { 1844 clear_highpage(head + i); 1845 flush_dcache_page(head + i); 1846 } 1847 SetPageUptodate(head); 1848 } 1849 } 1850 1851 /* Perhaps the file has been truncated since we checked */ 1852 if (sgp <= SGP_CACHE && 1853 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1854 if (alloced) { 1855 ClearPageDirty(page); 1856 delete_from_page_cache(page); 1857 spin_lock_irq(&info->lock); 1858 shmem_recalc_inode(inode); 1859 spin_unlock_irq(&info->lock); 1860 } 1861 error = -EINVAL; 1862 goto unlock; 1863 } 1864 *pagep = page + index - hindex; 1865 return 0; 1866 1867 /* 1868 * Error recovery. 1869 */ 1870 unacct: 1871 if (sbinfo->max_blocks) 1872 percpu_counter_sub(&sbinfo->used_blocks, 1873 1 << compound_order(page)); 1874 shmem_unacct_blocks(info->flags, 1 << compound_order(page)); 1875 1876 if (PageTransHuge(page)) { 1877 unlock_page(page); 1878 put_page(page); 1879 goto alloc_nohuge; 1880 } 1881 failed: 1882 if (swap.val && !shmem_confirm_swap(mapping, index, swap)) 1883 error = -EEXIST; 1884 unlock: 1885 if (page) { 1886 unlock_page(page); 1887 put_page(page); 1888 } 1889 if (error == -ENOSPC && !once++) { 1890 spin_lock_irq(&info->lock); 1891 shmem_recalc_inode(inode); 1892 spin_unlock_irq(&info->lock); 1893 goto repeat; 1894 } 1895 if (error == -EEXIST) /* from above or from radix_tree_insert */ 1896 goto repeat; 1897 return error; 1898 } 1899 1900 /* 1901 * This is like autoremove_wake_function, but it removes the wait queue 1902 * entry unconditionally - even if something else had already woken the 1903 * target. 1904 */ 1905 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 1906 { 1907 int ret = default_wake_function(wait, mode, sync, key); 1908 list_del_init(&wait->entry); 1909 return ret; 1910 } 1911 1912 static int shmem_fault(struct vm_fault *vmf) 1913 { 1914 struct vm_area_struct *vma = vmf->vma; 1915 struct inode *inode = file_inode(vma->vm_file); 1916 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 1917 enum sgp_type sgp; 1918 int error; 1919 int ret = VM_FAULT_LOCKED; 1920 1921 /* 1922 * Trinity finds that probing a hole which tmpfs is punching can 1923 * prevent the hole-punch from ever completing: which in turn 1924 * locks writers out with its hold on i_mutex. So refrain from 1925 * faulting pages into the hole while it's being punched. Although 1926 * shmem_undo_range() does remove the additions, it may be unable to 1927 * keep up, as each new page needs its own unmap_mapping_range() call, 1928 * and the i_mmap tree grows ever slower to scan if new vmas are added. 1929 * 1930 * It does not matter if we sometimes reach this check just before the 1931 * hole-punch begins, so that one fault then races with the punch: 1932 * we just need to make racing faults a rare case. 1933 * 1934 * The implementation below would be much simpler if we just used a 1935 * standard mutex or completion: but we cannot take i_mutex in fault, 1936 * and bloating every shmem inode for this unlikely case would be sad. 1937 */ 1938 if (unlikely(inode->i_private)) { 1939 struct shmem_falloc *shmem_falloc; 1940 1941 spin_lock(&inode->i_lock); 1942 shmem_falloc = inode->i_private; 1943 if (shmem_falloc && 1944 shmem_falloc->waitq && 1945 vmf->pgoff >= shmem_falloc->start && 1946 vmf->pgoff < shmem_falloc->next) { 1947 wait_queue_head_t *shmem_falloc_waitq; 1948 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 1949 1950 ret = VM_FAULT_NOPAGE; 1951 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && 1952 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 1953 /* It's polite to up mmap_sem if we can */ 1954 up_read(&vma->vm_mm->mmap_sem); 1955 ret = VM_FAULT_RETRY; 1956 } 1957 1958 shmem_falloc_waitq = shmem_falloc->waitq; 1959 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 1960 TASK_UNINTERRUPTIBLE); 1961 spin_unlock(&inode->i_lock); 1962 schedule(); 1963 1964 /* 1965 * shmem_falloc_waitq points into the shmem_fallocate() 1966 * stack of the hole-punching task: shmem_falloc_waitq 1967 * is usually invalid by the time we reach here, but 1968 * finish_wait() does not dereference it in that case; 1969 * though i_lock needed lest racing with wake_up_all(). 1970 */ 1971 spin_lock(&inode->i_lock); 1972 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 1973 spin_unlock(&inode->i_lock); 1974 return ret; 1975 } 1976 spin_unlock(&inode->i_lock); 1977 } 1978 1979 sgp = SGP_CACHE; 1980 1981 if ((vma->vm_flags & VM_NOHUGEPAGE) || 1982 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) 1983 sgp = SGP_NOHUGE; 1984 else if (vma->vm_flags & VM_HUGEPAGE) 1985 sgp = SGP_HUGE; 1986 1987 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, 1988 gfp, vma, vmf, &ret); 1989 if (error) 1990 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1991 return ret; 1992 } 1993 1994 unsigned long shmem_get_unmapped_area(struct file *file, 1995 unsigned long uaddr, unsigned long len, 1996 unsigned long pgoff, unsigned long flags) 1997 { 1998 unsigned long (*get_area)(struct file *, 1999 unsigned long, unsigned long, unsigned long, unsigned long); 2000 unsigned long addr; 2001 unsigned long offset; 2002 unsigned long inflated_len; 2003 unsigned long inflated_addr; 2004 unsigned long inflated_offset; 2005 2006 if (len > TASK_SIZE) 2007 return -ENOMEM; 2008 2009 get_area = current->mm->get_unmapped_area; 2010 addr = get_area(file, uaddr, len, pgoff, flags); 2011 2012 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 2013 return addr; 2014 if (IS_ERR_VALUE(addr)) 2015 return addr; 2016 if (addr & ~PAGE_MASK) 2017 return addr; 2018 if (addr > TASK_SIZE - len) 2019 return addr; 2020 2021 if (shmem_huge == SHMEM_HUGE_DENY) 2022 return addr; 2023 if (len < HPAGE_PMD_SIZE) 2024 return addr; 2025 if (flags & MAP_FIXED) 2026 return addr; 2027 /* 2028 * Our priority is to support MAP_SHARED mapped hugely; 2029 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2030 * But if caller specified an address hint, respect that as before. 2031 */ 2032 if (uaddr) 2033 return addr; 2034 2035 if (shmem_huge != SHMEM_HUGE_FORCE) { 2036 struct super_block *sb; 2037 2038 if (file) { 2039 VM_BUG_ON(file->f_op != &shmem_file_operations); 2040 sb = file_inode(file)->i_sb; 2041 } else { 2042 /* 2043 * Called directly from mm/mmap.c, or drivers/char/mem.c 2044 * for "/dev/zero", to create a shared anonymous object. 2045 */ 2046 if (IS_ERR(shm_mnt)) 2047 return addr; 2048 sb = shm_mnt->mnt_sb; 2049 } 2050 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2051 return addr; 2052 } 2053 2054 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2055 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2056 return addr; 2057 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2058 return addr; 2059 2060 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2061 if (inflated_len > TASK_SIZE) 2062 return addr; 2063 if (inflated_len < len) 2064 return addr; 2065 2066 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); 2067 if (IS_ERR_VALUE(inflated_addr)) 2068 return addr; 2069 if (inflated_addr & ~PAGE_MASK) 2070 return addr; 2071 2072 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2073 inflated_addr += offset - inflated_offset; 2074 if (inflated_offset > offset) 2075 inflated_addr += HPAGE_PMD_SIZE; 2076 2077 if (inflated_addr > TASK_SIZE - len) 2078 return addr; 2079 return inflated_addr; 2080 } 2081 2082 #ifdef CONFIG_NUMA 2083 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2084 { 2085 struct inode *inode = file_inode(vma->vm_file); 2086 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2087 } 2088 2089 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2090 unsigned long addr) 2091 { 2092 struct inode *inode = file_inode(vma->vm_file); 2093 pgoff_t index; 2094 2095 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2096 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2097 } 2098 #endif 2099 2100 int shmem_lock(struct file *file, int lock, struct user_struct *user) 2101 { 2102 struct inode *inode = file_inode(file); 2103 struct shmem_inode_info *info = SHMEM_I(inode); 2104 int retval = -ENOMEM; 2105 2106 spin_lock_irq(&info->lock); 2107 if (lock && !(info->flags & VM_LOCKED)) { 2108 if (!user_shm_lock(inode->i_size, user)) 2109 goto out_nomem; 2110 info->flags |= VM_LOCKED; 2111 mapping_set_unevictable(file->f_mapping); 2112 } 2113 if (!lock && (info->flags & VM_LOCKED) && user) { 2114 user_shm_unlock(inode->i_size, user); 2115 info->flags &= ~VM_LOCKED; 2116 mapping_clear_unevictable(file->f_mapping); 2117 } 2118 retval = 0; 2119 2120 out_nomem: 2121 spin_unlock_irq(&info->lock); 2122 return retval; 2123 } 2124 2125 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2126 { 2127 file_accessed(file); 2128 vma->vm_ops = &shmem_vm_ops; 2129 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 2130 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2131 (vma->vm_end & HPAGE_PMD_MASK)) { 2132 khugepaged_enter(vma, vma->vm_flags); 2133 } 2134 return 0; 2135 } 2136 2137 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2138 umode_t mode, dev_t dev, unsigned long flags) 2139 { 2140 struct inode *inode; 2141 struct shmem_inode_info *info; 2142 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2143 2144 if (shmem_reserve_inode(sb)) 2145 return NULL; 2146 2147 inode = new_inode(sb); 2148 if (inode) { 2149 inode->i_ino = get_next_ino(); 2150 inode_init_owner(inode, dir, mode); 2151 inode->i_blocks = 0; 2152 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2153 inode->i_generation = get_seconds(); 2154 info = SHMEM_I(inode); 2155 memset(info, 0, (char *)inode - (char *)info); 2156 spin_lock_init(&info->lock); 2157 info->seals = F_SEAL_SEAL; 2158 info->flags = flags & VM_NORESERVE; 2159 INIT_LIST_HEAD(&info->shrinklist); 2160 INIT_LIST_HEAD(&info->swaplist); 2161 simple_xattrs_init(&info->xattrs); 2162 cache_no_acl(inode); 2163 2164 switch (mode & S_IFMT) { 2165 default: 2166 inode->i_op = &shmem_special_inode_operations; 2167 init_special_inode(inode, mode, dev); 2168 break; 2169 case S_IFREG: 2170 inode->i_mapping->a_ops = &shmem_aops; 2171 inode->i_op = &shmem_inode_operations; 2172 inode->i_fop = &shmem_file_operations; 2173 mpol_shared_policy_init(&info->policy, 2174 shmem_get_sbmpol(sbinfo)); 2175 break; 2176 case S_IFDIR: 2177 inc_nlink(inode); 2178 /* Some things misbehave if size == 0 on a directory */ 2179 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2180 inode->i_op = &shmem_dir_inode_operations; 2181 inode->i_fop = &simple_dir_operations; 2182 break; 2183 case S_IFLNK: 2184 /* 2185 * Must not load anything in the rbtree, 2186 * mpol_free_shared_policy will not be called. 2187 */ 2188 mpol_shared_policy_init(&info->policy, NULL); 2189 break; 2190 } 2191 } else 2192 shmem_free_inode(sb); 2193 return inode; 2194 } 2195 2196 bool shmem_mapping(struct address_space *mapping) 2197 { 2198 return mapping->a_ops == &shmem_aops; 2199 } 2200 2201 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, 2202 pmd_t *dst_pmd, 2203 struct vm_area_struct *dst_vma, 2204 unsigned long dst_addr, 2205 unsigned long src_addr, 2206 struct page **pagep) 2207 { 2208 struct inode *inode = file_inode(dst_vma->vm_file); 2209 struct shmem_inode_info *info = SHMEM_I(inode); 2210 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2211 struct address_space *mapping = inode->i_mapping; 2212 gfp_t gfp = mapping_gfp_mask(mapping); 2213 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2214 struct mem_cgroup *memcg; 2215 spinlock_t *ptl; 2216 void *page_kaddr; 2217 struct page *page; 2218 pte_t _dst_pte, *dst_pte; 2219 int ret; 2220 2221 ret = -ENOMEM; 2222 if (shmem_acct_block(info->flags, 1)) 2223 goto out; 2224 if (sbinfo->max_blocks) { 2225 if (percpu_counter_compare(&sbinfo->used_blocks, 2226 sbinfo->max_blocks) >= 0) 2227 goto out_unacct_blocks; 2228 percpu_counter_inc(&sbinfo->used_blocks); 2229 } 2230 2231 if (!*pagep) { 2232 page = shmem_alloc_page(gfp, info, pgoff); 2233 if (!page) 2234 goto out_dec_used_blocks; 2235 2236 page_kaddr = kmap_atomic(page); 2237 ret = copy_from_user(page_kaddr, (const void __user *)src_addr, 2238 PAGE_SIZE); 2239 kunmap_atomic(page_kaddr); 2240 2241 /* fallback to copy_from_user outside mmap_sem */ 2242 if (unlikely(ret)) { 2243 *pagep = page; 2244 if (sbinfo->max_blocks) 2245 percpu_counter_add(&sbinfo->used_blocks, -1); 2246 shmem_unacct_blocks(info->flags, 1); 2247 /* don't free the page */ 2248 return -EFAULT; 2249 } 2250 } else { 2251 page = *pagep; 2252 *pagep = NULL; 2253 } 2254 2255 VM_BUG_ON(PageLocked(page) || PageSwapBacked(page)); 2256 __SetPageLocked(page); 2257 __SetPageSwapBacked(page); 2258 __SetPageUptodate(page); 2259 2260 ret = mem_cgroup_try_charge(page, dst_mm, gfp, &memcg, false); 2261 if (ret) 2262 goto out_release; 2263 2264 ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK); 2265 if (!ret) { 2266 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL); 2267 radix_tree_preload_end(); 2268 } 2269 if (ret) 2270 goto out_release_uncharge; 2271 2272 mem_cgroup_commit_charge(page, memcg, false, false); 2273 2274 _dst_pte = mk_pte(page, dst_vma->vm_page_prot); 2275 if (dst_vma->vm_flags & VM_WRITE) 2276 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte)); 2277 2278 ret = -EEXIST; 2279 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); 2280 if (!pte_none(*dst_pte)) 2281 goto out_release_uncharge_unlock; 2282 2283 lru_cache_add_anon(page); 2284 2285 spin_lock(&info->lock); 2286 info->alloced++; 2287 inode->i_blocks += BLOCKS_PER_PAGE; 2288 shmem_recalc_inode(inode); 2289 spin_unlock(&info->lock); 2290 2291 inc_mm_counter(dst_mm, mm_counter_file(page)); 2292 page_add_file_rmap(page, false); 2293 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 2294 2295 /* No need to invalidate - it was non-present before */ 2296 update_mmu_cache(dst_vma, dst_addr, dst_pte); 2297 unlock_page(page); 2298 pte_unmap_unlock(dst_pte, ptl); 2299 ret = 0; 2300 out: 2301 return ret; 2302 out_release_uncharge_unlock: 2303 pte_unmap_unlock(dst_pte, ptl); 2304 out_release_uncharge: 2305 mem_cgroup_cancel_charge(page, memcg, false); 2306 out_release: 2307 unlock_page(page); 2308 put_page(page); 2309 out_dec_used_blocks: 2310 if (sbinfo->max_blocks) 2311 percpu_counter_add(&sbinfo->used_blocks, -1); 2312 out_unacct_blocks: 2313 shmem_unacct_blocks(info->flags, 1); 2314 goto out; 2315 } 2316 2317 #ifdef CONFIG_TMPFS 2318 static const struct inode_operations shmem_symlink_inode_operations; 2319 static const struct inode_operations shmem_short_symlink_operations; 2320 2321 #ifdef CONFIG_TMPFS_XATTR 2322 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2323 #else 2324 #define shmem_initxattrs NULL 2325 #endif 2326 2327 static int 2328 shmem_write_begin(struct file *file, struct address_space *mapping, 2329 loff_t pos, unsigned len, unsigned flags, 2330 struct page **pagep, void **fsdata) 2331 { 2332 struct inode *inode = mapping->host; 2333 struct shmem_inode_info *info = SHMEM_I(inode); 2334 pgoff_t index = pos >> PAGE_SHIFT; 2335 2336 /* i_mutex is held by caller */ 2337 if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) { 2338 if (info->seals & F_SEAL_WRITE) 2339 return -EPERM; 2340 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2341 return -EPERM; 2342 } 2343 2344 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2345 } 2346 2347 static int 2348 shmem_write_end(struct file *file, struct address_space *mapping, 2349 loff_t pos, unsigned len, unsigned copied, 2350 struct page *page, void *fsdata) 2351 { 2352 struct inode *inode = mapping->host; 2353 2354 if (pos + copied > inode->i_size) 2355 i_size_write(inode, pos + copied); 2356 2357 if (!PageUptodate(page)) { 2358 struct page *head = compound_head(page); 2359 if (PageTransCompound(page)) { 2360 int i; 2361 2362 for (i = 0; i < HPAGE_PMD_NR; i++) { 2363 if (head + i == page) 2364 continue; 2365 clear_highpage(head + i); 2366 flush_dcache_page(head + i); 2367 } 2368 } 2369 if (copied < PAGE_SIZE) { 2370 unsigned from = pos & (PAGE_SIZE - 1); 2371 zero_user_segments(page, 0, from, 2372 from + copied, PAGE_SIZE); 2373 } 2374 SetPageUptodate(head); 2375 } 2376 set_page_dirty(page); 2377 unlock_page(page); 2378 put_page(page); 2379 2380 return copied; 2381 } 2382 2383 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2384 { 2385 struct file *file = iocb->ki_filp; 2386 struct inode *inode = file_inode(file); 2387 struct address_space *mapping = inode->i_mapping; 2388 pgoff_t index; 2389 unsigned long offset; 2390 enum sgp_type sgp = SGP_READ; 2391 int error = 0; 2392 ssize_t retval = 0; 2393 loff_t *ppos = &iocb->ki_pos; 2394 2395 /* 2396 * Might this read be for a stacking filesystem? Then when reading 2397 * holes of a sparse file, we actually need to allocate those pages, 2398 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2399 */ 2400 if (!iter_is_iovec(to)) 2401 sgp = SGP_CACHE; 2402 2403 index = *ppos >> PAGE_SHIFT; 2404 offset = *ppos & ~PAGE_MASK; 2405 2406 for (;;) { 2407 struct page *page = NULL; 2408 pgoff_t end_index; 2409 unsigned long nr, ret; 2410 loff_t i_size = i_size_read(inode); 2411 2412 end_index = i_size >> PAGE_SHIFT; 2413 if (index > end_index) 2414 break; 2415 if (index == end_index) { 2416 nr = i_size & ~PAGE_MASK; 2417 if (nr <= offset) 2418 break; 2419 } 2420 2421 error = shmem_getpage(inode, index, &page, sgp); 2422 if (error) { 2423 if (error == -EINVAL) 2424 error = 0; 2425 break; 2426 } 2427 if (page) { 2428 if (sgp == SGP_CACHE) 2429 set_page_dirty(page); 2430 unlock_page(page); 2431 } 2432 2433 /* 2434 * We must evaluate after, since reads (unlike writes) 2435 * are called without i_mutex protection against truncate 2436 */ 2437 nr = PAGE_SIZE; 2438 i_size = i_size_read(inode); 2439 end_index = i_size >> PAGE_SHIFT; 2440 if (index == end_index) { 2441 nr = i_size & ~PAGE_MASK; 2442 if (nr <= offset) { 2443 if (page) 2444 put_page(page); 2445 break; 2446 } 2447 } 2448 nr -= offset; 2449 2450 if (page) { 2451 /* 2452 * If users can be writing to this page using arbitrary 2453 * virtual addresses, take care about potential aliasing 2454 * before reading the page on the kernel side. 2455 */ 2456 if (mapping_writably_mapped(mapping)) 2457 flush_dcache_page(page); 2458 /* 2459 * Mark the page accessed if we read the beginning. 2460 */ 2461 if (!offset) 2462 mark_page_accessed(page); 2463 } else { 2464 page = ZERO_PAGE(0); 2465 get_page(page); 2466 } 2467 2468 /* 2469 * Ok, we have the page, and it's up-to-date, so 2470 * now we can copy it to user space... 2471 */ 2472 ret = copy_page_to_iter(page, offset, nr, to); 2473 retval += ret; 2474 offset += ret; 2475 index += offset >> PAGE_SHIFT; 2476 offset &= ~PAGE_MASK; 2477 2478 put_page(page); 2479 if (!iov_iter_count(to)) 2480 break; 2481 if (ret < nr) { 2482 error = -EFAULT; 2483 break; 2484 } 2485 cond_resched(); 2486 } 2487 2488 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2489 file_accessed(file); 2490 return retval ? retval : error; 2491 } 2492 2493 /* 2494 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree. 2495 */ 2496 static pgoff_t shmem_seek_hole_data(struct address_space *mapping, 2497 pgoff_t index, pgoff_t end, int whence) 2498 { 2499 struct page *page; 2500 struct pagevec pvec; 2501 pgoff_t indices[PAGEVEC_SIZE]; 2502 bool done = false; 2503 int i; 2504 2505 pagevec_init(&pvec, 0); 2506 pvec.nr = 1; /* start small: we may be there already */ 2507 while (!done) { 2508 pvec.nr = find_get_entries(mapping, index, 2509 pvec.nr, pvec.pages, indices); 2510 if (!pvec.nr) { 2511 if (whence == SEEK_DATA) 2512 index = end; 2513 break; 2514 } 2515 for (i = 0; i < pvec.nr; i++, index++) { 2516 if (index < indices[i]) { 2517 if (whence == SEEK_HOLE) { 2518 done = true; 2519 break; 2520 } 2521 index = indices[i]; 2522 } 2523 page = pvec.pages[i]; 2524 if (page && !radix_tree_exceptional_entry(page)) { 2525 if (!PageUptodate(page)) 2526 page = NULL; 2527 } 2528 if (index >= end || 2529 (page && whence == SEEK_DATA) || 2530 (!page && whence == SEEK_HOLE)) { 2531 done = true; 2532 break; 2533 } 2534 } 2535 pagevec_remove_exceptionals(&pvec); 2536 pagevec_release(&pvec); 2537 pvec.nr = PAGEVEC_SIZE; 2538 cond_resched(); 2539 } 2540 return index; 2541 } 2542 2543 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2544 { 2545 struct address_space *mapping = file->f_mapping; 2546 struct inode *inode = mapping->host; 2547 pgoff_t start, end; 2548 loff_t new_offset; 2549 2550 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2551 return generic_file_llseek_size(file, offset, whence, 2552 MAX_LFS_FILESIZE, i_size_read(inode)); 2553 inode_lock(inode); 2554 /* We're holding i_mutex so we can access i_size directly */ 2555 2556 if (offset < 0) 2557 offset = -EINVAL; 2558 else if (offset >= inode->i_size) 2559 offset = -ENXIO; 2560 else { 2561 start = offset >> PAGE_SHIFT; 2562 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2563 new_offset = shmem_seek_hole_data(mapping, start, end, whence); 2564 new_offset <<= PAGE_SHIFT; 2565 if (new_offset > offset) { 2566 if (new_offset < inode->i_size) 2567 offset = new_offset; 2568 else if (whence == SEEK_DATA) 2569 offset = -ENXIO; 2570 else 2571 offset = inode->i_size; 2572 } 2573 } 2574 2575 if (offset >= 0) 2576 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2577 inode_unlock(inode); 2578 return offset; 2579 } 2580 2581 /* 2582 * We need a tag: a new tag would expand every radix_tree_node by 8 bytes, 2583 * so reuse a tag which we firmly believe is never set or cleared on shmem. 2584 */ 2585 #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE 2586 #define LAST_SCAN 4 /* about 150ms max */ 2587 2588 static void shmem_tag_pins(struct address_space *mapping) 2589 { 2590 struct radix_tree_iter iter; 2591 void **slot; 2592 pgoff_t start; 2593 struct page *page; 2594 2595 lru_add_drain(); 2596 start = 0; 2597 rcu_read_lock(); 2598 2599 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2600 page = radix_tree_deref_slot(slot); 2601 if (!page || radix_tree_exception(page)) { 2602 if (radix_tree_deref_retry(page)) { 2603 slot = radix_tree_iter_retry(&iter); 2604 continue; 2605 } 2606 } else if (page_count(page) - page_mapcount(page) > 1) { 2607 spin_lock_irq(&mapping->tree_lock); 2608 radix_tree_tag_set(&mapping->page_tree, iter.index, 2609 SHMEM_TAG_PINNED); 2610 spin_unlock_irq(&mapping->tree_lock); 2611 } 2612 2613 if (need_resched()) { 2614 slot = radix_tree_iter_resume(slot, &iter); 2615 cond_resched_rcu(); 2616 } 2617 } 2618 rcu_read_unlock(); 2619 } 2620 2621 /* 2622 * Setting SEAL_WRITE requires us to verify there's no pending writer. However, 2623 * via get_user_pages(), drivers might have some pending I/O without any active 2624 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages 2625 * and see whether it has an elevated ref-count. If so, we tag them and wait for 2626 * them to be dropped. 2627 * The caller must guarantee that no new user will acquire writable references 2628 * to those pages to avoid races. 2629 */ 2630 static int shmem_wait_for_pins(struct address_space *mapping) 2631 { 2632 struct radix_tree_iter iter; 2633 void **slot; 2634 pgoff_t start; 2635 struct page *page; 2636 int error, scan; 2637 2638 shmem_tag_pins(mapping); 2639 2640 error = 0; 2641 for (scan = 0; scan <= LAST_SCAN; scan++) { 2642 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2643 break; 2644 2645 if (!scan) 2646 lru_add_drain_all(); 2647 else if (schedule_timeout_killable((HZ << scan) / 200)) 2648 scan = LAST_SCAN; 2649 2650 start = 0; 2651 rcu_read_lock(); 2652 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2653 start, SHMEM_TAG_PINNED) { 2654 2655 page = radix_tree_deref_slot(slot); 2656 if (radix_tree_exception(page)) { 2657 if (radix_tree_deref_retry(page)) { 2658 slot = radix_tree_iter_retry(&iter); 2659 continue; 2660 } 2661 2662 page = NULL; 2663 } 2664 2665 if (page && 2666 page_count(page) - page_mapcount(page) != 1) { 2667 if (scan < LAST_SCAN) 2668 goto continue_resched; 2669 2670 /* 2671 * On the last scan, we clean up all those tags 2672 * we inserted; but make a note that we still 2673 * found pages pinned. 2674 */ 2675 error = -EBUSY; 2676 } 2677 2678 spin_lock_irq(&mapping->tree_lock); 2679 radix_tree_tag_clear(&mapping->page_tree, 2680 iter.index, SHMEM_TAG_PINNED); 2681 spin_unlock_irq(&mapping->tree_lock); 2682 continue_resched: 2683 if (need_resched()) { 2684 slot = radix_tree_iter_resume(slot, &iter); 2685 cond_resched_rcu(); 2686 } 2687 } 2688 rcu_read_unlock(); 2689 } 2690 2691 return error; 2692 } 2693 2694 #define F_ALL_SEALS (F_SEAL_SEAL | \ 2695 F_SEAL_SHRINK | \ 2696 F_SEAL_GROW | \ 2697 F_SEAL_WRITE) 2698 2699 int shmem_add_seals(struct file *file, unsigned int seals) 2700 { 2701 struct inode *inode = file_inode(file); 2702 struct shmem_inode_info *info = SHMEM_I(inode); 2703 int error; 2704 2705 /* 2706 * SEALING 2707 * Sealing allows multiple parties to share a shmem-file but restrict 2708 * access to a specific subset of file operations. Seals can only be 2709 * added, but never removed. This way, mutually untrusted parties can 2710 * share common memory regions with a well-defined policy. A malicious 2711 * peer can thus never perform unwanted operations on a shared object. 2712 * 2713 * Seals are only supported on special shmem-files and always affect 2714 * the whole underlying inode. Once a seal is set, it may prevent some 2715 * kinds of access to the file. Currently, the following seals are 2716 * defined: 2717 * SEAL_SEAL: Prevent further seals from being set on this file 2718 * SEAL_SHRINK: Prevent the file from shrinking 2719 * SEAL_GROW: Prevent the file from growing 2720 * SEAL_WRITE: Prevent write access to the file 2721 * 2722 * As we don't require any trust relationship between two parties, we 2723 * must prevent seals from being removed. Therefore, sealing a file 2724 * only adds a given set of seals to the file, it never touches 2725 * existing seals. Furthermore, the "setting seals"-operation can be 2726 * sealed itself, which basically prevents any further seal from being 2727 * added. 2728 * 2729 * Semantics of sealing are only defined on volatile files. Only 2730 * anonymous shmem files support sealing. More importantly, seals are 2731 * never written to disk. Therefore, there's no plan to support it on 2732 * other file types. 2733 */ 2734 2735 if (file->f_op != &shmem_file_operations) 2736 return -EINVAL; 2737 if (!(file->f_mode & FMODE_WRITE)) 2738 return -EPERM; 2739 if (seals & ~(unsigned int)F_ALL_SEALS) 2740 return -EINVAL; 2741 2742 inode_lock(inode); 2743 2744 if (info->seals & F_SEAL_SEAL) { 2745 error = -EPERM; 2746 goto unlock; 2747 } 2748 2749 if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) { 2750 error = mapping_deny_writable(file->f_mapping); 2751 if (error) 2752 goto unlock; 2753 2754 error = shmem_wait_for_pins(file->f_mapping); 2755 if (error) { 2756 mapping_allow_writable(file->f_mapping); 2757 goto unlock; 2758 } 2759 } 2760 2761 info->seals |= seals; 2762 error = 0; 2763 2764 unlock: 2765 inode_unlock(inode); 2766 return error; 2767 } 2768 EXPORT_SYMBOL_GPL(shmem_add_seals); 2769 2770 int shmem_get_seals(struct file *file) 2771 { 2772 if (file->f_op != &shmem_file_operations) 2773 return -EINVAL; 2774 2775 return SHMEM_I(file_inode(file))->seals; 2776 } 2777 EXPORT_SYMBOL_GPL(shmem_get_seals); 2778 2779 long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 2780 { 2781 long error; 2782 2783 switch (cmd) { 2784 case F_ADD_SEALS: 2785 /* disallow upper 32bit */ 2786 if (arg > UINT_MAX) 2787 return -EINVAL; 2788 2789 error = shmem_add_seals(file, arg); 2790 break; 2791 case F_GET_SEALS: 2792 error = shmem_get_seals(file); 2793 break; 2794 default: 2795 error = -EINVAL; 2796 break; 2797 } 2798 2799 return error; 2800 } 2801 2802 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2803 loff_t len) 2804 { 2805 struct inode *inode = file_inode(file); 2806 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2807 struct shmem_inode_info *info = SHMEM_I(inode); 2808 struct shmem_falloc shmem_falloc; 2809 pgoff_t start, index, end; 2810 int error; 2811 2812 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2813 return -EOPNOTSUPP; 2814 2815 inode_lock(inode); 2816 2817 if (mode & FALLOC_FL_PUNCH_HOLE) { 2818 struct address_space *mapping = file->f_mapping; 2819 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2820 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2821 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2822 2823 /* protected by i_mutex */ 2824 if (info->seals & F_SEAL_WRITE) { 2825 error = -EPERM; 2826 goto out; 2827 } 2828 2829 shmem_falloc.waitq = &shmem_falloc_waitq; 2830 shmem_falloc.start = unmap_start >> PAGE_SHIFT; 2831 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2832 spin_lock(&inode->i_lock); 2833 inode->i_private = &shmem_falloc; 2834 spin_unlock(&inode->i_lock); 2835 2836 if ((u64)unmap_end > (u64)unmap_start) 2837 unmap_mapping_range(mapping, unmap_start, 2838 1 + unmap_end - unmap_start, 0); 2839 shmem_truncate_range(inode, offset, offset + len - 1); 2840 /* No need to unmap again: hole-punching leaves COWed pages */ 2841 2842 spin_lock(&inode->i_lock); 2843 inode->i_private = NULL; 2844 wake_up_all(&shmem_falloc_waitq); 2845 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 2846 spin_unlock(&inode->i_lock); 2847 error = 0; 2848 goto out; 2849 } 2850 2851 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2852 error = inode_newsize_ok(inode, offset + len); 2853 if (error) 2854 goto out; 2855 2856 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2857 error = -EPERM; 2858 goto out; 2859 } 2860 2861 start = offset >> PAGE_SHIFT; 2862 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2863 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2864 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2865 error = -ENOSPC; 2866 goto out; 2867 } 2868 2869 shmem_falloc.waitq = NULL; 2870 shmem_falloc.start = start; 2871 shmem_falloc.next = start; 2872 shmem_falloc.nr_falloced = 0; 2873 shmem_falloc.nr_unswapped = 0; 2874 spin_lock(&inode->i_lock); 2875 inode->i_private = &shmem_falloc; 2876 spin_unlock(&inode->i_lock); 2877 2878 for (index = start; index < end; index++) { 2879 struct page *page; 2880 2881 /* 2882 * Good, the fallocate(2) manpage permits EINTR: we may have 2883 * been interrupted because we are using up too much memory. 2884 */ 2885 if (signal_pending(current)) 2886 error = -EINTR; 2887 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2888 error = -ENOMEM; 2889 else 2890 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2891 if (error) { 2892 /* Remove the !PageUptodate pages we added */ 2893 if (index > start) { 2894 shmem_undo_range(inode, 2895 (loff_t)start << PAGE_SHIFT, 2896 ((loff_t)index << PAGE_SHIFT) - 1, true); 2897 } 2898 goto undone; 2899 } 2900 2901 /* 2902 * Inform shmem_writepage() how far we have reached. 2903 * No need for lock or barrier: we have the page lock. 2904 */ 2905 shmem_falloc.next++; 2906 if (!PageUptodate(page)) 2907 shmem_falloc.nr_falloced++; 2908 2909 /* 2910 * If !PageUptodate, leave it that way so that freeable pages 2911 * can be recognized if we need to rollback on error later. 2912 * But set_page_dirty so that memory pressure will swap rather 2913 * than free the pages we are allocating (and SGP_CACHE pages 2914 * might still be clean: we now need to mark those dirty too). 2915 */ 2916 set_page_dirty(page); 2917 unlock_page(page); 2918 put_page(page); 2919 cond_resched(); 2920 } 2921 2922 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2923 i_size_write(inode, offset + len); 2924 inode->i_ctime = current_time(inode); 2925 undone: 2926 spin_lock(&inode->i_lock); 2927 inode->i_private = NULL; 2928 spin_unlock(&inode->i_lock); 2929 out: 2930 inode_unlock(inode); 2931 return error; 2932 } 2933 2934 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2935 { 2936 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2937 2938 buf->f_type = TMPFS_MAGIC; 2939 buf->f_bsize = PAGE_SIZE; 2940 buf->f_namelen = NAME_MAX; 2941 if (sbinfo->max_blocks) { 2942 buf->f_blocks = sbinfo->max_blocks; 2943 buf->f_bavail = 2944 buf->f_bfree = sbinfo->max_blocks - 2945 percpu_counter_sum(&sbinfo->used_blocks); 2946 } 2947 if (sbinfo->max_inodes) { 2948 buf->f_files = sbinfo->max_inodes; 2949 buf->f_ffree = sbinfo->free_inodes; 2950 } 2951 /* else leave those fields 0 like simple_statfs */ 2952 return 0; 2953 } 2954 2955 /* 2956 * File creation. Allocate an inode, and we're done.. 2957 */ 2958 static int 2959 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) 2960 { 2961 struct inode *inode; 2962 int error = -ENOSPC; 2963 2964 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2965 if (inode) { 2966 error = simple_acl_create(dir, inode); 2967 if (error) 2968 goto out_iput; 2969 error = security_inode_init_security(inode, dir, 2970 &dentry->d_name, 2971 shmem_initxattrs, NULL); 2972 if (error && error != -EOPNOTSUPP) 2973 goto out_iput; 2974 2975 error = 0; 2976 dir->i_size += BOGO_DIRENT_SIZE; 2977 dir->i_ctime = dir->i_mtime = current_time(dir); 2978 d_instantiate(dentry, inode); 2979 dget(dentry); /* Extra count - pin the dentry in core */ 2980 } 2981 return error; 2982 out_iput: 2983 iput(inode); 2984 return error; 2985 } 2986 2987 static int 2988 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 2989 { 2990 struct inode *inode; 2991 int error = -ENOSPC; 2992 2993 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2994 if (inode) { 2995 error = security_inode_init_security(inode, dir, 2996 NULL, 2997 shmem_initxattrs, NULL); 2998 if (error && error != -EOPNOTSUPP) 2999 goto out_iput; 3000 error = simple_acl_create(dir, inode); 3001 if (error) 3002 goto out_iput; 3003 d_tmpfile(dentry, inode); 3004 } 3005 return error; 3006 out_iput: 3007 iput(inode); 3008 return error; 3009 } 3010 3011 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 3012 { 3013 int error; 3014 3015 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 3016 return error; 3017 inc_nlink(dir); 3018 return 0; 3019 } 3020 3021 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, 3022 bool excl) 3023 { 3024 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 3025 } 3026 3027 /* 3028 * Link a file.. 3029 */ 3030 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 3031 { 3032 struct inode *inode = d_inode(old_dentry); 3033 int ret; 3034 3035 /* 3036 * No ordinary (disk based) filesystem counts links as inodes; 3037 * but each new link needs a new dentry, pinning lowmem, and 3038 * tmpfs dentries cannot be pruned until they are unlinked. 3039 */ 3040 ret = shmem_reserve_inode(inode->i_sb); 3041 if (ret) 3042 goto out; 3043 3044 dir->i_size += BOGO_DIRENT_SIZE; 3045 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 3046 inc_nlink(inode); 3047 ihold(inode); /* New dentry reference */ 3048 dget(dentry); /* Extra pinning count for the created dentry */ 3049 d_instantiate(dentry, inode); 3050 out: 3051 return ret; 3052 } 3053 3054 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 3055 { 3056 struct inode *inode = d_inode(dentry); 3057 3058 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 3059 shmem_free_inode(inode->i_sb); 3060 3061 dir->i_size -= BOGO_DIRENT_SIZE; 3062 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 3063 drop_nlink(inode); 3064 dput(dentry); /* Undo the count from "create" - this does all the work */ 3065 return 0; 3066 } 3067 3068 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 3069 { 3070 if (!simple_empty(dentry)) 3071 return -ENOTEMPTY; 3072 3073 drop_nlink(d_inode(dentry)); 3074 drop_nlink(dir); 3075 return shmem_unlink(dir, dentry); 3076 } 3077 3078 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 3079 { 3080 bool old_is_dir = d_is_dir(old_dentry); 3081 bool new_is_dir = d_is_dir(new_dentry); 3082 3083 if (old_dir != new_dir && old_is_dir != new_is_dir) { 3084 if (old_is_dir) { 3085 drop_nlink(old_dir); 3086 inc_nlink(new_dir); 3087 } else { 3088 drop_nlink(new_dir); 3089 inc_nlink(old_dir); 3090 } 3091 } 3092 old_dir->i_ctime = old_dir->i_mtime = 3093 new_dir->i_ctime = new_dir->i_mtime = 3094 d_inode(old_dentry)->i_ctime = 3095 d_inode(new_dentry)->i_ctime = current_time(old_dir); 3096 3097 return 0; 3098 } 3099 3100 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) 3101 { 3102 struct dentry *whiteout; 3103 int error; 3104 3105 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 3106 if (!whiteout) 3107 return -ENOMEM; 3108 3109 error = shmem_mknod(old_dir, whiteout, 3110 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 3111 dput(whiteout); 3112 if (error) 3113 return error; 3114 3115 /* 3116 * Cheat and hash the whiteout while the old dentry is still in 3117 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 3118 * 3119 * d_lookup() will consistently find one of them at this point, 3120 * not sure which one, but that isn't even important. 3121 */ 3122 d_rehash(whiteout); 3123 return 0; 3124 } 3125 3126 /* 3127 * The VFS layer already does all the dentry stuff for rename, 3128 * we just have to decrement the usage count for the target if 3129 * it exists so that the VFS layer correctly free's it when it 3130 * gets overwritten. 3131 */ 3132 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) 3133 { 3134 struct inode *inode = d_inode(old_dentry); 3135 int they_are_dirs = S_ISDIR(inode->i_mode); 3136 3137 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3138 return -EINVAL; 3139 3140 if (flags & RENAME_EXCHANGE) 3141 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 3142 3143 if (!simple_empty(new_dentry)) 3144 return -ENOTEMPTY; 3145 3146 if (flags & RENAME_WHITEOUT) { 3147 int error; 3148 3149 error = shmem_whiteout(old_dir, old_dentry); 3150 if (error) 3151 return error; 3152 } 3153 3154 if (d_really_is_positive(new_dentry)) { 3155 (void) shmem_unlink(new_dir, new_dentry); 3156 if (they_are_dirs) { 3157 drop_nlink(d_inode(new_dentry)); 3158 drop_nlink(old_dir); 3159 } 3160 } else if (they_are_dirs) { 3161 drop_nlink(old_dir); 3162 inc_nlink(new_dir); 3163 } 3164 3165 old_dir->i_size -= BOGO_DIRENT_SIZE; 3166 new_dir->i_size += BOGO_DIRENT_SIZE; 3167 old_dir->i_ctime = old_dir->i_mtime = 3168 new_dir->i_ctime = new_dir->i_mtime = 3169 inode->i_ctime = current_time(old_dir); 3170 return 0; 3171 } 3172 3173 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 3174 { 3175 int error; 3176 int len; 3177 struct inode *inode; 3178 struct page *page; 3179 struct shmem_inode_info *info; 3180 3181 len = strlen(symname) + 1; 3182 if (len > PAGE_SIZE) 3183 return -ENAMETOOLONG; 3184 3185 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 3186 if (!inode) 3187 return -ENOSPC; 3188 3189 error = security_inode_init_security(inode, dir, &dentry->d_name, 3190 shmem_initxattrs, NULL); 3191 if (error) { 3192 if (error != -EOPNOTSUPP) { 3193 iput(inode); 3194 return error; 3195 } 3196 error = 0; 3197 } 3198 3199 info = SHMEM_I(inode); 3200 inode->i_size = len-1; 3201 if (len <= SHORT_SYMLINK_LEN) { 3202 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3203 if (!inode->i_link) { 3204 iput(inode); 3205 return -ENOMEM; 3206 } 3207 inode->i_op = &shmem_short_symlink_operations; 3208 } else { 3209 inode_nohighmem(inode); 3210 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3211 if (error) { 3212 iput(inode); 3213 return error; 3214 } 3215 inode->i_mapping->a_ops = &shmem_aops; 3216 inode->i_op = &shmem_symlink_inode_operations; 3217 memcpy(page_address(page), symname, len); 3218 SetPageUptodate(page); 3219 set_page_dirty(page); 3220 unlock_page(page); 3221 put_page(page); 3222 } 3223 dir->i_size += BOGO_DIRENT_SIZE; 3224 dir->i_ctime = dir->i_mtime = current_time(dir); 3225 d_instantiate(dentry, inode); 3226 dget(dentry); 3227 return 0; 3228 } 3229 3230 static void shmem_put_link(void *arg) 3231 { 3232 mark_page_accessed(arg); 3233 put_page(arg); 3234 } 3235 3236 static const char *shmem_get_link(struct dentry *dentry, 3237 struct inode *inode, 3238 struct delayed_call *done) 3239 { 3240 struct page *page = NULL; 3241 int error; 3242 if (!dentry) { 3243 page = find_get_page(inode->i_mapping, 0); 3244 if (!page) 3245 return ERR_PTR(-ECHILD); 3246 if (!PageUptodate(page)) { 3247 put_page(page); 3248 return ERR_PTR(-ECHILD); 3249 } 3250 } else { 3251 error = shmem_getpage(inode, 0, &page, SGP_READ); 3252 if (error) 3253 return ERR_PTR(error); 3254 unlock_page(page); 3255 } 3256 set_delayed_call(done, shmem_put_link, page); 3257 return page_address(page); 3258 } 3259 3260 #ifdef CONFIG_TMPFS_XATTR 3261 /* 3262 * Superblocks without xattr inode operations may get some security.* xattr 3263 * support from the LSM "for free". As soon as we have any other xattrs 3264 * like ACLs, we also need to implement the security.* handlers at 3265 * filesystem level, though. 3266 */ 3267 3268 /* 3269 * Callback for security_inode_init_security() for acquiring xattrs. 3270 */ 3271 static int shmem_initxattrs(struct inode *inode, 3272 const struct xattr *xattr_array, 3273 void *fs_info) 3274 { 3275 struct shmem_inode_info *info = SHMEM_I(inode); 3276 const struct xattr *xattr; 3277 struct simple_xattr *new_xattr; 3278 size_t len; 3279 3280 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3281 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3282 if (!new_xattr) 3283 return -ENOMEM; 3284 3285 len = strlen(xattr->name) + 1; 3286 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3287 GFP_KERNEL); 3288 if (!new_xattr->name) { 3289 kfree(new_xattr); 3290 return -ENOMEM; 3291 } 3292 3293 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3294 XATTR_SECURITY_PREFIX_LEN); 3295 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3296 xattr->name, len); 3297 3298 simple_xattr_list_add(&info->xattrs, new_xattr); 3299 } 3300 3301 return 0; 3302 } 3303 3304 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3305 struct dentry *unused, struct inode *inode, 3306 const char *name, void *buffer, size_t size) 3307 { 3308 struct shmem_inode_info *info = SHMEM_I(inode); 3309 3310 name = xattr_full_name(handler, name); 3311 return simple_xattr_get(&info->xattrs, name, buffer, size); 3312 } 3313 3314 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3315 struct dentry *unused, struct inode *inode, 3316 const char *name, const void *value, 3317 size_t size, int flags) 3318 { 3319 struct shmem_inode_info *info = SHMEM_I(inode); 3320 3321 name = xattr_full_name(handler, name); 3322 return simple_xattr_set(&info->xattrs, name, value, size, flags); 3323 } 3324 3325 static const struct xattr_handler shmem_security_xattr_handler = { 3326 .prefix = XATTR_SECURITY_PREFIX, 3327 .get = shmem_xattr_handler_get, 3328 .set = shmem_xattr_handler_set, 3329 }; 3330 3331 static const struct xattr_handler shmem_trusted_xattr_handler = { 3332 .prefix = XATTR_TRUSTED_PREFIX, 3333 .get = shmem_xattr_handler_get, 3334 .set = shmem_xattr_handler_set, 3335 }; 3336 3337 static const struct xattr_handler *shmem_xattr_handlers[] = { 3338 #ifdef CONFIG_TMPFS_POSIX_ACL 3339 &posix_acl_access_xattr_handler, 3340 &posix_acl_default_xattr_handler, 3341 #endif 3342 &shmem_security_xattr_handler, 3343 &shmem_trusted_xattr_handler, 3344 NULL 3345 }; 3346 3347 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3348 { 3349 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3350 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3351 } 3352 #endif /* CONFIG_TMPFS_XATTR */ 3353 3354 static const struct inode_operations shmem_short_symlink_operations = { 3355 .get_link = simple_get_link, 3356 #ifdef CONFIG_TMPFS_XATTR 3357 .listxattr = shmem_listxattr, 3358 #endif 3359 }; 3360 3361 static const struct inode_operations shmem_symlink_inode_operations = { 3362 .get_link = shmem_get_link, 3363 #ifdef CONFIG_TMPFS_XATTR 3364 .listxattr = shmem_listxattr, 3365 #endif 3366 }; 3367 3368 static struct dentry *shmem_get_parent(struct dentry *child) 3369 { 3370 return ERR_PTR(-ESTALE); 3371 } 3372 3373 static int shmem_match(struct inode *ino, void *vfh) 3374 { 3375 __u32 *fh = vfh; 3376 __u64 inum = fh[2]; 3377 inum = (inum << 32) | fh[1]; 3378 return ino->i_ino == inum && fh[0] == ino->i_generation; 3379 } 3380 3381 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3382 struct fid *fid, int fh_len, int fh_type) 3383 { 3384 struct inode *inode; 3385 struct dentry *dentry = NULL; 3386 u64 inum; 3387 3388 if (fh_len < 3) 3389 return NULL; 3390 3391 inum = fid->raw[2]; 3392 inum = (inum << 32) | fid->raw[1]; 3393 3394 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3395 shmem_match, fid->raw); 3396 if (inode) { 3397 dentry = d_find_alias(inode); 3398 iput(inode); 3399 } 3400 3401 return dentry; 3402 } 3403 3404 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3405 struct inode *parent) 3406 { 3407 if (*len < 3) { 3408 *len = 3; 3409 return FILEID_INVALID; 3410 } 3411 3412 if (inode_unhashed(inode)) { 3413 /* Unfortunately insert_inode_hash is not idempotent, 3414 * so as we hash inodes here rather than at creation 3415 * time, we need a lock to ensure we only try 3416 * to do it once 3417 */ 3418 static DEFINE_SPINLOCK(lock); 3419 spin_lock(&lock); 3420 if (inode_unhashed(inode)) 3421 __insert_inode_hash(inode, 3422 inode->i_ino + inode->i_generation); 3423 spin_unlock(&lock); 3424 } 3425 3426 fh[0] = inode->i_generation; 3427 fh[1] = inode->i_ino; 3428 fh[2] = ((__u64)inode->i_ino) >> 32; 3429 3430 *len = 3; 3431 return 1; 3432 } 3433 3434 static const struct export_operations shmem_export_ops = { 3435 .get_parent = shmem_get_parent, 3436 .encode_fh = shmem_encode_fh, 3437 .fh_to_dentry = shmem_fh_to_dentry, 3438 }; 3439 3440 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 3441 bool remount) 3442 { 3443 char *this_char, *value, *rest; 3444 struct mempolicy *mpol = NULL; 3445 uid_t uid; 3446 gid_t gid; 3447 3448 while (options != NULL) { 3449 this_char = options; 3450 for (;;) { 3451 /* 3452 * NUL-terminate this option: unfortunately, 3453 * mount options form a comma-separated list, 3454 * but mpol's nodelist may also contain commas. 3455 */ 3456 options = strchr(options, ','); 3457 if (options == NULL) 3458 break; 3459 options++; 3460 if (!isdigit(*options)) { 3461 options[-1] = '\0'; 3462 break; 3463 } 3464 } 3465 if (!*this_char) 3466 continue; 3467 if ((value = strchr(this_char,'=')) != NULL) { 3468 *value++ = 0; 3469 } else { 3470 pr_err("tmpfs: No value for mount option '%s'\n", 3471 this_char); 3472 goto error; 3473 } 3474 3475 if (!strcmp(this_char,"size")) { 3476 unsigned long long size; 3477 size = memparse(value,&rest); 3478 if (*rest == '%') { 3479 size <<= PAGE_SHIFT; 3480 size *= totalram_pages; 3481 do_div(size, 100); 3482 rest++; 3483 } 3484 if (*rest) 3485 goto bad_val; 3486 sbinfo->max_blocks = 3487 DIV_ROUND_UP(size, PAGE_SIZE); 3488 } else if (!strcmp(this_char,"nr_blocks")) { 3489 sbinfo->max_blocks = memparse(value, &rest); 3490 if (*rest) 3491 goto bad_val; 3492 } else if (!strcmp(this_char,"nr_inodes")) { 3493 sbinfo->max_inodes = memparse(value, &rest); 3494 if (*rest) 3495 goto bad_val; 3496 } else if (!strcmp(this_char,"mode")) { 3497 if (remount) 3498 continue; 3499 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 3500 if (*rest) 3501 goto bad_val; 3502 } else if (!strcmp(this_char,"uid")) { 3503 if (remount) 3504 continue; 3505 uid = simple_strtoul(value, &rest, 0); 3506 if (*rest) 3507 goto bad_val; 3508 sbinfo->uid = make_kuid(current_user_ns(), uid); 3509 if (!uid_valid(sbinfo->uid)) 3510 goto bad_val; 3511 } else if (!strcmp(this_char,"gid")) { 3512 if (remount) 3513 continue; 3514 gid = simple_strtoul(value, &rest, 0); 3515 if (*rest) 3516 goto bad_val; 3517 sbinfo->gid = make_kgid(current_user_ns(), gid); 3518 if (!gid_valid(sbinfo->gid)) 3519 goto bad_val; 3520 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3521 } else if (!strcmp(this_char, "huge")) { 3522 int huge; 3523 huge = shmem_parse_huge(value); 3524 if (huge < 0) 3525 goto bad_val; 3526 if (!has_transparent_hugepage() && 3527 huge != SHMEM_HUGE_NEVER) 3528 goto bad_val; 3529 sbinfo->huge = huge; 3530 #endif 3531 #ifdef CONFIG_NUMA 3532 } else if (!strcmp(this_char,"mpol")) { 3533 mpol_put(mpol); 3534 mpol = NULL; 3535 if (mpol_parse_str(value, &mpol)) 3536 goto bad_val; 3537 #endif 3538 } else { 3539 pr_err("tmpfs: Bad mount option %s\n", this_char); 3540 goto error; 3541 } 3542 } 3543 sbinfo->mpol = mpol; 3544 return 0; 3545 3546 bad_val: 3547 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n", 3548 value, this_char); 3549 error: 3550 mpol_put(mpol); 3551 return 1; 3552 3553 } 3554 3555 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 3556 { 3557 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3558 struct shmem_sb_info config = *sbinfo; 3559 unsigned long inodes; 3560 int error = -EINVAL; 3561 3562 config.mpol = NULL; 3563 if (shmem_parse_options(data, &config, true)) 3564 return error; 3565 3566 spin_lock(&sbinfo->stat_lock); 3567 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3568 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 3569 goto out; 3570 if (config.max_inodes < inodes) 3571 goto out; 3572 /* 3573 * Those tests disallow limited->unlimited while any are in use; 3574 * but we must separately disallow unlimited->limited, because 3575 * in that case we have no record of how much is already in use. 3576 */ 3577 if (config.max_blocks && !sbinfo->max_blocks) 3578 goto out; 3579 if (config.max_inodes && !sbinfo->max_inodes) 3580 goto out; 3581 3582 error = 0; 3583 sbinfo->huge = config.huge; 3584 sbinfo->max_blocks = config.max_blocks; 3585 sbinfo->max_inodes = config.max_inodes; 3586 sbinfo->free_inodes = config.max_inodes - inodes; 3587 3588 /* 3589 * Preserve previous mempolicy unless mpol remount option was specified. 3590 */ 3591 if (config.mpol) { 3592 mpol_put(sbinfo->mpol); 3593 sbinfo->mpol = config.mpol; /* transfers initial ref */ 3594 } 3595 out: 3596 spin_unlock(&sbinfo->stat_lock); 3597 return error; 3598 } 3599 3600 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3601 { 3602 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3603 3604 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3605 seq_printf(seq, ",size=%luk", 3606 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3607 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3608 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3609 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 3610 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3611 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3612 seq_printf(seq, ",uid=%u", 3613 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3614 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3615 seq_printf(seq, ",gid=%u", 3616 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3617 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3618 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3619 if (sbinfo->huge) 3620 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3621 #endif 3622 shmem_show_mpol(seq, sbinfo->mpol); 3623 return 0; 3624 } 3625 3626 #define MFD_NAME_PREFIX "memfd:" 3627 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) 3628 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) 3629 3630 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING) 3631 3632 SYSCALL_DEFINE2(memfd_create, 3633 const char __user *, uname, 3634 unsigned int, flags) 3635 { 3636 struct shmem_inode_info *info; 3637 struct file *file; 3638 int fd, error; 3639 char *name; 3640 long len; 3641 3642 if (flags & ~(unsigned int)MFD_ALL_FLAGS) 3643 return -EINVAL; 3644 3645 /* length includes terminating zero */ 3646 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); 3647 if (len <= 0) 3648 return -EFAULT; 3649 if (len > MFD_NAME_MAX_LEN + 1) 3650 return -EINVAL; 3651 3652 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY); 3653 if (!name) 3654 return -ENOMEM; 3655 3656 strcpy(name, MFD_NAME_PREFIX); 3657 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { 3658 error = -EFAULT; 3659 goto err_name; 3660 } 3661 3662 /* terminating-zero may have changed after strnlen_user() returned */ 3663 if (name[len + MFD_NAME_PREFIX_LEN - 1]) { 3664 error = -EFAULT; 3665 goto err_name; 3666 } 3667 3668 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); 3669 if (fd < 0) { 3670 error = fd; 3671 goto err_name; 3672 } 3673 3674 file = shmem_file_setup(name, 0, VM_NORESERVE); 3675 if (IS_ERR(file)) { 3676 error = PTR_ERR(file); 3677 goto err_fd; 3678 } 3679 info = SHMEM_I(file_inode(file)); 3680 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; 3681 file->f_flags |= O_RDWR | O_LARGEFILE; 3682 if (flags & MFD_ALLOW_SEALING) 3683 info->seals &= ~F_SEAL_SEAL; 3684 3685 fd_install(fd, file); 3686 kfree(name); 3687 return fd; 3688 3689 err_fd: 3690 put_unused_fd(fd); 3691 err_name: 3692 kfree(name); 3693 return error; 3694 } 3695 3696 #endif /* CONFIG_TMPFS */ 3697 3698 static void shmem_put_super(struct super_block *sb) 3699 { 3700 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3701 3702 percpu_counter_destroy(&sbinfo->used_blocks); 3703 mpol_put(sbinfo->mpol); 3704 kfree(sbinfo); 3705 sb->s_fs_info = NULL; 3706 } 3707 3708 int shmem_fill_super(struct super_block *sb, void *data, int silent) 3709 { 3710 struct inode *inode; 3711 struct shmem_sb_info *sbinfo; 3712 int err = -ENOMEM; 3713 3714 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3715 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3716 L1_CACHE_BYTES), GFP_KERNEL); 3717 if (!sbinfo) 3718 return -ENOMEM; 3719 3720 sbinfo->mode = S_IRWXUGO | S_ISVTX; 3721 sbinfo->uid = current_fsuid(); 3722 sbinfo->gid = current_fsgid(); 3723 sb->s_fs_info = sbinfo; 3724 3725 #ifdef CONFIG_TMPFS 3726 /* 3727 * Per default we only allow half of the physical ram per 3728 * tmpfs instance, limiting inodes to one per page of lowmem; 3729 * but the internal instance is left unlimited. 3730 */ 3731 if (!(sb->s_flags & MS_KERNMOUNT)) { 3732 sbinfo->max_blocks = shmem_default_max_blocks(); 3733 sbinfo->max_inodes = shmem_default_max_inodes(); 3734 if (shmem_parse_options(data, sbinfo, false)) { 3735 err = -EINVAL; 3736 goto failed; 3737 } 3738 } else { 3739 sb->s_flags |= MS_NOUSER; 3740 } 3741 sb->s_export_op = &shmem_export_ops; 3742 sb->s_flags |= MS_NOSEC; 3743 #else 3744 sb->s_flags |= MS_NOUSER; 3745 #endif 3746 3747 spin_lock_init(&sbinfo->stat_lock); 3748 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3749 goto failed; 3750 sbinfo->free_inodes = sbinfo->max_inodes; 3751 spin_lock_init(&sbinfo->shrinklist_lock); 3752 INIT_LIST_HEAD(&sbinfo->shrinklist); 3753 3754 sb->s_maxbytes = MAX_LFS_FILESIZE; 3755 sb->s_blocksize = PAGE_SIZE; 3756 sb->s_blocksize_bits = PAGE_SHIFT; 3757 sb->s_magic = TMPFS_MAGIC; 3758 sb->s_op = &shmem_ops; 3759 sb->s_time_gran = 1; 3760 #ifdef CONFIG_TMPFS_XATTR 3761 sb->s_xattr = shmem_xattr_handlers; 3762 #endif 3763 #ifdef CONFIG_TMPFS_POSIX_ACL 3764 sb->s_flags |= MS_POSIXACL; 3765 #endif 3766 uuid_gen(&sb->s_uuid); 3767 3768 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3769 if (!inode) 3770 goto failed; 3771 inode->i_uid = sbinfo->uid; 3772 inode->i_gid = sbinfo->gid; 3773 sb->s_root = d_make_root(inode); 3774 if (!sb->s_root) 3775 goto failed; 3776 return 0; 3777 3778 failed: 3779 shmem_put_super(sb); 3780 return err; 3781 } 3782 3783 static struct kmem_cache *shmem_inode_cachep; 3784 3785 static struct inode *shmem_alloc_inode(struct super_block *sb) 3786 { 3787 struct shmem_inode_info *info; 3788 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3789 if (!info) 3790 return NULL; 3791 return &info->vfs_inode; 3792 } 3793 3794 static void shmem_destroy_callback(struct rcu_head *head) 3795 { 3796 struct inode *inode = container_of(head, struct inode, i_rcu); 3797 if (S_ISLNK(inode->i_mode)) 3798 kfree(inode->i_link); 3799 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3800 } 3801 3802 static void shmem_destroy_inode(struct inode *inode) 3803 { 3804 if (S_ISREG(inode->i_mode)) 3805 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3806 call_rcu(&inode->i_rcu, shmem_destroy_callback); 3807 } 3808 3809 static void shmem_init_inode(void *foo) 3810 { 3811 struct shmem_inode_info *info = foo; 3812 inode_init_once(&info->vfs_inode); 3813 } 3814 3815 static int shmem_init_inodecache(void) 3816 { 3817 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3818 sizeof(struct shmem_inode_info), 3819 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3820 return 0; 3821 } 3822 3823 static void shmem_destroy_inodecache(void) 3824 { 3825 kmem_cache_destroy(shmem_inode_cachep); 3826 } 3827 3828 static const struct address_space_operations shmem_aops = { 3829 .writepage = shmem_writepage, 3830 .set_page_dirty = __set_page_dirty_no_writeback, 3831 #ifdef CONFIG_TMPFS 3832 .write_begin = shmem_write_begin, 3833 .write_end = shmem_write_end, 3834 #endif 3835 #ifdef CONFIG_MIGRATION 3836 .migratepage = migrate_page, 3837 #endif 3838 .error_remove_page = generic_error_remove_page, 3839 }; 3840 3841 static const struct file_operations shmem_file_operations = { 3842 .mmap = shmem_mmap, 3843 .get_unmapped_area = shmem_get_unmapped_area, 3844 #ifdef CONFIG_TMPFS 3845 .llseek = shmem_file_llseek, 3846 .read_iter = shmem_file_read_iter, 3847 .write_iter = generic_file_write_iter, 3848 .fsync = noop_fsync, 3849 .splice_read = generic_file_splice_read, 3850 .splice_write = iter_file_splice_write, 3851 .fallocate = shmem_fallocate, 3852 #endif 3853 }; 3854 3855 static const struct inode_operations shmem_inode_operations = { 3856 .getattr = shmem_getattr, 3857 .setattr = shmem_setattr, 3858 #ifdef CONFIG_TMPFS_XATTR 3859 .listxattr = shmem_listxattr, 3860 .set_acl = simple_set_acl, 3861 #endif 3862 }; 3863 3864 static const struct inode_operations shmem_dir_inode_operations = { 3865 #ifdef CONFIG_TMPFS 3866 .create = shmem_create, 3867 .lookup = simple_lookup, 3868 .link = shmem_link, 3869 .unlink = shmem_unlink, 3870 .symlink = shmem_symlink, 3871 .mkdir = shmem_mkdir, 3872 .rmdir = shmem_rmdir, 3873 .mknod = shmem_mknod, 3874 .rename = shmem_rename2, 3875 .tmpfile = shmem_tmpfile, 3876 #endif 3877 #ifdef CONFIG_TMPFS_XATTR 3878 .listxattr = shmem_listxattr, 3879 #endif 3880 #ifdef CONFIG_TMPFS_POSIX_ACL 3881 .setattr = shmem_setattr, 3882 .set_acl = simple_set_acl, 3883 #endif 3884 }; 3885 3886 static const struct inode_operations shmem_special_inode_operations = { 3887 #ifdef CONFIG_TMPFS_XATTR 3888 .listxattr = shmem_listxattr, 3889 #endif 3890 #ifdef CONFIG_TMPFS_POSIX_ACL 3891 .setattr = shmem_setattr, 3892 .set_acl = simple_set_acl, 3893 #endif 3894 }; 3895 3896 static const struct super_operations shmem_ops = { 3897 .alloc_inode = shmem_alloc_inode, 3898 .destroy_inode = shmem_destroy_inode, 3899 #ifdef CONFIG_TMPFS 3900 .statfs = shmem_statfs, 3901 .remount_fs = shmem_remount_fs, 3902 .show_options = shmem_show_options, 3903 #endif 3904 .evict_inode = shmem_evict_inode, 3905 .drop_inode = generic_delete_inode, 3906 .put_super = shmem_put_super, 3907 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3908 .nr_cached_objects = shmem_unused_huge_count, 3909 .free_cached_objects = shmem_unused_huge_scan, 3910 #endif 3911 }; 3912 3913 static const struct vm_operations_struct shmem_vm_ops = { 3914 .fault = shmem_fault, 3915 .map_pages = filemap_map_pages, 3916 #ifdef CONFIG_NUMA 3917 .set_policy = shmem_set_policy, 3918 .get_policy = shmem_get_policy, 3919 #endif 3920 }; 3921 3922 static struct dentry *shmem_mount(struct file_system_type *fs_type, 3923 int flags, const char *dev_name, void *data) 3924 { 3925 return mount_nodev(fs_type, flags, data, shmem_fill_super); 3926 } 3927 3928 static struct file_system_type shmem_fs_type = { 3929 .owner = THIS_MODULE, 3930 .name = "tmpfs", 3931 .mount = shmem_mount, 3932 .kill_sb = kill_litter_super, 3933 .fs_flags = FS_USERNS_MOUNT, 3934 }; 3935 3936 int __init shmem_init(void) 3937 { 3938 int error; 3939 3940 /* If rootfs called this, don't re-init */ 3941 if (shmem_inode_cachep) 3942 return 0; 3943 3944 error = shmem_init_inodecache(); 3945 if (error) 3946 goto out3; 3947 3948 error = register_filesystem(&shmem_fs_type); 3949 if (error) { 3950 pr_err("Could not register tmpfs\n"); 3951 goto out2; 3952 } 3953 3954 shm_mnt = kern_mount(&shmem_fs_type); 3955 if (IS_ERR(shm_mnt)) { 3956 error = PTR_ERR(shm_mnt); 3957 pr_err("Could not kern_mount tmpfs\n"); 3958 goto out1; 3959 } 3960 3961 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3962 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY) 3963 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3964 else 3965 shmem_huge = 0; /* just in case it was patched */ 3966 #endif 3967 return 0; 3968 3969 out1: 3970 unregister_filesystem(&shmem_fs_type); 3971 out2: 3972 shmem_destroy_inodecache(); 3973 out3: 3974 shm_mnt = ERR_PTR(error); 3975 return error; 3976 } 3977 3978 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) 3979 static ssize_t shmem_enabled_show(struct kobject *kobj, 3980 struct kobj_attribute *attr, char *buf) 3981 { 3982 int values[] = { 3983 SHMEM_HUGE_ALWAYS, 3984 SHMEM_HUGE_WITHIN_SIZE, 3985 SHMEM_HUGE_ADVISE, 3986 SHMEM_HUGE_NEVER, 3987 SHMEM_HUGE_DENY, 3988 SHMEM_HUGE_FORCE, 3989 }; 3990 int i, count; 3991 3992 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { 3993 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; 3994 3995 count += sprintf(buf + count, fmt, 3996 shmem_format_huge(values[i])); 3997 } 3998 buf[count - 1] = '\n'; 3999 return count; 4000 } 4001 4002 static ssize_t shmem_enabled_store(struct kobject *kobj, 4003 struct kobj_attribute *attr, const char *buf, size_t count) 4004 { 4005 char tmp[16]; 4006 int huge; 4007 4008 if (count + 1 > sizeof(tmp)) 4009 return -EINVAL; 4010 memcpy(tmp, buf, count); 4011 tmp[count] = '\0'; 4012 if (count && tmp[count - 1] == '\n') 4013 tmp[count - 1] = '\0'; 4014 4015 huge = shmem_parse_huge(tmp); 4016 if (huge == -EINVAL) 4017 return -EINVAL; 4018 if (!has_transparent_hugepage() && 4019 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 4020 return -EINVAL; 4021 4022 shmem_huge = huge; 4023 if (shmem_huge < SHMEM_HUGE_DENY) 4024 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4025 return count; 4026 } 4027 4028 struct kobj_attribute shmem_enabled_attr = 4029 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 4030 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ 4031 4032 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 4033 bool shmem_huge_enabled(struct vm_area_struct *vma) 4034 { 4035 struct inode *inode = file_inode(vma->vm_file); 4036 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 4037 loff_t i_size; 4038 pgoff_t off; 4039 4040 if (shmem_huge == SHMEM_HUGE_FORCE) 4041 return true; 4042 if (shmem_huge == SHMEM_HUGE_DENY) 4043 return false; 4044 switch (sbinfo->huge) { 4045 case SHMEM_HUGE_NEVER: 4046 return false; 4047 case SHMEM_HUGE_ALWAYS: 4048 return true; 4049 case SHMEM_HUGE_WITHIN_SIZE: 4050 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); 4051 i_size = round_up(i_size_read(inode), PAGE_SIZE); 4052 if (i_size >= HPAGE_PMD_SIZE && 4053 i_size >> PAGE_SHIFT >= off) 4054 return true; 4055 case SHMEM_HUGE_ADVISE: 4056 /* TODO: implement fadvise() hints */ 4057 return (vma->vm_flags & VM_HUGEPAGE); 4058 default: 4059 VM_BUG_ON(1); 4060 return false; 4061 } 4062 } 4063 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 4064 4065 #else /* !CONFIG_SHMEM */ 4066 4067 /* 4068 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 4069 * 4070 * This is intended for small system where the benefits of the full 4071 * shmem code (swap-backed and resource-limited) are outweighed by 4072 * their complexity. On systems without swap this code should be 4073 * effectively equivalent, but much lighter weight. 4074 */ 4075 4076 static struct file_system_type shmem_fs_type = { 4077 .name = "tmpfs", 4078 .mount = ramfs_mount, 4079 .kill_sb = kill_litter_super, 4080 .fs_flags = FS_USERNS_MOUNT, 4081 }; 4082 4083 int __init shmem_init(void) 4084 { 4085 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4086 4087 shm_mnt = kern_mount(&shmem_fs_type); 4088 BUG_ON(IS_ERR(shm_mnt)); 4089 4090 return 0; 4091 } 4092 4093 int shmem_unuse(swp_entry_t swap, struct page *page) 4094 { 4095 return 0; 4096 } 4097 4098 int shmem_lock(struct file *file, int lock, struct user_struct *user) 4099 { 4100 return 0; 4101 } 4102 4103 void shmem_unlock_mapping(struct address_space *mapping) 4104 { 4105 } 4106 4107 #ifdef CONFIG_MMU 4108 unsigned long shmem_get_unmapped_area(struct file *file, 4109 unsigned long addr, unsigned long len, 4110 unsigned long pgoff, unsigned long flags) 4111 { 4112 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 4113 } 4114 #endif 4115 4116 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4117 { 4118 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4119 } 4120 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4121 4122 #define shmem_vm_ops generic_file_vm_ops 4123 #define shmem_file_operations ramfs_file_operations 4124 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 4125 #define shmem_acct_size(flags, size) 0 4126 #define shmem_unacct_size(flags, size) do {} while (0) 4127 4128 #endif /* CONFIG_SHMEM */ 4129 4130 /* common code */ 4131 4132 static const struct dentry_operations anon_ops = { 4133 .d_dname = simple_dname 4134 }; 4135 4136 static struct file *__shmem_file_setup(const char *name, loff_t size, 4137 unsigned long flags, unsigned int i_flags) 4138 { 4139 struct file *res; 4140 struct inode *inode; 4141 struct path path; 4142 struct super_block *sb; 4143 struct qstr this; 4144 4145 if (IS_ERR(shm_mnt)) 4146 return ERR_CAST(shm_mnt); 4147 4148 if (size < 0 || size > MAX_LFS_FILESIZE) 4149 return ERR_PTR(-EINVAL); 4150 4151 if (shmem_acct_size(flags, size)) 4152 return ERR_PTR(-ENOMEM); 4153 4154 res = ERR_PTR(-ENOMEM); 4155 this.name = name; 4156 this.len = strlen(name); 4157 this.hash = 0; /* will go */ 4158 sb = shm_mnt->mnt_sb; 4159 path.mnt = mntget(shm_mnt); 4160 path.dentry = d_alloc_pseudo(sb, &this); 4161 if (!path.dentry) 4162 goto put_memory; 4163 d_set_d_op(path.dentry, &anon_ops); 4164 4165 res = ERR_PTR(-ENOSPC); 4166 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 4167 if (!inode) 4168 goto put_memory; 4169 4170 inode->i_flags |= i_flags; 4171 d_instantiate(path.dentry, inode); 4172 inode->i_size = size; 4173 clear_nlink(inode); /* It is unlinked */ 4174 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4175 if (IS_ERR(res)) 4176 goto put_path; 4177 4178 res = alloc_file(&path, FMODE_WRITE | FMODE_READ, 4179 &shmem_file_operations); 4180 if (IS_ERR(res)) 4181 goto put_path; 4182 4183 return res; 4184 4185 put_memory: 4186 shmem_unacct_size(flags, size); 4187 put_path: 4188 path_put(&path); 4189 return res; 4190 } 4191 4192 /** 4193 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4194 * kernel internal. There will be NO LSM permission checks against the 4195 * underlying inode. So users of this interface must do LSM checks at a 4196 * higher layer. The users are the big_key and shm implementations. LSM 4197 * checks are provided at the key or shm level rather than the inode. 4198 * @name: name for dentry (to be seen in /proc/<pid>/maps 4199 * @size: size to be set for the file 4200 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4201 */ 4202 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4203 { 4204 return __shmem_file_setup(name, size, flags, S_PRIVATE); 4205 } 4206 4207 /** 4208 * shmem_file_setup - get an unlinked file living in tmpfs 4209 * @name: name for dentry (to be seen in /proc/<pid>/maps 4210 * @size: size to be set for the file 4211 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4212 */ 4213 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4214 { 4215 return __shmem_file_setup(name, size, flags, 0); 4216 } 4217 EXPORT_SYMBOL_GPL(shmem_file_setup); 4218 4219 /** 4220 * shmem_zero_setup - setup a shared anonymous mapping 4221 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 4222 */ 4223 int shmem_zero_setup(struct vm_area_struct *vma) 4224 { 4225 struct file *file; 4226 loff_t size = vma->vm_end - vma->vm_start; 4227 4228 /* 4229 * Cloning a new file under mmap_sem leads to a lock ordering conflict 4230 * between XFS directory reading and selinux: since this file is only 4231 * accessible to the user through its mapping, use S_PRIVATE flag to 4232 * bypass file security, in the same way as shmem_kernel_file_setup(). 4233 */ 4234 file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE); 4235 if (IS_ERR(file)) 4236 return PTR_ERR(file); 4237 4238 if (vma->vm_file) 4239 fput(vma->vm_file); 4240 vma->vm_file = file; 4241 vma->vm_ops = &shmem_vm_ops; 4242 4243 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 4244 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4245 (vma->vm_end & HPAGE_PMD_MASK)) { 4246 khugepaged_enter(vma, vma->vm_flags); 4247 } 4248 4249 return 0; 4250 } 4251 4252 /** 4253 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4254 * @mapping: the page's address_space 4255 * @index: the page index 4256 * @gfp: the page allocator flags to use if allocating 4257 * 4258 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4259 * with any new page allocations done using the specified allocation flags. 4260 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4261 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4262 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4263 * 4264 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4265 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4266 */ 4267 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4268 pgoff_t index, gfp_t gfp) 4269 { 4270 #ifdef CONFIG_SHMEM 4271 struct inode *inode = mapping->host; 4272 struct page *page; 4273 int error; 4274 4275 BUG_ON(mapping->a_ops != &shmem_aops); 4276 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4277 gfp, NULL, NULL, NULL); 4278 if (error) 4279 page = ERR_PTR(error); 4280 else 4281 unlock_page(page); 4282 return page; 4283 #else 4284 /* 4285 * The tiny !SHMEM case uses ramfs without swap 4286 */ 4287 return read_cache_page_gfp(mapping, index, gfp); 4288 #endif 4289 } 4290 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4291