1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/random.h> 33 #include <linux/sched/signal.h> 34 #include <linux/export.h> 35 #include <linux/swap.h> 36 #include <linux/uio.h> 37 #include <linux/khugepaged.h> 38 #include <linux/hugetlb.h> 39 #include <linux/frontswap.h> 40 #include <linux/fs_parser.h> 41 #include <linux/swapfile.h> 42 43 static struct vfsmount *shm_mnt; 44 45 #ifdef CONFIG_SHMEM 46 /* 47 * This virtual memory filesystem is heavily based on the ramfs. It 48 * extends ramfs by the ability to use swap and honor resource limits 49 * which makes it a completely usable filesystem. 50 */ 51 52 #include <linux/xattr.h> 53 #include <linux/exportfs.h> 54 #include <linux/posix_acl.h> 55 #include <linux/posix_acl_xattr.h> 56 #include <linux/mman.h> 57 #include <linux/string.h> 58 #include <linux/slab.h> 59 #include <linux/backing-dev.h> 60 #include <linux/shmem_fs.h> 61 #include <linux/writeback.h> 62 #include <linux/pagevec.h> 63 #include <linux/percpu_counter.h> 64 #include <linux/falloc.h> 65 #include <linux/splice.h> 66 #include <linux/security.h> 67 #include <linux/swapops.h> 68 #include <linux/mempolicy.h> 69 #include <linux/namei.h> 70 #include <linux/ctype.h> 71 #include <linux/migrate.h> 72 #include <linux/highmem.h> 73 #include <linux/seq_file.h> 74 #include <linux/magic.h> 75 #include <linux/syscalls.h> 76 #include <linux/fcntl.h> 77 #include <uapi/linux/memfd.h> 78 #include <linux/userfaultfd_k.h> 79 #include <linux/rmap.h> 80 #include <linux/uuid.h> 81 82 #include <linux/uaccess.h> 83 84 #include "internal.h" 85 86 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 87 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 88 89 /* Pretend that each entry is of this size in directory's i_size */ 90 #define BOGO_DIRENT_SIZE 20 91 92 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 93 #define SHORT_SYMLINK_LEN 128 94 95 /* 96 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 97 * inode->i_private (with i_rwsem making sure that it has only one user at 98 * a time): we would prefer not to enlarge the shmem inode just for that. 99 */ 100 struct shmem_falloc { 101 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 102 pgoff_t start; /* start of range currently being fallocated */ 103 pgoff_t next; /* the next page offset to be fallocated */ 104 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 105 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 106 }; 107 108 struct shmem_options { 109 unsigned long long blocks; 110 unsigned long long inodes; 111 struct mempolicy *mpol; 112 kuid_t uid; 113 kgid_t gid; 114 umode_t mode; 115 bool full_inums; 116 int huge; 117 int seen; 118 #define SHMEM_SEEN_BLOCKS 1 119 #define SHMEM_SEEN_INODES 2 120 #define SHMEM_SEEN_HUGE 4 121 #define SHMEM_SEEN_INUMS 8 122 }; 123 124 #ifdef CONFIG_TMPFS 125 static unsigned long shmem_default_max_blocks(void) 126 { 127 return totalram_pages() / 2; 128 } 129 130 static unsigned long shmem_default_max_inodes(void) 131 { 132 unsigned long nr_pages = totalram_pages(); 133 134 return min(nr_pages - totalhigh_pages(), nr_pages / 2); 135 } 136 #endif 137 138 static int shmem_swapin_page(struct inode *inode, pgoff_t index, 139 struct page **pagep, enum sgp_type sgp, 140 gfp_t gfp, struct vm_area_struct *vma, 141 vm_fault_t *fault_type); 142 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 143 struct page **pagep, enum sgp_type sgp, 144 gfp_t gfp, struct vm_area_struct *vma, 145 struct vm_fault *vmf, vm_fault_t *fault_type); 146 147 int shmem_getpage(struct inode *inode, pgoff_t index, 148 struct page **pagep, enum sgp_type sgp) 149 { 150 return shmem_getpage_gfp(inode, index, pagep, sgp, 151 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); 152 } 153 154 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 155 { 156 return sb->s_fs_info; 157 } 158 159 /* 160 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 161 * for shared memory and for shared anonymous (/dev/zero) mappings 162 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 163 * consistent with the pre-accounting of private mappings ... 164 */ 165 static inline int shmem_acct_size(unsigned long flags, loff_t size) 166 { 167 return (flags & VM_NORESERVE) ? 168 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 169 } 170 171 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 172 { 173 if (!(flags & VM_NORESERVE)) 174 vm_unacct_memory(VM_ACCT(size)); 175 } 176 177 static inline int shmem_reacct_size(unsigned long flags, 178 loff_t oldsize, loff_t newsize) 179 { 180 if (!(flags & VM_NORESERVE)) { 181 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 182 return security_vm_enough_memory_mm(current->mm, 183 VM_ACCT(newsize) - VM_ACCT(oldsize)); 184 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 185 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 186 } 187 return 0; 188 } 189 190 /* 191 * ... whereas tmpfs objects are accounted incrementally as 192 * pages are allocated, in order to allow large sparse files. 193 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 194 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 195 */ 196 static inline int shmem_acct_block(unsigned long flags, long pages) 197 { 198 if (!(flags & VM_NORESERVE)) 199 return 0; 200 201 return security_vm_enough_memory_mm(current->mm, 202 pages * VM_ACCT(PAGE_SIZE)); 203 } 204 205 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 206 { 207 if (flags & VM_NORESERVE) 208 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 209 } 210 211 static inline bool shmem_inode_acct_block(struct inode *inode, long pages) 212 { 213 struct shmem_inode_info *info = SHMEM_I(inode); 214 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 215 216 if (shmem_acct_block(info->flags, pages)) 217 return false; 218 219 if (sbinfo->max_blocks) { 220 if (percpu_counter_compare(&sbinfo->used_blocks, 221 sbinfo->max_blocks - pages) > 0) 222 goto unacct; 223 percpu_counter_add(&sbinfo->used_blocks, pages); 224 } 225 226 return true; 227 228 unacct: 229 shmem_unacct_blocks(info->flags, pages); 230 return false; 231 } 232 233 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages) 234 { 235 struct shmem_inode_info *info = SHMEM_I(inode); 236 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 237 238 if (sbinfo->max_blocks) 239 percpu_counter_sub(&sbinfo->used_blocks, pages); 240 shmem_unacct_blocks(info->flags, pages); 241 } 242 243 static const struct super_operations shmem_ops; 244 const struct address_space_operations shmem_aops; 245 static const struct file_operations shmem_file_operations; 246 static const struct inode_operations shmem_inode_operations; 247 static const struct inode_operations shmem_dir_inode_operations; 248 static const struct inode_operations shmem_special_inode_operations; 249 static const struct vm_operations_struct shmem_vm_ops; 250 static struct file_system_type shmem_fs_type; 251 252 bool vma_is_shmem(struct vm_area_struct *vma) 253 { 254 return vma->vm_ops == &shmem_vm_ops; 255 } 256 257 static LIST_HEAD(shmem_swaplist); 258 static DEFINE_MUTEX(shmem_swaplist_mutex); 259 260 /* 261 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and 262 * produces a novel ino for the newly allocated inode. 263 * 264 * It may also be called when making a hard link to permit the space needed by 265 * each dentry. However, in that case, no new inode number is needed since that 266 * internally draws from another pool of inode numbers (currently global 267 * get_next_ino()). This case is indicated by passing NULL as inop. 268 */ 269 #define SHMEM_INO_BATCH 1024 270 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop) 271 { 272 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 273 ino_t ino; 274 275 if (!(sb->s_flags & SB_KERNMOUNT)) { 276 raw_spin_lock(&sbinfo->stat_lock); 277 if (sbinfo->max_inodes) { 278 if (!sbinfo->free_inodes) { 279 raw_spin_unlock(&sbinfo->stat_lock); 280 return -ENOSPC; 281 } 282 sbinfo->free_inodes--; 283 } 284 if (inop) { 285 ino = sbinfo->next_ino++; 286 if (unlikely(is_zero_ino(ino))) 287 ino = sbinfo->next_ino++; 288 if (unlikely(!sbinfo->full_inums && 289 ino > UINT_MAX)) { 290 /* 291 * Emulate get_next_ino uint wraparound for 292 * compatibility 293 */ 294 if (IS_ENABLED(CONFIG_64BIT)) 295 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n", 296 __func__, MINOR(sb->s_dev)); 297 sbinfo->next_ino = 1; 298 ino = sbinfo->next_ino++; 299 } 300 *inop = ino; 301 } 302 raw_spin_unlock(&sbinfo->stat_lock); 303 } else if (inop) { 304 /* 305 * __shmem_file_setup, one of our callers, is lock-free: it 306 * doesn't hold stat_lock in shmem_reserve_inode since 307 * max_inodes is always 0, and is called from potentially 308 * unknown contexts. As such, use a per-cpu batched allocator 309 * which doesn't require the per-sb stat_lock unless we are at 310 * the batch boundary. 311 * 312 * We don't need to worry about inode{32,64} since SB_KERNMOUNT 313 * shmem mounts are not exposed to userspace, so we don't need 314 * to worry about things like glibc compatibility. 315 */ 316 ino_t *next_ino; 317 318 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu()); 319 ino = *next_ino; 320 if (unlikely(ino % SHMEM_INO_BATCH == 0)) { 321 raw_spin_lock(&sbinfo->stat_lock); 322 ino = sbinfo->next_ino; 323 sbinfo->next_ino += SHMEM_INO_BATCH; 324 raw_spin_unlock(&sbinfo->stat_lock); 325 if (unlikely(is_zero_ino(ino))) 326 ino++; 327 } 328 *inop = ino; 329 *next_ino = ++ino; 330 put_cpu(); 331 } 332 333 return 0; 334 } 335 336 static void shmem_free_inode(struct super_block *sb) 337 { 338 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 339 if (sbinfo->max_inodes) { 340 raw_spin_lock(&sbinfo->stat_lock); 341 sbinfo->free_inodes++; 342 raw_spin_unlock(&sbinfo->stat_lock); 343 } 344 } 345 346 /** 347 * shmem_recalc_inode - recalculate the block usage of an inode 348 * @inode: inode to recalc 349 * 350 * We have to calculate the free blocks since the mm can drop 351 * undirtied hole pages behind our back. 352 * 353 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 354 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 355 * 356 * It has to be called with the spinlock held. 357 */ 358 static void shmem_recalc_inode(struct inode *inode) 359 { 360 struct shmem_inode_info *info = SHMEM_I(inode); 361 long freed; 362 363 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 364 if (freed > 0) { 365 info->alloced -= freed; 366 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 367 shmem_inode_unacct_blocks(inode, freed); 368 } 369 } 370 371 bool shmem_charge(struct inode *inode, long pages) 372 { 373 struct shmem_inode_info *info = SHMEM_I(inode); 374 unsigned long flags; 375 376 if (!shmem_inode_acct_block(inode, pages)) 377 return false; 378 379 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */ 380 inode->i_mapping->nrpages += pages; 381 382 spin_lock_irqsave(&info->lock, flags); 383 info->alloced += pages; 384 inode->i_blocks += pages * BLOCKS_PER_PAGE; 385 shmem_recalc_inode(inode); 386 spin_unlock_irqrestore(&info->lock, flags); 387 388 return true; 389 } 390 391 void shmem_uncharge(struct inode *inode, long pages) 392 { 393 struct shmem_inode_info *info = SHMEM_I(inode); 394 unsigned long flags; 395 396 /* nrpages adjustment done by __delete_from_page_cache() or caller */ 397 398 spin_lock_irqsave(&info->lock, flags); 399 info->alloced -= pages; 400 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 401 shmem_recalc_inode(inode); 402 spin_unlock_irqrestore(&info->lock, flags); 403 404 shmem_inode_unacct_blocks(inode, pages); 405 } 406 407 /* 408 * Replace item expected in xarray by a new item, while holding xa_lock. 409 */ 410 static int shmem_replace_entry(struct address_space *mapping, 411 pgoff_t index, void *expected, void *replacement) 412 { 413 XA_STATE(xas, &mapping->i_pages, index); 414 void *item; 415 416 VM_BUG_ON(!expected); 417 VM_BUG_ON(!replacement); 418 item = xas_load(&xas); 419 if (item != expected) 420 return -ENOENT; 421 xas_store(&xas, replacement); 422 return 0; 423 } 424 425 /* 426 * Sometimes, before we decide whether to proceed or to fail, we must check 427 * that an entry was not already brought back from swap by a racing thread. 428 * 429 * Checking page is not enough: by the time a SwapCache page is locked, it 430 * might be reused, and again be SwapCache, using the same swap as before. 431 */ 432 static bool shmem_confirm_swap(struct address_space *mapping, 433 pgoff_t index, swp_entry_t swap) 434 { 435 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap); 436 } 437 438 /* 439 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 440 * 441 * SHMEM_HUGE_NEVER: 442 * disables huge pages for the mount; 443 * SHMEM_HUGE_ALWAYS: 444 * enables huge pages for the mount; 445 * SHMEM_HUGE_WITHIN_SIZE: 446 * only allocate huge pages if the page will be fully within i_size, 447 * also respect fadvise()/madvise() hints; 448 * SHMEM_HUGE_ADVISE: 449 * only allocate huge pages if requested with fadvise()/madvise(); 450 */ 451 452 #define SHMEM_HUGE_NEVER 0 453 #define SHMEM_HUGE_ALWAYS 1 454 #define SHMEM_HUGE_WITHIN_SIZE 2 455 #define SHMEM_HUGE_ADVISE 3 456 457 /* 458 * Special values. 459 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 460 * 461 * SHMEM_HUGE_DENY: 462 * disables huge on shm_mnt and all mounts, for emergency use; 463 * SHMEM_HUGE_FORCE: 464 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 465 * 466 */ 467 #define SHMEM_HUGE_DENY (-1) 468 #define SHMEM_HUGE_FORCE (-2) 469 470 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 471 /* ifdef here to avoid bloating shmem.o when not necessary */ 472 473 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER; 474 475 bool shmem_is_huge(struct vm_area_struct *vma, 476 struct inode *inode, pgoff_t index) 477 { 478 loff_t i_size; 479 480 if (shmem_huge == SHMEM_HUGE_DENY) 481 return false; 482 if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) || 483 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))) 484 return false; 485 if (shmem_huge == SHMEM_HUGE_FORCE) 486 return true; 487 488 switch (SHMEM_SB(inode->i_sb)->huge) { 489 case SHMEM_HUGE_ALWAYS: 490 return true; 491 case SHMEM_HUGE_WITHIN_SIZE: 492 index = round_up(index + 1, HPAGE_PMD_NR); 493 i_size = round_up(i_size_read(inode), PAGE_SIZE); 494 if (i_size >> PAGE_SHIFT >= index) 495 return true; 496 fallthrough; 497 case SHMEM_HUGE_ADVISE: 498 if (vma && (vma->vm_flags & VM_HUGEPAGE)) 499 return true; 500 fallthrough; 501 default: 502 return false; 503 } 504 } 505 506 #if defined(CONFIG_SYSFS) 507 static int shmem_parse_huge(const char *str) 508 { 509 if (!strcmp(str, "never")) 510 return SHMEM_HUGE_NEVER; 511 if (!strcmp(str, "always")) 512 return SHMEM_HUGE_ALWAYS; 513 if (!strcmp(str, "within_size")) 514 return SHMEM_HUGE_WITHIN_SIZE; 515 if (!strcmp(str, "advise")) 516 return SHMEM_HUGE_ADVISE; 517 if (!strcmp(str, "deny")) 518 return SHMEM_HUGE_DENY; 519 if (!strcmp(str, "force")) 520 return SHMEM_HUGE_FORCE; 521 return -EINVAL; 522 } 523 #endif 524 525 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 526 static const char *shmem_format_huge(int huge) 527 { 528 switch (huge) { 529 case SHMEM_HUGE_NEVER: 530 return "never"; 531 case SHMEM_HUGE_ALWAYS: 532 return "always"; 533 case SHMEM_HUGE_WITHIN_SIZE: 534 return "within_size"; 535 case SHMEM_HUGE_ADVISE: 536 return "advise"; 537 case SHMEM_HUGE_DENY: 538 return "deny"; 539 case SHMEM_HUGE_FORCE: 540 return "force"; 541 default: 542 VM_BUG_ON(1); 543 return "bad_val"; 544 } 545 } 546 #endif 547 548 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 549 struct shrink_control *sc, unsigned long nr_to_split) 550 { 551 LIST_HEAD(list), *pos, *next; 552 LIST_HEAD(to_remove); 553 struct inode *inode; 554 struct shmem_inode_info *info; 555 struct page *page; 556 unsigned long batch = sc ? sc->nr_to_scan : 128; 557 int split = 0; 558 559 if (list_empty(&sbinfo->shrinklist)) 560 return SHRINK_STOP; 561 562 spin_lock(&sbinfo->shrinklist_lock); 563 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 564 info = list_entry(pos, struct shmem_inode_info, shrinklist); 565 566 /* pin the inode */ 567 inode = igrab(&info->vfs_inode); 568 569 /* inode is about to be evicted */ 570 if (!inode) { 571 list_del_init(&info->shrinklist); 572 goto next; 573 } 574 575 /* Check if there's anything to gain */ 576 if (round_up(inode->i_size, PAGE_SIZE) == 577 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 578 list_move(&info->shrinklist, &to_remove); 579 goto next; 580 } 581 582 list_move(&info->shrinklist, &list); 583 next: 584 sbinfo->shrinklist_len--; 585 if (!--batch) 586 break; 587 } 588 spin_unlock(&sbinfo->shrinklist_lock); 589 590 list_for_each_safe(pos, next, &to_remove) { 591 info = list_entry(pos, struct shmem_inode_info, shrinklist); 592 inode = &info->vfs_inode; 593 list_del_init(&info->shrinklist); 594 iput(inode); 595 } 596 597 list_for_each_safe(pos, next, &list) { 598 int ret; 599 600 info = list_entry(pos, struct shmem_inode_info, shrinklist); 601 inode = &info->vfs_inode; 602 603 if (nr_to_split && split >= nr_to_split) 604 goto move_back; 605 606 page = find_get_page(inode->i_mapping, 607 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 608 if (!page) 609 goto drop; 610 611 /* No huge page at the end of the file: nothing to split */ 612 if (!PageTransHuge(page)) { 613 put_page(page); 614 goto drop; 615 } 616 617 /* 618 * Move the inode on the list back to shrinklist if we failed 619 * to lock the page at this time. 620 * 621 * Waiting for the lock may lead to deadlock in the 622 * reclaim path. 623 */ 624 if (!trylock_page(page)) { 625 put_page(page); 626 goto move_back; 627 } 628 629 ret = split_huge_page(page); 630 unlock_page(page); 631 put_page(page); 632 633 /* If split failed move the inode on the list back to shrinklist */ 634 if (ret) 635 goto move_back; 636 637 split++; 638 drop: 639 list_del_init(&info->shrinklist); 640 goto put; 641 move_back: 642 /* 643 * Make sure the inode is either on the global list or deleted 644 * from any local list before iput() since it could be deleted 645 * in another thread once we put the inode (then the local list 646 * is corrupted). 647 */ 648 spin_lock(&sbinfo->shrinklist_lock); 649 list_move(&info->shrinklist, &sbinfo->shrinklist); 650 sbinfo->shrinklist_len++; 651 spin_unlock(&sbinfo->shrinklist_lock); 652 put: 653 iput(inode); 654 } 655 656 return split; 657 } 658 659 static long shmem_unused_huge_scan(struct super_block *sb, 660 struct shrink_control *sc) 661 { 662 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 663 664 if (!READ_ONCE(sbinfo->shrinklist_len)) 665 return SHRINK_STOP; 666 667 return shmem_unused_huge_shrink(sbinfo, sc, 0); 668 } 669 670 static long shmem_unused_huge_count(struct super_block *sb, 671 struct shrink_control *sc) 672 { 673 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 674 return READ_ONCE(sbinfo->shrinklist_len); 675 } 676 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 677 678 #define shmem_huge SHMEM_HUGE_DENY 679 680 bool shmem_is_huge(struct vm_area_struct *vma, 681 struct inode *inode, pgoff_t index) 682 { 683 return false; 684 } 685 686 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 687 struct shrink_control *sc, unsigned long nr_to_split) 688 { 689 return 0; 690 } 691 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 692 693 /* 694 * Like add_to_page_cache_locked, but error if expected item has gone. 695 */ 696 static int shmem_add_to_page_cache(struct page *page, 697 struct address_space *mapping, 698 pgoff_t index, void *expected, gfp_t gfp, 699 struct mm_struct *charge_mm) 700 { 701 XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page)); 702 unsigned long nr = compound_nr(page); 703 int error; 704 705 VM_BUG_ON_PAGE(PageTail(page), page); 706 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 707 VM_BUG_ON_PAGE(!PageLocked(page), page); 708 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 709 VM_BUG_ON(expected && PageTransHuge(page)); 710 711 page_ref_add(page, nr); 712 page->mapping = mapping; 713 page->index = index; 714 715 if (!PageSwapCache(page)) { 716 error = mem_cgroup_charge(page_folio(page), charge_mm, gfp); 717 if (error) { 718 if (PageTransHuge(page)) { 719 count_vm_event(THP_FILE_FALLBACK); 720 count_vm_event(THP_FILE_FALLBACK_CHARGE); 721 } 722 goto error; 723 } 724 } 725 cgroup_throttle_swaprate(page, gfp); 726 727 do { 728 xas_lock_irq(&xas); 729 if (expected != xas_find_conflict(&xas)) { 730 xas_set_err(&xas, -EEXIST); 731 goto unlock; 732 } 733 if (expected && xas_find_conflict(&xas)) { 734 xas_set_err(&xas, -EEXIST); 735 goto unlock; 736 } 737 xas_store(&xas, page); 738 if (xas_error(&xas)) 739 goto unlock; 740 if (PageTransHuge(page)) { 741 count_vm_event(THP_FILE_ALLOC); 742 __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr); 743 } 744 mapping->nrpages += nr; 745 __mod_lruvec_page_state(page, NR_FILE_PAGES, nr); 746 __mod_lruvec_page_state(page, NR_SHMEM, nr); 747 unlock: 748 xas_unlock_irq(&xas); 749 } while (xas_nomem(&xas, gfp)); 750 751 if (xas_error(&xas)) { 752 error = xas_error(&xas); 753 goto error; 754 } 755 756 return 0; 757 error: 758 page->mapping = NULL; 759 page_ref_sub(page, nr); 760 return error; 761 } 762 763 /* 764 * Like delete_from_page_cache, but substitutes swap for page. 765 */ 766 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 767 { 768 struct address_space *mapping = page->mapping; 769 int error; 770 771 VM_BUG_ON_PAGE(PageCompound(page), page); 772 773 xa_lock_irq(&mapping->i_pages); 774 error = shmem_replace_entry(mapping, page->index, page, radswap); 775 page->mapping = NULL; 776 mapping->nrpages--; 777 __dec_lruvec_page_state(page, NR_FILE_PAGES); 778 __dec_lruvec_page_state(page, NR_SHMEM); 779 xa_unlock_irq(&mapping->i_pages); 780 put_page(page); 781 BUG_ON(error); 782 } 783 784 /* 785 * Remove swap entry from page cache, free the swap and its page cache. 786 */ 787 static int shmem_free_swap(struct address_space *mapping, 788 pgoff_t index, void *radswap) 789 { 790 void *old; 791 792 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); 793 if (old != radswap) 794 return -ENOENT; 795 free_swap_and_cache(radix_to_swp_entry(radswap)); 796 return 0; 797 } 798 799 /* 800 * Determine (in bytes) how many of the shmem object's pages mapped by the 801 * given offsets are swapped out. 802 * 803 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 804 * as long as the inode doesn't go away and racy results are not a problem. 805 */ 806 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 807 pgoff_t start, pgoff_t end) 808 { 809 XA_STATE(xas, &mapping->i_pages, start); 810 struct page *page; 811 unsigned long swapped = 0; 812 813 rcu_read_lock(); 814 xas_for_each(&xas, page, end - 1) { 815 if (xas_retry(&xas, page)) 816 continue; 817 if (xa_is_value(page)) 818 swapped++; 819 820 if (need_resched()) { 821 xas_pause(&xas); 822 cond_resched_rcu(); 823 } 824 } 825 826 rcu_read_unlock(); 827 828 return swapped << PAGE_SHIFT; 829 } 830 831 /* 832 * Determine (in bytes) how many of the shmem object's pages mapped by the 833 * given vma is swapped out. 834 * 835 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 836 * as long as the inode doesn't go away and racy results are not a problem. 837 */ 838 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 839 { 840 struct inode *inode = file_inode(vma->vm_file); 841 struct shmem_inode_info *info = SHMEM_I(inode); 842 struct address_space *mapping = inode->i_mapping; 843 unsigned long swapped; 844 845 /* Be careful as we don't hold info->lock */ 846 swapped = READ_ONCE(info->swapped); 847 848 /* 849 * The easier cases are when the shmem object has nothing in swap, or 850 * the vma maps it whole. Then we can simply use the stats that we 851 * already track. 852 */ 853 if (!swapped) 854 return 0; 855 856 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 857 return swapped << PAGE_SHIFT; 858 859 /* Here comes the more involved part */ 860 return shmem_partial_swap_usage(mapping, vma->vm_pgoff, 861 vma->vm_pgoff + vma_pages(vma)); 862 } 863 864 /* 865 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 866 */ 867 void shmem_unlock_mapping(struct address_space *mapping) 868 { 869 struct pagevec pvec; 870 pgoff_t index = 0; 871 872 pagevec_init(&pvec); 873 /* 874 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 875 */ 876 while (!mapping_unevictable(mapping)) { 877 if (!pagevec_lookup(&pvec, mapping, &index)) 878 break; 879 check_move_unevictable_pages(&pvec); 880 pagevec_release(&pvec); 881 cond_resched(); 882 } 883 } 884 885 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) 886 { 887 struct folio *folio; 888 struct page *page; 889 890 /* 891 * At first avoid shmem_getpage(,,,SGP_READ): that fails 892 * beyond i_size, and reports fallocated pages as holes. 893 */ 894 folio = __filemap_get_folio(inode->i_mapping, index, 895 FGP_ENTRY | FGP_LOCK, 0); 896 if (!xa_is_value(folio)) 897 return folio; 898 /* 899 * But read a page back from swap if any of it is within i_size 900 * (although in some cases this is just a waste of time). 901 */ 902 page = NULL; 903 shmem_getpage(inode, index, &page, SGP_READ); 904 return page ? page_folio(page) : NULL; 905 } 906 907 /* 908 * Remove range of pages and swap entries from page cache, and free them. 909 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 910 */ 911 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 912 bool unfalloc) 913 { 914 struct address_space *mapping = inode->i_mapping; 915 struct shmem_inode_info *info = SHMEM_I(inode); 916 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 917 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 918 struct folio_batch fbatch; 919 pgoff_t indices[PAGEVEC_SIZE]; 920 struct folio *folio; 921 bool same_folio; 922 long nr_swaps_freed = 0; 923 pgoff_t index; 924 int i; 925 926 if (lend == -1) 927 end = -1; /* unsigned, so actually very big */ 928 929 if (info->fallocend > start && info->fallocend <= end && !unfalloc) 930 info->fallocend = start; 931 932 folio_batch_init(&fbatch); 933 index = start; 934 while (index < end && find_lock_entries(mapping, index, end - 1, 935 &fbatch, indices)) { 936 for (i = 0; i < folio_batch_count(&fbatch); i++) { 937 folio = fbatch.folios[i]; 938 939 index = indices[i]; 940 941 if (xa_is_value(folio)) { 942 if (unfalloc) 943 continue; 944 nr_swaps_freed += !shmem_free_swap(mapping, 945 index, folio); 946 continue; 947 } 948 index += folio_nr_pages(folio) - 1; 949 950 if (!unfalloc || !folio_test_uptodate(folio)) 951 truncate_inode_folio(mapping, folio); 952 folio_unlock(folio); 953 } 954 folio_batch_remove_exceptionals(&fbatch); 955 folio_batch_release(&fbatch); 956 cond_resched(); 957 index++; 958 } 959 960 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT); 961 folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT); 962 if (folio) { 963 same_folio = lend < folio_pos(folio) + folio_size(folio); 964 folio_mark_dirty(folio); 965 if (!truncate_inode_partial_folio(folio, lstart, lend)) { 966 start = folio->index + folio_nr_pages(folio); 967 if (same_folio) 968 end = folio->index; 969 } 970 folio_unlock(folio); 971 folio_put(folio); 972 folio = NULL; 973 } 974 975 if (!same_folio) 976 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); 977 if (folio) { 978 folio_mark_dirty(folio); 979 if (!truncate_inode_partial_folio(folio, lstart, lend)) 980 end = folio->index; 981 folio_unlock(folio); 982 folio_put(folio); 983 } 984 985 index = start; 986 while (index < end) { 987 cond_resched(); 988 989 if (!find_get_entries(mapping, index, end - 1, &fbatch, 990 indices)) { 991 /* If all gone or hole-punch or unfalloc, we're done */ 992 if (index == start || end != -1) 993 break; 994 /* But if truncating, restart to make sure all gone */ 995 index = start; 996 continue; 997 } 998 for (i = 0; i < folio_batch_count(&fbatch); i++) { 999 folio = fbatch.folios[i]; 1000 1001 index = indices[i]; 1002 if (xa_is_value(folio)) { 1003 if (unfalloc) 1004 continue; 1005 if (shmem_free_swap(mapping, index, folio)) { 1006 /* Swap was replaced by page: retry */ 1007 index--; 1008 break; 1009 } 1010 nr_swaps_freed++; 1011 continue; 1012 } 1013 1014 folio_lock(folio); 1015 1016 if (!unfalloc || !folio_test_uptodate(folio)) { 1017 if (folio_mapping(folio) != mapping) { 1018 /* Page was replaced by swap: retry */ 1019 folio_unlock(folio); 1020 index--; 1021 break; 1022 } 1023 VM_BUG_ON_FOLIO(folio_test_writeback(folio), 1024 folio); 1025 truncate_inode_folio(mapping, folio); 1026 } 1027 index = folio->index + folio_nr_pages(folio) - 1; 1028 folio_unlock(folio); 1029 } 1030 folio_batch_remove_exceptionals(&fbatch); 1031 folio_batch_release(&fbatch); 1032 index++; 1033 } 1034 1035 spin_lock_irq(&info->lock); 1036 info->swapped -= nr_swaps_freed; 1037 shmem_recalc_inode(inode); 1038 spin_unlock_irq(&info->lock); 1039 } 1040 1041 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 1042 { 1043 shmem_undo_range(inode, lstart, lend, false); 1044 inode->i_ctime = inode->i_mtime = current_time(inode); 1045 } 1046 EXPORT_SYMBOL_GPL(shmem_truncate_range); 1047 1048 static int shmem_getattr(struct user_namespace *mnt_userns, 1049 const struct path *path, struct kstat *stat, 1050 u32 request_mask, unsigned int query_flags) 1051 { 1052 struct inode *inode = path->dentry->d_inode; 1053 struct shmem_inode_info *info = SHMEM_I(inode); 1054 1055 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 1056 spin_lock_irq(&info->lock); 1057 shmem_recalc_inode(inode); 1058 spin_unlock_irq(&info->lock); 1059 } 1060 generic_fillattr(&init_user_ns, inode, stat); 1061 1062 if (shmem_is_huge(NULL, inode, 0)) 1063 stat->blksize = HPAGE_PMD_SIZE; 1064 1065 return 0; 1066 } 1067 1068 static int shmem_setattr(struct user_namespace *mnt_userns, 1069 struct dentry *dentry, struct iattr *attr) 1070 { 1071 struct inode *inode = d_inode(dentry); 1072 struct shmem_inode_info *info = SHMEM_I(inode); 1073 int error; 1074 1075 error = setattr_prepare(&init_user_ns, dentry, attr); 1076 if (error) 1077 return error; 1078 1079 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 1080 loff_t oldsize = inode->i_size; 1081 loff_t newsize = attr->ia_size; 1082 1083 /* protected by i_rwsem */ 1084 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 1085 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 1086 return -EPERM; 1087 1088 if (newsize != oldsize) { 1089 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1090 oldsize, newsize); 1091 if (error) 1092 return error; 1093 i_size_write(inode, newsize); 1094 inode->i_ctime = inode->i_mtime = current_time(inode); 1095 } 1096 if (newsize <= oldsize) { 1097 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1098 if (oldsize > holebegin) 1099 unmap_mapping_range(inode->i_mapping, 1100 holebegin, 0, 1); 1101 if (info->alloced) 1102 shmem_truncate_range(inode, 1103 newsize, (loff_t)-1); 1104 /* unmap again to remove racily COWed private pages */ 1105 if (oldsize > holebegin) 1106 unmap_mapping_range(inode->i_mapping, 1107 holebegin, 0, 1); 1108 } 1109 } 1110 1111 setattr_copy(&init_user_ns, inode, attr); 1112 if (attr->ia_valid & ATTR_MODE) 1113 error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); 1114 return error; 1115 } 1116 1117 static void shmem_evict_inode(struct inode *inode) 1118 { 1119 struct shmem_inode_info *info = SHMEM_I(inode); 1120 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1121 1122 if (shmem_mapping(inode->i_mapping)) { 1123 shmem_unacct_size(info->flags, inode->i_size); 1124 inode->i_size = 0; 1125 shmem_truncate_range(inode, 0, (loff_t)-1); 1126 if (!list_empty(&info->shrinklist)) { 1127 spin_lock(&sbinfo->shrinklist_lock); 1128 if (!list_empty(&info->shrinklist)) { 1129 list_del_init(&info->shrinklist); 1130 sbinfo->shrinklist_len--; 1131 } 1132 spin_unlock(&sbinfo->shrinklist_lock); 1133 } 1134 while (!list_empty(&info->swaplist)) { 1135 /* Wait while shmem_unuse() is scanning this inode... */ 1136 wait_var_event(&info->stop_eviction, 1137 !atomic_read(&info->stop_eviction)); 1138 mutex_lock(&shmem_swaplist_mutex); 1139 /* ...but beware of the race if we peeked too early */ 1140 if (!atomic_read(&info->stop_eviction)) 1141 list_del_init(&info->swaplist); 1142 mutex_unlock(&shmem_swaplist_mutex); 1143 } 1144 } 1145 1146 simple_xattrs_free(&info->xattrs); 1147 WARN_ON(inode->i_blocks); 1148 shmem_free_inode(inode->i_sb); 1149 clear_inode(inode); 1150 } 1151 1152 static int shmem_find_swap_entries(struct address_space *mapping, 1153 pgoff_t start, unsigned int nr_entries, 1154 struct page **entries, pgoff_t *indices, 1155 unsigned int type, bool frontswap) 1156 { 1157 XA_STATE(xas, &mapping->i_pages, start); 1158 struct page *page; 1159 swp_entry_t entry; 1160 unsigned int ret = 0; 1161 1162 if (!nr_entries) 1163 return 0; 1164 1165 rcu_read_lock(); 1166 xas_for_each(&xas, page, ULONG_MAX) { 1167 if (xas_retry(&xas, page)) 1168 continue; 1169 1170 if (!xa_is_value(page)) 1171 continue; 1172 1173 entry = radix_to_swp_entry(page); 1174 if (swp_type(entry) != type) 1175 continue; 1176 if (frontswap && 1177 !frontswap_test(swap_info[type], swp_offset(entry))) 1178 continue; 1179 1180 indices[ret] = xas.xa_index; 1181 entries[ret] = page; 1182 1183 if (need_resched()) { 1184 xas_pause(&xas); 1185 cond_resched_rcu(); 1186 } 1187 if (++ret == nr_entries) 1188 break; 1189 } 1190 rcu_read_unlock(); 1191 1192 return ret; 1193 } 1194 1195 /* 1196 * Move the swapped pages for an inode to page cache. Returns the count 1197 * of pages swapped in, or the error in case of failure. 1198 */ 1199 static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec, 1200 pgoff_t *indices) 1201 { 1202 int i = 0; 1203 int ret = 0; 1204 int error = 0; 1205 struct address_space *mapping = inode->i_mapping; 1206 1207 for (i = 0; i < pvec.nr; i++) { 1208 struct page *page = pvec.pages[i]; 1209 1210 if (!xa_is_value(page)) 1211 continue; 1212 error = shmem_swapin_page(inode, indices[i], 1213 &page, SGP_CACHE, 1214 mapping_gfp_mask(mapping), 1215 NULL, NULL); 1216 if (error == 0) { 1217 unlock_page(page); 1218 put_page(page); 1219 ret++; 1220 } 1221 if (error == -ENOMEM) 1222 break; 1223 error = 0; 1224 } 1225 return error ? error : ret; 1226 } 1227 1228 /* 1229 * If swap found in inode, free it and move page from swapcache to filecache. 1230 */ 1231 static int shmem_unuse_inode(struct inode *inode, unsigned int type, 1232 bool frontswap, unsigned long *fs_pages_to_unuse) 1233 { 1234 struct address_space *mapping = inode->i_mapping; 1235 pgoff_t start = 0; 1236 struct pagevec pvec; 1237 pgoff_t indices[PAGEVEC_SIZE]; 1238 bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0); 1239 int ret = 0; 1240 1241 pagevec_init(&pvec); 1242 do { 1243 unsigned int nr_entries = PAGEVEC_SIZE; 1244 1245 if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE) 1246 nr_entries = *fs_pages_to_unuse; 1247 1248 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries, 1249 pvec.pages, indices, 1250 type, frontswap); 1251 if (pvec.nr == 0) { 1252 ret = 0; 1253 break; 1254 } 1255 1256 ret = shmem_unuse_swap_entries(inode, pvec, indices); 1257 if (ret < 0) 1258 break; 1259 1260 if (frontswap_partial) { 1261 *fs_pages_to_unuse -= ret; 1262 if (*fs_pages_to_unuse == 0) { 1263 ret = FRONTSWAP_PAGES_UNUSED; 1264 break; 1265 } 1266 } 1267 1268 start = indices[pvec.nr - 1]; 1269 } while (true); 1270 1271 return ret; 1272 } 1273 1274 /* 1275 * Read all the shared memory data that resides in the swap 1276 * device 'type' back into memory, so the swap device can be 1277 * unused. 1278 */ 1279 int shmem_unuse(unsigned int type, bool frontswap, 1280 unsigned long *fs_pages_to_unuse) 1281 { 1282 struct shmem_inode_info *info, *next; 1283 int error = 0; 1284 1285 if (list_empty(&shmem_swaplist)) 1286 return 0; 1287 1288 mutex_lock(&shmem_swaplist_mutex); 1289 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) { 1290 if (!info->swapped) { 1291 list_del_init(&info->swaplist); 1292 continue; 1293 } 1294 /* 1295 * Drop the swaplist mutex while searching the inode for swap; 1296 * but before doing so, make sure shmem_evict_inode() will not 1297 * remove placeholder inode from swaplist, nor let it be freed 1298 * (igrab() would protect from unlink, but not from unmount). 1299 */ 1300 atomic_inc(&info->stop_eviction); 1301 mutex_unlock(&shmem_swaplist_mutex); 1302 1303 error = shmem_unuse_inode(&info->vfs_inode, type, frontswap, 1304 fs_pages_to_unuse); 1305 cond_resched(); 1306 1307 mutex_lock(&shmem_swaplist_mutex); 1308 next = list_next_entry(info, swaplist); 1309 if (!info->swapped) 1310 list_del_init(&info->swaplist); 1311 if (atomic_dec_and_test(&info->stop_eviction)) 1312 wake_up_var(&info->stop_eviction); 1313 if (error) 1314 break; 1315 } 1316 mutex_unlock(&shmem_swaplist_mutex); 1317 1318 return error; 1319 } 1320 1321 /* 1322 * Move the page from the page cache to the swap cache. 1323 */ 1324 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1325 { 1326 struct shmem_inode_info *info; 1327 struct address_space *mapping; 1328 struct inode *inode; 1329 swp_entry_t swap; 1330 pgoff_t index; 1331 1332 /* 1333 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or 1334 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages, 1335 * and its shmem_writeback() needs them to be split when swapping. 1336 */ 1337 if (PageTransCompound(page)) { 1338 /* Ensure the subpages are still dirty */ 1339 SetPageDirty(page); 1340 if (split_huge_page(page) < 0) 1341 goto redirty; 1342 ClearPageDirty(page); 1343 } 1344 1345 BUG_ON(!PageLocked(page)); 1346 mapping = page->mapping; 1347 index = page->index; 1348 inode = mapping->host; 1349 info = SHMEM_I(inode); 1350 if (info->flags & VM_LOCKED) 1351 goto redirty; 1352 if (!total_swap_pages) 1353 goto redirty; 1354 1355 /* 1356 * Our capabilities prevent regular writeback or sync from ever calling 1357 * shmem_writepage; but a stacking filesystem might use ->writepage of 1358 * its underlying filesystem, in which case tmpfs should write out to 1359 * swap only in response to memory pressure, and not for the writeback 1360 * threads or sync. 1361 */ 1362 if (!wbc->for_reclaim) { 1363 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1364 goto redirty; 1365 } 1366 1367 /* 1368 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1369 * value into swapfile.c, the only way we can correctly account for a 1370 * fallocated page arriving here is now to initialize it and write it. 1371 * 1372 * That's okay for a page already fallocated earlier, but if we have 1373 * not yet completed the fallocation, then (a) we want to keep track 1374 * of this page in case we have to undo it, and (b) it may not be a 1375 * good idea to continue anyway, once we're pushing into swap. So 1376 * reactivate the page, and let shmem_fallocate() quit when too many. 1377 */ 1378 if (!PageUptodate(page)) { 1379 if (inode->i_private) { 1380 struct shmem_falloc *shmem_falloc; 1381 spin_lock(&inode->i_lock); 1382 shmem_falloc = inode->i_private; 1383 if (shmem_falloc && 1384 !shmem_falloc->waitq && 1385 index >= shmem_falloc->start && 1386 index < shmem_falloc->next) 1387 shmem_falloc->nr_unswapped++; 1388 else 1389 shmem_falloc = NULL; 1390 spin_unlock(&inode->i_lock); 1391 if (shmem_falloc) 1392 goto redirty; 1393 } 1394 clear_highpage(page); 1395 flush_dcache_page(page); 1396 SetPageUptodate(page); 1397 } 1398 1399 swap = get_swap_page(page); 1400 if (!swap.val) 1401 goto redirty; 1402 1403 /* 1404 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1405 * if it's not already there. Do it now before the page is 1406 * moved to swap cache, when its pagelock no longer protects 1407 * the inode from eviction. But don't unlock the mutex until 1408 * we've incremented swapped, because shmem_unuse_inode() will 1409 * prune a !swapped inode from the swaplist under this mutex. 1410 */ 1411 mutex_lock(&shmem_swaplist_mutex); 1412 if (list_empty(&info->swaplist)) 1413 list_add(&info->swaplist, &shmem_swaplist); 1414 1415 if (add_to_swap_cache(page, swap, 1416 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN, 1417 NULL) == 0) { 1418 spin_lock_irq(&info->lock); 1419 shmem_recalc_inode(inode); 1420 info->swapped++; 1421 spin_unlock_irq(&info->lock); 1422 1423 swap_shmem_alloc(swap); 1424 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1425 1426 mutex_unlock(&shmem_swaplist_mutex); 1427 BUG_ON(page_mapped(page)); 1428 swap_writepage(page, wbc); 1429 return 0; 1430 } 1431 1432 mutex_unlock(&shmem_swaplist_mutex); 1433 put_swap_page(page, swap); 1434 redirty: 1435 set_page_dirty(page); 1436 if (wbc->for_reclaim) 1437 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1438 unlock_page(page); 1439 return 0; 1440 } 1441 1442 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1443 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1444 { 1445 char buffer[64]; 1446 1447 if (!mpol || mpol->mode == MPOL_DEFAULT) 1448 return; /* show nothing */ 1449 1450 mpol_to_str(buffer, sizeof(buffer), mpol); 1451 1452 seq_printf(seq, ",mpol=%s", buffer); 1453 } 1454 1455 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1456 { 1457 struct mempolicy *mpol = NULL; 1458 if (sbinfo->mpol) { 1459 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1460 mpol = sbinfo->mpol; 1461 mpol_get(mpol); 1462 raw_spin_unlock(&sbinfo->stat_lock); 1463 } 1464 return mpol; 1465 } 1466 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1467 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1468 { 1469 } 1470 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1471 { 1472 return NULL; 1473 } 1474 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1475 #ifndef CONFIG_NUMA 1476 #define vm_policy vm_private_data 1477 #endif 1478 1479 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1480 struct shmem_inode_info *info, pgoff_t index) 1481 { 1482 /* Create a pseudo vma that just contains the policy */ 1483 vma_init(vma, NULL); 1484 /* Bias interleave by inode number to distribute better across nodes */ 1485 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1486 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1487 } 1488 1489 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1490 { 1491 /* Drop reference taken by mpol_shared_policy_lookup() */ 1492 mpol_cond_put(vma->vm_policy); 1493 } 1494 1495 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1496 struct shmem_inode_info *info, pgoff_t index) 1497 { 1498 struct vm_area_struct pvma; 1499 struct page *page; 1500 struct vm_fault vmf = { 1501 .vma = &pvma, 1502 }; 1503 1504 shmem_pseudo_vma_init(&pvma, info, index); 1505 page = swap_cluster_readahead(swap, gfp, &vmf); 1506 shmem_pseudo_vma_destroy(&pvma); 1507 1508 return page; 1509 } 1510 1511 /* 1512 * Make sure huge_gfp is always more limited than limit_gfp. 1513 * Some of the flags set permissions, while others set limitations. 1514 */ 1515 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) 1516 { 1517 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; 1518 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; 1519 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; 1520 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK); 1521 1522 /* Allow allocations only from the originally specified zones. */ 1523 result |= zoneflags; 1524 1525 /* 1526 * Minimize the result gfp by taking the union with the deny flags, 1527 * and the intersection of the allow flags. 1528 */ 1529 result |= (limit_gfp & denyflags); 1530 result |= (huge_gfp & limit_gfp) & allowflags; 1531 1532 return result; 1533 } 1534 1535 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1536 struct shmem_inode_info *info, pgoff_t index) 1537 { 1538 struct vm_area_struct pvma; 1539 struct address_space *mapping = info->vfs_inode.i_mapping; 1540 pgoff_t hindex; 1541 struct page *page; 1542 1543 hindex = round_down(index, HPAGE_PMD_NR); 1544 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1, 1545 XA_PRESENT)) 1546 return NULL; 1547 1548 shmem_pseudo_vma_init(&pvma, info, hindex); 1549 page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, true); 1550 shmem_pseudo_vma_destroy(&pvma); 1551 if (page) 1552 prep_transhuge_page(page); 1553 else 1554 count_vm_event(THP_FILE_FALLBACK); 1555 return page; 1556 } 1557 1558 static struct page *shmem_alloc_page(gfp_t gfp, 1559 struct shmem_inode_info *info, pgoff_t index) 1560 { 1561 struct vm_area_struct pvma; 1562 struct page *page; 1563 1564 shmem_pseudo_vma_init(&pvma, info, index); 1565 page = alloc_page_vma(gfp, &pvma, 0); 1566 shmem_pseudo_vma_destroy(&pvma); 1567 1568 return page; 1569 } 1570 1571 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1572 struct inode *inode, 1573 pgoff_t index, bool huge) 1574 { 1575 struct shmem_inode_info *info = SHMEM_I(inode); 1576 struct page *page; 1577 int nr; 1578 int err = -ENOSPC; 1579 1580 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1581 huge = false; 1582 nr = huge ? HPAGE_PMD_NR : 1; 1583 1584 if (!shmem_inode_acct_block(inode, nr)) 1585 goto failed; 1586 1587 if (huge) 1588 page = shmem_alloc_hugepage(gfp, info, index); 1589 else 1590 page = shmem_alloc_page(gfp, info, index); 1591 if (page) { 1592 __SetPageLocked(page); 1593 __SetPageSwapBacked(page); 1594 return page; 1595 } 1596 1597 err = -ENOMEM; 1598 shmem_inode_unacct_blocks(inode, nr); 1599 failed: 1600 return ERR_PTR(err); 1601 } 1602 1603 /* 1604 * When a page is moved from swapcache to shmem filecache (either by the 1605 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1606 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1607 * ignorance of the mapping it belongs to. If that mapping has special 1608 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1609 * we may need to copy to a suitable page before moving to filecache. 1610 * 1611 * In a future release, this may well be extended to respect cpuset and 1612 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1613 * but for now it is a simple matter of zone. 1614 */ 1615 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1616 { 1617 return page_zonenum(page) > gfp_zone(gfp); 1618 } 1619 1620 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1621 struct shmem_inode_info *info, pgoff_t index) 1622 { 1623 struct page *oldpage, *newpage; 1624 struct folio *old, *new; 1625 struct address_space *swap_mapping; 1626 swp_entry_t entry; 1627 pgoff_t swap_index; 1628 int error; 1629 1630 oldpage = *pagep; 1631 entry.val = page_private(oldpage); 1632 swap_index = swp_offset(entry); 1633 swap_mapping = page_mapping(oldpage); 1634 1635 /* 1636 * We have arrived here because our zones are constrained, so don't 1637 * limit chance of success by further cpuset and node constraints. 1638 */ 1639 gfp &= ~GFP_CONSTRAINT_MASK; 1640 newpage = shmem_alloc_page(gfp, info, index); 1641 if (!newpage) 1642 return -ENOMEM; 1643 1644 get_page(newpage); 1645 copy_highpage(newpage, oldpage); 1646 flush_dcache_page(newpage); 1647 1648 __SetPageLocked(newpage); 1649 __SetPageSwapBacked(newpage); 1650 SetPageUptodate(newpage); 1651 set_page_private(newpage, entry.val); 1652 SetPageSwapCache(newpage); 1653 1654 /* 1655 * Our caller will very soon move newpage out of swapcache, but it's 1656 * a nice clean interface for us to replace oldpage by newpage there. 1657 */ 1658 xa_lock_irq(&swap_mapping->i_pages); 1659 error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage); 1660 if (!error) { 1661 old = page_folio(oldpage); 1662 new = page_folio(newpage); 1663 mem_cgroup_migrate(old, new); 1664 __inc_lruvec_page_state(newpage, NR_FILE_PAGES); 1665 __dec_lruvec_page_state(oldpage, NR_FILE_PAGES); 1666 } 1667 xa_unlock_irq(&swap_mapping->i_pages); 1668 1669 if (unlikely(error)) { 1670 /* 1671 * Is this possible? I think not, now that our callers check 1672 * both PageSwapCache and page_private after getting page lock; 1673 * but be defensive. Reverse old to newpage for clear and free. 1674 */ 1675 oldpage = newpage; 1676 } else { 1677 lru_cache_add(newpage); 1678 *pagep = newpage; 1679 } 1680 1681 ClearPageSwapCache(oldpage); 1682 set_page_private(oldpage, 0); 1683 1684 unlock_page(oldpage); 1685 put_page(oldpage); 1686 put_page(oldpage); 1687 return error; 1688 } 1689 1690 /* 1691 * Swap in the page pointed to by *pagep. 1692 * Caller has to make sure that *pagep contains a valid swapped page. 1693 * Returns 0 and the page in pagep if success. On failure, returns the 1694 * error code and NULL in *pagep. 1695 */ 1696 static int shmem_swapin_page(struct inode *inode, pgoff_t index, 1697 struct page **pagep, enum sgp_type sgp, 1698 gfp_t gfp, struct vm_area_struct *vma, 1699 vm_fault_t *fault_type) 1700 { 1701 struct address_space *mapping = inode->i_mapping; 1702 struct shmem_inode_info *info = SHMEM_I(inode); 1703 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL; 1704 struct page *page; 1705 swp_entry_t swap; 1706 int error; 1707 1708 VM_BUG_ON(!*pagep || !xa_is_value(*pagep)); 1709 swap = radix_to_swp_entry(*pagep); 1710 *pagep = NULL; 1711 1712 /* Look it up and read it in.. */ 1713 page = lookup_swap_cache(swap, NULL, 0); 1714 if (!page) { 1715 /* Or update major stats only when swapin succeeds?? */ 1716 if (fault_type) { 1717 *fault_type |= VM_FAULT_MAJOR; 1718 count_vm_event(PGMAJFAULT); 1719 count_memcg_event_mm(charge_mm, PGMAJFAULT); 1720 } 1721 /* Here we actually start the io */ 1722 page = shmem_swapin(swap, gfp, info, index); 1723 if (!page) { 1724 error = -ENOMEM; 1725 goto failed; 1726 } 1727 } 1728 1729 /* We have to do this with page locked to prevent races */ 1730 lock_page(page); 1731 if (!PageSwapCache(page) || page_private(page) != swap.val || 1732 !shmem_confirm_swap(mapping, index, swap)) { 1733 error = -EEXIST; 1734 goto unlock; 1735 } 1736 if (!PageUptodate(page)) { 1737 error = -EIO; 1738 goto failed; 1739 } 1740 wait_on_page_writeback(page); 1741 1742 /* 1743 * Some architectures may have to restore extra metadata to the 1744 * physical page after reading from swap. 1745 */ 1746 arch_swap_restore(swap, page); 1747 1748 if (shmem_should_replace_page(page, gfp)) { 1749 error = shmem_replace_page(&page, gfp, info, index); 1750 if (error) 1751 goto failed; 1752 } 1753 1754 error = shmem_add_to_page_cache(page, mapping, index, 1755 swp_to_radix_entry(swap), gfp, 1756 charge_mm); 1757 if (error) 1758 goto failed; 1759 1760 spin_lock_irq(&info->lock); 1761 info->swapped--; 1762 shmem_recalc_inode(inode); 1763 spin_unlock_irq(&info->lock); 1764 1765 if (sgp == SGP_WRITE) 1766 mark_page_accessed(page); 1767 1768 delete_from_swap_cache(page); 1769 set_page_dirty(page); 1770 swap_free(swap); 1771 1772 *pagep = page; 1773 return 0; 1774 failed: 1775 if (!shmem_confirm_swap(mapping, index, swap)) 1776 error = -EEXIST; 1777 unlock: 1778 if (page) { 1779 unlock_page(page); 1780 put_page(page); 1781 } 1782 1783 return error; 1784 } 1785 1786 /* 1787 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1788 * 1789 * If we allocate a new one we do not mark it dirty. That's up to the 1790 * vm. If we swap it in we mark it dirty since we also free the swap 1791 * entry since a page cannot live in both the swap and page cache. 1792 * 1793 * vma, vmf, and fault_type are only supplied by shmem_fault: 1794 * otherwise they are NULL. 1795 */ 1796 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1797 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1798 struct vm_area_struct *vma, struct vm_fault *vmf, 1799 vm_fault_t *fault_type) 1800 { 1801 struct address_space *mapping = inode->i_mapping; 1802 struct shmem_inode_info *info = SHMEM_I(inode); 1803 struct shmem_sb_info *sbinfo; 1804 struct mm_struct *charge_mm; 1805 struct page *page; 1806 pgoff_t hindex = index; 1807 gfp_t huge_gfp; 1808 int error; 1809 int once = 0; 1810 int alloced = 0; 1811 1812 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1813 return -EFBIG; 1814 repeat: 1815 if (sgp <= SGP_CACHE && 1816 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1817 return -EINVAL; 1818 } 1819 1820 sbinfo = SHMEM_SB(inode->i_sb); 1821 charge_mm = vma ? vma->vm_mm : NULL; 1822 1823 page = pagecache_get_page(mapping, index, 1824 FGP_ENTRY | FGP_HEAD | FGP_LOCK, 0); 1825 1826 if (page && vma && userfaultfd_minor(vma)) { 1827 if (!xa_is_value(page)) { 1828 unlock_page(page); 1829 put_page(page); 1830 } 1831 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); 1832 return 0; 1833 } 1834 1835 if (xa_is_value(page)) { 1836 error = shmem_swapin_page(inode, index, &page, 1837 sgp, gfp, vma, fault_type); 1838 if (error == -EEXIST) 1839 goto repeat; 1840 1841 *pagep = page; 1842 return error; 1843 } 1844 1845 if (page) { 1846 hindex = page->index; 1847 if (sgp == SGP_WRITE) 1848 mark_page_accessed(page); 1849 if (PageUptodate(page)) 1850 goto out; 1851 /* fallocated page */ 1852 if (sgp != SGP_READ) 1853 goto clear; 1854 unlock_page(page); 1855 put_page(page); 1856 } 1857 1858 /* 1859 * SGP_READ: succeed on hole, with NULL page, letting caller zero. 1860 * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail. 1861 */ 1862 *pagep = NULL; 1863 if (sgp == SGP_READ) 1864 return 0; 1865 if (sgp == SGP_NOALLOC) 1866 return -ENOENT; 1867 1868 /* 1869 * Fast cache lookup and swap lookup did not find it: allocate. 1870 */ 1871 1872 if (vma && userfaultfd_missing(vma)) { 1873 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 1874 return 0; 1875 } 1876 1877 /* Never use a huge page for shmem_symlink() */ 1878 if (S_ISLNK(inode->i_mode)) 1879 goto alloc_nohuge; 1880 if (!shmem_is_huge(vma, inode, index)) 1881 goto alloc_nohuge; 1882 1883 huge_gfp = vma_thp_gfp_mask(vma); 1884 huge_gfp = limit_gfp_mask(huge_gfp, gfp); 1885 page = shmem_alloc_and_acct_page(huge_gfp, inode, index, true); 1886 if (IS_ERR(page)) { 1887 alloc_nohuge: 1888 page = shmem_alloc_and_acct_page(gfp, inode, 1889 index, false); 1890 } 1891 if (IS_ERR(page)) { 1892 int retry = 5; 1893 1894 error = PTR_ERR(page); 1895 page = NULL; 1896 if (error != -ENOSPC) 1897 goto unlock; 1898 /* 1899 * Try to reclaim some space by splitting a huge page 1900 * beyond i_size on the filesystem. 1901 */ 1902 while (retry--) { 1903 int ret; 1904 1905 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1906 if (ret == SHRINK_STOP) 1907 break; 1908 if (ret) 1909 goto alloc_nohuge; 1910 } 1911 goto unlock; 1912 } 1913 1914 if (PageTransHuge(page)) 1915 hindex = round_down(index, HPAGE_PMD_NR); 1916 else 1917 hindex = index; 1918 1919 if (sgp == SGP_WRITE) 1920 __SetPageReferenced(page); 1921 1922 error = shmem_add_to_page_cache(page, mapping, hindex, 1923 NULL, gfp & GFP_RECLAIM_MASK, 1924 charge_mm); 1925 if (error) 1926 goto unacct; 1927 lru_cache_add(page); 1928 1929 spin_lock_irq(&info->lock); 1930 info->alloced += compound_nr(page); 1931 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1932 shmem_recalc_inode(inode); 1933 spin_unlock_irq(&info->lock); 1934 alloced = true; 1935 1936 if (PageTransHuge(page) && 1937 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1938 hindex + HPAGE_PMD_NR - 1) { 1939 /* 1940 * Part of the huge page is beyond i_size: subject 1941 * to shrink under memory pressure. 1942 */ 1943 spin_lock(&sbinfo->shrinklist_lock); 1944 /* 1945 * _careful to defend against unlocked access to 1946 * ->shrink_list in shmem_unused_huge_shrink() 1947 */ 1948 if (list_empty_careful(&info->shrinklist)) { 1949 list_add_tail(&info->shrinklist, 1950 &sbinfo->shrinklist); 1951 sbinfo->shrinklist_len++; 1952 } 1953 spin_unlock(&sbinfo->shrinklist_lock); 1954 } 1955 1956 /* 1957 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1958 */ 1959 if (sgp == SGP_FALLOC) 1960 sgp = SGP_WRITE; 1961 clear: 1962 /* 1963 * Let SGP_WRITE caller clear ends if write does not fill page; 1964 * but SGP_FALLOC on a page fallocated earlier must initialize 1965 * it now, lest undo on failure cancel our earlier guarantee. 1966 */ 1967 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1968 int i; 1969 1970 for (i = 0; i < compound_nr(page); i++) { 1971 clear_highpage(page + i); 1972 flush_dcache_page(page + i); 1973 } 1974 SetPageUptodate(page); 1975 } 1976 1977 /* Perhaps the file has been truncated since we checked */ 1978 if (sgp <= SGP_CACHE && 1979 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1980 if (alloced) { 1981 ClearPageDirty(page); 1982 delete_from_page_cache(page); 1983 spin_lock_irq(&info->lock); 1984 shmem_recalc_inode(inode); 1985 spin_unlock_irq(&info->lock); 1986 } 1987 error = -EINVAL; 1988 goto unlock; 1989 } 1990 out: 1991 *pagep = page + index - hindex; 1992 return 0; 1993 1994 /* 1995 * Error recovery. 1996 */ 1997 unacct: 1998 shmem_inode_unacct_blocks(inode, compound_nr(page)); 1999 2000 if (PageTransHuge(page)) { 2001 unlock_page(page); 2002 put_page(page); 2003 goto alloc_nohuge; 2004 } 2005 unlock: 2006 if (page) { 2007 unlock_page(page); 2008 put_page(page); 2009 } 2010 if (error == -ENOSPC && !once++) { 2011 spin_lock_irq(&info->lock); 2012 shmem_recalc_inode(inode); 2013 spin_unlock_irq(&info->lock); 2014 goto repeat; 2015 } 2016 if (error == -EEXIST) 2017 goto repeat; 2018 return error; 2019 } 2020 2021 /* 2022 * This is like autoremove_wake_function, but it removes the wait queue 2023 * entry unconditionally - even if something else had already woken the 2024 * target. 2025 */ 2026 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 2027 { 2028 int ret = default_wake_function(wait, mode, sync, key); 2029 list_del_init(&wait->entry); 2030 return ret; 2031 } 2032 2033 static vm_fault_t shmem_fault(struct vm_fault *vmf) 2034 { 2035 struct vm_area_struct *vma = vmf->vma; 2036 struct inode *inode = file_inode(vma->vm_file); 2037 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 2038 int err; 2039 vm_fault_t ret = VM_FAULT_LOCKED; 2040 2041 /* 2042 * Trinity finds that probing a hole which tmpfs is punching can 2043 * prevent the hole-punch from ever completing: which in turn 2044 * locks writers out with its hold on i_rwsem. So refrain from 2045 * faulting pages into the hole while it's being punched. Although 2046 * shmem_undo_range() does remove the additions, it may be unable to 2047 * keep up, as each new page needs its own unmap_mapping_range() call, 2048 * and the i_mmap tree grows ever slower to scan if new vmas are added. 2049 * 2050 * It does not matter if we sometimes reach this check just before the 2051 * hole-punch begins, so that one fault then races with the punch: 2052 * we just need to make racing faults a rare case. 2053 * 2054 * The implementation below would be much simpler if we just used a 2055 * standard mutex or completion: but we cannot take i_rwsem in fault, 2056 * and bloating every shmem inode for this unlikely case would be sad. 2057 */ 2058 if (unlikely(inode->i_private)) { 2059 struct shmem_falloc *shmem_falloc; 2060 2061 spin_lock(&inode->i_lock); 2062 shmem_falloc = inode->i_private; 2063 if (shmem_falloc && 2064 shmem_falloc->waitq && 2065 vmf->pgoff >= shmem_falloc->start && 2066 vmf->pgoff < shmem_falloc->next) { 2067 struct file *fpin; 2068 wait_queue_head_t *shmem_falloc_waitq; 2069 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 2070 2071 ret = VM_FAULT_NOPAGE; 2072 fpin = maybe_unlock_mmap_for_io(vmf, NULL); 2073 if (fpin) 2074 ret = VM_FAULT_RETRY; 2075 2076 shmem_falloc_waitq = shmem_falloc->waitq; 2077 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 2078 TASK_UNINTERRUPTIBLE); 2079 spin_unlock(&inode->i_lock); 2080 schedule(); 2081 2082 /* 2083 * shmem_falloc_waitq points into the shmem_fallocate() 2084 * stack of the hole-punching task: shmem_falloc_waitq 2085 * is usually invalid by the time we reach here, but 2086 * finish_wait() does not dereference it in that case; 2087 * though i_lock needed lest racing with wake_up_all(). 2088 */ 2089 spin_lock(&inode->i_lock); 2090 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 2091 spin_unlock(&inode->i_lock); 2092 2093 if (fpin) 2094 fput(fpin); 2095 return ret; 2096 } 2097 spin_unlock(&inode->i_lock); 2098 } 2099 2100 err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, SGP_CACHE, 2101 gfp, vma, vmf, &ret); 2102 if (err) 2103 return vmf_error(err); 2104 return ret; 2105 } 2106 2107 unsigned long shmem_get_unmapped_area(struct file *file, 2108 unsigned long uaddr, unsigned long len, 2109 unsigned long pgoff, unsigned long flags) 2110 { 2111 unsigned long (*get_area)(struct file *, 2112 unsigned long, unsigned long, unsigned long, unsigned long); 2113 unsigned long addr; 2114 unsigned long offset; 2115 unsigned long inflated_len; 2116 unsigned long inflated_addr; 2117 unsigned long inflated_offset; 2118 2119 if (len > TASK_SIZE) 2120 return -ENOMEM; 2121 2122 get_area = current->mm->get_unmapped_area; 2123 addr = get_area(file, uaddr, len, pgoff, flags); 2124 2125 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 2126 return addr; 2127 if (IS_ERR_VALUE(addr)) 2128 return addr; 2129 if (addr & ~PAGE_MASK) 2130 return addr; 2131 if (addr > TASK_SIZE - len) 2132 return addr; 2133 2134 if (shmem_huge == SHMEM_HUGE_DENY) 2135 return addr; 2136 if (len < HPAGE_PMD_SIZE) 2137 return addr; 2138 if (flags & MAP_FIXED) 2139 return addr; 2140 /* 2141 * Our priority is to support MAP_SHARED mapped hugely; 2142 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2143 * But if caller specified an address hint and we allocated area there 2144 * successfully, respect that as before. 2145 */ 2146 if (uaddr == addr) 2147 return addr; 2148 2149 if (shmem_huge != SHMEM_HUGE_FORCE) { 2150 struct super_block *sb; 2151 2152 if (file) { 2153 VM_BUG_ON(file->f_op != &shmem_file_operations); 2154 sb = file_inode(file)->i_sb; 2155 } else { 2156 /* 2157 * Called directly from mm/mmap.c, or drivers/char/mem.c 2158 * for "/dev/zero", to create a shared anonymous object. 2159 */ 2160 if (IS_ERR(shm_mnt)) 2161 return addr; 2162 sb = shm_mnt->mnt_sb; 2163 } 2164 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2165 return addr; 2166 } 2167 2168 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2169 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2170 return addr; 2171 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2172 return addr; 2173 2174 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2175 if (inflated_len > TASK_SIZE) 2176 return addr; 2177 if (inflated_len < len) 2178 return addr; 2179 2180 inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags); 2181 if (IS_ERR_VALUE(inflated_addr)) 2182 return addr; 2183 if (inflated_addr & ~PAGE_MASK) 2184 return addr; 2185 2186 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2187 inflated_addr += offset - inflated_offset; 2188 if (inflated_offset > offset) 2189 inflated_addr += HPAGE_PMD_SIZE; 2190 2191 if (inflated_addr > TASK_SIZE - len) 2192 return addr; 2193 return inflated_addr; 2194 } 2195 2196 #ifdef CONFIG_NUMA 2197 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2198 { 2199 struct inode *inode = file_inode(vma->vm_file); 2200 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2201 } 2202 2203 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2204 unsigned long addr) 2205 { 2206 struct inode *inode = file_inode(vma->vm_file); 2207 pgoff_t index; 2208 2209 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2210 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2211 } 2212 #endif 2213 2214 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 2215 { 2216 struct inode *inode = file_inode(file); 2217 struct shmem_inode_info *info = SHMEM_I(inode); 2218 int retval = -ENOMEM; 2219 2220 /* 2221 * What serializes the accesses to info->flags? 2222 * ipc_lock_object() when called from shmctl_do_lock(), 2223 * no serialization needed when called from shm_destroy(). 2224 */ 2225 if (lock && !(info->flags & VM_LOCKED)) { 2226 if (!user_shm_lock(inode->i_size, ucounts)) 2227 goto out_nomem; 2228 info->flags |= VM_LOCKED; 2229 mapping_set_unevictable(file->f_mapping); 2230 } 2231 if (!lock && (info->flags & VM_LOCKED) && ucounts) { 2232 user_shm_unlock(inode->i_size, ucounts); 2233 info->flags &= ~VM_LOCKED; 2234 mapping_clear_unevictable(file->f_mapping); 2235 } 2236 retval = 0; 2237 2238 out_nomem: 2239 return retval; 2240 } 2241 2242 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2243 { 2244 struct shmem_inode_info *info = SHMEM_I(file_inode(file)); 2245 int ret; 2246 2247 ret = seal_check_future_write(info->seals, vma); 2248 if (ret) 2249 return ret; 2250 2251 /* arm64 - allow memory tagging on RAM-based files */ 2252 vma->vm_flags |= VM_MTE_ALLOWED; 2253 2254 file_accessed(file); 2255 vma->vm_ops = &shmem_vm_ops; 2256 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 2257 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2258 (vma->vm_end & HPAGE_PMD_MASK)) { 2259 khugepaged_enter(vma, vma->vm_flags); 2260 } 2261 return 0; 2262 } 2263 2264 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2265 umode_t mode, dev_t dev, unsigned long flags) 2266 { 2267 struct inode *inode; 2268 struct shmem_inode_info *info; 2269 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2270 ino_t ino; 2271 2272 if (shmem_reserve_inode(sb, &ino)) 2273 return NULL; 2274 2275 inode = new_inode(sb); 2276 if (inode) { 2277 inode->i_ino = ino; 2278 inode_init_owner(&init_user_ns, inode, dir, mode); 2279 inode->i_blocks = 0; 2280 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2281 inode->i_generation = prandom_u32(); 2282 info = SHMEM_I(inode); 2283 memset(info, 0, (char *)inode - (char *)info); 2284 spin_lock_init(&info->lock); 2285 atomic_set(&info->stop_eviction, 0); 2286 info->seals = F_SEAL_SEAL; 2287 info->flags = flags & VM_NORESERVE; 2288 INIT_LIST_HEAD(&info->shrinklist); 2289 INIT_LIST_HEAD(&info->swaplist); 2290 simple_xattrs_init(&info->xattrs); 2291 cache_no_acl(inode); 2292 mapping_set_large_folios(inode->i_mapping); 2293 2294 switch (mode & S_IFMT) { 2295 default: 2296 inode->i_op = &shmem_special_inode_operations; 2297 init_special_inode(inode, mode, dev); 2298 break; 2299 case S_IFREG: 2300 inode->i_mapping->a_ops = &shmem_aops; 2301 inode->i_op = &shmem_inode_operations; 2302 inode->i_fop = &shmem_file_operations; 2303 mpol_shared_policy_init(&info->policy, 2304 shmem_get_sbmpol(sbinfo)); 2305 break; 2306 case S_IFDIR: 2307 inc_nlink(inode); 2308 /* Some things misbehave if size == 0 on a directory */ 2309 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2310 inode->i_op = &shmem_dir_inode_operations; 2311 inode->i_fop = &simple_dir_operations; 2312 break; 2313 case S_IFLNK: 2314 /* 2315 * Must not load anything in the rbtree, 2316 * mpol_free_shared_policy will not be called. 2317 */ 2318 mpol_shared_policy_init(&info->policy, NULL); 2319 break; 2320 } 2321 2322 lockdep_annotate_inode_mutex_key(inode); 2323 } else 2324 shmem_free_inode(sb); 2325 return inode; 2326 } 2327 2328 #ifdef CONFIG_USERFAULTFD 2329 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, 2330 pmd_t *dst_pmd, 2331 struct vm_area_struct *dst_vma, 2332 unsigned long dst_addr, 2333 unsigned long src_addr, 2334 bool zeropage, 2335 struct page **pagep) 2336 { 2337 struct inode *inode = file_inode(dst_vma->vm_file); 2338 struct shmem_inode_info *info = SHMEM_I(inode); 2339 struct address_space *mapping = inode->i_mapping; 2340 gfp_t gfp = mapping_gfp_mask(mapping); 2341 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2342 void *page_kaddr; 2343 struct page *page; 2344 int ret; 2345 pgoff_t max_off; 2346 2347 if (!shmem_inode_acct_block(inode, 1)) { 2348 /* 2349 * We may have got a page, returned -ENOENT triggering a retry, 2350 * and now we find ourselves with -ENOMEM. Release the page, to 2351 * avoid a BUG_ON in our caller. 2352 */ 2353 if (unlikely(*pagep)) { 2354 put_page(*pagep); 2355 *pagep = NULL; 2356 } 2357 return -ENOMEM; 2358 } 2359 2360 if (!*pagep) { 2361 ret = -ENOMEM; 2362 page = shmem_alloc_page(gfp, info, pgoff); 2363 if (!page) 2364 goto out_unacct_blocks; 2365 2366 if (!zeropage) { /* COPY */ 2367 page_kaddr = kmap_atomic(page); 2368 ret = copy_from_user(page_kaddr, 2369 (const void __user *)src_addr, 2370 PAGE_SIZE); 2371 kunmap_atomic(page_kaddr); 2372 2373 /* fallback to copy_from_user outside mmap_lock */ 2374 if (unlikely(ret)) { 2375 *pagep = page; 2376 ret = -ENOENT; 2377 /* don't free the page */ 2378 goto out_unacct_blocks; 2379 } 2380 } else { /* ZEROPAGE */ 2381 clear_highpage(page); 2382 } 2383 } else { 2384 page = *pagep; 2385 *pagep = NULL; 2386 } 2387 2388 VM_BUG_ON(PageLocked(page)); 2389 VM_BUG_ON(PageSwapBacked(page)); 2390 __SetPageLocked(page); 2391 __SetPageSwapBacked(page); 2392 __SetPageUptodate(page); 2393 2394 ret = -EFAULT; 2395 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2396 if (unlikely(pgoff >= max_off)) 2397 goto out_release; 2398 2399 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL, 2400 gfp & GFP_RECLAIM_MASK, dst_mm); 2401 if (ret) 2402 goto out_release; 2403 2404 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, 2405 page, true, false); 2406 if (ret) 2407 goto out_delete_from_cache; 2408 2409 spin_lock_irq(&info->lock); 2410 info->alloced++; 2411 inode->i_blocks += BLOCKS_PER_PAGE; 2412 shmem_recalc_inode(inode); 2413 spin_unlock_irq(&info->lock); 2414 2415 unlock_page(page); 2416 return 0; 2417 out_delete_from_cache: 2418 delete_from_page_cache(page); 2419 out_release: 2420 unlock_page(page); 2421 put_page(page); 2422 out_unacct_blocks: 2423 shmem_inode_unacct_blocks(inode, 1); 2424 return ret; 2425 } 2426 #endif /* CONFIG_USERFAULTFD */ 2427 2428 #ifdef CONFIG_TMPFS 2429 static const struct inode_operations shmem_symlink_inode_operations; 2430 static const struct inode_operations shmem_short_symlink_operations; 2431 2432 #ifdef CONFIG_TMPFS_XATTR 2433 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2434 #else 2435 #define shmem_initxattrs NULL 2436 #endif 2437 2438 static int 2439 shmem_write_begin(struct file *file, struct address_space *mapping, 2440 loff_t pos, unsigned len, unsigned flags, 2441 struct page **pagep, void **fsdata) 2442 { 2443 struct inode *inode = mapping->host; 2444 struct shmem_inode_info *info = SHMEM_I(inode); 2445 pgoff_t index = pos >> PAGE_SHIFT; 2446 int ret = 0; 2447 2448 /* i_rwsem is held by caller */ 2449 if (unlikely(info->seals & (F_SEAL_GROW | 2450 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) { 2451 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) 2452 return -EPERM; 2453 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2454 return -EPERM; 2455 } 2456 2457 ret = shmem_getpage(inode, index, pagep, SGP_WRITE); 2458 2459 if (ret) 2460 return ret; 2461 2462 if (PageHWPoison(*pagep)) { 2463 unlock_page(*pagep); 2464 put_page(*pagep); 2465 *pagep = NULL; 2466 return -EIO; 2467 } 2468 2469 return 0; 2470 } 2471 2472 static int 2473 shmem_write_end(struct file *file, struct address_space *mapping, 2474 loff_t pos, unsigned len, unsigned copied, 2475 struct page *page, void *fsdata) 2476 { 2477 struct inode *inode = mapping->host; 2478 2479 if (pos + copied > inode->i_size) 2480 i_size_write(inode, pos + copied); 2481 2482 if (!PageUptodate(page)) { 2483 struct page *head = compound_head(page); 2484 if (PageTransCompound(page)) { 2485 int i; 2486 2487 for (i = 0; i < HPAGE_PMD_NR; i++) { 2488 if (head + i == page) 2489 continue; 2490 clear_highpage(head + i); 2491 flush_dcache_page(head + i); 2492 } 2493 } 2494 if (copied < PAGE_SIZE) { 2495 unsigned from = pos & (PAGE_SIZE - 1); 2496 zero_user_segments(page, 0, from, 2497 from + copied, PAGE_SIZE); 2498 } 2499 SetPageUptodate(head); 2500 } 2501 set_page_dirty(page); 2502 unlock_page(page); 2503 put_page(page); 2504 2505 return copied; 2506 } 2507 2508 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2509 { 2510 struct file *file = iocb->ki_filp; 2511 struct inode *inode = file_inode(file); 2512 struct address_space *mapping = inode->i_mapping; 2513 pgoff_t index; 2514 unsigned long offset; 2515 enum sgp_type sgp = SGP_READ; 2516 int error = 0; 2517 ssize_t retval = 0; 2518 loff_t *ppos = &iocb->ki_pos; 2519 2520 /* 2521 * Might this read be for a stacking filesystem? Then when reading 2522 * holes of a sparse file, we actually need to allocate those pages, 2523 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2524 */ 2525 if (!iter_is_iovec(to)) 2526 sgp = SGP_CACHE; 2527 2528 index = *ppos >> PAGE_SHIFT; 2529 offset = *ppos & ~PAGE_MASK; 2530 2531 for (;;) { 2532 struct page *page = NULL; 2533 pgoff_t end_index; 2534 unsigned long nr, ret; 2535 loff_t i_size = i_size_read(inode); 2536 2537 end_index = i_size >> PAGE_SHIFT; 2538 if (index > end_index) 2539 break; 2540 if (index == end_index) { 2541 nr = i_size & ~PAGE_MASK; 2542 if (nr <= offset) 2543 break; 2544 } 2545 2546 error = shmem_getpage(inode, index, &page, sgp); 2547 if (error) { 2548 if (error == -EINVAL) 2549 error = 0; 2550 break; 2551 } 2552 if (page) { 2553 if (sgp == SGP_CACHE) 2554 set_page_dirty(page); 2555 unlock_page(page); 2556 2557 if (PageHWPoison(page)) { 2558 put_page(page); 2559 error = -EIO; 2560 break; 2561 } 2562 } 2563 2564 /* 2565 * We must evaluate after, since reads (unlike writes) 2566 * are called without i_rwsem protection against truncate 2567 */ 2568 nr = PAGE_SIZE; 2569 i_size = i_size_read(inode); 2570 end_index = i_size >> PAGE_SHIFT; 2571 if (index == end_index) { 2572 nr = i_size & ~PAGE_MASK; 2573 if (nr <= offset) { 2574 if (page) 2575 put_page(page); 2576 break; 2577 } 2578 } 2579 nr -= offset; 2580 2581 if (page) { 2582 /* 2583 * If users can be writing to this page using arbitrary 2584 * virtual addresses, take care about potential aliasing 2585 * before reading the page on the kernel side. 2586 */ 2587 if (mapping_writably_mapped(mapping)) 2588 flush_dcache_page(page); 2589 /* 2590 * Mark the page accessed if we read the beginning. 2591 */ 2592 if (!offset) 2593 mark_page_accessed(page); 2594 } else { 2595 page = ZERO_PAGE(0); 2596 get_page(page); 2597 } 2598 2599 /* 2600 * Ok, we have the page, and it's up-to-date, so 2601 * now we can copy it to user space... 2602 */ 2603 ret = copy_page_to_iter(page, offset, nr, to); 2604 retval += ret; 2605 offset += ret; 2606 index += offset >> PAGE_SHIFT; 2607 offset &= ~PAGE_MASK; 2608 2609 put_page(page); 2610 if (!iov_iter_count(to)) 2611 break; 2612 if (ret < nr) { 2613 error = -EFAULT; 2614 break; 2615 } 2616 cond_resched(); 2617 } 2618 2619 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2620 file_accessed(file); 2621 return retval ? retval : error; 2622 } 2623 2624 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2625 { 2626 struct address_space *mapping = file->f_mapping; 2627 struct inode *inode = mapping->host; 2628 2629 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2630 return generic_file_llseek_size(file, offset, whence, 2631 MAX_LFS_FILESIZE, i_size_read(inode)); 2632 if (offset < 0) 2633 return -ENXIO; 2634 2635 inode_lock(inode); 2636 /* We're holding i_rwsem so we can access i_size directly */ 2637 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence); 2638 if (offset >= 0) 2639 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2640 inode_unlock(inode); 2641 return offset; 2642 } 2643 2644 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2645 loff_t len) 2646 { 2647 struct inode *inode = file_inode(file); 2648 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2649 struct shmem_inode_info *info = SHMEM_I(inode); 2650 struct shmem_falloc shmem_falloc; 2651 pgoff_t start, index, end, undo_fallocend; 2652 int error; 2653 2654 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2655 return -EOPNOTSUPP; 2656 2657 inode_lock(inode); 2658 2659 if (mode & FALLOC_FL_PUNCH_HOLE) { 2660 struct address_space *mapping = file->f_mapping; 2661 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2662 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2663 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2664 2665 /* protected by i_rwsem */ 2666 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { 2667 error = -EPERM; 2668 goto out; 2669 } 2670 2671 shmem_falloc.waitq = &shmem_falloc_waitq; 2672 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; 2673 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2674 spin_lock(&inode->i_lock); 2675 inode->i_private = &shmem_falloc; 2676 spin_unlock(&inode->i_lock); 2677 2678 if ((u64)unmap_end > (u64)unmap_start) 2679 unmap_mapping_range(mapping, unmap_start, 2680 1 + unmap_end - unmap_start, 0); 2681 shmem_truncate_range(inode, offset, offset + len - 1); 2682 /* No need to unmap again: hole-punching leaves COWed pages */ 2683 2684 spin_lock(&inode->i_lock); 2685 inode->i_private = NULL; 2686 wake_up_all(&shmem_falloc_waitq); 2687 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 2688 spin_unlock(&inode->i_lock); 2689 error = 0; 2690 goto out; 2691 } 2692 2693 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2694 error = inode_newsize_ok(inode, offset + len); 2695 if (error) 2696 goto out; 2697 2698 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2699 error = -EPERM; 2700 goto out; 2701 } 2702 2703 start = offset >> PAGE_SHIFT; 2704 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2705 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2706 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2707 error = -ENOSPC; 2708 goto out; 2709 } 2710 2711 shmem_falloc.waitq = NULL; 2712 shmem_falloc.start = start; 2713 shmem_falloc.next = start; 2714 shmem_falloc.nr_falloced = 0; 2715 shmem_falloc.nr_unswapped = 0; 2716 spin_lock(&inode->i_lock); 2717 inode->i_private = &shmem_falloc; 2718 spin_unlock(&inode->i_lock); 2719 2720 /* 2721 * info->fallocend is only relevant when huge pages might be 2722 * involved: to prevent split_huge_page() freeing fallocated 2723 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size. 2724 */ 2725 undo_fallocend = info->fallocend; 2726 if (info->fallocend < end) 2727 info->fallocend = end; 2728 2729 for (index = start; index < end; ) { 2730 struct page *page; 2731 2732 /* 2733 * Good, the fallocate(2) manpage permits EINTR: we may have 2734 * been interrupted because we are using up too much memory. 2735 */ 2736 if (signal_pending(current)) 2737 error = -EINTR; 2738 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2739 error = -ENOMEM; 2740 else 2741 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2742 if (error) { 2743 info->fallocend = undo_fallocend; 2744 /* Remove the !PageUptodate pages we added */ 2745 if (index > start) { 2746 shmem_undo_range(inode, 2747 (loff_t)start << PAGE_SHIFT, 2748 ((loff_t)index << PAGE_SHIFT) - 1, true); 2749 } 2750 goto undone; 2751 } 2752 2753 index++; 2754 /* 2755 * Here is a more important optimization than it appears: 2756 * a second SGP_FALLOC on the same huge page will clear it, 2757 * making it PageUptodate and un-undoable if we fail later. 2758 */ 2759 if (PageTransCompound(page)) { 2760 index = round_up(index, HPAGE_PMD_NR); 2761 /* Beware 32-bit wraparound */ 2762 if (!index) 2763 index--; 2764 } 2765 2766 /* 2767 * Inform shmem_writepage() how far we have reached. 2768 * No need for lock or barrier: we have the page lock. 2769 */ 2770 if (!PageUptodate(page)) 2771 shmem_falloc.nr_falloced += index - shmem_falloc.next; 2772 shmem_falloc.next = index; 2773 2774 /* 2775 * If !PageUptodate, leave it that way so that freeable pages 2776 * can be recognized if we need to rollback on error later. 2777 * But set_page_dirty so that memory pressure will swap rather 2778 * than free the pages we are allocating (and SGP_CACHE pages 2779 * might still be clean: we now need to mark those dirty too). 2780 */ 2781 set_page_dirty(page); 2782 unlock_page(page); 2783 put_page(page); 2784 cond_resched(); 2785 } 2786 2787 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2788 i_size_write(inode, offset + len); 2789 inode->i_ctime = current_time(inode); 2790 undone: 2791 spin_lock(&inode->i_lock); 2792 inode->i_private = NULL; 2793 spin_unlock(&inode->i_lock); 2794 out: 2795 inode_unlock(inode); 2796 return error; 2797 } 2798 2799 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2800 { 2801 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2802 2803 buf->f_type = TMPFS_MAGIC; 2804 buf->f_bsize = PAGE_SIZE; 2805 buf->f_namelen = NAME_MAX; 2806 if (sbinfo->max_blocks) { 2807 buf->f_blocks = sbinfo->max_blocks; 2808 buf->f_bavail = 2809 buf->f_bfree = sbinfo->max_blocks - 2810 percpu_counter_sum(&sbinfo->used_blocks); 2811 } 2812 if (sbinfo->max_inodes) { 2813 buf->f_files = sbinfo->max_inodes; 2814 buf->f_ffree = sbinfo->free_inodes; 2815 } 2816 /* else leave those fields 0 like simple_statfs */ 2817 2818 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); 2819 2820 return 0; 2821 } 2822 2823 /* 2824 * File creation. Allocate an inode, and we're done.. 2825 */ 2826 static int 2827 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir, 2828 struct dentry *dentry, umode_t mode, dev_t dev) 2829 { 2830 struct inode *inode; 2831 int error = -ENOSPC; 2832 2833 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2834 if (inode) { 2835 error = simple_acl_create(dir, inode); 2836 if (error) 2837 goto out_iput; 2838 error = security_inode_init_security(inode, dir, 2839 &dentry->d_name, 2840 shmem_initxattrs, NULL); 2841 if (error && error != -EOPNOTSUPP) 2842 goto out_iput; 2843 2844 error = 0; 2845 dir->i_size += BOGO_DIRENT_SIZE; 2846 dir->i_ctime = dir->i_mtime = current_time(dir); 2847 d_instantiate(dentry, inode); 2848 dget(dentry); /* Extra count - pin the dentry in core */ 2849 } 2850 return error; 2851 out_iput: 2852 iput(inode); 2853 return error; 2854 } 2855 2856 static int 2857 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, 2858 struct dentry *dentry, umode_t mode) 2859 { 2860 struct inode *inode; 2861 int error = -ENOSPC; 2862 2863 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2864 if (inode) { 2865 error = security_inode_init_security(inode, dir, 2866 NULL, 2867 shmem_initxattrs, NULL); 2868 if (error && error != -EOPNOTSUPP) 2869 goto out_iput; 2870 error = simple_acl_create(dir, inode); 2871 if (error) 2872 goto out_iput; 2873 d_tmpfile(dentry, inode); 2874 } 2875 return error; 2876 out_iput: 2877 iput(inode); 2878 return error; 2879 } 2880 2881 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir, 2882 struct dentry *dentry, umode_t mode) 2883 { 2884 int error; 2885 2886 if ((error = shmem_mknod(&init_user_ns, dir, dentry, 2887 mode | S_IFDIR, 0))) 2888 return error; 2889 inc_nlink(dir); 2890 return 0; 2891 } 2892 2893 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir, 2894 struct dentry *dentry, umode_t mode, bool excl) 2895 { 2896 return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); 2897 } 2898 2899 /* 2900 * Link a file.. 2901 */ 2902 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2903 { 2904 struct inode *inode = d_inode(old_dentry); 2905 int ret = 0; 2906 2907 /* 2908 * No ordinary (disk based) filesystem counts links as inodes; 2909 * but each new link needs a new dentry, pinning lowmem, and 2910 * tmpfs dentries cannot be pruned until they are unlinked. 2911 * But if an O_TMPFILE file is linked into the tmpfs, the 2912 * first link must skip that, to get the accounting right. 2913 */ 2914 if (inode->i_nlink) { 2915 ret = shmem_reserve_inode(inode->i_sb, NULL); 2916 if (ret) 2917 goto out; 2918 } 2919 2920 dir->i_size += BOGO_DIRENT_SIZE; 2921 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2922 inc_nlink(inode); 2923 ihold(inode); /* New dentry reference */ 2924 dget(dentry); /* Extra pinning count for the created dentry */ 2925 d_instantiate(dentry, inode); 2926 out: 2927 return ret; 2928 } 2929 2930 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2931 { 2932 struct inode *inode = d_inode(dentry); 2933 2934 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2935 shmem_free_inode(inode->i_sb); 2936 2937 dir->i_size -= BOGO_DIRENT_SIZE; 2938 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2939 drop_nlink(inode); 2940 dput(dentry); /* Undo the count from "create" - this does all the work */ 2941 return 0; 2942 } 2943 2944 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2945 { 2946 if (!simple_empty(dentry)) 2947 return -ENOTEMPTY; 2948 2949 drop_nlink(d_inode(dentry)); 2950 drop_nlink(dir); 2951 return shmem_unlink(dir, dentry); 2952 } 2953 2954 static int shmem_whiteout(struct user_namespace *mnt_userns, 2955 struct inode *old_dir, struct dentry *old_dentry) 2956 { 2957 struct dentry *whiteout; 2958 int error; 2959 2960 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 2961 if (!whiteout) 2962 return -ENOMEM; 2963 2964 error = shmem_mknod(&init_user_ns, old_dir, whiteout, 2965 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 2966 dput(whiteout); 2967 if (error) 2968 return error; 2969 2970 /* 2971 * Cheat and hash the whiteout while the old dentry is still in 2972 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 2973 * 2974 * d_lookup() will consistently find one of them at this point, 2975 * not sure which one, but that isn't even important. 2976 */ 2977 d_rehash(whiteout); 2978 return 0; 2979 } 2980 2981 /* 2982 * The VFS layer already does all the dentry stuff for rename, 2983 * we just have to decrement the usage count for the target if 2984 * it exists so that the VFS layer correctly free's it when it 2985 * gets overwritten. 2986 */ 2987 static int shmem_rename2(struct user_namespace *mnt_userns, 2988 struct inode *old_dir, struct dentry *old_dentry, 2989 struct inode *new_dir, struct dentry *new_dentry, 2990 unsigned int flags) 2991 { 2992 struct inode *inode = d_inode(old_dentry); 2993 int they_are_dirs = S_ISDIR(inode->i_mode); 2994 2995 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 2996 return -EINVAL; 2997 2998 if (flags & RENAME_EXCHANGE) 2999 return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); 3000 3001 if (!simple_empty(new_dentry)) 3002 return -ENOTEMPTY; 3003 3004 if (flags & RENAME_WHITEOUT) { 3005 int error; 3006 3007 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry); 3008 if (error) 3009 return error; 3010 } 3011 3012 if (d_really_is_positive(new_dentry)) { 3013 (void) shmem_unlink(new_dir, new_dentry); 3014 if (they_are_dirs) { 3015 drop_nlink(d_inode(new_dentry)); 3016 drop_nlink(old_dir); 3017 } 3018 } else if (they_are_dirs) { 3019 drop_nlink(old_dir); 3020 inc_nlink(new_dir); 3021 } 3022 3023 old_dir->i_size -= BOGO_DIRENT_SIZE; 3024 new_dir->i_size += BOGO_DIRENT_SIZE; 3025 old_dir->i_ctime = old_dir->i_mtime = 3026 new_dir->i_ctime = new_dir->i_mtime = 3027 inode->i_ctime = current_time(old_dir); 3028 return 0; 3029 } 3030 3031 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, 3032 struct dentry *dentry, const char *symname) 3033 { 3034 int error; 3035 int len; 3036 struct inode *inode; 3037 struct page *page; 3038 3039 len = strlen(symname) + 1; 3040 if (len > PAGE_SIZE) 3041 return -ENAMETOOLONG; 3042 3043 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0, 3044 VM_NORESERVE); 3045 if (!inode) 3046 return -ENOSPC; 3047 3048 error = security_inode_init_security(inode, dir, &dentry->d_name, 3049 shmem_initxattrs, NULL); 3050 if (error && error != -EOPNOTSUPP) { 3051 iput(inode); 3052 return error; 3053 } 3054 3055 inode->i_size = len-1; 3056 if (len <= SHORT_SYMLINK_LEN) { 3057 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3058 if (!inode->i_link) { 3059 iput(inode); 3060 return -ENOMEM; 3061 } 3062 inode->i_op = &shmem_short_symlink_operations; 3063 } else { 3064 inode_nohighmem(inode); 3065 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3066 if (error) { 3067 iput(inode); 3068 return error; 3069 } 3070 inode->i_mapping->a_ops = &shmem_aops; 3071 inode->i_op = &shmem_symlink_inode_operations; 3072 memcpy(page_address(page), symname, len); 3073 SetPageUptodate(page); 3074 set_page_dirty(page); 3075 unlock_page(page); 3076 put_page(page); 3077 } 3078 dir->i_size += BOGO_DIRENT_SIZE; 3079 dir->i_ctime = dir->i_mtime = current_time(dir); 3080 d_instantiate(dentry, inode); 3081 dget(dentry); 3082 return 0; 3083 } 3084 3085 static void shmem_put_link(void *arg) 3086 { 3087 mark_page_accessed(arg); 3088 put_page(arg); 3089 } 3090 3091 static const char *shmem_get_link(struct dentry *dentry, 3092 struct inode *inode, 3093 struct delayed_call *done) 3094 { 3095 struct page *page = NULL; 3096 int error; 3097 if (!dentry) { 3098 page = find_get_page(inode->i_mapping, 0); 3099 if (!page) 3100 return ERR_PTR(-ECHILD); 3101 if (PageHWPoison(page) || 3102 !PageUptodate(page)) { 3103 put_page(page); 3104 return ERR_PTR(-ECHILD); 3105 } 3106 } else { 3107 error = shmem_getpage(inode, 0, &page, SGP_READ); 3108 if (error) 3109 return ERR_PTR(error); 3110 if (!page) 3111 return ERR_PTR(-ECHILD); 3112 if (PageHWPoison(page)) { 3113 unlock_page(page); 3114 put_page(page); 3115 return ERR_PTR(-ECHILD); 3116 } 3117 unlock_page(page); 3118 } 3119 set_delayed_call(done, shmem_put_link, page); 3120 return page_address(page); 3121 } 3122 3123 #ifdef CONFIG_TMPFS_XATTR 3124 /* 3125 * Superblocks without xattr inode operations may get some security.* xattr 3126 * support from the LSM "for free". As soon as we have any other xattrs 3127 * like ACLs, we also need to implement the security.* handlers at 3128 * filesystem level, though. 3129 */ 3130 3131 /* 3132 * Callback for security_inode_init_security() for acquiring xattrs. 3133 */ 3134 static int shmem_initxattrs(struct inode *inode, 3135 const struct xattr *xattr_array, 3136 void *fs_info) 3137 { 3138 struct shmem_inode_info *info = SHMEM_I(inode); 3139 const struct xattr *xattr; 3140 struct simple_xattr *new_xattr; 3141 size_t len; 3142 3143 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3144 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3145 if (!new_xattr) 3146 return -ENOMEM; 3147 3148 len = strlen(xattr->name) + 1; 3149 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3150 GFP_KERNEL); 3151 if (!new_xattr->name) { 3152 kvfree(new_xattr); 3153 return -ENOMEM; 3154 } 3155 3156 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3157 XATTR_SECURITY_PREFIX_LEN); 3158 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3159 xattr->name, len); 3160 3161 simple_xattr_list_add(&info->xattrs, new_xattr); 3162 } 3163 3164 return 0; 3165 } 3166 3167 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3168 struct dentry *unused, struct inode *inode, 3169 const char *name, void *buffer, size_t size) 3170 { 3171 struct shmem_inode_info *info = SHMEM_I(inode); 3172 3173 name = xattr_full_name(handler, name); 3174 return simple_xattr_get(&info->xattrs, name, buffer, size); 3175 } 3176 3177 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3178 struct user_namespace *mnt_userns, 3179 struct dentry *unused, struct inode *inode, 3180 const char *name, const void *value, 3181 size_t size, int flags) 3182 { 3183 struct shmem_inode_info *info = SHMEM_I(inode); 3184 3185 name = xattr_full_name(handler, name); 3186 return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL); 3187 } 3188 3189 static const struct xattr_handler shmem_security_xattr_handler = { 3190 .prefix = XATTR_SECURITY_PREFIX, 3191 .get = shmem_xattr_handler_get, 3192 .set = shmem_xattr_handler_set, 3193 }; 3194 3195 static const struct xattr_handler shmem_trusted_xattr_handler = { 3196 .prefix = XATTR_TRUSTED_PREFIX, 3197 .get = shmem_xattr_handler_get, 3198 .set = shmem_xattr_handler_set, 3199 }; 3200 3201 static const struct xattr_handler *shmem_xattr_handlers[] = { 3202 #ifdef CONFIG_TMPFS_POSIX_ACL 3203 &posix_acl_access_xattr_handler, 3204 &posix_acl_default_xattr_handler, 3205 #endif 3206 &shmem_security_xattr_handler, 3207 &shmem_trusted_xattr_handler, 3208 NULL 3209 }; 3210 3211 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3212 { 3213 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3214 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3215 } 3216 #endif /* CONFIG_TMPFS_XATTR */ 3217 3218 static const struct inode_operations shmem_short_symlink_operations = { 3219 .get_link = simple_get_link, 3220 #ifdef CONFIG_TMPFS_XATTR 3221 .listxattr = shmem_listxattr, 3222 #endif 3223 }; 3224 3225 static const struct inode_operations shmem_symlink_inode_operations = { 3226 .get_link = shmem_get_link, 3227 #ifdef CONFIG_TMPFS_XATTR 3228 .listxattr = shmem_listxattr, 3229 #endif 3230 }; 3231 3232 static struct dentry *shmem_get_parent(struct dentry *child) 3233 { 3234 return ERR_PTR(-ESTALE); 3235 } 3236 3237 static int shmem_match(struct inode *ino, void *vfh) 3238 { 3239 __u32 *fh = vfh; 3240 __u64 inum = fh[2]; 3241 inum = (inum << 32) | fh[1]; 3242 return ino->i_ino == inum && fh[0] == ino->i_generation; 3243 } 3244 3245 /* Find any alias of inode, but prefer a hashed alias */ 3246 static struct dentry *shmem_find_alias(struct inode *inode) 3247 { 3248 struct dentry *alias = d_find_alias(inode); 3249 3250 return alias ?: d_find_any_alias(inode); 3251 } 3252 3253 3254 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3255 struct fid *fid, int fh_len, int fh_type) 3256 { 3257 struct inode *inode; 3258 struct dentry *dentry = NULL; 3259 u64 inum; 3260 3261 if (fh_len < 3) 3262 return NULL; 3263 3264 inum = fid->raw[2]; 3265 inum = (inum << 32) | fid->raw[1]; 3266 3267 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3268 shmem_match, fid->raw); 3269 if (inode) { 3270 dentry = shmem_find_alias(inode); 3271 iput(inode); 3272 } 3273 3274 return dentry; 3275 } 3276 3277 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3278 struct inode *parent) 3279 { 3280 if (*len < 3) { 3281 *len = 3; 3282 return FILEID_INVALID; 3283 } 3284 3285 if (inode_unhashed(inode)) { 3286 /* Unfortunately insert_inode_hash is not idempotent, 3287 * so as we hash inodes here rather than at creation 3288 * time, we need a lock to ensure we only try 3289 * to do it once 3290 */ 3291 static DEFINE_SPINLOCK(lock); 3292 spin_lock(&lock); 3293 if (inode_unhashed(inode)) 3294 __insert_inode_hash(inode, 3295 inode->i_ino + inode->i_generation); 3296 spin_unlock(&lock); 3297 } 3298 3299 fh[0] = inode->i_generation; 3300 fh[1] = inode->i_ino; 3301 fh[2] = ((__u64)inode->i_ino) >> 32; 3302 3303 *len = 3; 3304 return 1; 3305 } 3306 3307 static const struct export_operations shmem_export_ops = { 3308 .get_parent = shmem_get_parent, 3309 .encode_fh = shmem_encode_fh, 3310 .fh_to_dentry = shmem_fh_to_dentry, 3311 }; 3312 3313 enum shmem_param { 3314 Opt_gid, 3315 Opt_huge, 3316 Opt_mode, 3317 Opt_mpol, 3318 Opt_nr_blocks, 3319 Opt_nr_inodes, 3320 Opt_size, 3321 Opt_uid, 3322 Opt_inode32, 3323 Opt_inode64, 3324 }; 3325 3326 static const struct constant_table shmem_param_enums_huge[] = { 3327 {"never", SHMEM_HUGE_NEVER }, 3328 {"always", SHMEM_HUGE_ALWAYS }, 3329 {"within_size", SHMEM_HUGE_WITHIN_SIZE }, 3330 {"advise", SHMEM_HUGE_ADVISE }, 3331 {} 3332 }; 3333 3334 const struct fs_parameter_spec shmem_fs_parameters[] = { 3335 fsparam_u32 ("gid", Opt_gid), 3336 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge), 3337 fsparam_u32oct("mode", Opt_mode), 3338 fsparam_string("mpol", Opt_mpol), 3339 fsparam_string("nr_blocks", Opt_nr_blocks), 3340 fsparam_string("nr_inodes", Opt_nr_inodes), 3341 fsparam_string("size", Opt_size), 3342 fsparam_u32 ("uid", Opt_uid), 3343 fsparam_flag ("inode32", Opt_inode32), 3344 fsparam_flag ("inode64", Opt_inode64), 3345 {} 3346 }; 3347 3348 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) 3349 { 3350 struct shmem_options *ctx = fc->fs_private; 3351 struct fs_parse_result result; 3352 unsigned long long size; 3353 char *rest; 3354 int opt; 3355 3356 opt = fs_parse(fc, shmem_fs_parameters, param, &result); 3357 if (opt < 0) 3358 return opt; 3359 3360 switch (opt) { 3361 case Opt_size: 3362 size = memparse(param->string, &rest); 3363 if (*rest == '%') { 3364 size <<= PAGE_SHIFT; 3365 size *= totalram_pages(); 3366 do_div(size, 100); 3367 rest++; 3368 } 3369 if (*rest) 3370 goto bad_value; 3371 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE); 3372 ctx->seen |= SHMEM_SEEN_BLOCKS; 3373 break; 3374 case Opt_nr_blocks: 3375 ctx->blocks = memparse(param->string, &rest); 3376 if (*rest) 3377 goto bad_value; 3378 ctx->seen |= SHMEM_SEEN_BLOCKS; 3379 break; 3380 case Opt_nr_inodes: 3381 ctx->inodes = memparse(param->string, &rest); 3382 if (*rest) 3383 goto bad_value; 3384 ctx->seen |= SHMEM_SEEN_INODES; 3385 break; 3386 case Opt_mode: 3387 ctx->mode = result.uint_32 & 07777; 3388 break; 3389 case Opt_uid: 3390 ctx->uid = make_kuid(current_user_ns(), result.uint_32); 3391 if (!uid_valid(ctx->uid)) 3392 goto bad_value; 3393 break; 3394 case Opt_gid: 3395 ctx->gid = make_kgid(current_user_ns(), result.uint_32); 3396 if (!gid_valid(ctx->gid)) 3397 goto bad_value; 3398 break; 3399 case Opt_huge: 3400 ctx->huge = result.uint_32; 3401 if (ctx->huge != SHMEM_HUGE_NEVER && 3402 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 3403 has_transparent_hugepage())) 3404 goto unsupported_parameter; 3405 ctx->seen |= SHMEM_SEEN_HUGE; 3406 break; 3407 case Opt_mpol: 3408 if (IS_ENABLED(CONFIG_NUMA)) { 3409 mpol_put(ctx->mpol); 3410 ctx->mpol = NULL; 3411 if (mpol_parse_str(param->string, &ctx->mpol)) 3412 goto bad_value; 3413 break; 3414 } 3415 goto unsupported_parameter; 3416 case Opt_inode32: 3417 ctx->full_inums = false; 3418 ctx->seen |= SHMEM_SEEN_INUMS; 3419 break; 3420 case Opt_inode64: 3421 if (sizeof(ino_t) < 8) { 3422 return invalfc(fc, 3423 "Cannot use inode64 with <64bit inums in kernel\n"); 3424 } 3425 ctx->full_inums = true; 3426 ctx->seen |= SHMEM_SEEN_INUMS; 3427 break; 3428 } 3429 return 0; 3430 3431 unsupported_parameter: 3432 return invalfc(fc, "Unsupported parameter '%s'", param->key); 3433 bad_value: 3434 return invalfc(fc, "Bad value for '%s'", param->key); 3435 } 3436 3437 static int shmem_parse_options(struct fs_context *fc, void *data) 3438 { 3439 char *options = data; 3440 3441 if (options) { 3442 int err = security_sb_eat_lsm_opts(options, &fc->security); 3443 if (err) 3444 return err; 3445 } 3446 3447 while (options != NULL) { 3448 char *this_char = options; 3449 for (;;) { 3450 /* 3451 * NUL-terminate this option: unfortunately, 3452 * mount options form a comma-separated list, 3453 * but mpol's nodelist may also contain commas. 3454 */ 3455 options = strchr(options, ','); 3456 if (options == NULL) 3457 break; 3458 options++; 3459 if (!isdigit(*options)) { 3460 options[-1] = '\0'; 3461 break; 3462 } 3463 } 3464 if (*this_char) { 3465 char *value = strchr(this_char, '='); 3466 size_t len = 0; 3467 int err; 3468 3469 if (value) { 3470 *value++ = '\0'; 3471 len = strlen(value); 3472 } 3473 err = vfs_parse_fs_string(fc, this_char, value, len); 3474 if (err < 0) 3475 return err; 3476 } 3477 } 3478 return 0; 3479 } 3480 3481 /* 3482 * Reconfigure a shmem filesystem. 3483 * 3484 * Note that we disallow change from limited->unlimited blocks/inodes while any 3485 * are in use; but we must separately disallow unlimited->limited, because in 3486 * that case we have no record of how much is already in use. 3487 */ 3488 static int shmem_reconfigure(struct fs_context *fc) 3489 { 3490 struct shmem_options *ctx = fc->fs_private; 3491 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb); 3492 unsigned long inodes; 3493 struct mempolicy *mpol = NULL; 3494 const char *err; 3495 3496 raw_spin_lock(&sbinfo->stat_lock); 3497 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3498 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) { 3499 if (!sbinfo->max_blocks) { 3500 err = "Cannot retroactively limit size"; 3501 goto out; 3502 } 3503 if (percpu_counter_compare(&sbinfo->used_blocks, 3504 ctx->blocks) > 0) { 3505 err = "Too small a size for current use"; 3506 goto out; 3507 } 3508 } 3509 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) { 3510 if (!sbinfo->max_inodes) { 3511 err = "Cannot retroactively limit inodes"; 3512 goto out; 3513 } 3514 if (ctx->inodes < inodes) { 3515 err = "Too few inodes for current use"; 3516 goto out; 3517 } 3518 } 3519 3520 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums && 3521 sbinfo->next_ino > UINT_MAX) { 3522 err = "Current inum too high to switch to 32-bit inums"; 3523 goto out; 3524 } 3525 3526 if (ctx->seen & SHMEM_SEEN_HUGE) 3527 sbinfo->huge = ctx->huge; 3528 if (ctx->seen & SHMEM_SEEN_INUMS) 3529 sbinfo->full_inums = ctx->full_inums; 3530 if (ctx->seen & SHMEM_SEEN_BLOCKS) 3531 sbinfo->max_blocks = ctx->blocks; 3532 if (ctx->seen & SHMEM_SEEN_INODES) { 3533 sbinfo->max_inodes = ctx->inodes; 3534 sbinfo->free_inodes = ctx->inodes - inodes; 3535 } 3536 3537 /* 3538 * Preserve previous mempolicy unless mpol remount option was specified. 3539 */ 3540 if (ctx->mpol) { 3541 mpol = sbinfo->mpol; 3542 sbinfo->mpol = ctx->mpol; /* transfers initial ref */ 3543 ctx->mpol = NULL; 3544 } 3545 raw_spin_unlock(&sbinfo->stat_lock); 3546 mpol_put(mpol); 3547 return 0; 3548 out: 3549 raw_spin_unlock(&sbinfo->stat_lock); 3550 return invalfc(fc, "%s", err); 3551 } 3552 3553 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3554 { 3555 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3556 3557 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3558 seq_printf(seq, ",size=%luk", 3559 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3560 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3561 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3562 if (sbinfo->mode != (0777 | S_ISVTX)) 3563 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3564 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3565 seq_printf(seq, ",uid=%u", 3566 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3567 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3568 seq_printf(seq, ",gid=%u", 3569 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3570 3571 /* 3572 * Showing inode{64,32} might be useful even if it's the system default, 3573 * since then people don't have to resort to checking both here and 3574 * /proc/config.gz to confirm 64-bit inums were successfully applied 3575 * (which may not even exist if IKCONFIG_PROC isn't enabled). 3576 * 3577 * We hide it when inode64 isn't the default and we are using 32-bit 3578 * inodes, since that probably just means the feature isn't even under 3579 * consideration. 3580 * 3581 * As such: 3582 * 3583 * +-----------------+-----------------+ 3584 * | TMPFS_INODE64=y | TMPFS_INODE64=n | 3585 * +------------------+-----------------+-----------------+ 3586 * | full_inums=true | show | show | 3587 * | full_inums=false | show | hide | 3588 * +------------------+-----------------+-----------------+ 3589 * 3590 */ 3591 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums) 3592 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32)); 3593 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3594 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3595 if (sbinfo->huge) 3596 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3597 #endif 3598 shmem_show_mpol(seq, sbinfo->mpol); 3599 return 0; 3600 } 3601 3602 #endif /* CONFIG_TMPFS */ 3603 3604 static void shmem_put_super(struct super_block *sb) 3605 { 3606 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3607 3608 free_percpu(sbinfo->ino_batch); 3609 percpu_counter_destroy(&sbinfo->used_blocks); 3610 mpol_put(sbinfo->mpol); 3611 kfree(sbinfo); 3612 sb->s_fs_info = NULL; 3613 } 3614 3615 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) 3616 { 3617 struct shmem_options *ctx = fc->fs_private; 3618 struct inode *inode; 3619 struct shmem_sb_info *sbinfo; 3620 3621 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3622 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3623 L1_CACHE_BYTES), GFP_KERNEL); 3624 if (!sbinfo) 3625 return -ENOMEM; 3626 3627 sb->s_fs_info = sbinfo; 3628 3629 #ifdef CONFIG_TMPFS 3630 /* 3631 * Per default we only allow half of the physical ram per 3632 * tmpfs instance, limiting inodes to one per page of lowmem; 3633 * but the internal instance is left unlimited. 3634 */ 3635 if (!(sb->s_flags & SB_KERNMOUNT)) { 3636 if (!(ctx->seen & SHMEM_SEEN_BLOCKS)) 3637 ctx->blocks = shmem_default_max_blocks(); 3638 if (!(ctx->seen & SHMEM_SEEN_INODES)) 3639 ctx->inodes = shmem_default_max_inodes(); 3640 if (!(ctx->seen & SHMEM_SEEN_INUMS)) 3641 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64); 3642 } else { 3643 sb->s_flags |= SB_NOUSER; 3644 } 3645 sb->s_export_op = &shmem_export_ops; 3646 sb->s_flags |= SB_NOSEC; 3647 #else 3648 sb->s_flags |= SB_NOUSER; 3649 #endif 3650 sbinfo->max_blocks = ctx->blocks; 3651 sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes; 3652 if (sb->s_flags & SB_KERNMOUNT) { 3653 sbinfo->ino_batch = alloc_percpu(ino_t); 3654 if (!sbinfo->ino_batch) 3655 goto failed; 3656 } 3657 sbinfo->uid = ctx->uid; 3658 sbinfo->gid = ctx->gid; 3659 sbinfo->full_inums = ctx->full_inums; 3660 sbinfo->mode = ctx->mode; 3661 sbinfo->huge = ctx->huge; 3662 sbinfo->mpol = ctx->mpol; 3663 ctx->mpol = NULL; 3664 3665 raw_spin_lock_init(&sbinfo->stat_lock); 3666 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3667 goto failed; 3668 spin_lock_init(&sbinfo->shrinklist_lock); 3669 INIT_LIST_HEAD(&sbinfo->shrinklist); 3670 3671 sb->s_maxbytes = MAX_LFS_FILESIZE; 3672 sb->s_blocksize = PAGE_SIZE; 3673 sb->s_blocksize_bits = PAGE_SHIFT; 3674 sb->s_magic = TMPFS_MAGIC; 3675 sb->s_op = &shmem_ops; 3676 sb->s_time_gran = 1; 3677 #ifdef CONFIG_TMPFS_XATTR 3678 sb->s_xattr = shmem_xattr_handlers; 3679 #endif 3680 #ifdef CONFIG_TMPFS_POSIX_ACL 3681 sb->s_flags |= SB_POSIXACL; 3682 #endif 3683 uuid_gen(&sb->s_uuid); 3684 3685 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3686 if (!inode) 3687 goto failed; 3688 inode->i_uid = sbinfo->uid; 3689 inode->i_gid = sbinfo->gid; 3690 sb->s_root = d_make_root(inode); 3691 if (!sb->s_root) 3692 goto failed; 3693 return 0; 3694 3695 failed: 3696 shmem_put_super(sb); 3697 return -ENOMEM; 3698 } 3699 3700 static int shmem_get_tree(struct fs_context *fc) 3701 { 3702 return get_tree_nodev(fc, shmem_fill_super); 3703 } 3704 3705 static void shmem_free_fc(struct fs_context *fc) 3706 { 3707 struct shmem_options *ctx = fc->fs_private; 3708 3709 if (ctx) { 3710 mpol_put(ctx->mpol); 3711 kfree(ctx); 3712 } 3713 } 3714 3715 static const struct fs_context_operations shmem_fs_context_ops = { 3716 .free = shmem_free_fc, 3717 .get_tree = shmem_get_tree, 3718 #ifdef CONFIG_TMPFS 3719 .parse_monolithic = shmem_parse_options, 3720 .parse_param = shmem_parse_one, 3721 .reconfigure = shmem_reconfigure, 3722 #endif 3723 }; 3724 3725 static struct kmem_cache *shmem_inode_cachep; 3726 3727 static struct inode *shmem_alloc_inode(struct super_block *sb) 3728 { 3729 struct shmem_inode_info *info; 3730 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3731 if (!info) 3732 return NULL; 3733 return &info->vfs_inode; 3734 } 3735 3736 static void shmem_free_in_core_inode(struct inode *inode) 3737 { 3738 if (S_ISLNK(inode->i_mode)) 3739 kfree(inode->i_link); 3740 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3741 } 3742 3743 static void shmem_destroy_inode(struct inode *inode) 3744 { 3745 if (S_ISREG(inode->i_mode)) 3746 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3747 } 3748 3749 static void shmem_init_inode(void *foo) 3750 { 3751 struct shmem_inode_info *info = foo; 3752 inode_init_once(&info->vfs_inode); 3753 } 3754 3755 static void shmem_init_inodecache(void) 3756 { 3757 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3758 sizeof(struct shmem_inode_info), 3759 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3760 } 3761 3762 static void shmem_destroy_inodecache(void) 3763 { 3764 kmem_cache_destroy(shmem_inode_cachep); 3765 } 3766 3767 /* Keep the page in page cache instead of truncating it */ 3768 static int shmem_error_remove_page(struct address_space *mapping, 3769 struct page *page) 3770 { 3771 return 0; 3772 } 3773 3774 const struct address_space_operations shmem_aops = { 3775 .writepage = shmem_writepage, 3776 .set_page_dirty = __set_page_dirty_no_writeback, 3777 #ifdef CONFIG_TMPFS 3778 .write_begin = shmem_write_begin, 3779 .write_end = shmem_write_end, 3780 #endif 3781 #ifdef CONFIG_MIGRATION 3782 .migratepage = migrate_page, 3783 #endif 3784 .error_remove_page = shmem_error_remove_page, 3785 }; 3786 EXPORT_SYMBOL(shmem_aops); 3787 3788 static const struct file_operations shmem_file_operations = { 3789 .mmap = shmem_mmap, 3790 .get_unmapped_area = shmem_get_unmapped_area, 3791 #ifdef CONFIG_TMPFS 3792 .llseek = shmem_file_llseek, 3793 .read_iter = shmem_file_read_iter, 3794 .write_iter = generic_file_write_iter, 3795 .fsync = noop_fsync, 3796 .splice_read = generic_file_splice_read, 3797 .splice_write = iter_file_splice_write, 3798 .fallocate = shmem_fallocate, 3799 #endif 3800 }; 3801 3802 static const struct inode_operations shmem_inode_operations = { 3803 .getattr = shmem_getattr, 3804 .setattr = shmem_setattr, 3805 #ifdef CONFIG_TMPFS_XATTR 3806 .listxattr = shmem_listxattr, 3807 .set_acl = simple_set_acl, 3808 #endif 3809 }; 3810 3811 static const struct inode_operations shmem_dir_inode_operations = { 3812 #ifdef CONFIG_TMPFS 3813 .create = shmem_create, 3814 .lookup = simple_lookup, 3815 .link = shmem_link, 3816 .unlink = shmem_unlink, 3817 .symlink = shmem_symlink, 3818 .mkdir = shmem_mkdir, 3819 .rmdir = shmem_rmdir, 3820 .mknod = shmem_mknod, 3821 .rename = shmem_rename2, 3822 .tmpfile = shmem_tmpfile, 3823 #endif 3824 #ifdef CONFIG_TMPFS_XATTR 3825 .listxattr = shmem_listxattr, 3826 #endif 3827 #ifdef CONFIG_TMPFS_POSIX_ACL 3828 .setattr = shmem_setattr, 3829 .set_acl = simple_set_acl, 3830 #endif 3831 }; 3832 3833 static const struct inode_operations shmem_special_inode_operations = { 3834 #ifdef CONFIG_TMPFS_XATTR 3835 .listxattr = shmem_listxattr, 3836 #endif 3837 #ifdef CONFIG_TMPFS_POSIX_ACL 3838 .setattr = shmem_setattr, 3839 .set_acl = simple_set_acl, 3840 #endif 3841 }; 3842 3843 static const struct super_operations shmem_ops = { 3844 .alloc_inode = shmem_alloc_inode, 3845 .free_inode = shmem_free_in_core_inode, 3846 .destroy_inode = shmem_destroy_inode, 3847 #ifdef CONFIG_TMPFS 3848 .statfs = shmem_statfs, 3849 .show_options = shmem_show_options, 3850 #endif 3851 .evict_inode = shmem_evict_inode, 3852 .drop_inode = generic_delete_inode, 3853 .put_super = shmem_put_super, 3854 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3855 .nr_cached_objects = shmem_unused_huge_count, 3856 .free_cached_objects = shmem_unused_huge_scan, 3857 #endif 3858 }; 3859 3860 static const struct vm_operations_struct shmem_vm_ops = { 3861 .fault = shmem_fault, 3862 .map_pages = filemap_map_pages, 3863 #ifdef CONFIG_NUMA 3864 .set_policy = shmem_set_policy, 3865 .get_policy = shmem_get_policy, 3866 #endif 3867 }; 3868 3869 int shmem_init_fs_context(struct fs_context *fc) 3870 { 3871 struct shmem_options *ctx; 3872 3873 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL); 3874 if (!ctx) 3875 return -ENOMEM; 3876 3877 ctx->mode = 0777 | S_ISVTX; 3878 ctx->uid = current_fsuid(); 3879 ctx->gid = current_fsgid(); 3880 3881 fc->fs_private = ctx; 3882 fc->ops = &shmem_fs_context_ops; 3883 return 0; 3884 } 3885 3886 static struct file_system_type shmem_fs_type = { 3887 .owner = THIS_MODULE, 3888 .name = "tmpfs", 3889 .init_fs_context = shmem_init_fs_context, 3890 #ifdef CONFIG_TMPFS 3891 .parameters = shmem_fs_parameters, 3892 #endif 3893 .kill_sb = kill_litter_super, 3894 .fs_flags = FS_USERNS_MOUNT, 3895 }; 3896 3897 int __init shmem_init(void) 3898 { 3899 int error; 3900 3901 shmem_init_inodecache(); 3902 3903 error = register_filesystem(&shmem_fs_type); 3904 if (error) { 3905 pr_err("Could not register tmpfs\n"); 3906 goto out2; 3907 } 3908 3909 shm_mnt = kern_mount(&shmem_fs_type); 3910 if (IS_ERR(shm_mnt)) { 3911 error = PTR_ERR(shm_mnt); 3912 pr_err("Could not kern_mount tmpfs\n"); 3913 goto out1; 3914 } 3915 3916 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3917 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) 3918 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3919 else 3920 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */ 3921 #endif 3922 return 0; 3923 3924 out1: 3925 unregister_filesystem(&shmem_fs_type); 3926 out2: 3927 shmem_destroy_inodecache(); 3928 shm_mnt = ERR_PTR(error); 3929 return error; 3930 } 3931 3932 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) 3933 static ssize_t shmem_enabled_show(struct kobject *kobj, 3934 struct kobj_attribute *attr, char *buf) 3935 { 3936 static const int values[] = { 3937 SHMEM_HUGE_ALWAYS, 3938 SHMEM_HUGE_WITHIN_SIZE, 3939 SHMEM_HUGE_ADVISE, 3940 SHMEM_HUGE_NEVER, 3941 SHMEM_HUGE_DENY, 3942 SHMEM_HUGE_FORCE, 3943 }; 3944 int len = 0; 3945 int i; 3946 3947 for (i = 0; i < ARRAY_SIZE(values); i++) { 3948 len += sysfs_emit_at(buf, len, 3949 shmem_huge == values[i] ? "%s[%s]" : "%s%s", 3950 i ? " " : "", 3951 shmem_format_huge(values[i])); 3952 } 3953 3954 len += sysfs_emit_at(buf, len, "\n"); 3955 3956 return len; 3957 } 3958 3959 static ssize_t shmem_enabled_store(struct kobject *kobj, 3960 struct kobj_attribute *attr, const char *buf, size_t count) 3961 { 3962 char tmp[16]; 3963 int huge; 3964 3965 if (count + 1 > sizeof(tmp)) 3966 return -EINVAL; 3967 memcpy(tmp, buf, count); 3968 tmp[count] = '\0'; 3969 if (count && tmp[count - 1] == '\n') 3970 tmp[count - 1] = '\0'; 3971 3972 huge = shmem_parse_huge(tmp); 3973 if (huge == -EINVAL) 3974 return -EINVAL; 3975 if (!has_transparent_hugepage() && 3976 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3977 return -EINVAL; 3978 3979 shmem_huge = huge; 3980 if (shmem_huge > SHMEM_HUGE_DENY) 3981 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3982 return count; 3983 } 3984 3985 struct kobj_attribute shmem_enabled_attr = 3986 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3987 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ 3988 3989 #else /* !CONFIG_SHMEM */ 3990 3991 /* 3992 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 3993 * 3994 * This is intended for small system where the benefits of the full 3995 * shmem code (swap-backed and resource-limited) are outweighed by 3996 * their complexity. On systems without swap this code should be 3997 * effectively equivalent, but much lighter weight. 3998 */ 3999 4000 static struct file_system_type shmem_fs_type = { 4001 .name = "tmpfs", 4002 .init_fs_context = ramfs_init_fs_context, 4003 .parameters = ramfs_fs_parameters, 4004 .kill_sb = kill_litter_super, 4005 .fs_flags = FS_USERNS_MOUNT, 4006 }; 4007 4008 int __init shmem_init(void) 4009 { 4010 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4011 4012 shm_mnt = kern_mount(&shmem_fs_type); 4013 BUG_ON(IS_ERR(shm_mnt)); 4014 4015 return 0; 4016 } 4017 4018 int shmem_unuse(unsigned int type, bool frontswap, 4019 unsigned long *fs_pages_to_unuse) 4020 { 4021 return 0; 4022 } 4023 4024 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 4025 { 4026 return 0; 4027 } 4028 4029 void shmem_unlock_mapping(struct address_space *mapping) 4030 { 4031 } 4032 4033 #ifdef CONFIG_MMU 4034 unsigned long shmem_get_unmapped_area(struct file *file, 4035 unsigned long addr, unsigned long len, 4036 unsigned long pgoff, unsigned long flags) 4037 { 4038 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 4039 } 4040 #endif 4041 4042 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4043 { 4044 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4045 } 4046 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4047 4048 #define shmem_vm_ops generic_file_vm_ops 4049 #define shmem_file_operations ramfs_file_operations 4050 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 4051 #define shmem_acct_size(flags, size) 0 4052 #define shmem_unacct_size(flags, size) do {} while (0) 4053 4054 #endif /* CONFIG_SHMEM */ 4055 4056 /* common code */ 4057 4058 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size, 4059 unsigned long flags, unsigned int i_flags) 4060 { 4061 struct inode *inode; 4062 struct file *res; 4063 4064 if (IS_ERR(mnt)) 4065 return ERR_CAST(mnt); 4066 4067 if (size < 0 || size > MAX_LFS_FILESIZE) 4068 return ERR_PTR(-EINVAL); 4069 4070 if (shmem_acct_size(flags, size)) 4071 return ERR_PTR(-ENOMEM); 4072 4073 inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0, 4074 flags); 4075 if (unlikely(!inode)) { 4076 shmem_unacct_size(flags, size); 4077 return ERR_PTR(-ENOSPC); 4078 } 4079 inode->i_flags |= i_flags; 4080 inode->i_size = size; 4081 clear_nlink(inode); /* It is unlinked */ 4082 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4083 if (!IS_ERR(res)) 4084 res = alloc_file_pseudo(inode, mnt, name, O_RDWR, 4085 &shmem_file_operations); 4086 if (IS_ERR(res)) 4087 iput(inode); 4088 return res; 4089 } 4090 4091 /** 4092 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4093 * kernel internal. There will be NO LSM permission checks against the 4094 * underlying inode. So users of this interface must do LSM checks at a 4095 * higher layer. The users are the big_key and shm implementations. LSM 4096 * checks are provided at the key or shm level rather than the inode. 4097 * @name: name for dentry (to be seen in /proc/<pid>/maps 4098 * @size: size to be set for the file 4099 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4100 */ 4101 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4102 { 4103 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE); 4104 } 4105 4106 /** 4107 * shmem_file_setup - get an unlinked file living in tmpfs 4108 * @name: name for dentry (to be seen in /proc/<pid>/maps 4109 * @size: size to be set for the file 4110 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4111 */ 4112 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4113 { 4114 return __shmem_file_setup(shm_mnt, name, size, flags, 0); 4115 } 4116 EXPORT_SYMBOL_GPL(shmem_file_setup); 4117 4118 /** 4119 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs 4120 * @mnt: the tmpfs mount where the file will be created 4121 * @name: name for dentry (to be seen in /proc/<pid>/maps 4122 * @size: size to be set for the file 4123 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4124 */ 4125 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name, 4126 loff_t size, unsigned long flags) 4127 { 4128 return __shmem_file_setup(mnt, name, size, flags, 0); 4129 } 4130 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt); 4131 4132 /** 4133 * shmem_zero_setup - setup a shared anonymous mapping 4134 * @vma: the vma to be mmapped is prepared by do_mmap 4135 */ 4136 int shmem_zero_setup(struct vm_area_struct *vma) 4137 { 4138 struct file *file; 4139 loff_t size = vma->vm_end - vma->vm_start; 4140 4141 /* 4142 * Cloning a new file under mmap_lock leads to a lock ordering conflict 4143 * between XFS directory reading and selinux: since this file is only 4144 * accessible to the user through its mapping, use S_PRIVATE flag to 4145 * bypass file security, in the same way as shmem_kernel_file_setup(). 4146 */ 4147 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags); 4148 if (IS_ERR(file)) 4149 return PTR_ERR(file); 4150 4151 if (vma->vm_file) 4152 fput(vma->vm_file); 4153 vma->vm_file = file; 4154 vma->vm_ops = &shmem_vm_ops; 4155 4156 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 4157 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4158 (vma->vm_end & HPAGE_PMD_MASK)) { 4159 khugepaged_enter(vma, vma->vm_flags); 4160 } 4161 4162 return 0; 4163 } 4164 4165 /** 4166 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4167 * @mapping: the page's address_space 4168 * @index: the page index 4169 * @gfp: the page allocator flags to use if allocating 4170 * 4171 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4172 * with any new page allocations done using the specified allocation flags. 4173 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4174 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4175 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4176 * 4177 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4178 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4179 */ 4180 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4181 pgoff_t index, gfp_t gfp) 4182 { 4183 #ifdef CONFIG_SHMEM 4184 struct inode *inode = mapping->host; 4185 struct page *page; 4186 int error; 4187 4188 BUG_ON(!shmem_mapping(mapping)); 4189 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4190 gfp, NULL, NULL, NULL); 4191 if (error) 4192 return ERR_PTR(error); 4193 4194 unlock_page(page); 4195 if (PageHWPoison(page)) { 4196 put_page(page); 4197 return ERR_PTR(-EIO); 4198 } 4199 4200 return page; 4201 #else 4202 /* 4203 * The tiny !SHMEM case uses ramfs without swap 4204 */ 4205 return read_cache_page_gfp(mapping, index, gfp); 4206 #endif 4207 } 4208 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4209