1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/fileattr.h> 32 #include <linux/mm.h> 33 #include <linux/random.h> 34 #include <linux/sched/signal.h> 35 #include <linux/export.h> 36 #include <linux/swap.h> 37 #include <linux/uio.h> 38 #include <linux/hugetlb.h> 39 #include <linux/fs_parser.h> 40 #include <linux/swapfile.h> 41 #include "swap.h" 42 43 static struct vfsmount *shm_mnt; 44 45 #ifdef CONFIG_SHMEM 46 /* 47 * This virtual memory filesystem is heavily based on the ramfs. It 48 * extends ramfs by the ability to use swap and honor resource limits 49 * which makes it a completely usable filesystem. 50 */ 51 52 #include <linux/xattr.h> 53 #include <linux/exportfs.h> 54 #include <linux/posix_acl.h> 55 #include <linux/posix_acl_xattr.h> 56 #include <linux/mman.h> 57 #include <linux/string.h> 58 #include <linux/slab.h> 59 #include <linux/backing-dev.h> 60 #include <linux/shmem_fs.h> 61 #include <linux/writeback.h> 62 #include <linux/pagevec.h> 63 #include <linux/percpu_counter.h> 64 #include <linux/falloc.h> 65 #include <linux/splice.h> 66 #include <linux/security.h> 67 #include <linux/swapops.h> 68 #include <linux/mempolicy.h> 69 #include <linux/namei.h> 70 #include <linux/ctype.h> 71 #include <linux/migrate.h> 72 #include <linux/highmem.h> 73 #include <linux/seq_file.h> 74 #include <linux/magic.h> 75 #include <linux/syscalls.h> 76 #include <linux/fcntl.h> 77 #include <uapi/linux/memfd.h> 78 #include <linux/userfaultfd_k.h> 79 #include <linux/rmap.h> 80 #include <linux/uuid.h> 81 82 #include <linux/uaccess.h> 83 84 #include "internal.h" 85 86 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 87 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 88 89 /* Pretend that each entry is of this size in directory's i_size */ 90 #define BOGO_DIRENT_SIZE 20 91 92 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 93 #define SHORT_SYMLINK_LEN 128 94 95 /* 96 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 97 * inode->i_private (with i_rwsem making sure that it has only one user at 98 * a time): we would prefer not to enlarge the shmem inode just for that. 99 */ 100 struct shmem_falloc { 101 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 102 pgoff_t start; /* start of range currently being fallocated */ 103 pgoff_t next; /* the next page offset to be fallocated */ 104 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 105 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 106 }; 107 108 struct shmem_options { 109 unsigned long long blocks; 110 unsigned long long inodes; 111 struct mempolicy *mpol; 112 kuid_t uid; 113 kgid_t gid; 114 umode_t mode; 115 bool full_inums; 116 int huge; 117 int seen; 118 #define SHMEM_SEEN_BLOCKS 1 119 #define SHMEM_SEEN_INODES 2 120 #define SHMEM_SEEN_HUGE 4 121 #define SHMEM_SEEN_INUMS 8 122 }; 123 124 #ifdef CONFIG_TMPFS 125 static unsigned long shmem_default_max_blocks(void) 126 { 127 return totalram_pages() / 2; 128 } 129 130 static unsigned long shmem_default_max_inodes(void) 131 { 132 unsigned long nr_pages = totalram_pages(); 133 134 return min(nr_pages - totalhigh_pages(), nr_pages / 2); 135 } 136 #endif 137 138 static int shmem_swapin_folio(struct inode *inode, pgoff_t index, 139 struct folio **foliop, enum sgp_type sgp, 140 gfp_t gfp, struct vm_area_struct *vma, 141 vm_fault_t *fault_type); 142 143 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 144 { 145 return sb->s_fs_info; 146 } 147 148 /* 149 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 150 * for shared memory and for shared anonymous (/dev/zero) mappings 151 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 152 * consistent with the pre-accounting of private mappings ... 153 */ 154 static inline int shmem_acct_size(unsigned long flags, loff_t size) 155 { 156 return (flags & VM_NORESERVE) ? 157 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 158 } 159 160 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 161 { 162 if (!(flags & VM_NORESERVE)) 163 vm_unacct_memory(VM_ACCT(size)); 164 } 165 166 static inline int shmem_reacct_size(unsigned long flags, 167 loff_t oldsize, loff_t newsize) 168 { 169 if (!(flags & VM_NORESERVE)) { 170 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 171 return security_vm_enough_memory_mm(current->mm, 172 VM_ACCT(newsize) - VM_ACCT(oldsize)); 173 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 174 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 175 } 176 return 0; 177 } 178 179 /* 180 * ... whereas tmpfs objects are accounted incrementally as 181 * pages are allocated, in order to allow large sparse files. 182 * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 183 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 184 */ 185 static inline int shmem_acct_block(unsigned long flags, long pages) 186 { 187 if (!(flags & VM_NORESERVE)) 188 return 0; 189 190 return security_vm_enough_memory_mm(current->mm, 191 pages * VM_ACCT(PAGE_SIZE)); 192 } 193 194 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 195 { 196 if (flags & VM_NORESERVE) 197 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 198 } 199 200 static inline bool shmem_inode_acct_block(struct inode *inode, long pages) 201 { 202 struct shmem_inode_info *info = SHMEM_I(inode); 203 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 204 205 if (shmem_acct_block(info->flags, pages)) 206 return false; 207 208 if (sbinfo->max_blocks) { 209 if (percpu_counter_compare(&sbinfo->used_blocks, 210 sbinfo->max_blocks - pages) > 0) 211 goto unacct; 212 percpu_counter_add(&sbinfo->used_blocks, pages); 213 } 214 215 return true; 216 217 unacct: 218 shmem_unacct_blocks(info->flags, pages); 219 return false; 220 } 221 222 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages) 223 { 224 struct shmem_inode_info *info = SHMEM_I(inode); 225 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 226 227 if (sbinfo->max_blocks) 228 percpu_counter_sub(&sbinfo->used_blocks, pages); 229 shmem_unacct_blocks(info->flags, pages); 230 } 231 232 static const struct super_operations shmem_ops; 233 const struct address_space_operations shmem_aops; 234 static const struct file_operations shmem_file_operations; 235 static const struct inode_operations shmem_inode_operations; 236 static const struct inode_operations shmem_dir_inode_operations; 237 static const struct inode_operations shmem_special_inode_operations; 238 static const struct vm_operations_struct shmem_vm_ops; 239 static struct file_system_type shmem_fs_type; 240 241 bool vma_is_shmem(struct vm_area_struct *vma) 242 { 243 return vma->vm_ops == &shmem_vm_ops; 244 } 245 246 static LIST_HEAD(shmem_swaplist); 247 static DEFINE_MUTEX(shmem_swaplist_mutex); 248 249 /* 250 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and 251 * produces a novel ino for the newly allocated inode. 252 * 253 * It may also be called when making a hard link to permit the space needed by 254 * each dentry. However, in that case, no new inode number is needed since that 255 * internally draws from another pool of inode numbers (currently global 256 * get_next_ino()). This case is indicated by passing NULL as inop. 257 */ 258 #define SHMEM_INO_BATCH 1024 259 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop) 260 { 261 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 262 ino_t ino; 263 264 if (!(sb->s_flags & SB_KERNMOUNT)) { 265 raw_spin_lock(&sbinfo->stat_lock); 266 if (sbinfo->max_inodes) { 267 if (!sbinfo->free_inodes) { 268 raw_spin_unlock(&sbinfo->stat_lock); 269 return -ENOSPC; 270 } 271 sbinfo->free_inodes--; 272 } 273 if (inop) { 274 ino = sbinfo->next_ino++; 275 if (unlikely(is_zero_ino(ino))) 276 ino = sbinfo->next_ino++; 277 if (unlikely(!sbinfo->full_inums && 278 ino > UINT_MAX)) { 279 /* 280 * Emulate get_next_ino uint wraparound for 281 * compatibility 282 */ 283 if (IS_ENABLED(CONFIG_64BIT)) 284 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n", 285 __func__, MINOR(sb->s_dev)); 286 sbinfo->next_ino = 1; 287 ino = sbinfo->next_ino++; 288 } 289 *inop = ino; 290 } 291 raw_spin_unlock(&sbinfo->stat_lock); 292 } else if (inop) { 293 /* 294 * __shmem_file_setup, one of our callers, is lock-free: it 295 * doesn't hold stat_lock in shmem_reserve_inode since 296 * max_inodes is always 0, and is called from potentially 297 * unknown contexts. As such, use a per-cpu batched allocator 298 * which doesn't require the per-sb stat_lock unless we are at 299 * the batch boundary. 300 * 301 * We don't need to worry about inode{32,64} since SB_KERNMOUNT 302 * shmem mounts are not exposed to userspace, so we don't need 303 * to worry about things like glibc compatibility. 304 */ 305 ino_t *next_ino; 306 307 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu()); 308 ino = *next_ino; 309 if (unlikely(ino % SHMEM_INO_BATCH == 0)) { 310 raw_spin_lock(&sbinfo->stat_lock); 311 ino = sbinfo->next_ino; 312 sbinfo->next_ino += SHMEM_INO_BATCH; 313 raw_spin_unlock(&sbinfo->stat_lock); 314 if (unlikely(is_zero_ino(ino))) 315 ino++; 316 } 317 *inop = ino; 318 *next_ino = ++ino; 319 put_cpu(); 320 } 321 322 return 0; 323 } 324 325 static void shmem_free_inode(struct super_block *sb) 326 { 327 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 328 if (sbinfo->max_inodes) { 329 raw_spin_lock(&sbinfo->stat_lock); 330 sbinfo->free_inodes++; 331 raw_spin_unlock(&sbinfo->stat_lock); 332 } 333 } 334 335 /** 336 * shmem_recalc_inode - recalculate the block usage of an inode 337 * @inode: inode to recalc 338 * 339 * We have to calculate the free blocks since the mm can drop 340 * undirtied hole pages behind our back. 341 * 342 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 343 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 344 * 345 * It has to be called with the spinlock held. 346 */ 347 static void shmem_recalc_inode(struct inode *inode) 348 { 349 struct shmem_inode_info *info = SHMEM_I(inode); 350 long freed; 351 352 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 353 if (freed > 0) { 354 info->alloced -= freed; 355 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 356 shmem_inode_unacct_blocks(inode, freed); 357 } 358 } 359 360 bool shmem_charge(struct inode *inode, long pages) 361 { 362 struct shmem_inode_info *info = SHMEM_I(inode); 363 unsigned long flags; 364 365 if (!shmem_inode_acct_block(inode, pages)) 366 return false; 367 368 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */ 369 inode->i_mapping->nrpages += pages; 370 371 spin_lock_irqsave(&info->lock, flags); 372 info->alloced += pages; 373 inode->i_blocks += pages * BLOCKS_PER_PAGE; 374 shmem_recalc_inode(inode); 375 spin_unlock_irqrestore(&info->lock, flags); 376 377 return true; 378 } 379 380 void shmem_uncharge(struct inode *inode, long pages) 381 { 382 struct shmem_inode_info *info = SHMEM_I(inode); 383 unsigned long flags; 384 385 /* nrpages adjustment done by __filemap_remove_folio() or caller */ 386 387 spin_lock_irqsave(&info->lock, flags); 388 info->alloced -= pages; 389 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 390 shmem_recalc_inode(inode); 391 spin_unlock_irqrestore(&info->lock, flags); 392 393 shmem_inode_unacct_blocks(inode, pages); 394 } 395 396 /* 397 * Replace item expected in xarray by a new item, while holding xa_lock. 398 */ 399 static int shmem_replace_entry(struct address_space *mapping, 400 pgoff_t index, void *expected, void *replacement) 401 { 402 XA_STATE(xas, &mapping->i_pages, index); 403 void *item; 404 405 VM_BUG_ON(!expected); 406 VM_BUG_ON(!replacement); 407 item = xas_load(&xas); 408 if (item != expected) 409 return -ENOENT; 410 xas_store(&xas, replacement); 411 return 0; 412 } 413 414 /* 415 * Sometimes, before we decide whether to proceed or to fail, we must check 416 * that an entry was not already brought back from swap by a racing thread. 417 * 418 * Checking page is not enough: by the time a SwapCache page is locked, it 419 * might be reused, and again be SwapCache, using the same swap as before. 420 */ 421 static bool shmem_confirm_swap(struct address_space *mapping, 422 pgoff_t index, swp_entry_t swap) 423 { 424 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap); 425 } 426 427 /* 428 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 429 * 430 * SHMEM_HUGE_NEVER: 431 * disables huge pages for the mount; 432 * SHMEM_HUGE_ALWAYS: 433 * enables huge pages for the mount; 434 * SHMEM_HUGE_WITHIN_SIZE: 435 * only allocate huge pages if the page will be fully within i_size, 436 * also respect fadvise()/madvise() hints; 437 * SHMEM_HUGE_ADVISE: 438 * only allocate huge pages if requested with fadvise()/madvise(); 439 */ 440 441 #define SHMEM_HUGE_NEVER 0 442 #define SHMEM_HUGE_ALWAYS 1 443 #define SHMEM_HUGE_WITHIN_SIZE 2 444 #define SHMEM_HUGE_ADVISE 3 445 446 /* 447 * Special values. 448 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 449 * 450 * SHMEM_HUGE_DENY: 451 * disables huge on shm_mnt and all mounts, for emergency use; 452 * SHMEM_HUGE_FORCE: 453 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 454 * 455 */ 456 #define SHMEM_HUGE_DENY (-1) 457 #define SHMEM_HUGE_FORCE (-2) 458 459 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 460 /* ifdef here to avoid bloating shmem.o when not necessary */ 461 462 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER; 463 464 bool shmem_is_huge(struct vm_area_struct *vma, 465 struct inode *inode, pgoff_t index) 466 { 467 loff_t i_size; 468 469 if (!S_ISREG(inode->i_mode)) 470 return false; 471 if (shmem_huge == SHMEM_HUGE_DENY) 472 return false; 473 if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) || 474 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))) 475 return false; 476 if (shmem_huge == SHMEM_HUGE_FORCE) 477 return true; 478 479 switch (SHMEM_SB(inode->i_sb)->huge) { 480 case SHMEM_HUGE_ALWAYS: 481 return true; 482 case SHMEM_HUGE_WITHIN_SIZE: 483 index = round_up(index + 1, HPAGE_PMD_NR); 484 i_size = round_up(i_size_read(inode), PAGE_SIZE); 485 if (i_size >> PAGE_SHIFT >= index) 486 return true; 487 fallthrough; 488 case SHMEM_HUGE_ADVISE: 489 if (vma && (vma->vm_flags & VM_HUGEPAGE)) 490 return true; 491 fallthrough; 492 default: 493 return false; 494 } 495 } 496 497 #if defined(CONFIG_SYSFS) 498 static int shmem_parse_huge(const char *str) 499 { 500 if (!strcmp(str, "never")) 501 return SHMEM_HUGE_NEVER; 502 if (!strcmp(str, "always")) 503 return SHMEM_HUGE_ALWAYS; 504 if (!strcmp(str, "within_size")) 505 return SHMEM_HUGE_WITHIN_SIZE; 506 if (!strcmp(str, "advise")) 507 return SHMEM_HUGE_ADVISE; 508 if (!strcmp(str, "deny")) 509 return SHMEM_HUGE_DENY; 510 if (!strcmp(str, "force")) 511 return SHMEM_HUGE_FORCE; 512 return -EINVAL; 513 } 514 #endif 515 516 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 517 static const char *shmem_format_huge(int huge) 518 { 519 switch (huge) { 520 case SHMEM_HUGE_NEVER: 521 return "never"; 522 case SHMEM_HUGE_ALWAYS: 523 return "always"; 524 case SHMEM_HUGE_WITHIN_SIZE: 525 return "within_size"; 526 case SHMEM_HUGE_ADVISE: 527 return "advise"; 528 case SHMEM_HUGE_DENY: 529 return "deny"; 530 case SHMEM_HUGE_FORCE: 531 return "force"; 532 default: 533 VM_BUG_ON(1); 534 return "bad_val"; 535 } 536 } 537 #endif 538 539 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 540 struct shrink_control *sc, unsigned long nr_to_split) 541 { 542 LIST_HEAD(list), *pos, *next; 543 LIST_HEAD(to_remove); 544 struct inode *inode; 545 struct shmem_inode_info *info; 546 struct folio *folio; 547 unsigned long batch = sc ? sc->nr_to_scan : 128; 548 int split = 0; 549 550 if (list_empty(&sbinfo->shrinklist)) 551 return SHRINK_STOP; 552 553 spin_lock(&sbinfo->shrinklist_lock); 554 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 555 info = list_entry(pos, struct shmem_inode_info, shrinklist); 556 557 /* pin the inode */ 558 inode = igrab(&info->vfs_inode); 559 560 /* inode is about to be evicted */ 561 if (!inode) { 562 list_del_init(&info->shrinklist); 563 goto next; 564 } 565 566 /* Check if there's anything to gain */ 567 if (round_up(inode->i_size, PAGE_SIZE) == 568 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 569 list_move(&info->shrinklist, &to_remove); 570 goto next; 571 } 572 573 list_move(&info->shrinklist, &list); 574 next: 575 sbinfo->shrinklist_len--; 576 if (!--batch) 577 break; 578 } 579 spin_unlock(&sbinfo->shrinklist_lock); 580 581 list_for_each_safe(pos, next, &to_remove) { 582 info = list_entry(pos, struct shmem_inode_info, shrinklist); 583 inode = &info->vfs_inode; 584 list_del_init(&info->shrinklist); 585 iput(inode); 586 } 587 588 list_for_each_safe(pos, next, &list) { 589 int ret; 590 pgoff_t index; 591 592 info = list_entry(pos, struct shmem_inode_info, shrinklist); 593 inode = &info->vfs_inode; 594 595 if (nr_to_split && split >= nr_to_split) 596 goto move_back; 597 598 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT; 599 folio = filemap_get_folio(inode->i_mapping, index); 600 if (!folio) 601 goto drop; 602 603 /* No huge page at the end of the file: nothing to split */ 604 if (!folio_test_large(folio)) { 605 folio_put(folio); 606 goto drop; 607 } 608 609 /* 610 * Move the inode on the list back to shrinklist if we failed 611 * to lock the page at this time. 612 * 613 * Waiting for the lock may lead to deadlock in the 614 * reclaim path. 615 */ 616 if (!folio_trylock(folio)) { 617 folio_put(folio); 618 goto move_back; 619 } 620 621 ret = split_folio(folio); 622 folio_unlock(folio); 623 folio_put(folio); 624 625 /* If split failed move the inode on the list back to shrinklist */ 626 if (ret) 627 goto move_back; 628 629 split++; 630 drop: 631 list_del_init(&info->shrinklist); 632 goto put; 633 move_back: 634 /* 635 * Make sure the inode is either on the global list or deleted 636 * from any local list before iput() since it could be deleted 637 * in another thread once we put the inode (then the local list 638 * is corrupted). 639 */ 640 spin_lock(&sbinfo->shrinklist_lock); 641 list_move(&info->shrinklist, &sbinfo->shrinklist); 642 sbinfo->shrinklist_len++; 643 spin_unlock(&sbinfo->shrinklist_lock); 644 put: 645 iput(inode); 646 } 647 648 return split; 649 } 650 651 static long shmem_unused_huge_scan(struct super_block *sb, 652 struct shrink_control *sc) 653 { 654 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 655 656 if (!READ_ONCE(sbinfo->shrinklist_len)) 657 return SHRINK_STOP; 658 659 return shmem_unused_huge_shrink(sbinfo, sc, 0); 660 } 661 662 static long shmem_unused_huge_count(struct super_block *sb, 663 struct shrink_control *sc) 664 { 665 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 666 return READ_ONCE(sbinfo->shrinklist_len); 667 } 668 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 669 670 #define shmem_huge SHMEM_HUGE_DENY 671 672 bool shmem_is_huge(struct vm_area_struct *vma, 673 struct inode *inode, pgoff_t index) 674 { 675 return false; 676 } 677 678 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 679 struct shrink_control *sc, unsigned long nr_to_split) 680 { 681 return 0; 682 } 683 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 684 685 /* 686 * Like filemap_add_folio, but error if expected item has gone. 687 */ 688 static int shmem_add_to_page_cache(struct folio *folio, 689 struct address_space *mapping, 690 pgoff_t index, void *expected, gfp_t gfp, 691 struct mm_struct *charge_mm) 692 { 693 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); 694 long nr = folio_nr_pages(folio); 695 int error; 696 697 VM_BUG_ON_FOLIO(index != round_down(index, nr), folio); 698 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 699 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio); 700 VM_BUG_ON(expected && folio_test_large(folio)); 701 702 folio_ref_add(folio, nr); 703 folio->mapping = mapping; 704 folio->index = index; 705 706 if (!folio_test_swapcache(folio)) { 707 error = mem_cgroup_charge(folio, charge_mm, gfp); 708 if (error) { 709 if (folio_test_pmd_mappable(folio)) { 710 count_vm_event(THP_FILE_FALLBACK); 711 count_vm_event(THP_FILE_FALLBACK_CHARGE); 712 } 713 goto error; 714 } 715 } 716 folio_throttle_swaprate(folio, gfp); 717 718 do { 719 xas_lock_irq(&xas); 720 if (expected != xas_find_conflict(&xas)) { 721 xas_set_err(&xas, -EEXIST); 722 goto unlock; 723 } 724 if (expected && xas_find_conflict(&xas)) { 725 xas_set_err(&xas, -EEXIST); 726 goto unlock; 727 } 728 xas_store(&xas, folio); 729 if (xas_error(&xas)) 730 goto unlock; 731 if (folio_test_pmd_mappable(folio)) { 732 count_vm_event(THP_FILE_ALLOC); 733 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr); 734 } 735 mapping->nrpages += nr; 736 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); 737 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr); 738 unlock: 739 xas_unlock_irq(&xas); 740 } while (xas_nomem(&xas, gfp)); 741 742 if (xas_error(&xas)) { 743 error = xas_error(&xas); 744 goto error; 745 } 746 747 return 0; 748 error: 749 folio->mapping = NULL; 750 folio_ref_sub(folio, nr); 751 return error; 752 } 753 754 /* 755 * Like delete_from_page_cache, but substitutes swap for @folio. 756 */ 757 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) 758 { 759 struct address_space *mapping = folio->mapping; 760 long nr = folio_nr_pages(folio); 761 int error; 762 763 xa_lock_irq(&mapping->i_pages); 764 error = shmem_replace_entry(mapping, folio->index, folio, radswap); 765 folio->mapping = NULL; 766 mapping->nrpages -= nr; 767 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 768 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); 769 xa_unlock_irq(&mapping->i_pages); 770 folio_put(folio); 771 BUG_ON(error); 772 } 773 774 /* 775 * Remove swap entry from page cache, free the swap and its page cache. 776 */ 777 static int shmem_free_swap(struct address_space *mapping, 778 pgoff_t index, void *radswap) 779 { 780 void *old; 781 782 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); 783 if (old != radswap) 784 return -ENOENT; 785 free_swap_and_cache(radix_to_swp_entry(radswap)); 786 return 0; 787 } 788 789 /* 790 * Determine (in bytes) how many of the shmem object's pages mapped by the 791 * given offsets are swapped out. 792 * 793 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 794 * as long as the inode doesn't go away and racy results are not a problem. 795 */ 796 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 797 pgoff_t start, pgoff_t end) 798 { 799 XA_STATE(xas, &mapping->i_pages, start); 800 struct page *page; 801 unsigned long swapped = 0; 802 803 rcu_read_lock(); 804 xas_for_each(&xas, page, end - 1) { 805 if (xas_retry(&xas, page)) 806 continue; 807 if (xa_is_value(page)) 808 swapped++; 809 810 if (need_resched()) { 811 xas_pause(&xas); 812 cond_resched_rcu(); 813 } 814 } 815 816 rcu_read_unlock(); 817 818 return swapped << PAGE_SHIFT; 819 } 820 821 /* 822 * Determine (in bytes) how many of the shmem object's pages mapped by the 823 * given vma is swapped out. 824 * 825 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 826 * as long as the inode doesn't go away and racy results are not a problem. 827 */ 828 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 829 { 830 struct inode *inode = file_inode(vma->vm_file); 831 struct shmem_inode_info *info = SHMEM_I(inode); 832 struct address_space *mapping = inode->i_mapping; 833 unsigned long swapped; 834 835 /* Be careful as we don't hold info->lock */ 836 swapped = READ_ONCE(info->swapped); 837 838 /* 839 * The easier cases are when the shmem object has nothing in swap, or 840 * the vma maps it whole. Then we can simply use the stats that we 841 * already track. 842 */ 843 if (!swapped) 844 return 0; 845 846 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 847 return swapped << PAGE_SHIFT; 848 849 /* Here comes the more involved part */ 850 return shmem_partial_swap_usage(mapping, vma->vm_pgoff, 851 vma->vm_pgoff + vma_pages(vma)); 852 } 853 854 /* 855 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 856 */ 857 void shmem_unlock_mapping(struct address_space *mapping) 858 { 859 struct folio_batch fbatch; 860 pgoff_t index = 0; 861 862 folio_batch_init(&fbatch); 863 /* 864 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 865 */ 866 while (!mapping_unevictable(mapping) && 867 filemap_get_folios(mapping, &index, ~0UL, &fbatch)) { 868 check_move_unevictable_folios(&fbatch); 869 folio_batch_release(&fbatch); 870 cond_resched(); 871 } 872 } 873 874 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) 875 { 876 struct folio *folio; 877 878 /* 879 * At first avoid shmem_get_folio(,,,SGP_READ): that fails 880 * beyond i_size, and reports fallocated pages as holes. 881 */ 882 folio = __filemap_get_folio(inode->i_mapping, index, 883 FGP_ENTRY | FGP_LOCK, 0); 884 if (!xa_is_value(folio)) 885 return folio; 886 /* 887 * But read a page back from swap if any of it is within i_size 888 * (although in some cases this is just a waste of time). 889 */ 890 folio = NULL; 891 shmem_get_folio(inode, index, &folio, SGP_READ); 892 return folio; 893 } 894 895 /* 896 * Remove range of pages and swap entries from page cache, and free them. 897 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 898 */ 899 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 900 bool unfalloc) 901 { 902 struct address_space *mapping = inode->i_mapping; 903 struct shmem_inode_info *info = SHMEM_I(inode); 904 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 905 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 906 struct folio_batch fbatch; 907 pgoff_t indices[PAGEVEC_SIZE]; 908 struct folio *folio; 909 bool same_folio; 910 long nr_swaps_freed = 0; 911 pgoff_t index; 912 int i; 913 914 if (lend == -1) 915 end = -1; /* unsigned, so actually very big */ 916 917 if (info->fallocend > start && info->fallocend <= end && !unfalloc) 918 info->fallocend = start; 919 920 folio_batch_init(&fbatch); 921 index = start; 922 while (index < end && find_lock_entries(mapping, index, end - 1, 923 &fbatch, indices)) { 924 for (i = 0; i < folio_batch_count(&fbatch); i++) { 925 folio = fbatch.folios[i]; 926 927 index = indices[i]; 928 929 if (xa_is_value(folio)) { 930 if (unfalloc) 931 continue; 932 nr_swaps_freed += !shmem_free_swap(mapping, 933 index, folio); 934 continue; 935 } 936 index += folio_nr_pages(folio) - 1; 937 938 if (!unfalloc || !folio_test_uptodate(folio)) 939 truncate_inode_folio(mapping, folio); 940 folio_unlock(folio); 941 } 942 folio_batch_remove_exceptionals(&fbatch); 943 folio_batch_release(&fbatch); 944 cond_resched(); 945 index++; 946 } 947 948 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT); 949 folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT); 950 if (folio) { 951 same_folio = lend < folio_pos(folio) + folio_size(folio); 952 folio_mark_dirty(folio); 953 if (!truncate_inode_partial_folio(folio, lstart, lend)) { 954 start = folio->index + folio_nr_pages(folio); 955 if (same_folio) 956 end = folio->index; 957 } 958 folio_unlock(folio); 959 folio_put(folio); 960 folio = NULL; 961 } 962 963 if (!same_folio) 964 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); 965 if (folio) { 966 folio_mark_dirty(folio); 967 if (!truncate_inode_partial_folio(folio, lstart, lend)) 968 end = folio->index; 969 folio_unlock(folio); 970 folio_put(folio); 971 } 972 973 index = start; 974 while (index < end) { 975 cond_resched(); 976 977 if (!find_get_entries(mapping, index, end - 1, &fbatch, 978 indices)) { 979 /* If all gone or hole-punch or unfalloc, we're done */ 980 if (index == start || end != -1) 981 break; 982 /* But if truncating, restart to make sure all gone */ 983 index = start; 984 continue; 985 } 986 for (i = 0; i < folio_batch_count(&fbatch); i++) { 987 folio = fbatch.folios[i]; 988 989 index = indices[i]; 990 if (xa_is_value(folio)) { 991 if (unfalloc) 992 continue; 993 if (shmem_free_swap(mapping, index, folio)) { 994 /* Swap was replaced by page: retry */ 995 index--; 996 break; 997 } 998 nr_swaps_freed++; 999 continue; 1000 } 1001 1002 folio_lock(folio); 1003 1004 if (!unfalloc || !folio_test_uptodate(folio)) { 1005 if (folio_mapping(folio) != mapping) { 1006 /* Page was replaced by swap: retry */ 1007 folio_unlock(folio); 1008 index--; 1009 break; 1010 } 1011 VM_BUG_ON_FOLIO(folio_test_writeback(folio), 1012 folio); 1013 truncate_inode_folio(mapping, folio); 1014 } 1015 index = folio->index + folio_nr_pages(folio) - 1; 1016 folio_unlock(folio); 1017 } 1018 folio_batch_remove_exceptionals(&fbatch); 1019 folio_batch_release(&fbatch); 1020 index++; 1021 } 1022 1023 spin_lock_irq(&info->lock); 1024 info->swapped -= nr_swaps_freed; 1025 shmem_recalc_inode(inode); 1026 spin_unlock_irq(&info->lock); 1027 } 1028 1029 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 1030 { 1031 shmem_undo_range(inode, lstart, lend, false); 1032 inode->i_ctime = inode->i_mtime = current_time(inode); 1033 } 1034 EXPORT_SYMBOL_GPL(shmem_truncate_range); 1035 1036 static int shmem_getattr(struct user_namespace *mnt_userns, 1037 const struct path *path, struct kstat *stat, 1038 u32 request_mask, unsigned int query_flags) 1039 { 1040 struct inode *inode = path->dentry->d_inode; 1041 struct shmem_inode_info *info = SHMEM_I(inode); 1042 1043 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 1044 spin_lock_irq(&info->lock); 1045 shmem_recalc_inode(inode); 1046 spin_unlock_irq(&info->lock); 1047 } 1048 if (info->fsflags & FS_APPEND_FL) 1049 stat->attributes |= STATX_ATTR_APPEND; 1050 if (info->fsflags & FS_IMMUTABLE_FL) 1051 stat->attributes |= STATX_ATTR_IMMUTABLE; 1052 if (info->fsflags & FS_NODUMP_FL) 1053 stat->attributes |= STATX_ATTR_NODUMP; 1054 stat->attributes_mask |= (STATX_ATTR_APPEND | 1055 STATX_ATTR_IMMUTABLE | 1056 STATX_ATTR_NODUMP); 1057 generic_fillattr(&init_user_ns, inode, stat); 1058 1059 if (shmem_is_huge(NULL, inode, 0)) 1060 stat->blksize = HPAGE_PMD_SIZE; 1061 1062 if (request_mask & STATX_BTIME) { 1063 stat->result_mask |= STATX_BTIME; 1064 stat->btime.tv_sec = info->i_crtime.tv_sec; 1065 stat->btime.tv_nsec = info->i_crtime.tv_nsec; 1066 } 1067 1068 return 0; 1069 } 1070 1071 static int shmem_setattr(struct user_namespace *mnt_userns, 1072 struct dentry *dentry, struct iattr *attr) 1073 { 1074 struct inode *inode = d_inode(dentry); 1075 struct shmem_inode_info *info = SHMEM_I(inode); 1076 int error; 1077 1078 error = setattr_prepare(&init_user_ns, dentry, attr); 1079 if (error) 1080 return error; 1081 1082 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 1083 loff_t oldsize = inode->i_size; 1084 loff_t newsize = attr->ia_size; 1085 1086 /* protected by i_rwsem */ 1087 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 1088 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 1089 return -EPERM; 1090 1091 if (newsize != oldsize) { 1092 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1093 oldsize, newsize); 1094 if (error) 1095 return error; 1096 i_size_write(inode, newsize); 1097 inode->i_ctime = inode->i_mtime = current_time(inode); 1098 } 1099 if (newsize <= oldsize) { 1100 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1101 if (oldsize > holebegin) 1102 unmap_mapping_range(inode->i_mapping, 1103 holebegin, 0, 1); 1104 if (info->alloced) 1105 shmem_truncate_range(inode, 1106 newsize, (loff_t)-1); 1107 /* unmap again to remove racily COWed private pages */ 1108 if (oldsize > holebegin) 1109 unmap_mapping_range(inode->i_mapping, 1110 holebegin, 0, 1); 1111 } 1112 } 1113 1114 setattr_copy(&init_user_ns, inode, attr); 1115 if (attr->ia_valid & ATTR_MODE) 1116 error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); 1117 return error; 1118 } 1119 1120 static void shmem_evict_inode(struct inode *inode) 1121 { 1122 struct shmem_inode_info *info = SHMEM_I(inode); 1123 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1124 1125 if (shmem_mapping(inode->i_mapping)) { 1126 shmem_unacct_size(info->flags, inode->i_size); 1127 inode->i_size = 0; 1128 mapping_set_exiting(inode->i_mapping); 1129 shmem_truncate_range(inode, 0, (loff_t)-1); 1130 if (!list_empty(&info->shrinklist)) { 1131 spin_lock(&sbinfo->shrinklist_lock); 1132 if (!list_empty(&info->shrinklist)) { 1133 list_del_init(&info->shrinklist); 1134 sbinfo->shrinklist_len--; 1135 } 1136 spin_unlock(&sbinfo->shrinklist_lock); 1137 } 1138 while (!list_empty(&info->swaplist)) { 1139 /* Wait while shmem_unuse() is scanning this inode... */ 1140 wait_var_event(&info->stop_eviction, 1141 !atomic_read(&info->stop_eviction)); 1142 mutex_lock(&shmem_swaplist_mutex); 1143 /* ...but beware of the race if we peeked too early */ 1144 if (!atomic_read(&info->stop_eviction)) 1145 list_del_init(&info->swaplist); 1146 mutex_unlock(&shmem_swaplist_mutex); 1147 } 1148 } 1149 1150 simple_xattrs_free(&info->xattrs); 1151 WARN_ON(inode->i_blocks); 1152 shmem_free_inode(inode->i_sb); 1153 clear_inode(inode); 1154 } 1155 1156 static int shmem_find_swap_entries(struct address_space *mapping, 1157 pgoff_t start, struct folio_batch *fbatch, 1158 pgoff_t *indices, unsigned int type) 1159 { 1160 XA_STATE(xas, &mapping->i_pages, start); 1161 struct folio *folio; 1162 swp_entry_t entry; 1163 1164 rcu_read_lock(); 1165 xas_for_each(&xas, folio, ULONG_MAX) { 1166 if (xas_retry(&xas, folio)) 1167 continue; 1168 1169 if (!xa_is_value(folio)) 1170 continue; 1171 1172 entry = radix_to_swp_entry(folio); 1173 /* 1174 * swapin error entries can be found in the mapping. But they're 1175 * deliberately ignored here as we've done everything we can do. 1176 */ 1177 if (swp_type(entry) != type) 1178 continue; 1179 1180 indices[folio_batch_count(fbatch)] = xas.xa_index; 1181 if (!folio_batch_add(fbatch, folio)) 1182 break; 1183 1184 if (need_resched()) { 1185 xas_pause(&xas); 1186 cond_resched_rcu(); 1187 } 1188 } 1189 rcu_read_unlock(); 1190 1191 return xas.xa_index; 1192 } 1193 1194 /* 1195 * Move the swapped pages for an inode to page cache. Returns the count 1196 * of pages swapped in, or the error in case of failure. 1197 */ 1198 static int shmem_unuse_swap_entries(struct inode *inode, 1199 struct folio_batch *fbatch, pgoff_t *indices) 1200 { 1201 int i = 0; 1202 int ret = 0; 1203 int error = 0; 1204 struct address_space *mapping = inode->i_mapping; 1205 1206 for (i = 0; i < folio_batch_count(fbatch); i++) { 1207 struct folio *folio = fbatch->folios[i]; 1208 1209 if (!xa_is_value(folio)) 1210 continue; 1211 error = shmem_swapin_folio(inode, indices[i], 1212 &folio, SGP_CACHE, 1213 mapping_gfp_mask(mapping), 1214 NULL, NULL); 1215 if (error == 0) { 1216 folio_unlock(folio); 1217 folio_put(folio); 1218 ret++; 1219 } 1220 if (error == -ENOMEM) 1221 break; 1222 error = 0; 1223 } 1224 return error ? error : ret; 1225 } 1226 1227 /* 1228 * If swap found in inode, free it and move page from swapcache to filecache. 1229 */ 1230 static int shmem_unuse_inode(struct inode *inode, unsigned int type) 1231 { 1232 struct address_space *mapping = inode->i_mapping; 1233 pgoff_t start = 0; 1234 struct folio_batch fbatch; 1235 pgoff_t indices[PAGEVEC_SIZE]; 1236 int ret = 0; 1237 1238 do { 1239 folio_batch_init(&fbatch); 1240 shmem_find_swap_entries(mapping, start, &fbatch, indices, type); 1241 if (folio_batch_count(&fbatch) == 0) { 1242 ret = 0; 1243 break; 1244 } 1245 1246 ret = shmem_unuse_swap_entries(inode, &fbatch, indices); 1247 if (ret < 0) 1248 break; 1249 1250 start = indices[folio_batch_count(&fbatch) - 1]; 1251 } while (true); 1252 1253 return ret; 1254 } 1255 1256 /* 1257 * Read all the shared memory data that resides in the swap 1258 * device 'type' back into memory, so the swap device can be 1259 * unused. 1260 */ 1261 int shmem_unuse(unsigned int type) 1262 { 1263 struct shmem_inode_info *info, *next; 1264 int error = 0; 1265 1266 if (list_empty(&shmem_swaplist)) 1267 return 0; 1268 1269 mutex_lock(&shmem_swaplist_mutex); 1270 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) { 1271 if (!info->swapped) { 1272 list_del_init(&info->swaplist); 1273 continue; 1274 } 1275 /* 1276 * Drop the swaplist mutex while searching the inode for swap; 1277 * but before doing so, make sure shmem_evict_inode() will not 1278 * remove placeholder inode from swaplist, nor let it be freed 1279 * (igrab() would protect from unlink, but not from unmount). 1280 */ 1281 atomic_inc(&info->stop_eviction); 1282 mutex_unlock(&shmem_swaplist_mutex); 1283 1284 error = shmem_unuse_inode(&info->vfs_inode, type); 1285 cond_resched(); 1286 1287 mutex_lock(&shmem_swaplist_mutex); 1288 next = list_next_entry(info, swaplist); 1289 if (!info->swapped) 1290 list_del_init(&info->swaplist); 1291 if (atomic_dec_and_test(&info->stop_eviction)) 1292 wake_up_var(&info->stop_eviction); 1293 if (error) 1294 break; 1295 } 1296 mutex_unlock(&shmem_swaplist_mutex); 1297 1298 return error; 1299 } 1300 1301 /* 1302 * Move the page from the page cache to the swap cache. 1303 */ 1304 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1305 { 1306 struct folio *folio = page_folio(page); 1307 struct shmem_inode_info *info; 1308 struct address_space *mapping; 1309 struct inode *inode; 1310 swp_entry_t swap; 1311 pgoff_t index; 1312 1313 /* 1314 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or 1315 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages, 1316 * and its shmem_writeback() needs them to be split when swapping. 1317 */ 1318 if (folio_test_large(folio)) { 1319 /* Ensure the subpages are still dirty */ 1320 folio_test_set_dirty(folio); 1321 if (split_huge_page(page) < 0) 1322 goto redirty; 1323 folio = page_folio(page); 1324 folio_clear_dirty(folio); 1325 } 1326 1327 BUG_ON(!folio_test_locked(folio)); 1328 mapping = folio->mapping; 1329 index = folio->index; 1330 inode = mapping->host; 1331 info = SHMEM_I(inode); 1332 if (info->flags & VM_LOCKED) 1333 goto redirty; 1334 if (!total_swap_pages) 1335 goto redirty; 1336 1337 /* 1338 * Our capabilities prevent regular writeback or sync from ever calling 1339 * shmem_writepage; but a stacking filesystem might use ->writepage of 1340 * its underlying filesystem, in which case tmpfs should write out to 1341 * swap only in response to memory pressure, and not for the writeback 1342 * threads or sync. 1343 */ 1344 if (!wbc->for_reclaim) { 1345 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1346 goto redirty; 1347 } 1348 1349 /* 1350 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1351 * value into swapfile.c, the only way we can correctly account for a 1352 * fallocated folio arriving here is now to initialize it and write it. 1353 * 1354 * That's okay for a folio already fallocated earlier, but if we have 1355 * not yet completed the fallocation, then (a) we want to keep track 1356 * of this folio in case we have to undo it, and (b) it may not be a 1357 * good idea to continue anyway, once we're pushing into swap. So 1358 * reactivate the folio, and let shmem_fallocate() quit when too many. 1359 */ 1360 if (!folio_test_uptodate(folio)) { 1361 if (inode->i_private) { 1362 struct shmem_falloc *shmem_falloc; 1363 spin_lock(&inode->i_lock); 1364 shmem_falloc = inode->i_private; 1365 if (shmem_falloc && 1366 !shmem_falloc->waitq && 1367 index >= shmem_falloc->start && 1368 index < shmem_falloc->next) 1369 shmem_falloc->nr_unswapped++; 1370 else 1371 shmem_falloc = NULL; 1372 spin_unlock(&inode->i_lock); 1373 if (shmem_falloc) 1374 goto redirty; 1375 } 1376 folio_zero_range(folio, 0, folio_size(folio)); 1377 flush_dcache_folio(folio); 1378 folio_mark_uptodate(folio); 1379 } 1380 1381 swap = folio_alloc_swap(folio); 1382 if (!swap.val) 1383 goto redirty; 1384 1385 /* 1386 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1387 * if it's not already there. Do it now before the folio is 1388 * moved to swap cache, when its pagelock no longer protects 1389 * the inode from eviction. But don't unlock the mutex until 1390 * we've incremented swapped, because shmem_unuse_inode() will 1391 * prune a !swapped inode from the swaplist under this mutex. 1392 */ 1393 mutex_lock(&shmem_swaplist_mutex); 1394 if (list_empty(&info->swaplist)) 1395 list_add(&info->swaplist, &shmem_swaplist); 1396 1397 if (add_to_swap_cache(folio, swap, 1398 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN, 1399 NULL) == 0) { 1400 spin_lock_irq(&info->lock); 1401 shmem_recalc_inode(inode); 1402 info->swapped++; 1403 spin_unlock_irq(&info->lock); 1404 1405 swap_shmem_alloc(swap); 1406 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap)); 1407 1408 mutex_unlock(&shmem_swaplist_mutex); 1409 BUG_ON(folio_mapped(folio)); 1410 swap_writepage(&folio->page, wbc); 1411 return 0; 1412 } 1413 1414 mutex_unlock(&shmem_swaplist_mutex); 1415 put_swap_folio(folio, swap); 1416 redirty: 1417 folio_mark_dirty(folio); 1418 if (wbc->for_reclaim) 1419 return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ 1420 folio_unlock(folio); 1421 return 0; 1422 } 1423 1424 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1425 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1426 { 1427 char buffer[64]; 1428 1429 if (!mpol || mpol->mode == MPOL_DEFAULT) 1430 return; /* show nothing */ 1431 1432 mpol_to_str(buffer, sizeof(buffer), mpol); 1433 1434 seq_printf(seq, ",mpol=%s", buffer); 1435 } 1436 1437 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1438 { 1439 struct mempolicy *mpol = NULL; 1440 if (sbinfo->mpol) { 1441 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1442 mpol = sbinfo->mpol; 1443 mpol_get(mpol); 1444 raw_spin_unlock(&sbinfo->stat_lock); 1445 } 1446 return mpol; 1447 } 1448 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1449 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1450 { 1451 } 1452 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1453 { 1454 return NULL; 1455 } 1456 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1457 #ifndef CONFIG_NUMA 1458 #define vm_policy vm_private_data 1459 #endif 1460 1461 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1462 struct shmem_inode_info *info, pgoff_t index) 1463 { 1464 /* Create a pseudo vma that just contains the policy */ 1465 vma_init(vma, NULL); 1466 /* Bias interleave by inode number to distribute better across nodes */ 1467 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1468 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1469 } 1470 1471 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1472 { 1473 /* Drop reference taken by mpol_shared_policy_lookup() */ 1474 mpol_cond_put(vma->vm_policy); 1475 } 1476 1477 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1478 struct shmem_inode_info *info, pgoff_t index) 1479 { 1480 struct vm_area_struct pvma; 1481 struct page *page; 1482 struct vm_fault vmf = { 1483 .vma = &pvma, 1484 }; 1485 1486 shmem_pseudo_vma_init(&pvma, info, index); 1487 page = swap_cluster_readahead(swap, gfp, &vmf); 1488 shmem_pseudo_vma_destroy(&pvma); 1489 1490 if (!page) 1491 return NULL; 1492 return page_folio(page); 1493 } 1494 1495 /* 1496 * Make sure huge_gfp is always more limited than limit_gfp. 1497 * Some of the flags set permissions, while others set limitations. 1498 */ 1499 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) 1500 { 1501 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; 1502 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; 1503 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; 1504 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK); 1505 1506 /* Allow allocations only from the originally specified zones. */ 1507 result |= zoneflags; 1508 1509 /* 1510 * Minimize the result gfp by taking the union with the deny flags, 1511 * and the intersection of the allow flags. 1512 */ 1513 result |= (limit_gfp & denyflags); 1514 result |= (huge_gfp & limit_gfp) & allowflags; 1515 1516 return result; 1517 } 1518 1519 static struct folio *shmem_alloc_hugefolio(gfp_t gfp, 1520 struct shmem_inode_info *info, pgoff_t index) 1521 { 1522 struct vm_area_struct pvma; 1523 struct address_space *mapping = info->vfs_inode.i_mapping; 1524 pgoff_t hindex; 1525 struct folio *folio; 1526 1527 hindex = round_down(index, HPAGE_PMD_NR); 1528 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1, 1529 XA_PRESENT)) 1530 return NULL; 1531 1532 shmem_pseudo_vma_init(&pvma, info, hindex); 1533 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true); 1534 shmem_pseudo_vma_destroy(&pvma); 1535 if (!folio) 1536 count_vm_event(THP_FILE_FALLBACK); 1537 return folio; 1538 } 1539 1540 static struct folio *shmem_alloc_folio(gfp_t gfp, 1541 struct shmem_inode_info *info, pgoff_t index) 1542 { 1543 struct vm_area_struct pvma; 1544 struct folio *folio; 1545 1546 shmem_pseudo_vma_init(&pvma, info, index); 1547 folio = vma_alloc_folio(gfp, 0, &pvma, 0, false); 1548 shmem_pseudo_vma_destroy(&pvma); 1549 1550 return folio; 1551 } 1552 1553 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode, 1554 pgoff_t index, bool huge) 1555 { 1556 struct shmem_inode_info *info = SHMEM_I(inode); 1557 struct folio *folio; 1558 int nr; 1559 int err = -ENOSPC; 1560 1561 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1562 huge = false; 1563 nr = huge ? HPAGE_PMD_NR : 1; 1564 1565 if (!shmem_inode_acct_block(inode, nr)) 1566 goto failed; 1567 1568 if (huge) 1569 folio = shmem_alloc_hugefolio(gfp, info, index); 1570 else 1571 folio = shmem_alloc_folio(gfp, info, index); 1572 if (folio) { 1573 __folio_set_locked(folio); 1574 __folio_set_swapbacked(folio); 1575 return folio; 1576 } 1577 1578 err = -ENOMEM; 1579 shmem_inode_unacct_blocks(inode, nr); 1580 failed: 1581 return ERR_PTR(err); 1582 } 1583 1584 /* 1585 * When a page is moved from swapcache to shmem filecache (either by the 1586 * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of 1587 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1588 * ignorance of the mapping it belongs to. If that mapping has special 1589 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1590 * we may need to copy to a suitable page before moving to filecache. 1591 * 1592 * In a future release, this may well be extended to respect cpuset and 1593 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1594 * but for now it is a simple matter of zone. 1595 */ 1596 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp) 1597 { 1598 return folio_zonenum(folio) > gfp_zone(gfp); 1599 } 1600 1601 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp, 1602 struct shmem_inode_info *info, pgoff_t index) 1603 { 1604 struct folio *old, *new; 1605 struct address_space *swap_mapping; 1606 swp_entry_t entry; 1607 pgoff_t swap_index; 1608 int error; 1609 1610 old = *foliop; 1611 entry = folio_swap_entry(old); 1612 swap_index = swp_offset(entry); 1613 swap_mapping = swap_address_space(entry); 1614 1615 /* 1616 * We have arrived here because our zones are constrained, so don't 1617 * limit chance of success by further cpuset and node constraints. 1618 */ 1619 gfp &= ~GFP_CONSTRAINT_MASK; 1620 VM_BUG_ON_FOLIO(folio_test_large(old), old); 1621 new = shmem_alloc_folio(gfp, info, index); 1622 if (!new) 1623 return -ENOMEM; 1624 1625 folio_get(new); 1626 folio_copy(new, old); 1627 flush_dcache_folio(new); 1628 1629 __folio_set_locked(new); 1630 __folio_set_swapbacked(new); 1631 folio_mark_uptodate(new); 1632 folio_set_swap_entry(new, entry); 1633 folio_set_swapcache(new); 1634 1635 /* 1636 * Our caller will very soon move newpage out of swapcache, but it's 1637 * a nice clean interface for us to replace oldpage by newpage there. 1638 */ 1639 xa_lock_irq(&swap_mapping->i_pages); 1640 error = shmem_replace_entry(swap_mapping, swap_index, old, new); 1641 if (!error) { 1642 mem_cgroup_migrate(old, new); 1643 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1); 1644 __lruvec_stat_mod_folio(new, NR_SHMEM, 1); 1645 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1); 1646 __lruvec_stat_mod_folio(old, NR_SHMEM, -1); 1647 } 1648 xa_unlock_irq(&swap_mapping->i_pages); 1649 1650 if (unlikely(error)) { 1651 /* 1652 * Is this possible? I think not, now that our callers check 1653 * both PageSwapCache and page_private after getting page lock; 1654 * but be defensive. Reverse old to newpage for clear and free. 1655 */ 1656 old = new; 1657 } else { 1658 folio_add_lru(new); 1659 *foliop = new; 1660 } 1661 1662 folio_clear_swapcache(old); 1663 old->private = NULL; 1664 1665 folio_unlock(old); 1666 folio_put_refs(old, 2); 1667 return error; 1668 } 1669 1670 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index, 1671 struct folio *folio, swp_entry_t swap) 1672 { 1673 struct address_space *mapping = inode->i_mapping; 1674 struct shmem_inode_info *info = SHMEM_I(inode); 1675 swp_entry_t swapin_error; 1676 void *old; 1677 1678 swapin_error = make_swapin_error_entry(&folio->page); 1679 old = xa_cmpxchg_irq(&mapping->i_pages, index, 1680 swp_to_radix_entry(swap), 1681 swp_to_radix_entry(swapin_error), 0); 1682 if (old != swp_to_radix_entry(swap)) 1683 return; 1684 1685 folio_wait_writeback(folio); 1686 delete_from_swap_cache(folio); 1687 spin_lock_irq(&info->lock); 1688 /* 1689 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks won't 1690 * be 0 when inode is released and thus trigger WARN_ON(inode->i_blocks) in 1691 * shmem_evict_inode. 1692 */ 1693 info->alloced--; 1694 info->swapped--; 1695 shmem_recalc_inode(inode); 1696 spin_unlock_irq(&info->lock); 1697 swap_free(swap); 1698 } 1699 1700 /* 1701 * Swap in the folio pointed to by *foliop. 1702 * Caller has to make sure that *foliop contains a valid swapped folio. 1703 * Returns 0 and the folio in foliop if success. On failure, returns the 1704 * error code and NULL in *foliop. 1705 */ 1706 static int shmem_swapin_folio(struct inode *inode, pgoff_t index, 1707 struct folio **foliop, enum sgp_type sgp, 1708 gfp_t gfp, struct vm_area_struct *vma, 1709 vm_fault_t *fault_type) 1710 { 1711 struct address_space *mapping = inode->i_mapping; 1712 struct shmem_inode_info *info = SHMEM_I(inode); 1713 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL; 1714 struct folio *folio = NULL; 1715 swp_entry_t swap; 1716 int error; 1717 1718 VM_BUG_ON(!*foliop || !xa_is_value(*foliop)); 1719 swap = radix_to_swp_entry(*foliop); 1720 *foliop = NULL; 1721 1722 if (is_swapin_error_entry(swap)) 1723 return -EIO; 1724 1725 /* Look it up and read it in.. */ 1726 folio = swap_cache_get_folio(swap, NULL, 0); 1727 if (!folio) { 1728 /* Or update major stats only when swapin succeeds?? */ 1729 if (fault_type) { 1730 *fault_type |= VM_FAULT_MAJOR; 1731 count_vm_event(PGMAJFAULT); 1732 count_memcg_event_mm(charge_mm, PGMAJFAULT); 1733 } 1734 /* Here we actually start the io */ 1735 folio = shmem_swapin(swap, gfp, info, index); 1736 if (!folio) { 1737 error = -ENOMEM; 1738 goto failed; 1739 } 1740 } 1741 1742 /* We have to do this with folio locked to prevent races */ 1743 folio_lock(folio); 1744 if (!folio_test_swapcache(folio) || 1745 folio_swap_entry(folio).val != swap.val || 1746 !shmem_confirm_swap(mapping, index, swap)) { 1747 error = -EEXIST; 1748 goto unlock; 1749 } 1750 if (!folio_test_uptodate(folio)) { 1751 error = -EIO; 1752 goto failed; 1753 } 1754 folio_wait_writeback(folio); 1755 1756 /* 1757 * Some architectures may have to restore extra metadata to the 1758 * folio after reading from swap. 1759 */ 1760 arch_swap_restore(swap, folio); 1761 1762 if (shmem_should_replace_folio(folio, gfp)) { 1763 error = shmem_replace_folio(&folio, gfp, info, index); 1764 if (error) 1765 goto failed; 1766 } 1767 1768 error = shmem_add_to_page_cache(folio, mapping, index, 1769 swp_to_radix_entry(swap), gfp, 1770 charge_mm); 1771 if (error) 1772 goto failed; 1773 1774 spin_lock_irq(&info->lock); 1775 info->swapped--; 1776 shmem_recalc_inode(inode); 1777 spin_unlock_irq(&info->lock); 1778 1779 if (sgp == SGP_WRITE) 1780 folio_mark_accessed(folio); 1781 1782 delete_from_swap_cache(folio); 1783 folio_mark_dirty(folio); 1784 swap_free(swap); 1785 1786 *foliop = folio; 1787 return 0; 1788 failed: 1789 if (!shmem_confirm_swap(mapping, index, swap)) 1790 error = -EEXIST; 1791 if (error == -EIO) 1792 shmem_set_folio_swapin_error(inode, index, folio, swap); 1793 unlock: 1794 if (folio) { 1795 folio_unlock(folio); 1796 folio_put(folio); 1797 } 1798 1799 return error; 1800 } 1801 1802 /* 1803 * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate 1804 * 1805 * If we allocate a new one we do not mark it dirty. That's up to the 1806 * vm. If we swap it in we mark it dirty since we also free the swap 1807 * entry since a page cannot live in both the swap and page cache. 1808 * 1809 * vma, vmf, and fault_type are only supplied by shmem_fault: 1810 * otherwise they are NULL. 1811 */ 1812 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, 1813 struct folio **foliop, enum sgp_type sgp, gfp_t gfp, 1814 struct vm_area_struct *vma, struct vm_fault *vmf, 1815 vm_fault_t *fault_type) 1816 { 1817 struct address_space *mapping = inode->i_mapping; 1818 struct shmem_inode_info *info = SHMEM_I(inode); 1819 struct shmem_sb_info *sbinfo; 1820 struct mm_struct *charge_mm; 1821 struct folio *folio; 1822 pgoff_t hindex = index; 1823 gfp_t huge_gfp; 1824 int error; 1825 int once = 0; 1826 int alloced = 0; 1827 1828 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1829 return -EFBIG; 1830 repeat: 1831 if (sgp <= SGP_CACHE && 1832 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1833 return -EINVAL; 1834 } 1835 1836 sbinfo = SHMEM_SB(inode->i_sb); 1837 charge_mm = vma ? vma->vm_mm : NULL; 1838 1839 folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0); 1840 if (folio && vma && userfaultfd_minor(vma)) { 1841 if (!xa_is_value(folio)) { 1842 folio_unlock(folio); 1843 folio_put(folio); 1844 } 1845 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); 1846 return 0; 1847 } 1848 1849 if (xa_is_value(folio)) { 1850 error = shmem_swapin_folio(inode, index, &folio, 1851 sgp, gfp, vma, fault_type); 1852 if (error == -EEXIST) 1853 goto repeat; 1854 1855 *foliop = folio; 1856 return error; 1857 } 1858 1859 if (folio) { 1860 hindex = folio->index; 1861 if (sgp == SGP_WRITE) 1862 folio_mark_accessed(folio); 1863 if (folio_test_uptodate(folio)) 1864 goto out; 1865 /* fallocated folio */ 1866 if (sgp != SGP_READ) 1867 goto clear; 1868 folio_unlock(folio); 1869 folio_put(folio); 1870 } 1871 1872 /* 1873 * SGP_READ: succeed on hole, with NULL folio, letting caller zero. 1874 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail. 1875 */ 1876 *foliop = NULL; 1877 if (sgp == SGP_READ) 1878 return 0; 1879 if (sgp == SGP_NOALLOC) 1880 return -ENOENT; 1881 1882 /* 1883 * Fast cache lookup and swap lookup did not find it: allocate. 1884 */ 1885 1886 if (vma && userfaultfd_missing(vma)) { 1887 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 1888 return 0; 1889 } 1890 1891 if (!shmem_is_huge(vma, inode, index)) 1892 goto alloc_nohuge; 1893 1894 huge_gfp = vma_thp_gfp_mask(vma); 1895 huge_gfp = limit_gfp_mask(huge_gfp, gfp); 1896 folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true); 1897 if (IS_ERR(folio)) { 1898 alloc_nohuge: 1899 folio = shmem_alloc_and_acct_folio(gfp, inode, index, false); 1900 } 1901 if (IS_ERR(folio)) { 1902 int retry = 5; 1903 1904 error = PTR_ERR(folio); 1905 folio = NULL; 1906 if (error != -ENOSPC) 1907 goto unlock; 1908 /* 1909 * Try to reclaim some space by splitting a large folio 1910 * beyond i_size on the filesystem. 1911 */ 1912 while (retry--) { 1913 int ret; 1914 1915 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1916 if (ret == SHRINK_STOP) 1917 break; 1918 if (ret) 1919 goto alloc_nohuge; 1920 } 1921 goto unlock; 1922 } 1923 1924 hindex = round_down(index, folio_nr_pages(folio)); 1925 1926 if (sgp == SGP_WRITE) 1927 __folio_set_referenced(folio); 1928 1929 error = shmem_add_to_page_cache(folio, mapping, hindex, 1930 NULL, gfp & GFP_RECLAIM_MASK, 1931 charge_mm); 1932 if (error) 1933 goto unacct; 1934 folio_add_lru(folio); 1935 1936 spin_lock_irq(&info->lock); 1937 info->alloced += folio_nr_pages(folio); 1938 inode->i_blocks += (blkcnt_t)BLOCKS_PER_PAGE << folio_order(folio); 1939 shmem_recalc_inode(inode); 1940 spin_unlock_irq(&info->lock); 1941 alloced = true; 1942 1943 if (folio_test_pmd_mappable(folio) && 1944 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1945 folio_next_index(folio) - 1) { 1946 /* 1947 * Part of the large folio is beyond i_size: subject 1948 * to shrink under memory pressure. 1949 */ 1950 spin_lock(&sbinfo->shrinklist_lock); 1951 /* 1952 * _careful to defend against unlocked access to 1953 * ->shrink_list in shmem_unused_huge_shrink() 1954 */ 1955 if (list_empty_careful(&info->shrinklist)) { 1956 list_add_tail(&info->shrinklist, 1957 &sbinfo->shrinklist); 1958 sbinfo->shrinklist_len++; 1959 } 1960 spin_unlock(&sbinfo->shrinklist_lock); 1961 } 1962 1963 /* 1964 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio. 1965 */ 1966 if (sgp == SGP_FALLOC) 1967 sgp = SGP_WRITE; 1968 clear: 1969 /* 1970 * Let SGP_WRITE caller clear ends if write does not fill folio; 1971 * but SGP_FALLOC on a folio fallocated earlier must initialize 1972 * it now, lest undo on failure cancel our earlier guarantee. 1973 */ 1974 if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) { 1975 long i, n = folio_nr_pages(folio); 1976 1977 for (i = 0; i < n; i++) 1978 clear_highpage(folio_page(folio, i)); 1979 flush_dcache_folio(folio); 1980 folio_mark_uptodate(folio); 1981 } 1982 1983 /* Perhaps the file has been truncated since we checked */ 1984 if (sgp <= SGP_CACHE && 1985 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1986 if (alloced) { 1987 folio_clear_dirty(folio); 1988 filemap_remove_folio(folio); 1989 spin_lock_irq(&info->lock); 1990 shmem_recalc_inode(inode); 1991 spin_unlock_irq(&info->lock); 1992 } 1993 error = -EINVAL; 1994 goto unlock; 1995 } 1996 out: 1997 *foliop = folio; 1998 return 0; 1999 2000 /* 2001 * Error recovery. 2002 */ 2003 unacct: 2004 shmem_inode_unacct_blocks(inode, folio_nr_pages(folio)); 2005 2006 if (folio_test_large(folio)) { 2007 folio_unlock(folio); 2008 folio_put(folio); 2009 goto alloc_nohuge; 2010 } 2011 unlock: 2012 if (folio) { 2013 folio_unlock(folio); 2014 folio_put(folio); 2015 } 2016 if (error == -ENOSPC && !once++) { 2017 spin_lock_irq(&info->lock); 2018 shmem_recalc_inode(inode); 2019 spin_unlock_irq(&info->lock); 2020 goto repeat; 2021 } 2022 if (error == -EEXIST) 2023 goto repeat; 2024 return error; 2025 } 2026 2027 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop, 2028 enum sgp_type sgp) 2029 { 2030 return shmem_get_folio_gfp(inode, index, foliop, sgp, 2031 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); 2032 } 2033 2034 /* 2035 * This is like autoremove_wake_function, but it removes the wait queue 2036 * entry unconditionally - even if something else had already woken the 2037 * target. 2038 */ 2039 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 2040 { 2041 int ret = default_wake_function(wait, mode, sync, key); 2042 list_del_init(&wait->entry); 2043 return ret; 2044 } 2045 2046 static vm_fault_t shmem_fault(struct vm_fault *vmf) 2047 { 2048 struct vm_area_struct *vma = vmf->vma; 2049 struct inode *inode = file_inode(vma->vm_file); 2050 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 2051 struct folio *folio = NULL; 2052 int err; 2053 vm_fault_t ret = VM_FAULT_LOCKED; 2054 2055 /* 2056 * Trinity finds that probing a hole which tmpfs is punching can 2057 * prevent the hole-punch from ever completing: which in turn 2058 * locks writers out with its hold on i_rwsem. So refrain from 2059 * faulting pages into the hole while it's being punched. Although 2060 * shmem_undo_range() does remove the additions, it may be unable to 2061 * keep up, as each new page needs its own unmap_mapping_range() call, 2062 * and the i_mmap tree grows ever slower to scan if new vmas are added. 2063 * 2064 * It does not matter if we sometimes reach this check just before the 2065 * hole-punch begins, so that one fault then races with the punch: 2066 * we just need to make racing faults a rare case. 2067 * 2068 * The implementation below would be much simpler if we just used a 2069 * standard mutex or completion: but we cannot take i_rwsem in fault, 2070 * and bloating every shmem inode for this unlikely case would be sad. 2071 */ 2072 if (unlikely(inode->i_private)) { 2073 struct shmem_falloc *shmem_falloc; 2074 2075 spin_lock(&inode->i_lock); 2076 shmem_falloc = inode->i_private; 2077 if (shmem_falloc && 2078 shmem_falloc->waitq && 2079 vmf->pgoff >= shmem_falloc->start && 2080 vmf->pgoff < shmem_falloc->next) { 2081 struct file *fpin; 2082 wait_queue_head_t *shmem_falloc_waitq; 2083 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 2084 2085 ret = VM_FAULT_NOPAGE; 2086 fpin = maybe_unlock_mmap_for_io(vmf, NULL); 2087 if (fpin) 2088 ret = VM_FAULT_RETRY; 2089 2090 shmem_falloc_waitq = shmem_falloc->waitq; 2091 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 2092 TASK_UNINTERRUPTIBLE); 2093 spin_unlock(&inode->i_lock); 2094 schedule(); 2095 2096 /* 2097 * shmem_falloc_waitq points into the shmem_fallocate() 2098 * stack of the hole-punching task: shmem_falloc_waitq 2099 * is usually invalid by the time we reach here, but 2100 * finish_wait() does not dereference it in that case; 2101 * though i_lock needed lest racing with wake_up_all(). 2102 */ 2103 spin_lock(&inode->i_lock); 2104 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 2105 spin_unlock(&inode->i_lock); 2106 2107 if (fpin) 2108 fput(fpin); 2109 return ret; 2110 } 2111 spin_unlock(&inode->i_lock); 2112 } 2113 2114 err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE, 2115 gfp, vma, vmf, &ret); 2116 if (err) 2117 return vmf_error(err); 2118 if (folio) 2119 vmf->page = folio_file_page(folio, vmf->pgoff); 2120 return ret; 2121 } 2122 2123 unsigned long shmem_get_unmapped_area(struct file *file, 2124 unsigned long uaddr, unsigned long len, 2125 unsigned long pgoff, unsigned long flags) 2126 { 2127 unsigned long (*get_area)(struct file *, 2128 unsigned long, unsigned long, unsigned long, unsigned long); 2129 unsigned long addr; 2130 unsigned long offset; 2131 unsigned long inflated_len; 2132 unsigned long inflated_addr; 2133 unsigned long inflated_offset; 2134 2135 if (len > TASK_SIZE) 2136 return -ENOMEM; 2137 2138 get_area = current->mm->get_unmapped_area; 2139 addr = get_area(file, uaddr, len, pgoff, flags); 2140 2141 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 2142 return addr; 2143 if (IS_ERR_VALUE(addr)) 2144 return addr; 2145 if (addr & ~PAGE_MASK) 2146 return addr; 2147 if (addr > TASK_SIZE - len) 2148 return addr; 2149 2150 if (shmem_huge == SHMEM_HUGE_DENY) 2151 return addr; 2152 if (len < HPAGE_PMD_SIZE) 2153 return addr; 2154 if (flags & MAP_FIXED) 2155 return addr; 2156 /* 2157 * Our priority is to support MAP_SHARED mapped hugely; 2158 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2159 * But if caller specified an address hint and we allocated area there 2160 * successfully, respect that as before. 2161 */ 2162 if (uaddr == addr) 2163 return addr; 2164 2165 if (shmem_huge != SHMEM_HUGE_FORCE) { 2166 struct super_block *sb; 2167 2168 if (file) { 2169 VM_BUG_ON(file->f_op != &shmem_file_operations); 2170 sb = file_inode(file)->i_sb; 2171 } else { 2172 /* 2173 * Called directly from mm/mmap.c, or drivers/char/mem.c 2174 * for "/dev/zero", to create a shared anonymous object. 2175 */ 2176 if (IS_ERR(shm_mnt)) 2177 return addr; 2178 sb = shm_mnt->mnt_sb; 2179 } 2180 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2181 return addr; 2182 } 2183 2184 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2185 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2186 return addr; 2187 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2188 return addr; 2189 2190 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2191 if (inflated_len > TASK_SIZE) 2192 return addr; 2193 if (inflated_len < len) 2194 return addr; 2195 2196 inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags); 2197 if (IS_ERR_VALUE(inflated_addr)) 2198 return addr; 2199 if (inflated_addr & ~PAGE_MASK) 2200 return addr; 2201 2202 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2203 inflated_addr += offset - inflated_offset; 2204 if (inflated_offset > offset) 2205 inflated_addr += HPAGE_PMD_SIZE; 2206 2207 if (inflated_addr > TASK_SIZE - len) 2208 return addr; 2209 return inflated_addr; 2210 } 2211 2212 #ifdef CONFIG_NUMA 2213 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2214 { 2215 struct inode *inode = file_inode(vma->vm_file); 2216 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2217 } 2218 2219 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2220 unsigned long addr) 2221 { 2222 struct inode *inode = file_inode(vma->vm_file); 2223 pgoff_t index; 2224 2225 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2226 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2227 } 2228 #endif 2229 2230 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 2231 { 2232 struct inode *inode = file_inode(file); 2233 struct shmem_inode_info *info = SHMEM_I(inode); 2234 int retval = -ENOMEM; 2235 2236 /* 2237 * What serializes the accesses to info->flags? 2238 * ipc_lock_object() when called from shmctl_do_lock(), 2239 * no serialization needed when called from shm_destroy(). 2240 */ 2241 if (lock && !(info->flags & VM_LOCKED)) { 2242 if (!user_shm_lock(inode->i_size, ucounts)) 2243 goto out_nomem; 2244 info->flags |= VM_LOCKED; 2245 mapping_set_unevictable(file->f_mapping); 2246 } 2247 if (!lock && (info->flags & VM_LOCKED) && ucounts) { 2248 user_shm_unlock(inode->i_size, ucounts); 2249 info->flags &= ~VM_LOCKED; 2250 mapping_clear_unevictable(file->f_mapping); 2251 } 2252 retval = 0; 2253 2254 out_nomem: 2255 return retval; 2256 } 2257 2258 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2259 { 2260 struct shmem_inode_info *info = SHMEM_I(file_inode(file)); 2261 int ret; 2262 2263 ret = seal_check_future_write(info->seals, vma); 2264 if (ret) 2265 return ret; 2266 2267 /* arm64 - allow memory tagging on RAM-based files */ 2268 vma->vm_flags |= VM_MTE_ALLOWED; 2269 2270 file_accessed(file); 2271 vma->vm_ops = &shmem_vm_ops; 2272 return 0; 2273 } 2274 2275 #ifdef CONFIG_TMPFS_XATTR 2276 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2277 2278 /* 2279 * chattr's fsflags are unrelated to extended attributes, 2280 * but tmpfs has chosen to enable them under the same config option. 2281 */ 2282 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) 2283 { 2284 unsigned int i_flags = 0; 2285 2286 if (fsflags & FS_NOATIME_FL) 2287 i_flags |= S_NOATIME; 2288 if (fsflags & FS_APPEND_FL) 2289 i_flags |= S_APPEND; 2290 if (fsflags & FS_IMMUTABLE_FL) 2291 i_flags |= S_IMMUTABLE; 2292 /* 2293 * But FS_NODUMP_FL does not require any action in i_flags. 2294 */ 2295 inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE); 2296 } 2297 #else 2298 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) 2299 { 2300 } 2301 #define shmem_initxattrs NULL 2302 #endif 2303 2304 static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir, 2305 umode_t mode, dev_t dev, unsigned long flags) 2306 { 2307 struct inode *inode; 2308 struct shmem_inode_info *info; 2309 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2310 ino_t ino; 2311 2312 if (shmem_reserve_inode(sb, &ino)) 2313 return NULL; 2314 2315 inode = new_inode(sb); 2316 if (inode) { 2317 inode->i_ino = ino; 2318 inode_init_owner(&init_user_ns, inode, dir, mode); 2319 inode->i_blocks = 0; 2320 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2321 inode->i_generation = prandom_u32(); 2322 info = SHMEM_I(inode); 2323 memset(info, 0, (char *)inode - (char *)info); 2324 spin_lock_init(&info->lock); 2325 atomic_set(&info->stop_eviction, 0); 2326 info->seals = F_SEAL_SEAL; 2327 info->flags = flags & VM_NORESERVE; 2328 info->i_crtime = inode->i_mtime; 2329 info->fsflags = (dir == NULL) ? 0 : 2330 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED; 2331 if (info->fsflags) 2332 shmem_set_inode_flags(inode, info->fsflags); 2333 INIT_LIST_HEAD(&info->shrinklist); 2334 INIT_LIST_HEAD(&info->swaplist); 2335 simple_xattrs_init(&info->xattrs); 2336 cache_no_acl(inode); 2337 mapping_set_large_folios(inode->i_mapping); 2338 2339 switch (mode & S_IFMT) { 2340 default: 2341 inode->i_op = &shmem_special_inode_operations; 2342 init_special_inode(inode, mode, dev); 2343 break; 2344 case S_IFREG: 2345 inode->i_mapping->a_ops = &shmem_aops; 2346 inode->i_op = &shmem_inode_operations; 2347 inode->i_fop = &shmem_file_operations; 2348 mpol_shared_policy_init(&info->policy, 2349 shmem_get_sbmpol(sbinfo)); 2350 break; 2351 case S_IFDIR: 2352 inc_nlink(inode); 2353 /* Some things misbehave if size == 0 on a directory */ 2354 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2355 inode->i_op = &shmem_dir_inode_operations; 2356 inode->i_fop = &simple_dir_operations; 2357 break; 2358 case S_IFLNK: 2359 /* 2360 * Must not load anything in the rbtree, 2361 * mpol_free_shared_policy will not be called. 2362 */ 2363 mpol_shared_policy_init(&info->policy, NULL); 2364 break; 2365 } 2366 2367 lockdep_annotate_inode_mutex_key(inode); 2368 } else 2369 shmem_free_inode(sb); 2370 return inode; 2371 } 2372 2373 #ifdef CONFIG_USERFAULTFD 2374 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, 2375 pmd_t *dst_pmd, 2376 struct vm_area_struct *dst_vma, 2377 unsigned long dst_addr, 2378 unsigned long src_addr, 2379 bool zeropage, bool wp_copy, 2380 struct page **pagep) 2381 { 2382 struct inode *inode = file_inode(dst_vma->vm_file); 2383 struct shmem_inode_info *info = SHMEM_I(inode); 2384 struct address_space *mapping = inode->i_mapping; 2385 gfp_t gfp = mapping_gfp_mask(mapping); 2386 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2387 void *page_kaddr; 2388 struct folio *folio; 2389 int ret; 2390 pgoff_t max_off; 2391 2392 if (!shmem_inode_acct_block(inode, 1)) { 2393 /* 2394 * We may have got a page, returned -ENOENT triggering a retry, 2395 * and now we find ourselves with -ENOMEM. Release the page, to 2396 * avoid a BUG_ON in our caller. 2397 */ 2398 if (unlikely(*pagep)) { 2399 put_page(*pagep); 2400 *pagep = NULL; 2401 } 2402 return -ENOMEM; 2403 } 2404 2405 if (!*pagep) { 2406 ret = -ENOMEM; 2407 folio = shmem_alloc_folio(gfp, info, pgoff); 2408 if (!folio) 2409 goto out_unacct_blocks; 2410 2411 if (!zeropage) { /* COPY */ 2412 page_kaddr = kmap_local_folio(folio, 0); 2413 ret = copy_from_user(page_kaddr, 2414 (const void __user *)src_addr, 2415 PAGE_SIZE); 2416 kunmap_local(page_kaddr); 2417 2418 /* fallback to copy_from_user outside mmap_lock */ 2419 if (unlikely(ret)) { 2420 *pagep = &folio->page; 2421 ret = -ENOENT; 2422 /* don't free the page */ 2423 goto out_unacct_blocks; 2424 } 2425 2426 flush_dcache_folio(folio); 2427 } else { /* ZEROPAGE */ 2428 clear_user_highpage(&folio->page, dst_addr); 2429 } 2430 } else { 2431 folio = page_folio(*pagep); 2432 VM_BUG_ON_FOLIO(folio_test_large(folio), folio); 2433 *pagep = NULL; 2434 } 2435 2436 VM_BUG_ON(folio_test_locked(folio)); 2437 VM_BUG_ON(folio_test_swapbacked(folio)); 2438 __folio_set_locked(folio); 2439 __folio_set_swapbacked(folio); 2440 __folio_mark_uptodate(folio); 2441 2442 ret = -EFAULT; 2443 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2444 if (unlikely(pgoff >= max_off)) 2445 goto out_release; 2446 2447 ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, 2448 gfp & GFP_RECLAIM_MASK, dst_mm); 2449 if (ret) 2450 goto out_release; 2451 2452 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, 2453 &folio->page, true, wp_copy); 2454 if (ret) 2455 goto out_delete_from_cache; 2456 2457 spin_lock_irq(&info->lock); 2458 info->alloced++; 2459 inode->i_blocks += BLOCKS_PER_PAGE; 2460 shmem_recalc_inode(inode); 2461 spin_unlock_irq(&info->lock); 2462 2463 folio_unlock(folio); 2464 return 0; 2465 out_delete_from_cache: 2466 filemap_remove_folio(folio); 2467 out_release: 2468 folio_unlock(folio); 2469 folio_put(folio); 2470 out_unacct_blocks: 2471 shmem_inode_unacct_blocks(inode, 1); 2472 return ret; 2473 } 2474 #endif /* CONFIG_USERFAULTFD */ 2475 2476 #ifdef CONFIG_TMPFS 2477 static const struct inode_operations shmem_symlink_inode_operations; 2478 static const struct inode_operations shmem_short_symlink_operations; 2479 2480 static int 2481 shmem_write_begin(struct file *file, struct address_space *mapping, 2482 loff_t pos, unsigned len, 2483 struct page **pagep, void **fsdata) 2484 { 2485 struct inode *inode = mapping->host; 2486 struct shmem_inode_info *info = SHMEM_I(inode); 2487 pgoff_t index = pos >> PAGE_SHIFT; 2488 struct folio *folio; 2489 int ret = 0; 2490 2491 /* i_rwsem is held by caller */ 2492 if (unlikely(info->seals & (F_SEAL_GROW | 2493 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) { 2494 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) 2495 return -EPERM; 2496 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2497 return -EPERM; 2498 } 2499 2500 ret = shmem_get_folio(inode, index, &folio, SGP_WRITE); 2501 2502 if (ret) 2503 return ret; 2504 2505 *pagep = folio_file_page(folio, index); 2506 if (PageHWPoison(*pagep)) { 2507 folio_unlock(folio); 2508 folio_put(folio); 2509 *pagep = NULL; 2510 return -EIO; 2511 } 2512 2513 return 0; 2514 } 2515 2516 static int 2517 shmem_write_end(struct file *file, struct address_space *mapping, 2518 loff_t pos, unsigned len, unsigned copied, 2519 struct page *page, void *fsdata) 2520 { 2521 struct inode *inode = mapping->host; 2522 2523 if (pos + copied > inode->i_size) 2524 i_size_write(inode, pos + copied); 2525 2526 if (!PageUptodate(page)) { 2527 struct page *head = compound_head(page); 2528 if (PageTransCompound(page)) { 2529 int i; 2530 2531 for (i = 0; i < HPAGE_PMD_NR; i++) { 2532 if (head + i == page) 2533 continue; 2534 clear_highpage(head + i); 2535 flush_dcache_page(head + i); 2536 } 2537 } 2538 if (copied < PAGE_SIZE) { 2539 unsigned from = pos & (PAGE_SIZE - 1); 2540 zero_user_segments(page, 0, from, 2541 from + copied, PAGE_SIZE); 2542 } 2543 SetPageUptodate(head); 2544 } 2545 set_page_dirty(page); 2546 unlock_page(page); 2547 put_page(page); 2548 2549 return copied; 2550 } 2551 2552 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2553 { 2554 struct file *file = iocb->ki_filp; 2555 struct inode *inode = file_inode(file); 2556 struct address_space *mapping = inode->i_mapping; 2557 pgoff_t index; 2558 unsigned long offset; 2559 int error = 0; 2560 ssize_t retval = 0; 2561 loff_t *ppos = &iocb->ki_pos; 2562 2563 index = *ppos >> PAGE_SHIFT; 2564 offset = *ppos & ~PAGE_MASK; 2565 2566 for (;;) { 2567 struct folio *folio = NULL; 2568 struct page *page = NULL; 2569 pgoff_t end_index; 2570 unsigned long nr, ret; 2571 loff_t i_size = i_size_read(inode); 2572 2573 end_index = i_size >> PAGE_SHIFT; 2574 if (index > end_index) 2575 break; 2576 if (index == end_index) { 2577 nr = i_size & ~PAGE_MASK; 2578 if (nr <= offset) 2579 break; 2580 } 2581 2582 error = shmem_get_folio(inode, index, &folio, SGP_READ); 2583 if (error) { 2584 if (error == -EINVAL) 2585 error = 0; 2586 break; 2587 } 2588 if (folio) { 2589 folio_unlock(folio); 2590 2591 page = folio_file_page(folio, index); 2592 if (PageHWPoison(page)) { 2593 folio_put(folio); 2594 error = -EIO; 2595 break; 2596 } 2597 } 2598 2599 /* 2600 * We must evaluate after, since reads (unlike writes) 2601 * are called without i_rwsem protection against truncate 2602 */ 2603 nr = PAGE_SIZE; 2604 i_size = i_size_read(inode); 2605 end_index = i_size >> PAGE_SHIFT; 2606 if (index == end_index) { 2607 nr = i_size & ~PAGE_MASK; 2608 if (nr <= offset) { 2609 if (folio) 2610 folio_put(folio); 2611 break; 2612 } 2613 } 2614 nr -= offset; 2615 2616 if (folio) { 2617 /* 2618 * If users can be writing to this page using arbitrary 2619 * virtual addresses, take care about potential aliasing 2620 * before reading the page on the kernel side. 2621 */ 2622 if (mapping_writably_mapped(mapping)) 2623 flush_dcache_page(page); 2624 /* 2625 * Mark the page accessed if we read the beginning. 2626 */ 2627 if (!offset) 2628 folio_mark_accessed(folio); 2629 /* 2630 * Ok, we have the page, and it's up-to-date, so 2631 * now we can copy it to user space... 2632 */ 2633 ret = copy_page_to_iter(page, offset, nr, to); 2634 folio_put(folio); 2635 2636 } else if (user_backed_iter(to)) { 2637 /* 2638 * Copy to user tends to be so well optimized, but 2639 * clear_user() not so much, that it is noticeably 2640 * faster to copy the zero page instead of clearing. 2641 */ 2642 ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to); 2643 } else { 2644 /* 2645 * But submitting the same page twice in a row to 2646 * splice() - or others? - can result in confusion: 2647 * so don't attempt that optimization on pipes etc. 2648 */ 2649 ret = iov_iter_zero(nr, to); 2650 } 2651 2652 retval += ret; 2653 offset += ret; 2654 index += offset >> PAGE_SHIFT; 2655 offset &= ~PAGE_MASK; 2656 2657 if (!iov_iter_count(to)) 2658 break; 2659 if (ret < nr) { 2660 error = -EFAULT; 2661 break; 2662 } 2663 cond_resched(); 2664 } 2665 2666 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2667 file_accessed(file); 2668 return retval ? retval : error; 2669 } 2670 2671 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2672 { 2673 struct address_space *mapping = file->f_mapping; 2674 struct inode *inode = mapping->host; 2675 2676 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2677 return generic_file_llseek_size(file, offset, whence, 2678 MAX_LFS_FILESIZE, i_size_read(inode)); 2679 if (offset < 0) 2680 return -ENXIO; 2681 2682 inode_lock(inode); 2683 /* We're holding i_rwsem so we can access i_size directly */ 2684 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence); 2685 if (offset >= 0) 2686 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2687 inode_unlock(inode); 2688 return offset; 2689 } 2690 2691 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2692 loff_t len) 2693 { 2694 struct inode *inode = file_inode(file); 2695 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2696 struct shmem_inode_info *info = SHMEM_I(inode); 2697 struct shmem_falloc shmem_falloc; 2698 pgoff_t start, index, end, undo_fallocend; 2699 int error; 2700 2701 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2702 return -EOPNOTSUPP; 2703 2704 inode_lock(inode); 2705 2706 if (mode & FALLOC_FL_PUNCH_HOLE) { 2707 struct address_space *mapping = file->f_mapping; 2708 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2709 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2710 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2711 2712 /* protected by i_rwsem */ 2713 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { 2714 error = -EPERM; 2715 goto out; 2716 } 2717 2718 shmem_falloc.waitq = &shmem_falloc_waitq; 2719 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; 2720 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2721 spin_lock(&inode->i_lock); 2722 inode->i_private = &shmem_falloc; 2723 spin_unlock(&inode->i_lock); 2724 2725 if ((u64)unmap_end > (u64)unmap_start) 2726 unmap_mapping_range(mapping, unmap_start, 2727 1 + unmap_end - unmap_start, 0); 2728 shmem_truncate_range(inode, offset, offset + len - 1); 2729 /* No need to unmap again: hole-punching leaves COWed pages */ 2730 2731 spin_lock(&inode->i_lock); 2732 inode->i_private = NULL; 2733 wake_up_all(&shmem_falloc_waitq); 2734 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 2735 spin_unlock(&inode->i_lock); 2736 error = 0; 2737 goto out; 2738 } 2739 2740 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2741 error = inode_newsize_ok(inode, offset + len); 2742 if (error) 2743 goto out; 2744 2745 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2746 error = -EPERM; 2747 goto out; 2748 } 2749 2750 start = offset >> PAGE_SHIFT; 2751 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2752 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2753 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2754 error = -ENOSPC; 2755 goto out; 2756 } 2757 2758 shmem_falloc.waitq = NULL; 2759 shmem_falloc.start = start; 2760 shmem_falloc.next = start; 2761 shmem_falloc.nr_falloced = 0; 2762 shmem_falloc.nr_unswapped = 0; 2763 spin_lock(&inode->i_lock); 2764 inode->i_private = &shmem_falloc; 2765 spin_unlock(&inode->i_lock); 2766 2767 /* 2768 * info->fallocend is only relevant when huge pages might be 2769 * involved: to prevent split_huge_page() freeing fallocated 2770 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size. 2771 */ 2772 undo_fallocend = info->fallocend; 2773 if (info->fallocend < end) 2774 info->fallocend = end; 2775 2776 for (index = start; index < end; ) { 2777 struct folio *folio; 2778 2779 /* 2780 * Good, the fallocate(2) manpage permits EINTR: we may have 2781 * been interrupted because we are using up too much memory. 2782 */ 2783 if (signal_pending(current)) 2784 error = -EINTR; 2785 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2786 error = -ENOMEM; 2787 else 2788 error = shmem_get_folio(inode, index, &folio, 2789 SGP_FALLOC); 2790 if (error) { 2791 info->fallocend = undo_fallocend; 2792 /* Remove the !uptodate folios we added */ 2793 if (index > start) { 2794 shmem_undo_range(inode, 2795 (loff_t)start << PAGE_SHIFT, 2796 ((loff_t)index << PAGE_SHIFT) - 1, true); 2797 } 2798 goto undone; 2799 } 2800 2801 /* 2802 * Here is a more important optimization than it appears: 2803 * a second SGP_FALLOC on the same large folio will clear it, 2804 * making it uptodate and un-undoable if we fail later. 2805 */ 2806 index = folio_next_index(folio); 2807 /* Beware 32-bit wraparound */ 2808 if (!index) 2809 index--; 2810 2811 /* 2812 * Inform shmem_writepage() how far we have reached. 2813 * No need for lock or barrier: we have the page lock. 2814 */ 2815 if (!folio_test_uptodate(folio)) 2816 shmem_falloc.nr_falloced += index - shmem_falloc.next; 2817 shmem_falloc.next = index; 2818 2819 /* 2820 * If !uptodate, leave it that way so that freeable folios 2821 * can be recognized if we need to rollback on error later. 2822 * But mark it dirty so that memory pressure will swap rather 2823 * than free the folios we are allocating (and SGP_CACHE folios 2824 * might still be clean: we now need to mark those dirty too). 2825 */ 2826 folio_mark_dirty(folio); 2827 folio_unlock(folio); 2828 folio_put(folio); 2829 cond_resched(); 2830 } 2831 2832 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2833 i_size_write(inode, offset + len); 2834 undone: 2835 spin_lock(&inode->i_lock); 2836 inode->i_private = NULL; 2837 spin_unlock(&inode->i_lock); 2838 out: 2839 if (!error) 2840 file_modified(file); 2841 inode_unlock(inode); 2842 return error; 2843 } 2844 2845 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2846 { 2847 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2848 2849 buf->f_type = TMPFS_MAGIC; 2850 buf->f_bsize = PAGE_SIZE; 2851 buf->f_namelen = NAME_MAX; 2852 if (sbinfo->max_blocks) { 2853 buf->f_blocks = sbinfo->max_blocks; 2854 buf->f_bavail = 2855 buf->f_bfree = sbinfo->max_blocks - 2856 percpu_counter_sum(&sbinfo->used_blocks); 2857 } 2858 if (sbinfo->max_inodes) { 2859 buf->f_files = sbinfo->max_inodes; 2860 buf->f_ffree = sbinfo->free_inodes; 2861 } 2862 /* else leave those fields 0 like simple_statfs */ 2863 2864 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); 2865 2866 return 0; 2867 } 2868 2869 /* 2870 * File creation. Allocate an inode, and we're done.. 2871 */ 2872 static int 2873 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir, 2874 struct dentry *dentry, umode_t mode, dev_t dev) 2875 { 2876 struct inode *inode; 2877 int error = -ENOSPC; 2878 2879 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2880 if (inode) { 2881 error = simple_acl_create(dir, inode); 2882 if (error) 2883 goto out_iput; 2884 error = security_inode_init_security(inode, dir, 2885 &dentry->d_name, 2886 shmem_initxattrs, NULL); 2887 if (error && error != -EOPNOTSUPP) 2888 goto out_iput; 2889 2890 error = 0; 2891 dir->i_size += BOGO_DIRENT_SIZE; 2892 dir->i_ctime = dir->i_mtime = current_time(dir); 2893 d_instantiate(dentry, inode); 2894 dget(dentry); /* Extra count - pin the dentry in core */ 2895 } 2896 return error; 2897 out_iput: 2898 iput(inode); 2899 return error; 2900 } 2901 2902 static int 2903 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, 2904 struct dentry *dentry, umode_t mode) 2905 { 2906 struct inode *inode; 2907 int error = -ENOSPC; 2908 2909 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2910 if (inode) { 2911 error = security_inode_init_security(inode, dir, 2912 NULL, 2913 shmem_initxattrs, NULL); 2914 if (error && error != -EOPNOTSUPP) 2915 goto out_iput; 2916 error = simple_acl_create(dir, inode); 2917 if (error) 2918 goto out_iput; 2919 d_tmpfile(dentry, inode); 2920 } 2921 return error; 2922 out_iput: 2923 iput(inode); 2924 return error; 2925 } 2926 2927 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir, 2928 struct dentry *dentry, umode_t mode) 2929 { 2930 int error; 2931 2932 if ((error = shmem_mknod(&init_user_ns, dir, dentry, 2933 mode | S_IFDIR, 0))) 2934 return error; 2935 inc_nlink(dir); 2936 return 0; 2937 } 2938 2939 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir, 2940 struct dentry *dentry, umode_t mode, bool excl) 2941 { 2942 return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); 2943 } 2944 2945 /* 2946 * Link a file.. 2947 */ 2948 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2949 { 2950 struct inode *inode = d_inode(old_dentry); 2951 int ret = 0; 2952 2953 /* 2954 * No ordinary (disk based) filesystem counts links as inodes; 2955 * but each new link needs a new dentry, pinning lowmem, and 2956 * tmpfs dentries cannot be pruned until they are unlinked. 2957 * But if an O_TMPFILE file is linked into the tmpfs, the 2958 * first link must skip that, to get the accounting right. 2959 */ 2960 if (inode->i_nlink) { 2961 ret = shmem_reserve_inode(inode->i_sb, NULL); 2962 if (ret) 2963 goto out; 2964 } 2965 2966 dir->i_size += BOGO_DIRENT_SIZE; 2967 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2968 inc_nlink(inode); 2969 ihold(inode); /* New dentry reference */ 2970 dget(dentry); /* Extra pinning count for the created dentry */ 2971 d_instantiate(dentry, inode); 2972 out: 2973 return ret; 2974 } 2975 2976 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2977 { 2978 struct inode *inode = d_inode(dentry); 2979 2980 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2981 shmem_free_inode(inode->i_sb); 2982 2983 dir->i_size -= BOGO_DIRENT_SIZE; 2984 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2985 drop_nlink(inode); 2986 dput(dentry); /* Undo the count from "create" - this does all the work */ 2987 return 0; 2988 } 2989 2990 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2991 { 2992 if (!simple_empty(dentry)) 2993 return -ENOTEMPTY; 2994 2995 drop_nlink(d_inode(dentry)); 2996 drop_nlink(dir); 2997 return shmem_unlink(dir, dentry); 2998 } 2999 3000 static int shmem_whiteout(struct user_namespace *mnt_userns, 3001 struct inode *old_dir, struct dentry *old_dentry) 3002 { 3003 struct dentry *whiteout; 3004 int error; 3005 3006 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 3007 if (!whiteout) 3008 return -ENOMEM; 3009 3010 error = shmem_mknod(&init_user_ns, old_dir, whiteout, 3011 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 3012 dput(whiteout); 3013 if (error) 3014 return error; 3015 3016 /* 3017 * Cheat and hash the whiteout while the old dentry is still in 3018 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 3019 * 3020 * d_lookup() will consistently find one of them at this point, 3021 * not sure which one, but that isn't even important. 3022 */ 3023 d_rehash(whiteout); 3024 return 0; 3025 } 3026 3027 /* 3028 * The VFS layer already does all the dentry stuff for rename, 3029 * we just have to decrement the usage count for the target if 3030 * it exists so that the VFS layer correctly free's it when it 3031 * gets overwritten. 3032 */ 3033 static int shmem_rename2(struct user_namespace *mnt_userns, 3034 struct inode *old_dir, struct dentry *old_dentry, 3035 struct inode *new_dir, struct dentry *new_dentry, 3036 unsigned int flags) 3037 { 3038 struct inode *inode = d_inode(old_dentry); 3039 int they_are_dirs = S_ISDIR(inode->i_mode); 3040 3041 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3042 return -EINVAL; 3043 3044 if (flags & RENAME_EXCHANGE) 3045 return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); 3046 3047 if (!simple_empty(new_dentry)) 3048 return -ENOTEMPTY; 3049 3050 if (flags & RENAME_WHITEOUT) { 3051 int error; 3052 3053 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry); 3054 if (error) 3055 return error; 3056 } 3057 3058 if (d_really_is_positive(new_dentry)) { 3059 (void) shmem_unlink(new_dir, new_dentry); 3060 if (they_are_dirs) { 3061 drop_nlink(d_inode(new_dentry)); 3062 drop_nlink(old_dir); 3063 } 3064 } else if (they_are_dirs) { 3065 drop_nlink(old_dir); 3066 inc_nlink(new_dir); 3067 } 3068 3069 old_dir->i_size -= BOGO_DIRENT_SIZE; 3070 new_dir->i_size += BOGO_DIRENT_SIZE; 3071 old_dir->i_ctime = old_dir->i_mtime = 3072 new_dir->i_ctime = new_dir->i_mtime = 3073 inode->i_ctime = current_time(old_dir); 3074 return 0; 3075 } 3076 3077 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, 3078 struct dentry *dentry, const char *symname) 3079 { 3080 int error; 3081 int len; 3082 struct inode *inode; 3083 struct folio *folio; 3084 3085 len = strlen(symname) + 1; 3086 if (len > PAGE_SIZE) 3087 return -ENAMETOOLONG; 3088 3089 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0, 3090 VM_NORESERVE); 3091 if (!inode) 3092 return -ENOSPC; 3093 3094 error = security_inode_init_security(inode, dir, &dentry->d_name, 3095 shmem_initxattrs, NULL); 3096 if (error && error != -EOPNOTSUPP) { 3097 iput(inode); 3098 return error; 3099 } 3100 3101 inode->i_size = len-1; 3102 if (len <= SHORT_SYMLINK_LEN) { 3103 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3104 if (!inode->i_link) { 3105 iput(inode); 3106 return -ENOMEM; 3107 } 3108 inode->i_op = &shmem_short_symlink_operations; 3109 } else { 3110 inode_nohighmem(inode); 3111 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE); 3112 if (error) { 3113 iput(inode); 3114 return error; 3115 } 3116 inode->i_mapping->a_ops = &shmem_aops; 3117 inode->i_op = &shmem_symlink_inode_operations; 3118 memcpy(folio_address(folio), symname, len); 3119 folio_mark_uptodate(folio); 3120 folio_mark_dirty(folio); 3121 folio_unlock(folio); 3122 folio_put(folio); 3123 } 3124 dir->i_size += BOGO_DIRENT_SIZE; 3125 dir->i_ctime = dir->i_mtime = current_time(dir); 3126 d_instantiate(dentry, inode); 3127 dget(dentry); 3128 return 0; 3129 } 3130 3131 static void shmem_put_link(void *arg) 3132 { 3133 folio_mark_accessed(arg); 3134 folio_put(arg); 3135 } 3136 3137 static const char *shmem_get_link(struct dentry *dentry, 3138 struct inode *inode, 3139 struct delayed_call *done) 3140 { 3141 struct folio *folio = NULL; 3142 int error; 3143 3144 if (!dentry) { 3145 folio = filemap_get_folio(inode->i_mapping, 0); 3146 if (!folio) 3147 return ERR_PTR(-ECHILD); 3148 if (PageHWPoison(folio_page(folio, 0)) || 3149 !folio_test_uptodate(folio)) { 3150 folio_put(folio); 3151 return ERR_PTR(-ECHILD); 3152 } 3153 } else { 3154 error = shmem_get_folio(inode, 0, &folio, SGP_READ); 3155 if (error) 3156 return ERR_PTR(error); 3157 if (!folio) 3158 return ERR_PTR(-ECHILD); 3159 if (PageHWPoison(folio_page(folio, 0))) { 3160 folio_unlock(folio); 3161 folio_put(folio); 3162 return ERR_PTR(-ECHILD); 3163 } 3164 folio_unlock(folio); 3165 } 3166 set_delayed_call(done, shmem_put_link, folio); 3167 return folio_address(folio); 3168 } 3169 3170 #ifdef CONFIG_TMPFS_XATTR 3171 3172 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa) 3173 { 3174 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3175 3176 fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE); 3177 3178 return 0; 3179 } 3180 3181 static int shmem_fileattr_set(struct user_namespace *mnt_userns, 3182 struct dentry *dentry, struct fileattr *fa) 3183 { 3184 struct inode *inode = d_inode(dentry); 3185 struct shmem_inode_info *info = SHMEM_I(inode); 3186 3187 if (fileattr_has_fsx(fa)) 3188 return -EOPNOTSUPP; 3189 if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE) 3190 return -EOPNOTSUPP; 3191 3192 info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) | 3193 (fa->flags & SHMEM_FL_USER_MODIFIABLE); 3194 3195 shmem_set_inode_flags(inode, info->fsflags); 3196 inode->i_ctime = current_time(inode); 3197 return 0; 3198 } 3199 3200 /* 3201 * Superblocks without xattr inode operations may get some security.* xattr 3202 * support from the LSM "for free". As soon as we have any other xattrs 3203 * like ACLs, we also need to implement the security.* handlers at 3204 * filesystem level, though. 3205 */ 3206 3207 /* 3208 * Callback for security_inode_init_security() for acquiring xattrs. 3209 */ 3210 static int shmem_initxattrs(struct inode *inode, 3211 const struct xattr *xattr_array, 3212 void *fs_info) 3213 { 3214 struct shmem_inode_info *info = SHMEM_I(inode); 3215 const struct xattr *xattr; 3216 struct simple_xattr *new_xattr; 3217 size_t len; 3218 3219 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3220 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3221 if (!new_xattr) 3222 return -ENOMEM; 3223 3224 len = strlen(xattr->name) + 1; 3225 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3226 GFP_KERNEL); 3227 if (!new_xattr->name) { 3228 kvfree(new_xattr); 3229 return -ENOMEM; 3230 } 3231 3232 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3233 XATTR_SECURITY_PREFIX_LEN); 3234 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3235 xattr->name, len); 3236 3237 simple_xattr_list_add(&info->xattrs, new_xattr); 3238 } 3239 3240 return 0; 3241 } 3242 3243 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3244 struct dentry *unused, struct inode *inode, 3245 const char *name, void *buffer, size_t size) 3246 { 3247 struct shmem_inode_info *info = SHMEM_I(inode); 3248 3249 name = xattr_full_name(handler, name); 3250 return simple_xattr_get(&info->xattrs, name, buffer, size); 3251 } 3252 3253 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3254 struct user_namespace *mnt_userns, 3255 struct dentry *unused, struct inode *inode, 3256 const char *name, const void *value, 3257 size_t size, int flags) 3258 { 3259 struct shmem_inode_info *info = SHMEM_I(inode); 3260 3261 name = xattr_full_name(handler, name); 3262 return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL); 3263 } 3264 3265 static const struct xattr_handler shmem_security_xattr_handler = { 3266 .prefix = XATTR_SECURITY_PREFIX, 3267 .get = shmem_xattr_handler_get, 3268 .set = shmem_xattr_handler_set, 3269 }; 3270 3271 static const struct xattr_handler shmem_trusted_xattr_handler = { 3272 .prefix = XATTR_TRUSTED_PREFIX, 3273 .get = shmem_xattr_handler_get, 3274 .set = shmem_xattr_handler_set, 3275 }; 3276 3277 static const struct xattr_handler *shmem_xattr_handlers[] = { 3278 #ifdef CONFIG_TMPFS_POSIX_ACL 3279 &posix_acl_access_xattr_handler, 3280 &posix_acl_default_xattr_handler, 3281 #endif 3282 &shmem_security_xattr_handler, 3283 &shmem_trusted_xattr_handler, 3284 NULL 3285 }; 3286 3287 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3288 { 3289 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3290 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3291 } 3292 #endif /* CONFIG_TMPFS_XATTR */ 3293 3294 static const struct inode_operations shmem_short_symlink_operations = { 3295 .getattr = shmem_getattr, 3296 .get_link = simple_get_link, 3297 #ifdef CONFIG_TMPFS_XATTR 3298 .listxattr = shmem_listxattr, 3299 #endif 3300 }; 3301 3302 static const struct inode_operations shmem_symlink_inode_operations = { 3303 .getattr = shmem_getattr, 3304 .get_link = shmem_get_link, 3305 #ifdef CONFIG_TMPFS_XATTR 3306 .listxattr = shmem_listxattr, 3307 #endif 3308 }; 3309 3310 static struct dentry *shmem_get_parent(struct dentry *child) 3311 { 3312 return ERR_PTR(-ESTALE); 3313 } 3314 3315 static int shmem_match(struct inode *ino, void *vfh) 3316 { 3317 __u32 *fh = vfh; 3318 __u64 inum = fh[2]; 3319 inum = (inum << 32) | fh[1]; 3320 return ino->i_ino == inum && fh[0] == ino->i_generation; 3321 } 3322 3323 /* Find any alias of inode, but prefer a hashed alias */ 3324 static struct dentry *shmem_find_alias(struct inode *inode) 3325 { 3326 struct dentry *alias = d_find_alias(inode); 3327 3328 return alias ?: d_find_any_alias(inode); 3329 } 3330 3331 3332 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3333 struct fid *fid, int fh_len, int fh_type) 3334 { 3335 struct inode *inode; 3336 struct dentry *dentry = NULL; 3337 u64 inum; 3338 3339 if (fh_len < 3) 3340 return NULL; 3341 3342 inum = fid->raw[2]; 3343 inum = (inum << 32) | fid->raw[1]; 3344 3345 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3346 shmem_match, fid->raw); 3347 if (inode) { 3348 dentry = shmem_find_alias(inode); 3349 iput(inode); 3350 } 3351 3352 return dentry; 3353 } 3354 3355 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3356 struct inode *parent) 3357 { 3358 if (*len < 3) { 3359 *len = 3; 3360 return FILEID_INVALID; 3361 } 3362 3363 if (inode_unhashed(inode)) { 3364 /* Unfortunately insert_inode_hash is not idempotent, 3365 * so as we hash inodes here rather than at creation 3366 * time, we need a lock to ensure we only try 3367 * to do it once 3368 */ 3369 static DEFINE_SPINLOCK(lock); 3370 spin_lock(&lock); 3371 if (inode_unhashed(inode)) 3372 __insert_inode_hash(inode, 3373 inode->i_ino + inode->i_generation); 3374 spin_unlock(&lock); 3375 } 3376 3377 fh[0] = inode->i_generation; 3378 fh[1] = inode->i_ino; 3379 fh[2] = ((__u64)inode->i_ino) >> 32; 3380 3381 *len = 3; 3382 return 1; 3383 } 3384 3385 static const struct export_operations shmem_export_ops = { 3386 .get_parent = shmem_get_parent, 3387 .encode_fh = shmem_encode_fh, 3388 .fh_to_dentry = shmem_fh_to_dentry, 3389 }; 3390 3391 enum shmem_param { 3392 Opt_gid, 3393 Opt_huge, 3394 Opt_mode, 3395 Opt_mpol, 3396 Opt_nr_blocks, 3397 Opt_nr_inodes, 3398 Opt_size, 3399 Opt_uid, 3400 Opt_inode32, 3401 Opt_inode64, 3402 }; 3403 3404 static const struct constant_table shmem_param_enums_huge[] = { 3405 {"never", SHMEM_HUGE_NEVER }, 3406 {"always", SHMEM_HUGE_ALWAYS }, 3407 {"within_size", SHMEM_HUGE_WITHIN_SIZE }, 3408 {"advise", SHMEM_HUGE_ADVISE }, 3409 {} 3410 }; 3411 3412 const struct fs_parameter_spec shmem_fs_parameters[] = { 3413 fsparam_u32 ("gid", Opt_gid), 3414 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge), 3415 fsparam_u32oct("mode", Opt_mode), 3416 fsparam_string("mpol", Opt_mpol), 3417 fsparam_string("nr_blocks", Opt_nr_blocks), 3418 fsparam_string("nr_inodes", Opt_nr_inodes), 3419 fsparam_string("size", Opt_size), 3420 fsparam_u32 ("uid", Opt_uid), 3421 fsparam_flag ("inode32", Opt_inode32), 3422 fsparam_flag ("inode64", Opt_inode64), 3423 {} 3424 }; 3425 3426 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) 3427 { 3428 struct shmem_options *ctx = fc->fs_private; 3429 struct fs_parse_result result; 3430 unsigned long long size; 3431 char *rest; 3432 int opt; 3433 3434 opt = fs_parse(fc, shmem_fs_parameters, param, &result); 3435 if (opt < 0) 3436 return opt; 3437 3438 switch (opt) { 3439 case Opt_size: 3440 size = memparse(param->string, &rest); 3441 if (*rest == '%') { 3442 size <<= PAGE_SHIFT; 3443 size *= totalram_pages(); 3444 do_div(size, 100); 3445 rest++; 3446 } 3447 if (*rest) 3448 goto bad_value; 3449 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE); 3450 ctx->seen |= SHMEM_SEEN_BLOCKS; 3451 break; 3452 case Opt_nr_blocks: 3453 ctx->blocks = memparse(param->string, &rest); 3454 if (*rest || ctx->blocks > S64_MAX) 3455 goto bad_value; 3456 ctx->seen |= SHMEM_SEEN_BLOCKS; 3457 break; 3458 case Opt_nr_inodes: 3459 ctx->inodes = memparse(param->string, &rest); 3460 if (*rest) 3461 goto bad_value; 3462 ctx->seen |= SHMEM_SEEN_INODES; 3463 break; 3464 case Opt_mode: 3465 ctx->mode = result.uint_32 & 07777; 3466 break; 3467 case Opt_uid: 3468 ctx->uid = make_kuid(current_user_ns(), result.uint_32); 3469 if (!uid_valid(ctx->uid)) 3470 goto bad_value; 3471 break; 3472 case Opt_gid: 3473 ctx->gid = make_kgid(current_user_ns(), result.uint_32); 3474 if (!gid_valid(ctx->gid)) 3475 goto bad_value; 3476 break; 3477 case Opt_huge: 3478 ctx->huge = result.uint_32; 3479 if (ctx->huge != SHMEM_HUGE_NEVER && 3480 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 3481 has_transparent_hugepage())) 3482 goto unsupported_parameter; 3483 ctx->seen |= SHMEM_SEEN_HUGE; 3484 break; 3485 case Opt_mpol: 3486 if (IS_ENABLED(CONFIG_NUMA)) { 3487 mpol_put(ctx->mpol); 3488 ctx->mpol = NULL; 3489 if (mpol_parse_str(param->string, &ctx->mpol)) 3490 goto bad_value; 3491 break; 3492 } 3493 goto unsupported_parameter; 3494 case Opt_inode32: 3495 ctx->full_inums = false; 3496 ctx->seen |= SHMEM_SEEN_INUMS; 3497 break; 3498 case Opt_inode64: 3499 if (sizeof(ino_t) < 8) { 3500 return invalfc(fc, 3501 "Cannot use inode64 with <64bit inums in kernel\n"); 3502 } 3503 ctx->full_inums = true; 3504 ctx->seen |= SHMEM_SEEN_INUMS; 3505 break; 3506 } 3507 return 0; 3508 3509 unsupported_parameter: 3510 return invalfc(fc, "Unsupported parameter '%s'", param->key); 3511 bad_value: 3512 return invalfc(fc, "Bad value for '%s'", param->key); 3513 } 3514 3515 static int shmem_parse_options(struct fs_context *fc, void *data) 3516 { 3517 char *options = data; 3518 3519 if (options) { 3520 int err = security_sb_eat_lsm_opts(options, &fc->security); 3521 if (err) 3522 return err; 3523 } 3524 3525 while (options != NULL) { 3526 char *this_char = options; 3527 for (;;) { 3528 /* 3529 * NUL-terminate this option: unfortunately, 3530 * mount options form a comma-separated list, 3531 * but mpol's nodelist may also contain commas. 3532 */ 3533 options = strchr(options, ','); 3534 if (options == NULL) 3535 break; 3536 options++; 3537 if (!isdigit(*options)) { 3538 options[-1] = '\0'; 3539 break; 3540 } 3541 } 3542 if (*this_char) { 3543 char *value = strchr(this_char, '='); 3544 size_t len = 0; 3545 int err; 3546 3547 if (value) { 3548 *value++ = '\0'; 3549 len = strlen(value); 3550 } 3551 err = vfs_parse_fs_string(fc, this_char, value, len); 3552 if (err < 0) 3553 return err; 3554 } 3555 } 3556 return 0; 3557 } 3558 3559 /* 3560 * Reconfigure a shmem filesystem. 3561 * 3562 * Note that we disallow change from limited->unlimited blocks/inodes while any 3563 * are in use; but we must separately disallow unlimited->limited, because in 3564 * that case we have no record of how much is already in use. 3565 */ 3566 static int shmem_reconfigure(struct fs_context *fc) 3567 { 3568 struct shmem_options *ctx = fc->fs_private; 3569 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb); 3570 unsigned long inodes; 3571 struct mempolicy *mpol = NULL; 3572 const char *err; 3573 3574 raw_spin_lock(&sbinfo->stat_lock); 3575 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3576 3577 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) { 3578 if (!sbinfo->max_blocks) { 3579 err = "Cannot retroactively limit size"; 3580 goto out; 3581 } 3582 if (percpu_counter_compare(&sbinfo->used_blocks, 3583 ctx->blocks) > 0) { 3584 err = "Too small a size for current use"; 3585 goto out; 3586 } 3587 } 3588 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) { 3589 if (!sbinfo->max_inodes) { 3590 err = "Cannot retroactively limit inodes"; 3591 goto out; 3592 } 3593 if (ctx->inodes < inodes) { 3594 err = "Too few inodes for current use"; 3595 goto out; 3596 } 3597 } 3598 3599 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums && 3600 sbinfo->next_ino > UINT_MAX) { 3601 err = "Current inum too high to switch to 32-bit inums"; 3602 goto out; 3603 } 3604 3605 if (ctx->seen & SHMEM_SEEN_HUGE) 3606 sbinfo->huge = ctx->huge; 3607 if (ctx->seen & SHMEM_SEEN_INUMS) 3608 sbinfo->full_inums = ctx->full_inums; 3609 if (ctx->seen & SHMEM_SEEN_BLOCKS) 3610 sbinfo->max_blocks = ctx->blocks; 3611 if (ctx->seen & SHMEM_SEEN_INODES) { 3612 sbinfo->max_inodes = ctx->inodes; 3613 sbinfo->free_inodes = ctx->inodes - inodes; 3614 } 3615 3616 /* 3617 * Preserve previous mempolicy unless mpol remount option was specified. 3618 */ 3619 if (ctx->mpol) { 3620 mpol = sbinfo->mpol; 3621 sbinfo->mpol = ctx->mpol; /* transfers initial ref */ 3622 ctx->mpol = NULL; 3623 } 3624 raw_spin_unlock(&sbinfo->stat_lock); 3625 mpol_put(mpol); 3626 return 0; 3627 out: 3628 raw_spin_unlock(&sbinfo->stat_lock); 3629 return invalfc(fc, "%s", err); 3630 } 3631 3632 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3633 { 3634 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3635 3636 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3637 seq_printf(seq, ",size=%luk", 3638 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3639 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3640 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3641 if (sbinfo->mode != (0777 | S_ISVTX)) 3642 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3643 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3644 seq_printf(seq, ",uid=%u", 3645 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3646 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3647 seq_printf(seq, ",gid=%u", 3648 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3649 3650 /* 3651 * Showing inode{64,32} might be useful even if it's the system default, 3652 * since then people don't have to resort to checking both here and 3653 * /proc/config.gz to confirm 64-bit inums were successfully applied 3654 * (which may not even exist if IKCONFIG_PROC isn't enabled). 3655 * 3656 * We hide it when inode64 isn't the default and we are using 32-bit 3657 * inodes, since that probably just means the feature isn't even under 3658 * consideration. 3659 * 3660 * As such: 3661 * 3662 * +-----------------+-----------------+ 3663 * | TMPFS_INODE64=y | TMPFS_INODE64=n | 3664 * +------------------+-----------------+-----------------+ 3665 * | full_inums=true | show | show | 3666 * | full_inums=false | show | hide | 3667 * +------------------+-----------------+-----------------+ 3668 * 3669 */ 3670 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums) 3671 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32)); 3672 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3673 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3674 if (sbinfo->huge) 3675 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3676 #endif 3677 shmem_show_mpol(seq, sbinfo->mpol); 3678 return 0; 3679 } 3680 3681 #endif /* CONFIG_TMPFS */ 3682 3683 static void shmem_put_super(struct super_block *sb) 3684 { 3685 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3686 3687 free_percpu(sbinfo->ino_batch); 3688 percpu_counter_destroy(&sbinfo->used_blocks); 3689 mpol_put(sbinfo->mpol); 3690 kfree(sbinfo); 3691 sb->s_fs_info = NULL; 3692 } 3693 3694 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) 3695 { 3696 struct shmem_options *ctx = fc->fs_private; 3697 struct inode *inode; 3698 struct shmem_sb_info *sbinfo; 3699 3700 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3701 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3702 L1_CACHE_BYTES), GFP_KERNEL); 3703 if (!sbinfo) 3704 return -ENOMEM; 3705 3706 sb->s_fs_info = sbinfo; 3707 3708 #ifdef CONFIG_TMPFS 3709 /* 3710 * Per default we only allow half of the physical ram per 3711 * tmpfs instance, limiting inodes to one per page of lowmem; 3712 * but the internal instance is left unlimited. 3713 */ 3714 if (!(sb->s_flags & SB_KERNMOUNT)) { 3715 if (!(ctx->seen & SHMEM_SEEN_BLOCKS)) 3716 ctx->blocks = shmem_default_max_blocks(); 3717 if (!(ctx->seen & SHMEM_SEEN_INODES)) 3718 ctx->inodes = shmem_default_max_inodes(); 3719 if (!(ctx->seen & SHMEM_SEEN_INUMS)) 3720 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64); 3721 } else { 3722 sb->s_flags |= SB_NOUSER; 3723 } 3724 sb->s_export_op = &shmem_export_ops; 3725 sb->s_flags |= SB_NOSEC; 3726 #else 3727 sb->s_flags |= SB_NOUSER; 3728 #endif 3729 sbinfo->max_blocks = ctx->blocks; 3730 sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes; 3731 if (sb->s_flags & SB_KERNMOUNT) { 3732 sbinfo->ino_batch = alloc_percpu(ino_t); 3733 if (!sbinfo->ino_batch) 3734 goto failed; 3735 } 3736 sbinfo->uid = ctx->uid; 3737 sbinfo->gid = ctx->gid; 3738 sbinfo->full_inums = ctx->full_inums; 3739 sbinfo->mode = ctx->mode; 3740 sbinfo->huge = ctx->huge; 3741 sbinfo->mpol = ctx->mpol; 3742 ctx->mpol = NULL; 3743 3744 raw_spin_lock_init(&sbinfo->stat_lock); 3745 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3746 goto failed; 3747 spin_lock_init(&sbinfo->shrinklist_lock); 3748 INIT_LIST_HEAD(&sbinfo->shrinklist); 3749 3750 sb->s_maxbytes = MAX_LFS_FILESIZE; 3751 sb->s_blocksize = PAGE_SIZE; 3752 sb->s_blocksize_bits = PAGE_SHIFT; 3753 sb->s_magic = TMPFS_MAGIC; 3754 sb->s_op = &shmem_ops; 3755 sb->s_time_gran = 1; 3756 #ifdef CONFIG_TMPFS_XATTR 3757 sb->s_xattr = shmem_xattr_handlers; 3758 #endif 3759 #ifdef CONFIG_TMPFS_POSIX_ACL 3760 sb->s_flags |= SB_POSIXACL; 3761 #endif 3762 uuid_gen(&sb->s_uuid); 3763 3764 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3765 if (!inode) 3766 goto failed; 3767 inode->i_uid = sbinfo->uid; 3768 inode->i_gid = sbinfo->gid; 3769 sb->s_root = d_make_root(inode); 3770 if (!sb->s_root) 3771 goto failed; 3772 return 0; 3773 3774 failed: 3775 shmem_put_super(sb); 3776 return -ENOMEM; 3777 } 3778 3779 static int shmem_get_tree(struct fs_context *fc) 3780 { 3781 return get_tree_nodev(fc, shmem_fill_super); 3782 } 3783 3784 static void shmem_free_fc(struct fs_context *fc) 3785 { 3786 struct shmem_options *ctx = fc->fs_private; 3787 3788 if (ctx) { 3789 mpol_put(ctx->mpol); 3790 kfree(ctx); 3791 } 3792 } 3793 3794 static const struct fs_context_operations shmem_fs_context_ops = { 3795 .free = shmem_free_fc, 3796 .get_tree = shmem_get_tree, 3797 #ifdef CONFIG_TMPFS 3798 .parse_monolithic = shmem_parse_options, 3799 .parse_param = shmem_parse_one, 3800 .reconfigure = shmem_reconfigure, 3801 #endif 3802 }; 3803 3804 static struct kmem_cache *shmem_inode_cachep; 3805 3806 static struct inode *shmem_alloc_inode(struct super_block *sb) 3807 { 3808 struct shmem_inode_info *info; 3809 info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); 3810 if (!info) 3811 return NULL; 3812 return &info->vfs_inode; 3813 } 3814 3815 static void shmem_free_in_core_inode(struct inode *inode) 3816 { 3817 if (S_ISLNK(inode->i_mode)) 3818 kfree(inode->i_link); 3819 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3820 } 3821 3822 static void shmem_destroy_inode(struct inode *inode) 3823 { 3824 if (S_ISREG(inode->i_mode)) 3825 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3826 } 3827 3828 static void shmem_init_inode(void *foo) 3829 { 3830 struct shmem_inode_info *info = foo; 3831 inode_init_once(&info->vfs_inode); 3832 } 3833 3834 static void shmem_init_inodecache(void) 3835 { 3836 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3837 sizeof(struct shmem_inode_info), 3838 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3839 } 3840 3841 static void shmem_destroy_inodecache(void) 3842 { 3843 kmem_cache_destroy(shmem_inode_cachep); 3844 } 3845 3846 /* Keep the page in page cache instead of truncating it */ 3847 static int shmem_error_remove_page(struct address_space *mapping, 3848 struct page *page) 3849 { 3850 return 0; 3851 } 3852 3853 const struct address_space_operations shmem_aops = { 3854 .writepage = shmem_writepage, 3855 .dirty_folio = noop_dirty_folio, 3856 #ifdef CONFIG_TMPFS 3857 .write_begin = shmem_write_begin, 3858 .write_end = shmem_write_end, 3859 #endif 3860 #ifdef CONFIG_MIGRATION 3861 .migrate_folio = migrate_folio, 3862 #endif 3863 .error_remove_page = shmem_error_remove_page, 3864 }; 3865 EXPORT_SYMBOL(shmem_aops); 3866 3867 static const struct file_operations shmem_file_operations = { 3868 .mmap = shmem_mmap, 3869 .get_unmapped_area = shmem_get_unmapped_area, 3870 #ifdef CONFIG_TMPFS 3871 .llseek = shmem_file_llseek, 3872 .read_iter = shmem_file_read_iter, 3873 .write_iter = generic_file_write_iter, 3874 .fsync = noop_fsync, 3875 .splice_read = generic_file_splice_read, 3876 .splice_write = iter_file_splice_write, 3877 .fallocate = shmem_fallocate, 3878 #endif 3879 }; 3880 3881 static const struct inode_operations shmem_inode_operations = { 3882 .getattr = shmem_getattr, 3883 .setattr = shmem_setattr, 3884 #ifdef CONFIG_TMPFS_XATTR 3885 .listxattr = shmem_listxattr, 3886 .set_acl = simple_set_acl, 3887 .fileattr_get = shmem_fileattr_get, 3888 .fileattr_set = shmem_fileattr_set, 3889 #endif 3890 }; 3891 3892 static const struct inode_operations shmem_dir_inode_operations = { 3893 #ifdef CONFIG_TMPFS 3894 .getattr = shmem_getattr, 3895 .create = shmem_create, 3896 .lookup = simple_lookup, 3897 .link = shmem_link, 3898 .unlink = shmem_unlink, 3899 .symlink = shmem_symlink, 3900 .mkdir = shmem_mkdir, 3901 .rmdir = shmem_rmdir, 3902 .mknod = shmem_mknod, 3903 .rename = shmem_rename2, 3904 .tmpfile = shmem_tmpfile, 3905 #endif 3906 #ifdef CONFIG_TMPFS_XATTR 3907 .listxattr = shmem_listxattr, 3908 .fileattr_get = shmem_fileattr_get, 3909 .fileattr_set = shmem_fileattr_set, 3910 #endif 3911 #ifdef CONFIG_TMPFS_POSIX_ACL 3912 .setattr = shmem_setattr, 3913 .set_acl = simple_set_acl, 3914 #endif 3915 }; 3916 3917 static const struct inode_operations shmem_special_inode_operations = { 3918 .getattr = shmem_getattr, 3919 #ifdef CONFIG_TMPFS_XATTR 3920 .listxattr = shmem_listxattr, 3921 #endif 3922 #ifdef CONFIG_TMPFS_POSIX_ACL 3923 .setattr = shmem_setattr, 3924 .set_acl = simple_set_acl, 3925 #endif 3926 }; 3927 3928 static const struct super_operations shmem_ops = { 3929 .alloc_inode = shmem_alloc_inode, 3930 .free_inode = shmem_free_in_core_inode, 3931 .destroy_inode = shmem_destroy_inode, 3932 #ifdef CONFIG_TMPFS 3933 .statfs = shmem_statfs, 3934 .show_options = shmem_show_options, 3935 #endif 3936 .evict_inode = shmem_evict_inode, 3937 .drop_inode = generic_delete_inode, 3938 .put_super = shmem_put_super, 3939 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3940 .nr_cached_objects = shmem_unused_huge_count, 3941 .free_cached_objects = shmem_unused_huge_scan, 3942 #endif 3943 }; 3944 3945 static const struct vm_operations_struct shmem_vm_ops = { 3946 .fault = shmem_fault, 3947 .map_pages = filemap_map_pages, 3948 #ifdef CONFIG_NUMA 3949 .set_policy = shmem_set_policy, 3950 .get_policy = shmem_get_policy, 3951 #endif 3952 }; 3953 3954 int shmem_init_fs_context(struct fs_context *fc) 3955 { 3956 struct shmem_options *ctx; 3957 3958 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL); 3959 if (!ctx) 3960 return -ENOMEM; 3961 3962 ctx->mode = 0777 | S_ISVTX; 3963 ctx->uid = current_fsuid(); 3964 ctx->gid = current_fsgid(); 3965 3966 fc->fs_private = ctx; 3967 fc->ops = &shmem_fs_context_ops; 3968 return 0; 3969 } 3970 3971 static struct file_system_type shmem_fs_type = { 3972 .owner = THIS_MODULE, 3973 .name = "tmpfs", 3974 .init_fs_context = shmem_init_fs_context, 3975 #ifdef CONFIG_TMPFS 3976 .parameters = shmem_fs_parameters, 3977 #endif 3978 .kill_sb = kill_litter_super, 3979 .fs_flags = FS_USERNS_MOUNT, 3980 }; 3981 3982 void __init shmem_init(void) 3983 { 3984 int error; 3985 3986 shmem_init_inodecache(); 3987 3988 error = register_filesystem(&shmem_fs_type); 3989 if (error) { 3990 pr_err("Could not register tmpfs\n"); 3991 goto out2; 3992 } 3993 3994 shm_mnt = kern_mount(&shmem_fs_type); 3995 if (IS_ERR(shm_mnt)) { 3996 error = PTR_ERR(shm_mnt); 3997 pr_err("Could not kern_mount tmpfs\n"); 3998 goto out1; 3999 } 4000 4001 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4002 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) 4003 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4004 else 4005 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */ 4006 #endif 4007 return; 4008 4009 out1: 4010 unregister_filesystem(&shmem_fs_type); 4011 out2: 4012 shmem_destroy_inodecache(); 4013 shm_mnt = ERR_PTR(error); 4014 } 4015 4016 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) 4017 static ssize_t shmem_enabled_show(struct kobject *kobj, 4018 struct kobj_attribute *attr, char *buf) 4019 { 4020 static const int values[] = { 4021 SHMEM_HUGE_ALWAYS, 4022 SHMEM_HUGE_WITHIN_SIZE, 4023 SHMEM_HUGE_ADVISE, 4024 SHMEM_HUGE_NEVER, 4025 SHMEM_HUGE_DENY, 4026 SHMEM_HUGE_FORCE, 4027 }; 4028 int len = 0; 4029 int i; 4030 4031 for (i = 0; i < ARRAY_SIZE(values); i++) { 4032 len += sysfs_emit_at(buf, len, 4033 shmem_huge == values[i] ? "%s[%s]" : "%s%s", 4034 i ? " " : "", 4035 shmem_format_huge(values[i])); 4036 } 4037 4038 len += sysfs_emit_at(buf, len, "\n"); 4039 4040 return len; 4041 } 4042 4043 static ssize_t shmem_enabled_store(struct kobject *kobj, 4044 struct kobj_attribute *attr, const char *buf, size_t count) 4045 { 4046 char tmp[16]; 4047 int huge; 4048 4049 if (count + 1 > sizeof(tmp)) 4050 return -EINVAL; 4051 memcpy(tmp, buf, count); 4052 tmp[count] = '\0'; 4053 if (count && tmp[count - 1] == '\n') 4054 tmp[count - 1] = '\0'; 4055 4056 huge = shmem_parse_huge(tmp); 4057 if (huge == -EINVAL) 4058 return -EINVAL; 4059 if (!has_transparent_hugepage() && 4060 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 4061 return -EINVAL; 4062 4063 shmem_huge = huge; 4064 if (shmem_huge > SHMEM_HUGE_DENY) 4065 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4066 return count; 4067 } 4068 4069 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled); 4070 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ 4071 4072 #else /* !CONFIG_SHMEM */ 4073 4074 /* 4075 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 4076 * 4077 * This is intended for small system where the benefits of the full 4078 * shmem code (swap-backed and resource-limited) are outweighed by 4079 * their complexity. On systems without swap this code should be 4080 * effectively equivalent, but much lighter weight. 4081 */ 4082 4083 static struct file_system_type shmem_fs_type = { 4084 .name = "tmpfs", 4085 .init_fs_context = ramfs_init_fs_context, 4086 .parameters = ramfs_fs_parameters, 4087 .kill_sb = kill_litter_super, 4088 .fs_flags = FS_USERNS_MOUNT, 4089 }; 4090 4091 void __init shmem_init(void) 4092 { 4093 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4094 4095 shm_mnt = kern_mount(&shmem_fs_type); 4096 BUG_ON(IS_ERR(shm_mnt)); 4097 } 4098 4099 int shmem_unuse(unsigned int type) 4100 { 4101 return 0; 4102 } 4103 4104 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 4105 { 4106 return 0; 4107 } 4108 4109 void shmem_unlock_mapping(struct address_space *mapping) 4110 { 4111 } 4112 4113 #ifdef CONFIG_MMU 4114 unsigned long shmem_get_unmapped_area(struct file *file, 4115 unsigned long addr, unsigned long len, 4116 unsigned long pgoff, unsigned long flags) 4117 { 4118 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 4119 } 4120 #endif 4121 4122 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4123 { 4124 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4125 } 4126 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4127 4128 #define shmem_vm_ops generic_file_vm_ops 4129 #define shmem_file_operations ramfs_file_operations 4130 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 4131 #define shmem_acct_size(flags, size) 0 4132 #define shmem_unacct_size(flags, size) do {} while (0) 4133 4134 #endif /* CONFIG_SHMEM */ 4135 4136 /* common code */ 4137 4138 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size, 4139 unsigned long flags, unsigned int i_flags) 4140 { 4141 struct inode *inode; 4142 struct file *res; 4143 4144 if (IS_ERR(mnt)) 4145 return ERR_CAST(mnt); 4146 4147 if (size < 0 || size > MAX_LFS_FILESIZE) 4148 return ERR_PTR(-EINVAL); 4149 4150 if (shmem_acct_size(flags, size)) 4151 return ERR_PTR(-ENOMEM); 4152 4153 inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0, 4154 flags); 4155 if (unlikely(!inode)) { 4156 shmem_unacct_size(flags, size); 4157 return ERR_PTR(-ENOSPC); 4158 } 4159 inode->i_flags |= i_flags; 4160 inode->i_size = size; 4161 clear_nlink(inode); /* It is unlinked */ 4162 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4163 if (!IS_ERR(res)) 4164 res = alloc_file_pseudo(inode, mnt, name, O_RDWR, 4165 &shmem_file_operations); 4166 if (IS_ERR(res)) 4167 iput(inode); 4168 return res; 4169 } 4170 4171 /** 4172 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4173 * kernel internal. There will be NO LSM permission checks against the 4174 * underlying inode. So users of this interface must do LSM checks at a 4175 * higher layer. The users are the big_key and shm implementations. LSM 4176 * checks are provided at the key or shm level rather than the inode. 4177 * @name: name for dentry (to be seen in /proc/<pid>/maps 4178 * @size: size to be set for the file 4179 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4180 */ 4181 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4182 { 4183 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE); 4184 } 4185 4186 /** 4187 * shmem_file_setup - get an unlinked file living in tmpfs 4188 * @name: name for dentry (to be seen in /proc/<pid>/maps 4189 * @size: size to be set for the file 4190 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4191 */ 4192 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4193 { 4194 return __shmem_file_setup(shm_mnt, name, size, flags, 0); 4195 } 4196 EXPORT_SYMBOL_GPL(shmem_file_setup); 4197 4198 /** 4199 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs 4200 * @mnt: the tmpfs mount where the file will be created 4201 * @name: name for dentry (to be seen in /proc/<pid>/maps 4202 * @size: size to be set for the file 4203 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4204 */ 4205 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name, 4206 loff_t size, unsigned long flags) 4207 { 4208 return __shmem_file_setup(mnt, name, size, flags, 0); 4209 } 4210 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt); 4211 4212 /** 4213 * shmem_zero_setup - setup a shared anonymous mapping 4214 * @vma: the vma to be mmapped is prepared by do_mmap 4215 */ 4216 int shmem_zero_setup(struct vm_area_struct *vma) 4217 { 4218 struct file *file; 4219 loff_t size = vma->vm_end - vma->vm_start; 4220 4221 /* 4222 * Cloning a new file under mmap_lock leads to a lock ordering conflict 4223 * between XFS directory reading and selinux: since this file is only 4224 * accessible to the user through its mapping, use S_PRIVATE flag to 4225 * bypass file security, in the same way as shmem_kernel_file_setup(). 4226 */ 4227 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags); 4228 if (IS_ERR(file)) 4229 return PTR_ERR(file); 4230 4231 if (vma->vm_file) 4232 fput(vma->vm_file); 4233 vma->vm_file = file; 4234 vma->vm_ops = &shmem_vm_ops; 4235 4236 return 0; 4237 } 4238 4239 /** 4240 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4241 * @mapping: the page's address_space 4242 * @index: the page index 4243 * @gfp: the page allocator flags to use if allocating 4244 * 4245 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4246 * with any new page allocations done using the specified allocation flags. 4247 * But read_cache_page_gfp() uses the ->read_folio() method: which does not 4248 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4249 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4250 * 4251 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4252 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4253 */ 4254 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4255 pgoff_t index, gfp_t gfp) 4256 { 4257 #ifdef CONFIG_SHMEM 4258 struct inode *inode = mapping->host; 4259 struct folio *folio; 4260 struct page *page; 4261 int error; 4262 4263 BUG_ON(!shmem_mapping(mapping)); 4264 error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE, 4265 gfp, NULL, NULL, NULL); 4266 if (error) 4267 return ERR_PTR(error); 4268 4269 folio_unlock(folio); 4270 page = folio_file_page(folio, index); 4271 if (PageHWPoison(page)) { 4272 folio_put(folio); 4273 return ERR_PTR(-EIO); 4274 } 4275 4276 return page; 4277 #else 4278 /* 4279 * The tiny !SHMEM case uses ramfs without swap 4280 */ 4281 return read_cache_page_gfp(mapping, index, gfp); 4282 #endif 4283 } 4284 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4285