1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/fileattr.h> 32 #include <linux/mm.h> 33 #include <linux/random.h> 34 #include <linux/sched/signal.h> 35 #include <linux/export.h> 36 #include <linux/swap.h> 37 #include <linux/uio.h> 38 #include <linux/hugetlb.h> 39 #include <linux/fs_parser.h> 40 #include <linux/swapfile.h> 41 #include <linux/iversion.h> 42 #include "swap.h" 43 44 static struct vfsmount *shm_mnt; 45 46 #ifdef CONFIG_SHMEM 47 /* 48 * This virtual memory filesystem is heavily based on the ramfs. It 49 * extends ramfs by the ability to use swap and honor resource limits 50 * which makes it a completely usable filesystem. 51 */ 52 53 #include <linux/xattr.h> 54 #include <linux/exportfs.h> 55 #include <linux/posix_acl.h> 56 #include <linux/posix_acl_xattr.h> 57 #include <linux/mman.h> 58 #include <linux/string.h> 59 #include <linux/slab.h> 60 #include <linux/backing-dev.h> 61 #include <linux/shmem_fs.h> 62 #include <linux/writeback.h> 63 #include <linux/pagevec.h> 64 #include <linux/percpu_counter.h> 65 #include <linux/falloc.h> 66 #include <linux/splice.h> 67 #include <linux/security.h> 68 #include <linux/swapops.h> 69 #include <linux/mempolicy.h> 70 #include <linux/namei.h> 71 #include <linux/ctype.h> 72 #include <linux/migrate.h> 73 #include <linux/highmem.h> 74 #include <linux/seq_file.h> 75 #include <linux/magic.h> 76 #include <linux/syscalls.h> 77 #include <linux/fcntl.h> 78 #include <uapi/linux/memfd.h> 79 #include <linux/userfaultfd_k.h> 80 #include <linux/rmap.h> 81 #include <linux/uuid.h> 82 83 #include <linux/uaccess.h> 84 85 #include "internal.h" 86 87 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 88 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 89 90 /* Pretend that each entry is of this size in directory's i_size */ 91 #define BOGO_DIRENT_SIZE 20 92 93 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 94 #define SHORT_SYMLINK_LEN 128 95 96 /* 97 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 98 * inode->i_private (with i_rwsem making sure that it has only one user at 99 * a time): we would prefer not to enlarge the shmem inode just for that. 100 */ 101 struct shmem_falloc { 102 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 103 pgoff_t start; /* start of range currently being fallocated */ 104 pgoff_t next; /* the next page offset to be fallocated */ 105 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 106 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 107 }; 108 109 struct shmem_options { 110 unsigned long long blocks; 111 unsigned long long inodes; 112 struct mempolicy *mpol; 113 kuid_t uid; 114 kgid_t gid; 115 umode_t mode; 116 bool full_inums; 117 int huge; 118 int seen; 119 #define SHMEM_SEEN_BLOCKS 1 120 #define SHMEM_SEEN_INODES 2 121 #define SHMEM_SEEN_HUGE 4 122 #define SHMEM_SEEN_INUMS 8 123 }; 124 125 #ifdef CONFIG_TMPFS 126 static unsigned long shmem_default_max_blocks(void) 127 { 128 return totalram_pages() / 2; 129 } 130 131 static unsigned long shmem_default_max_inodes(void) 132 { 133 unsigned long nr_pages = totalram_pages(); 134 135 return min(nr_pages - totalhigh_pages(), nr_pages / 2); 136 } 137 #endif 138 139 static int shmem_swapin_folio(struct inode *inode, pgoff_t index, 140 struct folio **foliop, enum sgp_type sgp, 141 gfp_t gfp, struct vm_area_struct *vma, 142 vm_fault_t *fault_type); 143 144 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 145 { 146 return sb->s_fs_info; 147 } 148 149 /* 150 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 151 * for shared memory and for shared anonymous (/dev/zero) mappings 152 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 153 * consistent with the pre-accounting of private mappings ... 154 */ 155 static inline int shmem_acct_size(unsigned long flags, loff_t size) 156 { 157 return (flags & VM_NORESERVE) ? 158 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 159 } 160 161 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 162 { 163 if (!(flags & VM_NORESERVE)) 164 vm_unacct_memory(VM_ACCT(size)); 165 } 166 167 static inline int shmem_reacct_size(unsigned long flags, 168 loff_t oldsize, loff_t newsize) 169 { 170 if (!(flags & VM_NORESERVE)) { 171 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 172 return security_vm_enough_memory_mm(current->mm, 173 VM_ACCT(newsize) - VM_ACCT(oldsize)); 174 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 175 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 176 } 177 return 0; 178 } 179 180 /* 181 * ... whereas tmpfs objects are accounted incrementally as 182 * pages are allocated, in order to allow large sparse files. 183 * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 184 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 185 */ 186 static inline int shmem_acct_block(unsigned long flags, long pages) 187 { 188 if (!(flags & VM_NORESERVE)) 189 return 0; 190 191 return security_vm_enough_memory_mm(current->mm, 192 pages * VM_ACCT(PAGE_SIZE)); 193 } 194 195 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 196 { 197 if (flags & VM_NORESERVE) 198 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 199 } 200 201 static inline bool shmem_inode_acct_block(struct inode *inode, long pages) 202 { 203 struct shmem_inode_info *info = SHMEM_I(inode); 204 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 205 206 if (shmem_acct_block(info->flags, pages)) 207 return false; 208 209 if (sbinfo->max_blocks) { 210 if (percpu_counter_compare(&sbinfo->used_blocks, 211 sbinfo->max_blocks - pages) > 0) 212 goto unacct; 213 percpu_counter_add(&sbinfo->used_blocks, pages); 214 } 215 216 return true; 217 218 unacct: 219 shmem_unacct_blocks(info->flags, pages); 220 return false; 221 } 222 223 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages) 224 { 225 struct shmem_inode_info *info = SHMEM_I(inode); 226 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 227 228 if (sbinfo->max_blocks) 229 percpu_counter_sub(&sbinfo->used_blocks, pages); 230 shmem_unacct_blocks(info->flags, pages); 231 } 232 233 static const struct super_operations shmem_ops; 234 const struct address_space_operations shmem_aops; 235 static const struct file_operations shmem_file_operations; 236 static const struct inode_operations shmem_inode_operations; 237 static const struct inode_operations shmem_dir_inode_operations; 238 static const struct inode_operations shmem_special_inode_operations; 239 static const struct vm_operations_struct shmem_vm_ops; 240 static struct file_system_type shmem_fs_type; 241 242 bool vma_is_shmem(struct vm_area_struct *vma) 243 { 244 return vma->vm_ops == &shmem_vm_ops; 245 } 246 247 static LIST_HEAD(shmem_swaplist); 248 static DEFINE_MUTEX(shmem_swaplist_mutex); 249 250 /* 251 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and 252 * produces a novel ino for the newly allocated inode. 253 * 254 * It may also be called when making a hard link to permit the space needed by 255 * each dentry. However, in that case, no new inode number is needed since that 256 * internally draws from another pool of inode numbers (currently global 257 * get_next_ino()). This case is indicated by passing NULL as inop. 258 */ 259 #define SHMEM_INO_BATCH 1024 260 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop) 261 { 262 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 263 ino_t ino; 264 265 if (!(sb->s_flags & SB_KERNMOUNT)) { 266 raw_spin_lock(&sbinfo->stat_lock); 267 if (sbinfo->max_inodes) { 268 if (!sbinfo->free_inodes) { 269 raw_spin_unlock(&sbinfo->stat_lock); 270 return -ENOSPC; 271 } 272 sbinfo->free_inodes--; 273 } 274 if (inop) { 275 ino = sbinfo->next_ino++; 276 if (unlikely(is_zero_ino(ino))) 277 ino = sbinfo->next_ino++; 278 if (unlikely(!sbinfo->full_inums && 279 ino > UINT_MAX)) { 280 /* 281 * Emulate get_next_ino uint wraparound for 282 * compatibility 283 */ 284 if (IS_ENABLED(CONFIG_64BIT)) 285 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n", 286 __func__, MINOR(sb->s_dev)); 287 sbinfo->next_ino = 1; 288 ino = sbinfo->next_ino++; 289 } 290 *inop = ino; 291 } 292 raw_spin_unlock(&sbinfo->stat_lock); 293 } else if (inop) { 294 /* 295 * __shmem_file_setup, one of our callers, is lock-free: it 296 * doesn't hold stat_lock in shmem_reserve_inode since 297 * max_inodes is always 0, and is called from potentially 298 * unknown contexts. As such, use a per-cpu batched allocator 299 * which doesn't require the per-sb stat_lock unless we are at 300 * the batch boundary. 301 * 302 * We don't need to worry about inode{32,64} since SB_KERNMOUNT 303 * shmem mounts are not exposed to userspace, so we don't need 304 * to worry about things like glibc compatibility. 305 */ 306 ino_t *next_ino; 307 308 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu()); 309 ino = *next_ino; 310 if (unlikely(ino % SHMEM_INO_BATCH == 0)) { 311 raw_spin_lock(&sbinfo->stat_lock); 312 ino = sbinfo->next_ino; 313 sbinfo->next_ino += SHMEM_INO_BATCH; 314 raw_spin_unlock(&sbinfo->stat_lock); 315 if (unlikely(is_zero_ino(ino))) 316 ino++; 317 } 318 *inop = ino; 319 *next_ino = ++ino; 320 put_cpu(); 321 } 322 323 return 0; 324 } 325 326 static void shmem_free_inode(struct super_block *sb) 327 { 328 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 329 if (sbinfo->max_inodes) { 330 raw_spin_lock(&sbinfo->stat_lock); 331 sbinfo->free_inodes++; 332 raw_spin_unlock(&sbinfo->stat_lock); 333 } 334 } 335 336 /** 337 * shmem_recalc_inode - recalculate the block usage of an inode 338 * @inode: inode to recalc 339 * 340 * We have to calculate the free blocks since the mm can drop 341 * undirtied hole pages behind our back. 342 * 343 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 344 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 345 * 346 * It has to be called with the spinlock held. 347 */ 348 static void shmem_recalc_inode(struct inode *inode) 349 { 350 struct shmem_inode_info *info = SHMEM_I(inode); 351 long freed; 352 353 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 354 if (freed > 0) { 355 info->alloced -= freed; 356 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 357 shmem_inode_unacct_blocks(inode, freed); 358 } 359 } 360 361 bool shmem_charge(struct inode *inode, long pages) 362 { 363 struct shmem_inode_info *info = SHMEM_I(inode); 364 unsigned long flags; 365 366 if (!shmem_inode_acct_block(inode, pages)) 367 return false; 368 369 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */ 370 inode->i_mapping->nrpages += pages; 371 372 spin_lock_irqsave(&info->lock, flags); 373 info->alloced += pages; 374 inode->i_blocks += pages * BLOCKS_PER_PAGE; 375 shmem_recalc_inode(inode); 376 spin_unlock_irqrestore(&info->lock, flags); 377 378 return true; 379 } 380 381 void shmem_uncharge(struct inode *inode, long pages) 382 { 383 struct shmem_inode_info *info = SHMEM_I(inode); 384 unsigned long flags; 385 386 /* nrpages adjustment done by __filemap_remove_folio() or caller */ 387 388 spin_lock_irqsave(&info->lock, flags); 389 info->alloced -= pages; 390 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 391 shmem_recalc_inode(inode); 392 spin_unlock_irqrestore(&info->lock, flags); 393 394 shmem_inode_unacct_blocks(inode, pages); 395 } 396 397 /* 398 * Replace item expected in xarray by a new item, while holding xa_lock. 399 */ 400 static int shmem_replace_entry(struct address_space *mapping, 401 pgoff_t index, void *expected, void *replacement) 402 { 403 XA_STATE(xas, &mapping->i_pages, index); 404 void *item; 405 406 VM_BUG_ON(!expected); 407 VM_BUG_ON(!replacement); 408 item = xas_load(&xas); 409 if (item != expected) 410 return -ENOENT; 411 xas_store(&xas, replacement); 412 return 0; 413 } 414 415 /* 416 * Sometimes, before we decide whether to proceed or to fail, we must check 417 * that an entry was not already brought back from swap by a racing thread. 418 * 419 * Checking page is not enough: by the time a SwapCache page is locked, it 420 * might be reused, and again be SwapCache, using the same swap as before. 421 */ 422 static bool shmem_confirm_swap(struct address_space *mapping, 423 pgoff_t index, swp_entry_t swap) 424 { 425 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap); 426 } 427 428 /* 429 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 430 * 431 * SHMEM_HUGE_NEVER: 432 * disables huge pages for the mount; 433 * SHMEM_HUGE_ALWAYS: 434 * enables huge pages for the mount; 435 * SHMEM_HUGE_WITHIN_SIZE: 436 * only allocate huge pages if the page will be fully within i_size, 437 * also respect fadvise()/madvise() hints; 438 * SHMEM_HUGE_ADVISE: 439 * only allocate huge pages if requested with fadvise()/madvise(); 440 */ 441 442 #define SHMEM_HUGE_NEVER 0 443 #define SHMEM_HUGE_ALWAYS 1 444 #define SHMEM_HUGE_WITHIN_SIZE 2 445 #define SHMEM_HUGE_ADVISE 3 446 447 /* 448 * Special values. 449 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 450 * 451 * SHMEM_HUGE_DENY: 452 * disables huge on shm_mnt and all mounts, for emergency use; 453 * SHMEM_HUGE_FORCE: 454 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 455 * 456 */ 457 #define SHMEM_HUGE_DENY (-1) 458 #define SHMEM_HUGE_FORCE (-2) 459 460 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 461 /* ifdef here to avoid bloating shmem.o when not necessary */ 462 463 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER; 464 465 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode, 466 pgoff_t index, bool shmem_huge_force) 467 { 468 loff_t i_size; 469 470 if (!S_ISREG(inode->i_mode)) 471 return false; 472 if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) || 473 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))) 474 return false; 475 if (shmem_huge_force) 476 return true; 477 if (shmem_huge == SHMEM_HUGE_FORCE) 478 return true; 479 if (shmem_huge == SHMEM_HUGE_DENY) 480 return false; 481 482 switch (SHMEM_SB(inode->i_sb)->huge) { 483 case SHMEM_HUGE_ALWAYS: 484 return true; 485 case SHMEM_HUGE_WITHIN_SIZE: 486 index = round_up(index + 1, HPAGE_PMD_NR); 487 i_size = round_up(i_size_read(inode), PAGE_SIZE); 488 if (i_size >> PAGE_SHIFT >= index) 489 return true; 490 fallthrough; 491 case SHMEM_HUGE_ADVISE: 492 if (vma && (vma->vm_flags & VM_HUGEPAGE)) 493 return true; 494 fallthrough; 495 default: 496 return false; 497 } 498 } 499 500 #if defined(CONFIG_SYSFS) 501 static int shmem_parse_huge(const char *str) 502 { 503 if (!strcmp(str, "never")) 504 return SHMEM_HUGE_NEVER; 505 if (!strcmp(str, "always")) 506 return SHMEM_HUGE_ALWAYS; 507 if (!strcmp(str, "within_size")) 508 return SHMEM_HUGE_WITHIN_SIZE; 509 if (!strcmp(str, "advise")) 510 return SHMEM_HUGE_ADVISE; 511 if (!strcmp(str, "deny")) 512 return SHMEM_HUGE_DENY; 513 if (!strcmp(str, "force")) 514 return SHMEM_HUGE_FORCE; 515 return -EINVAL; 516 } 517 #endif 518 519 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 520 static const char *shmem_format_huge(int huge) 521 { 522 switch (huge) { 523 case SHMEM_HUGE_NEVER: 524 return "never"; 525 case SHMEM_HUGE_ALWAYS: 526 return "always"; 527 case SHMEM_HUGE_WITHIN_SIZE: 528 return "within_size"; 529 case SHMEM_HUGE_ADVISE: 530 return "advise"; 531 case SHMEM_HUGE_DENY: 532 return "deny"; 533 case SHMEM_HUGE_FORCE: 534 return "force"; 535 default: 536 VM_BUG_ON(1); 537 return "bad_val"; 538 } 539 } 540 #endif 541 542 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 543 struct shrink_control *sc, unsigned long nr_to_split) 544 { 545 LIST_HEAD(list), *pos, *next; 546 LIST_HEAD(to_remove); 547 struct inode *inode; 548 struct shmem_inode_info *info; 549 struct folio *folio; 550 unsigned long batch = sc ? sc->nr_to_scan : 128; 551 int split = 0; 552 553 if (list_empty(&sbinfo->shrinklist)) 554 return SHRINK_STOP; 555 556 spin_lock(&sbinfo->shrinklist_lock); 557 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 558 info = list_entry(pos, struct shmem_inode_info, shrinklist); 559 560 /* pin the inode */ 561 inode = igrab(&info->vfs_inode); 562 563 /* inode is about to be evicted */ 564 if (!inode) { 565 list_del_init(&info->shrinklist); 566 goto next; 567 } 568 569 /* Check if there's anything to gain */ 570 if (round_up(inode->i_size, PAGE_SIZE) == 571 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 572 list_move(&info->shrinklist, &to_remove); 573 goto next; 574 } 575 576 list_move(&info->shrinklist, &list); 577 next: 578 sbinfo->shrinklist_len--; 579 if (!--batch) 580 break; 581 } 582 spin_unlock(&sbinfo->shrinklist_lock); 583 584 list_for_each_safe(pos, next, &to_remove) { 585 info = list_entry(pos, struct shmem_inode_info, shrinklist); 586 inode = &info->vfs_inode; 587 list_del_init(&info->shrinklist); 588 iput(inode); 589 } 590 591 list_for_each_safe(pos, next, &list) { 592 int ret; 593 pgoff_t index; 594 595 info = list_entry(pos, struct shmem_inode_info, shrinklist); 596 inode = &info->vfs_inode; 597 598 if (nr_to_split && split >= nr_to_split) 599 goto move_back; 600 601 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT; 602 folio = filemap_get_folio(inode->i_mapping, index); 603 if (!folio) 604 goto drop; 605 606 /* No huge page at the end of the file: nothing to split */ 607 if (!folio_test_large(folio)) { 608 folio_put(folio); 609 goto drop; 610 } 611 612 /* 613 * Move the inode on the list back to shrinklist if we failed 614 * to lock the page at this time. 615 * 616 * Waiting for the lock may lead to deadlock in the 617 * reclaim path. 618 */ 619 if (!folio_trylock(folio)) { 620 folio_put(folio); 621 goto move_back; 622 } 623 624 ret = split_folio(folio); 625 folio_unlock(folio); 626 folio_put(folio); 627 628 /* If split failed move the inode on the list back to shrinklist */ 629 if (ret) 630 goto move_back; 631 632 split++; 633 drop: 634 list_del_init(&info->shrinklist); 635 goto put; 636 move_back: 637 /* 638 * Make sure the inode is either on the global list or deleted 639 * from any local list before iput() since it could be deleted 640 * in another thread once we put the inode (then the local list 641 * is corrupted). 642 */ 643 spin_lock(&sbinfo->shrinklist_lock); 644 list_move(&info->shrinklist, &sbinfo->shrinklist); 645 sbinfo->shrinklist_len++; 646 spin_unlock(&sbinfo->shrinklist_lock); 647 put: 648 iput(inode); 649 } 650 651 return split; 652 } 653 654 static long shmem_unused_huge_scan(struct super_block *sb, 655 struct shrink_control *sc) 656 { 657 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 658 659 if (!READ_ONCE(sbinfo->shrinklist_len)) 660 return SHRINK_STOP; 661 662 return shmem_unused_huge_shrink(sbinfo, sc, 0); 663 } 664 665 static long shmem_unused_huge_count(struct super_block *sb, 666 struct shrink_control *sc) 667 { 668 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 669 return READ_ONCE(sbinfo->shrinklist_len); 670 } 671 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 672 673 #define shmem_huge SHMEM_HUGE_DENY 674 675 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode, 676 pgoff_t index, bool shmem_huge_force) 677 { 678 return false; 679 } 680 681 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 682 struct shrink_control *sc, unsigned long nr_to_split) 683 { 684 return 0; 685 } 686 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 687 688 /* 689 * Like filemap_add_folio, but error if expected item has gone. 690 */ 691 static int shmem_add_to_page_cache(struct folio *folio, 692 struct address_space *mapping, 693 pgoff_t index, void *expected, gfp_t gfp, 694 struct mm_struct *charge_mm) 695 { 696 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); 697 long nr = folio_nr_pages(folio); 698 int error; 699 700 VM_BUG_ON_FOLIO(index != round_down(index, nr), folio); 701 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 702 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio); 703 VM_BUG_ON(expected && folio_test_large(folio)); 704 705 folio_ref_add(folio, nr); 706 folio->mapping = mapping; 707 folio->index = index; 708 709 if (!folio_test_swapcache(folio)) { 710 error = mem_cgroup_charge(folio, charge_mm, gfp); 711 if (error) { 712 if (folio_test_pmd_mappable(folio)) { 713 count_vm_event(THP_FILE_FALLBACK); 714 count_vm_event(THP_FILE_FALLBACK_CHARGE); 715 } 716 goto error; 717 } 718 } 719 folio_throttle_swaprate(folio, gfp); 720 721 do { 722 xas_lock_irq(&xas); 723 if (expected != xas_find_conflict(&xas)) { 724 xas_set_err(&xas, -EEXIST); 725 goto unlock; 726 } 727 if (expected && xas_find_conflict(&xas)) { 728 xas_set_err(&xas, -EEXIST); 729 goto unlock; 730 } 731 xas_store(&xas, folio); 732 if (xas_error(&xas)) 733 goto unlock; 734 if (folio_test_pmd_mappable(folio)) { 735 count_vm_event(THP_FILE_ALLOC); 736 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr); 737 } 738 mapping->nrpages += nr; 739 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); 740 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr); 741 unlock: 742 xas_unlock_irq(&xas); 743 } while (xas_nomem(&xas, gfp)); 744 745 if (xas_error(&xas)) { 746 error = xas_error(&xas); 747 goto error; 748 } 749 750 return 0; 751 error: 752 folio->mapping = NULL; 753 folio_ref_sub(folio, nr); 754 return error; 755 } 756 757 /* 758 * Like delete_from_page_cache, but substitutes swap for @folio. 759 */ 760 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) 761 { 762 struct address_space *mapping = folio->mapping; 763 long nr = folio_nr_pages(folio); 764 int error; 765 766 xa_lock_irq(&mapping->i_pages); 767 error = shmem_replace_entry(mapping, folio->index, folio, radswap); 768 folio->mapping = NULL; 769 mapping->nrpages -= nr; 770 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 771 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); 772 xa_unlock_irq(&mapping->i_pages); 773 folio_put(folio); 774 BUG_ON(error); 775 } 776 777 /* 778 * Remove swap entry from page cache, free the swap and its page cache. 779 */ 780 static int shmem_free_swap(struct address_space *mapping, 781 pgoff_t index, void *radswap) 782 { 783 void *old; 784 785 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); 786 if (old != radswap) 787 return -ENOENT; 788 free_swap_and_cache(radix_to_swp_entry(radswap)); 789 return 0; 790 } 791 792 /* 793 * Determine (in bytes) how many of the shmem object's pages mapped by the 794 * given offsets are swapped out. 795 * 796 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 797 * as long as the inode doesn't go away and racy results are not a problem. 798 */ 799 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 800 pgoff_t start, pgoff_t end) 801 { 802 XA_STATE(xas, &mapping->i_pages, start); 803 struct page *page; 804 unsigned long swapped = 0; 805 806 rcu_read_lock(); 807 xas_for_each(&xas, page, end - 1) { 808 if (xas_retry(&xas, page)) 809 continue; 810 if (xa_is_value(page)) 811 swapped++; 812 813 if (need_resched()) { 814 xas_pause(&xas); 815 cond_resched_rcu(); 816 } 817 } 818 819 rcu_read_unlock(); 820 821 return swapped << PAGE_SHIFT; 822 } 823 824 /* 825 * Determine (in bytes) how many of the shmem object's pages mapped by the 826 * given vma is swapped out. 827 * 828 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 829 * as long as the inode doesn't go away and racy results are not a problem. 830 */ 831 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 832 { 833 struct inode *inode = file_inode(vma->vm_file); 834 struct shmem_inode_info *info = SHMEM_I(inode); 835 struct address_space *mapping = inode->i_mapping; 836 unsigned long swapped; 837 838 /* Be careful as we don't hold info->lock */ 839 swapped = READ_ONCE(info->swapped); 840 841 /* 842 * The easier cases are when the shmem object has nothing in swap, or 843 * the vma maps it whole. Then we can simply use the stats that we 844 * already track. 845 */ 846 if (!swapped) 847 return 0; 848 849 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 850 return swapped << PAGE_SHIFT; 851 852 /* Here comes the more involved part */ 853 return shmem_partial_swap_usage(mapping, vma->vm_pgoff, 854 vma->vm_pgoff + vma_pages(vma)); 855 } 856 857 /* 858 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 859 */ 860 void shmem_unlock_mapping(struct address_space *mapping) 861 { 862 struct folio_batch fbatch; 863 pgoff_t index = 0; 864 865 folio_batch_init(&fbatch); 866 /* 867 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 868 */ 869 while (!mapping_unevictable(mapping) && 870 filemap_get_folios(mapping, &index, ~0UL, &fbatch)) { 871 check_move_unevictable_folios(&fbatch); 872 folio_batch_release(&fbatch); 873 cond_resched(); 874 } 875 } 876 877 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) 878 { 879 struct folio *folio; 880 881 /* 882 * At first avoid shmem_get_folio(,,,SGP_READ): that fails 883 * beyond i_size, and reports fallocated pages as holes. 884 */ 885 folio = __filemap_get_folio(inode->i_mapping, index, 886 FGP_ENTRY | FGP_LOCK, 0); 887 if (!xa_is_value(folio)) 888 return folio; 889 /* 890 * But read a page back from swap if any of it is within i_size 891 * (although in some cases this is just a waste of time). 892 */ 893 folio = NULL; 894 shmem_get_folio(inode, index, &folio, SGP_READ); 895 return folio; 896 } 897 898 /* 899 * Remove range of pages and swap entries from page cache, and free them. 900 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 901 */ 902 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 903 bool unfalloc) 904 { 905 struct address_space *mapping = inode->i_mapping; 906 struct shmem_inode_info *info = SHMEM_I(inode); 907 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 908 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 909 struct folio_batch fbatch; 910 pgoff_t indices[PAGEVEC_SIZE]; 911 struct folio *folio; 912 bool same_folio; 913 long nr_swaps_freed = 0; 914 pgoff_t index; 915 int i; 916 917 if (lend == -1) 918 end = -1; /* unsigned, so actually very big */ 919 920 if (info->fallocend > start && info->fallocend <= end && !unfalloc) 921 info->fallocend = start; 922 923 folio_batch_init(&fbatch); 924 index = start; 925 while (index < end && find_lock_entries(mapping, index, end - 1, 926 &fbatch, indices)) { 927 for (i = 0; i < folio_batch_count(&fbatch); i++) { 928 folio = fbatch.folios[i]; 929 930 index = indices[i]; 931 932 if (xa_is_value(folio)) { 933 if (unfalloc) 934 continue; 935 nr_swaps_freed += !shmem_free_swap(mapping, 936 index, folio); 937 continue; 938 } 939 index += folio_nr_pages(folio) - 1; 940 941 if (!unfalloc || !folio_test_uptodate(folio)) 942 truncate_inode_folio(mapping, folio); 943 folio_unlock(folio); 944 } 945 folio_batch_remove_exceptionals(&fbatch); 946 folio_batch_release(&fbatch); 947 cond_resched(); 948 index++; 949 } 950 951 /* 952 * When undoing a failed fallocate, we want none of the partial folio 953 * zeroing and splitting below, but shall want to truncate the whole 954 * folio when !uptodate indicates that it was added by this fallocate, 955 * even when [lstart, lend] covers only a part of the folio. 956 */ 957 if (unfalloc) 958 goto whole_folios; 959 960 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT); 961 folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT); 962 if (folio) { 963 same_folio = lend < folio_pos(folio) + folio_size(folio); 964 folio_mark_dirty(folio); 965 if (!truncate_inode_partial_folio(folio, lstart, lend)) { 966 start = folio->index + folio_nr_pages(folio); 967 if (same_folio) 968 end = folio->index; 969 } 970 folio_unlock(folio); 971 folio_put(folio); 972 folio = NULL; 973 } 974 975 if (!same_folio) 976 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); 977 if (folio) { 978 folio_mark_dirty(folio); 979 if (!truncate_inode_partial_folio(folio, lstart, lend)) 980 end = folio->index; 981 folio_unlock(folio); 982 folio_put(folio); 983 } 984 985 whole_folios: 986 987 index = start; 988 while (index < end) { 989 cond_resched(); 990 991 if (!find_get_entries(mapping, index, end - 1, &fbatch, 992 indices)) { 993 /* If all gone or hole-punch or unfalloc, we're done */ 994 if (index == start || end != -1) 995 break; 996 /* But if truncating, restart to make sure all gone */ 997 index = start; 998 continue; 999 } 1000 for (i = 0; i < folio_batch_count(&fbatch); i++) { 1001 folio = fbatch.folios[i]; 1002 1003 index = indices[i]; 1004 if (xa_is_value(folio)) { 1005 if (unfalloc) 1006 continue; 1007 if (shmem_free_swap(mapping, index, folio)) { 1008 /* Swap was replaced by page: retry */ 1009 index--; 1010 break; 1011 } 1012 nr_swaps_freed++; 1013 continue; 1014 } 1015 1016 folio_lock(folio); 1017 1018 if (!unfalloc || !folio_test_uptodate(folio)) { 1019 if (folio_mapping(folio) != mapping) { 1020 /* Page was replaced by swap: retry */ 1021 folio_unlock(folio); 1022 index--; 1023 break; 1024 } 1025 VM_BUG_ON_FOLIO(folio_test_writeback(folio), 1026 folio); 1027 truncate_inode_folio(mapping, folio); 1028 } 1029 index = folio->index + folio_nr_pages(folio) - 1; 1030 folio_unlock(folio); 1031 } 1032 folio_batch_remove_exceptionals(&fbatch); 1033 folio_batch_release(&fbatch); 1034 index++; 1035 } 1036 1037 spin_lock_irq(&info->lock); 1038 info->swapped -= nr_swaps_freed; 1039 shmem_recalc_inode(inode); 1040 spin_unlock_irq(&info->lock); 1041 } 1042 1043 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 1044 { 1045 shmem_undo_range(inode, lstart, lend, false); 1046 inode->i_ctime = inode->i_mtime = current_time(inode); 1047 inode_inc_iversion(inode); 1048 } 1049 EXPORT_SYMBOL_GPL(shmem_truncate_range); 1050 1051 static int shmem_getattr(struct user_namespace *mnt_userns, 1052 const struct path *path, struct kstat *stat, 1053 u32 request_mask, unsigned int query_flags) 1054 { 1055 struct inode *inode = path->dentry->d_inode; 1056 struct shmem_inode_info *info = SHMEM_I(inode); 1057 1058 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 1059 spin_lock_irq(&info->lock); 1060 shmem_recalc_inode(inode); 1061 spin_unlock_irq(&info->lock); 1062 } 1063 if (info->fsflags & FS_APPEND_FL) 1064 stat->attributes |= STATX_ATTR_APPEND; 1065 if (info->fsflags & FS_IMMUTABLE_FL) 1066 stat->attributes |= STATX_ATTR_IMMUTABLE; 1067 if (info->fsflags & FS_NODUMP_FL) 1068 stat->attributes |= STATX_ATTR_NODUMP; 1069 stat->attributes_mask |= (STATX_ATTR_APPEND | 1070 STATX_ATTR_IMMUTABLE | 1071 STATX_ATTR_NODUMP); 1072 generic_fillattr(&init_user_ns, inode, stat); 1073 1074 if (shmem_is_huge(NULL, inode, 0, false)) 1075 stat->blksize = HPAGE_PMD_SIZE; 1076 1077 if (request_mask & STATX_BTIME) { 1078 stat->result_mask |= STATX_BTIME; 1079 stat->btime.tv_sec = info->i_crtime.tv_sec; 1080 stat->btime.tv_nsec = info->i_crtime.tv_nsec; 1081 } 1082 1083 return 0; 1084 } 1085 1086 static int shmem_setattr(struct user_namespace *mnt_userns, 1087 struct dentry *dentry, struct iattr *attr) 1088 { 1089 struct inode *inode = d_inode(dentry); 1090 struct shmem_inode_info *info = SHMEM_I(inode); 1091 int error; 1092 bool update_mtime = false; 1093 bool update_ctime = true; 1094 1095 error = setattr_prepare(&init_user_ns, dentry, attr); 1096 if (error) 1097 return error; 1098 1099 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 1100 loff_t oldsize = inode->i_size; 1101 loff_t newsize = attr->ia_size; 1102 1103 /* protected by i_rwsem */ 1104 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 1105 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 1106 return -EPERM; 1107 1108 if (newsize != oldsize) { 1109 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1110 oldsize, newsize); 1111 if (error) 1112 return error; 1113 i_size_write(inode, newsize); 1114 update_mtime = true; 1115 } else { 1116 update_ctime = false; 1117 } 1118 if (newsize <= oldsize) { 1119 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1120 if (oldsize > holebegin) 1121 unmap_mapping_range(inode->i_mapping, 1122 holebegin, 0, 1); 1123 if (info->alloced) 1124 shmem_truncate_range(inode, 1125 newsize, (loff_t)-1); 1126 /* unmap again to remove racily COWed private pages */ 1127 if (oldsize > holebegin) 1128 unmap_mapping_range(inode->i_mapping, 1129 holebegin, 0, 1); 1130 } 1131 } 1132 1133 setattr_copy(&init_user_ns, inode, attr); 1134 if (attr->ia_valid & ATTR_MODE) 1135 error = posix_acl_chmod(&init_user_ns, dentry, inode->i_mode); 1136 if (!error && update_ctime) { 1137 inode->i_ctime = current_time(inode); 1138 if (update_mtime) 1139 inode->i_mtime = inode->i_ctime; 1140 inode_inc_iversion(inode); 1141 } 1142 return error; 1143 } 1144 1145 static void shmem_evict_inode(struct inode *inode) 1146 { 1147 struct shmem_inode_info *info = SHMEM_I(inode); 1148 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1149 1150 if (shmem_mapping(inode->i_mapping)) { 1151 shmem_unacct_size(info->flags, inode->i_size); 1152 inode->i_size = 0; 1153 mapping_set_exiting(inode->i_mapping); 1154 shmem_truncate_range(inode, 0, (loff_t)-1); 1155 if (!list_empty(&info->shrinklist)) { 1156 spin_lock(&sbinfo->shrinklist_lock); 1157 if (!list_empty(&info->shrinklist)) { 1158 list_del_init(&info->shrinklist); 1159 sbinfo->shrinklist_len--; 1160 } 1161 spin_unlock(&sbinfo->shrinklist_lock); 1162 } 1163 while (!list_empty(&info->swaplist)) { 1164 /* Wait while shmem_unuse() is scanning this inode... */ 1165 wait_var_event(&info->stop_eviction, 1166 !atomic_read(&info->stop_eviction)); 1167 mutex_lock(&shmem_swaplist_mutex); 1168 /* ...but beware of the race if we peeked too early */ 1169 if (!atomic_read(&info->stop_eviction)) 1170 list_del_init(&info->swaplist); 1171 mutex_unlock(&shmem_swaplist_mutex); 1172 } 1173 } 1174 1175 simple_xattrs_free(&info->xattrs); 1176 WARN_ON(inode->i_blocks); 1177 shmem_free_inode(inode->i_sb); 1178 clear_inode(inode); 1179 } 1180 1181 static int shmem_find_swap_entries(struct address_space *mapping, 1182 pgoff_t start, struct folio_batch *fbatch, 1183 pgoff_t *indices, unsigned int type) 1184 { 1185 XA_STATE(xas, &mapping->i_pages, start); 1186 struct folio *folio; 1187 swp_entry_t entry; 1188 1189 rcu_read_lock(); 1190 xas_for_each(&xas, folio, ULONG_MAX) { 1191 if (xas_retry(&xas, folio)) 1192 continue; 1193 1194 if (!xa_is_value(folio)) 1195 continue; 1196 1197 entry = radix_to_swp_entry(folio); 1198 /* 1199 * swapin error entries can be found in the mapping. But they're 1200 * deliberately ignored here as we've done everything we can do. 1201 */ 1202 if (swp_type(entry) != type) 1203 continue; 1204 1205 indices[folio_batch_count(fbatch)] = xas.xa_index; 1206 if (!folio_batch_add(fbatch, folio)) 1207 break; 1208 1209 if (need_resched()) { 1210 xas_pause(&xas); 1211 cond_resched_rcu(); 1212 } 1213 } 1214 rcu_read_unlock(); 1215 1216 return xas.xa_index; 1217 } 1218 1219 /* 1220 * Move the swapped pages for an inode to page cache. Returns the count 1221 * of pages swapped in, or the error in case of failure. 1222 */ 1223 static int shmem_unuse_swap_entries(struct inode *inode, 1224 struct folio_batch *fbatch, pgoff_t *indices) 1225 { 1226 int i = 0; 1227 int ret = 0; 1228 int error = 0; 1229 struct address_space *mapping = inode->i_mapping; 1230 1231 for (i = 0; i < folio_batch_count(fbatch); i++) { 1232 struct folio *folio = fbatch->folios[i]; 1233 1234 if (!xa_is_value(folio)) 1235 continue; 1236 error = shmem_swapin_folio(inode, indices[i], 1237 &folio, SGP_CACHE, 1238 mapping_gfp_mask(mapping), 1239 NULL, NULL); 1240 if (error == 0) { 1241 folio_unlock(folio); 1242 folio_put(folio); 1243 ret++; 1244 } 1245 if (error == -ENOMEM) 1246 break; 1247 error = 0; 1248 } 1249 return error ? error : ret; 1250 } 1251 1252 /* 1253 * If swap found in inode, free it and move page from swapcache to filecache. 1254 */ 1255 static int shmem_unuse_inode(struct inode *inode, unsigned int type) 1256 { 1257 struct address_space *mapping = inode->i_mapping; 1258 pgoff_t start = 0; 1259 struct folio_batch fbatch; 1260 pgoff_t indices[PAGEVEC_SIZE]; 1261 int ret = 0; 1262 1263 do { 1264 folio_batch_init(&fbatch); 1265 shmem_find_swap_entries(mapping, start, &fbatch, indices, type); 1266 if (folio_batch_count(&fbatch) == 0) { 1267 ret = 0; 1268 break; 1269 } 1270 1271 ret = shmem_unuse_swap_entries(inode, &fbatch, indices); 1272 if (ret < 0) 1273 break; 1274 1275 start = indices[folio_batch_count(&fbatch) - 1]; 1276 } while (true); 1277 1278 return ret; 1279 } 1280 1281 /* 1282 * Read all the shared memory data that resides in the swap 1283 * device 'type' back into memory, so the swap device can be 1284 * unused. 1285 */ 1286 int shmem_unuse(unsigned int type) 1287 { 1288 struct shmem_inode_info *info, *next; 1289 int error = 0; 1290 1291 if (list_empty(&shmem_swaplist)) 1292 return 0; 1293 1294 mutex_lock(&shmem_swaplist_mutex); 1295 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) { 1296 if (!info->swapped) { 1297 list_del_init(&info->swaplist); 1298 continue; 1299 } 1300 /* 1301 * Drop the swaplist mutex while searching the inode for swap; 1302 * but before doing so, make sure shmem_evict_inode() will not 1303 * remove placeholder inode from swaplist, nor let it be freed 1304 * (igrab() would protect from unlink, but not from unmount). 1305 */ 1306 atomic_inc(&info->stop_eviction); 1307 mutex_unlock(&shmem_swaplist_mutex); 1308 1309 error = shmem_unuse_inode(&info->vfs_inode, type); 1310 cond_resched(); 1311 1312 mutex_lock(&shmem_swaplist_mutex); 1313 next = list_next_entry(info, swaplist); 1314 if (!info->swapped) 1315 list_del_init(&info->swaplist); 1316 if (atomic_dec_and_test(&info->stop_eviction)) 1317 wake_up_var(&info->stop_eviction); 1318 if (error) 1319 break; 1320 } 1321 mutex_unlock(&shmem_swaplist_mutex); 1322 1323 return error; 1324 } 1325 1326 /* 1327 * Move the page from the page cache to the swap cache. 1328 */ 1329 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1330 { 1331 struct folio *folio = page_folio(page); 1332 struct shmem_inode_info *info; 1333 struct address_space *mapping; 1334 struct inode *inode; 1335 swp_entry_t swap; 1336 pgoff_t index; 1337 1338 /* 1339 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or 1340 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages, 1341 * and its shmem_writeback() needs them to be split when swapping. 1342 */ 1343 if (folio_test_large(folio)) { 1344 /* Ensure the subpages are still dirty */ 1345 folio_test_set_dirty(folio); 1346 if (split_huge_page(page) < 0) 1347 goto redirty; 1348 folio = page_folio(page); 1349 folio_clear_dirty(folio); 1350 } 1351 1352 BUG_ON(!folio_test_locked(folio)); 1353 mapping = folio->mapping; 1354 index = folio->index; 1355 inode = mapping->host; 1356 info = SHMEM_I(inode); 1357 if (info->flags & VM_LOCKED) 1358 goto redirty; 1359 if (!total_swap_pages) 1360 goto redirty; 1361 1362 /* 1363 * Our capabilities prevent regular writeback or sync from ever calling 1364 * shmem_writepage; but a stacking filesystem might use ->writepage of 1365 * its underlying filesystem, in which case tmpfs should write out to 1366 * swap only in response to memory pressure, and not for the writeback 1367 * threads or sync. 1368 */ 1369 if (!wbc->for_reclaim) { 1370 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1371 goto redirty; 1372 } 1373 1374 /* 1375 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1376 * value into swapfile.c, the only way we can correctly account for a 1377 * fallocated folio arriving here is now to initialize it and write it. 1378 * 1379 * That's okay for a folio already fallocated earlier, but if we have 1380 * not yet completed the fallocation, then (a) we want to keep track 1381 * of this folio in case we have to undo it, and (b) it may not be a 1382 * good idea to continue anyway, once we're pushing into swap. So 1383 * reactivate the folio, and let shmem_fallocate() quit when too many. 1384 */ 1385 if (!folio_test_uptodate(folio)) { 1386 if (inode->i_private) { 1387 struct shmem_falloc *shmem_falloc; 1388 spin_lock(&inode->i_lock); 1389 shmem_falloc = inode->i_private; 1390 if (shmem_falloc && 1391 !shmem_falloc->waitq && 1392 index >= shmem_falloc->start && 1393 index < shmem_falloc->next) 1394 shmem_falloc->nr_unswapped++; 1395 else 1396 shmem_falloc = NULL; 1397 spin_unlock(&inode->i_lock); 1398 if (shmem_falloc) 1399 goto redirty; 1400 } 1401 folio_zero_range(folio, 0, folio_size(folio)); 1402 flush_dcache_folio(folio); 1403 folio_mark_uptodate(folio); 1404 } 1405 1406 swap = folio_alloc_swap(folio); 1407 if (!swap.val) 1408 goto redirty; 1409 1410 /* 1411 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1412 * if it's not already there. Do it now before the folio is 1413 * moved to swap cache, when its pagelock no longer protects 1414 * the inode from eviction. But don't unlock the mutex until 1415 * we've incremented swapped, because shmem_unuse_inode() will 1416 * prune a !swapped inode from the swaplist under this mutex. 1417 */ 1418 mutex_lock(&shmem_swaplist_mutex); 1419 if (list_empty(&info->swaplist)) 1420 list_add(&info->swaplist, &shmem_swaplist); 1421 1422 if (add_to_swap_cache(folio, swap, 1423 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN, 1424 NULL) == 0) { 1425 spin_lock_irq(&info->lock); 1426 shmem_recalc_inode(inode); 1427 info->swapped++; 1428 spin_unlock_irq(&info->lock); 1429 1430 swap_shmem_alloc(swap); 1431 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap)); 1432 1433 mutex_unlock(&shmem_swaplist_mutex); 1434 BUG_ON(folio_mapped(folio)); 1435 swap_writepage(&folio->page, wbc); 1436 return 0; 1437 } 1438 1439 mutex_unlock(&shmem_swaplist_mutex); 1440 put_swap_folio(folio, swap); 1441 redirty: 1442 folio_mark_dirty(folio); 1443 if (wbc->for_reclaim) 1444 return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ 1445 folio_unlock(folio); 1446 return 0; 1447 } 1448 1449 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1450 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1451 { 1452 char buffer[64]; 1453 1454 if (!mpol || mpol->mode == MPOL_DEFAULT) 1455 return; /* show nothing */ 1456 1457 mpol_to_str(buffer, sizeof(buffer), mpol); 1458 1459 seq_printf(seq, ",mpol=%s", buffer); 1460 } 1461 1462 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1463 { 1464 struct mempolicy *mpol = NULL; 1465 if (sbinfo->mpol) { 1466 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1467 mpol = sbinfo->mpol; 1468 mpol_get(mpol); 1469 raw_spin_unlock(&sbinfo->stat_lock); 1470 } 1471 return mpol; 1472 } 1473 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1474 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1475 { 1476 } 1477 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1478 { 1479 return NULL; 1480 } 1481 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1482 #ifndef CONFIG_NUMA 1483 #define vm_policy vm_private_data 1484 #endif 1485 1486 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1487 struct shmem_inode_info *info, pgoff_t index) 1488 { 1489 /* Create a pseudo vma that just contains the policy */ 1490 vma_init(vma, NULL); 1491 /* Bias interleave by inode number to distribute better across nodes */ 1492 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1493 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1494 } 1495 1496 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1497 { 1498 /* Drop reference taken by mpol_shared_policy_lookup() */ 1499 mpol_cond_put(vma->vm_policy); 1500 } 1501 1502 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1503 struct shmem_inode_info *info, pgoff_t index) 1504 { 1505 struct vm_area_struct pvma; 1506 struct page *page; 1507 struct vm_fault vmf = { 1508 .vma = &pvma, 1509 }; 1510 1511 shmem_pseudo_vma_init(&pvma, info, index); 1512 page = swap_cluster_readahead(swap, gfp, &vmf); 1513 shmem_pseudo_vma_destroy(&pvma); 1514 1515 if (!page) 1516 return NULL; 1517 return page_folio(page); 1518 } 1519 1520 /* 1521 * Make sure huge_gfp is always more limited than limit_gfp. 1522 * Some of the flags set permissions, while others set limitations. 1523 */ 1524 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) 1525 { 1526 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; 1527 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; 1528 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; 1529 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK); 1530 1531 /* Allow allocations only from the originally specified zones. */ 1532 result |= zoneflags; 1533 1534 /* 1535 * Minimize the result gfp by taking the union with the deny flags, 1536 * and the intersection of the allow flags. 1537 */ 1538 result |= (limit_gfp & denyflags); 1539 result |= (huge_gfp & limit_gfp) & allowflags; 1540 1541 return result; 1542 } 1543 1544 static struct folio *shmem_alloc_hugefolio(gfp_t gfp, 1545 struct shmem_inode_info *info, pgoff_t index) 1546 { 1547 struct vm_area_struct pvma; 1548 struct address_space *mapping = info->vfs_inode.i_mapping; 1549 pgoff_t hindex; 1550 struct folio *folio; 1551 1552 hindex = round_down(index, HPAGE_PMD_NR); 1553 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1, 1554 XA_PRESENT)) 1555 return NULL; 1556 1557 shmem_pseudo_vma_init(&pvma, info, hindex); 1558 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true); 1559 shmem_pseudo_vma_destroy(&pvma); 1560 if (!folio) 1561 count_vm_event(THP_FILE_FALLBACK); 1562 return folio; 1563 } 1564 1565 static struct folio *shmem_alloc_folio(gfp_t gfp, 1566 struct shmem_inode_info *info, pgoff_t index) 1567 { 1568 struct vm_area_struct pvma; 1569 struct folio *folio; 1570 1571 shmem_pseudo_vma_init(&pvma, info, index); 1572 folio = vma_alloc_folio(gfp, 0, &pvma, 0, false); 1573 shmem_pseudo_vma_destroy(&pvma); 1574 1575 return folio; 1576 } 1577 1578 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode, 1579 pgoff_t index, bool huge) 1580 { 1581 struct shmem_inode_info *info = SHMEM_I(inode); 1582 struct folio *folio; 1583 int nr; 1584 int err = -ENOSPC; 1585 1586 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1587 huge = false; 1588 nr = huge ? HPAGE_PMD_NR : 1; 1589 1590 if (!shmem_inode_acct_block(inode, nr)) 1591 goto failed; 1592 1593 if (huge) 1594 folio = shmem_alloc_hugefolio(gfp, info, index); 1595 else 1596 folio = shmem_alloc_folio(gfp, info, index); 1597 if (folio) { 1598 __folio_set_locked(folio); 1599 __folio_set_swapbacked(folio); 1600 return folio; 1601 } 1602 1603 err = -ENOMEM; 1604 shmem_inode_unacct_blocks(inode, nr); 1605 failed: 1606 return ERR_PTR(err); 1607 } 1608 1609 /* 1610 * When a page is moved from swapcache to shmem filecache (either by the 1611 * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of 1612 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1613 * ignorance of the mapping it belongs to. If that mapping has special 1614 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1615 * we may need to copy to a suitable page before moving to filecache. 1616 * 1617 * In a future release, this may well be extended to respect cpuset and 1618 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1619 * but for now it is a simple matter of zone. 1620 */ 1621 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp) 1622 { 1623 return folio_zonenum(folio) > gfp_zone(gfp); 1624 } 1625 1626 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp, 1627 struct shmem_inode_info *info, pgoff_t index) 1628 { 1629 struct folio *old, *new; 1630 struct address_space *swap_mapping; 1631 swp_entry_t entry; 1632 pgoff_t swap_index; 1633 int error; 1634 1635 old = *foliop; 1636 entry = folio_swap_entry(old); 1637 swap_index = swp_offset(entry); 1638 swap_mapping = swap_address_space(entry); 1639 1640 /* 1641 * We have arrived here because our zones are constrained, so don't 1642 * limit chance of success by further cpuset and node constraints. 1643 */ 1644 gfp &= ~GFP_CONSTRAINT_MASK; 1645 VM_BUG_ON_FOLIO(folio_test_large(old), old); 1646 new = shmem_alloc_folio(gfp, info, index); 1647 if (!new) 1648 return -ENOMEM; 1649 1650 folio_get(new); 1651 folio_copy(new, old); 1652 flush_dcache_folio(new); 1653 1654 __folio_set_locked(new); 1655 __folio_set_swapbacked(new); 1656 folio_mark_uptodate(new); 1657 folio_set_swap_entry(new, entry); 1658 folio_set_swapcache(new); 1659 1660 /* 1661 * Our caller will very soon move newpage out of swapcache, but it's 1662 * a nice clean interface for us to replace oldpage by newpage there. 1663 */ 1664 xa_lock_irq(&swap_mapping->i_pages); 1665 error = shmem_replace_entry(swap_mapping, swap_index, old, new); 1666 if (!error) { 1667 mem_cgroup_migrate(old, new); 1668 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1); 1669 __lruvec_stat_mod_folio(new, NR_SHMEM, 1); 1670 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1); 1671 __lruvec_stat_mod_folio(old, NR_SHMEM, -1); 1672 } 1673 xa_unlock_irq(&swap_mapping->i_pages); 1674 1675 if (unlikely(error)) { 1676 /* 1677 * Is this possible? I think not, now that our callers check 1678 * both PageSwapCache and page_private after getting page lock; 1679 * but be defensive. Reverse old to newpage for clear and free. 1680 */ 1681 old = new; 1682 } else { 1683 folio_add_lru(new); 1684 *foliop = new; 1685 } 1686 1687 folio_clear_swapcache(old); 1688 old->private = NULL; 1689 1690 folio_unlock(old); 1691 folio_put_refs(old, 2); 1692 return error; 1693 } 1694 1695 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index, 1696 struct folio *folio, swp_entry_t swap) 1697 { 1698 struct address_space *mapping = inode->i_mapping; 1699 struct shmem_inode_info *info = SHMEM_I(inode); 1700 swp_entry_t swapin_error; 1701 void *old; 1702 1703 swapin_error = make_swapin_error_entry(&folio->page); 1704 old = xa_cmpxchg_irq(&mapping->i_pages, index, 1705 swp_to_radix_entry(swap), 1706 swp_to_radix_entry(swapin_error), 0); 1707 if (old != swp_to_radix_entry(swap)) 1708 return; 1709 1710 folio_wait_writeback(folio); 1711 delete_from_swap_cache(folio); 1712 spin_lock_irq(&info->lock); 1713 /* 1714 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks won't 1715 * be 0 when inode is released and thus trigger WARN_ON(inode->i_blocks) in 1716 * shmem_evict_inode. 1717 */ 1718 info->alloced--; 1719 info->swapped--; 1720 shmem_recalc_inode(inode); 1721 spin_unlock_irq(&info->lock); 1722 swap_free(swap); 1723 } 1724 1725 /* 1726 * Swap in the folio pointed to by *foliop. 1727 * Caller has to make sure that *foliop contains a valid swapped folio. 1728 * Returns 0 and the folio in foliop if success. On failure, returns the 1729 * error code and NULL in *foliop. 1730 */ 1731 static int shmem_swapin_folio(struct inode *inode, pgoff_t index, 1732 struct folio **foliop, enum sgp_type sgp, 1733 gfp_t gfp, struct vm_area_struct *vma, 1734 vm_fault_t *fault_type) 1735 { 1736 struct address_space *mapping = inode->i_mapping; 1737 struct shmem_inode_info *info = SHMEM_I(inode); 1738 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL; 1739 struct folio *folio = NULL; 1740 swp_entry_t swap; 1741 int error; 1742 1743 VM_BUG_ON(!*foliop || !xa_is_value(*foliop)); 1744 swap = radix_to_swp_entry(*foliop); 1745 *foliop = NULL; 1746 1747 if (is_swapin_error_entry(swap)) 1748 return -EIO; 1749 1750 /* Look it up and read it in.. */ 1751 folio = swap_cache_get_folio(swap, NULL, 0); 1752 if (!folio) { 1753 /* Or update major stats only when swapin succeeds?? */ 1754 if (fault_type) { 1755 *fault_type |= VM_FAULT_MAJOR; 1756 count_vm_event(PGMAJFAULT); 1757 count_memcg_event_mm(charge_mm, PGMAJFAULT); 1758 } 1759 /* Here we actually start the io */ 1760 folio = shmem_swapin(swap, gfp, info, index); 1761 if (!folio) { 1762 error = -ENOMEM; 1763 goto failed; 1764 } 1765 } 1766 1767 /* We have to do this with folio locked to prevent races */ 1768 folio_lock(folio); 1769 if (!folio_test_swapcache(folio) || 1770 folio_swap_entry(folio).val != swap.val || 1771 !shmem_confirm_swap(mapping, index, swap)) { 1772 error = -EEXIST; 1773 goto unlock; 1774 } 1775 if (!folio_test_uptodate(folio)) { 1776 error = -EIO; 1777 goto failed; 1778 } 1779 folio_wait_writeback(folio); 1780 1781 /* 1782 * Some architectures may have to restore extra metadata to the 1783 * folio after reading from swap. 1784 */ 1785 arch_swap_restore(swap, folio); 1786 1787 if (shmem_should_replace_folio(folio, gfp)) { 1788 error = shmem_replace_folio(&folio, gfp, info, index); 1789 if (error) 1790 goto failed; 1791 } 1792 1793 error = shmem_add_to_page_cache(folio, mapping, index, 1794 swp_to_radix_entry(swap), gfp, 1795 charge_mm); 1796 if (error) 1797 goto failed; 1798 1799 spin_lock_irq(&info->lock); 1800 info->swapped--; 1801 shmem_recalc_inode(inode); 1802 spin_unlock_irq(&info->lock); 1803 1804 if (sgp == SGP_WRITE) 1805 folio_mark_accessed(folio); 1806 1807 delete_from_swap_cache(folio); 1808 folio_mark_dirty(folio); 1809 swap_free(swap); 1810 1811 *foliop = folio; 1812 return 0; 1813 failed: 1814 if (!shmem_confirm_swap(mapping, index, swap)) 1815 error = -EEXIST; 1816 if (error == -EIO) 1817 shmem_set_folio_swapin_error(inode, index, folio, swap); 1818 unlock: 1819 if (folio) { 1820 folio_unlock(folio); 1821 folio_put(folio); 1822 } 1823 1824 return error; 1825 } 1826 1827 /* 1828 * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate 1829 * 1830 * If we allocate a new one we do not mark it dirty. That's up to the 1831 * vm. If we swap it in we mark it dirty since we also free the swap 1832 * entry since a page cannot live in both the swap and page cache. 1833 * 1834 * vma, vmf, and fault_type are only supplied by shmem_fault: 1835 * otherwise they are NULL. 1836 */ 1837 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, 1838 struct folio **foliop, enum sgp_type sgp, gfp_t gfp, 1839 struct vm_area_struct *vma, struct vm_fault *vmf, 1840 vm_fault_t *fault_type) 1841 { 1842 struct address_space *mapping = inode->i_mapping; 1843 struct shmem_inode_info *info = SHMEM_I(inode); 1844 struct shmem_sb_info *sbinfo; 1845 struct mm_struct *charge_mm; 1846 struct folio *folio; 1847 pgoff_t hindex = index; 1848 gfp_t huge_gfp; 1849 int error; 1850 int once = 0; 1851 int alloced = 0; 1852 1853 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1854 return -EFBIG; 1855 repeat: 1856 if (sgp <= SGP_CACHE && 1857 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1858 return -EINVAL; 1859 } 1860 1861 sbinfo = SHMEM_SB(inode->i_sb); 1862 charge_mm = vma ? vma->vm_mm : NULL; 1863 1864 folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0); 1865 if (folio && vma && userfaultfd_minor(vma)) { 1866 if (!xa_is_value(folio)) { 1867 folio_unlock(folio); 1868 folio_put(folio); 1869 } 1870 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); 1871 return 0; 1872 } 1873 1874 if (xa_is_value(folio)) { 1875 error = shmem_swapin_folio(inode, index, &folio, 1876 sgp, gfp, vma, fault_type); 1877 if (error == -EEXIST) 1878 goto repeat; 1879 1880 *foliop = folio; 1881 return error; 1882 } 1883 1884 if (folio) { 1885 hindex = folio->index; 1886 if (sgp == SGP_WRITE) 1887 folio_mark_accessed(folio); 1888 if (folio_test_uptodate(folio)) 1889 goto out; 1890 /* fallocated folio */ 1891 if (sgp != SGP_READ) 1892 goto clear; 1893 folio_unlock(folio); 1894 folio_put(folio); 1895 } 1896 1897 /* 1898 * SGP_READ: succeed on hole, with NULL folio, letting caller zero. 1899 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail. 1900 */ 1901 *foliop = NULL; 1902 if (sgp == SGP_READ) 1903 return 0; 1904 if (sgp == SGP_NOALLOC) 1905 return -ENOENT; 1906 1907 /* 1908 * Fast cache lookup and swap lookup did not find it: allocate. 1909 */ 1910 1911 if (vma && userfaultfd_missing(vma)) { 1912 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 1913 return 0; 1914 } 1915 1916 if (!shmem_is_huge(vma, inode, index, false)) 1917 goto alloc_nohuge; 1918 1919 huge_gfp = vma_thp_gfp_mask(vma); 1920 huge_gfp = limit_gfp_mask(huge_gfp, gfp); 1921 folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true); 1922 if (IS_ERR(folio)) { 1923 alloc_nohuge: 1924 folio = shmem_alloc_and_acct_folio(gfp, inode, index, false); 1925 } 1926 if (IS_ERR(folio)) { 1927 int retry = 5; 1928 1929 error = PTR_ERR(folio); 1930 folio = NULL; 1931 if (error != -ENOSPC) 1932 goto unlock; 1933 /* 1934 * Try to reclaim some space by splitting a large folio 1935 * beyond i_size on the filesystem. 1936 */ 1937 while (retry--) { 1938 int ret; 1939 1940 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1941 if (ret == SHRINK_STOP) 1942 break; 1943 if (ret) 1944 goto alloc_nohuge; 1945 } 1946 goto unlock; 1947 } 1948 1949 hindex = round_down(index, folio_nr_pages(folio)); 1950 1951 if (sgp == SGP_WRITE) 1952 __folio_set_referenced(folio); 1953 1954 error = shmem_add_to_page_cache(folio, mapping, hindex, 1955 NULL, gfp & GFP_RECLAIM_MASK, 1956 charge_mm); 1957 if (error) 1958 goto unacct; 1959 folio_add_lru(folio); 1960 1961 spin_lock_irq(&info->lock); 1962 info->alloced += folio_nr_pages(folio); 1963 inode->i_blocks += (blkcnt_t)BLOCKS_PER_PAGE << folio_order(folio); 1964 shmem_recalc_inode(inode); 1965 spin_unlock_irq(&info->lock); 1966 alloced = true; 1967 1968 if (folio_test_pmd_mappable(folio) && 1969 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1970 folio_next_index(folio) - 1) { 1971 /* 1972 * Part of the large folio is beyond i_size: subject 1973 * to shrink under memory pressure. 1974 */ 1975 spin_lock(&sbinfo->shrinklist_lock); 1976 /* 1977 * _careful to defend against unlocked access to 1978 * ->shrink_list in shmem_unused_huge_shrink() 1979 */ 1980 if (list_empty_careful(&info->shrinklist)) { 1981 list_add_tail(&info->shrinklist, 1982 &sbinfo->shrinklist); 1983 sbinfo->shrinklist_len++; 1984 } 1985 spin_unlock(&sbinfo->shrinklist_lock); 1986 } 1987 1988 /* 1989 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio. 1990 */ 1991 if (sgp == SGP_FALLOC) 1992 sgp = SGP_WRITE; 1993 clear: 1994 /* 1995 * Let SGP_WRITE caller clear ends if write does not fill folio; 1996 * but SGP_FALLOC on a folio fallocated earlier must initialize 1997 * it now, lest undo on failure cancel our earlier guarantee. 1998 */ 1999 if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) { 2000 long i, n = folio_nr_pages(folio); 2001 2002 for (i = 0; i < n; i++) 2003 clear_highpage(folio_page(folio, i)); 2004 flush_dcache_folio(folio); 2005 folio_mark_uptodate(folio); 2006 } 2007 2008 /* Perhaps the file has been truncated since we checked */ 2009 if (sgp <= SGP_CACHE && 2010 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 2011 if (alloced) { 2012 folio_clear_dirty(folio); 2013 filemap_remove_folio(folio); 2014 spin_lock_irq(&info->lock); 2015 shmem_recalc_inode(inode); 2016 spin_unlock_irq(&info->lock); 2017 } 2018 error = -EINVAL; 2019 goto unlock; 2020 } 2021 out: 2022 *foliop = folio; 2023 return 0; 2024 2025 /* 2026 * Error recovery. 2027 */ 2028 unacct: 2029 shmem_inode_unacct_blocks(inode, folio_nr_pages(folio)); 2030 2031 if (folio_test_large(folio)) { 2032 folio_unlock(folio); 2033 folio_put(folio); 2034 goto alloc_nohuge; 2035 } 2036 unlock: 2037 if (folio) { 2038 folio_unlock(folio); 2039 folio_put(folio); 2040 } 2041 if (error == -ENOSPC && !once++) { 2042 spin_lock_irq(&info->lock); 2043 shmem_recalc_inode(inode); 2044 spin_unlock_irq(&info->lock); 2045 goto repeat; 2046 } 2047 if (error == -EEXIST) 2048 goto repeat; 2049 return error; 2050 } 2051 2052 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop, 2053 enum sgp_type sgp) 2054 { 2055 return shmem_get_folio_gfp(inode, index, foliop, sgp, 2056 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); 2057 } 2058 2059 /* 2060 * This is like autoremove_wake_function, but it removes the wait queue 2061 * entry unconditionally - even if something else had already woken the 2062 * target. 2063 */ 2064 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 2065 { 2066 int ret = default_wake_function(wait, mode, sync, key); 2067 list_del_init(&wait->entry); 2068 return ret; 2069 } 2070 2071 static vm_fault_t shmem_fault(struct vm_fault *vmf) 2072 { 2073 struct vm_area_struct *vma = vmf->vma; 2074 struct inode *inode = file_inode(vma->vm_file); 2075 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 2076 struct folio *folio = NULL; 2077 int err; 2078 vm_fault_t ret = VM_FAULT_LOCKED; 2079 2080 /* 2081 * Trinity finds that probing a hole which tmpfs is punching can 2082 * prevent the hole-punch from ever completing: which in turn 2083 * locks writers out with its hold on i_rwsem. So refrain from 2084 * faulting pages into the hole while it's being punched. Although 2085 * shmem_undo_range() does remove the additions, it may be unable to 2086 * keep up, as each new page needs its own unmap_mapping_range() call, 2087 * and the i_mmap tree grows ever slower to scan if new vmas are added. 2088 * 2089 * It does not matter if we sometimes reach this check just before the 2090 * hole-punch begins, so that one fault then races with the punch: 2091 * we just need to make racing faults a rare case. 2092 * 2093 * The implementation below would be much simpler if we just used a 2094 * standard mutex or completion: but we cannot take i_rwsem in fault, 2095 * and bloating every shmem inode for this unlikely case would be sad. 2096 */ 2097 if (unlikely(inode->i_private)) { 2098 struct shmem_falloc *shmem_falloc; 2099 2100 spin_lock(&inode->i_lock); 2101 shmem_falloc = inode->i_private; 2102 if (shmem_falloc && 2103 shmem_falloc->waitq && 2104 vmf->pgoff >= shmem_falloc->start && 2105 vmf->pgoff < shmem_falloc->next) { 2106 struct file *fpin; 2107 wait_queue_head_t *shmem_falloc_waitq; 2108 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 2109 2110 ret = VM_FAULT_NOPAGE; 2111 fpin = maybe_unlock_mmap_for_io(vmf, NULL); 2112 if (fpin) 2113 ret = VM_FAULT_RETRY; 2114 2115 shmem_falloc_waitq = shmem_falloc->waitq; 2116 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 2117 TASK_UNINTERRUPTIBLE); 2118 spin_unlock(&inode->i_lock); 2119 schedule(); 2120 2121 /* 2122 * shmem_falloc_waitq points into the shmem_fallocate() 2123 * stack of the hole-punching task: shmem_falloc_waitq 2124 * is usually invalid by the time we reach here, but 2125 * finish_wait() does not dereference it in that case; 2126 * though i_lock needed lest racing with wake_up_all(). 2127 */ 2128 spin_lock(&inode->i_lock); 2129 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 2130 spin_unlock(&inode->i_lock); 2131 2132 if (fpin) 2133 fput(fpin); 2134 return ret; 2135 } 2136 spin_unlock(&inode->i_lock); 2137 } 2138 2139 err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE, 2140 gfp, vma, vmf, &ret); 2141 if (err) 2142 return vmf_error(err); 2143 if (folio) 2144 vmf->page = folio_file_page(folio, vmf->pgoff); 2145 return ret; 2146 } 2147 2148 unsigned long shmem_get_unmapped_area(struct file *file, 2149 unsigned long uaddr, unsigned long len, 2150 unsigned long pgoff, unsigned long flags) 2151 { 2152 unsigned long (*get_area)(struct file *, 2153 unsigned long, unsigned long, unsigned long, unsigned long); 2154 unsigned long addr; 2155 unsigned long offset; 2156 unsigned long inflated_len; 2157 unsigned long inflated_addr; 2158 unsigned long inflated_offset; 2159 2160 if (len > TASK_SIZE) 2161 return -ENOMEM; 2162 2163 get_area = current->mm->get_unmapped_area; 2164 addr = get_area(file, uaddr, len, pgoff, flags); 2165 2166 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 2167 return addr; 2168 if (IS_ERR_VALUE(addr)) 2169 return addr; 2170 if (addr & ~PAGE_MASK) 2171 return addr; 2172 if (addr > TASK_SIZE - len) 2173 return addr; 2174 2175 if (shmem_huge == SHMEM_HUGE_DENY) 2176 return addr; 2177 if (len < HPAGE_PMD_SIZE) 2178 return addr; 2179 if (flags & MAP_FIXED) 2180 return addr; 2181 /* 2182 * Our priority is to support MAP_SHARED mapped hugely; 2183 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2184 * But if caller specified an address hint and we allocated area there 2185 * successfully, respect that as before. 2186 */ 2187 if (uaddr == addr) 2188 return addr; 2189 2190 if (shmem_huge != SHMEM_HUGE_FORCE) { 2191 struct super_block *sb; 2192 2193 if (file) { 2194 VM_BUG_ON(file->f_op != &shmem_file_operations); 2195 sb = file_inode(file)->i_sb; 2196 } else { 2197 /* 2198 * Called directly from mm/mmap.c, or drivers/char/mem.c 2199 * for "/dev/zero", to create a shared anonymous object. 2200 */ 2201 if (IS_ERR(shm_mnt)) 2202 return addr; 2203 sb = shm_mnt->mnt_sb; 2204 } 2205 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2206 return addr; 2207 } 2208 2209 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2210 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2211 return addr; 2212 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2213 return addr; 2214 2215 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2216 if (inflated_len > TASK_SIZE) 2217 return addr; 2218 if (inflated_len < len) 2219 return addr; 2220 2221 inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags); 2222 if (IS_ERR_VALUE(inflated_addr)) 2223 return addr; 2224 if (inflated_addr & ~PAGE_MASK) 2225 return addr; 2226 2227 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2228 inflated_addr += offset - inflated_offset; 2229 if (inflated_offset > offset) 2230 inflated_addr += HPAGE_PMD_SIZE; 2231 2232 if (inflated_addr > TASK_SIZE - len) 2233 return addr; 2234 return inflated_addr; 2235 } 2236 2237 #ifdef CONFIG_NUMA 2238 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2239 { 2240 struct inode *inode = file_inode(vma->vm_file); 2241 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2242 } 2243 2244 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2245 unsigned long addr) 2246 { 2247 struct inode *inode = file_inode(vma->vm_file); 2248 pgoff_t index; 2249 2250 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2251 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2252 } 2253 #endif 2254 2255 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 2256 { 2257 struct inode *inode = file_inode(file); 2258 struct shmem_inode_info *info = SHMEM_I(inode); 2259 int retval = -ENOMEM; 2260 2261 /* 2262 * What serializes the accesses to info->flags? 2263 * ipc_lock_object() when called from shmctl_do_lock(), 2264 * no serialization needed when called from shm_destroy(). 2265 */ 2266 if (lock && !(info->flags & VM_LOCKED)) { 2267 if (!user_shm_lock(inode->i_size, ucounts)) 2268 goto out_nomem; 2269 info->flags |= VM_LOCKED; 2270 mapping_set_unevictable(file->f_mapping); 2271 } 2272 if (!lock && (info->flags & VM_LOCKED) && ucounts) { 2273 user_shm_unlock(inode->i_size, ucounts); 2274 info->flags &= ~VM_LOCKED; 2275 mapping_clear_unevictable(file->f_mapping); 2276 } 2277 retval = 0; 2278 2279 out_nomem: 2280 return retval; 2281 } 2282 2283 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2284 { 2285 struct shmem_inode_info *info = SHMEM_I(file_inode(file)); 2286 int ret; 2287 2288 ret = seal_check_future_write(info->seals, vma); 2289 if (ret) 2290 return ret; 2291 2292 /* arm64 - allow memory tagging on RAM-based files */ 2293 vma->vm_flags |= VM_MTE_ALLOWED; 2294 2295 file_accessed(file); 2296 vma->vm_ops = &shmem_vm_ops; 2297 return 0; 2298 } 2299 2300 #ifdef CONFIG_TMPFS_XATTR 2301 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2302 2303 /* 2304 * chattr's fsflags are unrelated to extended attributes, 2305 * but tmpfs has chosen to enable them under the same config option. 2306 */ 2307 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) 2308 { 2309 unsigned int i_flags = 0; 2310 2311 if (fsflags & FS_NOATIME_FL) 2312 i_flags |= S_NOATIME; 2313 if (fsflags & FS_APPEND_FL) 2314 i_flags |= S_APPEND; 2315 if (fsflags & FS_IMMUTABLE_FL) 2316 i_flags |= S_IMMUTABLE; 2317 /* 2318 * But FS_NODUMP_FL does not require any action in i_flags. 2319 */ 2320 inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE); 2321 } 2322 #else 2323 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) 2324 { 2325 } 2326 #define shmem_initxattrs NULL 2327 #endif 2328 2329 static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir, 2330 umode_t mode, dev_t dev, unsigned long flags) 2331 { 2332 struct inode *inode; 2333 struct shmem_inode_info *info; 2334 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2335 ino_t ino; 2336 2337 if (shmem_reserve_inode(sb, &ino)) 2338 return NULL; 2339 2340 inode = new_inode(sb); 2341 if (inode) { 2342 inode->i_ino = ino; 2343 inode_init_owner(&init_user_ns, inode, dir, mode); 2344 inode->i_blocks = 0; 2345 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2346 inode->i_generation = get_random_u32(); 2347 info = SHMEM_I(inode); 2348 memset(info, 0, (char *)inode - (char *)info); 2349 spin_lock_init(&info->lock); 2350 atomic_set(&info->stop_eviction, 0); 2351 info->seals = F_SEAL_SEAL; 2352 info->flags = flags & VM_NORESERVE; 2353 info->i_crtime = inode->i_mtime; 2354 info->fsflags = (dir == NULL) ? 0 : 2355 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED; 2356 if (info->fsflags) 2357 shmem_set_inode_flags(inode, info->fsflags); 2358 INIT_LIST_HEAD(&info->shrinklist); 2359 INIT_LIST_HEAD(&info->swaplist); 2360 simple_xattrs_init(&info->xattrs); 2361 cache_no_acl(inode); 2362 mapping_set_large_folios(inode->i_mapping); 2363 2364 switch (mode & S_IFMT) { 2365 default: 2366 inode->i_op = &shmem_special_inode_operations; 2367 init_special_inode(inode, mode, dev); 2368 break; 2369 case S_IFREG: 2370 inode->i_mapping->a_ops = &shmem_aops; 2371 inode->i_op = &shmem_inode_operations; 2372 inode->i_fop = &shmem_file_operations; 2373 mpol_shared_policy_init(&info->policy, 2374 shmem_get_sbmpol(sbinfo)); 2375 break; 2376 case S_IFDIR: 2377 inc_nlink(inode); 2378 /* Some things misbehave if size == 0 on a directory */ 2379 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2380 inode->i_op = &shmem_dir_inode_operations; 2381 inode->i_fop = &simple_dir_operations; 2382 break; 2383 case S_IFLNK: 2384 /* 2385 * Must not load anything in the rbtree, 2386 * mpol_free_shared_policy will not be called. 2387 */ 2388 mpol_shared_policy_init(&info->policy, NULL); 2389 break; 2390 } 2391 2392 lockdep_annotate_inode_mutex_key(inode); 2393 } else 2394 shmem_free_inode(sb); 2395 return inode; 2396 } 2397 2398 #ifdef CONFIG_USERFAULTFD 2399 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, 2400 pmd_t *dst_pmd, 2401 struct vm_area_struct *dst_vma, 2402 unsigned long dst_addr, 2403 unsigned long src_addr, 2404 bool zeropage, bool wp_copy, 2405 struct page **pagep) 2406 { 2407 struct inode *inode = file_inode(dst_vma->vm_file); 2408 struct shmem_inode_info *info = SHMEM_I(inode); 2409 struct address_space *mapping = inode->i_mapping; 2410 gfp_t gfp = mapping_gfp_mask(mapping); 2411 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2412 void *page_kaddr; 2413 struct folio *folio; 2414 int ret; 2415 pgoff_t max_off; 2416 2417 if (!shmem_inode_acct_block(inode, 1)) { 2418 /* 2419 * We may have got a page, returned -ENOENT triggering a retry, 2420 * and now we find ourselves with -ENOMEM. Release the page, to 2421 * avoid a BUG_ON in our caller. 2422 */ 2423 if (unlikely(*pagep)) { 2424 put_page(*pagep); 2425 *pagep = NULL; 2426 } 2427 return -ENOMEM; 2428 } 2429 2430 if (!*pagep) { 2431 ret = -ENOMEM; 2432 folio = shmem_alloc_folio(gfp, info, pgoff); 2433 if (!folio) 2434 goto out_unacct_blocks; 2435 2436 if (!zeropage) { /* COPY */ 2437 page_kaddr = kmap_local_folio(folio, 0); 2438 /* 2439 * The read mmap_lock is held here. Despite the 2440 * mmap_lock being read recursive a deadlock is still 2441 * possible if a writer has taken a lock. For example: 2442 * 2443 * process A thread 1 takes read lock on own mmap_lock 2444 * process A thread 2 calls mmap, blocks taking write lock 2445 * process B thread 1 takes page fault, read lock on own mmap lock 2446 * process B thread 2 calls mmap, blocks taking write lock 2447 * process A thread 1 blocks taking read lock on process B 2448 * process B thread 1 blocks taking read lock on process A 2449 * 2450 * Disable page faults to prevent potential deadlock 2451 * and retry the copy outside the mmap_lock. 2452 */ 2453 pagefault_disable(); 2454 ret = copy_from_user(page_kaddr, 2455 (const void __user *)src_addr, 2456 PAGE_SIZE); 2457 pagefault_enable(); 2458 kunmap_local(page_kaddr); 2459 2460 /* fallback to copy_from_user outside mmap_lock */ 2461 if (unlikely(ret)) { 2462 *pagep = &folio->page; 2463 ret = -ENOENT; 2464 /* don't free the page */ 2465 goto out_unacct_blocks; 2466 } 2467 2468 flush_dcache_folio(folio); 2469 } else { /* ZEROPAGE */ 2470 clear_user_highpage(&folio->page, dst_addr); 2471 } 2472 } else { 2473 folio = page_folio(*pagep); 2474 VM_BUG_ON_FOLIO(folio_test_large(folio), folio); 2475 *pagep = NULL; 2476 } 2477 2478 VM_BUG_ON(folio_test_locked(folio)); 2479 VM_BUG_ON(folio_test_swapbacked(folio)); 2480 __folio_set_locked(folio); 2481 __folio_set_swapbacked(folio); 2482 __folio_mark_uptodate(folio); 2483 2484 ret = -EFAULT; 2485 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2486 if (unlikely(pgoff >= max_off)) 2487 goto out_release; 2488 2489 ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, 2490 gfp & GFP_RECLAIM_MASK, dst_mm); 2491 if (ret) 2492 goto out_release; 2493 2494 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, 2495 &folio->page, true, wp_copy); 2496 if (ret) 2497 goto out_delete_from_cache; 2498 2499 spin_lock_irq(&info->lock); 2500 info->alloced++; 2501 inode->i_blocks += BLOCKS_PER_PAGE; 2502 shmem_recalc_inode(inode); 2503 spin_unlock_irq(&info->lock); 2504 2505 folio_unlock(folio); 2506 return 0; 2507 out_delete_from_cache: 2508 filemap_remove_folio(folio); 2509 out_release: 2510 folio_unlock(folio); 2511 folio_put(folio); 2512 out_unacct_blocks: 2513 shmem_inode_unacct_blocks(inode, 1); 2514 return ret; 2515 } 2516 #endif /* CONFIG_USERFAULTFD */ 2517 2518 #ifdef CONFIG_TMPFS 2519 static const struct inode_operations shmem_symlink_inode_operations; 2520 static const struct inode_operations shmem_short_symlink_operations; 2521 2522 static int 2523 shmem_write_begin(struct file *file, struct address_space *mapping, 2524 loff_t pos, unsigned len, 2525 struct page **pagep, void **fsdata) 2526 { 2527 struct inode *inode = mapping->host; 2528 struct shmem_inode_info *info = SHMEM_I(inode); 2529 pgoff_t index = pos >> PAGE_SHIFT; 2530 struct folio *folio; 2531 int ret = 0; 2532 2533 /* i_rwsem is held by caller */ 2534 if (unlikely(info->seals & (F_SEAL_GROW | 2535 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) { 2536 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) 2537 return -EPERM; 2538 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2539 return -EPERM; 2540 } 2541 2542 ret = shmem_get_folio(inode, index, &folio, SGP_WRITE); 2543 2544 if (ret) 2545 return ret; 2546 2547 *pagep = folio_file_page(folio, index); 2548 if (PageHWPoison(*pagep)) { 2549 folio_unlock(folio); 2550 folio_put(folio); 2551 *pagep = NULL; 2552 return -EIO; 2553 } 2554 2555 return 0; 2556 } 2557 2558 static int 2559 shmem_write_end(struct file *file, struct address_space *mapping, 2560 loff_t pos, unsigned len, unsigned copied, 2561 struct page *page, void *fsdata) 2562 { 2563 struct inode *inode = mapping->host; 2564 2565 if (pos + copied > inode->i_size) 2566 i_size_write(inode, pos + copied); 2567 2568 if (!PageUptodate(page)) { 2569 struct page *head = compound_head(page); 2570 if (PageTransCompound(page)) { 2571 int i; 2572 2573 for (i = 0; i < HPAGE_PMD_NR; i++) { 2574 if (head + i == page) 2575 continue; 2576 clear_highpage(head + i); 2577 flush_dcache_page(head + i); 2578 } 2579 } 2580 if (copied < PAGE_SIZE) { 2581 unsigned from = pos & (PAGE_SIZE - 1); 2582 zero_user_segments(page, 0, from, 2583 from + copied, PAGE_SIZE); 2584 } 2585 SetPageUptodate(head); 2586 } 2587 set_page_dirty(page); 2588 unlock_page(page); 2589 put_page(page); 2590 2591 return copied; 2592 } 2593 2594 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2595 { 2596 struct file *file = iocb->ki_filp; 2597 struct inode *inode = file_inode(file); 2598 struct address_space *mapping = inode->i_mapping; 2599 pgoff_t index; 2600 unsigned long offset; 2601 int error = 0; 2602 ssize_t retval = 0; 2603 loff_t *ppos = &iocb->ki_pos; 2604 2605 index = *ppos >> PAGE_SHIFT; 2606 offset = *ppos & ~PAGE_MASK; 2607 2608 for (;;) { 2609 struct folio *folio = NULL; 2610 struct page *page = NULL; 2611 pgoff_t end_index; 2612 unsigned long nr, ret; 2613 loff_t i_size = i_size_read(inode); 2614 2615 end_index = i_size >> PAGE_SHIFT; 2616 if (index > end_index) 2617 break; 2618 if (index == end_index) { 2619 nr = i_size & ~PAGE_MASK; 2620 if (nr <= offset) 2621 break; 2622 } 2623 2624 error = shmem_get_folio(inode, index, &folio, SGP_READ); 2625 if (error) { 2626 if (error == -EINVAL) 2627 error = 0; 2628 break; 2629 } 2630 if (folio) { 2631 folio_unlock(folio); 2632 2633 page = folio_file_page(folio, index); 2634 if (PageHWPoison(page)) { 2635 folio_put(folio); 2636 error = -EIO; 2637 break; 2638 } 2639 } 2640 2641 /* 2642 * We must evaluate after, since reads (unlike writes) 2643 * are called without i_rwsem protection against truncate 2644 */ 2645 nr = PAGE_SIZE; 2646 i_size = i_size_read(inode); 2647 end_index = i_size >> PAGE_SHIFT; 2648 if (index == end_index) { 2649 nr = i_size & ~PAGE_MASK; 2650 if (nr <= offset) { 2651 if (folio) 2652 folio_put(folio); 2653 break; 2654 } 2655 } 2656 nr -= offset; 2657 2658 if (folio) { 2659 /* 2660 * If users can be writing to this page using arbitrary 2661 * virtual addresses, take care about potential aliasing 2662 * before reading the page on the kernel side. 2663 */ 2664 if (mapping_writably_mapped(mapping)) 2665 flush_dcache_page(page); 2666 /* 2667 * Mark the page accessed if we read the beginning. 2668 */ 2669 if (!offset) 2670 folio_mark_accessed(folio); 2671 /* 2672 * Ok, we have the page, and it's up-to-date, so 2673 * now we can copy it to user space... 2674 */ 2675 ret = copy_page_to_iter(page, offset, nr, to); 2676 folio_put(folio); 2677 2678 } else if (user_backed_iter(to)) { 2679 /* 2680 * Copy to user tends to be so well optimized, but 2681 * clear_user() not so much, that it is noticeably 2682 * faster to copy the zero page instead of clearing. 2683 */ 2684 ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to); 2685 } else { 2686 /* 2687 * But submitting the same page twice in a row to 2688 * splice() - or others? - can result in confusion: 2689 * so don't attempt that optimization on pipes etc. 2690 */ 2691 ret = iov_iter_zero(nr, to); 2692 } 2693 2694 retval += ret; 2695 offset += ret; 2696 index += offset >> PAGE_SHIFT; 2697 offset &= ~PAGE_MASK; 2698 2699 if (!iov_iter_count(to)) 2700 break; 2701 if (ret < nr) { 2702 error = -EFAULT; 2703 break; 2704 } 2705 cond_resched(); 2706 } 2707 2708 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2709 file_accessed(file); 2710 return retval ? retval : error; 2711 } 2712 2713 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2714 { 2715 struct address_space *mapping = file->f_mapping; 2716 struct inode *inode = mapping->host; 2717 2718 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2719 return generic_file_llseek_size(file, offset, whence, 2720 MAX_LFS_FILESIZE, i_size_read(inode)); 2721 if (offset < 0) 2722 return -ENXIO; 2723 2724 inode_lock(inode); 2725 /* We're holding i_rwsem so we can access i_size directly */ 2726 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence); 2727 if (offset >= 0) 2728 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2729 inode_unlock(inode); 2730 return offset; 2731 } 2732 2733 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2734 loff_t len) 2735 { 2736 struct inode *inode = file_inode(file); 2737 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2738 struct shmem_inode_info *info = SHMEM_I(inode); 2739 struct shmem_falloc shmem_falloc; 2740 pgoff_t start, index, end, undo_fallocend; 2741 int error; 2742 2743 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2744 return -EOPNOTSUPP; 2745 2746 inode_lock(inode); 2747 2748 if (mode & FALLOC_FL_PUNCH_HOLE) { 2749 struct address_space *mapping = file->f_mapping; 2750 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2751 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2752 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2753 2754 /* protected by i_rwsem */ 2755 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { 2756 error = -EPERM; 2757 goto out; 2758 } 2759 2760 shmem_falloc.waitq = &shmem_falloc_waitq; 2761 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; 2762 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2763 spin_lock(&inode->i_lock); 2764 inode->i_private = &shmem_falloc; 2765 spin_unlock(&inode->i_lock); 2766 2767 if ((u64)unmap_end > (u64)unmap_start) 2768 unmap_mapping_range(mapping, unmap_start, 2769 1 + unmap_end - unmap_start, 0); 2770 shmem_truncate_range(inode, offset, offset + len - 1); 2771 /* No need to unmap again: hole-punching leaves COWed pages */ 2772 2773 spin_lock(&inode->i_lock); 2774 inode->i_private = NULL; 2775 wake_up_all(&shmem_falloc_waitq); 2776 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 2777 spin_unlock(&inode->i_lock); 2778 error = 0; 2779 goto out; 2780 } 2781 2782 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2783 error = inode_newsize_ok(inode, offset + len); 2784 if (error) 2785 goto out; 2786 2787 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2788 error = -EPERM; 2789 goto out; 2790 } 2791 2792 start = offset >> PAGE_SHIFT; 2793 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2794 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2795 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2796 error = -ENOSPC; 2797 goto out; 2798 } 2799 2800 shmem_falloc.waitq = NULL; 2801 shmem_falloc.start = start; 2802 shmem_falloc.next = start; 2803 shmem_falloc.nr_falloced = 0; 2804 shmem_falloc.nr_unswapped = 0; 2805 spin_lock(&inode->i_lock); 2806 inode->i_private = &shmem_falloc; 2807 spin_unlock(&inode->i_lock); 2808 2809 /* 2810 * info->fallocend is only relevant when huge pages might be 2811 * involved: to prevent split_huge_page() freeing fallocated 2812 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size. 2813 */ 2814 undo_fallocend = info->fallocend; 2815 if (info->fallocend < end) 2816 info->fallocend = end; 2817 2818 for (index = start; index < end; ) { 2819 struct folio *folio; 2820 2821 /* 2822 * Good, the fallocate(2) manpage permits EINTR: we may have 2823 * been interrupted because we are using up too much memory. 2824 */ 2825 if (signal_pending(current)) 2826 error = -EINTR; 2827 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2828 error = -ENOMEM; 2829 else 2830 error = shmem_get_folio(inode, index, &folio, 2831 SGP_FALLOC); 2832 if (error) { 2833 info->fallocend = undo_fallocend; 2834 /* Remove the !uptodate folios we added */ 2835 if (index > start) { 2836 shmem_undo_range(inode, 2837 (loff_t)start << PAGE_SHIFT, 2838 ((loff_t)index << PAGE_SHIFT) - 1, true); 2839 } 2840 goto undone; 2841 } 2842 2843 /* 2844 * Here is a more important optimization than it appears: 2845 * a second SGP_FALLOC on the same large folio will clear it, 2846 * making it uptodate and un-undoable if we fail later. 2847 */ 2848 index = folio_next_index(folio); 2849 /* Beware 32-bit wraparound */ 2850 if (!index) 2851 index--; 2852 2853 /* 2854 * Inform shmem_writepage() how far we have reached. 2855 * No need for lock or barrier: we have the page lock. 2856 */ 2857 if (!folio_test_uptodate(folio)) 2858 shmem_falloc.nr_falloced += index - shmem_falloc.next; 2859 shmem_falloc.next = index; 2860 2861 /* 2862 * If !uptodate, leave it that way so that freeable folios 2863 * can be recognized if we need to rollback on error later. 2864 * But mark it dirty so that memory pressure will swap rather 2865 * than free the folios we are allocating (and SGP_CACHE folios 2866 * might still be clean: we now need to mark those dirty too). 2867 */ 2868 folio_mark_dirty(folio); 2869 folio_unlock(folio); 2870 folio_put(folio); 2871 cond_resched(); 2872 } 2873 2874 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2875 i_size_write(inode, offset + len); 2876 undone: 2877 spin_lock(&inode->i_lock); 2878 inode->i_private = NULL; 2879 spin_unlock(&inode->i_lock); 2880 out: 2881 if (!error) 2882 file_modified(file); 2883 inode_unlock(inode); 2884 return error; 2885 } 2886 2887 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2888 { 2889 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2890 2891 buf->f_type = TMPFS_MAGIC; 2892 buf->f_bsize = PAGE_SIZE; 2893 buf->f_namelen = NAME_MAX; 2894 if (sbinfo->max_blocks) { 2895 buf->f_blocks = sbinfo->max_blocks; 2896 buf->f_bavail = 2897 buf->f_bfree = sbinfo->max_blocks - 2898 percpu_counter_sum(&sbinfo->used_blocks); 2899 } 2900 if (sbinfo->max_inodes) { 2901 buf->f_files = sbinfo->max_inodes; 2902 buf->f_ffree = sbinfo->free_inodes; 2903 } 2904 /* else leave those fields 0 like simple_statfs */ 2905 2906 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); 2907 2908 return 0; 2909 } 2910 2911 /* 2912 * File creation. Allocate an inode, and we're done.. 2913 */ 2914 static int 2915 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir, 2916 struct dentry *dentry, umode_t mode, dev_t dev) 2917 { 2918 struct inode *inode; 2919 int error = -ENOSPC; 2920 2921 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2922 if (inode) { 2923 error = simple_acl_create(dir, inode); 2924 if (error) 2925 goto out_iput; 2926 error = security_inode_init_security(inode, dir, 2927 &dentry->d_name, 2928 shmem_initxattrs, NULL); 2929 if (error && error != -EOPNOTSUPP) 2930 goto out_iput; 2931 2932 error = 0; 2933 dir->i_size += BOGO_DIRENT_SIZE; 2934 dir->i_ctime = dir->i_mtime = current_time(dir); 2935 inode_inc_iversion(dir); 2936 d_instantiate(dentry, inode); 2937 dget(dentry); /* Extra count - pin the dentry in core */ 2938 } 2939 return error; 2940 out_iput: 2941 iput(inode); 2942 return error; 2943 } 2944 2945 static int 2946 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, 2947 struct file *file, umode_t mode) 2948 { 2949 struct inode *inode; 2950 int error = -ENOSPC; 2951 2952 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2953 if (inode) { 2954 error = security_inode_init_security(inode, dir, 2955 NULL, 2956 shmem_initxattrs, NULL); 2957 if (error && error != -EOPNOTSUPP) 2958 goto out_iput; 2959 error = simple_acl_create(dir, inode); 2960 if (error) 2961 goto out_iput; 2962 d_tmpfile(file, inode); 2963 } 2964 return finish_open_simple(file, error); 2965 out_iput: 2966 iput(inode); 2967 return error; 2968 } 2969 2970 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir, 2971 struct dentry *dentry, umode_t mode) 2972 { 2973 int error; 2974 2975 if ((error = shmem_mknod(&init_user_ns, dir, dentry, 2976 mode | S_IFDIR, 0))) 2977 return error; 2978 inc_nlink(dir); 2979 return 0; 2980 } 2981 2982 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir, 2983 struct dentry *dentry, umode_t mode, bool excl) 2984 { 2985 return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); 2986 } 2987 2988 /* 2989 * Link a file.. 2990 */ 2991 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2992 { 2993 struct inode *inode = d_inode(old_dentry); 2994 int ret = 0; 2995 2996 /* 2997 * No ordinary (disk based) filesystem counts links as inodes; 2998 * but each new link needs a new dentry, pinning lowmem, and 2999 * tmpfs dentries cannot be pruned until they are unlinked. 3000 * But if an O_TMPFILE file is linked into the tmpfs, the 3001 * first link must skip that, to get the accounting right. 3002 */ 3003 if (inode->i_nlink) { 3004 ret = shmem_reserve_inode(inode->i_sb, NULL); 3005 if (ret) 3006 goto out; 3007 } 3008 3009 dir->i_size += BOGO_DIRENT_SIZE; 3010 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 3011 inode_inc_iversion(dir); 3012 inc_nlink(inode); 3013 ihold(inode); /* New dentry reference */ 3014 dget(dentry); /* Extra pinning count for the created dentry */ 3015 d_instantiate(dentry, inode); 3016 out: 3017 return ret; 3018 } 3019 3020 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 3021 { 3022 struct inode *inode = d_inode(dentry); 3023 3024 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 3025 shmem_free_inode(inode->i_sb); 3026 3027 dir->i_size -= BOGO_DIRENT_SIZE; 3028 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 3029 inode_inc_iversion(dir); 3030 drop_nlink(inode); 3031 dput(dentry); /* Undo the count from "create" - this does all the work */ 3032 return 0; 3033 } 3034 3035 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 3036 { 3037 if (!simple_empty(dentry)) 3038 return -ENOTEMPTY; 3039 3040 drop_nlink(d_inode(dentry)); 3041 drop_nlink(dir); 3042 return shmem_unlink(dir, dentry); 3043 } 3044 3045 static int shmem_whiteout(struct user_namespace *mnt_userns, 3046 struct inode *old_dir, struct dentry *old_dentry) 3047 { 3048 struct dentry *whiteout; 3049 int error; 3050 3051 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 3052 if (!whiteout) 3053 return -ENOMEM; 3054 3055 error = shmem_mknod(&init_user_ns, old_dir, whiteout, 3056 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 3057 dput(whiteout); 3058 if (error) 3059 return error; 3060 3061 /* 3062 * Cheat and hash the whiteout while the old dentry is still in 3063 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 3064 * 3065 * d_lookup() will consistently find one of them at this point, 3066 * not sure which one, but that isn't even important. 3067 */ 3068 d_rehash(whiteout); 3069 return 0; 3070 } 3071 3072 /* 3073 * The VFS layer already does all the dentry stuff for rename, 3074 * we just have to decrement the usage count for the target if 3075 * it exists so that the VFS layer correctly free's it when it 3076 * gets overwritten. 3077 */ 3078 static int shmem_rename2(struct user_namespace *mnt_userns, 3079 struct inode *old_dir, struct dentry *old_dentry, 3080 struct inode *new_dir, struct dentry *new_dentry, 3081 unsigned int flags) 3082 { 3083 struct inode *inode = d_inode(old_dentry); 3084 int they_are_dirs = S_ISDIR(inode->i_mode); 3085 3086 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3087 return -EINVAL; 3088 3089 if (flags & RENAME_EXCHANGE) 3090 return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); 3091 3092 if (!simple_empty(new_dentry)) 3093 return -ENOTEMPTY; 3094 3095 if (flags & RENAME_WHITEOUT) { 3096 int error; 3097 3098 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry); 3099 if (error) 3100 return error; 3101 } 3102 3103 if (d_really_is_positive(new_dentry)) { 3104 (void) shmem_unlink(new_dir, new_dentry); 3105 if (they_are_dirs) { 3106 drop_nlink(d_inode(new_dentry)); 3107 drop_nlink(old_dir); 3108 } 3109 } else if (they_are_dirs) { 3110 drop_nlink(old_dir); 3111 inc_nlink(new_dir); 3112 } 3113 3114 old_dir->i_size -= BOGO_DIRENT_SIZE; 3115 new_dir->i_size += BOGO_DIRENT_SIZE; 3116 old_dir->i_ctime = old_dir->i_mtime = 3117 new_dir->i_ctime = new_dir->i_mtime = 3118 inode->i_ctime = current_time(old_dir); 3119 inode_inc_iversion(old_dir); 3120 inode_inc_iversion(new_dir); 3121 return 0; 3122 } 3123 3124 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, 3125 struct dentry *dentry, const char *symname) 3126 { 3127 int error; 3128 int len; 3129 struct inode *inode; 3130 struct folio *folio; 3131 3132 len = strlen(symname) + 1; 3133 if (len > PAGE_SIZE) 3134 return -ENAMETOOLONG; 3135 3136 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0, 3137 VM_NORESERVE); 3138 if (!inode) 3139 return -ENOSPC; 3140 3141 error = security_inode_init_security(inode, dir, &dentry->d_name, 3142 shmem_initxattrs, NULL); 3143 if (error && error != -EOPNOTSUPP) { 3144 iput(inode); 3145 return error; 3146 } 3147 3148 inode->i_size = len-1; 3149 if (len <= SHORT_SYMLINK_LEN) { 3150 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3151 if (!inode->i_link) { 3152 iput(inode); 3153 return -ENOMEM; 3154 } 3155 inode->i_op = &shmem_short_symlink_operations; 3156 } else { 3157 inode_nohighmem(inode); 3158 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE); 3159 if (error) { 3160 iput(inode); 3161 return error; 3162 } 3163 inode->i_mapping->a_ops = &shmem_aops; 3164 inode->i_op = &shmem_symlink_inode_operations; 3165 memcpy(folio_address(folio), symname, len); 3166 folio_mark_uptodate(folio); 3167 folio_mark_dirty(folio); 3168 folio_unlock(folio); 3169 folio_put(folio); 3170 } 3171 dir->i_size += BOGO_DIRENT_SIZE; 3172 dir->i_ctime = dir->i_mtime = current_time(dir); 3173 inode_inc_iversion(dir); 3174 d_instantiate(dentry, inode); 3175 dget(dentry); 3176 return 0; 3177 } 3178 3179 static void shmem_put_link(void *arg) 3180 { 3181 folio_mark_accessed(arg); 3182 folio_put(arg); 3183 } 3184 3185 static const char *shmem_get_link(struct dentry *dentry, 3186 struct inode *inode, 3187 struct delayed_call *done) 3188 { 3189 struct folio *folio = NULL; 3190 int error; 3191 3192 if (!dentry) { 3193 folio = filemap_get_folio(inode->i_mapping, 0); 3194 if (!folio) 3195 return ERR_PTR(-ECHILD); 3196 if (PageHWPoison(folio_page(folio, 0)) || 3197 !folio_test_uptodate(folio)) { 3198 folio_put(folio); 3199 return ERR_PTR(-ECHILD); 3200 } 3201 } else { 3202 error = shmem_get_folio(inode, 0, &folio, SGP_READ); 3203 if (error) 3204 return ERR_PTR(error); 3205 if (!folio) 3206 return ERR_PTR(-ECHILD); 3207 if (PageHWPoison(folio_page(folio, 0))) { 3208 folio_unlock(folio); 3209 folio_put(folio); 3210 return ERR_PTR(-ECHILD); 3211 } 3212 folio_unlock(folio); 3213 } 3214 set_delayed_call(done, shmem_put_link, folio); 3215 return folio_address(folio); 3216 } 3217 3218 #ifdef CONFIG_TMPFS_XATTR 3219 3220 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa) 3221 { 3222 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3223 3224 fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE); 3225 3226 return 0; 3227 } 3228 3229 static int shmem_fileattr_set(struct user_namespace *mnt_userns, 3230 struct dentry *dentry, struct fileattr *fa) 3231 { 3232 struct inode *inode = d_inode(dentry); 3233 struct shmem_inode_info *info = SHMEM_I(inode); 3234 3235 if (fileattr_has_fsx(fa)) 3236 return -EOPNOTSUPP; 3237 if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE) 3238 return -EOPNOTSUPP; 3239 3240 info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) | 3241 (fa->flags & SHMEM_FL_USER_MODIFIABLE); 3242 3243 shmem_set_inode_flags(inode, info->fsflags); 3244 inode->i_ctime = current_time(inode); 3245 inode_inc_iversion(inode); 3246 return 0; 3247 } 3248 3249 /* 3250 * Superblocks without xattr inode operations may get some security.* xattr 3251 * support from the LSM "for free". As soon as we have any other xattrs 3252 * like ACLs, we also need to implement the security.* handlers at 3253 * filesystem level, though. 3254 */ 3255 3256 /* 3257 * Callback for security_inode_init_security() for acquiring xattrs. 3258 */ 3259 static int shmem_initxattrs(struct inode *inode, 3260 const struct xattr *xattr_array, 3261 void *fs_info) 3262 { 3263 struct shmem_inode_info *info = SHMEM_I(inode); 3264 const struct xattr *xattr; 3265 struct simple_xattr *new_xattr; 3266 size_t len; 3267 3268 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3269 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3270 if (!new_xattr) 3271 return -ENOMEM; 3272 3273 len = strlen(xattr->name) + 1; 3274 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3275 GFP_KERNEL); 3276 if (!new_xattr->name) { 3277 kvfree(new_xattr); 3278 return -ENOMEM; 3279 } 3280 3281 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3282 XATTR_SECURITY_PREFIX_LEN); 3283 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3284 xattr->name, len); 3285 3286 simple_xattr_add(&info->xattrs, new_xattr); 3287 } 3288 3289 return 0; 3290 } 3291 3292 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3293 struct dentry *unused, struct inode *inode, 3294 const char *name, void *buffer, size_t size) 3295 { 3296 struct shmem_inode_info *info = SHMEM_I(inode); 3297 3298 name = xattr_full_name(handler, name); 3299 return simple_xattr_get(&info->xattrs, name, buffer, size); 3300 } 3301 3302 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3303 struct user_namespace *mnt_userns, 3304 struct dentry *unused, struct inode *inode, 3305 const char *name, const void *value, 3306 size_t size, int flags) 3307 { 3308 struct shmem_inode_info *info = SHMEM_I(inode); 3309 int err; 3310 3311 name = xattr_full_name(handler, name); 3312 err = simple_xattr_set(&info->xattrs, name, value, size, flags, NULL); 3313 if (!err) { 3314 inode->i_ctime = current_time(inode); 3315 inode_inc_iversion(inode); 3316 } 3317 return err; 3318 } 3319 3320 static const struct xattr_handler shmem_security_xattr_handler = { 3321 .prefix = XATTR_SECURITY_PREFIX, 3322 .get = shmem_xattr_handler_get, 3323 .set = shmem_xattr_handler_set, 3324 }; 3325 3326 static const struct xattr_handler shmem_trusted_xattr_handler = { 3327 .prefix = XATTR_TRUSTED_PREFIX, 3328 .get = shmem_xattr_handler_get, 3329 .set = shmem_xattr_handler_set, 3330 }; 3331 3332 static const struct xattr_handler *shmem_xattr_handlers[] = { 3333 #ifdef CONFIG_TMPFS_POSIX_ACL 3334 &posix_acl_access_xattr_handler, 3335 &posix_acl_default_xattr_handler, 3336 #endif 3337 &shmem_security_xattr_handler, 3338 &shmem_trusted_xattr_handler, 3339 NULL 3340 }; 3341 3342 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3343 { 3344 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3345 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3346 } 3347 #endif /* CONFIG_TMPFS_XATTR */ 3348 3349 static const struct inode_operations shmem_short_symlink_operations = { 3350 .getattr = shmem_getattr, 3351 .get_link = simple_get_link, 3352 #ifdef CONFIG_TMPFS_XATTR 3353 .listxattr = shmem_listxattr, 3354 #endif 3355 }; 3356 3357 static const struct inode_operations shmem_symlink_inode_operations = { 3358 .getattr = shmem_getattr, 3359 .get_link = shmem_get_link, 3360 #ifdef CONFIG_TMPFS_XATTR 3361 .listxattr = shmem_listxattr, 3362 #endif 3363 }; 3364 3365 static struct dentry *shmem_get_parent(struct dentry *child) 3366 { 3367 return ERR_PTR(-ESTALE); 3368 } 3369 3370 static int shmem_match(struct inode *ino, void *vfh) 3371 { 3372 __u32 *fh = vfh; 3373 __u64 inum = fh[2]; 3374 inum = (inum << 32) | fh[1]; 3375 return ino->i_ino == inum && fh[0] == ino->i_generation; 3376 } 3377 3378 /* Find any alias of inode, but prefer a hashed alias */ 3379 static struct dentry *shmem_find_alias(struct inode *inode) 3380 { 3381 struct dentry *alias = d_find_alias(inode); 3382 3383 return alias ?: d_find_any_alias(inode); 3384 } 3385 3386 3387 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3388 struct fid *fid, int fh_len, int fh_type) 3389 { 3390 struct inode *inode; 3391 struct dentry *dentry = NULL; 3392 u64 inum; 3393 3394 if (fh_len < 3) 3395 return NULL; 3396 3397 inum = fid->raw[2]; 3398 inum = (inum << 32) | fid->raw[1]; 3399 3400 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3401 shmem_match, fid->raw); 3402 if (inode) { 3403 dentry = shmem_find_alias(inode); 3404 iput(inode); 3405 } 3406 3407 return dentry; 3408 } 3409 3410 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3411 struct inode *parent) 3412 { 3413 if (*len < 3) { 3414 *len = 3; 3415 return FILEID_INVALID; 3416 } 3417 3418 if (inode_unhashed(inode)) { 3419 /* Unfortunately insert_inode_hash is not idempotent, 3420 * so as we hash inodes here rather than at creation 3421 * time, we need a lock to ensure we only try 3422 * to do it once 3423 */ 3424 static DEFINE_SPINLOCK(lock); 3425 spin_lock(&lock); 3426 if (inode_unhashed(inode)) 3427 __insert_inode_hash(inode, 3428 inode->i_ino + inode->i_generation); 3429 spin_unlock(&lock); 3430 } 3431 3432 fh[0] = inode->i_generation; 3433 fh[1] = inode->i_ino; 3434 fh[2] = ((__u64)inode->i_ino) >> 32; 3435 3436 *len = 3; 3437 return 1; 3438 } 3439 3440 static const struct export_operations shmem_export_ops = { 3441 .get_parent = shmem_get_parent, 3442 .encode_fh = shmem_encode_fh, 3443 .fh_to_dentry = shmem_fh_to_dentry, 3444 }; 3445 3446 enum shmem_param { 3447 Opt_gid, 3448 Opt_huge, 3449 Opt_mode, 3450 Opt_mpol, 3451 Opt_nr_blocks, 3452 Opt_nr_inodes, 3453 Opt_size, 3454 Opt_uid, 3455 Opt_inode32, 3456 Opt_inode64, 3457 }; 3458 3459 static const struct constant_table shmem_param_enums_huge[] = { 3460 {"never", SHMEM_HUGE_NEVER }, 3461 {"always", SHMEM_HUGE_ALWAYS }, 3462 {"within_size", SHMEM_HUGE_WITHIN_SIZE }, 3463 {"advise", SHMEM_HUGE_ADVISE }, 3464 {} 3465 }; 3466 3467 const struct fs_parameter_spec shmem_fs_parameters[] = { 3468 fsparam_u32 ("gid", Opt_gid), 3469 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge), 3470 fsparam_u32oct("mode", Opt_mode), 3471 fsparam_string("mpol", Opt_mpol), 3472 fsparam_string("nr_blocks", Opt_nr_blocks), 3473 fsparam_string("nr_inodes", Opt_nr_inodes), 3474 fsparam_string("size", Opt_size), 3475 fsparam_u32 ("uid", Opt_uid), 3476 fsparam_flag ("inode32", Opt_inode32), 3477 fsparam_flag ("inode64", Opt_inode64), 3478 {} 3479 }; 3480 3481 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) 3482 { 3483 struct shmem_options *ctx = fc->fs_private; 3484 struct fs_parse_result result; 3485 unsigned long long size; 3486 char *rest; 3487 int opt; 3488 3489 opt = fs_parse(fc, shmem_fs_parameters, param, &result); 3490 if (opt < 0) 3491 return opt; 3492 3493 switch (opt) { 3494 case Opt_size: 3495 size = memparse(param->string, &rest); 3496 if (*rest == '%') { 3497 size <<= PAGE_SHIFT; 3498 size *= totalram_pages(); 3499 do_div(size, 100); 3500 rest++; 3501 } 3502 if (*rest) 3503 goto bad_value; 3504 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE); 3505 ctx->seen |= SHMEM_SEEN_BLOCKS; 3506 break; 3507 case Opt_nr_blocks: 3508 ctx->blocks = memparse(param->string, &rest); 3509 if (*rest || ctx->blocks > S64_MAX) 3510 goto bad_value; 3511 ctx->seen |= SHMEM_SEEN_BLOCKS; 3512 break; 3513 case Opt_nr_inodes: 3514 ctx->inodes = memparse(param->string, &rest); 3515 if (*rest) 3516 goto bad_value; 3517 ctx->seen |= SHMEM_SEEN_INODES; 3518 break; 3519 case Opt_mode: 3520 ctx->mode = result.uint_32 & 07777; 3521 break; 3522 case Opt_uid: 3523 ctx->uid = make_kuid(current_user_ns(), result.uint_32); 3524 if (!uid_valid(ctx->uid)) 3525 goto bad_value; 3526 break; 3527 case Opt_gid: 3528 ctx->gid = make_kgid(current_user_ns(), result.uint_32); 3529 if (!gid_valid(ctx->gid)) 3530 goto bad_value; 3531 break; 3532 case Opt_huge: 3533 ctx->huge = result.uint_32; 3534 if (ctx->huge != SHMEM_HUGE_NEVER && 3535 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 3536 has_transparent_hugepage())) 3537 goto unsupported_parameter; 3538 ctx->seen |= SHMEM_SEEN_HUGE; 3539 break; 3540 case Opt_mpol: 3541 if (IS_ENABLED(CONFIG_NUMA)) { 3542 mpol_put(ctx->mpol); 3543 ctx->mpol = NULL; 3544 if (mpol_parse_str(param->string, &ctx->mpol)) 3545 goto bad_value; 3546 break; 3547 } 3548 goto unsupported_parameter; 3549 case Opt_inode32: 3550 ctx->full_inums = false; 3551 ctx->seen |= SHMEM_SEEN_INUMS; 3552 break; 3553 case Opt_inode64: 3554 if (sizeof(ino_t) < 8) { 3555 return invalfc(fc, 3556 "Cannot use inode64 with <64bit inums in kernel\n"); 3557 } 3558 ctx->full_inums = true; 3559 ctx->seen |= SHMEM_SEEN_INUMS; 3560 break; 3561 } 3562 return 0; 3563 3564 unsupported_parameter: 3565 return invalfc(fc, "Unsupported parameter '%s'", param->key); 3566 bad_value: 3567 return invalfc(fc, "Bad value for '%s'", param->key); 3568 } 3569 3570 static int shmem_parse_options(struct fs_context *fc, void *data) 3571 { 3572 char *options = data; 3573 3574 if (options) { 3575 int err = security_sb_eat_lsm_opts(options, &fc->security); 3576 if (err) 3577 return err; 3578 } 3579 3580 while (options != NULL) { 3581 char *this_char = options; 3582 for (;;) { 3583 /* 3584 * NUL-terminate this option: unfortunately, 3585 * mount options form a comma-separated list, 3586 * but mpol's nodelist may also contain commas. 3587 */ 3588 options = strchr(options, ','); 3589 if (options == NULL) 3590 break; 3591 options++; 3592 if (!isdigit(*options)) { 3593 options[-1] = '\0'; 3594 break; 3595 } 3596 } 3597 if (*this_char) { 3598 char *value = strchr(this_char, '='); 3599 size_t len = 0; 3600 int err; 3601 3602 if (value) { 3603 *value++ = '\0'; 3604 len = strlen(value); 3605 } 3606 err = vfs_parse_fs_string(fc, this_char, value, len); 3607 if (err < 0) 3608 return err; 3609 } 3610 } 3611 return 0; 3612 } 3613 3614 /* 3615 * Reconfigure a shmem filesystem. 3616 * 3617 * Note that we disallow change from limited->unlimited blocks/inodes while any 3618 * are in use; but we must separately disallow unlimited->limited, because in 3619 * that case we have no record of how much is already in use. 3620 */ 3621 static int shmem_reconfigure(struct fs_context *fc) 3622 { 3623 struct shmem_options *ctx = fc->fs_private; 3624 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb); 3625 unsigned long inodes; 3626 struct mempolicy *mpol = NULL; 3627 const char *err; 3628 3629 raw_spin_lock(&sbinfo->stat_lock); 3630 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3631 3632 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) { 3633 if (!sbinfo->max_blocks) { 3634 err = "Cannot retroactively limit size"; 3635 goto out; 3636 } 3637 if (percpu_counter_compare(&sbinfo->used_blocks, 3638 ctx->blocks) > 0) { 3639 err = "Too small a size for current use"; 3640 goto out; 3641 } 3642 } 3643 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) { 3644 if (!sbinfo->max_inodes) { 3645 err = "Cannot retroactively limit inodes"; 3646 goto out; 3647 } 3648 if (ctx->inodes < inodes) { 3649 err = "Too few inodes for current use"; 3650 goto out; 3651 } 3652 } 3653 3654 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums && 3655 sbinfo->next_ino > UINT_MAX) { 3656 err = "Current inum too high to switch to 32-bit inums"; 3657 goto out; 3658 } 3659 3660 if (ctx->seen & SHMEM_SEEN_HUGE) 3661 sbinfo->huge = ctx->huge; 3662 if (ctx->seen & SHMEM_SEEN_INUMS) 3663 sbinfo->full_inums = ctx->full_inums; 3664 if (ctx->seen & SHMEM_SEEN_BLOCKS) 3665 sbinfo->max_blocks = ctx->blocks; 3666 if (ctx->seen & SHMEM_SEEN_INODES) { 3667 sbinfo->max_inodes = ctx->inodes; 3668 sbinfo->free_inodes = ctx->inodes - inodes; 3669 } 3670 3671 /* 3672 * Preserve previous mempolicy unless mpol remount option was specified. 3673 */ 3674 if (ctx->mpol) { 3675 mpol = sbinfo->mpol; 3676 sbinfo->mpol = ctx->mpol; /* transfers initial ref */ 3677 ctx->mpol = NULL; 3678 } 3679 raw_spin_unlock(&sbinfo->stat_lock); 3680 mpol_put(mpol); 3681 return 0; 3682 out: 3683 raw_spin_unlock(&sbinfo->stat_lock); 3684 return invalfc(fc, "%s", err); 3685 } 3686 3687 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3688 { 3689 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3690 3691 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3692 seq_printf(seq, ",size=%luk", 3693 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3694 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3695 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3696 if (sbinfo->mode != (0777 | S_ISVTX)) 3697 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3698 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3699 seq_printf(seq, ",uid=%u", 3700 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3701 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3702 seq_printf(seq, ",gid=%u", 3703 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3704 3705 /* 3706 * Showing inode{64,32} might be useful even if it's the system default, 3707 * since then people don't have to resort to checking both here and 3708 * /proc/config.gz to confirm 64-bit inums were successfully applied 3709 * (which may not even exist if IKCONFIG_PROC isn't enabled). 3710 * 3711 * We hide it when inode64 isn't the default and we are using 32-bit 3712 * inodes, since that probably just means the feature isn't even under 3713 * consideration. 3714 * 3715 * As such: 3716 * 3717 * +-----------------+-----------------+ 3718 * | TMPFS_INODE64=y | TMPFS_INODE64=n | 3719 * +------------------+-----------------+-----------------+ 3720 * | full_inums=true | show | show | 3721 * | full_inums=false | show | hide | 3722 * +------------------+-----------------+-----------------+ 3723 * 3724 */ 3725 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums) 3726 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32)); 3727 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3728 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3729 if (sbinfo->huge) 3730 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3731 #endif 3732 shmem_show_mpol(seq, sbinfo->mpol); 3733 return 0; 3734 } 3735 3736 #endif /* CONFIG_TMPFS */ 3737 3738 static void shmem_put_super(struct super_block *sb) 3739 { 3740 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3741 3742 free_percpu(sbinfo->ino_batch); 3743 percpu_counter_destroy(&sbinfo->used_blocks); 3744 mpol_put(sbinfo->mpol); 3745 kfree(sbinfo); 3746 sb->s_fs_info = NULL; 3747 } 3748 3749 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) 3750 { 3751 struct shmem_options *ctx = fc->fs_private; 3752 struct inode *inode; 3753 struct shmem_sb_info *sbinfo; 3754 3755 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3756 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3757 L1_CACHE_BYTES), GFP_KERNEL); 3758 if (!sbinfo) 3759 return -ENOMEM; 3760 3761 sb->s_fs_info = sbinfo; 3762 3763 #ifdef CONFIG_TMPFS 3764 /* 3765 * Per default we only allow half of the physical ram per 3766 * tmpfs instance, limiting inodes to one per page of lowmem; 3767 * but the internal instance is left unlimited. 3768 */ 3769 if (!(sb->s_flags & SB_KERNMOUNT)) { 3770 if (!(ctx->seen & SHMEM_SEEN_BLOCKS)) 3771 ctx->blocks = shmem_default_max_blocks(); 3772 if (!(ctx->seen & SHMEM_SEEN_INODES)) 3773 ctx->inodes = shmem_default_max_inodes(); 3774 if (!(ctx->seen & SHMEM_SEEN_INUMS)) 3775 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64); 3776 } else { 3777 sb->s_flags |= SB_NOUSER; 3778 } 3779 sb->s_export_op = &shmem_export_ops; 3780 sb->s_flags |= SB_NOSEC | SB_I_VERSION; 3781 #else 3782 sb->s_flags |= SB_NOUSER; 3783 #endif 3784 sbinfo->max_blocks = ctx->blocks; 3785 sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes; 3786 if (sb->s_flags & SB_KERNMOUNT) { 3787 sbinfo->ino_batch = alloc_percpu(ino_t); 3788 if (!sbinfo->ino_batch) 3789 goto failed; 3790 } 3791 sbinfo->uid = ctx->uid; 3792 sbinfo->gid = ctx->gid; 3793 sbinfo->full_inums = ctx->full_inums; 3794 sbinfo->mode = ctx->mode; 3795 sbinfo->huge = ctx->huge; 3796 sbinfo->mpol = ctx->mpol; 3797 ctx->mpol = NULL; 3798 3799 raw_spin_lock_init(&sbinfo->stat_lock); 3800 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3801 goto failed; 3802 spin_lock_init(&sbinfo->shrinklist_lock); 3803 INIT_LIST_HEAD(&sbinfo->shrinklist); 3804 3805 sb->s_maxbytes = MAX_LFS_FILESIZE; 3806 sb->s_blocksize = PAGE_SIZE; 3807 sb->s_blocksize_bits = PAGE_SHIFT; 3808 sb->s_magic = TMPFS_MAGIC; 3809 sb->s_op = &shmem_ops; 3810 sb->s_time_gran = 1; 3811 #ifdef CONFIG_TMPFS_XATTR 3812 sb->s_xattr = shmem_xattr_handlers; 3813 #endif 3814 #ifdef CONFIG_TMPFS_POSIX_ACL 3815 sb->s_flags |= SB_POSIXACL; 3816 #endif 3817 uuid_gen(&sb->s_uuid); 3818 3819 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3820 if (!inode) 3821 goto failed; 3822 inode->i_uid = sbinfo->uid; 3823 inode->i_gid = sbinfo->gid; 3824 sb->s_root = d_make_root(inode); 3825 if (!sb->s_root) 3826 goto failed; 3827 return 0; 3828 3829 failed: 3830 shmem_put_super(sb); 3831 return -ENOMEM; 3832 } 3833 3834 static int shmem_get_tree(struct fs_context *fc) 3835 { 3836 return get_tree_nodev(fc, shmem_fill_super); 3837 } 3838 3839 static void shmem_free_fc(struct fs_context *fc) 3840 { 3841 struct shmem_options *ctx = fc->fs_private; 3842 3843 if (ctx) { 3844 mpol_put(ctx->mpol); 3845 kfree(ctx); 3846 } 3847 } 3848 3849 static const struct fs_context_operations shmem_fs_context_ops = { 3850 .free = shmem_free_fc, 3851 .get_tree = shmem_get_tree, 3852 #ifdef CONFIG_TMPFS 3853 .parse_monolithic = shmem_parse_options, 3854 .parse_param = shmem_parse_one, 3855 .reconfigure = shmem_reconfigure, 3856 #endif 3857 }; 3858 3859 static struct kmem_cache *shmem_inode_cachep; 3860 3861 static struct inode *shmem_alloc_inode(struct super_block *sb) 3862 { 3863 struct shmem_inode_info *info; 3864 info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); 3865 if (!info) 3866 return NULL; 3867 return &info->vfs_inode; 3868 } 3869 3870 static void shmem_free_in_core_inode(struct inode *inode) 3871 { 3872 if (S_ISLNK(inode->i_mode)) 3873 kfree(inode->i_link); 3874 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3875 } 3876 3877 static void shmem_destroy_inode(struct inode *inode) 3878 { 3879 if (S_ISREG(inode->i_mode)) 3880 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3881 } 3882 3883 static void shmem_init_inode(void *foo) 3884 { 3885 struct shmem_inode_info *info = foo; 3886 inode_init_once(&info->vfs_inode); 3887 } 3888 3889 static void shmem_init_inodecache(void) 3890 { 3891 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3892 sizeof(struct shmem_inode_info), 3893 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3894 } 3895 3896 static void shmem_destroy_inodecache(void) 3897 { 3898 kmem_cache_destroy(shmem_inode_cachep); 3899 } 3900 3901 /* Keep the page in page cache instead of truncating it */ 3902 static int shmem_error_remove_page(struct address_space *mapping, 3903 struct page *page) 3904 { 3905 return 0; 3906 } 3907 3908 const struct address_space_operations shmem_aops = { 3909 .writepage = shmem_writepage, 3910 .dirty_folio = noop_dirty_folio, 3911 #ifdef CONFIG_TMPFS 3912 .write_begin = shmem_write_begin, 3913 .write_end = shmem_write_end, 3914 #endif 3915 #ifdef CONFIG_MIGRATION 3916 .migrate_folio = migrate_folio, 3917 #endif 3918 .error_remove_page = shmem_error_remove_page, 3919 }; 3920 EXPORT_SYMBOL(shmem_aops); 3921 3922 static const struct file_operations shmem_file_operations = { 3923 .mmap = shmem_mmap, 3924 .get_unmapped_area = shmem_get_unmapped_area, 3925 #ifdef CONFIG_TMPFS 3926 .llseek = shmem_file_llseek, 3927 .read_iter = shmem_file_read_iter, 3928 .write_iter = generic_file_write_iter, 3929 .fsync = noop_fsync, 3930 .splice_read = generic_file_splice_read, 3931 .splice_write = iter_file_splice_write, 3932 .fallocate = shmem_fallocate, 3933 #endif 3934 }; 3935 3936 static const struct inode_operations shmem_inode_operations = { 3937 .getattr = shmem_getattr, 3938 .setattr = shmem_setattr, 3939 #ifdef CONFIG_TMPFS_XATTR 3940 .listxattr = shmem_listxattr, 3941 .set_acl = simple_set_acl, 3942 .fileattr_get = shmem_fileattr_get, 3943 .fileattr_set = shmem_fileattr_set, 3944 #endif 3945 }; 3946 3947 static const struct inode_operations shmem_dir_inode_operations = { 3948 #ifdef CONFIG_TMPFS 3949 .getattr = shmem_getattr, 3950 .create = shmem_create, 3951 .lookup = simple_lookup, 3952 .link = shmem_link, 3953 .unlink = shmem_unlink, 3954 .symlink = shmem_symlink, 3955 .mkdir = shmem_mkdir, 3956 .rmdir = shmem_rmdir, 3957 .mknod = shmem_mknod, 3958 .rename = shmem_rename2, 3959 .tmpfile = shmem_tmpfile, 3960 #endif 3961 #ifdef CONFIG_TMPFS_XATTR 3962 .listxattr = shmem_listxattr, 3963 .fileattr_get = shmem_fileattr_get, 3964 .fileattr_set = shmem_fileattr_set, 3965 #endif 3966 #ifdef CONFIG_TMPFS_POSIX_ACL 3967 .setattr = shmem_setattr, 3968 .set_acl = simple_set_acl, 3969 #endif 3970 }; 3971 3972 static const struct inode_operations shmem_special_inode_operations = { 3973 .getattr = shmem_getattr, 3974 #ifdef CONFIG_TMPFS_XATTR 3975 .listxattr = shmem_listxattr, 3976 #endif 3977 #ifdef CONFIG_TMPFS_POSIX_ACL 3978 .setattr = shmem_setattr, 3979 .set_acl = simple_set_acl, 3980 #endif 3981 }; 3982 3983 static const struct super_operations shmem_ops = { 3984 .alloc_inode = shmem_alloc_inode, 3985 .free_inode = shmem_free_in_core_inode, 3986 .destroy_inode = shmem_destroy_inode, 3987 #ifdef CONFIG_TMPFS 3988 .statfs = shmem_statfs, 3989 .show_options = shmem_show_options, 3990 #endif 3991 .evict_inode = shmem_evict_inode, 3992 .drop_inode = generic_delete_inode, 3993 .put_super = shmem_put_super, 3994 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3995 .nr_cached_objects = shmem_unused_huge_count, 3996 .free_cached_objects = shmem_unused_huge_scan, 3997 #endif 3998 }; 3999 4000 static const struct vm_operations_struct shmem_vm_ops = { 4001 .fault = shmem_fault, 4002 .map_pages = filemap_map_pages, 4003 #ifdef CONFIG_NUMA 4004 .set_policy = shmem_set_policy, 4005 .get_policy = shmem_get_policy, 4006 #endif 4007 }; 4008 4009 int shmem_init_fs_context(struct fs_context *fc) 4010 { 4011 struct shmem_options *ctx; 4012 4013 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL); 4014 if (!ctx) 4015 return -ENOMEM; 4016 4017 ctx->mode = 0777 | S_ISVTX; 4018 ctx->uid = current_fsuid(); 4019 ctx->gid = current_fsgid(); 4020 4021 fc->fs_private = ctx; 4022 fc->ops = &shmem_fs_context_ops; 4023 return 0; 4024 } 4025 4026 static struct file_system_type shmem_fs_type = { 4027 .owner = THIS_MODULE, 4028 .name = "tmpfs", 4029 .init_fs_context = shmem_init_fs_context, 4030 #ifdef CONFIG_TMPFS 4031 .parameters = shmem_fs_parameters, 4032 #endif 4033 .kill_sb = kill_litter_super, 4034 .fs_flags = FS_USERNS_MOUNT, 4035 }; 4036 4037 void __init shmem_init(void) 4038 { 4039 int error; 4040 4041 shmem_init_inodecache(); 4042 4043 error = register_filesystem(&shmem_fs_type); 4044 if (error) { 4045 pr_err("Could not register tmpfs\n"); 4046 goto out2; 4047 } 4048 4049 shm_mnt = kern_mount(&shmem_fs_type); 4050 if (IS_ERR(shm_mnt)) { 4051 error = PTR_ERR(shm_mnt); 4052 pr_err("Could not kern_mount tmpfs\n"); 4053 goto out1; 4054 } 4055 4056 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4057 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) 4058 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4059 else 4060 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */ 4061 #endif 4062 return; 4063 4064 out1: 4065 unregister_filesystem(&shmem_fs_type); 4066 out2: 4067 shmem_destroy_inodecache(); 4068 shm_mnt = ERR_PTR(error); 4069 } 4070 4071 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) 4072 static ssize_t shmem_enabled_show(struct kobject *kobj, 4073 struct kobj_attribute *attr, char *buf) 4074 { 4075 static const int values[] = { 4076 SHMEM_HUGE_ALWAYS, 4077 SHMEM_HUGE_WITHIN_SIZE, 4078 SHMEM_HUGE_ADVISE, 4079 SHMEM_HUGE_NEVER, 4080 SHMEM_HUGE_DENY, 4081 SHMEM_HUGE_FORCE, 4082 }; 4083 int len = 0; 4084 int i; 4085 4086 for (i = 0; i < ARRAY_SIZE(values); i++) { 4087 len += sysfs_emit_at(buf, len, 4088 shmem_huge == values[i] ? "%s[%s]" : "%s%s", 4089 i ? " " : "", 4090 shmem_format_huge(values[i])); 4091 } 4092 4093 len += sysfs_emit_at(buf, len, "\n"); 4094 4095 return len; 4096 } 4097 4098 static ssize_t shmem_enabled_store(struct kobject *kobj, 4099 struct kobj_attribute *attr, const char *buf, size_t count) 4100 { 4101 char tmp[16]; 4102 int huge; 4103 4104 if (count + 1 > sizeof(tmp)) 4105 return -EINVAL; 4106 memcpy(tmp, buf, count); 4107 tmp[count] = '\0'; 4108 if (count && tmp[count - 1] == '\n') 4109 tmp[count - 1] = '\0'; 4110 4111 huge = shmem_parse_huge(tmp); 4112 if (huge == -EINVAL) 4113 return -EINVAL; 4114 if (!has_transparent_hugepage() && 4115 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 4116 return -EINVAL; 4117 4118 shmem_huge = huge; 4119 if (shmem_huge > SHMEM_HUGE_DENY) 4120 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4121 return count; 4122 } 4123 4124 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled); 4125 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ 4126 4127 #else /* !CONFIG_SHMEM */ 4128 4129 /* 4130 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 4131 * 4132 * This is intended for small system where the benefits of the full 4133 * shmem code (swap-backed and resource-limited) are outweighed by 4134 * their complexity. On systems without swap this code should be 4135 * effectively equivalent, but much lighter weight. 4136 */ 4137 4138 static struct file_system_type shmem_fs_type = { 4139 .name = "tmpfs", 4140 .init_fs_context = ramfs_init_fs_context, 4141 .parameters = ramfs_fs_parameters, 4142 .kill_sb = kill_litter_super, 4143 .fs_flags = FS_USERNS_MOUNT, 4144 }; 4145 4146 void __init shmem_init(void) 4147 { 4148 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4149 4150 shm_mnt = kern_mount(&shmem_fs_type); 4151 BUG_ON(IS_ERR(shm_mnt)); 4152 } 4153 4154 int shmem_unuse(unsigned int type) 4155 { 4156 return 0; 4157 } 4158 4159 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 4160 { 4161 return 0; 4162 } 4163 4164 void shmem_unlock_mapping(struct address_space *mapping) 4165 { 4166 } 4167 4168 #ifdef CONFIG_MMU 4169 unsigned long shmem_get_unmapped_area(struct file *file, 4170 unsigned long addr, unsigned long len, 4171 unsigned long pgoff, unsigned long flags) 4172 { 4173 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 4174 } 4175 #endif 4176 4177 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4178 { 4179 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4180 } 4181 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4182 4183 #define shmem_vm_ops generic_file_vm_ops 4184 #define shmem_file_operations ramfs_file_operations 4185 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 4186 #define shmem_acct_size(flags, size) 0 4187 #define shmem_unacct_size(flags, size) do {} while (0) 4188 4189 #endif /* CONFIG_SHMEM */ 4190 4191 /* common code */ 4192 4193 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size, 4194 unsigned long flags, unsigned int i_flags) 4195 { 4196 struct inode *inode; 4197 struct file *res; 4198 4199 if (IS_ERR(mnt)) 4200 return ERR_CAST(mnt); 4201 4202 if (size < 0 || size > MAX_LFS_FILESIZE) 4203 return ERR_PTR(-EINVAL); 4204 4205 if (shmem_acct_size(flags, size)) 4206 return ERR_PTR(-ENOMEM); 4207 4208 inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0, 4209 flags); 4210 if (unlikely(!inode)) { 4211 shmem_unacct_size(flags, size); 4212 return ERR_PTR(-ENOSPC); 4213 } 4214 inode->i_flags |= i_flags; 4215 inode->i_size = size; 4216 clear_nlink(inode); /* It is unlinked */ 4217 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4218 if (!IS_ERR(res)) 4219 res = alloc_file_pseudo(inode, mnt, name, O_RDWR, 4220 &shmem_file_operations); 4221 if (IS_ERR(res)) 4222 iput(inode); 4223 return res; 4224 } 4225 4226 /** 4227 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4228 * kernel internal. There will be NO LSM permission checks against the 4229 * underlying inode. So users of this interface must do LSM checks at a 4230 * higher layer. The users are the big_key and shm implementations. LSM 4231 * checks are provided at the key or shm level rather than the inode. 4232 * @name: name for dentry (to be seen in /proc/<pid>/maps 4233 * @size: size to be set for the file 4234 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4235 */ 4236 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4237 { 4238 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE); 4239 } 4240 4241 /** 4242 * shmem_file_setup - get an unlinked file living in tmpfs 4243 * @name: name for dentry (to be seen in /proc/<pid>/maps 4244 * @size: size to be set for the file 4245 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4246 */ 4247 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4248 { 4249 return __shmem_file_setup(shm_mnt, name, size, flags, 0); 4250 } 4251 EXPORT_SYMBOL_GPL(shmem_file_setup); 4252 4253 /** 4254 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs 4255 * @mnt: the tmpfs mount where the file will be created 4256 * @name: name for dentry (to be seen in /proc/<pid>/maps 4257 * @size: size to be set for the file 4258 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4259 */ 4260 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name, 4261 loff_t size, unsigned long flags) 4262 { 4263 return __shmem_file_setup(mnt, name, size, flags, 0); 4264 } 4265 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt); 4266 4267 /** 4268 * shmem_zero_setup - setup a shared anonymous mapping 4269 * @vma: the vma to be mmapped is prepared by do_mmap 4270 */ 4271 int shmem_zero_setup(struct vm_area_struct *vma) 4272 { 4273 struct file *file; 4274 loff_t size = vma->vm_end - vma->vm_start; 4275 4276 /* 4277 * Cloning a new file under mmap_lock leads to a lock ordering conflict 4278 * between XFS directory reading and selinux: since this file is only 4279 * accessible to the user through its mapping, use S_PRIVATE flag to 4280 * bypass file security, in the same way as shmem_kernel_file_setup(). 4281 */ 4282 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags); 4283 if (IS_ERR(file)) 4284 return PTR_ERR(file); 4285 4286 if (vma->vm_file) 4287 fput(vma->vm_file); 4288 vma->vm_file = file; 4289 vma->vm_ops = &shmem_vm_ops; 4290 4291 return 0; 4292 } 4293 4294 /** 4295 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4296 * @mapping: the page's address_space 4297 * @index: the page index 4298 * @gfp: the page allocator flags to use if allocating 4299 * 4300 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4301 * with any new page allocations done using the specified allocation flags. 4302 * But read_cache_page_gfp() uses the ->read_folio() method: which does not 4303 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4304 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4305 * 4306 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4307 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4308 */ 4309 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4310 pgoff_t index, gfp_t gfp) 4311 { 4312 #ifdef CONFIG_SHMEM 4313 struct inode *inode = mapping->host; 4314 struct folio *folio; 4315 struct page *page; 4316 int error; 4317 4318 BUG_ON(!shmem_mapping(mapping)); 4319 error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE, 4320 gfp, NULL, NULL, NULL); 4321 if (error) 4322 return ERR_PTR(error); 4323 4324 folio_unlock(folio); 4325 page = folio_file_page(folio, index); 4326 if (PageHWPoison(page)) { 4327 folio_put(folio); 4328 return ERR_PTR(-EIO); 4329 } 4330 4331 return page; 4332 #else 4333 /* 4334 * The tiny !SHMEM case uses ramfs without swap 4335 */ 4336 return read_cache_page_gfp(mapping, index, gfp); 4337 #endif 4338 } 4339 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4340